-
Notifications
You must be signed in to change notification settings - Fork 134
Description
代码如下:
特征提取:
fet = FeatureExtractionTool(config='/data/huangjingwang/SCI/MMSA/cut/config2.json')
infile_path=os.path.abspath('/data/huangjingwang/SCI/MMSA/cut/test_output/output_video_002.mp4')
outfile_path=os.path.abspath('/data/huangjingwang/SCI/MMSA/cut/test_output/feature_get.pkl')
text_path=os.path.abspath("/data/huangjingwang/SCI/MMSA/cut/input.txt")
fet.run_single(in_file=infile_path, out_file=outfile_path,text="What is love")
模型训练:
config=get_config_regression('mtfn','sims')
config['custom_feature'] = '/data/huangjingwang/SCI/MMSA/dataset/SIMS/unaligned_39.pkl'
MMSA_run('mtfn', 'sims',
gpu_ids=[0],
config=config,
res_save_dir='/data/huangjingwang/SCI/MMSA/results/SIMS',
model_save_dir='/data/huangjingwang/SCI/MMSA/models/SMIS')
模型测试:
MMSA_test(config=config,
weights_path='/data/huangjingwang/SCI/MMSA/models/SMIS/mtfn-sims.pth',
feature_path='/data/huangjingwang/SCI/MMSA/cut/test_output/feature_get.pkl')
config2.json内容:
{
"audio": {
"tool": "librosa",
"sample_rate": 22050,
"features": {
"mfcc":{
"n_mfcc":20,
"htk": true
},
"zero_crossing_rate":{},
"chroma_cqt":{}
}
},
"video": {
"tool": "openface",
"fps": 30,
"ultiFace": {"enable": false,
"device": "cuda:0",
"facedetScale": 0.25,
"minTrack": 10,
"numFailedDet": 10,
"minFaceSize": 1,
"cropScale": 0.4},
"average_over": 1,
"args": {
"hogalign": false,
"simalign": false,
"nobadaligned": false,
"landmark_2D": true,
"landmark_3D": true,
"pdmparams": true,
"head_pose": true,
"action_units": true,
"gaze": true,
"tracked": false
}
},
"text": {
"model": "bert",
"device": "cpu",
"pretrained": "bert-base-chinese",
"args": {}
}
}
报错信息:Traceback (most recent call last):
File "/data/huangjingwang/SCI/MMSA/cut/test2.py", line 21, in
MMSA_test(config=config,
File "/data/huangjingwang/.conda/envs/hjw_nlp/lib/python3.11/site-packages/MMSA/run.py", line 350, in MMSA_test
output = model(text, audio, vision)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/huangjingwang/.conda/envs/hjw_nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/huangjingwang/.conda/envs/hjw_nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/huangjingwang/.conda/envs/hjw_nlp/lib/python3.11/site-packages/MMSA/models/AMIO.py", line 49, in forward
return self.Model(text_x, audio_x, video_x, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/huangjingwang/.conda/envs/hjw_nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/huangjingwang/.conda/envs/hjw_nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/huangjingwang/.conda/envs/hjw_nlp/lib/python3.11/site-packages/MMSA/models/multiTask/MTFN.py", line 185, in forward
_text_h = torch.cat((add_one, text_h), dim=1)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: Tensors must have same number of dimensions: got 2 and 1