| Reading metadata...: 2165it [00:00, 13200.08it/s] | 0/60000 [00:00<?, ?it/s] | |
| Reading metadata...: 1it [00:00, 6.42it/s] | |
| [INFO|trainer_utils.py:759] 2023-11-18 13:12:05,720 >> The following columns in the training set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`, you can safely ignore this message. | |
| Traceback (most recent call last): | |
| File "/mnt/e/run_speech_recognition_seq2seq_streaming.py", line 679, in <module> | |
| main() | |
| File "/mnt/e/run_speech_recognition_seq2seq_streaming.py", line 628, in main | |
| train_result = trainer.train(resume_from_checkpoint=checkpoint) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 1547, in train | |
| return inner_training_loop( | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 1872, in _inner_training_loop | |
| tr_loss_step = self.training_step(model, inputs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 2739, in training_step | |
| loss = self.compute_loss(model, inputs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 2762, in compute_loss | |
| outputs = model(**inputs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn | |
| ret_val = func(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1818, in forward | |
| loss = self.module(*inputs, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 1683, in forward | |
| outputs = self.model( | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 1543, in forward | |
| encoder_outputs = self.encoder( | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 1119, in forward | |
| inputs_embeds = nn.functional.gelu(self.conv1(input_features)) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/conv.py", line 310, in forward | |
| return self._conv_forward(input, self.weight, self.bias) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/conv.py", line 306, in _conv_forward | |
| return F.conv1d(input, weight, bias, self.stride, | |
| RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (CUDABFloat16Type) should be the same |