| Step | \n", "Training Loss | \n", "
|---|---|
| 1 | \n", "2.421700 | \n", "
"
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
"\u001b[31m│\u001b[0m in \u001b[92m╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
"│ in <cell line: 27>:27 │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:1696 in train │\n",
"│ │\n",
"│ 1693 │ │ inner_training_loop = find_executable_batch_size( │\n",
"│ 1694 │ │ │ self._inner_training_loop, self._train_batch_size, args.auto_find_batch_size │\n",
"│ 1695 │ │ ) │\n",
"│ ❱ 1696 │ │ return inner_training_loop( │\n",
"│ 1697 │ │ │ args=args, │\n",
"│ 1698 │ │ │ resume_from_checkpoint=resume_from_checkpoint, │\n",
"│ 1699 │ │ │ trial=trial, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:1973 in _inner_training_loop │\n",
"│ │\n",
"│ 1970 │ │ │ │ │ with model.no_sync(): │\n",
"│ 1971 │ │ │ │ │ │ tr_loss_step = self.training_step(model, inputs) │\n",
"│ 1972 │ │ │ │ else: │\n",
"│ ❱ 1973 │ │ │ │ │ tr_loss_step = self.training_step(model, inputs) │\n",
"│ 1974 │ │ │ │ │\n",
"│ 1975 │ │ │ │ if ( │\n",
"│ 1976 │ │ │ │ │ args.logging_nan_inf_filter │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2787 in training_step │\n",
"│ │\n",
"│ 2784 │ │ │ return loss_mb.reduce_mean().detach().to(self.args.device) │\n",
"│ 2785 │ │ │\n",
"│ 2786 │ │ with self.compute_loss_context_manager(): │\n",
"│ ❱ 2787 │ │ │ loss = self.compute_loss(model, inputs) │\n",
"│ 2788 │ │ │\n",
"│ 2789 │ │ if self.args.n_gpu > 1: │\n",
"│ 2790 │ │ │ loss = loss.mean() # mean() to average on multi-gpu parallel training │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2819 in compute_loss │\n",
"│ │\n",
"│ 2816 │ │ │ labels = inputs.pop(\"labels\") │\n",
"│ 2817 │ │ else: │\n",
"│ 2818 │ │ │ labels = None │\n",
"│ ❱ 2819 │ │ outputs = model(**inputs) │\n",
"│ 2820 │ │ # Save past state if it exists │\n",
"│ 2821 │ │ # TODO: this needs to be fixed and made cleaner later. │\n",
"│ 2822 │ │ if self.args.past_index >= 0: │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/peft/peft_model.py:686 in forward │\n",
"│ │\n",
"│ 683 │ ): │\n",
"│ 684 │ │ peft_config = self.active_peft_config │\n",
"│ 685 │ │ if not isinstance(peft_config, PromptLearningConfig): │\n",
"│ ❱ 686 │ │ │ return self.base_model( │\n",
"│ 687 │ │ │ │ input_ids=input_ids, │\n",
"│ 688 │ │ │ │ attention_mask=attention_mask, │\n",
"│ 689 │ │ │ │ inputs_embeds=inputs_embeds, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in new_forward │\n",
"│ │\n",
"│ 162 │ │ │ with torch.no_grad(): │\n",
"│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 164 │ │ else: │\n",
"│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
"│ 167 │ │\n",
"│ 168 │ module.forward = new_forward │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/models/llama/modeling_llama.py:687 in │\n",
"│ forward │\n",
"│ │\n",
"│ 684 │ │ return_dict = return_dict if return_dict is not None else self.config.use_return │\n",
"│ 685 │ │ │\n",
"│ 686 │ │ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) │\n",
"│ ❱ 687 │ │ outputs = self.model( │\n",
"│ 688 │ │ │ input_ids=input_ids, │\n",
"│ 689 │ │ │ attention_mask=attention_mask, │\n",
"│ 690 │ │ │ position_ids=position_ids, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in new_forward │\n",
"│ │\n",
"│ 162 │ │ │ with torch.no_grad(): │\n",
"│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 164 │ │ else: │\n",
"│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
"│ 167 │ │\n",
"│ 168 │ module.forward = new_forward │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/models/llama/modeling_llama.py:569 in │\n",
"│ forward │\n",
"│ │\n",
"│ 566 │ │ │ │ │ │\n",
"│ 567 │ │ │ │ │ return custom_forward │\n",
"│ 568 │ │ │ │ │\n",
"│ ❱ 569 │ │ │ │ layer_outputs = torch.utils.checkpoint.checkpoint( │\n",
"│ 570 │ │ │ │ │ create_custom_forward(decoder_layer), │\n",
"│ 571 │ │ │ │ │ hidden_states, │\n",
"│ 572 │ │ │ │ │ attention_mask, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:249 in checkpoint │\n",
"│ │\n",
"│ 246 │ │ raise ValueError(\"Unexpected keyword arguments: \" + \",\".join(arg for arg in kwar │\n",
"│ 247 │ │\n",
"│ 248 │ if use_reentrant: │\n",
"│ ❱ 249 │ │ return CheckpointFunction.apply(function, preserve, *args) │\n",
"│ 250 │ else: │\n",
"│ 251 │ │ return _checkpoint_without_reentrant( │\n",
"│ 252 │ │ │ function, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/autograd/function.py:506 in apply │\n",
"│ │\n",
"│ 503 │ │ if not torch._C._are_functorch_transforms_active(): │\n",
"│ 504 │ │ │ # See NOTE: [functorch vjp and autograd interaction] │\n",
"│ 505 │ │ │ args = _functorch.utils.unwrap_dead_wrappers(args) │\n",
"│ ❱ 506 │ │ │ return super().apply(*args, **kwargs) # type: ignore[misc] │\n",
"│ 507 │ │ │\n",
"│ 508 │ │ if cls.setup_context == _SingleLevelFunction.setup_context: │\n",
"│ 509 │ │ │ raise RuntimeError( │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:107 in forward │\n",
"│ │\n",
"│ 104 │ │ ctx.save_for_backward(*tensor_inputs) │\n",
"│ 105 │ │ │\n",
"│ 106 │ │ with torch.no_grad(): │\n",
"│ ❱ 107 │ │ │ outputs = run_function(*args) │\n",
"│ 108 │ │ return outputs │\n",
"│ 109 │ │\n",
"│ 110 │ @staticmethod │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/models/llama/modeling_llama.py:565 in │\n",
"│ custom_forward │\n",
"│ │\n",
"│ 562 │ │ │ │ def create_custom_forward(module): │\n",
"│ 563 │ │ │ │ │ def custom_forward(*inputs): │\n",
"│ 564 │ │ │ │ │ │ # None for past_key_value │\n",
"│ ❱ 565 │ │ │ │ │ │ return module(*inputs, output_attentions, None) │\n",
"│ 566 │ │ │ │ │ │\n",
"│ 567 │ │ │ │ │ return custom_forward │\n",
"│ 568 │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in new_forward │\n",
"│ │\n",
"│ 162 │ │ │ with torch.no_grad(): │\n",
"│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 164 │ │ else: │\n",
"│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
"│ 167 │ │\n",
"│ 168 │ module.forward = new_forward │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/models/llama/modeling_llama.py:292 in │\n",
"│ forward │\n",
"│ │\n",
"│ 289 │ │ hidden_states = self.input_layernorm(hidden_states) │\n",
"│ 290 │ │ │\n",
"│ 291 │ │ # Self Attention │\n",
"│ ❱ 292 │ │ hidden_states, self_attn_weights, present_key_value = self.self_attn( │\n",
"│ 293 │ │ │ hidden_states=hidden_states, │\n",
"│ 294 │ │ │ attention_mask=attention_mask, │\n",
"│ 295 │ │ │ position_ids=position_ids, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in new_forward │\n",
"│ │\n",
"│ 162 │ │ │ with torch.no_grad(): │\n",
"│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 164 │ │ else: │\n",
"│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
"│ 167 │ │\n",
"│ 168 │ module.forward = new_forward │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/models/llama/modeling_llama.py:194 in │\n",
"│ forward │\n",
"│ │\n",
"│ 191 │ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: │\n",
"│ 192 │ │ bsz, q_len, _ = hidden_states.size() │\n",
"│ 193 │ │ │\n",
"│ ❱ 194 │ │ query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self. │\n",
"│ 195 │ │ key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.he │\n",
"│ 196 │ │ value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self. │\n",
"│ 197 │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/peft/tuners/lora.py:709 in forward │\n",
"│ │\n",
"│ 706 │ │ │ self.active_adapter = adapter_name │\n",
"│ 707 │ │ │\n",
"│ 708 │ │ def forward(self, x: torch.Tensor): │\n",
"│ ❱ 709 │ │ │ result = super().forward(x) │\n",
"│ 710 │ │ │ │\n",
"│ 711 │ │ │ if self.disable_adapters or self.active_adapter not in self.lora_A.keys(): │\n",
"│ 712 │ │ │ │ return result │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/bitsandbytes/nn/modules.py:388 in forward │\n",
"│ │\n",
"│ 385 │ │ if self.bias is not None and self.bias.dtype != x.dtype: │\n",
"│ 386 │ │ │ self.bias.data = self.bias.data.to(x.dtype) │\n",
"│ 387 │ │ │\n",
"│ ❱ 388 │ │ out = bnb.matmul(x, self.weight, bias=self.bias, state=self.state) │\n",
"│ 389 │ │ │\n",
"│ 390 │ │ if not self.state.has_fp16_weights: │\n",
"│ 391 │ │ │ if self.state.CB is not None and self.state.CxB is not None: │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:559 in matmul │\n",
"│ │\n",
"│ 556 │ state = state or MatmulLtState() │\n",
"│ 557 │ if threshold > 0.0: │\n",
"│ 558 │ │ state.threshold = threshold │\n",
"│ ❱ 559 │ return MatMul8bitLt.apply(A, B, out, bias, state) │\n",
"│ 560 │\n",
"│ 561 │\n",
"│ 562 def matmul_4bit(A: tensor, B: tensor, quant_state: List, out: tensor = None, bias=None): │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/autograd/function.py:506 in apply │\n",
"│ │\n",
"│ 503 │ │ if not torch._C._are_functorch_transforms_active(): │\n",
"│ 504 │ │ │ # See NOTE: [functorch vjp and autograd interaction] │\n",
"│ 505 │ │ │ args = _functorch.utils.unwrap_dead_wrappers(args) │\n",
"│ ❱ 506 │ │ │ return super().apply(*args, **kwargs) # type: ignore[misc] │\n",
"│ 507 │ │ │\n",
"│ 508 │ │ if cls.setup_context == _SingleLevelFunction.setup_context: │\n",
"│ 509 │ │ │ raise RuntimeError( │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:323 in forward │\n",
"│ │\n",
"│ 320 │ │ # 1. Quantize A │\n",
"│ 321 │ │ if len(A.shape) == 3: │\n",
"│ 322 │ │ │ A = A.view(-1, A.shape[-1]).contiguous() │\n",
"│ ❱ 323 │ │ CA, CAt, SCA, SCAt, coo_tensorA = F.double_quant(A.to(torch.float16), threshold= │\n",
"│ 324 │ │ │\n",
"│ 325 │ │ if state.threshold > 0.0 and coo_tensorA is not None: │\n",
"│ 326 │ │ │ if state.has_fp16_weights: │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/bitsandbytes/functional.py:2029 in double_quant │\n",
"│ │\n",
"│ 2026 │ │ │ │ ct.c_int32(rows), │\n",
"│ 2027 │ │ │ │ ct.c_int32(cols), │\n",
"│ 2028 │ │ │ ) │\n",
"│ ❱ 2029 │ │ │ val, idx = torch.sort(coo_tensor.rowidx) │\n",
"│ 2030 │ │ │ coo_tensor.rowidx = val │\n",
"│ 2031 │ │ │ coo_tensor.colidx = coo_tensor.colidx[idx] │\n",
"│ 2032 │ │ │ coo_tensor.values = coo_tensor.values[idx] │\n",
"╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
"KeyboardInterrupt\n",
"
\n"
]
},
"metadata": {}
}
]
}
]
}