How to append new token and train?
#70
by
NickyNicky
- opened
my code is and error train
def smart_tokenizer_and_embedding_resize( # https://github.com/artidoro/qlora/blob/main/qlora.py
special_tokens_dict,
tokenizer,
model,
):
"""Resize tokenizer and embedding.
Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
"""
num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)
print("We have add_special_tokens:", num_new_tokens, "tokens")
old_embeddings= model.get_input_embeddings()
print("old_embeddings: ",old_embeddings.weight.data.shape)
# model.resize_token_embeddings(len(tokenizer)+num_new_tokens) #aqui elimina como 12 embeddings pero es el codigo original
model.resize_token_embeddings(old_embeddings.weight.data.shape[0]+num_new_tokens) # implementacion mia donde no elimina los embeddings
print("model: ",model.get_input_embeddings().weight.data.shape)
if num_new_tokens > 0:
input_embeddings = model.get_input_embeddings().weight.data
output_embeddings = model.get_output_embeddings().weight.data
print("input_embeddings:",input_embeddings.shape)
print("output_embeddings:",output_embeddings.shape)
# print("")
# print("output_embeddings:",output_embeddings)
# print("-->>",input_embeddings[:-num_new_tokens]).mean(dim=0, keepdim=True)
input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)
output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)
print("input_embeddings_avg:",input_embeddings_avg.shape)
print("output_embeddings_avg:",output_embeddings_avg.shape)
input_embeddings[-num_new_tokens:] = input_embeddings_avg
output_embeddings[-num_new_tokens:] = output_embeddings_avg
print("")
print(model)
# {'text': " Who made you?<|endoftext|>\n<bot>: I'm h2oGPT, a large language model by H2O.ai, the visionary leader in democratizing AI.\n<|endoftext|>"}
dic = {
'additional_special_tokens': ['<human>:','<bot>:']
}
smart_tokenizer_and_embedding_resize(
special_tokens_dict= dic,
tokenizer=tokenizer,
model=model,
)
########## DATASET
Dataset({
features: ['input_ids', 'attention_mask', 'labels'],
num_rows: 12332
})
########## TRAIN
from torch.optim import AdamW
# optimizer = AdamW(model.parameters(), lr=1e-3)
# Crear el optimizador
# optimizer = AdamW(model.parameters(), lr=1e-4)
training_args = transformers.TrainingArguments(
per_device_train_batch_size=6, # es el tamaño del lote de entrenamiento por dispositivo.
per_device_eval_batch_size=6, # es el tamaño del lote de evaluación por dispositivo.
gradient_accumulation_steps=3,
logging_steps=4,
num_train_epochs=1,
learning_rate= 1e-3, # #2e-8, 3e-5, 1e-5, 2e-5, (1e-3), 1e-2
# fp16=False,
# # fp16=True, #True,
# auto_find_batch_size=False, # Desactivar la búsqueda automática del tamaño de lote
fp16=True,
auto_find_batch_size=True, # Desactivar la búsqueda automática del tamaño de lote
save_total_limit=4,
output_dir="./outputs",
optim="adamw_torch",
lr_scheduler_type = 'cosine',
warmup_ratio = 0.03, #0.05,
)
trainer = transformers.Trainer(
model=model,
train_dataset=lm_dataset,
# eval_dataset=val_dataset,
args=training_args,
data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),
)
# Configurar el optimizador en el objeto Trainer
# trainer.optimizer = optimizer
trainer.train()
########## Error
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ in <cell line: 49>:49 │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:1645 in train │
│ │
│ 1642 │ │ inner_training_loop = find_executable_batch_size( │
│ 1643 │ │ │ self._inner_training_loop, self._train_batch_size, args.auto_find_batch_size │
│ 1644 │ │ ) │
│ ❱ 1645 │ │ return inner_training_loop( │
│ 1646 │ │ │ args=args, │
│ 1647 │ │ │ resume_from_checkpoint=resume_from_checkpoint, │
│ 1648 │ │ │ trial=trial, │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/memory.py:132 in decorator │
│ │
│ 129 │ │ │ if batch_size == 0: │
│ 130 │ │ │ │ raise RuntimeError("No executable batch size found, reached zero.") │
│ 131 │ │ │ try: │
│ ❱ 132 │ │ │ │ return function(batch_size, *args, **kwargs) │
│ 133 │ │ │ except Exception as e: │
│ 134 │ │ │ │ if should_reduce_batch_size(e): │
│ 135 │ │ │ │ │ gc.collect() │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:1938 in _inner_training_loop │
│ │
│ 1935 │ │ │ │ │ self.control = self.callback_handler.on_step_begin(args, self.state, │
│ 1936 │ │ │ │ │
│ 1937 │ │ │ │ with self.accelerator.accumulate(model): │
│ ❱ 1938 │ │ │ │ │ tr_loss_step = self.training_step(model, inputs) │
│ 1939 │ │ │ │ │
│ 1940 │ │ │ │ if ( │
│ 1941 │ │ │ │ │ args.logging_nan_inf_filter │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2759 in training_step │
│ │
│ 2756 │ │ │ return loss_mb.reduce_mean().detach().to(self.args.device) │
│ 2757 │ │ │
│ 2758 │ │ with self.compute_loss_context_manager(): │
│ ❱ 2759 │ │ │ loss = self.compute_loss(model, inputs) │
│ 2760 │ │ │
│ 2761 │ │ if self.args.n_gpu > 1: │
│ 2762 │ │ │ loss = loss.mean() # mean() to average on multi-gpu parallel training │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2784 in compute_loss │
│ │
│ 2781 │ │ │ labels = inputs.pop("labels") │
│ 2782 │ │ else: │
│ 2783 │ │ │ labels = None │
│ ❱ 2784 │ │ outputs = model(**inputs) │
│ 2785 │ │ # Save past state if it exists │
│ 2786 │ │ # TODO: this needs to be fixed and made cleaner later. │
│ 2787 │ │ if self.args.past_index >= 0: │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1194 in _call_impl │
│ │
│ 1191 │ │ # this function, and just call forward. │
│ 1192 │ │ if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks o │
│ 1193 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1194 │ │ │ return forward_call(*input, **kwargs) │
│ 1195 │ │ # Do not call functions when jit is used │
│ 1196 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1197 │ │ if self._backward_hooks or _global_backward_hooks: │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:553 in forward │
│ │
│ 550 │ model_forward = ConvertOutputsToFp32(model_forward) │
│ 551 │ │
│ 552 │ def forward(*args, **kwargs): │
│ ❱ 553 │ │ return model_forward(*args, **kwargs) │
│ 554 │ │
│ 555 │ # To act like a decorator so that it can be popped when doing `extract_model_from_pa │
│ 556 │ forward.__wrapped__ = model_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:541 in __call__ │
│ │
│ 538 │ │ update_wrapper(self, model_forward) │
│ 539 │ │
│ 540 │ def __call__(self, *args, **kwargs): │
│ ❱ 541 │ │ return convert_to_fp32(self.model_forward(*args, **kwargs)) │
│ 542 │ │
│ 543 │ def __getstate__(self): │
│ 544 │ │ raise pickle.PicklingError( │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/amp/autocast_mode.py:14 in decorate_autocast │
│ │
│ 11 │ @functools.wraps(func) │
│ 12 │ def decorate_autocast(*args, **kwargs): │
│ 13 │ │ with autocast_instance: │
│ ❱ 14 │ │ │ return func(*args, **kwargs) │
│ 15 │ decorate_autocast.__script_unsupported = '@autocast() decorator is not supported in │
│ 16 │ return decorate_autocast │
│ 17 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:553 in forward │
│ │
│ 550 │ model_forward = ConvertOutputsToFp32(model_forward) │
│ 551 │ │
│ 552 │ def forward(*args, **kwargs): │
│ ❱ 553 │ │ return model_forward(*args, **kwargs) │
│ 554 │ │
│ 555 │ # To act like a decorator so that it can be popped when doing `extract_model_from_pa │
│ 556 │ forward.__wrapped__ = model_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:541 in __call__ │
│ │
│ 538 │ │ update_wrapper(self, model_forward) │
│ 539 │ │
│ 540 │ def __call__(self, *args, **kwargs): │
│ ❱ 541 │ │ return convert_to_fp32(self.model_forward(*args, **kwargs)) │
│ 542 │ │
│ 543 │ def __getstate__(self): │
│ 544 │ │ raise pickle.PicklingError( │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/amp/autocast_mode.py:14 in decorate_autocast │
│ │
│ 11 │ @functools.wraps(func) │
│ 12 │ def decorate_autocast(*args, **kwargs): │
│ 13 │ │ with autocast_instance: │
│ ❱ 14 │ │ │ return func(*args, **kwargs) │
│ 15 │ decorate_autocast.__script_unsupported = '@autocast() decorator is not supported in │
│ 16 │ return decorate_autocast │
│ 17 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:553 in forward │
│ │
│ 550 │ model_forward = ConvertOutputsToFp32(model_forward) │
│ 551 │ │
│ 552 │ def forward(*args, **kwargs): │
│ ❱ 553 │ │ return model_forward(*args, **kwargs) │
│ 554 │ │
│ 555 │ # To act like a decorator so that it can be popped when doing `extract_model_from_pa │
│ 556 │ forward.__wrapped__ = model_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:541 in __call__ │
│ │
│ 538 │ │ update_wrapper(self, model_forward) │
│ 539 │ │
│ 540 │ def __call__(self, *args, **kwargs): │
│ ❱ 541 │ │ return convert_to_fp32(self.model_forward(*args, **kwargs)) │
│ 542 │ │
│ 543 │ def __getstate__(self): │
│ 544 │ │ raise pickle.PicklingError( │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/amp/autocast_mode.py:14 in decorate_autocast │
│ │
│ 11 │ @functools.wraps(func) │
│ 12 │ def decorate_autocast(*args, **kwargs): │
│ 13 │ │ with autocast_instance: │
│ ❱ 14 │ │ │ return func(*args, **kwargs) │
│ 15 │ decorate_autocast.__script_unsupported = '@autocast() decorator is not supported in │
│ 16 │ return decorate_autocast │
│ 17 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:553 in forward │
│ │
│ 550 │ model_forward = ConvertOutputsToFp32(model_forward) │
│ 551 │ │
│ 552 │ def forward(*args, **kwargs): │
│ ❱ 553 │ │ return model_forward(*args, **kwargs) │
│ 554 │ │
│ 555 │ # To act like a decorator so that it can be popped when doing `extract_model_from_pa │
│ 556 │ forward.__wrapped__ = model_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:541 in __call__ │
│ │
│ 538 │ │ update_wrapper(self, model_forward) │
│ 539 │ │
│ 540 │ def __call__(self, *args, **kwargs): │
│ ❱ 541 │ │ return convert_to_fp32(self.model_forward(*args, **kwargs)) │
│ 542 │ │
│ 543 │ def __getstate__(self): │
│ 544 │ │ raise pickle.PicklingError( │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/amp/autocast_mode.py:14 in decorate_autocast │
│ │
│ 11 │ @functools.wraps(func) │
│ 12 │ def decorate_autocast(*args, **kwargs): │
│ 13 │ │ with autocast_instance: │
│ ❱ 14 │ │ │ return func(*args, **kwargs) │
│ 15 │ decorate_autocast.__script_unsupported = '@autocast() decorator is not supported in │
│ 16 │ return decorate_autocast │
│ 17 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:553 in forward │
│ │
│ 550 │ model_forward = ConvertOutputsToFp32(model_forward) │
│ 551 │ │
│ 552 │ def forward(*args, **kwargs): │
│ ❱ 553 │ │ return model_forward(*args, **kwargs) │
│ 554 │ │
│ 555 │ # To act like a decorator so that it can be popped when doing `extract_model_from_pa │
│ 556 │ forward.__wrapped__ = model_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:541 in __call__ │
│ │
│ 538 │ │ update_wrapper(self, model_forward) │
│ 539 │ │
│ 540 │ def __call__(self, *args, **kwargs): │
│ ❱ 541 │ │ return convert_to_fp32(self.model_forward(*args, **kwargs)) │
│ 542 │ │
│ 543 │ def __getstate__(self): │
│ 544 │ │ raise pickle.PicklingError( │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/amp/autocast_mode.py:14 in decorate_autocast │
│ │
│ 11 │ @functools.wraps(func) │
│ 12 │ def decorate_autocast(*args, **kwargs): │
│ 13 │ │ with autocast_instance: │
│ ❱ 14 │ │ │ return func(*args, **kwargs) │
│ 15 │ decorate_autocast.__script_unsupported = '@autocast() decorator is not supported in │
│ 16 │ return decorate_autocast │
│ 17 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:553 in forward │
│ │
│ 550 │ model_forward = ConvertOutputsToFp32(model_forward) │
│ 551 │ │
│ 552 │ def forward(*args, **kwargs): │
│ ❱ 553 │ │ return model_forward(*args, **kwargs) │
│ 554 │ │
│ 555 │ # To act like a decorator so that it can be popped when doing `extract_model_from_pa │
│ 556 │ forward.__wrapped__ = model_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py:541 in __call__ │
│ │
│ 538 │ │ update_wrapper(self, model_forward) │
│ 539 │ │
│ 540 │ def __call__(self, *args, **kwargs): │
│ ❱ 541 │ │ return convert_to_fp32(self.model_forward(*args, **kwargs)) │
│ 542 │ │
│ 543 │ def __getstate__(self): │
│ 544 │ │ raise pickle.PicklingError( │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/amp/autocast_mode.py:14 in decorate_autocast │
│ │
│ 11 │ @functools.wraps(func) │
│ 12 │ def decorate_autocast(*args, **kwargs): │
│ 13 │ │ with autocast_instance: │
│ ❱ 14 │ │ │ return func(*args, **kwargs) │
│ 15 │ decorate_autocast.__script_unsupported = '@autocast() decorator is not supported in │
│ 16 │ return decorate_autocast │
│ 17 │
│ │
│ /usr/local/lib/python3.10/dist-packages/peft/peft_model.py:847 in forward │
│ │
│ 844 │ │ │ if self.base_model.config.model_type == "mpt": │
│ 845 │ │ │ │ if inputs_embeds is not None: │
│ 846 │ │ │ │ │ raise AssertionError("forward in MPTForCausalLM does not support inp │
│ ❱ 847 │ │ │ │ return self.base_model( │
│ 848 │ │ │ │ │ input_ids=input_ids, │
│ 849 │ │ │ │ │ attention_mask=attention_mask, │
│ 850 │ │ │ │ │ labels=labels, │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1194 in _call_impl │
│ │
│ 1191 │ │ # this function, and just call forward. │
│ 1192 │ │ if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks o │
│ 1193 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1194 │ │ │ return forward_call(*input, **kwargs) │
│ 1195 │ │ # Do not call functions when jit is used │
│ 1196 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1197 │ │ if self._backward_hooks or _global_backward_hooks: │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in new_forward │
│ │
│ 162 │ │ │ with torch.no_grad(): │
│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │
│ 164 │ │ else: │
│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │
│ 166 │ │ return module._hf_hook.post_forward(module, output) │
│ 167 │ │
│ 168 │ module.forward = new_forward │
│ │
│ /root/.cache/huggingface/modules/transformers_modules/mosaicml/mpt-7b-instruct/1fc4634127ec64a45 │
│ 716003578b9cfae23265849/modeling_mpt.py:271 in forward │
│ │
│ 268 │ │ if inputs_embeds is not None: │
│ 269 │ │ │ raise NotImplementedError('inputs_embeds has to be None (for hf/peft support │
│ 270 │ │ outputs = self.transformer(input_ids=input_ids, past_key_values=past_key_values, │
│ ❱ 271 │ │ logits = self.transformer.wte(outputs.last_hidden_state.to(self.transformer.wte. │
│ 272 │ │ if self.logit_scale is not None: │
│ 273 │ │ │ if self.logit_scale == 0: │
│ 274 │ │ │ │ warnings.warn(f'Multiplying logits by self.logit_scale={self.logit_scale │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1194 in _call_impl │
│ │
│ 1191 │ │ # this function, and just call forward. │
│ 1192 │ │ if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks o │
│ 1193 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1194 │ │ │ return forward_call(*input, **kwargs) │
│ 1195 │ │ # Do not call functions when jit is used │
│ 1196 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1197 │ │ if self._backward_hooks or _global_backward_hooks: │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
TypeError: Embedding.forward() takes 2 positional arguments but 3 were given
HI, I got the same error
I got the same error too :(