dev

app.py

@@ -82,8 +82,8 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
             print("Loading model from checkpoint...")
             model = AutoModelForSeq2SeqLM.from_pretrained(training_args.output_dir)        
     
-        #max_length = 128
-        max_length = model.get_input_embeddings().weight.shape[0]
+        max_length = 128
+        #max_length = model.get_input_embeddings().weight.shape[0]
         try:
             tokenized_train_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')
             tokenized_test_dataset = load_from_disk(f'/data/{hub_id.strip()}_test_dataset')
@@ -129,6 +129,10 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
             tokenized_datasets['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset')
             tokenized_datasets['test'].save_to_disk(f'/data/{hub_id.strip()}_test_dataset')
         
+            embedding_size = model.get_input_embeddings().weight.shape[0]
+            if len(tokenizer) > embedding_size:
+                model.resize_token_embeddings(len(tokenizer))            
+        
             # Create Trainer
             trainer = Trainer(
                 model=model,