deve

app.py

@@ -115,15 +115,16 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
             )
         
             # Setup the decoder input IDs (shifted right)
-            labels = tokenizer(
-                examples['target'], 
-                max_length=max_length,  # Set to None for dynamic padding
-                truncation=True,
-                padding='max_length',
-                #text_target=examples['target'],
-                #return_tensors='pt',
-                #padding=True, 
-            )
+            with tokenizer.as_target_tokenizer():
+                labels = tokenizer(
+                    examples['target'], 
+                    max_length=max_length,  # Set to None for dynamic padding
+                    truncation=True,
+                    padding='max_length',
+                    #text_target=examples['target'],
+                    #return_tensors='pt',
+                    #padding=True, 
+                )
             #labels["input_ids"] = [
              #   [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels["input_ids"]
             #]        
@@ -150,7 +151,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                     train_dataset=train_dataset,
                     eval_dataset=saved_test_dataset,
                     #compute_metrics=compute_metrics,
-                    #data_collator=data_collator,
+                    data_collator=data_collator,
                     #processing_class=tokenizer,
                 )