dev

app.py

@@ -116,7 +116,18 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         max_length = model.get_input_embeddings().weight.shape[0]
         try:
             saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')
-            if 'test' in saved_dataset.keys():                
+            if 'test' in saved_dataset.key():
+                print("FOUND TEST")
+                # Create Trainer
+                trainer = Trainer(
+                    model=model,
+                    args=training_args,
+                    train_dataset=tokenized_train_dataset,
+                    eval_dataset=tokenized_test_dataset,
+                    compute_metrics=compute_metrics,
+                )                   
+            elif 'validation' in saved_dataset.keys():
+                print("FOUND VALIDATION")
                 third_third = dataset['train'].select(range(third_size*2, train_size))
                 dataset['train'] = third_third
                 tokenized_second_half = dataset.map(tokenize_function, batched=True)
@@ -125,20 +136,13 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 tokenized_test_dataset = dataset['test']
             else:
                 second_third = dataset['train'].select(range(third_size, third_size*2))
-                dataset['train'] = second_third                
+                dataset['train'] = second_third
+                del dataset['test']
                 tokenized_sh_fq_dataset = tokenize_function(dataset, batched=True)
                 dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_sh_fq_dataset['train']])
-                tokenized_half.save_to_disk(f'/data/{hub_id.strip()}_train_dataset')
+                dataset.save_to_disk(f'/data/{hub_id.strip()}_train_dataset')
                 return
-            
-            # Create Trainer
-            trainer = Trainer(
-                model=model,
-                args=training_args,
-                train_dataset=tokenized_train_dataset,
-                eval_dataset=tokenized_test_dataset,
-                compute_metrics=compute_metrics,
-            )            
+                
         except:                                            
             tokenizer = AutoTokenizer.from_pretrained('google/t5-efficient-tiny-nh8')
             # Tokenize the dataset