dev

app.py

@@ -111,46 +111,51 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
             return model_inputs
     
         #max_length = 512
-        # Load the dataset
-        dataset = load_dataset(dataset_name.strip())
+        # Load the dataset        
         train_size = len(dataset['train'])
         third_size = train_size // 3
         max_length = model.get_input_embeddings().weight.shape[0]
         try:
             saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')                                       
-            if 'validation' in saved_dataset.keys():
-                if 'test' in saved_dataset.keys():
+            try:
+                load_from_disk(f'/data/{hub_id.strip()}_validation_dataset')                                       
+                dataset = load_dataset(dataset_name.strip())
+                try:                    
+                    saved_test_dataset = load_from_disk(f'/data/{hub_id.strip()}_test_dataset')
                     print("FOUND TEST")
-                    dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset3')
+                    train_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset3')
                     # Create Trainer
                     trainer = Trainer(
                         model=model,
                         args=training_args,
-                        train_dataset=tokenized_train_dataset,
-                        eval_dataset=tokenized_test_dataset,
+                        train_dataset=train_dataset,
+                        eval_dataset=saved_test_dataset,
                         compute_metrics=compute_metrics,
                     )
-                else:
+                except:
+                    dataset = load_dataset(dataset_name.strip())
                     print("FOUND VALIDATION")
                     saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset2')     
                     third_third = dataset['train'].select(range(third_size*2, train_size))
                     dataset['train'] = third_third
                     tokenized_second_half = dataset.map(tokenize_function, batched=True)
                     dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_second_half['train']])
-                    tokenized_train_dataset = dataset['train']
-                    tokenized_test_dataset = dataset['test']
-                    dataset.save_to_disk(f'/data/{hub_id.strip()}_train_dataset3')
+                    dataset['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset3')
+                    dataset['test'].save_to_disk(f'/data/{hub_id.strip()}_test_dataset')
                     return 'THIRD THIRD LOADED'
-            else:
+            except:
+                dataset = load_dataset(dataset_name.strip())
                 second_third = dataset['train'].select(range(third_size, third_size*2))
                 dataset['train'] = second_third
                 del dataset['test']
                 tokenized_sh_fq_dataset = dataset.map(tokenize_function, batched=True)
                 dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_sh_fq_dataset['train']])
-                dataset.save_to_disk(f'/data/{hub_id.strip()}_train_dataset2')
+                dataset['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset2')
+                dataset['validation'].save_to_disk(f'/data/{hub_id.strip()}_validation_dataset')
                 return 'SECOND THIRD LOADED'
                 
-        except:                                            
+        except:
+            dataset = load_dataset(dataset_name.strip())
             # Tokenize the dataset                    
             first_third = dataset['train'].select(range(third_size))
             dataset['train'] = first_third