dev

app.py

@@ -112,14 +112,11 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
     
         #max_length = 512
         # Load the dataset        
-        train_size = len(dataset['train'])
-        third_size = train_size // 3
         max_length = model.get_input_embeddings().weight.shape[0]
         try:
             saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')                                       
             try:
-                load_from_disk(f'/data/{hub_id.strip()}_validation_dataset')                                       
-                dataset = load_dataset(dataset_name.strip())
+                load_from_disk(f'/data/{hub_id.strip()}_validation_dataset')                                                      
                 try:                    
                     saved_test_dataset = load_from_disk(f'/data/{hub_id.strip()}_test_dataset')
                     print("FOUND TEST")
@@ -134,6 +131,8 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                     )
                 except:
                     dataset = load_dataset(dataset_name.strip())
+                    train_size = len(dataset['train'])
+                    third_size = train_size // 3                    
                     print("FOUND VALIDATION")
                     saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset2')     
                     third_third = dataset['train'].select(range(third_size*2, train_size))
@@ -145,6 +144,8 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                     return 'THIRD THIRD LOADED'
             except:
                 dataset = load_dataset(dataset_name.strip())
+                train_size = len(dataset['train'])
+                third_size = train_size // 3                
                 second_third = dataset['train'].select(range(third_size, third_size*2))
                 dataset['train'] = second_third
                 del dataset['test']
@@ -156,6 +157,8 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 
         except:
             dataset = load_dataset(dataset_name.strip())
+            train_size = len(dataset['train'])
+            third_size = train_size // 3            
             # Tokenize the dataset                    
             first_third = dataset['train'].select(range(third_size))
             dataset['train'] = first_third