dev

app.py

@@ -115,7 +115,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         max_length = model.get_input_embeddings().weight.shape[0]
         try:
             saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')                                       
-            if os.access(f'/data/{hub_id.strip()}_validation_dataset'):                    
+            if os.path.isfile(f'/data/{hub_id.strip()}_validation_dataset'):                    
                 dataset = load_dataset(dataset_name.strip())
                 train_size = len(dataset['train'])
                 third_size = train_size // 3
@@ -133,7 +133,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 dataset['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset3')
                 return 'THIRD THIRD LOADED'
             
-            if not os.access(f'/data/{hub_id.strip()}_train_dataset3'):
+            if not os.path.isfile(f'/data/{hub_id.strip()}_train_dataset3'):
                 train_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset3')
                 if len(dataset['train']) == len(train_dataset['train']):                            
                     dataset = load_dataset(dataset_name.strip())
@@ -154,7 +154,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                     eval_dataset=saved_test_dataset,
                     compute_metrics=compute_metrics,
                 )                                      
-            if os.access(f'/data/{hub_id.strip()}_train_dataset' and not os.access(f'/data/{hub_id.strip()}_train_dataset3')):
+            if os.path.isfile(f'/data/{hub_id.strip()}_train_dataset' and not os.access(f'/data/{hub_id.strip()}_train_dataset3')):
                 dataset = load_dataset(dataset_name.strip())
                 train_size = len(dataset['train'])
                 third_size = train_size // 3