dev

app.py

@@ -236,14 +236,18 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 print(f'{entry}: {current_dir}')
             except:
                 pass
+        
+        def get_checkpoint_int(s):
+            int_index = s.find('-')
+            return int(s[int_index+1:])
+            
         try:
             train_result = trainer.train(resume_from_checkpoint=True)
         except:
             checkpoint_dir = training_args.output_dir
             # If the trainer_state.json is missing, look for the previous checkpoint
-            print(f"Checkpoint {checkpoint_dir} is missing 'trainer_state.json'. Looking for previous checkpoints...")
-            previous_checkpoints = sorted(glob.glob(os.path.join(os.path.dirname(checkpoint_dir), 'checkpoint-*')), key=os.path.getmtime)
-            print(previous_checkpoints)
+            previous_checkpoints = sorted(os.listdir("/data/results"), key=get_checkpoint_int)
+
             if previous_checkpoints:
                 # Load the most recent previous checkpoint
                 last_checkpoint = previous_checkpoints[-2]