dev

app.py

@@ -246,16 +246,19 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         except:
             checkpoint_dir = training_args.output_dir
             # If the trainer_state.json is missing, look for the previous checkpoint
-            previous_checkpoints = sorted(os.listdir("/data/results"), key=get_checkpoint_int)
+            previous_checkpoints = sorted(os.listdir("/data/results"), key=get_checkpoint_int, reverse=True)
 
-            if previous_checkpoints:
-                # Load the most recent previous checkpoint
-                last_checkpoint = previous_checkpoints[-2]
-                print(f"Loading previous checkpoint: {last_checkpoint}")
-                train_result = trainer.train(resume_from_checkpoint=last_checkpoint)
-            else:
-                print("No previous checkpoints found. Starting training from scratch.")
-                train_result = trainer.train()
+            for check in previous_checkpoints[1:]:
+                try:                        
+                    print(f"Loading previous checkpoint: {check}")
+                    train_result = trainer.train(resume_from_checkpoint=check)
+                    trainer.push_to_hub(commit_message="Training complete!")
+                    return 'DONE!'#train_result
+                except:
+                    pass
+
+            print("No previous checkpoints found. Starting training from scratch.")
+            train_result = trainer.train()
         trainer.push_to_hub(commit_message="Training complete!")
     except Exception as e:
         return f"An error occurred: {str(e)}, TB: {traceback.format_exc()}"