deve

app.py

@@ -95,8 +95,8 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         #if os.path.exists(training_args.output_dir) and os.listdir(training_args.output_dir):
             #print("Loading model from checkpoint...")
             #model = AutoModelForSeq2SeqLM.from_pretrained(training_args.output_dir)        
-    
-        tokenizer = AutoTokenizer.from_pretrained('google/t5-efficient-tiny', use_fast=True, trust_remote_code=True)
+        config = AutoConfig.from_pretrained("google/t5-efficient-tiny-nh8")
+        tokenizer = AutoTokenizer.from_pretrained('google/t5-efficient-tiny-nh8', use_fast=True, trust_remote_code=True)
     
         #max_length = model.get_input_embeddings().weight.shape[0]
         max_length = 512
@@ -268,7 +268,7 @@ def run_train(dataset_name, hub_id, api_key, num_epochs, batch_size, lr, grad):
             elif 'encoder.block.0.layer.0.DenseReluDense.wo.weight' in name:  # Another example layer
                 torch.nn.init.kaiming_normal_(param.data)  # Kaiming initialization
     
-    config = AutoConfig.from_pretrained("google/t5-efficient-tiny")
+    config = AutoConfig.from_pretrained("google/t5-efficient-tiny-nh8")
     model = AutoModelForSeq2SeqLM.from_config(config)
     initialize_weights(model)
     lora_config = LoraConfig(