dev

app.py

@@ -117,7 +117,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         try:
             tokenized_first_half = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')
             second_half = dataset['train'].select(range(half_size, train_size))
-            tokenized_second_half = tokenize_function(second_half)
+            tokenized_second_half = tokenize_function(second_half.to_dict())
             tokenized_train_dataset = concatenate_datasets([tokenized_first_half, tokenized_second_half])
             tokenized_test_dataset = tokenize_function(dataset['test'])
             
@@ -133,7 +133,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
             tokenizer = AutoTokenizer.from_pretrained('google/t5-efficient-tiny-nh8')
             # Tokenize the dataset                    
             first_half = dataset['train'].select(range(half_size))
-            tokenized_half = tokenize_function(first_half)
+            tokenized_half = tokenize_function(first_half.to_dict())
             
             tokenized_half.save_to_disk(f'/data/{hub_id.strip()}_train_dataset')