Update README.md

README.md

@@ -1,3 +1,103 @@
----
-license: mit
----
+---
+license: mit
+datasets:
+- open-thoughts/OpenThoughts-114k
+language:
+- ar
+metrics:
+- code_eval
+base_model:
+- deepseek-ai/DeepSeek-R1
+new_version: deepseek-ai/DeepSeek-R1
+library_name: adapter-transformers
+tags:
+- code
+---
+---
+license: mit
+---# Step 1: Install required libraries
+!pip install transformers datasets torch sentencepiece
+
+# Step 2: Import Libraries
+from datasets import load_dataset
+from transformers import MarianMTModel, MarianTokenizer
+import torch
+from transformers import Trainer, TrainingArguments
+
+# Step 3: Load the Dataset
+dataset = load_dataset(cfilt/iitb-engl"ish-hindi")
+
+# Check the structure of the dataset
+print(dataset)
+
+# Step 4: Prepare Tokenizer and Model
+model_name = "Helsinki-NLP/opus-mt-en-hi"
+tokenizer = MarianTokenizer.from_pretrained(model_name)
+model = MarianMTModel.from_pretrained(model_name)
+
+# Step 5: Preprocess the Dataset
+def preprocess_function(examples):
+    # Tokenize the English input and Hindi target
+    model_inputs = tokenizer(examples["en"], truncation=True, padding="max_length", max_length=128)
+    # Tokenize the Hindi target for training
+    with tokenizer.as_target_tokenizer():
+        labels = tokenizer(examples["hi"], truncation=True, padding="max_length", max_length=128)
+    
+    model_inputs["labels"] = labels["input_ids"]
+    return model_inputs
+
+# Apply preprocessing to the dataset
+tokenized_datasets = dataset.map(preprocess_function, batched=True)
+
+# Step 6: Training the Model
+training_args = TrainingArguments(
+    output_dir="./results",          # output directory for results
+    evaluation_strategy="epoch",     # evaluate after every epoch
+    learning_rate=2e-5,              # learning rate
+    per_device_train_batch_size=16,  # batch size for training
+    per_device_eval_batch_size=16,   # batch size for evaluation
+    num_train_epochs=3,              # number of training epochs
+    logging_dir="./logs",            # directory for storing logs
+    save_steps=500,                  # save checkpoint every 500 steps
+)
+
+# Initialize the Trainer
+trainer = Trainer(
+    model=model,                         # the pre-trained model
+    args=training_args,                  # training arguments
+    train_dataset=tokenized_datasets["train"],   # training dataset
+    eval_dataset=tokenized_datasets["validation"],  # validation dataset
+)
+
+# Train the model
+trainer.train()
+
+# Step 7: Evaluate the Model
+results = trainer.evaluate(tokenized_datasets["test"])
+print("Evaluation Results:", results)
+
+# Step 8: Translate Text Using the Model
+def translate(texts):
+    # Tokenize the input English text
+    inputs = tokenizer(texts, return_tensors="pt", padding=True, truncation=True)
+    
+    # Generate the translation (output of the model)
+    with torch.no_grad():
+        translated = model.generate(**inputs)
+    
+    # Decode the generated ids back into text
+    translations = tokenizer.decode(translated[0], skip_special_tokens=True)
+    return translations
+
+# Example translation
+english_text = ["Hello, how are you?", "I am learning NLP."]
+translations = translate(english_text)
+print(translations)
+
+# Step 9: Save the Model and Tokenizer
+model.save_pretrained("./hindi_translation_model")
+tokenizer.save_pretrained("./hindi_translation_tokenizer")
+
+# Step 10: Load the model and tokenizer for future use
+model = MarianMTModel.from_pretrained("./hindi_translation_model")
+tokenizer = MarianTokenizer.from_pretrained("./hindi_translation_tokenizer")