Upload 24 files

.github/ci_cd_pipeline.yml

@@ -0,0 +1,76 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Run unit tests
+        run: |
+          python -m unittest discover tests
+
+      - name: Train Full Fine-Tuning model
+        run: |
+          python scripts/train_full_finetune.py
+        env:
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+
+      - name: Train PERT model
+        run: |
+          python scripts/train_pert.py
+        env:
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+
+      - name: Train LoRA model
+        run: |
+          python scripts/train_lora.py
+        env:
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+
+      - name: Distill Student model
+        run: |
+          python scripts/distill_student.py
+        env:
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+
+      - name: Prompt Engineering Evaluation
+        run: |
+          python evaluations/prompt_engineering.py
+        env:
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+
+      - name: RAG Evaluation using LangChain
+        run: |
+          python langchain/rag.py
+
+      - name: Evaluate models
+        run: |
+          python scripts/evaluate.py
+
+      - name: Deploy model
+        if: ${{ success() }}
+        run: |
+          python scripts/deploy_model.py
+        env:
+          HF_AUTH_TOKEN: ${{ secrets.HF_AUTH_TOKEN }}

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.8-slim
+
+WORKDIR /app
+
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+COPY . .
+
+CMD ["python", "app.py"]

app.py

@@ -0,0 +1,16 @@
+from flask import Flask, jsonify, request
+import mlflow.pyfunc
+
+app = Flask(__name__)
+
+# Load the model as a PyFuncModel.
+model = mlflow.pyfunc.load_model(model_uri="models:/deployed_model/1")
+
+@app.route('/predict', methods=['POST'])
+def predict():
+    data = request.get_json()
+    predictions = model.predict(data)
+    return jsonify(predictions.tolist())
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000)

config/config.yaml

@@ -0,0 +1,38 @@
+training:
+  num_epochs: 3
+  batch_size: 16
+  learning_rate: 2e-5
+
+dataset:
+  name: imdb
+  split: train[:10%]
+
+model:
+  adapter:
+    reduction_factor: 16
+  lora:
+    r: 4
+    alpha: 32
+  student:
+    hidden_size: 384
+
+evaluation:
+  models:
+    - bert-base-uncased
+    - distilbert-base-uncased
+    - roberta-base
+    - gpt2
+    - bart-base
+    - electra-small-discriminator
+    - t5-small
+    - xlm-roberta-base
+    - albert-base-v2
+    - xlnet-base-cased
+    - deberta-base
+    - camembert-base
+    - marianmt-en-de
+    - m2m100_418M
+
+wandb:
+  project: fine_tuning_comparison
+  entity: your_wandb_username

data/dataset.py

@@ -0,0 +1,46 @@
+from datasets import load_dataset
+from transformers import BertTokenizer
+
+def load_and_tokenize_data(config):
+    """
+    Load and tokenize data based on the provided configuration.
+
+    Args:
+        config (dict): Configuration dictionary containing dataset and tokenizer details.
+
+    Returns:
+        tuple: A tuple containing the tokenized train and test datasets.
+    """
+    # Load the dataset
+    dataset = load_dataset(config['dataset']['name'], split=config['dataset']['split'])
+    dataset = dataset.train_test_split(test_size=0.2)
+    train_dataset = dataset['train']
+    test_dataset = dataset['test']
+
+    # Initialize the tokenizer
+    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+
+    # Define the tokenization function
+    def tokenize_function(examples):
+        return tokenizer(examples['text'], padding='max_length', truncation=True)
+
+    # Apply tokenization to the train and test datasets
+    train_dataset = train_dataset.map(tokenize_function, batched=True)
+    test_dataset = test_dataset.map(tokenize_function, batched=True)
+
+    # Set the format to PyTorch tensors
+    train_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])
+    test_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])
+
+    return train_dataset, test_dataset
+
+# Example usage
+if __name__ == "__main__":
+    config = {
+        'dataset': {
+            'name': 'imdb',
+            'split': 'train[:10%]'
+        }
+    }
+    train_dataset, test_dataset = load_and_tokenize_data(config)
+    print("Train dataset and Test dataset have been loaded and tokenized successfully.")

docker-compose.yml

@@ -0,0 +1,39 @@
+training:
+  num_epochs: 3
+  batch_size: 16
+  learning_rate: 2e-5
+
+dataset:
+  name: imdb
+  split: train[:10%]
+
+model:
+  adapter:
+    reduction_factor: 16
+  lora:
+    r: 4
+    alpha: 32
+  student:
+    hidden_size: 384
+
+evaluation:
+  models:
+    - bert-base-uncased
+    - distilbert-base-uncased
+    - roberta-base
+    - gpt2
+    - bart-base
+    - electra-small-discriminator
+    - t5-small
+    - xlm-roberta-base
+    - albert-base-v2
+    - xlnet-base-cased
+    - deberta-base
+    - camembert-base
+    - marianmt-en-de
+    - m2m100_418M
+
+wandb:
+  project: fine_tuning_comparison
+  entity: your_wandb_username
+

evaluations/evaluate_models.py

@@ -0,0 +1,62 @@
+import wandb
+import yaml
+from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments
+from data.datasets import load_and_tokenize_data
+from utils.monitor import measure_resources
+
+# Charger la configuration
+with open('config/config.yaml', 'r') as f:
+    config = yaml.safe_load(f)
+
+# Initialiser wandb
+wandb.init(project=config['wandb']['project'], entity=config['wandb']['entity'])
+
+# Charger les donn�es
+train_dataset, test_dataset = load_and_tokenize_data(config)
+
+def evaluate_model(model_name):
+    # Charger le mod�le et le tokenizer
+    model = AutoModelForSequenceClassification.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+    # Tokenizer les donn�es
+    train_dataset = train_dataset.map(lambda x: tokenizer(x['text'], padding='max_length', truncation=True), batched=True)
+    test_dataset = test_dataset.map(lambda x: tokenizer(x['text'], padding='max_length', truncation=True), batched=True)
+
+    # D�finir les arguments de formation
+    training_args = TrainingArguments(
+        output_dir=f'./results/{model_name}',
+        num_train_epochs=config['training']['num_epochs'],
+        per_device_train_batch_size=config['training']['batch_size'],
+        per_device_eval_batch_size=config['training']['batch_size'],
+        evaluation_strategy='epoch',
+        save_steps=10_000,
+        save_total_limit=2,
+        logging_dir='./logs',
+        logging_steps=10,
+    )
+
+    # Cr�er le Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=test_dataset,
+    )
+
+    # Mesurer les ressources et �valuer le mod�le
+    peak_memory, training_time = measure_resources(trainer, model_name)
+
+    # �valuation des performances
+    metrics = trainer.evaluate()
+
+    wandb.log({
+        'model_name': model_name,
+        'peak_memory_MB': peak_memory,
+        'training_time_seconds': training_time,
+        **metrics
+    })
+
+# �valuer chaque mod�le
+for model_name in config['evaluation']['models']:
+    evaluate_model(model_name)

evaluations/prompt_engineering.py

@@ -0,0 +1,46 @@
+import wandb
+import yaml
+from transformers import pipeline
+
+# Charger la configuration
+with open('config/config.yaml', 'r') as f:
+    config = yaml.safe_load(f)
+
+# Initialiser wandb
+wandb.init(project=config['wandb']['project'], entity=config['wandb']['entity'])
+
+# Charger le mod�le fine-tuned
+model_name = "results_student"  # Remplacer par le chemin vers le mod�le student
+tokenizer_name = "distilbert-base-uncased"
+
+# Configuration du pipeline
+nlp = pipeline("text-classification", model=model_name, tokenizer=tokenizer_name)
+
+# Simuler des exemples pour l'�valuation
+examples = [
+    {"reference": "This is a great movie.", "candidate": "This is a fantastic movie."},
+    {"reference": "I love this film.", "candidate": "I enjoy this movie."}
+]
+
+def evaluate_prompt(example, shots=0):
+    prompt = example["candidate"]
+    if shots == 1:
+        prompt = "Classify the sentiment of the following text: " + prompt
+    elif shots > 1:
+        prompt = "Classify the sentiment of the following text based on these examples:\n" + \
+                 "Example: This movie is terrible. -> Negative\n" + \
+                 "Example: I love this movie. -> Positive\n" + \
+                 prompt
+
+    result = nlp(prompt)[0]
+    return result
+
+# �valuer les prompts
+for example in examples:
+    for shots in [0, 1, 5]:
+        result = evaluate_prompt(example, shots)
+        wandb.log({
+            'example': example['candidate'],
+            'shots': shots,
+            'result': result
+        })

evaluations/rag.py

@@ -0,0 +1,29 @@
+rom langchain.chains import RAGChain
+from langchain.llms import HuggingFace
+from langchain.retrievers import BM25Retriever
+from langchain.prompts import PromptTemplate
+import yaml
+
+# Charger la configuration
+with open('config/config.yaml', 'r') as f:
+    config = yaml.safe_load(f)
+
+# Configuration du mod�le
+llm = HuggingFace("distilbert-base-uncased")
+
+# Configuration du retriever
+retriever = BM25Retriever.from_documents(["This is a great movie.", "I love this film."])
+
+# Cr�ation du template de prompt
+template = PromptTemplate("Classify the sentiment of the following text: {text}")
+
+# Cr�ation de la cha�ne RAG
+rag_chain = RAGChain(llm=llm, retriever=retriever, prompt_template=template)
+
+# Exemples de textes � classifier
+texts = ["This is a fantastic movie.", "I enjoy this movie."]
+
+# Utiliser RAG pour obtenir des classifications avec contexte
+for text in texts:
+    result = rag_chain.run({"text": text})
+    print(f"Text: {text}, Result: {result}")

models/full_finetune_model.py

@@ -0,0 +1,5 @@
+from transformers import BertForSequenceClassification
+
+def get_full_finetune_model():
+    model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
+    return model

models/lora_model.py

@@ -0,0 +1,8 @@
+from transformers import BertForSequenceClassification, LoRAConfig
+
+def get_lora_model(config):
+    model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
+    lora_config = LoRAConfig(r=config['model']['lora']['r'], alpha=config['model']['lora']['alpha'])
+    model.add_lora('imdb_lora', config=lora_config)
+    model.train_lora('imdb_lora')
+    return model

models/pert_model.py

@@ -0,0 +1,8 @@
+from transformers import BertForSequenceClassification, AdapterConfig
+
+def get_pert_model(config):
+    model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
+    adapter_config = AdapterConfig(mh_adapter=True, output_adapter=True, reduction_factor=config['model']['adapter']['reduction_factor'])
+    model.add_adapter('imdb_adapter', config=adapter_config)
+    model.train_adapter('imdb_adapter')
+    return model

models/student_model.py

@@ -0,0 +1,6 @@
+from transformers import DistilBertForSequenceClassification
+
+def get_student_model(config):
+    model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')
+    model.config.hidden_size = config['model']['student']['hidden_size']
+    return model

requirements.txt

@@ -0,0 +1,37 @@
+transformers
+accelerate
+torch
+datasets
+wandb
+pyyaml
+langchain
+flask
+mlflow
+streamlit
+streamlit==1.32.1
+flask
+mlflow
+fastapi==0.74.* 
+requests==2.27.* 
+uvicorn[standard]==0.17.* 
+sentencepiece==0.2.* 
+python-dotenv
+wikipedia
+tiktoken
+neo4j
+streamlit
+Pillow
+fastapi
+PyPDF2
+pydantic
+uvicorn
+sse-starlette
+boto3
+streamlit==1.32.1
+# missing from the langchain base image?
+langchain-openai==0.2.4
+langchain-community==0.3.3
+langchain-google-genai==2.0.3
+langchain-ollama==0.2.0
+langchain-huggingface==0.1.1
+langchain-aws==0.2.4

scripts/deploy_model.py

@@ -0,0 +1,16 @@
+from huggingface_hub import HfApi
+import os
+
+def deploy_model_to_huggingface(model_path, model_name):
+    api = HfApi()
+    api.upload_folder(
+        folder_path=model_path,
+        repo_id=model_name,
+        commit_message="Deploying fine-tuned model",
+        use_auth_token=os.getenv("HF_AUTH_TOKEN")
+    )
+
+if __name__ == "__main__":
+    model_path = "./results_student"
+    model_name = "your_model_name"
+    deploy_model_to_huggingface(model_path, model_name)

scripts/distill_student.py

@@ -0,0 +1,63 @@
+import torch
+import wandb
+import yaml
+from transformers import Trainer, TrainingArguments
+from data.datasets import load_and_tokenize_data
+from models.full_finetune_model import get_full_finetune_model
+from models.student_model import get_student_model
+
+# Charger la configuration
+with open('config/config.yaml', 'r') as f:
+    config = yaml.safe_load(f)
+
+# Initialiser wandb
+wandb.init(project=config['wandb']['project'], entity=config['wandb']['entity'])
+
+# Charger les donn�es
+train_dataset, test_dataset = load_and_tokenize_data(config)
+
+# Charger le mod�le teacher et le mod�le student
+teacher_model = get_full_finetune_model()
+student_model = get_student_model(config)
+
+# D�finir les arguments de formation pour la distillation
+training_args = TrainingArguments(
+    output_dir='./results_student',
+    num_train_epochs=config['training']['num_epochs'],
+    per_device_train_batch_size=config['training']['batch_size'],
+    per_device_eval_batch_size=config['training']['batch_size'],
+    evaluation_strategy='epoch',
+    save_steps=10_000,
+    save_total_limit=2,
+    logging_dir='./logs',
+    logging_steps=10,
+)
+
+# D�finir le distillateur
+class DistillationTrainer(Trainer):
+    def compute_loss(self, model, inputs, return_outputs=False):
+        # Forward pass of teacher model
+        with torch.no_grad():
+            teacher_outputs = teacher_model(**inputs)
+
+        # Forward pass of student model
+        student_outputs = model(**inputs)
+
+        # Compute distillation loss
+        loss = torch.nn.functional.kl_div(
+            torch.nn.functional.log_softmax(student_outputs.logits, dim=-1),
+            torch.nn.functional.softmax(teacher_outputs.logits, dim=-1),
+            reduction='batchmean'
+        )
+        return (loss, student_outputs) if return_outputs else loss
+
+# Cr�er le Trainer pour la distillation
+trainer = DistillationTrainer(
+    model=student_model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=test_dataset,
+)
+
+# Mesurer les ressources et entra�ner le mod�le student
+measure_resources(trainer, "Distillation")

scripts/evaluate.py

@@ -0,0 +1,44 @@
+import wandb
+from datasets import load_metric
+from transformers import pipeline
+import yaml
+
+# Charger la configuration
+with open('config/config.yaml', 'r') as f:
+    config = yaml.safe_load(f)
+
+# Charger le mod�le fine-tuned
+model_name = "results_student"  # Remplacer par le chemin vers le mod�le student
+tokenizer_name = "distilbert-base-uncased"
+
+# Configuration de l'�valuation
+bleu = load_metric("bleu")
+rouge = load_metric("rouge")
+
+# Initialiser wandb
+wandb.init(project=config['wandb']['project'], entity=config['wandb']['entity'])
+
+def evaluate_model(model_name, tokenizer_name):
+    nlp = pipeline("text-classification", model=model_name, tokenizer=tokenizer_name)
+    
+    # Simuler des exemples pour l'�valuation
+    examples = [
+        {"reference": "This is a great movie.", "candidate": "This is a fantastic movie."},
+        {"reference": "I love this film.", "candidate": "I enjoy this movie."}
+    ]
+    
+    references = [e["reference"] for e in examples]
+    candidates = [nlp(e["candidate"])[0]["label"] for e in examples]
+    
+    # Calcul des scores BLEU et ROUGE
+    bleu_score = bleu.compute(predictions=candidates, references=references)
+    rouge_score = rouge.compute(predictions=candidates, references=references)
+    
+    # Enregistrer les scores sur wandb
+    wandb.log({
+        "bleu_score": bleu_score,
+        "rouge_score": rouge_score
+    })
+
+# �valuer les mod�les
+evaluate_model(model_name, tokenizer_name)

scripts/train_full_finetune.py

@@ -0,0 +1,42 @@
+import wandb
+import yaml
+from transformers import Trainer, TrainingArguments
+from utils.monitor import measure_resources
+from data.datasets import load_and_tokenize_data
+from models.full_finetune_model import get_full_finetune_model
+
+# Charger la configuration
+with open('config/config.yaml', 'r') as f:
+    config = yaml.safe_load(f)
+
+# Initialiser wandb
+wandb.init(project=config['wandb']['project'], entity=config['wandb']['entity'])
+
+# Charger les donn�es
+train_dataset, test_dataset = load_and_tokenize_data(config)
+
+# Charger le mod�le
+model = get_full_finetune_model()
+
+# D�finir les arguments de formation
+training_args = TrainingArguments(
+    output_dir='./results',
+    num_train_epochs=config['training']['num_epochs'],
+    per_device_train_batch_size=config['training']['batch_size'],
+    per_device_eval_batch_size=config['training']['batch_size'],
+    evaluation_strategy='epoch',
+    save_steps=10_000,
+    save_total_limit=2,
+    logging_dir='./logs',
+    logging_steps=10,
+)
+
+# Cr�er le Trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=test_dataset,
+)
+#Mesurer les ressources et entra�ner le mod�le
+measure_resources(trainer, "Full Fine-Tuning")

scripts/train_lora.py

@@ -0,0 +1,43 @@
+import wandb
+import yaml
+from transformers import Trainer, TrainingArguments
+from utils.monitor import measure_resources
+from data.datasets import load_and_tokenize_data
+from models.lora_model import get_lora_model
+
+# Charger la configuration
+with open('config/config.yaml', 'r') as f:
+    config = yaml.safe_load(f)
+
+# Initialiser wandb
+wandb.init(project=config['wandb']['project'], entity=config['wandb']['entity'])
+
+# Charger les donn�es
+train_dataset, test_dataset = load_and_tokenize_data(config)
+
+# Charger le mod�le
+model = get_lora_model(config)
+
+# D�finir les arguments de formation
+training_args = TrainingArguments(
+    output_dir='./results',
+    num_train_epochs=config['training']['num_epochs'],
+    per_device_train_batch_size=config['training']['batch_size'],
+    per_device_eval_batch_size=config['training']['batch_size'],
+    evaluation_strategy='epoch',
+    save_steps=10_000,
+    save_total_limit=2,
+    logging_dir='./logs',
+    logging_steps=10,
+)
+
+# Cr�er le Trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=test_dataset,
+)
+
+# Mesurer les ressources et entra�ner le mod�le
+measure_resources(trainer, "LoRA")

scripts/train_pert.py

@@ -0,0 +1,43 @@
+import wandb
+import yaml
+from transformers import Trainer, TrainingArguments
+from utils.monitor import measure_resources
+from data.datasets import load_and_tokenize_data
+from models.pert_model import get_pert_model
+
+# Charger la configuration
+with open('config/config.yaml', 'r') as f:
+    config = yaml.safe_load(f)
+
+# Initialiser wandb
+wandb.init(project=config['wandb']['project'], entity=config['wandb']['entity'])
+
+# Charger les donn�es
+train_dataset, test_dataset = load_and_tokenize_data(config)
+
+# Charger le mod�le
+model = get_pert_model(config)
+
+# D�finir les arguments de formation
+training_args = TrainingArguments(
+    output_dir='./results',
+    num_train_epochs=config['training']['num_epochs'],
+    per_device_train_batch_size=config['training']['batch_size'],
+    per_device_eval_batch_size=config['training']['batch_size'],
+    evaluation_strategy='epoch',
+    save_steps=10_000,
+    save_total_limit=2,
+    logging_dir='./logs',
+    logging_steps=10,
+)
+
+# Cr�er le Trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=test_dataset,
+)
+
+# Mesurer les ressources et entra�ner le mod�le
+measure_resources(trainer, "PERT (Adapters)")

tests/test_datasets.py

@@ -0,0 +1,16 @@
+import unittest
+from data.datasets import load_dataset, preprocess_data
+
+class TestDatasets(unittest.TestCase):
+
+    def test_load_dataset(self):
+        df = load_dataset('data/test.csv')
+        self.assertIsNotNone(df)
+
+    def test_preprocess_data(self):
+        df = load_dataset('data/test.csv')
+        preprocessed_df = preprocess_data(df)
+        self.assertFalse(preprocessed_df.isnull().values.any())
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_metrics.py

@@ -0,0 +1,13 @@
+import unittest
+from evaluations.evaluate_models import evaluate_model
+
+class TestMetrics(unittest.TestCase):
+
+    def test_evaluate_model(self):
+        model = train_full_finetune_model(train_data, val_data)
+        results = evaluate_model(model, val_data)
+        self.assertIn('accuracy', results)
+        self.assertIn('f1_score', results)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_models.py

@@ -0,0 +1,13 @@
+import unittest
+from models.full_finetune_model import train_full_finetune_model
+
+class TestModels(unittest.TestCase):
+
+    def test_train_full_finetune_model(self):
+        train_data = preprocess_data(load_dataset('data/train.csv'))
+        val_data = preprocess_data(load_dataset('data/val.csv'))
+        model = train_full_finetune_model(train_data, val_data)
+        self.assertIsNotNone(model)
+
+if __name__ == '__main__':
+    unittest.main()

utils/monitor.py

@@ -0,0 +1,21 @@
+import time
+import torch
+
+def measure_resources(trainer, method_name):
+    start_time = time.time()
+    torch.cuda.reset_peak_memory_stats()
+    
+    # D�marrer l'entra�nement et enregistrer les m�triques
+    trainer.train()
+    
+    end_time = time.time()
+    peak_memory = torch.cuda.max_memory_allocated() / (1024 * 1024)  # Convertir en MB
+    training_time = end_time - start_time
+    
+    # Enregistrer les m�triques sur wandb
+    wandb.log({
+        'method': method_name,
+        'peak_memory_MB': peak_memory,
+        'training_time_seconds': training_time
+    })
+    return peak_memory, training_time