First Commit

.dockerignore

@@ -0,0 +1,7 @@
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+*.db
+*.sqlite3
+*.log

Dockerfile

@@ -0,0 +1,46 @@
+# Use the official Python 3.9 image
+FROM python:3.9-slim
+
+# Create a non-root user
+RUN useradd -m appuser
+
+# Set the working directory inside the container
+WORKDIR /app
+
+# Set environment variables for cache
+ENV TRANSFORMERS_CACHE=/app/cache/huggingface/transformers
+ENV HF_HOME=/app/cache/huggingface
+ENV SENTENCE_TRANSFORMERS_HOME=/app/cache/sentence_transformers
+
+# Ensure the cache directory can be written to
+RUN mkdir -p /app/cache/huggingface/transformers && \
+    mkdir -p /app/cache/sentence_transformers && \
+    chown -R appuser:appuser /app/cache
+
+# Copy the requirements file into the container
+COPY requirements.txt ./requirements.txt
+
+# Install system dependencies and Python packages
+RUN apt-get update && \
+    apt-get -y install gcc libpq-dev && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Copy the entire project into the container
+COPY . /app
+
+# Switch to the non-root user
+USER appuser
+
+# Create a script to load the models
+RUN echo "from sentence_transformers import SentenceTransformer; \
+    SentenceTransformer('sentence-transformers/msmarco-distilbert-base-v3'); \
+    SentenceTransformer('sentence-transformers/all-mpnet-base-v2'); \
+    SentenceTransformer('sentence-transformers/paraphrase-MiniLM-L6-v2');" > load_models.py
+
+# Run the model loading script
+RUN python load_models.py
+# RUN python -c "SentenceTransformer('sentence-transformers/paraphrase-MiniLM-L6-v2');"
+# Start the application
+# CMD ["python", "semantic_search.py"]
+# CMD ["gunicorn","-b" , "0.0.0.0:7860", "semantic_search:app"]
+CMD ["uvicorn", "semantic_search:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,13 +1,12 @@
 ---
-title: Semantic Eval
-emoji: 🐢
-colorFrom: yellow
-colorTo: yellow
-sdk: gradio
-sdk_version: 5.10.0
-app_file: app.py
+title: Semantic Search Evaluation
+emoji: 🌖
+colorFrom: indigo
+colorTo: red
+sdk: docker
 pinned: false
 short_description: Evaluation of semantic search
+app_port: 8080
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

requirements.txt

@@ -0,0 +1,5 @@
+flask
+sentence_transformers
+pandas
+# gunicorn
+uvicorn

semantic_search.py

@@ -0,0 +1,94 @@
+from flask import Flask, request, jsonify
+from sentence_transformers import SentenceTransformer, util
+import pandas as pd
+import json
+app = Flask(__name__)
+class EnhancedSemanticSearchEvaluator:
+    def __init__(self, relevance_threshold=3, top_k=300, similarity_threshold=0.5):
+        self.models = {
+            "Model_1": SentenceTransformer('sentence-transformers/msmarco-distilbert-base-v3'),
+            "Model_2": SentenceTransformer('sentence-transformers/all-mpnet-base-v2'),
+            "Model_3": SentenceTransformer('sentence-transformers/paraphrase-MiniLM-L6-v2')
+        }
+        self.relevance_threshold = relevance_threshold
+        self.top_k = top_k
+        self.similarity_threshold = similarity_threshold
+
+    def compute_similarity(self, model, query, matches):
+        query_embedding = model.encode(query, convert_to_tensor=True)
+        match_embeddings = model.encode(
+            [match['metadata'] for match in matches], convert_to_tensor=True
+        )
+        scores = util.pytorch_cos_sim(query_embedding, match_embeddings).squeeze(0).tolist()
+        return scores
+
+    def rank_results(self, model, query, matches):
+        similarity_scores = self.compute_similarity(model, query, matches)
+        for match, score in zip(matches, similarity_scores):
+            match['similarity_score'] = score
+        ranked_matches = sorted(matches, key=lambda x: x['similarity_score'], reverse=True)
+        return ranked_matches
+
+    def evaluate_results(self, query, results):
+        all_metrics = {}
+        results_status = {}
+
+        for model_name, model in self.models.items():
+            ranked_matches = self.rank_results(model, query, results['matches'])
+
+            results_with_scores = []
+            for rank, match in enumerate(ranked_matches[:self.top_k], start=1):
+                doc_id = match['id']
+                similarity_score = match['similarity_score']
+
+                if similarity_score >= 0.7:
+                    llm_score = 5
+                elif similarity_score >= 0.5:
+                    llm_score = 4
+                elif similarity_score >= 0.3:
+                    llm_score = 3
+                elif similarity_score >= 0.1:
+                    llm_score = 2
+                else:
+                    llm_score = 1
+
+                results_with_scores.append({
+                    "Rank": rank,
+                    "Document ID": doc_id,
+                    "Similarity Score": similarity_score,
+                    "LLM Score": llm_score
+                })
+
+            results_df = pd.DataFrame(results_with_scores)
+            results_df['Pass'] = results_df['LLM Score'] >= self.relevance_threshold
+
+            pass_rate = results_df['Pass'].mean()
+            metrics = {
+                "Pass Rate": pass_rate,
+                "Precision@K": results_df.head(self.top_k)['Pass'].mean(),
+                "Recall@K": results_df.head(self.top_k)['Pass'].sum() / max(results_df['Pass'].sum(), 1),
+                "F1@K": (
+                    2 * (results_df.head(self.top_k)['Pass'].mean() * (results_df.head(self.top_k)['Pass'].sum() / max(results_df['Pass'].sum(), 1))) /
+                    (results_df.head(self.top_k)['Pass'].mean() + (results_df.head(self.top_k)['Pass'].sum() / max(results_df['Pass'].sum(), 1)))
+                    if (results_df.head(self.top_k)['Pass'].mean() + (results_df.head(self.top_k)['Pass'].sum() / max(results_df['Pass'].sum(), 1))) > 0 else 0)
+                
+            }
+
+            all_metrics[model_name] = metrics
+            results_status[model_name] = "Test Passed" if pass_rate > 0.5 else "Test Failed"
+
+        return results_status
+
+evaluator = EnhancedSemanticSearchEvaluator()
+
+@app.route('/evaluate', methods=['POST'])
+def evaluate():
+    content = request.json
+    query = content['query']
+    results = content['results']
+    evaluator = EnhancedSemanticSearchEvaluator()
+    evaluation_result = evaluator.evaluate_results(query, results)
+    return jsonify(evaluation_result)
+
+# if __name__ == '__main__':
+#     app.run(debug=True, host='0.0.0.0', port=8000)