remove similarity_threshold

app.py

@@ -78,7 +78,7 @@ def upsert_data(dataset_split):
 upsert_data(dataset['train'])
 
 # Define retrieval function with similarity threshold
-def retrieve_relevant_text(input_text, similarity_threshold=1.0):  # Lower threshold to capture more results
+def retrieve_relevant_text(input_text):
     input_embedding = embed_text([input_text])[0]
     results = collection.query(
         query_embeddings=[input_embedding],
@@ -90,20 +90,20 @@ def retrieve_relevant_text(input_text, similarity_threshold=1.0):  # Lower thres
     #print("Retrieved items and their similarity scores:")
     for metadata, distance in zip(results['metadatas'][0], results['distances'][0]):
         #print(f"Code: {metadata['code']}, Similarity Score: {distance}")
-        if distance <= similarity_threshold:
-            output.append({
-                "similarity_score": distance,
-                "code": metadata['code'],
-                "code_system": metadata['code_system'],
-                "description": metadata['description']
-            })
+        #if distance <= similarity_threshold:
+        output.append({
+            "similarity_score": distance,
+            "code": metadata['code'],
+            "code_system": metadata['code_system'],
+            "description": metadata['description']
+        })
     
-    if not output:
-        print("No results met the similarity threshold.")
+    # if not output:
+    #     print("No results met the similarity threshold.")
     return output
 
 # Evaluate retrieval efficiency on the validation/test set
-def evaluate_efficiency(dataset_split, similarity_threshold=1.0):
+def evaluate_efficiency(dataset_split):
     y_true = []
     y_pred = []
     total_similarity = 0
@@ -115,7 +115,7 @@ def evaluate_efficiency(dataset_split, similarity_threshold=1.0):
             annotations_list = [annotation['code'] for annotation in note.get('annotations', []) if 'code' in annotation]
             
             if text and annotations_list:
-                retrieved_results = retrieve_relevant_text(text, similarity_threshold=similarity_threshold)
+                retrieved_results = retrieve_relevant_text(text)
                 retrieved_codes = [result['code'] for result in retrieved_results]
                 
                 # Sum up similarity scores for average calculation
@@ -153,7 +153,7 @@ def evaluate_efficiency(dataset_split, similarity_threshold=1.0):
     return precision, recall, f1, avg_similarity
 
 # Calculate retrieval efficiency metrics
-precision, recall, f1, avg_similarity = evaluate_efficiency(dataset['validation'], similarity_threshold=1.0)
+precision, recall, f1, avg_similarity = evaluate_efficiency(dataset['validation'])
 
 # Gradio interface
 def gradio_interface(input_text):