WIP

app.py

@@ -45,8 +45,7 @@ with st.container(): # user input
     col1, col2 = st.columns((6, 1))
 
     with col1:
-        description_input = st.text_area(label="Enter a disease description 👇", placeholder='A disease that causes memory loss and other cognitive impairments.')
-
+        description_input = st.text_area(label="Enter a disease description 👇", placeholder='A disorder manifested in memory loss and other cognitive impairments among elderly patients (60+ years old), especially women.')
     with col2:
         st.text('') # dummy to center vertically
         st.text('') # dummy to center vertically
@@ -67,25 +66,34 @@ with st.container():
             )
             status.info(f'Found {len(diseases_related_to_the_user_text)} diseases related to the description you entered.')
             status.json(diseases_related_to_the_user_text, expanded=False)
+            status.divider()
             # 3. Get the similarities of the embeddings of those diseases (cosine similarity of the embeddings of the nodes of such diseases)
             status.write("Getting the similarities among the diseases to filter out less promising ones...")
             diseases_uris = [disease["uri"] for disease in diseases_related_to_the_user_text]
-            get_similarities_among_diseases_uris(diseases_uris)
+            similarities = get_similarities_among_diseases_uris(diseases_uris)
+            status.info(f'Obtained similarity information among the diseases by measuring the cosine similarity of their embeddings. Using the similarity information to filter out less promising diseases.')
+            status.json(similarities, expanded=False)
+            status.divider()
             # 4. Potentially filter out the diseases that are not similar enough (e.g. similarity < 0.8)
             # 5. Augment the set of diseases: add new diseases that are similar to the ones that are already in the set, until we get 10-15 diseases
             status.write("Augmenting the set of diseases by finding others with related embeddings...")
             augmented_set_of_diseases = augment_the_set_of_diseaces(diseases_uris)
             # print(augmented_set_of_diseases)
+            status.info(f'Augmented set of diseases: {len(augmented_set_of_diseases)} diseases.')
             # 6. Query the embeddings of the diseases related to each clinical trial (also in the DB), to get the most similar clinical trials to our set of diseases
             status.write("Getting the clinical trials related to the diseases found...")
             clinical_trials_related_to_the_diseases = get_clinical_trials_related_to_diseases(
                 augmented_set_of_diseases, encoder
             )
+            status.info(f'Found {len(clinical_trials_related_to_the_diseases)} clinical trials related to the diseases.')
+            status.json(clinical_trials_related_to_the_diseases, expanded=False)
+            status.divider()
             status.write("Getting the details of the clinical trials...")
             json_of_clinical_trials = get_clinical_records_by_ids(
                 [trial["nct_id"] for trial in clinical_trials_related_to_the_diseases]
             )
             status.json(json_of_clinical_trials, expanded=False)
+            status.divider()
             # 7. Use an LLM to get a summary of the clinical trials, in plain text format.
             status.write("Getting a summary of the clinical trials...")
             response, stats_dict = get_short_summary_out_of_json_files(json_of_clinical_trials)
@@ -109,9 +117,9 @@ with st.container():
         st.info(
             """This is a graph of the relevant diseases that we found, based on the description that you entered. The diseases are connected by edges if they are similar to each other. The color of the edges represents the similarity of the diseases.
                 
-We use the embeddings of the diseases to determine the similarity between them. The embeddings are generated using a Representation Learning algorithm that learns the topological relations among the nodes in the graph, depending on how they are connected. We utilize the (PyKeen)[https://github.com/pykeen/pykeen] implementation of TransH to train an embedding model.
+We use the embeddings of the diseases to determine the similarity between them. The embeddings are generated using a Representation Learning algorithm that learns the topological relations among the nodes in the graph, depending on how they are connected. We utilize the [PyKeen](https://github.com/pykeen/pykeen) implementation of TransH to train an embedding model.
 
-(TransH)[https://ojs.aaai.org/index.php/AAAI/article/view/8870] utilizes hyperplanes to model relations between entities. It is a multi-relational model that can handle many-to-many relations between entities. The model is trained on the triples of the graph, where the triples are the subject, relation, and object of the graph. The model learns the embeddings of the entities and the relations, such that the embeddings of the subject and object are close to each other when the relation is true.
+[TransH](https://ojs.aaai.org/index.php/AAAI/article/view/8870) utilizes hyperplanes to model relations between entities. It is a multi-relational model that can handle many-to-many relations between entities. The model is trained on the triples of the graph, where the triples are the subject, relation, and object of the graph. The model learns the embeddings of the entities and the relations, such that the embeddings of the subject and object are close to each other when the relation is true.
 
 Specifically, it optimizes the following cost function:
 $$"""

llm_res.py

@@ -221,7 +221,7 @@ def process_dictionaty_with_llm_to_generate_response(json_data):
     return filtered_data
 
 def get_short_summary_out_of_json_files(data_json):
-#     prompt_template = """ You are an expert clinician working on the analysis of reports of clinical trials.
+    prompt_template = """You are an expert clinician working on the analysis of reports of clinical trials.
 
 #       # Task
 #       You will be given a set of descriptions of clinical trials. Your job is to come up with a short summary (100-200 words) of the descriptions of the clinical trials. Your users are clinical researchers who are experts in medicine, so you should be technical and specific, including scientific terms. Always be faithful to the original information written in the reports.

utils.py

@@ -125,15 +125,15 @@ def get_similarities_among_diseases_uris(
 
 
 def augment_the_set_of_diseaces(diseases: List[str]) -> str:
-    print(diseases)
-    for i in range(15-len(diseases)):
+    augmented_diseases = diseases.copy()
+    for i in range(15-len(augmented_diseases)):
         with engine.connect() as conn:
             with conn.begin():
                 sql = f"""
-                    SELECT TOP 1 e2.uri AS new_disease, (SUM(VECTOR_COSINE(e1.embedding, e2.embedding))/ {len(diseases)})  AS score
+                    SELECT TOP 1 e2.uri AS new_disease, (SUM(VECTOR_COSINE(e1.embedding, e2.embedding))/ {len(augmented_diseases)})  AS score
                     FROM Test.EntityEmbeddings e1, Test.EntityEmbeddings e2
-                    WHERE e1.uri IN ({','.join([f"'{disease}'" for disease in diseases])})
-                    AND e2.uri NOT IN ({','.join([f"'{disease}'" for disease in diseases])})
+                    WHERE e1.uri IN ({','.join([f"'{disease}'" for disease in augmented_diseases])})
+                    AND e2.uri NOT IN ({','.join([f"'{disease}'" for disease in augmented_diseases])})
                     AND e2.label != 'nan'
                     GROUP BY e2.label
                     ORDER BY score DESC
@@ -142,9 +142,9 @@ def augment_the_set_of_diseaces(diseases: List[str]) -> str:
                 result = conn.execute(text(sql))
                 data = result.fetchall()
 
-        diseases.append(data[0][0].split('/')[-1])
+        augmented_diseases.append(data[0][0])
 
-    return diseases
+    return augmented_diseases
 
 def get_embedding(string: str, encoder) -> List[float]:
     # Embed the string using sentence-transformers
@@ -162,14 +162,14 @@ def get_diseases_related_to_a_textual_description(
     with engine.connect() as conn:
         with conn.begin():
             sql = f"""
-                    SELECT TOP 5 d.uri, VECTOR_COSINE(d.embedding, TO_VECTOR('{string_representation}', DOUBLE)) AS distance
+                    SELECT TOP 10 d.uri, VECTOR_COSINE(d.embedding, TO_VECTOR('{string_representation}', DOUBLE)) AS distance
                     FROM Test.DiseaseDescriptions d
                     ORDER BY distance DESC
                 """
             result = conn.execute(text(sql))
             data = result.fetchall()
 
-    return [{"uri": row[0], "distance": row[1]} for row in data]
+    return [{"uri": row[0], "distance": row[1]} for row in data if row[1] > 0.8]
 
 def get_clinical_trials_related_to_diseases(
     diseases: List[str], encoder