Change validation logic

validate.py

@@ -1,4 +1,3 @@
-from collections import defaultdict
 from dotenv import load_dotenv
 from langchain_google_genai import ChatGoogleGenerativeAI
 from langchain_openai import ChatOpenAI
@@ -21,7 +20,7 @@ class Validation():
         else:
             self.llm = ChatOpenAI(temperature=0, model_name=llm, api_key=os.environ['PERPLEXITY_API_KEY'], base_url="https://api.perplexity.ai")
 
-    def validate(self, df, api):
+    def validate(self, df, text, api):
 
         df = df.fillna('')
         df['Genes'] = df['Genes'].str.replace(' ', '').str.upper()
@@ -83,6 +82,7 @@ class Validation():
         df.reset_index(drop=True, inplace=True)
         df_clean = df.copy()
 
+        # WARNING: DEPRECATED
         # Validate genes and SNPs with APIs
         if api:
             dbsnp = {}
@@ -110,14 +110,34 @@ class Validation():
                     else:
                         df = df.drop(i)
 
+        # WARNING: DEPRECATED
         # Check with GWAS ground truth
+        if False:
+            for i in df.index:
+                gene = df.loc[i, 'Genes']
+                snp = df.loc[i, 'rsID']
+                perms = permutate(gene)
+
+                for perm in perms:
+                    if perm in ground_truth and snp in ground_truth[perm]:
+                        df.loc[i, 'Genes'] = perm
+                        if gene != perm:
+                            print(f'{gene} corrected to {perm} with {snp}')
+                        else:
+                            print(f'{gene} and {snp} safe')
+                        break
+                else:
+                    print(f'{gene} and {snp} not found')
+                    df = df.drop(i)
+
+        # Check with Text
         for i in df.index:
             gene = df.loc[i, 'Genes']
             snp = df.loc[i, 'rsID']
             perms = permutate(gene)
 
             for perm in perms:
-                if perm in ground_truth and snp in ground_truth[perm]:
+                if perm in text and snp in text:
                     df.loc[i, 'Genes'] = perm
                     if gene != perm:
                         print(f'{gene} corrected to {perm} with {snp}')
@@ -128,6 +148,25 @@ class Validation():
                 print(f'{gene} and {snp} not found')
                 df = df.drop(i)
 
+        # Drop (duplicate) entries with empty values
+        genes = []
+        snps = []
+        for i in df.index:
+            gene = df.loc[i, 'Genes']
+            snp = df.loc[i, 'rsID']
+
+            if len(gene) == 0 and len(snp) == 0:
+                df = df.drop(i)
+            elif len(gene) == 0:
+                if snp in snps:
+                    df = df.drop(i)
+            elif len(snp) == 0:
+                if gene in genes:
+                    df = df.drop(i)
+            else:
+                genes.append(gene)
+                snps.append(snp)
+
         df.reset_index(drop=True, inplace=True)
 
         return df, df_clean