Update RAG_utils.py

RAG_utils.py

@@ -618,80 +618,42 @@ class PDFQueryEngine:
             logger.error(f"Error during query engine setup: {e}")
             raise
 
-    # def evaluate_with_llm(self, reg_result: Any, peer_result: Any, guidelines_result: Any, queries: List[str]) -> Tuple[int, List[int], int, float, List[str]]:
-    #     """
-    #     Evaluate documents using a language model based on various criteria.
-    #     Args:
-    #         reg_result (Any): Result related to registration.
-    #         peer_result (Any): Result related to peer review.
-    #         guidelines_result (Any): Result related to following guidelines.
-    #         queries (List[str]): A list of queries to be processed.
-    #     Returns:
-    #         Tuple[int, List[int], int, float, List[str]]: A tuple containing the total score, a list of scores per criteria.
-    #     """
-    
-    #     logger.info("Starting evaluation with LLM.")
-    #     self.config_manager.load_config("few_shot", "few_shot.json")
-    #     query_engine = self.setup_query_engine()
-
-    #     total_score = 0
-    #     criteria_met = 0
-    #     reasoning = []
+    def evaluate_with_llm(self, reg_result: Any, peer_result: Any, guidelines_result: Any, queries: List[str]) -> Tuple[int, List[int], int, float, List[str]]:
+        """
+        Evaluate documents using a language model based on various criteria.
+        Args:
+            reg_result (Any): Result related to registration.
+            peer_result (Any): Result related to peer review.
+            guidelines_result (Any): Result related to following guidelines.
+            queries (List[str]): A list of queries to be processed.
+        Returns:
+            Tuple[int, List[int], int, float, List[str]]: A tuple containing the total score, a list of scores per criteria.
+        """
     
-    #     for j, query in enumerate(queries):
-    #         # Handle special cases based on the value of j and other conditions
-    #         if j == 1 and reg_result:
-    #             extracted_data = {"score": 1, "reasoning": reg_result[0]}
-    #         elif j == 2 and guidelines_result:
-    #             extracted_data = {"score": 1, "reasoning": "The article is published in a journal following EQUATOR-NETWORK reporting guidelines"}
-    #         elif j == 8 and (guidelines_result or peer_result):
-    #             extracted_data = {"score": 1, "reasoning": "The article is published in a peer-reviewed journal."}
-    #         else:
-                
-    #             # Execute the query
-    #             result = query_engine.query(query).response
-    #             extracted_data = self.base_utils.extract_score_reasoning(result)
-
-
-    #         # Validate and accumulate the scores
-    #         extracted_data_score = 0 if extracted_data.get("score") is None else int(extracted_data.get("score"))
-    #         if extracted_data_score > 0:
-    #             criteria_met += 1
-    #         reasoning.append(extracted_data["reasoning"])
-    #         total_score += extracted_data_score
-
-    #     score_percentage = (float(total_score) / len(queries)) * 100
-    #     logger.info("Evaluation completed.")
-    #     return total_score, criteria_met, score_percentage, reasoning
-        
-    async def evaluate_with_llm_async(self, reg_result: Any, peer_result: Any, guidelines_result: Any, queries: List[str]) -> Tuple[int, List[int], int, float, List[str]]:
         logger.info("Starting evaluation with LLM.")
         self.config_manager.load_config("few_shot", "few_shot.json")
-        # Setup your query engine, if it's using aiohttp this is where you'd configure it
+        query_engine = self.setup_query_engine()
 
         total_score = 0
         criteria_met = 0
         reasoning = []
-
-        async def handle_query(session, j, query):
+    
+        for j, query in enumerate(queries):
+            # Handle special cases based on the value of j and other conditions
             if j == 1 and reg_result:
-                return {"score": 1, "reasoning": reg_result[0]}
+                extracted_data = {"score": 1, "reasoning": reg_result[0]}
             elif j == 2 and guidelines_result:
-                return {"score": 1, "reasoning": "The article is published in a journal following EQUATOR-NETWORK reporting guidelines"}
+                extracted_data = {"score": 1, "reasoning": "The article is published in a journal following EQUATOR-NETWORK reporting guidelines"}
             elif j == 8 and (guidelines_result or peer_result):
-                return {"score": 1, "reasoning": "The article is published in a peer-reviewed journal."}
+                extracted_data = {"score": 1, "reasoning": "The article is published in a peer-reviewed journal."}
             else:
-                # Here, adapt your query engine or direct API call to use aiohttp
-                async with session.post('Your API Endpoint', json={'query': query}) as response:
-                    result = await response.json()
-                    return self.base_utils.extract_score_reasoning(result)
+                
+                # Execute the query
+                result = query_engine.query(query).response
+                extracted_data = self.base_utils.extract_score_reasoning(result)
 
-        async with aiohttp.ClientSession() as session:
-            tasks = [handle_query(session, j, query) for j, query in enumerate(queries)]
-            results = await asyncio.gather(*tasks)
 
-        # Process results
-        for extracted_data in results:
+            # Validate and accumulate the scores
             extracted_data_score = 0 if extracted_data.get("score") is None else int(extracted_data.get("score"))
             if extracted_data_score > 0:
                 criteria_met += 1
@@ -700,8 +662,8 @@ class PDFQueryEngine:
 
         score_percentage = (float(total_score) / len(queries)) * 100
         logger.info("Evaluation completed.")
-        return total_score, criteria_met, len(queries), score_percentage, reasoning
-
+        return total_score, criteria_met, score_percentage, reasoning
+        
 
 
 class MixtralLLM(CustomLLM):