Upload generation_test_llmware_script.py

generation_test_llmware_script.py

@@ -11,7 +11,7 @@ def load_rag_benchmark_tester_ds():
 
     dataset = load_dataset(ds_name)
 
-    print("update: loading test dataset - ", dataset)
+    print("update: loading RAG Benchmark test dataset - ", dataset)
 
     test_set = []
     for i, samples in enumerate(dataset["train"]):
@@ -25,9 +25,10 @@ def load_rag_benchmark_tester_ds():
 
 def run_test(model_name, prompt_list):
 
-    print("\nupdate: Starting RAG Benchmark Inference Test")
+    print("\nupdate: Starting RAG Benchmark Inference Test - ", model_name)
 
-    prompter = Prompt().load_model(model_name,from_hf=True)
+    # pull DRAGON / BLING model directly from catalog, e.g., no from_hf=True
+    prompter = Prompt().load_model(model_name)
 
     for i, entries in enumerate(prompt_list):
 
@@ -36,21 +37,25 @@ def run_test(model_name, prompt_list):
 
         response = prompter.prompt_main(prompt,context=context,prompt_name="default_with_context", temperature=0.3)
 
+        print("\nupdate: model inference output - ", i, response["llm_response"])
+        print("update: gold_answer              - ", i, entries["answer"])
+
         fc = prompter.evidence_check_numbers(response)
         sc = prompter.evidence_comparison_stats(response)
         sr = prompter.evidence_check_sources(response)
 
-        print("\nupdate: model inference output - ", i, response["llm_response"])
-        print("update: gold_answer              - ", i, entries["answer"])
+        print("\nFact-Checking Tools")
 
         for entries in fc:
-            print("update: fact check - ", entries["fact_check"])
+            for f, facts in enumerate(entries["fact_check"]):
+                print("update: fact check - ", f, facts)
 
         for entries in sc:
             print("update: comparison stats - ", entries["comparison_stats"])
 
         for entries in sr:
-            print("update: sources - ", entries["source_review"])
+            for s, sources in enumerate(entries["source_review"]):
+                print("update: sources - ", s, sources)
 
     return 0
 
@@ -59,6 +64,7 @@ if __name__ == "__main__":
 
     core_test_set = load_rag_benchmark_tester_ds()
 
-    model_name = "llmware/dragon-red-pajama-7b-v0"
-    
-    output = run_test(model_name, core_test_set)
+    # one of the 7 gpu dragon models
+    gpu_model_name = "llmware/dragon-red-pajama-7b-v0"
+
+    output = run_test(gpu_model_name, core_test_set)