fixed bug for eval-mgtv-llama3_8b.sh

llm_toolkit/eval_logical_reasoning.py

@@ -23,7 +23,11 @@ load_in_4bit = os.getenv("LOAD_IN_4BIT") == "true"
 data_path = os.getenv("LOGICAL_REASONING_DATA_PATH")
 results_path = os.getenv("LOGICAL_REASONING_RESULTS_PATH")
 use_english_datasets = os.getenv("USE_ENGLISH_DATASETS") == "true"
-dtype = torch.bfloat16 if os.getenv("USE_BF16_FOR_INFERENCE") == "true" else torch.float16  
+using_p1 = os.getenv("USING_P1_PROMPT_TEMPLATE") == "true"
+
+dtype = (
+    torch.bfloat16 if os.getenv("USE_BF16_FOR_INFERENCE") == "true" else torch.float16
+)
 
 print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path)
 
@@ -34,7 +38,10 @@ print(f"(1) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
 print(f"{start_gpu_memory} GB of memory reserved.")
 
 model, tokenizer = load_model(
-    model_name, load_in_4bit=load_in_4bit, adapter_name_or_path=adapter_name_or_path, dtype=dtype
+    model_name,
+    load_in_4bit=load_in_4bit,
+    adapter_name_or_path=adapter_name_or_path,
+    dtype=dtype,
 )
 
 gpu_stats = torch.cuda.get_device_properties(0)
@@ -44,13 +51,16 @@ print(f"(2) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
 print(f"{start_gpu_memory} GB of memory reserved.")
 
 datasets = load_logical_reasoning_dataset(
-    data_path, tokenizer=tokenizer, chinese_prompt=not use_english_datasets
+    data_path,
+    tokenizer=tokenizer,
+    chinese_prompt=not use_english_datasets,
+    using_p1=using_p1,
 )
+
 if len(sys.argv) > 1:
     num = int(sys.argv[1])
     if num > 0:
         print(f"--- evaluating {num} entries")
-        # create new dataset exluding those idx
         datasets["test"] = datasets["test"].select(range(num))
 
 print_row_details(datasets["test"].to_pandas(), indices=[0, -1])

scripts/eval-mgtv-llama3_8b.sh

@@ -24,10 +24,14 @@ export MODEL_PREFIX=llama3-8b_lora_sft_bf16
 
 export LOGICAL_REASONING_RESULTS_PATH=results/$MODEL_PREFIX-p1.csv
 export ADAPTER_PATH_BASE=llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full
+export USING_P1_PROMPT_TEMPLATE=true
+export START_EPOCH=1
 echo "Eval $MODEL_NAME with $ADAPTER_PATH_BASE"
 python llm_toolkit/eval_logical_reasoning_all_epochs.py
 
 export LOGICAL_REASONING_RESULTS_PATH=results/$MODEL_PREFIX-p2.csv
 export ADAPTER_PATH_BASE=llama-factory/saves/llama3-8b/lora/sft_bf16_p2_full
+export USING_P1_PROMPT_TEMPLATE=false
+export START_EPOCH=0
 echo "Eval $MODEL_NAME with $ADAPTER_PATH_BASE"
 python llm_toolkit/eval_logical_reasoning_all_epochs.py