eval with few-shots prompting

.env.example

@@ -7,4 +7,4 @@ HF_TOKEN=
 LOAD_IN_4BIT=false
 
 DATA_PATH=datasets/mac/mac.tsv
-RESULTS_PATH=results/mac-results.csv
+RESULTS_PATH=results/mac-results_few_shots.csv

llm_toolkit/eval.py

@@ -28,7 +28,6 @@ results_path = os.getenv("RESULTS_PATH")
 batch_size = int(os.getenv("BATCH_SIZE", 1))
 use_english_datasets = os.getenv("USE_ENGLISH_DATASETS") == "true"
 max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", 2048))
-start_repetition_penalty = float(os.getenv("START_REPETITION_PENALTY", 1.0))
 
 print(
     model_name,
@@ -62,41 +61,68 @@ if is_cuda:
     print(f"(2) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
     print(f"{start_gpu_memory} GB of memory reserved.")
 
-datasets = load_translation_dataset(data_path, tokenizer)
 
-if len(sys.argv) > 1:
-    num = int(sys.argv[1])
-    if num > 0:
-        print(f"--- evaluating {num} entries")
-        datasets["test"] = datasets["test"].select(range(num))
-
-print_row_details(datasets["test"].to_pandas(), indices=[0, -1])
-
-
-def on_repetition_penalty_step_completed(model_name, predictions):
+def on_num_shots_step_completed(model_name, dataset, predictions):
     save_results(
         model_name,
         results_path,
-        datasets["test"],
+        dataset,
         predictions,
     )
 
-    metrics = calc_metrics(datasets["test"]["english"], predictions, debug=True)
+    metrics = calc_metrics(dataset["english"], predictions, debug=True)
     print(f"{model_name} metrics: {metrics}")
 
 
 if adapter_name_or_path is not None:
     model_name += "/" + adapter_name_or_path.split("/")[-1]
 
-evaluate_model_with_repetition_penalty(
+
+def evaluate_model_with_num_shots(
     model,
     tokenizer,
     model_name,
-    datasets["test"],
-    on_repetition_penalty_step_completed,
-    start_repetition_penalty=start_repetition_penalty,
-    end_repetition_penalty=1.3,
-    step_repetition_penalty=0.02,
+    data_path,
+    range_num_shots=[0, 1, 3, 5, 10, 50],
+    batch_size=1,
+    max_new_tokens=2048,
+    device="cuda",
+):
+    print(f"Evaluating model: {model_name} on {device}")
+
+    for num_shots in range_num_shots:
+        print(f"*** Evaluating with num_shots: {num_shots}")
+
+        datasets = load_translation_dataset(data_path, tokenizer, num_shots=num_shots)
+        print_row_details(datasets["test"].to_pandas())
+
+        predictions = eval_model(
+            model,
+            tokenizer,
+            datasets["test"],
+            device=device,
+            num_shots=num_shots,
+            batch_size=batch_size,
+            max_new_tokens=max_new_tokens,
+        )
+
+        model_name_with_rp = f"{model_name}/shots-{num_shots:02d}"
+
+        try:
+            on_num_shots_step_completed(
+                model_name_with_rp,
+                datasets["test"],
+                predictions,
+            )
+        except Exception as e:
+            print(e)
+
+
+evaluate_model_with_num_shots(
+    model,
+    tokenizer,
+    model_name,
+    data_path,
     batch_size=batch_size,
     max_new_tokens=max_new_tokens,
     device=device,

llm_toolkit/translation_utils.py

@@ -91,7 +91,7 @@ def save_results(model_name, results_path, dataset, predictions, debug=False):
     df.to_csv(results_path, index=False)
 
 
-def load_translation_dataset(data_path, tokenizer=None):
+def load_translation_dataset(data_path, tokenizer=None, num_shots=5):
     train_data_file = data_path.replace(".tsv", "-train.tsv")
     test_data_file = data_path.replace(".tsv", "-test.tsv")
 
@@ -122,7 +122,17 @@ def load_translation_dataset(data_path, tokenizer=None):
     )
 
     if tokenizer:
-        translation_prompt = "Please translate the following Chinese text into English and provide only the translated content, nothing else.\n{}"
+        translation_prompt = "You will be given a Chinese sentence to translate. If it is an incomplete sentence, or if you are unsure about the meaning, simply copy the input text as your output. Do not output any additional sentence such as explanation or reasoning.\n\n"
+        if num_shots > 0:
+            example_translations = "Example Translations:\n"
+            for i in range(num_shots):
+                example_translations += f"Chinese: {datasets['train'][i]['chinese']}\n"
+                example_translations += (
+                    f"English: {datasets['train'][i]['english']}\n\n"
+                )
+            translation_prompt = translation_prompt + example_translations
+
+        translation_prompt = translation_prompt + "Chinese: {}\nEnglish:"
 
         def formatting_prompts_func(examples):
             inputs = examples["chinese"]
@@ -131,7 +141,7 @@ def load_translation_dataset(data_path, tokenizer=None):
             messages = [
                 {
                     "role": "system",
-                    "content": "You are an expert in translating Chinese to English.",
+                    "content": "You are a helpful assistant that translates Chinese to English.",
                 },
                 None,
             ]

results/mac-results_metrics.csv

@@ -1,4 +1,5 @@
 model,rpp,meteor,bleu_1,rouge_l,ews_score,repetition_score,total_repetitions,rap,num_max_output_tokens
+01-ai/Yi-1.5-34B-Chat,1.00,0.3755299490820739,0.105238895159134,0.35899145567662555,0.0,0.43777581641659313,0.43777581641659313,0.3686697552536047,4
 01-ai/Yi-1.5-9B-Chat,1.00,0.3463725436435439,0.09312113035602035,0.33287597095291194,0.0,0.35127978817299205,0.35127978817299205,0.34125573890735983,2
 01-ai/Yi-1.5-9B-Chat,1.02,0.3471185374158656,0.09126513887574451,0.3325894211716421,0.0,0.264783759929391,0.264783759929391,0.3432230230787291,4
 01-ai/Yi-1.5-9B-Chat,1.04,0.3471882673119874,0.09019886552461354,0.33194600115482237,0.0,0.37775816416593117,0.37775816416593117,0.3416859125059273,8
@@ -11,6 +12,10 @@ model,rpp,meteor,bleu_1,rouge_l,ews_score,repetition_score,total_repetitions,rap
 01-ai/Yi-1.5-9B-Chat,1.18,0.3365273541015462,0.06786555450924157,0.31409006189011535,0.0,0.30979699911738745,0.30979699911738745,0.33212663825955735,26
 01-ai/Yi-1.5-9B-Chat,1.20,0.3355307477803475,0.06314678954328107,0.3110853370928648,0.0,0.22241835834068843,0.22241835834068843,0.33235553904085485,36
 01-ai/Yi-1.5-9B-Chat,1.22,0.33363375306882515,0.06214712430276763,0.3086913255065605,0.11827007943512798,0.25772285966460723,0.3715798764342454,0.32842981018186884,33
+01-ai/Yi-1.5-9B-Chat,1.24,0.32989818112961883,0.0570096687900327,0.30266509467774577,0.0970873786407767,0.09796999117387467,0.19064430714916153,0.3272144846442304,43
+01-ai/Yi-1.5-9B-Chat,1.26,0.3273962142810874,0.055237435454242444,0.3014277229065493,0.10944395410414828,0.40158870255957635,0.5110326566637247,0.32045973971100133,42
+01-ai/Yi-1.5-9B-Chat,1.28,0.323837066013506,0.051278882542576266,0.2953285400642388,0.03000882612533098,0.1059135039717564,0.13592233009708737,0.32194938854685645,42
+01-ai/Yi-1.5-9B-Chat,1.30,0.32093338081030204,0.047270589835958714,0.2909437718168147,0.02912621359223301,0.3698146513680494,0.3989408649602824,0.31557209228338484,49
 Qwen/Qwen2-72B-Instruct,1.00,0.3928168861285181,0.12345162681603773,0.3843593208981698,0.0,0.17563989408649602,0.17563989408649602,0.389868803763904,0
 Qwen/Qwen2-72B-Instruct,1.02,0.3936651928828143,0.12446659906815814,0.3844415446718956,0.0,0.147396293027361,0.147396293027361,0.39117939588436124,0
 Qwen/Qwen2-72B-Instruct,1.04,0.39263683565035906,0.12496255366843562,0.38481746782098636,0.0,0.15798764342453664,0.15798764342453664,0.38998196316138,0
@@ -43,9 +48,23 @@ Qwen/Qwen2-7B-Instruct,1.24,0.352755026120472,0.08591470945904531,0.339935275908
 Qwen/Qwen2-7B-Instruct,1.26,0.3483233677173315,0.07972359456247886,0.3352291660383133,0.0,0.08561341571050309,0.08561341571050309,0.3470385209221742,0
 Qwen/Qwen2-7B-Instruct,1.28,0.34450122231539704,0.07518096876457613,0.33099576010918924,0.0,0.09179170344218888,0.09179170344218888,0.34313954918633316,1
 Qwen/Qwen2-7B-Instruct,1.30,0.3401098279932269,0.07026740554261787,0.32623150769341913,0.0,0.09002647837599294,0.09002647837599294,0.3387911491977248,3
+internlm/internlm2_5-7b-chat,1.00,0.3671999390104764,0.11113560012478008,0.35982569682606647,0.0,0.14916151809355693,0.14916151809355693,0.36485386875796905,0
+internlm/internlm2_5-7b-chat,1.02,0.3669871024548418,0.11052834918066008,0.36047543966393214,0.0,0.11738746690203,0.11738746690203,0.36513645123895805,0
+internlm/internlm2_5-7b-chat,1.04,0.36431458209018286,0.10500475486417327,0.35796005034467937,0.0,0.09620476610767872,0.09620476610767872,0.36280598035582484,0
+internlm/internlm2_5-7b-chat,1.06,0.35852062221276365,0.1007746558766565,0.35340523977241706,0.0,0.09532215357458076,0.09532215357458076,0.3570495131117442,0
+internlm/internlm2_5-7b-chat,1.08,0.35020798408499415,0.0934055868567434,0.3442244092114163,0.0,0.11562224183583407,0.11562224183583407,0.3484682207171237,0
+internlm/internlm2_5-7b-chat,1.10,0.34191587814517765,0.07320041801997382,0.32676144300373294,0.0,0.1262135922330097,0.1262135922330097,0.3400635236612148,1
+internlm/internlm2_5-7b-chat,1.12,0.3343574558292014,0.052927581567650116,0.298405410760479,0.0,0.12886142983230361,0.12886142983230361,0.33250849918959285,6
+internlm/internlm2_5-7b-chat,1.14,0.31458167312569474,0.03414901931637522,0.2613753073822461,0.01323918799646955,0.1562224183583407,0.16946160635481025,0.31230250164005646,50
+internlm/internlm2_5-7b-chat,1.16,0.2895154344950096,0.021563541388547387,0.21947098060882128,0.1297440423654016,0.18446601941747573,0.3142100617828773,0.28567708514627493,155
 shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.3815423445635067,0.11524878188694271,0.37042646286690667,0.0,0.1968225948808473,0.1968225948808473,0.37833975022913946,0
 shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.02,0.3814400195917603,0.11481993983759356,0.3699786095406088,0.0,0.2118270079435128,0.2118270079435128,0.3779989256169545,0
 shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.04,0.3804874897233306,0.11332999451398112,0.36959666465385377,0.0,0.1879964695498676,0.1879964695498676,0.3774345028777672,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.06,0.3794618813204196,0.11264147540594219,0.3691306547739815,0.0,0.1879964695498676,0.1879964695498676,0.37641712383599774,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.08,0.3776298001192138,0.11139124543548622,0.3679447010986864,0.0,0.21977052074139453,0.21977052074139453,0.3740978880627333,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.10,0.3774364800258169,0.11069681096128998,0.36781965660594534,0.0,0.21712268314210062,0.21712268314210062,0.37394806502228206,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.12,0.3762356152613159,0.10985732529615087,0.36685494442700517,0.0,0.21712268314210062,0.21712268314210062,0.37275829911781017,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.14,0.37576687641484824,0.10927960097864851,0.36660320893621023,0.0,0.1968225948808473,0.1968225948808473,0.3726127602686422,0
 shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.00,0.3579680086793429,0.10207096308148353,0.3460132814937531,0.0,0.19240953221535745,0.19240953221535745,0.3550294775004645,0
 shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.02,0.3572226770743513,0.10061303169730976,0.3450507994469454,0.0,0.1615180935569285,0.1615180935569285,0.3547540871288482,0
 shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.04,0.35670586983276636,0.10074138007196803,0.3450245802338977,0.0,0.1615180935569285,0.1615180935569285,0.3542408512875192,0

scripts/eval-mac.sh

@@ -14,12 +14,13 @@ grep MemTotal /proc/meminfo
 # pip install torch torchvision torchaudio
 # pip install -r requirements.txt
 
-#./scripts/eval-model.sh shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat
+./scripts/eval-model.sh 01-ai/Yi-1.5-9B-Chat
+
+./scripts/eval-model.sh internlm/internlm2_5-7b-chat
 
 ./scripts/eval-model.sh Qwen/Qwen2-7B-Instruct
 
-./scripts/eval-model.sh shenzhi-wang/Llama3.1-8B-Chinese-Chat
+./scripts/eval-model.sh shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat
 
-#./scripts/eval-model.sh 01-ai/Yi-1.5-9B-Chat
+./scripts/eval-model.sh shenzhi-wang/Llama3.1-8B-Chinese-Chat
 
-#./scripts/eval-model.sh internlm/internlm2_5-7b-chat