completed llama-3.1-70b generic prompt

eval_modules/calc_repetitions_v2d.py

@@ -0,0 +1,1281 @@
+import os
+import re
+import math
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mtick
+import seaborn as sns
+import nltk
+import evaluate
+import traceback
+
+bert_score = evaluate.load("bertscore")
+meteor = evaluate.load("meteor")
+
+print(f"loading: {__file__}")
+
+# pattern_non_word_char_repetition = re.compile(r"\s{5,}")
+# pattern_text_repetitions = re.compile(r"(.{5}.*)\s*((\1)\s*)+", re.M | re.DOTALL)
+
+# final version
+pattern_non_word_char_repetition = re.compile(r"[\s\W]{5,}")
+pattern_text_repetitions = re.compile(
+    r"(?P<repeat>.{5}.*?)(?:[\s\W]*(?P=repeat))+", re.M | re.DOTALL | re.IGNORECASE
+)
+# Explanation of the Regex Pattern:
+#   (?P<repeat>.{5}.*?): Captures any sequence of characters with minimal length of 5 and names this group repeat.
+#     .*?: Matches zero or more characters, non-greedily (as few as possible).
+#   (?:[\s\W]+(?P=repeat))+: A non-capturing group that matches one or more repetitions of:
+#     [\s\W]+: One or more whitespace or non-word characters (spaces, punctuation, etc.).
+#     (?P=repeat): A backreference to the named group repeat.
+
+
+def del_non_word_char_repetition(text, debug=False):
+    count = 0
+
+    if isinstance(text, str):
+        if debug:
+            print("----detect non-word characters repetition----")
+        count = len(text)
+        text = pattern_non_word_char_repetition.sub("\t", text)
+        count -= len(text)
+        if debug and count:
+            print(f"removed non-word characters repetition: {count}")
+    return text, count
+
+
+# final version for repetition detection
+def detect_text_repetitions(text, debug=False):
+    count = 0
+
+    if isinstance(text, str):
+        if debug:
+            print("----detect text repetitions----")
+        matches = pattern_text_repetitions.finditer(text)
+        for match in matches:
+            if debug:
+                print(match)
+                for groupNum in range(0, len(match.groups())):
+                    groupNum = groupNum + 1
+                    print(
+                        "Group {groupNum} found at {start}-{end}: `{group}`".format(
+                            groupNum=groupNum,
+                            start=match.start(groupNum),
+                            end=match.end(groupNum),
+                            group=match.group(groupNum),
+                        )
+                    )
+
+            start, end = match.span()
+            count += end - start - len(match.group(1))
+
+    return count
+
+
+def detect_repetitions(text, debug=False):
+    text, count_non_word_char_repetition = del_non_word_char_repetition(
+        text, debug=debug
+    )
+    count_text_repetitions = detect_text_repetitions(text, debug=debug)
+    total_repetitions = count_non_word_char_repetition + count_text_repetitions
+
+    result = (count_non_word_char_repetition, count_text_repetitions, total_repetitions)
+
+    if debug:
+        print(result)
+    return result
+
+
+def detect_scores(text, debug=False):
+    newline_score, repetition_score, total_repetitions = detect_repetitions(
+        text, debug=debug
+    )
+    return pd.Series([newline_score, repetition_score, total_repetitions])
+
+
+def load_with_newline_and_repetition_scores(result_file, force_recalculate=False):
+    print(f"loading result file: {result_file}")
+    df = pd.read_csv(result_file, comment="#", on_bad_lines="warn")
+
+    if (
+        force_recalculate
+        or "newline_score" not in df.columns
+        or "repetition_score" not in df.columns
+        or "total_repetitions" not in df.columns
+        or "nrr" not in df.columns
+        or "rr" not in df.columns
+    ):
+        if (
+            force_recalculate
+            or "newline_score" not in df.columns
+            or "repetition_score" not in df.columns
+            or "total_repetitions" not in df.columns
+        ):
+            df[["newline_score", "repetition_score", "total_repetitions"]] = df[
+                "answer"
+            ].apply(detect_scores)
+
+        df["answer_len"] = df["answer"].apply(
+            lambda x: len(x) if isinstance(x, str) else 0
+        )
+
+        df["nrr"] = df.apply(
+            lambda x: (
+                1
+                if x["answer_len"] == 0
+                else 1 - (x["newline_score"] + x["repetition_score"]) / x["answer_len"]
+            ),
+            axis=1,
+        )
+
+        df["rr"] = df["nrr"].apply(lambda x: 1 - x)
+
+        df.to_csv(result_file, index=False)
+
+    return df
+
+
+def replace_last(source_string, old_string, new_string):
+    head, _sep, tail = source_string.rpartition(old_string)
+    return head + new_string + tail
+
+
+def load_for_repetition_penalty(
+    csv_result_file, repetition_penalty, force_recalculate=False
+):
+    result_file = replace_last(
+        csv_result_file, ".csv", f"_RP_{repetition_penalty:.3f}.csv"
+    )
+    return load_with_newline_and_repetition_scores(
+        result_file, force_recalculate=force_recalculate
+    )
+
+
+def calc_adjusted_performance(f, r):
+    return f / math.log10(10 + r)
+
+
+def calculate_adjusted_performance(row):
+    r = row["total_repetitions"]
+    adjusted_precision = calc_adjusted_performance(row["precision"], r)
+    adjusted_recall = calc_adjusted_performance(row["recall"], r)
+    return pd.Series([adjusted_precision, adjusted_recall])
+
+
+def load_performance_df(csv_result_file, repetition_penalty):
+    result_file = replace_last(
+        csv_result_file, ".csv", f"_RP_{repetition_penalty:.3f}-t2_evaluated.json"
+    )
+    result_file = result_file.replace("/results/", "/eval/")
+    print(f"loading json file: {result_file}")
+    df = pd.read_json(result_file)
+
+    return df
+
+
+def calculate_performance_score(
+    csv_result_file, repetition_penalty, force_recalculate=False
+):
+    result_file = replace_last(
+        csv_result_file, ".csv", f"_rpp_{repetition_penalty:.2f}.csv"
+    )
+
+    if os.path.exists(result_file):
+        print(f"loading result file: {result_file}")
+        df = load_with_newline_and_repetition_scores(
+            result_file, force_recalculate=force_recalculate
+        )
+    else:
+        print(f"re-creating result file: {result_file}")
+        df = pd.DataFrame()
+        force_recalculate = True
+
+    if force_recalculate or "f2" in df.columns or "f1" not in df.columns:
+        try:
+            perf_df = load_performance_df(csv_result_file, repetition_penalty)
+            df.drop(
+                columns=[
+                    "precision",
+                    "recall",
+                    "f1",
+                    "f2",
+                    "entities_in_answer",
+                    "entities_in_question",
+                    "word_count",
+                ],
+                errors="ignore",
+                inplace=True,
+            )
+
+            df["id"] = perf_df["id"]
+            df["question"] = perf_df["question"]
+            df["answer"] = perf_df["pred_answer"]
+            df["word_count"] = df["answer"].apply(
+                lambda x: len(nltk.word_tokenize(x)) if isinstance(x, str) else 0
+            )
+            df["ground_truth"] = perf_df["ground_truth"]
+
+            df["eval_gemini_1.0_pro"] = perf_df["eval_gemini_1.0_pro"]
+            df["precision"] = perf_df["score"].apply(lambda x: x[0])
+            df["recall"] = perf_df["score"].apply(lambda x: x[1])
+            df["f1"] = perf_df["score"].apply(lambda x: x[2])
+        except Exception as e:
+            print(f"\tignored error: {e}")
+            # traceback.print_exc()
+
+        df[["newline_score", "repetition_score", "total_repetitions"]] = df[
+            "answer"
+        ].apply(detect_scores)
+
+        df[["adjusted_precision", "adjusted_recall"]] = df.apply(
+            calculate_adjusted_performance, axis=1
+        )
+
+        df.to_csv(result_file, index=False)
+        print(f"performance scores saved to result file: {result_file}")
+
+    # print(f"df len: {len(df)}")
+
+    return df
+
+
+def adjust_perf_scores_with_repetition_penalty(result, precision, recall):
+    newline_score = [
+        df["newline_score"].mean() for df in result["df_list_repetition_penalty"]
+    ]
+
+    repetition_score = [
+        df["repetition_score"].mean() for df in result["df_list_repetition_penalty"]
+    ]
+
+    precision = [
+        f / math.log10(10 + n + r)
+        for f, n, r in zip(precision, newline_score, repetition_score)
+    ]
+    recall = [
+        f / math.log10(10 + n + r)
+        for f, n, r in zip(recall, newline_score, repetition_score)
+    ]
+
+    return precision, recall
+
+
+def plot_performance_scores(
+    result,
+    models=None,
+    title="Performance",
+):
+    if models is None:
+        models = result.keys()
+    for model in models:
+        print(f"model: {model}")
+        df = result[model]["df_overall"]
+
+        # Calculate the statistics
+        precision = [
+            df["precision"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        recall = [
+            df["recall"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
+        best_f1 = max(f1)
+        best_f1_index = f1.index(best_f1)
+
+        precision, recall = adjust_perf_scores_with_repetition_penalty(
+            result[model], precision, recall
+        )
+        afrp = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
+
+        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
+        best_afrp = max(afrp)
+        best_afrp_index = afrp.index(best_afrp)
+
+        adjusted_precision = [
+            df["adjusted_precision"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        adjusted_recall = [
+            df["adjusted_recall"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        afrp2 = [
+            2 * (p * r) / (p + r) for p, r in zip(adjusted_precision, adjusted_recall)
+        ]
+        best_afrp2 = max(afrp2)
+        best_afrp2_index = afrp2.index(best_afrp2)
+
+        repetition_penalties = list(df["repetition_penalty"])
+
+        # line plot for precision, recall, f1
+        plt.figure(figsize=(10, 6))
+
+        plt.axvspan(
+            repetition_penalties[best_f1_index] - 0.01,
+            repetition_penalties[best_f1_index] + 0.01,
+            alpha=0.5,
+            edgecolor="none",
+            facecolor="blue",
+        )
+
+        # plt.axvspan(
+        #     repetition_penalties[best_afrp2_index] - 0.01,
+        #     repetition_penalties[best_afrp2_index] + 0.01,
+        #     alpha=0.5,
+        #     edgecolor="none",
+        #     facecolor="green",
+        # )
+
+        plt.axvspan(
+            repetition_penalties[best_afrp_index] - 0.01,
+            repetition_penalties[best_afrp_index] + 0.01,
+            alpha=0.5,
+            edgecolor="none",
+            facecolor="orange",
+        )
+
+        plt.plot(repetition_penalties, f1, label="F1", marker="D", color="blue")
+        # plt.plot(
+        #     repetition_penalties,
+        #     afrp2,
+        #     label="Per-question RAP - F1",
+        #     marker="s",
+        #     color="green",
+        # )
+        plt.plot(
+            repetition_penalties,
+            afrp,
+            label="RAP - F1",
+            marker="o",
+            color="orange",
+        )
+        plt.xlabel("Repetition Penalties")
+        plt.ylabel("Score")
+        # plt.xlim(0.99, 1.31)
+        # y in percentage
+        plt.gca().yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
+        plt.title(f"{model} {title}")
+        plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+        plt.show()
+
+
+def plot_best_afrp(
+    result,
+    models=None,
+    title="Models with Best RAP - F1",
+    ref_result=None,
+):
+    # Initialize lists to store the statistics
+    model_names = []
+    best_f1 = []
+    best_afrp = []
+    best_repetition_penalty = []
+    best_mtr = []
+
+    if models is None:
+        models = result.keys()
+    for model in models:
+        print(f"model: {model}")
+        df = result[model]["df_overall"]
+
+        # Calculate the statistics
+        precision = [
+            df["precision"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        recall = [
+            df["recall"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
+        f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
+
+        newline_score = [
+            df["newline_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        # print(f"newline_score: {newline_score}")
+
+        repetition_score = [
+            df["repetition_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        # print(f"repetition_score: {repetition_score}")
+
+        afrp = [
+            f / math.log10(10 + n + r)
+            for f, n, r in zip(f1, newline_score, repetition_score)
+        ]
+
+        best_afrp.append(max(afrp))
+        best_afrp_index = afrp.index(best_afrp[-1])
+        best_repetition_penalty.append(df["repetition_penalty"][best_afrp_index])
+
+        best_f1.append(f1[best_afrp_index])
+        best_mtr.append(
+            newline_score[best_afrp_index] + repetition_score[best_afrp_index]
+        )
+
+        # print(
+        #     f"best repetition penalty: {best_repetition_penalty[-1]}, best afrp: {best_afrp[-1]}, f1: {best_f1[-1]}"
+        # )
+
+        df = result[model]["df_list_repetition_penalty"][best_afrp_index]
+
+        model_names.append(
+            f"{model} (RP={best_repetition_penalty[-1]})"
+        )  # Add the model name to the list
+
+    if ref_result is not None:
+        print("ref_result:", ref_result)
+        for model in ref_result.keys():
+            model_names.append(model)
+            df = pd.read_csv(ref_result[model])
+            # df = df[df["id"].isin(wikidata_df["id"])]
+
+            p = df["precision"].mean()
+            r = df["recall"].mean()
+
+            f1 = 2 * p * r / (p + r) if p + r > 0 else 0
+            best_f1.append(f1)
+            best_afrp.append(f1)
+            best_mtr.append(0)
+
+    print("model_names:", model_names)
+    # print("best_f1:", best_f1)
+    # print("best_afrp:", best_afrp)
+
+    # Create a DataFrame with the statistics
+    data = pd.DataFrame(
+        {
+            "Model": model_names,
+            "RAP - F1": best_afrp,
+            "F1": best_f1,
+        }
+    )
+
+    # Melt the DataFrame to a long format
+    data_melted = data.melt(id_vars="Model", var_name="Metric", value_name="Score")
+
+    # Pivot the DataFrame to a wide format
+    data_pivoted = data_melted.pivot(index="Metric", columns="Model", values="Score")
+
+    # make sure the columns are following the order of the models
+    data_pivoted = data_pivoted[model_names]
+
+    # make sure three groups in the order of precision, recall, f1
+    data_pivoted = data_pivoted.reindex(["RAP - F1", "F1"])
+
+    # Plot the statistics
+    plt.figure(figsize=(15, 6))
+    ax = data_pivoted.plot(kind="bar", ax=plt.gca(), width=0.9)
+    plt.title(title)
+    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+    # Set the rotation of the x-axis labels to 0 degrees
+    plt.xticks(rotation=0)
+
+    # Format the y-axis to display as percentage
+    ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
+
+    # get the max value of the y-axis
+    a1 = max(best_afrp)
+    a2 = max(best_f1)
+
+    max_value = max([a1, a2]) * 1.12
+    print("max_value:", max_value)
+
+    # Set the y-axis limit up to 70%
+    ax.set_ylim(0, max_value)
+
+    # Add the values above each bar
+    for p in ax.patches:
+        ax.annotate(
+            f"{p.get_height() * 100:.1f}",
+            (p.get_x() + p.get_width() / 2.0, p.get_height()),
+            ha="center",
+            va="bottom",
+            xytext=(0, 10),
+            textcoords="offset points",
+            rotation=90,
+        )
+
+    plt.show()
+    return data_pivoted, best_mtr
+
+
+def plot_best_performance(
+    result,
+    models=None,
+    title="Models with Best F1 Score",
+    adjusted_f1=False,
+    ref_result=None,
+):
+    # Initialize lists to store the statistics
+    model_names = []
+    best_precision = []
+    best_recall = []
+    best_f1 = []
+    best_repetition_penalty = []
+    best_mtr = []
+
+    if models is None:
+        models = result.keys()
+    for model in models:
+        print(f"model: {model}")
+        df = result[model]["df_overall"]
+
+        # Calculate the statistics
+        precision = [
+            df["precision"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        recall = [
+            df["recall"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        newline_score = [
+            df["newline_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+
+        repetition_score = [
+            df["repetition_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+
+        if adjusted_f1:
+            precision, recall = adjust_perf_scores_with_repetition_penalty(
+                result[model], precision, recall
+            )
+
+        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
+        f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
+
+        best_f1.append(max(f1))
+        best_f1_index = f1.index(best_f1[-1])
+        best_repetition_penalty.append(df["repetition_penalty"][best_f1_index])
+
+        best_precision.append(precision[best_f1_index])
+        best_recall.append(recall[best_f1_index])
+        best_mtr.append(newline_score[best_f1_index] + repetition_score[best_f1_index])
+
+        print(
+            f"best repetition penalty: {best_repetition_penalty[-1]}, best f1: {best_f1[-1]}, precision: {best_precision[-1]}, recall: {best_recall[-1]}"
+        )
+
+        df = result[model]["df_list_repetition_penalty"][best_f1_index]
+
+        model_names.append(
+            f"{model} (RP={best_repetition_penalty[-1]})"
+        )  # Add the model name to the list
+
+        # print sum for columns: newline_score, repetition_score
+        print(
+            f"newline_score: {df['newline_score'].sum()}, repetition_score: {df['repetition_score'].sum()}"
+        )
+
+    if ref_result is not None:
+        print("ref_result:", ref_result)
+        for model in ref_result.keys():
+            model_names.append(model)
+            df = pd.read_csv(ref_result[model])
+            # df = df[df["id"].isin(wikidata_df["id"])]
+
+            best_precision.append(df["precision"].mean())
+            best_recall.append(df["recall"].mean())
+            f1 = (
+                2
+                * (best_precision[-1] * best_recall[-1])
+                / (best_precision[-1] + best_recall[-1])
+            )
+            # best_f1.append(df["f1"].mean())
+            best_f1.append(f1)
+            best_mtr.append(0)
+
+    # Create a DataFrame with the statistics
+    data = (
+        pd.DataFrame(
+            {
+                "Model": model_names,
+                "Adjusted Precision with RP": best_precision,
+                "Adjusted Recall with RP": best_recall,
+                "Adjusted F1 with RP": best_f1,
+            }
+        )
+        if adjusted_f1
+        else pd.DataFrame(
+            {
+                "Model": model_names,
+                "Precision": best_precision,
+                "Recall": best_recall,
+                "F1": best_f1,
+            }
+        )
+    )
+    columns = list(data.columns)
+
+    # Melt the DataFrame to a long format
+    data_melted = data.melt(id_vars="Model", var_name="Metric", value_name="Score")
+
+    # Pivot the DataFrame to a wide format
+    data_pivoted = data_melted.pivot(index="Metric", columns="Model", values="Score")
+
+    # make sure the columns are following the order of the models
+    data_pivoted = data_pivoted[model_names]
+
+    # make sure three groups in the order of precision, recall, f1
+    data_pivoted = data_pivoted.reindex(columns[1:])
+
+    # Plot the statistics
+    plt.figure(figsize=(10, 6))
+    ax = data_pivoted.plot(kind="bar", ax=plt.gca(), width=0.9)
+    plt.title(title)
+    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+    # Set the rotation of the x-axis labels to 0 degrees
+    plt.xticks(rotation=0)
+
+    # Format the y-axis to display as percentage
+    ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
+
+    # get the max value of the y-axis
+    a1 = max(best_precision)
+    a2 = max(best_recall)
+    a3 = max(best_f1)
+
+    max_value = max([a1, a2, a3]) * 1.12
+    print("max_value:", max_value)
+
+    # Set the y-axis limit up to 70%
+    ax.set_ylim(0, max_value)
+
+    # Add the values above each bar
+    for p in ax.patches:
+        ax.annotate(
+            f"{p.get_height() * 100:.1f}",
+            (p.get_x() + p.get_width() / 2.0, p.get_height()),
+            ha="center",
+            va="bottom",
+            xytext=(0, 10),
+            textcoords="offset points",
+            rotation=90,
+        )
+
+    plt.show()
+    return data_pivoted, best_mtr
+
+
+def plot_best_performance_ms_macro(
+    result,
+    models=None,
+    title="Models with Best RAP - Performance",
+    ref_result=None,
+    skip_generic_prompt=False,
+    include_adjusted_performance=True,
+):
+    # Initialize lists to store the statistics
+    model_names = []
+    best_f1 = []
+    best_afrp = []
+    best_repetition_penalty = []
+    best_bleu1 = []
+    best_rougeL = []
+    best_mtr = []
+
+    if models is None:
+        models = result.keys()
+    for model in models:
+        if skip_generic_prompt and "generic prompt" in model:
+            continue
+        print(f"model: {model}")
+        df = result[model]["df_overall"]
+
+        # Calculate the statistics
+        bleu1 = [x for x in df["bleu1"]]
+        rougeL = [x for x in df["rougeL"]]
+        f1 = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
+
+        newline_score = [
+            df["newline_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        # print(f"newline_score: {newline_score}")
+
+        repetition_score = [
+            df["repetition_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        # print(f"repetition_score: {repetition_score}")
+
+        afrp = [
+            f / math.log10(10 + n + r)
+            for f, n, r in zip(f1, newline_score, repetition_score)
+        ]
+
+        best_afrp.append(max(afrp if include_adjusted_performance else f1))
+        best_afrp_index = (
+            afrp.index(best_afrp[-1])
+            if include_adjusted_performance
+            else f1.index(best_afrp[-1])
+        )
+        best_repetition_penalty.append(df["repetition_penalty"][best_afrp_index])
+
+        best_f1.append(f1[best_afrp_index])
+        best_bleu1.append(bleu1[best_afrp_index])
+        best_rougeL.append(rougeL[best_afrp_index])
+        best_mtr.append(
+            newline_score[best_afrp_index] + repetition_score[best_afrp_index]
+        )
+
+        # print(
+        #     f"best repetition penalty: {best_repetition_penalty[-1]}, best afrp: {best_afrp[-1]}, f1: {best_f1[-1]}"
+        # )
+
+        df = result[model]["df_list_repetition_penalty"][best_afrp_index]
+
+        model_names.append(
+            f"{model} (RP={best_repetition_penalty[-1]})"
+        )  # Add the model name to the list
+
+    if ref_result is not None:
+        print("ref_result:", ref_result)
+        for model in ref_result.keys():
+            model_names.append(model)
+            df = pd.read_csv(ref_result[model], comment="#", on_bad_lines="warn")
+            # df = df[df["id"].isin(wikidata_df["id"])]
+
+            p = df["bleu1"][0]
+            best_bleu1.append(p)
+
+            r = df["rougeL"][0]
+            best_rougeL.append(r)
+
+            f1 = 2 * p * r / (p + r) if p + r > 0 else 0
+            best_f1.append(f1)
+            best_afrp.append(f1)
+            best_mtr.append(0)
+
+    # print("model_names:", model_names)
+    # print("best_f1:", best_f1)
+    # print("best_afrp:", best_afrp)
+
+    # Create a DataFrame with the statistics
+    data = (
+        pd.DataFrame(
+            {
+                "Model": model_names,
+                "RAP - Perf Score": best_afrp,
+                "Overall Perf Score": best_f1,
+            }
+        )
+        if include_adjusted_performance
+        else pd.DataFrame(
+            {
+                "Model": model_names,
+                "Bleu-1": best_bleu1,
+                "Rouge-L": best_rougeL,
+                "Overall Perf Score": best_f1,
+            }
+        )
+    )
+
+    # Melt the DataFrame to a long format
+    data_melted = data.melt(id_vars="Model", var_name="Metric", value_name="Score")
+
+    # Pivot the DataFrame to a wide format
+    data_pivoted = data_melted.pivot(index="Metric", columns="Model", values="Score")
+
+    # make sure the columns are following the order of the models
+    data_pivoted = data_pivoted[model_names]
+
+    columns = list(data.columns)
+    data_pivoted = data_pivoted.reindex(columns[1:])
+
+    # Plot the statistics
+    plt.figure(figsize=(10, 6))
+    ax = data_pivoted.plot(kind="bar", ax=plt.gca(), width=0.9)
+    plt.title(title)
+    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+    # Set the rotation of the x-axis labels to 0 degrees
+    plt.xticks(rotation=0)
+
+    # Format the y-axis to display as percentage
+    ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
+
+    # get the max value of the y-axis
+    a1 = max(best_afrp)
+    a2 = max(best_f1)
+    a3 = max(best_bleu1)
+    a4 = max(best_rougeL)
+
+    max_value = (
+        max([a1, a2] if include_adjusted_performance else [a1, a2, a3, a4]) * 1.12
+    )
+    print("max_value:", max_value)
+
+    # Set the y-axis limit up to 70%
+    ax.set_ylim(0, max_value)
+
+    # Add the values above each bar
+    for p in ax.patches:
+        ax.annotate(
+            f"{p.get_height() * 100:.1f}",
+            (p.get_x() + p.get_width() / 2.0, p.get_height()),
+            ha="center",
+            va="bottom",
+            xytext=(0, 10),
+            textcoords="offset points",
+            rotation=90,
+        )
+
+    plt.show()
+    return data_pivoted, best_mtr
+
+
+all_open_source_models = [
+    "gemma-1.1-2b-it",
+    "Phi-3-mini-128k-instruct",
+    "gemma-1.1-7b-it",
+    "Llama-2-7b-chat-hf",
+    "Mistral-7B-Instruct-v0.2",
+    "Meta-Llama-3-8B-Instruct",
+    "Llama-2-13b-chat-hf",
+    "Llama-2-70b-chat-hf",
+    "Meta-Llama-3-70B-Instruct",
+]
+
+
+def load_for_repetition_penalty_ms_macro(
+    csv_result_file, repetition_penalty, force_recalculate=False
+):
+    result_file = replace_last(
+        csv_result_file, ".csv", f"_rpp_{repetition_penalty:.2f}.csv"
+    )
+    df = load_with_newline_and_repetition_scores(
+        result_file, force_recalculate=force_recalculate
+    )
+
+    return df
+
+
+# MS MACRO
+def plot_performance_scores_ms_macro(
+    result,
+    models=None,
+    title="Performance",
+):
+    if models is None:
+        models = result.keys()
+    for model in models:
+        print(f"model: {model}")
+        df = result[model]["df_overall"]
+        # print(result[model]["df_list_repetition_penalty"][0].describe())
+
+        # Calculate the statistics
+        bleu1 = list(df["bleu1"])
+        rougeL = list(df["rougeL"])
+        f1 = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
+        best_f1 = max(f1)
+        best_f1_index = f1.index(best_f1)
+
+        bleu1, rougeL = adjust_perf_scores_with_repetition_penalty(
+            result[model], bleu1, rougeL
+        )
+        afrp = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
+
+        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
+        best_afrp = max(afrp)
+        best_afrp_index = afrp.index(best_afrp)
+
+        repetition_penalties = list(df["repetition_penalty"])
+
+        # line plot for precision, recall, f1
+        plt.figure(figsize=(10, 6))
+
+        plt.axvspan(
+            repetition_penalties[best_f1_index] - 0.01,
+            repetition_penalties[best_f1_index] + 0.01,
+            alpha=0.5,
+            edgecolor="none",
+            facecolor="blue",
+        )
+
+        plt.axvspan(
+            repetition_penalties[best_afrp_index] - 0.01,
+            repetition_penalties[best_afrp_index] + 0.01,
+            alpha=0.5,
+            edgecolor="none",
+            facecolor="orange",
+        )
+
+        plt.plot(
+            repetition_penalties,
+            f1,
+            label="Overall Perf Score",
+            marker="D",
+            color="blue",
+        )
+        plt.plot(
+            repetition_penalties,
+            afrp,
+            label="RAP - Perf Score",
+            marker="o",
+            color="orange",
+        )
+
+        plt.xlabel("Repetition Penalties")
+        plt.ylabel("Score")
+        # plt.xlim(0.99, 1.31)
+        # y in percentage
+        plt.gca().yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
+        plt.title(f"{model} {title}")
+        plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+        plt.show()
+
+
+def plot_repetition_factors(result, groups):
+    for group in groups:
+        # Plot the statistics
+        plt.figure(figsize=(10, 6))
+
+        max_value = 0
+        for model in result.keys():
+            if not group in model.lower():
+                continue
+            print(f"model: {model}")
+            df = result[model]["df_overall"]
+            repetition_panelties = [
+                repetition_penalty for repetition_penalty in df["repetition_penalty"]
+            ]
+
+            mean_score = [
+                # math.log10(10 + df["total_repetitions"].mean())
+                df["total_repetitions"].mean()
+                for df in result[model]["df_list_repetition_penalty"]
+            ]
+
+            sns.lineplot(x=repetition_panelties, y=mean_score, label=model)
+
+            new_max = max(mean_score)
+            if new_max > max_value:
+                max_value = new_max
+
+        max_value = max_value * 1.05
+        # if max_value < 1.5:
+        #     max_value = 1.5
+        # set ylimit
+        plt.ylim(0, max_value)
+
+        # show grid
+        plt.grid(True)
+        plt.xlabel("Repetition Penalties")
+        plt.ylabel("Mean Total Repetitions")
+        plt.title("Mean Total Repetitions vs Repetition Penalties")
+        plt.legend()
+
+        plt.show()
+
+
+def plot_repetition_factors_by_group(result, group_filter=None):
+    markers = ["D", "o", "s", "x"]
+    colors = ["blue", "orange", "green", "red"]
+
+    # Plot the statistics
+    plt.figure(figsize=(10, 6))
+    index = 0
+    max_value = 0
+
+    for model in result.keys():
+        if group_filter is not None and group_filter not in model:
+            continue
+
+        print(f"model: {model}")
+
+        df = result[model]["df_overall"]
+        repetition_panelties = [
+            repetition_penalty for repetition_penalty in df["repetition_penalty"]
+        ]
+
+        # Calculate the statistics
+        mean_score = [
+            # math.log10(10 + df["total_repetitions"].mean())
+            df["total_repetitions"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        if len(mean_score) != len(repetition_panelties):
+            print(
+                f"model: {model} has different length of repetition penalties and mean score"
+            )
+            print("repetition_panelties:", len(repetition_panelties))
+            print("mean_score:", len(mean_score))
+            continue
+
+        new_max = max(mean_score)
+        if new_max > max_value:
+            max_value = new_max
+
+        sns.lineplot(
+            x=repetition_panelties,
+            y=mean_score,
+            label=model,
+            marker=markers[index],
+            color=colors[index],
+        )
+
+        index += 1
+
+    max_value = max_value * 1.05
+    # if max_value < 1.5:
+    #     max_value = 1.5
+    # set ylimit
+    plt.ylim(0, max_value)
+    max_value = 0
+
+    plt.xlabel("Repetition Penalties")
+    plt.ylabel("Mean Total Repetitions")
+    plt.title("Mean Total Repetitions vs Repetition Penalties")
+    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+    plt.show()
+
+
+ms_marco_csv_result_files = [
+    "data/results_v2/gemma-1.1-2b-it(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/gemma-1.1-2b-it(RAG - Chat Template)_mm.csv",
+    "data/results_v2/gemma-1.1-2b-it(Non-RAG)_mm.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(Non-RAG)_mm.csv",
+    "data/results_v2/gemma-1.1-7b-it(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/gemma-1.1-7b-it(RAG - Chat Template)_mm.csv",
+    "data/results_v2/gemma-1.1-7b-it(Non-RAG)_mm.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(Non-RAG)_mm.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(Non-RAG)_mm.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(Non-RAG)_mm.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(Non-RAG)_mm.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(Non-RAG)_mm.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(Non-RAG)_mm.csv",
+]
+
+webqsp_csv_result_files = [
+    "data/results_v2/gemma-1.1-2b-it(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/gemma-1.1-2b-it(RAG - Chat Template)_wd.csv",
+    "data/results_v2/gemma-1.1-2b-it(Non-RAG)_wd.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(Non-RAG)_wd.csv",
+    "data/results_v2/gemma-1.1-7b-it(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/gemma-1.1-7b-it(RAG - Chat Template)_wd.csv",
+    "data/results_v2/gemma-1.1-7b-it(Non-RAG)_wd.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(Non-RAG)_wd.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(Non-RAG)_wd.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(Non-RAG)_wd.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(Non-RAG)_wd.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(Non-RAG)_wd.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(Non-RAG)_wd.csv",
+]
+
+
+def calc_rap_scores(result, precision="precision", recall="recall"):
+    newline_score = [
+        df["newline_score"].mean() for df in result["df_list_repetition_penalty"]
+    ]
+
+    repetition_score = [
+        df["repetition_score"].mean() for df in result["df_list_repetition_penalty"]
+    ]
+
+    if precision in result["df_list_repetition_penalty"][0].columns:
+        precision = [
+            df[precision].mean() for df in result["df_list_repetition_penalty"]
+        ]
+        recall = [df[recall].mean() for df in result["df_list_repetition_penalty"]]
+    else:
+        precision = result["df_overall"][precision]
+        recall = result["df_overall"][recall]
+
+    f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
+
+    # rap = [
+    #     f / math.log10(10 + n + r)
+    #     for f, n, r in zip(f1, newline_score, repetition_score)
+    # ]
+
+    nrr = [
+        1 - (n + r) / s
+        for f, n, r, s in zip(
+            f1, newline_score, repetition_score, result["df_overall"]["answer_len"]
+        )
+    ]
+
+    rap = [f * n * n * n for f, n in zip(f1, nrr)]
+
+    return newline_score, repetition_score, f1, rap, nrr
+
+
+def get_model_name(csv_result_file):
+    parts = re.split(r"[_/]", csv_result_file)
+    print(f"parts: {parts}")
+    model_name = parts[3]
+    return model_name
+
+
+def load_webqsp_result(csv_result_files, force_recalculate=False, save=False):
+    result = {}
+    for i, csv_result_file in enumerate(csv_result_files):
+        try:
+            df = pd.read_csv(csv_result_file)
+            model_name = get_model_name(csv_result_file)
+            print(f"\tmodel_name: {model_name}")
+
+            dfs = [
+                calculate_performance_score(
+                    csv_result_file,
+                    repetition_penalty,
+                    force_recalculate=force_recalculate,
+                )
+                for repetition_penalty in df["repetition_penalty"]
+            ]
+
+            answer_lens = []
+            for df_rpp in dfs:
+                df_rpp["answer_len"] = df_rpp["answer"].apply(
+                    lambda x: len(x) if isinstance(x, str) else 0
+                )
+                answer_lens.append(df_rpp["answer_len"].mean())
+            df["answer_len"] = answer_lens
+
+            result[model_name] = {
+                "df_overall": df,
+                "df_list_repetition_penalty": dfs,
+                "file": csv_result_file,
+            }
+            newline_score, repetition_score, perf, rap, nrr = calc_rap_scores(
+                result[model_name]
+            )
+            df["newline_score"] = newline_score
+            df["repetition_score"] = repetition_score
+            df["total_repetitions"] = df["newline_score"] + df["repetition_score"]
+            df["perf"] = perf
+            df["nrr"] = nrr
+            df["rap"] = rap
+            df["rr"] = df["nrr"].apply(lambda x: 1 - x)
+            if save:
+                df.to_csv(csv_result_file, index=False)
+        except Exception as e:
+            print(f"Error: {e}")
+            traceback.print_exc()
+
+    return result
+
+
+def load_ms_marco_result(
+    csv_result_files, force_recalculate=False, calc_bertscore=False, save=False
+):
+    result = {}
+    for csv_result_file in csv_result_files:
+        try:
+            df = pd.read_csv(csv_result_file)
+            model_name = get_model_name(csv_result_file)
+            print(f"\tmodel_name: {model_name}")
+
+            dfs = [
+                load_for_repetition_penalty_ms_macro(
+                    csv_result_file,
+                    repetition_penalty,
+                    force_recalculate=force_recalculate,
+                )
+                for repetition_penalty in df["repetition_penalty"]
+            ]
+
+            answer_lens = []
+            for df_rpp in dfs:
+                answer_lens.append(df_rpp["answer_len"].mean())
+            df["answer_len"] = answer_lens
+
+            col = "bert_score" if calc_bertscore else "meteor"
+            score_unavailable = col not in df.columns
+
+            if score_unavailable:
+                save = True
+                bert_meteor_scores = []
+                bert_score_references = None
+                for df_rpp in dfs:
+                    if calc_bertscore:
+                        bert_meteor_score = 0
+
+                        for i, row in df_rpp.iterrows():
+                            answer = row["answer"]
+                            if not isinstance(answer, str):
+                                answer = ""
+                            bert_meteor_score += bert_score.compute(
+                                predictions=[answer],
+                                references=[row["ground_truth"][0]],
+                                lang="en",
+                                model_type="microsoft/deberta-large-mnli",
+                            )["f1"][0]
+                        # get average of bertscore
+                        bert_meteor_score = bert_meteor_score / len(df_rpp)
+
+                        print(f"bert_score: {bert_meteor_score}")
+                    else:
+                        bert_meteor_score = meteor.compute(
+                            predictions=df_rpp["answer"],
+                            references=df_rpp["ground_truth"],
+                        )["meteor"]
+
+                    bert_meteor_scores.append(bert_meteor_score)
+
+                df[col] = bert_meteor_scores
+
+            result[model_name] = {
+                "df_overall": df,
+                "df_list_repetition_penalty": dfs,
+                "file": csv_result_file,
+            }
+            newline_score, repetition_score, perf, rap, nrr = calc_rap_scores(
+                result[model_name],
+                precision=col,
+                recall=col,
+            )
+            df["newline_score"] = newline_score
+            df["repetition_score"] = repetition_score
+            df["total_repetitions"] = df["newline_score"] + df["repetition_score"]
+            df["perf"] = perf
+            df["nrr"] = nrr
+            df["rap"] = rap
+            df["rr"] = df["nrr"].apply(lambda x: 1 - x)
+
+            if save:
+                df.to_csv(csv_result_file, index=False)
+        except Exception as e:
+            print("An error occurred:", e)
+            traceback.print_exc()
+            print(f"csv_result_file: {csv_result_file}")
+
+    return result

llm_toolkit/translation_utils_v2.py

@@ -0,0 +1,766 @@
+import os
+import re
+import glob
+import pandas as pd
+import evaluate
+import seaborn as sns
+import matplotlib.pyplot as plt
+from datasets import load_dataset
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from tqdm import tqdm
+from eval_modules.calc_repetitions_v2d import *
+from llm_toolkit.llm_utils import load_tokenizer, print_row_details
+
+print(f"loading {__file__}")
+
+bleu = evaluate.load("bleu")
+rouge = evaluate.load("rouge")
+meteor = evaluate.load("meteor")
+accuracy = evaluate.load("accuracy")
+sacrebleu = evaluate.load("sacrebleu")
+comet = evaluate.load("comet")
+
+
+def extract_answer(text, debug=False):
+    if text and isinstance(text, str):
+        # Remove the begin and end tokens
+        text = re.sub(
+            r".*?(assistant|\[/INST\]).+?\b", "", text, flags=re.DOTALL | re.MULTILINE
+        )
+        if debug:
+            print("--------\nstep 1:", text)
+
+        text = re.sub(r"<.+?>.*", "", text, flags=re.DOTALL | re.MULTILINE)
+        if debug:
+            print("--------\nstep 2:", text)
+
+        text = re.sub(
+            r".*?end_header_id\|>\n\n", "", text, flags=re.DOTALL | re.MULTILINE
+        )
+        if debug:
+            print("--------\nstep 3:", text)
+
+    return text
+
+
+def calc_metrics(references, predictions, sources=None, debug=False):
+    assert len(references) == len(
+        predictions
+    ), f"lengths are difference: {len(references)} != {len(predictions)}"
+
+    predictions = [extract_answer(text) for text in predictions]
+    results = {}
+
+    results["comet"] = comet.compute(
+        predictions=predictions, references=references, sources=sources
+    )["mean_score"]
+
+    results["meteor"] = meteor.compute(predictions=predictions, references=references)[
+        "meteor"
+    ]
+
+    results["sacrebleu"] = sacrebleu.compute(
+        predictions=predictions, references=references
+    )
+
+    results["bleu_scores"] = bleu.compute(
+        predictions=predictions, references=references, max_order=4
+    )
+    results["rouge_scores"] = rouge.compute(
+        predictions=predictions, references=references
+    )
+
+    correct = [1 if ref == pred else 0 for ref, pred in zip(references, predictions)]
+    accuracy = sum(correct) / len(references)
+
+    results["accuracy"] = accuracy
+    if debug:
+        correct_ids = [i for i, c in enumerate(correct) if c == 1]
+        results["correct_ids"] = correct_ids
+
+    return results
+
+
+def save_results(model_name, results_path, dataset, predictions, debug=False):
+    if not os.path.exists(results_path):
+        # Get the directory part of the file path
+        dir_path = os.path.dirname(results_path)
+
+        # Create all directories in the path (if they don't exist)
+        os.makedirs(dir_path, exist_ok=True)
+        df = dataset.to_pandas()
+        df.drop(columns=["text", "prompt"], inplace=True, errors="ignore")
+    else:
+        df = pd.read_csv(results_path, on_bad_lines="warn")
+
+    df[model_name] = predictions
+
+    if debug:
+        print(df.head(1))
+
+    df.to_csv(results_path, index=False)
+
+
+system_prompt = "You are a helpful assistant that translates Chinese to English."
+
+
+def get_few_shot_prompt(dataset, num_shots=5):
+    translation_prompt = "You will be given a Chinese sentence to translate. If it is an incomplete sentence, or if you are unsure about the meaning, simply copy the input text as your output. Do not output any additional sentence such as explanation or reasoning.\n\n"
+    if num_shots > 0:
+        example_translations = "Example Translations:\n"
+        for i in range(num_shots):
+            example_translations += f"Chinese: {dataset[i]['chinese']}\n"
+            example_translations += f"English: {dataset[i]['english']}\n"
+        translation_prompt = translation_prompt + example_translations + "\n"
+
+    translation_prompt = translation_prompt + "Chinese: {input}\nEnglish:"
+    return translation_prompt
+
+
+def load_translation_dataset(
+    data_path, tokenizer=None, num_shots=0, for_openai=False, using_chat_template=True
+):
+    train_data_file = data_path.replace(".tsv", "-train.tsv")
+    test_data_file = data_path.replace(".tsv", "-test.tsv")
+
+    if not os.path.exists(train_data_file):
+        print("generating train/test data files")
+        dataset = load_dataset(
+            "csv", data_files=data_path, delimiter="\t", split="train"
+        )
+        print(len(dataset))
+        dataset = dataset.filter(lambda x: x["chinese"] and x["english"])
+
+        datasets = dataset.train_test_split(test_size=0.2)
+        print(len(dataset))
+
+        # Convert to pandas DataFrame
+        train_df = pd.DataFrame(datasets["train"])
+        test_df = pd.DataFrame(datasets["test"])
+
+        # Save to TSV
+        train_df.to_csv(train_data_file, sep="\t", index=False)
+        test_df.to_csv(test_data_file, sep="\t", index=False)
+
+    print("loading train/test data files")
+    datasets = load_dataset(
+        "csv",
+        data_files={"train": train_data_file, "test": test_data_file},
+        delimiter="\t",
+    )
+
+    if tokenizer or for_openai:
+        translation_prompt = get_few_shot_prompt(datasets["train"], num_shots)
+
+        def formatting_prompts_func(examples):
+            inputs = examples["chinese"]
+            outputs = examples["english"]
+
+            messages = [
+                {
+                    "role": "system",
+                    "content": system_prompt,
+                },
+                None,
+            ]
+
+            model_name = os.getenv("MODEL_NAME")
+
+            # if "mistral" in model_name.lower():
+            # messages = messages[1:]
+
+            texts = []
+            prompts = []
+            for input, output in zip(inputs, outputs):
+                prompt = translation_prompt.format(input=input)
+                messages[-1] = {"role": "user", "content": prompt}
+
+                if for_openai:
+                    prompts.append(messages.copy())
+                    text = messages.copy()
+                    text.append(
+                        {
+                            "role": "assistant",
+                            "content": output,
+                        }
+                    )
+                    texts.append(text)
+                else:
+                    prompt = (
+                        tokenizer.apply_chat_template(
+                            messages, tokenize=False, add_generation_prompt=True
+                        )
+                        if using_chat_template
+                        else prompt
+                    )
+
+                    prompts.append(prompt)
+                    texts.append(prompt + output + tokenizer.eos_token)
+
+            return {"text": texts, "prompt": prompts}
+
+        datasets = datasets.map(
+            formatting_prompts_func,
+            batched=True,
+        )
+
+    print(datasets)
+    return datasets
+
+
+def count_entries_with_max_tokens(entries, max_tokens):
+    """
+    Count the number of entries with the max output tokens or more.
+
+    Parameters:
+    entries (list of int): List of token counts for each entry.
+    max_tokens (int): The maximum token threshold.
+
+    Returns:
+    int: The number of entries with token counts greater than or equal to max_tokens.
+    """
+    count = 0
+    for tokens in entries:
+        if tokens >= max_tokens:
+            count += 1
+    return count
+
+
+def detect_repetition_scores(row, col, debug=False):
+    # print(f"row: {row}")
+    text = row[col] if isinstance(row[col], str) else ""
+    newline_score, repetition_score, total_repetitions = detect_repetitions(
+        text, debug=debug
+    )
+    newline_score -= row["ground_truth_ews_score"]
+    repetition_score -= row["ground_truth_repetition_score"]
+    total_repetitions -= row["ground_truth_total_repetitions"]
+
+    return pd.Series(
+        [
+            newline_score if newline_score > 0 else 0,
+            repetition_score if repetition_score > 0 else 0,
+            total_repetitions if total_repetitions > 0 else 0,
+            len(text),
+        ]
+    )
+
+
+def count_chinese_characters(text):
+    if isinstance(text, str) is False:
+        return 0
+
+    # Define a regular expression pattern for Chinese characters
+    chinese_char_pattern = r"[\u4e00-\u9fff]"
+
+    # Use re.findall to find all Chinese characters in the text
+    chinese_chars = re.findall(chinese_char_pattern, text)
+
+    # Return the count of Chinese characters
+    return len(chinese_chars)
+
+
+def get_metrics(df, max_output_tokens=2048, variant="rpp", existing_metrics_df=None):
+    metrics_df = pd.DataFrame(df.columns.T)[2:]
+    metrics_df.rename(columns={0: "model"}, inplace=True)
+    metrics_df[variant] = metrics_df["model"].apply(
+        lambda x: x.split(f"{variant}-")[-1]
+    )
+    metrics_df["model"] = metrics_df["model"].apply(
+        lambda x: x.split(f"/{variant}-")[0].split("/checkpoint")[0]
+    )
+
+    metrics_df.reset_index(inplace=True)
+    metrics_df = metrics_df.drop(columns=["index"])
+
+    models = [
+        model
+        for model in metrics_df["model"].unique()
+        if ("/" in model or "gpt" in model)
+        and "ground_truth_" not in model
+        and "count_" not in model
+        and "output_" not in model
+    ]
+    print(models)
+
+    tokenizers = {model: load_tokenizer(model) for model in models}
+
+    comet = []
+    meteor = []
+    spbleu = []
+    bleu_1 = []
+    rouge_l = []
+    ews_score = []
+    repetition_score = []
+    total_repetitions = []
+    nrr = []
+    num_max_output_tokens = []
+    translation_completeness = []
+    columns = df.columns[2:]
+
+    df[
+        [
+            "ground_truth_ews_score",
+            "ground_truth_repetition_score",
+            "ground_truth_total_repetitions",
+        ]
+    ] = df["english"].apply(detect_scores)
+
+    new_col = f"count_chinese_characters-ground_truth"
+    df[new_col] = df["chinese"].apply(count_chinese_characters)
+
+    for col in columns:
+        metrics = None
+        if existing_metrics_df is not None:
+            parts = col.split(f"/{variant}-")
+            if len(parts) == 1:
+                break
+            print(parts)
+            val = float(parts[1]) if variant == "rpp" else int(parts[1])
+            result = existing_metrics_df[
+                existing_metrics_df["model"] == parts[0].split("/checkpoint")[0]
+            ]
+
+            for i, row in result.iterrows():
+                # print(i, row[variant], val)
+                if row[variant] == val:
+                    print(f"Using existing metrics for {col}")
+                    metrics = row.to_dict()
+                    # print(metrics)
+                    break
+
+        if metrics is None:
+            print(f"Calculating metrics for {col}")
+            metrics = calc_metrics(
+                df["english"], df[col], sources=df["chinese"], debug=True
+            )
+        print(f"{col}: {metrics}")
+
+        comet.append(metrics["comet"])
+        meteor.append(metrics["meteor"])
+        spbleu.append(
+            metrics["spbleu"] if "spbleu" in metrics else metrics["sacrebleu"]["score"]
+        )
+        bleu_1.append(
+            metrics["bleu_1"] if "bleu_1" in metrics else metrics["bleu_scores"]["bleu"]
+        )
+        rouge_l.append(
+            metrics["rouge_l"]
+            if "rouge_l" in metrics
+            else metrics["rouge_scores"]["rougeL"]
+        )
+
+        df[["ews_score", "repetition_score", "total_repetitions", "answer_len"]] = df.apply(
+            lambda x: detect_repetition_scores(x, col), axis=1
+        )
+        ews_score.append(df["ews_score"].mean())
+        repetition_score.append(df["repetition_score"].mean())
+        total_repetitions.append(df["total_repetitions"].mean())
+
+        nrr.append(1 - df["total_repetitions"].mean() / df["answer_len"].mean())
+
+        model = col.split(f"/{variant}")[0].split("/checkpoint")[0]
+
+        new_col = f"ground_truth_tokens-{model}"
+        df[new_col] = df["english"].apply(
+            lambda x: len(tokenizers[model](x)["input_ids"])
+        )
+
+        new_col = f"count_chinese_characters-{col}"
+        df[new_col] = df[col].apply(
+            lambda x: 1 if count_chinese_characters(x) > 0 else 0
+        )
+        translation_completeness.append(1 - df[new_col].sum() / len(df))
+
+        new_col = f"output_tokens-{col}"
+        df[new_col] = df[col].apply(
+            lambda x: (
+                len(tokenizers[model](x)["input_ids"]) if isinstance(x, str) else 0
+            )
+        )
+
+        num_max_output_tokens.append(
+            count_entries_with_max_tokens(df[new_col], max_output_tokens)
+        )
+
+    metrics_df["comet"] = comet
+    metrics_df["meteor"] = meteor
+    metrics_df["spbleu"] = spbleu
+    metrics_df["bleu_1"] = bleu_1
+    metrics_df["rouge_l"] = rouge_l
+    metrics_df["ews_score"] = ews_score
+    metrics_df["repetition_score"] = repetition_score
+    metrics_df["total_repetitions"] = total_repetitions
+    metrics_df["nrr"] = nrr
+    metrics_df["rap"] = metrics_df.apply(
+        lambda x: x["comet"] * math.exp(x["nrr"] - 1), axis=1
+    )
+
+    metrics_df["translation_completeness"] = translation_completeness
+    metrics_df["num_max_output_tokens"] = num_max_output_tokens
+
+    if variant != "rpp":
+        metrics_df[variant] = metrics_df[variant].astype(int)
+
+    return metrics_df
+
+
+def analyze_translation_results(df, col, max_new_tokens=300, repetition_threshold=100):
+    df[["ews_score", "repetition_score", "total_repetitions", "answer_len"]] = df.apply(
+        lambda x: detect_repetition_scores(x, col), axis=1
+    )
+    rows = df.query(f"total_repetitions > {repetition_threshold}")
+    print(
+        f"*** Found {len(rows)} rows with total_repetitions > {repetition_threshold} for {col}"
+    )
+
+    for i in range(len(rows)):
+        row = rows.iloc[i]
+        print(row["chinese"])
+        print("=" * 80)
+        print(row["english"])
+        print("=" * 80)
+        output = row[col]
+        print(output)
+        print("=" * 80)
+        detect_repetitions(output, debug=True)
+
+    output_tokens = f"output_tokens-{col}"
+    df2 = df[df[output_tokens] >= max_new_tokens][
+        ["chinese", "english", col, output_tokens]
+    ]
+
+    print(
+        f"\n*** Found {len(df2)} rows with output_tokens >= {max_new_tokens} for {col}"
+    )
+    print_row_details(df2, range(len(df2)))
+
+    count_chinese_characters = f"count_chinese_characters-{col}"
+    df3 = df[df[count_chinese_characters] > 0][
+        ["chinese", "english", col, count_chinese_characters]
+    ]
+
+    print(f"\n*** Found {len(df3)} rows with incomplete translations for {col}")
+    print_row_details(df3, range(len(df3)))
+
+
+def plot_metrics(metrics_df, figsize=(14, 5), ylim=(0, 0.44)):
+    plt.figure(figsize=figsize)
+    df_melted = pd.melt(
+        metrics_df, id_vars="model", value_vars=["meteor", "bleu_1", "rouge_l"]
+    )
+
+    barplot = sns.barplot(x="variable", y="value", hue="model", data=df_melted)
+
+    # Set different hatches for each model
+    hatches = ["/", "\\", "|", "-", "+", "x", "o", "O", ".", "*", "//", "\\\\"]
+
+    # Create a dictionary to map models to hatches
+    model_hatches = {
+        model: hatches[i % len(hatches)]
+        for i, model in enumerate(metrics_df["model"].unique())
+    }
+
+    # Apply hatches based on the model
+    num_vars = len(df_melted["variable"].unique())
+    for i, bar in enumerate(barplot.patches):
+        model = df_melted["model"].iloc[i // num_vars]
+        bar.set_hatch(model_hatches[model])
+
+    # Manually update legend to match the bar hatches
+    handles, labels = barplot.get_legend_handles_labels()
+    for handle, model in zip(handles, metrics_df["model"].unique()):
+        handle.set_hatch(model_hatches[model])
+
+    barplot.set_xticklabels(["METEOR", "BLEU-1", "ROUGE-L"])
+    for p in barplot.patches:
+        if p.get_height() == 0:
+            continue
+        barplot.annotate(
+            f"{p.get_height():.2f}",
+            (p.get_x() + p.get_width() / 2.0, p.get_height()),
+            ha="center",
+            va="center",
+            xytext=(0, 10),
+            textcoords="offset points",
+        )
+
+    barplot.set(ylim=ylim, ylabel="Scores", xlabel="Metrics")
+    plt.legend(bbox_to_anchor=(0.5, -0.1), loc="upper center")
+    plt.show()
+
+
+def plot_times(perf_df, ylim=0.421):
+    # Adjusted code to put "train-time" bars in red at the bottom
+
+    fig, ax1 = plt.subplots(figsize=(12, 10))
+
+    color_train = "tab:red"
+    color_eval = "orange"
+    ax1.set_xlabel("Models")
+    ax1.set_ylabel("Time (mins)")
+    ax1.set_xticks(range(len(perf_df["model"])))  # Set x-ticks positions
+    ax1.set_xticklabels(perf_df["model"], rotation=90)
+
+    # Plot "train-time" first so it's at the bottom
+    ax1.bar(
+        perf_df["model"],
+        perf_df["train-time(mins)"],
+        color=color_train,
+        label="train-time",
+    )
+
+    # Then, plot "eval-time" on top of "train-time"
+    ax1.bar(
+        perf_df["model"],
+        perf_df["eval-time(mins)"],
+        bottom=perf_df["train-time(mins)"],
+        color=color_eval,
+        label="eval-time",
+    )
+
+    ax1.tick_params(axis="y")
+    ax1.legend(loc="upper left")
+
+    if "meteor" in perf_df.columns:
+        ax2 = ax1.twinx()
+        color_meteor = "tab:blue"
+        ax2.set_ylabel("METEOR", color=color_meteor)
+        ax2.plot(
+            perf_df["model"],
+            perf_df["meteor"],
+            color=color_meteor,
+            marker="o",
+            label="meteor",
+        )
+        ax2.tick_params(axis="y", labelcolor=color_meteor)
+        ax2.legend(loc="upper right")
+        ax2.set_ylim(ax2.get_ylim()[0], ylim)
+
+    # Show numbers in bars
+    for p in ax1.patches:
+        height = p.get_height()
+        if height == 0:  # Skip bars with height 0
+            continue
+        ax1.annotate(
+            f"{height:.2f}",
+            (p.get_x() + p.get_width() / 2.0, p.get_y() + height),
+            ha="center",
+            va="center",
+            xytext=(0, -10),
+            textcoords="offset points",
+        )
+
+    fig.tight_layout()
+    plt.show()
+
+
+def translate_via_openai(
+    text, translation_prompt, max_tokens=None, model="gpt-4o-mini", base_url=None
+):
+    llm = ChatOpenAI(
+        model=model,
+        temperature=0,
+        max_tokens=max_tokens,
+        timeout=None,
+        max_retries=2,
+        base_url=base_url,
+    )
+
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                "You are a helpful assistant that translates Chinese to English.",
+            ),
+            (
+                "human",
+                translation_prompt,
+            ),
+        ]
+    )
+
+    chain = prompt | llm
+    response = chain.invoke(
+        {
+            "input": text,
+        }
+    )
+
+    return response.content
+
+
+def eval_openai(num_shots, datasets, model="gpt-4o-mini", max_new_tokens=300):
+    translation_prompt = get_few_shot_prompt(datasets["train"], num_shots=num_shots)
+    eval_dataset = datasets["test"]
+    total = len(eval_dataset)
+    predictions = []
+
+    for i in tqdm(range(total)):
+        output = translate_via_openai(
+            eval_dataset["chinese"][i],
+            translation_prompt,
+            model=model,
+            max_tokens=max_new_tokens,
+        )
+        predictions.append(output)
+
+    return predictions
+
+
+def convert_time_to_seconds(time_str):
+    # print(f"converting time_str: {time_str}")
+    # Split the time string into its components
+    time_parts = list(map(int, time_str.split(":")))
+
+    # Initialize total minutes
+    total_seconds = 0
+
+    # Calculate total minutes based on the number of parts
+    if len(time_parts) == 3:  # HH:MM:SS
+        hours, minutes, seconds = time_parts
+        total_seconds = hours * 3600 + minutes * 60 + seconds
+    elif len(time_parts) == 2:  # MM:SS
+        minutes, seconds = time_parts
+        total_seconds = minutes * 60 + seconds
+    elif len(time_parts) == 1:  # SS
+        seconds = time_parts[0]
+        total_seconds = seconds
+
+    return total_seconds
+
+
+def process_log_file(log_file, total_entries, variant):
+    time_pattern = re.compile(r"\[(.{5,10})<00:00")
+    metrics_pattern = re.compile(rf"(.*)/{variant}-(.*) metrics:")
+
+    model = []
+    shots = []
+    eval_time = []
+
+    i = 0
+
+    with open(log_file, "r") as f:
+        try:
+            for line in f:
+                i += 1
+                matches = time_pattern.search(line)
+                if matches:
+                    time_pattern_matches = matches
+                else:
+                    matches = metrics_pattern.search(line)
+                    if matches:
+                        metrics_pattern_matches = matches
+                        groups = metrics_pattern_matches.groups()
+
+                        model.append(groups[0].split("/checkpoint")[0])
+                        shots.append(groups[1])
+
+                        groups = time_pattern_matches.groups()
+                        time_str = groups[0]
+                        eval_time.append(
+                            convert_time_to_seconds(time_str) / total_entries
+                        )
+        except Exception as e:
+            print(f"Error processing log file: {log_file} at line {i}: {line}")
+            print(e)
+
+    df = pd.DataFrame(
+        {
+            "model": model,
+            variant: shots,
+            "eval_time": eval_time,
+        }
+    )
+    return df
+
+
+def load_eval_times(logs_folder, total_entries=1133, variant="shots"):
+    # Get a list of all files in the logs folder
+    log_files = glob.glob(os.path.join(logs_folder, "*"))
+    log_files.sort()
+
+    time_df = pd.DataFrame({"model": [], variant: [], "eval_time": []})
+
+    for log_file in log_files:
+        print(f"Loading content of {log_file}")
+        df = process_log_file(log_file, total_entries, variant)
+        time_df = pd.concat([time_df, df], ignore_index=True)
+
+    time_df[variant] = time_df[variant].apply(
+        lambda x: x if variant == "rpp" else int(x)
+    )
+    # Keep the last occurrence of each duplicate
+    return time_df.drop_duplicates(subset=["model", variant], keep="last")
+
+
+def load_alpaca_data(data_path):
+    alpaca_data_path = "data/alpaca_mac.json"
+
+    if os.path.exists(alpaca_data_path):
+        print("loading existing data from:", alpaca_data_path)
+        data = pd.read_json(alpaca_data_path, orient="records", lines=False)
+        return data
+
+    datasets = load_translation_dataset(data_path)
+    prompt_template = get_few_shot_prompt(datasets["train"], num_shots=0)
+
+    df_train = datasets["train"].to_pandas()
+    df_train["instruction"] = df_train.apply(
+        lambda x: prompt_template.format(input=x["chinese"]), axis=1
+    )
+
+    df_alpaca = pd.DataFrame(
+        {
+            "system": [system_prompt] * len(df_train),
+            "instruction": df_train["instruction"].to_list(),
+            "input": [""] * len(df_train),
+            "output": df_train["english"].to_list(),
+        }
+    )
+
+    df_alpaca.to_json(alpaca_data_path, orient="records", lines=False, indent=2)
+
+    return df_alpaca
+
+
+def load_openai_training_data(
+    data_path, openai_data_path="datasets/mac/openai-training.jsonl"
+):
+    if os.path.exists(openai_data_path):
+        print("loading existing data from:", openai_data_path)
+        data = pd.read_json(openai_data_path, orient="records", lines=True)
+        return data
+
+    datasets = load_translation_dataset(data_path)
+    prompt_template = get_few_shot_prompt(datasets["train"], num_shots=0)
+
+    df_train = datasets["train"].to_pandas()
+    messages = []
+
+    for i, row in df_train.iterrows():
+        messages.append(
+            [
+                {
+                    "role": "system",
+                    "content": system_prompt,
+                },
+                {
+                    "role": "user",
+                    "content": prompt_template.format(input=row["chinese"]),
+                },
+                {
+                    "role": "assistant",
+                    "content": row["english"],
+                },
+            ]
+        )
+
+    df_openai = pd.DataFrame(
+        {
+            "messages": messages,
+        }
+    )
+    df_openai.to_json(openai_data_path, orient="records", lines=True)
+    return df_openai

results/mac-results_rpp_with_mnt_2048_generic_prompt_metrics.csv

@@ -1,21 +1,25 @@
-model,rpp,comet,meteor,spbleu,bleu_1,rouge_l,ews_score,repetition_score,total_repetitions,rap,translation_completeness,num_max_output_tokens
-Qwen/Qwen2-72B-Instruct,1.00,0.7457879471336964,0.4655033970145371,11.09789843749336,0.1109789843749336,0.4309698494389731,0.0,48.81906443071492,48.81906443071492,0.4214638674295989,0.9902912621359223,10
-internlm/internlm2_5-7b-chat,1.00,0.7357995069773978,0.4297612514398102,15.060226683930628,0.1506022668393063,0.4097577795330234,0.0,11.742277140335393,11.742277140335393,0.5502106789386991,1.0,2
-internlm/internlm2_5-7b-chat,1.02,0.7377187550620283,0.4246676977198055,14.728605282752795,0.147286052827528,0.4063246630867048,0.0,5.420123565754634,5.420123565754634,0.6209294646606592,1.0,1
-internlm/internlm2_5-7b-chat,1.04,0.7371160490183523,0.4173352728374962,13.846403511622256,0.1384640351162226,0.3988121301027288,0.0,5.3097969991173875,5.3097969991173875,0.6220549062227617,1.0,1
-internlm/internlm2_5-7b-chat,1.06,0.7338597697698218,0.3997609847704189,12.213374588416173,0.1221337458841617,0.3841365748920261,0.0,5.316857899382171,5.316857899382171,0.6192022797578944,1.0,1
-internlm/internlm2_5-7b-chat,1.08,0.7318234702626478,0.3881614120395272,11.369735763522288,0.1136973576352228,0.372963223209074,0.0,5.340688437775817,5.340688437775817,0.6171325620248217,1.0,1
-internlm/internlm2_5-7b-chat,1.10,0.7288648442604431,0.3784182249483568,10.377989030628608,0.103779890306286,0.3618424457502351,0.0,5.316857899382171,5.316857899382171,0.6149877562344177,1.0,1
-microsoft/Phi-3.5-mini-instruct,1.00,0.710605339281136,0.3788926591792472,9.70032874202361,0.097003287420236,0.3556134739443916,1.7987643424536628,16.421006178287733,18.219770520741395,0.4898856540979652,1.0,4
-microsoft/Phi-3.5-mini-instruct,1.02,0.7150978385770836,0.3741049510326346,9.910633597905436,0.0991063359790543,0.3453160556383774,0.0,10.690203000882612,10.690203000882612,0.543484568672847,1.0,2
-microsoft/Phi-3.5-mini-instruct,1.04,0.7074641684778791,0.3538698731015666,9.19721270538052,0.0919721270538052,0.3225824135517728,0.0,0.10944395410414828,0.10944395410414828,0.7041355304734901,1.0,0
-microsoft/Phi-3.5-mini-instruct,1.06,0.6962301708225224,0.3252854575717334,6.967166383106307,0.069671663831063,0.2948764736589108,0.0,0.09355692850838482,0.09355692850838482,0.6934257926979845,1.0,0
-microsoft/Phi-3.5-mini-instruct,1.08,0.6823413657174107,0.301599095293242,5.452744292893752,0.0545274429289375,0.2726387617958179,0.0,0.0264783759929391,0.0264783759929391,0.6815586491033614,1.0,0
-microsoft/Phi-3.5-mini-instruct,1.10,0.6717851540206916,0.2885734336603344,4.751039447225815,0.0475103944722581,0.2604284999048123,0.0,0.04589585172109444,0.04589585172109444,0.670451845604288,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.739080294072365,0.4490104515425626,6.7013404492782405,0.06701340449278238,0.41961816376805966,0.0,144.23654015887027,144.23654015887027,0.33775915922136823,0.999117387466902,15
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.00,0.7222260562908512,0.4039898602650971,13.461179673541356,0.1346117967354136,0.3819960428004565,0.0,5.837599293909974,5.837599293909974,0.6020108921217914,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.02,0.723643534970515,0.4051102919608809,13.18537912294539,0.1318537912294539,0.3824621732976229,0.0,5.7855251544571935,5.7855251544571935,0.6039124380139065,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.04,0.7238812581796301,0.4039456988919502,13.314773371306682,0.1331477337130668,0.3813737464821349,0.0,5.804060017652251,5.804060017652251,0.6038540012915103,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.06,0.7252625281686607,0.4012797167602334,13.19924345265053,0.1319924345265053,0.3798291332004637,0.0,5.795233892321271,5.795233892321271,0.6051287090305499,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.08,0.7261167238322592,0.3987395126194482,12.656486100206328,0.1265648610020633,0.376975448872996,0.0,5.804060017652251,5.804060017652251,0.6057188028233036,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.10,0.7264630642225547,0.3964859769229444,12.284961706379857,0.1228496170637985,0.3744555065346823,0.0,0.16328331862312445,0.16328331862312445,0.7213887920596253,1.0,0
+model,rpp,comet,meteor,spbleu,bleu_1,rouge_l,ews_score,repetition_score,total_repetitions,nrr,rap,translation_completeness,num_max_output_tokens
+internlm/internlm2_5-7b-chat,1.00,0.7357995069773978,0.4297612514398102,15.060226683930628,0.1506022668393063,0.4097577795330234,0.04942630185348632,9.235657546337158,9.285083848190645,0.9247496423462088,0.6824623187873116,1.0,2
+internlm/internlm2_5-7b-chat,1.02,0.7377187550620283,0.4246676977198055,14.728605282752795,0.147286052827528,0.4063246630867048,0.06972639011473963,5.35657546337158,5.4210061782877315,0.953789668507456,0.7044041934116965,1.0,1
+internlm/internlm2_5-7b-chat,1.04,0.7371160490183523,0.4173352728374962,13.846403511622256,0.1384640351162226,0.3988121301027288,0.06884377758164166,5.315092674315975,5.383053839364519,0.9549885977018281,0.7046730517422506,1.0,1
+internlm/internlm2_5-7b-chat,1.06,0.7338597697698218,0.3997609847704189,12.213374588416173,0.1221337458841617,0.3841365748920261,0.05825242718446602,5.275375110326567,5.332744924977935,0.9561764257893248,0.7023939203825759,1.0,1
+internlm/internlm2_5-7b-chat,1.08,0.7318234702626478,0.3881614120395272,11.369735763522288,0.1136973576352228,0.372963223209074,0.06707855251544571,5.283318623124448,5.345101500441306,0.9570359334539392,0.7010472282786626,1.0,1
+internlm/internlm2_5-7b-chat,1.10,0.7288648442604431,0.3784182249483568,10.377989030628608,0.103779890306286,0.3618424457502351,0.05207413945278023,5.288614298323036,5.340688437775817,0.957823935317488,0.6987634348896543,1.0,1
+microsoft/Phi-3.5-mini-instruct,1.00,0.710605339281136,0.3788926591792472,9.70032874202361,0.097003287420236,0.3556134739443916,5.390997352162401,12.997352162400706,18.368049426301855,0.8624429902835613,0.6192816563968827,1.0,4
+microsoft/Phi-3.5-mini-instruct,1.02,0.7150978385770836,0.3741049510326346,9.910633597905436,0.0991063359790543,0.3453160556383774,3.586054721977052,7.001765225066196,10.567519858781994,0.9183516206245184,0.6590312746582365,1.0,2
+microsoft/Phi-3.5-mini-instruct,1.04,0.7074641684778791,0.3538698731015666,9.19721270538052,0.0919721270538052,0.3225824135517728,0.05119152691968226,0.05560458958517211,0.10150044130626655,0.9991834532118691,0.7068867266696017,1.0,0
+microsoft/Phi-3.5-mini-instruct,1.06,0.6962301708225224,0.3252854575717334,6.967166383106307,0.069671663831063,0.2948764736589108,0.0353045013239188,0.06796116504854369,0.09796999117387467,0.9992538065947363,0.6957108422443861,1.0,0
+microsoft/Phi-3.5-mini-instruct,1.08,0.6823413657174107,0.301599095293242,5.452744292893752,0.0545274429289375,0.2726387617958179,0.07678729037952339,0.04766107678729038,0.11297440423654016,0.9991814653050001,0.6817830741574145,1.0,0
+microsoft/Phi-3.5-mini-instruct,1.10,0.6717851540206916,0.2885734336603344,4.751039447225815,0.0475103944722581,0.2604284999048123,0.08031774051191527,0.02383053839364519,0.10414827890556046,0.999281171568508,0.6713024292710505,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.739080294072365,0.4490104515425626,6.7013404492782405,0.0670134044927823,0.4196181637680596,0.36716681376875554,139.80935569285083,140.15798764342455,0.5164419894213406,0.4557063153750335,0.999117387466902,15
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.02,0.743018615750854,0.4514907128972251,8.545954556237808,0.085459545562378,0.4214940415288087,1.0035304501323918,67.00353045013239,67.98852603706973,0.7071153729164131,0.5543722941592363,1.0,6
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.04,0.7432195577780335,0.4517500968367987,10.080425294411064,0.1008042529441106,0.4200973007348334,0.01059135039717564,35.19770520741395,35.18358340688438,0.8244802169835412,0.6235766669370041,1.0,6
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.06,0.7430821573139815,0.4484154407825542,10.37470506193322,0.1037470506193321,0.4160289393328045,1.8005295675198587,26.880847308031775,28.656663724624888,0.8478345432646117,0.6381932626507077,1.0,3
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.08,0.7435937259684909,0.4407733547418294,10.930453247368872,0.1093045324736887,0.4113063412348818,0.09267431597528684,12.007943512797882,12.072374227714034,0.9329421050825354,0.6953650250037441,1.0,3
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.10,0.7427059700687901,0.4358940590119784,11.381344076286156,0.1138134407628615,0.4062980635945339,0.0176522506619594,11.914386584289497,11.905560458958517,0.9312420672746087,0.6933551155892559,1.0,3
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.00,0.7222260562908512,0.4039898602650971,13.461179673541356,0.1346117967354136,0.3819960428004565,0.05736981465136805,5.87378640776699,5.9179170344218885,0.9486112388485238,0.6860492554476049,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.02,0.723643534970515,0.4051102919608809,13.18537912294539,0.1318537912294539,0.3824621732976229,0.06266548984995587,5.840247131509267,5.8914386584289495,0.9486127363429205,0.6873967610154514,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.04,0.7238812581796301,0.4039456988919502,13.314773371306682,0.1331477337130668,0.3813737464821349,0.05736981465136805,5.845542806707855,5.889673433362754,0.948840810819099,0.6877794238881113,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.06,0.7252625281686607,0.4012797167602334,13.19924345265053,0.1319924345265053,0.3798291332004637,0.06266548984995587,5.847308031774051,5.884377758164166,0.9494061847846709,0.6894815110844906,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.08,0.7261167238322592,0.3987395126194482,12.656486100206328,0.1265648610020633,0.376975448872996,0.05648720211827008,5.820829655781112,5.864077669902913,0.9499856972945303,0.6906937144718889,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.10,0.7264630642225547,0.3964859769229444,12.284961706379857,0.1228496170637985,0.3744555065346823,0.04942630185348632,0.09267431597528684,0.12886142983230361,0.9988510902838437,0.7256289030293478,1.0,0

results/mac-results_rpp_with_mnt_2048_metrics.csv

@@ -1,43 +1,31 @@
-model,rpp,comet,meteor,spbleu,bleu_1,rouge_l,ews_score,repetition_score,total_repetitions,rap,translation_completeness,num_max_output_tokens
-Qwen/Qwen2-72B-Instruct,1.00,0.7570169081439618,0.4789208192601603,18.953782447211417,0.1895378244721141,0.4520943414365094,0.0,0.09885260370697264,0.09885260370697264,0.7537966634953808,1.0,0
-Qwen/Qwen2-72B-Instruct,1.02,0.7573581492858762,0.4796836675632054,19.0137490702917,0.190137490702917,0.4520458023287854,0.0,0.08120035304501325,0.08120035304501325,0.7547074311557754,1.0,0
-Qwen/Qwen2-72B-Instruct,1.04,0.7574983964372501,0.476362681282195,18.52063321160408,0.1852063321160408,0.4489040568717591,0.0,0.10503089143865843,0.10503089143865843,0.7540766638515739,1.0,0
-Qwen/Qwen2-72B-Instruct,1.06,0.7563851308422812,0.4679636383950649,17.982473951038504,0.1798247395103851,0.441062203630729,0.0,0.07325684024713151,0.07325684024713151,0.7539950363948406,1.0,0
-Qwen/Qwen2-72B-Instruct,1.08,0.7554470169417962,0.4597578197711645,17.067954025424825,0.1706795402542483,0.4317496244624469,0.0,0.04766107678729038,0.04766107678729038,0.753890250753016,1.0,0
-Qwen/Qwen2-72B-Instruct,1.10,0.7550096405046973,0.4515778511124262,16.22452191616505,0.1622452191616504,0.4242083184666598,0.0,0.05736981465136805,0.05736981465136805,0.7531385283434515,1.0,0
-Qwen/Qwen2-7B-Instruct,1.00,0.7457721890965501,0.442240791493943,14.38814929350883,0.1438814929350883,0.4160546438584113,0.0,12.81288614298323,12.81288614298323,0.5490966301804292,0.9947043248014121,2
-Qwen/Qwen2-7B-Instruct,1.02,0.7474080664545482,0.4400998640836595,15.16172261831792,0.1516172261831792,0.4160749445105633,0.0,7.1562224183583405,7.1562224183583405,0.605472251398711,0.9947043248014121,1
-Qwen/Qwen2-7B-Instruct,1.04,0.7484375637974869,0.4390136558190875,14.958631815014014,0.1495863181501401,0.4138111632957468,0.0,0.1853486319505737,0.1853486319505737,0.742515336500048,0.999117387466902,0
-Qwen/Qwen2-7B-Instruct,1.06,0.7471612840233287,0.4328321576515084,14.28087386760537,0.1428087386760537,0.4069074178015074,0.0,0.2030008826125331,0.2030008826125331,0.7406965407448669,0.9982347749338041,0
-Qwen/Qwen2-7B-Instruct,1.08,0.74519401609845,0.423560805217557,13.659683698817108,0.1365968369881711,0.3966315425573522,0.0,0.22153574580759047,0.22153574580759047,0.7381694832783203,1.0,0
-Qwen/Qwen2-7B-Instruct,1.10,0.7432071187016402,0.4135053136541433,12.922649874705083,0.1292264987470507,0.3876193185383525,0.0,0.17740511915269197,0.17740511915269197,0.737574218462684,1.0,0
-internlm/internlm2_5-7b-chat,1.00,0.739699612254078,0.4289996929258777,14.734881589173108,0.1473488158917311,0.4096466800937898,0.0,12.751103265666373,12.751103265666373,0.5450982095784719,1.0,2
-internlm/internlm2_5-7b-chat,1.02,0.740223803961056,0.4266246904302194,14.583816688798017,0.1458381668879802,0.4071727106228415,0.0,9.824360105913504,9.824360105913504,0.5706323412193933,1.0,1
-internlm/internlm2_5-7b-chat,1.04,0.7398856264610577,0.4154585167056314,13.534659133050225,0.1353465913305021,0.3968657713589718,0.0,6.527802294792586,6.527802294792586,0.6073521998554903,1.0,1
-internlm/internlm2_5-7b-chat,1.06,0.7379362287241489,0.4039588647855378,12.346740971499404,0.1234674097149939,0.3872447044295494,0.0,6.533980582524272,6.533980582524272,0.6056712925221626,0.999117387466902,1
-internlm/internlm2_5-7b-chat,1.08,0.7319988705684732,0.3873176839854818,11.075674965706344,0.1107567496570634,0.3724352909668609,0.0,9.820829655781113,9.820829655781113,0.5643254582029298,0.999117387466902,1
-internlm/internlm2_5-7b-chat,1.10,0.7295350462119345,0.3769306874386757,10.305163787094209,0.1030516378709421,0.3634496155759507,0.0,6.525154457193292,6.525154457193292,0.5988898943353679,0.999117387466902,1
-microsoft/Phi-3.5-mini-instruct,1.00,0.7107840433177544,0.3796831545348129,8.71296896471494,0.0871296896471493,0.3589874395901284,0.0,28.42630185348632,28.42630185348632,0.4485492990665038,1.0,6
-microsoft/Phi-3.5-mini-instruct,1.02,0.7164765837070485,0.3780585837553919,10.291240080163629,0.1029124008016362,0.3546952732427276,0.0,10.696381288614297,10.696381288614297,0.5444787777612768,1.0,2
-microsoft/Phi-3.5-mini-instruct,1.04,0.7111233387336411,0.3547161333845742,8.966881655527896,0.0896688165552789,0.3300979657678754,0.0,3.7484554280670785,3.7484554280670785,0.6247493045810047,1.0,1
-microsoft/Phi-3.5-mini-instruct,1.06,0.7024363270136286,0.3298733737040869,7.076233088011138,0.0707623308801113,0.3019513312669543,0.0,0.10767872903795234,0.10767872903795234,0.6991841213752065,1.0,0
-microsoft/Phi-3.5-mini-instruct,1.08,0.6882111219210848,0.3054541022592767,5.105510599247868,0.0510551059924786,0.2736030007297014,0.0,3.29567519858782,3.29567519858782,0.6124452738619192,1.0,1
-microsoft/Phi-3.5-mini-instruct,1.10,0.6712992989638161,0.2903831801547132,4.091958857999118,0.0409195885799911,0.251653275009876,0.0,0.07766990291262135,0.07766990291262135,0.6690512007506374,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.7501818982248062,0.4611110508507017,17.87914973742753,0.1787914973742752,0.4340662057009564,0.0,0.088261253309797,0.088261253309797,0.747329847981863,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.02,0.7485114382045625,0.4571517219079576,17.436884594979905,0.174368845949799,0.4311385932640979,0.0,0.09267431597528684,0.09267431597528684,0.7455246732136059,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.04,0.7500591586357918,0.4560467960364254,17.440173470996626,0.1744017347099662,0.4302844557731285,0.0,0.13062665489849956,0.13062665489849956,0.7458552866168927,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.06,0.748812871571673,0.4520416361219855,16.89523258317781,0.168952325831778,0.4260026774745837,0.0,0.12533097969991175,0.12533097969991175,0.7447841638756355,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.08,0.7473851635144647,0.4442106511292453,16.16623784482793,0.1616623784482792,0.4195129470585874,0.0,0.18711385701676964,0.18711385701676964,0.7414159053529263,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.10,0.7465709781131172,0.4379837926138161,15.60172257624066,0.1560172257624066,0.4132562932940978,0.0,0.08649602824360106,0.08649602824360106,0.743788967986371,1.0,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.00,0.7426396049131678,0.433632501662176,15.209540658023398,0.1520954065802339,0.4089208235151474,0.0,5.798764342453663,5.798764342453663,0.619577239776096,1.0,1
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.02,0.7436477056353469,0.4329054166518245,15.19102241646024,0.1519102241646024,0.4068967964789407,0.0,5.77846425419241,5.77846425419241,0.6207074516423631,1.0,1
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.04,0.7440943776351209,0.4320478700956207,15.05135166158296,0.1505135166158296,0.4062008380201262,0.0,0.11827007943512798,0.11827007943512798,0.7403141356055205,1.0,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.06,0.7426502735395928,0.4275429314912545,14.449130821290163,0.1444913082129016,0.4001409979222783,0.0,0.176522506619594,0.176522506619594,0.7370491441666694,1.0,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.08,0.7408098006080129,0.4206626658729054,13.933703757385222,0.1393370375738522,0.3964824268676203,0.0,0.21888790820829657,0.21888790820829657,0.7339083936807739,1.0,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.10,0.7392685912871718,0.4111211240399151,13.303738403756984,0.1330373840375698,0.3870959581563503,0.0,0.13857016769638128,0.13857016769638128,0.7348764469929082,1.0,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.00,0.7240239171358935,0.4068335357738006,13.565136550617618,0.1356513655061761,0.3866395067055498,0.0,0.1059135039717564,0.1059135039717564,0.7207261791424407,1.0,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.02,0.7263097057327799,0.4064914781094827,13.42987641622816,0.1342987641622816,0.3863697821025159,0.0,6.238305383936452,6.238305383936452,0.5999878425713037,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.04,0.7276128307708258,0.4054859896994975,13.295092218891954,0.1329509221889195,0.3851203729935697,0.0,0.1297440423654016,0.1297440423654016,0.7235619893938705,1.0,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.06,0.7276865132383193,0.4014727027723293,13.10860799057166,0.1310860799057166,0.3804952786306688,0.0,0.20741394527802295,0.20741394527802295,0.7212559915451495,1.0,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.08,0.726393195584298,0.3987018836449559,12.850537785783194,0.1285053778578319,0.3788945955746495,0.0,0.2903795233892321,0.2903795233892321,0.717473994791502,1.0,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.10,0.7244012304511832,0.3932239948456176,12.361161644811926,0.1236116164481192,0.3733413807007665,0.0,0.1500441306266549,0.1500441306266549,0.7197459635880831,1.0,0
+model,rpp,comet,meteor,spbleu,bleu_1,rouge_l,ews_score,repetition_score,total_repetitions,nrr,rap,translation_completeness,num_max_output_tokens
+internlm/internlm2_5-7b-chat,1.00,0.739699612254078,0.4289996929258777,14.734881589173108,0.1473488158917311,0.4096466800937898,0.05383936451897617,12.606354810238305,12.646954986760813,0.8963919016630513,0.6668973100142501,1.0,2
+internlm/internlm2_5-7b-chat,1.02,0.740223803961056,0.4266246904302194,14.583816688798017,0.1458381668879802,0.4071727106228415,0.06266548984995587,9.849073256840247,9.910856134157106,0.916784004505773,0.6811186927169239,1.0,1
+internlm/internlm2_5-7b-chat,1.04,0.7398856264610577,0.4154585167056314,13.534659133050225,0.1353465913305021,0.3968657713589718,0.07237422771403354,6.529567519858782,6.596646072374227,0.9439099437148217,0.6995278161884907,1.0,1
+internlm/internlm2_5-7b-chat,1.06,0.7379362287241489,0.4039588647855378,12.346740971499404,0.1234674097149939,0.3872447044295494,0.06796116504854369,6.533980582524272,6.596646072374227,0.9449043529541853,0.6983788795529117,0.999117387466902,1
+internlm/internlm2_5-7b-chat,1.08,0.7319988705684732,0.3873176839854818,11.075674965706344,0.1107567496570634,0.3724352909668609,0.05207413945278023,9.83495145631068,9.881729920564872,0.920975175928344,0.6763793903551222,0.999117387466902,1
+internlm/internlm2_5-7b-chat,1.10,0.7295350462119345,0.3769306874386757,10.305163787094209,0.1030516378709421,0.3634496155759507,0.07855251544571933,6.527802294792586,6.596646072374227,0.9470732363646663,0.6919271286047086,0.999117387466902,1
+microsoft/Phi-3.5-mini-instruct,1.00,0.7107840433177544,0.3796831545348129,8.71296896471494,0.0871296896471493,0.3589874395901284,10.670785525154457,17.93821712268314,28.58340688437776,0.7979259092866101,0.5807350079921869,1.0,6
+microsoft/Phi-3.5-mini-instruct,1.02,0.7164765837070485,0.3780585837553919,10.291240080163629,0.1029124008016362,0.3546952732427276,3.585172109443954,7.1403353927625774,10.705207413945278,0.914860909301493,0.6580010154348458,1.0,2
+microsoft/Phi-3.5-mini-instruct,1.04,0.7111233387336411,0.3547161333845742,8.966881655527896,0.0896688165552789,0.3300979657678754,3.6125330979699912,0.07325684024713151,3.685789938217123,0.9702657286890148,0.6902898733301204,1.0,1
+microsoft/Phi-3.5-mini-instruct,1.06,0.7024363270136286,0.3298733737040869,7.076233088011138,0.0707623308801113,0.3019513312669543,0.04589585172109444,0.05207413945278023,0.0970873786407767,0.9992496538175567,0.7019094542904326,1.0,0
+microsoft/Phi-3.5-mini-instruct,1.08,0.6882111219210848,0.3054541022592767,5.105510599247868,0.0510551059924786,0.2736030007297014,3.3609885260370698,0.06443071491615181,3.414827890556046,0.9764915329416268,0.6722210212114398,1.0,1
+microsoft/Phi-3.5-mini-instruct,1.10,0.6712992989638161,0.2903831801547132,4.091958857999118,0.0409195885799911,0.251653275009876,0.32215357458075905,0.06531332744924978,0.3786407766990291,0.9977125977744483,0.6697655223041806,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.7501818982248062,0.4611110508507017,17.87914973742753,0.1787914973742752,0.4340662057009564,0.00706090026478376,0.1262135922330097,0.11650485436893204,0.9990152266843727,0.7494435027453233,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.02,0.7485114382045625,0.4571517219079576,17.436884594979905,0.174368845949799,0.4311385932640979,0.00706090026478376,0.11562224183583407,0.1059135039717564,0.9991036950172912,0.7478408442461326,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.04,0.7500591586357918,0.4560467960364254,17.440173470996626,0.1744017347099662,0.4302844557731285,0.00706090026478376,0.13062665489849956,0.1209179170344219,0.9989818138577363,0.7492958474569671,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.06,0.748812871571673,0.4520416361219855,16.89523258317781,0.168952325831778,0.4260026774745837,0.00706090026478376,0.0997352162400706,0.09002647837599294,0.999249541999897,0.7482511297698003,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.08,0.7473851635144647,0.4442106511292453,16.16623784482793,0.1616623784482792,0.4195129470585874,0.01059135039717564,0.13062665489849956,0.12444836716681378,0.9989631285573515,0.7466106228007235,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.10,0.7465709781131172,0.4379837926138161,15.60172257624066,0.1560172257624066,0.4132562932940978,0.01059135039717564,0.07855251544571933,0.06531332744924978,0.9994618690596525,0.7461693332490859,1.0,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.00,0.7426396049131678,0.433632501662176,15.209540658023398,0.1520954065802339,0.4089208235151474,0.00353045013239188,3.901147396293027,3.889673433362754,0.9677537371860069,0.7190742424186368,1.0,1
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.02,0.7436477056353469,0.4329054166518245,15.19102241646024,0.1519102241646024,0.4068967964789407,0.0,3.8905560458958517,3.8693733451015007,0.9679787303975633,0.7202123788620469,1.0,1
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.04,0.7440943776351209,0.4320478700956207,15.05135166158296,0.1505135166158296,0.4062008380201262,0.00353045013239188,0.1526919682259488,0.13503971756398941,0.9988310348779463,0.7432250654569489,1.0,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.06,0.7426502735395928,0.4275429314912545,14.449130821290163,0.1444913082129016,0.4001409979222783,0.00706090026478376,0.13768755516328332,0.13327449249779347,0.9988583611812559,0.7418029189370554,1.0,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.08,0.7408098006080129,0.4206626658729054,13.933703757385222,0.1393370375738522,0.3964824268676203,0.00353045013239188,0.1297440423654016,0.11738746690203,0.9990003006614552,0.7400695835992323,1.0,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.10,0.7392685912871718,0.4111211240399151,13.303738403756984,0.1330373840375698,0.3870959581563503,0.00353045013239188,0.12180052956751986,0.10944395410414828,0.9990805075005376,0.7385891517800307,1.0,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.00,0.7240239171358935,0.4068335357738006,13.565136550617618,0.1356513655061761,0.3866395067055498,0.0529567519858782,0.1209179170344219,0.1676963812886143,0.9984771126055,0.7229221493862132,1.0,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.02,0.7263097057327799,0.4064914781094827,13.42987641622816,0.1342987641622816,0.3863697821025159,0.06001765225066196,6.236540158870256,6.294792586054722,0.9458252309188138,0.6880088812871149,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.04,0.7276128307708258,0.4054859896994975,13.295092218891954,0.1329509221889195,0.3851203729935697,0.05207413945278023,0.1297440423654016,0.16946160635481025,0.9984590566537452,0.726492484037711,1.0,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.06,0.7276865132383193,0.4014727027723293,13.10860799057166,0.1310860799057166,0.3804952786306688,0.05207413945278023,0.13415710503089143,0.18446601941747573,0.9983249715485598,0.7264686378975903,1.0,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.08,0.726393195584298,0.3987018836449559,12.850537785783194,0.1285053778578319,0.3788945955746495,0.05648720211827008,0.15357458075904679,0.21006178287731686,0.9981128634521912,0.7250236850699381,1.0,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.10,0.7244012304511832,0.3932239948456176,12.361161644811926,0.1236116164481192,0.3733413807007665,0.05030891438658429,0.08561341571050309,0.13592233009708737,0.9987782625942087,0.7235167427869905,1.0,0