v2e results

app.py

@@ -2,8 +2,9 @@ import os
 import sys
 import evaluate
 import gradio as gr
-from huggingface_hub import InferenceClient
+from huggingface_hub import InferenceClient, login
 from dotenv import find_dotenv, load_dotenv
+from huggingface_hub import login
 
 found_dotenv = find_dotenv(".env")
 
@@ -18,11 +19,14 @@ sys.path.append(path)
 
 from llm_toolkit.llm_utils import *
 from llm_toolkit.translation_utils import *
-from eval_modules.calc_repetitions_v2d import detect_repetitions
+from eval_modules.calc_repetitions_v2e import detect_repetitions
 
 model_name = os.getenv("MODEL_NAME") or "microsoft/Phi-3.5-mini-instruct"
 num_shots = int(os.getenv("NUM_SHOTS", 10))
 data_path = os.getenv("DATA_PATH")
+hf_token = os.getenv("HF_TOKEN")
+
+login(token=hf_token, add_to_git_credential=True)
 
 comet = evaluate.load("comet", config_name="Unbabel/wmt22-cometkiwi-da", gpus=1)
 meteor = evaluate.load("meteor")
@@ -59,7 +63,7 @@ def calc_perf_scores(prediction, source, reference, debug=False):
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
 # client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-client = InferenceClient(model_name)
+client = InferenceClient(model_name, token=hf_token)
 
 datasets = load_translation_dataset(data_path)
 print_row_details(datasets["test"].to_pandas())

eval_modules/calc_repetitions_v2d.py

@@ -1,1281 +0,0 @@
-import os
-import re
-import math
-import pandas as pd
-import numpy as np
-import matplotlib.pyplot as plt
-import matplotlib.ticker as mtick
-import seaborn as sns
-import nltk
-import evaluate
-import traceback
-
-bert_score = evaluate.load("bertscore")
-meteor = evaluate.load("meteor")
-
-print(f"loading: {__file__}")
-
-# pattern_non_word_char_repetition = re.compile(r"\s{5,}")
-# pattern_text_repetitions = re.compile(r"(.{5}.*)\s*((\1)\s*)+", re.M | re.DOTALL)
-
-# final version
-pattern_non_word_char_repetition = re.compile(r"[\s\W]{5,}")
-pattern_text_repetitions = re.compile(
-    r"(?P<repeat>.{5}.*?)(?:[\s\W]*(?P=repeat))+", re.M | re.DOTALL | re.IGNORECASE
-)
-# Explanation of the Regex Pattern:
-#   (?P<repeat>.{5}.*?): Captures any sequence of characters with minimal length of 5 and names this group repeat.
-#     .*?: Matches zero or more characters, non-greedily (as few as possible).
-#   (?:[\s\W]+(?P=repeat))+: A non-capturing group that matches one or more repetitions of:
-#     [\s\W]+: One or more whitespace or non-word characters (spaces, punctuation, etc.).
-#     (?P=repeat): A backreference to the named group repeat.
-
-
-def del_non_word_char_repetition(text, debug=False):
-    count = 0
-
-    if isinstance(text, str):
-        if debug:
-            print("----detect non-word characters repetition----")
-        count = len(text)
-        text = pattern_non_word_char_repetition.sub("\t", text)
-        count -= len(text)
-        if debug and count:
-            print(f"removed non-word characters repetition: {count}")
-    return text, count
-
-
-# final version for repetition detection
-def detect_text_repetitions(text, debug=False):
-    count = 0
-
-    if isinstance(text, str):
-        if debug:
-            print("----detect text repetitions----")
-        matches = pattern_text_repetitions.finditer(text)
-        for match in matches:
-            if debug:
-                print(match)
-                for groupNum in range(0, len(match.groups())):
-                    groupNum = groupNum + 1
-                    print(
-                        "Group {groupNum} found at {start}-{end}: `{group}`".format(
-                            groupNum=groupNum,
-                            start=match.start(groupNum),
-                            end=match.end(groupNum),
-                            group=match.group(groupNum),
-                        )
-                    )
-
-            start, end = match.span()
-            count += end - start - len(match.group(1))
-
-    return count
-
-
-def detect_repetitions(text, debug=False):
-    text, count_non_word_char_repetition = del_non_word_char_repetition(
-        text, debug=debug
-    )
-    count_text_repetitions = detect_text_repetitions(text, debug=debug)
-    total_repetitions = count_non_word_char_repetition + count_text_repetitions
-
-    result = (count_non_word_char_repetition, count_text_repetitions, total_repetitions)
-
-    if debug:
-        print(result)
-    return result
-
-
-def detect_scores(text, debug=False):
-    newline_score, repetition_score, total_repetitions = detect_repetitions(
-        text, debug=debug
-    )
-    return pd.Series([newline_score, repetition_score, total_repetitions])
-
-
-def load_with_newline_and_repetition_scores(result_file, force_recalculate=False):
-    print(f"loading result file: {result_file}")
-    df = pd.read_csv(result_file, comment="#", on_bad_lines="warn")
-
-    if (
-        force_recalculate
-        or "newline_score" not in df.columns
-        or "repetition_score" not in df.columns
-        or "total_repetitions" not in df.columns
-        or "nrr" not in df.columns
-        or "rr" not in df.columns
-    ):
-        if (
-            force_recalculate
-            or "newline_score" not in df.columns
-            or "repetition_score" not in df.columns
-            or "total_repetitions" not in df.columns
-        ):
-            df[["newline_score", "repetition_score", "total_repetitions"]] = df[
-                "answer"
-            ].apply(detect_scores)
-
-        df["answer_len"] = df["answer"].apply(
-            lambda x: len(x) if isinstance(x, str) else 0
-        )
-
-        df["nrr"] = df.apply(
-            lambda x: (
-                1
-                if x["answer_len"] == 0
-                else 1 - (x["newline_score"] + x["repetition_score"]) / x["answer_len"]
-            ),
-            axis=1,
-        )
-
-        df["rr"] = df["nrr"].apply(lambda x: 1 - x)
-
-        df.to_csv(result_file, index=False)
-
-    return df
-
-
-def replace_last(source_string, old_string, new_string):
-    head, _sep, tail = source_string.rpartition(old_string)
-    return head + new_string + tail
-
-
-def load_for_repetition_penalty(
-    csv_result_file, repetition_penalty, force_recalculate=False
-):
-    result_file = replace_last(
-        csv_result_file, ".csv", f"_RP_{repetition_penalty:.3f}.csv"
-    )
-    return load_with_newline_and_repetition_scores(
-        result_file, force_recalculate=force_recalculate
-    )
-
-
-def calc_adjusted_performance(f, r):
-    return f / math.log10(10 + r)
-
-
-def calculate_adjusted_performance(row):
-    r = row["total_repetitions"]
-    adjusted_precision = calc_adjusted_performance(row["precision"], r)
-    adjusted_recall = calc_adjusted_performance(row["recall"], r)
-    return pd.Series([adjusted_precision, adjusted_recall])
-
-
-def load_performance_df(csv_result_file, repetition_penalty):
-    result_file = replace_last(
-        csv_result_file, ".csv", f"_RP_{repetition_penalty:.3f}-t2_evaluated.json"
-    )
-    result_file = result_file.replace("/results/", "/eval/")
-    print(f"loading json file: {result_file}")
-    df = pd.read_json(result_file)
-
-    return df
-
-
-def calculate_performance_score(
-    csv_result_file, repetition_penalty, force_recalculate=False
-):
-    result_file = replace_last(
-        csv_result_file, ".csv", f"_rpp_{repetition_penalty:.2f}.csv"
-    )
-
-    if os.path.exists(result_file):
-        print(f"loading result file: {result_file}")
-        df = load_with_newline_and_repetition_scores(
-            result_file, force_recalculate=force_recalculate
-        )
-    else:
-        print(f"re-creating result file: {result_file}")
-        df = pd.DataFrame()
-        force_recalculate = True
-
-    if force_recalculate or "f2" in df.columns or "f1" not in df.columns:
-        try:
-            perf_df = load_performance_df(csv_result_file, repetition_penalty)
-            df.drop(
-                columns=[
-                    "precision",
-                    "recall",
-                    "f1",
-                    "f2",
-                    "entities_in_answer",
-                    "entities_in_question",
-                    "word_count",
-                ],
-                errors="ignore",
-                inplace=True,
-            )
-
-            df["id"] = perf_df["id"]
-            df["question"] = perf_df["question"]
-            df["answer"] = perf_df["pred_answer"]
-            df["word_count"] = df["answer"].apply(
-                lambda x: len(nltk.word_tokenize(x)) if isinstance(x, str) else 0
-            )
-            df["ground_truth"] = perf_df["ground_truth"]
-
-            df["eval_gemini_1.0_pro"] = perf_df["eval_gemini_1.0_pro"]
-            df["precision"] = perf_df["score"].apply(lambda x: x[0])
-            df["recall"] = perf_df["score"].apply(lambda x: x[1])
-            df["f1"] = perf_df["score"].apply(lambda x: x[2])
-        except Exception as e:
-            print(f"\tignored error: {e}")
-            # traceback.print_exc()
-
-        df[["newline_score", "repetition_score", "total_repetitions"]] = df[
-            "answer"
-        ].apply(detect_scores)
-
-        df[["adjusted_precision", "adjusted_recall"]] = df.apply(
-            calculate_adjusted_performance, axis=1
-        )
-
-        df.to_csv(result_file, index=False)
-        print(f"performance scores saved to result file: {result_file}")
-
-    # print(f"df len: {len(df)}")
-
-    return df
-
-
-def adjust_perf_scores_with_repetition_penalty(result, precision, recall):
-    newline_score = [
-        df["newline_score"].mean() for df in result["df_list_repetition_penalty"]
-    ]
-
-    repetition_score = [
-        df["repetition_score"].mean() for df in result["df_list_repetition_penalty"]
-    ]
-
-    precision = [
-        f / math.log10(10 + n + r)
-        for f, n, r in zip(precision, newline_score, repetition_score)
-    ]
-    recall = [
-        f / math.log10(10 + n + r)
-        for f, n, r in zip(recall, newline_score, repetition_score)
-    ]
-
-    return precision, recall
-
-
-def plot_performance_scores(
-    result,
-    models=None,
-    title="Performance",
-):
-    if models is None:
-        models = result.keys()
-    for model in models:
-        print(f"model: {model}")
-        df = result[model]["df_overall"]
-
-        # Calculate the statistics
-        precision = [
-            df["precision"].mean() for df in result[model]["df_list_repetition_penalty"]
-        ]
-        recall = [
-            df["recall"].mean() for df in result[model]["df_list_repetition_penalty"]
-        ]
-        f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
-        best_f1 = max(f1)
-        best_f1_index = f1.index(best_f1)
-
-        precision, recall = adjust_perf_scores_with_repetition_penalty(
-            result[model], precision, recall
-        )
-        afrp = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
-
-        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
-        best_afrp = max(afrp)
-        best_afrp_index = afrp.index(best_afrp)
-
-        adjusted_precision = [
-            df["adjusted_precision"].mean()
-            for df in result[model]["df_list_repetition_penalty"]
-        ]
-        adjusted_recall = [
-            df["adjusted_recall"].mean()
-            for df in result[model]["df_list_repetition_penalty"]
-        ]
-        afrp2 = [
-            2 * (p * r) / (p + r) for p, r in zip(adjusted_precision, adjusted_recall)
-        ]
-        best_afrp2 = max(afrp2)
-        best_afrp2_index = afrp2.index(best_afrp2)
-
-        repetition_penalties = list(df["repetition_penalty"])
-
-        # line plot for precision, recall, f1
-        plt.figure(figsize=(10, 6))
-
-        plt.axvspan(
-            repetition_penalties[best_f1_index] - 0.01,
-            repetition_penalties[best_f1_index] + 0.01,
-            alpha=0.5,
-            edgecolor="none",
-            facecolor="blue",
-        )
-
-        # plt.axvspan(
-        #     repetition_penalties[best_afrp2_index] - 0.01,
-        #     repetition_penalties[best_afrp2_index] + 0.01,
-        #     alpha=0.5,
-        #     edgecolor="none",
-        #     facecolor="green",
-        # )
-
-        plt.axvspan(
-            repetition_penalties[best_afrp_index] - 0.01,
-            repetition_penalties[best_afrp_index] + 0.01,
-            alpha=0.5,
-            edgecolor="none",
-            facecolor="orange",
-        )
-
-        plt.plot(repetition_penalties, f1, label="F1", marker="D", color="blue")
-        # plt.plot(
-        #     repetition_penalties,
-        #     afrp2,
-        #     label="Per-question RAP - F1",
-        #     marker="s",
-        #     color="green",
-        # )
-        plt.plot(
-            repetition_penalties,
-            afrp,
-            label="RAP - F1",
-            marker="o",
-            color="orange",
-        )
-        plt.xlabel("Repetition Penalties")
-        plt.ylabel("Score")
-        # plt.xlim(0.99, 1.31)
-        # y in percentage
-        plt.gca().yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
-        plt.title(f"{model} {title}")
-        plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
-
-        plt.show()
-
-
-def plot_best_afrp(
-    result,
-    models=None,
-    title="Models with Best RAP - F1",
-    ref_result=None,
-):
-    # Initialize lists to store the statistics
-    model_names = []
-    best_f1 = []
-    best_afrp = []
-    best_repetition_penalty = []
-    best_mtr = []
-
-    if models is None:
-        models = result.keys()
-    for model in models:
-        print(f"model: {model}")
-        df = result[model]["df_overall"]
-
-        # Calculate the statistics
-        precision = [
-            df["precision"].mean() for df in result[model]["df_list_repetition_penalty"]
-        ]
-        recall = [
-            df["recall"].mean() for df in result[model]["df_list_repetition_penalty"]
-        ]
-        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
-        f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
-
-        newline_score = [
-            df["newline_score"].mean()
-            for df in result[model]["df_list_repetition_penalty"]
-        ]
-        # print(f"newline_score: {newline_score}")
-
-        repetition_score = [
-            df["repetition_score"].mean()
-            for df in result[model]["df_list_repetition_penalty"]
-        ]
-        # print(f"repetition_score: {repetition_score}")
-
-        afrp = [
-            f / math.log10(10 + n + r)
-            for f, n, r in zip(f1, newline_score, repetition_score)
-        ]
-
-        best_afrp.append(max(afrp))
-        best_afrp_index = afrp.index(best_afrp[-1])
-        best_repetition_penalty.append(df["repetition_penalty"][best_afrp_index])
-
-        best_f1.append(f1[best_afrp_index])
-        best_mtr.append(
-            newline_score[best_afrp_index] + repetition_score[best_afrp_index]
-        )
-
-        # print(
-        #     f"best repetition penalty: {best_repetition_penalty[-1]}, best afrp: {best_afrp[-1]}, f1: {best_f1[-1]}"
-        # )
-
-        df = result[model]["df_list_repetition_penalty"][best_afrp_index]
-
-        model_names.append(
-            f"{model} (RP={best_repetition_penalty[-1]})"
-        )  # Add the model name to the list
-
-    if ref_result is not None:
-        print("ref_result:", ref_result)
-        for model in ref_result.keys():
-            model_names.append(model)
-            df = pd.read_csv(ref_result[model])
-            # df = df[df["id"].isin(wikidata_df["id"])]
-
-            p = df["precision"].mean()
-            r = df["recall"].mean()
-
-            f1 = 2 * p * r / (p + r) if p + r > 0 else 0
-            best_f1.append(f1)
-            best_afrp.append(f1)
-            best_mtr.append(0)
-
-    print("model_names:", model_names)
-    # print("best_f1:", best_f1)
-    # print("best_afrp:", best_afrp)
-
-    # Create a DataFrame with the statistics
-    data = pd.DataFrame(
-        {
-            "Model": model_names,
-            "RAP - F1": best_afrp,
-            "F1": best_f1,
-        }
-    )
-
-    # Melt the DataFrame to a long format
-    data_melted = data.melt(id_vars="Model", var_name="Metric", value_name="Score")
-
-    # Pivot the DataFrame to a wide format
-    data_pivoted = data_melted.pivot(index="Metric", columns="Model", values="Score")
-
-    # make sure the columns are following the order of the models
-    data_pivoted = data_pivoted[model_names]
-
-    # make sure three groups in the order of precision, recall, f1
-    data_pivoted = data_pivoted.reindex(["RAP - F1", "F1"])
-
-    # Plot the statistics
-    plt.figure(figsize=(15, 6))
-    ax = data_pivoted.plot(kind="bar", ax=plt.gca(), width=0.9)
-    plt.title(title)
-    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
-
-    # Set the rotation of the x-axis labels to 0 degrees
-    plt.xticks(rotation=0)
-
-    # Format the y-axis to display as percentage
-    ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
-
-    # get the max value of the y-axis
-    a1 = max(best_afrp)
-    a2 = max(best_f1)
-
-    max_value = max([a1, a2]) * 1.12
-    print("max_value:", max_value)
-
-    # Set the y-axis limit up to 70%
-    ax.set_ylim(0, max_value)
-
-    # Add the values above each bar
-    for p in ax.patches:
-        ax.annotate(
-            f"{p.get_height() * 100:.1f}",
-            (p.get_x() + p.get_width() / 2.0, p.get_height()),
-            ha="center",
-            va="bottom",
-            xytext=(0, 10),
-            textcoords="offset points",
-            rotation=90,
-        )
-
-    plt.show()
-    return data_pivoted, best_mtr
-
-
-def plot_best_performance(
-    result,
-    models=None,
-    title="Models with Best F1 Score",
-    adjusted_f1=False,
-    ref_result=None,
-):
-    # Initialize lists to store the statistics
-    model_names = []
-    best_precision = []
-    best_recall = []
-    best_f1 = []
-    best_repetition_penalty = []
-    best_mtr = []
-
-    if models is None:
-        models = result.keys()
-    for model in models:
-        print(f"model: {model}")
-        df = result[model]["df_overall"]
-
-        # Calculate the statistics
-        precision = [
-            df["precision"].mean() for df in result[model]["df_list_repetition_penalty"]
-        ]
-        recall = [
-            df["recall"].mean() for df in result[model]["df_list_repetition_penalty"]
-        ]
-        newline_score = [
-            df["newline_score"].mean()
-            for df in result[model]["df_list_repetition_penalty"]
-        ]
-
-        repetition_score = [
-            df["repetition_score"].mean()
-            for df in result[model]["df_list_repetition_penalty"]
-        ]
-
-        if adjusted_f1:
-            precision, recall = adjust_perf_scores_with_repetition_penalty(
-                result[model], precision, recall
-            )
-
-        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
-        f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
-
-        best_f1.append(max(f1))
-        best_f1_index = f1.index(best_f1[-1])
-        best_repetition_penalty.append(df["repetition_penalty"][best_f1_index])
-
-        best_precision.append(precision[best_f1_index])
-        best_recall.append(recall[best_f1_index])
-        best_mtr.append(newline_score[best_f1_index] + repetition_score[best_f1_index])
-
-        print(
-            f"best repetition penalty: {best_repetition_penalty[-1]}, best f1: {best_f1[-1]}, precision: {best_precision[-1]}, recall: {best_recall[-1]}"
-        )
-
-        df = result[model]["df_list_repetition_penalty"][best_f1_index]
-
-        model_names.append(
-            f"{model} (RP={best_repetition_penalty[-1]})"
-        )  # Add the model name to the list
-
-        # print sum for columns: newline_score, repetition_score
-        print(
-            f"newline_score: {df['newline_score'].sum()}, repetition_score: {df['repetition_score'].sum()}"
-        )
-
-    if ref_result is not None:
-        print("ref_result:", ref_result)
-        for model in ref_result.keys():
-            model_names.append(model)
-            df = pd.read_csv(ref_result[model])
-            # df = df[df["id"].isin(wikidata_df["id"])]
-
-            best_precision.append(df["precision"].mean())
-            best_recall.append(df["recall"].mean())
-            f1 = (
-                2
-                * (best_precision[-1] * best_recall[-1])
-                / (best_precision[-1] + best_recall[-1])
-            )
-            # best_f1.append(df["f1"].mean())
-            best_f1.append(f1)
-            best_mtr.append(0)
-
-    # Create a DataFrame with the statistics
-    data = (
-        pd.DataFrame(
-            {
-                "Model": model_names,
-                "Adjusted Precision with RP": best_precision,
-                "Adjusted Recall with RP": best_recall,
-                "Adjusted F1 with RP": best_f1,
-            }
-        )
-        if adjusted_f1
-        else pd.DataFrame(
-            {
-                "Model": model_names,
-                "Precision": best_precision,
-                "Recall": best_recall,
-                "F1": best_f1,
-            }
-        )
-    )
-    columns = list(data.columns)
-
-    # Melt the DataFrame to a long format
-    data_melted = data.melt(id_vars="Model", var_name="Metric", value_name="Score")
-
-    # Pivot the DataFrame to a wide format
-    data_pivoted = data_melted.pivot(index="Metric", columns="Model", values="Score")
-
-    # make sure the columns are following the order of the models
-    data_pivoted = data_pivoted[model_names]
-
-    # make sure three groups in the order of precision, recall, f1
-    data_pivoted = data_pivoted.reindex(columns[1:])
-
-    # Plot the statistics
-    plt.figure(figsize=(10, 6))
-    ax = data_pivoted.plot(kind="bar", ax=plt.gca(), width=0.9)
-    plt.title(title)
-    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
-
-    # Set the rotation of the x-axis labels to 0 degrees
-    plt.xticks(rotation=0)
-
-    # Format the y-axis to display as percentage
-    ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
-
-    # get the max value of the y-axis
-    a1 = max(best_precision)
-    a2 = max(best_recall)
-    a3 = max(best_f1)
-
-    max_value = max([a1, a2, a3]) * 1.12
-    print("max_value:", max_value)
-
-    # Set the y-axis limit up to 70%
-    ax.set_ylim(0, max_value)
-
-    # Add the values above each bar
-    for p in ax.patches:
-        ax.annotate(
-            f"{p.get_height() * 100:.1f}",
-            (p.get_x() + p.get_width() / 2.0, p.get_height()),
-            ha="center",
-            va="bottom",
-            xytext=(0, 10),
-            textcoords="offset points",
-            rotation=90,
-        )
-
-    plt.show()
-    return data_pivoted, best_mtr
-
-
-def plot_best_performance_ms_macro(
-    result,
-    models=None,
-    title="Models with Best RAP - Performance",
-    ref_result=None,
-    skip_generic_prompt=False,
-    include_adjusted_performance=True,
-):
-    # Initialize lists to store the statistics
-    model_names = []
-    best_f1 = []
-    best_afrp = []
-    best_repetition_penalty = []
-    best_bleu1 = []
-    best_rougeL = []
-    best_mtr = []
-
-    if models is None:
-        models = result.keys()
-    for model in models:
-        if skip_generic_prompt and "generic prompt" in model:
-            continue
-        print(f"model: {model}")
-        df = result[model]["df_overall"]
-
-        # Calculate the statistics
-        bleu1 = [x for x in df["bleu1"]]
-        rougeL = [x for x in df["rougeL"]]
-        f1 = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
-
-        newline_score = [
-            df["newline_score"].mean()
-            for df in result[model]["df_list_repetition_penalty"]
-        ]
-        # print(f"newline_score: {newline_score}")
-
-        repetition_score = [
-            df["repetition_score"].mean()
-            for df in result[model]["df_list_repetition_penalty"]
-        ]
-        # print(f"repetition_score: {repetition_score}")
-
-        afrp = [
-            f / math.log10(10 + n + r)
-            for f, n, r in zip(f1, newline_score, repetition_score)
-        ]
-
-        best_afrp.append(max(afrp if include_adjusted_performance else f1))
-        best_afrp_index = (
-            afrp.index(best_afrp[-1])
-            if include_adjusted_performance
-            else f1.index(best_afrp[-1])
-        )
-        best_repetition_penalty.append(df["repetition_penalty"][best_afrp_index])
-
-        best_f1.append(f1[best_afrp_index])
-        best_bleu1.append(bleu1[best_afrp_index])
-        best_rougeL.append(rougeL[best_afrp_index])
-        best_mtr.append(
-            newline_score[best_afrp_index] + repetition_score[best_afrp_index]
-        )
-
-        # print(
-        #     f"best repetition penalty: {best_repetition_penalty[-1]}, best afrp: {best_afrp[-1]}, f1: {best_f1[-1]}"
-        # )
-
-        df = result[model]["df_list_repetition_penalty"][best_afrp_index]
-
-        model_names.append(
-            f"{model} (RP={best_repetition_penalty[-1]})"
-        )  # Add the model name to the list
-
-    if ref_result is not None:
-        print("ref_result:", ref_result)
-        for model in ref_result.keys():
-            model_names.append(model)
-            df = pd.read_csv(ref_result[model], comment="#", on_bad_lines="warn")
-            # df = df[df["id"].isin(wikidata_df["id"])]
-
-            p = df["bleu1"][0]
-            best_bleu1.append(p)
-
-            r = df["rougeL"][0]
-            best_rougeL.append(r)
-
-            f1 = 2 * p * r / (p + r) if p + r > 0 else 0
-            best_f1.append(f1)
-            best_afrp.append(f1)
-            best_mtr.append(0)
-
-    # print("model_names:", model_names)
-    # print("best_f1:", best_f1)
-    # print("best_afrp:", best_afrp)
-
-    # Create a DataFrame with the statistics
-    data = (
-        pd.DataFrame(
-            {
-                "Model": model_names,
-                "RAP - Perf Score": best_afrp,
-                "Overall Perf Score": best_f1,
-            }
-        )
-        if include_adjusted_performance
-        else pd.DataFrame(
-            {
-                "Model": model_names,
-                "Bleu-1": best_bleu1,
-                "Rouge-L": best_rougeL,
-                "Overall Perf Score": best_f1,
-            }
-        )
-    )
-
-    # Melt the DataFrame to a long format
-    data_melted = data.melt(id_vars="Model", var_name="Metric", value_name="Score")
-
-    # Pivot the DataFrame to a wide format
-    data_pivoted = data_melted.pivot(index="Metric", columns="Model", values="Score")
-
-    # make sure the columns are following the order of the models
-    data_pivoted = data_pivoted[model_names]
-
-    columns = list(data.columns)
-    data_pivoted = data_pivoted.reindex(columns[1:])
-
-    # Plot the statistics
-    plt.figure(figsize=(10, 6))
-    ax = data_pivoted.plot(kind="bar", ax=plt.gca(), width=0.9)
-    plt.title(title)
-    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
-
-    # Set the rotation of the x-axis labels to 0 degrees
-    plt.xticks(rotation=0)
-
-    # Format the y-axis to display as percentage
-    ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
-
-    # get the max value of the y-axis
-    a1 = max(best_afrp)
-    a2 = max(best_f1)
-    a3 = max(best_bleu1)
-    a4 = max(best_rougeL)
-
-    max_value = (
-        max([a1, a2] if include_adjusted_performance else [a1, a2, a3, a4]) * 1.12
-    )
-    print("max_value:", max_value)
-
-    # Set the y-axis limit up to 70%
-    ax.set_ylim(0, max_value)
-
-    # Add the values above each bar
-    for p in ax.patches:
-        ax.annotate(
-            f"{p.get_height() * 100:.1f}",
-            (p.get_x() + p.get_width() / 2.0, p.get_height()),
-            ha="center",
-            va="bottom",
-            xytext=(0, 10),
-            textcoords="offset points",
-            rotation=90,
-        )
-
-    plt.show()
-    return data_pivoted, best_mtr
-
-
-all_open_source_models = [
-    "gemma-1.1-2b-it",
-    "Phi-3-mini-128k-instruct",
-    "gemma-1.1-7b-it",
-    "Llama-2-7b-chat-hf",
-    "Mistral-7B-Instruct-v0.2",
-    "Meta-Llama-3-8B-Instruct",
-    "Llama-2-13b-chat-hf",
-    "Llama-2-70b-chat-hf",
-    "Meta-Llama-3-70B-Instruct",
-]
-
-
-def load_for_repetition_penalty_ms_macro(
-    csv_result_file, repetition_penalty, force_recalculate=False
-):
-    result_file = replace_last(
-        csv_result_file, ".csv", f"_rpp_{repetition_penalty:.2f}.csv"
-    )
-    df = load_with_newline_and_repetition_scores(
-        result_file, force_recalculate=force_recalculate
-    )
-
-    return df
-
-
-# MS MACRO
-def plot_performance_scores_ms_macro(
-    result,
-    models=None,
-    title="Performance",
-):
-    if models is None:
-        models = result.keys()
-    for model in models:
-        print(f"model: {model}")
-        df = result[model]["df_overall"]
-        # print(result[model]["df_list_repetition_penalty"][0].describe())
-
-        # Calculate the statistics
-        bleu1 = list(df["bleu1"])
-        rougeL = list(df["rougeL"])
-        f1 = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
-        best_f1 = max(f1)
-        best_f1_index = f1.index(best_f1)
-
-        bleu1, rougeL = adjust_perf_scores_with_repetition_penalty(
-            result[model], bleu1, rougeL
-        )
-        afrp = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
-
-        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
-        best_afrp = max(afrp)
-        best_afrp_index = afrp.index(best_afrp)
-
-        repetition_penalties = list(df["repetition_penalty"])
-
-        # line plot for precision, recall, f1
-        plt.figure(figsize=(10, 6))
-
-        plt.axvspan(
-            repetition_penalties[best_f1_index] - 0.01,
-            repetition_penalties[best_f1_index] + 0.01,
-            alpha=0.5,
-            edgecolor="none",
-            facecolor="blue",
-        )
-
-        plt.axvspan(
-            repetition_penalties[best_afrp_index] - 0.01,
-            repetition_penalties[best_afrp_index] + 0.01,
-            alpha=0.5,
-            edgecolor="none",
-            facecolor="orange",
-        )
-
-        plt.plot(
-            repetition_penalties,
-            f1,
-            label="Overall Perf Score",
-            marker="D",
-            color="blue",
-        )
-        plt.plot(
-            repetition_penalties,
-            afrp,
-            label="RAP - Perf Score",
-            marker="o",
-            color="orange",
-        )
-
-        plt.xlabel("Repetition Penalties")
-        plt.ylabel("Score")
-        # plt.xlim(0.99, 1.31)
-        # y in percentage
-        plt.gca().yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
-        plt.title(f"{model} {title}")
-        plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
-
-        plt.show()
-
-
-def plot_repetition_factors(result, groups):
-    for group in groups:
-        # Plot the statistics
-        plt.figure(figsize=(10, 6))
-
-        max_value = 0
-        for model in result.keys():
-            if not group in model.lower():
-                continue
-            print(f"model: {model}")
-            df = result[model]["df_overall"]
-            repetition_panelties = [
-                repetition_penalty for repetition_penalty in df["repetition_penalty"]
-            ]
-
-            mean_score = [
-                # math.log10(10 + df["total_repetitions"].mean())
-                df["total_repetitions"].mean()
-                for df in result[model]["df_list_repetition_penalty"]
-            ]
-
-            sns.lineplot(x=repetition_panelties, y=mean_score, label=model)
-
-            new_max = max(mean_score)
-            if new_max > max_value:
-                max_value = new_max
-
-        max_value = max_value * 1.05
-        # if max_value < 1.5:
-        #     max_value = 1.5
-        # set ylimit
-        plt.ylim(0, max_value)
-
-        # show grid
-        plt.grid(True)
-        plt.xlabel("Repetition Penalties")
-        plt.ylabel("Mean Total Repetitions")
-        plt.title("Mean Total Repetitions vs Repetition Penalties")
-        plt.legend()
-
-        plt.show()
-
-
-def plot_repetition_factors_by_group(result, group_filter=None):
-    markers = ["D", "o", "s", "x"]
-    colors = ["blue", "orange", "green", "red"]
-
-    # Plot the statistics
-    plt.figure(figsize=(10, 6))
-    index = 0
-    max_value = 0
-
-    for model in result.keys():
-        if group_filter is not None and group_filter not in model:
-            continue
-
-        print(f"model: {model}")
-
-        df = result[model]["df_overall"]
-        repetition_panelties = [
-            repetition_penalty for repetition_penalty in df["repetition_penalty"]
-        ]
-
-        # Calculate the statistics
-        mean_score = [
-            # math.log10(10 + df["total_repetitions"].mean())
-            df["total_repetitions"].mean()
-            for df in result[model]["df_list_repetition_penalty"]
-        ]
-        if len(mean_score) != len(repetition_panelties):
-            print(
-                f"model: {model} has different length of repetition penalties and mean score"
-            )
-            print("repetition_panelties:", len(repetition_panelties))
-            print("mean_score:", len(mean_score))
-            continue
-
-        new_max = max(mean_score)
-        if new_max > max_value:
-            max_value = new_max
-
-        sns.lineplot(
-            x=repetition_panelties,
-            y=mean_score,
-            label=model,
-            marker=markers[index],
-            color=colors[index],
-        )
-
-        index += 1
-
-    max_value = max_value * 1.05
-    # if max_value < 1.5:
-    #     max_value = 1.5
-    # set ylimit
-    plt.ylim(0, max_value)
-    max_value = 0
-
-    plt.xlabel("Repetition Penalties")
-    plt.ylabel("Mean Total Repetitions")
-    plt.title("Mean Total Repetitions vs Repetition Penalties")
-    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
-
-    plt.show()
-
-
-ms_marco_csv_result_files = [
-    "data/results_v2/gemma-1.1-2b-it(RAG - Generic Prompt)_mm.csv",
-    "data/results_v2/gemma-1.1-2b-it(RAG - Chat Template)_mm.csv",
-    "data/results_v2/gemma-1.1-2b-it(Non-RAG)_mm.csv",
-    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Generic Prompt)_mm.csv",
-    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Chat Template)_mm.csv",
-    "data/results_v2/Phi-3-mini-128k-instruct(Non-RAG)_mm.csv",
-    "data/results_v2/gemma-1.1-7b-it(RAG - Generic Prompt)_mm.csv",
-    "data/results_v2/gemma-1.1-7b-it(RAG - Chat Template)_mm.csv",
-    "data/results_v2/gemma-1.1-7b-it(Non-RAG)_mm.csv",
-    "data/results_v2/Llama-2-7b-chat-hf(RAG - Generic Prompt)_mm.csv",
-    "data/results_v2/Llama-2-7b-chat-hf(RAG - Chat Template)_mm.csv",
-    "data/results_v2/Llama-2-7b-chat-hf(Non-RAG)_mm.csv",
-    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Generic Prompt)_mm.csv",
-    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Chat Template)_mm.csv",
-    "data/results_v2/Mistral-7B-Instruct-v0.2(Non-RAG)_mm.csv",
-    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Generic Prompt)_mm.csv",
-    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Chat Template)_mm.csv",
-    "data/results_v2/Meta-Llama-3-8B-Instruct(Non-RAG)_mm.csv",
-    "data/results_v2/Llama-2-13b-chat-hf(RAG - Generic Prompt)_mm.csv",
-    "data/results_v2/Llama-2-13b-chat-hf(RAG - Chat Template)_mm.csv",
-    "data/results_v2/Llama-2-13b-chat-hf(Non-RAG)_mm.csv",
-    "data/results_v2/Llama-2-70b-chat-hf(RAG - Generic Prompt)_mm.csv",
-    "data/results_v2/Llama-2-70b-chat-hf(RAG - Chat Template)_mm.csv",
-    "data/results_v2/Llama-2-70b-chat-hf(Non-RAG)_mm.csv",
-    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Generic Prompt)_mm.csv",
-    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Chat Template)_mm.csv",
-    "data/results_v2/Meta-Llama-3-70B-Instruct(Non-RAG)_mm.csv",
-]
-
-webqsp_csv_result_files = [
-    "data/results_v2/gemma-1.1-2b-it(RAG - Generic Prompt)_wd.csv",
-    "data/results_v2/gemma-1.1-2b-it(RAG - Chat Template)_wd.csv",
-    "data/results_v2/gemma-1.1-2b-it(Non-RAG)_wd.csv",
-    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Generic Prompt)_wd.csv",
-    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Chat Template)_wd.csv",
-    "data/results_v2/Phi-3-mini-128k-instruct(Non-RAG)_wd.csv",
-    "data/results_v2/gemma-1.1-7b-it(RAG - Generic Prompt)_wd.csv",
-    "data/results_v2/gemma-1.1-7b-it(RAG - Chat Template)_wd.csv",
-    "data/results_v2/gemma-1.1-7b-it(Non-RAG)_wd.csv",
-    "data/results_v2/Llama-2-7b-chat-hf(RAG - Generic Prompt)_wd.csv",
-    "data/results_v2/Llama-2-7b-chat-hf(RAG - Chat Template)_wd.csv",
-    "data/results_v2/Llama-2-7b-chat-hf(Non-RAG)_wd.csv",
-    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Generic Prompt)_wd.csv",
-    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Chat Template)_wd.csv",
-    "data/results_v2/Mistral-7B-Instruct-v0.2(Non-RAG)_wd.csv",
-    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Generic Prompt)_wd.csv",
-    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Chat Template)_wd.csv",
-    "data/results_v2/Meta-Llama-3-8B-Instruct(Non-RAG)_wd.csv",
-    "data/results_v2/Llama-2-13b-chat-hf(RAG - Generic Prompt)_wd.csv",
-    "data/results_v2/Llama-2-13b-chat-hf(RAG - Chat Template)_wd.csv",
-    "data/results_v2/Llama-2-13b-chat-hf(Non-RAG)_wd.csv",
-    "data/results_v2/Llama-2-70b-chat-hf(RAG - Generic Prompt)_wd.csv",
-    "data/results_v2/Llama-2-70b-chat-hf(RAG - Chat Template)_wd.csv",
-    "data/results_v2/Llama-2-70b-chat-hf(Non-RAG)_wd.csv",
-    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Generic Prompt)_wd.csv",
-    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Chat Template)_wd.csv",
-    "data/results_v2/Meta-Llama-3-70B-Instruct(Non-RAG)_wd.csv",
-]
-
-
-def calc_rap_scores(result, precision="precision", recall="recall"):
-    newline_score = [
-        df["newline_score"].mean() for df in result["df_list_repetition_penalty"]
-    ]
-
-    repetition_score = [
-        df["repetition_score"].mean() for df in result["df_list_repetition_penalty"]
-    ]
-
-    if precision in result["df_list_repetition_penalty"][0].columns:
-        precision = [
-            df[precision].mean() for df in result["df_list_repetition_penalty"]
-        ]
-        recall = [df[recall].mean() for df in result["df_list_repetition_penalty"]]
-    else:
-        precision = result["df_overall"][precision]
-        recall = result["df_overall"][recall]
-
-    f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
-
-    # rap = [
-    #     f / math.log10(10 + n + r)
-    #     for f, n, r in zip(f1, newline_score, repetition_score)
-    # ]
-
-    nrr = [
-        1 - (n + r) / s
-        for f, n, r, s in zip(
-            f1, newline_score, repetition_score, result["df_overall"]["answer_len"]
-        )
-    ]
-
-    rap = [f * n * n * n for f, n in zip(f1, nrr)]
-
-    return newline_score, repetition_score, f1, rap, nrr
-
-
-def get_model_name(csv_result_file):
-    parts = re.split(r"[_/]", csv_result_file)
-    print(f"parts: {parts}")
-    model_name = parts[3]
-    return model_name
-
-
-def load_webqsp_result(csv_result_files, force_recalculate=False, save=False):
-    result = {}
-    for i, csv_result_file in enumerate(csv_result_files):
-        try:
-            df = pd.read_csv(csv_result_file)
-            model_name = get_model_name(csv_result_file)
-            print(f"\tmodel_name: {model_name}")
-
-            dfs = [
-                calculate_performance_score(
-                    csv_result_file,
-                    repetition_penalty,
-                    force_recalculate=force_recalculate,
-                )
-                for repetition_penalty in df["repetition_penalty"]
-            ]
-
-            answer_lens = []
-            for df_rpp in dfs:
-                df_rpp["answer_len"] = df_rpp["answer"].apply(
-                    lambda x: len(x) if isinstance(x, str) else 0
-                )
-                answer_lens.append(df_rpp["answer_len"].mean())
-            df["answer_len"] = answer_lens
-
-            result[model_name] = {
-                "df_overall": df,
-                "df_list_repetition_penalty": dfs,
-                "file": csv_result_file,
-            }
-            newline_score, repetition_score, perf, rap, nrr = calc_rap_scores(
-                result[model_name]
-            )
-            df["newline_score"] = newline_score
-            df["repetition_score"] = repetition_score
-            df["total_repetitions"] = df["newline_score"] + df["repetition_score"]
-            df["perf"] = perf
-            df["nrr"] = nrr
-            df["rap"] = rap
-            df["rr"] = df["nrr"].apply(lambda x: 1 - x)
-            if save:
-                df.to_csv(csv_result_file, index=False)
-        except Exception as e:
-            print(f"Error: {e}")
-            traceback.print_exc()
-
-    return result
-
-
-def load_ms_marco_result(
-    csv_result_files, force_recalculate=False, calc_bertscore=False, save=False
-):
-    result = {}
-    for csv_result_file in csv_result_files:
-        try:
-            df = pd.read_csv(csv_result_file)
-            model_name = get_model_name(csv_result_file)
-            print(f"\tmodel_name: {model_name}")
-
-            dfs = [
-                load_for_repetition_penalty_ms_macro(
-                    csv_result_file,
-                    repetition_penalty,
-                    force_recalculate=force_recalculate,
-                )
-                for repetition_penalty in df["repetition_penalty"]
-            ]
-
-            answer_lens = []
-            for df_rpp in dfs:
-                answer_lens.append(df_rpp["answer_len"].mean())
-            df["answer_len"] = answer_lens
-
-            col = "bert_score" if calc_bertscore else "meteor"
-            score_unavailable = col not in df.columns
-
-            if score_unavailable:
-                save = True
-                bert_meteor_scores = []
-                bert_score_references = None
-                for df_rpp in dfs:
-                    if calc_bertscore:
-                        bert_meteor_score = 0
-
-                        for i, row in df_rpp.iterrows():
-                            answer = row["answer"]
-                            if not isinstance(answer, str):
-                                answer = ""
-                            bert_meteor_score += bert_score.compute(
-                                predictions=[answer],
-                                references=[row["ground_truth"][0]],
-                                lang="en",
-                                model_type="microsoft/deberta-large-mnli",
-                            )["f1"][0]
-                        # get average of bertscore
-                        bert_meteor_score = bert_meteor_score / len(df_rpp)
-
-                        print(f"bert_score: {bert_meteor_score}")
-                    else:
-                        bert_meteor_score = meteor.compute(
-                            predictions=df_rpp["answer"],
-                            references=df_rpp["ground_truth"],
-                        )["meteor"]
-
-                    bert_meteor_scores.append(bert_meteor_score)
-
-                df[col] = bert_meteor_scores
-
-            result[model_name] = {
-                "df_overall": df,
-                "df_list_repetition_penalty": dfs,
-                "file": csv_result_file,
-            }
-            newline_score, repetition_score, perf, rap, nrr = calc_rap_scores(
-                result[model_name],
-                precision=col,
-                recall=col,
-            )
-            df["newline_score"] = newline_score
-            df["repetition_score"] = repetition_score
-            df["total_repetitions"] = df["newline_score"] + df["repetition_score"]
-            df["perf"] = perf
-            df["nrr"] = nrr
-            df["rap"] = rap
-            df["rr"] = df["nrr"].apply(lambda x: 1 - x)
-
-            if save:
-                df.to_csv(csv_result_file, index=False)
-        except Exception as e:
-            print("An error occurred:", e)
-            traceback.print_exc()
-            print(f"csv_result_file: {csv_result_file}")
-
-    return result

eval_modules/calc_repetitions_v2e.py

@@ -1 +0,0 @@
-/Users/inflaton/code/engd/papers/rapget-v2/eval_modules/calc_repetitions_v2e.py

eval_modules/calc_repetitions_v2e.py

@@ -0,0 +1,1310 @@
+import os
+import re
+import math
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mtick
+import seaborn as sns
+import nltk
+import evaluate
+import traceback
+
+bert_score = evaluate.load("bertscore")
+meteor = evaluate.load("meteor")
+
+print(f"loading: {__file__}")
+
+# pattern_non_word_char_repetition = re.compile(r"\s{5,}")
+# pattern_text_repetitions = re.compile(r"(.{5}.*)\s*((\1)\s*)+", re.M | re.DOTALL)
+
+# final version
+pattern_non_word_char_repetition = re.compile(r"[\s\W]{5,}")
+pattern_text_repetitions = re.compile(
+    r"(?P<repeat>.{5}.*?)(?:[\s\W]*(?P=repeat))+", re.M | re.DOTALL | re.IGNORECASE
+)
+# Explanation of the Regex Pattern:
+#   (?P<repeat>.{5}.*?): Captures any sequence of characters with minimal length of 5 and names this group repeat.
+#     .*?: Matches zero or more characters, non-greedily (as few as possible).
+#   (?:[\s\W]+(?P=repeat))+: A non-capturing group that matches one or more repetitions of:
+#     [\s\W]+: One or more whitespace or non-word characters (spaces, punctuation, etc.).
+#     (?P=repeat): A backreference to the named group repeat.
+
+
+def del_non_word_char_repetition(text, debug=False):
+    count = 0
+
+    if isinstance(text, str):
+        if debug:
+            print("----detect non-word characters repetition----")
+        count = len(text)
+        text = pattern_non_word_char_repetition.sub("\t", text)
+        count -= len(text)
+        if debug and count:
+            print(f"removed non-word characters repetition: {count}")
+    return text, count
+
+
+# final version for repetition detection
+def detect_text_repetitions(text, debug=False):
+    count = 0
+
+    if isinstance(text, str):
+        if debug:
+            print("----detect text repetitions----")
+        matches = pattern_text_repetitions.finditer(text)
+        for match in matches:
+            if debug:
+                print(match)
+                for groupNum in range(0, len(match.groups())):
+                    groupNum = groupNum + 1
+                    print(
+                        "Group {groupNum} found at {start}-{end}: `{group}`".format(
+                            groupNum=groupNum,
+                            start=match.start(groupNum),
+                            end=match.end(groupNum),
+                            group=match.group(groupNum),
+                        )
+                    )
+
+            start, end = match.span()
+            count += end - start - len(match.group(1))
+
+    return count
+
+
+def detect_repetitions(text, debug=False):
+    if isinstance(text, str) is False:
+        return 0, 0, 0
+    text, count_non_word_char_repetition = del_non_word_char_repetition(
+        text, debug=debug
+    )
+    count_text_repetitions = detect_text_repetitions(text, debug=debug)
+    total_repetitions = count_non_word_char_repetition + count_text_repetitions
+
+    result = (count_non_word_char_repetition, count_text_repetitions, total_repetitions)
+
+    if debug:
+        print(result)
+    return result
+
+
+def detect_scores(
+    row, debug=False, answer_col="answer", ground_truth_col="ground_truth"
+):
+    newline_score, repetition_score, total_repetitions = detect_repetitions(
+        row[answer_col], debug=debug
+    )
+
+    if ground_truth_col:
+        ground_truth_newline_score, ground_truth_repetition_score, _ = (
+            detect_repetitions(row[ground_truth_col], debug=debug)
+        )
+
+        newline_score -= ground_truth_newline_score
+        if newline_score < 0:
+            newline_score = 0
+
+        repetition_score -= ground_truth_repetition_score
+        if repetition_score < 0:
+            repetition_score = 0
+
+        total_repetitions = newline_score + repetition_score
+
+    return pd.Series([newline_score, repetition_score, total_repetitions])
+
+
+def load_with_newline_and_repetition_scores(result_file, force_recalculate=False):
+    print(f"loading result file: {result_file}")
+    df = pd.read_csv(result_file, comment="#", on_bad_lines="warn")
+
+    if (
+        force_recalculate
+        or "newline_score" not in df.columns
+        or "repetition_score" not in df.columns
+        or "total_repetitions" not in df.columns
+        or "nrr" not in df.columns
+        or "rr" not in df.columns
+    ):
+        if (
+            force_recalculate
+            or "newline_score" not in df.columns
+            or "repetition_score" not in df.columns
+            or "total_repetitions" not in df.columns
+        ):
+            df[["newline_score", "repetition_score", "total_repetitions"]] = df.apply(
+                detect_scores, axis=1
+            )
+
+        df["answer_len"] = df["answer"].apply(
+            lambda x: len(x) if isinstance(x, str) else 0
+        )
+
+        df["nrr"] = df.apply(
+            lambda x: (
+                1
+                if x["answer_len"] == 0
+                else 1 - (x["newline_score"] + x["repetition_score"]) / x["answer_len"]
+            ),
+            axis=1,
+        )
+
+        df["rr"] = df["nrr"].apply(lambda x: 1 - x)
+
+        df.to_csv(result_file, index=False)
+
+    return df
+
+
+def replace_last(source_string, old_string, new_string):
+    head, _sep, tail = source_string.rpartition(old_string)
+    return head + new_string + tail
+
+
+def load_for_repetition_penalty(
+    csv_result_file, repetition_penalty, force_recalculate=False
+):
+    result_file = replace_last(
+        csv_result_file, ".csv", f"_RP_{repetition_penalty:.3f}.csv"
+    )
+    return load_with_newline_and_repetition_scores(
+        result_file, force_recalculate=force_recalculate
+    )
+
+
+def calc_adjusted_performance(f, r, l=1):
+    n = 1 - r / l if l > 0 else 0
+    return f * n * n * n
+
+
+def calculate_adjusted_performance(row):
+    r = row["total_repetitions"]
+    l = row["answer_len"]
+    adjusted_precision = calc_adjusted_performance(row["precision"], r, l)
+    adjusted_recall = calc_adjusted_performance(row["recall"], r, l)
+    return pd.Series([adjusted_precision, adjusted_recall])
+
+
+def load_performance_df(csv_result_file, repetition_penalty):
+    result_file = replace_last(
+        csv_result_file, ".csv", f"_RP_{repetition_penalty:.3f}-t2_evaluated.json"
+    )
+    result_file = result_file.replace("/results/", "/eval/")
+    print(f"loading json file: {result_file}")
+    df = pd.read_json(result_file)
+
+    return df
+
+
+def calculate_performance_score(
+    csv_result_file, repetition_penalty, force_recalculate=False
+):
+    result_file = replace_last(
+        csv_result_file, ".csv", f"_rpp_{repetition_penalty:.2f}.csv"
+    )
+
+    if os.path.exists(result_file):
+        print(f"loading result file: {result_file}")
+        df = load_with_newline_and_repetition_scores(
+            result_file, force_recalculate=force_recalculate
+        )
+    else:
+        print(f"re-creating result file: {result_file}")
+        df = pd.DataFrame()
+        force_recalculate = True
+
+    if force_recalculate or "f2" in df.columns or "f1" not in df.columns:
+        try:
+            perf_df = load_performance_df(csv_result_file, repetition_penalty)
+            df.drop(
+                columns=[
+                    "precision",
+                    "recall",
+                    "f1",
+                    "f2",
+                    "entities_in_answer",
+                    "entities_in_question",
+                    "word_count",
+                ],
+                errors="ignore",
+                inplace=True,
+            )
+
+            df["id"] = perf_df["id"]
+            df["question"] = perf_df["question"]
+            df["answer"] = perf_df["pred_answer"]
+            df["word_count"] = df["answer"].apply(
+                lambda x: len(nltk.word_tokenize(x)) if isinstance(x, str) else 0
+            )
+            df["ground_truth"] = perf_df["ground_truth"]
+
+            df["eval_gemini_1.0_pro"] = perf_df["eval_gemini_1.0_pro"]
+            df["precision"] = perf_df["score"].apply(lambda x: x[0])
+            df["recall"] = perf_df["score"].apply(lambda x: x[1])
+            df["f1"] = perf_df["score"].apply(lambda x: x[2])
+        except Exception as e:
+            print(f"\tignored error: {e}")
+            # traceback.print_exc()
+
+        df[["newline_score", "repetition_score", "total_repetitions"]] = df.apply(
+            detect_scores, axis=1
+        )
+        df["answer_len"] = df["answer"].apply(
+            lambda x: len(x) if isinstance(x, str) else 0
+        )
+
+        df[["adjusted_precision", "adjusted_recall"]] = df.apply(
+            calculate_adjusted_performance, axis=1
+        )
+
+        df.to_csv(result_file, index=False)
+        print(f"performance scores saved to result file: {result_file}")
+
+    # print(f"df len: {len(df)}")
+
+    return df
+
+
+def adjust_perf_scores_with_repetition_penalty(result, precision, recall):
+    newline_score = [
+        df["newline_score"].mean() for df in result["df_list_repetition_penalty"]
+    ]
+
+    repetition_score = [
+        df["repetition_score"].mean() for df in result["df_list_repetition_penalty"]
+    ]
+
+    answer_len = [
+        df["answer_len"].mean() for df in result["df_list_repetition_penalty"]
+    ]
+
+    precision = [
+        calc_adjusted_performance(f, n + r, l)
+        for f, n, r, l in zip(precision, newline_score, repetition_score, answer_len)
+    ]
+    recall = [
+        calc_adjusted_performance(f, n + r, l)
+        for f, n, r, l in zip(recall, newline_score, repetition_score, answer_len)
+    ]
+
+    return precision, recall
+
+
+def plot_performance_scores(
+    result,
+    models=None,
+    title="Performance",
+):
+    if models is None:
+        models = result.keys()
+    for model in models:
+        print(f"model: {model}")
+        df = result[model]["df_overall"]
+
+        # Calculate the statistics
+        precision = [
+            df["precision"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        recall = [
+            df["recall"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
+        best_f1 = max(f1)
+        best_f1_index = f1.index(best_f1)
+
+        precision, recall = adjust_perf_scores_with_repetition_penalty(
+            result[model], precision, recall
+        )
+        afrp = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
+
+        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
+        best_afrp = max(afrp)
+        best_afrp_index = afrp.index(best_afrp)
+
+        adjusted_precision = [
+            df["adjusted_precision"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        adjusted_recall = [
+            df["adjusted_recall"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        afrp2 = [
+            2 * (p * r) / (p + r) for p, r in zip(adjusted_precision, adjusted_recall)
+        ]
+        best_afrp2 = max(afrp2)
+        best_afrp2_index = afrp2.index(best_afrp2)
+
+        repetition_penalties = list(df["repetition_penalty"])
+
+        # line plot for precision, recall, f1
+        plt.figure(figsize=(10, 6))
+
+        plt.axvspan(
+            repetition_penalties[best_f1_index] - 0.01,
+            repetition_penalties[best_f1_index] + 0.01,
+            alpha=0.5,
+            edgecolor="none",
+            facecolor="blue",
+        )
+
+        # plt.axvspan(
+        #     repetition_penalties[best_afrp2_index] - 0.01,
+        #     repetition_penalties[best_afrp2_index] + 0.01,
+        #     alpha=0.5,
+        #     edgecolor="none",
+        #     facecolor="green",
+        # )
+
+        plt.axvspan(
+            repetition_penalties[best_afrp_index] - 0.01,
+            repetition_penalties[best_afrp_index] + 0.01,
+            alpha=0.5,
+            edgecolor="none",
+            facecolor="orange",
+        )
+
+        plt.plot(repetition_penalties, f1, label="F1", marker="D", color="blue")
+        # plt.plot(
+        #     repetition_penalties,
+        #     afrp2,
+        #     label="Per-question RAP - F1",
+        #     marker="s",
+        #     color="green",
+        # )
+        plt.plot(
+            repetition_penalties,
+            afrp,
+            label="RAP - F1",
+            marker="o",
+            color="orange",
+        )
+        plt.xlabel("Repetition Penalties")
+        plt.ylabel("Score")
+        # plt.xlim(0.99, 1.31)
+        # y in percentage
+        plt.gca().yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
+        plt.title(f"{model} {title}")
+        plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+        plt.show()
+
+
+def plot_best_afrp(
+    result,
+    models=None,
+    title="Models with Best RAP - F1",
+    ref_result=None,
+):
+    # Initialize lists to store the statistics
+    model_names = []
+    best_f1 = []
+    best_afrp = []
+    best_repetition_penalty = []
+    best_mtr = []
+
+    if models is None:
+        models = result.keys()
+    for model in models:
+        print(f"model: {model}")
+        df = result[model]["df_overall"]
+
+        # Calculate the statistics
+        precision = [
+            df["precision"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        recall = [
+            df["recall"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
+        f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
+
+        newline_score = [
+            df["newline_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        # print(f"newline_score: {newline_score}")
+
+        repetition_score = [
+            df["repetition_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        # print(f"repetition_score: {repetition_score}")
+
+        answer_len = [
+            df["answer_len"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+
+        afrp = [
+            calc_adjusted_performance(f, n + r, l)
+            for f, n, r, l in zip(f1, newline_score, repetition_score, answer_len)
+        ]
+
+        best_afrp.append(max(afrp))
+        best_afrp_index = afrp.index(best_afrp[-1])
+        best_repetition_penalty.append(df["repetition_penalty"][best_afrp_index])
+
+        best_f1.append(f1[best_afrp_index])
+        best_mtr.append(
+            newline_score[best_afrp_index] + repetition_score[best_afrp_index]
+        )
+
+        # print(
+        #     f"best repetition penalty: {best_repetition_penalty[-1]}, best afrp: {best_afrp[-1]}, f1: {best_f1[-1]}"
+        # )
+
+        df = result[model]["df_list_repetition_penalty"][best_afrp_index]
+
+        model_names.append(
+            f"{model} (RP={best_repetition_penalty[-1]})"
+        )  # Add the model name to the list
+
+    if ref_result is not None:
+        print("ref_result:", ref_result)
+        for model in ref_result.keys():
+            model_names.append(model)
+            df = pd.read_csv(ref_result[model])
+            # df = df[df["id"].isin(wikidata_df["id"])]
+
+            p = df["precision"].mean()
+            r = df["recall"].mean()
+
+            f1 = 2 * p * r / (p + r) if p + r > 0 else 0
+            best_f1.append(f1)
+            best_afrp.append(f1)
+            best_mtr.append(0)
+
+    print("model_names:", model_names)
+    # print("best_f1:", best_f1)
+    # print("best_afrp:", best_afrp)
+
+    # Create a DataFrame with the statistics
+    data = pd.DataFrame(
+        {
+            "Model": model_names,
+            "RAP - F1": best_afrp,
+            "F1": best_f1,
+        }
+    )
+
+    # Melt the DataFrame to a long format
+    data_melted = data.melt(id_vars="Model", var_name="Metric", value_name="Score")
+
+    # Pivot the DataFrame to a wide format
+    data_pivoted = data_melted.pivot(index="Metric", columns="Model", values="Score")
+
+    # make sure the columns are following the order of the models
+    data_pivoted = data_pivoted[model_names]
+
+    # make sure three groups in the order of precision, recall, f1
+    data_pivoted = data_pivoted.reindex(["RAP - F1", "F1"])
+
+    # Plot the statistics
+    plt.figure(figsize=(15, 6))
+    ax = data_pivoted.plot(kind="bar", ax=plt.gca(), width=0.9)
+    plt.title(title)
+    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+    # Set the rotation of the x-axis labels to 0 degrees
+    plt.xticks(rotation=0)
+
+    # Format the y-axis to display as percentage
+    ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
+
+    # get the max value of the y-axis
+    a1 = max(best_afrp)
+    a2 = max(best_f1)
+
+    max_value = max([a1, a2]) * 1.12
+    print("max_value:", max_value)
+
+    # Set the y-axis limit up to 70%
+    ax.set_ylim(0, max_value)
+
+    # Add the values above each bar
+    for p in ax.patches:
+        ax.annotate(
+            f"{p.get_height() * 100:.1f}",
+            (p.get_x() + p.get_width() / 2.0, p.get_height()),
+            ha="center",
+            va="bottom",
+            xytext=(0, 10),
+            textcoords="offset points",
+            rotation=90,
+        )
+
+    plt.show()
+    return data_pivoted, best_mtr
+
+
+def plot_best_performance(
+    result,
+    models=None,
+    title="Models with Best F1 Score",
+    adjusted_f1=False,
+    ref_result=None,
+):
+    # Initialize lists to store the statistics
+    model_names = []
+    best_precision = []
+    best_recall = []
+    best_f1 = []
+    best_repetition_penalty = []
+    best_mtr = []
+
+    if models is None:
+        models = result.keys()
+    for model in models:
+        print(f"model: {model}")
+        df = result[model]["df_overall"]
+
+        # Calculate the statistics
+        precision = [
+            df["precision"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        recall = [
+            df["recall"].mean() for df in result[model]["df_list_repetition_penalty"]
+        ]
+        newline_score = [
+            df["newline_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+
+        repetition_score = [
+            df["repetition_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+
+        if adjusted_f1:
+            precision, recall = adjust_perf_scores_with_repetition_penalty(
+                result[model], precision, recall
+            )
+
+        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
+        f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
+
+        best_f1.append(max(f1))
+        best_f1_index = f1.index(best_f1[-1])
+        best_repetition_penalty.append(df["repetition_penalty"][best_f1_index])
+
+        best_precision.append(precision[best_f1_index])
+        best_recall.append(recall[best_f1_index])
+        best_mtr.append(newline_score[best_f1_index] + repetition_score[best_f1_index])
+
+        print(
+            f"best repetition penalty: {best_repetition_penalty[-1]}, best f1: {best_f1[-1]}, precision: {best_precision[-1]}, recall: {best_recall[-1]}"
+        )
+
+        df = result[model]["df_list_repetition_penalty"][best_f1_index]
+
+        model_names.append(
+            f"{model} (RP={best_repetition_penalty[-1]})"
+        )  # Add the model name to the list
+
+        # print sum for columns: newline_score, repetition_score
+        print(
+            f"newline_score: {df['newline_score'].sum()}, repetition_score: {df['repetition_score'].sum()}"
+        )
+
+    if ref_result is not None:
+        print("ref_result:", ref_result)
+        for model in ref_result.keys():
+            model_names.append(model)
+            df = pd.read_csv(ref_result[model])
+            # df = df[df["id"].isin(wikidata_df["id"])]
+
+            best_precision.append(df["precision"].mean())
+            best_recall.append(df["recall"].mean())
+            f1 = (
+                2
+                * (best_precision[-1] * best_recall[-1])
+                / (best_precision[-1] + best_recall[-1])
+            )
+            # best_f1.append(df["f1"].mean())
+            best_f1.append(f1)
+            best_mtr.append(0)
+
+    # Create a DataFrame with the statistics
+    data = (
+        pd.DataFrame(
+            {
+                "Model": model_names,
+                "Adjusted Precision with RP": best_precision,
+                "Adjusted Recall with RP": best_recall,
+                "Adjusted F1 with RP": best_f1,
+            }
+        )
+        if adjusted_f1
+        else pd.DataFrame(
+            {
+                "Model": model_names,
+                "Precision": best_precision,
+                "Recall": best_recall,
+                "F1": best_f1,
+            }
+        )
+    )
+    columns = list(data.columns)
+
+    # Melt the DataFrame to a long format
+    data_melted = data.melt(id_vars="Model", var_name="Metric", value_name="Score")
+
+    # Pivot the DataFrame to a wide format
+    data_pivoted = data_melted.pivot(index="Metric", columns="Model", values="Score")
+
+    # make sure the columns are following the order of the models
+    data_pivoted = data_pivoted[model_names]
+
+    # make sure three groups in the order of precision, recall, f1
+    data_pivoted = data_pivoted.reindex(columns[1:])
+
+    # Plot the statistics
+    plt.figure(figsize=(10, 6))
+    ax = data_pivoted.plot(kind="bar", ax=plt.gca(), width=0.9)
+    plt.title(title)
+    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+    # Set the rotation of the x-axis labels to 0 degrees
+    plt.xticks(rotation=0)
+
+    # Format the y-axis to display as percentage
+    ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
+
+    # get the max value of the y-axis
+    a1 = max(best_precision)
+    a2 = max(best_recall)
+    a3 = max(best_f1)
+
+    max_value = max([a1, a2, a3]) * 1.12
+    print("max_value:", max_value)
+
+    # Set the y-axis limit up to 70%
+    ax.set_ylim(0, max_value)
+
+    # Add the values above each bar
+    for p in ax.patches:
+        ax.annotate(
+            f"{p.get_height() * 100:.1f}",
+            (p.get_x() + p.get_width() / 2.0, p.get_height()),
+            ha="center",
+            va="bottom",
+            xytext=(0, 10),
+            textcoords="offset points",
+            rotation=90,
+        )
+
+    plt.show()
+    return data_pivoted, best_mtr
+
+
+def plot_best_performance_ms_macro(
+    result,
+    models=None,
+    title="Models with Best RAP - Performance",
+    ref_result=None,
+    skip_generic_prompt=False,
+    include_adjusted_performance=True,
+):
+    # Initialize lists to store the statistics
+    model_names = []
+    best_f1 = []
+    best_afrp = []
+    best_repetition_penalty = []
+    best_bleu1 = []
+    best_rougeL = []
+    best_mtr = []
+
+    if models is None:
+        models = result.keys()
+    for model in models:
+        if skip_generic_prompt and "generic prompt" in model:
+            continue
+        print(f"model: {model}")
+        df = result[model]["df_overall"]
+
+        # Calculate the statistics
+        bleu1 = [x for x in df["bleu1"]]
+        rougeL = [x for x in df["rougeL"]]
+        f1 = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
+
+        newline_score = [
+            df["newline_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        # print(f"newline_score: {newline_score}")
+
+        repetition_score = [
+            df["repetition_score"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        # print(f"repetition_score: {repetition_score}")
+
+        answer_len = [
+            df["answer_len"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+
+        afrp = [
+            calc_adjusted_performance(f, n + r, l)
+            for f, n, r, l in zip(f1, newline_score, repetition_score, answer_len)
+        ]
+
+        best_afrp.append(max(afrp if include_adjusted_performance else f1))
+        best_afrp_index = (
+            afrp.index(best_afrp[-1])
+            if include_adjusted_performance
+            else f1.index(best_afrp[-1])
+        )
+        best_repetition_penalty.append(df["repetition_penalty"][best_afrp_index])
+
+        best_f1.append(f1[best_afrp_index])
+        best_bleu1.append(bleu1[best_afrp_index])
+        best_rougeL.append(rougeL[best_afrp_index])
+        best_mtr.append(
+            newline_score[best_afrp_index] + repetition_score[best_afrp_index]
+        )
+
+        # print(
+        #     f"best repetition penalty: {best_repetition_penalty[-1]}, best afrp: {best_afrp[-1]}, f1: {best_f1[-1]}"
+        # )
+
+        df = result[model]["df_list_repetition_penalty"][best_afrp_index]
+
+        model_names.append(
+            f"{model} (RP={best_repetition_penalty[-1]})"
+        )  # Add the model name to the list
+
+    if ref_result is not None:
+        print("ref_result:", ref_result)
+        for model in ref_result.keys():
+            model_names.append(model)
+            df = pd.read_csv(ref_result[model], comment="#", on_bad_lines="warn")
+            # df = df[df["id"].isin(wikidata_df["id"])]
+
+            p = df["bleu1"][0]
+            best_bleu1.append(p)
+
+            r = df["rougeL"][0]
+            best_rougeL.append(r)
+
+            f1 = 2 * p * r / (p + r) if p + r > 0 else 0
+            best_f1.append(f1)
+            best_afrp.append(f1)
+            best_mtr.append(0)
+
+    # print("model_names:", model_names)
+    # print("best_f1:", best_f1)
+    # print("best_afrp:", best_afrp)
+
+    # Create a DataFrame with the statistics
+    data = (
+        pd.DataFrame(
+            {
+                "Model": model_names,
+                "RAP - Perf Score": best_afrp,
+                "Overall Perf Score": best_f1,
+            }
+        )
+        if include_adjusted_performance
+        else pd.DataFrame(
+            {
+                "Model": model_names,
+                "Bleu-1": best_bleu1,
+                "Rouge-L": best_rougeL,
+                "Overall Perf Score": best_f1,
+            }
+        )
+    )
+
+    # Melt the DataFrame to a long format
+    data_melted = data.melt(id_vars="Model", var_name="Metric", value_name="Score")
+
+    # Pivot the DataFrame to a wide format
+    data_pivoted = data_melted.pivot(index="Metric", columns="Model", values="Score")
+
+    # make sure the columns are following the order of the models
+    data_pivoted = data_pivoted[model_names]
+
+    columns = list(data.columns)
+    data_pivoted = data_pivoted.reindex(columns[1:])
+
+    # Plot the statistics
+    plt.figure(figsize=(10, 6))
+    ax = data_pivoted.plot(kind="bar", ax=plt.gca(), width=0.9)
+    plt.title(title)
+    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+    # Set the rotation of the x-axis labels to 0 degrees
+    plt.xticks(rotation=0)
+
+    # Format the y-axis to display as percentage
+    ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
+
+    # get the max value of the y-axis
+    a1 = max(best_afrp)
+    a2 = max(best_f1)
+    a3 = max(best_bleu1)
+    a4 = max(best_rougeL)
+
+    max_value = (
+        max([a1, a2] if include_adjusted_performance else [a1, a2, a3, a4]) * 1.12
+    )
+    print("max_value:", max_value)
+
+    # Set the y-axis limit up to 70%
+    ax.set_ylim(0, max_value)
+
+    # Add the values above each bar
+    for p in ax.patches:
+        ax.annotate(
+            f"{p.get_height() * 100:.1f}",
+            (p.get_x() + p.get_width() / 2.0, p.get_height()),
+            ha="center",
+            va="bottom",
+            xytext=(0, 10),
+            textcoords="offset points",
+            rotation=90,
+        )
+
+    plt.show()
+    return data_pivoted, best_mtr
+
+
+all_open_source_models = [
+    "gemma-1.1-2b-it",
+    "Phi-3-mini-128k-instruct",
+    "gemma-1.1-7b-it",
+    "Llama-2-7b-chat-hf",
+    "Mistral-7B-Instruct-v0.2",
+    "Meta-Llama-3-8B-Instruct",
+    "Llama-2-13b-chat-hf",
+    "Llama-2-70b-chat-hf",
+    "Meta-Llama-3-70B-Instruct",
+]
+
+
+def load_for_repetition_penalty_ms_macro(
+    csv_result_file, repetition_penalty, force_recalculate=False
+):
+    result_file = replace_last(
+        csv_result_file, ".csv", f"_rpp_{repetition_penalty:.2f}.csv"
+    )
+    df = load_with_newline_and_repetition_scores(
+        result_file, force_recalculate=force_recalculate
+    )
+
+    return df
+
+
+# MS MACRO
+def plot_performance_scores_ms_macro(
+    result,
+    models=None,
+    title="Performance",
+):
+    if models is None:
+        models = result.keys()
+    for model in models:
+        print(f"model: {model}")
+        df = result[model]["df_overall"]
+        # print(result[model]["df_list_repetition_penalty"][0].describe())
+
+        # Calculate the statistics
+        bleu1 = list(df["bleu1"])
+        rougeL = list(df["rougeL"])
+        f1 = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
+        best_f1 = max(f1)
+        best_f1_index = f1.index(best_f1)
+
+        bleu1, rougeL = adjust_perf_scores_with_repetition_penalty(
+            result[model], bleu1, rougeL
+        )
+        afrp = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
+
+        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
+        best_afrp = max(afrp)
+        best_afrp_index = afrp.index(best_afrp)
+
+        repetition_penalties = list(df["repetition_penalty"])
+
+        # line plot for precision, recall, f1
+        plt.figure(figsize=(10, 6))
+
+        plt.axvspan(
+            repetition_penalties[best_f1_index] - 0.01,
+            repetition_penalties[best_f1_index] + 0.01,
+            alpha=0.5,
+            edgecolor="none",
+            facecolor="blue",
+        )
+
+        plt.axvspan(
+            repetition_penalties[best_afrp_index] - 0.01,
+            repetition_penalties[best_afrp_index] + 0.01,
+            alpha=0.5,
+            edgecolor="none",
+            facecolor="orange",
+        )
+
+        plt.plot(
+            repetition_penalties,
+            f1,
+            label="Overall Perf Score",
+            marker="D",
+            color="blue",
+        )
+        plt.plot(
+            repetition_penalties,
+            afrp,
+            label="RAP - Perf Score",
+            marker="o",
+            color="orange",
+        )
+
+        plt.xlabel("Repetition Penalties")
+        plt.ylabel("Score")
+        # plt.xlim(0.99, 1.31)
+        # y in percentage
+        plt.gca().yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
+        plt.title(f"{model} {title}")
+        plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+        plt.show()
+
+
+def plot_repetition_factors(result, groups):
+    for group in groups:
+        # Plot the statistics
+        plt.figure(figsize=(10, 6))
+
+        max_value = 0
+        for model in result.keys():
+            if not group in model.lower():
+                continue
+            print(f"model: {model}")
+            df = result[model]["df_overall"]
+            repetition_panelties = [
+                repetition_penalty for repetition_penalty in df["repetition_penalty"]
+            ]
+
+            mean_score = [
+                df["total_repetitions"].mean()
+                for df in result[model]["df_list_repetition_penalty"]
+            ]
+
+            sns.lineplot(x=repetition_panelties, y=mean_score, label=model)
+
+            new_max = max(mean_score)
+            if new_max > max_value:
+                max_value = new_max
+
+        max_value = max_value * 1.05
+        # if max_value < 1.5:
+        #     max_value = 1.5
+        # set ylimit
+        plt.ylim(0, max_value)
+
+        # show grid
+        plt.grid(True)
+        plt.xlabel("Repetition Penalties")
+        plt.ylabel("Mean Total Repetitions")
+        plt.title("Mean Total Repetitions vs Repetition Penalties")
+        plt.legend()
+
+        plt.show()
+
+
+def plot_repetition_factors_by_group(result, group_filter=None):
+    markers = ["D", "o", "s", "x"]
+    colors = ["blue", "orange", "green", "red"]
+
+    # Plot the statistics
+    plt.figure(figsize=(10, 6))
+    index = 0
+    max_value = 0
+
+    for model in result.keys():
+        if group_filter is not None and group_filter not in model:
+            continue
+
+        print(f"model: {model}")
+
+        df = result[model]["df_overall"]
+        repetition_panelties = [
+            repetition_penalty for repetition_penalty in df["repetition_penalty"]
+        ]
+
+        # Calculate the statistics
+        mean_score = [
+            df["total_repetitions"].mean()
+            for df in result[model]["df_list_repetition_penalty"]
+        ]
+        if len(mean_score) != len(repetition_panelties):
+            print(
+                f"model: {model} has different length of repetition penalties and mean score"
+            )
+            print("repetition_panelties:", len(repetition_panelties))
+            print("mean_score:", len(mean_score))
+            continue
+
+        new_max = max(mean_score)
+        if new_max > max_value:
+            max_value = new_max
+
+        sns.lineplot(
+            x=repetition_panelties,
+            y=mean_score,
+            label=model,
+            marker=markers[index],
+            color=colors[index],
+        )
+
+        index += 1
+
+    max_value = max_value * 1.05
+    # if max_value < 1.5:
+    #     max_value = 1.5
+    # set ylimit
+    plt.ylim(0, max_value)
+    max_value = 0
+
+    plt.xlabel("Repetition Penalties")
+    plt.ylabel("Mean Total Repetitions")
+    plt.title("Mean Total Repetitions vs Repetition Penalties")
+    plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
+
+    plt.show()
+
+
+ms_marco_csv_result_files = [
+    "data/results_v2/gemma-1.1-2b-it(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/gemma-1.1-2b-it(RAG - Chat Template)_mm.csv",
+    "data/results_v2/gemma-1.1-2b-it(Non-RAG)_mm.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(Non-RAG)_mm.csv",
+    "data/results_v2/gemma-1.1-7b-it(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/gemma-1.1-7b-it(RAG - Chat Template)_mm.csv",
+    "data/results_v2/gemma-1.1-7b-it(Non-RAG)_mm.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(Non-RAG)_mm.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(Non-RAG)_mm.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(Non-RAG)_mm.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(Non-RAG)_mm.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(Non-RAG)_mm.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Generic Prompt)_mm.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Chat Template)_mm.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(Non-RAG)_mm.csv",
+]
+
+webqsp_csv_result_files = [
+    "data/results_v2/gemma-1.1-2b-it(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/gemma-1.1-2b-it(RAG - Chat Template)_wd.csv",
+    "data/results_v2/gemma-1.1-2b-it(Non-RAG)_wd.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Phi-3-mini-128k-instruct(Non-RAG)_wd.csv",
+    "data/results_v2/gemma-1.1-7b-it(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/gemma-1.1-7b-it(RAG - Chat Template)_wd.csv",
+    "data/results_v2/gemma-1.1-7b-it(Non-RAG)_wd.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Llama-2-7b-chat-hf(Non-RAG)_wd.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Mistral-7B-Instruct-v0.2(Non-RAG)_wd.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Meta-Llama-3-8B-Instruct(Non-RAG)_wd.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Llama-2-13b-chat-hf(Non-RAG)_wd.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Llama-2-70b-chat-hf(Non-RAG)_wd.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Generic Prompt)_wd.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(RAG - Chat Template)_wd.csv",
+    "data/results_v2/Meta-Llama-3-70B-Instruct(Non-RAG)_wd.csv",
+]
+
+
+def calc_rap_scores(result, precision="precision", recall="recall"):
+    newline_score = [
+        df["newline_score"].mean() for df in result["df_list_repetition_penalty"]
+    ]
+
+    repetition_score = [
+        df["repetition_score"].mean() for df in result["df_list_repetition_penalty"]
+    ]
+
+    if precision in result["df_list_repetition_penalty"][0].columns:
+        precision = [
+            df[precision].mean() for df in result["df_list_repetition_penalty"]
+        ]
+        recall = [df[recall].mean() for df in result["df_list_repetition_penalty"]]
+    else:
+        precision = result["df_overall"][precision]
+        recall = result["df_overall"][recall]
+
+    f1 = [2 * (p * r) / (p + r) for p, r in zip(precision, recall)]
+
+    nrr = [
+        1 - (n + r) / s
+        for f, n, r, s in zip(
+            f1, newline_score, repetition_score, result["df_overall"]["answer_len"]
+        )
+    ]
+
+    rap = [calc_adjusted_performance(f, 1 - n) for f, n in zip(f1, nrr)]
+
+    return newline_score, repetition_score, f1, rap, nrr
+
+
+def get_model_name(csv_result_file):
+    parts = re.split(r"[_/]", csv_result_file)
+    print(f"parts: {parts}")
+    model_name = parts[3]
+    return model_name
+
+
+def load_webqsp_result(csv_result_files, force_recalculate=False, save=False):
+    result = {}
+    for i, csv_result_file in enumerate(csv_result_files):
+        try:
+            df = pd.read_csv(csv_result_file)
+            model_name = get_model_name(csv_result_file)
+            print(f"\tmodel_name: {model_name}")
+
+            dfs = [
+                calculate_performance_score(
+                    csv_result_file,
+                    repetition_penalty,
+                    force_recalculate=force_recalculate,
+                )
+                for repetition_penalty in df["repetition_penalty"]
+            ]
+
+            answer_lens = []
+            for df_rpp in dfs:
+                answer_lens.append(df_rpp["answer_len"].mean())
+            df["answer_len"] = answer_lens
+
+            result[model_name] = {
+                "df_overall": df,
+                "df_list_repetition_penalty": dfs,
+                "file": csv_result_file,
+            }
+            newline_score, repetition_score, perf, rap, nrr = calc_rap_scores(
+                result[model_name]
+            )
+            df["newline_score"] = newline_score
+            df["repetition_score"] = repetition_score
+            df["total_repetitions"] = df["newline_score"] + df["repetition_score"]
+            df["perf"] = perf
+            df["nrr"] = nrr
+            df["rap"] = rap
+            df["rr"] = df["nrr"].apply(lambda x: 1 - x)
+            if save:
+                df.to_csv(csv_result_file, index=False)
+        except Exception as e:
+            print(f"Error: {e}")
+            traceback.print_exc()
+
+    return result
+
+
+def load_ms_marco_result(
+    csv_result_files, force_recalculate=False, calc_bertscore=False, save=False
+):
+    result = {}
+    for csv_result_file in csv_result_files:
+        try:
+            df = pd.read_csv(csv_result_file)
+            model_name = get_model_name(csv_result_file)
+            print(f"\tmodel_name: {model_name}")
+
+            dfs = [
+                load_for_repetition_penalty_ms_macro(
+                    csv_result_file,
+                    repetition_penalty,
+                    force_recalculate=force_recalculate,
+                )
+                for repetition_penalty in df["repetition_penalty"]
+            ]
+
+            answer_lens = []
+            for df_rpp in dfs:
+                answer_lens.append(df_rpp["answer_len"].mean())
+            df["answer_len"] = answer_lens
+
+            col = "bert_score" if calc_bertscore else "meteor"
+            score_unavailable = col not in df.columns
+
+            if score_unavailable:
+                save = True
+                bert_meteor_scores = []
+                bert_score_references = None
+                for df_rpp in dfs:
+                    if calc_bertscore:
+                        bert_meteor_score = 0
+
+                        for i, row in df_rpp.iterrows():
+                            answer = row["answer"]
+                            if not isinstance(answer, str):
+                                answer = ""
+                            bert_meteor_score += bert_score.compute(
+                                predictions=[answer],
+                                references=[row["ground_truth"][0]],
+                                lang="en",
+                                model_type="microsoft/deberta-large-mnli",
+                            )["f1"][0]
+                        # get average of bertscore
+                        bert_meteor_score = bert_meteor_score / len(df_rpp)
+
+                        print(f"bert_score: {bert_meteor_score}")
+                    else:
+                        bert_meteor_score = meteor.compute(
+                            predictions=df_rpp["answer"],
+                            references=df_rpp["ground_truth"],
+                        )["meteor"]
+
+                    bert_meteor_scores.append(bert_meteor_score)
+
+                df[col] = bert_meteor_scores
+
+            result[model_name] = {
+                "df_overall": df,
+                "df_list_repetition_penalty": dfs,
+                "file": csv_result_file,
+            }
+            newline_score, repetition_score, perf, rap, nrr = calc_rap_scores(
+                result[model_name],
+                precision=col,
+                recall=col,
+            )
+            df["newline_score"] = newline_score
+            df["repetition_score"] = repetition_score
+            df["total_repetitions"] = df["newline_score"] + df["repetition_score"]
+            df["perf"] = perf
+            df["nrr"] = nrr
+            df["rap"] = rap
+            df["rr"] = df["nrr"].apply(lambda x: 1 - x)
+
+            if save:
+                df.to_csv(csv_result_file, index=False)
+        except Exception as e:
+            print("An error occurred:", e)
+            traceback.print_exc()
+            print(f"csv_result_file: {csv_result_file}")
+
+    return result

notebooks/03a_RAPGeT_v2_Data Analysis_Chat_Template.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6273fc3413aa0c507438f5061c2047948cee5516d16244aacbd2f6f72b19dfff
-size 1562071
+oid sha256:205197936482de4ebc17e7cad622a0e699303d062112cc45df85477e7f1f8328
+size 1557858

notebooks/03b_RAPGeT_v2_Data Analysis_Generic_Prompt.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b948f85d9b7ba464ddadbd63be90d285641d64d366140a55a2999f69b21b2021
-size 14255738
+oid sha256:5e3885c9aa8fd8a1f83f6693df9c68a278575b6b1caf9e087c00eb6264d3e886
+size 26471820

notebooks/03c_RAPGeT_v2_Data Analysis.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f31409b55a833748353233d379d184488373cab3e54697002a503877914a2af
-size 1954627
+oid sha256:26dcf7f7d287ca5c6135b26533495ec4d56ed3b0a7eac7d0e10f12aec4d95257
+size 1714764

results/mac-results_rpp_with_mnt_2048_generic_prompt_metrics.csv

@@ -1,26 +1,27 @@
 model,rpp,comet,meteor,spbleu,bleu_1,rouge_l,ews_score,repetition_score,total_repetitions,rr,rap,translation_completeness,num_max_output_tokens
-internlm/internlm2_5-7b-chat,1.00,0.7357995069773978,0.4297612514398102,15.060226683930628,0.1506022668393063,0.4097577795330234,0.04942630185348632,9.235657546337158,9.285083848190645,0.07525035765379114,0.6824623187873116,1.0,2
-internlm/internlm2_5-7b-chat,1.02,0.7377187550620283,0.4246676977198055,14.728605282752795,0.147286052827528,0.4063246630867048,0.06972639011473963,5.35657546337158,5.426301853486319,0.04625547346404442,0.7043723959353786,1.0,1
-internlm/internlm2_5-7b-chat,1.04,0.7371160490183523,0.4173352728374962,13.846403511622256,0.1384640351162226,0.3988121301027288,0.06884377758164166,5.315092674315975,5.383936451897617,0.04501878242643857,0.704667851183933,1.0,1
-internlm/internlm2_5-7b-chat,1.06,0.7338597697698218,0.3997609847704189,12.213374588416173,0.1221337458841617,0.3841365748920261,0.05825242718446602,5.275375110326567,5.333627537511033,0.043830827367611756,0.7023888258277158,1.0,1
-internlm/internlm2_5-7b-chat,1.08,0.7318234702626478,0.3881614120395272,11.369735763522288,0.1136973576352228,0.372963223209074,0.06707855251544571,5.283318623124448,5.350397175639894,0.04300663332269164,0.7010173875930041,1.0,1
-internlm/internlm2_5-7b-chat,1.10,0.7288648442604431,0.3784182249483568,10.377989030628608,0.103779890306286,0.3618424457502351,0.05207413945278023,5.288614298323036,5.340688437775817,0.042176064682512025,0.6987634348896543,1.0,1
-microsoft/Phi-3.5-mini-instruct,1.00,0.710605339281136,0.3788926591792472,9.70032874202361,0.097003287420236,0.3556134739443916,5.390997352162401,12.997352162400706,18.388349514563107,0.13770903562694164,0.6191875166952294,1.0,4
-microsoft/Phi-3.5-mini-instruct,1.02,0.7150978385770836,0.3741049510326346,9.910633597905436,0.0991063359790543,0.3453160556383774,3.586054721977052,7.001765225066196,10.587819947043249,0.08180522500528503,0.6589279165887452,1.0,2
-microsoft/Phi-3.5-mini-instruct,1.04,0.7074641684778791,0.3538698731015666,9.19721270538052,0.0919721270538052,0.3225824135517728,0.05119152691968226,0.05560458958517211,0.10679611650485436,0.000859149229250836,0.7068566122109297,1.0,0
-microsoft/Phi-3.5-mini-instruct,1.06,0.6962301708225224,0.3252854575717334,6.967166383106307,0.069671663831063,0.2948764736589108,0.0353045013239188,0.06796116504854369,0.10326566637246248,0.0007865281839265906,0.6956827814674672,1.0,0
-microsoft/Phi-3.5-mini-instruct,1.08,0.6823413657174107,0.301599095293242,5.452744292893752,0.0545274429289375,0.2726387617958179,0.07678729037952339,0.04766107678729038,0.12444836716681378,0.0009016671249608319,0.68172639822959,1.0,0
-microsoft/Phi-3.5-mini-instruct,1.10,0.6717851540206916,0.2885734336603344,4.751039447225815,0.0475103944722581,0.2604284999048123,0.08031774051191527,0.02383053839364519,0.10414827890556046,0.0007188284314919954,0.6713024292710504,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.739080294072365,0.4490104515425626,6.7013404492782405,0.0670134044927823,0.4196181637680596,0.36716681376875554,139.80935569285083,140.1765225066196,0.48362195756964893,0.45567717525911156,0.999117387466902,15
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.02,0.743018615750854,0.4514907128972251,8.545954556237808,0.085459545562378,0.4214940415288087,1.0035304501323918,67.00353045013239,68.00706090026479,0.2929644725635723,0.5543280318044165,1.0,6
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.04,0.7432195577780335,0.4517500968367987,10.080425294411064,0.1008042529441106,0.4200973007348334,0.01059135039717564,35.19770520741395,35.208296557811124,0.17564306911947306,0.6234997933386273,1.0,6
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.06,0.7430821573139815,0.4484154407825542,10.37470506193322,0.1037470506193321,0.4160289393328045,1.8005295675198587,26.880847308031775,28.68137687555163,0.1522966823356282,0.6381095208514015,1.0,3
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.08,0.7435937259684909,0.4407733547418294,10.930453247368872,0.1093045324736887,0.4113063412348818,0.09267431597528684,12.007943512797882,12.100617828773169,0.06721477842655646,0.6952559422553878,1.0,3
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.10,0.7427059700687901,0.4358940590119784,11.381344076286156,0.1138134407628615,0.4062980635945339,0.03971756398940865,0.6681376875551632,0.707855251544572,0.003961824217515018,0.7397693206556093,1.0,1
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.00,0.3888604919913587,0.2055875758168277,0.2434587181959752,0.0024345871819597,0.1844552188025856,638.2797881729921,3889.9232127096207,4528.203000882612,0.9210262088655917,0.15480936210106447,0.9240953221535746,570
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.00,0.7222260562908512,0.4039898602650971,13.461179673541356,0.1346117967354136,0.3819960428004565,0.05736981465136805,5.87378640776699,5.931156222418358,0.05150372482295595,0.6859703892398439,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.02,0.723643534970515,0.4051102919608809,13.18537912294539,0.1318537912294539,0.3824621732976229,0.06266548984995587,5.840247131509267,5.902912621359223,0.05148734372113075,0.6873279697459551,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.04,0.7238812581796301,0.4039456988919502,13.314773371306682,0.1331477337130668,0.3813737464821349,0.05736981465136805,5.845542806707855,5.902912621359223,0.05127418810757766,0.6877003345402561,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.06,0.7252625281686607,0.4012797167602334,13.19924345265053,0.1319924345265053,0.3798291332004637,0.06266548984995587,5.847308031774051,5.909973521624007,0.05081388730791121,0.6893297921407147,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.08,0.7261167238322592,0.3987395126194482,12.656486100206328,0.1265648610020633,0.376975448872996,0.05648720211827008,5.820829655781112,5.877316857899382,0.05012721880128273,0.6906157284372703,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.10,0.7264630642225547,0.3964859769229444,12.284961706379857,0.1228496170637985,0.3744555065346823,0.04942630185348632,0.09267431597528684,0.14210061782877317,0.001266948385624464,0.7255432558140477,1.0,0
+internlm/internlm2_5-7b-chat,1.00,0.7357995069773978,0.4297612514398102,15.060226683930628,0.1506022668393063,0.4097577795330234,0.04942630185348632,9.235657546337158,9.285083848190645,0.07525035765379114,0.581878095297299,1.0,2
+internlm/internlm2_5-7b-chat,1.02,0.7377187550620283,0.4246676977198055,14.728605282752795,0.147286052827528,0.4063246630867048,0.06972639011473963,5.35657546337158,5.426301853486319,0.04625547346404442,0.6400103546749837,1.0,1
+internlm/internlm2_5-7b-chat,1.04,0.7371160490183523,0.4173352728374962,13.846403511622256,0.1384640351162226,0.3988121301027288,0.06884377758164166,5.315092674315975,5.383936451897617,0.04501878242643857,0.6419783129560218,1.0,1
+internlm/internlm2_5-7b-chat,1.06,0.7338597697698218,0.3997609847704189,12.213374588416173,0.1221337458841617,0.3841365748920261,0.05825242718446602,5.275375110326567,5.333627537511033,0.043830827367611756,0.6415304775228277,1.0,1
+internlm/internlm2_5-7b-chat,1.08,0.7318234702626478,0.3881614120395272,11.369735763522288,0.1136973576352228,0.372963223209074,0.06707855251544571,5.283318623124448,5.350397175639894,0.04300663332269164,0.6414061446416202,1.0,1
+internlm/internlm2_5-7b-chat,1.10,0.7288648442604431,0.3784182249483568,10.377989030628608,0.103779890306286,0.3618424457502351,0.05207413945278023,5.288614298323036,5.340688437775817,0.042176064682512025,0.6404777687452995,1.0,1
+microsoft/Phi-3.5-mini-instruct,1.00,0.710605339281136,0.3788926591792472,9.70032874202361,0.097003287420236,0.3556134739443916,5.390997352162401,12.997352162400706,18.388349514563107,0.13770903562694164,0.4556065638568846,1.0,4
+microsoft/Phi-3.5-mini-instruct,1.02,0.7150978385770836,0.3741049510326346,9.910633597905436,0.0991063359790543,0.3453160556383774,3.586054721977052,7.001765225066196,10.587819947043249,0.08180522500528503,0.5535666483700645,1.0,2
+microsoft/Phi-3.5-mini-instruct,1.04,0.7074641684778791,0.3538698731015666,9.19721270538052,0.0919721270538052,0.3225824135517728,0.05119152691968226,0.05560458958517211,0.10679611650485436,0.000859149229250836,0.7056422827612971,1.0,0
+microsoft/Phi-3.5-mini-instruct,1.06,0.6962301708225224,0.3252854575717334,6.967166383106307,0.069671663831063,0.2948764736589108,0.0353045013239188,0.06796116504854369,0.10326566637246248,0.0007865281839265906,0.6945886486476809,1.0,0
+microsoft/Phi-3.5-mini-instruct,1.08,0.6823413657174107,0.301599095293242,5.452744292893752,0.0545274429289375,0.2726387617958179,0.07678729037952339,0.04766107678729038,0.12444836716681378,0.0009016671249608319,0.6804972951227785,1.0,0
+microsoft/Phi-3.5-mini-instruct,1.10,0.6717851540206916,0.2885734336603344,4.751039447225815,0.0475103944722581,0.2604284999048123,0.08031774051191527,0.02383053839364519,0.10414827890556046,0.0007188284314919954,0.6703375003284932,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.739080294072365,0.4490104515425626,6.7013404492782405,0.0670134044927823,0.4196181637680596,0.36716681376875554,139.80935569285083,140.1765225066196,0.48362195756964893,0.10176417668651536,0.999117387466902,15
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.02,0.743018615750854,0.4514907128972251,8.545954556237808,0.085459545562378,0.4214940415288087,1.0035304501323918,67.00353045013239,68.00706090026479,0.2929644725635723,0.2626173445806161,1.0,6
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.04,0.7432195577780335,0.4517500968367987,10.080425294411064,0.1008042529441106,0.4200973007348334,0.01059135039717564,35.19770520741395,35.208296557811124,0.17564306911947306,0.4163542580947422,1.0,6
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.06,0.7430821573139815,0.4484154407825542,10.37470506193322,0.1037470506193321,0.4160289393328045,1.8005295675198587,26.880847308031775,28.68137687555163,0.1522966823356282,0.45265620897504616,1.0,3
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.08,0.7435937259684909,0.4407733547418294,10.930453247368872,0.1093045324736887,0.4113063412348818,0.09267431597528684,12.007943512797882,12.100617828773169,0.06721477842655646,0.6035047423944578,1.0,3
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.10,0.7427059700687901,0.4358940590119784,11.381344076286156,0.1138134407628615,0.4062980635945339,0.03971756398940865,0.6681376875551632,0.707855251544572,0.003961824217515018,0.7339134850401315,1.0,1
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.00,0.3888604919913587,0.2055875758168277,0.2434587181959752,0.0024345871819597,0.1844552188025856,638.2797881729921,3889.9232127096207,4528.203000882612,0.9210262088655917,0.00019153263422509847,0.9240953221535746,570
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.02,0.401959364669889,0.2020993340187826,0.2473696547531083,0.002473696547531,0.1795542969510355,611.315975286849,3759.7599293909975,4371.075904677847,0.8883366762655638,0.0005596465146821489,0.912621359223301,562
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.00,0.7222260562908512,0.4039898602650971,13.461179673541356,0.1346117967354136,0.3819960428004565,0.05736981465136805,5.87378640776699,5.931156222418358,0.05150372482295595,0.6162827926700337,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.02,0.723643534970515,0.4051102919608809,13.18537912294539,0.1318537912294539,0.3824621732976229,0.06266548984995587,5.840247131509267,5.902912621359223,0.05148734372113075,0.617524335498735,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.04,0.7238812581796301,0.4039456988919502,13.314773371306682,0.1331477337130668,0.3813737464821349,0.05736981465136805,5.845542806707855,5.902912621359223,0.05127418810757766,0.6181437496179476,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.06,0.7252625281686607,0.4012797167602334,13.19924345265053,0.1319924345265053,0.3798291332004637,0.06266548984995587,5.847308031774051,5.909973521624007,0.05081388730791121,0.6202251404786316,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.08,0.7261167238322592,0.3987395126194482,12.656486100206328,0.1265648610020633,0.376975448872996,0.05648720211827008,5.820829655781112,5.877316857899382,0.05012721880128273,0.6223042523816,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.10,0.7264630642225547,0.3964859769229444,12.284961706379857,0.1228496170637985,0.3744555065346823,0.04942630185348632,0.09267431597528684,0.14210061782877317,0.001266948385624464,0.7237053873903428,1.0,0

results/mac-results_rpp_with_mnt_2048_metrics.csv

@@ -1,31 +1,31 @@
 model,rpp,comet,meteor,spbleu,bleu_1,rouge_l,ews_score,repetition_score,total_repetitions,rr,rap,translation_completeness,num_max_output_tokens
-internlm/internlm2_5-7b-chat,1.00,0.739699612254078,0.4289996929258777,14.734881589173108,0.1473488158917311,0.4096466800937898,0.05383936451897617,12.606354810238305,12.660194174757281,0.10371655820679682,0.6668249823411998,1.0,2
-internlm/internlm2_5-7b-chat,1.02,0.740223803961056,0.4266246904302194,14.583816688798017,0.1458381668879802,0.4071727106228415,0.06266548984995587,9.849073256840247,9.911738746690203,0.0832234063051179,0.6811136450938015,1.0,1
-internlm/internlm2_5-7b-chat,1.04,0.7398856264610577,0.4154585167056314,13.534659133050225,0.1353465913305021,0.3968657713589718,0.07237422771403354,6.529567519858782,6.601941747572815,0.05613508442776736,0.6994963184593858,1.0,1
-internlm/internlm2_5-7b-chat,1.06,0.7379362287241489,0.4039588647855378,12.346740971499404,0.1234674097149939,0.3872447044295494,0.06796116504854369,6.533980582524272,6.601941747572815,0.05513987689359035,0.6983479910444814,0.999117387466902,1
-internlm/internlm2_5-7b-chat,1.08,0.7319988705684732,0.3873176839854818,11.075674965706344,0.1107567496570634,0.3724352909668609,0.05207413945278023,9.83495145631068,9.88702559576346,0.07906717392378437,0.676350746394496,0.999117387466902,1
-internlm/internlm2_5-7b-chat,1.10,0.7295350462119345,0.3769306874386757,10.305163787094209,0.1030516378709421,0.3634496155759507,0.07855251544571933,6.527802294792586,6.606354810238305,0.053004659594657756,0.6918732323764201,0.999117387466902,1
-microsoft/Phi-3.5-mini-instruct,1.00,0.7107840433177544,0.3796831545348129,8.71296896471494,0.0871296896471493,0.3589874395901284,10.670785525154457,17.93821712268314,28.6090026478376,0.20225504327262062,0.5806299320134263,1.0,6
-microsoft/Phi-3.5-mini-instruct,1.02,0.7164765837070485,0.3780585837553919,10.291240080163629,0.1029124008016362,0.3546952732427276,3.585172109443954,7.1403353927625774,10.725507502206531,0.08530053839296368,0.6578947912630083,1.0,2
-microsoft/Phi-3.5-mini-instruct,1.04,0.7111233387336411,0.3547161333845742,8.966881655527896,0.0896688165552789,0.3300979657678754,3.6125330979699912,0.07325684024713151,3.685789938217123,0.02973427131098516,0.6902898733301204,1.0,1
-microsoft/Phi-3.5-mini-instruct,1.06,0.7024363270136286,0.3298733737040869,7.076233088011138,0.0707623308801113,0.3019513312669543,0.04589585172109444,0.05207413945278023,0.09796999117387467,0.0007571675113745661,0.701904666351495,1.0,0
-microsoft/Phi-3.5-mini-instruct,1.08,0.6882111219210848,0.3054541022592767,5.105510599247868,0.0510551059924786,0.2736030007297014,3.3609885260370698,0.06443071491615181,3.4254192409532216,0.023581380370521147,0.6721720091371258,1.0,1
-microsoft/Phi-3.5-mini-instruct,1.10,0.6712992989638161,0.2903831801547132,4.091958857999118,0.0409195885799911,0.251653275009876,0.32215357458075905,0.06531332744924978,0.38746690203000883,0.0023407216247487324,0.6697298117609694,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.7501818982248062,0.4611110508507017,17.87914973742753,0.1787914973742752,0.4340662057009564,0.00706090026478376,0.1262135922330097,0.13327449249779347,0.0011265209898463904,0.7493372784005554,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.02,0.7485114382045625,0.4571517219079576,17.436884594979905,0.174368845949799,0.4311385932640979,0.00706090026478376,0.11562224183583407,0.12268314210061783,0.0010382199383043404,0.7477347219762754,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.04,0.7500591586357918,0.4560467960364254,17.440173470996626,0.1744017347099662,0.4302844557731285,0.00706090026478376,0.13062665489849956,0.13768755516328332,0.0011593944393659004,0.7491900481363686,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.06,0.748812871571673,0.4520416361219855,16.89523258317781,0.168952325831778,0.4260026774745837,0.00706090026478376,0.0997352162400706,0.10679611650485436,0.0008902491962006224,0.7481465381600518,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.08,0.7473851635144647,0.4442106511292453,16.16623784482793,0.1616623784482792,0.4195129470585874,0.01059135039717564,0.13062665489849956,0.1412180052956752,0.001176591707969938,0.7465063134536485,1.0,0
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.10,0.7465709781131172,0.4379837926138161,15.60172257624066,0.1560172257624066,0.4132562932940978,0.01059135039717564,0.07855251544571933,0.08914386584289496,0.000734476013176936,0.7460228409589962,1.0,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.00,0.7426396049131678,0.433632501662176,15.209540658023398,0.1520954065802339,0.4089208235151474,0.00353045013239188,3.901147396293027,3.904677846425419,0.03237065275450547,0.7189848023792343,1.0,1
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.02,0.7436477056353469,0.4329054166518245,15.19102241646024,0.1519102241646024,0.4068967964789407,0.0,3.8905560458958517,3.8905560458958517,0.03219656852361788,0.7200861374743239,1.0,1
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.04,0.7440943776351209,0.4320478700956207,15.05135166158296,0.1505135166158296,0.4062008380201262,0.00353045013239188,0.1526919682259488,0.1562224183583407,0.001352332200022921,0.7430887949425674,1.0,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.06,0.7426502735395928,0.4275429314912545,14.449130821290163,0.1444913082129016,0.4001409979222783,0.00706090026478376,0.13768755516328332,0.14474845542806708,0.0012399256044637321,0.7417300130954148,1.0,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.08,0.7408098006080129,0.4206626658729054,13.933703757385222,0.1393370375738522,0.3964824268676203,0.00353045013239188,0.1297440423654016,0.13327449249779347,0.001134996993385448,0.7399694606935023,1.0,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.10,0.7392685912871718,0.4111211240399151,13.303738403756984,0.1330373840375698,0.3870959581563503,0.00353045013239188,0.12180052956751986,0.12533097969991175,0.0010529672171262895,0.7384905753804022,1.0,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.00,0.7240239171358935,0.4068335357738006,13.565136550617618,0.1356513655061761,0.3866395067055498,0.0529567519858782,0.1209179170344219,0.17387466902030008,0.001578993772192076,0.7228815899808891,1.0,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.02,0.7263097057327799,0.4064914781094827,13.42987641622816,0.1342987641622816,0.3863697821025159,0.06001765225066196,6.236540158870256,6.296557811120918,0.0541899611084103,0.6879984291168549,1.0,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.04,0.7276128307708258,0.4054859896994975,13.295092218891954,0.1329509221889195,0.3851203729935697,0.05207413945278023,0.1297440423654016,0.18181818181818182,0.0016533037985858635,0.7264108595993547,1.0,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.06,0.7276865132383193,0.4014727027723293,13.10860799057166,0.1310860799057166,0.3804952786306688,0.05207413945278023,0.13415710503089143,0.18623124448367168,0.001691057431836761,0.7264569934393594,1.0,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.08,0.726393195584298,0.3987018836449559,12.850537785783194,0.1285053778578319,0.3788945955746495,0.05648720211827008,0.15357458075904679,0.21006178287731686,0.0018871365478087807,0.7250236850699382,1.0,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.10,0.7244012304511832,0.3932239948456176,12.361161644811926,0.1236116164481192,0.3733413807007665,0.05030891438658429,0.08561341571050309,0.13592233009708737,0.0012217374057913526,0.7235167427869905,1.0,0
+internlm/internlm2_5-7b-chat,1.00,0.739699612254078,0.4289996929258777,14.734881589173108,0.1473488158917311,0.4096466800937898,0.05383936451897617,12.606354810238305,12.660194174757281,0.10371655820679682,0.5325881640465967,1.0,2
+internlm/internlm2_5-7b-chat,1.02,0.740223803961056,0.4266246904302194,14.583816688798017,0.1458381668879802,0.4071727106228415,0.06266548984995587,9.849073256840247,9.911738746690203,0.0832234063051179,0.5703659582906754,1.0,1
+internlm/internlm2_5-7b-chat,1.04,0.7398856264610577,0.4154585167056314,13.534659133050225,0.1353465913305021,0.3968657713589718,0.07237422771403354,6.529567519858782,6.601941747572815,0.05613508442776736,0.6221485884888651,1.0,1
+internlm/internlm2_5-7b-chat,1.06,0.7379362287241489,0.4039588647855378,12.346740971499404,0.1234674097149939,0.3872447044295494,0.06796116504854369,6.533980582524272,6.601941747572815,0.05513987689359035,0.6224742543197805,0.999117387466902,1
+internlm/internlm2_5-7b-chat,1.08,0.7319988705684732,0.3873176839854818,11.075674965706344,0.1107567496570634,0.3724352909668609,0.05207413945278023,9.83495145631068,9.88702559576346,0.07906717392378437,0.5717343310562308,0.999117387466902,1
+internlm/internlm2_5-7b-chat,1.10,0.7295350462119345,0.3769306874386757,10.305163787094209,0.1030516378709421,0.3634496155759507,0.07855251544571933,6.527802294792586,6.606354810238305,0.053004659594657756,0.6195690090849183,0.999117387466902,1
+microsoft/Phi-3.5-mini-instruct,1.00,0.7107840433177544,0.3796831545348129,8.71296896471494,0.0871296896471493,0.3589874395901284,10.670785525154457,17.93821712268314,28.6090026478376,0.20225504327262062,0.3608526271635592,1.0,6
+microsoft/Phi-3.5-mini-instruct,1.02,0.7164765837070485,0.3780585837553919,10.291240080163629,0.1029124008016362,0.3546952732427276,3.585172109443954,7.1403353927625774,10.725507502206531,0.08530053839296368,0.5483240204881398,1.0,2
+microsoft/Phi-3.5-mini-instruct,1.04,0.7111233387336411,0.3547161333845742,8.966881655527896,0.0896688165552789,0.3300979657678754,3.6125330979699912,0.07325684024713151,3.685789938217123,0.02973427131098516,0.6495566110355127,1.0,1
+microsoft/Phi-3.5-mini-instruct,1.06,0.7024363270136286,0.3298733737040869,7.076233088011138,0.0707623308801113,0.3019513312669543,0.04589585172109444,0.05207413945278023,0.09796999117387467,0.0007571675113745661,0.7008419489376413,1.0,0
+microsoft/Phi-3.5-mini-instruct,1.08,0.6882111219210848,0.3054541022592767,5.105510599247868,0.0510551059924786,0.2736030007297014,3.3609885260370698,0.06443071491615181,3.4254192409532216,0.023581380370521147,0.6406632969877,1.0,1
+microsoft/Phi-3.5-mini-instruct,1.10,0.6712992989638161,0.2903831801547132,4.091958857999118,0.0409195885799911,0.251653275009876,0.32215357458075905,0.06531332744924978,0.38746690203000883,0.0023407216247487324,0.6665963500989894,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.7501818982248062,0.4611110508507017,17.87914973742753,0.1787914973742752,0.4340662057009564,0.00706090026478376,0.1262135922330097,0.13327449249779347,0.0011265209898463904,0.7476494662426587,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.02,0.7485114382045625,0.4571517219079576,17.436884594979905,0.174368845949799,0.4311385932640979,0.00706090026478376,0.11562224183583407,0.12268314210061783,0.0010382199383043404,0.7461824993322019,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.04,0.7500591586357918,0.4560467960364254,17.440173470996626,0.1744017347099662,0.4302844557731285,0.00706090026478376,0.13062665489849956,0.13768755516328332,0.0011593944393659004,0.7474533388920676,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.06,0.748812871571673,0.4520416361219855,16.89523258317781,0.168952325831778,0.4260026774745837,0.00706090026478376,0.0997352162400706,0.10679611650485436,0.0008902491962006224,0.7468147612728927,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.08,0.7473851635144647,0.4442106511292453,16.16623784482793,0.1616623784482792,0.4195129470585874,0.01059135039717564,0.13062665489849956,0.1412180052956752,0.001176591707969938,0.7447501647073623,1.0,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.10,0.7465709781131172,0.4379837926138161,15.60172257624066,0.1560172257624066,0.4132562932940978,0.01059135039717564,0.07855251544571933,0.08914386584289496,0.000734476013176936,0.7449271706150111,1.0,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.00,0.7426396049131678,0.433632501662176,15.209540658023398,0.1520954065802339,0.4089208235151474,0.00353045013239188,3.901147396293027,3.904677846425419,0.03237065275450547,0.6728297734832243,1.0,1
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.02,0.7436477056353469,0.4329054166518245,15.19102241646024,0.1519102241646024,0.4068967964789407,0.0,3.8905560458958517,3.8905560458958517,0.03219656852361788,0.6741068111074712,1.0,1
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.04,0.7440943776351209,0.4320478700956207,15.05135166158296,0.1505135166158296,0.4062008380201262,0.00353045013239188,0.1526919682259488,0.1562224183583407,0.001352332200022921,0.7410796698393737,1.0,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.06,0.7426502735395928,0.4275429314912545,14.449130821290163,0.1444913082129016,0.4001409979222783,0.00706090026478376,0.13768755516328332,0.14474845542806708,0.0012399256044637321,0.7398912041420567,1.0,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.08,0.7408098006080129,0.4206626658729054,13.933703757385222,0.1393370375738522,0.3964824268676203,0.00353045013239188,0.1297440423654016,0.13327449249779347,0.001134996993385448,0.7382902118097237,1.0,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.10,0.7392685912871718,0.4111211240399151,13.303738403756984,0.1330373840375698,0.3870959581563503,0.00353045013239188,0.12180052956751986,0.12533097969991175,0.0010529672171262895,0.7369357726201563,1.0,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.00,0.7240239171358935,0.4068335357738006,13.565136550617618,0.1356513655061761,0.3866395067055498,0.0529567519858782,0.1209179170344219,0.17387466902030008,0.001578993772192076,0.7205996419729696,1.0,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.02,0.7263097057327799,0.4064914781094827,13.42987641622816,0.1342987641622816,0.3863697821025159,0.06001765225066196,6.236540158870256,6.296557811120918,0.0541899611084103,0.6145165811709306,1.0,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.04,0.7276128307708258,0.4054859896994975,13.295092218891954,0.1329509221889195,0.3851203729935697,0.05207413945278023,0.1297440423654016,0.18181818181818182,0.0016533037985858635,0.7240098989116803,1.0,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.06,0.7276865132383193,0.4014727027723293,13.10860799057166,0.1310860799057166,0.3804952786306688,0.05207413945278023,0.13415710503089143,0.18623124448367168,0.001691057431836761,0.7240010735018495,1.0,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.08,0.726393195584298,0.3987018836449559,12.850537785783194,0.1285053778578319,0.3788945955746495,0.05648720211827008,0.15357458075904679,0.21006178287731686,0.0018871365478087807,0.7222885419382362,1.0,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.10,0.7244012304511832,0.3932239948456176,12.361161644811926,0.1236116164481192,0.3733413807007665,0.05030891438658429,0.08561341571050309,0.13592233009708737,0.0012217374057913526,0.721749388705754,1.0,0