new mistral results analyzed

llm_toolkit/translation_utils.py

@@ -216,12 +216,18 @@ def get_metrics(df, max_output_tokens=2048):
         repetition_score.append(df["repetition_score"].mean())
         total_repetitions.append(df["total_repetitions"].mean())
 
-        df["output_tokens"] = df[col].apply(
-            lambda x: len(tokenizers[col.split("/rpp")[0]](x)["input_ids"])
+        model = col.split("/rpp")[0]
+
+        new_col = f"ground_truth_tokens-{model}"
+        df[new_col] = df["english"].apply(
+            lambda x: len(tokenizers[model](x)["input_ids"])
         )
 
+        new_col = f"output_tokens-{model}"
+        df[new_col] = df[col].apply(lambda x: len(tokenizers[model](x)["input_ids"]))
+
         num_entries_with_max_output_tokens.append(
-            count_entries_with_max_tokens(df["output_tokens"], max_output_tokens)
+            count_entries_with_max_tokens(df[new_col], max_output_tokens)
         )
 
     metrics_df["meteor"] = meteor
@@ -230,9 +236,13 @@ def get_metrics(df, max_output_tokens=2048):
     metrics_df["ews_score"] = ews_score
     metrics_df["repetition_score"] = repetition_score
     metrics_df["total_repetitions"] = total_repetitions
-    metrics_df[
-        "num_entries_with_max_output_tokens"
-    ] = num_entries_with_max_output_tokens
+    metrics_df["num_entries_with_max_output_tokens"] = (
+        num_entries_with_max_output_tokens
+    )
+
+    metrics_df["rap"] = metrics_df.apply(
+        lambda x: x["meteor"] / math.log10(10 + x["total_repetitions"]), axis=1
+    )
 
     return metrics_df