Update the parameters

generation_steps/examples.py

@@ -1,7 +1,7 @@
 import config
 from api_wrappers import hf_data_loader
 
-N_EXAMPLES = 5
+N_EXAMPLES = 15
 
 
 def get_example_prompt_end_to_start(start_msg, end_msg):

generation_steps/metrics_analysis.py

@@ -34,6 +34,10 @@ def rouge2_fn(pred, ref):
     return ROUGE.compute(predictions=[pred], references=[ref])["rouge2"]
 
 
+def rougeL_fn(pred, ref):
+    return ROUGE.compute(predictions=[pred], references=[ref])["rougeL"]
+
+
 BERTSCORE = evaluate.load('bertscore', cache_dir=config.CACHE_DIR)
 
 
@@ -45,13 +49,30 @@ def gptscore_fn(pred, ref):
     return gpt_eval.compute(prediction=pred, reference=ref)
 
 
+CHRF = evaluate.load("chrf")
+
+
+def chrf_fn(pred, ref):
+    return CHRF.compute(predictions=[pred], references=[[ref]])["score"]
+
+
+TER = evaluate.load("ter")
+
+
+def ter_fn(pred, ref):
+    return TER.compute(predictions=[pred], references=[[ref]])["score"]
+
+
 METRICS = {
-    "gptscore": gptscore_fn,
+    # "gptscore": gptscore_fn,
     "bleu": bleu_fn,
     "meteor": meteor_fn,
     "rouge1": rouge1_fn,
     "rouge2": rouge2_fn,
+    "rougeL": rougeL_fn,
     "bertscore": bertscore_fn,
+    "chrF": chrf_fn,
+    "ter": ter_fn
 }
 
 
@@ -98,6 +119,13 @@ def correlations_for_group(group):
             f"{metric}_pearson": group[f"{metric}_related"].corr(group[f"{metric}_independent"], method="pearson"),
             f"{metric}_spearman": group[f"{metric}_related"].corr(group[f"{metric}_independent"], method="spearman")
         })
+        for other_metric in METRICS:
+            correlations.append({
+                f"ind_{metric}_rel_{other_metric}_pearson": group[f"{other_metric}_related"].corr(
+                    group[f"{metric}_independent"], method="pearson"),
+                f"ind_{metric}_rel_{other_metric}_spearman": group[f"{other_metric}_related"].corr(
+                    group[f"{metric}_independent"], method="spearman")
+            })
     return pd.Series(functools.reduce(operator.ior, correlations, {}))
 
 

generation_steps/synthetic_end_to_start.py

@@ -7,7 +7,7 @@ import statistics
 from api_wrappers import grazie_wrapper, hf_data_loader
 from generation_steps import examples
 
-GENERATION_MULTIPLIER = 3
+GENERATION_MULTIPLIER = 2
 REL_INSERTIONS_THRESHOLD = 0.5
 GENERATION_ATTEMPTS = 5
 

generation_steps/synthetic_start_to_end.py

@@ -7,7 +7,7 @@ import statistics
 from api_wrappers import grazie_wrapper
 from generation_steps import examples
 
-GENERATION_MULTIPLIER = 3
+GENERATION_MULTIPLIER = 2
 REL_DELETIONS_THRESHOLD = 0.75
 GENERATION_ATTEMPTS = 5