Compute & compare metrics

generate_synthetic_dataset.py

@@ -1,26 +1,14 @@
 import config
 from api_wrappers import hf_data_loader
-from generation_steps import synthetic_end_to_start, examples, synthetic_start_to_end
+from generation_steps import synthetic_end_to_start, synthetic_start_to_end, metrics_analysis
 
 
 def run():
     df = hf_data_loader.load_processed_rewriting_dataset_as_pandas()
-    print(f"NUMBER OF EXAMPLES PER PROMPT = {examples.N_EXAMPLES}")
-    print()
 
-    print(f"End -> start synthesis:")
-    print(f"GENERATION_MULTIPLIER = {synthetic_end_to_start.GENERATION_MULTIPLIER}")
-    print(f"REL_INSERTIONS_THRESHOLD = {synthetic_end_to_start.REL_INSERTIONS_THRESHOLD}")
-    print(f"GENERATION_ATTEMPTS = {synthetic_end_to_start.GENERATION_ATTEMPTS}")
     df = synthetic_end_to_start.transform(df)
-    print("Done")
-
-    print(f"Start -> send synthesis:")
-    print(f"GENERATION_MULTIPLIER = {synthetic_start_to_end.GENERATION_MULTIPLIER}")
-    print(f"REL_DELETIONS_THRESHOLD = {synthetic_start_to_end.REL_DELETIONS_THRESHOLD}")
-    print(f"GENERATION_ATTEMPTS = {synthetic_start_to_end.GENERATION_ATTEMPTS}")
     df = synthetic_start_to_end.transform(df)
-    print("Done")
+    df = metrics_analysis.transform(df)
 
     df.to_csv(config.SYNTHETIC_DATASET_ARTIFACT)
 

generation_steps/metrics_analysis.py

@@ -0,0 +1,102 @@
+import evaluate
+import pandas as pd
+from tqdm import tqdm
+
+import config
+from api_wrappers import hf_data_loader
+
+BLEU = evaluate.load('bleu', cache_dir=config.CACHE_DIR)
+
+
+def bleu_fn(pred, ref):
+    return BLEU.compute(predictions=[pred], references=[ref])["bleu"]
+
+
+METEOR = evaluate.load('meteor', cache_dir=config.CACHE_DIR)
+
+
+def meteor_fn(pred, ref):
+    return METEOR.compute(predictions=[pred], references=[ref])["meteor"]
+
+
+ROUGE = evaluate.load('rouge', cache_dir=config.CACHE_DIR)
+
+
+def rouge1_fn(pred, ref):
+    return ROUGE.compute(predictions=[pred], references=[ref])["rouge1"]
+
+
+def rouge2_fn(pred, ref):
+    return ROUGE.compute(predictions=[pred], references=[ref])["rouge2"]
+
+
+BERTSCORE = evaluate.load('bertscore', cache_dir=config.CACHE_DIR)
+
+
+def bertscore_fn(pred, ref):
+    return BERTSCORE.compute(predictions=[pred], references=[ref], model_type="distilbert-base-uncased")["f1"][0]
+
+
+METRICS = {
+    "bleu": bleu_fn,
+    "meteor": meteor_fn,
+    "rouge1": rouge1_fn,
+    "rouge2": rouge2_fn,
+    "bertscore": bertscore_fn
+}
+
+
+def attach_references(df):
+    reference_df = hf_data_loader.load_full_commit_dataset_as_pandas().set_index(["hash", "repo"])[["reference"]]
+    df = df.set_index(["hash", "repo"])
+    return df.join(other=reference_df, how="left").reset_index()
+
+
+def compute_metrics(df):
+    tqdm.pandas()
+
+    def apply_metric_fn_to_row(row, fn, col_pred, col_ref):
+        return fn(row[col_pred], row[col_ref])
+
+    for metric in METRICS:
+        print(f"Computing {metric}")
+        metric_fn = METRICS[metric]
+        df[f"{metric}_related"] = df.progress_apply(
+            lambda row: apply_metric_fn_to_row(row=row,
+                                               fn=metric_fn,
+                                               col_pred="commit_msg_start",
+                                               col_ref="commit_msg_end"),
+            axis=1
+        )
+        df[f"{metric}_independent"] = df.progress_apply(
+            lambda row: apply_metric_fn_to_row(row=row,
+                                               fn=metric_fn,
+                                               col_pred="commit_msg_start",
+                                               col_ref="reference"),
+            axis=1
+        )
+
+        df[f"{metric}_pearson"] = df[f"{metric}_related"].corr(df[f"{metric}_independent"], method="pearson")
+        df[f"{metric}_spearman"] = df[f"{metric}_related"].corr(df[f"{metric}_independent"], method="spearman")
+
+    return df
+
+
+def transform(df):
+    print("Computing metrics")
+
+    df = attach_references(df)
+    df = compute_metrics(df)
+
+    print("Done")
+    return df
+
+
+def main():
+    df = pd.read_csv(config.SYNTHETIC_DATASET_ARTIFACT, index_col=[0])
+    df = transform(df)
+    df.to_csv(config.SYNTHETIC_DATASET_ARTIFACT)
+
+
+if __name__ == '__main__':
+    main()

generation_steps/synthetic_end_to_start.py

@@ -1,12 +1,13 @@
 import pandas as pd
 from tqdm import tqdm
 
+import config
 import generate_annotated_diffs
 import statistics
 from api_wrappers import grazie_wrapper
 from generation_steps import examples
 
-GENERATION_MULTIPLIER = 1
+GENERATION_MULTIPLIER = 3
 REL_INSERTIONS_THRESHOLD = 0.5
 GENERATION_ATTEMPTS = 5
 
@@ -62,6 +63,12 @@ COLS_TO_KEEP = ["hash", "repo", "commit_msg_end", "mods", "session"]
 
 
 def transform(df):
+    print(f"End -> start synthesis:")
+    print(f"NUMBER OF EXAMPLES PER PROMPT = {examples.N_EXAMPLES}")
+    print(f"GENERATION_MULTIPLIER = {GENERATION_MULTIPLIER}")
+    print(f"REL_INSERTIONS_THRESHOLD = {REL_INSERTIONS_THRESHOLD}")
+    print(f"GENERATION_ATTEMPTS = {GENERATION_ATTEMPTS}")
+
     df['end_to_start'] = False
 
     generated_data = {
@@ -83,4 +90,17 @@ def transform(df):
     generated_df = pd.DataFrame.from_dict(generated_data)
     generated_df['end_to_start'] = True
 
-    return pd.concat([df, generated_df], ignore_index=True)
+    result = pd.concat([df, generated_df], ignore_index=True)
+
+    print("Done")
+    return result
+
+
+def main():
+    df = pd.read_csv(config.SYNTHETIC_DATASET_ARTIFACT, index_col=[0])
+    df = transform(df)
+    df.to_csv(config.SYNTHETIC_DATASET_ARTIFACT)
+
+
+if __name__ == '__main__':
+    main()

generation_steps/synthetic_start_to_end.py

@@ -1,12 +1,13 @@
 import pandas as pd
 from tqdm import tqdm
 
+import config
 import generate_annotated_diffs
 import statistics
 from api_wrappers import grazie_wrapper
 from generation_steps import examples
 
-GENERATION_MULTIPLIER = 1
+GENERATION_MULTIPLIER = 3
 REL_DELETIONS_THRESHOLD = 0.75
 GENERATION_ATTEMPTS = 5
 
@@ -62,6 +63,12 @@ COLS_TO_KEEP = ["hash", "repo", "commit_msg_start", "mods", "session", "end_to_s
 
 
 def transform(df):
+    print(f"Start -> send synthesis:")
+    print(f"NUMBER OF EXAMPLES PER PROMPT = {examples.N_EXAMPLES}")
+    print(f"GENERATION_MULTIPLIER = {GENERATION_MULTIPLIER}")
+    print(f"REL_DELETIONS_THRESHOLD = {REL_DELETIONS_THRESHOLD}")
+    print(f"GENERATION_ATTEMPTS = {GENERATION_ATTEMPTS}")
+
     df['start_to_end'] = False
 
     generated_data = {
@@ -83,4 +90,17 @@ def transform(df):
     generated_df = pd.DataFrame.from_dict(generated_data)
     generated_df['start_to_end'] = True
 
-    return pd.concat([df, generated_df], ignore_index=True)
+    result = pd.concat([df, generated_df], ignore_index=True)
+
+    print("Done")
+    return result
+
+
+def main():
+    df = pd.read_csv(config.SYNTHETIC_DATASET_ARTIFACT, index_col=[0])
+    df = transform(df)
+    df.to_csv(config.SYNTHETIC_DATASET_ARTIFACT)
+
+
+if __name__ == '__main__':
+    main()