Generate charts for the presentation & diploma;some refactoring; add (commented) Student's t-test

change_visualizer.py

@@ -2,7 +2,7 @@ import gradio as gr
 
 import analysis_util
 import generate_annotated_diffs
-import statistics
+import dataset_statistics
 
 df_manual = generate_annotated_diffs.manual_data_with_annotated_diffs()
 df_manual["end_to_start"] = False
@@ -33,12 +33,14 @@ def synthetic():
     return df_synthetic[(df_synthetic['end_to_start'] == True) | (df_synthetic['start_to_end'] == True)]
 
 
-STATISTICS = {"manual": statistics.get_statistics_for_df(df_manual),
-              "e2s": statistics.get_statistics_for_df(e2s()),
-              "s2e": statistics.get_statistics_for_df(s2e()),
-              "e2s_s2e": statistics.get_statistics_for_df(e2s_s2e()),
-              "synthetic": statistics.get_statistics_for_df(synthetic()),
-              "all": statistics.get_statistics_for_df(df_synthetic)}
+STATISTICS = {"manual": dataset_statistics.get_statistics_for_df(df_manual),
+              "e2s": dataset_statistics.get_statistics_for_df(e2s()),
+              "s2e": dataset_statistics.get_statistics_for_df(s2e()),
+              "e2s_s2e": dataset_statistics.get_statistics_for_df(e2s_s2e()),
+              "synthetic": dataset_statistics.get_statistics_for_df(synthetic()),
+              "all": dataset_statistics.get_statistics_for_df(df_synthetic)}
+
+STATISTICS_T_TEST = dataset_statistics.t_test(STATISTICS, main_group='manual')
 
 STAT_NAMES = list(STATISTICS['manual'].keys())
 
@@ -135,6 +137,23 @@ if __name__ == '__main__':
                           value=stats['changes'].mean().item(), precision=3, min_width=00)
 
 
+            def layout_for_statistics_t_test(statistics_group_name):
+                gr.Markdown(f"### {statistics_group_name}")
+                stats = STATISTICS_T_TEST[statistics_group_name]
+                gr.Number(label="Deletions number (rel to the initial msg length)", interactive=False,
+                          value=stats['deletions_norm'], precision=3, min_width=00)
+                gr.Number(label="Insertions number (rel to the result length)", interactive=False,
+                          value=stats['insertions_norm'], precision=3, min_width=00)
+                gr.Number(label="Changes number (rel to the initial msg length)", interactive=False,
+                          value=stats['changes_norm'], precision=3, min_width=00)
+                gr.Number(label="Deletions number", interactive=False,
+                          value=stats['deletions'], precision=3, min_width=00)
+                gr.Number(label="Insertions number", interactive=False,
+                          value=stats['insertions'], precision=3, min_width=00)
+                gr.Number(label="Changes number", interactive=False,
+                          value=stats['changes'], precision=3, min_width=00)
+
+
             with gr.Row():
                 with gr.Column(scale=1, min_width=100):
                     layout_for_statistics("manual")
@@ -149,10 +168,25 @@ if __name__ == '__main__':
                 with gr.Column(scale=1, min_width=100):
                     layout_for_statistics("all")
 
+            # gr.Markdown(f"### Student t-test (p-value)")
+            # with gr.Row():
+            #     with gr.Column(scale=1, min_width=100):
+            #         layout_for_statistics_t_test("manual")
+            #     with gr.Column(scale=1, min_width=100):
+            #         layout_for_statistics_t_test("e2s")
+            #     with gr.Column(scale=1, min_width=100):
+            #         layout_for_statistics_t_test("s2e")
+            #     with gr.Column(scale=1, min_width=100):
+            #         layout_for_statistics_t_test("e2s_s2e")
+            #     with gr.Column(scale=1, min_width=100):
+            #         layout_for_statistics_t_test("synthetic")
+            #     with gr.Column(scale=1, min_width=100):
+            #         layout_for_statistics_t_test("all")
+
             with gr.Row():
                 with gr.Column(scale=1):
                     for stat_name in filter(lambda s: "_norm" not in s, STAT_NAMES):
-                        chart = statistics.build_plotly_chart(
+                        chart = dataset_statistics.build_plotly_chart(
                             stat_golden=STATISTICS['manual'][stat_name],
                             stat_e2s=STATISTICS['e2s'][stat_name],
                             stat_s2e=STATISTICS['s2e'][stat_name],
@@ -164,7 +198,7 @@ if __name__ == '__main__':
                 with gr.Column(scale=1):
                     with gr.Column(scale=1):
                         for stat_name in filter(lambda s: "_norm" in s, STAT_NAMES):
-                            chart = statistics.build_plotly_chart(
+                            chart = dataset_statistics.build_plotly_chart(
                                 stat_golden=STATISTICS['manual'][stat_name],
                                 stat_e2s=STATISTICS['e2s'][stat_name],
                                 stat_s2e=STATISTICS['s2e'][stat_name],

config.py

@@ -35,3 +35,6 @@ START_TO_END_ARTIFACT = OUTPUT_DIR / "start_to_end.csv"
 SYNTHETIC_DATASET_ARTIFACT = OUTPUT_DIR / "synthetic.csv"
 METRICS_CORRELATIONS_ARTIFACT = OUTPUT_DIR / "metrics_correlations.csv"
 DATA_FOR_LABELING_ARTIFACT = OUTPUT_DIR / "data_for_labeling.csv"
+
+OUTPUT_CHARTS_DIR = OUTPUT_DIR / "charts"
+OUTPUT_CHARTS_DIR.mkdir(exist_ok=True)

statistics.py → dataset_statistics.py

@@ -1,7 +1,12 @@
+import pickle
+
 import Levenshtein
 import numpy as np
 import pandas as pd
 import plotly.figure_factory as ff
+from scipy.stats import stats
+
+import config
 
 
 def get_statistics(start_msg, end_msg, annotated_msg):
@@ -38,11 +43,28 @@ def build_plotly_chart(stat_golden, stat_e2s, stat_s2e, stat_e2s_s2e, stat_name)
     hist_data = [stat_golden, stat_e2s, stat_s2e, stat_e2s_s2e,
                  np.concatenate((stat_e2s, stat_s2e, stat_e2s_s2e), axis=0)]
 
-    group_labels = ['Golden', 'e2s', 's2e', 'e2s+s 2e', 'Synthetic']
+    group_labels = ['Golden', 'e2s', 's2e', 'e2s+s2e', 'Synthetic']
 
     fig = ff.create_distplot(hist_data, group_labels,
-                             bin_size=.1, show_rug=False, show_hist=False)
+                             bin_size=.05, show_rug=False, show_hist=False)
 
     fig.update_layout(title_text=stat_name)
 
+    with open(config.OUTPUT_CHARTS_DIR / f"{stat_name}_data.pkl", "wb") as f:
+        pickle.dump(hist_data, f)
+
     return fig
+
+
+def t_test(group_stats, main_group="manual"):
+    results = {}
+    for group in group_stats:
+        results[group] = {}
+        for stat in group_stats[group]:
+            a = group_stats[main_group][stat]
+            b = group_stats[group][stat]
+
+            p = stats.ttest_ind(a, b, equal_var=False, random_state=config.RANDOM_STATE).pvalue
+            results[group][stat] = p
+
+    return results

generation_steps/synthetic_end_to_start.py

@@ -5,7 +5,7 @@ from tqdm import tqdm
 
 import config
 import generate_annotated_diffs
-import statistics
+import dataset_statistics
 from api_wrappers import grazie_wrapper, hf_data_loader
 from generation_steps import examples
 

generation_steps/synthetic_start_to_end.py

@@ -3,7 +3,7 @@ from tqdm import tqdm
 
 import config
 import generate_annotated_diffs
-import statistics
+import dataset_statistics
 from api_wrappers import grazie_wrapper
 from generation_steps import examples
 

requirements.txt

@@ -163,4 +163,5 @@ zipp==3.18.1
 
 plotly==5.22.0
 tenacity==8.2.3
-Levenshtein==0.25.1
+Levenshtein==0.25.1
+kaleido==0.2.1