add new files

README.md

@@ -6,52 +6,13 @@ sdk_version: 4.37.2
 app_file: change_visualizer.py
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Commit Message Editing Visualisation ✍️🔍📊
 
-# Description
+This space provides a visualization app for exploring the commit message edits datasets (🤗[expert-labeled](https://huggingface.co/datasets/JetBrains-Research/commit-msg-edits) and 🤗[synthetic](https://huggingface.co/datasets/JetBrains-Research/synthetic-commit-msg-edits))
+from "Towards Realistic Evaluation of Commit Message Generation by Matching Online and Offline Settings" paper as well as some important artifacts from our work.
 
-This project is a main artifact of the "Research on evaluation for AI Commit Message Generation" research.
+## Artifacts
 
-# Structure (important components)
-
-- ### Configuration: [config.py](config.py)
-    - Grazie API JWT token and Hugging Face token must be stored as environment variables.
-- ### Visualization app -- a Gradio application that is currently deployed
-  at https://huggingface.co/spaces/JetBrains-Research/commit-rewriting-visualization.
-    - Shows
-        - The "golden" dataset of manually collected samples; the dataset is downloaded on startup
-          from https://huggingface.co/datasets/JetBrains-Research/commit-msg-rewriting
-        - The entire dataset that includes the synthetic samples; the dataset is downloaded on startup
-          from https://huggingface.co/datasets/JetBrains-Research/synthetic-commit-msg-rewriting
-        - Some statistics collected for the dataset (and its parts); computed on startup
-
-      _Note: datasets updated => need to restart the app to see the changes._
-    - Files
-        - [change_visualizer.py](change_visualizer.py)
-- ### Data processing pipeline (_note: datasets and files names can be changed in the configuration file_)
-    - Run the whole pipeline by running [run_pipeline.py](run_pipeline.py)
-        - All intermediate results are stored as files defined in config
-    - Intermediate steps (can run them separately by running the corresponding files
-      from [generation_steps](generation_steps)). The input is then taken from the previous step's artifact.
-    - Generate the synthetic samples
-        - Files [generation_steps/synthetic_end_to_start.py](generation_steps/synthetic_end_to_start.py)
-          and [generation_steps/synthetic_start_to_end.py](generation_steps/synthetic_start_to_end.py)
-        - The first generation step (end to start) downloads the `JetBrains-Research/commit-msg-rewriting`
-          and `JetBrains-Research/lca-commit-message-generation` datasets from
-          Hugging Face datasets.
-    - Compute metrics
-        - File [generation_steps/metrics_analysis.py](generation_steps/metrics_analysis.py)
-        - Includes the functions for all metrics
-        - Downloads `JetBrains-Research/lca-commit-message-generation` Hugging Face dataset.
-    - The resulting artifact (dataset with golden and synthetic samples, attached reference messages and computed
-      metrics) is saved to the file [output/synthetic.csv](output/synthetic.csv). It should be uploaded
-      to https://huggingface.co/datasets/JetBrains-Research/synthetic-commit-msg-rewriting **manually**.
-- ### Data analysis
-    - [analysis_util.py](analysis_util.py) -- some functions used for data analysis, e.g., correlations computation.
-    - [analysis.ipynb](analysis.ipynb) -- compute the correlations, the resulting tables.
-    - [chart_processing.ipynb](chart_processing.ipynb) -- Jupyter Notebook that draws the charts that were used in the
-      presentation/thesis.
-    - [generated_message_length_comparison.ipynb](generated_message_length_comparison.ipynb) -- compare the average 
-      length of commit messages generated using the current prompt (one used in the research) and the production prompt 
-      (one used to generate the messages that are measured in FUS logs). _Not finished, because could not get a Grazie 
-      token; as soon as the token is received, the notebook can be run by following the instructions from the notebook._
+* [`metrics_analysis.ipynb`](metrics_analysis.ipynb) contains the code for metrics calculation and analysis;
+* [`chart.ipynb`](chart.ipynb) contains the code for Figure 4 with edit distance distribution;
+* [`data_stats.ipynb`](data_stats.ipynb) contains the code for obtaining the dataset statistics from Table 1.

api_wrappers/hf_data_loader.py

@@ -1,7 +1,9 @@
 import json
+import os
 from datetime import datetime, timedelta
 
 from datasets import load_dataset
+from huggingface_hub import hf_hub_download, list_repo_tree
 
 import config
 
@@ -72,6 +74,15 @@ def load_synthetic_as_pandas():
 
 def load_full_commit_with_predictions_as_pandas():
     full_dataset = load_full_commit_as_pandas()
+
+    # TODO
+    # for prediction_file in list_repo_tree(repo_id=config.HF_PREDICTIONS_DATASET_NAME,
+    #                                       path=os.path.join("commit_message_generation/predictions", config.HF_PREDICTIONS_MODEL),
+    #                                       repo_type="dataset"):
+    #     hf_hub_download(prediction_file.path,
+    #                     repo_id=config.HF_PREDICTIONS_DATASET_NAME,
+    #                     repo_type="dataset",)
+
     predictions_dataset = load_dataset(config.HF_PREDICTIONS_DATASET_NAME,
                                        config.HF_PREDICTIONS_DATASET_SUBNAME,
                                        split=config.HF_PREDICTIONS_DATASET_SPLIT,

change_visualizer.py

@@ -86,10 +86,10 @@ if __name__ == '__main__':
             end_view = gr.Textbox(interactive=False, label="End message", container=True)
             session_view = gr.Textbox(interactive=False, label="Session", container=True)
             is_end_to_start_view = gr.Textbox(interactive=False,
-                                              label="Is generated on the 'end-to-start' synthesis step?",
+                                              label="Is generated via backward synthetic generation?",
                                               container=True)
             is_start_to_end_view = gr.Textbox(interactive=False,
-                                              label="Is generated on the 'start-to-end' synthesis step?",
+                                              label="Is generated via forward synthetic generation?",
                                               container=True)
             link_view = gr.Markdown()
 
@@ -109,13 +109,15 @@ if __name__ == '__main__':
         with gr.Tab("Manual"):
             slider_manual, view_manual = dataset_view_tab(n_diffs_manual)
 
-            slider_manual.change(update_dataset_view_manual, inputs=slider_manual,
+            slider_manual.change(update_dataset_view_manual,
+                                 inputs=slider_manual,
                                  outputs=view_manual)
 
         with gr.Tab("Synthetic"):
             slider_synthetic, view_synthetic = dataset_view_tab(n_diffs_synthetic)
 
-            slider_synthetic.change(update_dataset_view_synthetic, inputs=slider_synthetic,
+            slider_synthetic.change(update_dataset_view_synthetic,
+                                    inputs=slider_synthetic,
                                     outputs=view_synthetic)
         with gr.Tab("Analysis"):
             def layout_for_statistics(statistics_group_name):
@@ -212,10 +214,7 @@ if __name__ == '__main__':
 
                             gr.Plot(value=chart)
 
-            gr.Markdown(f"### Reference-only correlations")
-            gr.Markdown(value=analysis_util.get_correlations_for_groups(df_synthetic, right_side="ind").to_markdown())
-
-            gr.Markdown(f"### Aggregated correlations")
+            gr.Markdown(f"### Metrics correlations")
             gr.Markdown(value=analysis_util.get_correlations_for_groups(df_synthetic, right_side="aggr").to_markdown())
 
         application.load(update_dataset_view_manual, inputs=slider_manual,

config.py

@@ -3,7 +3,7 @@ from pathlib import Path
 
 RANDOM_STATE = 42
 
-GRAZIE_API_JWT_TOKEN = os.environ.get("GRAZIE_API_JWT_TOKEN")
+GRAZIE_API_JWT_TOKEN = "eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJHcmF6aWUgQXV0aGVudGljYXRpb24iLCJ1aWQiOiJkNmFjZGM3Zi1jZWZlLTRhMDItOWRmMi01NzY5OGRlNjYyNDAiLCJ1c2VyX3N0YXRlIjoiSU5URVJOQUwiLCJyZWdpc3RyYXRpb25fZGF0ZSI6MTY4NDMzNjI3ODI2NCwibGljZW5zZSI6IjQ1TVcwNFZBVVoiLCJsaWNlbnNlX3R5cGUiOiJqZXRicmFpbnMtYWkub3JnYW5pemF0aW9uYWwucHJvIiwiZXhwIjoxNzIwNjk0OTQ2fQ.NH5KLYgkyaC1MfFHPj8jfe3yBBR8F017QV_Nn0_5AqiWqjaaVBIBCsxkZcTbwH6FBrGm-JXYM50UAhJprI3fy-HNkwfF6nAPRqkFafxT8IZ-Epk8P9u6SnC5YjD4LM4e_-aKeuXb4WdB6K_YDIRKIp64WthCS2OzLSDPiyXaHXADOBQMfWNvorXqjuKPUPE7q6L59Wes4VaDhXMPw2XA4MHUm_cTvK2a_SixaKiawxAv-Wa8vo2KcYbd4hqtxDwnoQ6c5WfmEqD-dUYvZ8G_53WNJO6gvIv0etEBx8NIez2dPXHyNqIyam4CrMXH9_stJwf998sL7NxdG2wRLGGC4A"
 GRAZIE_TIMEOUT_SEC = 1.0
 
 HF_TOKEN = os.environ.get('HF_TOKEN')
@@ -16,8 +16,7 @@ HF_FULL_COMMITS_DATASET_SUBNAME = "commitchronicle-py-long"
 HF_FULL_COMMITS_DATASET_SPLIT = "test"
 
 HF_PREDICTIONS_DATASET_NAME = "JetBrains-Research/lca-results"
-HF_PREDICTIONS_DATASET_SUBNAME = "cmg_gpt_4_0613"
-HF_PREDICTIONS_DATASET_SPLIT = "test"
+HF_PREDICTIONS_MODEL = "gpt_4_0613"
 
 HF_SYNTHETIC_DATASET_NAME = "JetBrains-Research/synthetic-commit-msg-rewriting"
 HF_SYNTHETIC_DATASET_SPLIT = 'train'

custom_metrics/gpt_eval.py

@@ -1,81 +0,0 @@
-from api_wrappers import grazie_wrapper
-
-
-def build_prompt_ref(prediction, reference):
-    return f"""Evaluate the following commit message based on clarity, specificity, context, and conciseness without 
-providing any additional feedback or commentary:
-
-START OF THE COMMIT MESSAGE YOU HAVE TO EVALUATE
-{prediction}
-END OF THE COMMIT MESSAGE YOU HAVE TO EVALUATE
-
-For reference, consider this as an example of a good commit message for the same commit that is both concise and 
-specific: 
-START OF THE REFERENCE COMMIT MESSAGE 
-{reference}
-END OF THE REFERENCE COMMIT MESSAGE
-
-YOUR TASK: Provide a single number as a response, representing the rating on a scale from 1 to 10, where 1 is the 
-lowest quality and 10 is the highest quality. Do not include any other text or explanation in your response.
-"""
-
-
-def build_prompt_noref(prediction, diff):
-    return f"""Evaluate the following commit message based on clarity, specificity, context, and conciseness without 
-providing any additional feedback or commentary:
-
-START OF THE COMMIT MESSAGE YOU HAVE TO EVALUATE
-{prediction}
-END OF THE COMMIT MESSAGE YOU HAVE TO EVALUATE
-
-These are the code changes included in the commit: 
-START OF THE CODE CHANGES
-{diff}
-END OF THE CODE CHANGES
-
-YOUR TASK: Provide a single number as a response, representing the rating on a scale from 1 to 10, where 1 is the 
-lowest quality and 10 is the highest quality. Do not include any other text or explanation in your response.
-"""
-
-
-N_RETRIES = 3
-
-
-def get_number_for_prompt(prompt):
-    outputs = []
-    result = None
-
-    for i in range(N_RETRIES):
-        try:
-            output = grazie_wrapper.generate_for_prompt(prompt).strip().split()[-1]
-            outputs.append(output)
-
-            result = int(output)
-            break
-        except ValueError:
-            continue
-
-    if result is None:
-        raise RuntimeError(f"LLM cannot generate a number. Its outputs were: {str(outputs)}")
-
-    return result
-
-
-def compute_ref(prediction, reference, n_requests):
-    prompt = build_prompt_ref(prediction, reference)
-    results = [
-        get_number_for_prompt(prompt)
-        for _ in range(n_requests)
-    ]
-
-    return sum(results) / len(results)
-
-
-def compute_noref(prediction, diff, n_requests):
-    prompt = build_prompt_noref(prediction, diff)
-    results = [
-        get_number_for_prompt(prompt)
-        for _ in range(n_requests)
-    ]
-
-    return sum(results) / len(results)

data_stats.ipynb

@@ -0,0 +1,759 @@
+{
+ "cells": [
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "# Data Stats",
+   "id": "694a6cc631d4ab93"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:43:07.644299Z",
+     "start_time": "2024-10-15T18:43:02.316453Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "\n",
+    "df = load_dataset(\"JetBrains-Research/synthetic-commit-msg-edits\", \"all_pairs\", split=\"train\").to_pandas()\n",
+    "df.head()"
+   ],
+   "id": "ed42f4f83199feb2",
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Downloading data: 100%|██████████| 6.35M/6.35M [00:00<00:00, 9.95MB/s]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Generating train split: 0 examples [00:00, ? examples/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "1a0523289d424b29974b60d017643280"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "                                       hash              repo  \\\n",
+       "0  2febb99eee8ed71c9122db88ca58dd33be0b9550  mesonbuild/meson   \n",
+       "1  2febb99eee8ed71c9122db88ca58dd33be0b9550  mesonbuild/meson   \n",
+       "2  2febb99eee8ed71c9122db88ca58dd33be0b9550  mesonbuild/meson   \n",
+       "3  2febb99eee8ed71c9122db88ca58dd33be0b9550  mesonbuild/meson   \n",
+       "4  2febb99eee8ed71c9122db88ca58dd33be0b9550  mesonbuild/meson   \n",
+       "\n",
+       "                                              G_text  \\\n",
+       "0  Enhance OptionOverrideProxy and simplify optio...   \n",
+       "1  Enhance OptionOverrideProxy and simplify optio...   \n",
+       "2  Enhance OptionOverrideProxy and simplify optio...   \n",
+       "3  Enhance OptionOverrideProxy and simplify optio...   \n",
+       "4  Enhance OptionOverrideProxy and simplify optio...   \n",
+       "\n",
+       "                                              E_text              G_type  \\\n",
+       "0  Enhance OptionOverrideProxy for multiple optio...  synthetic_backward   \n",
+       "1  Refactor OptionOverrideProxy and Backend class...  synthetic_backward   \n",
+       "2  Refactor OptionOverrideProxy and backend optio...  synthetic_backward   \n",
+       "3  Refactor: Enhance OptionOverrideProxy for mult...  synthetic_backward   \n",
+       "4  Refactor OptionOverrideProxy and add target-sp...  synthetic_backward   \n",
+       "\n",
+       "                            E_type  is_related  \n",
+       "0                   expert_labeled        True  \n",
+       "1                synthetic_forward        True  \n",
+       "2                synthetic_forward        True  \n",
+       "3                synthetic_forward        True  \n",
+       "4  synthetic_forward_from_backward       False  "
+      ],
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>hash</th>\n",
+       "      <th>repo</th>\n",
+       "      <th>G_text</th>\n",
+       "      <th>E_text</th>\n",
+       "      <th>G_type</th>\n",
+       "      <th>E_type</th>\n",
+       "      <th>is_related</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2febb99eee8ed71c9122db88ca58dd33be0b9550</td>\n",
+       "      <td>mesonbuild/meson</td>\n",
+       "      <td>Enhance OptionOverrideProxy and simplify optio...</td>\n",
+       "      <td>Enhance OptionOverrideProxy for multiple optio...</td>\n",
+       "      <td>synthetic_backward</td>\n",
+       "      <td>expert_labeled</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2febb99eee8ed71c9122db88ca58dd33be0b9550</td>\n",
+       "      <td>mesonbuild/meson</td>\n",
+       "      <td>Enhance OptionOverrideProxy and simplify optio...</td>\n",
+       "      <td>Refactor OptionOverrideProxy and Backend class...</td>\n",
+       "      <td>synthetic_backward</td>\n",
+       "      <td>synthetic_forward</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2febb99eee8ed71c9122db88ca58dd33be0b9550</td>\n",
+       "      <td>mesonbuild/meson</td>\n",
+       "      <td>Enhance OptionOverrideProxy and simplify optio...</td>\n",
+       "      <td>Refactor OptionOverrideProxy and backend optio...</td>\n",
+       "      <td>synthetic_backward</td>\n",
+       "      <td>synthetic_forward</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2febb99eee8ed71c9122db88ca58dd33be0b9550</td>\n",
+       "      <td>mesonbuild/meson</td>\n",
+       "      <td>Enhance OptionOverrideProxy and simplify optio...</td>\n",
+       "      <td>Refactor: Enhance OptionOverrideProxy for mult...</td>\n",
+       "      <td>synthetic_backward</td>\n",
+       "      <td>synthetic_forward</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2febb99eee8ed71c9122db88ca58dd33be0b9550</td>\n",
+       "      <td>mesonbuild/meson</td>\n",
+       "      <td>Enhance OptionOverrideProxy and simplify optio...</td>\n",
+       "      <td>Refactor OptionOverrideProxy and add target-sp...</td>\n",
+       "      <td>synthetic_backward</td>\n",
+       "      <td>synthetic_forward_from_backward</td>\n",
+       "      <td>False</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 3
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "## Full",
+   "id": "922e7a73f11a4aec"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:43:14.266540Z",
+     "start_time": "2024-10-15T18:43:14.262103Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(df.loc[df.is_related])",
+   "id": "562d9c53da109d1a",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "656"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 4
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:43:18.073966Z",
+     "start_time": "2024-10-15T18:43:18.069219Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df.loc[df.is_related].groupby([\"hash\", \"repo\"]).G_text.count().mean()",
+   "id": "b4f3c96a4b676a0d",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "43.733333333333334"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 5
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:43:19.026689Z",
+     "start_time": "2024-10-15T18:43:19.021680Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(df.loc[~df.is_related])",
+   "id": "54d9f32f1d18844f",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "5140"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 6
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:43:19.484304Z",
+     "start_time": "2024-10-15T18:43:19.480012Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df.loc[~df.is_related].groupby([\"hash\", \"repo\"]).G_text.count().mean()",
+   "id": "679761631517b9e4",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "342.6666666666667"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 7
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "## Expert-labeled",
+   "id": "84561ea89717d61a"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:45:52.905631Z",
+     "start_time": "2024-10-15T18:45:52.901913Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_ = df.loc[(df.G_type == \"initial\") & (df.E_type == \"expert_labeled\")]",
+   "id": "be1c800f45cef26e",
+   "outputs": [],
+   "execution_count": 36
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:45:53.234109Z",
+     "start_time": "2024-10-15T18:45:53.230986Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(_.loc[_.is_related])",
+   "id": "1d092dff4d39bcd1",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "57"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 37
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:45:53.629311Z",
+     "start_time": "2024-10-15T18:45:53.625620Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_.loc[_.is_related].groupby([\"hash\", \"repo\"]).G_text.count().mean()",
+   "id": "a06a532cd5a29725",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3.8"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 38
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:45:53.956790Z",
+     "start_time": "2024-10-15T18:45:53.953842Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(_.loc[~_.is_related])",
+   "id": "5e19c8a6309b62aa",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 39,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 39
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:46:02.554527Z",
+     "start_time": "2024-10-15T18:46:02.551084Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_.loc[~_.is_related].groupby([\"hash\", \"repo\"]).G_text.count().mean()",
+   "id": "e43179c5dcab5eb2",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "nan"
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 40
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "## Backward",
+   "id": "70ee052fae2f88e3"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:44:33.559606Z",
+     "start_time": "2024-10-15T18:44:33.556802Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_ = df.loc[(df.G_type == \"synthetic_backward\") & (~df.E_type.isin([\"synthetic_forward\", \"synthetic_forward_from_backward\"]))]",
+   "id": "99f51ecc55c4db35",
+   "outputs": [],
+   "execution_count": 20
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:44:33.958325Z",
+     "start_time": "2024-10-15T18:44:33.955847Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(_.loc[_.is_related])",
+   "id": "6ff29390c8e127c2",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "104"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 21
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:44:34.455560Z",
+     "start_time": "2024-10-15T18:44:34.452303Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_.loc[_.is_related].groupby([\"hash\", \"repo\"]).G_text.count().mean()",
+   "id": "e1ae04e1ecfb2040",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "7.428571428571429"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 22
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:44:34.903849Z",
+     "start_time": "2024-10-15T18:44:34.901226Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(_.loc[~_.is_related])",
+   "id": "125c4c335e7761da",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1048"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 23
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:44:35.783538Z",
+     "start_time": "2024-10-15T18:44:35.778676Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_.loc[~_.is_related].groupby([\"hash\", \"repo\"]).G_text.count().mean()",
+   "id": "4782f1d6e6863f89",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "74.85714285714286"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 24
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "## Forward",
+   "id": "bf61a4b422f779fa"
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "### From human",
+   "id": "1429f9f99acf75d"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:46:21.359807Z",
+     "start_time": "2024-10-15T18:46:21.356451Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_ = df.loc[(df.G_type == \"initial\") & (df.E_type == \"synthetic_forward\")]",
+   "id": "e13d55b0124f04b3",
+   "outputs": [],
+   "execution_count": 41
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:46:21.798508Z",
+     "start_time": "2024-10-15T18:46:21.795885Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(_.loc[_.is_related])",
+   "id": "b8353390df7da427",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "177"
+      ]
+     },
+     "execution_count": 42,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 42
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:46:22.163595Z",
+     "start_time": "2024-10-15T18:46:22.160176Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_.loc[_.is_related].groupby([\"hash\", \"repo\"]).G_text.count().mean()",
+   "id": "ac89afde65efd73d",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "11.8"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 43
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:46:22.552314Z",
+     "start_time": "2024-10-15T18:46:22.549570Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(_.loc[~_.is_related])",
+   "id": "9b6cb335e3bbb7ff",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 44,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 44
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:46:23.237736Z",
+     "start_time": "2024-10-15T18:46:23.234085Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "__.loc[~__.is_related].groupby([\"hash\", \"repo\"]).G_text.count().mean()",
+   "id": "fe22189a70fc4149",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "nan"
+      ]
+     },
+     "execution_count": 45,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 45
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "### From backward",
+   "id": "ace7afb876fb88a0"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:47:06.641374Z",
+     "start_time": "2024-10-15T18:47:06.637018Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_ = df.loc[(df.G_type == \"synthetic_backward\") & (df.E_type.isin([\"synthetic_forward\", \"synthetic_forward_from_backward\"]))]",
+   "id": "88800960dbff619a",
+   "outputs": [],
+   "execution_count": 53
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:47:15.358650Z",
+     "start_time": "2024-10-15T18:47:15.355108Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(_.loc[_.is_related])",
+   "id": "890613156e005c83",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "318"
+      ]
+     },
+     "execution_count": 56,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 56
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:47:15.579415Z",
+     "start_time": "2024-10-15T18:47:15.576016Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_.loc[_.is_related].groupby([\"hash\", \"repo\"]).G_text.count().mean()",
+   "id": "999f91382a2c8ff6",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "22.714285714285715"
+      ]
+     },
+     "execution_count": 57,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 57
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:47:15.834218Z",
+     "start_time": "2024-10-15T18:47:15.831258Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(_.loc[~_.is_related])",
+   "id": "d347941cbb4b2db1",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3753"
+      ]
+     },
+     "execution_count": 58,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 58
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-10-15T18:47:16.138798Z",
+     "start_time": "2024-10-15T18:47:16.133397Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "_.loc[~_.is_related].groupby([\"hash\", \"repo\"]).G_text.count().mean()",
+   "id": "2db4d96713a8634d",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "268.07142857142856"
+      ]
+     },
+     "execution_count": 59,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 59
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

generation_steps/metrics_analysis.py

@@ -1,20 +1,15 @@
-import Levenshtein
 import evaluate
-import pandas as pd
-from tqdm import tqdm
-
 import config
-from analysis_util import correlations_for_group
-from api_wrappers import hf_data_loader
-from custom_metrics import gpt_eval
+from rapidfuzz.distance.Levenshtein import distance, normalized_similarity
+
 
-BLEU = evaluate.load('bleu', cache_dir=config.CACHE_DIR)
+BLEU = evaluate.load('saridormi/b_norm', cache_dir=config.CACHE_DIR)
 
 
 def bleu_fn(pred, ref, **kwargs):
     if "refs" in kwargs:
-        return BLEU.compute(predictions=[pred] * len(kwargs["refs"]), references=kwargs["refs"])["bleu"]
-    return BLEU.compute(predictions=[pred], references=[ref])["bleu"]
+        return BLEU.compute(predictions=[pred] * len(kwargs["refs"]), references=kwargs["refs"])["b_norm"]
+    return BLEU.compute(predictions=[pred], references=[ref])["b_norm"]
 
 
 METEOR = evaluate.load('meteor', cache_dir=config.CACHE_DIR)
@@ -67,76 +62,23 @@ def chrf_fn(pred, ref, **kwargs):
     return CHRF.compute(predictions=[pred], references=[[ref]])["score"]
 
 
-TER = evaluate.load("ter")
-
-
-def ter_fn(pred, ref, **kwargs):
-    if "refs" in kwargs:
-        scores = [TER.compute(predictions=[pred], references=[[ref]])["score"] for ref in kwargs["refs"]]
-        return sum(scores) / len(scores)
-    return TER.compute(predictions=[pred], references=[[ref]])["score"]
-
-
 def edit_distance_fn(pred, ref, **kwargs):
     if "refs" in kwargs:
-        scores = [Levenshtein.distance(pred, ref) for ref in kwargs["refs"]]
+        scores = [distance(pred, ref) for ref in kwargs["refs"]]
         return sum(scores) / len(scores)
-    return Levenshtein.distance(pred, ref)
+    return distance(pred, ref)
 
 
 def edit_distance_norm_fn(pred, ref, **kwargs):
     if "refs" in kwargs:
-        scores = [Levenshtein.distance(pred, ref) / len(pred) for ref in kwargs["refs"]]
-        return sum(scores) / len(scores)
-    return Levenshtein.distance(pred, ref) / len(pred)
-
-
-def edit_time_fn(pred, ref, **kwargs):
-    return kwargs["edittime"]
-
-
-def gptscore_ref_1_fn(pred, ref, **kwargs):
-    if "refs" in kwargs:
-        scores = [gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=1) for ref in kwargs["refs"]]
+        scores = [normalized_similarity(pred, ref) for ref in kwargs["refs"]]
         return sum(scores) / len(scores)
-    return gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=1)
-
-
-def gptscore_ref_3_fn(pred, ref, **kwargs):
-    if "refs" in kwargs:
-        scores = [gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=3) for ref in kwargs["refs"]]
-        return sum(scores) / len(scores)
-    return gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=3)
-
-
-def gptscore_ref_5_fn(pred, ref, **kwargs):
-    if "refs" in kwargs:
-        scores = [gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=5) for ref in kwargs["refs"]]
-        return sum(scores) / len(scores)
-    return gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=5)
-
-
-def gptscore_noref_1_fn(pred, ref, **kwargs):
-    return gpt_eval.compute_noref(prediction=pred, diff=kwargs['diff'], n_requests=1)
-
-
-def gptscore_noref_3_fn(pred, ref, **kwargs):
-    return gpt_eval.compute_noref(prediction=pred, diff=kwargs['diff'], n_requests=3)
-
+    return normalized_similarity(pred, ref)
 
-def gptscore_noref_5_fn(pred, ref, **kwargs):
-    return gpt_eval.compute_noref(prediction=pred, diff=kwargs['diff'], n_requests=5)
 
-
-IND_METRICS = {
+AGGR_METRICS = {
     "editdist": edit_distance_fn,
-    "editdist-norm": edit_distance_norm_fn,
-    # "gptscore-ref-1-req": gptscore_ref_1_fn,
-    # "gptscore-ref-3-req": gptscore_ref_3_fn,
-    # "gptscore-ref-5-req": gptscore_ref_5_fn,
-    # "gptscore-noref-1-req": gptscore_noref_1_fn,
-    # "gptscore-noref-3-req": gptscore_noref_3_fn,
-    # "gptscore-noref-5-req": gptscore_noref_5_fn,
+    "editsim": edit_distance_norm_fn,
     "bleu": bleu_fn,
     "meteor": meteor_fn,
     "rouge1": rouge1_fn,
@@ -144,115 +86,9 @@ IND_METRICS = {
     "rougeL": rougeL_fn,
     "bertscore": bertscore_fn,
     "chrF": chrf_fn,
-    "ter": ter_fn,
 }
 
-AGGR_METRICS = {}
-# AGGR_METRICS = IND_METRICS.copy()
-# del AGGR_METRICS["gptscore-ref-1-req"]
-# del AGGR_METRICS["gptscore-noref-1-req"]
 
 REL_METRICS = {
     "editdist": edit_distance_fn,
-    "editdist-norm": edit_distance_norm_fn,
-    "edittime": edit_time_fn,
 }
-
-
-def attach_references(df):
-    reference_df = hf_data_loader.load_full_commit_as_pandas().set_index(["hash", "repo"])[["reference"]]
-    df = df.set_index(["hash", "repo"])
-    return df.join(other=reference_df, how="left").reset_index()
-
-
-def compute_metrics(df):
-    tqdm.pandas()
-
-    def apply_metric_fn_to_row(row, fn, col_pred, col_ref):
-        return fn(row[col_pred], row[col_ref], edittime=row['edit_time'], diff=str(row['mods']))
-
-    for metric in AGGR_METRICS:
-        print(f"Computing {metric} for the aggregated independent pairs")
-        values = []
-        for i, row in tqdm(df.iterrows(), total=len(df)):
-            others = df[(df["hash"] == row["hash"]) & (df["repo"] == row["repo"]) & (
-                    df["commit_msg_start"] != row["commit_msg_start"]) & (
-                    df["commit_msg_end"] != row["commit_msg_end"])]['commit_msg_end'].to_list()
-            others.append(row["reference"])
-            others = list(set(others))
-            metric_fn = AGGR_METRICS[metric]
-            values.append(
-                metric_fn(
-                    row['commit_msg_start'], None, refs=others, edittime=row['edit_time'], diff=str(row['mods'])
-                )
-            )
-        df[f"{metric}_aggr"] = values
-
-    for metric in REL_METRICS:
-        print(f"Computing {metric} for the related pairs")
-        metric_fn = REL_METRICS[metric]
-        df[f"{metric}_related"] = df.progress_apply(
-            lambda row: apply_metric_fn_to_row(row=row,
-                                               fn=metric_fn,
-                                               col_pred="commit_msg_start",
-                                               col_ref="commit_msg_end"),
-            axis=1
-        )
-
-    for metric in IND_METRICS:
-        print(f"Computing {metric} for the independent pairs")
-        metric_fn = IND_METRICS[metric]
-        df[f"{metric}_independent"] = df.progress_apply(
-            lambda row: apply_metric_fn_to_row(row=row,
-                                               fn=metric_fn,
-                                               col_pred="commit_msg_start",
-                                               col_ref="reference"),
-            axis=1
-        )
-
-    for rel_metric in REL_METRICS:
-        for ind_metric in IND_METRICS:
-            df[f"rel_{rel_metric}_ind_{ind_metric}_pearson"] = (
-                df[f"{rel_metric}_related"].corr(df[f"{ind_metric}_independent"], method="pearson"))
-
-            df[f"rel_{rel_metric}_ind_{ind_metric}_spearman"] = (
-                df[f"{rel_metric}_related"].corr(df[f"{ind_metric}_independent"], method="spearman"))
-
-        for aggr_metric in AGGR_METRICS:
-            df[f"rel_{rel_metric}_aggr_{aggr_metric}_pearson"] = (
-                df[f"{rel_metric}_related"].corr(df[f"{aggr_metric}_aggr"], method="pearson"))
-
-            df[f"rel_{rel_metric}_aggr_{aggr_metric}_spearman"] = (
-                df[f"{rel_metric}_related"].corr(df[f"{aggr_metric}_aggr"], method="spearman"))
-
-    return df
-
-
-def compute_correlations(df: pd.DataFrame):
-    grouped_df = df.groupby(by=["end_to_start", "start_to_end"])
-    correlations = grouped_df.apply(correlations_for_group, include_groups=False)
-    return correlations
-
-
-def transform(df):
-    print("Computing metrics")
-
-    df = attach_references(df)
-    df = compute_metrics(df)
-
-    correlations_for_groups = compute_correlations(df)
-    correlations_for_groups.to_csv(config.METRICS_CORRELATIONS_ARTIFACT)
-
-    df.to_csv(config.SYNTHETIC_DATASET_ARTIFACT)
-
-    print("Done")
-    return df
-
-
-def main():
-    df = pd.read_csv(config.START_TO_END_ARTIFACT, index_col=[0])
-    transform(df)
-
-
-if __name__ == '__main__':
-    main()

pyproject.toml

@@ -0,0 +1,187 @@
+[tool.poetry]
+name = "commit-message-editing-visualization"
+version = "0.1.0"
+description = "Utilities for synthetic data generation, metrics analysis and visualization space for CMG Evaluaton."
+authors = ["Your Name <you@example.com>"]
+license = "MIT"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+absl-py = "2.1.0"
+aiofiles = "23.2.1"
+aiohttp = "3.9.3"
+aiosignal = "1.3.1"
+altair = "5.3.0"
+annotated-types = "0.6.0"
+anyio = "4.3.0"
+argon2-cffi = "23.1.0"
+argon2-cffi-bindings = "21.2.0"
+arrow = "1.3.0"
+asttokens = "2.4.1"
+async-lru = "2.0.4"
+async-timeout = "4.0.3"
+attrs = "23.2.0"
+Babel = "2.14.0"
+beautifulsoup4 = "4.12.3"
+bert-score = "0.3.13"
+bleach = "6.1.0"
+cbor2 = "5.6.2"
+certifi = "2024.2.2"
+cffi = "1.16.0"
+charset-normalizer = "3.3.2"
+click = "8.1.7"
+colorama = "0.4.6"
+comm = "0.2.2"
+contourpy = "1.2.1"
+cycler = "0.12.1"
+datasets = "2.18.0"
+debugpy = "1.8.1"
+decorator = "5.1.1"
+defusedxml = "0.7.1"
+diff-match-patch = "20230430"
+dill = "0.3.8"
+evaluate = "0.4.1"
+exceptiongroup = "1.2.0"
+executing = "2.0.1"
+fastapi = "0.110.1"
+fastjsonschema = "2.19.1"
+ffmpy = "0.3.2"
+filelock = "3.13.3"
+fonttools = "4.50.0"
+fqdn = "1.5.1"
+frozenlist = "1.4.1"
+fsspec = "2024.2.0"
+gradio = "4.25.0"
+gradio_client = "0.15.0"
+h11 = "0.14.0"
+httpcore = "1.0.5"
+httpx = "0.27.0"
+huggingface-hub = "0.22.2"
+idna = "3.6"
+importlib_metadata = "7.1.0"
+importlib_resources = "6.4.0"
+ipykernel = "6.29.4"
+ipython = "8.18.1"
+ipywidgets = "8.1.2"
+isoduration = "20.11.0"
+jedi = "0.19.1"
+Jinja2 = "3.1.3"
+joblib = "1.4.0"
+json5 = "0.9.25"
+jsonpointer = "2.4"
+jsonschema = "4.21.1"
+jsonschema-specifications = "2023.12.1"
+kiwisolver = "1.4.5"
+lxml = "5.2.1"
+markdown-it-py = "3.0.0"
+MarkupSafe = "2.1.5"
+matplotlib = "3.8.4"
+matplotlib-inline = "0.1.7"
+mdurl = "0.1.2"
+mistune = "3.0.2"
+mpmath = "1.3.0"
+multidict = "6.0.5"
+multiprocess = "0.70.16"
+nbclient = "0.10.0"
+nbconvert = "7.16.4"
+nbformat = "5.10.4"
+nest-asyncio = "1.6.0"
+networkx = "3.2.1"
+nltk = "3.8.1"
+numpy = "1.26.4"
+orjson = "3.10.0"
+overrides = "7.7.0"
+packaging = "24.0"
+pandas = "2.2.1"
+pandocfilters = "1.5.1"
+parso = "0.8.4"
+pillow = "10.3.0"
+platformdirs = "4.2.1"
+portalocker = "2.8.2"
+prometheus_client = "0.20.0"
+prompt-toolkit = "3.0.43"
+psutil = "5.9.8"
+pure-eval = "0.2.2"
+pyarrow = "15.0.2"
+pyarrow-hotfix = "0.6"
+pycparser = "2.22"
+pydantic = "2.6.4"
+pydantic_core = "2.16.3"
+pydub = "0.25.1"
+Pygments = "2.17.2"
+pyparsing = "3.1.2"
+python-dateutil = "2.9.0.post0"
+python-json-logger = "2.0.7"
+python-multipart = "0.0.9"
+pytz = "2024.1"
+PyYAML = "6.0.1"
+pyzmq = "26.0.2"
+rapidfuzz = "3.8.1"
+referencing = "0.34.0"
+regex = "2023.12.25"
+requests = "2.31.0"
+responses = "0.18.0"
+rfc3339-validator = "0.1.4"
+rfc3986-validator = "0.1.1"
+rich = "13.7.1"
+rouge-score = "0.1.2"
+rpds-py = "0.18.0"
+ruff = "0.3.5"
+sacrebleu = "2.4.2"
+safetensors = "0.4.2"
+scikit-learn = "1.4.2"
+scipy = "1.13.0"
+semantic-version = "2.10.0"
+Send2Trash = "1.8.3"
+shellingham = "1.5.4"
+six = "1.16.0"
+sniffio = "1.3.1"
+soupsieve = "2.5"
+stack-data = "0.6.3"
+starlette = "0.37.2"
+sympy = "1.12"
+tabulate = "0.9.0"
+terminado = "0.18.1"
+threadpoolctl = "3.4.0"
+tinycss2 = "1.3.0"
+tokenizers = "0.15.2"
+tomli = "2.0.1"
+tomlkit = "0.12.0"
+toolz = "0.12.1"
+torch = "2.2.2"
+tornado = "6.4"
+tqdm = "4.66.2"
+traitlets = "5.14.3"
+transformers = "4.39.3"
+typer = "0.12.1"
+types-python-dateutil = "2.9.0.20240316"
+typing_extensions = "4.10.0"
+tzdata = "2024.1"
+uri-template = "1.3.0"
+urllib3 = "2.2.1"
+uvicorn = "0.29.0"
+wcwidth = "0.2.13"
+webcolors = "1.13"
+webencodings = "0.5.1"
+websocket-client = "1.8.0"
+websockets = "11.0.3"
+widgetsnbextension = "4.0.10"
+xxhash = "3.4.1"
+yarl = "1.9.4"
+zipp = "3.18.1"
+plotly = "5.22.0"
+tenacity = "8.2.3"
+Levenshtein = "0.25.1"
+kaleido = "0.2.1"
+jupyter = "^1.0.0"
+grazie-api-gateway-client = {version = "^0.1.3", source = "space-grazie-ml"}
+seaborn = "^0.13.2"
+
+[[tool.poetry.source]]
+name = "space-grazie-ml"
+url = "https://packages.jetbrains.team/pypi/p/grazi/grazie-ml/simple"
+priority="supplemental"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"