finished brainstorming

requirements.txt

@@ -1,4 +1,6 @@
 langchain
 openai
+ipykernel
+pandas
+tabulate
 gradio
-# ipykernel

src/notebooks/brainstorm2.ipynb

@@ -0,0 +1,307 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import pandas as pd\n",
+    "import langchain\n",
+    "from langchain.agents import OpenAIFunctionsAgent, AgentExecutor\n",
+    "from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
+    "from langchain.tools import PythonAstREPLTool\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from pydantic import BaseModel, Field\n",
+    "from langchain.memory import ConversationBufferMemory"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "langchain.debug = True\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"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\"\n",
+    "data_dir_path = os.path.join(os.getcwd())\n",
+    "pd.set_option('display.max_rows', 20)\n",
+    "pd.set_option('display.max_columns', 20)\n",
+    "\n",
+    "NUM_ROWS_IN_HEAD = 5\n",
+    "\n",
+    "# {dataframe_heads_str}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "PROMPT_TEMPLATE = \"\"\"You are DataMapperGPT. Your job is to work with a human, who is a data engineer, to compare multiple source dataframes and map their structures to the schema of the target dataframe.\n",
+    "The ultimate goal is to generate a mapping from the source dataframes to the target dataframe.\n",
+    "\n",
+    "This is the result of running `df.head().to_markdown()` on each of the dataframes:\n",
+    "\n",
+    "{dataframe_heads_str}\n",
+    "You can use these samples to draw conclusions about the structure of the data. Do not get more than 5 rows at a time.\n",
+    "\n",
+    "Please work step by step through this process. You can make intermediate queries, validate your logic, and then move on to the next step.\n",
+    "\n",
+    "Be precise, analytical, thorough.\n",
+    "\n",
+    "Here is a history of the conversation with the user so far:\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class PythonInputs(BaseModel):\n",
+    "    query: str = Field(description=\"code snippet to run\")\n",
+    "\n",
+    "format_df_for_prompt = lambda df: f'<df>\\n{df.head(NUM_ROWS_IN_HEAD).to_markdown()}\\n</df>'\n",
+    "\n",
+    "entries_a_df = pd.read_csv(os.path.join(data_dir_path, 'legal_entries_a.csv'))\n",
+    "entries_b_df = pd.read_csv(os.path.join(data_dir_path, 'legal_entries_b.csv'))\n",
+    "template_df = pd.read_csv(os.path.join(data_dir_path, 'legal_template.csv'))\n",
+    "\n",
+    "df_name_to_df_map = {\"source_df_1\": entries_a_df, \"source_df_2\": entries_b_df, \"template_df\": template_df}\n",
+    "\n",
+    "dataframe_heads_str_list: str = []\n",
+    "for df_name, df in df_name_to_df_map.items():\n",
+    "    dataframe_heads_str_list.append(f'<{df_name}>\\n{df.head(NUM_ROWS_IN_HEAD).to_markdown()}\\n</{df_name}>')\n",
+    "\n",
+    "prompt_template = PROMPT_TEMPLATE.format(dataframe_heads_str=\"\\n\\n\".join(dataframe_heads_str_list))\n",
+    "\n",
+    "prompt = ChatPromptTemplate.from_messages([\n",
+    "    (\"system\", prompt_template),\n",
+    "    MessagesPlaceholder(variable_name=\"agent_scratchpad\"),\n",
+    "    (\"human\", \"{input}\")\n",
+    "])\n",
+    "memory = ConversationBufferMemory(memory_key=\"chat_history\", return_messages=True)\n",
+    "\n",
+    "repl = PythonAstREPLTool(locals=df_name_to_df_map, name=\"python_repl\",\n",
+    "                            description=\"Runs code and returns the output of the final line\",\n",
+    "                            args_schema=PythonInputs)\n",
+    "tools = [repl]\n",
+    "agent = OpenAIFunctionsAgent(llm=ChatOpenAI(temperature=0, model=\"gpt-3.5-turbo-0613\"), prompt=prompt, tools=tools, memory=memory, handle_parsing_errors=True)\n",
+    "agent_executor = AgentExecutor(agent=agent, tools=tools, max_iterations=5, early_stopping_method=\"generate\", handle_parsing_errors=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:AgentExecutor] Entering Chain run with input:\n",
+      "\u001b[0m{\n",
+      "  \"input\": \"What are the key differences between the dataframe schemas?\",\n",
+      "  \"chat_history\": []\n",
+      "}\n",
+      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:AgentExecutor > 2:llm:ChatOpenAI] Entering LLM run with input:\n",
+      "\u001b[0m{\n",
+      "  \"prompts\": [\n",
+      "    \"System: You are DataMapperGPT. Your job is to work with a human, who is a data engineer, to compare multiple source dataframes and map their structures to the schema of the target dataframe.\\nThe ultimate goal is to generate a mapping from the source dataframes to the target dataframe.\\n\\nThis is the result of running `df.head().to_markdown()` on each of the dataframes:\\n\\n<source_df_1>\\n|    | case_date   | lastname   | firstname   | case_type   | case_id   |   court_fee | jurisdiction   |\\n|---:|:------------|:-----------|:------------|:------------|:----------|------------:|:---------------|\\n|  0 | 2023-05-12  | Kim        | Miguel      | Civil       | CR-1095   |         100 | BOSTON         |\\n|  1 | 2023-04-20  | Lee        | John        | Criminl     | CR-8597   |         150 | houston        |\\n|  2 | 2023-02-10  | Smith      | Dmitri      | Criminal    | CR-6833   |         200 | chicago        |\\n|  3 | 2023-03-16  | Patel      | Dmitri      | Criminal    | CR-2899   |         100 | BOSTON         |\\n|  4 | 2023-06-15  | Ivanov     | Jane        | Family      | CR-5997   |         200 | houston        |\\n</source_df_1>\\n\\n<source_df_2>\\n|    | Date_of_Case   |   Fee | FullName     | CaseNumber   | CaseKind   | Location   |\\n|---:|:---------------|------:|:-------------|:-------------|:-----------|:-----------|\\n|  0 | 2023/05/12     |   100 | Miguel Kim   | CASE-8206    | Civil      | BOST       |\\n|  1 | 2023/04/20     |   150 | John Lee     | CASE-4328    | Criminl    | HOUST      |\\n|  2 | 2023/02/10     |   200 | Dmitri Smith | CASE-1915    | Criminal   | CHIC       |\\n|  3 | 2023/03/16     |   100 | Dmitri Patel | CASE-4283    | Criminal   | BOSTO      |\\n|  4 | 2023/06/15     |   200 | Jane Ivanov  | CASE-7732    | Family     | HOUST      |\\n</source_df_2>\\n\\n<template_df>\\n|    | CaseDate   | FullName     | CaseType   | CaseID    |   Fee | Jurisdiction   |\\n|---:|:-----------|:-------------|:-----------|:----------|------:|:---------------|\\n|  0 | 2023-05-12 | Miguel Kim   | Civil      | CASE-6761 |   100 | Boston         |\\n|  1 | 2023-04-20 | John Lee     | Criminl    | CASE-6089 |   150 | Houston        |\\n|  2 | 2023-02-10 | Dmitri Smith | Criminal   | CASE-9565 |   200 | Chicago        |\\n|  3 | 2023-03-16 | Dmitri Patel | Criminal   | CASE-6222 |   100 | Boston         |\\n|  4 | 2023-06-15 | Jane Ivanov  | Family     | CASE-2702 |   200 | Houston        |\\n</template_df>\\nYou can use these samples to draw conclusions about the structure of the data. Do not get more than 5 rows at a time.\\n\\nPlease work step by step through this process. You can make intermediate queries, validate your logic, and then move on to the next step.\\n\\nBe precise, analytical, thorough.\\n\\nHere is a history of the conversation with the user so far:\\n\\nHuman: What are the key differences between the dataframe schemas?\"\n",
+      "  ]\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:AgentExecutor > 2:llm:ChatOpenAI] [16.60s] Exiting LLM run with output:\n",
+      "\u001b[0m{\n",
+      "  \"generations\": [\n",
+      "    [\n",
+      "      {\n",
+      "        \"text\": \"\",\n",
+      "        \"generation_info\": {\n",
+      "          \"finish_reason\": \"function_call\"\n",
+      "        },\n",
+      "        \"message\": {\n",
+      "          \"lc\": 1,\n",
+      "          \"type\": \"constructor\",\n",
+      "          \"id\": [\n",
+      "            \"langchain\",\n",
+      "            \"schema\",\n",
+      "            \"messages\",\n",
+      "            \"AIMessage\"\n",
+      "          ],\n",
+      "          \"kwargs\": {\n",
+      "            \"content\": \"\",\n",
+      "            \"additional_kwargs\": {\n",
+      "              \"function_call\": {\n",
+      "                \"name\": \"python_repl\",\n",
+      "                \"arguments\": \"{\\n  \\\"query\\\": \\\"import pandas as pd\\\\n\\\\nsource_df_1 = pd.DataFrame({'case_date': ['2023-05-12', '2023-04-20', '2023-02-10', '2023-03-16', '2023-06-15'], 'lastname': ['Kim', 'Lee', 'Smith', 'Patel', 'Ivanov'], 'firstname': ['Miguel', 'John', 'Dmitri', 'Dmitri', 'Jane'], 'case_type': ['Civil', 'Criminl', 'Criminal', 'Criminal', 'Family'], 'case_id': ['CR-1095', 'CR-8597', 'CR-6833', 'CR-2899', 'CR-5997'], 'court_fee': [100, 150, 200, 100, 200], 'jurisdiction': ['BOSTON', 'houston', 'chicago', 'BOSTON', 'houston']})\\\\n\\\\nsource_df_2 = pd.DataFrame({'Date_of_Case': ['2023/05/12', '2023/04/20', '2023/02/10', '2023/03/16', '2023/06/15'], 'Fee': [100, 150, 200, 100, 200], 'FullName': ['Miguel Kim', 'John Lee', 'Dmitri Smith', 'Dmitri Patel', 'Jane Ivanov'], 'CaseNumber': ['CASE-8206', 'CASE-4328', 'CASE-1915', 'CASE-4283', 'CASE-7732'], 'CaseKind': ['Civil', 'Criminl', 'Criminal', 'Criminal', 'Family'], 'Location': ['BOST', 'HOUST', 'CHIC', 'BOSTO', 'HOUST']})\\\\n\\\\ntemplate_df = pd.DataFrame({'CaseDate': ['2023-05-12', '2023-04-20', '2023-02-10', '2023-03-16', '2023-06-15'], 'FullName': ['Miguel Kim', 'John Lee', 'Dmitri Smith', 'Dmitri Patel', 'Jane Ivanov'], 'CaseType': ['Civil', 'Criminl', 'Criminal', 'Criminal', 'Family'], 'CaseID': ['CASE-6761', 'CASE-6089', 'CASE-9565', 'CASE-6222', 'CASE-2702'], 'Fee': [100, 150, 200, 100, 200], 'Jurisdiction': ['Boston', 'Houston', 'Chicago', 'Boston', 'Houston']})\\\\n\\\\nsource_df_1.head().to_markdown()\\\"\\n}\"\n",
+      "              }\n",
+      "            }\n",
+      "          }\n",
+      "        }\n",
+      "      }\n",
+      "    ]\n",
+      "  ],\n",
+      "  \"llm_output\": {\n",
+      "    \"token_usage\": {\n",
+      "      \"prompt_tokens\": 932,\n",
+      "      \"completion_tokens\": 599,\n",
+      "      \"total_tokens\": 1531\n",
+      "    },\n",
+      "    \"model_name\": \"gpt-3.5-turbo-0613\"\n",
+      "  },\n",
+      "  \"run\": null\n",
+      "}\n",
+      "\u001b[32;1m\u001b[1;3m[tool/start]\u001b[0m \u001b[1m[1:chain:AgentExecutor > 3:tool:python_repl] Entering Tool run with input:\n",
+      "\u001b[0m\"{'query': \"import pandas as pd\\n\\nsource_df_1 = pd.DataFrame({'case_date': ['2023-05-12', '2023-04-20', '2023-02-10', '2023-03-16', '2023-06-15'], 'lastname': ['Kim', 'Lee', 'Smith', 'Patel', 'Ivanov'], 'firstname': ['Miguel', 'John', 'Dmitri', 'Dmitri', 'Jane'], 'case_type': ['Civil', 'Criminl', 'Criminal', 'Criminal', 'Family'], 'case_id': ['CR-1095', 'CR-8597', 'CR-6833', 'CR-2899', 'CR-5997'], 'court_fee': [100, 150, 200, 100, 200], 'jurisdiction': ['BOSTON', 'houston', 'chicago', 'BOSTON', 'houston']})\\n\\nsource_df_2 = pd.DataFrame({'Date_of_Case': ['2023/05/12', '2023/04/20', '2023/02/10', '2023/03/16', '2023/06/15'], 'Fee': [100, 150, 200, 100, 200], 'FullName': ['Miguel Kim', 'John Lee', 'Dmitri Smith', 'Dmitri Patel', 'Jane Ivanov'], 'CaseNumber': ['CASE-8206', 'CASE-4328', 'CASE-1915', 'CASE-4283', 'CASE-7732'], 'CaseKind': ['Civil', 'Criminl', 'Criminal', 'Criminal', 'Family'], 'Location': ['BOST', 'HOUST', 'CHIC', 'BOSTO', 'HOUST']})\\n\\ntemplate_df = pd.DataFrame({'CaseDate': ['2023-05-12', '2023-04-20', '2023-02-10', '2023-03-16', '2023-06-15'], 'FullName': ['Miguel Kim', 'John Lee', 'Dmitri Smith', 'Dmitri Patel', 'Jane Ivanov'], 'CaseType': ['Civil', 'Criminl', 'Criminal', 'Criminal', 'Family'], 'CaseID': ['CASE-6761', 'CASE-6089', 'CASE-9565', 'CASE-6222', 'CASE-2702'], 'Fee': [100, 150, 200, 100, 200], 'Jurisdiction': ['Boston', 'Houston', 'Chicago', 'Boston', 'Houston']})\\n\\nsource_df_1.head().to_markdown()\"}\"\n",
+      "\u001b[36;1m\u001b[1;3m[tool/end]\u001b[0m \u001b[1m[1:chain:AgentExecutor > 3:tool:python_repl] [7ms] Exiting Tool run with output:\n",
+      "\u001b[0m\"|    | case_date   | lastname   | firstname   | case_type   | case_id   |   court_fee | jurisdiction   |\n",
+      "|---:|:------------|:-----------|:------------|:------------|:----------|------------:|:---------------|\n",
+      "|  0 | 2023-05-12  | Kim        | Miguel      | Civil       | CR-1095   |         100 | BOSTON         |\n",
+      "|  1 | 2023-04-20  | Lee        | John        | Criminl     | CR-8597   |         150 | houston        |\n",
+      "|  2 | 2023-02-10  | Smith      | Dmitri      | Criminal    | CR-6833   |         200 | chicago        |\n",
+      "|  3 | 2023-03-16  | Patel      | Dmitri      | Criminal    | CR-2899   |         100 | BOSTON         |\n",
+      "|  4 | 2023-06-15  | Ivanov     | Jane        | Family      | CR-5997   |         200 | houston        |\"\n",
+      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:AgentExecutor > 4:llm:ChatOpenAI] Entering LLM run with input:\n",
+      "\u001b[0m{\n",
+      "  \"prompts\": [\n",
+      "    \"System: You are DataMapperGPT. Your job is to work with a human, who is a data engineer, to compare multiple source dataframes and map their structures to the schema of the target dataframe.\\nThe ultimate goal is to generate a mapping from the source dataframes to the target dataframe.\\n\\nThis is the result of running `df.head().to_markdown()` on each of the dataframes:\\n\\n<source_df_1>\\n|    | case_date   | lastname   | firstname   | case_type   | case_id   |   court_fee | jurisdiction   |\\n|---:|:------------|:-----------|:------------|:------------|:----------|------------:|:---------------|\\n|  0 | 2023-05-12  | Kim        | Miguel      | Civil       | CR-1095   |         100 | BOSTON         |\\n|  1 | 2023-04-20  | Lee        | John        | Criminl     | CR-8597   |         150 | houston        |\\n|  2 | 2023-02-10  | Smith      | Dmitri      | Criminal    | CR-6833   |         200 | chicago        |\\n|  3 | 2023-03-16  | Patel      | Dmitri      | Criminal    | CR-2899   |         100 | BOSTON         |\\n|  4 | 2023-06-15  | Ivanov     | Jane        | Family      | CR-5997   |         200 | houston        |\\n</source_df_1>\\n\\n<source_df_2>\\n|    | Date_of_Case   |   Fee | FullName     | CaseNumber   | CaseKind   | Location   |\\n|---:|:---------------|------:|:-------------|:-------------|:-----------|:-----------|\\n|  0 | 2023/05/12     |   100 | Miguel Kim   | CASE-8206    | Civil      | BOST       |\\n|  1 | 2023/04/20     |   150 | John Lee     | CASE-4328    | Criminl    | HOUST      |\\n|  2 | 2023/02/10     |   200 | Dmitri Smith | CASE-1915    | Criminal   | CHIC       |\\n|  3 | 2023/03/16     |   100 | Dmitri Patel | CASE-4283    | Criminal   | BOSTO      |\\n|  4 | 2023/06/15     |   200 | Jane Ivanov  | CASE-7732    | Family     | HOUST      |\\n</source_df_2>\\n\\n<template_df>\\n|    | CaseDate   | FullName     | CaseType   | CaseID    |   Fee | Jurisdiction   |\\n|---:|:-----------|:-------------|:-----------|:----------|------:|:---------------|\\n|  0 | 2023-05-12 | Miguel Kim   | Civil      | CASE-6761 |   100 | Boston         |\\n|  1 | 2023-04-20 | John Lee     | Criminl    | CASE-6089 |   150 | Houston        |\\n|  2 | 2023-02-10 | Dmitri Smith | Criminal   | CASE-9565 |   200 | Chicago        |\\n|  3 | 2023-03-16 | Dmitri Patel | Criminal   | CASE-6222 |   100 | Boston         |\\n|  4 | 2023-06-15 | Jane Ivanov  | Family     | CASE-2702 |   200 | Houston        |\\n</template_df>\\nYou can use these samples to draw conclusions about the structure of the data. Do not get more than 5 rows at a time.\\n\\nPlease work step by step through this process. You can make intermediate queries, validate your logic, and then move on to the next step.\\n\\nBe precise, analytical, thorough.\\n\\nHere is a history of the conversation with the user so far:\\n\\nAI: {'name': 'python_repl', 'arguments': '{\\\\n  \\\"query\\\": \\\"import pandas as pd\\\\\\\\n\\\\\\\\nsource_df_1 = pd.DataFrame({\\\\'case_date\\\\': [\\\\'2023-05-12\\\\', \\\\'2023-04-20\\\\', \\\\'2023-02-10\\\\', \\\\'2023-03-16\\\\', \\\\'2023-06-15\\\\'], \\\\'lastname\\\\': [\\\\'Kim\\\\', \\\\'Lee\\\\', \\\\'Smith\\\\', \\\\'Patel\\\\', \\\\'Ivanov\\\\'], \\\\'firstname\\\\': [\\\\'Miguel\\\\', \\\\'John\\\\', \\\\'Dmitri\\\\', \\\\'Dmitri\\\\', \\\\'Jane\\\\'], \\\\'case_type\\\\': [\\\\'Civil\\\\', \\\\'Criminl\\\\', \\\\'Criminal\\\\', \\\\'Criminal\\\\', \\\\'Family\\\\'], \\\\'case_id\\\\': [\\\\'CR-1095\\\\', \\\\'CR-8597\\\\', \\\\'CR-6833\\\\', \\\\'CR-2899\\\\', \\\\'CR-5997\\\\'], \\\\'court_fee\\\\': [100, 150, 200, 100, 200], \\\\'jurisdiction\\\\': [\\\\'BOSTON\\\\', \\\\'houston\\\\', \\\\'chicago\\\\', \\\\'BOSTON\\\\', \\\\'houston\\\\']})\\\\\\\\n\\\\\\\\nsource_df_2 = pd.DataFrame({\\\\'Date_of_Case\\\\': [\\\\'2023/05/12\\\\', \\\\'2023/04/20\\\\', \\\\'2023/02/10\\\\', \\\\'2023/03/16\\\\', \\\\'2023/06/15\\\\'], \\\\'Fee\\\\': [100, 150, 200, 100, 200], \\\\'FullName\\\\': [\\\\'Miguel Kim\\\\', \\\\'John Lee\\\\', \\\\'Dmitri Smith\\\\', \\\\'Dmitri Patel\\\\', \\\\'Jane Ivanov\\\\'], \\\\'CaseNumber\\\\': [\\\\'CASE-8206\\\\', \\\\'CASE-4328\\\\', \\\\'CASE-1915\\\\', \\\\'CASE-4283\\\\', \\\\'CASE-7732\\\\'], \\\\'CaseKind\\\\': [\\\\'Civil\\\\', \\\\'Criminl\\\\', \\\\'Criminal\\\\', \\\\'Criminal\\\\', \\\\'Family\\\\'], \\\\'Location\\\\': [\\\\'BOST\\\\', \\\\'HOUST\\\\', \\\\'CHIC\\\\', \\\\'BOSTO\\\\', \\\\'HOUST\\\\']})\\\\\\\\n\\\\\\\\ntemplate_df = pd.DataFrame({\\\\'CaseDate\\\\': [\\\\'2023-05-12\\\\', \\\\'2023-04-20\\\\', \\\\'2023-02-10\\\\', \\\\'2023-03-16\\\\', \\\\'2023-06-15\\\\'], \\\\'FullName\\\\': [\\\\'Miguel Kim\\\\', \\\\'John Lee\\\\', \\\\'Dmitri Smith\\\\', \\\\'Dmitri Patel\\\\', \\\\'Jane Ivanov\\\\'], \\\\'CaseType\\\\': [\\\\'Civil\\\\', \\\\'Criminl\\\\', \\\\'Criminal\\\\', \\\\'Criminal\\\\', \\\\'Family\\\\'], \\\\'CaseID\\\\': [\\\\'CASE-6761\\\\', \\\\'CASE-6089\\\\', \\\\'CASE-9565\\\\', \\\\'CASE-6222\\\\', \\\\'CASE-2702\\\\'], \\\\'Fee\\\\': [100, 150, 200, 100, 200], \\\\'Jurisdiction\\\\': [\\\\'Boston\\\\', \\\\'Houston\\\\', \\\\'Chicago\\\\', \\\\'Boston\\\\', \\\\'Houston\\\\']})\\\\\\\\n\\\\\\\\nsource_df_1.head().to_markdown()\\\"\\\\n}'}\\nFunction: |    | case_date   | lastname   | firstname   | case_type   | case_id   |   court_fee | jurisdiction   |\\n|---:|:------------|:-----------|:------------|:------------|:----------|------------:|:---------------|\\n|  0 | 2023-05-12  | Kim        | Miguel      | Civil       | CR-1095   |         100 | BOSTON         |\\n|  1 | 2023-04-20  | Lee        | John        | Criminl     | CR-8597   |         150 | houston        |\\n|  2 | 2023-02-10  | Smith      | Dmitri      | Criminal    | CR-6833   |         200 | chicago        |\\n|  3 | 2023-03-16  | Patel      | Dmitri      | Criminal    | CR-2899   |         100 | BOSTON         |\\n|  4 | 2023-06-15  | Ivanov     | Jane        | Family      | CR-5997   |         200 | houston        |\\nHuman: What are the key differences between the dataframe schemas?\"\n",
+      "  ]\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:AgentExecutor > 4:llm:ChatOpenAI] [1.18s] Exiting LLM run with output:\n",
+      "\u001b[0m{\n",
+      "  \"generations\": [\n",
+      "    [\n",
+      "      {\n",
+      "        \"text\": \"\",\n",
+      "        \"generation_info\": {\n",
+      "          \"finish_reason\": \"function_call\"\n",
+      "        },\n",
+      "        \"message\": {\n",
+      "          \"lc\": 1,\n",
+      "          \"type\": \"constructor\",\n",
+      "          \"id\": [\n",
+      "            \"langchain\",\n",
+      "            \"schema\",\n",
+      "            \"messages\",\n",
+      "            \"AIMessage\"\n",
+      "          ],\n",
+      "          \"kwargs\": {\n",
+      "            \"content\": \"\",\n",
+      "            \"additional_kwargs\": {\n",
+      "              \"function_call\": {\n",
+      "                \"name\": \"python_repl\",\n",
+      "                \"arguments\": \"{\\n  \\\"query\\\": \\\"set(source_df_1.columns) - set(template_df.columns)\\\"\\n}\"\n",
+      "              }\n",
+      "            }\n",
+      "          }\n",
+      "        }\n",
+      "      }\n",
+      "    ]\n",
+      "  ],\n",
+      "  \"llm_output\": {\n",
+      "    \"token_usage\": {\n",
+      "      \"prompt_tokens\": 1784,\n",
+      "      \"completion_tokens\": 27,\n",
+      "      \"total_tokens\": 1811\n",
+      "    },\n",
+      "    \"model_name\": \"gpt-3.5-turbo-0613\"\n",
+      "  },\n",
+      "  \"run\": null\n",
+      "}\n",
+      "\u001b[32;1m\u001b[1;3m[tool/start]\u001b[0m \u001b[1m[1:chain:AgentExecutor > 5:tool:python_repl] Entering Tool run with input:\n",
+      "\u001b[0m\"{'query': 'set(source_df_1.columns) - set(template_df.columns)'}\"\n",
+      "\u001b[36;1m\u001b[1;3m[tool/end]\u001b[0m \u001b[1m[1:chain:AgentExecutor > 5:tool:python_repl] [0ms] Exiting Tool run with output:\n",
+      "\u001b[0m\"{'case_id', 'firstname', 'court_fee', 'case_type', 'lastname', 'case_date', 'jurisdiction'}\"\n",
+      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:AgentExecutor > 6:llm:ChatOpenAI] Entering LLM run with input:\n",
+      "\u001b[0m{\n",
+      "  \"prompts\": [\n",
+      "    \"System: You are DataMapperGPT. Your job is to work with a human, who is a data engineer, to compare multiple source dataframes and map their structures to the schema of the target dataframe.\\nThe ultimate goal is to generate a mapping from the source dataframes to the target dataframe.\\n\\nThis is the result of running `df.head().to_markdown()` on each of the dataframes:\\n\\n<source_df_1>\\n|    | case_date   | lastname   | firstname   | case_type   | case_id   |   court_fee | jurisdiction   |\\n|---:|:------------|:-----------|:------------|:------------|:----------|------------:|:---------------|\\n|  0 | 2023-05-12  | Kim        | Miguel      | Civil       | CR-1095   |         100 | BOSTON         |\\n|  1 | 2023-04-20  | Lee        | John        | Criminl     | CR-8597   |         150 | houston        |\\n|  2 | 2023-02-10  | Smith      | Dmitri      | Criminal    | CR-6833   |         200 | chicago        |\\n|  3 | 2023-03-16  | Patel      | Dmitri      | Criminal    | CR-2899   |         100 | BOSTON         |\\n|  4 | 2023-06-15  | Ivanov     | Jane        | Family      | CR-5997   |         200 | houston        |\\n</source_df_1>\\n\\n<source_df_2>\\n|    | Date_of_Case   |   Fee | FullName     | CaseNumber   | CaseKind   | Location   |\\n|---:|:---------------|------:|:-------------|:-------------|:-----------|:-----------|\\n|  0 | 2023/05/12     |   100 | Miguel Kim   | CASE-8206    | Civil      | BOST       |\\n|  1 | 2023/04/20     |   150 | John Lee     | CASE-4328    | Criminl    | HOUST      |\\n|  2 | 2023/02/10     |   200 | Dmitri Smith | CASE-1915    | Criminal   | CHIC       |\\n|  3 | 2023/03/16     |   100 | Dmitri Patel | CASE-4283    | Criminal   | BOSTO      |\\n|  4 | 2023/06/15     |   200 | Jane Ivanov  | CASE-7732    | Family     | HOUST      |\\n</source_df_2>\\n\\n<template_df>\\n|    | CaseDate   | FullName     | CaseType   | CaseID    |   Fee | Jurisdiction   |\\n|---:|:-----------|:-------------|:-----------|:----------|------:|:---------------|\\n|  0 | 2023-05-12 | Miguel Kim   | Civil      | CASE-6761 |   100 | Boston         |\\n|  1 | 2023-04-20 | John Lee     | Criminl    | CASE-6089 |   150 | Houston        |\\n|  2 | 2023-02-10 | Dmitri Smith | Criminal   | CASE-9565 |   200 | Chicago        |\\n|  3 | 2023-03-16 | Dmitri Patel | Criminal   | CASE-6222 |   100 | Boston         |\\n|  4 | 2023-06-15 | Jane Ivanov  | Family     | CASE-2702 |   200 | Houston        |\\n</template_df>\\nYou can use these samples to draw conclusions about the structure of the data. Do not get more than 5 rows at a time.\\n\\nPlease work step by step through this process. You can make intermediate queries, validate your logic, and then move on to the next step.\\n\\nBe precise, analytical, thorough.\\n\\nHere is a history of the conversation with the user so far:\\n\\nAI: {'name': 'python_repl', 'arguments': '{\\\\n  \\\"query\\\": \\\"import pandas as pd\\\\\\\\n\\\\\\\\nsource_df_1 = pd.DataFrame({\\\\'case_date\\\\': [\\\\'2023-05-12\\\\', \\\\'2023-04-20\\\\', \\\\'2023-02-10\\\\', \\\\'2023-03-16\\\\', \\\\'2023-06-15\\\\'], \\\\'lastname\\\\': [\\\\'Kim\\\\', \\\\'Lee\\\\', \\\\'Smith\\\\', \\\\'Patel\\\\', \\\\'Ivanov\\\\'], \\\\'firstname\\\\': [\\\\'Miguel\\\\', \\\\'John\\\\', \\\\'Dmitri\\\\', \\\\'Dmitri\\\\', \\\\'Jane\\\\'], \\\\'case_type\\\\': [\\\\'Civil\\\\', \\\\'Criminl\\\\', \\\\'Criminal\\\\', \\\\'Criminal\\\\', \\\\'Family\\\\'], \\\\'case_id\\\\': [\\\\'CR-1095\\\\', \\\\'CR-8597\\\\', \\\\'CR-6833\\\\', \\\\'CR-2899\\\\', \\\\'CR-5997\\\\'], \\\\'court_fee\\\\': [100, 150, 200, 100, 200], \\\\'jurisdiction\\\\': [\\\\'BOSTON\\\\', \\\\'houston\\\\', \\\\'chicago\\\\', \\\\'BOSTON\\\\', \\\\'houston\\\\']})\\\\\\\\n\\\\\\\\nsource_df_2 = pd.DataFrame({\\\\'Date_of_Case\\\\': [\\\\'2023/05/12\\\\', \\\\'2023/04/20\\\\', \\\\'2023/02/10\\\\', \\\\'2023/03/16\\\\', \\\\'2023/06/15\\\\'], \\\\'Fee\\\\': [100, 150, 200, 100, 200], \\\\'FullName\\\\': [\\\\'Miguel Kim\\\\', \\\\'John Lee\\\\', \\\\'Dmitri Smith\\\\', \\\\'Dmitri Patel\\\\', \\\\'Jane Ivanov\\\\'], \\\\'CaseNumber\\\\': [\\\\'CASE-8206\\\\', \\\\'CASE-4328\\\\', \\\\'CASE-1915\\\\', \\\\'CASE-4283\\\\', \\\\'CASE-7732\\\\'], \\\\'CaseKind\\\\': [\\\\'Civil\\\\', \\\\'Criminl\\\\', \\\\'Criminal\\\\', \\\\'Criminal\\\\', \\\\'Family\\\\'], \\\\'Location\\\\': [\\\\'BOST\\\\', \\\\'HOUST\\\\', \\\\'CHIC\\\\', \\\\'BOSTO\\\\', \\\\'HOUST\\\\']})\\\\\\\\n\\\\\\\\ntemplate_df = pd.DataFrame({\\\\'CaseDate\\\\': [\\\\'2023-05-12\\\\', \\\\'2023-04-20\\\\', \\\\'2023-02-10\\\\', \\\\'2023-03-16\\\\', \\\\'2023-06-15\\\\'], \\\\'FullName\\\\': [\\\\'Miguel Kim\\\\', \\\\'John Lee\\\\', \\\\'Dmitri Smith\\\\', \\\\'Dmitri Patel\\\\', \\\\'Jane Ivanov\\\\'], \\\\'CaseType\\\\': [\\\\'Civil\\\\', \\\\'Criminl\\\\', \\\\'Criminal\\\\', \\\\'Criminal\\\\', \\\\'Family\\\\'], \\\\'CaseID\\\\': [\\\\'CASE-6761\\\\', \\\\'CASE-6089\\\\', \\\\'CASE-9565\\\\', \\\\'CASE-6222\\\\', \\\\'CASE-2702\\\\'], \\\\'Fee\\\\': [100, 150, 200, 100, 200], \\\\'Jurisdiction\\\\': [\\\\'Boston\\\\', \\\\'Houston\\\\', \\\\'Chicago\\\\', \\\\'Boston\\\\', \\\\'Houston\\\\']})\\\\\\\\n\\\\\\\\nsource_df_1.head().to_markdown()\\\"\\\\n}'}\\nFunction: |    | case_date   | lastname   | firstname   | case_type   | case_id   |   court_fee | jurisdiction   |\\n|---:|:------------|:-----------|:------------|:------------|:----------|------------:|:---------------|\\n|  0 | 2023-05-12  | Kim        | Miguel      | Civil       | CR-1095   |         100 | BOSTON         |\\n|  1 | 2023-04-20  | Lee        | John        | Criminl     | CR-8597   |         150 | houston        |\\n|  2 | 2023-02-10  | Smith      | Dmitri      | Criminal    | CR-6833   |         200 | chicago        |\\n|  3 | 2023-03-16  | Patel      | Dmitri      | Criminal    | CR-2899   |         100 | BOSTON         |\\n|  4 | 2023-06-15  | Ivanov     | Jane        | Family      | CR-5997   |         200 | houston        |\\nAI: {'name': 'python_repl', 'arguments': '{\\\\n  \\\"query\\\": \\\"set(source_df_1.columns) - set(template_df.columns)\\\"\\\\n}'}\\nFunction: {'case_id', 'firstname', 'court_fee', 'case_type', 'lastname', 'case_date', 'jurisdiction'}\\nHuman: What are the key differences between the dataframe schemas?\"\n",
+      "  ]\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:AgentExecutor > 6:llm:ChatOpenAI] [8.40s] Exiting LLM run with output:\n",
+      "\u001b[0m{\n",
+      "  \"generations\": [\n",
+      "    [\n",
+      "      {\n",
+      "        \"text\": \"The key differences between the dataframe schemas are as follows:\\n\\n1. The column names in `source_df_1` are different from the column names in `template_df`. The column names in `source_df_1` are: `case_date`, `lastname`, `firstname`, `case_type`, `case_id`, `court_fee`, and `jurisdiction`. The corresponding column names in `template_df` are: `CaseDate`, `FullName`, `CaseType`, `CaseID`, `Fee`, and `Jurisdiction`.\\n\\n2. The order of the columns is different between `source_df_1` and `template_df`.\\n\\n3. The values in the `case_date` column of `source_df_1` are in the format 'YYYY-MM-DD', while the values in the `CaseDate` column of `template_df` are in the format 'YYYY-MM-DD'.\\n\\n4. The values in the `court_fee` column of `source_df_1` are integers, while the values in the `Fee` column of `template_df` are also integers.\\n\\n5. The values in the `jurisdiction` column of `source_df_1` are in uppercase, while the values in the `Jurisdiction` column of `template_df` are in title case.\\n\\nThese are the key differences between the dataframe schemas.\",\n",
+      "        \"generation_info\": {\n",
+      "          \"finish_reason\": \"stop\"\n",
+      "        },\n",
+      "        \"message\": {\n",
+      "          \"lc\": 1,\n",
+      "          \"type\": \"constructor\",\n",
+      "          \"id\": [\n",
+      "            \"langchain\",\n",
+      "            \"schema\",\n",
+      "            \"messages\",\n",
+      "            \"AIMessage\"\n",
+      "          ],\n",
+      "          \"kwargs\": {\n",
+      "            \"content\": \"The key differences between the dataframe schemas are as follows:\\n\\n1. The column names in `source_df_1` are different from the column names in `template_df`. The column names in `source_df_1` are: `case_date`, `lastname`, `firstname`, `case_type`, `case_id`, `court_fee`, and `jurisdiction`. The corresponding column names in `template_df` are: `CaseDate`, `FullName`, `CaseType`, `CaseID`, `Fee`, and `Jurisdiction`.\\n\\n2. The order of the columns is different between `source_df_1` and `template_df`.\\n\\n3. The values in the `case_date` column of `source_df_1` are in the format 'YYYY-MM-DD', while the values in the `CaseDate` column of `template_df` are in the format 'YYYY-MM-DD'.\\n\\n4. The values in the `court_fee` column of `source_df_1` are integers, while the values in the `Fee` column of `template_df` are also integers.\\n\\n5. The values in the `jurisdiction` column of `source_df_1` are in uppercase, while the values in the `Jurisdiction` column of `template_df` are in title case.\\n\\nThese are the key differences between the dataframe schemas.\",\n",
+      "            \"additional_kwargs\": {}\n",
+      "          }\n",
+      "        }\n",
+      "      }\n",
+      "    ]\n",
+      "  ],\n",
+      "  \"llm_output\": {\n",
+      "    \"token_usage\": {\n",
+      "      \"prompt_tokens\": 1846,\n",
+      "      \"completion_tokens\": 271,\n",
+      "      \"total_tokens\": 2117\n",
+      "    },\n",
+      "    \"model_name\": \"gpt-3.5-turbo-0613\"\n",
+      "  },\n",
+      "  \"run\": null\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:AgentExecutor] [26.19s] Exiting Chain run with output:\n",
+      "\u001b[0m{\n",
+      "  \"output\": \"The key differences between the dataframe schemas are as follows:\\n\\n1. The column names in `source_df_1` are different from the column names in `template_df`. The column names in `source_df_1` are: `case_date`, `lastname`, `firstname`, `case_type`, `case_id`, `court_fee`, and `jurisdiction`. The corresponding column names in `template_df` are: `CaseDate`, `FullName`, `CaseType`, `CaseID`, `Fee`, and `Jurisdiction`.\\n\\n2. The order of the columns is different between `source_df_1` and `template_df`.\\n\\n3. The values in the `case_date` column of `source_df_1` are in the format 'YYYY-MM-DD', while the values in the `CaseDate` column of `template_df` are in the format 'YYYY-MM-DD'.\\n\\n4. The values in the `court_fee` column of `source_df_1` are integers, while the values in the `Fee` column of `template_df` are also integers.\\n\\n5. The values in the `jurisdiction` column of `source_df_1` are in uppercase, while the values in the `Jurisdiction` column of `template_df` are in title case.\\n\\nThese are the key differences between the dataframe schemas.\"\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "question = \"What are the key differences between the dataframe schemas?\"\n",
+    "res = agent_executor.run(input=question, chat_history=memory.chat_memory.messages)\n",
+    "memory.chat_memory.add_user_message(question)\n",
+    "memory.chat_memory.add_ai_message(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "get_differences_between_dataframes\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/notebooks/brainstorm4.ipynb

@@ -0,0 +1,509 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import pandas as pd\n",
+    "import gradio as gr\n",
+    "from pydantic import BaseModel, Field\n",
+    "\n",
+    "import langchain\n",
+    "from langchain.output_parsers import PydanticOutputParser\n",
+    "from langchain.prompts import ChatPromptTemplate\n",
+    "from langchain.prompts import ChatPromptTemplate\n",
+    "from langchain.tools import PythonAstREPLTool\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.schema.output_parser import StrOutputParser"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "langchain.debug = False\n",
+    "# Throwaway key with strict usage limit\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"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\"\n",
+    "pd.set_option('display.max_columns', 20)\n",
+    "pd.set_option('display.max_rows', 20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_dir_path = os.path.join(os.getcwd(), 'data')\n",
+    "NUM_ROWS_TO_RETURN = 5\n",
+    "\n",
+    "table_1_df = pd.read_csv(os.path.join(data_dir_path, 'legal_entries_a.csv'))\n",
+    "table_2_df = pd.read_csv(os.path.join(data_dir_path, 'legal_entries_b.csv'))\n",
+    "template_df = pd.read_csv(os.path.join(data_dir_path, 'legal_template.csv'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "transform_model = ChatOpenAI(\n",
+    "    model_name='gpt-4',\n",
+    "    temperature=0,\n",
+    ")\n",
+    "\n",
+    "natural_language_model = ChatOpenAI(\n",
+    "    model_name='gpt-4',\n",
+    "    temperature=0.1,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# TODO: add validation to models, coupled with retry mechanism in chain\n",
+    "class TableMappingEntry(BaseModel):\n",
+    "    '''A single row in a table mapping. Describes how a single column in a source table maps to a single column in a target table, including any necessary transformations, and their explanations.'''\n",
+    "    source_column_name: str = Field(..., description=\"Name of the column in the source table.\")\n",
+    "    target_column_name: str = Field(..., description=\"Name of the column in the target table, to which the source column maps.\")\n",
+    "    value_transformations: str = Field(..., description=\"Transformations needed make the source values match the target values. If unncecessary, write 'NO_TRANSFORM'.\")\n",
+    "    explanation: str = Field(..., description=\"One-sentence explanation of this row (source-target mapping/transformation). Include any information that might be relevant to a software engineer building an ETL pipeline with this document.\")\n",
+    "\n",
+    "class TableMapping(BaseModel):\n",
+    "    '''A list of table mappings collectively describe how a source table should be transformed to match the schema of a target table.'''\n",
+    "    table_mappings: list[TableMappingEntry] = Field(..., description=\"A list of table mappings.\")\n",
+    "    \n",
+    "analyst_prompt_str = '''\n",
+    "You are a Data Scientist, who specializes in generating schema mappings for use by Software Engineers in ETL pipelines.\n",
+    "\n",
+    "Head of `source_csv`:\n",
+    "\n",
+    "{source_1_csv_str}\n",
+    "\n",
+    "Head of `target_csv`:\n",
+    "\n",
+    "{target_csv_str}\n",
+    "\n",
+    "Your job is to generate a thorough, precise summary of how `source_csv` should be transformed to adhere exactly to the `target_csv` schema.\n",
+    "\n",
+    "For each column in the `source_csv`, you must communicate which column in the `target_csv` it maps to, and how the values in the `source_csv` column should be transformed to match those in the `target_csv`.\n",
+    "You can assume the rows are aligned: that is, the first row in `source_csv` corresponds to the first row in `target_csv`, and so on.\n",
+    "\n",
+    "Remember:\n",
+    "1. Which column in `target_csv` it maps to. You should consider the semantic meaning of the columns, not just the character similarity. \n",
+    "\n",
+    "Example mappings:\n",
+    "- 'MunICipality' in `source_csv` should map to 'City' in `target_csv`.\n",
+    "- 'fullname' in `source_csv` should map to both 'FirstName' and 'LastName' in `target_csv`. You must explain this transformation, as well, including the target sequencing of first and last name.\n",
+    "\n",
+    "Example transformations:\n",
+    "- If date in `source_csv` is `2020-01-01` and date in `target_csv` is `01/01/2020`, explain exactly how this should be transformed and the reasoning behind it.\n",
+    "- If city in `source_csv` is `New York` and city in `target_csv` is `NEW YORK` or `NYC`, explain exactly how this should be transformed and the reasoning behind it.\n",
+    "\n",
+    "Lastly, point out any other oddities, such as duplicate columns, erroneous columns, etc.\n",
+    "\n",
+    "{format_instructions}\n",
+    "\n",
+    "Remember:\n",
+    "- Be concise: you are speaking to engineers, not customers.\n",
+    "- Be precise: all of these values are case sensitive. Consider casing for city names, exact prefixes for identifiers, ordering of people's names, etc.\n",
+    "- DO NOT include commas, quotes, or any other characters that might interfere with JSON serialization or CSV generation\n",
+    "\n",
+    "Your response:\n",
+    "'''\n",
+    "\n",
+    "def get_data_str_from_df_for_prompt(df, use_head=True, num_rows_to_return=NUM_ROWS_TO_RETURN):\n",
+    "    data = df.head(num_rows_to_return) if use_head else df.tail(num_rows_to_return)\n",
+    "    return f'<df>\\n{data.to_markdown()}\\n</df>'\n",
+    "\n",
+    "table_mapping_parser = PydanticOutputParser(pydantic_object=TableMapping)\n",
+    "analyst_prompt = ChatPromptTemplate.from_template(\n",
+    "    template=analyst_prompt_str, \n",
+    "    partial_variables={'format_instructions': table_mapping_parser.get_format_instructions()},\n",
+    ")\n",
+    "\n",
+    "mapping_chain = analyst_prompt | transform_model | table_mapping_parser\n",
+    "table_mapping: TableMapping = mapping_chain.invoke({\"source_1_csv_str\": get_data_str_from_df_for_prompt(table_1_df), \"target_csv_str\": get_data_str_from_df_for_prompt(template_df)})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# spec writer\n",
+    "spec_writer_prompt_str = '''\n",
+    "You are an expert product manager and technical writer for a software company, who generates clean, concise, precise specification documents for your employees.\n",
+    "Your job is to write a plaintext spec for a python script for a software engineer to develop a component within an ETL pipeline.\n",
+    "\n",
+    "This document must include 100% of the information your employee needs to write a successful script to transform source_df to target_df.\n",
+    "However, DO NOT include the original table_mapping. Your job is to translate everything into natural language.\n",
+    "\n",
+    "Here is a stringified pydantic object that describes the mapping and the transformation steps:\n",
+    "\n",
+    "{table_mapping}\n",
+    "\n",
+    "You must translate this into clean, concise, and complete instructions for your employee.\n",
+    "\n",
+    "This document should be formatted like a technical document in plaintext. Do not include code or data.\n",
+    "\n",
+    "This document must include:\n",
+    "- Overview\n",
+    "- Input (source_df), Output (target_df)\n",
+    "- Exact column mapping\n",
+    "- Exact transformation steps for each column\n",
+    "- Precise instructions for what this script should do\n",
+    "- Script input: Pandas Dataframe named `source_df`.\n",
+    "- Script output: Pandas Dataframe named `target_df`.\n",
+    "- Do not modify the source_df. Create a new dataframe named target_df.\n",
+    "- This script should never include the source data. It should only include the transormations required to create the target_df.\n",
+    "- Return the target_df.\n",
+    "\n",
+    "You will never see this employee. They cannot contact you. You will never see their code. You must include 100% of the information they need to write a successful script.\n",
+    "Remember:\n",
+    "- Clean: No extra information, no formatting aside from plaintext\n",
+    "- Concise: Your employees benefit from brevity\n",
+    "- Precise: your words must be unambiguous, exact, and full represent a perfect translation of the table_mapping object.\n",
+    "\n",
+    "Your response:\n",
+    "'''\n",
+    "spec_writer_prompt = ChatPromptTemplate.from_template(spec_writer_prompt_str)\n",
+    "\n",
+    "spec_writer_chain = spec_writer_prompt | natural_language_model | StrOutputParser()\n",
+    "spec_str = spec_writer_chain.invoke({\"table_mapping\": str(table_mapping)})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "engineer_prompt_str = '''\n",
+    "You are a Senior Software Engineer, who specializes in writing Python code for ETL pipelines.\n",
+    "Your Product Manager has written a spec for a new transormation script. You must follow this document exactly, write python code that implements the spec, validate that code, and then return it.\n",
+    "Your output should only be python code in Markdown format, eg:\n",
+    "    ```python\n",
+    "    ....\n",
+    "    ```\"\"\"\n",
+    "Do not return any additional text / explanation. This code will be executed by a robot without human intervention.\n",
+    "\n",
+    "Here is the technical specification for your code:\n",
+    "\n",
+    "{spec_str}\n",
+    "\n",
+    "Remember: return only clean python code in markdown format. The python interpreter running this code will already have `source_df` as a local variable.\n",
+    "\n",
+    "Your must return `target_df` at the end.\n",
+    "'''\n",
+    "engineer_prompt = ChatPromptTemplate.from_template(engineer_prompt_str)\n",
+    "\n",
+    "# engineer_chain = engineer_prompt | transform_model | StrOutputParser() | PythonAstREPLTool(locals={'source_df': table_1_df}).run\n",
+    "# table_1_df_transformed = engineer_chain.invoke({\"spec_str\": spec_str})\n",
+    "engineer_chain = engineer_prompt | transform_model | StrOutputParser()\n",
+    "transform_code = engineer_chain.invoke({\"spec_str\": spec_str})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7874\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7874/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def show_mapping(file):\n",
+    "    # TODO: add code\n",
+    "    return pd.DataFrame(table_mapping.dict()['table_mappings'])\n",
+    "demo = gr.Interface(fn=show_mapping, inputs=[\"file\"], outputs='dataframe')\n",
+    "demo.launch()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7885\n",
+      "\n",
+      "Thanks for being a Gradio user! If you have questions or feedback, please join our Discord server and chat with us: https://discord.gg/feTf9x3ZSB\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7885/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def _sanitize_python_output(text: str):\n",
+    "    _, after = text.split(\"```python\")\n",
+    "    return after.split(\"```\")[0]\n",
+    "\n",
+    "def show_code(button):\n",
+    "    # TODO: add code\n",
+    "    return _sanitize_python_output(transform_code)\n",
+    "check_mapping_text = 'How does that mapping look? \\n\\nFeel free to update it: your changes will be incorporated! \\n\\nWhen you are ready, click the Submit below, and the mapping code will be generated for your approval.'\n",
+    "demo = gr.Interface(fn=show_code, inputs=[gr.Textbox(value=check_mapping_text, interactive=False)], outputs=[gr.Code(language=\"python\")])\n",
+    "demo.launch()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/lx/3ksh07r96gn2v7b8mb__3mpc0000gn/T/ipykernel_94012/4236222443.py:4: GradioDeprecationWarning: `layout` parameter is deprecated, and it has no effect\n",
+      "  demo = gr.Interface(\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7892\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7892/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 41,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def get_transformed_table(button):\n",
+    "    return template_df, PythonAstREPLTool(locals={'source_df': table_1_df}).run(transform_code)\n",
+    "check_mapping_text = 'How does that code look? \\n\\nWhen you are ready, click the Submit button and the transformed source file will be transformed.'\n",
+    "demo = gr.Interface(\n",
+    "    fn=get_transformed_table,\n",
+    "    inputs=[gr.Textbox(value=check_mapping_text, interactive=False)],\n",
+    "    outputs=[gr.Dataframe(label='Template Table (target)'), gr.Dataframe(label='Table 1 (transformed)')],\n",
+    "    layout=\"column\",\n",
+    "    examples=[[1]],\n",
+    ")\n",
+    "demo.launch()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 89,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/lx/3ksh07r96gn2v7b8mb__3mpc0000gn/T/ipykernel_94012/2180252060.py:18: GradioDeprecationWarning: Usage of gradio.inputs is deprecated, and will not be supported in the future, please import your component from gradio.components\n",
+      "  gr.inputs.File(label=\"Template\", type=\"file\", file_count='single')\n",
+      "/var/folders/lx/3ksh07r96gn2v7b8mb__3mpc0000gn/T/ipykernel_94012/2180252060.py:18: GradioDeprecationWarning: `optional` parameter is deprecated, and it has no effect\n",
+      "  gr.inputs.File(label=\"Template\", type=\"file\", file_count='single')\n",
+      "/var/folders/lx/3ksh07r96gn2v7b8mb__3mpc0000gn/T/ipykernel_94012/2180252060.py:18: GradioDeprecationWarning: `keep_filename` parameter is deprecated, and it has no effect\n",
+      "  gr.inputs.File(label=\"Template\", type=\"file\", file_count='single')\n",
+      "/var/folders/lx/3ksh07r96gn2v7b8mb__3mpc0000gn/T/ipykernel_94012/2180252060.py:19: GradioDeprecationWarning: Usage of gradio.inputs is deprecated, and will not be supported in the future, please import your component from gradio.components\n",
+      "  gr.inputs.File(label=\"Source\", type=\"file\", file_count='single')\n",
+      "/var/folders/lx/3ksh07r96gn2v7b8mb__3mpc0000gn/T/ipykernel_94012/2180252060.py:19: GradioDeprecationWarning: `optional` parameter is deprecated, and it has no effect\n",
+      "  gr.inputs.File(label=\"Source\", type=\"file\", file_count='single')\n",
+      "/var/folders/lx/3ksh07r96gn2v7b8mb__3mpc0000gn/T/ipykernel_94012/2180252060.py:19: GradioDeprecationWarning: `keep_filename` parameter is deprecated, and it has no effect\n",
+      "  gr.inputs.File(label=\"Source\", type=\"file\", file_count='single')\n",
+      "/Users/andybryant/Desktop/projects/zero-mapper/venv/lib/python3.9/site-packages/gradio/utils.py:841: UserWarning: Expected 1 arguments for function <function generate_code at 0x12cb559d0>, received 0.\n",
+      "  warnings.warn(\n",
+      "/Users/andybryant/Desktop/projects/zero-mapper/venv/lib/python3.9/site-packages/gradio/utils.py:845: UserWarning: Expected at least 1 arguments for function <function generate_code at 0x12cb559d0>, received 0.\n",
+      "  warnings.warn(\n",
+      "/var/folders/lx/3ksh07r96gn2v7b8mb__3mpc0000gn/T/ipykernel_94012/2180252060.py:39: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'trigger': 'transform_source'}\n",
+      "  gr.Button(value=\"Transform Source\", variant=\"primary\", trigger=\"transform_source\")\n",
+      "/var/folders/lx/3ksh07r96gn2v7b8mb__3mpc0000gn/T/ipykernel_94012/2180252060.py:40: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'trigger': 'save_code'}\n",
+      "  gr.Button(value=\"Save Code\", variant=\"secondary\", trigger=\"save_code\")\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7934\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7934/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 89,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def _sanitize_python_output(text: str):\n",
+    "    _, after = text.split(\"```python\")\n",
+    "    return after.split(\"```\")[0]\n",
+    "\n",
+    "def do_stuff(val):\n",
+    "    print(val)\n",
+    "\n",
+    "def generate_code(val):\n",
+    "    return '# check this out'\n",
+    "\n",
+    "def save_csv_file(df, filename):\n",
+    "    df.to_csv(os.path.join(data_dir_path, 'output', filename) + '.csv')\n",
+    "\n",
+    "with gr.Blocks() as demo:\n",
+    "    with gr.Column():\n",
+    "        gr.Markdown(\"## To begin, upload a Template CSV and a Source CSV file.\")\n",
+    "        with gr.Row():\n",
+    "            gr.inputs.File(label=\"Template\", type=\"file\", file_count='single')\n",
+    "            gr.inputs.File(label=\"Source\", type=\"file\", file_count='single')\n",
+    "\n",
+    "    with gr.Column():\n",
+    "        gr.Markdown(\"## Mapping from Source to Template\")\n",
+    "        with gr.Row():\n",
+    "            table_mapping_df = pd.DataFrame(table_mapping.dict()['table_mappings'])\n",
+    "            gr.DataFrame(value=table_mapping_df)\n",
+    "            save_mapping_btn = gr.Button(value=\"Save Mapping\", variant=\"secondary\")\n",
+    "            save_mapping_btn.click(fn=lambda : save_csv_file(table_mapping_df, 'table_mapping'))\n",
+    "\n",
+    "    with gr.Row():\n",
+    "        test = gr.Markdown()\n",
+    "        generate_code_btn = gr.Button(value=\"Generate Code from Mapping\", variant=\"primary\")\n",
+    "        generate_code_btn.click(fn=generate_code, outputs=test)\n",
+    "\n",
+    "    with gr.Column():\n",
+    "        gr.Markdown(\"## Here is the code that will be used to transform the source file into the template schema:\")\n",
+    "        gr.Code(language=\"python\", value=_sanitize_python_output(transform_code))\n",
+    "\n",
+    "    with gr.Row():\n",
+    "        gr.Button(value=\"Transform Source\", variant=\"primary\", trigger=\"transform_source\")\n",
+    "        gr.Button(value=\"Save Code\", variant=\"secondary\", trigger=\"save_code\")\n",
+    "    \n",
+    "    with gr.Row():\n",
+    "        with gr.Column():\n",
+    "            gr.Dataframe(label='Target (template)', type='pandas', value=template_df)\n",
+    "        with gr.Column():\n",
+    "            gr.Dataframe(label='Source (transformed)', type='pandas', value=PythonAstREPLTool(locals={'source_df': table_1_df}).run(transform_code))\n",
+    "\n",
+    "demo.launch()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/notebooks/data/legal_entries_a.csv

@@ -0,0 +1,101 @@
+case_date,lastname,firstname,case_type,case_id,court_fee,jurisdiction,judge_last_name
+2023-01-16,Okafor,Jane,Civil,CR-6190,250,BOSTON,Connor
+2023-08-10,Malcolm,Elena,Civil,CR-3092,100,chicago,James
+2023-06-14,Nasser,Alan,Civil,CR-5947,150,BOSTON,Skaarsgard
+2023-07-17,Smith,Miguel,Family,CR-7727,250,LOS angeles,Brown
+2023-07-25,Kim,John,Criminal,CR-4120,150,BOSTON,Skaarsgard
+2023-07-14,Brown,John,Civil,CR-8850,100,LOS angeles,Brown
+2023-01-19,Nasser,Dmitri,Criminal,CR-2308,100,chicago,Connor
+2023-02-26,Rodriguez,Alan,Criminal,CR-4477,100,chicago,Morgan
+2023-02-10,Brown,Alice,Criminal,CR-9490,200,chicago,Morgan
+2023-09-12,Smith,Nadia,Family,CR-4111,100,LOS angeles,Skaarsgard
+2023-02-25,Kim,Chen,Criminal,CR-9421,150,BOSTON,Oleg
+2023-09-15,Kim,John,Family,CR-3270,200,houston,Morgan
+2023-07-22,Patel,Nadia,Family,CR-1501,200,houston,Skaarsgard
+2023-01-27,Lee,Lakshmi,Family,CR-8321,150,houston,Brown
+2023-01-14,Brown,John,Family,CR-2748,100,LOS angeles,James
+2023-07-13,Malcolm,Miguel,Family,CR-3163,100,LOS angeles,Skaarsgard
+2023-02-26,Smith,Alice,Civil,CR-4296,150,BOSTON,James
+2023-09-25,Patel,Terrance,Criminal,CR-2230,200,houston,Morgan
+2023-02-13,Ivanov,Alan,Family,CR-9353,100,new York,Morgan
+2023-04-18,Chatterjee,Alice,Civil,CR-8786,100,chicago,Skaarsgard
+2023-09-11,Brown,Jane,Criminal,CR-6001,100,LOS angeles,Connor
+2023-02-16,Okafor,Jane,Criminal,CR-9434,250,BOSTON,Oleg
+2023-07-22,Chatterjee,Dmitri,Criminal,CR-1042,100,BOSTON,Brown
+2023-08-28,Smith,Miguel,Family,CR-1427,150,LOS angeles,Brown
+2023-06-14,Johnson,Miguel,Civil,CR-7553,200,chicago,Skaarsgard
+2023-02-24,Ivanov,Chen,Civil,CR-2242,250,LOS angeles,Connor
+2023-06-23,Rodriguez,Terrance,Criminal,CR-6940,250,houston,Connor
+2023-01-10,Johnson,Elena,Civil,CR-4064,150,houston,Oleg
+2023-01-15,Patel,Chen,Civil,CR-3129,100,new York,Morgan
+2023-08-16,Malcolm,Oluwaseun,Civil,CR-2758,150,BOSTON,Connor
+2023-02-24,Ivanov,Lakshmi,Criminal,CR-9562,250,BOSTON,Brown
+2023-05-15,Okafor,Terrance,Criminal,CR-2292,250,BOSTON,Morgan
+2023-06-26,Patel,Jane,Criminal,CR-7889,250,LOS angeles,Brown
+2023-02-14,Rodriguez,John,Family,CR-5178,150,houston,Morgan
+2023-05-15,Patel,Terrance,Civil,CR-5004,150,houston,Morgan
+2023-03-19,Johnson,Alice,Family,CR-2883,200,new York,Morgan
+2023-02-12,Rodriguez,Alan,Family,CR-4416,200,BOSTON,Brown
+2023-07-25,Malcolm,Chen,Civil,CR-9332,200,houston,Morgan
+2023-09-15,Chatterjee,Miguel,Civil,CR-7699,250,BOSTON,Skaarsgard
+2023-03-13,Lee,Nadia,Civil,CR-7258,100,new York,James
+2023-05-27,Brown,Nadia,Civil,CR-7490,200,houston,Morgan
+2023-02-22,Johnson,Alice,Civil,CR-8231,100,chicago,Connor
+2023-03-18,Malcolm,Nadia,Criminal,CR-2720,100,new York,Oleg
+2023-06-11,Brown,Nadia,Criminal,CR-4277,100,BOSTON,Connor
+2023-02-22,Okafor,Oluwaseun,Criminal,CR-9738,100,new York,Skaarsgard
+2023-08-19,Patel,Jane,Civil,CR-2452,250,BOSTON,Skaarsgard
+2023-09-27,Lee,Alan,Family,CR-1899,100,new York,Skaarsgard
+2023-04-21,Malcolm,Dmitri,Family,CR-8404,150,LOS angeles,Brown
+2023-03-10,Chatterjee,Alice,Family,CR-4240,100,LOS angeles,Morgan
+2023-05-13,Kim,Elena,Family,CR-6153,250,chicago,Brown
+2023-09-10,Patel,Alan,Criminal,CR-3485,200,chicago,Oleg
+2023-08-18,Kim,Lakshmi,Criminal,CR-5520,200,LOS angeles,James
+2023-02-21,Patel,Alan,Criminal,CR-9879,250,LOS angeles,Brown
+2023-05-12,Brown,Jane,Criminal,CR-5259,200,new York,James
+2023-01-20,Patel,Oluwaseun,Criminal,CR-8333,100,BOSTON,James
+2023-01-23,Nasser,Chen,Civil,CR-2711,200,LOS angeles,Morgan
+2023-03-12,Brown,Miguel,Family,CR-5100,100,LOS angeles,Morgan
+2023-01-15,Rodriguez,Terrance,Criminal,CR-4849,100,LOS angeles,Morgan
+2023-05-17,Lee,Jane,Criminal,CR-8058,150,new York,Skaarsgard
+2023-04-18,Okafor,Chen,Civil,CR-9076,100,new York,Brown
+2023-02-22,Chatterjee,Lakshmi,Criminal,CR-5230,200,BOSTON,Morgan
+2023-08-18,Brown,John,Criminal,CR-7094,200,LOS angeles,Connor
+2023-08-17,Lee,Oluwaseun,Civil,CR-8915,150,BOSTON,Oleg
+2023-08-18,Malcolm,Alan,Family,CR-9030,100,chicago,Brown
+2023-02-13,Malcolm,Chen,Criminal,CR-1482,150,houston,Morgan
+2023-02-16,Brown,John,Criminal,CR-3535,100,BOSTON,Morgan
+2023-08-20,Johnson,Chen,Criminal,CR-2029,250,houston,James
+2023-01-10,Kim,Alan,Civil,CR-1812,250,houston,Oleg
+2023-02-18,Chatterjee,Alice,Civil,CR-5295,150,chicago,Oleg
+2023-08-25,Lee,Miguel,Criminal,CR-6850,150,LOS angeles,Brown
+2023-05-12,Malcolm,Alan,Criminal,CR-7973,150,BOSTON,Connor
+2023-05-19,Johnson,Chen,Family,CR-5221,200,houston,Skaarsgard
+2023-06-17,Okafor,John,Criminal,CR-4117,250,BOSTON,Oleg
+2023-03-18,Patel,Elena,Family,CR-2368,100,houston,Skaarsgard
+2023-06-22,Rodriguez,Lakshmi,Family,CR-8384,200,new York,Oleg
+2023-07-14,Smith,Miguel,Civil,CR-4476,100,new York,Morgan
+2023-03-26,Brown,Chen,Civil,CR-4545,100,houston,Morgan
+2023-06-22,Chatterjee,Dmitri,Civil,CR-4421,250,houston,Skaarsgard
+2023-03-20,Patel,Miguel,Criminal,CR-6559,150,new York,Morgan
+2023-07-11,Kim,Oluwaseun,Civil,CR-1803,250,BOSTON,Brown
+2023-03-13,Okafor,Elena,Civil,CR-8622,150,new York,Brown
+2023-05-27,Lee,Alice,Criminal,CR-9488,200,LOS angeles,Connor
+2023-05-14,Patel,Alice,Civil,CR-4581,150,chicago,Brown
+2023-06-27,Malcolm,Terrance,Criminal,CR-2388,250,chicago,James
+2023-02-13,Ivanov,Terrance,Criminal,CR-6529,100,LOS angeles,Connor
+2023-01-21,Patel,Terrance,Family,CR-4443,150,BOSTON,Oleg
+2023-09-22,Malcolm,John,Civil,CR-8721,200,new York,Skaarsgard
+2023-02-12,Malcolm,Miguel,Family,CR-3780,250,new York,Oleg
+2023-04-26,Kim,Alan,Criminal,CR-2663,250,new York,Connor
+2023-03-16,Ivanov,Lakshmi,Criminal,CR-8702,150,LOS angeles,Morgan
+2023-07-22,Ivanov,Jane,Criminal,CR-1232,100,BOSTON,Connor
+2023-05-28,Okafor,Nadia,Family,CR-5215,150,houston,Brown
+2023-03-14,Okafor,Oluwaseun,Criminal,CR-6631,250,BOSTON,Skaarsgard
+2023-06-17,Nasser,Alan,Civil,CR-1405,150,BOSTON,Morgan
+2023-08-13,Kim,Oluwaseun,Civil,CR-8816,100,LOS angeles,James
+2023-07-20,Brown,Oluwaseun,Family,CR-2665,150,new York,Morgan
+2023-05-16,Patel,Alan,Family,CR-2874,100,new York,Connor
+2023-07-15,Chatterjee,Nadia,Family,CR-2037,100,houston,James
+2023-04-18,Johnson,Dmitri,Criminal,CR-5402,200,houston,Morgan
+2023-08-14,Johnson,Chen,Civil,CR-3569,250,BOSTON,Morgan

src/notebooks/data/legal_entries_b.csv

@@ -0,0 +1,101 @@
+Date_of_Case,Fee,FullName,CaseNumber,CaseKind,Location,Weather
+2023/01/16,250.0,Jane Okafor,case--6190,Civil,BOSTO,snowy
+2023/08/10,100.0,Elena Malcolm,case--3092,Civil,CHIC,sunny
+2023/06/14,150.0,Alan Nasser,case--5947,Civil,BOSTO,rainy
+2023/07/17,250.0,Miguel Smith,case--7727,Family,LOSAN,snowy
+2023/07/25,150.0,John Kim,case--4120,Criminal,BOSTO,rainy
+2023/07/14,100.0,John Brown,case--8850,Civil,LOSAN,snowy
+2023/01/19,100.0,Dmitri Nasser,case--2308,Criminal,CHIC,sunny
+2023/02/26,100.0,Alan Rodriguez,case--4477,Criminal,CHIC,cloudy
+2023/02/10,200.0,Alice Brown,case--9490,Criminal,CHICA,snowy
+2023/09/12,100.0,Nadia Smith,case--4111,Family,LOSA,sunny
+2023/02/25,150.0,Chen Kim,case--9421,Criminal,BOST,sunny
+2023/09/15,200.0,John Kim,case--3270,Family,HOUST,sunny
+2023/07/22,200.0,Nadia Patel,case--1501,Family,HOUS,cloudy
+2023/01/27,150.0,Lakshmi Lee,case--8321,Family,HOUST,snowy
+2023/01/14,100.0,John Brown,case--2748,Family,LOSAN,snowy
+2023/07/13,100.0,Miguel Malcolm,case--3163,Family,LOSA,rainy
+2023/02/26,150.0,Alice Smith,case--4296,Civil,BOSTO,cloudy
+2023/09/25,200.0,Terrance Patel,case--2230,Criminal,HOUS,snowy
+2023/02/13,100.0,Alan Ivanov,case--9353,Family,NEWY,sunny
+2023/04/18,100.0,Alice Chatterjee,case--8786,Civil,CHICA,rainy
+2023/09/11,100.0,Jane Brown,case--6001,Criminal,LOSA,snowy
+2023/02/16,250.0,Jane Okafor,case--9434,Criminal,BOST,snowy
+2023/07/22,100.0,Dmitri Chatterjee,case--1042,Criminal,BOST,rainy
+2023/08/28,150.0,Miguel Smith,case--1427,Family,LOSA,cloudy
+2023/06/14,200.0,Miguel Johnson,case--7553,Civil,CHIC,sunny
+2023/02/24,250.0,Chen Ivanov,case--2242,Civil,LOSA,rainy
+2023/06/23,250.0,Terrance Rodriguez,case--6940,Criminal,HOUST,rainy
+2023/01/10,150.0,Elena Johnson,case--4064,Civil,HOUS,rainy
+2023/01/15,100.0,Chen Patel,case--3129,Civil,NEWY,rainy
+2023/08/16,150.0,Oluwaseun Malcolm,case--2758,Civil,BOSTO,snowy
+2023/02/24,250.0,Lakshmi Ivanov,case--9562,Criminal,BOSTO,sunny
+2023/05/15,250.0,Terrance Okafor,case--2292,Criminal,BOST,rainy
+2023/06/26,250.0,Jane Patel,case--7889,Criminal,LOSAN,cloudy
+2023/02/14,150.0,John Rodriguez,case--5178,Family,HOUS,sunny
+2023/05/15,150.0,Terrance Patel,case--5004,Civil,HOUST,snowy
+2023/03/19,200.0,Alice Johnson,case--2883,Family,NEWYO,snowy
+2023/02/12,200.0,Alan Rodriguez,case--4416,Family,BOSTO,rainy
+2023/07/25,200.0,Chen Malcolm,case--9332,Civil,HOUS,snowy
+2023/09/15,250.0,Miguel Chatterjee,case--7699,Civil,BOST,rainy
+2023/03/13,100.0,Nadia Lee,case--7258,Civil,NEWYO,snowy
+2023/05/27,200.0,Nadia Brown,case--7490,Civil,HOUS,snowy
+2023/02/22,100.0,Alice Johnson,case--8231,Civil,CHIC,cloudy
+2023/03/18,100.0,Nadia Malcolm,case--2720,Criminal,NEWY,cloudy
+2023/06/11,100.0,Nadia Brown,case--4277,Criminal,BOST,snowy
+2023/02/22,100.0,Oluwaseun Okafor,case--9738,Criminal,NEWYO,snowy
+2023/08/19,250.0,Jane Patel,case--2452,Civil,BOSTO,snowy
+2023/09/27,100.0,Alan Lee,case--1899,Family,NEWY,rainy
+2023/04/21,150.0,Dmitri Malcolm,case--8404,Family,LOSAN,rainy
+2023/03/10,100.0,Alice Chatterjee,case--4240,Family,LOSA,snowy
+2023/05/13,250.0,Elena Kim,case--6153,Family,CHIC,rainy
+2023/09/10,200.0,Alan Patel,case--3485,Criminal,CHIC,cloudy
+2023/08/18,200.0,Lakshmi Kim,case--5520,Criminal,LOSAN,sunny
+2023/02/21,250.0,Alan Patel,case--9879,Criminal,LOSA,sunny
+2023/05/12,200.0,Jane Brown,case--5259,Criminal,NEWYO,rainy
+2023/01/20,100.0,Oluwaseun Patel,case--8333,Criminal,BOSTO,cloudy
+2023/01/23,200.0,Chen Nasser,case--2711,Civil,LOSAN,sunny
+2023/03/12,100.0,Miguel Brown,case--5100,Family,LOSAN,sunny
+2023/01/15,100.0,Terrance Rodriguez,case--4849,Criminal,LOSAN,rainy
+2023/05/17,150.0,Jane Lee,case--8058,Criminal,NEWY,cloudy
+2023/04/18,100.0,Chen Okafor,case--9076,Civil,NEWYO,sunny
+2023/02/22,200.0,Lakshmi Chatterjee,case--5230,Criminal,BOST,rainy
+2023/08/18,200.0,John Brown,case--7094,Criminal,LOSA,cloudy
+2023/08/17,150.0,Oluwaseun Lee,case--8915,Civil,BOSTO,sunny
+2023/08/18,100.0,Alan Malcolm,case--9030,Family,CHIC,sunny
+2023/02/13,150.0,Chen Malcolm,case--1482,Criminal,HOUS,cloudy
+2023/02/16,100.0,John Brown,case--3535,Criminal,BOST,rainy
+2023/08/20,250.0,Chen Johnson,case--2029,Criminal,HOUST,sunny
+2023/01/10,250.0,Alan Kim,case--1812,Civil,HOUST,sunny
+2023/02/18,150.0,Alice Chatterjee,case--5295,Civil,CHICA,snowy
+2023/08/25,150.0,Miguel Lee,case--6850,Criminal,LOSA,sunny
+2023/05/12,150.0,Alan Malcolm,case--7973,Criminal,BOST,cloudy
+2023/05/19,200.0,Chen Johnson,case--5221,Family,HOUS,snowy
+2023/06/17,250.0,John Okafor,case--4117,Criminal,BOSTO,sunny
+2023/03/18,100.0,Elena Patel,case--2368,Family,HOUST,rainy
+2023/06/22,200.0,Lakshmi Rodriguez,case--8384,Family,NEWY,cloudy
+2023/07/14,100.0,Miguel Smith,case--4476,Civil,NEWYO,cloudy
+2023/03/26,100.0,Chen Brown,case--4545,Civil,HOUST,snowy
+2023/06/22,250.0,Dmitri Chatterjee,case--4421,Civil,HOUS,snowy
+2023/03/20,150.0,Miguel Patel,case--6559,Criminal,NEWYO,snowy
+2023/07/11,250.0,Oluwaseun Kim,case--1803,Civil,BOSTO,sunny
+2023/03/13,150.0,Elena Okafor,case--8622,Civil,NEWYO,cloudy
+2023/05/27,200.0,Alice Lee,case--9488,Criminal,LOSAN,cloudy
+2023/05/14,150.0,Alice Patel,case--4581,Civil,CHICA,sunny
+2023/06/27,250.0,Terrance Malcolm,case--2388,Criminal,CHIC,sunny
+2023/02/13,100.0,Terrance Ivanov,case--6529,Criminal,LOSAN,snowy
+2023/01/21,150.0,Terrance Patel,case--4443,Family,BOST,sunny
+2023/09/22,200.0,John Malcolm,case--8721,Civil,NEWYO,snowy
+2023/02/12,250.0,Miguel Malcolm,case--3780,Family,NEWYO,cloudy
+2023/04/26,250.0,Alan Kim,case--2663,Criminal,NEWY,rainy
+2023/03/16,150.0,Lakshmi Ivanov,case--8702,Criminal,LOSA,snowy
+2023/07/22,100.0,Jane Ivanov,case--1232,Criminal,BOSTO,rainy
+2023/05/28,150.0,Nadia Okafor,case--5215,Family,HOUS,cloudy
+2023/03/14,250.0,Oluwaseun Okafor,case--6631,Criminal,BOST,rainy
+2023/06/17,150.0,Alan Nasser,case--1405,Civil,BOST,snowy
+2023/08/13,100.0,Oluwaseun Kim,case--8816,Civil,LOSAN,cloudy
+2023/07/20,150.0,Oluwaseun Brown,case--2665,Family,NEWYO,sunny
+2023/05/16,100.0,Alan Patel,case--2874,Family,NEWYO,sunny
+2023/07/15,100.0,Nadia Chatterjee,case--2037,Family,HOUST,rainy
+2023/04/18,200.0,Dmitri Johnson,case--5402,Criminal,HOUS,snowy
+2023/08/14,250.0,Chen Johnson,case--3569,Civil,BOST,sunny

src/notebooks/data/legal_template.csv

@@ -0,0 +1,101 @@
+CaseDate,FullName,CaseType,CaseID,Fee,Jurisdiction
+2023-01-16,Jane Okafor,Civil,CASE-6190,250,Boston
+2023-08-10,Elena Malcolm,Civil,CASE-3092,100,Chicago
+2023-06-14,Alan Nasser,Civil,CASE-5947,150,Boston
+2023-07-17,Miguel Smith,Family,CASE-7727,250,Los Angeles
+2023-07-25,John Kim,Criminal,CASE-4120,150,Boston
+2023-07-14,John Brown,Civil,CASE-8850,100,Los Angeles
+2023-01-19,Dmitri Nasser,Criminal,CASE-2308,100,Chicago
+2023-02-26,Alan Rodriguez,Criminal,CASE-4477,100,Chicago
+2023-02-10,Alice Brown,Criminal,CASE-9490,200,Chicago
+2023-09-12,Nadia Smith,Family,CASE-4111,100,Los Angeles
+2023-02-25,Chen Kim,Criminal,CASE-9421,150,Boston
+2023-09-15,John Kim,Family,CASE-3270,200,Houston
+2023-07-22,Nadia Patel,Family,CASE-1501,200,Houston
+2023-01-27,Lakshmi Lee,Family,CASE-8321,150,Houston
+2023-01-14,John Brown,Family,CASE-2748,100,Los Angeles
+2023-07-13,Miguel Malcolm,Family,CASE-3163,100,Los Angeles
+2023-02-26,Alice Smith,Civil,CASE-4296,150,Boston
+2023-09-25,Terrance Patel,Criminal,CASE-2230,200,Houston
+2023-02-13,Alan Ivanov,Family,CASE-9353,100,New York
+2023-04-18,Alice Chatterjee,Civil,CASE-8786,100,Chicago
+2023-09-11,Jane Brown,Criminal,CASE-6001,100,Los Angeles
+2023-02-16,Jane Okafor,Criminal,CASE-9434,250,Boston
+2023-07-22,Dmitri Chatterjee,Criminal,CASE-1042,100,Boston
+2023-08-28,Miguel Smith,Family,CASE-1427,150,Los Angeles
+2023-06-14,Miguel Johnson,Civil,CASE-7553,200,Chicago
+2023-02-24,Chen Ivanov,Civil,CASE-2242,250,Los Angeles
+2023-06-23,Terrance Rodriguez,Criminal,CASE-6940,250,Houston
+2023-01-10,Elena Johnson,Civil,CASE-4064,150,Houston
+2023-01-15,Chen Patel,Civil,CASE-3129,100,New York
+2023-08-16,Oluwaseun Malcolm,Civil,CASE-2758,150,Boston
+2023-02-24,Lakshmi Ivanov,Criminal,CASE-9562,250,Boston
+2023-05-15,Terrance Okafor,Criminal,CASE-2292,250,Boston
+2023-06-26,Jane Patel,Criminal,CASE-7889,250,Los Angeles
+2023-02-14,John Rodriguez,Family,CASE-5178,150,Houston
+2023-05-15,Terrance Patel,Civil,CASE-5004,150,Houston
+2023-03-19,Alice Johnson,Family,CASE-2883,200,New York
+2023-02-12,Alan Rodriguez,Family,CASE-4416,200,Boston
+2023-07-25,Chen Malcolm,Civil,CASE-9332,200,Houston
+2023-09-15,Miguel Chatterjee,Civil,CASE-7699,250,Boston
+2023-03-13,Nadia Lee,Civil,CASE-7258,100,New York
+2023-05-27,Nadia Brown,Civil,CASE-7490,200,Houston
+2023-02-22,Alice Johnson,Civil,CASE-8231,100,Chicago
+2023-03-18,Nadia Malcolm,Criminal,CASE-2720,100,New York
+2023-06-11,Nadia Brown,Criminal,CASE-4277,100,Boston
+2023-02-22,Oluwaseun Okafor,Criminal,CASE-9738,100,New York
+2023-08-19,Jane Patel,Civil,CASE-2452,250,Boston
+2023-09-27,Alan Lee,Family,CASE-1899,100,New York
+2023-04-21,Dmitri Malcolm,Family,CASE-8404,150,Los Angeles
+2023-03-10,Alice Chatterjee,Family,CASE-4240,100,Los Angeles
+2023-05-13,Elena Kim,Family,CASE-6153,250,Chicago
+2023-09-10,Alan Patel,Criminal,CASE-3485,200,Chicago
+2023-08-18,Lakshmi Kim,Criminal,CASE-5520,200,Los Angeles
+2023-02-21,Alan Patel,Criminal,CASE-9879,250,Los Angeles
+2023-05-12,Jane Brown,Criminal,CASE-5259,200,New York
+2023-01-20,Oluwaseun Patel,Criminal,CASE-8333,100,Boston
+2023-01-23,Chen Nasser,Civil,CASE-2711,200,Los Angeles
+2023-03-12,Miguel Brown,Family,CASE-5100,100,Los Angeles
+2023-01-15,Terrance Rodriguez,Criminal,CASE-4849,100,Los Angeles
+2023-05-17,Jane Lee,Criminal,CASE-8058,150,New York
+2023-04-18,Chen Okafor,Civil,CASE-9076,100,New York
+2023-02-22,Lakshmi Chatterjee,Criminal,CASE-5230,200,Boston
+2023-08-18,John Brown,Criminal,CASE-7094,200,Los Angeles
+2023-08-17,Oluwaseun Lee,Civil,CASE-8915,150,Boston
+2023-08-18,Alan Malcolm,Family,CASE-9030,100,Chicago
+2023-02-13,Chen Malcolm,Criminal,CASE-1482,150,Houston
+2023-02-16,John Brown,Criminal,CASE-3535,100,Boston
+2023-08-20,Chen Johnson,Criminal,CASE-2029,250,Houston
+2023-01-10,Alan Kim,Civil,CASE-1812,250,Houston
+2023-02-18,Alice Chatterjee,Civil,CASE-5295,150,Chicago
+2023-08-25,Miguel Lee,Criminal,CASE-6850,150,Los Angeles
+2023-05-12,Alan Malcolm,Criminal,CASE-7973,150,Boston
+2023-05-19,Chen Johnson,Family,CASE-5221,200,Houston
+2023-06-17,John Okafor,Criminal,CASE-4117,250,Boston
+2023-03-18,Elena Patel,Family,CASE-2368,100,Houston
+2023-06-22,Lakshmi Rodriguez,Family,CASE-8384,200,New York
+2023-07-14,Miguel Smith,Civil,CASE-4476,100,New York
+2023-03-26,Chen Brown,Civil,CASE-4545,100,Houston
+2023-06-22,Dmitri Chatterjee,Civil,CASE-4421,250,Houston
+2023-03-20,Miguel Patel,Criminal,CASE-6559,150,New York
+2023-07-11,Oluwaseun Kim,Civil,CASE-1803,250,Boston
+2023-03-13,Elena Okafor,Civil,CASE-8622,150,New York
+2023-05-27,Alice Lee,Criminal,CASE-9488,200,Los Angeles
+2023-05-14,Alice Patel,Civil,CASE-4581,150,Chicago
+2023-06-27,Terrance Malcolm,Criminal,CASE-2388,250,Chicago
+2023-02-13,Terrance Ivanov,Criminal,CASE-6529,100,Los Angeles
+2023-01-21,Terrance Patel,Family,CASE-4443,150,Boston
+2023-09-22,John Malcolm,Civil,CASE-8721,200,New York
+2023-02-12,Miguel Malcolm,Family,CASE-3780,250,New York
+2023-04-26,Alan Kim,Criminal,CASE-2663,250,New York
+2023-03-16,Lakshmi Ivanov,Criminal,CASE-8702,150,Los Angeles
+2023-07-22,Jane Ivanov,Criminal,CASE-1232,100,Boston
+2023-05-28,Nadia Okafor,Family,CASE-5215,150,Houston
+2023-03-14,Oluwaseun Okafor,Criminal,CASE-6631,250,Boston
+2023-06-17,Alan Nasser,Civil,CASE-1405,150,Boston
+2023-08-13,Oluwaseun Kim,Civil,CASE-8816,100,Los Angeles
+2023-07-20,Oluwaseun Brown,Family,CASE-2665,150,New York
+2023-05-16,Alan Patel,Family,CASE-2874,100,New York
+2023-07-15,Nadia Chatterjee,Family,CASE-2037,100,Houston
+2023-04-18,Dmitri Johnson,Criminal,CASE-5402,200,Houston
+2023-08-14,Chen Johnson,Civil,CASE-3569,250,Boston

src/notebooks/data/output/table_mapping

@@ -0,0 +1,9 @@
+,source_column_name,target_column_name,value_transformations,explanation
+0,case_date,CaseDate,NO_TRANSFORM,The 'case_date' column in the source table maps directly to the 'CaseDate' column in the target table with no transformation needed.
+1,lastname,FullName,"Concatenate 'lastname' and 'firstname' from source table, with a space in between, and reverse the order.",The 'lastname' and 'firstname' columns in the source table are combined and reversed to form the 'FullName' column in the target table.
+2,firstname,FullName,"Concatenate 'lastname' and 'firstname' from source table, with a space in between, and reverse the order.",The 'lastname' and 'firstname' columns in the source table are combined and reversed to form the 'FullName' column in the target table.
+3,case_type,CaseType,"Correct spelling errors in source table ('Familly' to 'Family', 'Criminl' to 'Criminal', 'Civl' to 'Civil').","The 'case_type' column in the source table maps to the 'CaseType' column in the target table, with spelling corrections needed."
+4,case_id,CaseID,Replace 'CR-' prefix in source table with 'CASE-' prefix.,"The 'case_id' column in the source table maps to the 'CaseID' column in the target table, with a prefix change needed."
+5,court_fee,Fee,NO_TRANSFORM,The 'court_fee' column in the source table maps directly to the 'Fee' column in the target table with no transformation needed.
+6,jurisdiction,Jurisdiction,Capitalize the first letter of each word in the 'jurisdiction' column of the source table.,"The 'jurisdiction' column in the source table maps to the 'Jurisdiction' column in the target table, with capitalization needed."
+7,judge_last_name,NO_TARGET,NO_TRANSFORM,The 'judge_last_name' column in the source table does not have a corresponding column in the target table and can be ignored.

src/notebooks/data/output/table_mapping.csv

@@ -0,0 +1,9 @@
+,source_column_name,target_column_name,value_transformations,explanation
+0,case_date,CaseDate,NO_TRANSFORM,The 'case_date' column in the source table maps directly to the 'CaseDate' column in the target table with no transformation needed.
+1,lastname,FullName,"Concatenate 'lastname' and 'firstname' from source table, with a space in between, and reverse the order.",The 'lastname' and 'firstname' columns in the source table are combined and reversed to form the 'FullName' column in the target table.
+2,firstname,FullName,"Concatenate 'lastname' and 'firstname' from source table, with a space in between, and reverse the order.",The 'lastname' and 'firstname' columns in the source table are combined and reversed to form the 'FullName' column in the target table.
+3,case_type,CaseType,"Correct spelling errors in source table ('Familly' to 'Family', 'Criminl' to 'Criminal', 'Civl' to 'Civil').","The 'case_type' column in the source table maps to the 'CaseType' column in the target table, with spelling corrections needed."
+4,case_id,CaseID,Replace 'CR-' prefix in source table with 'CASE-' prefix.,"The 'case_id' column in the source table maps to the 'CaseID' column in the target table, with a prefix change needed."
+5,court_fee,Fee,NO_TRANSFORM,The 'court_fee' column in the source table maps directly to the 'Fee' column in the target table with no transformation needed.
+6,jurisdiction,Jurisdiction,Capitalize the first letter of each word in the 'jurisdiction' column of the source table.,"The 'jurisdiction' column in the source table maps to the 'Jurisdiction' column in the target table, with capitalization needed."
+7,judge_last_name,NO_TARGET,NO_TRANSFORM,The 'judge_last_name' column in the source table does not have a corresponding column in the target table and can be ignored.

src/notebooks/generate.ipynb

@@ -0,0 +1,120 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import random\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "num_records = 100\n",
+    "first_names = [\"Alan\", \"Miguel\", \"Lakshmi\", \"Chen\", \"Oluwaseun\", \"Dmitri\", \"Nadia\", \"John\", \"Jane\", \"Alice\", \"Terrance\", \"Elena\"]\n",
+    "last_names = [\"Patel\", \"Rodriguez\", \"Kim\", \"Okafor\", \"Nasser\", \"Ivanov\", \"Smith\", \"Brown\", \"Johnson\", \"Lee\", \"Malcolm\", \"Chatterjee\"]\n",
+    "# new last names\n",
+    "judge_last_names = [\"James\", \"Skaarsgard\", \"Oleg\", \"Morgan\", \"Brown\", \"Connor\"]\n",
+    "case_types = [\"Criminal\", \"Civil\", \"Familly\", \"Criminl\", \"Civl\", \"Family\"]\n",
+    "cities = ['new York', 'LOS angeles', 'chicago', 'houston', 'BOSTON']\n",
+    "weather_types = ['sunny', 'cloudy', 'rainy', 'snowy']\n",
+    "get_random_date = lambda: f'2023-0{random.randint(1,9)}-{random.randint(10,28)}'\n",
+    "get_random_case_number = lambda: f'{random.choice([\"CR\", \"CASE\"])}-{random.randint(1000,9999)}'\n",
+    "get_random_fee = lambda: random.choice([100, 150, 200, 250])\n",
+    "\n",
+    "get_random_int = lambda: random.randint(1000, 9999)\n",
+    "\n",
+    "# Define column names with standard casing\n",
+    "case_date_col = 'case_date'\n",
+    "lastname_col = 'lastname'\n",
+    "firstname_col = 'firstname'\n",
+    "case_type_col = 'case_type'\n",
+    "case_id_col = 'case_id'\n",
+    "court_fee_col = 'court_fee'\n",
+    "jurisdiction_col = 'jurisdiction'\n",
+    "case_id_prefix = 'CR'\n",
+    "\n",
+    "# Use inconsistent casing for values\n",
+    "legal_entries_a_data = {\n",
+    "    case_date_col: [get_random_date() for _ in range(num_records)],\n",
+    "    lastname_col: [random.choice(last_names) for _ in range(num_records)],\n",
+    "    firstname_col: [random.choice(first_names) for _ in range(num_records)],\n",
+    "    case_type_col: [random.choice(case_types) for _ in range(num_records)],\n",
+    "    case_id_col: [f'{case_id_prefix}-{get_random_int()}' for _ in range(num_records)],\n",
+    "    court_fee_col: [get_random_fee() for _ in range(num_records)],\n",
+    "    jurisdiction_col: [random.choice(cities) for _ in range(num_records)],\n",
+    "    'judge_last_name': [random.choice(judge_last_names) for _ in range(num_records)]\n",
+    "}\n",
+    "\n",
+    "legal_entries_a = pd.DataFrame(legal_entries_a_data)\n",
+    "\n",
+    "# Apply the transform_row function for legal_entries_A to create legal_entries_B\n",
+    "def transform_row(row):\n",
+    "    return pd.Series({\n",
+    "        'Date_of_Case' : row[case_date_col].replace(\"-\", \"/\"),\n",
+    "        'Fee' : float(row[court_fee_col]),\n",
+    "        'FullName' : f\"{row[firstname_col]} {row[lastname_col]}\".title(),\n",
+    "        'CaseNumber' : f'{row[case_id_col].replace(case_id_prefix, \"case-\")}',\n",
+    "        'CaseKind' : row[case_type_col].capitalize(),\n",
+    "        'Date_of_Case' : row[case_date_col].replace(\"-\", \"/\"),\n",
+    "        'Location' : row[jurisdiction_col].replace(\" \", \"\")[:random.randint(4,5)].upper(),\n",
+    "        'Weather': random.choice(weather_types)\n",
+    "    })\n",
+    "\n",
+    "def transform_to_template(row):\n",
+    "    return pd.Series({\n",
+    "        'CaseDate': row[case_date_col],\n",
+    "        'FullName': f\"{row[firstname_col]} {row[lastname_col]}\",\n",
+    "        'CaseType': \"Family\" if \"Fam\" in row[case_type_col] else \"Civil\" if 'Civ' in row[case_type_col] else row[case_type_col].capitalize(),\n",
+    "        'CaseID': f'{row[case_id_col].replace(case_id_prefix, \"CASE\")}',\n",
+    "        'Fee': row[court_fee_col],\n",
+    "        'Jurisdiction': row[jurisdiction_col].title()\n",
+    "    })\n",
+    "\n",
+    "legal_entries_b = legal_entries_a.apply(transform_row, axis=1)\n",
+    "legal_template = legal_entries_a.apply(transform_to_template, axis=1)\n",
+    "\n",
+    "data_dir_path = os.path.join(os.getcwd(), \"data\")\n",
+    "legal_entries_a.to_csv(os.path.join(data_dir_path, \"legal_entries_a.csv\"), index=False)\n",
+    "legal_entries_b.to_csv(os.path.join(data_dir_path, \"legal_entries_b.csv\"), index=False)\n",
+    "legal_template.to_csv(os.path.join(data_dir_path, \"legal_template.csv\"), index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/notebooks/table_mapping

@@ -0,0 +1,9 @@
+,source_column_name,target_column_name,value_transformations,explanation
+0,case_date,CaseDate,NO_TRANSFORM,The 'case_date' column in the source table maps directly to the 'CaseDate' column in the target table with no transformation needed.
+1,lastname,FullName,"Concatenate 'lastname' and 'firstname' from source table, with a space in between, and reverse the order.",The 'lastname' and 'firstname' columns in the source table are combined and reversed to form the 'FullName' column in the target table.
+2,firstname,FullName,"Concatenate 'lastname' and 'firstname' from source table, with a space in between, and reverse the order.",The 'lastname' and 'firstname' columns in the source table are combined and reversed to form the 'FullName' column in the target table.
+3,case_type,CaseType,"Correct spelling errors in source table ('Familly' to 'Family', 'Criminl' to 'Criminal', 'Civl' to 'Civil').","The 'case_type' column in the source table maps to the 'CaseType' column in the target table, with spelling corrections needed."
+4,case_id,CaseID,Replace 'CR-' prefix in source table with 'CASE-' prefix.,"The 'case_id' column in the source table maps to the 'CaseID' column in the target table, with a prefix change needed."
+5,court_fee,Fee,NO_TRANSFORM,The 'court_fee' column in the source table maps directly to the 'Fee' column in the target table with no transformation needed.
+6,jurisdiction,Jurisdiction,Capitalize the first letter of each word in the 'jurisdiction' column of the source table.,"The 'jurisdiction' column in the source table maps to the 'Jurisdiction' column in the target table, with capitalization needed."
+7,judge_last_name,NO_TARGET,NO_TRANSFORM,The 'judge_last_name' column in the source table does not have a corresponding column in the target table and can be ignored.