added hf config and dotenv

.github/workflows/check_size.yaml

@@ -0,0 +1,15 @@
+name: Check file size
+on:               
+  pull_request:
+    branches: [main]
+
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check large files
+        uses: ActionsDesk/lfs-warning@v2.0
+        with:
+          filesizelimit: 10485760 

.github/workflows/sync_hf.yaml

@@ -0,0 +1,19 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push https://andymbryant:$HF_TOKEN@huggingface.co/spaces/andymbryant/data-mapper main

.gitignore

@@ -1,3 +1,4 @@
 venv/
 __pycache__/
 *.pyc
+.env

README.md

@@ -0,0 +1,8 @@
+---
+colorTo: coral
+colorFrom: green
+emoji: ✨
+license: BSD 2-Clause License
+title: Data Mapper
+sdk: gradio
+---

requirements.txt

@@ -4,3 +4,4 @@ ipykernel
 pandas
 tabulate
 gradio
+python-dotenv

src/core.py

@@ -1,4 +1,5 @@
 import os
+from dotenv import load_dotenv
 import pandas as pd
 from langchain.output_parsers import PydanticOutputParser
 from langchain.prompts import ChatPromptTemplate
@@ -10,7 +11,7 @@ from src.types import TableMapping
 from src.vars import NUM_ROWS_TO_RETURN
 from src.prompt import DATA_SCIENTIST_PROMPT_STR, SPEC_WRITER_PROMPT_STR, ENGINEER_PROMPT_STR
 
-os.environ["OPENAI_API_KEY"] = "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"
+load_dotenv()
 
 DATA_DIR_PATH = os.path.join(os.path.dirname(__file__), 'data')
 SYNTHETIC_DATA_DIR_PATH = os.path.join(DATA_DIR_PATH, 'synthetic')
@@ -20,7 +21,7 @@ TRANSFORM_MODEL = ChatOpenAI(
     temperature=0,
 )
 
-natural_language_model = ChatOpenAI(
+NATURAL_LANGUAGE_MODEL = ChatOpenAI(
     model_name='gpt-4',
     temperature=0.1,
 )
@@ -46,7 +47,7 @@ def get_table_mapping(source_df, template_df) -> TableMapping:
 
 def get_code_spec(table_mapping: TableMapping) -> str:
     writer_prompt = ChatPromptTemplate.from_template(SPEC_WRITER_PROMPT_STR)
-    writer_chain = writer_prompt | natural_language_model | StrOutputParser()
+    writer_chain = writer_prompt | NATURAL_LANGUAGE_MODEL | StrOutputParser()
     return writer_chain.invoke({"table_mapping": str(table_mapping)})
 
 

src/notebooks/brainstorm.ipynb

@@ -20,7 +20,9 @@
     "from pygments import highlight\n",
     "from pygments.lexers import PythonLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "import json"
+    "import json\n",
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
    ]
   },
   {
@@ -30,7 +32,6 @@
    "outputs": [],
    "source": [
     "langchain.debug = True\n",
-    "os.environ[\"OPENAI_API_KEY\"] = \"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\"\n",
     "data_dir_path = os.path.join(os.getcwd(), \"data\")"
    ]
   },

src/notebooks/brainstorm2.ipynb

@@ -14,7 +14,9 @@
     "from langchain.tools import PythonAstREPLTool\n",
     "from langchain.chat_models import ChatOpenAI\n",
     "from pydantic import BaseModel, Field\n",
-    "from langchain.memory import ConversationBufferMemory"
+    "from langchain.memory import ConversationBufferMemory\n",
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
    ]
   },
   {
@@ -24,7 +26,6 @@
    "outputs": [],
    "source": [
     "langchain.debug = True\n",
-    "os.environ[\"OPENAI_API_KEY\"] = \"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\"\n",
     "data_dir_path = os.path.join(os.getcwd())\n",
     "pd.set_option('display.max_rows', 20)\n",
     "pd.set_option('display.max_columns', 20)\n",

src/notebooks/brainstorm3.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -17,7 +17,9 @@
     "from langchain.memory import ConversationBufferMemory\n",
     "from langchain.schema.output_parser import StrOutputParser\n",
     "import json\n",
-    "import gradio as gr"
+    "import gradio as gr\n",
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
    ]
   },
   {
@@ -27,7 +29,6 @@
    "outputs": [],
    "source": [
     "langchain.debug = True\n",
-    "os.environ[\"OPENAI_API_KEY\"] = \"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\"\n",
     "data_dir_path = os.path.join(os.getcwd())\n",
     "pd.set_option('display.max_rows', 20)\n",
     "pd.set_option('display.max_columns', 20)\n",

src/notebooks/brainstorm4.ipynb

@@ -2,9 +2,20 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 90,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 90,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "import os\n",
     "import pandas as pd\n",
@@ -17,7 +28,9 @@
     "from langchain.prompts import ChatPromptTemplate\n",
     "from langchain.tools import PythonAstREPLTool\n",
     "from langchain.chat_models import ChatOpenAI\n",
-    "from langchain.schema.output_parser import StrOutputParser"
+    "from langchain.schema.output_parser import StrOutputParser\n",
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
    ]
   },
   {
@@ -27,8 +40,6 @@
    "outputs": [],
    "source": [
     "langchain.debug = False\n",
-    "# Throwaway key with strict usage limit\n",
-    "os.environ[\"OPENAI_API_KEY\"] = \"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\"\n",
     "pd.set_option('display.max_columns', 20)\n",
     "pd.set_option('display.max_rows', 20)"
    ]