duckdb-fastapi-gradio

Running

App Files Files Community

liquidcarbon commited on Apr 26

Commit

0af2a8e

•

1 Parent(s): 5a864fe

aliases

Browse files

Files changed (4) hide show

Dockerfile +2 -1
history.json +3 -0
main.py +126 -69
sql.py +161 -14

Dockerfile CHANGED Viewed

@@ -25,9 +25,10 @@ RUN $PUP
 RUN pup py3.11
-RUN pup fetch appenv duckdb gradio itables
 COPY --chown=user *.py .
 EXPOSE 7860
 CMD ["appenv/.venv/bin/uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

 RUN pup py3.11
+RUN pup fetch appenv duckdb gradio itables python-ulid
 COPY --chown=user *.py .
+COPY --chown=user *.json .
 EXPOSE 7860
 CMD ["appenv/.venv/bin/uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

history.json ADDED Viewed

	@@ -0,0 +1,3 @@

+{"id": "01HWDNFA8QB96PV3RQCEPSG3C2", "q": "SELECT 42 AS answer", "alias": "example1", "template": "SELECT {x} AS {colname}", "definitions": "# Define variables: one '=' per line\nx=42\ncolname='answer'", "kwargs": {"x": 42, "colname": "answer"}, "rows": 1, "cols": 1, "source_id": null, "start": "2024-04-26 16:58:09.479", "end": "2024-04-26 16:58:09.479"}
+{"id": "01HWDNFA8QTEA1Y1V229501RMF", "q": "SELECT\n    Symbol,\n    Number,\n    Mass,\n    Abundance\nFROM 'https://raw.githubusercontent.com/ekwan/cctk/master/cctk/data/isotopes.csv'", "alias": "example2", "template": "SELECT\n    Symbol,\n    Number,\n    Mass,\n    Abundance\nFROM '{url}'", "definitions": "url = https://raw.githubusercontent.com/ekwan/cctk/master/cctk/data/isotopes.csv", "kwargs": {"url": "https://raw.githubusercontent.com/ekwan/cctk/master/cctk/data/isotopes.csv"}, "rows": 354, "cols": 4, "source_id": null, "start": "2024-04-26 16:58:13.425", "end": "2024-04-26 16:58:13.725"}
+{"id": "01HWE3CN3BPK6S4XT9B7H3GMSA", "q": "SELECT *\nFROM 'history.json'\nORDER BY id DESC", "alias": "example3", "template": "SELECT *\nFROM 'history.json'\nORDER BY id DESC", "kwargs": {}, "definitions": "", "rows": 2, "cols": 11, "source_id": null, "start": "2024-04-26 21:01:13.707", "end": "2024-04-26 21:01:13.709"}

main.py CHANGED Viewed

@@ -10,101 +10,158 @@ import gradio as gr
 import pandas as pd
 from fastapi import FastAPI
 from fastapi.responses import HTMLResponse, RedirectResponse
-from itables import options as itoptions, to_html_datatable
 from sql import Q
 itoptions.classes = "display compact cell-border"
 itoptions.column_filters = "footer"
-HEAD = """
 """
-# <link rel="stylesheet" href="https://cdn.datatables.net/2.0.5/css/dataTables.dataTables.min.css" />
-EXAMPLE1 = """
-SELECT
-  Symbol,
-  Number,
-  Mass,
-  Abundance
-FROM 'https://raw.githubusercontent.com/ekwan/cctk/master/cctk/data/isotopes.csv'
 """
-EXAMPLE2 = """
-SELECT
-    42 AS answer,
-    'Life, Universe & Everything' AS question
 """
 app = FastAPI()
-@app.get("/q/{base64query}", response_class=HTMLResponse)
-def query_db(base64query: str|None = None):
-    """Endpoint for running b64-encoded SQL queries."""
-    decoded = Q.from_base64(base64query)
-    df = decoded.df()
-    html = to_html_datatable(df, buttons=["copyHtml5"])
-    return f"""
-        <head>
-        {HEAD}
-        </head>
-        <div style="width:95vh;">{html}</div>
-        <hr><pre>{decoded}</pre>
-    """
-def query_from_request(query, request: gr.Request):
-    """Process query from input block or from initial request.
-    https://github.com/gradio-app/gradio/issues/7464#issuecomment-1960161591
-    """
-    if not query:
-        query_params = request.query_params
-        base64query = dict(query_params).get("q")
-    else:
-        base64query = Q(query).base64
-    if base64query in (None, "example"):
-        decoded = Q(EXAMPLE2)
-        base64query = decoded.base64
     else:
-        decoded = Q.from_base64(base64query)
-    _host = request.headers.get("Host")
-    if "huggingface.co/spaces" in _host:
-        # modify URL to access endpoints that aren't available on default app landing page
-        split_url = _host.rsplit("/", maxsplit=2)
-        hf_user, hf_space = split_url[1], split_url[2]
-        host = f"https://{hf_user}-{hf_space}.hf.space"
-    else:
-        host = _host
-    editor_url = f"{host}/sql/?q={base64query}"
-    query_url = f"{host}/q/{base64query}"
     result = f"""
     <div id="resultContainer">
-    <iframe src="/q/{base64query}" width="90%" height="90%"></iframe>
-    </div>"""
-    return (decoded, editor_url, query_url, result)
 with gr.Blocks(
     title="Gradio DuckDB Editor",
-    css="#resultContainer {height: 75vh;}"
 ) as gradio_sql_interface:
     with gr.Row():
-        with gr.Column(scale=1, min_width=420):
-            header = gr.Markdown("# SQL Editor")
-            sql_code = gr.Code(language="sql", label="SQL Query", lines=32, interactive=True)
-            button = gr.Button("run")
             editor_url = gr.Code(label="Share Editor URL", lines=1)
-            query_url = gr.Code(label="Share Query Results URL", lines=1)
-        with gr.Column(scale=1, min_width=540):
-            markdown = gr.Markdown("# RESULTS")
-            results = gr.HTML()
-    button.click(query_from_request, [sql_code], [sql_code, editor_url, query_url, results])
-    gradio_sql_interface.load(query_from_request, [sql_code], [sql_code, editor_url, query_url, results])
 app = gr.mount_gradio_app(app, gradio_sql_interface, path="/sql")
 @app.get("/")
-@app.get("/sql")
-@app.get("/sql/")
 def redirect_to_example():
-    return RedirectResponse("/sql/?q=example")

 import pandas as pd
 from fastapi import FastAPI
 from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.middleware.cors import CORSMiddleware
 from sql import Q
+from itables import options as itoptions, to_html_datatable
 itoptions.classes = "display compact cell-border"
 itoptions.column_filters = "footer"
+itoptions.layout = {
+    "top": {
+        "buttons": ["copyHtml5", "csvHtml5", "pageLength"],
+    },
+    "top2": "search",
+    "top3": "info",
+    "topStart": None,
+    "topEnd": None,
+    "bottom": "paging",
+    "bottomStart": None,
+    "bottomEnd": None,
+}
+CSS = """
+#resultContainer {
+    height: calc(90vh - 40px);
+    box-shadow: var(--block-shadow) !important;
+    border-width: var(--block-border-width) !important;
+    border-color: var(--block-border-color) !important;
+    border-radius: var(--block-radius) !important;
+    background: var(--block-background-fill) !important;
+}
 """
+HEAD = """
 """
+RESULT_TEMPLATE = f"""
+<!doctype html>
+<html>
+    <head>{HEAD}</head>
+    <body>
+        <div style="">{{datatable}}</div>
+        <hr><h3>Formatted query:</h3>
+        <pre>{{q}}</pre>
+        <hr>ran at {{ran}}</h3>
+    </body>
+</html>
 """
 app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["GET", "POST"],
+)
+@app.get("/q")
+def retrieve_query(query_id: str|None = None, alias: str|None = None):
+    """Endpoint for retrieving saved SQL queries."""
+    q = Q.from_history(query_id=query_id, alias=alias)
+    return HTMLResponse(content=run_query(q))
+def run_query(q: Q, save=True, html_template=RESULT_TEMPLATE):
+    try:
+        df = q.df(save=save, _raise=True)
+    except Exception as e:
+        df = pd.DataFrame({"error": [str(e)]})
+    result_datatable = to_html_datatable(df)
+    html = html_template.format(
+        datatable=result_datatable,
+        q=q,
+        ran=q.end.datetime.strftime("%F %T")
+    )
+    return html
+def query_from_request(sql_input, definitions, request: gr.Request):
+    """Process query from input block or from initial request."""
+    host = change_hf_host(request)
+    if not sql_input and not definitions:
+        # landing page or saved query
+        url_query_params = dict(request.query_params)
+        query_id = url_query_params.get("q")
+        alias = url_query_params.get("alias")
+        q = Q.from_history(query_id=query_id, alias=alias)
+        iframe_src = f"/q?query_id={q.source_id}"
     else:
+        # new query - run button was pressed, register unexecuted query
+        q = Q.from_template_and_definitions(sql_input, definitions)
+        q.save()
+        query_id = q.id
+        alias = q.alias
+        iframe_src = f"/q?query_id={query_id}"
     result = f"""
     <div id="resultContainer">
+    <iframe src="{iframe_src}" width="99%" height="99%"></iframe>
+    </div>
+    """
+    sql_input = q.template
+    definitions = f"{q.definitions}"
+    editor_url = "".join([
+        f"http://{host}/sql?",
+        f"q={query_id}" if query_id else "",
+        f"&alias={alias}" if alias else "",
+    ])
+    result_url = f"http://{host}{iframe_src}"
+    return (sql_input, definitions, result, editor_url, result_url)
 with gr.Blocks(
     title="Gradio DuckDB Editor",
+    theme=gr.themes.Soft(),
+    css=CSS,
 ) as gradio_sql_interface:
     with gr.Row():
+        with gr.Column(scale=2, min_width=450):
+            caption1 = gr.Markdown("# SQL Editor\nClick buttons below to see examples")
+            with gr.Row():
+                ex1_button = gr.Button("Variables", link="/sql?alias=example1")
+                ex2_button = gr.Button("URL", link="/sql?alias=example2")
+                ex3_button = gr.Button("Local File", link="/sql?alias=example3")
+            definitions = gr.Code(label="Definitions", lines=2, interactive=True)
+            sql_input = gr.Code(label="SQL Query", language="sql", lines=25, interactive=True)
+            run_button = gr.Button("run", variant="primary")
             editor_url = gr.Code(label="Share Editor URL", lines=1)
+            result_url = gr.Code(label="Share Query Results URL", lines=1)
+        with gr.Column(scale=3, min_width=600):
+            caption2 = gr.Markdown("# RESULTS")
+            result = gr.HTML(elem_classes="block")
+    magic = dict(
+        fn=query_from_request,
+        inputs=[sql_input, definitions],
+        outputs=[sql_input, definitions, result, editor_url, result_url]
+    )
+    run_button.click(**magic)
+    gradio_sql_interface.load(**magic)
+def change_hf_host(request: gr.Request):
+    """Access endpoints that hidden from default HF app landing page."""
+    _host = request.headers.get("Host")
+    if "huggingface.co/spaces" in _host:
+        split_url = _host.rsplit("/", maxsplit=2)
+        hf_user, hf_space = split_url[1], split_url[2]
+        host = f"https://{hf_user}-{hf_space}.hf.space"
+    else:
+        host = _host
+    return host
 app = gr.mount_gradio_app(app, gradio_sql_interface, path="/sql")
 @app.get("/")
 def redirect_to_example():
+    return RedirectResponse("/sql?alias=example1")

sql.py CHANGED Viewed

@@ -1,30 +1,130 @@
 import base64
 import duckdb
 class Q(str):
     UNSAFE = ["CREATE", "DELETE", "DROP", "INSERT", "UPDATE"]
-    def __init__(self, query: str):
-        self.is_safe = not any([cmd in query.upper() for cmd in self.UNSAFE])
-    def run(self, sql_engine=None):
         try:
             if sql_engine is None:
-                return self.run_duckdb()
             else:
-                return self.run_sql(sql_engine)
         except Exception as e:
-            pass
     def run_duckdb(self):
         return duckdb.sql(self)
-    def df(self, sql_engine=None):
-        result = self.run(sql_engine=sql_engine)
-        if result is None: return
-        result_df = result.df()
-        return result_df
     @property
     def base64(self):
         return base64.b64encode(self.encode()).decode()
@@ -32,4 +132,51 @@ class Q(str):
     @classmethod
     def from_base64(cls, b64):
         """Initializing from base64-encoded URL paths."""
-        return cls(base64.b64decode(b64).decode())

+import ast
 import base64
 import duckdb
+import json
+import re
+import textwrap
+from ulid import ULID
+HISTORY_FILE = "history.json"
+class SQLError(Exception):
+    pass
+class NotFoundError(Exception):
+    pass
 class Q(str):
     UNSAFE = ["CREATE", "DELETE", "DROP", "INSERT", "UPDATE"]
+    rows=None
+    def __new__(cls, template: str, **kwargs):
+        """Create a new Q-string."""
+        _template = textwrap.dedent(template).strip()
+        try:
+            instance = str.__new__(cls, _template.format(**kwargs))
+        except KeyError:
+            instance = str.__new__(cls, _template)
+        instance.id = str(ULID())
+        instance.alias = kwargs.pop("alias") if kwargs.get("alias") else None
+        instance.template = _template
+        instance.kwargs = kwargs
+        instance.definitions = "\n".join([f"{k} = {repr(v)}" for k, v in kwargs.items()])
+        for attr in ("rows", "cols", "source_id", "start", "end"):
+            setattr(instance, attr, None)
+        return instance
+    def __repr__(self):
+        """Neat repr for inspecting Q objects."""
+        strings = []
+        for k, v in self.__dict__.items():
+            value_repr = "\n" + textwrap.indent(v, "  ") if "\n" in str(v) else v
+            strings.append(f"{k}: {value_repr}")
+        return "\n".join(strings)
+    def run(self, sql_engine=None, save=False, _raise=False):
+        self.start = ULID()
         try:
             if sql_engine is None:
+                res = self.run_duckdb()
             else:
+                res = self.run_sql(sql_engine)
+            self.rows, self.cols = res.shape
+            return res
         except Exception as e:
+            if _raise:
+                raise e
+            return str(e)
+        finally:
+            self.end = ULID()
+            if save:
+                self.save()
     def run_duckdb(self):
         return duckdb.sql(self)
+    def df(self, sql_engine=None, save=False, _raise=False):
+        res = self.run(sql_engine=sql_engine, save=save, _raise=_raise)
+        if not getattr(self, "rows", None):
+            return
+        else:
+            result_df = res.df()
+            result_df.q = self
+            return result_df
+    def save(self, file=HISTORY_FILE):
+        with open(file, "a") as f:
+            f.write(self.json)
+            f.write("\n")
+    @property
+    def json(self):
+        serialized = {"id": self.id, "q": self}
+        serialized.update(self.__dict__)
+        return json.dumps(serialized, default=lambda x: x.datetime.strftime("%F %T.%f")[:-3])
+    @property
+    def is_safe(self):
+        return not any(cmd in self.template.upper() for cmd in self.UNSAFE)
+    @classmethod
+    def from_dict(cls, query_dict: dict):
+        q = query_dict.pop("q")
+        return cls(q, **query_dict)
+    @classmethod
+    def from_template_and_definitions(cls, template: str, definitions: str, alias: str|None = None):
+        query_dict = {"q": template, "alias": alias}
+        query_dict.update(parse_definitions(definitions))
+        instance = Q.from_dict(query_dict)
+        instance.definitions = definitions
+        return instance
+    @classmethod
+    def from_history(cls, query_id=None, alias=None):
+        search_query = Q(f"""
+            SELECT id, template, kwargs
+            FROM '{HISTORY_FILE}'
+            WHERE id='{query_id}' OR alias='{alias}'
+            LIMIT 1
+        """)
+        query = search_query.run()
+        if search_query.rows == 1:
+            source_id, template, kwargs = query.fetchall()[0]
+            kwargs = {k: v for k, v in kwargs.items() if v is not None}
+            instance = cls(template, **kwargs)
+            instance.source_id = source_id
+            return instance
+        elif search_query.rows == 0:
+            raise NotFoundError(f"id '{query_id}' / alias '{alias}' not found")
+        else:
+            raise SQLError(query)
+    # @property
+    # def definitions(self):
+    #     return "\n".join([""]+[f"{k} = {v}" for k, v in self.kwargs.items()])
     @property
     def base64(self):
         return base64.b64encode(self.encode()).decode()
     @classmethod
     def from_base64(cls, b64):
         """Initializing from base64-encoded URL paths."""
+        return cls(base64.b64decode(b64).decode())
+def parse_definitions(definitions) -> dict:
+    """Parse a string literal of "key=value" pairs, one per line, into kwargs."""
+    kwargs = {}
+    lines = definitions.split("\n")
+    for _line in lines:
+        line = re.sub("\s+", "", _line)
+        if line == "" or line.startswith("#"):
+            continue
+        if "=" in line:
+            key, value = line.split("=", maxsplit=1)
+            kwargs[key] = ast.literal_eval(value)
+    return kwargs
+EX1 = Q.from_template_and_definitions(
+    template="SELECT {x} AS {colname}",
+    definitions="\n".join([
+        "# Define variables: one '=' per line",
+        "x=42",
+        "colname='answer'",
+    ]),
+    alias="example1",
+)
+EX2 = Q(
+    """
+    SELECT
+        Symbol,
+        Number,
+        Mass,
+        Abundance
+    FROM '{url}'
+    """,
+    url="https://raw.githubusercontent.com/ekwan/cctk/master/cctk/data/isotopes.csv",
+    alias="example2",
+)
+EX3 = Q(
+    """
+    SELECT /
+    FROM 'history.json'
+    ORDER BY id DESC
+    """,
+    alias="example3",
+)