curation app

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.10
+
+WORKDIR /code
+
+COPY --link --chown=1000 . .
+
+RUN mkdir -p /tmp/cache/
+RUN mkdir -p /.cache
+RUN chmod a+rwx -R /tmp/cache/
+RUN chmod a+rwx -R /.cache
+ENV HF_HUB_CACHE=HF_HOME
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+ENV PYTHONUNBUFFERED=1 PORT=7860
+CMD ["python", "app.py"]

README.md

@@ -8,4 +8,4 @@ pinned: false
 license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,229 @@
+import json
+import random
+from collections import Counter
+
+import dotenv
+from datasets import load_dataset
+from fasthtml.common import *
+from fasthtml_hf import setup_hf_backup
+from fastlite import database
+
+dotenv.load_dotenv()
+
+fact_dataset = load_dataset("griffin/iclr2025_data_scores", split="train").to_list()
+fact_dataset = [{"example_id": i, **example} for i, example in enumerate(fact_dataset)]
+
+db = database("data/examples.db")
+examples = db.t.examples
+if examples not in db.t:
+    examples.create(
+        id=int,
+        example_id=int,
+        question_type=str,
+        question=str,
+        answer=str,
+        decision=str,
+        pk="id",
+    )
+
+evaluated_ids = set(row["example_id"] for row in examples.rows)
+question_types = sorted(set(ex["question_type"] for ex in fact_dataset))
+
+
+def get_stats():
+    total_examples = Counter(ex["question_type"] for ex in fact_dataset)
+    curated_examples = Counter(row["question_type"] for row in examples.rows)
+    stats = {
+        qt: {"total": total_examples[qt], "curated": curated_examples[qt]}
+        for qt in question_types
+    }
+    return stats
+
+
+def get_example(selected_type=None):
+    available_examples = [
+        ex for ex in fact_dataset if ex["example_id"] not in evaluated_ids
+    ]
+    if selected_type:
+        available_examples = [
+            ex for ex in available_examples if ex["question_type"] == selected_type
+        ]
+
+    if not available_examples:
+        return None
+
+    example = random.choice(available_examples)
+
+    keep_keys = [
+        "example_id",
+        "question_type",
+        "question",
+        "rationale",
+        "answer",
+        "log_ll",
+        "oracle_log_ll",
+        "oracle_advantage",
+        "prediction",
+        "prediction_oracle",
+        "accuracy",
+        "accuracy_oracle",
+        "accuracy_status",
+    ]
+
+    return {k: example[k] for k in keep_keys if k in example}
+
+
+# app
+style = Style("""
+            body { background-color: #1e1e1e; color: #d4d4d4; font-family: Arial, sans-serif; }
+            h1, h2, h3 { color: #61dafb; }
+            .example-container { margin-top: 20px; }
+            .example-table { border-collapse: collapse; width: 100%; }
+            .example-table th, .example-table td { border: 1px solid #3a3a3a; padding: 8px; text-align: left; }
+            .example-table th { background-color: #2a2a2a; color: #61dafb; }
+            .example-table td { color: #d4d4d4; }
+            #evaluation-form { margin-top: 20px; }
+            #evaluation-form button { margin-right: 10px; background-color: #0e639c; color: white; border: none; padding: 10px 20px; cursor: pointer; }
+            #evaluation-form button:hover { background-color: #1177bb; }
+            select { background-color: #2a2a2a; color: #d4d4d4; border: 1px solid #3a3a3a; padding: 5px; }
+            a { color: #61dafb; text-decoration: none; }
+            a:hover { text-decoration: underline; }
+        """)
+
+app, rt = fast_app(live=True, hdrs=(style,))
+
+
+def render_stats(stats):
+    return Table(
+        Tr(Th("Question Type"), Th("Curated"), Th("Total")),
+        *[
+            Tr(
+                Td(qt),
+                Td(
+                    f"{stats[qt]['curated']} ({stats[qt]['curated']/stats[qt]['total']:.1%})"
+                ),
+                Td(stats[qt]["total"]),
+            )
+            for qt in question_types
+        ],
+        cls="stats-table",
+    )
+
+
+def render_example(example):
+    return Div(
+        H3("Example Details"),
+        Table(
+            *[Tr(Th(key), Td(str(value))) for key, value in example.items()],
+            cls="example-table",
+        ),
+        id="example-details",
+    )
+
+
+@rt("/")
+async def get(question_type: str = None):
+    stats = get_stats()
+    example = get_example(question_type)
+
+    dropdown = Select(
+        Option("Question Types", value="", selected=question_type is None),
+        *[Option(qt, value=qt, selected=qt == question_type) for qt in question_types],
+        name="question_type",
+        hx_get="/",
+        hx_target="body",
+        hx_push_url="true",
+    )
+
+    if example is None:
+        content = Div(
+            H2("All examples of this type have been evaluated!"), render_stats(stats)
+        )
+    else:
+        content = Div(
+            H2("Example"),
+            Div(
+                render_example(example),
+                Form(
+                    Button(
+                        "Good Example",
+                        name="decision",
+                        value="good",
+                        hx_post="/evaluate",
+                        hx_target="#example-details",
+                    ),
+                    Button(
+                        "Bad Example",
+                        name="decision",
+                        value="bad",
+                        hx_post="/evaluate",
+                        hx_target="#example-details",
+                    ),
+                    Hidden(name="example", value=json.dumps(example)),
+                    id="evaluation-form",
+                ),
+                id="example-container",
+            ),
+        )
+
+    view_stats_link = A("Curation Stats", href="/stats", cls="view-stats-link")
+
+    return Titled(
+        "Example Curation",
+        H2("Question Type Selection"),
+        dropdown,
+        content,
+        view_stats_link,
+    )
+
+
+@rt("/stats")
+async def get():
+    stats = get_stats()
+    stats = render_stats(stats)
+
+    return Titled(
+        "Curation Statistics",
+        Div(
+            stats,
+            A("Back to Curation", href="/", cls="back-link"),
+            cls="container",
+        ),
+    )
+
+
+@rt("/evaluate")
+async def post(decision: str, example: str):
+    example_dict = json.loads(example)
+
+    # Insert the evaluated example into the database
+    examples.insert(
+        {
+            "id": len(list(examples.rows)) + 1,  # Auto-increment ID
+            "example_id": example_dict["example_id"],
+            "question_type": example_dict["question_type"],
+            "question": example_dict["question"],
+            "answer": example_dict["answer"],
+            "decision": decision,
+        }
+    )
+
+    # Add the evaluated example's ID to the set of evaluated IDs
+    evaluated_ids.add(example_dict["example_id"])
+    # Get a new example
+    new_example = get_example(example_dict["question_type"])
+
+    if new_example is None:
+        return Div(H2("All examples of this type have been evaluated!"))
+    else:
+        return render_example(new_example)
+
+
+# serve()
+if __name__ == "__main__":
+    import os
+
+    import uvicorn
+
+    setup_hf_backup(app)
+    uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", 7860)))

config.ini

@@ -0,0 +1,5 @@
+[DEFAULT]
+dataset_id = single-fact-curation-iclr-eval
+db_dir = data
+private_backup = True
+interval = 2 # number of minutes between periodic backups

requirements.txt

@@ -0,0 +1,8 @@
+fasthtml-hf>=0.1.1
+python-fasthtml>=0.0.8
+huggingface-hub>=0.20.0
+uvicorn>=0.29
+pandas
+numpy
+python-dotenv
+datasets