Upload folder using huggingface_hub

.gitattributes

@@ -1,36 +1 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
 textgames-scrabble-black2-ss.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -4,6 +4,7 @@
 agents/*.sh
 user_outputs/
 model_outputs/__runs__
+runner_out/
 
 .idea/
 

README.md

@@ -9,24 +9,37 @@ hf_oauth: true
 ---
 # TextGames
 
-## Setup
-```
-❱❱❱ pip install -r requirements.txt
-```
-
-## Play (Terminal)
-```
-❱❱❱ python play.py
-```
-
-## Play (Web UI)
-```
-❱❱❱ pip install gradio
-❱❱❱ GRADIO_SERVER_PORT=1080  python play_gradio.py
-```
-Open `localhost:1080` to access.
-
-## Optional Environment Varibles
-```
-TEXTGAMES_SHOW_HIDDEN_LEVEL=1
-```
+## Play on Hosted Servers
+
+- HuggingFace Space
+
+    https://huggingface.co/spaces/fhudi/textgames
+    (login required)
+
+## Play on localhost
+
+- Setup
+    ```
+    ❱❱❱ pip install -r requirements.txt
+    ```
+
+- Play (Terminal)
+    ```
+    ❱❱❱ python play.py
+    ```
+
+- Play (Web UI)
+    ```
+    ❱❱❱ pip install gradio
+    ❱❱❱ GRADIO_SERVER_PORT=1080  python play_gradio.py
+    ```
+    Open `localhost:1080` to access.
+
+---
+
+## Extras
+
+- Optional Environment Varibles
+    ```
+    TEXTGAMES_SHOW_HIDDEN_LEVEL=1
+    ```

Try Gemma-2-9B.ipynb

@@ -1,218 +1 @@
-{
- "nbformat": 4,
- "nbformat_minor": 0,
- "metadata": {
-  "colab": {
-   "private_outputs": true,
-   "provenance": [],
-   "authorship_tag": "ABX9TyPmvDoFpmwAf1QFBJZy7XSQ"
-  },
-  "kernelspec": {
-   "name": "python3",
-   "display_name": "Python 3"
-  },
-  "language_info": {
-   "name": "python"
-  }
- },
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "Rli_enT6lBDT"
-   },
-   "outputs": [],
-   "source": [
-    "##%%\n",
-    "import os\n",
-    "import torch\n",
-    "import random\n",
-    "import numpy as np\n",
-    "import argparse\n",
-    "import json\n",
-    "import cohere\n",
-    "from openai import OpenAI\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "##%%\n",
-    "from tqdm import tqdm\n",
-    "\n",
-    "from collections import Counter\n",
-    "\n",
-    "from transformers import LlamaForCausalLM, AutoTokenizer, AutoModelForCausalLM, AutoModelForSeq2SeqLM\n",
-    "import hashlib\n",
-    "\n",
-    "from textgames import GAME_NAMES, GAME_IDS, LEVELS, LEVELS_HIDDEN, LEVEL_IDS, new_game\n"
-   ],
-   "metadata": {
-    "id": "dp1F32B8oSfD"
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "##%%\n",
-    "gen_model_checkpoint = \"google/gemma-2-9b-it\"\n",
-    "quantize = True"
-   ],
-   "metadata": {
-    "id": "jZF8bkUcojTX"
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "kwargs = {\n",
-    "    \"device_map\": \"auto\",\n",
-    "} if quantize else {}"
-   ],
-   "metadata": {
-    "id": "VAF5sR9arYzS"
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "##%%\n",
-    "gen_model = AutoModelForCausalLM.from_pretrained(gen_model_checkpoint, **kwargs)\n",
-    "tokenizer = AutoTokenizer.from_pretrained(gen_model_checkpoint, **kwargs)"
-   ],
-   "metadata": {
-    "id": "tzqldl8ooRVL"
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "gen_model.device"
-   ],
-   "metadata": {
-    "id": "FeBUXdkWsWrL"
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "def get_gemma_response(text):\n",
-    "    # global gen_model, tokenizer\n",
-    "    messages = [\n",
-    "        {\"role\": \"user\", \"content\": text},\n",
-    "    ]\n",
-    "\n",
-    "    input_ids = tokenizer.apply_chat_template(\n",
-    "        messages,\n",
-    "        add_generation_prompt=True,\n",
-    "        return_tensors=\"pt\"\n",
-    "    ).to(gen_model.device)\n",
-    "\n",
-    "    terminators = [\n",
-    "        tokenizer.eos_token_id,\n",
-    "        tokenizer.convert_tokens_to_ids(\"<|eot_id|>\")\n",
-    "    ]\n",
-    "\n",
-    "    outputs = gen_model.generate(\n",
-    "        input_ids,\n",
-    "        max_new_tokens=100,\n",
-    "        eos_token_id=terminators,\n",
-    "        do_sample=True,\n",
-    "        temperature=0.2,\n",
-    "        top_p=1\n",
-    "    )\n",
-    "\n",
-    "    response = outputs[0][input_ids.shape[-1]:]\n",
-    "    return tokenizer.decode(response, skip_special_tokens=True)"
-   ],
-   "metadata": {
-    "id": "R5D4K-P2sPaj"
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "text = \\\n",
-    "\"\"\"\n",
-    "Given a set of rules to calculate point, sort the set of words in decreasing order.\n",
-    "When there 2 or more words with same point, sort lexicographically.\n",
-    "\n",
-    "Rules:\n",
-    "- every pair of consecutive consonant gets 5 points\n",
-    "- every pair of consecutive vowel gets 3 points\n",
-    "- add 1 point if there exists exactly 1 'g' in the word\n",
-    "- word less than 5 characters gets 10 points\n",
-    "- word starts with gen gets 100 points\n",
-    "- word ends with ta gets -1000 point\n",
-    "\n",
-    "Words:\n",
-    "- genta\n",
-    "- winata\n",
-    "- hudi\n",
-    "- alham\n",
-    "- aji\n",
-    "- ruochen\n",
-    "\n",
-    "Print only the answer.\n",
-    "\"\"\"\n",
-    "\n",
-    "# Answer:\n",
-    "# - aji      10\n",
-    "# - hudi     10\n",
-    "# - ruochen   5  3\n",
-    "# - alham     5\n",
-    "# - genta     5  1  100 -1000\n",
-    "# - winata  -1000"
-   ],
-   "metadata": {
-    "id": "T_tk4hTGsxsR"
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "print(get_gemma_response(text))"
-   ],
-   "metadata": {
-    "id": "05OI36v6vGoY"
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "code",
-   "source": [
-    "print(get_gemma_response(text))"
-   ],
-   "metadata": {
-    "id": "riwXqTc-tmNr"
-   },
-   "execution_count": null,
-   "outputs": []
-  },
-  {
-   "cell_type": "code",
-   "source": [],
-   "metadata": {
-    "id": "T72sUG4_vYUa"
-   },
-   "execution_count": null,
-   "outputs": []
-  }
- ]
-}
+{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"Rli_enT6lBDT","executionInfo":{"status":"ok","timestamp":1737395007014,"user_tz":-540,"elapsed":5212,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}}},"outputs":[],"source":["##%%\n","import os\n","import pickle\n","import json\n","# import random\n","# import torch\n","# import numpy as np\n","# import argparse\n","# import cohere\n","# from openai import OpenAI\n"]},{"cell_type":"code","source":["##%%\n","# import hashlib\n","from tqdm import tqdm\n","from itertools import product\n","# from collections import Counter\n","\n","# from transformers import LlamaForCausalLM, AutoTokenizer, AutoModelForCausalLM, AutoModelForSeq2SeqLM\n","from transformers import AutoTokenizer, AutoModelForCausalLM\n","from textgames import GAME_NAMES, LEVEL_IDS, game_filename, _game_class_from_name\n"],"metadata":{"id":"dp1F32B8oSfD","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1737395010583,"user_tz":-540,"elapsed":3547,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"e9adeb5f-70eb-4ca9-dcbb-428e4b28ab41"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stderr","text":["/home/is/frederikus-h/miniconda3/envs/textgame/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n","  from .autonotebook import tqdm as notebook_tqdm\n"]}]},{"cell_type":"code","source":["os.environ.setdefault(\"TEXTGAMES_OUTPUT_DIR\", \"user_outputs\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2wEu1V1wvxn0","executionInfo":{"status":"ok","timestamp":1737395010664,"user_tz":-540,"elapsed":67,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"cdcad20f-e357-4009-9f4f-0d4495ebd894"},"execution_count":3,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'user_outputs'"]},"metadata":{},"execution_count":3}]},{"cell_type":"code","source":["##%%\n","gen_model_checkpoint = \"google/gemma-2-9b-it\"\n","quantize = True"],"metadata":{"id":"jZF8bkUcojTX","executionInfo":{"status":"ok","timestamp":1737395010678,"user_tz":-540,"elapsed":13,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["kwargs = {\n","    \"device_map\": \"auto\",\n","} if quantize else {}"],"metadata":{"id":"VAF5sR9arYzS","executionInfo":{"status":"ok","timestamp":1737395010683,"user_tz":-540,"elapsed":2,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["##%%\n","gen_model = AutoModelForCausalLM.from_pretrained(gen_model_checkpoint, **kwargs)\n","tokenizer = AutoTokenizer.from_pretrained(gen_model_checkpoint, **kwargs)"],"metadata":{"id":"tzqldl8ooRVL","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1737395038547,"user_tz":-540,"elapsed":27859,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"902b638c-e6ce-4f8a-bba2-e9f7241c9a27"},"execution_count":6,"outputs":[{"output_type":"stream","name":"stderr","text":["Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:24<00:00,  6.19s/it]\n"]}]},{"cell_type":"code","source":["gen_model.device"],"metadata":{"id":"FeBUXdkWsWrL","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1737395038552,"user_tz":-540,"elapsed":3,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"6437d1b7-02f8-47f5-d519-e979cefde795"},"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/plain":["device(type='cuda', index=0)"]},"metadata":{},"execution_count":7}]},{"cell_type":"code","source":["def get_gemma_response(text):\n","    # global gen_model, tokenizer\n","    messages = [\n","        {\"role\": \"user\", \"content\": text},\n","    ]\n","\n","    input_ids = tokenizer.apply_chat_template(\n","        messages,\n","        add_generation_prompt=True,\n","        return_tensors=\"pt\"\n","    ).to(gen_model.device)\n","\n","    terminators = [\n","        tokenizer.eos_token_id,\n","        tokenizer.convert_tokens_to_ids(\"<|eot_id|>\")\n","    ]\n","\n","    outputs = gen_model.generate(\n","        input_ids,\n","        max_new_tokens=100,\n","        eos_token_id=terminators,\n","        do_sample=True,\n","        temperature=.001,\n","        top_p=1,\n","    )\n","\n","    response = outputs[0][input_ids.shape[-1]:]\n","    return tokenizer.decode(response, skip_special_tokens=True)"],"metadata":{"id":"R5D4K-P2sPaj","executionInfo":{"status":"ok","timestamp":1737395038554,"user_tz":-540,"elapsed":1,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}}},"execution_count":8,"outputs":[]},{"cell_type":"markdown","source":["---\n","Example Call"],"metadata":{"id":"s5FEwOOvxf4h"}},{"cell_type":"code","source":["# @title\n","text = \\\n","\"\"\"\n","Given a set of rules to calculate point, sort the set of words in decreasing order.\n","When there 2 or more words with same point, sort lexicographically.\n","\n","Rules:\n","- every pair of consecutive consonant gets 5 points\n","- every pair of consecutive vowel gets 3 points\n","- add 1 point if there exists exactly 1 'g' in the word\n","- word less than 5 characters gets 10 points\n","- word starts with gen gets 100 points\n","- word ends with ta gets -1000 point\n","\n","Words:\n","- genta\n","- winata\n","- hudi\n","- alham\n","- aji\n","- ruochen\n","\n","Print only the answer.\n","\"\"\"\n","\n","print(text)"],"metadata":{"id":"T_tk4hTGsxsR","colab":{"base_uri":"https://localhost:8080/"},"cellView":"form","executionInfo":{"status":"ok","timestamp":1737392776367,"user_tz":-540,"elapsed":27,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"d5ea884f-d0fa-4134-ecd9-690eab51c976"},"execution_count":14,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Given a set of rules to calculate point, sort the set of words in decreasing order.\n","When there 2 or more words with same point, sort lexicographically.\n","\n","Rules:\n","- every pair of consecutive consonant gets 5 points\n","- every pair of consecutive vowel gets 3 points\n","- add 1 point if there exists exactly 1 'g' in the word\n","- word less than 5 characters gets 10 points\n","- word starts with gen gets 100 points\n","- word ends with ta gets -1000 point\n","\n","Words:\n","- genta\n","- winata\n","- hudi\n","- alham\n","- aji\n","- ruochen\n","\n","Print only the answer.\n","\n"]}]},{"cell_type":"code","source":["# Gold Answer:\n","# - aji      10\n","# - hudi     10\n","# - ruochen   5  3\n","# - alham     5\n","# - genta     5  1  100 -1000\n","# - winata  -1000"],"metadata":{"id":"G-5yS4S-rdsN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(get_gemma_response(text))"],"metadata":{"id":"05OI36v6vGoY","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1737392724119,"user_tz":-540,"elapsed":6741,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"fe5d6ed2-d063-4f1c-b2e1-b3af8dbc456e"},"execution_count":9,"outputs":[{"output_type":"stream","name":"stdout","text":["genta\n","winata\n","ruochen\n","hudi\n","alham\n","aji \n","\n"]}]},{"cell_type":"markdown","source":["---\n","Automate run all sessions"],"metadata":{"id":"cxJ4WqHpxi75"}},{"cell_type":"code","source":["for game_name, difficulty_level in product([GAME_NAMES[4], *GAME_NAMES[:4], *GAME_NAMES[5:]], LEVEL_IDS[:3]):\n","    game_cls = _game_class_from_name(game_name)\n","    with open(f\"problemsets/{game_filename(game_name)}_{difficulty_level}.json\", \"r\", encoding=\"utf8\") as f:\n","        sid_prompt_dict = json.load(f)\n","\n","    correct_cnt = 0\n","    for sid, prompt in tqdm(list(sid_prompt_dict.items()), desc=f\"{game_filename(game_name)}_-_{difficulty_level}\"):\n","        cur_game = game_cls()\n","        cur_game.load_game(prompt)\n","        response = get_gemma_response(cur_game.get_prompt()).strip()\n","        solved, val_msg = cur_game.validate(response)\n","        with open(f\"model_outputs/results_gemma_2_9B_it.pkl\", \"ab\") as o:\n","            pickle.dump((f\"{game_filename(game_name)}_{difficulty_level}\", sid, response, (solved, val_msg)), o)\n","        if solved:\n","            correct_cnt += 1\n","\n","    print(f\"{game_name}_-_{difficulty_level}\")\n","    print(f\"  Acc.: {correct_cnt / len(sid_prompt_dict):.2%}\")"],"metadata":{"id":"hCTXYpXa1UQ6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"GC-zkVI52IJX"},"execution_count":null,"outputs":[]}]}

agents/check_param.py

@@ -0,0 +1,9 @@
+import os
+from transformers import AutoModelForCausalLM
+model_name = os.getenv('MODEL_NAME')
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    device_map="auto",
+    torch_dtype="bfloat16",
+)
+print(model_name, sum(p.numel() for p in model.parameters()), model.num_parameters())

oauth_environ_google.sh

@@ -1 +1 @@
-export $(cat oauth_environ_google.env | xargs)
+export $(cat ${ENVFILE} | xargs)

play_helper.py

@@ -7,6 +7,8 @@ import gradio as gr
 import hashlib
 from io import BytesIO
 
+from datetime import datetime
+
 from textgames import GAME_NAMES as _GAME_NAMES, LEVEL_IDS, LEVELS, new_game, preload_game, game_filename
 from textgames.islands.islands import Islands
 from textgames.sudoku.sudoku import Sudoku
@@ -39,8 +41,10 @@ def declare_components(demo, greet, use_login_button=False):
                 logout_btn = gr.Button("Logout", link="/logout", variant='huggingface', size='sm', elem_id="btn-logout")
                 reset_sid_btn = gr.Button(interactive=False, visible=False, size='sm')
         with gr.Column(scale=2):
-            solved_games_df = gr.DataFrame(headers=[g.split('\t', 1)[0] for g in GAME_NAMES], label="Attempted Games",
-                                           row_count=(1, 'fixed'), interactive=False, elem_id="df-solved-games")
+            solved_games_df = gr.DataFrame(
+                pd.DataFrame({g.split('\t', 1)[0]: ['∅'] for g in GAME_NAMES}), label="Attempted Games",
+                row_count=(1, 'fixed'), col_count=(8, 'fixed'), interactive=False, elem_id="df-solved-games",
+            )
     level_radio = gr.Radio(LEVELS, label="Level", elem_id="radio-level-name", visible=False)
     game_radio = gr.Radio(GAME_NAMES, label="Game", elem_id="radio-game-name", visible=False)
     new_game_btn = gr.Button("Start Game", elem_id="btn-start-game", visible=False)
@@ -69,7 +73,7 @@ def declare_components(demo, greet, use_login_button=False):
     ).then(
         lambda: gr.update(interactive=False), None, [new_game_btn],
     ).then(
-        check_played_game, [solved_games, user_state], [solved_games, solved_games_df]
+        check_played_game, [user_state, solved_games, solved_games_df], [solved_games, solved_games_df]
     ).then(
         lambda uid: ([gr.update(visible=True, interactive=True)] if uid else
                      [gr.update(visible=False, interactive=False)]) * 3,
@@ -643,12 +647,12 @@ def start_new_game(game_name, level, session_state_component, is_solved_componen
                      js=js_submit)
     give_up_checkbox = gr.Checkbox(False, visible=False, interactive=False)
     give_up_btn.click(
-        lambda: (gr.update(interactive=False), gr.update(interactive=False)), None, [submit_btn, give_up_btn]
-    ).then(
+    #     lambda: (gr.update(interactive=False), gr.update(interactive=False)), None, [submit_btn, give_up_btn]
+    # ).then(
         lambda x: x, [give_up_checkbox], [give_up_checkbox],
         js="(x) => confirm('🥹 Give-up? 💸')"
-    ).then(
-        lambda: (gr.update(interactive=True), gr.update(interactive=True)), None, [submit_btn, give_up_btn]
+    # ).then(
+    #     lambda: (gr.update(interactive=True), gr.update(interactive=True)), None, [submit_btn, give_up_btn]
     )
 
     def _forfeiting(confirmed, _solved_games):
@@ -657,7 +661,7 @@ def start_new_game(game_name, level, session_state_component, is_solved_componen
             cur_game.finish_stats_(forfeit=True)
             if level in LEVELS and level not in _solved_games[game_name]:
                 _solved_games[game_name].append(level)
-            upload_to_drive(fp_out)
+            upload_to_drive(fp_out, update=True)
             return 0, _solved_games
         return 1, _solved_games
     give_up_checkbox.change(
@@ -696,7 +700,8 @@ def start_new_game(game_name, level, session_state_component, is_solved_componen
                            'game_name': game_name, 'difficulty_level': difficulty_level,
                            }, f)
                 f.write("\n")
-            upload_to_drive(fp_out)
+            print(f"   >>> Solved @ {datetime.now()}:", uid, sid, game_name, level, sep="  ")
+            upload_to_drive(fp_out, update=True)
             upload_to_drive(_leaderboards, update=True)
             return gr.update(interactive=True)
         return gr.update()
@@ -715,13 +720,15 @@ def start_new_game(game_name, level, session_state_component, is_solved_componen
 
 # %%
 def check_to_start_new_game(game_name, level, user=None, uid=None, sid=None):
-    print(game_name, level, uid, sid)
+    if not sid and isinstance(user, dict):
+        sid = user.get('sid', None)
+    print(f"  >>> Starts @ {datetime.now()}:", uid, sid, game_name, level, sep="  ")
     if game_name is None or level is None:
         raise gr.Error("please choose both Game & Level")
     fp = _get_file_output(game_name, LEVEL_IDS[LEVELS.index(level)], f"{uid}_{sid}")
     if os.path.exists(fp):
         # raise gr.Error(f"You have done this game already.<br/>{game_name} - {level}")
-        gr.Warning("You have done this game already. Only first attempt is recorded in the scoreboard.")
+        gr.Warning("You have done this game already.<br/>Only the first attempt is recorded on the leaderboard.")
     if user is None:
         gr.Warning("no user, game will be generated randomly")
     # else:
@@ -733,11 +740,11 @@ def check_to_start_new_game(game_name, level, user=None, uid=None, sid=None):
 
 
 # %%
-def check_played_game(solved_games, user, progress=gr.Progress()):
+def check_played_game(user, solved_games, solved_games_df, progress=gr.Progress()):
     uid = user['email']
     sid = user.get('sid', None)
     matches = _files.list(
-        q=f"'{_folder_id}' in parents and mimeType='application/octet-stream' and name contains '{uid}_-_'",
+        q=f"'{_folder_id}' in parents and mimeType='application/octet-stream' and name contains '{uid}_{sid}_-_'",
         fields=f"files(name, id, {_cksm_methods_str})",
     ).execute()
     matches = matches['files']
@@ -747,10 +754,14 @@ def check_played_game(solved_games, user, progress=gr.Progress()):
         for level, level_id in zip(LEVELS, LEVEL_IDS):
             fp_out = _get_file_output(game_name, level_id, f"{uid}_{sid}")
             _matches = list(filter(lambda m: fp_out.endswith(m['name']), matches))
-            if os.path.exists(fp_out):
-                upload_to_drive(fp_out, _matches)
-            else:
-                download_from_drive(fp_out, _matches)
+            if _matches and not os.path.exists(fp_out):
+                os.system(f"touch \"{fp_out}\"")
+            elif not _matches and os.path.exists(fp_out):
+                upload_to_drive(fp_out, _matches, update=True)
+            # if os.path.exists(fp_out):
+            #     upload_to_drive(fp_out, _matches, update=True)
+            # else:
+            #     download_from_drive(fp_out, _matches)
             if os.path.exists(fp_out):
                 cur.append(level)
         ret[game_name] = cur or '∅'

play_with_hf.py

@@ -64,12 +64,12 @@ with gr.Blocks(title="TextGames", css=css, delete_cache=(3600, 3600)) as demo:
 
     reset_sid_checkbox = gr.Checkbox(False, visible=False, interactive=False)
     reset_sid_btn.click(
-        lambda: [gr.update(interactive=False)]*2, None, [reset_sid_btn, new_game_btn]
-    ).then(
+    #     lambda: [gr.update(interactive=False)]*2, None, [reset_sid_btn, new_game_btn]
+    # ).then(
         lambda x: x, [reset_sid_checkbox], [reset_sid_checkbox],
-        js="(x) => confirm('Reset Progress? (cannot be undone)')"
-    ).then(
-        lambda: [gr.update(interactive=True)]*2, None, [reset_sid_btn, new_game_btn]
+        js="(x) => confirm('Only your best session is recorded on the leaderboard. Are you sure you want to start from the beginning? (cannot be undone)')"
+    # ).then(
+    #     lambda: [gr.update(interactive=True)]*2, None, [reset_sid_btn, new_game_btn]
     )
 
     def _resetting(confirmed, user):
@@ -78,13 +78,15 @@ with gr.Blocks(title="TextGames", css=css, delete_cache=(3600, 3600)) as demo:
             gr.Warning("You need to log in first!")
         elif confirmed:
             user['sid'] = get_sid(uid, force_generate_sid=True)
+            gr.Info("Successfully resets the game with new session. Enjoy the game! 💪")
         return user, False
     reset_sid_checkbox.change(
         lambda: [gr.update(interactive=False)]*3, None, [logout_btn, reset_sid_btn, new_game_btn]
     ).then(
         _resetting, [reset_sid_checkbox, user_state], [user_state, reset_sid_checkbox]
     ).then(
-        check_played_game, [solved_games, user_state], [solved_games, solved_games_df]
+        check_played_game, [user_state, solved_games, solved_games_df], [solved_games, solved_games_df]
+
     ).then(
         lambda: [gr.update(interactive=True)]*3, None, [logout_btn, reset_sid_btn, new_game_btn]
     )
@@ -96,30 +98,72 @@ with gr.Blocks(title="TextGames", css=css, delete_cache=(3600, 3600)) as demo:
             start_new_game(game_name, level, session_state, is_solved, solved_games, user=user, uid=_uid_state)
 
 #%%
-with demo.route("Leaderboards", "/leaderboard") as demo_leaderboard:
-    gr.Markdown("Under Construction. Will be available soon.")
-    leaderboards = []
-    for tab in ["🚅 Easy", "🚀 Medium", "🛸 Hard"]:
-        with gr.Tab(tab):
-            leaderboards.append(gr.DataFrame(label="Rankings"))
-
-    # if os.path.exists(_leaderboards):
-    #     datas = []
-    #     with open(_leaderboards, "r", encoding="utf8") as f:
-    #         for line in f:
-    #             datas.append(json.loads(line))
-    #     concat = [{'Level': d['difficulty_level'], 'User': d['uid'], 'Game': d['game_name'].split('\t', 1)[0], 'Attempts': d['turns'],
-    #                "Time": d['ed'] - d['st']} for d in datas]
-    # else:
-    def add_dummies():
-        return pd.DataFrame({
-            'User': ['dummy'],
-            'Solved': [' '.join([g.split('\t', 1)[0] for g in GAME_NAMES])],
-            'Attempts': [8],
-            'Time': [7200.8],
-        })
-    for l in leaderboards:
-        demo_leaderboard.load(add_dummies, None, [l])
+with (demo.route("Leaderboards", "/leaderboards") as demo_leaderboard):
+    # gr.Markdown("Under Construction. Will be available soon.")
+    def reload_leaderboard():
+        ret_leaderboards = {}
+
+        def add_dummies():
+            return pd.DataFrame({
+                'User': ['dummy'],
+                'Solved': [sorted([g.split('\t', 1)[0] for g in GAME_NAMES])],
+                'Attempts': [888],
+                'Time': [8888.8888],
+            })
+
+        if not os.path.exists(_leaderboards):
+            for lv in ['1', '2', '3']:
+                ret_leaderboards[lv] = add_dummies()
+
+        else:
+            datas = []
+            with open(_leaderboards, "r", encoding="utf8") as f:
+                for line in f:
+                    datas.append(json.loads(line))
+            concat = [{'Level': d['difficulty_level'], 'User': d['uid'], 'Session': d['sid'],
+                       'Solved': d['game_name'].split('\t', 1)[0], 'Attempts': d['turns'], "Time": d['ed'] - d['st']
+                       } for d in datas]
+            df_leaderboards_all = pd.DataFrame(concat)
+
+            def get_best(_cur_df):
+                def _per_session(_df):
+                    best = _df.groupby("Solved").apply(
+                        lambda _df: _df.sort_values(["Attempts", "Time"]).iloc[0]
+                    ).reset_index(drop=True)
+                    ret = pd.DataFrame({
+                        "Solved": [sorted(best.Solved.unique())], "Attempts": best.Attempts.sum(), "Time": best.Time.sum(),
+                    })
+                    return ret
+                flat = _cur_df.groupby("Session").apply(_per_session)
+                srt = flat.sort_values(["Solved", "Attempts", "Time"], key=lambda c: {
+                    "Solved": lambda s: -s.apply(len),
+                }.get(c.name, lambda s: s)(c))
+                return srt.iloc[0]
+
+            for lv in ['1', '2', '3']:
+                cur_df = df_leaderboards_all.loc[df_leaderboards_all.Level.eq(lv)].groupby("User").apply(get_best)
+                ret_leaderboards[lv] = cur_df.reset_index() if len(cur_df) else add_dummies()
+
+        return ret_leaderboards
+
+    df_leaderboards = {}
+
+    # for lv, tab_name in [('1', "🚅 Easy"), ('2', "🚀 Medium"), ('3', "🛸 Hard")]:
+    with gr.Tab("🚅 Easy") as tab1:
+        lb_df_1 = gr.DataFrame(label="Rankings", col_count=(4, 'fixed'), interactive=False, show_search='filter')
+        tab1.select(lambda: df_leaderboards['1'], None, [lb_df_1])
+    with gr.Tab("🚀 Medium") as tab2:
+        lb_df_2 = gr.DataFrame(label="Rankings", col_count=(4, 'fixed'), interactive=False, show_search='filter')
+        tab2.select(lambda: df_leaderboards['2'], None, [lb_df_2])
+    with gr.Tab("🛸 Hard") as tab3:
+        lb_df_3 = gr.DataFrame(label="Rankings", col_count=(4, 'fixed'), interactive=False, show_search='filter')
+        tab3.select(lambda: df_leaderboards['3'], None, [lb_df_3])
+
+    def onload(progress=gr.Progress()):
+        global df_leaderboards
+        df_leaderboards = reload_leaderboard()
+        return df_leaderboards['1']
+    demo_leaderboard.load(onload, None, [lb_df_1])
 
 
 #%%
@@ -130,3 +174,18 @@ demo.launch(
 )
 
 
+#%%
+
+#%%
+
+
+#%%
+
+
+#%%
+
+
+#%%
+
+
+#%%

textgames_check_model_outputs.py

@@ -0,0 +1,172 @@
+# %%
+import json
+import pickle
+import re
+from pathlib import Path
+
+
+# %%
+def load_pickle(fp):
+    with open(fp, "rb") as f:
+        try:
+            while True:
+                yield pickle.load(f)
+        except EOFError:
+            pass
+
+
+# %%
+fd = Path("model_outputs")
+
+# %%
+
+
+# %%
+
+
+# %%
+
+
+# # %%
+# # concat pickle results (1/22)
+# list(fd.glob("results_gemma_*"))[0]
+#
+# # %%
+# fps = sorted(fd.glob("results_gemma_*"))
+# all_responses = dict()
+# errors = set()
+# for fp in fps:
+#     responses = list(load_pickle(str(fp)))
+#     print(fp.name, len(responses), responses[0][0], responses[-1][0])
+#     for r in responses:
+#         if r[-1]:
+#             errors.add((r[0], str(r[-1])))
+#         all_responses.setdefault(r[:2], set())
+#         all_responses[r[:2]].add(r)
+# errors = sorted(errors)
+#
+# # %%
+# assert all(len(v) == 1 for v in all_responses.values()), f"Duplicated response(s) found"
+#
+# # %%
+# duplicated = {k: v for k, v in all_responses.items() if len(v) > 1}
+#
+# # %%
+# concatenated = [list(v)[0] for v in all_responses.values()]
+#
+# # %%
+# with open(fd / "gemma2_9b_results_depre_250122/results_gemma-2-9b-it.single_turn.jsonl", "w", encoding="utf8") as o:
+#     for i in concatenated:
+#         json.dump({
+#             "game": i[0],
+#             "session": i[1],
+#             "turn": 1,
+#             "response": i[2],
+#             "solved": i[3][0],
+#             "val_msg": i[3][1],
+#             "error": repr(i[4]) if i[4] else i[4],
+#         }, o, ensure_ascii=False)
+#         o.write("\n")
+
+# %%
+
+
+# %%
+
+
+# %%
+
+
+# %%
+
+
+# %%
+
+
+# %%
+# Rerun gemma, resolving errors
+
+# %%
+import os
+import json
+import pandas as pd
+
+# %%
+os.environ["TG_GAME_ST"] = "7"
+os.environ["TG_GAME_ED"] = "8"
+
+# %%
+st, ed = os.getenv("TG_GAME_ST", None), os.getenv("TG_GAME_ED", None)
+st, ed = ((None if x is None else int(x)) for x in (st, ed))
+fp_out = f"model_outputs/results_gemma-2-9b-it{'' if st is None else f'.{st}'}.jsonl"
+
+
+# %%
+from tqdm import tqdm
+from itertools import product
+
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+from textgames import THE_GAMES, GAME_NAMES, LEVEL_IDS, game_filename, _game_class_from_name
+
+os.environ.setdefault("TEXTGAMES_OUTPUT_DIR", "user_outputs")
+
+
+# %%
+with open(fd / "gemma2_9b_results_depre_250122/results_gemma-2-9b-it.single_turn.jsonl", "r", encoding="utf-8") as f:
+    df = pd.read_json(f, lines=True)
+
+# %%
+df.columns
+
+
+# %%
+from agents import run_with_agent
+from agents.gemma_2_9b_it import gemma_postproc
+
+
+# %%
+def get_buffered_response(texts, game_name, difficulty_level, turn):
+    if turn > 1:
+        return None
+    cur_df = df.loc[(df.game == f"{game_filename(game_name)}_{difficulty_level}")].set_index(["session", "turn"])
+    with open(f"problemsets/{game_filename(game_name)}_{difficulty_level}.json", "r", encoding="utf8") as f:
+        _sid_prompt_dict = json.load(f)
+    prompt_sid_dict = {v: k for k, v in _sid_prompt_dict.items()}
+    sid = prompt_sid_dict[texts[0]]
+    try:
+        return cur_df.loc[(sid, turn)].response
+    except KeyError:
+        return None
+
+
+# %%
+run_with_agent(fp_out, get_buffered_response, get_postprocess=gemma_postproc, game_names_list=GAME_NAMES[st:ed], n_turns=1)
+
+# %%
+
+
+# %%
+# type(cur_df.loc[(sid, 1)].response)
+
+# %%
+
+
+# %%
+
+
+# %%
+
+
+# %%
+
+
+# %%
+
+
+# %%
+
+
+# %%
+
+