using new columns

ZeroEval-main/result_dirs/zebra-grid.summary.json

@@ -1,4 +1,38 @@
 [
+  {
+    "Model": "o1-2024-12-17",
+    "Mode": "greedy",
+    "Puzzle Acc": "81.00",
+    "Cell Acc": "78.74",
+    "No answer": "0.20",
+    "Easy Puzzle Acc": "98.21",
+    "Hard Puzzle Acc": "74.31",
+    "Small Puzzle Acc": "97.19",
+    "Medium Puzzle Acc": "92.14",
+    "Large Puzzle Acc": "78.00",
+    "XL Puzzle Acc": "42.50",
+    "Total Puzzles": 1000,
+    "Reason Lens": "1197.51",
+    "N_Mode": "single",
+    "N_Size": 1
+  },
+  {
+    "Model": "deepseek-R1",
+    "Mode": "greedy",
+    "Puzzle Acc": "78.70",
+    "Cell Acc": "80.54",
+    "No answer": "0.00",
+    "Easy Puzzle Acc": "98.57",
+    "Hard Puzzle Acc": "70.97",
+    "Small Puzzle Acc": "98.44",
+    "Medium Puzzle Acc": "95.71",
+    "Large Puzzle Acc": "73.50",
+    "XL Puzzle Acc": "28.50",
+    "Total Puzzles": 1000,
+    "Reason Lens": "586.33",
+    "N_Mode": "single",
+    "N_Size": 1
+  },
   {
     "Model": "o1-preview-2024-09-12",
     "Mode": "greedy",
@@ -7,8 +41,65 @@
     "No answer": "0.30",
     "Easy Puzzle Acc": "98.57",
     "Hard Puzzle Acc": "60.83",
+    "Small Puzzle Acc": "98.12",
+    "Medium Puzzle Acc": "88.21",
+    "Large Puzzle Acc": "59.50",
+    "XL Puzzle Acc": "17.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1565.88"
+    "Reason Lens": "1565.88",
+    "N_Mode": "single",
+    "N_Size": 1
+  },
+  {
+    "Model": "o1-preview-2024-09-12-v2",
+    "Mode": "greedy",
+    "Puzzle Acc": "70.40",
+    "Cell Acc": "74.18",
+    "No answer": "0.40",
+    "Easy Puzzle Acc": "98.21",
+    "Hard Puzzle Acc": "59.58",
+    "Small Puzzle Acc": "97.81",
+    "Medium Puzzle Acc": "88.57",
+    "Large Puzzle Acc": "55.50",
+    "XL Puzzle Acc": "16.00",
+    "Total Puzzles": 1000,
+    "Reason Lens": "1559.71",
+    "N_Mode": "single",
+    "N_Size": 1
+  },
+  {
+    "Model": "o1-mini-2024-09-12-v3",
+    "Mode": "greedy",
+    "Puzzle Acc": "59.70",
+    "Cell Acc": "70.32",
+    "No answer": "1.00",
+    "Easy Puzzle Acc": "86.07",
+    "Hard Puzzle Acc": "49.44",
+    "Small Puzzle Acc": "87.50",
+    "Medium Puzzle Acc": "76.79",
+    "Large Puzzle Acc": "39.00",
+    "XL Puzzle Acc": "12.00",
+    "Total Puzzles": 1000,
+    "Reason Lens": "1166.38",
+    "N_Mode": "single",
+    "N_Size": 1
+  },
+  {
+    "Model": "o1-mini-2024-09-12-v2",
+    "Mode": "greedy",
+    "Puzzle Acc": "56.80",
+    "Cell Acc": "69.87",
+    "No answer": "1.30",
+    "Easy Puzzle Acc": "82.86",
+    "Hard Puzzle Acc": "46.67",
+    "Small Puzzle Acc": "83.44",
+    "Medium Puzzle Acc": "76.43",
+    "Large Puzzle Acc": "36.00",
+    "XL Puzzle Acc": "7.50",
+    "Total Puzzles": 1000,
+    "Reason Lens": "1164.95",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "o1-mini-2024-09-12",
@@ -18,30 +109,65 @@
     "No answer": "0.80",
     "Easy Puzzle Acc": "87.14",
     "Hard Puzzle Acc": "39.17",
+    "Small Puzzle Acc": "87.81",
+    "Medium Puzzle Acc": "67.50",
+    "Large Puzzle Acc": "24.50",
+    "XL Puzzle Acc": "3.50",
     "Total Puzzles": 1000,
-    "Reason Lens": "993.28"
+    "Reason Lens": "993.28",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
-    "Model": "claude-3-5-sonnet-20240620",
+    "Model": "deepseek-v3",
     "Mode": "greedy",
-    "Puzzle Acc": "33.40",
-    "Cell Acc": "54.34",
+    "Puzzle Acc": "42.10",
+    "Cell Acc": "42.04",
+    "No answer": "27.90",
+    "Easy Puzzle Acc": "90.00",
+    "Hard Puzzle Acc": "23.47",
+    "Small Puzzle Acc": "85.62",
+    "Medium Puzzle Acc": "44.64",
+    "Large Puzzle Acc": "10.00",
+    "XL Puzzle Acc": "1.00",
+    "Total Puzzles": 1000,
+    "Reason Lens": "2158.00",
+    "N_Mode": "single",
+    "N_Size": 1
+  },
+  {
+    "Model": "claude-3-5-sonnet-20241022",
+    "Mode": "greedy",
+    "Puzzle Acc": "36.20",
+    "Cell Acc": "54.27",
     "No answer": "0.00",
-    "Easy Puzzle Acc": "87.50",
-    "Hard Puzzle Acc": "12.36",
+    "Easy Puzzle Acc": "91.07",
+    "Hard Puzzle Acc": "14.86",
+    "Small Puzzle Acc": "84.69",
+    "Medium Puzzle Acc": "28.93",
+    "Large Puzzle Acc": "4.00",
+    "XL Puzzle Acc": "1.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1141.94"
+    "Reason Lens": "861.18",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "claude-3-5-sonnet-20240620",
-    "Mode": "sampling",
+    "Mode": "greedy",
     "Puzzle Acc": "33.40",
-    "Cell Acc": "53.01",
-    "No answer": "0.10",
-    "Easy Puzzle Acc": "88.21",
-    "Hard Puzzle Acc": "12.08",
+    "Cell Acc": "54.34",
+    "No answer": "0.00",
+    "Easy Puzzle Acc": "87.50",
+    "Hard Puzzle Acc": "12.36",
+    "Small Puzzle Acc": "83.44",
+    "Medium Puzzle Acc": "21.79",
+    "Large Puzzle Acc": "3.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1153.83"
+    "Reason Lens": "1141.94",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Llama-3.1-405B-Inst-fp8@together",
@@ -51,19 +177,14 @@
     "No answer": "12.50",
     "Easy Puzzle Acc": "87.14",
     "Hard Puzzle Acc": "11.39",
+    "Small Puzzle Acc": "81.25",
+    "Medium Puzzle Acc": "22.50",
+    "Large Puzzle Acc": "1.50",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "314.66"
-  },
-  {
-    "Model": "Llama-3.1-405B-Inst-fp8@together",
-    "Mode": "sampling",
-    "Puzzle Acc": "32.60",
-    "Cell Acc": "47.04",
-    "No answer": "10.80",
-    "Easy Puzzle Acc": "86.07",
-    "Hard Puzzle Acc": "11.81",
-    "Total Puzzles": 1000,
-    "Reason Lens": "439.96"
+    "Reason Lens": "314.66",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gpt-4o-2024-08-06",
@@ -73,19 +194,14 @@
     "No answer": "3.60",
     "Easy Puzzle Acc": "84.64",
     "Hard Puzzle Acc": "11.11",
+    "Small Puzzle Acc": "80.00",
+    "Medium Puzzle Acc": "19.64",
+    "Large Puzzle Acc": "2.50",
+    "XL Puzzle Acc": "0.50",
     "Total Puzzles": 1000,
-    "Reason Lens": "1106.51"
-  },
-  {
-    "Model": "gpt-4o-2024-05-13",
-    "Mode": "sampling",
-    "Puzzle Acc": "30.80",
-    "Cell Acc": "46.19",
-    "No answer": "6.60",
-    "Easy Puzzle Acc": "81.07",
-    "Hard Puzzle Acc": "11.25",
-    "Total Puzzles": 1000,
-    "Reason Lens": "1549.74"
+    "Reason Lens": "1106.51",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gemini-1.5-pro-exp-0827",
@@ -95,8 +211,14 @@
     "No answer": "0.80",
     "Easy Puzzle Acc": "79.64",
     "Hard Puzzle Acc": "11.39",
+    "Small Puzzle Acc": "75.31",
+    "Medium Puzzle Acc": "20.71",
+    "Large Puzzle Acc": "3.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1594.47"
+    "Reason Lens": "1594.47",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Llama-3.1-405B-Inst@sambanova",
@@ -106,8 +228,14 @@
     "No answer": "24.70",
     "Easy Puzzle Acc": "84.64",
     "Hard Puzzle Acc": "8.89",
+    "Small Puzzle Acc": "79.06",
+    "Medium Puzzle Acc": "16.43",
+    "Large Puzzle Acc": "0.50",
+    "XL Puzzle Acc": "0.50",
     "Total Puzzles": 1000,
-    "Reason Lens": "2001.12"
+    "Reason Lens": "2001.12",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "chatgpt-4o-latest-24-09-07",
@@ -117,8 +245,14 @@
     "No answer": "4.20",
     "Easy Puzzle Acc": "81.43",
     "Hard Puzzle Acc": "9.86",
+    "Small Puzzle Acc": "76.88",
+    "Medium Puzzle Acc": "17.86",
+    "Large Puzzle Acc": "1.50",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1539.99"
+    "Reason Lens": "1539.99",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Mistral-Large-2",
@@ -128,8 +262,14 @@
     "No answer": "1.70",
     "Easy Puzzle Acc": "80.36",
     "Hard Puzzle Acc": "9.03",
+    "Small Puzzle Acc": "75.94",
+    "Medium Puzzle Acc": "15.00",
+    "Large Puzzle Acc": "2.50",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1592.39"
+    "Reason Lens": "1592.39",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gpt-4-turbo-2024-04-09",
@@ -139,8 +279,14 @@
     "No answer": "0.10",
     "Easy Puzzle Acc": "80.71",
     "Hard Puzzle Acc": "8.06",
+    "Small Puzzle Acc": "75.31",
+    "Medium Puzzle Acc": "15.00",
+    "Large Puzzle Acc": "0.50",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1148.46"
+    "Reason Lens": "1148.46",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gpt-4o-2024-05-13",
@@ -150,8 +296,31 @@
     "No answer": "19.30",
     "Easy Puzzle Acc": "77.86",
     "Hard Puzzle Acc": "8.89",
+    "Small Puzzle Acc": "73.75",
+    "Medium Puzzle Acc": "16.43",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
+    "Total Puzzles": 1000,
+    "Reason Lens": "1643.51",
+    "N_Mode": "single",
+    "N_Size": 1
+  },
+  {
+    "Model": "grok-2-1212",
+    "Mode": "greedy",
+    "Puzzle Acc": "27.70",
+    "Cell Acc": "48.16",
+    "No answer": "3.50",
+    "Easy Puzzle Acc": "76.43",
+    "Hard Puzzle Acc": "8.75",
+    "Small Puzzle Acc": "71.88",
+    "Medium Puzzle Acc": "13.93",
+    "Large Puzzle Acc": "4.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1643.51"
+    "Reason Lens": "2551.39",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gpt-4-0314",
@@ -161,8 +330,14 @@
     "No answer": "0.20",
     "Easy Puzzle Acc": "77.14",
     "Hard Puzzle Acc": "7.64",
+    "Small Puzzle Acc": "71.25",
+    "Medium Puzzle Acc": "13.57",
+    "Large Puzzle Acc": "2.50",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1203.17"
+    "Reason Lens": "1203.17",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "claude-3-opus-20240229",
@@ -172,8 +347,14 @@
     "No answer": "0.00",
     "Easy Puzzle Acc": "78.21",
     "Hard Puzzle Acc": "7.08",
+    "Small Puzzle Acc": "73.44",
+    "Medium Puzzle Acc": "12.14",
+    "Large Puzzle Acc": "0.50",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "855.72"
+    "Reason Lens": "855.72",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Qwen2.5-72B-Instruct",
@@ -183,19 +364,14 @@
     "No answer": "11.90",
     "Easy Puzzle Acc": "76.43",
     "Hard Puzzle Acc": "7.22",
+    "Small Puzzle Acc": "72.50",
+    "Medium Puzzle Acc": "12.14",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1795.90"
-  },
-  {
-    "Model": "gpt-4-turbo-2024-04-09",
-    "Mode": "sampling",
-    "Puzzle Acc": "26.40",
-    "Cell Acc": "47.93",
-    "No answer": "0.00",
-    "Easy Puzzle Acc": "74.29",
-    "Hard Puzzle Acc": "7.78",
-    "Total Puzzles": 1000,
-    "Reason Lens": "1165.90"
+    "Reason Lens": "1795.90",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Qwen2.5-32B-Instruct",
@@ -205,8 +381,14 @@
     "No answer": "6.30",
     "Easy Puzzle Acc": "77.50",
     "Hard Puzzle Acc": "6.11",
+    "Small Puzzle Acc": "72.19",
+    "Medium Puzzle Acc": "10.36",
+    "Large Puzzle Acc": "0.50",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1333.07"
+    "Reason Lens": "1333.07",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gemini-1.5-pro-exp-0801",
@@ -216,8 +398,14 @@
     "No answer": "0.00",
     "Easy Puzzle Acc": "72.50",
     "Hard Puzzle Acc": "6.81",
+    "Small Puzzle Acc": "66.56",
+    "Medium Puzzle Acc": "13.93",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1389.75"
+    "Reason Lens": "1389.75",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Llama-3.1-405B-Inst@hyperbolic",
@@ -227,8 +415,14 @@
     "No answer": "6.25",
     "Easy Puzzle Acc": "66.67",
     "Hard Puzzle Acc": "15.38",
+    "Small Puzzle Acc": "50.00",
+    "Medium Puzzle Acc": "33.33",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 16,
-    "Reason Lens": "1517.13"
+    "Reason Lens": "1517.13",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gemini-1.5-flash-exp-0827",
@@ -238,8 +432,14 @@
     "No answer": "8.50",
     "Easy Puzzle Acc": "70.71",
     "Hard Puzzle Acc": "7.22",
+    "Small Puzzle Acc": "65.00",
+    "Medium Puzzle Acc": "13.57",
+    "Large Puzzle Acc": "2.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1705.11"
+    "Reason Lens": "1705.11",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Meta-Llama-3.1-70B-Instruct",
@@ -249,8 +449,14 @@
     "No answer": "43.00",
     "Easy Puzzle Acc": "73.57",
     "Hard Puzzle Acc": "5.97",
+    "Small Puzzle Acc": "67.81",
+    "Medium Puzzle Acc": "10.36",
+    "Large Puzzle Acc": "1.50",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1483.68"
+    "Reason Lens": "1483.68",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "deepseek-v2-chat-0628",
@@ -260,8 +466,14 @@
     "No answer": "5.20",
     "Easy Puzzle Acc": "68.57",
     "Hard Puzzle Acc": "4.86",
+    "Small Puzzle Acc": "63.44",
+    "Medium Puzzle Acc": "8.57",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1260.23"
+    "Reason Lens": "1260.23",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "deepseek-v2.5-0908",
@@ -271,8 +483,14 @@
     "No answer": "12.70",
     "Easy Puzzle Acc": "68.21",
     "Hard Puzzle Acc": "4.17",
+    "Small Puzzle Acc": "62.19",
+    "Medium Puzzle Acc": "7.86",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1294.46"
+    "Reason Lens": "1294.46",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Qwen2-72B-Instruct",
@@ -282,8 +500,14 @@
     "No answer": "10.20",
     "Easy Puzzle Acc": "63.93",
     "Hard Puzzle Acc": "4.86",
+    "Small Puzzle Acc": "60.94",
+    "Medium Puzzle Acc": "6.79",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1813.82"
+    "Reason Lens": "1813.82",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "deepseek-v2-coder-0614",
@@ -293,8 +517,14 @@
     "No answer": "4.90",
     "Easy Puzzle Acc": "64.64",
     "Hard Puzzle Acc": "4.17",
+    "Small Puzzle Acc": "59.69",
+    "Medium Puzzle Acc": "7.14",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1324.55"
+    "Reason Lens": "1324.55",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "deepseek-v2-coder-0724",
@@ -304,8 +534,14 @@
     "No answer": "3.40",
     "Easy Puzzle Acc": "61.79",
     "Hard Puzzle Acc": "4.44",
+    "Small Puzzle Acc": "57.50",
+    "Medium Puzzle Acc": "7.14",
+    "Large Puzzle Acc": "0.50",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1230.63"
+    "Reason Lens": "1230.63",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gpt-4o-mini-2024-07-18",
@@ -315,19 +551,14 @@
     "No answer": "0.10",
     "Easy Puzzle Acc": "62.50",
     "Hard Puzzle Acc": "3.61",
+    "Small Puzzle Acc": "58.75",
+    "Medium Puzzle Acc": "4.64",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "943.52"
-  },
-  {
-    "Model": "gemini-1.5-pro",
-    "Mode": "sampling",
-    "Puzzle Acc": "19.70",
-    "Cell Acc": "45.24",
-    "No answer": "0.40",
-    "Easy Puzzle Acc": "60.00",
-    "Hard Puzzle Acc": "4.03",
-    "Total Puzzles": 1000,
-    "Reason Lens": "1356.77"
+    "Reason Lens": "943.52",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gemini-1.5-flash",
@@ -337,8 +568,14 @@
     "No answer": "22.70",
     "Easy Puzzle Acc": "59.29",
     "Hard Puzzle Acc": "3.89",
+    "Small Puzzle Acc": "55.00",
+    "Medium Puzzle Acc": "6.43",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1538.18"
+    "Reason Lens": "1538.18",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gemini-1.5-pro",
@@ -348,8 +585,14 @@
     "No answer": "0.80",
     "Easy Puzzle Acc": "55.71",
     "Hard Puzzle Acc": "5.28",
+    "Small Puzzle Acc": "52.19",
+    "Medium Puzzle Acc": "9.64",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1336.17"
+    "Reason Lens": "1336.17",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "yi-large-preview",
@@ -359,8 +602,14 @@
     "No answer": "1.40",
     "Easy Puzzle Acc": "58.93",
     "Hard Puzzle Acc": "3.33",
+    "Small Puzzle Acc": "53.75",
+    "Medium Puzzle Acc": "6.07",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "833.36"
+    "Reason Lens": "833.36",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "yi-large",
@@ -370,41 +619,48 @@
     "No answer": "1.80",
     "Easy Puzzle Acc": "58.21",
     "Hard Puzzle Acc": "3.47",
+    "Small Puzzle Acc": "54.37",
+    "Medium Puzzle Acc": "5.00",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "757.01"
+    "Reason Lens": "757.01",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
-    "Model": "claude-3-sonnet-20240229",
+    "Model": "claude-3-5-haiku-20241022",
     "Mode": "greedy",
     "Puzzle Acc": "18.70",
-    "Cell Acc": "43.66",
-    "No answer": "0.00",
-    "Easy Puzzle Acc": "58.93",
-    "Hard Puzzle Acc": "3.06",
+    "Cell Acc": "43.22",
+    "No answer": "0.10",
+    "Easy Puzzle Acc": "57.86",
+    "Hard Puzzle Acc": "3.47",
+    "Small Puzzle Acc": "53.12",
+    "Medium Puzzle Acc": "6.07",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1095.37"
+    "Reason Lens": "660.91",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
-    "Model": "Qwen2-72B-Instruct",
-    "Mode": "sampling",
+    "Model": "claude-3-sonnet-20240229",
+    "Mode": "greedy",
     "Puzzle Acc": "18.70",
-    "Cell Acc": "40.57",
-    "No answer": "3.20",
-    "Easy Puzzle Acc": "57.50",
-    "Hard Puzzle Acc": "3.61",
-    "Total Puzzles": 1000,
-    "Reason Lens": "1894.72"
-  },
-  {
-    "Model": "gemini-1.5-flash",
-    "Mode": "sampling",
-    "Puzzle Acc": "18.40",
-    "Cell Acc": "36.03",
-    "No answer": "12.80",
-    "Easy Puzzle Acc": "57.86",
+    "Cell Acc": "43.66",
+    "No answer": "0.00",
+    "Easy Puzzle Acc": "58.93",
     "Hard Puzzle Acc": "3.06",
+    "Small Puzzle Acc": "54.06",
+    "Medium Puzzle Acc": "4.29",
+    "Large Puzzle Acc": "1.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1713.03"
+    "Reason Lens": "1095.37",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Meta-Llama-3-70B-Instruct",
@@ -414,8 +670,14 @@
     "No answer": "0.20",
     "Easy Puzzle Acc": "52.86",
     "Hard Puzzle Acc": "2.78",
+    "Small Puzzle Acc": "48.44",
+    "Medium Puzzle Acc": "4.64",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "809.95"
+    "Reason Lens": "809.95",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Athene-70B",
@@ -425,8 +687,14 @@
     "No answer": "21.10",
     "Easy Puzzle Acc": "52.50",
     "Hard Puzzle Acc": "2.78",
+    "Small Puzzle Acc": "48.75",
+    "Medium Puzzle Acc": "3.93",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "391.19"
+    "Reason Lens": "391.19",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gemma-2-27b-it",
@@ -436,8 +704,14 @@
     "No answer": "1.10",
     "Easy Puzzle Acc": "50.71",
     "Hard Puzzle Acc": "2.92",
+    "Small Puzzle Acc": "46.56",
+    "Medium Puzzle Acc": "5.00",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1014.56"
+    "Reason Lens": "1014.56",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "claude-3-haiku-20240307",
@@ -447,8 +721,14 @@
     "No answer": "0.10",
     "Easy Puzzle Acc": "47.86",
     "Hard Puzzle Acc": "1.25",
+    "Small Puzzle Acc": "43.75",
+    "Medium Puzzle Acc": "1.07",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1015.06"
+    "Reason Lens": "1015.06",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "command-r-plus",
@@ -458,8 +738,14 @@
     "No answer": "0.20",
     "Easy Puzzle Acc": "44.64",
     "Hard Puzzle Acc": "1.94",
+    "Small Puzzle Acc": "40.94",
+    "Medium Puzzle Acc": "2.86",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "810.53"
+    "Reason Lens": "810.53",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "reka-core-20240501",
@@ -469,19 +755,14 @@
     "No answer": "4.00",
     "Easy Puzzle Acc": "43.21",
     "Hard Puzzle Acc": "1.25",
+    "Small Puzzle Acc": "39.38",
+    "Medium Puzzle Acc": "1.43",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1078.29"
-  },
-  {
-    "Model": "Meta-Llama-3.1-8B-Instruct",
-    "Mode": "greedy",
-    "Puzzle Acc": "12.80",
-    "Cell Acc": "13.68",
-    "No answer": "61.50",
-    "Easy Puzzle Acc": "43.57",
-    "Hard Puzzle Acc": "0.83",
-    "Total Puzzles": 1000,
-    "Reason Lens": "1043.90"
+    "Reason Lens": "1078.29",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gemma-2-9b-it",
@@ -491,8 +772,31 @@
     "No answer": "0.00",
     "Easy Puzzle Acc": "41.79",
     "Hard Puzzle Acc": "1.53",
+    "Small Puzzle Acc": "37.81",
+    "Medium Puzzle Acc": "2.50",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
+    "Total Puzzles": 1000,
+    "Reason Lens": "849.84",
+    "N_Mode": "single",
+    "N_Size": 1
+  },
+  {
+    "Model": "Meta-Llama-3.1-8B-Instruct",
+    "Mode": "greedy",
+    "Puzzle Acc": "12.80",
+    "Cell Acc": "13.68",
+    "No answer": "61.50",
+    "Easy Puzzle Acc": "43.57",
+    "Hard Puzzle Acc": "0.83",
+    "Small Puzzle Acc": "39.38",
+    "Medium Puzzle Acc": "0.71",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "849.84"
+    "Reason Lens": "1043.90",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Qwen2.5-7B-Instruct",
@@ -502,8 +806,14 @@
     "No answer": "9.50",
     "Easy Puzzle Acc": "38.93",
     "Hard Puzzle Acc": "1.53",
+    "Small Puzzle Acc": "36.25",
+    "Medium Puzzle Acc": "1.43",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "850.93"
+    "Reason Lens": "850.93",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Meta-Llama-3-8B-Instruct",
@@ -513,8 +823,14 @@
     "No answer": "29.20",
     "Easy Puzzle Acc": "40.71",
     "Hard Puzzle Acc": "0.69",
+    "Small Puzzle Acc": "36.88",
+    "Medium Puzzle Acc": "0.36",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1216.40"
+    "Reason Lens": "1216.40",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Mistral-Nemo-Instruct-2407",
@@ -524,8 +840,14 @@
     "No answer": "1.60",
     "Easy Puzzle Acc": "38.93",
     "Hard Puzzle Acc": "1.25",
+    "Small Puzzle Acc": "35.31",
+    "Medium Puzzle Acc": "1.79",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "925.88"
+    "Reason Lens": "925.88",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Phi-3-mini-4k-instruct",
@@ -535,8 +857,14 @@
     "No answer": "59.00",
     "Easy Puzzle Acc": "38.21",
     "Hard Puzzle Acc": "1.25",
+    "Small Puzzle Acc": "35.94",
+    "Medium Puzzle Acc": "0.36",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "790.29"
+    "Reason Lens": "790.29",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Yi-1.5-34B-Chat",
@@ -546,19 +874,14 @@
     "No answer": "4.40",
     "Easy Puzzle Acc": "37.50",
     "Hard Puzzle Acc": "1.39",
+    "Small Puzzle Acc": "35.00",
+    "Medium Puzzle Acc": "1.07",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "869.65"
-  },
-  {
-    "Model": "Meta-Llama-3-8B-Instruct",
-    "Mode": "sampling",
-    "Puzzle Acc": "11.00",
-    "Cell Acc": "26.11",
-    "No answer": "22.30",
-    "Easy Puzzle Acc": "36.79",
-    "Hard Puzzle Acc": "0.97",
-    "Total Puzzles": 1000,
-    "Reason Lens": "1282.40"
+    "Reason Lens": "869.65",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gpt-3.5-turbo-0125",
@@ -568,8 +891,14 @@
     "No answer": "0.10",
     "Easy Puzzle Acc": "33.57",
     "Hard Puzzle Acc": "0.97",
+    "Small Puzzle Acc": "30.31",
+    "Medium Puzzle Acc": "1.07",
+    "Large Puzzle Acc": "0.50",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "820.66"
+    "Reason Lens": "820.66",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "command-r",
@@ -579,8 +908,14 @@
     "No answer": "1.50",
     "Easy Puzzle Acc": "32.14",
     "Hard Puzzle Acc": "1.25",
+    "Small Puzzle Acc": "30.31",
+    "Medium Puzzle Acc": "0.71",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1005.17"
+    "Reason Lens": "1005.17",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "reka-flash-20240226",
@@ -590,8 +925,14 @@
     "No answer": "18.70",
     "Easy Puzzle Acc": "30.71",
     "Hard Puzzle Acc": "0.97",
+    "Small Puzzle Acc": "28.44",
+    "Medium Puzzle Acc": "0.71",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1074.80"
+    "Reason Lens": "1074.80",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "mathstral-7B-v0.1",
@@ -601,8 +942,14 @@
     "No answer": "36.00",
     "Easy Puzzle Acc": "30.00",
     "Hard Puzzle Acc": "0.83",
+    "Small Puzzle Acc": "27.19",
+    "Medium Puzzle Acc": "1.07",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1148.16"
+    "Reason Lens": "1148.16",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Mixtral-8x7B-Instruct-v0.1",
@@ -612,8 +959,14 @@
     "No answer": "20.30",
     "Easy Puzzle Acc": "28.93",
     "Hard Puzzle Acc": "0.83",
+    "Small Puzzle Acc": "26.25",
+    "Medium Puzzle Acc": "1.07",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1177.21"
+    "Reason Lens": "1177.21",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Qwen2-7B-Instruct",
@@ -623,8 +976,31 @@
     "No answer": "24.40",
     "Easy Puzzle Acc": "29.29",
     "Hard Puzzle Acc": "0.28",
+    "Small Puzzle Acc": "26.25",
+    "Medium Puzzle Acc": "0.00",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
+    "Total Puzzles": 1000,
+    "Reason Lens": "1473.23",
+    "N_Mode": "single",
+    "N_Size": 1
+  },
+  {
+    "Model": "Llama-3.2-3B-Instruct@together",
+    "Mode": "greedy",
+    "Puzzle Acc": "7.40",
+    "Cell Acc": "13.14",
+    "No answer": "54.50",
+    "Easy Puzzle Acc": "25.71",
+    "Hard Puzzle Acc": "0.28",
+    "Small Puzzle Acc": "23.12",
+    "Medium Puzzle Acc": "0.00",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1473.23"
+    "Reason Lens": "963.47",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Phi-3.5-mini-instruct",
@@ -634,8 +1010,14 @@
     "No answer": "80.60",
     "Easy Puzzle Acc": "21.79",
     "Hard Puzzle Acc": "0.42",
+    "Small Puzzle Acc": "19.38",
+    "Medium Puzzle Acc": "0.71",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "718.43"
+    "Reason Lens": "718.43",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Qwen2.5-3B-Instruct",
@@ -645,8 +1027,14 @@
     "No answer": "56.70",
     "Easy Puzzle Acc": "17.14",
     "Hard Puzzle Acc": "0.00",
+    "Small Puzzle Acc": "15.00",
+    "Medium Puzzle Acc": "0.00",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "906.58"
+    "Reason Lens": "906.58",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "gemma-2-2b-it",
@@ -656,8 +1044,14 @@
     "No answer": "57.20",
     "Easy Puzzle Acc": "14.29",
     "Hard Puzzle Acc": "0.28",
+    "Small Puzzle Acc": "13.12",
+    "Medium Puzzle Acc": "0.00",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1032.89"
+    "Reason Lens": "1032.89",
+    "N_Mode": "single",
+    "N_Size": 1
   },
   {
     "Model": "Yi-1.5-9B-Chat",
@@ -667,7 +1061,13 @@
     "No answer": "11.30",
     "Easy Puzzle Acc": "8.21",
     "Hard Puzzle Acc": "0.00",
+    "Small Puzzle Acc": "7.19",
+    "Medium Puzzle Acc": "0.00",
+    "Large Puzzle Acc": "0.00",
+    "XL Puzzle Acc": "0.00",
     "Total Puzzles": 1000,
-    "Reason Lens": "1592.60"
+    "Reason Lens": "1592.60",
+    "N_Mode": "single",
+    "N_Size": 1
   }
 ]

app.py

@@ -66,9 +66,12 @@ def _tab_leaderboard():
         # default_main_df_no_task = default_main_df.copy()
         default_mode = "greedy"
         default_main_df = df_filters(default_mode, False)
+        print(default_main_df.columns)
+        # drop the Mode column
+        default_main_df = default_main_df.drop(columns=["Mode"])
         with gr.Row():
             with gr.Column(scale=5):
-                mode_selection_radio = gr.Radio(["greedy", "all"], show_label=False, elem_id="rank-column-radio", value=default_mode)
+                mode_selection_radio = gr.Radio(["greedy", "all"], show_label=False, elem_id="rank-column-radio", value=default_mode, visible=False)
         # with gr.Row():
         #     with gr.Column(scale=2):
 
@@ -140,7 +143,7 @@ def _tab_submit():
     and apply for the access for the [private dataset](https://huggingface.co/datasets/WildEval/ZebraLogic) that contains the truth solutions.
     """
 
-    gr.Markdown("## 🚀 Submit Your Results\n\n" + markdown_text, elem_classes="markdown-text")
+    gr.Markdown("## 🚀 Evaluate your models\n\n" + markdown_text, elem_classes="markdown-text")
 
 
 
@@ -159,7 +162,7 @@ def build_demo():
                 _tab_leaderboard()
             with gr.TabItem("🔍 Explore", elem_id="od-benchmark-tab-table", id=1):
                 _tab_explore()
-            with gr.TabItem("🚀 Submit Your Results", elem_id="od-benchmark-tab-table", id=3):
+            with gr.TabItem("🚀 Evaluate your models", elem_id="od-benchmark-tab-table", id=3):
                 _tab_submit()
 
             with gr.TabItem("📮 About Us", elem_id="od-benchmark-tab-table", id=4):
@@ -200,7 +203,9 @@ def data_load(result_file):
                 pass
     original_df = pd.DataFrame(raw_data)
     original_df = original_df[original_df["Total Puzzles"] == 1000]
+
     original_df = post_processing(original_df, column_names_main, ordered_columns=main_ordered_columns, click_url=click_url, rank_column=RANKING_COLUMN)
+    print(f"original_df.columns: {original_df.columns}")
     # print(original_df.columns)
 
 

constants.py

@@ -36,12 +36,16 @@ CITATION_TEXT = """
 
 column_names = OrderedDict({
     "Model": "Model",
-    "Mode": "Mode",
+    # "Mode": "Mode",
     "Puzzle Acc": "Puzzle Acc",
+    "Small Puzzle Acc": "Small",
+    "Medium Puzzle Acc": "Medium",
+    "Large Puzzle Acc": "Large",
+    "XL Puzzle Acc": "XL",
     "Cell Acc": "Cell Acc",
-    "No answer": "No answer",
-    "Easy Puzzle Acc": "Easy Puzzle Acc",
-    "Hard Puzzle Acc": "Hard Puzzle Acc",
+    # "No answer": "No answer",
+    # "Easy Puzzle Acc": "Easy Puzzle Acc",
+    # "Hard Puzzle Acc": "Hard Puzzle Acc",
     # "Total Puzzles": "Total Puzzles",
     # "Reason Lens": "Reason Lens",
 })
@@ -64,10 +68,18 @@ ORDERED_COLUMN_NAMES = [
     "Model",
     "Mode",
     "Puzzle Acc",
-    "Easy Puzzle Acc",
-    "Hard Puzzle Acc",
+    "XL",
+    "Large",
+    "Medium",
+    "Small",
+    "XL Puzzle Acc",
+    "Large Puzzle Acc",
+    "Medium Puzzle Acc",
+    "Small Puzzle Acc",
+    # "Easy Puzzle Acc",
+    # "Hard Puzzle Acc",
     "Cell Acc",
-    "No answer",
+    # "No answer",
 ]
 
 

eval_utils.py

@@ -1,6 +1,6 @@
-import json 
+import json
 from collections import defaultdict
-import os 
+import os
 from tabulate import tabulate
 from datasets import load_dataset
 
@@ -10,28 +10,28 @@ def load_private_solutions():
     global private_solutions
     private_zebra_data = load_dataset("WildEval/ZebraLogic", "grid_mode", split="test")
     for item in private_zebra_data:
-        private_solutions[item["id"]] = item["solution"] 
-    return 
+        private_solutions[item["id"]] = item["solution"]
+    return
 
 def load_model_results(run_name_folders):
     model_results = {}
     for run_name, folder in run_name_folders.items():
-        # iterate all json files under the folder 
+        # iterate all json files under the folder
         for filename in os.listdir(folder):
             filepath = os.path.join(folder, filename)
             if not filename.endswith(".json"):
                 continue
-            model_name = filename.replace(".json", "")  
+            model_name = filename.replace(".json", "")
             model_name = f"{model_name}%{run_name}"
-            model_results[model_name] = filepath  
+            model_results[model_name] = filepath
     return model_results
- 
+
 def extract_last_complete_json(s):
     # Stack to keep track of opening and closing braces
     stack = []
     last_json_start = None
     last_json_str = None
-    
+
     for i, char in enumerate(s):
         if char == '{':
             stack.append(i)
@@ -44,14 +44,14 @@ def extract_last_complete_json(s):
                     # Complete JSON object found
                     last_json_str = s[last_json_start:i+1]
                     last_json_start = None
-    
+
     # Load the last JSON object
     if last_json_str:
         try:
             return json.loads(last_json_str.replace("\n", ""))
         except json.JSONDecodeError:
             pass
-    
+
     return None
 
 def eval_each_puzzle(id, prediction_table):
@@ -64,15 +64,15 @@ def eval_each_puzzle(id, prediction_table):
     columns = solution["header"]
     assert columns[0] == "House"
     solution_table = {}
-    this_total_cells = 0 
+    this_total_cells = 0
     for i in range(num_houses):
-        solution_table[f'House {i+1}'] = {columns[j]: solution["rows"][i][j] for j in range(1, len(columns))} 
+        solution_table[f'House {i+1}'] = {columns[j]: solution["rows"][i][j] for j in range(1, len(columns))}
         this_total_cells += len(columns) - 1
-    
-    this_correct_cells = 0 # number in the solution_table 
+
+    this_correct_cells = 0 # number in the solution_table
     for house in solution_table:
-        for column in solution_table[house]: 
-            # if prediction_table[house][column] not exist then pass 
+        for column in solution_table[house]:
+            # if prediction_table[house][column] not exist then pass
             if house in prediction_table and column in prediction_table[house]:
                 truth_cell = solution_table[house][column].lower().strip()
                 if prediction_table[house][column] is None or len(prediction_table[house][column]) == 0:
@@ -82,23 +82,24 @@ def eval_each_puzzle(id, prediction_table):
                 elif type(prediction_table[house][column]) == str:
                     predicted_cell = prediction_table[house][column].lower().strip()
                 if truth_cell == predicted_cell:
-                    this_correct_cells += 1  
+                    this_correct_cells += 1
     return this_total_cells, this_correct_cells, private_solutions[id]
 
+"""
 def eval_model(model, filepath):
     global private_solutions
     with open(filepath, "r") as f:
         print(f"Processing {filepath}")
         data = json.load(f)
 
-    solved_puzzles = 0 
+    solved_puzzles = 0
     num_total_puzzles = len(data)
     correct_cells = 0
     total_cells = 0
-    no_asnwer = 0 
+    no_asnwer = 0
 
     num_total_puzzles_by_size = defaultdict(int)
-    solved_puzzles_by_size = defaultdict(int) 
+    solved_puzzles_by_size = defaultdict(int)
     reason_lens = []
     for item in data:
         # solution = item["solution"]
@@ -106,20 +107,20 @@ def eval_model(model, filepath):
         size = item["size"]
         num_total_puzzles_by_size[size] += 1
 
-        # Process the solution 
+        # Process the solution
         solution_table = {}
         num_houses = len(solution["rows"])
         columns = solution["header"]
         assert columns[0] == "House"
         solution_table = {}
-        this_total_cells = 0 
+        this_total_cells = 0
         for i in range(num_houses):
-            solution_table[f'House {i+1}'] = {columns[j]: solution["rows"][i][j] for j in range(1, len(columns))} 
+            solution_table[f'House {i+1}'] = {columns[j]: solution["rows"][i][j] for j in range(1, len(columns))}
             this_total_cells += len(columns) - 1
         total_cells += this_total_cells
 
         # Read and Parse the prediction from model output
-        prediction_str = item["output"][0]     
+        prediction_str = item["output"][0]
         prediction_json = extract_last_complete_json(prediction_str)
         if prediction_json is None or "solution" not in prediction_json:
             # print("-"*100)
@@ -128,16 +129,16 @@ def eval_model(model, filepath):
             # json.loads(prediction_str)
             no_asnwer += 1
             # print(item["id"])
-            continue 
+            continue
         reason = prediction_json.get("reasoning", "")
         prediction_table = prediction_json["solution"]
-        
+
         reason_lens.append(len(reason))
 
-        this_correct_cells = 0 # number in the solution_table 
+        this_correct_cells = 0 # number in the solution_table
         for house in solution_table:
-            for column in solution_table[house]: 
-                # if prediction_table[house][column] not exist then pass 
+            for column in solution_table[house]:
+                # if prediction_table[house][column] not exist then pass
                 if house in prediction_table and column in prediction_table[house]:
                     truth_cell = solution_table[house][column].lower().strip()
                     if prediction_table[house][column] is None or len(prediction_table[house][column]) == 0:
@@ -149,24 +150,24 @@ def eval_model(model, filepath):
                     else:
                         raise ValueError(f"Unknown type: {type(prediction_table[house][column])}")
                     if truth_cell == predicted_cell:
-                        this_correct_cells += 1  
+                        this_correct_cells += 1
         correct_cells += this_correct_cells
-        
+
         # compute puzzle success rate
         if this_correct_cells == this_total_cells:
             solved_puzzles += 1
             solved_puzzles_by_size[size] += 1
 
-        
-         
 
-    # # print the success rate by size; order the dict by size first  
-    sizes = sorted(num_total_puzzles_by_size.keys()) 
-    easy_sizes =  ['2*2', '2*3', '2*4', '2*5', '2*6', '3*2', '3*3',] 
+
+
+    # # print the success rate by size; order the dict by size first
+    sizes = sorted(num_total_puzzles_by_size.keys())
+    easy_sizes =  ['2*2', '2*3', '2*4', '2*5', '2*6', '3*2', '3*3',]
     hard_sizes =  ['3*4', '3*5', '4*2', '3*6', '4*3', '4*4', '5*2', '6*2', '4*5', '4*6', '5*3', '5*4', '5*5', '5*6', '6*3', '6*4', '6*5', '6*6']
-    
+
     easy_solved_puzzles = sum([solved_puzzles_by_size[size] for size in easy_sizes])
-    easy_total_puzzles = sum([num_total_puzzles_by_size[size] for size in easy_sizes]) 
+    easy_total_puzzles = sum([num_total_puzzles_by_size[size] for size in easy_sizes])
     hard_solved_puzzles = sum([solved_puzzles_by_size[size] for size in hard_sizes])
     hard_total_puzzles = sum([num_total_puzzles_by_size[size] for size in hard_sizes])
 
@@ -179,20 +180,20 @@ def eval_model(model, filepath):
     result["Puzzle Acc"] = f"{solved_puzzles/num_total_puzzles*100:.2f}"
     result["Cell Acc"] = f"{correct_cells/total_cells*100:.2f}"
     result["No answer"] = f"{no_asnwer/num_total_puzzles*100:.2f}"
-    result["Easy Puzzle Acc"] = f"{easy_solved_puzzles/easy_total_puzzles*100:.2f}" 
+    result["Easy Puzzle Acc"] = f"{easy_solved_puzzles/easy_total_puzzles*100:.2f}"
     result["Hard Puzzle Acc"] = f"{hard_solved_puzzles/hard_total_puzzles*100:.2f}"
     result["Total Puzzles"] = num_total_puzzles
     result["Reason Lens"] = f"{sum(reason_lens)/len(reason_lens):.2f}"
     return result
 
 
-def gen_results(run_name_folders): 
+def gen_results(run_name_folders):
     model_results = load_model_results(run_name_folders)
 
     columns = ["Model", "Mode", "Puzzle Acc", "Cell Acc", "No answer", "Easy Puzzle Acc", "Hard Puzzle Acc", "Total Puzzles", "Reason Lens"]
     rows = []
-    for model_name, filepath in model_results.items(): 
-        result = eval_model(model_name, filepath) 
+    for model_name, filepath in model_results.items():
+        result = eval_model(model_name, filepath)
         rows.append(result)
 
     # sort the rows by puzzle accuracy
@@ -203,7 +204,7 @@ def gen_results(run_name_folders):
     print(tabulate(table_data, headers=columns, tablefmt="fancy_outline", stralign="center", numalign="center"))
     # print(tabulate(rows, headers=columns, tablefmt="github"))
 
-    # write to json file 
+    # write to json file
     with open("result_dirs/zebra-grid.summary.json", "w") as f:
         json.dump(rows, f, indent=2)
 
@@ -212,6 +213,7 @@ if __name__ == "__main__":
     run_name_folders = {
         "greedy": "result_dirs/zebra-grid",
         "sampling": "result_dirs/zebra-grid/sampling",
-    } 
+    }
     load_private_solutions()
     gen_results(run_name_folders)
+"""