llm-qa-bench

Sleeping

App Files Files Community

dh-mc commited on Jun 19

Commit

8f1a330

•

1 Parent(s): e8b4f46

latest code/data

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

.gitattributes +0 -0
Llama-2-eval/data/datasets/ms_macro/dataset_info.json +0 -95
Llama-2-eval/data/datasets/ms_macro/state.json +0 -13
Llama-2-eval/data/results/results_full-a40.csv +0 -10
Llama-2-eval/data/results/results_full-l40.csv +0 -10
Llama-2-eval/notebook/baseline.ipynb +0 -1983
Llama-2-eval/notebook/metrics.ipynb +0 -1293
Makefile +25 -0
README.md +132 -17
app.py +62 -92
app_modules/llm_loader.py +8 -3
app_modules/utils.py +1 -153
data/datasets/WebQSP.test.wikidata.json +0 -0
data/{logs/Phi-3-mini-128k-instruct_mm_false_RP_1.060.txt → eval/Llama-2-13b-chat-hf_wd_true_RP_1.000-t2.json} +2 -2
data/{logs/Phi-3-mini-128k-instruct_mm_false_RP_1.120.txt → eval/Llama-2-13b-chat-hf_wd_true_RP_1.000-t2_evaluated.json} +2 -2
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.020-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.020-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.040-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.040-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.060-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.060-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.080-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.080-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.100-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.100-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.120-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.120-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.140-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.140-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.160-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.160-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.180-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.180-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.200-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.200-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.220-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.220-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.240-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.240-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.260-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.260-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.280-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.280-t2_evaluated.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.300-t2.json +3 -0
data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.300-t2_evaluated.json +3 -0
Llama-2-eval/data/datasets/ms_macro/data-00000-of-00001.arrow → data/eval/Llama-2-70b-chat-hf_wd_RP_1.000-t2.json +2 -2
data/{logs/Phi-3-mini-128k-instruct_mm_false_RP_1.000.txt → eval/Llama-2-70b-chat-hf_wd_RP_1.000-t2_evaluated.json} +2 -2
data/eval/Llama-2-70b-chat-hf_wd_RP_1.020-t2.json +3 -0
data/eval/Llama-2-70b-chat-hf_wd_RP_1.020-t2_evaluated.json +3 -0
data/eval/Llama-2-70b-chat-hf_wd_RP_1.040-t2.json +3 -0

.gitattributes CHANGED Viewed

The diff for this file is too large to render. See raw diff

Llama-2-eval/data/datasets/ms_macro/dataset_info.json DELETED Viewed

@@ -1,95 +0,0 @@
-{
-  "builder_name": "parquet",
-  "citation": "",
-  "config_name": "default",
-  "dataset_name": "ms-macro-wellformed_only",
-  "dataset_size": 726469485,
-  "description": "",
-  "download_checksums": {
-    "hf://datasets/zhengxuanzenwu/ms-macro-wellformed_only@d6a0dd610474a02e63224176514c0073bb723c7c/data/train-00000-of-00002-0a6f58dc7ee03f61.parquet": {
-      "num_bytes": 164629356,
-      "checksum": null
-    },
-    "hf://datasets/zhengxuanzenwu/ms-macro-wellformed_only@d6a0dd610474a02e63224176514c0073bb723c7c/data/train-00001-of-00002-5262fd5ec1911156.parquet": {
-      "num_bytes": 164721520,
-      "checksum": null
-    },
-    "hf://datasets/zhengxuanzenwu/ms-macro-wellformed_only@d6a0dd610474a02e63224176514c0073bb723c7c/data/test-00000-of-00001-f965dd5a841915d3.parquet": {
-      "num_bytes": 26541566,
-      "checksum": null
-    }
-  },
-  "download_size": 355892442,
-  "features": {
-    "answers": {
-      "feature": {
-        "dtype": "string",
-        "_type": "Value"
-      },
-      "_type": "Sequence"
-    },
-    "passages": {
-      "feature": {
-        "is_selected": {
-          "dtype": "int32",
-          "_type": "Value"
-        },
-        "passage_text": {
-          "dtype": "string",
-          "_type": "Value"
-        },
-        "url": {
-          "dtype": "string",
-          "_type": "Value"
-        }
-      },
-      "_type": "Sequence"
-    },
-    "query": {
-      "dtype": "string",
-      "_type": "Value"
-    },
-    "query_id": {
-      "dtype": "int32",
-      "_type": "Value"
-    },
-    "query_type": {
-      "dtype": "string",
-      "_type": "Value"
-    },
-    "wellFormedAnswers": {
-      "feature": {
-        "dtype": "string",
-        "_type": "Value"
-      },
-      "_type": "Sequence"
-    }
-  },
-  "homepage": "",
-  "license": "",
-  "size_in_bytes": 1082361927,
-  "splits": {
-    "train": {
-      "name": "train",
-      "num_bytes": 674327331,
-      "num_examples": 153725,
-      "shard_lengths": [
-        116863,
-        36862
-      ],
-      "dataset_name": "ms-macro-wellformed_only"
-    },
-    "test": {
-      "name": "test",
-      "num_bytes": 52142154,
-      "num_examples": 12467,
-      "dataset_name": "ms-macro-wellformed_only"
-    }
-  },
-  "version": {
-    "version_str": "0.0.0",
-    "major": 0,
-    "minor": 0,
-    "patch": 0
-  }
-}

Llama-2-eval/data/datasets/ms_macro/state.json DELETED Viewed

@@ -1,13 +0,0 @@
-{
-  "_data_files": [
-    {
-      "filename": "data-00000-of-00001.arrow"
-    }
-  ],
-  "_fingerprint": "fe2a26ddba75833a",
-  "_format_columns": null,
-  "_format_kwargs": {},
-  "_format_type": null,
-  "_output_all_columns": false,
-  "_split": "test"
-}

Llama-2-eval/data/results/results_full-a40.csv DELETED Viewed

@@ -1,10 +0,0 @@
-model_name,repetition_penalty,generation_time,evaluation_time,total_tokens,total_words,tokens_per_second,tokens_per_word,numeric_bleu,numeric_rougeL,description_bleu,description_rougeL,entity_bleu,entity_rougeL,person_bleu,person_rougeL,location_bleu,location_rougeL,overall_bleu,overall_rougeL,total_words_over_total_tokens
-gpt-4,,2696.407,1.772,34069,29552,12.635,1.153,0.1732,0.3337,0.1895,0.3248,0.1654,0.3117,0.1879,0.3286,0.4068,0.6213,0.1969,0.3843,0.867
-gpt-3.5-turbo,,1492.921,1.786,34353,29917,23.011,1.148,0.1606,0.3178,0.1623,0.2582,0.1296,0.2939,0.2024,0.3462,0.3632,0.5953,0.1761,0.3623,0.871
-Llama-2-13b-chat-hf,1.12,2133.992,1.66,33389,24007,15.646,1.391,0.163,0.3345,0.2031,0.3756,0.1632,0.2962,0.1388,0.3045,0.3423,0.5302,0.1846,0.3694,0.719
-vicuna-13b-v1.1,1.095,2212.946,1.682,35308,26456,15.955,1.335,0.1285,0.2319,0.1991,0.2812,0.1556,0.2644,0.2009,0.2768,0.3159,0.5761,0.1853,0.3276,0.749
-Llama-2-7b-chat-hf,1.19,1280.314,1.793,34349,23987,26.829,1.432,0.1274,0.2383,0.1836,0.2621,0.1572,0.2754,0.17,0.2911,0.3631,0.5383,0.1781,0.3209,0.698
-vicuna-7b-v1.1,1.095,975.73,1.574,25932,18714,26.577,1.386,0.1664,0.2838,0.2227,0.3118,0.166,0.2351,0.259,0.2753,0.4542,0.5838,0.2218,0.3379,0.722
-wizardLM-7B-HF,1.095,1265.93,1.667,33570,24003,26.518,1.399,0.1367,0.2584,0.2027,0.2882,0.1358,0.2592,0.1985,0.3085,0.4154,0.5794,0.1866,0.3384,0.715
-mpt-7b-instruct,1.05,2071.066,1.42,12374,9927,5.975,1.246,0.1804,0.285,0.2589,0.2556,0.2383,0.2468,0.2635,0.2571,0.3512,0.4042,0.2509,0.2897,0.802
-gpt4all-j,1.095,5603.316,1.706,31502,27099,5.622,1.162,0.1236,0.2406,0.1708,0.2511,0.143,0.255,0.194,0.2941,0.3721,0.5337,0.1737,0.3153,0.860

Llama-2-eval/data/results/results_full-l40.csv DELETED Viewed

@@ -1,10 +0,0 @@
-model_name,repetition_penalty,generation_time,evaluation_time,total_tokens,total_words,tokens_per_second,tokens_per_word,words_per_token_l40,words_per_second,numeric_bleu,numeric_rougeL,description_bleu,description_rougeL,entity_bleu,entity_rougeL,person_bleu,person_rougeL,location_bleu,location_rougeL,overall_bleu,overall_rougeL,total_words_over_total_tokens
-gpt-4,,2696.407,1.772,34069,29552,12.635,1.153,0.867,,0.1732,0.3337,0.1895,0.3248,0.1654,0.3117,0.1879,0.3286,0.4068,0.6213,0.1969,0.3843,0.867
-gpt-3.5-turbo,,1492.921,1.786,34353,29917,23.011,1.148,0.871,,0.1606,0.3178,0.1623,0.2582,0.1296,0.2939,0.2024,0.3462,0.3632,0.5953,0.1761,0.3623,0.871
-Llama-2-13b-chat-hf,1.12,1687.637,1.785,32808,23575,19.44,1.392,0.718,13.969,0.1612,0.3305,0.2061,0.3701,0.1675,0.3018,0.141,0.305,0.3394,0.5288,0.1866,0.368,0.719
-vicuna-13b-v1.1,1.095,1799.165,2.197,35543,26613,19.755,1.336,0.749,14.792,0.1274,0.2321,0.1994,0.2834,0.154,0.2631,0.1984,0.2773,0.3194,0.5759,0.1844,0.3256,0.749
-Llama-2-7b-chat-hf,1.19,1002.46,6.606,34686,24229,34.601,1.432,0.698,24.170,0.1269,0.2404,0.1824,0.2614,0.157,0.2769,0.1687,0.2896,0.3565,0.5378,0.177,0.3214,0.699
-vicuna-7b-v1.1,1.095,758.227,1.432,25827,18638,34.062,1.386,0.722,24.581,0.1673,0.2859,0.2221,0.3096,0.1655,0.2327,0.2576,0.2717,0.4564,0.5849,0.2216,0.3387,0.722
-wizardLM-7B-HF,1.095,998.702,1.683,33674,23996,33.718,1.403,0.713,24.027,0.1372,0.259,0.2046,0.2878,0.1354,0.2588,0.1982,0.3083,0.4154,0.5769,0.187,0.3383,0.713
-mpt-7b-instruct,1.05,1622.435,1.338,12607,10139,7.77,1.243,0.805,6.249,0.1751,0.2756,0.2569,0.2625,0.2349,0.2456,0.2466,0.2566,0.3522,0.4049,0.2455,0.2889,0.804
-gpt4all-j,1.095,3794.429,1.611,31719,27286,8.359,1.162,0.861,7.191,0.1262,0.2443,0.1669,0.251,0.1394,0.2505,0.1937,0.2968,0.3693,0.5348,0.1719,0.3151,0.860

Llama-2-eval/notebook/baseline.ipynb DELETED Viewed

@@ -1,1983 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "a6d96660",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "True"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import os\n",
-    "from dotenv import load_dotenv\n",
-    "\n",
-    "load_dotenv()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "7510ab87",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "DatasetDict({\n",
-       "    train: Dataset({\n",
-       "        features: ['answers', 'passages', 'query', 'query_id', 'query_type', 'wellFormedAnswers'],\n",
-       "        num_rows: 153725\n",
-       "    })\n",
-       "    test: Dataset({\n",
-       "        features: ['answers', 'passages', 'query', 'query_id', 'query_type', 'wellFormedAnswers'],\n",
-       "        num_rows: 12467\n",
-       "    })\n",
-       "})"
-      ]
-     },
-     "execution_count": 11,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from datasets import load_dataset\n",
-    "\n",
-    "dataset = load_dataset(\"zhengxuanzenwu/ms-macro-wellformed_only\")\n",
-    "dataset"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "1f4f0e76",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>answers</th>\n",
-       "      <th>passages</th>\n",
-       "      <th>query</th>\n",
-       "      <th>query_id</th>\n",
-       "      <th>query_type</th>\n",
-       "      <th>wellFormedAnswers</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>[2,662]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>albany mn population</td>\n",
-       "      <td>15177</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The population of Albany, Minnesota is 2,662. ]</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "      <td>{'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...</td>\n",
-       "      <td>current weather in volcano, ca</td>\n",
-       "      <td>114414</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>[Hippocrates]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...</td>\n",
-       "      <td>____________________ is considered the father ...</td>\n",
-       "      <td>9083</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[Hippocrates is considered the father of moder...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>[120 days from the date of the Note.]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>how many days is an appraisal good for a fanni...</td>\n",
-       "      <td>281439</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[An appraisal is good for 120 days from the da...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>[From $26,000 to $39,000 a year]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>average pharmacy tech salary</td>\n",
-       "      <td>40287</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The average salary for a pharmacy technician ...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                             answers  \\\n",
-       "0                                            [2,662]   \n",
-       "1  [The Volcano forecast for Apr 12 is 52 degrees...   \n",
-       "2                                      [Hippocrates]   \n",
-       "3              [120 days from the date of the Note.]   \n",
-       "4                   [From $26,000 to $39,000 a year]   \n",
-       "\n",
-       "                                            passages  \\\n",
-       "0  {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...   \n",
-       "1  {'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...   \n",
-       "2  {'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...   \n",
-       "3  {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "4  {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "\n",
-       "                                               query  query_id   query_type  \\\n",
-       "0                               albany mn population     15177      NUMERIC   \n",
-       "1                     current weather in volcano, ca    114414  DESCRIPTION   \n",
-       "2  ____________________ is considered the father ...      9083  DESCRIPTION   \n",
-       "3  how many days is an appraisal good for a fanni...    281439      NUMERIC   \n",
-       "4                       average pharmacy tech salary     40287      NUMERIC   \n",
-       "\n",
-       "                                   wellFormedAnswers  \n",
-       "0   [The population of Albany, Minnesota is 2,662. ]  \n",
-       "1  [The Volcano forecast for Apr 12 is 52 degrees...  \n",
-       "2  [Hippocrates is considered the father of moder...  \n",
-       "3  [An appraisal is good for 120 days from the da...  \n",
-       "4  [The average salary for a pharmacy technician ...  "
-      ]
-     },
-     "execution_count": 12,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df = dataset[\"test\"].to_pandas()\n",
-    "df.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "id": "3e9b4cef",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'answers': ['2,662'],\n",
-       " 'passages': {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0],\n",
-       "  'passage_text': ['City of Albany, MN Zip Codes. City of Albany, MN Demographic Information. * Demographic data is based on information taken from the 2000 Census. City of Albany, MN covers 1 Area Code. City of Albany, MN covers 1 Zip Code. 15 Cities within 15 Miles of the City of Albany, MN.',\n",
-       "   'Place of birth for U.S.-born residents: 70% of the 56307 zip code residents lived in the same house 5 years ago. Out of people who lived in different houses, 71% lived in this county. Out of people who lived in different counties, 50% lived in Minnesota. 92% of the 56307 zip code residents lived in the same house 1 year ago.',\n",
-       "   'For the unincorporated community in southeast Minnesota named West Albany, see West Albany, Minnesota. Albany is a city in Stearns County, Minnesota, United States. The population was 2,561 at the 2010 census. It is part of the St. Cloud Metropolitan Statistical Area.',\n",
-       "   'Albany, Minnesota, as per 2017 US Census estimate, has a community population of 2,662 people. Albany is located in Stearns County, 20 miles west of St. Cloud and 80 miles northwest of Minneapolis/St. Paul on Interstate 94 (I-94). Albany has direct access to State Highway 238, which originates in Albany.',\n",
-       "   'Sponsored Topics. Albany is a city in Stearns County, Minnesota, United States. The population was 2,561 at the 2010 census. It is part of the St. Cloud Metropolitan Statistical Area.',\n",
-       "   'Recent posts about Albany, Minnesota on our local forum with over 2,000,000 registered users. Albany is mentioned 87 times on our forum: Latest news from Albany, MN collected exclusively by city-data.com from local newspapers, TV, and radio stations. Ancestries: German (55.6%), Irish (10.0%), Polish (5.9%), Norwegian (5.4%), Swedish (2.8%), United States (2.6%).',\n",
-       "   \"For population 25 years and over in 56307: 1  High school or higher: 87.4%. 2  Bachelor's degree or higher: 15.4%. 3  Graduate or professional degree: 3.3 4 %. Unemployed: 3. 5 2%. Mean travel time to work (commute): 23.6 minutes.\",\n",
-       "   \"For population 25 years and over in Albany: 1  High school or higher: 86.7%. 2  Bachelor's degree or higher: 15.4%. 3  Graduate or professional degree: 4.4 4 %. Unemployed: 4. 5 3%. Mean travel time to work (commute): 23.0 minutes.\"],\n",
-       "  'url': ['http://zipcode.org/city/MN/ALBANY',\n",
-       "   'http://www.city-data.com/zips/56307.html',\n",
-       "   'https://en.wikipedia.org/wiki/Albany,_Minnesota',\n",
-       "   'http://ci.albany.mn.us/index.asp?SEC=A8341FEC-6B8C-47D2-926B-75A89ED4C539&Type=B_BASIC',\n",
-       "   'https://www.mapquest.com/us/mn/albany-282023394',\n",
-       "   'http://www.city-data.com/city/Albany-Minnesota.html',\n",
-       "   'http://www.city-data.com/zips/56307.html',\n",
-       "   'http://www.city-data.com/city/Albany-Minnesota.html']},\n",
-       " 'query': 'albany mn population',\n",
-       " 'query_id': 15177,\n",
-       " 'query_type': 'NUMERIC',\n",
-       " 'wellFormedAnswers': ['The population of Albany, Minnesota is 2,662. ']}"
-      ]
-     },
-     "execution_count": 15,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "test = dataset[\"test\"]\n",
-    "test[0]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 24,
-   "id": "104dfbea",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "12467"
-      ]
-     },
-     "execution_count": 24,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "test.num_rows"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 35,
-   "id": "01b3a886",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "({'NUMERIC': 3685,\n",
-       "  'DESCRIPTION': 5487,\n",
-       "  'ENTITY': 1077,\n",
-       "  'PERSON': 868,\n",
-       "  'LOCATION': 1350},\n",
-       " {'NUMERIC': 179,\n",
-       "  'DESCRIPTION': 218,\n",
-       "  'ENTITY': 2403,\n",
-       "  'LOCATION': 2559,\n",
-       "  'PERSON': 3966})"
-      ]
-     },
-     "execution_count": 35,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "counts = {}\n",
-    "indices = {}\n",
-    "size = 100\n",
-    "for i in range(test.num_rows):\n",
-    "    row = test[i]\n",
-    "    query_type = row[\"query_type\"]\n",
-    "    if query_type in counts:\n",
-    "        counts[query_type] += 1\n",
-    "    else:\n",
-    "        counts[query_type] = 1\n",
-    "    if counts[query_type] == size:\n",
-    "        indices[query_type] = i\n",
-    "counts, indices"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 39,
-   "id": "967bc1cd",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# create new dataset exluding those idx\n",
-    "baseline = test.select(\n",
-    "    (i for i in range(len(test)) if i <= indices[test[i][\"query_type\"]])\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 40,
-   "id": "9a5fcad5",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Dataset({\n",
-       "    features: ['answers', 'passages', 'query', 'query_id', 'query_type', 'wellFormedAnswers'],\n",
-       "    num_rows: 500\n",
-       "})"
-      ]
-     },
-     "execution_count": 40,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "baseline"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 41,
-   "id": "0524a973",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>answers</th>\n",
-       "      <th>passages</th>\n",
-       "      <th>query</th>\n",
-       "      <th>query_id</th>\n",
-       "      <th>query_type</th>\n",
-       "      <th>wellFormedAnswers</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>[2,662]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>albany mn population</td>\n",
-       "      <td>15177</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The population of Albany, Minnesota is 2,662. ]</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "      <td>{'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...</td>\n",
-       "      <td>current weather in volcano, ca</td>\n",
-       "      <td>114414</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>[Hippocrates]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...</td>\n",
-       "      <td>____________________ is considered the father ...</td>\n",
-       "      <td>9083</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[Hippocrates is considered the father of moder...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>[120 days from the date of the Note.]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>how many days is an appraisal good for a fanni...</td>\n",
-       "      <td>281439</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[An appraisal is good for 120 days from the da...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>[From $26,000 to $39,000 a year]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>average pharmacy tech salary</td>\n",
-       "      <td>40287</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The average salary for a pharmacy technician ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>495</th>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...</td>\n",
-       "      <td>the pool shower company</td>\n",
-       "      <td>518269</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>496</th>\n",
-       "      <td>[Hanson]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>longest tenured american football players</td>\n",
-       "      <td>442806</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Hanson is the longest tenured American footba...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>497</th>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "      <td>{'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...</td>\n",
-       "      <td>mt. view baptist in pendleton sc</td>\n",
-       "      <td>460250</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>498</th>\n",
-       "      <td>[Honeysuckle Weeks]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>what actress disappeared for a while</td>\n",
-       "      <td>549739</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The actress disappeared for a while Honeysuck...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>499</th>\n",
-       "      <td>[African-Nguni]</td>\n",
-       "      <td>{'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>what ethnicity is the surname sabol</td>\n",
-       "      <td>658265</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The ethnicity of the surname Sabol is African...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>500 rows × 6 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                               answers  \\\n",
-       "0                                              [2,662]   \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...   \n",
-       "2                                        [Hippocrates]   \n",
-       "3                [120 days from the date of the Note.]   \n",
-       "4                     [From $26,000 to $39,000 a year]   \n",
-       "..                                                 ...   \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...   \n",
-       "496                                           [Hanson]   \n",
-       "497  [Mount Able Baptist Church is located at the a...   \n",
-       "498                                [Honeysuckle Weeks]   \n",
-       "499                                    [African-Nguni]   \n",
-       "\n",
-       "                                              passages  \\\n",
-       "0    {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...   \n",
-       "1    {'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...   \n",
-       "2    {'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...   \n",
-       "3    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "4    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "..                                                 ...   \n",
-       "495  {'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...   \n",
-       "496  {'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...   \n",
-       "497  {'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...   \n",
-       "498  {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...   \n",
-       "499  {'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...   \n",
-       "\n",
-       "                                                 query  query_id   query_type  \\\n",
-       "0                                 albany mn population     15177      NUMERIC   \n",
-       "1                       current weather in volcano, ca    114414  DESCRIPTION   \n",
-       "2    ____________________ is considered the father ...      9083  DESCRIPTION   \n",
-       "3    how many days is an appraisal good for a fanni...    281439      NUMERIC   \n",
-       "4                         average pharmacy tech salary     40287      NUMERIC   \n",
-       "..                                                 ...       ...          ...   \n",
-       "495                            the pool shower company    518269       PERSON   \n",
-       "496          longest tenured american football players    442806       PERSON   \n",
-       "497                   mt. view baptist in pendleton sc    460250       PERSON   \n",
-       "498               what actress disappeared for a while    549739       PERSON   \n",
-       "499                what ethnicity is the surname sabol    658265       PERSON   \n",
-       "\n",
-       "                                     wellFormedAnswers  \n",
-       "0     [The population of Albany, Minnesota is 2,662. ]  \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...  \n",
-       "2    [Hippocrates is considered the father of moder...  \n",
-       "3    [An appraisal is good for 120 days from the da...  \n",
-       "4    [The average salary for a pharmacy technician ...  \n",
-       "..                                                 ...  \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...  \n",
-       "496  [Hanson is the longest tenured American footba...  \n",
-       "497  [Mount Able Baptist Church is located at the a...  \n",
-       "498  [The actress disappeared for a while Honeysuck...  \n",
-       "499  [The ethnicity of the surname Sabol is African...  \n",
-       "\n",
-       "[500 rows x 6 columns]"
-      ]
-     },
-     "execution_count": 41,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "baseline.to_pandas()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 42,
-   "id": "57a195e0",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "66abd394cb054cf1b7459e92d4763d02",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Saving the dataset (0/1 shards):   0%|          | 0/500 [00:00<?, ? examples/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "baseline.save_to_disk(\"../data/datasets/ms_macro/\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 44,
-   "id": "b72bf3f9",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Dataset({\n",
-       "    features: ['answers', 'passages', 'query', 'query_id', 'query_type', 'wellFormedAnswers'],\n",
-       "    num_rows: 500\n",
-       "})"
-      ]
-     },
-     "execution_count": 44,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from datasets import load_from_disk\n",
-    "\n",
-    "new_ds = load_from_disk(\"../data/datasets/ms_macro/\")\n",
-    "new_ds"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 45,
-   "id": "051bd771",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "({'NUMERIC': 100,\n",
-       "  'DESCRIPTION': 100,\n",
-       "  'ENTITY': 100,\n",
-       "  'PERSON': 100,\n",
-       "  'LOCATION': 100},\n",
-       " {'NUMERIC': 179,\n",
-       "  'DESCRIPTION': 215,\n",
-       "  'ENTITY': 443,\n",
-       "  'LOCATION': 461,\n",
-       "  'PERSON': 499})"
-      ]
-     },
-     "execution_count": 45,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "counts = {}\n",
-    "indices = {}\n",
-    "size = 100\n",
-    "for i in range(new_ds.num_rows):\n",
-    "    row = new_ds[i]\n",
-    "    query_type = row[\"query_type\"]\n",
-    "    if query_type in counts:\n",
-    "        counts[query_type] += 1\n",
-    "    else:\n",
-    "        counts[query_type] = 1\n",
-    "    if counts[query_type] == size:\n",
-    "        indices[query_type] = i\n",
-    "counts, indices"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 46,
-   "id": "db48dcc4",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>answers</th>\n",
-       "      <th>passages</th>\n",
-       "      <th>query</th>\n",
-       "      <th>query_id</th>\n",
-       "      <th>query_type</th>\n",
-       "      <th>wellFormedAnswers</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>[2,662]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>albany mn population</td>\n",
-       "      <td>15177</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The population of Albany, Minnesota is 2,662. ]</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "      <td>{'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...</td>\n",
-       "      <td>current weather in volcano, ca</td>\n",
-       "      <td>114414</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>[Hippocrates]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...</td>\n",
-       "      <td>____________________ is considered the father ...</td>\n",
-       "      <td>9083</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[Hippocrates is considered the father of moder...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>[120 days from the date of the Note.]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>how many days is an appraisal good for a fanni...</td>\n",
-       "      <td>281439</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[An appraisal is good for 120 days from the da...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>[From $26,000 to $39,000 a year]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>average pharmacy tech salary</td>\n",
-       "      <td>40287</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The average salary for a pharmacy technician ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>495</th>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...</td>\n",
-       "      <td>the pool shower company</td>\n",
-       "      <td>518269</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>496</th>\n",
-       "      <td>[Hanson]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>longest tenured american football players</td>\n",
-       "      <td>442806</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Hanson is the longest tenured American footba...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>497</th>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "      <td>{'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...</td>\n",
-       "      <td>mt. view baptist in pendleton sc</td>\n",
-       "      <td>460250</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>498</th>\n",
-       "      <td>[Honeysuckle Weeks]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>what actress disappeared for a while</td>\n",
-       "      <td>549739</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The actress disappeared for a while Honeysuck...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>499</th>\n",
-       "      <td>[African-Nguni]</td>\n",
-       "      <td>{'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>what ethnicity is the surname sabol</td>\n",
-       "      <td>658265</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The ethnicity of the surname Sabol is African...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>500 rows × 6 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                               answers  \\\n",
-       "0                                              [2,662]   \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...   \n",
-       "2                                        [Hippocrates]   \n",
-       "3                [120 days from the date of the Note.]   \n",
-       "4                     [From $26,000 to $39,000 a year]   \n",
-       "..                                                 ...   \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...   \n",
-       "496                                           [Hanson]   \n",
-       "497  [Mount Able Baptist Church is located at the a...   \n",
-       "498                                [Honeysuckle Weeks]   \n",
-       "499                                    [African-Nguni]   \n",
-       "\n",
-       "                                              passages  \\\n",
-       "0    {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...   \n",
-       "1    {'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...   \n",
-       "2    {'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...   \n",
-       "3    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "4    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "..                                                 ...   \n",
-       "495  {'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...   \n",
-       "496  {'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...   \n",
-       "497  {'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...   \n",
-       "498  {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...   \n",
-       "499  {'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...   \n",
-       "\n",
-       "                                                 query  query_id   query_type  \\\n",
-       "0                                 albany mn population     15177      NUMERIC   \n",
-       "1                       current weather in volcano, ca    114414  DESCRIPTION   \n",
-       "2    ____________________ is considered the father ...      9083  DESCRIPTION   \n",
-       "3    how many days is an appraisal good for a fanni...    281439      NUMERIC   \n",
-       "4                         average pharmacy tech salary     40287      NUMERIC   \n",
-       "..                                                 ...       ...          ...   \n",
-       "495                            the pool shower company    518269       PERSON   \n",
-       "496          longest tenured american football players    442806       PERSON   \n",
-       "497                   mt. view baptist in pendleton sc    460250       PERSON   \n",
-       "498               what actress disappeared for a while    549739       PERSON   \n",
-       "499                what ethnicity is the surname sabol    658265       PERSON   \n",
-       "\n",
-       "                                     wellFormedAnswers  \n",
-       "0     [The population of Albany, Minnesota is 2,662. ]  \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...  \n",
-       "2    [Hippocrates is considered the father of moder...  \n",
-       "3    [An appraisal is good for 120 days from the da...  \n",
-       "4    [The average salary for a pharmacy technician ...  \n",
-       "..                                                 ...  \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...  \n",
-       "496  [Hanson is the longest tenured American footba...  \n",
-       "497  [Mount Able Baptist Church is located at the a...  \n",
-       "498  [The actress disappeared for a while Honeysuck...  \n",
-       "499  [The ethnicity of the surname Sabol is African...  \n",
-       "\n",
-       "[500 rows x 6 columns]"
-      ]
-     },
-     "execution_count": 46,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "new_ds.to_pandas()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 47,
-   "id": "7ed0c22d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "\"\"\"\n",
-    "Official evaluation script for QAConv, modified from SQuAD 2.0.\n",
-    "\n",
-    " * Copyright (c) 2021, salesforce.com, inc.\n",
-    " * All rights reserved.\n",
-    " * SPDX-License-Identifier: BSD-3-Clause\n",
-    " * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n",
-    "\n",
-    "\"\"\"\n",
-    "\n",
-    "import collections\n",
-    "import re\n",
-    "import string\n",
-    "\n",
-    "\n",
-    "def normalize_answer(s):\n",
-    "    \"\"\"Lower text and remove punctuation, articles and extra whitespace.\"\"\"\n",
-    "\n",
-    "    def remove_articles(text):\n",
-    "        regex = re.compile(r\"\\b(a|an|the)\\b\", re.UNICODE)\n",
-    "        return re.sub(regex, \" \", text)\n",
-    "\n",
-    "    def white_space_fix(text):\n",
-    "        return \" \".join(text.split())\n",
-    "\n",
-    "    def remove_punc(text):\n",
-    "        exclude = set(string.punctuation)\n",
-    "        return \"\".join(ch for ch in text if ch not in exclude)\n",
-    "\n",
-    "    def lower(text):\n",
-    "        return text.lower()\n",
-    "\n",
-    "    return white_space_fix(remove_articles(remove_punc(lower(s))))\n",
-    "\n",
-    "\n",
-    "def get_tokens(s):\n",
-    "    if not s:\n",
-    "        return []\n",
-    "    return normalize_answer(s).split()\n",
-    "\n",
-    "\n",
-    "def compute_exact(a_gold, a_pred):\n",
-    "    return int(normalize_answer(a_gold) == normalize_answer(a_pred))\n",
-    "\n",
-    "\n",
-    "def compute_f1(a_gold, a_pred):\n",
-    "    gold_toks = get_tokens(a_gold)\n",
-    "    pred_toks = get_tokens(a_pred)\n",
-    "    common = collections.Counter(gold_toks) & collections.Counter(pred_toks)\n",
-    "    num_same = sum(common.values())\n",
-    "    if len(gold_toks) == 0 or len(pred_toks) == 0:\n",
-    "        # If either is no-answer, then F1 is 1 if they agree, 0 otherwise\n",
-    "        return int(gold_toks == pred_toks)\n",
-    "    if num_same == 0:\n",
-    "        return 0\n",
-    "    precision = 1.0 * num_same / len(pred_toks)\n",
-    "    recall = 1.0 * num_same / len(gold_toks)\n",
-    "    f1 = (2 * precision * recall) / (precision + recall)\n",
-    "    return f1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 49,
-   "id": "d9ff4756",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d8a8d425f60a467eb56f6a13a50ed94b",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Map:   0%|          | 0/500 [00:00<?, ? examples/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": [
-       "Dataset({\n",
-       "    features: ['answers', 'passages', 'query', 'query_id', 'query_type', 'wellFormedAnswers', 'EM', 'F1'],\n",
-       "    num_rows: 500\n",
-       "})"
-      ]
-     },
-     "execution_count": 49,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "result_all = new_ds.map(\n",
-    "    lambda record, idx: {\n",
-    "        \"EM\": compute_exact(record[\"wellFormedAnswers\"][0], record[\"answers\"][0]),\n",
-    "        \"F1\": compute_f1(record[\"wellFormedAnswers\"][0], record[\"answers\"][0]),\n",
-    "    },\n",
-    "    batched=False,\n",
-    "    with_indices=True,\n",
-    ")\n",
-    "result_all"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 50,
-   "id": "31402fb2",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>answers</th>\n",
-       "      <th>passages</th>\n",
-       "      <th>query</th>\n",
-       "      <th>query_id</th>\n",
-       "      <th>query_type</th>\n",
-       "      <th>wellFormedAnswers</th>\n",
-       "      <th>EM</th>\n",
-       "      <th>F1</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>[2,662]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>albany mn population</td>\n",
-       "      <td>15177</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The population of Albany, Minnesota is 2,662. ]</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.285714</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "      <td>{'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...</td>\n",
-       "      <td>current weather in volcano, ca</td>\n",
-       "      <td>114414</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "      <td>1</td>\n",
-       "      <td>1.000000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>[Hippocrates]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...</td>\n",
-       "      <td>____________________ is considered the father ...</td>\n",
-       "      <td>9083</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[Hippocrates is considered the father of moder...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.250000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>[120 days from the date of the Note.]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>how many days is an appraisal good for a fanni...</td>\n",
-       "      <td>281439</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[An appraisal is good for 120 days from the da...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.631579</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>[From $26,000 to $39,000 a year]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>average pharmacy tech salary</td>\n",
-       "      <td>40287</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The average salary for a pharmacy technician ...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.500000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>495</th>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...</td>\n",
-       "      <td>the pool shower company</td>\n",
-       "      <td>518269</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "      <td>1</td>\n",
-       "      <td>1.000000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>496</th>\n",
-       "      <td>[Hanson]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>longest tenured american football players</td>\n",
-       "      <td>442806</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Hanson is the longest tenured American footba...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.250000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>497</th>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "      <td>{'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...</td>\n",
-       "      <td>mt. view baptist in pendleton sc</td>\n",
-       "      <td>460250</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "      <td>1</td>\n",
-       "      <td>1.000000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>498</th>\n",
-       "      <td>[Honeysuckle Weeks]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>what actress disappeared for a while</td>\n",
-       "      <td>549739</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The actress disappeared for a while Honeysuck...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.500000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>499</th>\n",
-       "      <td>[African-Nguni]</td>\n",
-       "      <td>{'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>what ethnicity is the surname sabol</td>\n",
-       "      <td>658265</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The ethnicity of the surname Sabol is African...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.285714</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>500 rows × 8 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                               answers  \\\n",
-       "0                                              [2,662]   \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...   \n",
-       "2                                        [Hippocrates]   \n",
-       "3                [120 days from the date of the Note.]   \n",
-       "4                     [From $26,000 to $39,000 a year]   \n",
-       "..                                                 ...   \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...   \n",
-       "496                                           [Hanson]   \n",
-       "497  [Mount Able Baptist Church is located at the a...   \n",
-       "498                                [Honeysuckle Weeks]   \n",
-       "499                                    [African-Nguni]   \n",
-       "\n",
-       "                                              passages  \\\n",
-       "0    {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...   \n",
-       "1    {'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...   \n",
-       "2    {'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...   \n",
-       "3    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "4    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "..                                                 ...   \n",
-       "495  {'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...   \n",
-       "496  {'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...   \n",
-       "497  {'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...   \n",
-       "498  {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...   \n",
-       "499  {'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...   \n",
-       "\n",
-       "                                                 query  query_id   query_type  \\\n",
-       "0                                 albany mn population     15177      NUMERIC   \n",
-       "1                       current weather in volcano, ca    114414  DESCRIPTION   \n",
-       "2    ____________________ is considered the father ...      9083  DESCRIPTION   \n",
-       "3    how many days is an appraisal good for a fanni...    281439      NUMERIC   \n",
-       "4                         average pharmacy tech salary     40287      NUMERIC   \n",
-       "..                                                 ...       ...          ...   \n",
-       "495                            the pool shower company    518269       PERSON   \n",
-       "496          longest tenured american football players    442806       PERSON   \n",
-       "497                   mt. view baptist in pendleton sc    460250       PERSON   \n",
-       "498               what actress disappeared for a while    549739       PERSON   \n",
-       "499                what ethnicity is the surname sabol    658265       PERSON   \n",
-       "\n",
-       "                                     wellFormedAnswers  EM        F1  \n",
-       "0     [The population of Albany, Minnesota is 2,662. ]   0  0.285714  \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...   1  1.000000  \n",
-       "2    [Hippocrates is considered the father of moder...   0  0.250000  \n",
-       "3    [An appraisal is good for 120 days from the da...   0  0.631579  \n",
-       "4    [The average salary for a pharmacy technician ...   0  0.500000  \n",
-       "..                                                 ...  ..       ...  \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...   1  1.000000  \n",
-       "496  [Hanson is the longest tenured American footba...   0  0.250000  \n",
-       "497  [Mount Able Baptist Church is located at the a...   1  1.000000  \n",
-       "498  [The actress disappeared for a while Honeysuck...   0  0.500000  \n",
-       "499  [The ethnicity of the surname Sabol is African...   0  0.285714  \n",
-       "\n",
-       "[500 rows x 8 columns]"
-      ]
-     },
-     "execution_count": 50,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "result_all.to_pandas()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 53,
-   "id": "af2d4577",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Note: you may need to restart the kernel to use updated packages.\n"
-     ]
-    }
-   ],
-   "source": [
-    "%pip install -q evaluate rouge_score"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 54,
-   "id": "89494c3d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import evaluate\n",
-    "\n",
-    "bleu = evaluate.load(\"bleu\")\n",
-    "rouge = evaluate.load(\"rouge\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 56,
-   "id": "e447aa08",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "c87066449ebc44d39a66b1630977f2ac",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Map:   0%|          | 0/500 [00:00<?, ? examples/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>answers</th>\n",
-       "      <th>passages</th>\n",
-       "      <th>query</th>\n",
-       "      <th>query_id</th>\n",
-       "      <th>query_type</th>\n",
-       "      <th>wellFormedAnswers</th>\n",
-       "      <th>EM</th>\n",
-       "      <th>F1</th>\n",
-       "      <th>bleu</th>\n",
-       "      <th>precisions</th>\n",
-       "      <th>brevity_penalty</th>\n",
-       "      <th>length_ratio</th>\n",
-       "      <th>translation_length</th>\n",
-       "      <th>reference_length</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>[2,662]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>albany mn population</td>\n",
-       "      <td>15177</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The population of Albany, Minnesota is 2,662. ]</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.285714</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>[1.0, 0.0, 0.0, 0.0]</td>\n",
-       "      <td>0.000335</td>\n",
-       "      <td>0.111111</td>\n",
-       "      <td>1</td>\n",
-       "      <td>9</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "      <td>{'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...</td>\n",
-       "      <td>current weather in volcano, ca</td>\n",
-       "      <td>114414</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "      <td>1</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>[1.0, 1.0, 1.0, 1.0]</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>14</td>\n",
-       "      <td>14</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>[Hippocrates]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...</td>\n",
-       "      <td>____________________ is considered the father ...</td>\n",
-       "      <td>9083</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[Hippocrates is considered the father of moder...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.250000</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>[1.0, 0.0, 0.0, 0.0]</td>\n",
-       "      <td>0.000335</td>\n",
-       "      <td>0.111111</td>\n",
-       "      <td>1</td>\n",
-       "      <td>9</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>[120 days from the date of the Note.]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>how many days is an appraisal good for a fanni...</td>\n",
-       "      <td>281439</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[An appraisal is good for 120 days from the da...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.631579</td>\n",
-       "      <td>0.327096</td>\n",
-       "      <td>[1.0, 0.875, 0.8571428571428571, 0.83333333333...</td>\n",
-       "      <td>0.367879</td>\n",
-       "      <td>0.500000</td>\n",
-       "      <td>9</td>\n",
-       "      <td>18</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>[From $26,000 to $39,000 a year]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>average pharmacy tech salary</td>\n",
-       "      <td>40287</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The average salary for a pharmacy technician ...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.500000</td>\n",
-       "      <td>0.193040</td>\n",
-       "      <td>[0.875, 0.7142857142857143, 0.5, 0.4]</td>\n",
-       "      <td>0.324652</td>\n",
-       "      <td>0.470588</td>\n",
-       "      <td>8</td>\n",
-       "      <td>17</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>495</th>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...</td>\n",
-       "      <td>the pool shower company</td>\n",
-       "      <td>518269</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "      <td>1</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>[1.0, 1.0, 1.0, 1.0]</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>19</td>\n",
-       "      <td>19</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>496</th>\n",
-       "      <td>[Hanson]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>longest tenured american football players</td>\n",
-       "      <td>442806</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Hanson is the longest tenured American footba...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.250000</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>[1.0, 0.0, 0.0, 0.0]</td>\n",
-       "      <td>0.000335</td>\n",
-       "      <td>0.111111</td>\n",
-       "      <td>1</td>\n",
-       "      <td>9</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>497</th>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "      <td>{'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...</td>\n",
-       "      <td>mt. view baptist in pendleton sc</td>\n",
-       "      <td>460250</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "      <td>1</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>[1.0, 1.0, 1.0, 1.0]</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>21</td>\n",
-       "      <td>21</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>498</th>\n",
-       "      <td>[Honeysuckle Weeks]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>what actress disappeared for a while</td>\n",
-       "      <td>549739</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The actress disappeared for a while Honeysuck...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.500000</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>[1.0, 1.0, 0.0, 0.0]</td>\n",
-       "      <td>0.030197</td>\n",
-       "      <td>0.222222</td>\n",
-       "      <td>2</td>\n",
-       "      <td>9</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>499</th>\n",
-       "      <td>[African-Nguni]</td>\n",
-       "      <td>{'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>what ethnicity is the surname sabol</td>\n",
-       "      <td>658265</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The ethnicity of the surname Sabol is African...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.285714</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>[1.0, 0.0, 0.0, 0.0]</td>\n",
-       "      <td>0.000335</td>\n",
-       "      <td>0.111111</td>\n",
-       "      <td>1</td>\n",
-       "      <td>9</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>500 rows × 14 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                               answers  \\\n",
-       "0                                              [2,662]   \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...   \n",
-       "2                                        [Hippocrates]   \n",
-       "3                [120 days from the date of the Note.]   \n",
-       "4                     [From $26,000 to $39,000 a year]   \n",
-       "..                                                 ...   \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...   \n",
-       "496                                           [Hanson]   \n",
-       "497  [Mount Able Baptist Church is located at the a...   \n",
-       "498                                [Honeysuckle Weeks]   \n",
-       "499                                    [African-Nguni]   \n",
-       "\n",
-       "                                              passages  \\\n",
-       "0    {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...   \n",
-       "1    {'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...   \n",
-       "2    {'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...   \n",
-       "3    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "4    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "..                                                 ...   \n",
-       "495  {'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...   \n",
-       "496  {'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...   \n",
-       "497  {'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...   \n",
-       "498  {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...   \n",
-       "499  {'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...   \n",
-       "\n",
-       "                                                 query  query_id   query_type  \\\n",
-       "0                                 albany mn population     15177      NUMERIC   \n",
-       "1                       current weather in volcano, ca    114414  DESCRIPTION   \n",
-       "2    ____________________ is considered the father ...      9083  DESCRIPTION   \n",
-       "3    how many days is an appraisal good for a fanni...    281439      NUMERIC   \n",
-       "4                         average pharmacy tech salary     40287      NUMERIC   \n",
-       "..                                                 ...       ...          ...   \n",
-       "495                            the pool shower company    518269       PERSON   \n",
-       "496          longest tenured american football players    442806       PERSON   \n",
-       "497                   mt. view baptist in pendleton sc    460250       PERSON   \n",
-       "498               what actress disappeared for a while    549739       PERSON   \n",
-       "499                what ethnicity is the surname sabol    658265       PERSON   \n",
-       "\n",
-       "                                     wellFormedAnswers  EM        F1  \\\n",
-       "0     [The population of Albany, Minnesota is 2,662. ]   0  0.285714   \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...   1  1.000000   \n",
-       "2    [Hippocrates is considered the father of moder...   0  0.250000   \n",
-       "3    [An appraisal is good for 120 days from the da...   0  0.631579   \n",
-       "4    [The average salary for a pharmacy technician ...   0  0.500000   \n",
-       "..                                                 ...  ..       ...   \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...   1  1.000000   \n",
-       "496  [Hanson is the longest tenured American footba...   0  0.250000   \n",
-       "497  [Mount Able Baptist Church is located at the a...   1  1.000000   \n",
-       "498  [The actress disappeared for a while Honeysuck...   0  0.500000   \n",
-       "499  [The ethnicity of the surname Sabol is African...   0  0.285714   \n",
-       "\n",
-       "         bleu                                         precisions  \\\n",
-       "0    0.000000                               [1.0, 0.0, 0.0, 0.0]   \n",
-       "1    1.000000                               [1.0, 1.0, 1.0, 1.0]   \n",
-       "2    0.000000                               [1.0, 0.0, 0.0, 0.0]   \n",
-       "3    0.327096  [1.0, 0.875, 0.8571428571428571, 0.83333333333...   \n",
-       "4    0.193040              [0.875, 0.7142857142857143, 0.5, 0.4]   \n",
-       "..        ...                                                ...   \n",
-       "495  1.000000                               [1.0, 1.0, 1.0, 1.0]   \n",
-       "496  0.000000                               [1.0, 0.0, 0.0, 0.0]   \n",
-       "497  1.000000                               [1.0, 1.0, 1.0, 1.0]   \n",
-       "498  0.000000                               [1.0, 1.0, 0.0, 0.0]   \n",
-       "499  0.000000                               [1.0, 0.0, 0.0, 0.0]   \n",
-       "\n",
-       "     brevity_penalty  length_ratio  translation_length  reference_length  \n",
-       "0           0.000335      0.111111                   1                 9  \n",
-       "1           1.000000      1.000000                  14                14  \n",
-       "2           0.000335      0.111111                   1                 9  \n",
-       "3           0.367879      0.500000                   9                18  \n",
-       "4           0.324652      0.470588                   8                17  \n",
-       "..               ...           ...                 ...               ...  \n",
-       "495         1.000000      1.000000                  19                19  \n",
-       "496         0.000335      0.111111                   1                 9  \n",
-       "497         1.000000      1.000000                  21                21  \n",
-       "498         0.030197      0.222222                   2                 9  \n",
-       "499         0.000335      0.111111                   1                 9  \n",
-       "\n",
-       "[500 rows x 14 columns]"
-      ]
-     },
-     "execution_count": 56,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "result_all = result_all.map(\n",
-    "    lambda record: bleu.compute(\n",
-    "        predictions=[record[\"answers\"][0]], references=[record[\"wellFormedAnswers\"][0]]\n",
-    "    ),\n",
-    "    batched=False,\n",
-    ")\n",
-    "result_all.to_pandas()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 57,
-   "id": "fbbe31fd",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "88f839b74aa54fcd8c95215e22e30472",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Map:   0%|          | 0/500 [00:00<?, ? examples/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>answers</th>\n",
-       "      <th>passages</th>\n",
-       "      <th>query</th>\n",
-       "      <th>query_id</th>\n",
-       "      <th>query_type</th>\n",
-       "      <th>wellFormedAnswers</th>\n",
-       "      <th>EM</th>\n",
-       "      <th>F1</th>\n",
-       "      <th>bleu</th>\n",
-       "      <th>precisions</th>\n",
-       "      <th>brevity_penalty</th>\n",
-       "      <th>length_ratio</th>\n",
-       "      <th>translation_length</th>\n",
-       "      <th>reference_length</th>\n",
-       "      <th>rouge1</th>\n",
-       "      <th>rouge2</th>\n",
-       "      <th>rougeL</th>\n",
-       "      <th>rougeLsum</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>[2,662]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>albany mn population</td>\n",
-       "      <td>15177</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The population of Albany, Minnesota is 2,662. ]</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.285714</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>[1.0, 0.0, 0.0, 0.0]</td>\n",
-       "      <td>0.000335</td>\n",
-       "      <td>0.111111</td>\n",
-       "      <td>1</td>\n",
-       "      <td>9</td>\n",
-       "      <td>0.400000</td>\n",
-       "      <td>0.250000</td>\n",
-       "      <td>0.400000</td>\n",
-       "      <td>0.400000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "      <td>{'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...</td>\n",
-       "      <td>current weather in volcano, ca</td>\n",
-       "      <td>114414</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "      <td>1</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>[1.0, 1.0, 1.0, 1.0]</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>14</td>\n",
-       "      <td>14</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>[Hippocrates]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...</td>\n",
-       "      <td>____________________ is considered the father ...</td>\n",
-       "      <td>9083</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[Hippocrates is considered the father of moder...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.250000</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>[1.0, 0.0, 0.0, 0.0]</td>\n",
-       "      <td>0.000335</td>\n",
-       "      <td>0.111111</td>\n",
-       "      <td>1</td>\n",
-       "      <td>9</td>\n",
-       "      <td>0.222222</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>0.222222</td>\n",
-       "      <td>0.222222</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>[120 days from the date of the Note.]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>how many days is an appraisal good for a fanni...</td>\n",
-       "      <td>281439</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[An appraisal is good for 120 days from the da...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.631579</td>\n",
-       "      <td>0.327096</td>\n",
-       "      <td>[1.0, 0.875, 0.8571428571428571, 0.83333333333...</td>\n",
-       "      <td>0.367879</td>\n",
-       "      <td>0.500000</td>\n",
-       "      <td>9</td>\n",
-       "      <td>18</td>\n",
-       "      <td>0.640000</td>\n",
-       "      <td>0.608696</td>\n",
-       "      <td>0.640000</td>\n",
-       "      <td>0.640000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>[From $26,000 to $39,000 a year]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>average pharmacy tech salary</td>\n",
-       "      <td>40287</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The average salary for a pharmacy technician ...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.500000</td>\n",
-       "      <td>0.193040</td>\n",
-       "      <td>[0.875, 0.7142857142857143, 0.5, 0.4]</td>\n",
-       "      <td>0.324652</td>\n",
-       "      <td>0.470588</td>\n",
-       "      <td>8</td>\n",
-       "      <td>17</td>\n",
-       "      <td>0.583333</td>\n",
-       "      <td>0.454545</td>\n",
-       "      <td>0.583333</td>\n",
-       "      <td>0.583333</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>495</th>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...</td>\n",
-       "      <td>the pool shower company</td>\n",
-       "      <td>518269</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "      <td>1</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>[1.0, 1.0, 1.0, 1.0]</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>19</td>\n",
-       "      <td>19</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>496</th>\n",
-       "      <td>[Hanson]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>longest tenured american football players</td>\n",
-       "      <td>442806</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Hanson is the longest tenured American footba...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.250000</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>[1.0, 0.0, 0.0, 0.0]</td>\n",
-       "      <td>0.000335</td>\n",
-       "      <td>0.111111</td>\n",
-       "      <td>1</td>\n",
-       "      <td>9</td>\n",
-       "      <td>0.222222</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>0.222222</td>\n",
-       "      <td>0.222222</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>497</th>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "      <td>{'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...</td>\n",
-       "      <td>mt. view baptist in pendleton sc</td>\n",
-       "      <td>460250</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "      <td>1</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>[1.0, 1.0, 1.0, 1.0]</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>21</td>\n",
-       "      <td>21</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "      <td>1.000000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>498</th>\n",
-       "      <td>[Honeysuckle Weeks]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>what actress disappeared for a while</td>\n",
-       "      <td>549739</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The actress disappeared for a while Honeysuck...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.500000</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>[1.0, 1.0, 0.0, 0.0]</td>\n",
-       "      <td>0.030197</td>\n",
-       "      <td>0.222222</td>\n",
-       "      <td>2</td>\n",
-       "      <td>9</td>\n",
-       "      <td>0.400000</td>\n",
-       "      <td>0.250000</td>\n",
-       "      <td>0.400000</td>\n",
-       "      <td>0.400000</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>499</th>\n",
-       "      <td>[African-Nguni]</td>\n",
-       "      <td>{'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>what ethnicity is the surname sabol</td>\n",
-       "      <td>658265</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The ethnicity of the surname Sabol is African...</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0.285714</td>\n",
-       "      <td>0.000000</td>\n",
-       "      <td>[1.0, 0.0, 0.0, 0.0]</td>\n",
-       "      <td>0.000335</td>\n",
-       "      <td>0.111111</td>\n",
-       "      <td>1</td>\n",
-       "      <td>9</td>\n",
-       "      <td>0.363636</td>\n",
-       "      <td>0.222222</td>\n",
-       "      <td>0.363636</td>\n",
-       "      <td>0.363636</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>500 rows × 18 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                               answers  \\\n",
-       "0                                              [2,662]   \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...   \n",
-       "2                                        [Hippocrates]   \n",
-       "3                [120 days from the date of the Note.]   \n",
-       "4                     [From $26,000 to $39,000 a year]   \n",
-       "..                                                 ...   \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...   \n",
-       "496                                           [Hanson]   \n",
-       "497  [Mount Able Baptist Church is located at the a...   \n",
-       "498                                [Honeysuckle Weeks]   \n",
-       "499                                    [African-Nguni]   \n",
-       "\n",
-       "                                              passages  \\\n",
-       "0    {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...   \n",
-       "1    {'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...   \n",
-       "2    {'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...   \n",
-       "3    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "4    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "..                                                 ...   \n",
-       "495  {'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...   \n",
-       "496  {'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...   \n",
-       "497  {'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...   \n",
-       "498  {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...   \n",
-       "499  {'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...   \n",
-       "\n",
-       "                                                 query  query_id   query_type  \\\n",
-       "0                                 albany mn population     15177      NUMERIC   \n",
-       "1                       current weather in volcano, ca    114414  DESCRIPTION   \n",
-       "2    ____________________ is considered the father ...      9083  DESCRIPTION   \n",
-       "3    how many days is an appraisal good for a fanni...    281439      NUMERIC   \n",
-       "4                         average pharmacy tech salary     40287      NUMERIC   \n",
-       "..                                                 ...       ...          ...   \n",
-       "495                            the pool shower company    518269       PERSON   \n",
-       "496          longest tenured american football players    442806       PERSON   \n",
-       "497                   mt. view baptist in pendleton sc    460250       PERSON   \n",
-       "498               what actress disappeared for a while    549739       PERSON   \n",
-       "499                what ethnicity is the surname sabol    658265       PERSON   \n",
-       "\n",
-       "                                     wellFormedAnswers  EM        F1  \\\n",
-       "0     [The population of Albany, Minnesota is 2,662. ]   0  0.285714   \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...   1  1.000000   \n",
-       "2    [Hippocrates is considered the father of moder...   0  0.250000   \n",
-       "3    [An appraisal is good for 120 days from the da...   0  0.631579   \n",
-       "4    [The average salary for a pharmacy technician ...   0  0.500000   \n",
-       "..                                                 ...  ..       ...   \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...   1  1.000000   \n",
-       "496  [Hanson is the longest tenured American footba...   0  0.250000   \n",
-       "497  [Mount Able Baptist Church is located at the a...   1  1.000000   \n",
-       "498  [The actress disappeared for a while Honeysuck...   0  0.500000   \n",
-       "499  [The ethnicity of the surname Sabol is African...   0  0.285714   \n",
-       "\n",
-       "         bleu                                         precisions  \\\n",
-       "0    0.000000                               [1.0, 0.0, 0.0, 0.0]   \n",
-       "1    1.000000                               [1.0, 1.0, 1.0, 1.0]   \n",
-       "2    0.000000                               [1.0, 0.0, 0.0, 0.0]   \n",
-       "3    0.327096  [1.0, 0.875, 0.8571428571428571, 0.83333333333...   \n",
-       "4    0.193040              [0.875, 0.7142857142857143, 0.5, 0.4]   \n",
-       "..        ...                                                ...   \n",
-       "495  1.000000                               [1.0, 1.0, 1.0, 1.0]   \n",
-       "496  0.000000                               [1.0, 0.0, 0.0, 0.0]   \n",
-       "497  1.000000                               [1.0, 1.0, 1.0, 1.0]   \n",
-       "498  0.000000                               [1.0, 1.0, 0.0, 0.0]   \n",
-       "499  0.000000                               [1.0, 0.0, 0.0, 0.0]   \n",
-       "\n",
-       "     brevity_penalty  length_ratio  translation_length  reference_length  \\\n",
-       "0           0.000335      0.111111                   1                 9   \n",
-       "1           1.000000      1.000000                  14                14   \n",
-       "2           0.000335      0.111111                   1                 9   \n",
-       "3           0.367879      0.500000                   9                18   \n",
-       "4           0.324652      0.470588                   8                17   \n",
-       "..               ...           ...                 ...               ...   \n",
-       "495         1.000000      1.000000                  19                19   \n",
-       "496         0.000335      0.111111                   1                 9   \n",
-       "497         1.000000      1.000000                  21                21   \n",
-       "498         0.030197      0.222222                   2                 9   \n",
-       "499         0.000335      0.111111                   1                 9   \n",
-       "\n",
-       "       rouge1    rouge2    rougeL  rougeLsum  \n",
-       "0    0.400000  0.250000  0.400000   0.400000  \n",
-       "1    1.000000  1.000000  1.000000   1.000000  \n",
-       "2    0.222222  0.000000  0.222222   0.222222  \n",
-       "3    0.640000  0.608696  0.640000   0.640000  \n",
-       "4    0.583333  0.454545  0.583333   0.583333  \n",
-       "..        ...       ...       ...        ...  \n",
-       "495  1.000000  1.000000  1.000000   1.000000  \n",
-       "496  0.222222  0.000000  0.222222   0.222222  \n",
-       "497  1.000000  1.000000  1.000000   1.000000  \n",
-       "498  0.400000  0.250000  0.400000   0.400000  \n",
-       "499  0.363636  0.222222  0.363636   0.363636  \n",
-       "\n",
-       "[500 rows x 18 columns]"
-      ]
-     },
-     "execution_count": 57,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "result_all = result_all.map(\n",
-    "    lambda record: rouge.compute(\n",
-    "        predictions=[record[\"answers\"][0]], references=[record[\"wellFormedAnswers\"][0]]\n",
-    "    ),\n",
-    "    batched=False,\n",
-    ")\n",
-    "result_all.to_pandas()"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.4"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

Llama-2-eval/notebook/metrics.ipynb DELETED Viewed

@@ -1,1293 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "af2d4577",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Note: you may need to restart the kernel to use updated packages.\n"
-     ]
-    }
-   ],
-   "source": [
-    "%pip install -q evaluate rouge_score"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "a6d96660",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "True"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import os\n",
-    "from dotenv import load_dotenv\n",
-    "\n",
-    "load_dotenv()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "b72bf3f9",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Dataset({\n",
-       "    features: ['answers', 'passages', 'query', 'query_id', 'query_type', 'wellFormedAnswers'],\n",
-       "    num_rows: 500\n",
-       "})"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from datasets import load_from_disk\n",
-    "\n",
-    "new_ds = load_from_disk(\"../data/datasets/ms_macro/\")\n",
-    "new_ds"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "051bd771",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "({'NUMERIC': 100,\n",
-       "  'DESCRIPTION': 100,\n",
-       "  'ENTITY': 100,\n",
-       "  'PERSON': 100,\n",
-       "  'LOCATION': 100},\n",
-       " {'NUMERIC': 179,\n",
-       "  'DESCRIPTION': 215,\n",
-       "  'ENTITY': 443,\n",
-       "  'LOCATION': 461,\n",
-       "  'PERSON': 499})"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "counts = {}\n",
-    "indices = {}\n",
-    "size = 100\n",
-    "for i in range(new_ds.num_rows):\n",
-    "    row = new_ds[i]\n",
-    "    query_type = row[\"query_type\"]\n",
-    "    if query_type in counts:\n",
-    "        counts[query_type] += 1\n",
-    "    else:\n",
-    "        counts[query_type] = 1\n",
-    "    if counts[query_type] == size:\n",
-    "        indices[query_type] = i\n",
-    "counts, indices"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "db48dcc4",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>answers</th>\n",
-       "      <th>passages</th>\n",
-       "      <th>query</th>\n",
-       "      <th>query_id</th>\n",
-       "      <th>query_type</th>\n",
-       "      <th>wellFormedAnswers</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>[2,662]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>albany mn population</td>\n",
-       "      <td>15177</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The population of Albany, Minnesota is 2,662. ]</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "      <td>{'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...</td>\n",
-       "      <td>current weather in volcano, ca</td>\n",
-       "      <td>114414</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[The Volcano forecast for Apr 12 is 52 degrees...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>[Hippocrates]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...</td>\n",
-       "      <td>____________________ is considered the father ...</td>\n",
-       "      <td>9083</td>\n",
-       "      <td>DESCRIPTION</td>\n",
-       "      <td>[Hippocrates is considered the father of moder...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>[120 days from the date of the Note.]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>how many days is an appraisal good for a fanni...</td>\n",
-       "      <td>281439</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[An appraisal is good for 120 days from the da...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>[From $26,000 to $39,000 a year]</td>\n",
-       "      <td>{'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>average pharmacy tech salary</td>\n",
-       "      <td>40287</td>\n",
-       "      <td>NUMERIC</td>\n",
-       "      <td>[The average salary for a pharmacy technician ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>495</th>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...</td>\n",
-       "      <td>the pool shower company</td>\n",
-       "      <td>518269</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The Pool Shower, Inc. is a Georgia Domestic P...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>496</th>\n",
-       "      <td>[Hanson]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>longest tenured american football players</td>\n",
-       "      <td>442806</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Hanson is the longest tenured American footba...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>497</th>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "      <td>{'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...</td>\n",
-       "      <td>mt. view baptist in pendleton sc</td>\n",
-       "      <td>460250</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[Mount Able Baptist Church is located at the a...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>498</th>\n",
-       "      <td>[Honeysuckle Weeks]</td>\n",
-       "      <td>{'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...</td>\n",
-       "      <td>what actress disappeared for a while</td>\n",
-       "      <td>549739</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The actress disappeared for a while Honeysuck...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>499</th>\n",
-       "      <td>[African-Nguni]</td>\n",
-       "      <td>{'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...</td>\n",
-       "      <td>what ethnicity is the surname sabol</td>\n",
-       "      <td>658265</td>\n",
-       "      <td>PERSON</td>\n",
-       "      <td>[The ethnicity of the surname Sabol is African...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>500 rows × 6 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                               answers  \\\n",
-       "0                                              [2,662]   \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...   \n",
-       "2                                        [Hippocrates]   \n",
-       "3                [120 days from the date of the Note.]   \n",
-       "4                     [From $26,000 to $39,000 a year]   \n",
-       "..                                                 ...   \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...   \n",
-       "496                                           [Hanson]   \n",
-       "497  [Mount Able Baptist Church is located at the a...   \n",
-       "498                                [Honeysuckle Weeks]   \n",
-       "499                                    [African-Nguni]   \n",
-       "\n",
-       "                                              passages  \\\n",
-       "0    {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0], 'pas...   \n",
-       "1    {'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0]...   \n",
-       "2    {'is_selected': [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]...   \n",
-       "3    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "4    {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]...   \n",
-       "..                                                 ...   \n",
-       "495  {'is_selected': [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]...   \n",
-       "496  {'is_selected': [0, 0, 0, 0, 1, 0, 0, 0, 0, 0]...   \n",
-       "497  {'is_selected': [1, 0, 0, 0, 0, 0, 0, 0, 0], '...   \n",
-       "498  {'is_selected': [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]...   \n",
-       "499  {'is_selected': [0, 0, 1, 0, 0, 0, 0, 0], 'pas...   \n",
-       "\n",
-       "                                                 query  query_id   query_type  \\\n",
-       "0                                 albany mn population     15177      NUMERIC   \n",
-       "1                       current weather in volcano, ca    114414  DESCRIPTION   \n",
-       "2    ____________________ is considered the father ...      9083  DESCRIPTION   \n",
-       "3    how many days is an appraisal good for a fanni...    281439      NUMERIC   \n",
-       "4                         average pharmacy tech salary     40287      NUMERIC   \n",
-       "..                                                 ...       ...          ...   \n",
-       "495                            the pool shower company    518269       PERSON   \n",
-       "496          longest tenured american football players    442806       PERSON   \n",
-       "497                   mt. view baptist in pendleton sc    460250       PERSON   \n",
-       "498               what actress disappeared for a while    549739       PERSON   \n",
-       "499                what ethnicity is the surname sabol    658265       PERSON   \n",
-       "\n",
-       "                                     wellFormedAnswers  \n",
-       "0     [The population of Albany, Minnesota is 2,662. ]  \n",
-       "1    [The Volcano forecast for Apr 12 is 52 degrees...  \n",
-       "2    [Hippocrates is considered the father of moder...  \n",
-       "3    [An appraisal is good for 120 days from the da...  \n",
-       "4    [The average salary for a pharmacy technician ...  \n",
-       "..                                                 ...  \n",
-       "495  [The Pool Shower, Inc. is a Georgia Domestic P...  \n",
-       "496  [Hanson is the longest tenured American footba...  \n",
-       "497  [Mount Able Baptist Church is located at the a...  \n",
-       "498  [The actress disappeared for a while Honeysuck...  \n",
-       "499  [The ethnicity of the surname Sabol is African...  \n",
-       "\n",
-       "[500 rows x 6 columns]"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "new_ds.to_pandas()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "89494c3d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import evaluate\n",
-    "\n",
-    "bleu = evaluate.load(\"bleu\")\n",
-    "rouge = evaluate.load(\"rouge\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "24a818ba",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def calc_metrics(ds):\n",
-    "    predictions = [ds[i][\"answers\"][0] for i in range(ds.num_rows)]\n",
-    "    references = [ds[i][\"wellFormedAnswers\"][0] for i in range(ds.num_rows)]\n",
-    "    bleu_scores = bleu.compute(predictions=predictions, references=references)\n",
-    "    rouge_scores = rouge.compute(predictions=predictions, references=references)\n",
-    "    return {\"bleu_scores\": bleu_scores, \"rouge_scores\": rouge_scores}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "e447aa08",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'bleu_scores': {'bleu': 0.5842479720128682,\n",
-       "  'precisions': [0.7814257485940113,\n",
-       "   0.7185392334265505,\n",
-       "   0.6801561945331913,\n",
-       "   0.6543700340522134],\n",
-       "  'brevity_penalty': 0.8263321448047812,\n",
-       "  'length_ratio': 0.8398008680112331,\n",
-       "  'translation_length': 6579,\n",
-       "  'reference_length': 7834},\n",
-       " 'rouge_scores': {'rouge1': 0.6301946495853493,\n",
-       "  'rouge2': 0.5266427189500504,\n",
-       "  'rougeL': 0.623467453115133,\n",
-       "  'rougeLsum': 0.6239164817179192}}"
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "calc_metrics(new_ds)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "id": "b29d1f3e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def calc_all_metrics(ds):\n",
-    "    result = {}\n",
-    "    result[\"OVERALL\"] = calc_metrics(ds)\n",
-    "    for query_type in indices:\n",
-    "        result[query_type] = calc_metrics(\n",
-    "            ds.filter(lambda example: example[\"query_type\"] == query_type)\n",
-    "        )\n",
-    "\n",
-    "    return result"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "id": "1a4273da",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'OVERALL': {'bleu_scores': {'bleu': 0.5842479720128682,\n",
-       "   'precisions': [0.7814257485940113,\n",
-       "    0.7185392334265505,\n",
-       "    0.6801561945331913,\n",
-       "    0.6543700340522134],\n",
-       "   'brevity_penalty': 0.8263321448047812,\n",
-       "   'length_ratio': 0.8398008680112331,\n",
-       "   'translation_length': 6579,\n",
-       "   'reference_length': 7834},\n",
-       "  'rouge_scores': {'rouge1': 0.6301946495853493,\n",
-       "   'rouge2': 0.5266427189500504,\n",
-       "   'rougeL': 0.623467453115133,\n",
-       "   'rougeLsum': 0.6239164817179192}},\n",
-       " 'NUMERIC': {'bleu_scores': {'bleu': 0.3589193328591513,\n",
-       "   'precisions': [0.7536764705882353,\n",
-       "    0.6494413407821229,\n",
-       "    0.5884244372990354,\n",
-       "    0.5657657657657658],\n",
-       "   'brevity_penalty': 0.5649158870633492,\n",
-       "   'length_ratio': 0.6365054602184087,\n",
-       "   'translation_length': 816,\n",
-       "   'reference_length': 1282},\n",
-       "  'rouge_scores': {'rouge1': 0.5569863096088544,\n",
-       "   'rouge2': 0.4262959859853511,\n",
-       "   'rougeL': 0.5495190228731732,\n",
-       "   'rougeLsum': 0.5502805905003136}},\n",
-       " 'DESCRIPTION': {'bleu_scores': {'bleu': 0.7521919521555381,\n",
-       "   'precisions': [0.8093238135237295,\n",
-       "    0.761946514686541,\n",
-       "    0.7335164835164835,\n",
-       "    0.7077144226161955],\n",
-       "   'brevity_penalty': 1.0,\n",
-       "   'length_ratio': 1.0778632865550022,\n",
-       "   'translation_length': 2381,\n",
-       "   'reference_length': 2209},\n",
-       "  'rouge_scores': {'rouge1': 0.8503571429521525,\n",
-       "   'rouge2': 0.8009206345153658,\n",
-       "   'rougeL': 0.8406066569954856,\n",
-       "   'rougeLsum': 0.8405710628479812}},\n",
-       " 'ENTITY': {'bleu_scores': {'bleu': 0.5057439480363012,\n",
-       "   'precisions': [0.7135050741608119,\n",
-       "    0.6375952582557155,\n",
-       "    0.5884509624197983,\n",
-       "    0.5555555555555556],\n",
-       "   'brevity_penalty': 0.8143961563151505,\n",
-       "   'length_ratio': 0.8296632124352331,\n",
-       "   'translation_length': 1281,\n",
-       "   'reference_length': 1544},\n",
-       "  'rouge_scores': {'rouge1': 0.5877667231458372,\n",
-       "   'rouge2': 0.48898551862814277,\n",
-       "   'rougeL': 0.5796676511145928,\n",
-       "   'rougeLsum': 0.5784518864116339}},\n",
-       " 'LOCATION': {'bleu_scores': {'bleu': 0.4167786604147962,\n",
-       "   'precisions': [0.8600583090379009,\n",
-       "    0.7986348122866894,\n",
-       "    0.7573385518590998,\n",
-       "    0.7414529914529915],\n",
-       "   'brevity_penalty': 0.5288627994571649,\n",
-       "   'length_ratio': 0.6108637577916296,\n",
-       "   'translation_length': 686,\n",
-       "   'reference_length': 1123},\n",
-       "  'rouge_scores': {'rouge1': 0.5405464995752973,\n",
-       "   'rouge2': 0.3950940848806123,\n",
-       "   'rougeL': 0.5400724136440879,\n",
-       "   'rougeLsum': 0.5389556394979822}},\n",
-       " 'PERSON': {'bleu_scores': {'bleu': 0.5861084149356606,\n",
-       "   'precisions': [0.773851590106007,\n",
-       "    0.7178707224334601,\n",
-       "    0.6810766721044046,\n",
-       "    0.6522864538395168],\n",
-       "   'brevity_penalty': 0.8315596069910627,\n",
-       "   'length_ratio': 0.844272076372315,\n",
-       "   'translation_length': 1415,\n",
-       "   'reference_length': 1676},\n",
-       "  'rouge_scores': {'rouge1': 0.6119770025611677,\n",
-       "   'rouge2': 0.522853938087197,\n",
-       "   'rougeL': 0.6096713664231095,\n",
-       "   'rougeLsum': 0.6103086543984155}}}"
-      ]
-     },
-     "execution_count": 10,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "calc_all_metrics(new_ds)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "3698be27",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "loading env vars from: /Users/inflaton/code/emtech/gpt/Llama-2-eval/.env\n",
-      "App init started at 2023-10-10 12:04:33.775140\n",
-      "Running on:  macOS-14.0-arm64-arm-64bit\n",
-      "MPS is  available\n",
-      "CUDA is NOT available\n",
-      "hf_embeddings_device_type: mps\n",
-      "hf_pipeline_device_type: mps\n",
-      "initializing LLM: openai\n",
-      "  hf_pipeline_device_type: mps\n",
-      "     load_quantized_model: None\n",
-      "              torch_dtype: torch.float32\n",
-      "                 n_threds: 24\n",
-      "              using model: gpt-3.5-turbo\n",
-      "initialization complete\n",
-      "App init completed in 0.167s\n"
-     ]
-    }
-   ],
-   "source": [
-    "import json\n",
-    "import sys\n",
-    "import os\n",
-    "\n",
-    "os.environ[\"TEST_FIRST_5\"] = \"true\"\n",
-    "os.environ[\"LANGCHAIN_DEBUG\"] = \"true\"\n",
-    "\n",
-    "from pathlib import Path\n",
-    "\n",
-    "sys.path.append(str(Path.cwd().parent))\n",
-    "\n",
-    "from evaluate_llm_ms_macro import (\n",
-    "    QAChainWithMsMacroDataset,\n",
-    "    llm_loader,\n",
-    "    calc_all_metrics,\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "2395804d",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'question': 'albany mn population', 'chat_history': []}\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain] Entering Chain run with input:\n",
-      "\u001b[0m{\n",
-      "  \"question\": \"albany mn population\",\n",
-      "  \"chat_history\": []\n",
-      "}\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain] Entering Chain run with input:\n",
-      "\u001b[0m[inputs]\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain] Entering Chain run with input:\n",
-      "\u001b[0m{\n",
-      "  \"question\": \"albany mn population\",\n",
-      "  \"context\": \"City of Albany, MN Zip Codes. City of Albany, MN Demographic Information. * Demographic data is based on information taken from the 2000 Census. City of Albany, MN covers 1 Area Code. City of Albany, MN covers 1 Zip Code. 15 Cities within 15 Miles of the City of Albany, MN.\\n\\nPlace of birth for U.S.-born residents: 70% of the 56307 zip code residents lived in the same house 5 years ago. Out of people who lived in different houses, 71% lived in this county. Out of people who lived in different counties, 50% lived in Minnesota. 92% of the 56307 zip code residents lived in the same house 1 year ago.\\n\\nFor the unincorporated community in southeast Minnesota named West Albany, see West Albany, Minnesota. Albany is a city in Stearns County, Minnesota, United States. The population was 2,561 at the 2010 census. It is part of the St. Cloud Metropolitan Statistical Area.\\n\\nAlbany, Minnesota, as per 2017 US Census estimate, has a community population of 2,662 people. Albany is located in Stearns County, 20 miles west of St. Cloud and 80 miles northwest of Minneapolis/St. Paul on Interstate 94 (I-94). Albany has direct access to State Highway 238, which originates in Albany.\\n\\nSponsored Topics. Albany is a city in Stearns County, Minnesota, United States. The population was 2,561 at the 2010 census. It is part of the St. Cloud Metropolitan Statistical Area.\\n\\nRecent posts about Albany, Minnesota on our local forum with over 2,000,000 registered users. Albany is mentioned 87 times on our forum: Latest news from Albany, MN collected exclusively by city-data.com from local newspapers, TV, and radio stations. Ancestries: German (55.6%), Irish (10.0%), Polish (5.9%), Norwegian (5.4%), Swedish (2.8%), United States (2.6%).\\n\\nFor population 25 years and over in 56307: 1  High school or higher: 87.4%. 2  Bachelor's degree or higher: 15.4%. 3  Graduate or professional degree: 3.3 4 %. Unemployed: 3. 5 2%. Mean travel time to work (commute): 23.6 minutes.\\n\\nFor population 25 years and over in Albany: 1  High school or higher: 86.7%. 2  Bachelor's degree or higher: 15.4%. 3  Graduate or professional degree: 4.4 4 %. Unemployed: 4. 5 3%. Mean travel time to work (commute): 23.0 minutes.\"\n",
-      "}\n",
-      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain > 5:llm:ChatOpenAI] Entering LLM run with input:\n",
-      "\u001b[0m{\n",
-      "  \"prompts\": [\n",
-      "    \"System: Use the following pieces of context to answer the users question. \\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\\n----------------\\nCity of Albany, MN Zip Codes. City of Albany, MN Demographic Information. * Demographic data is based on information taken from the 2000 Census. City of Albany, MN covers 1 Area Code. City of Albany, MN covers 1 Zip Code. 15 Cities within 15 Miles of the City of Albany, MN.\\n\\nPlace of birth for U.S.-born residents: 70% of the 56307 zip code residents lived in the same house 5 years ago. Out of people who lived in different houses, 71% lived in this county. Out of people who lived in different counties, 50% lived in Minnesota. 92% of the 56307 zip code residents lived in the same house 1 year ago.\\n\\nFor the unincorporated community in southeast Minnesota named West Albany, see West Albany, Minnesota. Albany is a city in Stearns County, Minnesota, United States. The population was 2,561 at the 2010 census. It is part of the St. Cloud Metropolitan Statistical Area.\\n\\nAlbany, Minnesota, as per 2017 US Census estimate, has a community population of 2,662 people. Albany is located in Stearns County, 20 miles west of St. Cloud and 80 miles northwest of Minneapolis/St. Paul on Interstate 94 (I-94). Albany has direct access to State Highway 238, which originates in Albany.\\n\\nSponsored Topics. Albany is a city in Stearns County, Minnesota, United States. The population was 2,561 at the 2010 census. It is part of the St. Cloud Metropolitan Statistical Area.\\n\\nRecent posts about Albany, Minnesota on our local forum with over 2,000,000 registered users. Albany is mentioned 87 times on our forum: Latest news from Albany, MN collected exclusively by city-data.com from local newspapers, TV, and radio stations. Ancestries: German (55.6%), Irish (10.0%), Polish (5.9%), Norwegian (5.4%), Swedish (2.8%), United States (2.6%).\\n\\nFor population 25 years and over in 56307: 1  High school or higher: 87.4%. 2  Bachelor's degree or higher: 15.4%. 3  Graduate or professional degree: 3.3 4 %. Unemployed: 3. 5 2%. Mean travel time to work (commute): 23.6 minutes.\\n\\nFor population 25 years and over in Albany: 1  High school or higher: 86.7%. 2  Bachelor's degree or higher: 15.4%. 3  Graduate or professional degree: 4.4 4 %. Unemployed: 4. 5 3%. Mean travel time to work (commute): 23.0 minutes.\\nHuman: albany mn population\"\n",
-      "  ]\n",
-      "}\n",
-      "The population of Albany, Minnesota is approximately 2,561 as of the 2010 census. However, according to a 2017 US Census estimate, the community population has increased to 2,662 people.\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain > 5:llm:ChatOpenAI] [3.23s] Exiting LLM run with output:\n",
-      "\u001b[0m{\n",
-      "  \"generations\": [\n",
-      "    [\n",
-      "      {\n",
-      "        \"text\": \"The population of Albany, Minnesota is approximately 2,561 as of the 2010 census. However, according to a 2017 US Census estimate, the community population has increased to 2,662 people.\",\n",
-      "        \"generation_info\": {\n",
-      "          \"finish_reason\": \"stop\"\n",
-      "        },\n",
-      "        \"message\": {\n",
-      "          \"lc\": 1,\n",
-      "          \"type\": \"constructor\",\n",
-      "          \"id\": [\n",
-      "            \"langchain\",\n",
-      "            \"schema\",\n",
-      "            \"messages\",\n",
-      "            \"AIMessageChunk\"\n",
-      "          ],\n",
-      "          \"kwargs\": {\n",
-      "            \"example\": false,\n",
-      "            \"content\": \"The population of Albany, Minnesota is approximately 2,561 as of the 2010 census. However, according to a 2017 US Census estimate, the community population has increased to 2,662 people.\",\n",
-      "            \"additional_kwargs\": {}\n",
-      "          }\n",
-      "        }\n",
-      "      }\n",
-      "    ]\n",
-      "  ],\n",
-      "  \"llm_output\": null,\n",
-      "  \"run\": null\n",
-      "}\n",
-      "\n",
-      "\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain] [3.23s] Exiting Chain run with output:\n",
-      "\u001b[0m{\n",
-      "  \"text\": \"The population of Albany, Minnesota is approximately 2,561 as of the 2010 census. However, according to a 2017 US Census estimate, the community population has increased to 2,662 people.\"\n",
-      "}\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain] [3.23s] Exiting Chain run with output:\n",
-      "\u001b[0m{\n",
-      "  \"output_text\": \"The population of Albany, Minnesota is approximately 2,561 as of the 2010 census. However, according to a 2017 US Census estimate, the community population has increased to 2,662 people.\"\n",
-      "}\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain] [3.46s] Exiting Chain run with output:\n",
-      "\u001b[0m[outputs]\n",
-      "{'question': 'current weather in volcano, ca', 'chat_history': []}\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain] Entering Chain run with input:\n",
-      "\u001b[0m{\n",
-      "  \"question\": \"current weather in volcano, ca\",\n",
-      "  \"chat_history\": []\n",
-      "}\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain] Entering Chain run with input:\n",
-      "\u001b[0m[inputs]\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain] Entering Chain run with input:\n",
-      "\u001b[0m{\n",
-      "  \"question\": \"current weather in volcano, ca\",\n",
-      "  \"context\": \"Volcano 10 Day Weather. Sunday:The Volcano forecast for Apr 09 is 43 degrees and Sunny. There is 55 percentage chance of rain and 4 mph winds from the Southwest. Monday:The Volcano forecast for Apr 10 is 51 degrees and Sunny.\\n\\nCurrent U.S. National Radar--Current. The Current National Weather Radar is shown below with a UTC Time (subtract 5 hours from UTC to get Eastern Time). National Weather Forecast--Current. The Current National Weather Forecast and National Weather Map are shown below.\\n\\nVolcano 10 Day Weather. 1  Sunday:The Volcano forecast for Apr 09 is 43 degrees and Sunny. There is 55 percentage chance of rain and 4 mph winds from the Southwest. 2  Monday:The Volcano forecast for Apr 10 is 51 degrees and Sunny. There is 49 percentage chance of rain and 3 mph winds from the Southwest.\\n\\nVolcano, CA Weather Data. 1  Volcano, CA Current Weather Data. 2  Sponsored. 3  Volcano, CA Historical Weather Trends.  Volcano, CA area 1  Highlights. Volcano, CA Chance of Sunshine.  Volcano, CA Historical 1  Temperature. Volcano, CA Rainfall and Snowfall Average.  Volcano, CA Energy Demand.\\n\\nVolcano Weather. Volcano weather and daily current conditions with summary and 5 Day forecast including humidity, precipitation, high and low temperatures presented in Fahrenheit and Celsius, barometric pressure, heat index, wind chill, hourly forecast, sunrise, sunset, wind speed with direction, and more.\\n\\nHourly Forecast Detailed. 1  0am:The Volcano, CA forecast for Apr 03 is 48 degrees and Patchy rain possible. There is 83 percentage chance of rain and 2 mph winds from the East. 2  3am:The Volcano, CA forecast for Apr 03 is 44 degrees and Clear. There is 77 percentage chance of rain and 2 mph winds from the East.\\n\\nVolcano 7 Day Weather. 1  Monday:The Volcano forecast for Apr 03 is 58 degrees and Sunny. There is 34 percentage chance of rain and 5 mph winds from the West. 2  Tuesday:The Volcano forecast for Apr 04 is 59 degrees and Sunny. There is 33 percentage chance of rain and 5 mph winds from the West-Southwest.\\n\\nVolcano 10 Day Weather. 1  Sunday:The Volcano forecast for Apr 09 is 43 degrees and Sunny. 2  Monday:The Volcano forecast for Apr 10 is 51 degrees and Sunny. 3  Tuesday:The Volcano forecast for Apr 11 is 49 degrees and Patchy rain possible.  Wednesday:The Volcano forecast for Apr 12 is 52 degrees and Patchy light rain.\\n\\nVolcano, CA weather and traffic updates by locals. Write your own weather report, forecast, or traffic update: Please note by clicking on Post you acknowledge that you have read the Terms of Service and the report and/or forecast you are posting is in compliance with such terms. Be respectful.\\n\\nHourly Forecast Detailed. 1  0am:The Volcano, CA forecast for Apr 03 is 48 degrees and Patchy rain possible. 2  3am:The Volcano, CA forecast for Apr 03 is 44 degrees and Clear. 3  6am:The Volcano, CA forecast for Apr 03 is 41 degrees and Clear.  9am:The Volcano, CA forecast for Apr 03 is 48 degrees and Sunny.\"\n",
-      "}\n",
-      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain > 5:llm:ChatOpenAI] Entering LLM run with input:\n",
-      "\u001b[0m{\n",
-      "  \"prompts\": [\n",
-      "    \"System: Use the following pieces of context to answer the users question. \\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\\n----------------\\nVolcano 10 Day Weather. Sunday:The Volcano forecast for Apr 09 is 43 degrees and Sunny. There is 55 percentage chance of rain and 4 mph winds from the Southwest. Monday:The Volcano forecast for Apr 10 is 51 degrees and Sunny.\\n\\nCurrent U.S. National Radar--Current. The Current National Weather Radar is shown below with a UTC Time (subtract 5 hours from UTC to get Eastern Time). National Weather Forecast--Current. The Current National Weather Forecast and National Weather Map are shown below.\\n\\nVolcano 10 Day Weather. 1  Sunday:The Volcano forecast for Apr 09 is 43 degrees and Sunny. There is 55 percentage chance of rain and 4 mph winds from the Southwest. 2  Monday:The Volcano forecast for Apr 10 is 51 degrees and Sunny. There is 49 percentage chance of rain and 3 mph winds from the Southwest.\\n\\nVolcano, CA Weather Data. 1  Volcano, CA Current Weather Data. 2  Sponsored. 3  Volcano, CA Historical Weather Trends.  Volcano, CA area 1  Highlights. Volcano, CA Chance of Sunshine.  Volcano, CA Historical 1  Temperature. Volcano, CA Rainfall and Snowfall Average.  Volcano, CA Energy Demand.\\n\\nVolcano Weather. Volcano weather and daily current conditions with summary and 5 Day forecast including humidity, precipitation, high and low temperatures presented in Fahrenheit and Celsius, barometric pressure, heat index, wind chill, hourly forecast, sunrise, sunset, wind speed with direction, and more.\\n\\nHourly Forecast Detailed. 1  0am:The Volcano, CA forecast for Apr 03 is 48 degrees and Patchy rain possible. There is 83 percentage chance of rain and 2 mph winds from the East. 2  3am:The Volcano, CA forecast for Apr 03 is 44 degrees and Clear. There is 77 percentage chance of rain and 2 mph winds from the East.\\n\\nVolcano 7 Day Weather. 1  Monday:The Volcano forecast for Apr 03 is 58 degrees and Sunny. There is 34 percentage chance of rain and 5 mph winds from the West. 2  Tuesday:The Volcano forecast for Apr 04 is 59 degrees and Sunny. There is 33 percentage chance of rain and 5 mph winds from the West-Southwest.\\n\\nVolcano 10 Day Weather. 1  Sunday:The Volcano forecast for Apr 09 is 43 degrees and Sunny. 2  Monday:The Volcano forecast for Apr 10 is 51 degrees and Sunny. 3  Tuesday:The Volcano forecast for Apr 11 is 49 degrees and Patchy rain possible.  Wednesday:The Volcano forecast for Apr 12 is 52 degrees and Patchy light rain.\\n\\nVolcano, CA weather and traffic updates by locals. Write your own weather report, forecast, or traffic update: Please note by clicking on Post you acknowledge that you have read the Terms of Service and the report and/or forecast you are posting is in compliance with such terms. Be respectful.\\n\\nHourly Forecast Detailed. 1  0am:The Volcano, CA forecast for Apr 03 is 48 degrees and Patchy rain possible. 2  3am:The Volcano, CA forecast for Apr 03 is 44 degrees and Clear. 3  6am:The Volcano, CA forecast for Apr 03 is 41 degrees and Clear.  9am:The Volcano, CA forecast for Apr 03 is 48 degrees and Sunny.\\nHuman: current weather in volcano, ca\"\n",
-      "  ]\n",
-      "}\n",
-      "I don't have the current weather information for Volcano, CA.\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain > 5:llm:ChatOpenAI] [1.04s] Exiting LLM run with output:\n",
-      "\u001b[0m{\n",
-      "  \"generations\": [\n",
-      "    [\n",
-      "      {\n",
-      "        \"text\": \"I don't have the current weather information for Volcano, CA.\",\n",
-      "        \"generation_info\": {\n",
-      "          \"finish_reason\": \"stop\"\n",
-      "        },\n",
-      "        \"message\": {\n",
-      "          \"lc\": 1,\n",
-      "          \"type\": \"constructor\",\n",
-      "          \"id\": [\n",
-      "            \"langchain\",\n",
-      "            \"schema\",\n",
-      "            \"messages\",\n",
-      "            \"AIMessageChunk\"\n",
-      "          ],\n",
-      "          \"kwargs\": {\n",
-      "            \"example\": false,\n",
-      "            \"content\": \"I don't have the current weather information for Volcano, CA.\",\n",
-      "            \"additional_kwargs\": {}\n",
-      "          }\n",
-      "        }\n",
-      "      }\n",
-      "    ]\n",
-      "  ],\n",
-      "  \"llm_output\": null,\n",
-      "  \"run\": null\n",
-      "}\n",
-      "\n",
-      "\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain] [1.04s] Exiting Chain run with output:\n",
-      "\u001b[0m{\n",
-      "  \"text\": \"I don't have the current weather information for Volcano, CA.\"\n",
-      "}\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain] [1.04s] Exiting Chain run with output:\n",
-      "\u001b[0m{\n",
-      "  \"output_text\": \"I don't have the current weather information for Volcano, CA.\"\n",
-      "}\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain] [1.04s] Exiting Chain run with output:\n",
-      "\u001b[0m[outputs]\n",
-      "{'question': '____________________ is considered the father of modern medicine.', 'chat_history': []}\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain] Entering Chain run with input:\n",
-      "\u001b[0m{\n",
-      "  \"question\": \"____________________ is considered the father of modern medicine.\",\n",
-      "  \"chat_history\": []\n",
-      "}\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain] Entering Chain run with input:\n",
-      "\u001b[0m[inputs]\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain] Entering Chain run with input:\n",
-      "\u001b[0m{\n",
-      "  \"question\": \"____________________ is considered the father of modern medicine.\",\n",
-      "  \"context\": \"Hippocrates is widely considered to be the Father of Medicine. His contributions revolutionized the practice of medicine; but after his death the advancement stalled.\\n\\nMany of the invaluable lessons prescribed in that place of learning are assigned to Hippocrates. If that was the case, then it truly was Hippocrates, with his approach to healing and the role of the doctor, that influenced western medicine for thousands of years.\\n\\nDespite this, Hippocrates is attributed with a great many wonderful deeds and thoughts. He is recognised as the founder of the Hippocratic School of Medicine, a college that revolutionized the understanding of medicine in Ancient Greece.\\n\\nAt least that is what we’d like to think. While his fame was such to warrant a mention from the likes of Plato and Aristotle, not much is actually known about Hippocrates the father of Medicine. Consequently, he has become the projection of what people ideally want in a physician.\\n\\n460 – c. 370 BC) was a Greek physician of the Age of Pericles (Classical Greece), and is considered one of the most outstanding figures in the history of medicine.\\n\\nTRUE. Hippocrates is considered the father of modern medicine because he did not believe that illness was a punishment inflicted by the gods. True False. Weegy: TRUE. [ \\n\\nThe two sons of Hippocrates, Thessalus and Draco, and his son-in-law, Polybus, were his students. According to Galen, a later physician, Polybus was Hippocrates' true successor, while Thessalus and Draco each had a son named Hippocrates.\\n\\nHippocrates is mentioned in passing in the writings of two contemporaries: Plato, in Protagoras and Phaedrus, and, Aristotle 's Politics, which date from the 4th century BC. Soranus wrote that Hippocrates' father was Heraclides, a physician, and his mother was Praxitela, daughter of Tizane.\\n\\nReload the page to try again! Press Cmd-0 to reset your zoom. Press Ctrl-0 to reset your zoom. It looks like your browser might be zoomed in or out. Your browser needs to be zoomed to a normal size to record audio.\\n\\nHowever, the achievements of the writers of the Corpus, the practitioners of Hippocratic medicine, and the actions of Hippocrates himself were often commingled; thus very little is known about what Hippocrates actually thought, wrote, and did.\"\n",
-      "}\n",
-      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain > 5:llm:ChatOpenAI] Entering LLM run with input:\n",
-      "\u001b[0m{\n",
-      "  \"prompts\": [\n",
-      "    \"System: Use the following pieces of context to answer the users question. \\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\\n----------------\\nHippocrates is widely considered to be the Father of Medicine. His contributions revolutionized the practice of medicine; but after his death the advancement stalled.\\n\\nMany of the invaluable lessons prescribed in that place of learning are assigned to Hippocrates. If that was the case, then it truly was Hippocrates, with his approach to healing and the role of the doctor, that influenced western medicine for thousands of years.\\n\\nDespite this, Hippocrates is attributed with a great many wonderful deeds and thoughts. He is recognised as the founder of the Hippocratic School of Medicine, a college that revolutionized the understanding of medicine in Ancient Greece.\\n\\nAt least that is what we’d like to think. While his fame was such to warrant a mention from the likes of Plato and Aristotle, not much is actually known about Hippocrates the father of Medicine. Consequently, he has become the projection of what people ideally want in a physician.\\n\\n460 – c. 370 BC) was a Greek physician of the Age of Pericles (Classical Greece), and is considered one of the most outstanding figures in the history of medicine.\\n\\nTRUE. Hippocrates is considered the father of modern medicine because he did not believe that illness was a punishment inflicted by the gods. True False. Weegy: TRUE. [ \\n\\nThe two sons of Hippocrates, Thessalus and Draco, and his son-in-law, Polybus, were his students. According to Galen, a later physician, Polybus was Hippocrates' true successor, while Thessalus and Draco each had a son named Hippocrates.\\n\\nHippocrates is mentioned in passing in the writings of two contemporaries: Plato, in Protagoras and Phaedrus, and, Aristotle 's Politics, which date from the 4th century BC. Soranus wrote that Hippocrates' father was Heraclides, a physician, and his mother was Praxitela, daughter of Tizane.\\n\\nReload the page to try again! Press Cmd-0 to reset your zoom. Press Ctrl-0 to reset your zoom. It looks like your browser might be zoomed in or out. Your browser needs to be zoomed to a normal size to record audio.\\n\\nHowever, the achievements of the writers of the Corpus, the practitioners of Hippocratic medicine, and the actions of Hippocrates himself were often commingled; thus very little is known about what Hippocrates actually thought, wrote, and did.\\nHuman: ____________________ is considered the father of modern medicine.\"\n",
-      "  ]\n",
-      "}\n",
-      "Hippocrates is considered the father of modern medicine.\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain > 5:llm:ChatOpenAI] [654ms] Exiting LLM run with output:\n",
-      "\u001b[0m{\n",
-      "  \"generations\": [\n",
-      "    [\n",
-      "      {\n",
-      "        \"text\": \"Hippocrates is considered the father of modern medicine.\",\n",
-      "        \"generation_info\": {\n",
-      "          \"finish_reason\": \"stop\"\n",
-      "        },\n",
-      "        \"message\": {\n",
-      "          \"lc\": 1,\n",
-      "          \"type\": \"constructor\",\n",
-      "          \"id\": [\n",
-      "            \"langchain\",\n",
-      "            \"schema\",\n",
-      "            \"messages\",\n",
-      "            \"AIMessageChunk\"\n",
-      "          ],\n",
-      "          \"kwargs\": {\n",
-      "            \"example\": false,\n",
-      "            \"content\": \"Hippocrates is considered the father of modern medicine.\",\n",
-      "            \"additional_kwargs\": {}\n",
-      "          }\n",
-      "        }\n",
-      "      }\n",
-      "    ]\n",
-      "  ],\n",
-      "  \"llm_output\": null,\n",
-      "  \"run\": null\n",
-      "}\n",
-      "\n",
-      "\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain] [655ms] Exiting Chain run with output:\n",
-      "\u001b[0m{\n",
-      "  \"text\": \"Hippocrates is considered the father of modern medicine.\"\n",
-      "}\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain] [655ms] Exiting Chain run with output:\n",
-      "\u001b[0m{\n",
-      "  \"output_text\": \"Hippocrates is considered the father of modern medicine.\"\n",
-      "}\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain] [657ms] Exiting Chain run with output:\n",
-      "\u001b[0m[outputs]\n",
-      "{'question': 'how many days is an appraisal good for a fannie loan', 'chat_history': []}\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain] Entering Chain run with input:\n",
-      "\u001b[0m{\n",
-      "  \"question\": \"how many days is an appraisal good for a fannie loan\",\n",
-      "  \"chat_history\": []\n",
-      "}\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain] Entering Chain run with input:\n",
-      "\u001b[0m[inputs]\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain] Entering Chain run with input:\n",
-      "\u001b[0m{\n",
-      "  \"question\": \"how many days is an appraisal good for a fannie loan\",\n",
-      "  \"context\": \"New and Updated Underwriting and Eligibility Policies. Age of Credit Documents Selling Guide, B1-1-04, Allowable Age of Credit. Documents. The maximum age of credit documents is reduced from 120 days to 90 days for existing. construction and from 180 days to 120 days for new construction. Credit documents include. credit reports and employment, income, and asset documentation. The age of the documents is.\\n\\nIn no case may the appraisal be dated more than 1 year prior to the date of the Note. Property Inspection Reports/Condition and Marketability Reports (Fannie Mae Form 2070/Freddie Mac Form. 2075 may be dated no earlier than 120 days from the date of the Note. Continued on next page.\\n\\nFannie Mae will allow the use of an origination appraisal for a subsequent transaction if the following requirements are met: 1  The subsequent transaction may only be a Limited Cash-Out Refinance. 2  The appraisal report must not be more than 12 months old on the note date of the subsequent transaction.\\n\\nThe subsequent transaction may only be a Limited Cash-Out Refinance. The appraisal report must not be more than 12 months old on the note date of the subsequent transaction. If the appraisal report is greater than 4 months old on the date of the note and mortgage, then an appraisal update is required.\\n\\nIf they were sold with exposure to the market, listed in. MLS they should be considered. How long is the FHA case # good for (not the appraisal, but the actual case #)? the case number is valid for 6 months unless the appraiser expires prior to the 6 month time frame.\\n\\nNo the borrower can only pay for one appraisal. Your question about Comps is not acceptable, Comps over 1 year old for comps 1-3 are not. acceptable, but supporting comps are with an adequate explanation from the Appraiser. Comps, over one year old would be acceptable, onlywith a waiver request by the lender.\\n\\nThe appraisal may be dated no earlier than 120 days from the date of the Note, regardless of whether the. property was appraised as proposed or existing construction. When the appraisal will be more than 120 days old but less than 1 year old on the date of the Note, the.\\n\\nThis inspection and results of the analysis must be reported on the Appraisal Update and/or Completion Report (Form 1004D). 1  If the appraiser indicates on the Form 1004D that the property value has declined, then the lender must obtain a new appraisal for the property.\\n\\nUnfortunately, that is a complete new order (and expense), as one year is a lifetime where property values are concerned. Thanks for the information, however I asked for the PMi to be removed just over 3 months after the appraisal. In fact, the manager at Nationstar said it was 91 days and not valid per Fannie Mae.\\n\\nReputation: 6463. Actually, Fannie Mae and FHA went to 120 days, but at no time were they ever over 6 months for existing construction. Generally, comparables from August 2009 could not be included on an appraisal report today, so there is no way an appraisal issued then would be acceptable.\"\n",
-      "}\n",
-      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain > 5:llm:ChatOpenAI] Entering LLM run with input:\n",
-      "\u001b[0m{\n",
-      "  \"prompts\": [\n",
-      "    \"System: Use the following pieces of context to answer the users question. \\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\\n----------------\\nNew and Updated Underwriting and Eligibility Policies. Age of Credit Documents Selling Guide, B1-1-04, Allowable Age of Credit. Documents. The maximum age of credit documents is reduced from 120 days to 90 days for existing. construction and from 180 days to 120 days for new construction. Credit documents include. credit reports and employment, income, and asset documentation. The age of the documents is.\\n\\nIn no case may the appraisal be dated more than 1 year prior to the date of the Note. Property Inspection Reports/Condition and Marketability Reports (Fannie Mae Form 2070/Freddie Mac Form. 2075 may be dated no earlier than 120 days from the date of the Note. Continued on next page.\\n\\nFannie Mae will allow the use of an origination appraisal for a subsequent transaction if the following requirements are met: 1  The subsequent transaction may only be a Limited Cash-Out Refinance. 2  The appraisal report must not be more than 12 months old on the note date of the subsequent transaction.\\n\\nThe subsequent transaction may only be a Limited Cash-Out Refinance. The appraisal report must not be more than 12 months old on the note date of the subsequent transaction. If the appraisal report is greater than 4 months old on the date of the note and mortgage, then an appraisal update is required.\\n\\nIf they were sold with exposure to the market, listed in. MLS they should be considered. How long is the FHA case # good for (not the appraisal, but the actual case #)? the case number is valid for 6 months unless the appraiser expires prior to the 6 month time frame.\\n\\nNo the borrower can only pay for one appraisal. Your question about Comps is not acceptable, Comps over 1 year old for comps 1-3 are not. acceptable, but supporting comps are with an adequate explanation from the Appraiser. Comps, over one year old would be acceptable, onlywith a waiver request by the lender.\\n\\nThe appraisal may be dated no earlier than 120 days from the date of the Note, regardless of whether the. property was appraised as proposed or existing construction. When the appraisal will be more than 120 days old but less than 1 year old on the date of the Note, the.\\n\\nThis inspection and results of the analysis must be reported on the Appraisal Update and/or Completion Report (Form 1004D). 1  If the appraiser indicates on the Form 1004D that the property value has declined, then the lender must obtain a new appraisal for the property.\\n\\nUnfortunately, that is a complete new order (and expense), as one year is a lifetime where property values are concerned. Thanks for the information, however I asked for the PMi to be removed just over 3 months after the appraisal. In fact, the manager at Nationstar said it was 91 days and not valid per Fannie Mae.\\n\\nReputation: 6463. Actually, Fannie Mae and FHA went to 120 days, but at no time were they ever over 6 months for existing construction. Generally, comparables from August 2009 could not be included on an appraisal report today, so there is no way an appraisal issued then would be acceptable.\\nHuman: how many days is an appraisal good for a fannie loan\"\n",
-      "  ]\n",
-      "}\n",
-      "According to the provided information, an appraisal for a Fannie Mae loan is typically valid for up to 120 days from the date of the Note.\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain > 5:llm:ChatOpenAI] [1.02s] Exiting LLM run with output:\n",
-      "\u001b[0m{\n",
-      "  \"generations\": [\n",
-      "    [\n",
-      "      {\n",
-      "        \"text\": \"According to the provided information, an appraisal for a Fannie Mae loan is typically valid for up to 120 days from the date of the Note.\",\n",
-      "        \"generation_info\": {\n",
-      "          \"finish_reason\": \"stop\"\n",
-      "        },\n",
-      "        \"message\": {\n",
-      "          \"lc\": 1,\n",
-      "          \"type\": \"constructor\",\n",
-      "          \"id\": [\n",
-      "            \"langchain\",\n",
-      "            \"schema\",\n",
-      "            \"messages\",\n",
-      "            \"AIMessageChunk\"\n",
-      "          ],\n",
-      "          \"kwargs\": {\n",
-      "            \"example\": false,\n",
-      "            \"content\": \"According to the provided information, an appraisal for a Fannie Mae loan is typically valid for up to 120 days from the date of the Note.\",\n",
-      "            \"additional_kwargs\": {}\n",
-      "          }\n",
-      "        }\n",
-      "      }\n",
-      "    ]\n",
-      "  ],\n",
-      "  \"llm_output\": null,\n",
-      "  \"run\": null\n",
-      "}\n",
-      "\n",
-      "\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain] [1.02s] Exiting Chain run with output:\n",
-      "\u001b[0m{\n",
-      "  \"text\": \"According to the provided information, an appraisal for a Fannie Mae loan is typically valid for up to 120 days from the date of the Note.\"\n",
-      "}\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain] [1.02s] Exiting Chain run with output:\n",
-      "\u001b[0m{\n",
-      "  \"output_text\": \"According to the provided information, an appraisal for a Fannie Mae loan is typically valid for up to 120 days from the date of the Note.\"\n",
-      "}\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain] [1.02s] Exiting Chain run with output:\n",
-      "\u001b[0m[outputs]\n",
-      "{'question': 'average pharmacy tech salary', 'chat_history': []}\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain] Entering Chain run with input:\n",
-      "\u001b[0m{\n",
-      "  \"question\": \"average pharmacy tech salary\",\n",
-      "  \"chat_history\": []\n",
-      "}\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain] Entering Chain run with input:\n",
-      "\u001b[0m[inputs]\n",
-      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain] Entering Chain run with input:\n",
-      "\u001b[0m{\n",
-      "  \"question\": \"average pharmacy tech salary\",\n",
-      "  \"context\": \"If you are interested in becoming a pharmacy technician, you’re choosing a career that is in high demand. According to the U.S. Bureau of Labor Statistics (BLS), the career growth is expected to be “much faster than average”, with an employment increase of 32% predicted in the decade spanning 2010 to 2020*.\\n\\nWhat can a pharmacy technician really expect to earn in today’s economy? According to Salary.com, pharmacy technicians make anywhere from $26,000 to $39,000 a year, though most make around $32,000 annually. California has the highest average pharmacy technician wage, at $34,317, according to Open Farm Tech’s website.\\n\\nThe median annual wage for pharmacy technicians was $30,410 in May 2015. Employment of pharmacy technicians is projected to grow 9 percent from 2014 to 2024, faster than the average for all occupations. Increased demand for prescription medications will lead to more demand for pharmaceutical services.\\n\\nThe majority of pharmacy techs work in drug stores and hospitals, where the average annual salary was $28,940 and $34,410, respectively**. However, a higher salary can be had if you can find employment with outpatient care centers or physicians’ offices, where the annual pay is in the $37,000-$39,000 range.\\n\\nThe pharmacy technician salary** depends on a number of factors, from the area and type of employer, to your educational background. Browse pharmacy tech pay for a comparison between similar careers, geographic location, educational and certification requirements, and more.\\n\\nPharmacy Technician Salary. A Pharmacy Technician earns an average wage of $12.68 per hour. The skills that increase pay for this job the most are Mail Order Pharmacy and Long Term Care. People in this job generally don't have more than 20 years' experience. $18,722 - $48,714.\\n\\nPopular Companies. * Please note that all salary figures are approximations based upon third party submissions to Simply Hired. These figures are given to Simply Hired users for the purpose of generalized comparison only. Minimum wage may differ by jurisdiction and you should consult the employer for actual salary figures.\\n\\nPharmacy Technician average salary is $30,288, median salary is $30,534 with a salary range from $21,570 to $34,320. Pharmacy Technician salaries are collected from government agencies and companies. Each salary is associated with a real job position. Pharmacy Technician salary statistics is not exclusive and is for reference only.\\n\\nIt also states that pharmacy technicians working in an acute care hospital earn an average salary of $37,000 per year, while those working for the military or a pharmaceutical company earn an average salary of $38,000 per year. This represents a difference of more than $10,000, simply due to the health care setting.\\n\\nOccupational Employment and Wages, May 2016. 29-2052 Pharmacy Technicians. Prepare medications under the direction of a pharmacist. May measure, mix, count out, label, and record amounts and dosages of medications according to prescription orders. National estimates for this occupation. Industry profile for this occupation.\"\n",
-      "}\n",
-      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain > 5:llm:ChatOpenAI] Entering LLM run with input:\n",
-      "\u001b[0m{\n",
-      "  \"prompts\": [\n",
-      "    \"System: Use the following pieces of context to answer the users question. \\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\\n----------------\\nIf you are interested in becoming a pharmacy technician, you’re choosing a career that is in high demand. According to the U.S. Bureau of Labor Statistics (BLS), the career growth is expected to be “much faster than average”, with an employment increase of 32% predicted in the decade spanning 2010 to 2020*.\\n\\nWhat can a pharmacy technician really expect to earn in today’s economy? According to Salary.com, pharmacy technicians make anywhere from $26,000 to $39,000 a year, though most make around $32,000 annually. California has the highest average pharmacy technician wage, at $34,317, according to Open Farm Tech’s website.\\n\\nThe median annual wage for pharmacy technicians was $30,410 in May 2015. Employment of pharmacy technicians is projected to grow 9 percent from 2014 to 2024, faster than the average for all occupations. Increased demand for prescription medications will lead to more demand for pharmaceutical services.\\n\\nThe majority of pharmacy techs work in drug stores and hospitals, where the average annual salary was $28,940 and $34,410, respectively**. However, a higher salary can be had if you can find employment with outpatient care centers or physicians’ offices, where the annual pay is in the $37,000-$39,000 range.\\n\\nThe pharmacy technician salary** depends on a number of factors, from the area and type of employer, to your educational background. Browse pharmacy tech pay for a comparison between similar careers, geographic location, educational and certification requirements, and more.\\n\\nPharmacy Technician Salary. A Pharmacy Technician earns an average wage of $12.68 per hour. The skills that increase pay for this job the most are Mail Order Pharmacy and Long Term Care. People in this job generally don't have more than 20 years' experience. $18,722 - $48,714.\\n\\nPopular Companies. * Please note that all salary figures are approximations based upon third party submissions to Simply Hired. These figures are given to Simply Hired users for the purpose of generalized comparison only. Minimum wage may differ by jurisdiction and you should consult the employer for actual salary figures.\\n\\nPharmacy Technician average salary is $30,288, median salary is $30,534 with a salary range from $21,570 to $34,320. Pharmacy Technician salaries are collected from government agencies and companies. Each salary is associated with a real job position. Pharmacy Technician salary statistics is not exclusive and is for reference only.\\n\\nIt also states that pharmacy technicians working in an acute care hospital earn an average salary of $37,000 per year, while those working for the military or a pharmaceutical company earn an average salary of $38,000 per year. This represents a difference of more than $10,000, simply due to the health care setting.\\n\\nOccupational Employment and Wages, May 2016. 29-2052 Pharmacy Technicians. Prepare medications under the direction of a pharmacist. May measure, mix, count out, label, and record amounts and dosages of medications according to prescription orders. National estimates for this occupation. Industry profile for this occupation.\\nHuman: average pharmacy tech salary\"\n",
-      "  ]\n",
-      "}\n",
-      "The average salary for a pharmacy technician can vary depending on factors such as location, employer, and experience. However, based on the information provided, the average salary for a pharmacy technician is around $30,000 to $34,000 per year.\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain > 5:llm:ChatOpenAI] [1.45s] Exiting LLM run with output:\n",
-      "\u001b[0m{\n",
-      "  \"generations\": [\n",
-      "    [\n",
-      "      {\n",
-      "        \"text\": \"The average salary for a pharmacy technician can vary depending on factors such as location, employer, and experience. However, based on the information provided, the average salary for a pharmacy technician is around $30,000 to $34,000 per year.\",\n",
-      "        \"generation_info\": {\n",
-      "          \"finish_reason\": \"stop\"\n",
-      "        },\n",
-      "        \"message\": {\n",
-      "          \"lc\": 1,\n",
-      "          \"type\": \"constructor\",\n",
-      "          \"id\": [\n",
-      "            \"langchain\",\n",
-      "            \"schema\",\n",
-      "            \"messages\",\n",
-      "            \"AIMessageChunk\"\n",
-      "          ],\n",
-      "          \"kwargs\": {\n",
-      "            \"example\": false,\n",
-      "            \"content\": \"The average salary for a pharmacy technician can vary depending on factors such as location, employer, and experience. However, based on the information provided, the average salary for a pharmacy technician is around $30,000 to $34,000 per year.\",\n",
-      "            \"additional_kwargs\": {}\n",
-      "          }\n",
-      "        }\n",
-      "      }\n",
-      "    ]\n",
-      "  ],\n",
-      "  \"llm_output\": null,\n",
-      "  \"run\": null\n",
-      "}\n",
-      "\n",
-      "\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain > 4:chain:LLMChain] [1.45s] Exiting Chain run with output:\n",
-      "\u001b[0m{\n",
-      "  \"text\": \"The average salary for a pharmacy technician can vary depending on factors such as location, employer, and experience. However, based on the information provided, the average salary for a pharmacy technician is around $30,000 to $34,000 per year.\"\n",
-      "}\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain > 3:chain:StuffDocumentsChain] [1.46s] Exiting Chain run with output:\n",
-      "\u001b[0m{\n",
-      "  \"output_text\": \"The average salary for a pharmacy technician can vary depending on factors such as location, employer, and experience. However, based on the information provided, the average salary for a pharmacy technician is around $30,000 to $34,000 per year.\"\n",
-      "}\n",
-      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:ConversationalRetrievalChain] [1.46s] Exiting Chain run with output:\n",
-      "\u001b[0m[outputs]\n",
-      "Q-001: albany mn population\n",
-      "A-001: The population of Albany, Minnesota is approximately 2,561 as of the 2010 census. However, according to a 2017 US Census estimate, the community population has increased to 2,662 people.\n",
-      "G-001: The population of Albany, Minnesota is 2,662. \n",
-      "\n",
-      "Q-002: current weather in volcano, ca\n",
-      "A-002: I don't have the current weather information for Volcano, CA.\n",
-      "G-002: The Volcano forecast for Apr 12 is 52 degrees and Patchy light rain.\n",
-      "\n",
-      "Q-003: ____________________ is considered the father of modern medicine.\n",
-      "A-003: Hippocrates is considered the father of modern medicine.\n",
-      "G-003: Hippocrates is considered the father of modern medicine.\n",
-      "\n",
-      "Q-004: how many days is an appraisal good for a fannie loan\n",
-      "A-004: According to the provided information, an appraisal for a Fannie Mae loan is typically valid for up to 120 days from the date of the Note.\n",
-      "G-004: An appraisal is good for 120 days from the date of the Note for a Fannie loan.\n",
-      "\n",
-      "Q-005: average pharmacy tech salary\n",
-      "A-005: The average salary for a pharmacy technician can vary depending on factors such as location, employer, and experience. However, based on the information provided, the average salary for a pharmacy technician is around $30,000 to $34,000 per year.\n",
-      "G-005: The average salary for a pharmacy technician is $26,000 to $39,000 in a year.\n",
-      "\n",
-      "\n",
-      "\n",
-      "scores: {\n",
-      "  \"OVERALL\": {\n",
-      "    \"bleu_scores\": {\n",
-      "      \"bleu\": 0.3953488372093023,\n",
-      "      \"precisions\": [\n",
-      "        0.3953488372093023\n",
-      "      ],\n",
-      "      \"brevity_penalty\": 1.0,\n",
-      "      \"length_ratio\": 1.9253731343283582,\n",
-      "      \"translation_length\": 129,\n",
-      "      \"reference_length\": 67\n",
-      "    },\n",
-      "    \"rouge_scores\": {\n",
-      "      \"rouge1\": 0.5737456342107505,\n",
-      "      \"rouge2\": 0.4160794941282746,\n",
-      "      \"rougeL\": 0.5108953062441435,\n",
-      "      \"rougeLsum\": 0.4989862850327967\n",
-      "    }\n",
-      "  },\n",
-      "  \"NUMERIC\": {\n",
-      "    \"bleu_scores\": {\n",
-      "      \"bleu\": 0.36111111111111116,\n",
-      "      \"precisions\": [\n",
-      "        0.3611111111111111\n",
-      "      ],\n",
-      "      \"brevity_penalty\": 1.0,\n",
-      "      \"length_ratio\": 2.4545454545454546,\n",
-      "      \"translation_length\": 108,\n",
-      "      \"reference_length\": 44\n",
-      "    },\n",
-      "    \"rouge_scores\": {\n",
-      "      \"rouge1\": 0.5395760570179174,\n",
-      "      \"rouge2\": 0.3694751662231337,\n",
-      "      \"rougeL\": 0.4656557912371866,\n",
-      "      \"rougeLsum\": 0.4656557912371866\n",
-      "    }\n",
-      "  },\n",
-      "  \"DESCRIPTION\": {\n",
-      "    \"bleu_scores\": {\n",
-      "      \"bleu\": 0.5195179673581217,\n",
-      "      \"precisions\": [\n",
-      "        0.5714285714285714\n",
-      "      ],\n",
-      "      \"brevity_penalty\": 0.909156442876713,\n",
-      "      \"length_ratio\": 0.9130434782608695,\n",
-      "      \"translation_length\": 21,\n",
-      "      \"reference_length\": 23\n",
-      "    },\n",
-      "    \"rouge_scores\": {\n",
-      "      \"rouge1\": 0.625,\n",
-      "      \"rouge2\": 0.5,\n",
-      "      \"rougeL\": 0.5833333333333334,\n",
-      "      \"rougeLsum\": 0.5833333333333334\n",
-      "    }\n",
-      "  }\n",
-      "}\n",
-      "\n",
-      "CPU times: user 512 ms, sys: 63.7 ms, total: 576 ms\n",
-      "Wall time: 7.85 s\n"
-     ]
-    }
-   ],
-   "source": [
-    "%%time\n",
-    "\n",
-    "eval_ds = new_ds.select(range(5))\n",
-    "qa_chain = QAChainWithMsMacroDataset(eval_ds, llm_loader)\n",
-    "\n",
-    "answers = []\n",
-    "for i in range(eval_ds.num_rows):\n",
-    "    inputs = {\"question\": str(eval_ds[i][\"query\"]), \"chat_history\": []}\n",
-    "    result = qa_chain.call_chain(\n",
-    "        inputs,\n",
-    "        None,\n",
-    "        None,\n",
-    "        True,\n",
-    "    )\n",
-    "    answers.append(result[\"answer\"])\n",
-    "\n",
-    "result = calc_all_metrics(eval_ds, answers)\n",
-    "\n",
-    "for i in range(eval_ds.num_rows):\n",
-    "    n = i + 1\n",
-    "    print(f\"Q-{n:03d}: {eval_ds[i]['query']}\")\n",
-    "    print(f\"A-{n:03d}: {answers[i]}\")\n",
-    "    print(f\"G-{n:03d}: {eval_ds[i]['wellFormedAnswers'][0]}\\n\")\n",
-    "\n",
-    "print(f\"\\n\\nscores: {json.dumps(result, indent=2)}\\n\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "bae05024",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "System: Use the following pieces of context to answer the users question. \n",
-      "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n",
-      "----------------\n",
-      "City of Albany, MN Zip Codes. City of Albany, MN Demographic Information. * Demographic data is based on information taken from the 2000 Census. City of Albany, MN covers 1 Area Code. City of Albany, MN covers 1 Zip Code. 15 Cities within 15 Miles of the City of Albany, MN.\n",
-      "\n",
-      "Place of birth for U.S.-born residents: 70% of the 56307 zip code residents lived in the same house 5 years ago. Out of people who lived in different houses, 71% lived in this county. Out of people who lived in different counties, 50% lived in Minnesota. 92% of the 56307 zip code residents lived in the same house 1 year ago.\n",
-      "\n",
-      "For the unincorporated community in southeast Minnesota named West Albany, see West Albany, Minnesota. Albany is a city in Stearns County, Minnesota, United States. The population was 2,561 at the 2010 census. It is part of the St. Cloud Metropolitan Statistical Area.\n",
-      "\n",
-      "Albany, Minnesota, as per 2017 US Census estimate, has a community population of 2,662 people. Albany is located in Stearns County, 20 miles west of St. Cloud and 80 miles northwest of Minneapolis/St. Paul on Interstate 94 (I-94). Albany has direct access to State Highway 238, which originates in Albany.\n",
-      "\n",
-      "Sponsored Topics. Albany is a city in Stearns County, Minnesota, United States. The population was 2,561 at the 2010 census. It is part of the St. Cloud Metropolitan Statistical Area.\n",
-      "\n",
-      "Recent posts about Albany, Minnesota on our local forum with over 2,000,000 registered users. Albany is mentioned 87 times on our forum: Latest news from Albany, MN collected exclusively by city-data.com from local newspapers, TV, and radio stations. Ancestries: German (55.6%), Irish (10.0%), Polish (5.9%), Norwegian (5.4%), Swedish (2.8%), United States (2.6%).\n",
-      "\n",
-      "For population 25 years and over in 56307: 1  High school or higher: 87.4%. 2  Bachelor's degree or higher: 15.4%. 3  Graduate or professional degree: 3.3 4 %. Unemployed: 3. 5 2%. Mean travel time to work (commute): 23.6 minutes.\n",
-      "\n",
-      "For population 25 years and over in Albany: 1  High school or higher: 86.7%. 2  Bachelor's degree or higher: 15.4%. 3  Graduate or professional degree: 4.4 4 %. Unemployed: 4. 5 3%. Mean travel time to work (commute): 23.0 minutes.\n",
-      "Human: albany mn population\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\n",
-    "    \"System: Use the following pieces of context to answer the users question. \\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\\n----------------\\nCity of Albany, MN Zip Codes. City of Albany, MN Demographic Information. * Demographic data is based on information taken from the 2000 Census. City of Albany, MN covers 1 Area Code. City of Albany, MN covers 1 Zip Code. 15 Cities within 15 Miles of the City of Albany, MN.\\n\\nPlace of birth for U.S.-born residents: 70% of the 56307 zip code residents lived in the same house 5 years ago. Out of people who lived in different houses, 71% lived in this county. Out of people who lived in different counties, 50% lived in Minnesota. 92% of the 56307 zip code residents lived in the same house 1 year ago.\\n\\nFor the unincorporated community in southeast Minnesota named West Albany, see West Albany, Minnesota. Albany is a city in Stearns County, Minnesota, United States. The population was 2,561 at the 2010 census. It is part of the St. Cloud Metropolitan Statistical Area.\\n\\nAlbany, Minnesota, as per 2017 US Census estimate, has a community population of 2,662 people. Albany is located in Stearns County, 20 miles west of St. Cloud and 80 miles northwest of Minneapolis/St. Paul on Interstate 94 (I-94). Albany has direct access to State Highway 238, which originates in Albany.\\n\\nSponsored Topics. Albany is a city in Stearns County, Minnesota, United States. The population was 2,561 at the 2010 census. It is part of the St. Cloud Metropolitan Statistical Area.\\n\\nRecent posts about Albany, Minnesota on our local forum with over 2,000,000 registered users. Albany is mentioned 87 times on our forum: Latest news from Albany, MN collected exclusively by city-data.com from local newspapers, TV, and radio stations. Ancestries: German (55.6%), Irish (10.0%), Polish (5.9%), Norwegian (5.4%), Swedish (2.8%), United States (2.6%).\\n\\nFor population 25 years and over in 56307: 1  High school or higher: 87.4%. 2  Bachelor's degree or higher: 15.4%. 3  Graduate or professional degree: 3.3 4 %. Unemployed: 3. 5 2%. Mean travel time to work (commute): 23.6 minutes.\\n\\nFor population 25 years and over in Albany: 1  High school or higher: 86.7%. 2  Bachelor's degree or higher: 15.4%. 3  Graduate or professional degree: 4.4 4 %. Unemployed: 4. 5 3%. Mean travel time to work (commute): 23.0 minutes.\\nHuman: albany mn population\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "593f574a",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "System: Use the following pieces of context to answer the users question. \n",
-      "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n",
-      "----------------\n",
-      "Hippocrates is widely considered to be the Father of Medicine. His contributions revolutionized the practice of medicine; but after his death the advancement stalled.\n",
-      "\n",
-      "Many of the invaluable lessons prescribed in that place of learning are assigned to Hippocrates. If that was the case, then it truly was Hippocrates, with his approach to healing and the role of the doctor, that influenced western medicine for thousands of years.\n",
-      "\n",
-      "Despite this, Hippocrates is attributed with a great many wonderful deeds and thoughts. He is recognised as the founder of the Hippocratic School of Medicine, a college that revolutionized the understanding of medicine in Ancient Greece.\n",
-      "\n",
-      "At least that is what we’d like to think. While his fame was such to warrant a mention from the likes of Plato and Aristotle, not much is actually known about Hippocrates the father of Medicine. Consequently, he has become the projection of what people ideally want in a physician.\n",
-      "\n",
-      "460 – c. 370 BC) was a Greek physician of the Age of Pericles (Classical Greece), and is considered one of the most outstanding figures in the history of medicine.\n",
-      "\n",
-      "TRUE. Hippocrates is considered the father of modern medicine because he did not believe that illness was a punishment inflicted by the gods. True False. Weegy: TRUE. [ \n",
-      "\n",
-      "The two sons of Hippocrates, Thessalus and Draco, and his son-in-law, Polybus, were his students. According to Galen, a later physician, Polybus was Hippocrates' true successor, while Thessalus and Draco each had a son named Hippocrates.\n",
-      "\n",
-      "Hippocrates is mentioned in passing in the writings of two contemporaries: Plato, in Protagoras and Phaedrus, and, Aristotle 's Politics, which date from the 4th century BC. Soranus wrote that Hippocrates' father was Heraclides, a physician, and his mother was Praxitela, daughter of Tizane.\n",
-      "\n",
-      "Reload the page to try again! Press Cmd-0 to reset your zoom. Press Ctrl-0 to reset your zoom. It looks like your browser might be zoomed in or out. Your browser needs to be zoomed to a normal size to record audio.\n",
-      "\n",
-      "However, the achievements of the writers of the Corpus, the practitioners of Hippocratic medicine, and the actions of Hippocrates himself were often commingled; thus very little is known about what Hippocrates actually thought, wrote, and did.\n",
-      "Human: ____________________ is considered the father of modern medicine.\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\n",
-    "    \"System: Use the following pieces of context to answer the users question. \\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\\n----------------\\nHippocrates is widely considered to be the Father of Medicine. His contributions revolutionized the practice of medicine; but after his death the advancement stalled.\\n\\nMany of the invaluable lessons prescribed in that place of learning are assigned to Hippocrates. If that was the case, then it truly was Hippocrates, with his approach to healing and the role of the doctor, that influenced western medicine for thousands of years.\\n\\nDespite this, Hippocrates is attributed with a great many wonderful deeds and thoughts. He is recognised as the founder of the Hippocratic School of Medicine, a college that revolutionized the understanding of medicine in Ancient Greece.\\n\\nAt least that is what we’d like to think. While his fame was such to warrant a mention from the likes of Plato and Aristotle, not much is actually known about Hippocrates the father of Medicine. Consequently, he has become the projection of what people ideally want in a physician.\\n\\n460 – c. 370 BC) was a Greek physician of the Age of Pericles (Classical Greece), and is considered one of the most outstanding figures in the history of medicine.\\n\\nTRUE. Hippocrates is considered the father of modern medicine because he did not believe that illness was a punishment inflicted by the gods. True False. Weegy: TRUE. [ \\n\\nThe two sons of Hippocrates, Thessalus and Draco, and his son-in-law, Polybus, were his students. According to Galen, a later physician, Polybus was Hippocrates' true successor, while Thessalus and Draco each had a son named Hippocrates.\\n\\nHippocrates is mentioned in passing in the writings of two contemporaries: Plato, in Protagoras and Phaedrus, and, Aristotle 's Politics, which date from the 4th century BC. Soranus wrote that Hippocrates' father was Heraclides, a physician, and his mother was Praxitela, daughter of Tizane.\\n\\nReload the page to try again! Press Cmd-0 to reset your zoom. Press Ctrl-0 to reset your zoom. It looks like your browser might be zoomed in or out. Your browser needs to be zoomed to a normal size to record audio.\\n\\nHowever, the achievements of the writers of the Corpus, the practitioners of Hippocratic medicine, and the actions of Hippocrates himself were often commingled; thus very little is known about what Hippocrates actually thought, wrote, and did.\\nHuman: ____________________ is considered the father of modern medicine.\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "5b9204e0",
-   "metadata": {},
-   "source": [
-    "```\n",
-    "Q-003: ____________________ is considered the father of modern medicine.\n",
-    "A-003: Hippocrates is considered the father of modern medicine.\n",
-    "G-003: Hippocrates is considered the father of modern medicine.\n",
-    "```"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "5cfc8320",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'answers': ['The Volcano forecast for Apr 12 is 52 degrees and Patchy light rain.'],\n",
-       " 'passages': {'is_selected': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0],\n",
-       "  'passage_text': ['Volcano 10 Day Weather. Sunday:The Volcano forecast for Apr 09 is 43 degrees and Sunny. There is 55 percentage chance of rain and 4 mph winds from the Southwest. Monday:The Volcano forecast for Apr 10 is 51 degrees and Sunny.',\n",
-       "   'Current U.S. National Radar--Current. The Current National Weather Radar is shown below with a UTC Time (subtract 5 hours from UTC to get Eastern Time). National Weather Forecast--Current. The Current National Weather Forecast and National Weather Map are shown below.',\n",
-       "   'Volcano 10 Day Weather. 1  Sunday:The Volcano forecast for Apr 09 is 43 degrees and Sunny. There is 55 percentage chance of rain and 4 mph winds from the Southwest. 2  Monday:The Volcano forecast for Apr 10 is 51 degrees and Sunny. There is 49 percentage chance of rain and 3 mph winds from the Southwest.',\n",
-       "   'Volcano, CA Weather Data. 1  Volcano, CA Current Weather Data. 2  Sponsored. 3  Volcano, CA Historical Weather Trends.  Volcano, CA area 1  Highlights. Volcano, CA Chance of Sunshine.  Volcano, CA Historical 1  Temperature. Volcano, CA Rainfall and Snowfall Average.  Volcano, CA Energy Demand.',\n",
-       "   'Volcano Weather. Volcano weather and daily current conditions with summary and 5 Day forecast including humidity, precipitation, high and low temperatures presented in Fahrenheit and Celsius, barometric pressure, heat index, wind chill, hourly forecast, sunrise, sunset, wind speed with direction, and more.',\n",
-       "   'Hourly Forecast Detailed. 1  0am:The Volcano, CA forecast for Apr 03 is 48 degrees and Patchy rain possible. There is 83 percentage chance of rain and 2 mph winds from the East. 2  3am:The Volcano, CA forecast for Apr 03 is 44 degrees and Clear. There is 77 percentage chance of rain and 2 mph winds from the East.',\n",
-       "   'Volcano 7 Day Weather. 1  Monday:The Volcano forecast for Apr 03 is 58 degrees and Sunny. There is 34 percentage chance of rain and 5 mph winds from the West. 2  Tuesday:The Volcano forecast for Apr 04 is 59 degrees and Sunny. There is 33 percentage chance of rain and 5 mph winds from the West-Southwest.',\n",
-       "   'Volcano 10 Day Weather. 1  Sunday:The Volcano forecast for Apr 09 is 43 degrees and Sunny. 2  Monday:The Volcano forecast for Apr 10 is 51 degrees and Sunny. 3  Tuesday:The Volcano forecast for Apr 11 is 49 degrees and Patchy rain possible.  Wednesday:The Volcano forecast for Apr 12 is 52 degrees and Patchy light rain.',\n",
-       "   'Volcano, CA weather and traffic updates by locals. Write your own weather report, forecast, or traffic update: Please note by clicking on Post you acknowledge that you have read the Terms of Service and the report and/or forecast you are posting is in compliance with such terms. Be respectful.',\n",
-       "   'Hourly Forecast Detailed. 1  0am:The Volcano, CA forecast for Apr 03 is 48 degrees and Patchy rain possible. 2  3am:The Volcano, CA forecast for Apr 03 is 44 degrees and Clear. 3  6am:The Volcano, CA forecast for Apr 03 is 41 degrees and Clear.  9am:The Volcano, CA forecast for Apr 03 is 48 degrees and Sunny.'],\n",
-       "  'url': ['http://www.weatherman.com/us/ca/zip-codes/95689-10-day-weather',\n",
-       "   'http://www.fastweather.com/index.php?city=Volcano_CA&g',\n",
-       "   'http://www.weatherman.com/us/ca/zip-codes/95689-10-day-weather',\n",
-       "   'http://www.homefacts.com/weather/California/Amador-County/Volcano.html',\n",
-       "   'http://www.localconditions.com/weather-volcano-california/95689/',\n",
-       "   'http://www.weatherman.com/us/ca/volcano',\n",
-       "   'http://www.weatherman.com/us/ca/volcano',\n",
-       "   'http://www.weatherman.com/us/ca/zip-codes/95689-10-day-weather',\n",
-       "   'http://www.localconditions.com/weather-volcano-california/95689/',\n",
-       "   'http://www.weatherman.com/us/ca/volcano']},\n",
-       " 'query': 'current weather in volcano, ca',\n",
-       " 'query_id': 114414,\n",
-       " 'query_type': 'DESCRIPTION',\n",
-       " 'wellFormedAnswers': ['The Volcano forecast for Apr 12 is 52 degrees and Patchy light rain.']}"
-      ]
-     },
-     "execution_count": 11,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "test_ds = new_ds.select([1])\n",
-    "test_ds[0]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "56b91cae",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'bleu_scores': {'bleu': 1.0,\n",
-       "  'precisions': [1.0, 1.0, 1.0, 1.0],\n",
-       "  'brevity_penalty': 1.0,\n",
-       "  'length_ratio': 1.0,\n",
-       "  'translation_length': 14,\n",
-       "  'reference_length': 14},\n",
-       " 'rouge_scores': {'rouge1': 1.0,\n",
-       "  'rouge2': 1.0,\n",
-       "  'rougeL': 1.0,\n",
-       "  'rougeLsum': 1.0}}"
-      ]
-     },
-     "execution_count": 12,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "calc_metrics(test_ds)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 18,
-   "id": "56c6bf24",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['The',\n",
-       " 'Volcano',\n",
-       " 'forecast',\n",
-       " 'for',\n",
-       " 'Apr',\n",
-       " '12',\n",
-       " 'is',\n",
-       " '52',\n",
-       " 'degrees',\n",
-       " 'and',\n",
-       " 'Patchy',\n",
-       " 'light',\n",
-       " 'rain.']"
-      ]
-     },
-     "execution_count": 18,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "test_ds[0][\"answers\"][0].split()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "id": "77d08267",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "13"
-      ]
-     },
-     "execution_count": 19,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(test_ds[0][\"answers\"][0].split())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "id": "8c19694b",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'answers': ['From $26,000 to $39,000 a year'],\n",
-       " 'passages': {'is_selected': [0, 1, 0, 0, 0, 0, 0, 0, 0, 0],\n",
-       "  'passage_text': ['If you are interested in becoming a pharmacy technician, you’re choosing a career that is in high demand. According to the U.S. Bureau of Labor Statistics (BLS), the career growth is expected to be “much faster than average”, with an employment increase of 32% predicted in the decade spanning 2010 to 2020*.',\n",
-       "   'What can a pharmacy technician really expect to earn in today’s economy? According to Salary.com, pharmacy technicians make anywhere from $26,000 to $39,000 a year, though most make around $32,000 annually. California has the highest average pharmacy technician wage, at $34,317, according to Open Farm Tech’s website.',\n",
-       "   'The median annual wage for pharmacy technicians was $30,410 in May 2015. Employment of pharmacy technicians is projected to grow 9 percent from 2014 to 2024, faster than the average for all occupations. Increased demand for prescription medications will lead to more demand for pharmaceutical services.',\n",
-       "   'The majority of pharmacy techs work in drug stores and hospitals, where the average annual salary was $28,940 and $34,410, respectively**. However, a higher salary can be had if you can find employment with outpatient care centers or physicians’ offices, where the annual pay is in the $37,000-$39,000 range.',\n",
-       "   'The pharmacy technician salary** depends on a number of factors, from the area and type of employer, to your educational background. Browse pharmacy tech pay for a comparison between similar careers, geographic location, educational and certification requirements, and more.',\n",
-       "   \"Pharmacy Technician Salary. A Pharmacy Technician earns an average wage of $12.68 per hour. The skills that increase pay for this job the most are Mail Order Pharmacy and Long Term Care. People in this job generally don't have more than 20 years' experience. $18,722 - $48,714.\",\n",
-       "   'Popular Companies. * Please note that all salary figures are approximations based upon third party submissions to Simply Hired. These figures are given to Simply Hired users for the purpose of generalized comparison only. Minimum wage may differ by jurisdiction and you should consult the employer for actual salary figures.',\n",
-       "   'Pharmacy Technician average salary is $30,288, median salary is $30,534 with a salary range from $21,570 to $34,320. Pharmacy Technician salaries are collected from government agencies and companies. Each salary is associated with a real job position. Pharmacy Technician salary statistics is not exclusive and is for reference only.',\n",
-       "   'It also states that pharmacy technicians working in an acute care hospital earn an average salary of $37,000 per year, while those working for the military or a pharmaceutical company earn an average salary of $38,000 per year. This represents a difference of more than $10,000, simply due to the health care setting.',\n",
-       "   'Occupational Employment and Wages, May 2016. 29-2052 Pharmacy Technicians. Prepare medications under the direction of a pharmacist. May measure, mix, count out, label, and record amounts and dosages of medications according to prescription orders. National estimates for this occupation. Industry profile for this occupation.'],\n",
-       "  'url': ['http://www.pharmacytechschools.com/salary/',\n",
-       "   'http://www.pharmacytimes.com/contributor/alex-barker-pharmd/2015/06/guide-to-pharmacy-technician-salaries',\n",
-       "   'https://www.bls.gov/ooh/healthcare/pharmacy-technicians.htm',\n",
-       "   'http://www.pharmacytechschools.com/salary/',\n",
-       "   'http://www.pharmacytechschools.com/salary/',\n",
-       "   'http://www.payscale.com/research/US/Job=Pharmacy_Technician/Hourly_Rate',\n",
-       "   'http://www.simplyhired.com/salaries-k-certified-pharmacy-technician-jobs.html',\n",
-       "   'https://www.salarylist.com/jobs/Pharmacy-Technician-Salary.htm',\n",
-       "   'http://www.pharmacytimes.com/contributor/alex-barker-pharmd/2015/06/guide-to-pharmacy-technician-salaries',\n",
-       "   'https://www.bls.gov/oes/current/oes292052.htm']},\n",
-       " 'query': 'average pharmacy tech salary',\n",
-       " 'query_id': 40287,\n",
-       " 'query_type': 'NUMERIC',\n",
-       " 'wellFormedAnswers': ['The average salary for a pharmacy technician is $26,000 to $39,000 in a year.',\n",
-       "  'The average salary for a pharmacy technician is from $26,000 to $39,000 a year.']}"
-      ]
-     },
-     "execution_count": 22,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "test_ds = new_ds.select([4])\n",
-    "test_ds[0]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "id": "34209164",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'bleu_scores': {'bleu': 0.19303951204286907,\n",
-       "  'precisions': [0.875, 0.7142857142857143, 0.5, 0.4],\n",
-       "  'brevity_penalty': 0.32465246735834974,\n",
-       "  'length_ratio': 0.47058823529411764,\n",
-       "  'translation_length': 8,\n",
-       "  'reference_length': 17},\n",
-       " 'rouge_scores': {'rouge1': 0.5833333333333334,\n",
-       "  'rouge2': 0.4545454545454545,\n",
-       "  'rougeL': 0.5833333333333334,\n",
-       "  'rougeLsum': 0.5833333333333334}}"
-      ]
-     },
-     "execution_count": 23,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "calc_metrics(test_ds)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.9"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

Makefile ADDED Viewed

	@@ -0,0 +1,25 @@

+.PHONY: start
+start:
+	python app.py
+test:
+	python eval_modules/qa_chain_test.py
+tune:
+	./tune_rp.sh
+chat:
+	python eval_modules/qa_chain_test.py chat
+.PHONY: format
+format:
+	black .
+install:
+	pip install -r requirements.txt
+install-torch:
+	pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+install-torch-cuda:
+	pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121

README.md CHANGED Viewed

@@ -1,53 +1,168 @@
 ---
-title: Chat with LLMs
-emoji: 🤖💬
-colorFrom: purple
-colorTo: blue
 sdk: gradio
-sdk_version: 4.26.0
 app_file: app.py
-pinned: true
-short_description: 'Chat with LLMs'
 ---
 ## Running Locally
 1. Check pre-conditions:
 - [Git Large File Storage (LFS)](https://git-lfs.com/) must have been installed.
 - Run `python --version` to make sure you're running Python version 3.10 or above.
-- The latest PyTorch must have been installed. Here is a sample `conda` command for Linix/WSL2:
 ```
-conda install -y pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia
 ```
 2. Clone the repo
 ```
-git lfs install
-git clone https://huggingface.co/spaces/inflaton-ai/llm-qa-bench
 ```
-3. Install packages
 ```
 pip install -r requirements.txt
 ```
-4. Set up your environment variables
 - By default, environment variables are loaded from `.env.example` file
 - If you don't want to use the default settings, copy `.env.example` into `.env`. Your can then update it for your local runs.
-5. Run automated test:
 ```
-python qa_chain_test.py
 ```
-6. Start the local server at `http://localhost:7860`:
 ```
-python app.py
 ```

 ---
+title: LLLM QA Eval
+emoji: 💬
+colorFrom: yellow
+colorTo: purple
 sdk: gradio
+sdk_version: 4.36.1
 app_file: app.py
+pinned: false
+license: apache-2.0
 ---
+# Evaluate and Optimize Open-Source LLMs' Performance for Question Answering with RAG and Non-RAG
+This project contains the source code, datasets and results for the titled paper.
+## Results for [WebQSP Dataset](./data/datasets/WebQSP.test.wikidata.json)
+| Model Name                       | RAG                                                                                                   | RAG with Chat Template                                                                    | Non-RAG                                                                                                         | Note              |
+| -------------------------------- | ----------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | ----------------- |
+| Phi-3-mini-128k-instruct (batch) | [Phi-3-mini-128k-instruct_wd_rag_batch_4](./data/results/Phi-3-mini-128k-instruct_wd_rag_batch_4.csv) | [Phi-3-mini-128k-instruct_wd_true](./data/results/Phi-3-mini-128k-instruct_wd_true.csv)   | [Phi-3-mini-128k-instruct_wd_non_rag_batch_16](./data/results/Phi-3-mini-128k-instruct_wd_non_rag_batch_16.csv) | Evaluated 3 types |
+| gemma-1.1-2b-it                  | [gemma-1.1-2b-it_wd](./data/results/gemma-1.1-2b-it_wd.csv)                                           | [gemma-1.1-2b-it_wd_true](./data/results/gemma-1.1-2b-it_wd_true.csv)                     | [gemma-1.1-2b-it_wd_non_rag](./data/results/gemma-1.1-2b-it_wd_non_rag.csv)                                     | Evaluated 3 types |
+| gemma-1.1-7b-it                  | [gemma-1.1-7b-it_wd](./data/results/gemma-1.1-7b-it_wd.csv)                                           | [gemma-1.1-7b-it_wd_true](./data/results/gemma-1.1-7b-it_wd_true.csv)                     | [gemma-1.1-7b-it_wd_non_rag](./data/results/gemma-1.1-27b-it_wd_non_rag.csv)                                    | Evaluated 3 types |
+| Mistral-7B-Instruct-v0.2         | [Tune_2024-03-29_11-28-20](./data/results/Tune_2024-03-29_11-28-20.csv)                               | [Mistral-7B-Instruct-v0.2_wd_true](./data/results/Mistral-7B-Instruct-v0.2_wd_true.csv)   | [Tune_2024-04-16_12-24-27](./data/results/Tune_2024-04-16_12-24-27.csv.csv)                                     | Evaluated 3 types |
+| Llama-2-7b-chat-hf               | [Tune_2024-03-20_15-35-37](./data/results/Tune_2024-03-20_15-35-37.csv)                               | [Llama-2-7b-chat-hf_wd_true](./data/results/Llama-2-7b-chat-hf_wd_true.csv)               | [Tune_2024-04-09_09-19-22](./data/results/Tune_2024-04-09_09-19-22.csv)                                         | Evaluated 3 types |
+| Meta-Llama-3-8B-Instruct         | [Meta-Llama-3-8B-Instruct_wd](./data/results/Meta-Llama-3-8B-Instruct_wd.csv)                         | [Meta-Llama-3-8B-Instruct_wd_true](./data/results/Meta-Llama-3-8B-Instruct_wd_true.csv)   | [Meta-Llama-3-8B-Instruct_wd_non_rag](./data/results/Meta-Llama-3-8B-Instruct_wd_non_rag.csv) (generic prompt)  | Evaluated 3 types |
+|                                  |                                                                                                       |                                                                                           | [Meta-Llama-3-8B-Instruct_wd_1_non_rag](./data/results/Meta-Llama-3-8B-Instruct_wd_1_non_rag.csv)               | Evaluated Non-RAG |
+| Llama-2-13b-chat-hf              | [Tune_2024-03-25_23-32-57](./data/results/Tune_2024-03-25_23-32-57.csv)                               | [Llama-2-13b-chat-hf_wd_true](./data/results/Llama-2-13b-chat-hf_wd_true.csv)             | [Tune_2024-04-10_16-53-38](./data/results/Tune_2024-04-10_16-53-38.csv)                                         | Evaluated 3 types |
+| Llama-2-70b-chat-hf              | [Llama-2-70b-chat-hf_wd](./data/results/Llama-2-70b-chat-hf_wd.csv)                                   | [Llama-2-70b-chat-hf_wd_true](./data/results/Llama-2-70b-chat-hf_wd_true.csv)             | [Llama-2-70b-chat-hf_wd_non_rag](./data/results/Llama-2-70b-chat-hf_wd_non_rag.csv)                             | Evaluated 3 types |
+| Meta-Llama-3-70B-Instruct        | [Meta-Llama-3-70B-Instruct_wd](./data/results/Meta-Llama-3-70B-Instruct_wd.csv)                       | [Meta-Llama-3-70B-Instruct_wd_true](./data/results/Meta-Llama-3-70B-Instruct_wd_true.csv) | [Meta-Llama-3-70B-Instruct_wd_non_rag](./data/results/Meta-Llama-3-70B-Instruct_wd_non_rag.csv)                 | Evaluated 3 types |
+| gpt-3.5-turbo                    | [gpt-3.5-turbo_rag](./data/results/gpt-3.5-turbo_rag.csv)                                             |                                                                                           | [gpt-3.5-turbo_non_rag](./data/results/gpt-3.5-turbo_non_rag.csv)                                               | Evaluated both    |
+## Results for [MS MACRO Dataset](./data/datasets/ms_macro.json)
+| Model Name                | RAG                                                                                            | RAG with Chat Template                                                                  | Non-RAG                                                                                                      | Note |
+| ------------------------- | ---------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ---- |
+| gemma-1.1-2b-it           | [gemma-1.1-2b-it_mm_false](data/results/gemma-1.1-2b-it_mm_true_false.csv)                     | [gemma-1.1-2b-it_mm_true](data/results/gemma-1.1-2b-it_mm_true.csv)                     | [gemma-1.1-2b-it_mm_non_rag.csv](data/results/gemma-1.1-2b-it_mm_true_false_non_rag.csv)                     |      |
+| Phi-3-mini-128k-instruct  | [Phi-3-mini-128k-instruct_mm_false](data/results/Phi-3-mini-128k-instruct_mm_false.csv)        | [Phi-3-mini-128k-instruct_mm_true](data/results/Phi-3-mini-128k-instruct_mm_true.csv)   | [Phi-3-mini-128k-instruct_mm_non_rag.csv](data/results/Phi-3-mini-128k-instruct_mm_non_rag.csv)              |      |
+| gemma-1.1-7b-it           | [gemma-1.1-7b-it_mm_false](data/results/gemma-1.1-7b-it_mm_false.csv)                          | [gemma-1.1-7b-it_mm_true](data/results/gemma-1.1-7b-it_mm_true.csv)                     | [gemma-1.1-7b-it_mm_non_rag.csv](data/results/gemma-1.1-7b-it_mm_non_rag.csv)                                |      |
+| Mistral-7B-Instruct-v0.2  | [Mistral-7B-Instruct-v0.2_mm_false](data/results/Mistral-7B-Instruct-v0.2_mm_false.csv)        | [Mistral-7B-Instruct-v0.2_mm_true](data/results/Mistral-7B-Instruct-v0.2_mm_true.csv)   | [Mistral-7B-Instruct-v0.2_mm_false](data/results/Mistral-7B-Instruct-v0.2_mm_non_rag.csv)                    |      |
+| Llama-2-7b-chat-hf        | [Llama-2-7b-chat-hf_mm_false](data/results/Llama-2-7b-chat-hf_mm_true_false.csv)               | [Llama-2-7b-chat-hf_mm_true](data/results/Llama-2-7b-chat-hf_mm_true.csv)               | [Llama-2-7b-chat-hf_mm_non_rag.csv](data/results/Llama-2-7b-chat-hf_mm_true_false_non_rag.csv)               |      |
+| Meta-Llama-3-8B-Instruct  | [Meta-Llama-3-8B-Instruct_mm_false](data/results/Meta-Llama-3-8B-Instruct_mm_true_false.csv)   | [Meta-Llama-3-8B-Instruct_mm_true](data/results/Meta-Llama-3-8B-Instruct_mm_true.csv)   | [Meta-Llama-3-8B-Instruct_mm_non_rag.csv](data/results/Meta-Llama-3-8B-Instruct_mm_true_false_non_rag.csv)   |      |
+| Llama-2-13b-chat-hf       | [Llama-2-13b-chat-hf_mm_false](data/results/Llama-2-13b-chat-hf_mm_false.csv)                  | [Llama-2-13b-chat-hf_mm_true](data/results/Llama-2-13b-chat-hf_mm_true.csv)             | [Llama-2-13b-chat-hf_mm_non_rag.csv](data/results/Llama-2-13b-chat-hf_mm_non_rag.csv)                        |      |
+| Llama-2-70b-chat-hf       | [Llama-2-70b-chat-hf_mm_false](data/results/Llama-2-70b-chat-hf_mm_false.csv)                  | [Llama-2-70b-chat-hf_mm_true](data/results/Llama-2-70b-chat-hf_mm_true.csv)             | [Llama-2-70b-chat-hf_mm_non_rag.csv](data/results/Llama-2-70b-chat-hf_mm_non_rag.csv)                        |      |
+| Meta-Llama-3-70B-Instruct | [Meta-Llama-3-70B-Instruct_mm_false](data/results/Meta-Llama-3-70B-Instruct_mm_true_false.csv) | [Meta-Llama-3-70B-Instruct_mm_true](data/results/Meta-Llama-3-70B-Instruct_mm_true.csv) | [Meta-Llama-3-70B-Instruct_mm_non_rag.csv](data/results/Meta-Llama-3-70B-Instruct_mm_true_false_non_rag.csv) |      |
+| gpt-3.5-turbo             | [gpt-3.5-turbo_rag](./data/results/gpt-3.5-turbo_mm_RP_1.300.csv)                              |                                                                                         | [gpt-3.5-turbo_non_rag](./data/results/gpt-3.5-turbo_mm_non_rag_RP_1.300.csv)                                |      |
+## How it works
+We're using an AI methodology, namely Conversational Retrieval Augmentation (CRAG), which uses LLMs off the shelf (i.e., without any fine-tuning), then controls their behavior through clever prompting and conditioning on private “contextual” data, e.g., texts extracted from your PDF files.
+At a very high level, the workflow can be divided into three stages:
+1. Data preprocessing / embedding: This stage involves storing private data (your PDF files) to be retrieved later. Typically, the documents are broken into chunks, passed through an embedding model, then stored the created embeddings in a vectorstore.
+2. Prompt construction / retrieval: When a user submits a query, the application constructs a series of prompts to submit to the language model. A compiled prompt typically combines a prompt template and a set of relevant documents retrieved from the vectorstore.
+3. Prompt execution / inference: Once the prompts have been compiled, they are submitted to a pre-trained LLM for inference—including both proprietary model APIs and open-source or self-trained models.
+Tech stack used includes LangChain, Gradio, Chroma and FAISS.
+- LangChain is an open-source framework that makes it easier to build scalable AI/LLM apps and chatbots.
+- Gradio is an open-source Python library that is used to build machine learning and data science demos and web applications.
+- Chroma and FAISS are open-source vectorstores for storing embeddings for your files.
 ## Running Locally
 1. Check pre-conditions:
 - [Git Large File Storage (LFS)](https://git-lfs.com/) must have been installed.
 - Run `python --version` to make sure you're running Python version 3.10 or above.
+- [CMake](https://cmake.org/) must have been installed. Here is a sample command to install `CMake` on `ubuntu`:
 ```
+sudo apt install cmake
 ```
 2. Clone the repo
 ```
+git lfs install
+git clone --recursive https://github.com/smu-ai/Evaluation-of-Orca-2-Models-for-Conversational-RAG.git
+```
+3. Ensure the latest PyTorch must have been installed.
+```
+# using CUDA with Nvidia GPU
+make install-torch-cuda
+# using Apple Silicon or other CPU
+make install-torch
 ```
+4. Install packages
 ```
 pip install -r requirements.txt
 ```
+5. Set up your environment variables
 - By default, environment variables are loaded from `.env.example` file
 - If you don't want to use the default settings, copy `.env.example` into `.env`. Your can then update it for your local runs.
+6. Run automated test:
 ```
+make test
 ```
+7. Start the local server at `http://localhost:7860`:
+```
+make start
+```
+8. Tune repetition penalty parameters:
+```
+make tune
+```
+## Talk to Your Own PDF Files
+- The sample PDF files are downloaded from [PCI DSS official website](https://www.pcisecuritystandards.org/document_library/?category=pcidss) and the corresponding embeddings are stored in folders `data/chromadb_1024_512` and `data/faiss_1024_512` with Chroma & FAISS formats respectively, which allows you to run locally without any additional effort.
+- You can also put your own PDF files into any folder specified in `SOURCE_PDFS_PATH` and run the command below to generate embeddings which will be stored in folder `FAISS_INDEX_PATH` or `CHROMADB_INDEX_PATH`. If both `*_INDEX_PATH` env vars are set, `FAISS_INDEX_PATH` takes precedence. Make sure the folder specified by `*_INDEX_PATH` doesn't exist; other wise the command will simply try to load index from the folder and do a simple similarity search, as a way to verify if embeddings are generated and stored properly. Please note the HuggingFace Embedding model specified by `HF_EMBEDDINGS_MODEL_NAME` will be used to generate the embeddings.
+```
+python ingest.py
+```
+- Once embeddings are generated, you can test them out locally, or check them into your duplicated space. Please note HF Spaces git server does not allow PDF files to be checked in.
+## Play with Different Large Language Models
+The source code supports different LLM types - as shown at the top of `.env.example`
+```
+# LLM_MODEL_TYPE=openai
+# LLM_MODEL_TYPE=gpt4all-j
+# LLM_MODEL_TYPE=gpt4all
+# LLM_MODEL_TYPE=llamacpp
+# LLM_MODEL_TYPE=huggingface
+# LLM_MODEL_TYPE=mosaicml
+# LLM_MODEL_TYPE=stablelm
+# LLM_MODEL_TYPE=openllm
+LLM_MODEL_TYPE=hftgi
+```
+- By default, the app runs `microsoft/orca-2-13b` model with HF Text Generation Interface, which runs on a research server and might be down from time to time.
+- Uncomment/comment the above to play with different LLM types. You may also want to update other related env vars. E.g., here's the list of HF models which have been tested with the code:
 ```
+# HUGGINGFACE_MODEL_NAME_OR_PATH="microsoft/orca-2-7b"
+HUGGINGFACE_MODEL_NAME_OR_PATH="microsoft/orca-2-13b"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/wizardLM-7B-HF"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/vicuna-7B-1.1-HF"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-j"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-falcon"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="lmsys/fastchat-t5-3b-v1.0"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-7b-chat-hf"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-13b-chat-hf"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-70b-chat-hf"
 ```

app.py CHANGED Viewed

@@ -1,38 +1,12 @@
 import json
-import gradio as gr
-import torch
-from transformers import (
-    AutoModelForCausalLM,
-    AutoTokenizer,
-    TextIteratorStreamer,
-)
 import os
-from threading import Thread
-import subprocess
 from app_modules.utils import calc_bleu_rouge_scores, detect_repetitions
 from dotenv import find_dotenv, load_dotenv
 found_dotenv = find_dotenv(".env")
-if len(found_dotenv) == 0:
-    found_dotenv = find_dotenv(".env.example")
-print(f"loading env vars from: {found_dotenv}")
-load_dotenv(found_dotenv, override=False)
-subprocess.run(
-    "pip install flash-attn --no-build-isolation",
-    env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
-    shell=True,
-)
-token = os.getenv("HUGGINGFACE_AUTH_TOKEN")
-model_name = os.getenv(
-    "HUGGINGFACE_MODEL_NAME_OR_PATH", "google/gemma-1.1-2b-it"
-)  # "microsoft/Phi-3-mini-128k-instruct"
-print(f"       model_name: {model_name}")
 HF_RP = os.getenv("HF_RP", "1.2")
 repetition_penalty = float(HF_RP)
 print(f"       repetition_penalty: {repetition_penalty}")
@@ -47,52 +21,21 @@ print(f"Loaded {len(examples)} examples")
 qa_system_prompt = "Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer."
-model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-    token=token,
-    trust_remote_code=True,
-)
-tok = AutoTokenizer.from_pretrained(model_name, token=token)
-terminators = [
-    tok.eos_token_id,
-]
-# Check that MPS is available
-if not torch.backends.mps.is_available():
-    if not torch.backends.mps.is_built():
-        print(
-            "MPS not available because the current PyTorch install was not "
-            "built with MPS enabled."
-        )
-    else:
-        print(
-            "MPS not available because the current MacOS version is not 12.3+ "
-            "and/or you do not have an MPS-enabled device on this machine."
-        )
-    mps_device = None
-else:
-    mps_device = torch.device("mps")
-if mps_device is not None:
-    device = mps_device
-    print("Using MPS")
-elif torch.cuda.is_available():
-    device = torch.device("cuda")
-    print(f"Using GPU: {torch.cuda.get_device_name(device)}")
-else:
-    device = torch.device("cpu")
-    print("Using CPU")
-model = model.to(device)
 def chat(
     message,
-    history,
     temperature=0,
     repetition_penalty=1.1,
     do_sample=True,
     max_tokens=1024,
 ):
     print("repetition_penalty:", repetition_penalty)
     chat = []
@@ -109,36 +52,58 @@ def chat(
     chat.append({"role": "user", "content": message})
-    messages = tok.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
-    model_inputs = tok([messages], return_tensors="pt").to(device)
-    streamer = TextIteratorStreamer(
-        tok, timeout=200.0, skip_prompt=True, skip_special_tokens=True
-    )
-    if temperature == 0:
-        temperature = 0.01
-    generate_kwargs = dict(
-        model_inputs,
-        streamer=streamer,
-        max_new_tokens=max_tokens,
-        do_sample=do_sample,
         temperature=temperature,
-        eos_token_id=terminators,
-    )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
-    partial_text = ""
-    for new_text in streamer:
         partial_text += new_text
         yield partial_text
     answer = partial_text
-    (newline_score, repetition_score, total_repetitions) = detect_repetitions(answer)
     partial_text += "\n\nRepetition Metrics:\n"
-    partial_text += f"1. Newline Score: {newline_score:.3f}\n"
     partial_text += f"1. Repetition Score: {repetition_score:.3f}\n"
     partial_text += f"1. Total Repetitions: {total_repetitions:.3f}\n"
@@ -151,7 +116,7 @@ def chat(
         scores = calc_bleu_rouge_scores([answer], [questions[index][key]], debug=True)
         partial_text += "\n\n Performance Metrics:\n"
-        partial_text += f'1. BLEU: {scores["bleu_scores"]["bleu"]:.3f}\n'
         partial_text += f'1. RougeL: {scores["rouge_scores"]["rougeL"]:.3f}\n'
     yield partial_text
@@ -165,6 +130,7 @@ demo = gr.ChatInterface(
         label="⚙️ Parameters", open=False, render=False
     ),
     additional_inputs=[
         gr.Slider(
             minimum=0, maximum=1, step=0.1, value=0, label="Temperature", render=False
         ),
@@ -185,9 +151,13 @@ demo = gr.ChatInterface(
             label="Max new tokens",
             render=False,
         ),
     ],
-    stop_btn="Stop Generation",
-    title="Chat With LLMs",
-    description=f"Now Running [{model_name}](https://huggingface.co/{model_name})",
 )
 demo.launch()

 import json
 import os
+import gradio as gr
+from huggingface_hub import InferenceClient
 from app_modules.utils import calc_bleu_rouge_scores, detect_repetitions
 from dotenv import find_dotenv, load_dotenv
 found_dotenv = find_dotenv(".env")
 HF_RP = os.getenv("HF_RP", "1.2")
 repetition_penalty = float(HF_RP)
 print(f"       repetition_penalty: {repetition_penalty}")
 qa_system_prompt = "Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer."
+"""
+For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+"""
+client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 def chat(
     message,
+    history: list[tuple[str, str]],
+    system_message,
     temperature=0,
     repetition_penalty=1.1,
     do_sample=True,
     max_tokens=1024,
+    top_p=0.95,
 ):
     print("repetition_penalty:", repetition_penalty)
     chat = []
     chat.append({"role": "user", "content": message})
+    messages = [{"role": "system", "content": system_message}]
+    messages.append({"role": "user", "content": message})
+    partial_text = ""
+    # huggingface_hub.utils._errors.HfHubHTTPError: 422 Client Error: Unprocessable Entity for url: https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta (Request ID: NZamtWmdoSg3flfgRKT0e)
+    # Make sure 'text-generation' task is supported by the model.
+    # for message in client.text_generation(
+    #     messages,
+    #     stream=True,
+    #     temperature=temperature,
+    #     top_p=top_p,
+    #     repetition_penalty=repetition_penalty,
+    # ):
+    # https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta
+    # {
+    #     "id": "HuggingFaceH4/zephyr-7b-beta",
+    #     "sha": "b70e0c9a2d9e14bd1e812d3c398e5f313e93b473",
+    #     "pipeline_tag": "text-generation",
+    #     "library_name": "transformers",
+    #     "private": false,
+    #     "gated": false,
+    #     "siblings": [],
+    #     "safetensors": {
+    #         "parameters": {
+    #             "BF16": 7241732096
+    #         }
+    #     },
+    #     "cardData": {
+    #         "tags": [
+    #             "generated_from_trainer"
+    #         ],
+    #         "base_model": "mistralai/Mistral-7B-v0.1"
+    #     }
+    # }
+    for message in client.chat_completion(
+        messages,
+        max_tokens=max_tokens,
+        stream=True,
         temperature=temperature,
+        top_p=top_p,
+    ):
+        new_text = message.choices[0].delta.content
         partial_text += new_text
         yield partial_text
     answer = partial_text
+    (whitespace_score, repetition_score, total_repetitions) = detect_repetitions(answer)
     partial_text += "\n\nRepetition Metrics:\n"
+    partial_text += f"1. Whitespace Score: {whitespace_score:.3f}\n"
     partial_text += f"1. Repetition Score: {repetition_score:.3f}\n"
     partial_text += f"1. Total Repetitions: {total_repetitions:.3f}\n"
         scores = calc_bleu_rouge_scores([answer], [questions[index][key]], debug=True)
         partial_text += "\n\n Performance Metrics:\n"
+        partial_text += f'1. BLEU-1: {scores["bleu_scores"]["bleu"]:.3f}\n'
         partial_text += f'1. RougeL: {scores["rouge_scores"]["rougeL"]:.3f}\n'
     yield partial_text
         label="⚙️ Parameters", open=False, render=False
     ),
     additional_inputs=[
+        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
         gr.Slider(
             minimum=0, maximum=1, step=0.1, value=0, label="Temperature", render=False
         ),
             label="Max new tokens",
             render=False,
         ),
+        gr.Slider(
+            minimum=0.1,
+            maximum=1.0,
+            value=0.95,
+            step=0.05,
+            label="Top-p (nucleus sampling)",
+        ),
     ],
 )
 demo.launch()

app_modules/llm_loader.py CHANGED Viewed

@@ -3,7 +3,7 @@ import sys
 import threading
 from queue import Queue
 from typing import Any, Dict, List, Optional
 import torch
 from langchain.callbacks.base import BaseCallbackHandler
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
@@ -176,7 +176,6 @@ class LLMLoader:
             load_in_4bit=load_quantized_model == "4bit",
             bnb_4bit_use_double_quant=load_quantized_model == "4bit",
             load_in_8bit=load_quantized_model == "8bit",
-            bnb_8bit_use_double_quant=load_quantized_model == "8bit",
         )
         callbacks = []
@@ -212,13 +211,19 @@ class LLMLoader:
                 print(f"              using model: {MODEL_NAME}")
                 self.llm = ChatGoogleGenerativeAI(
                     model=MODEL_NAME,
-                    convert_system_message_to_human=True,
                     callbacks=callbacks,
                     streaming=True,
                     safety_settings={
                         HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
                     },
                 )
             elif self.llm_model_type.startswith("gpt4all"):
                 MODEL_PATH = ensure_model_is_downloaded(self.llm_model_type)
                 self.llm = GPT4All(

 import threading
 from queue import Queue
 from typing import Any, Dict, List, Optional
+import google.generativeai as genai
 import torch
 from langchain.callbacks.base import BaseCallbackHandler
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
             load_in_4bit=load_quantized_model == "4bit",
             bnb_4bit_use_double_quant=load_quantized_model == "4bit",
             load_in_8bit=load_quantized_model == "8bit",
         )
         callbacks = []
                 print(f"              using model: {MODEL_NAME}")
                 self.llm = ChatGoogleGenerativeAI(
                     model=MODEL_NAME,
                     callbacks=callbacks,
                     streaming=True,
                     safety_settings={
+                        HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
+                        HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
+                        HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
                         HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
                     },
                 )
+                # for m in genai.list_models():
+                #     if "generateContent" in m.supported_generation_methods:
+                #         print(m.name)
+                # exit()
             elif self.llm_model_type.startswith("gpt4all"):
                 MODEL_PATH = ensure_model_is_downloaded(self.llm_model_type)
                 self.llm = GPT4All(

app_modules/utils.py CHANGED Viewed

@@ -1,9 +1,9 @@
 # -*- coding:utf-8 -*-
 from __future__ import annotations
 import logging
 import os
-import math
 import platform
 import re
 from pathlib import Path
@@ -13,8 +13,6 @@ import requests
 import torch
 from tqdm import tqdm
 from langchain.memory import ConversationSummaryBufferMemory
-import matplotlib.pyplot as plt
-import matplotlib.ticker as mtick
 class LogRecord(logging.LogRecord):
@@ -278,153 +276,3 @@ def detect_repetition_scores(text, debug=False):
         text, debug=debug
     )
     return pd.Series([newline_score, repetition_score, total_repetitions])
-def detect_scores(text, debug=False):
-    newline_score, repetition_score, total_repetitions = detect_repetitions(
-        text, debug=debug
-    )
-    return pd.Series([newline_score, repetition_score, total_repetitions])
-def load_with_newline_and_repetition_scores(result_file, force_recalculate=False):
-    print(f"loading result file: {result_file}")
-    df = pd.read_csv(result_file, comment="#", on_bad_lines="warn")
-    if (
-        force_recalculate
-        or "newline_score" not in df.columns
-        or "repetition_score" not in df.columns
-        or "total_repetitions" not in df.columns
-    ):
-        df[["newline_score", "repetition_score", "total_repetitions"]] = df[
-            "answer"
-        ].apply(detect_scores)
-        df.to_csv(result_file, index=False)
-    return df
-def replace_last(source_string, old_string, new_string):
-    head, _sep, tail = source_string.rpartition(old_string)
-    return head + new_string + tail
-df_ms_macro = pd.read_json("./data/datasets/ms_macro.json")
-def load_for_repetition_penalty_ms_macro(
-    csv_result_file, repetition_penalty, force_recalculate=False
-):
-    result_file = replace_last(
-        csv_result_file, ".csv", f"_RP_{repetition_penalty:.3f}.csv"
-    )
-    df = load_with_newline_and_repetition_scores(
-        result_file, force_recalculate=force_recalculate
-    )
-    if df["ground_truth"][0] != df_ms_macro["wellFormedAnswers"][0]:
-        df["ground_truth"] = df_ms_macro["wellFormedAnswers"]
-        print("ground_truth updated for:", result_file)
-        df.to_csv(result_file, index=False)
-    return df
-def adjust_perf_scores_with_repetition_penalty(result, precision, recall):
-    newline_score = [
-        df["newline_score"].mean() for df in result["df_list_repetition_penalty"]
-    ]
-    print(f"newline_score: {newline_score}")
-    repetition_score = [
-        df["repetition_score"].mean() for df in result["df_list_repetition_penalty"]
-    ]
-    print(f"repetition_score: {repetition_score}")
-    precision = [
-        f / math.log10(10 + n + r)
-        for f, n, r in zip(precision, newline_score, repetition_score)
-    ]
-    recall = [
-        f / math.log10(10 + n + r)
-        for f, n, r in zip(recall, newline_score, repetition_score)
-    ]
-    return precision, recall
-# MS MACRO
-def plot_performance_scores_ms_macro(
-    result,
-    models=None,
-    title="Performance",
-):
-    if models is None:
-        models = result.keys()
-    for model in models:
-        print(f"model: {model}")
-        df = result[model]["df_overall"]
-        # print(result[model]["df_list_repetition_penalty"][0].describe())
-        # Calculate the statistics
-        bleu1 = list(df["bleu1"])
-        rougeL = list(df["rougeL"])
-        f1 = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
-        best_f1 = max(f1)
-        best_f1_index = f1.index(best_f1)
-        bleu1, rougeL = adjust_perf_scores_with_repetition_penalty(
-            result[model], bleu1, rougeL
-        )
-        afrp = [2 * (p * r) / (p + r) for p, r in zip(bleu1, rougeL)]
-        # f1 = [df["f1"].mean() for df in result[model]["df_list_repetition_penalty"]]
-        best_afrp = max(afrp)
-        best_afrp_index = afrp.index(best_afrp)
-        repetition_penalties = list(df["repetition_penalty"])
-        # line plot for precision, recall, f1
-        plt.figure(figsize=(10, 6))
-        plt.axvspan(
-            repetition_penalties[best_f1_index] - 0.01,
-            repetition_penalties[best_f1_index] + 0.01,
-            alpha=0.5,
-            edgecolor="none",
-            facecolor="blue",
-        )
-        plt.axvspan(
-            repetition_penalties[best_afrp_index] - 0.01,
-            repetition_penalties[best_afrp_index] + 0.01,
-            alpha=0.5,
-            edgecolor="none",
-            facecolor="orange",
-        )
-        plt.plot(
-            repetition_penalties,
-            f1,
-            label="Overall Perf Score",
-            marker="D",
-            color="blue",
-        )
-        plt.plot(
-            repetition_penalties,
-            afrp,
-            label="RF Adjusted Perf Score",
-            marker="o",
-            color="orange",
-        )
-        plt.xlabel("Repetition Penalties")
-        plt.ylabel("Score")
-        plt.xlim(0.99, 1.31)
-        # y in percentage
-        plt.gca().yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
-        plt.title(f"{model} {title}")
-        plt.legend(bbox_to_anchor=(1.0, 0.5), loc="center left")
-        plt.show()

 # -*- coding:utf-8 -*-
 from __future__ import annotations
+import json
 import logging
 import os
 import platform
 import re
 from pathlib import Path
 import torch
 from tqdm import tqdm
 from langchain.memory import ConversationSummaryBufferMemory
 class LogRecord(logging.LogRecord):
         text, debug=debug
     )
     return pd.Series([newline_score, repetition_score, total_repetitions])

data/datasets/WebQSP.test.wikidata.json ADDED Viewed

The diff for this file is too large to render. See raw diff

data/{logs/Phi-3-mini-128k-instruct_mm_false_RP_1.060.txt → eval/Llama-2-13b-chat-hf_wd_true_RP_1.000-t2.json} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f834d6a3c9429ba3f883a923e12c94c8b9a62acfe0cfeb3a1218403e31c7958
-size 952195

 version https://git-lfs.github.com/spec/v1
+oid sha256:1277996feced5bd3ef1a12f0bbfd612219ee32825ced7a5f67ba154b2d61b7a7
+size 655310

data/{logs/Phi-3-mini-128k-instruct_mm_false_RP_1.120.txt → eval/Llama-2-13b-chat-hf_wd_true_RP_1.000-t2_evaluated.json} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9b32298de75ee46fc56fe8434f4de0488954d4e643453af37c31895dc054bb3
-size 607038

 version https://git-lfs.github.com/spec/v1
+oid sha256:fb905b34dc2c46254530376a9609e2f6b9011b412c67994220549fb94b9ee72b
+size 716045

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.020-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ee64cfc0130b4b323a159ead1a9104989c800578856b1805cc056ab5257e45
+size 653448

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.020-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:689bc4f321e11fc4d42cd6a3ef1d15f1f87c4228d8248eb7a4e28d635746db73
+size 714002

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.040-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:b3b8f1059469d94a2f9f6a0d2216e5d1cf66d7b48cdf39b267a4cc67623e1f47
+size 650480

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.040-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:4c01183de7769798e48ac681749f6cb2722ce0c4763e01f03e17569d7b0e8ca5
+size 710922

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.060-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:0a12beb32ed61761899bba3a73d6959691f8785328d70c13c9037ed71116f5e2
+size 654870

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.060-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:f30bf65ae09a25f1ac818528d352d3cff9eced98fab349c47b55672a53aa457a
+size 715516

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.080-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:2a6404ba60e543621c5be62e9b4d52639671c42f61ed9249afb52428fd70015c
+size 644399

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.080-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:484ba047b61462ee92bbfa33b335bee05b13c2b667abae414417bfe33da46274
+size 705020

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.100-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:51cb665ebc45ca714f9e03415937c152923f75ca9ccacca2bdf2f2698a009948
+size 642376

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.100-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:96e0d343cded4a11df5bdc1dffa7c2ed9631afce46f706d0c0f8f35d4683df9d
+size 702987

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.120-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:1fe968825c4b87aa7b2a09bf9a0ddcef58cbf58707b8620c0e2c4344b6cbded8
+size 647286

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.120-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:dc525d15dc082962e317fb8b3fccf28cc9a6de4d6d765d743016dc34f13ec506
+size 707966

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.140-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:f354e4dd9a889af49831dd797199143be856040b51d152d935216ebcb0d3e0f0
+size 644443

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.140-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:25de5711464fa1015e193c0f452902742fbab711c464a4f87d9774914f3edf2d
+size 705131

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.160-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:539b143d92be17b1f397e020ff3518ea397fea3f985a36e2db4cd6efdc9b0755
+size 643126

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.160-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:b92bbb2bc9dffa027274dc593cbc61a838bde4a8e54a1b4210d4f921e280b436
+size 703815

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.180-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:6e56aa85968670bfe81b541bec3c57cf5763457c76847266411844abc544fc26
+size 652949

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.180-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:b2c1e8311f7b644fb2ce62b05db31cd81236ab8bfa94107ac8d33e7e03444871
+size 713794

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.200-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:0c4136642c87159fba5861af7b4be13a2f099943041703f4d5e4c86b85c49c9a
+size 647659

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.200-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:784ad453019ef8ac12c24032d16cbba3d1b11b25439903ebbee7c51382b13531
+size 708220

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.220-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a467439fdb97136ddfbda188a4e216d8aec1305837d535c2c5c3fe0546c490bb
+size 647989

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.220-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:9a1d85371b24da315049092b1adf3229c0a4663e64e453843dfadb596d9f0ef1
+size 708567

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.240-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:39c1241171262c6fb1712ed8089bef7e790b71bf2f327c95052d4087037005f2
+size 642213

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.240-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:4735ae18bf2c0f576f116ec178afb817a8973d9e84fd8fa51e8a3cc05c5b3a6e
+size 702922

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.260-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:79e64c9b8ba5c47da0be79d0e9f2551c2efe31352bd21b94362cf549acd7f3f6
+size 642810

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.260-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:24792de358440e5ac6f44e4711f067de8842a7ef0ee0dca1fda8effa307959b4
+size 703496

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.280-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a98e4e6f6f07902ba7d098f02b186f2150d1c355769a5576b0645cddee195062
+size 653962

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.280-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:0b99d3c8f0c0a3b35c9b707f82bd1657765392da680cf89de1ae71b10a8971b4
+size 714741

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.300-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:579e3e57776f6bfd53f312f565335a71338e392e2b39ffc7b588a2df5e001e9f
+size 644777

data/eval/Llama-2-13b-chat-hf_wd_true_RP_1.300-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:be28d56ac8257ca7489d51265d2d71a74abd54330abcd5e4f22b077ba344c651
+size 705446

Llama-2-eval/data/datasets/ms_macro/data-00000-of-00001.arrow → data/eval/Llama-2-70b-chat-hf_wd_RP_1.000-t2.json RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ef1814438c08fe1bcd56be04a29c7dbe96f09420be471fdfc36d61c1500f13c
-size 2068896

 version https://git-lfs.github.com/spec/v1
+oid sha256:256520aa638029eb70674b3a93acbd7d21277f2626d756e1cbd55d54dc40b55e
+size 1046106

data/{logs/Phi-3-mini-128k-instruct_mm_false_RP_1.000.txt → eval/Llama-2-70b-chat-hf_wd_RP_1.000-t2_evaluated.json} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34860965667ba1a520aa539d7893315c0769b8fcccc7eb9a2b83d3165629d434
-size 1412667

 version https://git-lfs.github.com/spec/v1
+oid sha256:7917c1b3993e69ca3d3b9c6b0245646e5f78ea2568b0b669f2e08b37ad9759f1
+size 1107148

data/eval/Llama-2-70b-chat-hf_wd_RP_1.020-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:7f43b5b7139632e00b30f719868b85d265592ba6e99f04e2e4abb404679ae4ad
+size 813840

data/eval/Llama-2-70b-chat-hf_wd_RP_1.020-t2_evaluated.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:6cb0c6be15cadd8ef841c8aaceb4d37ce08fbe242757ea9e03be4f3c13c250db
+size 874638

data/eval/Llama-2-70b-chat-hf_wd_RP_1.040-t2.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:447323f0b62e5f885fe58667b9698b1b9038f37381a19f0b5eb948b501d653a9
+size 663779