Spaces:

inflaton-ai
/

logical-reasoning

Build error

App Files Files Community

dh-mc commited on Jul 20, 2024

Commit

47b70ed

•

1 Parent(s): 26227a0

M3 results

Browse files

Files changed (5) hide show

competition/10c_InternLM_M3_eval.ipynb +1 -1
competition/10d_InternLM_M3_analysis.ipynb +1 -0
competition/14_GLM-4_M3_eval.ipynb +1 -1
llm_toolkit/llm_utils.py +14 -4
results/mgtv-results_m3.csv +0 -0

competition/10c_InternLM_M3_eval.ipynb CHANGED Viewed

@@ -1 +1 @@

- {"cells":[{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":476,"status":"ok","timestamp":1720679526275,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"uWKRSV6eZsCn"},"outputs":[],"source":["%load_ext autoreload\n","%autoreload 2"]},{"cell_type":"code","execution_count":2,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"eb33b19f-1206-41ee-84e2-e6258a12eef7","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2534,"status":"ok","timestamp":1720679529344,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"xwFh14uiZBrI","outputId":"d767799c-34c2-46a5-f052-378146a55321"},"outputs":[],"source":["from pathlib import Path\n","\n","try:\n"," from google.colab import drive\n","\n"," drive.mount(\"/content/drive\")\n"," workding_dir = \"/content/drive/MyDrive/logical-reasoning/\"\n","except ModuleNotFoundError:\n"," workding_dir = str(Path.cwd().parent)"]},{"cell_type":"code","execution_count":3,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"6d394937-6c99-4a7c-9d32-7600a280032f","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":5,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"G5pNu3zgZBrL","outputId":"160a554f-fb08-4aa0-bc00-0422fb7c1fac"},"outputs":[{"name":"stdout","output_type":"stream","text":["workding dir: /Users/inflaton/code/engd/projects/logical-reasoning\n"]}],"source":["import os\n","import sys\n","from pathlib import Path\n","\n","os.chdir(workding_dir)\n","sys.path.append(workding_dir)\n","print(\"workding dir:\", workding_dir)"]},{"cell_type":"code","execution_count":4,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"ac667aba-076e-4de6-9984-8f6a67cb09cd","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"0dVRAabNZBrL","outputId":"b977e116-df16-47cd-9160-a24f611da687"},"outputs":[{"data":{"text/plain":["False"]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["need_to_setup_env = False\n","need_to_setup_env"]},{"cell_type":"code","execution_count":5,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"72f9cf79-7b0d-4d9e-90a0-1fa5251b947f","showTitle":false,"title":""},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"hKUOfP2HZBrL"},"outputs":[],"source":["if need_to_setup_env:\n"," %pip install -r requirements.txt\n"," %cd /content/\n"," %rm -rf LLaMA-Factory\n"," !git clone https://github.com/hiyouga/LLaMA-Factory.git\n"," %cd LLaMA-Factory\n"," %ls\n"," %pip install -e .[torch,bitsandbytes]\n"," \n"," os.chdir(workding_dir)\n"," sys.path.append(workding_dir)\n"," print(\"workding dir:\", workding_dir)"]},{"cell_type":"code","execution_count":6,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"9f67ec60-2f24-411c-84eb-0dd664b44775","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"hPCC-6m7ZBrM","outputId":"c7aa2c96-5e99-440a-c148-201d79465ff9"},"outputs":[{"name":"stdout","output_type":"stream","text":["loading env vars from: /Users/inflaton/code/engd/projects/logical-reasoning/.env\n"]},{"data":{"text/plain":["True"]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["from dotenv import find_dotenv, load_dotenv\n","\n","found_dotenv = find_dotenv(\".env\")\n","\n","if len(found_dotenv) == 0:\n"," found_dotenv = find_dotenv(\".env.example\")\n","print(f\"loading env vars from: {found_dotenv}\")\n","load_dotenv(found_dotenv, override=True)"]},{"cell_type":"code","execution_count":7,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"f1597656-8042-4878-9d3b-9ebfb8dd86dc","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"1M3IraVtZBrM","outputId":"29ab35f6-2970-4ade-d85d-3174acf8cda0"},"outputs":[{"name":"stdout","output_type":"stream","text":["internlm/internlm2_5-7b-chat-1m llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88 False datasets/mgtv results/mgtv-results_m3.csv\n"]}],"source":["import os\n","\n","model_name = os.getenv(\"MODEL_NAME\")\n","adapter_name_or_path = os.getenv(\"ADAPTER_NAME_OR_PATH\")\n","load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n","data_path = os.getenv(\"LOGICAL_REASONING_DATA_PATH\")\n","results_path = os.getenv(\"LOGICAL_REASONING_RESULTS_PATH\")\n","use_english_datasets = os.getenv(\"USE_ENGLISH_DATASETS\") == \"true\"\n","\n","print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path)"]},{"cell_type":"code","execution_count":8,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"b2a43943-9324-4839-9a47-cfa72de2244b","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":564,"status":"ok","timestamp":1720679529907,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"UgMvt6dIZBrM","outputId":"ce37581c-fd26-46c2-ad87-d933d99f68f7"},"outputs":[{"name":"stdout","output_type":"stream","text":["Python 3.11.9\n","\u001b[33mWARNING: Package(s) not found: flash-attn\u001b[0m\u001b[33m\n","\u001b[0mCPU times: user 3.96 ms, sys: 7.26 ms, total: 11.2 ms\n","Wall time: 643 ms\n"]}],"source":["%%time\n","!python --version\n","!pip show flash-attn"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1685,"status":"ok","timestamp":1720679531591,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"ZuS_FsLyZBrN","outputId":"2cba0105-c505-4395-afbd-2f2fee6581d0"},"outputs":[{"name":"stdout","output_type":"stream","text":["loading /Users/inflaton/code/engd/projects/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\n","MPS is available\n"]}],"source":["from llm_toolkit.llm_utils import *\n","from llm_toolkit.logical_reasoning_utils import *\n","\n","device = check_gpu()"]},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["loading model: internlm/internlm2_5-7b-chat-1m with adapter: llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"74891a59a88c4845845a61327a619b22","version_major":2,"version_minor":0},"text/plain":["Loading checkpoint shards: 0%| | 0/8 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: user 2.44 s, sys: 4.12 s, total: 6.56 s\n","Wall time: 7.18 s\n"]}],"source":["%%time\n","\n","model, tokenizer = load_model(model_name, adapter_name_or_path=adapter_name_or_path, using_llama_factory=False)"]},{"cell_type":"code","execution_count":11,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["loading train/test data files\n","DatasetDict({\n"," train: Dataset({\n"," features: ['text', 'label', 'answer', 'title', 'puzzle', 'truth', 'train_text', 'prompt'],\n"," num_rows: 25000\n"," })\n"," test: Dataset({\n"," features: ['text', 'label', 'answer', 'title', 'puzzle', 'truth', 'train_text', 'prompt'],\n"," num_rows: 3000\n"," })\n","})\n"]}],"source":["datasets = load_logical_reasoning_dataset(\n"," data_path,\n"," tokenizer=tokenizer,\n"," chinese_prompt=not use_english_datasets,\n"," using_p1=False,\n",")"]},{"cell_type":"code","execution_count":12,"metadata":{},"outputs":[],"source":["def evaluate_model(model, tokenizer, model_name, dataset):\n"," print(f\"Evaluating model: {model_name} on {device}\")\n"," predictions = eval_model(model, tokenizer, dataset, device=device)\n","\n"," save_results(\n"," model_name,\n"," results_path,\n"," dataset,\n"," predictions,\n"," debug=False,\n"," )\n","\n"," metrics = calc_metrics(dataset[\"label\"], predictions, debug=False)\n"," print(metrics)"]},{"cell_type":"code","execution_count":13,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Evaluating model: internlm/internlm2_5-7b-chat-1m_llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88 on mps\n"]},{"name":"stderr","output_type":"stream","text":[" 0%| | 1/3000 [00:03<2:33:23, 3.07s/it]"]},{"name":"stdout","output_type":"stream","text":["--------\n","step 1: 不是</s>\n","--------\n","step 2: 不是\n","--------\n","step 3: 不是\n","--------\n","step 4: 不是\n","--------\n","step 5: 不是\n"]},{"name":"stderr","output_type":"stream","text":[" 57%|█████▋ | 1709/3000 [7:39:10<1:11:42, 3.33s/it] "]}],"source":["%%time\n","\n","evaluate_model(model, tokenizer, f\"{model_name}_{adapter_name_or_path}\", datasets[\"test\"])"]}],"metadata":{"accelerator":"GPU","application/vnd.databricks.v1+notebook":{"dashboards":[],"environmentMetadata":null,"language":"python","notebookMetadata":{"mostRecentlyExecutedCommandWithImplicitDF":{"commandId":-1,"dataframes":["_sqldf"]},"pythonIndentUnit":4},"notebookName":"10_eval-lf-medium-py3.11","widgets":{}},"colab":{"gpuType":"L4","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.9"}},"nbformat":4,"nbformat_minor":0}

+ {"cells":[{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":476,"status":"ok","timestamp":1720679526275,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"uWKRSV6eZsCn"},"outputs":[],"source":["%load_ext autoreload\n","%autoreload 2"]},{"cell_type":"code","execution_count":2,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"eb33b19f-1206-41ee-84e2-e6258a12eef7","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2534,"status":"ok","timestamp":1720679529344,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"xwFh14uiZBrI","outputId":"d767799c-34c2-46a5-f052-378146a55321"},"outputs":[],"source":["from pathlib import Path\n","\n","try:\n"," from google.colab import drive\n","\n"," drive.mount(\"/content/drive\")\n"," workding_dir = \"/content/drive/MyDrive/logical-reasoning/\"\n","except ModuleNotFoundError:\n"," workding_dir = str(Path.cwd().parent)"]},{"cell_type":"code","execution_count":3,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"6d394937-6c99-4a7c-9d32-7600a280032f","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":5,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"G5pNu3zgZBrL","outputId":"160a554f-fb08-4aa0-bc00-0422fb7c1fac"},"outputs":[{"name":"stdout","output_type":"stream","text":["workding dir: /Users/inflaton/code/engd/projects/logical-reasoning\n"]}],"source":["import os\n","import sys\n","from pathlib import Path\n","\n","os.chdir(workding_dir)\n","sys.path.append(workding_dir)\n","print(\"workding dir:\", workding_dir)"]},{"cell_type":"code","execution_count":4,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"ac667aba-076e-4de6-9984-8f6a67cb09cd","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"0dVRAabNZBrL","outputId":"b977e116-df16-47cd-9160-a24f611da687"},"outputs":[{"data":{"text/plain":["False"]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["need_to_setup_env = False\n","need_to_setup_env"]},{"cell_type":"code","execution_count":5,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"72f9cf79-7b0d-4d9e-90a0-1fa5251b947f","showTitle":false,"title":""},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"hKUOfP2HZBrL"},"outputs":[],"source":["if need_to_setup_env:\n"," %pip install -r requirements.txt\n"," %cd /content/\n"," %rm -rf LLaMA-Factory\n"," !git clone https://github.com/hiyouga/LLaMA-Factory.git\n"," %cd LLaMA-Factory\n"," %ls\n"," %pip install -e .[torch,bitsandbytes]\n"," \n"," os.chdir(workding_dir)\n"," sys.path.append(workding_dir)\n"," print(\"workding dir:\", workding_dir)"]},{"cell_type":"code","execution_count":6,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"9f67ec60-2f24-411c-84eb-0dd664b44775","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"hPCC-6m7ZBrM","outputId":"c7aa2c96-5e99-440a-c148-201d79465ff9"},"outputs":[{"name":"stdout","output_type":"stream","text":["loading env vars from: /Users/inflaton/code/engd/projects/logical-reasoning/.env\n"]},{"data":{"text/plain":["True"]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["from dotenv import find_dotenv, load_dotenv\n","\n","found_dotenv = find_dotenv(\".env\")\n","\n","if len(found_dotenv) == 0:\n"," found_dotenv = find_dotenv(\".env.example\")\n","print(f\"loading env vars from: {found_dotenv}\")\n","load_dotenv(found_dotenv, override=True)"]},{"cell_type":"code","execution_count":7,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"f1597656-8042-4878-9d3b-9ebfb8dd86dc","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"1M3IraVtZBrM","outputId":"29ab35f6-2970-4ade-d85d-3174acf8cda0"},"outputs":[{"name":"stdout","output_type":"stream","text":["internlm/internlm2_5-7b-chat-1m llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88 False datasets/mgtv results/mgtv-results_m3.csv\n"]}],"source":["import os\n","\n","model_name = os.getenv(\"MODEL_NAME\")\n","adapter_name_or_path = os.getenv(\"ADAPTER_NAME_OR_PATH\")\n","load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n","data_path = os.getenv(\"LOGICAL_REASONING_DATA_PATH\")\n","results_path = os.getenv(\"LOGICAL_REASONING_RESULTS_PATH\")\n","use_english_datasets = os.getenv(\"USE_ENGLISH_DATASETS\") == \"true\"\n","\n","print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path)"]},{"cell_type":"code","execution_count":8,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"b2a43943-9324-4839-9a47-cfa72de2244b","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":564,"status":"ok","timestamp":1720679529907,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"UgMvt6dIZBrM","outputId":"ce37581c-fd26-46c2-ad87-d933d99f68f7"},"outputs":[{"name":"stdout","output_type":"stream","text":["Python 3.11.9\n","\u001b[33mWARNING: Package(s) not found: flash-attn\u001b[0m\u001b[33m\n","\u001b[0mCPU times: user 38.9 ms, sys: 26.7 ms, total: 65.7 ms\n","Wall time: 2.97 s\n"]}],"source":["%%time\n","!python --version\n","!pip show flash-attn"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1685,"status":"ok","timestamp":1720679531591,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"ZuS_FsLyZBrN","outputId":"2cba0105-c505-4395-afbd-2f2fee6581d0"},"outputs":[{"name":"stdout","output_type":"stream","text":["loading /Users/inflaton/code/engd/projects/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\n","MPS is available\n"]}],"source":["from llm_toolkit.llm_utils import *\n","from llm_toolkit.logical_reasoning_utils import *\n","\n","device = check_gpu()"]},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["loading model: internlm/internlm2_5-7b-chat-1m with adapter: llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"443302f393fe45e3a5150cb5e1f35a11","version_major":2,"version_minor":0},"text/plain":["Loading checkpoint shards: 0%| | 0/8 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: user 3.13 s, sys: 6.52 s, total: 9.65 s\n","Wall time: 30.7 s\n"]}],"source":["%%time\n","\n","model, tokenizer = load_model(model_name, adapter_name_or_path=adapter_name_or_path, using_llama_factory=False)"]},{"cell_type":"code","execution_count":11,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["loading train/test data files\n","DatasetDict({\n"," train: Dataset({\n"," features: ['text', 'label', 'answer', 'title', 'puzzle', 'truth', 'train_text', 'prompt'],\n"," num_rows: 25000\n"," })\n"," test: Dataset({\n"," features: ['text', 'label', 'answer', 'title', 'puzzle', 'truth', 'train_text', 'prompt'],\n"," num_rows: 3000\n"," })\n","})\n"]}],"source":["datasets = load_logical_reasoning_dataset(\n"," data_path,\n"," tokenizer=tokenizer,\n"," chinese_prompt=not use_english_datasets,\n"," using_p1=False,\n",")"]},{"cell_type":"code","execution_count":12,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["--------------------------------------------------\n","text: 甄加索是自杀吗\n","--------------------------------------------------\n","label: 不是\n","--------------------------------------------------\n","answer: nan\n","--------------------------------------------------\n","title: 海岸之谜\n","--------------------------------------------------\n","puzzle: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n","--------------------------------------------------\n","truth: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n","--------------------------------------------------\n","train_text: <s><|im_start|>system\n","You are an expert in logical reasoning.<|im_end|>\n","<|im_start|>user\n","你是一个情景猜谜游戏的主持人。游戏规则如下：\n","\n","1. 参与者会得到一个谜面，谜面会描述一个简单又难以理解的事件。\n","2. 主持人知道谜底，谜底是谜面的答案。\n","3. 参与者可以询问任何封闭式问题来找寻事件的真相。\n","4. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。各回答的判断标准如下：\n"," - 若谜面和谜底能找到问题的答案，回答：是或者不是\n"," - 若谜面和谜底不能直接或者间接推断出问题的答案，回答：不重要\n"," - 若参与者提问不是一个封闭式问题或者问题难以理解，回答：问法错误\n"," - 若参与者提问基本还原了谜底真相，回答：回答正确\n","5. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n","\n","请严格按照这些规则回答参与者提出的问题。\n","\n","**谜面:** 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n","\n","**谜底:** 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n","\n","**参与者提出的问题:** 甄加索是自杀吗\n","<|im_end|>\n","<|im_start|>assistant\n","不是</s>\n","--------------------------------------------------\n","prompt: <s><|im_start|>system\n","You are an expert in logical reasoning.<|im_end|>\n","<|im_start|>user\n","你是一个情景猜谜游戏的主持人。游戏规则如下：\n","\n","1. 参与者会得到一个谜面，谜面会描述一个简单又难以理解的事件。\n","2. 主持人知道谜底，谜底是谜面的答案。\n","3. 参与者可以询问任何封闭式问题来找寻事件的真相。\n","4. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。各回答的判断标准如下：\n"," - 若谜面和谜底能找到问题的答案，回答：是或者不是\n"," - 若谜面和谜底不能直接或者间接推断出问题的答案，回答：不重要\n"," - 若参与者提问不是一个封闭式问题或者问题难以理解，回答：问法错误\n"," - 若参与者提问基本还原了谜底真相，回答：回答正确\n","5. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n","\n","请严格按照这些规则回答参与者提出的问题。\n","\n","**谜面:** 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n","\n","**谜底:** 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n","\n","**参与者提出的问题:** 甄加索是自杀吗\n","<|im_end|>\n","<|im_start|>assistant\n","\n"]}],"source":["print_row_details(datasets[\"test\"].to_pandas())"]},{"cell_type":"code","execution_count":13,"metadata":{},"outputs":[],"source":["def evaluate_model(model, tokenizer, model_name, dataset, batch_size=1):\n"," print(f\"Evaluating model: {model_name} on {device}\")\n"," predictions = eval_model(\n"," model, tokenizer, dataset, device=device, batch_size=batch_size\n"," )\n","\n"," save_results(\n"," model_name,\n"," results_path,\n"," dataset,\n"," predictions,\n"," debug=False,\n"," )\n","\n"," metrics = calc_metrics(dataset[\"label\"], predictions, debug=False)\n"," print(metrics)"]},{"cell_type":"code","execution_count":13,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Evaluating model: internlm/internlm2_5-7b-chat-1m_llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88 on mps\n"]},{"name":"stderr","output_type":"stream","text":[" 0%| | 1/3000 [00:03<2:33:23, 3.07s/it]"]},{"name":"stdout","output_type":"stream","text":["--------\n","step 1: 不是</s>\n","--------\n","step 2: 不是\n","--------\n","step 3: 不是\n","--------\n","step 4: 不是\n","--------\n","step 5: 不是\n"]},{"name":"stderr","output_type":"stream","text":["100%|██████████| 3000/3000 [10:20:50<00:00, 12.42s/it] "]},{"name":"stdout","output_type":"stream","text":["{'accuracy': 0.7836666666666666}\n","CPU times: user 12min 26s, sys: 11min 38s, total: 24min 4s\n","Wall time: 10h 20min 50s\n"]},{"name":"stderr","output_type":"stream","text":["\n"]}],"source":["%%time\n","\n","evaluate_model(model, tokenizer, f\"{model_name}_{adapter_name_or_path}\", datasets[\"test\"])"]},{"cell_type":"code","execution_count":15,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Evaluating model: internlm/internlm2_5-7b-chat-1m_llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88_batch_16 on mps\n"]},{"name":"stderr","output_type":"stream","text":[" 1%| | 1/188 [01:28<4:34:52, 88.20s/it]"]},{"name":"stdout","output_type":"stream","text":["--------\n","step 1: 不是\n","--------\n","step 2: 不是\n","--------\n","step 3: 不是\n","--------\n","step 4: 不是\n","--------\n","step 5: 不是\n"]},{"name":"stderr","output_type":"stream","text":[" 2%|▏ | 4/188 [04:32<3:21:43, 65.78s/it]"]}],"source":["%%time\n","\n","evaluate_model(model, tokenizer, f\"{model_name}_{adapter_name_or_path}_batch_16\", datasets[\"test\"], batch_size=16)"]}],"metadata":{"accelerator":"GPU","application/vnd.databricks.v1+notebook":{"dashboards":[],"environmentMetadata":null,"language":"python","notebookMetadata":{"mostRecentlyExecutedCommandWithImplicitDF":{"commandId":-1,"dataframes":["_sqldf"]},"pythonIndentUnit":4},"notebookName":"10_eval-lf-medium-py3.11","widgets":{}},"colab":{"gpuType":"L4","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.9"}},"nbformat":4,"nbformat_minor":0}

competition/10d_InternLM_M3_analysis.ipynb ADDED Viewed

	@@ -0,0 +1 @@

+ {"cells":[{"cell_type":"code","execution_count":1,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"0ea8b46b-839b-445b-8043-ccdf4e920ace","showTitle":false,"title":""},"id":"YLH80COBzi_F"},"outputs":[],"source":["%load_ext autoreload\n","%autoreload 2"]},{"cell_type":"code","execution_count":2,"metadata":{"id":"63B5exAuzq4M"},"outputs":[],"source":["from pathlib import Path\n","\n","try:\n"," from google.colab import drive\n"," drive.mount('/content/drive')\n"," workding_dir = \"/content/drive/MyDrive/logical-reasoning/\"\n","except ModuleNotFoundError:\n"," workding_dir = str(Path.cwd().parent)"]},{"cell_type":"code","execution_count":3,"metadata":{"executionInfo":{"elapsed":368,"status":"ok","timestamp":1719461634865,"user":{"displayName":"Donghao Huang","userId":"00463591218503521679"},"user_tz":-480},"id":"zFulf0bg0H-9","outputId":"debdd535-c828-40b9-efc0-8a180e5830dd"},"outputs":[{"name":"stdout","output_type":"stream","text":["workding dir: /Users/inflaton/code/engd/projects/logical-reasoning\n"]}],"source":["import os\n","import sys\n","\n","os.chdir(workding_dir)\n","sys.path.append(workding_dir)\n","print(\"workding dir:\", workding_dir)"]},{"cell_type":"code","execution_count":4,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"9f67ec60-2f24-411c-84eb-0dd664b44775","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":589,"status":"ok","timestamp":1719462011879,"user":{"displayName":"Donghao Huang","userId":"00463591218503521679"},"user_tz":-480},"id":"DIUiweYYzi_I","outputId":"e16e9247-9077-4b0c-f8ea-17059f05a1c4"},"outputs":[{"name":"stdout","output_type":"stream","text":["loading env vars from: /Users/inflaton/code/engd/projects/logical-reasoning/.env\n"]},{"data":{"text/plain":["True"]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["from dotenv import find_dotenv, load_dotenv\n","\n","found_dotenv = find_dotenv(\".env\")\n","\n","if len(found_dotenv) == 0:\n"," found_dotenv = find_dotenv(\".env.example\")\n","print(f\"loading env vars from: {found_dotenv}\")\n","load_dotenv(found_dotenv, override=True)"]},{"cell_type":"code","execution_count":5,"metadata":{"id":"W2QyVreqhOGM","outputId":"68b9590e-1ac6-4c6f-e0c4-e273ec816419"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>text</th>\n"," <th>label</th>\n"," <th>title</th>\n"," <th>puzzle</th>\n"," <th>truth</th>\n"," <th>internlm/internlm2_5-7b-chat-1m_llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>甄加索是自杀吗</td>\n"," <td>不是</td>\n"," <td>海岸之谜</td>\n"," <td>在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任...</td>\n"," <td>甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在...</td>\n"," <td>不是</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>甄加索有身体上的疾病吗</td>\n"," <td>是</td>\n"," <td>海岸之谜</td>\n"," <td>在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任...</td>\n"," <td>甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在...</td>\n"," <td>是</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>画作是甄的</td>\n"," <td>是</td>\n"," <td>海岸之谜</td>\n"," <td>在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任...</td>\n"," <td>甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在...</td>\n"," <td>是</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>甄有心脏病吗</td>\n"," <td>是</td>\n"," <td>海岸之谜</td>\n"," <td>在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任...</td>\n"," <td>甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在...</td>\n"," <td>是</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>车轮是凶手留下的</td>\n"," <td>不是</td>\n"," <td>海岸之谜</td>\n"," <td>在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任...</td>\n"," <td>甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在...</td>\n"," <td>不是</td>\n"," </tr>\n"," <tr>\n"," <th>...</th>\n"," <td>...</td>\n"," <td>...</td>\n"," <td>...</td>\n"," <td>...</td>\n"," <td>...</td>\n"," <td>...</td>\n"," </tr>\n"," <tr>\n"," <th>2995</th>\n"," <td>哭泣者必须在晚上祭奠吗</td>\n"," <td>是</td>\n"," <td>甄庄哭声</td>\n"," <td>在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着...</td>\n"," <td>原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖...</td>\n"," <td>不重要</td>\n"," </tr>\n"," <tr>\n"," <th>2996</th>\n"," <td>尸体在湖里吗</td>\n"," <td>不是</td>\n"," <td>甄庄哭声</td>\n"," <td>在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着...</td>\n"," <td>原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖...</td>\n"," <td>不是</td>\n"," </tr>\n"," <tr>\n"," <th>2997</th>\n"," <td>哭泣者和死者有特殊关系吗</td>\n"," <td>是</td>\n"," <td>甄庄哭声</td>\n"," <td>在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着...</td>\n"," <td>原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖...</td>\n"," <td>是</td>\n"," </tr>\n"," <tr>\n"," <th>2998</th>\n"," <td>是帽子的主人去世了吗</td>\n"," <td>不是</td>\n"," <td>甄庄哭声</td>\n"," <td>在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着...</td>\n"," <td>原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖...</td>\n"," <td>是</td>\n"," </tr>\n"," <tr>\n"," <th>2999</th>\n"," <td>死者受伤了吗</td>\n"," <td>不是</td>\n"," <td>甄庄哭声</td>\n"," <td>在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着...</td>\n"," <td>原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖...</td>\n"," <td>不是</td>\n"," </tr>\n"," </tbody>\n","</table>\n","<p>3000 rows × 6 columns</p>\n","</div>"],"text/plain":[" text label title \\\n","0 甄加索是自杀吗不是海岸之谜 \n","1 甄加索有身体上的疾病吗是海岸之谜 \n","2 画作是甄的是海岸之谜 \n","3 甄有心脏病吗是海岸之谜 \n","4 车轮是凶手留下的不是海岸之谜 \n","... ... ... ... \n","2995 哭泣者必须在晚上祭奠吗是甄庄哭声 \n","2996 尸体在湖里吗不是甄庄哭声 \n","2997 哭泣者和死者有特殊关系吗是甄庄哭声 \n","2998 是帽子的主人去世了吗不是甄庄哭声 \n","2999 死者受伤了吗不是甄庄哭声 \n","\n"," puzzle \\\n","0 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任... \n","1 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任... \n","2 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任... \n","3 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任... \n","4 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任... \n","... ... \n","2995 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着... \n","2996 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。��二天早晨，村长甄锐发现湖边的石头上放着... \n","2997 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着... \n","2998 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着... \n","2999 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着... \n","\n"," truth \\\n","0 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在... \n","1 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在... \n","2 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在... \n","3 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在... \n","4 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在... \n","... ... \n","2995 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖... \n","2996 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖... \n","2997 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖... \n","2998 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖... \n","2999 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖... \n","\n"," internlm/internlm2_5-7b-chat-1m_llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88 \n","0 不是 \n","1 是 \n","2 是 \n","3 是 \n","4 不是 \n","... ... \n","2995 不重要 \n","2996 不是 \n","2997 是 \n","2998 是 \n","2999 不是 \n","\n","[3000 rows x 6 columns]"]},"execution_count":5,"metadata":{},"output_type":"execute_result"}],"source":["import pandas as pd\n","\n","df = pd.read_csv(\"results/mgtv-results_m3.csv\")\n","df"]},{"cell_type":"code","execution_count":6,"metadata":{},"outputs":[],"source":["import matplotlib.pyplot as plt\n","from matplotlib import rcParams\n","\n","def plot_value_counts(df, column):\n"," font_family = rcParams[\"font.family\"]\n"," # Set the font to SimHei to support Chinese characters\n"," rcParams[\"font.family\"] = \"STHeiti\"\n"," rcParams[\"axes.unicode_minus\"] = False # This is to support the minus sign in Chinese.\n","\n"," plt.figure(figsize=(12, 6))\n"," df[column].value_counts().plot(kind=\"bar\")\n"," # add values on top of bars\n"," for i, v in enumerate(df[column].value_counts()):\n"," plt.text(i, v + 0.1, str(v), ha=\"center\")\n"," plt.show()\n"," \n"," rcParams[\"font.family\"] = font_family\n"]},{"cell_type":"code","execution_count":7,"metadata":{},"outputs":[{"data":{"text/plain":["['text',\n"," 'label',\n"," 'title',\n"," 'puzzle',\n"," 'truth',\n"," 'internlm/internlm2_5-7b-chat-1m_llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88']"]},"execution_count":7,"metadata":{},"output_type":"execute_result"}],"source":["df.columns.to_list()"]},{"cell_type":"code","execution_count":8,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["********** internlm/internlm2_5-7b-chat-1m_llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88 **********\n","internlm/internlm2_5-7b-chat-1m_llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88\n","不是 1505\n","是 1140\n","不重要 264\n","问法错误 53\n","回答正确 38\n","Name: count, dtype: int64\n"]},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAA94AAAI3CAYAAABtUYPVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABzyUlEQVR4nO3dd3QU1f/G8ScFkk1CEgglBELvIDZ6ExEECQIBxIJKl6bSVCyIBQXFgn4FVFBAmoUiKF0p0osCiiBICRAiCRBCQnq7vz842R9LAgTIZCnv1zl7YOfObD6ze3d2n507My7GGCMAAAAAAGAJV2cXAAAAAADArYzgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhdydXUBeyczM1H///adChQrJxcXF2eUAAAAAAG5xxhidO3dOQUFBcnW99H7tWyZ4//fffwoODnZ2GQAAAACA20x4eLhKly59yfZbJngXKlRI0vkV9vX1dXI1AAAAAIBbXVxcnIKDg+159FJumeCdNbzc19eX4A0AAAAAyDdXOtyZk6sBAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjfs7r//frm6uurDDz/M1laqVCm5u7vbb+PGjbO3TZ48WWXKlJG3t7dCQ0MVFRXlsOyUKVPk5uZmX9bHx8fydQEAAACAGwXBG3Zr1qzRqFGjsk0/e/asPDw8lJ6ebr+99NJLkqTly5drzJgxWrx4sU6fPq1atWopNDTUYfk9e/Zo6tSp9mXj4+PzZX0AAAAA4EZA8MYV7dmzR1WrVs2xbfz48froo49Uu3Zt2Ww2jR49WpmZmVq9erXD8tWqVcuvcgEAAADghkLwxhXt3bs3x+BtjNHWrVsVEhLiML1Lly5atWrVFZcHAAAAgNsBwRtXtGfPHn333XcqUqSI7rrrLq1YsUKSFB0dLX9/f3l6ejrMX7VqVR0+fFjS+WHqkZGRuueee1SiRAn16dOHoeYAAAAAbisEb1xRSEiINm3apMjISH300Ufq1auX9u7dq6SkJHl5eWWb32azKSkpSZKUkpKi5cuX66+//tKePXtkjFG/fv3yexUAAAAAwGkI3riiVq1aqUKFCipYsKAeeOABjRgxQl9//bVsNpuSk5OzzR8dHW0P5CVKlFCrVq3k4+OjokWL6osvvtCqVasUFxeX36sBAAAAAE5B8MZVq1Chgo4fP66AgACdPXtWaWlpDu1hYWGqUKFCjssWKFBApUuXVkRERH6UCgAAAABOR/DGZY0ePVo//fSTw7QNGzaoSpUqcnFxUZ06dRzOYC5J8+bNU8uWLRUREaGOHTs6tEVHR+vQoUMqX7681aUDAAAAwA2B4I3LuueeezRkyBDt2rVLiYmJmj59ur7++msNGDBAkjRs2DANGTJE+/btU3JyskaPHi2bzabmzZurVKlS+u+//zRy5EglJCTo4MGDeuSRR9S7d+9sJ2QDAAAAgFuVu7MLwI0tJCRE//33nzp37qyIiAjdfffdWrRokYKCgiRJbdq00dGjR9WqVStFR0erdevWmj9/vn35BQsWaNCgQSpZsqRsNpueeuopjRkzxlmrAwAAAAD5zsUYY5xdRF6Ii4uTn5+fYmNj5evr6+xyAAAAAAC3uNzmUPZ43wDKvbzE2SXcdo68F+LsEgAAAADcJjjGGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsNA1Be/7779frq6u+vDDDy85T1paml5//XWHaZMnT1aZMmXk7e2t0NBQRUVFOcz//PPPq3DhwgoICNCLL76o9PT0aykPAAAAAIAbxjUF7zVr1mjUqFGXnWfUqFHauXOn/f7y5cs1ZswYLV68WKdPn1atWrUUGhpqb3/99df177//at++fdq3b5/27t2bLbgDAAAAAHCzsWSo+caNG/Xxxx87TBs/frw++ugj1a5dWzabTaNHj1ZmZqZWr16t1NRUTZkyRdOnT1eJEiVUrFgxzZgxQ1999ZUSEhKsKBEAAAAAgHyR58E7Pj5e3bt31wsvvGCfZozR1q1bFRIS4jBvly5dtGrVKu3atUvVq1dXYGCgvS0gIED169fXpk2b8rpEAAAAAADyTZ4H78GDByskJEStWrWyT4uOjpa/v788PT0d5q1ataoOHz6sY8eOqVKlStkeK6s9JykpKYqLi3O4AQAAAABwo8nT4L1o0SJt2bJF77//vsP0pKQkeXl5ZZvfZrMpKSnpiu05GTt2rPz8/Oy34ODgvFkJAAAAAADyUJ4F75MnT2rQoEGaNWtWtj3bNptNycnJ2ZaJjo6Wl5fXFdtz8sorryg2NtZ+Cw8Pz5sVAQAAAAAgD7nn1QMNGDBAUVFRatiwoSQpMzNTGRkZCgwM1IkTJ3T27FmlpaWpQIEC9mXCwsJUoUIFlSlTRmFhYdkeMywsTJ07d87x73l4eMjDwyOvygcAAAAAwBJ5tsd7/vz5SktLU3JyspKTk7Vy5Uo99NBDioyMlIuLi+rUqaPVq1c7LDNv3jy1bNlSd911l/bu3auYmBh725kzZ7R161Y1atQor0oEAAAAACDfWXI5sZwMGzZMQ4YM0b59+5ScnKzRo0fLZrOpefPmKliwoHr27Km+ffsqOjpap0+fVq9evfTss89ecqg5AAAAAAA3g3wL3m3atNGQIUPUqlUrFSlSRDt27ND8+fPt7W+//bZKliypihUrqkqVKqpcubLefPPN/CoPAAAAAABLuBhjjLOLyAtxcXHy8/NTbGysfH19nV3OVSn38hJnl3DbOfJeyJVnAgAAAIDLyG0Ozbc93gAAAAAA3I4I3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABY6JqC9/333y9XV1d9+OGHDtM3bNig+vXry9fXVw0aNNCWLVsc2t9++22VKFFCvr6+6tWrl+Lj4+1tcXFx6tatm3x8fBQUFKT333//WkoDAAAAAOCGck3Be82aNRo1apTDtAMHDqhr16569913FRUVpZdeekkdO3bU8ePHJUlffPGFFi5cqC1btuj48eNycXFR37597cs/88wzKlCggCIiIrR582bNmzdPX3755XWsGgAAAAAAzueeVw80YcIEDRs2TC1btpQkderUSVu3btUPP/ygYcOGafz48ZozZ47Kly8v6XwQr1ixog4dOiRPT0+tWbNGR48elaenp/z8/DRjxgw99NBD6tevX16VCAAAAABAvsuz4F2+fHmFhIQ4TAsODlZ4eLhOnjyphIQE3Xvvvfa2AgUKqH379lq9erX8/PzUqlUreXp62turV68uHx8fHThwQJUrV86rMgEAAAAAyFd5dnK1IUOGZAvIixcvVp06dXTs2DFVqlQp2zJVq1bV4cOHr9gOAAAAAMDNyrKzmk+fPl2RkZEKDQ1VUlKSvLy8ss1js9mUlJR0xfacpKSkKC4uzuEGAAAAAMCNxpLgvX37do0YMULffvut3N3dZbPZlJycnG2+6OhoeXl5XbE9J2PHjpWfn5/9FhwcnOfrAQAAAADA9crz4H3s2DGFhobq66+/VvXq1SVJZcqUUVhYWLZ5w8LCVKFChSu25+SVV15RbGys/RYeHp63KwIAAAAAQB7I0+B97tw5tWvXTi+++KLatWtnn168eHEVLFhQ+/fvt09LT0/XokWL9MADD6hx48ZatWqV0tPT7e1///23UlJScjz2W5I8PDzk6+vrcAMAAAAA4EaTZ8E7IyNDXbt2VbNmzTR48OBs7UOHDlWfPn10/PhxnTt3Ts8995xatmyp8uXLq1SpUmrUqJGGDh2q+Ph4hYeHq2/fvho5cmRelQcAAAAAgFPkWfB+7rnntHz5cn3xxRdyd3e33x544AFJUv/+/dWqVSvdc889KlWqlFJTU/XFF1/Yl//yyy915swZBQYGql69eurUqZN69+6dV+UBAAAAAOAULsYY4+wi8kJcXJz8/PwUGxt70w07L/fyEmeXcNs58l7IlWcCAAAAgMvIbQ617HJiAHAjuv/+++Xq6qoPP/zQYfq+ffvk5eUlFxcXnT59+rKPsWXLFi1cuNB+Py4uTt26dZOPj4+CgoL0/vvvW1E6AAAAblIEbwC3lTVr1mjUqFHZplerVk2JiYkqW7bsZZdPSEjQU0895XAyyGeeeUYFChRQRESENm/erHnz5unLL7/M89oBAABwc3J3dgEAcDMZNmyYw+UPIyIitGbNGh09elSenp7y8/PTjBkz9NBDD6lfv35OrBQAAAA3CvZ4A0AuLVmyRGvXrlWnTp3s0zZu3KhWrVrJ09PTPq169ery8fHRgQMHnFEmAAAAbjAEbwDIhdOnT6t///6aOXOmvLy87NOPHTumSpUqZZu/atWqOnz4cH6WCAAAgBsUwRsAcuGZZ55R3759Va9ePYfpSUlJDkE8i81mU1JSUn6VBwAAgBsYx3gDwBVMnz5dJ06c0Ny5c7O12Ww2JSYmZpseHR2dYyAHAADA7YfgDQCXcezYMQ0fPlzx8fHy9vaWJKWlpWnOnDnaunWr6tatq2XLlmVbLiwsTBUqVMjvcgEAAHADIngDwGWUKVNG0dHRDtN69Oihdu3aqUuXLoqIiNALL7yg9PR0ubuf36T+/fffSklJyfHYbwAAANx+OMYbAK5DqVKl1KhRIw0dOlTx8fEKDw9X3759NXLkSGeXBgAAgBsEwRsArtOXX36pM2fOKDAwUPXq1VOnTp3Uu3dvZ5cFAACAG4SLMcY4u4i8EBcXJz8/P8XGxsrX19fZ5VyVci8vcXYJt50j74U4u4TbDv08/9HPAQAArJXbHMoebwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACx0TcH7/vvvl6urqz788EOH6Rs3btRdd90lm82munXratu2bQ7tkydPVpkyZeTt7a3Q0FBFRUXZ29LS0vT888+rcOHCCggI0Isvvqj09PRrKQ8AAAAAgBvGNQXvNWvWaNSoUQ7ToqKiFBoaqlGjRikuLk6vvfaaOnTooMjISEnS8uXLNWbMGC1evFinT59WrVq1FBoaal/+9ddf17///qt9+/Zp37592rt3r15//fXrWDUAAAAAAJwvz4aaT506VY8++qg6deqkAgUKqGPHjurevbsmTZokSRo/frw++ugj1a5dWzabTaNHj1ZmZqZWr16t1NRUTZkyRdOnT1eJEiVUrFgxzZgxQ1999ZUSEhLyqkQAAAAAAPJdngXv9evXq0OHDg7TunTpolWrVskYo61btyokJCTH9l27dql69eoKDAy0twUEBKh+/fratGlTXpUIAAAAAEC+y7PgfezYMVWqVMlhWtWqVXX48GFFR0fL399fnp6eObbntOyF7TlJSUlRXFycww0AAAAAgBtNngXvpKQkeXl5OUyz2WxKSkrKse1q2nMyduxY+fn52W/BwcF5syIAAAAAAOShPAveNptNycnJDtOio6Pl5eWVY9vVtOfklVdeUWxsrP0WHh6eNysCAAAAAEAeyrPgXaZMGYWFhTlMCwsLU4UKFRQQEKCzZ88qLS0tx/aclr2wPSceHh7y9fV1uAEAAAAAcKPJs+DdpEkTLV++3GHavHnz1LJlS7m4uKhOnTpavXp1ju133XWX9u7dq5iYGHvbmTNntHXrVjVq1CivSgQAAAAAIN/lWfDu3bu3vvnmGy1btkzp6elavHixvvvuOw0cOFCSNGzYMA0ZMkT79u1TcnKyRo8eLZvNpubNm6tgwYLq2bOn+vbtq+joaJ0+fVq9evXSs88+e8mh5gAAAAAA3AzyLHiXKFFCc+fO1SuvvCIfHx+9+eabWrhwoYoXLy5JatOmjYYMGaJWrVqpSJEi2rFjh+bPn29f/u2331bJkiVVsWJFValSRZUrV9abb76ZV+UBAAAAAOAULsYY4+wi8kJcXJz8/PwUGxt70x3vXe7lJc4u4bZz5L2QK8+EPEU/z3/0cwAAAGvlNofm2R5vAAAAAACQHcEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAvlafCOjo7Wk08+qSJFiqhMmTL66KOP7G3//POPmjRpIpvNppo1a2rp0qUOyy5cuFBVqlSRzWZTixYtdODAgbwsDQAAAAAAp8jT4N29e3dVqlRJ4eHh2r59u9avX69vvvlGKSkpatu2rR5//HHFxsbqiy++UJ8+fbR7925J0u7du9WvXz9NmTJFsbGx6tq1q9q0aaPk5OS8LA8AAAAAgHyXp8F73bp1evXVV+Xt7a0SJUroueee048//qgFCxaoVq1aGjRokAoWLKimTZtq1KhR+uCDDyRJEyZM0EsvvaT77rtPBQsWVP/+/VW3bl3NmTMnL8sDAAAAACDf5WnwDgkJ0UsvvaS4uDiFh4drzJgxKlGihNavX68OHTo4zNulSxetWrVKkq7YDgAAAADAzSpPg/fEiRM1f/58+fn5qUyZMjpx4oTeeOMNHTt2TJUqVXKYt2jRokpOTlZKSooiIiJUoUIFh/aqVavq8OHDeVkeAAAAAAD5Ls+Cd3p6uh5++GE98cQTio6O1tGjR9WmTRudPHlSSUlJ8vLyyraMzWZTUlKSMjMz5erqmmPbpaSkpCguLs7hBgAAAADAjSbPgvfixYtls9n0/vvv289qPnr0aPXs2VMFCxbM8URpZ86ckZeXl1xdXWWMcWiLjo7OMaxnGTt2rPz8/Oy34ODgvFoVAAAAAADyTJ4F7/3796tp06YO07y9veXv7y9JCgsLc2iLjIxUkSJFVLBgQZUqVUrHjh1zaA8LC8s2/PxCr7zyimJjY+238PDwvFkRAAAAAADyUJ4F7woVKmjfvn0O05KTk/XPP//oySef1PLlyx3a5s2bp5YtW0qSmjRpctn2nHh4eMjX19fhBgAAAADAjSbPgvfDDz+sHTt2aOLEiTp37pwiIiLUvXt3NWzYUJ07d9a2bds0Y8YMpaWlafPmzRo3bpxefPFFSdJzzz2nd955R1u3blVaWpqmTp2qv/76S48//nhelQcAAAAAgFPkWfD29PTU4sWL9dNPP6lEiRKqX7++ihcvrhkzZsjT01M///yzJk+erEKFCql3796aPHmyatasKUm644479Nlnn+mpp56Sr6+vZs2apaVLl8rDwyOvygMAAAAAwCnc8/LBKleurBUrVuTYVqNGDW3YsOGSy3bs2FEdO3bMy3IAAAAAAHC6PL2ONwAAAAAAcETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBClgbvo0eP6ptvvrHyTwAAAAAAcEOzNHgPHjxYUVFR9vsbN27UXXfdJZvNprp162rbtm0O80+ePFllypSRt7e3QkNDHZYFAAAAAOBmZFnwXrp0qQ4dOqRhw4ZJkqKiohQaGqpRo0YpLi5Or732mjp06KDIyEhJ0vLlyzVmzBgtXrxYp0+fVq1atRQaGmpVeQAAAAAA5AtLgndKSoqGDBmiL774Qu7u7pKkqVOn6tFHH1WnTp1UoEABdezYUd27d9ekSZMkSePHj9dHH32k2rVry2azafTo0crMzNTq1autKBEAAAAAgHxhSfAeN26cmjVrpsaNG9unrV+/Xh06dHCYr0uXLlq1apWMMdq6datCQkJybAcAAAAA4GaV58E7PDxcY8eO1YoVK1S4cGENHz5cmZmZOnbsmCpVquQwb9WqVXX48GFFR0fL399fnp6eObbnJCUlRXFxcQ43AAAAAABuNHkevEePHq1WrVrpjz/+0I4dO7RhwwZNnDhRSUlJ8vLycpjXZrMpKSkpx7YL23MyduxY+fn52W/BwcF5vSoAAAAAAFw397x+wEWLFmnfvn0qXLiwJOnrr79W165dZbPZlJyc7DBvdHS0vLy8cmy7sD0nr7zyiv3EbZIUFxdH+AYAAAAA3HDydI/3qVOn5O3tbQ/dklSzZk0dP35cZcqUUVhYmMP8YWFhqlChggICAnT27FmlpaXl2J4TDw8P+fr6OtwAAAAAALjR5GnwDggIUExMjGJiYuzT9u7dqzJlyqhJkyZavny5w/zz5s1Ty5Yt5eLiojp16mQ7g3lWOwAAAAAAN6s8Dd6urq7q0aOHevTooZMnTyosLEx9+vTR4MGD1bt3b33zzTdatmyZ0tPTtXjxYn333XcaOHCgJGnYsGEaMmSI9u3bp+TkZI0ePVo2m03NmzfPyxIBAAAAAMhXeX6M93vvvacRI0aoVq1a8vb21vPPP6++fftKkubOnatBgwYpNDRUtWrV0sKFC1W8eHFJUps2bXT06FG1atVK0dHRat26tebPn5/X5QEAAAAAkK9cjDHG2UXkhbi4OPn5+Sk2NvamO9673MtLnF3CbefIeyFXngl5in6e/+jnAAAA1sptDs3zy4kBAAAAAID/R/AGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEKWBu8PPvhAp0+fliT9888/atKkiWw2m2rWrKmlS5c6zLtw4UJVqVJFNptNLVq00IEDB6wsDQAAAACAfGFZ8P7rr7/0+uuvS5JSUlLUtm1bPf7444qNjdUXX3yhPn36aPfu3ZKk3bt3q1+/fpoyZYpiY2PVtWtXtWnTRsnJyVaVBwAAAABAvrAkeKempuqpp55Senq6JGnBggWqVauWBg0apIIFC6pp06YaNWqUPvjgA0nShAkT9NJLL+m+++5TwYIF1b9/f9WtW1dz5syxojwAAAAAAPKNJcH7tdde0x133KHSpUtLktavX68OHTo4zNOlSxetWrUqV+0AAAAAANys8jx4r1u3TvPmzdPEiRPt044dO6ZKlSo5zFe0aFElJycrJSVFERERqlChgkN71apVdfjw4bwuDwAAAACAfOWelw8WFxenHj16aNq0afLz87NPT0pKkpeXV7b5bTabkpKSlJmZKVdX1xzbLiUlJUUpKSkOfxsAAAAAgBtNnu7xfv7559W5c2c1b97cYbrNZsvxRGlnzpyRl5eXXF1dZYxxaIuOjs4xrGcZO3as/Pz87Lfg4OA8WQcAAAAAAPJSngXvRYsWafbs2frss8/k6ekpT09PHT16VKVLl9aePXsUFhbmMH9kZKSKFCmiggULqlSpUjp27JhDe1hYWLbh5xd65ZVXFBsba7+Fh4fn1aoAAAAAAJBn8ix4d+jQQWlpaUpOTrbfypYtq+PHj+vdd9/V8uXLHeafN2+eWrZsKUlq0qTJZdtz4uHhIV9fX4cbAAAAAAA3Gsuu432hTp06adu2bZoxY4bS0tK0efNmjRs3Ti+++KIk6bnnntM777yjrVu3Ki0tTVOnTtVff/2lxx9/PD/KAwAAAADAMvkSvD09PfXzzz9r8uTJKlSokHr37q3JkyerZs2akqQ77rhDn332mZ566in5+vpq1qxZWrp0qTw8PPKjPAAAAAAALJOnZzW/2JEjR+z/r1GjhjZs2HDJeTt27KiOHTtaWQ4AAAAAAPkuX/Z4AwAAAABwuyJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAHCbmzNnjg4cOODsMgAAuGURvAEAuEXFxMSoe/fuKlasmMqVK6f33ntPmZmZDvNERERo6NChKlCgwCUfZ8uWLVq4cKHF1QIAcOsieAMAcIvq0qWLSpUqpUOHDumXX37RwoULNWHCBId5hg4dqqFDh6pcuXI5PkZCQoKeeuoppaen50PFAADcmgjeAADcgnbu3KmTJ0/q3Xffla+vrypXrqxp06bpq6++ss+zatUq7d27V8OHD7/k4wwbNkxhYWH5UTIAALcsd2cXAAAA8l5GRoaGDRsmFxcX+7Tg4GBFRUVJktLS0vT888/riy++uOQw8yVLlmjt2rXq1KlTvtQMAMCtij3eAADcgurUqaOePXs6TFu8eLHq1KkjSfr000916NAhPfLII6pcubJWrlzpMO/p06fVv39/zZw5U15eXvlWNwAAtyKCNwAAt4HIyEgNHz5co0aNUkJCgt577z1NnDhRhw4d0gcffKBu3brpxIkT9vmfeeYZ9e3bV/Xq1XNi1QAA3BoI3gAA3OJSUlLUuXNnDRw4UPXr19fKlSvVokUL9e7dW97e3urYsaO6d++umTNnSpKmT5+uEydO6LXXXnNy5QAA3Bo4xhsAgFtc7969FRwcrFdffVWSdPDgQdWqVcthntq1a2vr1q06duyYhg8frvj4eHl7e0s6fzz4nDlztHXrVn3wwQf5Xj8AADc7gjcAALewN998U4cPH9bq1avtJ1orXbq0fvzxR4f5du/erfLly6tMmTKKjo52aOvRo4fatWunLl265FvdAADcShhqDgDALWr27NmaOXOmFi5cKE9PT/v09u3ba+vWrfr666+VmJion376SXPmzFH37t2dWC0AALcu9ngDAHALWr9+vXr16qW0tDQFBQU5tB06dEgrVqxQv379NHToUNWqVUuLFy9WsWLFnFQtAAC3NhdjjHF2EXkhLi5Ofn5+io2Nla+vr7PLuSrlXl7i7BJuO0feC3F2Cbcd+nn+o5/nP/p5/qOfAwCcKbc5lKHmAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYKE+Dd0xMjLp3765ixYqpXLlyeu+995SZmSlJ2rhxo+666y7ZbDbVrVtX27Ztc1h28uTJKlOmjLy9vRUaGqqoqKi8LA0AAAAAAKfI0+DdpUsXlSpVSocOHdIvv/yihQsXasKECYqKilJoaKhGjRqluLg4vfbaa+rQoYMiIyMlScuXL9eYMWO0ePFinT59WrVq1VJoaGhelgYAAAAAgFPkWfDeuXOnTp48qXfffVe+vr6qXLmypk2bpq+++kpTp07Vo48+qk6dOqlAgQLq2LGjunfvrkmTJkmSxo8fr48++ki1a9eWzWbT6NGjlZmZqdWrV+dVeQAAAAAAOEWeBe+MjAwNGzZMLi4u9mnBwcGKiorS+vXr1aFDB4f5u3TpolWrVskYo61btyokJCTHdgAAAAAAbmbuefVAderUUZ06dRymLV68WHXq1NHRo0dVqVIlh7aqVavq8OHDio6Olr+/vzw9PbO1z5kz55J/LyUlRSkpKfb7cXFxebAWAAAAAADkLcvOah4ZGanhw4dr1KhRSkpKkpeXl0O7zWZTUlJSjm0Xtl/K2LFj5efnZ78FBwfn+ToAAAAAAHC9LAneKSkp6ty5swYOHKj69evLZrMpOTnZYZ7o6Gh5eXnl2HZh+6W88sorio2Ntd/Cw8PzfD0AAAAAALheeTbU/EK9e/dWcHCwXn31VUlSmTJlFBYWpjJlytjnCQsLU4UKFRQQEKCzZ88qLS1NBQoUyNZ+KR4eHvLw8LCifAAAAAAA8kye7/F+8803dfjwYU2fPt1+orUmTZpo+fLlDvPNmzdPLVu2lIuLi+rUqZPtDOZZ7QAAAAAA3MzydI/37NmzNXPmTG3evNnhZGm9e/fW3XffrWbNmqlVq1Zavny5vvvuO+3YsUOSNGzYMA0ZMkQ//vijypUrpw8++EA2m03NmzfPy/IAAAAAAMh3eRa8169fr169eiktLU1BQUEObYcOHdLcuXM1aNAghYaGqlatWlq4cKGKFy8uSWrTpo2OHj2qVq1aKTo6Wq1bt9b8+fPzqjQAAAAAAJwmz4J306ZNHS7vdbGyZctq165dl2zv16+f+vXrl1flAAAAAABwQ7DscmIAAAAAAIDgDQAAAACApQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAALjlvPbaa3Jzc5O7u7vc3d1Vq1YtSdKqVat01113ycfHR02aNNHOnTudXCmA2wHBGwAAALecPXv2aPXq1UpPT1d6err+/vtvHT16VI8//rg++OADnTlzRkOHDtXDDz+shIQEZ5cL4BZH8AYAAMAtZ8+ePapWrZrDtNjYWE2fPl2tWrVSwYIF1blzZxUtWlT//POPk6oEcLtwd3YBAAAAQF5KTk5WTEyMSpQo4TC9du3aql27tiTp3LlzmjlzpmJjY1WjRg1nlAngNkLwBgAAwC1l3759SkhIUHBwsIwx6tatm9555x0VKFBA0vk93/7+/pKkyZMny8vLy4nVArgdMNQcAAAAtxRvb2/9+uuv+vfff7V582bt2bNHb775pr3dz89PZ8+e1bx58/Tmm29q3bp1zisWuAZhYWFq06aNChUqpKpVq+q7776zt61du1Z16tSRj4+PGjZsqN9//92JlSILe7wBAABwS6lcubIqV64sSQoODtbMmTNVtWpVvfvuu/Z5/Pz81LlzZ6WkpGj8+PFq1qyZs8oFrooxRu3bt9dTTz2lRYsWad++ferYsaPKlSunoKAgde/eXbNmzVKDBg20fft2Pf3001q/fr0CAgKcXfptjT3eAAAAuKUVLlxYxhj9+eefSkxMdGi78847deLECSdVBly9s2fP6tlnn9VLL70kDw8P3XnnnerQoYM2bNigbdu2qVmzZmratKkKFCigRo0a6d5779X69eudXfZtj+ANAACAW8Yff/yh/v37O0zbt2+fPD09NWHCBM2ePduhbfv27ZxcDTeVwoULq1+/fpKk1NRU/fLLL5o/f75atGihxo0ba/369Vq2bJlSU1O1cuVK/fTTT9lONIj8x1BzAAAA3DJq1Kih5cuXa9KkSerVq5f279+vp59+WiNGjFDTpk3Vvn171axZU/fee682bNigUaNG6ZdffnF22cA1qV+/vnbt2qUnn3xSd999t1xcXDRu3Di1bdvWPs/gwYPVsGFDJ1YJiT3eAAAAuIXYbDYtWbJEc+fOVUBAgNq1a6fHH39cgwYN0p133qnPP/9c/fv3l7+/v1544QVNmzZNVatWdXbZwDXZvHmzNm/erP3792vMmDH6448/9MILL2jJkiVKTEzUypUr5e7urpSUFGeXettjjzcAAABuKTVr1tSaNWtybGvbtq3D3kDgZubp6akGDRroxx9/VO3atbV9+3aNGTPG3sdbtWqlmJgYvfbaa/rwww+dXO3tjeANAACAXCv38hJnl3DbOfJeiLNLwA3kyJEj8vDwUMmSJe3TSpUqJTc3N/3555/69NNPHeavX7++Ro8eTfB2MoaaAwAAAMBNYuPGjXrhhRccph05ckSFChVSjRo1tG/fPoe2HTt2KDAwMD9LRA4I3gAAAABwk+jYsaO2bdumb775RklJSdq3b58ee+wxvf766xo6dKiee+45bdmyRSkpKVq3bp0GDx6sQYMGObvs2x5DzQEAAADgJuHt7a3Fixfr2Wef1aBBg1SkSBG98MIL6tGjhyRp5MiReuqpp3T8+HFVqVJF48aNU8eOHZ1aMwjeAAAAAODgpjiXwb1DVPTeIZKkj/+TPrbXHCB1/kQlJMVKenmX9PKuG399bvVzGTDUHAAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAvdUME7Li5O3bp1k4+Pj4KCgvT+++87uyQAAAAAAK7LDXU5sWeeeUaenp6KiIjQ2bNn1aVLF/n7+6tfv37OLg0AAAAAgGtywwTviIgIrVmzRkePHpWnp6f8/Pw0Y8YMPfTQQwRvAAAAAMBN64YZar5x40a1atVKnp6e9mnVq1eXj4+PDhw44MTKAAAAAAC4djdM8D527JgqVaqUbXrVqlV1+PBhJ1QEAAAAAMD1u2GGmiclJcnLyyvbdJvNpqSkpGzTU1JSlJKSYr8fGxsr6fwJ2m42mSmJzi7htnMz9pObHf08/9HP8x/9PP/Rz/Mf/Tz/0c/zH/08/92s/TyrbmPMZee7YYK3zWZTYmL2Dh4dHZ1jIB87dqzeeuutbNODg4MtqQ+3Fr9PnF0BYD36OW4H9HPcDujnuB3c7P383Llz8vPzu2T7DRO8y5Qpo2XLlmWbHhYWpgoVKmSb/sorr2jYsGH2+5mZmTpz5owCAgLk4uJiaa04Ly4uTsHBwQoPD5evr6+zywEsQT/H7YB+jtsB/Ry3A/p5/jPG6Ny5cwoKCrrsfDdM8G7cuLFeeOEFpaeny939fFl///23UlJScjz228PDQx4eHg7T/P3986NUXMTX15c3Nm559HPcDujnuB3Qz3E7oJ/nr8vt6c5yw5xcrVSpUmrUqJGGDh2q+Ph4hYeHq2/fvho5cqSzSwMAAAAA4JrdMMFbkr788kudOXNGgYGBqlevnjp16qTevXs7uywAAAAAAK7ZDTPUXDq/i3727NnOLgO55OHhoTfeeCPbkH/gVkI/x+2Afo7bAf0ctwP6+Y3LxVzpvOcAAAAAAOCa3VBDzQEAAAAAuNUQvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbxxzRISElS7dm2lp6fbp61du1avvfaawzTgZhYfHy9fX1+HPr106VJ169ZNqampTqwMuDqxsbEaPHjwJdu9vb116tQp+/2EhIT8KAsAgNsClxPDNUtPT5enp6eSkpI0Z84cde/eXWvXrtUDDzyg2NhY+fj4OLtE4LplZGTIw8NDCQkJGjdunF5//XWtX79ezZs3p5/jphIXF6eaNWsqPDxcU6ZMUWJiolxcXGSz2dS3b18VLVpUx44dk5eXl6KionTPPfdo06ZNKlu2rLNLB/JEenq6nnnmGX366acqVKiQdu/eLVdXV9WsWdPZpQHXbenSpfLy8pKra/b9qi4uLgoMDFTlypWdUBmyELxxRQkJCXrwwQfl4eEhSTLGyM/PTwsXLpSPj4/OnDmjwMBAnTlzRrt27dK9996ruLg4eXt7O7lyIPfi4+NVtWpVh34eEBCg33//XYUKFbL38+joaP3555+655576Oe4qaSkpKhWrVo6cOCAqlWrpqZNm0qS1qxZo4MHD6ps2bI6evSoJGnixIn6+uuvtWPHDmeWDFy1lJQUff7557LZbHJzc1NiYqJCQ0P1wAMPaMeOHfL399fZs2fl4+OjJ554Qps2bdKBAwdUoEABZ5cOXJNjx46pTJkyKlmypOrWrStjjH777Tfdd999+uOPP1SrVi0VLFhQ69ev1+7du1WmTBlnl3zbYqg5rsjDw0Ph4eHq3bu3Dh48qD59+ujff/+VJHl6eqpgwYJyd3e335ekggULOq1e4FrYbDZ5enpq5syZcnFx0axZs5SYmCjp/HugQIEC9n6e1b+zQjpwM3B3d3fow1OmTNGUKVPk5uZmn9a+fXtFRUVp/vz5euGFF5xZLnBNjDEaNmyYZs+erW+++UbDhg1TSkqKDh8+LE9PT3l4eMjDw0M///yz5s+fr6+++orQjZtaw4YNNWvWLHl5eemnn37Szz//rNKlS+vnn39W/fr1NX36dC1evFg9e/a0f6+Bc7g7uwDc+Nzd3eXv769u3brp/fffV7du3TR27Fh7myT7h1ZaWpoKFCjAhxhuOm5ubvL29lbjxo3t/2YN17o4cKempsrT09M+HbgZuLm5KSoqSgMHDlRkZKQGDhwoSQ79vEWLFmratKliYmLUuXNnZ5YLXBMPDw+5ublp3bp1kiQvLy+HH09dXV01bdo0jRw5UnPmzFHLli2dXDFwfWw2mxYtWqSjR4/q7bffliSdPn1ab7/9tvbv36+PP/5Yvr6+8vLyUrVq1Zxc7e2Nb424Jllf1FJSUvT2228rPj5eb7/9tiIiIhQYGOjk6oC8kdXPk5KS1KtXL8XExKhXr146deqUSpYs6eTqgKtns9nUuHFjLV26VI0bN5YxRmvWrJF0/hjAIUOG6Ny5c1q6dCkjOnBTcnFxkYuLi8M0Nzc3ubq6auTIkUpLS9PGjRu1bds2lStXzjlFAnng999/188//ywPDw/NnTtXZcuWtY9gcnFxkZubm/1fV1dXpaSkOLliMNQc1yTr1ACZmZk6cOCA2rdvrwMHDmjXrl268847nVwdkDey+rmrq6tKlSqloUOHqlSpUkpNTVWdOnWcXB1w9fz8/NStWzcVLlxY3bp105NPPilJGjJkiOLi4iRJ//33n/766y/t3LnTmaUC1+zi0xdlZmbKGKNDhw4pIyNDixYt0qxZs7LNB9xM0tLStHLlSh09elTHjx+Xi4uLevXqpR49esjPz089e/ZU2bJl9eyzz2rkyJH66KOPnF3ybY893rgmWR9WNptNM2fOtE9v27at/YsccLPL6ucFCxbU6NGj7dMaN26sYcOGObM04JocOXJE9erV04EDB1SvXj379OLFiysmJkbLli3Tb7/9pr59+2revHm6++67nVgtcPUyMzOz7fFOT09XRkaGvv32W/30009au3atnn76aR04cEDffPONkyoFrk/Dhg21efNmjR8/Xm3atJG7u7saNGhgH63XsGFDnTlzRlWrVtUDDzygt99+m226k7HHG1eUkZGh1NRU/fvvv/Z/s65pnJ6ert27d+uff/7Re++9p+TkZD3yyCNOrhi4ehkZGYqPj9fKlSvt/2b184yMDG3atEkrVqzQoEGDVL58ebVu3drJFQNXLzAwUDNnzlSZMmU0a9YszZgxQ8YYvfrqqwoKCtLIkSPl6empDh06aOPGjc4uF7hqycnJSk9Pdxhee+7cOaWlpSk1NVVpaWmqVq2a1q5dqzVr1mjy5MnOLhm4ZqdOndK+ffs0adIkHTx4UEePHlVYWJjCwsJ09OhRnTt3Tv/884/q1aunFi1aaPv27c4u+bbGHm9cUXJysv79919Vr15dxhhVq1ZNpUuXlnT+GO+dO3eqX79+8vHx0ZYtW5xcLXBtkpKSlJycrH79+kmS+vXrpyJFikg6/x7IOhlVRkaGli9f7sxSgWuSmZkpd3d3Va1a1X7sn4uLizIzMyWd/yF1/fr1+uWXXxQUFKQ///zTyRUDV8/Dw0O7d++29/GMjAyVLl1an332mVJTU5Wenq6EhATZbDZNnDhRPXv2VLdu3bg0JG5Krq6u+umnn/Tll19KkurXr6+oqCiHa3mnpKSoTZs2mjdvHuc1cDKu441cyTpbeZbU1FS5u7vL09NTqampio6O1vjx4/X5559r5MiRGjp0qBOrBfJORkaGPD09lZaWpoyMDE2bNk0vv/yyunXrpnHjxnECKtw0kpOTVb16dYWFhalZs2aKiopSWlqaSpQooc2bN6t06dI6cuSI3N3ddfr0aRUvXlzR0dEqXLiws0sHrtr333+v0NDQbJc3jYqK0oYNGzR48GAtW7ZMe/bs0WOPPeakKoHrEx8frzJlymj69Om6//779eCDD+q+++7Tc889J+n84XH16tXT3r175e/v79xiQfDGtUtMTJSPj4/S0tLsZ1HcvHmzevXqpQ0bNiggIMDJFQLXLz4+Xr6+vvYfmyTp0KFD6t27t5YsWcJeEtw0zpw5o4oVKyomJibHdj8/P4WHh8vX11fnzp3TjBkz9Mwzz3B5SNxUjh07ps6dO2vXrl2aN2+eEhMTNXbsWHl6etrnMcbo33//lbu7u3744Qc98MADTqwYuHZxcXEKCgpSkyZN9Oeff+rcuXOqXbu2wwlgp02bpp49e0qS/ve//zmrVIih5siljIwMRUREKDg42H7SEk9PT+3evdthOEvDhg31xx9/yMvLy1mlAnnKx8dH586dc7hmd8WKFbVmzZpsJ/ABbmRFihS5ZOiWpO+++042m02SVKhQIQ0aNCi/SgPyTGBgoOrUqaPvvvtOFStW1Jw5c5SYmKjhw4c7zJeRkaGtW7faD7UAbkYJCQny8PDQ8uXLlZGRodWrV2vChAmaMGGCQkJC1LFjR33yySeKj49XdHS0s8u97bHHG7kSHh6ucuXKKTIyUsWKFXNo69Onj+644w4NHjzYSdUBAABkt2LFCi1evFifffaZs0sB8lxycrJ27NihRo0aOUxfs2aNZs2apc8++4ydYTcQgjdy5cyZMypatKjOnTvnMLS2f//+mjZtmj7//HP16tXLiRUCAHLj1KlTuvfee/Xnn39e8vjt8PBw9e7dWytXrszn6gAAuDVxOTHkioeHh1xcXBxOUvL8889r3rx5+vXXXwndAHATOHfunGw2m44fPy6bzabw8HCdPn1ap06d0rFjx3Tq1CmNGTNGbm5u+v33351dLnDNkpKStGDBAklSmTJlFB4e7uSKAOc4efKk7rnnHu3fv9/Zpdz2CN7IlayT62Qd57pgwQJNmzZNS5cuVdOmTZ1ZGgAgF7755hu98MIL8vT0lIuLizw9PVWlShWVKFFCgYGBKl++vBYtWqSxY8fK19fX4WRUwM0mLS1Nzz77rCSpYMGCKlq0qDZu3KjWrVurbdu2atu2rVq1aqV27do5uVLg2u3atUu+vr7677//JEmxsbF66aWXdPr0afs8NptNu3bt4rw0NwCCN3LFzc1NxhjNnj1bktShQwf9+uuvqlevnpMrAwDkxr///mu/lnfWSTHLlSunjIwM1atXT1OnTpW3t7cCAgLk4eHhcEJB4GaRkZGhmjVrqn379oqJiVGLFi104sQJeXh4yGazKSYmRtWqVVOLFi20c+dOvfrqq84uGbhmXl5eio+Ptx8GWrBgQX300UdKTEy0z5P1I+rFl9ZD/uNTFVdl8ODB+vjjj1WrVi1J0sSJEx3a09PTdfbsWS1dutQZ5QEALqF27dpasmSJpP//AnbhdeizDicqVKiQ3N3dCd64KaWlpWnkyJHy9vbWwIEDNWLECPXu3Vuurq6y2WwKCAhQtWrVVLFiRRUsWDDbSamAm0nWCKasK1LYbDYZYxy231n/z7r0L5yHPd7INRcXF+3fv1/16tXT7NmztWXLFqWlpTncUlNTlZqa6uxSAQAXqVChgsLCwrRu3TpJ0rp165SYmKh169YpLi5O+/bt04EDB5ScnKz169c7XCoSuFl4enqqRYsWCgkJkZeXl1q3bm0/qzNDbXGryToUNOtf6Xw/v7CvZ/2fbbrz8XM2ciXr5PdFixbVF198oSZNmmjQoEHq06ePHnjgASdXBwC4khIlSmjnzp0aPny4UlNTNXz4cEVERGj48OE6duyYvvvuO7m7u+vEiRMaPnw4X9JwU0pLS1PJkiXl7u6ujIwMFShQQJmZmUpLS1NycrKSkpIUHR0tHx8fZWRkKDw8XMHBwc4uG7gmrq6uMsY4fBc3xqhLly7ZRjTB+QjeyJX09HRJ54+dcnNz05NPPilfX1+Fhobql19+Uf369Z1cIQDgcooUKaIyZcpo+/bt8vHx0fbt23XXXXdp+/btatiwoQYMGKDChQvrtdde0/bt21WxYkVnlwxcNVdXV4WHh9vPXXDo0CFVrlxZ8fHxio6OVkREhGbMmCFJKly4sBo3bqxjx445uWrg+lz4Pfzi7+TGGP3222+E7xsAwRu5kpaWJul8AM86RqR9+/Z69tln9cgjj1z2erAAAOez2Ww6e/aspPM/ol74b5bMzEwlJibKGKPMzMz8LhG4bm5ubipVqpSSkpI0efJkeXt7q0+fPvYg/uGHH6pNmzbauHGjWrRo4exygetijJGLi4vGjh172fnGjRtnH70K52EcGXIlOTlZkpSSkuIwffTo0bLZbOrevbszygIA5JKbm5uSkpJkjLH/mHrmzBmNGjVKx48f14IFC5SUlKSYmBilpKTYRzoBNyNjjHr27KmIiAglJCSoaNGiysjI0JAhQ+Th4aExY8Zo7ty5zi4TuC4XjkiVpM2bN+vYsWPZbi4uLgTvGwB7vJErqampqlGjhv3LWhY3NzeNGjVKy5YtU2ZmJscEAsANKi0tTZmZmUpOTpYxRhkZGerRo4dcXFzUq1cvJScn24+NjY+Pz/ZDK3AzyMzMVIsWLfTSSy/Zp23YsEEzZsxQRkaGkpOTNWPGDN17773q3bu3atSooZo1azqxYuDaZW2nU1JSlJKSosaNGzuE7Kz/u7i4ZBvhhPxH8EaulCpVSrt3786x7YknnlDXrl0J3QBwAzPGaN68eUpISJAxRgkJCXr33XezzRcVFaWjR48qISHBCVUC1yc+Pl533323unbtKjc3NyUkJCg8PFw//vijjDGKj4/Xjz/+KEkqWbKkvvzyS/3vf/9zctXAtYmPj7f36yJFiujgwYMOJ1WTzv/oWrFiRUYx3QBcDOMOAAC4LWRkZGj//v3asGGDunfvnu0LWpaEhARt3LhRDz74YD5XCOSN48ePq3fv3oqNjVW/fv3Us2dPnThxQnXr1tXx48clnR/Nl3VNe+BmlJCQoL///lv33HOPwyXFLhQdHa1ixYrpr7/+Uq1atfK5QlyI4A0AwG0iPDxc5cqVU2RkpIoVK+bQ1qdPH91xxx0aPHiwk6oD8t6nn36q3r17y8fHR8nJydq/f7/uvPNOZ5cF5Ju0tDT9+++/qlSp0iV/bEX+YGwwAAC3CW9vbxlj5OXl5TC9f//+mjlzpgoVKuSkygBrDB48WD4+PpIkT09PQjduOwUKFFDNmjUJ3TcAgjcAALcJDw8Pubi4OAyvff755zVv3jz9+uuv6tWrlxOrA/JORkaGDh8+LEmKi4uTr6+vIiMjHeY5fvy4wsPDnVEekGeKFCmSbVqXLl20bt06J1SDyyF4AwBwm8g6BtDd/fy5VRcsWKBp06Zp6dKlatq0qTNLA/JUXFycKleurPT0dHl5eSk+Pt5hRMdXX32lWrVq6c0333RekUAe8Pf3lzFGx44dkyQlJSVp6dKl2a5EBOfjGG8AAG4TGRkZKlCggGbMmKEnn3xSGRkZ+v3331W/fn1nlwbkqbS0NPn4+Ngvt+Tu7q7k5GTFx8erV69eWrp0qSpVqqQ//viDIbi4aZ07d06VK1dWwYIFVbNmTS1btkzTpk3TmDFjtG3bNvXo0UNubm4yxsjf31/Tpk1zdsm3NS4nBgDAbWbw4MH6+OOP7We4nThxokN7enq6zp49q6VLlzqjPOC6JCYmysvLy+GQCldXV7m7uys8PFzp6enavHmz2rVrR+jGTSsqKkqNGzeWJC1fvlw1atRQTEyMRo0apUGDBsnFxUUbNmzQxx9/rFdeeUWTJk1ycsVgqDkAALcRFxcX7d+/X/Xq1dPs2bO1ZcsWpaWlOdxSU1OVmprq7FKBq2aMUfny5VWvXj0lJyerXr16qlevnjIyMrRmzRp5enrqp59+0t13363o6GglJSU5u2Tgmnh7e2v48OEqWLCgvvvuO507d05dunRRjx499MMPPyg6Olre3t7q3r27/Pz81LFjR2eXfNtjjzcAALeJrKPLihYtqi+++EJNmjTRoEGD1KdPHz3wwANOrg64fmlpaXr55ZdVqFAhDRo0SIMGDZIxRr1799Ynn3yiTZs2afr06QoJCVFgYKAOHjyoO+64w9llA1fNx8dHAwYM0B133KGJEyeqfPnyeu211zR06FAtXryY0Rw3III3AAC3ifT0dEnnj/V2c3PTk08+KV9fX4WGhuqXX37hWG/c9AoWLKihQ4cqPT1dgwYNUvfu3e3B+4cfftC8efP03HPPafny5SpSpIj2799P8MZNrWfPnlq8eLH279+vLVu2yBgjFxcXubi4OLs0XISh5gAA3CayznKbFcAlqX379nr22Wf1yCOPKCYmxlmlAXkqNjZWaWlpSklJsff7tLQ0devWTXv37pW7u7t27dqlLVu2OLlS4Pp4enoqMjJSH3zwgeLi4vTKK69IEocL3YAI3gAA3CaSk5MlyX6m5yyjR4+WzWZT9+7dnVEWkOdsNpuOHj0qDw8PJSUlKSgoSImJiZLOB5Xx48dr9uzZGjBggJMrBa7dypUrJUkDBgzQvn37NHfuXM2bN09169aVt7e3kpKS9NNPPyk+Pl4///yzk6sFlxMDAOA2ERERoTZt2mjt2rUKCAhwaJs9e7aWLVumGTNmyNWV3+Vxc3v66acVHR2tJUuWaP369apataq8vb31ySef6JFHHlGVKlWcXSJwXRITE1WiRAklJyfr1KlT8vf3lyQtWbJE/fv316ZNm/T444+rQIECysjIUGJion7//XfnFn2bI3gDAAAZY5Senq4CBQo4uxTguixatEh9+/bV1q1bVb58ed15553q06eP6tevr1GjRmnt2rWqUKGC2rVrpwceeECtW7d2dsnANTlx4oSGDx+uLVu2aNGiRfbzFbRq1Up16tTR2LFjnVwhLkTwBgAAwC0jPT1d+/btU61atfTXX3/p4Ycf1uHDh+Xm5iZJio6O1tSpUzVhwgT16tVLb7zxhpMrBq7P559/rpkzZ2rTpk2Szl/X++mnn9aJEyfs/R7OR/AGAADALevgwYOqVKlStulpaWlyc3Pj0ArcEs6cOaMiRYrY7//xxx+69957nVgRLkbwBgAAwC0hIyNDn3zySa4OmXB1dVXt2rXVrFmzfKgMyHvx8fFas2aNChUqdNkfkNLS0pSWlqY2bdrkY3W4GMEbAAAAtwx3d3fVq1dPHh4el50vMjJSERERiomJYTgubkqRkZEKCgrKdrLMi505c0ZVqlTRP//8k0+VIScEbwAAANwyvLy8dPDgQQUFBV12vv3796t69erat28fZznHTSk2NlaFCxdWfHy8vLy8cpwnOTlZXl5eyszMzOfqcDF3ZxcAAAAA5JUL916npKTo888/l4uLi1xcXJS1v2nQoEEqUqSINm/eTOjGTcvd3V0uLi6Szg87//bbb+0jPZKTk/X000/b+z6cjz3eAAAAuGV4e3vrwIEDCgoKUkpKimw2m5588km5ubnJGKOZM2cqJSVF7u7sf8LNLWtvdnx8vDIyMuTv769+/frJGKOpU6cqNjZWLi4u8vLyUkZGhrPLve0RvAEAAHDLKFiwoHbv3q2qVavKGCM3Nzf7UNz09HQVLFiQYbe4JWT9sBQfHy9PT095eHgoLS1NklSoUCGdO3dOKSkpBO8bBNdPAAAAwC0hPT1dJUqUUFJSkiTZh9lmDbVl2C1uVa6urg59+8JDK3BjYIwNAAAAbgnu7u4KDw+379FOTU2VMUYvvfSSChQoYJ+enp7OUHPc9C4M1snJycrIyNATTzwhY4xSUlKUlJTEGftvIGxxAAAAcEuIj4/XW2+9pVWrVmn79u3KyMjQ66+/Lg8PD7m6uiozM1MjR45kqDluCRkZGfa93MYYTZgwwX528/vvv/+y1/ZG/uMYbwAAANwS+vXrp99++00vvviiunXrJk9PT2eXBFgmNjZWRYoU0d69e2Wz2XKcJyUlRdWqVdORI0cUHByczxXiQgRvAAAA3BJiYmJUqFAhubu7a9u2bXrooYfk7e192WWOHTuWT9UBeeu///5T6dKlc3U8t4uLCydYczKGmgMAAOCWULhwYfv/y5UrpxkzZlzyWO6UlBTFxMTkV2lAnitevLhOnz4tHx+fyx7LnZqaqjNnzuRjZcgJe7wBAAAAALAQR9wDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAHCBuLg4jR8/3tllALiFELxxw/jjjz9UrFgx/ffff7mav1y5cjp9+rSlNb355pv68MMP8+Sxli5dqvvuu0+SVLduXX322WdOqeNSjhw5olq1al3XY2zYsEH169eXr6+vGjRooC1btuR62V9++UWurq5yd3e336KionKcN+sstRfeXF1d1aFDB0nWPl95+djNmzfX77//ftl5Tp8+rWLFisnFxeWK816vrPdUXvQFK2VkZKhZs2YqXLiwVqxYkeeP7+Pjk+ePeTn16tXTmjVr8vVvXqtJkyape/fuSklJUZkyZbRo0aJcL9ujRw/NmzfPwuqktWvXql27dtf1GAsXLlTt2rXl6+urli1bav/+/bledsqUKXJzc7Nvl3LTl44ePZpte+bm5qbAwEAlJSVd0zplvU5S/nxWXqxr166aNm3aZed56aWXVKhQIb311luSru61y+28MTExuuOOO1SsWDH9/fffkqTVq1erQIECV3xtRo0apQMHDuSqntzKyMjQO++8o3LlyqlYsWLq1auXYmNjc7Xs5s2bVaxYMVWrVi1X80+fPl3PPvuspEu/9y78XpIb+fEelhxrt4pV27AjR47o4Ycflq+vrypXrqxZs2Y5tK9du1Z16tSRj4+PGjZsaPnnOm4sBG/cMAoVKqQqVarIZrM5uxRLzJ07V4888ogkqXLlyipevLiTK8pbBw4cUNeuXfXuu+8qKipKL730kjp27Kjjx4/navk9e/bojTfeUHp6uv1WokSJHOd9+umnHeZLTU1V9erVNXLkyLxcpRtC0aJFderUqav6cnSr+/PPP5WQkKDIyEg1aNDA2eVclyNHjujYsWNq1qyZs0vJlaztmLu7u6pUqaIiRYo4u6Q89dtvv2nYsGGaOnWqIiMj1bVrV7Vu3Vrx8fG5Wn7Pnj2aOnWqfduUm+XKli3rsD1LT09X37599cILL1zz5+GFnzf5LTExUb/88os6dux42fkmT56sI0eOWBqw1qxZo8qVK+vo0aOqUKGCJKlFixZKS0u77HK7d+/WvHnzNGbMmDyt591339WGDRu0ceNG7d+/X25uburZs2eulv322281cuRIbdiwIc/qcWY/cTartmHt27fXww8/rJMnT2rlypX64osv7D+sHjt2TN27d9f48eMVExOjjz76SE8//bSio6PztAbcuAjeuGFUqVJFGzdudLgG560iLS1NP//8szp37ixJmjNnjh599FEnV5W3JkyYoGHDhqlly5ay2Wzq1KmTunfvrh9++CFXy+/ZsyfXv+RfbP78+SpTpozq1q17Tcvj5hIXF6eSJUvKw8NDfn5+zi7nusybN0+hoaGXvf7qjSIqKkp//fWXHnzwQbm5uenXX39V06ZNnV1Wnvr44481btw41alTR15eXnrmmWdUv359rVy5MlfLX892LEtERIR+/vlnDRgw4JqWv/B1coalS5eqQYMGV/wsT09PV0BAgAICAiyrJS4uTkFBQfLy8pKXl1eulxs0aJDGjRsnf3//PK3n008/1TfffKNSpUqpSJEi+vzzz7V169ZcXV85Li5OpUqVUtGiRfOklou/l9xurNiGnT59WjExMXrmmWfk6emp8uXLq2fPnva96tu2bVOzZs3UtGlTFShQQI0aNdK9996r9evX51kNuLERvHHDOH36tMqVK2cf5rp37141b95cXl5eqlOnjnbs2CHp/Bcbd3d3HT16VIGBgRo8eLD9MWbNmqUaNWrIw8NDFStW1EcffaSsS9WvXbtWISEhevLJJxUYGKiEhAQ1b95cGzdu1NChQxUQEKASJUro448/zrG+Hj16aNasWRowYIB8fX1VoUIFLVmyRKmpqXr++edVuHBhVaxYUcuWLcu27K+//qoaNWqoZMmSkqR27dpp7dq1knTFGho0aKC3335bI0aMcPgScODAAT388MPy8fFR4cKF1a1bN504ccLeXq5cOX311Vfy8/PTDz/8YB+6tXDhQtWoUUM+Pj7q2LFjjr+0Zg3jmzt3ripWrChfX18999xzyszM1I8//qhq1arJ399fgwcPVkZGhiSpfPny9qHeWYKDgy85XPxie/fuVdWqVXM174WMMXr33Xf1+uuvZ2ubOnWqqlWrJk9PT91555368ccfc/WYK1asUOPGjeXl5aWiRYvqiSee0JEjRxzmmTx5sipUqCBfX1/16tVLycnJDu2fffaZKlWqJJvNprp162rVqlXq0qWL1q5dq4SEBLm7u+u3335TgwYNsj1vuXU9ffJqpKen64033lBwcLC8vLzUvHlz7dq1S3Xq1LE/L82bN9cvv/yirl27ysfHR3fccYe2bt2quLg4PfHEE/L19VXt2rW1fft2h8detWqV6tWrJ09PT1WsWFETJkzQhAkT9Oabb+ZYyx9//KEHHnhAS5cutT+HkpSQkKDnn39exYsXl4+Pjx5++GGFhYVl+5KampqqN954QxUqVJCnp6eqVKmi999/X+np6fbHzHp9Lnwffvnll6patao8PDxUvXp1TZ8+3d528bZlyZIlatOmTbbaQ0NDsw1TvHCPU2Zmpp555hn5+vqqTp06+uuvv+zzHT58WB07dpSPj4+KFSum559/Xtu2bVPz5s0lScOHD9cHH3zg8NhnzpxRUFCQEhMTJUnnzp3TwIEDVaxYMXl5een+++/X1q1b7fOfPHlS999/v7y9vdWlSxfFxMQ4PN6CBQsUEhKiggULSpJq1aplf/3LlSungwcPqnv37vL19VXZsmU1e/Zs+7KBgYGaMWOGHn30Ud1111326du3b1fz5s1ls9lUokQJDRgwQHFxcfZ2Hx8fTZo0Sd7e3tq6dav9UI8rvf+k/x+uOmHCBHvQydqD+eWXX6ps2bIqWrSow17NO+64Qw888IDD4+THduxC48aN06BBg+Tt7e0wfdOmTWrcuLFsNpvKlCmjd999V5mZmdmWv/h1yqn9zjvvlIeHh8qXL68PPvjA/jmZ9fn7yiuvyMfHR+Hh4ZKu3P8udPFe1DfffFOFCxdW1apV7Z97/v7+9vdZ1ufbhe/rP/74I1fP1eWekwULFqh3796aNGmS/TtDbsycOVMeHh564okncjW/9P/P2549e9SyZUt5e3urZMmSGjFihFJSUiRJ8fHx6tOnj8NILnd3d5UsWfKK/Wvw4MH290+lSpUk5Xyo0tUM17/4e0mW2bNn6+6775anp6dKliyp/v37OxyqkJmZqXfffVdBQUEqXLiwRowY4dAP09LS9Nprr6lUqVLy9PRU3bp1c/wM+vfff9W5c2cVLlxYhQoVUosWLez942IvvfSS6tevrzNnztjXcf369WrQoIFsNpvKly+vcePG2ftxlt9++02NGjWSp6engoKC9PLLLys1NdXenhfbsAsVLVpUwcHBGjdunJKSkrRv3z7973//s7/mjRs31vr167Vs2TKlpqZq5cqV+umnny45ug+3IAPcIE6dOmXKli1rwsLCTGBgoKlTp47ZsmWLSUxMNFOmTDGlS5c2qamp9vnLli1rTp06Zb//5ZdfmsaNG5s///zTpKammt27d5vWrVubF154wRhjzJo1a4zNZjPvvPOOSU5ONsYYc99995m77rrLfPjhhyYxMdH89ddfpkKFCmbp0qXGGGPeeOMN88EHHxhjjOnevbspX768+d///meSkpLMr7/+aooUKWJ69OhhJkyYYJKSksxvv/1mihUrZuLj4x3WrWfPnubTTz+13w8JCTFr1qzJVQ0X12GMMeHh4aZ8+fLmu+++MwkJCeb06dPmvffeM1WqVDGxsbH256dJkybm9OnTJiMjw0ybNs3UrFnTtGrVyoSHh5szZ86Yvn37mq5duxpjjAkLCzM1a9a0P1elS5c27dq1M8ePHzeRkZGmYcOGpl+/fqZdu3YmIiLCnDx50jRr1sxMnz79kq9p69atzQ8//JCbl9/4+fmZ8uXLmyJFipjOnTubqKioXC23aNEi06JFC4dpb7zxhqlatarp2rWr+ffff01CQoJZsmSJKV26tPn2228v+3jfffedKVu2rFm4cKE5d+6cOXPmjJkwYYIJDg42Z86cMW+88Ya54447zOOPP25OnjxpIiMjTfv27c1LL71kf4yRI0eau+66y95/N23aZOrUqWMqVqxof92NOf/ab9++PVfrmdO819Mnc5L1nrqwLxhjzJNPPmlatGhhdu/ebRISEszy5ctN7dq1TcmSJU1YWJi9vqpVq5rvvvvOpKSkmFmzZpmyZcuazp07m7lz55qUlBTzww8/mAoVKtgfd/ny5aZEiRJm7ty55ty5c+bgwYOmS5cupnr16uaNN964ZJ1r1qwxISEh9vvp6enmvvvuM48++qg5ePCgOXfunPn+++/NXXfdZby9ve3zZWZmmpCQENOxY0fz999/m9TUVLN//37zyCOPmD59+tjnu3AZY4wZPXq0qV27ttm8ebNJTEw0v/32m6lataoZP368vZ4Lty2ZmZmmYsWK5uDBg/bHOHHiRLZt2NGjR03x4sVNenq6McaYZcuWmTZt2pj4+Hjz448/mp9//tkYY8zx48dNqVKlzLvvvmtOnjxpoqKizOjRo03NmjXNfffdZ4wx5tChQ6ZSpUomIyPD/vjjxo0zQ4cOtT9HTZs2NaNGjTKnTp0yCQkJZv78+aZ06dLm999/N8YYM2LECPPWW2+Zc+fOmY8++sihfmOMuf/++82iRYvs92vWrGl//cuWLWvq1atnZsyYYZKTk8369etN0aJFzd9//22fv3v37mbu3Ln2+zt37jQVKlQwK1asMMnJySYiIsIMGzbMNGjQwP6cuLm5mY4dO9q3a1nvvyeeeMKcOnXKREZGmpCQEPv778K+MW3aNFO2bFnTo0cPEx0dbX+OBg4caHr27GnOnDljjhw5YqpVq2bWrl1rcpKenm5q1Khhtm3blmP7hWJiYoyrq6spX768KV68uOndu7c5d+7cFZe7UGRkpAkKCrKvb9Y6lS1b1tSvX99s3LjRJCYmmp07d5rGjRubAQMGZHuMi1+nCz8rZ8yYYcqWLWtWrFhhEhISzI4dO0yDBg3M4MGDjTHnPwdsNpvp16+fSUhIMMbkrv9lSUxMNP7+/iY6OtoYY8w///xj7rzzThMTE2PWrl1rpk2bZp/34vfZxe/ry8ntczJt2jQzaNCgHB/j4r9vjDHnzp0zgYGBpmTJksbX19c8/fTTJjEx8Yr1hIWFmeLFi5u77rrLLFmyxCQkJJj9+/ebDh06mHbt2l1yuaNHj5oSJUrYv5NczsXvn5w+Ey7u/1nrfvGyxmT/XmKMMe+//76pVauWWb16tUlMTDSRkZHmrbfeMjVr1jRpaWmme/fu5o477jDDhg0zsbGxJiwszNSvX99MmjTJ/hiPP/64eeaZZ8zx48dNcnKy+eWXX0zlypXNTz/9ZJ9n//79JigoyIwfP95ERkaa5ORks3jxYlO+fHmzbt06e+0ZGRmmX79+pkWLFvb30po1a0z58uXNvffea9avX28SExPNjh07TKNGjRxe61WrVpkSJUqY77//3pw7d87s27fPtGvXznTo0ME+z/Vuw3Jy4MABU6hQISPJSDJNmjRx2A58//339jZJ9vcebg8Eb9wwLgzeksyWLVsc2uvWrevw5efCLxPJycmmdOnSDkHcGGNSUlJM6dKlzcmTJ82aNWuMv7+/wxff++67z/Tv399hmYkTJ5qBAwcaY7IH70cffdRh3gcffNA89thjDtPatm1r1q1bZ7+flpZmihYtao4fP26fdnHwvlwNF9dhjDEDBw50+KDLMnToUPPxxx/bn58LQ++0adNMQECAQwCLjY01/v7+JjMzM1vw9vT0NDExMfZ558yZY2w2mzlz5ox92g8//GB69eqVrY6sv3fnnXeatLS0HNsvlJSUZBYsWGBiYmLM2bNnzeuvv24aN258xeWMOd8vLgyzxpx/vurUqeMQQowxZuvWraZUqVLZpmdJTU01pUqVMrt3787WduLECftj16hRw2RmZtrb9u3bZ2rUqGGMMSYiIsIUK1bMREZGOiwfHh5uChUqlOfB+1r65KXkFLy3bdtmKlSokO3L544dO4yrq6tD8B4xYoTDPFWqVDEvv/yyw7QaNWqYY8eO2f+/fPlyh/b09HRTp06dqwreP/zwg2nUqJHDa2LM+R9lLvx9eeHChaZJkybZXv+MjAyHH3ou/EJ+4sQJU6xYMRMREeGwzOHDh03RokVNTExMjtuWsWPHmhdffNF+f8yYMdmei48++sg888wz9vsrV640HTt2zLYeAwcOzLasMeff7xcGn3bt2pklS5bY1+nC8P/DDz/Yf2S70KJFi0z79u2NMca8+uqr9u3HxU6ePGkKFy7sEBAu/tL63nvvOSzz4osvmnHjxtnvX/yltW3btg4/MGYJDQ01CxYsMMYYI8lhu5/T+2/Pnj3299/FwePiHzvGjBljSpcu7bBdGjdunHn77bdzXO833ngj12EwMjLSrFy50pw7d86cOnXK9OrVyzzxxBO5WjbLiy++mK3vr1mzxvj4+Dh8hhhzfvtdqlQps3fvXvu0nF6nrPd1SkqKKVGihNm1a5fD48TExJigoCCzb98+ExYWZlxcXBzeD7ntf8YYM3/+fPPggw/a7//777+mTp06Dq9BlusN3rl5Tq42eL/77rumQYMG5vDhwyYyMtK0a9fODBs27Ir1ZD1vF2+j09LSzN13322WLVuWbZn09HTTokULh8/2y8nL4J3T95KoqChTsmRJ+2fdhbKmde/e3bRp08ahbfny5aZt27bGmPOfF/Xr18+2/J9//mlq165tv9+xY0fz5ZdfZpvv5MmTJj093UybNs3069fPdOvWzbRv396hP69Zs8Z4e3vbP0eyxMbGmtKlS9tf+zvuuMMsXrzYYZ7U1FRTu3Zts2LFCmPM9W/DLnb27Fn7j7JxcXFm7969ZtCgQebIkSPGGGN+//13ExwcbJYsWWISExPNypUrzfDhw3P1wwtuDQw1xw0pKChI9evXd5hWqVIlRUZG5jj/7t27FRERodKlS8vT09N+8/X1VVRUlP78809JUo0aNVSgQAGHZTt16pTrv3PxsLpixYplOz4oICDAYfj2qlWrVLVqVZUqVeqS63s1NUjnh08NGTLEYV09PT01ceJE7dy50z7fxcOhWrdu7TB80dfXVzabLccTAN15550OQ9uLFSumWrVqORy3d/G6Ztm+fbtGjBihb7/9Vu7u7pdcjyyenp4KDQ2Vv7+//Pz89PbbbysxMVG7d+++7HIrVqyQh4dHjsMdO3ToIFdXx01cvXr15OPjc8kz1e7evVtBQUE5ntE7MDDQ/v+OHTvKxcXFfr9ChQr212vTpk267777sg0dK126tBo3bnzZ9bkW19Inr8Zvv/2mTp06ZTvJ0913360qVapccy2nT5/WyZMn1bp1a4d2Nzc3denS5aprfOKJJxxeE+n8IR0X9vdVq1bp8ccfz9YvXF1dL3myw40bN6pp06YKCgpymF6+fHndc8892rZtm6Ts25ZevXpp9uzZSklJkTFGU6dOVd++fR0e4+IhuS1btpSvr69atGihPXv2OKzfk08+ma22i88T8dxzz2nSpEmSzh9nW6VKFVWsWNH+GAsWLMi2zejatat27dolSRoyZIjmzJmjfv36ZTvmdMGCBWrbtq08PDxyfJ6kq9+OrVu3TqGhodlqWrJkif3QIun8tuhCF7//Lvd3Gjdu7PC6FCtWTA0bNnTYLl3q/fHTTz9p6tSp+vrrry+5DhcqUaKEWrVqJR8fHxUtWlRffPGFVq1a5TB0/nKio6M1e/Zsh8OnstSvXz/bZ4ivr6/atWundevW2add7nXavXu3SpUqle359Pf3V7t27eyHbZQsWdLh/ZDb/idl79OVK1dW69atVb9+fW3atOlSq35NcvucXI3vv/9eX3/9tcqXL68SJUpo+vTpmjZtWrYhzDmpVKmS6tSp4zDN3d1dXbt2tT+3FxoxYoQKFCigYcOGXVOtOcnp0IOc5PS9ZMOGDWrSpInDZ12WC6dd/D6/8PPvt99+0++//57tPV2vXj3t3bvXPsx7zZo1OQ7lL1asmP18F9OmTVNiYqLmz5+frT83atRIwcHBDtN8fX0VEhKidevWKTo6WhEREQoJCXGYp0CBAnrssccuOaT9ardhF5s2bZqaNGmiIUOGqFChQqpevbpeffVV+3tl9OjRGjNmjNq2bSubzaZWrVqpXr16eu2113L9N3BzI3jjhpTTCZMKFixoP1YqJ/Xq1VNycnK2W2pqqlq2bCnp/Ib5Sn/rcn/n4pPFuLu7Zzsjpqurq9LT0+33c3PW0KupIcuff/6ZbV1TUlI0Y8YM+zwXr+/VPK/Xsq7S+bN2hoaG6uuvv1b16tUvuw6XU6FCBR0/flwjRoxwuMzOhYHkUsd2S8pW18UeffRRh8dNSEhQRkbGFZeTsj+PBQoUsD+HLi4uufqSdrGs4xCzbkuXLs3Vctf6OuXW1azP1dRycUi+lMDAQPtzcqkzmF+uxgun5/b1vVBmZuYlfzxyc3Ozf9G9+L1WvHhxNW3aVHPnztWqVatUvnx5+1mVJSk8PFyHDh1y+LHCxcVF33zzjV566SWFhoZq1apVV1y/C7Vq1UqHDx9WWFiYJk6cmO1s0Z999lmO28isY1+LFSumzZs3q0aNGmrYsKHDOSOs2o6dOXMmx+3Y6NGjJZ3/Ue7iY5Wt3mZL0s6dO9W3b18tWLDgmo+/LFCggEqXLq2IiIhczf/JJ5+oe/fuOZ6ULLf99nKv07X25dz2v+TkZC1fvlyhoaEO09955x1NmjRJAwYMcPh8ul7Xuk27nCNHjjh8bgUEBMjf3z9Xl2O7mu3+lClTtHjxYn377bfZfgjMLQ8Pj2xnZ8/tjzw59ZO8+PyTpBdeeCHH7UxaWpr9vZyenm4/P8ylVKxYUbt27cp2fhXpys91bvv6xa5m25J1PoKs26RJk7R///5sPzYHBQUpKipK0dHRObbXr1/fksti4sZE8MYN6Wo/iGrVqqWjR4/q2LFjDtPj4+M1YMAA+0Y2pzMHX83fymn5y+3RTU9P16JFi6541tCrXd8mTZro559/zjZ93Lhx9r37UvZ6rVxX6fzJm9q1a6cXX3wx1yd4SU1NVatWrZSUlGSflpKSou3btzuc+CrrVrNmTUnnf1lPSUm55Jl7582bl+1LyZYtW5SQkKDKlSvr+++/d3hcb29v3XHHHQoPD8/xxD4XhpDLPY+NGzfWunXrsv1KHhERoY0bN15yuYEDBzrU07Zt20vOe6FreZ2uRvPmzfXjjz/aT9CVZefOnfr333+vuZaAgAAFBgZq+fLlDtMzMjIcTkAWGRlpf04udV345s2ba86cOdm+TC1evNih7mbNmmnmzJnZvvBlZmbq5MmTOT52o0aNtH79+mx7gCMiIvT777+rXr16knJe9/79++vzzz/XlClT1K9fP4e2efPmqWPHjjk+Pw899JD+97//6dNPP7Wv38XXgpXO7527kIuLiwYOHKgXXnhBhw4d0kMPPWRvu9Q2Y+nSpQ6P7e7ursGDBys0NFTffvutJOnUqVPasWNHttEJF7va7Vjjxo21ZMmSbNNHjBhhP6lXfm+zpfOvbYcOHfTFF19k24N5uWUuvnxWdHS0Dh06pPLly19x+djYWE2dOvWSez+3bt2qw4cPO0w7e/asfv75Z/ulBq/0OtWuXVtHjx7VwYMHHabHx8dryZIl9kvaXfyc5bb/LVu2THXr1s3xLOUNGjTQ999/rw8//DDH2q5Fbp6TqxUYGKh//vnHfj8mJkZxcXG5OpP4oUOHsp3sLD09Xd9//71DPb/++qtGjRqlJUuWXNdVXAIDA7Ot/4IFC6643KW+lzRq1Ei//fab/b13odx+/jVp0kRLly7Nto3duXOnw8kfmzZt6nCCyiwnT560L9uiRQt9+OGHatmypfbt2+cw3+bNm7N934uNjbX342LFiikgICDb2cIzMjI0d+7cS/aPq9m2tG3b1uEze+DAgapQoUK2Wk+ePKno6Gj5+/vn2L5jx44cRxng1kTwxk3L1dXVHqw8PT01atQodezYUdu3b1dqaqr+/PNPtW3bVkWLFr3mX5Sv1+rVq1W5cmWVLl36uh7nwnWVpNdee00TJkzQV199pbi4OJ0+fVpvvPGGpk2bZj/jaX7LyMhQ165d1axZsxyHSl5KwYIF5e/vr0GDBikmJkb//fefunfvrrp169qHyeZk9OjRl71ud8GCBdWpUyft379fiYmJWrx4sbp06aKPP/74kv3B09NTY8eOVYcOHfTjjz/q3LlziomJ0aeffqpGjRrp7NmzV1yfwMBADRw4UA899JC2bt2q5ORkbd68WZ06dcr24Xrx63ojuvfee9W0aVM9/PDD+vvvv5WUlKQVK1aoT58+1/1lYfz48erZs6fmzp2rhIQEHTp0SI899pjDWWdzIzQ0VDabTU888YQOHz6s+Ph4/fDDD3rnnXcchpp36dJF3t7e6tChg/7++2+lp6dr37596ty5s9544w37fMYYe4gvVaqUevXqpXbt2mnnzp1KTk7Wpk2b9NBDD2n48OGXvQZs8+bNdebMGW3atEnt27d3aJs7d262IfXffvutfvjhByUmJmr58uUqW7asJOnVV1/V7NmzNWbMGJ0+fVqnTp3SO++8Y7827IW6d++uX375Rf3793cYVdClSxedPXtWQ4cO1YkTJ5SYmKhvv/1Wffr0Ue3atSWdP/v0H3/8oTNnzmjjxo32v//jjz+qTZs28vT0zNXrcSkX9/cxY8Zo+PDhWrBggRITE3X8+HH1799fW7ZsueyhOVaKj49Xu3bt9Pzzz2fbc3s5pUqV0n///aeRI0cqISFBBw8e1COPPKLevXvn6nn77LPP9Nhjj10y4AUHB6tr165at26dkpKStHPnTrVt21adOnWyX77sSq+Th4eH3nnnHT388MNat26dkpOT9ddff6ldu3Zq27atatSokeNyue1/OfXptWvXauLEiUpMTNTSpUvtfSonV7s9zM1zcrUGDhyo3r17KywsTFFRUerRo4f69euXqxE6xYsX14ABA7R06VIlJibaz9odFBRkv8rB3r171a1bN/sVQ65HSEiI3nvvPf37779KSkrS2LFjHa6EcCmX+l5SqlQpDRgwQA8++KBWr16tpKQkRUZG6vXXX1ebNm1ytTe8QYMGql69up588kkdOXJEKSkp9lEQWdsZSXrvvff09ttv68MPP1RUVJRSUlL0008/qX79+g4/sHbq1EkTJ05U69at9ffff9unBwYG6pFHHtGGDRuUnJysnTt3KiQkRA8//LB9xML48eP15JNP2n+APXjwoB577DEVK1Ysx6tO5MaV+miPHj00c+ZMff/990pMTNSBAwf06KOPqnfv3nJzc9PQoUP13HPPacuWLUpJSdG6des0ePBgDRo06Jrqwc2H4I2b1v33369KlSpp8+bNkqQBAwZo8ODB6t69uwoVKqTOnTurc+fO9iGLzjBv3rwrDs/MjYYNG2rMmDH2QBscHKwVK1Zo4cKFCgoKUpUqVXTo0CGtXbs22yVo8stzzz2n5cuX64svvnAYfnXxpXlyMmXKFGVkZKhixYqqWbOmvLy8NHXq1EvOf+7cOQUEBOjhhx++5DxPPPGEunbtqvbt26tw4cJ6/fXX9fnnn1/x9ejTp48+/fRTvfPOOypWrJiqVq2qHTt2aP369bm+puubb76pPn366Mknn5S/v7+GDh2qTz75xOGLh3Q+mD3wwAO5vta5s3z99ddq1qyZHnroIQUEBOjDDz/UrFmzsl2G5mq1bNlSc+bM0UcffaSAgAC1bt1aDz30ULZjoa/E1dVVP//8s0qUKKGGDRsqMDBQ33//vRYuXOgQQtzc3LR06VJVq1ZNrVu3lo+Pjzp16qSmTZtqwoQJ9vkaNWokPz8/HT9+XNL5obJZ/cnX11c9e/bUs88+qxEjRlyxtoceekhPPfWUw3HGEREROnDggFq0aOEwb8OGDfXZZ5+paNGi+ueff+w/BpQsWVLr1q3T9u3bVb58edWoUUNxcXH66quvsv29QoUKadCgQerVq5fDdHd3dy1btkxJSUmqXbu2ihcvrilTpmjRokX2ftmkSRM9+uijKl++vO666y778Y55tR1r2rSpevfubd/rec899+iHH37Qp59+qoCAAN17771yd3fXkiVLnPJjaWZmph5//HHt2rVLL7/8ssN2rHfv3ldcfsGCBdq9e7dKliypxo0b65577nG4VNnlHDp0SC+88MIl26tUqaLPP/9cr776qgoXLqyOHTuqQ4cO+t///mefJzevU58+ffTWW2/p2WeflZ+fnx5++GE9+OCDmjhx4iWXyU3/u9Qw89q1a2vFihUqWrSo5s6dq08++eSSf6datWras2fPJS/VdLHcPCdX6/nnn1dISIiaNm2qu+++WzVr1tRbb72Vq2WLFi2qb7/9VpMmTbKf36JSpUqaP3++pPN7PkNCQnTy5Ek1b97coX9dyxD8rl276pFHHrEf73z27NlslxTMyeX6yTvvvKMhQ4bo2Weflb+/v+rUqaOzZ89q7dq1uR5JNXPmTJUrV06NGzeWv7+/Ro0apcmTJzuMxLjzzju1evVqrV69WhUrVlSxYsU0YcIEzZo1K9u5UEJCQjRlyhSFhITYz0dRrVo1TZo0Sa+99poKFy6sDh06qG3btg7b8datW2vq1Kl65513VLhwYTVt2lRlypTRggULcn2o08Uu3oZdrFixYlqwYIE+/fRTFSlSRK1bt1azZs30/vvvSzr/mTdy5Eg99dRT8vf313PPPadx48ZlGy2DW5eLuZaDEQFcUUZGhkqWLKk//vgj20lAANz6jDGqXLmyfvnlF4fhxp9++ql2796dY3C+0URHR6tSpUo6ceLEde/xhnWc/TotWrRIn332mX799dd8/9s3giNHjqhdu3YOe2VvRLfC95K1a9fqww8/1OLFi51dCnDV2OMNWOT06dMaOXLkTfvhlpceeOABh1/3L7xdeEw6rMXrkL9++eUXVaxYMdsxvoGBgXr++eedVNXVOX36tD799FNCt86f4fhS75/Y2NhLLnf06NFLLpfbY8ivxNmvU9bhXnllxowZl3zOhg8fnmd/J7d+++23S9ZzpXO45EZ+9BGJ7yWAs7HHGwAAC3Tu3FlPPPFEnnwxB3Djuln2eN8K2OONmxnBGwAAAAAACzHUHAAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAAL/R9zIOih3F5lXAAAAABJRU5ErkJggg==","text/plain":["<Figure size 1200x600 with 1 Axes>"]},"metadata":{},"output_type":"display_data"}],"source":["for col in df.columns[5:]:\n"," print(\"*\" * 10, col, \"*\" * 10)\n"," print(df[col].value_counts())\n"," plot_value_counts(df, col)"]},{"cell_type":"code","execution_count":9,"metadata":{},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n","\n","\n","def calc_metrics_for_col(df, col):\n"," y_true = df[\"label\"]\n"," y_pred = df[col]\n","\n"," accuracy = accuracy_score(y_true, y_pred)\n"," precision = precision_score(y_true, y_pred, average=\"weighted\", labels=np.unique(y_pred))\n"," recall = recall_score(y_true, y_pred, average=\"weighted\", labels=np.unique(y_pred))\n"," f1 = f1_score(y_true, y_pred, average=\"weighted\", labels=np.unique(y_pred))\n","\n"," return accuracy, float(precision), float(recall), float(f1)"]},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[{"name":"stderr","output_type":"stream","text":["/var/folders/7x/56svhln929zdh2xhr3mwqg4r0000gn/T/ipykernel_73014/961288552.py:9: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n"," perf_df = pd.concat([perf_df, pd.DataFrame([new_model_metrics])], ignore_index=True)\n"]},{"data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>epoch</th>\n"," <th>model</th>\n"," <th>accuracy</th>\n"," <th>precision</th>\n"," <th>recall</th>\n"," <th>f1</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>0</td>\n"," <td>internlm/internlm2_5-7b-chat-1m_llama-factory/...</td>\n"," <td>0.783667</td>\n"," <td>0.809455</td>\n"," <td>0.783667</td>\n"," <td>0.794048</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" epoch model accuracy \\\n","0 0 internlm/internlm2_5-7b-chat-1m_llama-factory/... 0.783667 \n","\n"," precision recall f1 \n","0 0.809455 0.783667 0.794048 "]},"execution_count":10,"metadata":{},"output_type":"execute_result"}],"source":["import pandas as pd\n","\n","perf_df = pd.DataFrame(columns=[\"epoch\", \"model\", \"accuracy\", \"precision\", \"recall\", \"f1\"])\n","for i, col in enumerate(df.columns[5:]):\n"," accuracy, precision, recall, f1 = calc_metrics_for_col(df, col)\n"," new_model_metrics = {\"epoch\": i, \"model\": col, \"accuracy\": accuracy, \"precision\": precision, \"recall\": recall, \"f1\": f1}\n","\n"," # Convert the dictionary to a DataFrame and concatenate it with the existing DataFrame\n"," perf_df = pd.concat([perf_df, pd.DataFrame([new_model_metrics])], ignore_index=True)\n","\n","perf_df"]},{"cell_type":"code","execution_count":13,"metadata":{},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABE0AAAHACAYAAABXiZaAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABkEklEQVR4nO3deVwVZf//8TeL7II7oKKYW1gIKkq4JLeSWH0t78rMuhXJpUVKI3Mpl8wMy0JbvLPMrbtM27U0syjLLTWVysQ1jSzBNUksULh+f/Rj6siwHEXRfD0fj/N4cOZcc81nzhzmzHmfOde4GGOMAAAAAAAA4MC1sgsAAAAAAAC4EBGaAAAAAAAA2CA0AQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgA1CEwAAAAAAABuEJgAAAAAAADbcK7uA8igsLNQvv/yiqlWrysXFpbLLAQAAAAD8wxlj9Ntvv6lu3bpydeV8g0vVRRGa/PLLLwoJCansMgAAAAAAl5iffvpJ9evXr+wyUEkuitCkatWqkv58sfr7+1dyNQAAAACAf7qcnByFhIRYn0dxabooQpOin+T4+/sTmgAAAAAAzhuGiLi08cMsAAAAAAAAG4QmAAAAAAAANghNAAAXpenTpys0NFReXl6Kjo7W+vXrS20/bdo0NW/eXN7e3goJCdEDDzygP/74w6k+d+/erX//+9+qXbu2/P39deuttyo7O7vC1w0AAAAXBkITAMBFZ+HChUpOTtb48eO1adMmRUREKD4+XgcOHLBtP3/+fI0aNUrjx49XRkaGZs2apYULF+rhhx8ud5+5ubnq1q2bXFxc9Nlnn2n16tXKz89Xjx49VFhYeF7WGwAAAOeXizHGVHYRZcnJyVFAQICOHTvGQLAAAEVHR6tt27Z64YUXJEmFhYUKCQnRfffdp1GjRhVrn5SUpIyMDKWlpVnTHnzwQa1bt06rVq0qV5/Lly/Xtddeq6NHj1rvRceOHVP16tW1fPlyxcXFnevVBgAA5xGfQyFxpgkA4CKTn5+vjRs3OoQUrq6uiouL09q1a23nad++vTZu3Gj93OaHH37Q0qVLdd1115W7z7y8PLm4uMjT09Nq4+XlJVdXVyt4AQAAwD8LoQkA4KJy6NAhFRQUKDAw0GF6YGCgsrKybOe5/fbb9dhjj6ljx46qUqWKGjdurNjYWOvnOeXp86qrrpKvr69GjhypEydOKDc3V8OHD1dBQYH2799/DtYUAAAAlY3QBADwj7dixQo98cQT+u9//6tNmzbp3Xff1ZIlSzRx4sRy91G7dm299dZb+uCDD+Tn56eAgAD9+uuvat26tVxdeTsFAAD4J3Kv7AIAAHBGrVq15ObmVuyqNdnZ2QoKCrKdZ+zYserbt68GDhwoSQoPD1dubq4GDx6sRx55pNx9duvWTbt379ahQ4fk7u6uatWqKSgoSJdddlkFryUAAAAuBHw1BgC4qHh4eKhNmzYOg7oWFhYqLS1NMTExtvOcOHGi2Nkgbm5ukiRjjNN91qpVS9WqVdNnn32mAwcO6IYbbqiIVQP+MZy5JHhsbKxcXFyK3a6//nqrTXZ2tvr376+6devKx8dH3bt3186dO237M8bo2muvlYuLi95///2KXjUAwCWGM00AABed5ORkJSQkKCoqSu3atdO0adOUm5urxMRESVK/fv1Ur149paSkSJJ69Oih1NRUtWrVStHR0dq1a5fGjh2rHj16WOFJWX1K0pw5cxQWFqbatWtr7dq1Gjp0qB544AE1b978/D8JwAWq6PLdM2bMUHR0tKZNm6b4+Hht375dderUKdb+3XffVX5+vnX/8OHDioiIUK9evST9GYL07NlTVapU0aJFi+Tv76/U1FTFxcVp69at8vX1dehv2rRpcnFxObcrCQC4ZBCaAAAuOr1799bBgwc1btw4ZWVlKTIyUsuWLbMGcs3MzHQ4s2TMmDFycXHRmDFj9PPPP6t27drq0aOHJk2aVO4+JWn79u0aPXq0jhw5otDQUD3yyCN64IEHzt+KAxeB1NRUDRo0yAocZ8yYoSVLlmj27Nm2lwSvUaOGw/0FCxbIx8fHCk127typr776Slu2bNEVV1whSXrxxRcVFBSkN954w/rZnSSlp6frmWee0ddff63g4OBztYoAgEuIizHGVHYRZeH62AAAABe+/Px8+fj46O2331bPnj2t6QkJCfr111+1aNGiMvsIDw9XTEyMXn75ZUnSd999p5YtW2rXrl1q3Lix1S4kJERdu3bV3LlzJf35M7yoqCilpKToxhtvlIuLi9577z2HOgDAGXwOhcSYJgAAAKggZ3JJ8L9bv369tmzZ4nD2yOWXX64GDRpo9OjROnr0qPLz8/Xkk09q3759Dpf7fuCBB9S+fXvdeOONFbdCAIBLHqEJgEtCRQ9KePz4cSUlJal+/fry9vZWixYtNGPGDNv+GJQQKBv/o5CkWbNmKTw8XO3atbOmValSRe+++6527NihGjVqyMfHR59//rmuvfZa62d4ixcv1meffaZp06ZVUuUAgH+qMxrTZPr06ZoyZYqysrIUERGh559/3uHN7XTTpk3Tiy++qMzMTNWqVUu33HKLUlJS5OXldcaFA0B5VfSghNKfg4Z+9tlneu211xQaGqrly5fr3nvvVd26dYtdSeWiG5Tw0YDKrgAV7dFjlV1Bqfgf/ec4k0uCF8nNzdWCBQv02GOPFXusTZs2Sk9P17Fjx5Sfn6/atWsrOjpaUVFRkqTPPvtMu3fvVrVq1Rzmu/nmm9WpUyetWLHirNYLAHDpcvpMk6IDm/Hjx2vTpk2KiIhQfHy8Dhw4YNt+/vz5GjVqlMaPH6+MjAzNmjVLCxcu1MMPP3zWxQNAefx9UMKib5t9fHw0e/Zs2/Y1atRQUFCQdfvkk08cBiWUpDVr1ighIUGxsbEKDQ3V4MGDFRERUezb8aJBCUtaFgD+R/9JzuSS4EXeeust5eXl6T//+U+JbQICAlS7dm3t3LlTX3/9tfVTnFGjRunbb79Venq6dZOkqVOnas6cOWe/YgCAS5bToYmzBzZr1qxRhw4ddPvttys0NFTdunVTnz59Sj3tFgAqSn5+vjZu3Ki4uDhrmqurq+Li4rR27dpy9TFr1izddtttDpe1bN++vRYvXqyff/5Zxhh9/vnn2rFjh7p162a1OXHihG6//XZNnz69zG9YgUsV/6P/PMnJyZo5c6bmzZunjIwM3XPPPcUuCT569Ohi882aNUs9e/ZUzZo1iz321ltvacWKFfrhhx+0aNEiXXPNNerZs6e1PYOCgnTllVc63CSpQYMGatSo0TlcWwDAP51TP88pOrD5+xtdWQc27du312uvvab169erXbt2+uGHH7R06VL17du3xOXk5eUpLy/Pup+Tk+NMmQBgKW1Qwm3btpU5f9GghLNmzXKY/vzzz2vw4MGqX7++3N3d5erqqpkzZ+rqq6+22jAoIVA2/kf/eZy9JLj05+W8V61apeXLl9v2uX//fiUnJys7O1vBwcHq16+fxo4de87XBQAAp0KTMzmwuf3223Xo0CF17NhRxhidOnVKd999d6k/z0lJSdGECROcKQ0Azgm7QQmlPz+QffXVV1q8eLEaNmyoL7/8UkOGDFHdunUVFxdnDUq4efPmSqocuDTwP3phSkpKUlJSku1jduOLNG/eXMaYEvu7//77df/99ztVQ2n9AQBQXuf86jkrVqzQE088of/+97/atGmT3n33XS1ZskQTJ04scZ7Ro0fr2LFj1u2nn34612UC+IeqiEEJBwwY4DD9999/18MPP6zU1FT16NFDLVu2VFJSknr37q2nn35akuOghO7u7nJ3/zOjvvnmmxUbG1txKwhc5PgfBQAAFzKnzjQ5kwObsWPHqm/fvho4cKAkKTw8XLm5uRo8eLAeeeSRYqdnSpKnp6c8PT2dKQ0AbP19UMKePXtK+mtQwpK+BS1S0qCEJ0+e1MmTJ4vtv9zc3FRYWCjpz0EJi/Z7RcLDwzV16lT16NHjLNcK+OfgfxQAAFzInApNzuTA5sSJE7YHLRKnTQI4P5KTk5WQkKCoqCi1a9dO06ZNKzYoYb169ZSSkuIwX0mDEvr7+6tz58566KGH5O3trYYNG+qLL77Qq6++qtTUVEmyrupxOgYlBIrjf9Q5GZeHVXYJqEBh2zIquwQAQCmcCk0k5w9sevToodTUVLVq1UrR0dHatWuXxo4dqx49eljhCQCcS+diUMIFCxZo9OjRuuOOO3TkyBE1bNhQkyZN0t13333O1wf4p+F/FAAAXKhczBmc7vHCCy9oypQp1oHNc889p+joaElSbGysQkNDNXfuXEnSqVOnNGnSJP3vf//Tzz//rNq1a6tHjx6aNGmSqlWrVq7l5eTkKCAgQMeOHZO/v7+z5QIAnPFoQGVXgIr26LHKrgAViDNN/lk40wS4cPE5FNIZhibnGy9WADiPCE3+eQhN/lEITf5ZCE2ACxefQyGdh6vnAAAAAAAAXIycHtMEAE4XOmpJZZeACrTXq7IrQEULnxde2SWgAr1Z2QUAAHAJ4UwTAAAAAAAAG4QmAAAAAAAANghNAAAAAAAAbBCaAAAAAAAA2CA0AQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgA1CEwAAAAAAABuEJgAAAAAAADYITQAAAAAAAGwQmgAAAAAAANggNAEAAAAAALBBaAIAAAAAAGCD0AQAAAAAAMAGoQkAAAAAAIANQhMAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwQWgCAAAAAABgg9AEAAAAAADABqEJAAAAAACADUITAAAAAAAAG4QmAAAAAAAANghNAAAAAAAAbBCaAAAAAAAA2CA0AQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgI0zCk2mT5+u0NBQeXl5KTo6WuvXry+xbWxsrFxcXIrdrr/++jMuGgAAAAAA4FxzOjRZuHChkpOTNX78eG3atEkRERGKj4/XgQMHbNu/++672r9/v3XbsmWL3Nzc1KtXr7MuHgAAAAAA4FxxOjRJTU3VoEGDlJiYqBYtWmjGjBny8fHR7NmzbdvXqFFDQUFB1u2TTz6Rj48PoQkAAAAAALigORWa5Ofna+PGjYqLi/urA1dXxcXFae3ateXqY9asWbrtttvk6+tbYpu8vDzl5OQ43AAAAAAAAM4np0KTQ4cOqaCgQIGBgQ7TAwMDlZWVVeb869ev15YtWzRw4MBS26WkpCggIMC6hYSEOFMmAAAAAADAWTuvV8+ZNWuWwsPD1a5du1LbjR49WseOHbNuP/3003mqEAAAAAAA4E/uzjSuVauW3NzclJ2d7TA9OztbQUFBpc6bm5urBQsW6LHHHitzOZ6envL09HSmNAAAAAAAgArl1JkmHh4eatOmjdLS0qxphYWFSktLU0xMTKnzvvXWW8rLy9N//vOfM6sUAAAAAADgPHLqTBNJSk5OVkJCgqKiotSuXTtNmzZNubm5SkxMlCT169dP9erVU0pKisN8s2bNUs+ePVWzZs2KqRwAAAAAAOAccjo06d27tw4ePKhx48YpKytLkZGRWrZsmTU4bGZmplxdHU9g2b59u1atWqXly5dXTNUAAAAAAADnmNOhiSQlJSUpKSnJ9rEVK1YUm9a8eXMZY85kUQAAAAAAAJXivF49BwAAAAAA4GJBaAIAAAAAAGCD0AQAAAAAAMAGoQkAAAAAAIANQhMAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwQWgCAAAAAABgg9AEAAAAAADABqEJAAAAAACADUITAAAAAAAAG4QmAAAAAAAANghNAAAAAAAAbBCaAAAAAAAA2CA0AQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgA1CEwAAAAAAABuEJgAAAAAAADYITQAAAAAAAGwQmgAAAAAAANggNAEAAAAAALBBaAIAAAAAAGCD0AQAAAAAAMAGoQkAAAAAAIANQhMAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwQWgCAAAAAABgg9AEAAAAAADAxhmFJtOnT1doaKi8vLwUHR2t9evXl9r+119/1ZAhQxQcHCxPT081a9ZMS5cuPaOCAQAAAAAAzgd3Z2dYuHChkpOTNWPGDEVHR2vatGmKj4/X9u3bVadOnWLt8/Pzdc0116hOnTp6++23Va9ePf3444+qVq1aRdQPAAAAAABwTjgdmqSmpmrQoEFKTEyUJM2YMUNLlizR7NmzNWrUqGLtZ8+erSNHjmjNmjWqUqWKJCk0NPTsqgYAAAAAADjHnPp5Tn5+vjZu3Ki4uLi/OnB1VVxcnNauXWs7z+LFixUTE6MhQ4YoMDBQV155pZ544gkVFBSUuJy8vDzl5OQ43AAAAAAAAM4np0KTQ4cOqaCgQIGBgQ7TAwMDlZWVZTvPDz/8oLffflsFBQVaunSpxo4dq2eeeUaPP/54ictJSUlRQECAdQsJCXGmTAAAAAAAgLN2zq+eU1hYqDp16ujll19WmzZt1Lt3bz3yyCOaMWNGifOMHj1ax44ds24//fTTuS4TAAAAAADAgVNjmtSqVUtubm7Kzs52mJ6dna2goCDbeYKDg1WlShW5ublZ08LCwpSVlaX8/Hx5eHgUm8fT01Oenp7OlAYAAAAAAFChnDrTxMPDQ23atFFaWpo1rbCwUGlpaYqJibGdp0OHDtq1a5cKCwutaTt27FBwcLBtYAIAAAAAAHAhcPrnOcnJyZo5c6bmzZunjIwM3XPPPcrNzbWuptOvXz+NHj3aan/PPffoyJEjGjp0qHbs2KElS5boiSee0JAhQypuLQAAAAAAACqY05cc7t27tw4ePKhx48YpKytLkZGRWrZsmTU4bGZmplxd/8piQkJC9PHHH+uBBx5Qy5YtVa9ePQ0dOlQjR46suLUAAAAAAACoYC7GGFPZRZQlJydHAQEBOnbsmPz9/Su7HACnCR21pLJLQAXa63V7ZZeAChbeqEFll4AK9GbKqcouARUobFtGZZcAoAR8DoV0Hq6eAwAAAAAAcDEiNAEAAAAAALBBaAIAAAAAAGCD0AQAAAAAAMAGoQkAAAAAAIANQhMAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwQWgCAAAAAABgg9AEAAAAAADABqEJAAAAAACADUITAAAAAAAAG4QmAAAAAAAANghNAAAAAAAAbBCaAAAAAAAA2CA0AQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgA1CEwAAAAAAABuEJgAAAAAAADYITQAAAAAAAGwQmgAAAAAAANggNAEAAAAAALBBaAIAAAAAAGDDvbILAAAAAADgYlRYWKj8/PzKLgNOqlKlitzc3MrVltAEAAAAAAAn5efna8+ePSosLKzsUnAGqlWrpqCgILm4uJTajtAEAAAAAAAnGGO0f/9+ubm5KSQkRK6ujHxxsTDG6MSJEzpw4IAkKTg4uNT2hCYAAAAAADjh1KlTOnHihOrWrSsfH5/KLgdO8vb2liQdOHBAderUKfWnOsRhAAAAAAA4oaCgQJLk4eFRyZXgTBWFXSdPniy1HaEJAAAAAABnoKzxMHDhKu+2IzQBAAAAAACwcUahyfTp0xUaGiovLy9FR0dr/fr1JbadO3euXFxcHG5eXl5nXDAAAAAAAMD54PRAsAsXLlRycrJmzJih6OhoTZs2TfHx8dq+fbvq1KljO4+/v7+2b99u3ecUJgAAAADAP03oqCXndXl7J19/Xpd3KXL6TJPU1FQNGjRIiYmJatGihWbMmCEfHx/Nnj27xHlcXFwUFBRk3QIDA8+qaAAAAAAAcPErayDWyuZUaJKfn6+NGzcqLi7urw5cXRUXF6e1a9eWON/x48fVsGFDhYSE6MYbb9T3339f6nLy8vKUk5PjcAMAAAAAAGdn2bJl6tixo6pVq6aaNWvq//7v/7R7927r8X379qlPnz6qUaOGfH19FRUVpXXr1lmPf/DBB2rbtq28vLxUq1Yt/fvf/7Yec3Fx0fvvv++wvGrVqmnu3LmSpL1798rFxUULFy5U586d5eXlpddff12HDx9Wnz59VK9ePfn4+Cg8PFxvvPGGQz+FhYV66qmn1KRJE3l6eqpBgwaaNGmSJKlLly5KSkpyaH/w4EF5eHgoLS3trJ4vp0KTQ4cOqaCgoNiZIoGBgcrKyrKdp3nz5po9e7YWLVqk1157TYWFhWrfvr327dtX4nJSUlIUEBBg3UJCQpwpEwAAAAAA2MjNzVVycrK+/vprpaWlydXVVf/+979VWFio48ePq3Pnzvr555+1ePFiffPNNxoxYoQKCwslSUuWLNG///1vXXfdddq8ebPS0tLUrl07p2sYNWqUhg4dqoyMDMXHx+uPP/5QmzZttGTJEm3ZskWDBw9W3759HcZPHT16tCZPnqyxY8dq69atmj9/vpVNDBw4UPPnz1deXp7V/rXXXlO9evXUpUuXs3q+nB7TxFkxMTGKiYmx7rdv315hYWF66aWXNHHiRNt5Ro8ereTkZOt+Tk4OwQkAAAAAAGfp5ptvdrg/e/Zs1a5dW1u3btWaNWt08OBBbdiwQTVq1JAkNWnSxGo7adIk3XbbbZowYYI1LSIiwukahg0bpptuuslh2vDhw62/77vvPn388cd688031a5dO/3222969tln9cILLyghIUGS1LhxY3Xs2FGSdNNNNykpKUmLFi3SrbfeKunPi9L079//rMdUdepMk1q1asnNzU3Z2dkO07OzsxUUFFSuPqpUqaJWrVpp165dJbbx9PSUv7+/ww0AAAAAAJydnTt3qk+fPrrsssvk7++v0NBQSVJmZqbS09PVqlUrKzA5XXp6urp27XrWNURFRTncLygo0MSJExUeHq4aNWrIz89PH3/8sTIzMyVJGRkZysvLK3HZXl5e6tu3rzXW6qZNm7Rlyxb179//rGt1KjTx8PBQmzZtHH4TVFhYqLS0NIezSUpTUFCg7777TsHBwc5VCgAAAAAAzkqPHj105MgRzZw5U+vWrbPGK8nPz5e3t3ep85b1uIuLi4wxDtPsBnr19fV1uD9lyhQ9++yzGjlypD7//HOlp6crPj5e+fn55Vqu9OdPdD755BPt27dPc+bMUZcuXdSwYcMy5yuL01fPSU5O1syZMzVv3jxlZGTonnvuUW5urhITEyVJ/fr10+jRo632jz32mJYvX64ffvhBmzZt0n/+8x/9+OOPGjhw4FkXDwAAAAAAyufw4cPavn27xowZo65duyosLExHjx61Hm/ZsqXS09N15MgR2/lbtmxZ6sCqtWvX1v79+637O3fu1IkTJ8qsa/Xq1brxxhv1n//8RxEREbrsssu0Y8cO6/GmTZvK29u71GWHh4crKipKM2fO1Pz583XnnXeWudzycHpMk969e+vgwYMaN26csrKyFBkZqWXLllkDsGRmZsrV9a8s5ujRoxo0aJCysrJUvXp1tWnTRmvWrFGLFi0qZAUAAAAAAEDZqlevrpo1a+rll19WcHCwMjMzNWrUKOvxPn366IknnlDPnj2VkpKi4OBgbd68WXXr1lVMTIzGjx+vrl27qnHjxrrtttt06tQpLV26VCNHjpT051VsXnjhBcXExKigoEAjR45UlSpVyqyradOmevvtt7VmzRpVr15dqampys7OtnIDLy8vjRw5UiNGjJCHh4c6dOiggwcP6vvvv9eAAQOsfgYOHKikpCT5+vo6XNXnbDh9pokkJSUl6ccff1ReXp7WrVun6Oho67EVK1ZYlxOSpKlTp1pts7KytGTJErVq1eqsCwcAAAAAAOXn6uqqBQsWaOPGjbryyiv1wAMPaMqUKdbjHh4eWr58uerUqaPrrrtO4eHhmjx5stzc3CRJsbGxeuutt7R48WJFRkaqS5cuDle4eeaZZxQSEqJOnTrp9ttv1/Dhw+Xj41NmXWPGjFHr1q0VHx+v2NhYBQUFqWfPng5txo4dqwcffFDjxo1TWFiYevfurQMHDji06dOnj9zd3dWnTx95eXmdxTP1Fxdz+g+OLkA5OTkKCAjQsWPHGBQWuACFjlpS2SWgAu31ur2yS0AFC2/UoLJLQAV6M+VUZZeAChS2LaOySwBQgtI+h/7xxx/as2ePGjVqVGEfznH29u7dq8aNG2vDhg1q3bp1qW3Luw3P+SWHAQAAAAAAzpWTJ0/q8OHDGjNmjK666qoyAxNnnNHPcwAAAAAAAC4Eq1evVnBwsDZs2KAZM2ZUaN+caQIAAAAAAC5asbGxxS51XFE40wQAAAAAAMAGoQkAAAAAAIANQhMAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAADnxIoVK+Ti4qJff/21QtueL+6VXQAAAAAAAP8Ijwac5+UdO7/LOwPt27fX/v37FRBQ9nPjTNvzhTNNAAAAAABAMfn5+Wfdh4eHh4KCguTi4lKhbc8XQhMAAAAAAC4BsbGxSkpKUlJSkgICAlSrVi2NHTtWxhhJUmhoqCZOnKh+/frJ399fgwcPliStWrVKnTp1kre3t0JCQnT//fcrNzfX6jcvL08jR45USEiIPD091aRJE82aNUtS8Z/c/Pjjj+rRo4eqV68uX19fXXHFFVq6dKltW0l65513dMUVV8jT01OhoaF65plnHNYpNDRUTzzxhO68805VrVpVDRo00Msvv1xhzxmhCQAAAAAAl4h58+bJ3d1d69ev17PPPqvU1FS98sor1uNPP/20IiIitHnzZo0dO1a7d+9W9+7ddfPNN+vbb7/VwoULtWrVKiUlJVnz9OvXT2+88Yaee+45ZWRk6KWXXpKfn5/t8ocMGaK8vDx9+eWX+u677/Tkk0+W2Hbjxo269dZbddttt+m7777To48+qrFjx2ru3LkO7Z555hlFRUVp8+bNuvfee3XPPfdo+/btZ/9kiTFNAAAAAAC4ZISEhGjq1KlycXFR8+bN9d1332nq1KkaNGiQJKlLly568MEHrfYDBw7UHXfcoWHDhkmSmjZtqueee06dO3fWiy++qMzMTL355pv65JNPFBcXJ0m67LLLSlx+Zmambr75ZoWHh5fZNjU1VV27dtXYsWMlSc2aNdPWrVs1ZcoU9e/f32p33XXX6d5775UkjRw5UlOnTtXnn3+u5s2bO/8EnYYzTQAAAAAAuERcddVVDmOGxMTEaOfOnSooKJAkRUVFObT/5ptvNHfuXPn5+Vm3+Ph4FRYWas+ePUpPT5ebm5s6d+5cruXff//9evzxx9WhQweNHz9e3377bYltMzIy1KFDB4dpHTp0cKhXklq2bGn97eLioqCgIB04cKBc9ZSF0AQAAAAAAEiSfH19He4fP35cd911l9LT063bN998o507d6px48by9vZ2qv+BAwfqhx9+UN++ffXdd98pKipKzz///FnVXKVKFYf7Li4uKiwsPKs+ixCaAAAAAABwiVi3bp3D/a+++kpNmzaVm5ubbfvWrVtr69atatKkSbGbh4eHwsPDVVhYqC+++KLcNYSEhOjuu+/Wu+++qwcffFAzZ860bRcWFqbVq1c7TFu9erWaNWtWYr0VjdAEAAAAAIBLRGZmppKTk7V9+3a98cYbev755zV06NAS248cOVJr1qxRUlKS0tPTtXPnTi1atMgaCDY0NFQJCQm688479f7772vPnj1asWKF3nzzTdv+hg0bpo8//lh79uzRpk2b9PnnnyssLMy27YMPPqi0tDRNnDhRO3bs0Lx58/TCCy9o+PDhZ/9ElBMDwQIAAAAAcIno16+ffv/9d7Vr105ubm4aOnSodWlhOy1bttQXX3yhRx55RJ06dZIxRo0bN1bv3r2tNi+++KIefvhh3XvvvTp8+LAaNGighx9+2La/goICDRkyRPv27ZO/v7+6d++uqVOn2rZt3bq13nzzTY0bN04TJ05UcHCwHnvsMYdBYM81F1N0QeYLWE5OjgICAnTs2DH5+/tXdjkAThM6aklll4AKtNfr9souARUsvFGDyi4BFejNlFOVXQIqUNi2jMouAUAJSvsc+scff2jPnj1q1KiRvLy8KqlC58XGxioyMlLTpk2r7FIqXXm3IT/PAQAAAAAAsEFoAgAAAAAAYIMxTQAAAAAAuASsWLGisku46HCmCQAAAAAAgA1CEwAAAAAAABuEJgAAAAAAADYITQAAAAAAAGwQmgAAAAAAANggNAEAAAAAALBBaAIAAAAAAM6JRx99VJGRkdb9/v37q2fPnpVWj7PcK7sAAAAAAAD+CcLnhZ/X5X2X8N15Xd6liDNNAAAAAAC4BOXn51d2CRe8MwpNpk+frtDQUHl5eSk6Olrr168v13wLFiyQi4vLRXUqDgAAAAAA/wSxsbFKSkrSsGHDVKtWLcXHx2vLli269tpr5efnp8DAQPXt21eHDh2y5iksLNRTTz2lJk2ayNPTUw0aNNCkSZOsx0eOHKlmzZrJx8dHl112mcaOHauTJ09WxuqdE06HJgsXLlRycrLGjx+vTZs2KSIiQvHx8Tpw4ECp8+3du1fDhw9Xp06dzrhYAAAAAABw5ubNmycPDw+tXr1akydPVpcuXdSqVSt9/fXXWrZsmbKzs3Xrrbda7UePHq3Jkydr7Nix2rp1q+bPn6/AwEDr8apVq2ru3LnaunWrnn32Wc2cOVNTp06tjFU7J5we0yQ1NVWDBg1SYmKiJGnGjBlasmSJZs+erVGjRtnOU1BQoDvuuEMTJkzQypUr9euvv55V0QAAAAAAwHlNmzbVU089JUl6/PHH1apVKz3xxBPW47Nnz1ZISIh27Nih4OBgPfvss3rhhReUkJAgSWrcuLE6duxotR8zZoz1d2hoqIYPH64FCxZoxIgR52mNzi2nQpP8/Hxt3LhRo0ePtqa5uroqLi5Oa9euLXG+xx57THXq1NGAAQO0cuXKMpeTl5envLw8635OTo4zZQIAAAAAABtt2rSx/v7mm2/0+eefy8/Pr1i73bt369dff1VeXp66du1aYn8LFy7Uc889p927d+v48eM6deqU/P39z0ntlcGp0OTQoUMqKChwOBVHkgIDA7Vt2zbbeVatWqVZs2YpPT293MtJSUnRhAkTnCkNAAAAAACUwdfX1/r7+PHj6tGjh5588sli7YKDg/XDDz+U2tfatWutX5XEx8crICBACxYs0DPPPFPhdVeWc3rJ4d9++019+/bVzJkzVatWrXLPN3r0aCUnJ1v3c3JyFBISci5KBAAAAADgktS6dWu98847Cg0Nlbt78XigadOm8vb2VlpamgYOHFjs8TVr1qhhw4Z65JFHrGk//vjjOa35fHMqNKlVq5bc3NyUnZ3tMD07O1tBQUHF2u/evVt79+5Vjx49rGmFhYV/LtjdXdu3b1fjxo2Lzefp6SlPT09nSgMAAAAAAE4YMmSIZs6cqT59+mjEiBGqUaOGdu3apQULFuiVV16Rl5eXRo4cqREjRsjDw0MdOnTQwYMH9f3332vAgAFq2rSpMjMztWDBArVt21ZLlizRe++9V9mrVaGcunqOh4eH2rRpo7S0NGtaYWGh0tLSFBMTU6z95Zdfru+++07p6enW7YYbbtC//vUvpaenc/YIAAAAAACVpG7dulq9erUKCgrUrVs3hYeHa9iwYapWrZpcXf+MC8aOHasHH3xQ48aNU1hYmHr37m1dPfeGG27QAw88oKSkJEVGRmrNmjUaO3ZsZa5ShXMxxhhnZli4cKESEhL00ksvqV27dpo2bZrefPNNbdu2TYGBgerXr5/q1aunlJQU2/n79++vX3/9Ve+//365l5mTk6OAgAAdO3bsHzWgDPBPETpqSWWXgAq01+v2yi4BFSy8UYPKLgEV6M2UU5VdAipQ2LaMyi4BQAlK+xz6xx9/aM+ePWrUqJG8vLwqqUKcjfJuQ6fHNOndu7cOHjyocePGKSsrS5GRkVq2bJk1OGxmZqaVSAEAAAAAAFyszmgg2KSkJCUlJdk+tmLFilLnnTt37pksEgAAAAAA4LzilBAAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwQWgCAAAAAABgg9AEAAAAAADABqEJAAAAAACXAGOMBg8erBo1asjFxUXp6emVXdIFz72yCwAAAAAA4J8g4/Kw87q8sG0ZTrVftmyZ5s6dqxUrVuiyyy7Tjh071KNHD23cuFH79+/Xe++9p549e56bYi9SnGkCAAAAAMAlYPfu3QoODlb79u0VFBSk3NxcRUREaPr06ZVd2gWLM00AAAAAAPiH69+/v+bNmydJcnFxUcOGDbV3715de+21lVzZhY3QBAAAAACAf7hnn31WjRs31ssvv6wNGzbIzc2tsku6KBCaAAAAAADwDxcQEKCqVavKzc1NQUFBlV3ORYMxTQAAAAAAAGwQmgAAAAAAANggNAEAAAAAALDBmCYAAAAAAFyCjh8/rl27dln39+zZo/T0dNWoUUMNGjSoxMouHIQmAAAAAABUgLBtGZVdglO+/vpr/etf/7LuJycnS5ISEhI0d+7cSqrqwkJoAgAAAADAJWDYsGEaNmyYdT82NlbGmMor6CLAmCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwQWgCAAAAAMAZYBDVi1d5tx2hCQAAAAAATnBzc5Mk5efnV3IlOFMnTpyQJFWpUqXUdlxyGAAAAAAAJ7i7u8vHx0cHDx5UlSpV5OrK+QgXC2OMTpw4oQMHDqhatWpWAFYSQhMAAAAAAJzg4uKi4OBg7dmzRz/++GNll4MzUK1aNQUFBZXZjtAEAAAAAAAneXh4qGnTpvxE5yJUpUqVMs8wKUJoAgAAAADAGXB1dZWXl1dll4FziB9eAQAAAAAA2CA0AQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgA1CEwAAAAAAABtnFJpMnz5doaGh8vLyUnR0tNavX19i23fffVdRUVGqVq2afH19FRkZqf/9739nXDAAAAAAAMD54HRosnDhQiUnJ2v8+PHatGmTIiIiFB8frwMHDti2r1Gjhh555BGtXbtW3377rRITE5WYmKiPP/74rIsHAAAAAAA4V5wOTVJTUzVo0CAlJiaqRYsWmjFjhnx8fDR79mzb9rGxsfr3v/+tsLAwNW7cWEOHDlXLli21atWqsy4eAAAAAADgXHEqNMnPz9fGjRsVFxf3VweuroqLi9PatWvLnN8Yo7S0NG3fvl1XX311ie3y8vKUk5PjcAMAAAAAADifnApNDh06pIKCAgUGBjpMDwwMVFZWVonzHTt2TH5+fvLw8ND111+v559/Xtdcc02J7VNSUhQQEGDdQkJCnCkTAAAAAADgrJ2Xq+dUrVpV6enp2rBhgyZNmqTk5GStWLGixPajR4/WsWPHrNtPP/10PsoEAAAAAACwuDvTuFatWnJzc1N2drbD9OzsbAUFBZU4n6urq5o0aSJJioyMVEZGhlJSUhQbG2vb3tPTU56ens6UBgAAAAAAUKGcOtPEw8NDbdq0UVpamjWtsLBQaWlpiomJKXc/hYWFysvLc2bRAAAAAAAA55VTZ5pIUnJyshISEhQVFaV27dpp2rRpys3NVWJioiSpX79+qlevnlJSUiT9OT5JVFSUGjdurLy8PC1dulT/+9//9OKLL1bsmgAAAAAAAFQgp0OT3r176+DBgxo3bpyysrIUGRmpZcuWWYPDZmZmytX1rxNYcnNzde+992rfvn3y9vbW5Zdfrtdee029e/euuLUAAAAAAACoYC7GGFPZRZQlJydHAQEBOnbsmPz9/Su7HACnCR21pLJLQAXa63V7ZZeAChbeqEFll4AK9GbKqcouARUobFtGZZcAoAR8DoV0nq6eAwAAAAAAcLEhNAEAAAAAALBBaAIAAAAAAGCD0AQAAAAAAMAGoQkAAAAAAIANQhMAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwQWgCAAAAAABgg9AEAAAAAADABqEJAAAAAACADUITAAAAAAAAG4QmAAAAAAAANghNAAAAAAAAbBCaAAAAAAAA2CA0AQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgA1CEwAAAAAAABuEJgAAAAAAADYITQAAAAAAAGwQmgAAAAAAANggNAEAAAAAALBBaAIAAAAAAGCD0AQAAAAAAMAGoQkAAAAAAIANQhMAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwcUahyfTp0xUaGiovLy9FR0dr/fr1JbadOXOmOnXqpOrVq6t69eqKi4srtT0AAAAAAMCFwOnQZOHChUpOTtb48eO1adMmRUREKD4+XgcOHLBtv2LFCvXp00eff/651q5dq5CQEHXr1k0///zzWRcPAAAAAABwrjgdmqSmpmrQoEFKTExUixYtNGPGDPn4+Gj27Nm27V9//XXde++9ioyM1OWXX65XXnlFhYWFSktLO+viAQAAAAAAzhWnQpP8/Hxt3LhRcXFxf3Xg6qq4uDitXbu2XH2cOHFCJ0+eVI0aNUpsk5eXp5ycHIcbAAAAAADA+eRUaHLo0CEVFBQoMDDQYXpgYKCysrLK1cfIkSNVt25dh+DldCkpKQoICLBuISEhzpQJAAAAAABw1s7r1XMmT56sBQsW6L333pOXl1eJ7UaPHq1jx45Zt59++uk8VgkAAAAAACC5O9O4Vq1acnNzU3Z2tsP07OxsBQUFlTrv008/rcmTJ+vTTz9Vy5YtS23r6ekpT09PZ0oDAAAAAACoUE6daeLh4aE2bdo4DOJaNKhrTExMifM99dRTmjhxopYtW6aoqKgzrxYAAAAAAOA8cepME0lKTk5WQkKCoqKi1K5dO02bNk25ublKTEyUJPXr10/16tVTSkqKJOnJJ5/UuHHjNH/+fIWGhlpjn/j5+cnPz68CVwUAAAAAAKDiOB2a9O7dWwcPHtS4ceOUlZWlyMhILVu2zBocNjMzU66uf53A8uKLLyo/P1+33HKLQz/jx4/Xo48+enbVAwAAAAAAnCNOhyaSlJSUpKSkJNvHVqxY4XB/7969Z7IIAAAAAACASnVer54DAAAAAABwsSA0AQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgA1CEwAAAAAAABuEJgAAAAAAADYITQAAAAAAAGwQmgAAAAAAANggNAEAAAAAALBBaAIAAAAAAGCD0AQAAAAAAMAGoQkAAAAAAIANQhMAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwQWgCAAAAAABgg9AEAAAAAADABqEJAAAAAACADUITAAAAAAAAG4QmAAAAAAAANghNAAAAAAAAbBCaAAAAAAAA2CA0AQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgA1CEwAAAAAAABuEJgAAAAAAADYITQAAAAAAAGwQmgAAAAAAANggNAEAAAAAALBxRqHJ9OnTFRoaKi8vL0VHR2v9+vUltv3+++918803KzQ0VC4uLpo2bdqZ1goAAAAAAHDeOB2aLFy4UMnJyRo/frw2bdqkiIgIxcfH68CBA7btT5w4ocsuu0yTJ09WUFDQWRcMAAAAAABwPjgdmqSmpmrQoEFKTExUixYtNGPGDPn4+Gj27Nm27du2baspU6botttuk6en51kXDAAAAAAAcD44FZrk5+dr48aNiouL+6sDV1fFxcVp7dq1FVZUXl6ecnJyHG4AAAAAAADnk1OhyaFDh1RQUKDAwECH6YGBgcrKyqqwolJSUhQQEGDdQkJCKqxvAAAAAACA8rggr54zevRoHTt2zLr99NNPlV0SAAAAAAC4xLg707hWrVpyc3NTdna2w/Ts7OwKHeTV09OT8U8AAAAAAEClcupMEw8PD7Vp00ZpaWnWtMLCQqWlpSkmJqbCiwMAAAAAAKgsTp1pIknJyclKSEhQVFSU2rVrp2nTpik3N1eJiYmSpH79+qlevXpKSUmR9OfgsVu3brX+/vnnn5Weni4/Pz81adKkAlcFAAAAAACg4jgdmvTu3VsHDx7UuHHjlJWVpcjISC1btswaHDYzM1Ourn+dwPLLL7+oVatW1v2nn35aTz/9tDp37qwVK1ac/RoAAAAAAACcA06HJpKUlJSkpKQk28dOD0JCQ0NljDmTxQAAAAAAAFSaC/LqOQAAAAAAAJWN0AQAAAAAAMAGoQkAAAAAAIANQhMAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwQWgCAAAAAABgg9AEAAAAAADABqEJAAAAAACADUITAAAAAAAAG4QmAAAAAAAANghNAAAAAAAAbBCaAAAAAAAA2CA0AQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgA1CEwAAAAAAABuEJgAAAAAAADYITQAAAAAAAGwQmgAAAAAAANggNAEAAAAAALBBaAIAAAAAAGCD0AQAAAAAAMAGoQkAAAAAAIANQhMAAAAAAAAbhCYAAAAAAAA2CE0AAAAAAABsEJoAAAAAAADYIDQBAAAAAACwQWgCAAAAAABgg9AEAAAAAADABqEJAAAAAACAjTMKTaZPn67Q0FB5eXkpOjpa69evL7X9W2+9pcsvv1xeXl4KDw/X0qVLz6hYAAAAAACA88Xp0GThwoVKTk7W+PHjtWnTJkVERCg+Pl4HDhywbb9mzRr16dNHAwYM0ObNm9WzZ0/17NlTW7ZsOeviAQAAAAAAzhWnQ5PU1FQNGjRIiYmJatGihWbMmCEfHx/Nnj3btv2zzz6r7t2766GHHlJYWJgmTpyo1q1b64UXXjjr4gEAAAAAAM4Vd2ca5+fna+PGjRo9erQ1zdXVVXFxcVq7dq3tPGvXrlVycrLDtPj4eL3//vslLicvL095eXnW/WPHjkmScnJynCkXwHlSmHeisktABcpxMZVdAipYwe8FlV0CKtDxArbnPwnHt8CFq+j/0xiOjS5lToUmhw4dUkFBgQIDAx2mBwYGatu2bbbzZGVl2bbPysoqcTkpKSmaMGFCsekhISHOlAsAOAMBlV0AzoGMyi4AFahdZReAihXAXhe40P32228K4H/1kuVUaHK+jB492uHslMLCQh05ckQ1a9aUi4tLJVYGAMDFJScnRyEhIfrpp5/k7+9f2eUAAHDRMMbot99+U926dSu7FFQip0KTWrVqyc3NTdnZ2Q7Ts7OzFRQUZDtPUFCQU+0lydPTU56eng7TqlWr5kypAADgb/z9/QlNAABwEmeYwKmBYD08PNSmTRulpaVZ0woLC5WWlqaYmBjbeWJiYhzaS9Inn3xSYnsAAAAAAIALgdM/z0lOTlZCQoKioqLUrl07TZs2Tbm5uUpMTJQk9evXT/Xq1VNKSookaejQoercubOeeeYZXX/99VqwYIG+/vprvfzyyxW7JgAAAAAAABXI6dCkd+/eOnjwoMaNG6esrCxFRkZq2bJl1mCvmZmZcnX96wSW9u3ba/78+RozZowefvhhNW3aVO+//76uvPLKilsLAABgy9PTU+PHjy/2s1cAAACUzcVw/SQAAAAAAIBinBrTBAAAAAAA4FJBaAIAAAAAAGCD0AQAAAAAAMAGoQmcEhsbq2HDhlV2GSUKDQ3VtGnTLpp+K5KLi4vef//9yi7jnHj00UcVGRlZoX3u3btXLi4uSk9Pr9B+LzR/f11cDOv86KOPKjAw8B/9er7QnYv/t4rWv39/9ezZs7LLOOfO5P/gQt5+WVlZuuaaa+Tr66tq1aqdUR/lOQ45ceKEbr75Zvn7+8vFxUW//vrrGS3rQvTyyy8rJCRErq6u5T4uOf05O9tjmvNxLHg+jrvmzp17xq9DAJcWQhM45d1339XEiRPL1fZi+IBWmnnz5qljx46SpA0bNmjw4MHlnnfFihUX5YHazJkz1alTJ1WvXl3Vq1dXXFyc1q9fX+75+/fvLxcXF4db9+7dS51n7ty5xeYpuh04cOBsV+mcK+/B4x9//KH+/fsrPDxc7u7ul8QHPmdlZGRowoQJeumll7R//35de+21Z91nZR0Uf/HFFwoJCTnvyz1Tv//+u3x9fbVr1y4NHz5caWlpTs1/MQTLp/vmm2/Up08fhYSEyNvbW2FhYXr22WfLPX/Rfv70W1ZWVrnmDw0NtZ1/yJAhTq/L37dfZUhMTNSYMWPKbDd16lTt379f6enp2rFjxzl7r5w3b55WrlypNWvWaP/+/QoICNC7776rbt26qWbNmqUem6xdu1ZdunSRr6+v/P39dfXVV+v333+vkLrO9jWXk5OjpKQkjRw5Uj///LNTxyXl0ahRI3366acV2ueFrHfv3tqxY4dT8zgTGH388ce66qqrVLVqVdWuXVs333yz9u7d69Dm9ddfV0REhHx8fBQcHKw777xThw8fdqomAOee05ccxqWtRo0albLckydPqkqVKud1mYsWLdINN9wgSapdu/Z5XXYRY4wKCgrk7n5+/lVXrFihPn36qH379vLy8tKTTz6pbt266fvvv1e9evXK1Uf37t01Z84c635Zlznt3bt3sWClf//++uOPP1SnTh3nV+ICVVBQIG9vb91///165513KrucC9Lu3bslSTfeeKNcXFwquRpHBQUFcnFxkatr+b5rWLRokXr06HGOq6o4n3zyiRo2bKgmTZpIkvz8/Cqljvz8fHl4eJyXZW3cuFF16tTRa6+9ppCQEK1Zs0aDBw+Wm5ubkpKSyt3P9u3b5e/vb90v735rw4YNKigosO5v2bJF11xzjXr16lX+lfj/Tt9+Z8uZ99yCggJ9+OGHWrJkSZltd+/erTZt2qhp06aSpK1bt55VnaUtJywsTFdeeaU1LTc3Vx07dtStt96qQYMG2c63du1ade/eXaNHj9bzzz8vd3d3ffPNN+X+vy/L2b7mMjMzdfLkSV1//fUKDg6ukJqKfPvttzp69Kg6d+5cof1eyLy9veXt7X1O+t6zZ49uvPFGJScn6/XXX9exY8f0wAMP6KabbtKmTZskSatXr1a/fv00depU9ejRQz///LPuvvtuDRo0SO++++45qQvAGTKAEzp37myGDh1qjDGmYcOGZtKkSSYxMdH4+fmZkJAQ89JLL1ltJTncOnfubD02c+ZMc/nllxtPT0/TvHlzM336dOuxPXv2GElmwYIF5uqrrzaenp5mzpw5JiEhwdx4441mypQpJigoyNSoUcPce++9Jj8/35q3YcOGZurUqQ41zJgxw1x//fXG29vbXH755WbNmjVm586dpnPnzsbHx8fExMSYXbt2Oazn77//bnx9fU1GRkaJ/c6cOdP07NnTeHt7myZNmphFixY51P/3W0JCgjHGmIKCAvPEE0+Y0NBQ4+XlZVq2bGneeustq9/PP//cSDJLly41rVu3NlWqVDGff/656dy5s7nvvvvMQw89ZKpXr24CAwPN+PHjHWqWZN577z2HGhYuXGg6duxovLy8TFRUlNm+fbtZv369adOmjfH19TXdu3c3Bw4cKHF7nzp1ylStWtXMmzevxDZ/V7SNzsaBAwdMlSpVzKuvvmpNGz9+vImIiDAzZsww9evXN97e3qZXr17m119/LbWvgoIC8+STT5rGjRsbDw8PExISYh5//HFjzF/P0TvvvGNiY2ONt7e3admypVmzZo01/6FDh8xtt91m6tata7y9vc2VV15p5s+f77C+p2/rPXv2lLmOJT1PRes5a9YsExISYnx9fc0999xjTp06ZZ588kkTGBhoateuba1Dedi9LjZv3myM+XP73nnnndbrsVmzZmbatGm2tU6aNMnUqVPHBAQEmAkTJpiTJ0+a4cOHm+rVq5t69eqZ2bNnO8w3YsQI07RpU+Pt7W0aNWpkxowZ4/C/arfupz+Xxhizfv16ExcXZ2rWrGn8/f3N1VdfbTZu3Ogw79GjR83gwYNNnTp1jKenp7niiivMBx98YP0//f1W9H9z5MgR07dvX1OtWjXj7e1tunfvbnbs2GH1OWfOHBMQEGAWLVpkwsLCjJubm/niiy+Mu7u72b9/v8Pyhw4dajp27OgwrXHjxuajjz4yxhjz1ltvmSuvvNJ4eXmZGjVqmK5du5rjx4+Xa/369Oljbr31Voe+8/PzTc2aNa3/y7L2K0eOHDG33367qVWrlvHy8jJNmjQptr3uvPNOM3LkSGtbREREWI+Vte/t3Lmz7bYzxpiVK1da+6D69eub++67z1p3Y/7ctz722GOmb9++pmrVqiYhIcF67pctW2Yuv/xy4+vra+Lj480vv/xSrKYinTt3NklJSWbo0KGmWrVqpk6dOubll182x48fN/379zd+fn6mcePGZunSpaY09957r/nXv/5VapsiRa+vo0ePlqt9WYYOHWoaN25sCgsLrWmSzH//+1/TvXt34+XlZRo1auSwbYuUtv0KCgrMhAkTTL169YyHh4eJiIiwXpvGlPyeW9b+r8iXX35pgoODTWFhocnLyzNDhgwxQUFBxtPT0zRo0MA88cQTxpg/t/Xp74slvVeWpnPnzmbIkCFmyJAhxt/f39SsWdOMGTPGet5Ofz3+/djj7+tbtB/8u+joaDNmzJgya7BT1O8bb7xhYmJirH3RihUrSp2vvK+5OXPm2L7f2L2fDB061GG9/37sZkzxYxpjjHnsscdM7969rfurVq0ynTt3Nt7e3qZatWqmW7du5siRI1Z/ZR2THD161AwYMMDUqlXLVK1a1fzrX/8y6enpDm0WL15soqKijKenp6lZs6bp2bNniTXOnDnTBAQEmE8//dSqobTXgTHl388XKfrfefXVV03Dhg2Nv7+/6d27t8nJyTHGOPee/9Zbbxl3d3dTUFDgsL4uLi7WvnPKlCnmsssuc5jvueeeM/Xq1bPtE0DlITSBU04PTWrUqGGmT59udu7caVJSUoyrq6vZtm2bMebPDwOSzKeffmr2799vDh8+bIwx5rXXXjPBwcHmnXfeMT/88IN55513TI0aNczcuXONMX8deISGhlptfvnlF5OQkGD8/f3N3XffbTIyMswHH3xgfHx8zMsvv2zVZxdu1KtXzyxcuNBs377d9OzZ04SGhpouXbqYZcuWma1bt5qrrrrKdO/e3WE9P/zwQ9OsWbNS+61fv76ZP3++2blzp7n//vuNn5+fOXz4sDl16pR55513jCSzfft2s3//fuvD/eOPP24uv/xys2zZMrN7924zZ84c4+npaR1UFR2Et2zZ0ixfvtzs2rXLHD582HTu3Nn4+/ubRx991OzYscPMmzfPuLi4mOXLlzvUdPqH46JlFa1nmzZtTGxsrFm1apXZtGmTadKkibn77rtL3N45OTnGy8vLfPDBB+V5eZiEhAQTEBBgateubZo1a2buvvtuc+jQoXLNW+Tpp582AQEB5sSJE9a08ePHG19fX9OlSxezefNm88UXX5gmTZqY22+/vdS+RowYYapXr27mzp1rdu3aZVauXGlmzpxpjHF8jj788EOzfft2c8stt5iGDRuakydPGmOM2bdvn5kyZYrZvHmz2b17t3nuueeMm5ubWbdunTHGmF9//dXExMSYQYMGmf3795v9+/ebU6dOlet5Kik08fPzM7fccov5/vvvzeLFi42Hh4eJj4839913n9m2bZuZPXu2kWS++uqrcj2fpYUm+fn5Zty4cWbDhg3mhx9+MK+99prx8fExCxcudKi1atWqZsiQIWbbtm1m1qxZRpKJj483kyZNMjt27DATJ040VapUMT/99JM138SJE83q1avNnj17zOLFi01gYKB58sknS6zzt99+sz4UFD2XxhiTlpZm/ve//5mMjAyzdetWM2DAABMYGGgdxBYUFJirrrrKXHHFFWb58uVm9+7d5oMPPjBLly41eXl5Ztq0acbf39/q87fffjPGGHPDDTeYsLAw8+WXX5r09HQTHx9vmjRpYh3Mzpkzx1SpUsW0b9/erF692mzbts3k5uaaZs2amaeeesqqOz8/39SqVcshhNiyZYupWrWqycvLM7/88otxd3c3qampZs+ePebbb78106dPt+ooa/0+/PBD4+3tbbU3xpgPPvjAeHt7W23K2q8MGTLEREZGmg0bNpg9e/aYTz75xCxevNjqr6CgwNSpU8cKDO1Ck9L2vYcPHzb169c3jz32mMO227Vrl/H19TVTp041O3bsMKtXrzatWrUy/fv3t/ou+mDy9NNPm127dpldu3ZZz31cXJzZsGGD2bhxowkLC3P4f7cLTapWrWomTpxovSbd3NzMtddea15++WWzY8cOc88995iaNWua3NzcEl+Hd9xxh7n55ptLfPzvivbXDRs2NEFBQSYuLs6sWrWqXPOeLi8vz9SsWdNMmjTJYbokU7NmTTNz5kyzfft2M2bMGOPm5ma2bt1qtSlr+6Wmphp/f3/zxhtvmG3btpkRI0aYKlWqWB8eS3rPLWv/V2T48OFm8ODBxpg/PwSGhISYL7/80uzdu9esXLnSCloOHDhgunfvbm699VbrfbGk98rSdO7c2fj5+ZmhQ4eabdu2Wfutv78eBw0aZGJiYhyOPYqUFJpkZ2cbSea5554zMTExpk6dOubqq682K1euLLOmv/dbv3598/bbb5utW7eagQMHmqpVq5b6Plje19yJEyfMp59+aiSZ9evXW+83FRWaREVFWdtq8+bNxtPT09xzzz0mPT3dbNmyxTz//PPm4MGDVn9lHZPExcWZHj16mA0bNpgdO3aYBx980NSsWdPaHh9++KFxc3Mz48aNM1u3bjXp6elWwHZ6jU8++aSpWbOmw2uvrNeBMeXbz58emvj5+ZmbbrrJfPfdd+bLL780QUFB5uGHHzbGOPee/8MPPxgPDw/zyiuvmFOnTplff/3V9OrVy1xzzTVWm1WrVpkqVaqYJUuWmMLCQpOVlWWuvvpqM2jQINs+AVQeQhM45fTQ5D//+Y/1WGFhoalTp4558cUXjTElH5g0bty42LdVEydONDExMQ7z2X3j3bBhQ4c3qF69ejl8M2IXbvz9W6O1a9caSWbWrFnWtDfeeMN4eXk5LGvQoEFm+PDh5e73+PHjRpL17Z3dN5B//PGH8fHxcTiTwRhjBgwYYPr06eMw3/vvv+/QpnPnzsW+yW7btq31zWJRTad/OH7llVcc1lOSSUtLs6alpKSY5s2bm5Lcc8895rLLLjO///57iW3+7o033jCLFi0y3377rXnvvfdMWFiYadu2bbmChCJhYWHmnnvucZg2fvx44+bmZvbt22dN++ijj4yrq2uxb/2L5OTkGE9PTyskOZ3dc/T9998bSdYZRnauv/568+CDD1r3Tz8YLY/SQhMfHx/rw7AxxsTHx5vQ0FCHb6uaN29uUlJSyrWs0kITO0OGDHE4gC/6vzt9+Z06dbLunzp1yvj6+po33nijxH6nTJli2rRpU2qt7733nsNZCnYKCgpM1apVrSDv448/Nq6urmb79u227U8/KDbGmB07dhhJZvXq1da0Q4cOGW9vb/Pmm29a80kq9s3ok08+acLCwqz777zzjvHz83M4e2LSpEnmlltuMcYYs3HjRiPJ7N27t9T1Kmn9Tp48aWrVquVw5lWfPn2s/V559is9evQwiYmJJS5z9erVpk6dOtY2tgtNnN33FtVQ9GG6yMqVK42rq6u1T2nYsKHDt8vG/PXc//0MwOnTp5vAwECHmk4PTf6+jyx6Tfbt29eatn//fiPJrF27tsTnwd3d3Xz88ce2j59u27ZtZsaMGebrr782q1evNomJicbd3b3YmVDlsXDhQuPm5mZ+/vlnh+mSigXb0dHRDvvIsrZf3bp1i4Uxbdu2Nffee68xpuT3XDun7/+MMaZp06bmww8/NMYYc99995kuXbo4fNv/dzfeeKPD2SRncrZO586dTVhYmMMyRo4c6fB/eXpo8Hcl7QeLjg9q1KhhZs+ebTZt2mSGDRtmPDw8HM5OKElRv5MnT7amnTx50tSvX7/EwNjZ19zmzZuLnd1QEaHJvn37jIeHh7Ud+vTpYzp06FBiHWUdk6xcudL4+/ubP/74w6FN48aNrTOSY2JizB133FHiMopqHDFihAkODjZbtmwpVkNpr4Py7udPD01Ofw9+6KGHTHR0tMNyy/uev2LFClOnTh3j5uZmJJmYmJhir/U333zT+Pn5GXd3dyPJ9OjRo9SzMgFUDgaCxVlp2bKl9beLi4uCgoJKHbwzNzdXu3fv1oABA+Tn52fdHn/8cWs8gyJRUVHF5r/iiivk5uZm3Q8ODi5zsNC/1xgYGChJCg8Pd5j2xx9/KCcnR9Kf44h88MEH1ngm5em3aMC40mrZtWuXTpw4oWuuucZh3V999dVyrfvflydV3LqX1MfkyZO1YMECvffee/Ly8ip1OUVuu+023XDDDQoPD1fPnj314YcfasOGDVqxYoUk6dprr7XW+4orrig2/9q1a5WRkaEBAwYUe6xBgwYO46rExMSosLBQ27dv18qVKx2e09dff10ZGRnKy8tT165dS635789R0W/Ei56TgoICTZw4UeHh4apRo4b8/Pz08ccfKzMzs9Q+r7jiCqsWZwczDQ0NVdWqVa37gYGBatGihcNv6kvbbs6aPn262rRpo9q1a8vPz08vv/xysfW74oorii3/768jNzc31axZ06GmhQsXqkOHDgoKCpKfn5/GjBlj9ZuZmemwvZ544okS68vOztagQYPUtGlTBQQEyN/fX8ePH7f6Sk9PV/369dWsWbNyr3NGRobc3d0VHR1tTatZs6aaN2+ujIwMa5qHh0ex/7v+/ftr165d+uqrryT9OdDsrbfeKl9fX6vN38dDioiIUNeuXRUeHq5evXpp5syZOnr0aLnXz93dXbfeeqtef/11SX/uQxctWqQ77rhDUvn2K/fcc48WLFigyMhIjRgxQmvWrHFYp0WLFun//u//Sh234Uz2vd98843mzp3rUFd8fLwKCwu1Z88eq53d/s7Hx0eNGzd2anl/31ZFr8nT93eSbPvZsmWLbrzxRo0fP17dunUrdTlFmjdvrrvuuktt2rRR+/btNXv2bLVv315Tp04t1/x/N2vWLF177bWqW7duscdiYmKK3f/767S07ZeTk6NffvlFHTp0cJjeoUMHhz6k4tuhPPu/jIwM/fLLL9Z+tn///kpPT1fz5s11//33a/ny5eV8Bpxz1VVXOYx7FBMTo507dzqMEeOswsJCSdJdd92lxMREtWrVSlOnTlXz5s01e/bscvfz9+3l7u6uqKioYs+1dGavuXNl8eLF6tixozVodnp6ulPvnZLj/+g333yj48ePq2bNmg7//3v27LH2S+VZxjPPPKOZM2dq1apVtscMpb0OyrufP93p78Hl2ffYvednZWVp0KBBSkhI0IYNG/TFF1/Iw8NDt9xyi4wxkv4c02fo0KEaN26cNm7cqGXLlmnv3r26++67S10egPOPgWBxVk4fKM7FxcU68LBz/PhxSX9epeXvb2SSHA7IJTl8CDnT5Z0+T9Gbq920on7Wr1+vU6dOqX379uXutzy1FK37kiVLig2qevpgqedz3e36ePrppzV58mR9+umnxQ6MnHHZZZepVq1a2rVrl7p27apXXnnFugqB3SCDr7zyiiIjI9WmTRunlhMVFeVwJYTAwMBiI9SXpLTXwpQpU/Tss89q2rRpCg8Pl6+vr4YNG6b8/PxS+1y6dKlOnjwpSU4PMme3nc9k25fHggULNHz4cD3zzDOKiYlR1apVNWXKFK1bt+6salq7dq3uuOMOTZgwQfHx8QoICNCCBQv0zDPPSJLq1q3rsL1KG2A6ISFBhw8f1rPPPquGDRvK09NTMTEx1jY4V4P4FfV9+oC0derUUY8ePTRnzhw1atRIH330kRUKStL+/fu1efNmXX/99ZL+3K998sknWrNmjZYvX67nn39ejzzyiNatW6dGjRqVuX6SdMcdd6hz5846cOCAPvnkE3l7e1uDJ5dnv3Lttdfqxx9/1NKlS/XJJ5+oa9euGjJkiJ5++mlJf35gmjx5cqnPxZm8Bo8fP6677rpL999/f7HHGjRoYP1d3v1d0QcNZ2os7f+7yNatW9W1a1cNHjy4XFeAKU27du20atUqp+b58ccf9emnn57xwI/l2X7lcfp2KM/+b/HixbrmmmusYL1169bas2ePPvroI3366ae69dZbFRcXp7fffvus6zvXikLzFi1aOEwPCwsrMyh3VkW+5lxdXYv9bxS9/5TX4sWLHb4oKs9+tbR9wvHjxxUcHOywbyxSFMyUZxmdOnXSkiVL9Oabb2rUqFFltq8IZ7Kvs3vPnz59ugICAvTUU09Z7YoGAF63bp2uuuoqpaSkqEOHDnrooYck/RlE+fr6qlOnTnr88ccrfLBfAGeO0ATnTNEVEP7+zU9gYKDq1q2rH374wfqm9EKzaNEiXX/99cVCHGfYrXuLFi3k6empzMzMC3p0+qeeekqTJk3Sxx9/bPsNsDP27dunw4cPW2/8pV2B5/jx43rzzTeVkpJi+3hmZqZ++eUX65vYr776Sq6urmrevLm8vb2LXTWiadOm8vb2VlpamgYOHHhG9a9evVo33nij/vOf/0j688PWjh07HA6qPTw8in272bBhwzNa3vm2evVqtW/fXvfee6817fSzns7EmjVr1LBhQz3yyCPWtB9//NH6293dvdxX+Vi9erX++9//6rrrrpMk/fTTTzp06JD1eMuWLbVv3z7t2LHD9mwTu+0TFhamU6dOad26dVY4evjwYW3fvr3YByY7AwcOVJ8+fVS/fn01btzY4Vv8Dz74QO3bt3cIglxcXNShQwd16NBB48aNU8OGDfXee+8pOTm5zPWTpPbt2yskJEQLFy7URx99pF69elkH9uXdr9SuXVsJCQlKSEhQp06d9NBDD+npp5/Wzp079eOPP+qaa64pc71LY/c8t27dWlu3bq2wK7qcC99//726dOmihIQETZo06az7S09Pd/qDzpw5c1SnTh0raDvdV199pX79+jncb9WqlSSVuf38/f1Vt25drV692uH1sXr1arVr167Uusqz/1u0aFGxy976+/urd+/e6t27t2655RZ1795dR44csQ1H7d4ry+P0YPerr75S06ZNz+p9OzQ0VHXr1tX27dsdpu/YscOpMwa/+uorXX311ZKkU6dOaePGjQ5Xxqno11zt2rW1ZcsWh2np6enlvvrR8ePH9fnnn+vFF1+0prVs2VJpaWmaMGHCGdXUunVrZWVlyd3dXaGhobZtipaRmJhYYj/t2rVTUlKSunfvLnd3dw0fPtzh8dJeB2e7ny9Jed/zT5w4Uezsr6LXZ1EIc+LEiWJXRyxqU1ZIDOD8IjTBOVOnTh15e3tr2bJlql+/vry8vBQQEKAJEybo/vvvV0BAgLp37668vDx9/fXXOnr0qJKTkyu7bC1evFiPPfbYWfXRsGFDubi46MMPP9R1110nb29vVa1aVcOHD9cDDzygwsJCdezYUceOHdPq1avl7++vhISEClqDM/fkk09q3Lhxmj9/vkJDQ5WVlSVJ1mmnpTl+/LgmTJigm2++WUFBQdq9e7dGjBihJk2aKD4+vsxlL1y4UKdOnbIO0E/n5eWlhIQEPf3008rJydH999+vW2+9VUFBQSW2HzlypEaMGCEPDw916NBBBw8e1Pfff2/78x87TZs21dtvv601a9aoevXqSk1NVXZ2tsMBV2hoqNatW6e9e/fKz89PNWrUKPFnDlu3blV+fr6OHDmi3377zTrbIjIyslz1VLSmTZvq1Vdf1ccff6xGjRrpf//7nzZs2KBGjRqddb+ZmZlasGCB2rZtqyVLlui99947477+97//KSoqSjk5OXrooYccvqHs3Lmzrr76at18881KTU1VkyZNtG3bNrm4uKh79+4KDQ3V8ePHlZaWpoiICPn4+Khp06a68cYbNWjQIL300kuqWrWqRo0apXr16unGG28ss6b4+Hj5+/vr8ccfL7avOP0b23Xr1iktLU3dunVTnTp1tG7dOh08eFBhYWHlWr8it99+u2bMmKEdO3bo888/t6aXZ78ybtw4tWnTRldccYXy8vL04YcfWstftGiR4uLi5OPj49yGOU1oaKi+/PJL3XbbbfL09FStWrU0cuRIXXXVVUpKStLAgQPl6+urrVu36pNPPtELL7xwVsurCFu2bFGXLl0UHx+v5ORka3/n5uZWrsvMT5s2TY0aNdIVV1yhP/74Q6+88oo+++wzp36SUlhYqDlz5ighIaHES8u/9dZbioqKUseOHfX6669r/fr1mjVrlqTybb+HHnpI48ePV+PGjRUZGak5c+YoPT3d+slXScra/x04cEBff/21Fi9ebM2Tmpqq4OBgtWrVSq6urnrrrbcUFBRknV1wOrv3yvJc7jozM1PJycm66667tGnTJj3//PPWmWwlOXLkiBW+S7LCkaCgIAUFBcnFxcV6riIiIhQZGal58+Zp27ZtTp0pM336dDVt2lRhYWGaOnWqjh49qjvvvFPS2b/m7HTp0kVTpkzRq6++qpiYGL322mvasmWLFayVZdmyZWrWrJlDuDF69GiFh4fr3nvv1d133y0PDw99/vnn6tWrl2rVqlVmn3FxcYqJiVHPnj311FNPqVmzZvrll1+0ZMkS/fvf/1ZUVJTGjx+vrl27qnHjxrrtttt06tQpLV26VCNHjnToq3379lq6dKmuvfZaubu7a9iwYdZjpb0OznY/X5Lyvudff/31mjp1qh577DH16dNHv/32mx5++GE1bNjQ2jY9evTQoEGD9OKLLyo+Pl779+/XsGHD1K5dO9uf6gGoPIxpgnPG3d1dzz33nF566SXVrVvXepMaOHCgXnnlFc2ZM0fh4eHq3Lmz5s6de9Yf1CrC7t27tWvXrnJ9yC9NvXr1NGHCBI0aNUqBgYHWt0wTJ07U2LFjlZKSorCwMHXv3l1Lliy5INZdkl588UXl5+frlltuUXBwsHUrOo2/NG5ubvr22291ww03qFmzZhowYIDatGmjlStXFvv5kZ1Zs2bppptuKvHgukmTJrrpppt03XXXqVu3bmrZsqX++9//ltrn2LFj9eCDD2rcuHEKCwtT7969nRoLZMyYMWrdurXi4+MVGxuroKAg9ezZ06HN8OHD5ebmphYtWqh27dqlnsZ93XXXqVWrVvrggw+0YsUKtWrVqtwHtufCXXfdpZtuukm9e/dWdHS0Dh8+7HDWyZm64YYb9MADDygpKUmRkZFas2aNxo4de0Z9zZo1S0ePHlXr1q3Vt29f3X///apTp45Dm3feeUdt27ZVnz591KJFC40YMcL6JrB9+/a6++671bt3b9WuXds6VXrOnDlq06aN/u///k8xMTEyxmjp0qXl+nbW1dVV/fv3V0FBgcMZALm5uUpLS3MITfz9/fXll1/quuuuU7NmzTRmzBg988wz1jfX5Vk/6c+f6GzdulX16tUrNj5FWfsVDw8PjR49Wi1bttTVV18tNzc3LViwQJLj+Ctn47HHHtPevXvVuHFj68Nfy5Yt9cUXX2jHjh3q1KmTWrVqpXHjxl0wHwbefvttHTx4UK+99prD/q5t27blmj8/P18PPvig9T72zTff6NNPPy1znIa/+/TTT5WZmWl9qLYzYcIELViwQC1bttSrr76qN954wwouyrP97r//fiUnJ1u1Llu2TIsXL1bTpk1Lna+s/d8HH3ygdu3aOXyIrlq1qp566ilFRUWpbdu22rt3r5YuXVpikFzSe2VZ+vXrp99//13t2rXTkCFDNHTo0GJnvJxu8eLFatWqlXVGz2233aZWrVppxowZVpthw4Zp9OjReuCBBxQREaG0tDR98sknDuPrlGXy5MmaPHmyIiIitGrVKi1evNh6js72NWcnPj5eY8eO1YgRI9S2bVv99ttvDvulsti9hpo1a6bly5frm2++Ubt27RQTE6NFixaVGOydzsXFRUuXLtXVV1+txMRENWvWTLfddpt+/PFHa2yh2NhYvfXWW1q8eLEiIyPVpUsXrV+/3ra/jh07asmSJRozZoyef/55a3pZr4Oz2c+XpLzv+V26dNH8+fP1/vvvq1WrVurevbs8PT21bNkyKxjv37+/UlNT9cILL+jKK69Ur1691Lx58zP+qR6Ac8fFcP4XYElNTdWnn36qpUuXVnYpAC5gAwYM0MGDBx2+ZX/33Xc1ZswYbd26tRIrK79Dhw4pODhY+/btsz7I4OJR2dvvhhtuUMeOHTVixIjzvuwL0d69e9WoUSNt3ry50s4edNapU6cUGBiojz76qMyfa11oYmNjFRkZqWnTplV2KQAuAfw8B/ib+vXra/To0ZVdBoAL1LFjx/Tdd99p/vz5DoGJ9OfP2J588slKqsx5R44cUWpqKoHJRaqyt1/Hjh3Vp0+fSlk2KsaRI0f0wAMPnNWZLgBwKeBMEwDlsnLlylIHwyu6igfOn9dff1133XWX7WMNGzbU999/f54r+ueLjY3V+vXrddddd53RpWVx8bjiiiscBjD+u5deeqnUwcwzMzNLHWxy69atDlcQwp8u5OftiSeeKPHy6J06ddKLL7541meanM1r7lLDmSYAzidCEwDl8vvvv+vnn38u8fEL+QoZ/1S//fabsrOzbR+rUqXKRXMVH+BC9OOPP5Z46dbAwEBVrVq1xHlPnTpV6mXPQ0NDyz0+xKXkQn7ejhw5oiNHjtg+5u3tXerV4crrbF5zAIBzh9AEAAAAAADABlfPAQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAl6DY2FgNGzas3O3nzp2ratWqnbN6AAAALkSEJgAAAAAAADYITQAAAAAAAGwQmgAAcAGJjY3Vfffdp2HDhql69eoKDAzUzJkzlZubq8TERFWtWlVNmjTRRx99ZM3zxRdfqF27dvL09FRwcLBGjRqlU6dOWY/n5uaqX79+8vPzU3BwsJ555pliy83Ly9Pw4cNVr149+fr6Kjo6WitWrDgfqwwAAHDBIjQBAOACM2/ePNWqVUvr16/Xfffdp3vuuUe9evVS+/bttWnTJnXr1k19+/bViRMn9PPPP+u6665T27Zt9c033+jFF1/UrFmz9Pjjj1v9PfTQQ/riiy+0aNEiLV++XCtWrNCmTZsclpmUlKS1a9dqwYIF+vbbb9WrVy91795dO3fuPN+rDwAAcMFwMcaYyi4CAAD8KTY2VgUFBVq5cqUkqaCgQAEBAbrpppv06quvSpKysrIUHBystWvX6oMPPtA777yjjIwMubi4SJL++9//auTIkTp27JhOnDihmjVr6rXXXlOvXr0kSUeOHFH9+vU1ePBgTZs2TZmZmbrsssuUmZmpunXrWrXExcWpXbt2euKJJzR37lwNGzZMv/766/l9QgAAACqRe2UXAAAAHLVs2dL6283NTTVr1lR4eLg1LTAwUJJ04MABZWRkKCYmxgpMJKlDhw46fvy49u3bp6NHjyo/P1/R0dHW4zVq1FDz5s2t+999950KCgrUrFkzhzry8vJUs2bNCl8/AACAiwWhCQAAF5gqVao43HdxcXGYVhSQFBYWVsjyjh8/Ljc3N23cuFFubm4Oj/n5+VXIMgAAAC5GhCYAAFzEwsLC9M4778gYY4Upq1evVtWqVVW/fn3VqFFDVapU0bp169SgQQNJ0tGjR7Vjxw517txZktSqVSsVFBTowIED6tSpU6WtCwAAwIWGgWABALiI3Xvvvfrpp5903333adu2bVq0aJHGjx+v5ORkubq6ys/PTwMGDNBDDz2kzz77TFu2bFH//v3l6vrXIUCzZs10xx13qF+/fnr33Xe1Z88erV+/XikpKVqyZEklrh0AAEDl4kwTAAAuYvXq1dPSpUv10EMPKSIiQjVq1NCAAQM0ZswYq82UKVN0/Phx9ejRQ1WrVtWDDz6oY8eOOfQzZ84cPf7443rwwQf1888/q1atWrrqqqv0f//3f+d7lQAAAC4YXD0HAAAAAADABj/PAQAAAAAAsEFoAgAAAAAAYIPQBAAAAAAAwAahCQAAAAAAgA1CEwAAAAAAABuEJgAAAAAAADYITQAAAAAAAGwQmgAAAAAAANggNAEAAAAAALBBaAIAAAAAAGCD0AQAAAAAAMAGoQkAAAAAAICN/wcQbNQ33nSCpQAAAABJRU5ErkJggg==","text/plain":["<Figure size 1200x500 with 1 Axes>"]},"metadata":{},"output_type":"display_data"}],"source":["# plot metrics for each model\n","import matplotlib.pyplot as plt\n","\n","fig, ax = plt.subplots(1, 1, figsize=(12, 5))\n","\n","perf_df.plot(x=\"model\", y=[\"accuracy\", \"precision\", \"recall\", \"f1\"], kind=\"bar\", ax=ax)\n","\n","# add values on top of bars\n","for p in ax.patches:\n"," ax.annotate(\n"," f\"{p.get_height():.3f}\",\n"," (p.get_x() + p.get_width() / 2, p.get_height()),\n"," ha=\"center\",\n"," va=\"bottom\",\n"," fontsize=10,\n"," )\n","\n","# add title and labels\n","# ax.set_title(\"Metrics for different settings\")\n","# ax.set_ylabel(\"Value\")\n","# ax.set_xlabel(\"Epoch (0: base model, 1-4: fine-tuned models)\")\n","# rotate x labels\n","plt.xticks(rotation=0)\n","\n","# set legend at the right to avoid overlapping with bars\n","plt.legend(loc=\"center left\", bbox_to_anchor=(1.0, 0.5))\n","# plt.tight_layout()\n","\n","plt.show()"]},{"cell_type":"code","execution_count":14,"metadata":{},"outputs":[],"source":["perf_df.to_csv(\"results/mgtv-results_p2_full_metrics.csv\", index=False)"]}],"metadata":{"accelerator":"GPU","application/vnd.databricks.v1+notebook":{"dashboards":[],"environmentMetadata":null,"language":"python","notebookMetadata":{"pythonIndentUnit":4},"notebookName":"07_MAC_+_Qwen2-7B-Instructi_Unsloth_train","widgets":{}},"colab":{"gpuType":"T4","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.9"}},"nbformat":4,"nbformat_minor":0}

competition/14_GLM-4_M3_eval.ipynb CHANGED Viewed

@@ -1 +1 @@

- {"cells":[{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":476,"status":"ok","timestamp":1720679526275,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"uWKRSV6eZsCn"},"outputs":[],"source":["%load_ext autoreload\n","%autoreload 2"]},{"cell_type":"code","execution_count":2,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"eb33b19f-1206-41ee-84e2-e6258a12eef7","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2534,"status":"ok","timestamp":1720679529344,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"xwFh14uiZBrI","outputId":"d767799c-34c2-46a5-f052-378146a55321"},"outputs":[],"source":["from pathlib import Path\n","\n","try:\n"," from google.colab import drive\n","\n"," drive.mount(\"/content/drive\")\n"," workding_dir = \"/content/drive/MyDrive/logical-reasoning/\"\n","except ModuleNotFoundError:\n"," workding_dir = str(Path.cwd().parent)"]},{"cell_type":"code","execution_count":3,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"6d394937-6c99-4a7c-9d32-7600a280032f","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":5,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"G5pNu3zgZBrL","outputId":"160a554f-fb08-4aa0-bc00-0422fb7c1fac"},"outputs":[{"name":"stdout","output_type":"stream","text":["workding dir: /Users/inflaton/code/engd/projects/logical-reasoning\n"]}],"source":["import os\n","import sys\n","from pathlib import Path\n","\n","os.chdir(workding_dir)\n","sys.path.append(workding_dir)\n","print(\"workding dir:\", workding_dir)"]},{"cell_type":"code","execution_count":4,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"ac667aba-076e-4de6-9984-8f6a67cb09cd","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"0dVRAabNZBrL","outputId":"b977e116-df16-47cd-9160-a24f611da687"},"outputs":[{"data":{"text/plain":["False"]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["need_to_setup_env = False\n","need_to_setup_env"]},{"cell_type":"code","execution_count":5,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"72f9cf79-7b0d-4d9e-90a0-1fa5251b947f","showTitle":false,"title":""},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"hKUOfP2HZBrL"},"outputs":[],"source":["if need_to_setup_env:\n"," %pip install -r requirements.txt\n"," %cd /content/\n"," %rm -rf LLaMA-Factory\n"," !git clone https://github.com/hiyouga/LLaMA-Factory.git\n"," %cd LLaMA-Factory\n"," %ls\n"," %pip install -e .[torch,bitsandbytes]\n"," \n"," os.chdir(workding_dir)\n"," sys.path.append(workding_dir)\n"," print(\"workding dir:\", workding_dir)"]},{"cell_type":"code","execution_count":6,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"9f67ec60-2f24-411c-84eb-0dd664b44775","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"hPCC-6m7ZBrM","outputId":"c7aa2c96-5e99-440a-c148-201d79465ff9"},"outputs":[{"name":"stdout","output_type":"stream","text":["loading env vars from: /Users/inflaton/code/engd/projects/logical-reasoning/.env\n"]},{"data":{"text/plain":["True"]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["from dotenv import find_dotenv, load_dotenv\n","\n","found_dotenv = find_dotenv(\".env\")\n","\n","if len(found_dotenv) == 0:\n"," found_dotenv = find_dotenv(\".env.example\")\n","print(f\"loading env vars from: {found_dotenv}\")\n","load_dotenv(found_dotenv, override=True)"]},{"cell_type":"code","execution_count":7,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"f1597656-8042-4878-9d3b-9ebfb8dd86dc","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"1M3IraVtZBrM","outputId":"29ab35f6-2970-4ade-d85d-3174acf8cda0"},"outputs":[{"name":"stdout","output_type":"stream","text":["THUDM/glm-4-9b-chat-1m None False datasets/mgtv results/mgtv-results_m3.csv\n"]}],"source":["import os\n","\n","model_name = os.getenv(\"MODEL_NAME\")\n","adapter_name_or_path = os.getenv(\"ADAPTER_NAME_OR_PATH\")\n","load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n","data_path = os.getenv(\"LOGICAL_REASONING_DATA_PATH\")\n","results_path = os.getenv(\"LOGICAL_REASONING_RESULTS_PATH\")\n","use_english_datasets = os.getenv(\"USE_ENGLISH_DATASETS\") == \"true\"\n","\n","print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path)"]},{"cell_type":"code","execution_count":8,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"b2a43943-9324-4839-9a47-cfa72de2244b","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":564,"status":"ok","timestamp":1720679529907,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"UgMvt6dIZBrM","outputId":"ce37581c-fd26-46c2-ad87-d933d99f68f7"},"outputs":[{"name":"stdout","output_type":"stream","text":["Python 3.11.9\n","\u001b[33mWARNING: Package(s) not found: flash-attn\u001b[0m\u001b[33m\n","\u001b[0mCPU times: user 4.12 ms, sys: 9.4 ms, total: 13.5 ms\n","Wall time: 651 ms\n"]}],"source":["%%time\n","!python --version\n","!pip show flash-attn"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1685,"status":"ok","timestamp":1720679531591,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"ZuS_FsLyZBrN","outputId":"2cba0105-c505-4395-afbd-2f2fee6581d0"},"outputs":[{"name":"stdout","output_type":"stream","text":["loading /Users/inflaton/code/engd/projects/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\n","MPS is available\n"]}],"source":["from llm_toolkit.llm_utils import *\n","from llm_toolkit.logical_reasoning_utils import *\n","\n","device = check_gpu()"]},{"cell_type":"code","execution_count":12,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["loading model: THUDM/glm-4-9b-chat-1m with adapter: None\n"]},{"name":"stderr","output_type":"stream","text":["Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"132c5a39ad11466db54e9cc820789fa1","version_major":2,"version_minor":0},"text/plain":["Downloading shards: 0%| | 0/10 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"a41d42be474b428fa9cb046fa047f52f","version_major":2,"version_minor":0},"text/plain":["model-00002-of-00010.safetensors: 91%|#########1| 1.68G/1.84G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"73fb4f921cbf4019857647180498a4b9","version_major":2,"version_minor":0},"text/plain":["model-00003-of-00010.safetensors: 0%| | 0.00/1.99G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"}],"source":["%%time\n","\n","model, tokenizer = load_model(model_name, adapter_name_or_path=adapter_name_or_path, using_llama_factory=False)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"ename":"NameError","evalue":"name 'tokenizer' is not defined","output_type":"error","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[11], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m datasets \u001b[38;5;241m=\u001b[39m load_logical_reasoning_dataset(\n\u001b[1;32m 2\u001b[0m data_path,\n\u001b[0;32m----> 3\u001b[0m tokenizer\u001b[38;5;241m=\u001b[39m\u001b[43mtokenizer\u001b[49m,\n\u001b[1;32m 4\u001b[0m chinese_prompt\u001b[38;5;241m=\u001b[39m\u001b[38;5;129;01mnot\u001b[39;00m use_english_datasets,\n\u001b[1;32m 5\u001b[0m using_p1\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 6\u001b[0m )\n","\u001b[0;31mNameError\u001b[0m: name 'tokenizer' is not defined"]}],"source":["datasets = load_logical_reasoning_dataset(\n"," data_path,\n"," tokenizer=tokenizer,\n"," chinese_prompt=not use_english_datasets,\n"," using_p1=False,\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def evaluate_model(model, tokenizer, model_name, dataset):\n"," print(f\"Evaluating model: {model_name} on {device}\")\n"," predictions = eval_model(model, tokenizer, dataset, device=device)\n","\n"," save_results(\n"," model_name,\n"," results_path,\n"," dataset,\n"," predictions,\n"," debug=False,\n"," )\n","\n"," metrics = calc_metrics(dataset[\"label\"], predictions, debug=False)\n"," print(metrics)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Evaluating model: internlm/internlm2_5-7b-chat-1m_llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88 on mps\n"]},{"name":"stderr","output_type":"stream","text":[" 0%| | 1/3000 [00:06<5:23:35, 6.47s/it]"]},{"name":"stdout","output_type":"stream","text":["--------\n","step 1: 不是</s>\n","--------\n","step 2: 不是\n","--------\n","step 3: 不是\n","--------\n","step 4: 不是\n","--------\n","step 5: 不是\n"]},{"name":"stderr","output_type":"stream","text":[" 22%|██▏ | 657/3000 [40:38<2:18:00, 3.53s/it]"]}],"source":["%%time\n","\n","evaluate_model(model, tokenizer, f\"{model_name}{'_' + adapter_name_or_path if adapter_name_or_path else ''}\", datasets[\"test\"])"]}],"metadata":{"accelerator":"GPU","application/vnd.databricks.v1+notebook":{"dashboards":[],"environmentMetadata":null,"language":"python","notebookMetadata":{"mostRecentlyExecutedCommandWithImplicitDF":{"commandId":-1,"dataframes":["_sqldf"]},"pythonIndentUnit":4},"notebookName":"10_eval-lf-medium-py3.11","widgets":{}},"colab":{"gpuType":"L4","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.9"}},"nbformat":4,"nbformat_minor":0}

+ {"cells":[{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":476,"status":"ok","timestamp":1720679526275,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"uWKRSV6eZsCn"},"outputs":[],"source":["%load_ext autoreload\n","%autoreload 2"]},{"cell_type":"code","execution_count":2,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"eb33b19f-1206-41ee-84e2-e6258a12eef7","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2534,"status":"ok","timestamp":1720679529344,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"xwFh14uiZBrI","outputId":"d767799c-34c2-46a5-f052-378146a55321"},"outputs":[],"source":["from pathlib import Path\n","\n","try:\n"," from google.colab import drive\n","\n"," drive.mount(\"/content/drive\")\n"," workding_dir = \"/content/drive/MyDrive/logical-reasoning/\"\n","except ModuleNotFoundError:\n"," workding_dir = str(Path.cwd().parent)"]},{"cell_type":"code","execution_count":3,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"6d394937-6c99-4a7c-9d32-7600a280032f","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":5,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"G5pNu3zgZBrL","outputId":"160a554f-fb08-4aa0-bc00-0422fb7c1fac"},"outputs":[{"name":"stdout","output_type":"stream","text":["workding dir: /Users/inflaton/code/engd/projects/logical-reasoning\n"]}],"source":["import os\n","import sys\n","from pathlib import Path\n","\n","os.chdir(workding_dir)\n","sys.path.append(workding_dir)\n","print(\"workding dir:\", workding_dir)"]},{"cell_type":"code","execution_count":4,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"ac667aba-076e-4de6-9984-8f6a67cb09cd","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"0dVRAabNZBrL","outputId":"b977e116-df16-47cd-9160-a24f611da687"},"outputs":[{"data":{"text/plain":["False"]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["need_to_setup_env = False\n","need_to_setup_env"]},{"cell_type":"code","execution_count":5,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"72f9cf79-7b0d-4d9e-90a0-1fa5251b947f","showTitle":false,"title":""},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"hKUOfP2HZBrL"},"outputs":[],"source":["if need_to_setup_env:\n"," %pip install -r requirements.txt\n"," %cd /content/\n"," %rm -rf LLaMA-Factory\n"," !git clone https://github.com/hiyouga/LLaMA-Factory.git\n"," %cd LLaMA-Factory\n"," %ls\n"," %pip install -e .[torch,bitsandbytes]\n"," \n"," os.chdir(workding_dir)\n"," sys.path.append(workding_dir)\n"," print(\"workding dir:\", workding_dir)"]},{"cell_type":"code","execution_count":6,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"9f67ec60-2f24-411c-84eb-0dd664b44775","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"hPCC-6m7ZBrM","outputId":"c7aa2c96-5e99-440a-c148-201d79465ff9"},"outputs":[{"name":"stdout","output_type":"stream","text":["loading env vars from: /Users/inflaton/code/engd/projects/logical-reasoning/.env\n"]},{"data":{"text/plain":["True"]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["from dotenv import find_dotenv, load_dotenv\n","\n","found_dotenv = find_dotenv(\".env\")\n","\n","if len(found_dotenv) == 0:\n"," found_dotenv = find_dotenv(\".env.example\")\n","print(f\"loading env vars from: {found_dotenv}\")\n","load_dotenv(found_dotenv, override=True)"]},{"cell_type":"code","execution_count":7,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"f1597656-8042-4878-9d3b-9ebfb8dd86dc","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"1M3IraVtZBrM","outputId":"29ab35f6-2970-4ade-d85d-3174acf8cda0"},"outputs":[{"name":"stdout","output_type":"stream","text":["THUDM/glm-4-9b-chat-1m None False datasets/mgtv results/mgtv-results_m3.csv\n"]}],"source":["import os\n","\n","model_name = os.getenv(\"MODEL_NAME\")\n","adapter_name_or_path = os.getenv(\"ADAPTER_NAME_OR_PATH\")\n","load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n","data_path = os.getenv(\"LOGICAL_REASONING_DATA_PATH\")\n","results_path = os.getenv(\"LOGICAL_REASONING_RESULTS_PATH\")\n","use_english_datasets = os.getenv(\"USE_ENGLISH_DATASETS\") == \"true\"\n","\n","print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path)"]},{"cell_type":"code","execution_count":8,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"b2a43943-9324-4839-9a47-cfa72de2244b","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":564,"status":"ok","timestamp":1720679529907,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"UgMvt6dIZBrM","outputId":"ce37581c-fd26-46c2-ad87-d933d99f68f7"},"outputs":[{"name":"stdout","output_type":"stream","text":["Python 3.11.9\n","\u001b[33mWARNING: Package(s) not found: flash-attn\u001b[0m\u001b[33m\n","\u001b[0mCPU times: user 4.12 ms, sys: 9.4 ms, total: 13.5 ms\n","Wall time: 651 ms\n"]}],"source":["%%time\n","!python --version\n","!pip show flash-attn"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1685,"status":"ok","timestamp":1720679531591,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"ZuS_FsLyZBrN","outputId":"2cba0105-c505-4395-afbd-2f2fee6581d0"},"outputs":[{"name":"stdout","output_type":"stream","text":["loading /Users/inflaton/code/engd/projects/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\n","MPS is available\n"]}],"source":["from llm_toolkit.llm_utils import *\n","from llm_toolkit.logical_reasoning_utils import *\n","\n","device = check_gpu()"]},{"cell_type":"code","execution_count":14,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["loading model: THUDM/glm-4-9b-chat-1m with adapter: None\n"]},{"name":"stderr","output_type":"stream","text":["Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"19aebcc5f4934933878ce3bb1fdc4b32","version_major":2,"version_minor":0},"text/plain":["Downloading shards: 0%| | 0/10 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c161cec46c72435a8b8d0beab1a329fa","version_major":2,"version_minor":0},"text/plain":["model-00006-of-00010.safetensors: 96%|#########6| 1.91G/1.99G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"13d593455bc94919846a6b5bd288740a","version_major":2,"version_minor":0},"text/plain":["model-00007-of-00010.safetensors: 0%| | 0.00/1.95G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"6154990ed2f047348720919a884f67d0","version_major":2,"version_minor":0},"text/plain":["model-00008-of-00010.safetensors: 0%| | 0.00/1.84G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["Error while downloading from https://cdn-lfs-us-1.huggingface.co/repos/8f/69/8f69006a64acf627b9dd5b8ed9962abf5ef416d57fc370004ac5f598cea0df41/ea76696ccd8b266eff2e755d9286a789a2b8d644a19b2cdccd365933e97cfa94?response-content-disposition=inline%3B+filename*%3DUTF-8%27%27model-00008-of-00010.safetensors%3B+filename%3D%22model-00008-of-00010.safetensors%22%3B&Expires=1721695460&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMTY5NTQ2MH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzhmLzY5LzhmNjkwMDZhNjRhY2Y2MjdiOWRkNWI4ZWQ5OTYyYWJmNWVmNDE2ZDU3ZmMzNzAwMDRhYzVmNTk4Y2VhMGRmNDEvZWE3NjY5NmNjZDhiMjY2ZWZmMmU3NTVkOTI4NmE3ODlhMmI4ZDY0NGExOWIyY2RjY2QzNjU5MzNlOTdjZmE5ND9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=BFlf2hKIdIMTSR%7E5dNEt5kHkOVcTfJOy3l8sseMydbGlzagX5bSyy18zLAc450pgxun6NKtv1ke8Db3nCDnl4DSSFOkzxH0zEzNzWN0Jt7P7axDubD%7EE7qeD1VMn1NB-r8OI0QDaF9Z%7EnAd9--fXyiXX7hacB3aCvmnsiwQbHkNy2DO89UKGcKIrf1yvXQMY-uO3RYtypVGqwjpgBBE9pf3n-Z1SEjAHwpxLeqowvj3Jc8yI5M-R60ymjlGe24Zcrdg25ScWJxnzKlqrAPB2p3P9clz7LdxLiI-7Ip0k8TsbkM-5BMe-yc99ED77-qzb6t9qQhlAFCOznu67KjFb-w__&Key-Pair-Id=K24J24Z295AEI9: HTTPSConnectionPool(host='cdn-lfs-us-1.huggingface.co', port=443): Read timed out.\n","Trying to resume download...\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"0c0b15e2f9e344b9ad4274fcc150d55d","version_major":2,"version_minor":0},"text/plain":["model-00008-of-00010.safetensors: 31%|### | 566M/1.84G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["Error while downloading from https://cdn-lfs-us-1.huggingface.co/repos/8f/69/8f69006a64acf627b9dd5b8ed9962abf5ef416d57fc370004ac5f598cea0df41/ea76696ccd8b266eff2e755d9286a789a2b8d644a19b2cdccd365933e97cfa94?response-content-disposition=inline%3B+filename*%3DUTF-8%27%27model-00008-of-00010.safetensors%3B+filename%3D%22model-00008-of-00010.safetensors%22%3B&Expires=1721695460&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMTY5NTQ2MH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzhmLzY5LzhmNjkwMDZhNjRhY2Y2MjdiOWRkNWI4ZWQ5OTYyYWJmNWVmNDE2ZDU3ZmMzNzAwMDRhYzVmNTk4Y2VhMGRmNDEvZWE3NjY5NmNjZDhiMjY2ZWZmMmU3NTVkOTI4NmE3ODlhMmI4ZDY0NGExOWIyY2RjY2QzNjU5MzNlOTdjZmE5ND9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=BFlf2hKIdIMTSR%7E5dNEt5kHkOVcTfJOy3l8sseMydbGlzagX5bSyy18zLAc450pgxun6NKtv1ke8Db3nCDnl4DSSFOkzxH0zEzNzWN0Jt7P7axDubD%7EE7qeD1VMn1NB-r8OI0QDaF9Z%7EnAd9--fXyiXX7hacB3aCvmnsiwQbHkNy2DO89UKGcKIrf1yvXQMY-uO3RYtypVGqwjpgBBE9pf3n-Z1SEjAHwpxLeqowvj3Jc8yI5M-R60ymjlGe24Zcrdg25ScWJxnzKlqrAPB2p3P9clz7LdxLiI-7Ip0k8TsbkM-5BMe-yc99ED77-qzb6t9qQhlAFCOznu67KjFb-w__&Key-Pair-Id=K24J24Z295AEI9: HTTPSConnectionPool(host='cdn-lfs-us-1.huggingface.co', port=443): Read timed out.\n","Trying to resume download...\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"82f48613f82047a7abc23a99939275ac","version_major":2,"version_minor":0},"text/plain":["model-00008-of-00010.safetensors: 34%|###3 | 619M/1.84G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["Error while downloading from https://cdn-lfs-us-1.huggingface.co/repos/8f/69/8f69006a64acf627b9dd5b8ed9962abf5ef416d57fc370004ac5f598cea0df41/ea76696ccd8b266eff2e755d9286a789a2b8d644a19b2cdccd365933e97cfa94?response-content-disposition=inline%3B+filename*%3DUTF-8%27%27model-00008-of-00010.safetensors%3B+filename%3D%22model-00008-of-00010.safetensors%22%3B&Expires=1721695460&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMTY5NTQ2MH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzhmLzY5LzhmNjkwMDZhNjRhY2Y2MjdiOWRkNWI4ZWQ5OTYyYWJmNWVmNDE2ZDU3ZmMzNzAwMDRhYzVmNTk4Y2VhMGRmNDEvZWE3NjY5NmNjZDhiMjY2ZWZmMmU3NTVkOTI4NmE3ODlhMmI4ZDY0NGExOWIyY2RjY2QzNjU5MzNlOTdjZmE5ND9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=BFlf2hKIdIMTSR%7E5dNEt5kHkOVcTfJOy3l8sseMydbGlzagX5bSyy18zLAc450pgxun6NKtv1ke8Db3nCDnl4DSSFOkzxH0zEzNzWN0Jt7P7axDubD%7EE7qeD1VMn1NB-r8OI0QDaF9Z%7EnAd9--fXyiXX7hacB3aCvmnsiwQbHkNy2DO89UKGcKIrf1yvXQMY-uO3RYtypVGqwjpgBBE9pf3n-Z1SEjAHwpxLeqowvj3Jc8yI5M-R60ymjlGe24Zcrdg25ScWJxnzKlqrAPB2p3P9clz7LdxLiI-7Ip0k8TsbkM-5BMe-yc99ED77-qzb6t9qQhlAFCOznu67KjFb-w__&Key-Pair-Id=K24J24Z295AEI9: HTTPSConnectionPool(host='cdn-lfs-us-1.huggingface.co', port=443): Read timed out.\n","Trying to resume download...\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d028a93b6cc146c5abc72f0469ea7481","version_major":2,"version_minor":0},"text/plain":["model-00008-of-00010.safetensors: 37%|###6 | 671M/1.84G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c838854922d141779d3b2e42067ae07c","version_major":2,"version_minor":0},"text/plain":["model-00009-of-00010.safetensors: 0%| | 0.00/1.99G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"}],"source":["%%time\n","\n","model, tokenizer = load_model(model_name, adapter_name_or_path=adapter_name_or_path, using_llama_factory=False)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"ename":"NameError","evalue":"name 'tokenizer' is not defined","output_type":"error","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[13], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m datasets \u001b[38;5;241m=\u001b[39m load_logical_reasoning_dataset(\n\u001b[1;32m 2\u001b[0m data_path,\n\u001b[0;32m----> 3\u001b[0m tokenizer\u001b[38;5;241m=\u001b[39m\u001b[43mtokenizer\u001b[49m,\n\u001b[1;32m 4\u001b[0m chinese_prompt\u001b[38;5;241m=\u001b[39m\u001b[38;5;129;01mnot\u001b[39;00m use_english_datasets,\n\u001b[1;32m 5\u001b[0m using_p1\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 6\u001b[0m )\n","\u001b[0;31mNameError\u001b[0m: name 'tokenizer' is not defined"]}],"source":["datasets = load_logical_reasoning_dataset(\n"," data_path,\n"," tokenizer=tokenizer,\n"," chinese_prompt=not use_english_datasets,\n"," using_p1=False,\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def evaluate_model(model, tokenizer, model_name, dataset, batch_size=1):\n"," print(f\"Evaluating model: {model_name} on {device}\")\n"," predictions = eval_model(\n"," model, tokenizer, dataset, device=device, batch_size=batch_size\n"," )\n","\n"," save_results(\n"," model_name,\n"," results_path,\n"," dataset,\n"," predictions,\n"," debug=False,\n"," )\n","\n"," metrics = calc_metrics(dataset[\"label\"], predictions, debug=False)\n"," print(metrics)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Evaluating model: internlm/internlm2_5-7b-chat-1m_llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full/checkpoint-88 on mps\n"]},{"name":"stderr","output_type":"stream","text":[" 0%| | 1/3000 [00:06<5:23:35, 6.47s/it]"]},{"name":"stdout","output_type":"stream","text":["--------\n","step 1: 不是</s>\n","--------\n","step 2: 不是\n","--------\n","step 3: 不是\n","--------\n","step 4: 不是\n","--------\n","step 5: 不是\n"]},{"name":"stderr","output_type":"stream","text":[" 22%|██▏ | 657/3000 [40:38<2:18:00, 3.53s/it]"]}],"source":["%%time\n","\n","evaluate_model(model, tokenizer, f\"{model_name}{'_' + adapter_name_or_path if adapter_name_or_path else ''}\", datasets[\"test\"], batch_size=16)"]}],"metadata":{"accelerator":"GPU","application/vnd.databricks.v1+notebook":{"dashboards":[],"environmentMetadata":null,"language":"python","notebookMetadata":{"mostRecentlyExecutedCommandWithImplicitDF":{"commandId":-1,"dataframes":["_sqldf"]},"pythonIndentUnit":4},"notebookName":"10_eval-lf-medium-py3.11","widgets":{}},"colab":{"gpuType":"L4","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.9"}},"nbformat":4,"nbformat_minor":0}

llm_toolkit/llm_utils.py CHANGED Viewed

@@ -137,6 +137,9 @@ def extract_answer(text, debug=False):
     return text
 def eval_model(
     model,
     tokenizer,
@@ -144,13 +147,17 @@ def eval_model(
     device="cuda",
     max_new_tokens=4096,
     repetition_penalty=1.0,
 ):
     total = len(eval_dataset)
     predictions = []
-    for i in tqdm(range(total)):
         inputs = tokenizer(
-            eval_dataset["prompt"][i : i + 1],
             return_tensors="pt",
         ).to(device)
         outputs = model.generate(
@@ -159,10 +166,13 @@ def eval_model(
             repetition_penalty=repetition_penalty,
             use_cache=False,
         )
-        decoded_output = tokenizer.batch_decode(outputs)
         debug = i == 0
         decoded_output = [
-            extract_answer(output, debug=debug) for output in decoded_output
         ]
         predictions.extend(decoded_output)

     return text
+from tqdm import tqdm
 def eval_model(
     model,
     tokenizer,
     device="cuda",
     max_new_tokens=4096,
     repetition_penalty=1.0,
+    batch_size=1,  # Add a parameter for batch size
 ):
     total = len(eval_dataset)
     predictions = []
+    for i in tqdm(range(0, total, batch_size)):  # Iterate in batches
+        batch_end = min(i + batch_size, total)  # Ensure not to exceed dataset
+        batch_prompts = eval_dataset["prompt"][i:batch_end]
         inputs = tokenizer(
+            batch_prompts,
             return_tensors="pt",
+            padding=True,  # Ensure all inputs in the batch have the same length
         ).to(device)
         outputs = model.generate(
             repetition_penalty=repetition_penalty,
             use_cache=False,
         )
+        decoded_output = tokenizer.batch_decode(
+            outputs, skip_special_tokens=True
+        )  # Skip special tokens for clean output
         debug = i == 0
         decoded_output = [
+            extract_answer(output, debug=debug and j == 0)
+            for j, output in enumerate(decoded_output)
         ]
         predictions.extend(decoded_output)

results/mgtv-results_m3.csv ADDED Viewed

The diff for this file is too large to render. See raw diff