{ "cells": [ { "cell_type": "code", "id": "initial_id", "metadata": { "collapsed": true, "jupyter": { "outputs_hidden": true }, "ExecuteTime": { "end_time": "2024-05-01T15:23:17.507403Z", "start_time": "2024-05-01T15:23:17.497406Z" } }, "source": [ "import pandas as pd\n", "\n", "import config" ], "outputs": [], "execution_count": 7 }, { "cell_type": "code", "id": "2ac8757a17e62293", "metadata": { "ExecuteTime": { "end_time": "2024-05-01T15:23:19.365525Z", "start_time": "2024-05-01T15:23:19.120308Z" } }, "source": [ "df = pd.read_csv(config.SYNTHETIC_DATASET_ARTIFACT, index_col=0)\n", "\n", "df.head()" ], "outputs": [ { "data": { "text/plain": [ " hash \\\n", "0 9a581830e4fa02eed501b4e1f546a2e2ea358e13 \n", "1 37067a53c4b3b99982ef8e1f431ba0c9302b66e8 \n", "2 82e350064cb8d1622c7cde275567ae594483fe62 \n", "3 cf98f5e3705603ae21bef9b0a577bcd001a8c92e \n", "4 c17a80f47b772d759aeb0878aa767a768a6fdd0c \n", "\n", " repo \\\n", "0 bitcoinunlimited/bitcoinunlimited \n", "1 mesonbuild/meson \n", "2 mycroftai/mycroft-core \n", "3 mesonbuild/meson \n", "4 mesonbuild/meson \n", "\n", " commit_msg_start \\\n", "0 Add extensive test option to parallel RPC test... \n", "1 Refactor argument parsing and command executio... \n", "2 Add helper functions for disk space management... \n", "3 Update path resolution for non-Windows systems... \n", "4 Add support for VS2017 architecture detection\\... \n", "\n", " commit_msg_end \\\n", "0 Add new block attack patterns\\n\\n- Added test ... \n", "1 Introduce unified argument parsing in meson\\n\\... \n", "2 Refactor file_utils.py\\n\\n- Add helper functio... \n", "3 Enable loading crossfiles for all platforms ex... \n", "4 Add support for VS2017 architecture detection.... \n", "\n", " session \\\n", "0 032e60d7-621a-46b6-972f-7590cfaf6458 \n", "1 5d7f1209-4ed9-4620-87ca-975f029c7f6f \n", "2 93b1c57c-e56c-4d75-89a6-ae1158b4fa74 \n", "3 5d7f1209-4ed9-4620-87ca-975f029c7f6f \n", "4 16e57250-21ff-4cdd-ae0d-760cabcc6160 \n", "\n", " commit_msg_history \\\n", "0 [{\"t\": \"-\", \"p\": 4, \"c\": \"e\", \"ts\": \"2024-04-0... \n", "1 [] \n", "2 [{\"t\": \"+\", \"p\": 0, \"c\": \"R\", \"ts\": \"2024-04-0... \n", "3 [] \n", "4 [{\"t\": \"-\", \"p\": 45, \"c\": \"\\n\", \"ts\": \"2024-04... \n", "\n", " loaded_ts submitted_ts edit_time_hist \\\n", "0 2024-04-04T19:48:31.180017 2024-04-04T19:50:32.925989 59468.0 \n", "1 2024-04-15T16:50:17.208813 2024-04-15T15:29:02.014310 0.0 \n", "2 2024-04-04T19:52:38.276314 2024-04-04T19:57:02.449096 133655.0 \n", "3 2024-04-15T17:42:14.482856 2024-04-15T15:29:02.014310 0.0 \n", "4 2024-04-15T15:47:31.022477 2024-04-15T15:53:08.796895 163218.0 \n", "\n", " edit_time ... rel_edittime_ind_rouge2_pearson \\\n", "0 121745.0 ... 0.281944 \n", "1 NaN ... 0.281944 \n", "2 264172.0 ... 0.281944 \n", "3 NaN ... 0.281944 \n", "4 337774.0 ... 0.281944 \n", "\n", " rel_edittime_ind_rouge2_spearman rel_edittime_ind_rougeL_pearson \\\n", "0 0.218822 0.091196 \n", "1 0.218822 0.091196 \n", "2 0.218822 0.091196 \n", "3 0.218822 0.091196 \n", "4 0.218822 0.091196 \n", "\n", " rel_edittime_ind_rougeL_spearman rel_edittime_ind_bertscore_pearson \\\n", "0 0.071344 0.158807 \n", "1 0.071344 0.158807 \n", "2 0.071344 0.158807 \n", "3 0.071344 0.158807 \n", "4 0.071344 0.158807 \n", "\n", " rel_edittime_ind_bertscore_spearman rel_edittime_ind_chrF_pearson \\\n", "0 0.140481 0.184202 \n", "1 0.140481 0.184202 \n", "2 0.140481 0.184202 \n", "3 0.140481 0.184202 \n", "4 0.140481 0.184202 \n", "\n", " rel_edittime_ind_chrF_spearman rel_edittime_ind_ter_pearson \\\n", "0 0.079802 0.062616 \n", "1 0.079802 0.062616 \n", "2 0.079802 0.062616 \n", "3 0.079802 0.062616 \n", "4 0.079802 0.062616 \n", "\n", " rel_edittime_ind_ter_spearman \n", "0 0.305601 \n", "1 0.305601 \n", "2 0.305601 \n", "3 0.305601 \n", "4 0.305601 \n", "\n", "[5 rows x 71 columns]" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
hashrepocommit_msg_startcommit_msg_endsessioncommit_msg_historyloaded_tssubmitted_tsedit_time_histedit_time...rel_edittime_ind_rouge2_pearsonrel_edittime_ind_rouge2_spearmanrel_edittime_ind_rougeL_pearsonrel_edittime_ind_rougeL_spearmanrel_edittime_ind_bertscore_pearsonrel_edittime_ind_bertscore_spearmanrel_edittime_ind_chrF_pearsonrel_edittime_ind_chrF_spearmanrel_edittime_ind_ter_pearsonrel_edittime_ind_ter_spearman
09a581830e4fa02eed501b4e1f546a2e2ea358e13bitcoinunlimited/bitcoinunlimitedAdd extensive test option to parallel RPC test...Add new block attack patterns\\n\\n- Added test ...032e60d7-621a-46b6-972f-7590cfaf6458[{\"t\": \"-\", \"p\": 4, \"c\": \"e\", \"ts\": \"2024-04-0...2024-04-04T19:48:31.1800172024-04-04T19:50:32.92598959468.0121745.0...0.2819440.2188220.0911960.0713440.1588070.1404810.1842020.0798020.0626160.305601
137067a53c4b3b99982ef8e1f431ba0c9302b66e8mesonbuild/mesonRefactor argument parsing and command executio...Introduce unified argument parsing in meson\\n\\...5d7f1209-4ed9-4620-87ca-975f029c7f6f[]2024-04-15T16:50:17.2088132024-04-15T15:29:02.0143100.0NaN...0.2819440.2188220.0911960.0713440.1588070.1404810.1842020.0798020.0626160.305601
282e350064cb8d1622c7cde275567ae594483fe62mycroftai/mycroft-coreAdd helper functions for disk space management...Refactor file_utils.py\\n\\n- Add helper functio...93b1c57c-e56c-4d75-89a6-ae1158b4fa74[{\"t\": \"+\", \"p\": 0, \"c\": \"R\", \"ts\": \"2024-04-0...2024-04-04T19:52:38.2763142024-04-04T19:57:02.449096133655.0264172.0...0.2819440.2188220.0911960.0713440.1588070.1404810.1842020.0798020.0626160.305601
3cf98f5e3705603ae21bef9b0a577bcd001a8c92emesonbuild/mesonUpdate path resolution for non-Windows systems...Enable loading crossfiles for all platforms ex...5d7f1209-4ed9-4620-87ca-975f029c7f6f[]2024-04-15T17:42:14.4828562024-04-15T15:29:02.0143100.0NaN...0.2819440.2188220.0911960.0713440.1588070.1404810.1842020.0798020.0626160.305601
4c17a80f47b772d759aeb0878aa767a768a6fdd0cmesonbuild/mesonAdd support for VS2017 architecture detection\\...Add support for VS2017 architecture detection....16e57250-21ff-4cdd-ae0d-760cabcc6160[{\"t\": \"-\", \"p\": 45, \"c\": \"\\n\", \"ts\": \"2024-04...2024-04-15T15:47:31.0224772024-04-15T15:53:08.796895163218.0337774.0...0.2819440.2188220.0911960.0713440.1588070.1404810.1842020.0798020.0626160.305601
\n", "

5 rows × 71 columns

\n", "
" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 8 }, { "metadata": { "ExecuteTime": { "end_time": "2024-05-01T15:11:08.418257Z", "start_time": "2024-05-01T15:11:08.408943Z" } }, "cell_type": "code", "source": "len(set(df['session'].to_list()))", "id": "4bcbc0f1d3d6d248", "outputs": [ { "data": { "text/plain": [ "9" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 6 }, { "cell_type": "code", "execution_count": 15, "id": "d19c12dd10b25c75", "metadata": { "ExecuteTime": { "end_time": "2024-05-01T13:02:40.761645Z", "start_time": "2024-05-01T13:02:40.740647Z" } }, "outputs": [ { "data": { "text/plain": [ "['editdist', 'edittime']" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "rel_metrics = [col.split(\"_\")[0] for col in df.columns if col.endswith(\"_related\")]\n", "rel_metrics" ] }, { "cell_type": "code", "execution_count": 16, "id": "79d644cd780b28a1", "metadata": { "ExecuteTime": { "end_time": "2024-05-01T13:02:44.072037Z", "start_time": "2024-05-01T13:02:44.055039Z" } }, "outputs": [ { "data": { "text/plain": [ "['gptscore-ref-1-req',\n", " 'gptscore-noref-1-req',\n", " 'editdist',\n", " 'bleu',\n", " 'meteor',\n", " 'rouge1',\n", " 'rouge2',\n", " 'rougeL',\n", " 'bertscore',\n", " 'chrF',\n", " 'ter']" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ind_metrics = [col.split(\"_\")[0] for col in df.columns if col.endswith(\"_independent\")]\n", "ind_metrics" ] }, { "cell_type": "code", "execution_count": 19, "id": "fdc5ae636bffbc8b", "metadata": { "ExecuteTime": { "end_time": "2024-05-01T13:03:52.623346Z", "start_time": "2024-05-01T13:03:52.577076Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
hasheditdist_relatededittime_relatedgptscore-ref-1-req_independentgptscore-noref-1-req_independenteditdist_independentbleu_independentmeteor_independentrouge1_independentrouge2_independentrougeL_independentbertscore_independentchrF_independentter_independent
countmeanmeanmeanmeanmeanmeanmeanmeanmeanmeanmeanmeanmean
end_to_startstart_to_end
FalseFalse43355.441860364099.06257.2558148.116279491.0697670.0128050.2249610.2020630.0407180.1364270.78026632.067005312.732989
True129406.627907NaN7.2170548.178295491.0697670.0128050.2249610.2020630.0407180.1364270.78026632.067005312.732989
TrueFalse129433.899225NaN7.3565898.302326534.0155040.0095420.2218930.2051510.0390330.1341140.77716231.753065317.717517
True387444.509044NaN7.3126618.276486534.0155040.0095420.2218930.2051510.0390330.1341140.77716231.753065317.717517
\n", "
" ], "text/plain": [ " hash editdist_related edittime_related \\\n", " count mean mean \n", "end_to_start start_to_end \n", "False False 43 355.441860 364099.0625 \n", " True 129 406.627907 NaN \n", "True False 129 433.899225 NaN \n", " True 387 444.509044 NaN \n", "\n", " gptscore-ref-1-req_independent \\\n", " mean \n", "end_to_start start_to_end \n", "False False 7.255814 \n", " True 7.217054 \n", "True False 7.356589 \n", " True 7.312661 \n", "\n", " gptscore-noref-1-req_independent \\\n", " mean \n", "end_to_start start_to_end \n", "False False 8.116279 \n", " True 8.178295 \n", "True False 8.302326 \n", " True 8.276486 \n", "\n", " editdist_independent bleu_independent \\\n", " mean mean \n", "end_to_start start_to_end \n", "False False 491.069767 0.012805 \n", " True 491.069767 0.012805 \n", "True False 534.015504 0.009542 \n", " True 534.015504 0.009542 \n", "\n", " meteor_independent rouge1_independent \\\n", " mean mean \n", "end_to_start start_to_end \n", "False False 0.224961 0.202063 \n", " True 0.224961 0.202063 \n", "True False 0.221893 0.205151 \n", " True 0.221893 0.205151 \n", "\n", " rouge2_independent rougeL_independent \\\n", " mean mean \n", "end_to_start start_to_end \n", "False False 0.040718 0.136427 \n", " True 0.040718 0.136427 \n", "True False 0.039033 0.134114 \n", " True 0.039033 0.134114 \n", "\n", " bertscore_independent chrF_independent \\\n", " mean mean \n", "end_to_start start_to_end \n", "False False 0.780266 32.067005 \n", " True 0.780266 32.067005 \n", "True False 0.777162 31.753065 \n", " True 0.777162 31.753065 \n", "\n", " ter_independent \n", " mean \n", "end_to_start start_to_end \n", "False False 312.732989 \n", " True 312.732989 \n", "True False 317.717517 \n", " True 317.717517 " ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "AGGREGATION = {\"hash\": [\"count\"]}\n", "\n", "for metric in rel_metrics:\n", " AGGREGATION[f\"{metric}_related\"] = [\"mean\"]\n", "\n", "for metric in ind_metrics:\n", " AGGREGATION[f\"{metric}_independent\"] = [\"mean\"]\n", "\n", "df.groupby(by=[\"end_to_start\", \"start_to_end\"]).agg(AGGREGATION)" ] }, { "cell_type": "code", "execution_count": 47, "id": "3429b60eab154b79", "metadata": { "ExecuteTime": { "end_time": "2024-05-01T13:42:57.052768Z", "start_time": "2024-05-01T13:42:56.812556Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
allgolden+s2e+e2s+e2s+s2e
spearmanpearsonspearmanpearsonspearmanpearsonspearmanpearsonspearmanpearson
relativeindependent
editdistbertscore-0.184962-0.129057-0.316215-0.254700-0.308494-0.113525-0.181393-0.165924-0.135421-0.091748
bleu0.2601180.1859950.2690280.2596900.5128410.5028270.1098310.0681380.2297120.145062
chrF-0.199200-0.129029-0.343201-0.300656-0.238124-0.064922-0.233123-0.201726-0.156914-0.093376
editdist0.9099340.9106410.7107720.6628080.9504940.9350640.8619300.8781180.9393180.962305
gptscore-noref-1-req0.0320480.0553640.1555100.0485880.0678570.047215-0.029048-0.0131280.0121020.066882
gptscore-ref-1-req0.0245500.035295-0.009830-0.062574-0.015178-0.0360010.0713450.0875840.0130120.033618
meteor0.3360160.3719490.0680340.1732370.2036160.4257750.3725980.3600510.3922620.401802
rouge1-0.077574-0.043738-0.187349-0.163230-0.139874-0.065543-0.082093-0.035603-0.054034-0.030799
rouge20.4142560.3407320.2761390.3320870.5235590.5375600.3239110.2828720.4338590.324538
rougeL0.006513-0.008078-0.041502-0.034867-0.022288-0.0046640.0124090.0163720.021983-0.010644
ter0.6180950.3855150.5756140.5013850.7740860.4625540.5293380.3885920.5916840.354459
edittimebertscore0.1404810.1588070.1404810.158807NaNNaNNaNNaNNaNNaN
bleu0.3023800.3261670.3023800.326167NaNNaNNaNNaNNaNNaN
chrF0.0798020.1842020.0798020.184202NaNNaNNaNNaNNaNNaN
editdist0.2526450.4111310.2526450.411131NaNNaNNaNNaNNaNNaN
gptscore-noref-1-req0.2064650.0262350.2064650.026235NaNNaNNaNNaNNaNNaN
gptscore-ref-1-req0.130419-0.0552180.130419-0.055218NaNNaNNaNNaNNaNNaN
meteor0.2533800.4035640.2533800.403564NaNNaNNaNNaNNaNNaN
rouge10.1559260.1369710.1559260.136971NaNNaNNaNNaNNaNNaN
rouge20.2188220.2819440.2188220.281944NaNNaNNaNNaNNaNNaN
rougeL0.0713440.0911960.0713440.091196NaNNaNNaNNaNNaNNaN
ter0.3056010.0626160.3056010.062616NaNNaNNaNNaNNaNNaN
\n", "
" ], "text/plain": [ " all golden \\\n", " spearman pearson spearman pearson \n", "relative independent \n", "editdist bertscore -0.184962 -0.129057 -0.316215 -0.254700 \n", " bleu 0.260118 0.185995 0.269028 0.259690 \n", " chrF -0.199200 -0.129029 -0.343201 -0.300656 \n", " editdist 0.909934 0.910641 0.710772 0.662808 \n", " gptscore-noref-1-req 0.032048 0.055364 0.155510 0.048588 \n", " gptscore-ref-1-req 0.024550 0.035295 -0.009830 -0.062574 \n", " meteor 0.336016 0.371949 0.068034 0.173237 \n", " rouge1 -0.077574 -0.043738 -0.187349 -0.163230 \n", " rouge2 0.414256 0.340732 0.276139 0.332087 \n", " rougeL 0.006513 -0.008078 -0.041502 -0.034867 \n", " ter 0.618095 0.385515 0.575614 0.501385 \n", "edittime bertscore 0.140481 0.158807 0.140481 0.158807 \n", " bleu 0.302380 0.326167 0.302380 0.326167 \n", " chrF 0.079802 0.184202 0.079802 0.184202 \n", " editdist 0.252645 0.411131 0.252645 0.411131 \n", " gptscore-noref-1-req 0.206465 0.026235 0.206465 0.026235 \n", " gptscore-ref-1-req 0.130419 -0.055218 0.130419 -0.055218 \n", " meteor 0.253380 0.403564 0.253380 0.403564 \n", " rouge1 0.155926 0.136971 0.155926 0.136971 \n", " rouge2 0.218822 0.281944 0.218822 0.281944 \n", " rougeL 0.071344 0.091196 0.071344 0.091196 \n", " ter 0.305601 0.062616 0.305601 0.062616 \n", "\n", " +s2e +e2s \\\n", " spearman pearson spearman pearson \n", "relative independent \n", "editdist bertscore -0.308494 -0.113525 -0.181393 -0.165924 \n", " bleu 0.512841 0.502827 0.109831 0.068138 \n", " chrF -0.238124 -0.064922 -0.233123 -0.201726 \n", " editdist 0.950494 0.935064 0.861930 0.878118 \n", " gptscore-noref-1-req 0.067857 0.047215 -0.029048 -0.013128 \n", " gptscore-ref-1-req -0.015178 -0.036001 0.071345 0.087584 \n", " meteor 0.203616 0.425775 0.372598 0.360051 \n", " rouge1 -0.139874 -0.065543 -0.082093 -0.035603 \n", " rouge2 0.523559 0.537560 0.323911 0.282872 \n", " rougeL -0.022288 -0.004664 0.012409 0.016372 \n", " ter 0.774086 0.462554 0.529338 0.388592 \n", "edittime bertscore NaN NaN NaN NaN \n", " bleu NaN NaN NaN NaN \n", " chrF NaN NaN NaN NaN \n", " editdist NaN NaN NaN NaN \n", " gptscore-noref-1-req NaN NaN NaN NaN \n", " gptscore-ref-1-req NaN NaN NaN NaN \n", " meteor NaN NaN NaN NaN \n", " rouge1 NaN NaN NaN NaN \n", " rouge2 NaN NaN NaN NaN \n", " rougeL NaN NaN NaN NaN \n", " ter NaN NaN NaN NaN \n", "\n", " +e2s+s2e \n", " spearman pearson \n", "relative independent \n", "editdist bertscore -0.135421 -0.091748 \n", " bleu 0.229712 0.145062 \n", " chrF -0.156914 -0.093376 \n", " editdist 0.939318 0.962305 \n", " gptscore-noref-1-req 0.012102 0.066882 \n", " gptscore-ref-1-req 0.013012 0.033618 \n", " meteor 0.392262 0.401802 \n", " rouge1 -0.054034 -0.030799 \n", " rouge2 0.433859 0.324538 \n", " rougeL 0.021983 -0.010644 \n", " ter 0.591684 0.354459 \n", "edittime bertscore NaN NaN \n", " bleu NaN NaN \n", " chrF NaN NaN \n", " editdist NaN NaN \n", " gptscore-noref-1-req NaN NaN \n", " gptscore-ref-1-req NaN NaN \n", " meteor NaN NaN \n", " rouge1 NaN NaN \n", " rouge2 NaN NaN \n", " rougeL NaN NaN \n", " ter NaN NaN " ] }, "execution_count": 47, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 50, "id": "a3531f28722fa5bc", "metadata": { "ExecuteTime": { "end_time": "2024-05-01T13:49:09.514129Z", "start_time": "2024-05-01T13:49:09.295101Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
allgolden+s2e+e2s+e2s+s2e
spearmanpearsonspearmanpearsonspearmanpearsonspearmanpearsonspearmanpearson
relativeindependent
editdistbertscore-0.184962-0.129057-0.316215-0.254700-0.308494-0.113525-0.181393-0.165924-0.135421-0.091748
bleu0.2601180.1859950.2690280.2596900.5128410.5028270.1098310.0681380.2297120.145062
chrF-0.199200-0.129029-0.343201-0.300656-0.238124-0.064922-0.233123-0.201726-0.156914-0.093376
editdist0.9099340.9106410.7107720.6628080.9504940.9350640.8619300.8781180.9393180.962305
gptscore-noref-1-req0.0320480.0553640.1555100.0485880.0678570.047215-0.029048-0.0131280.0121020.066882
gptscore-ref-1-req0.0245500.035295-0.009830-0.062574-0.015178-0.0360010.0713450.0875840.0130120.033618
meteor0.3360160.3719490.0680340.1732370.2036160.4257750.3725980.3600510.3922620.401802
rouge1-0.077574-0.043738-0.187349-0.163230-0.139874-0.065543-0.082093-0.035603-0.054034-0.030799
rouge20.4142560.3407320.2761390.3320870.5235590.5375600.3239110.2828720.4338590.324538
rougeL0.006513-0.008078-0.041502-0.034867-0.022288-0.0046640.0124090.0163720.021983-0.010644
ter0.6180950.3855150.5756140.5013850.7740860.4625540.5293380.3885920.5916840.354459
edittimebertscore0.1404810.1588070.1404810.158807NaNNaNNaNNaNNaNNaN
bleu0.3023800.3261670.3023800.326167NaNNaNNaNNaNNaNNaN
chrF0.0798020.1842020.0798020.184202NaNNaNNaNNaNNaNNaN
editdist0.2526450.4111310.2526450.411131NaNNaNNaNNaNNaNNaN
gptscore-noref-1-req0.2064650.0262350.2064650.026235NaNNaNNaNNaNNaNNaN
gptscore-ref-1-req0.130419-0.0552180.130419-0.055218NaNNaNNaNNaNNaNNaN
meteor0.2533800.4035640.2533800.403564NaNNaNNaNNaNNaNNaN
rouge10.1559260.1369710.1559260.136971NaNNaNNaNNaNNaNNaN
rouge20.2188220.2819440.2188220.281944NaNNaNNaNNaNNaNNaN
rougeL0.0713440.0911960.0713440.091196NaNNaNNaNNaNNaNNaN
ter0.3056010.0626160.3056010.062616NaNNaNNaNNaNNaNNaN
\n", "
" ], "text/plain": [ " all golden \\\n", " spearman pearson spearman pearson \n", "relative independent \n", "editdist bertscore -0.184962 -0.129057 -0.316215 -0.254700 \n", " bleu 0.260118 0.185995 0.269028 0.259690 \n", " chrF -0.199200 -0.129029 -0.343201 -0.300656 \n", " editdist 0.909934 0.910641 0.710772 0.662808 \n", " gptscore-noref-1-req 0.032048 0.055364 0.155510 0.048588 \n", " gptscore-ref-1-req 0.024550 0.035295 -0.009830 -0.062574 \n", " meteor 0.336016 0.371949 0.068034 0.173237 \n", " rouge1 -0.077574 -0.043738 -0.187349 -0.163230 \n", " rouge2 0.414256 0.340732 0.276139 0.332087 \n", " rougeL 0.006513 -0.008078 -0.041502 -0.034867 \n", " ter 0.618095 0.385515 0.575614 0.501385 \n", "edittime bertscore 0.140481 0.158807 0.140481 0.158807 \n", " bleu 0.302380 0.326167 0.302380 0.326167 \n", " chrF 0.079802 0.184202 0.079802 0.184202 \n", " editdist 0.252645 0.411131 0.252645 0.411131 \n", " gptscore-noref-1-req 0.206465 0.026235 0.206465 0.026235 \n", " gptscore-ref-1-req 0.130419 -0.055218 0.130419 -0.055218 \n", " meteor 0.253380 0.403564 0.253380 0.403564 \n", " rouge1 0.155926 0.136971 0.155926 0.136971 \n", " rouge2 0.218822 0.281944 0.218822 0.281944 \n", " rougeL 0.071344 0.091196 0.071344 0.091196 \n", " ter 0.305601 0.062616 0.305601 0.062616 \n", "\n", " +s2e +e2s \\\n", " spearman pearson spearman pearson \n", "relative independent \n", "editdist bertscore -0.308494 -0.113525 -0.181393 -0.165924 \n", " bleu 0.512841 0.502827 0.109831 0.068138 \n", " chrF -0.238124 -0.064922 -0.233123 -0.201726 \n", " editdist 0.950494 0.935064 0.861930 0.878118 \n", " gptscore-noref-1-req 0.067857 0.047215 -0.029048 -0.013128 \n", " gptscore-ref-1-req -0.015178 -0.036001 0.071345 0.087584 \n", " meteor 0.203616 0.425775 0.372598 0.360051 \n", " rouge1 -0.139874 -0.065543 -0.082093 -0.035603 \n", " rouge2 0.523559 0.537560 0.323911 0.282872 \n", " rougeL -0.022288 -0.004664 0.012409 0.016372 \n", " ter 0.774086 0.462554 0.529338 0.388592 \n", "edittime bertscore NaN NaN NaN NaN \n", " bleu NaN NaN NaN NaN \n", " chrF NaN NaN NaN NaN \n", " editdist NaN NaN NaN NaN \n", " gptscore-noref-1-req NaN NaN NaN NaN \n", " gptscore-ref-1-req NaN NaN NaN NaN \n", " meteor NaN NaN NaN NaN \n", " rouge1 NaN NaN NaN NaN \n", " rouge2 NaN NaN NaN NaN \n", " rougeL NaN NaN NaN NaN \n", " ter NaN NaN NaN NaN \n", "\n", " +e2s+s2e \n", " spearman pearson \n", "relative independent \n", "editdist bertscore -0.135421 -0.091748 \n", " bleu 0.229712 0.145062 \n", " chrF -0.156914 -0.093376 \n", " editdist 0.939318 0.962305 \n", " gptscore-noref-1-req 0.012102 0.066882 \n", " gptscore-ref-1-req 0.013012 0.033618 \n", " meteor 0.392262 0.401802 \n", " rouge1 -0.054034 -0.030799 \n", " rouge2 0.433859 0.324538 \n", " rougeL 0.021983 -0.010644 \n", " ter 0.591684 0.354459 \n", "edittime bertscore NaN NaN \n", " bleu NaN NaN \n", " chrF NaN NaN \n", " editdist NaN NaN \n", " gptscore-noref-1-req NaN NaN \n", " gptscore-ref-1-req NaN NaN \n", " meteor NaN NaN \n", " rouge1 NaN NaN \n", " rouge2 NaN NaN \n", " rougeL NaN NaN \n", " ter NaN NaN " ] }, "execution_count": 50, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from analysis_util import get_correlations_for_groups\n", "\n", "get_correlations_for_groups(df, right_side=\"ind\")" ] }, { "cell_type": "code", "execution_count": null, "id": "d5dc33a4251baf9a", "metadata": {}, "outputs": [], "source": [ "get_correlations_for_groups(df, right_side=\"aggr\")" ] }, { "metadata": { "ExecuteTime": { "end_time": "2024-05-01T15:25:18.226195Z", "start_time": "2024-05-01T15:25:17.464762Z" } }, "cell_type": "code", "source": [ "from matplotlib import pyplot as plt\n", "\n", "plt.scatter(x=df['edittime_related'], y=df['editdist_related'])" ], "id": "5df60ac60034b274", "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/plain": [ "
" ], "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGvCAYAAABxUC54AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuyklEQVR4nO3df3TU1Z3/8dckkAxiMjTYZCYVNaUVGYP8UIMRbbUGSaU55UjbpV/wsPUHbgqu+LOwu5pNtUZcf3T9UbCuBW20tG4PuqEapVCh1UhsIltCWLSYBZRMYk2ZIG4CzHy+f8SZMuTXzGQyczPzfJwz58Bn7kzen8+ZM/M6937uvTbLsiwBAAAYJC3RBQAAAJyMgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMM6oRBcQDb/fr4MHDyorK0s2my3R5QAAgDBYlqXDhw8rPz9faWkD95GMyIBy8OBBTZgwIdFlAACAKBw4cECnn376gG1GZEDJysqS1HOC2dnZCa4GAACEo7OzUxMmTAj+jg9kRAaUwLBOdnY2AQUAgBEmnNszuEkWAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADDOiFyoDYnl81uqb+lQ++Eu5WbZVVSQo/Q09kQCAMQOAQURqW1qVWVNs1q9XcFjLoddFWVulRa6ElgZACCZMMSDsNU2taq8ujEknEiSx9ul8upG1Ta1JqgyAECyIaAgLD6/pcqaZll9PBc4VlnTLJ+/rxYAAESGgIKw1Ld09Oo5OZElqdXbpfqWjvgVBQBIWgQUhKX9cP/hJJp2AAAMhICCsORm2WPaDgCAgRBQEJaighy5HHb1N5nYpp7ZPEUFOfEsCwCQpAgoCEt6mk0VZW5J6hVSAv+vKHOzHgoAICYIKAhbaaFLqxfNkNMROozjdNi1etEM1kEBAMQMC7UhIqWFLs12O1lJFgAwrAgoiFh6mk3FE8cnugwAQBJjiAcAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYJ6KA4vP5dNddd6mgoEBjxozRxIkTdc8998iyrGAby7J09913y+VyacyYMSopKdF7770X8j4dHR1auHChsrOzNW7cOF133XX65JNPYnNGAABgxIsooKxatUqrV6/W448/rt27d2vVqlV64IEH9NhjjwXbPPDAA3r00Ue1Zs0abd++XWPHjtWcOXPU1dUVbLNw4ULt2rVLmzZt0saNG7Vt2zYtWbIkdmcFAABGNJt1YvfHIL7xjW8oLy9PTz/9dPDY/PnzNWbMGFVXV8uyLOXn5+u2227T7bffLknyer3Ky8vTunXrtGDBAu3evVtut1tvv/22LrjgAklSbW2trrrqKn3wwQfKz88ftI7Ozk45HA55vV5lZ2dHes5G8Pkt1bd0qP1wl3Kz7CoqyFF6mi3RZQEAMGwi+f2OqAfl4osv1ubNm/Xuu+9Kkv77v/9bf/jDH/T1r39dktTS0iKPx6OSkpLgaxwOh2bOnKm6ujpJUl1dncaNGxcMJ5JUUlKitLQ0bd++vc+/293drc7OzpDHSFbb1KpLVm3Rd596Szev36HvPvWWLlm1RbVNrYkuDQAAI0QUUFasWKEFCxbonHPO0ejRozV9+nQtX75cCxculCR5PB5JUl5eXsjr8vLygs95PB7l5uaGPD9q1Cjl5OQE25ysqqpKDocj+JgwYUIkZRultqlV5dWNavV2hRz3eLtUXt1ISAEAQBEGlF/96ld67rnn9Pzzz6uxsVHPPPOMHnzwQT3zzDPDVZ8kaeXKlfJ6vcHHgQMHhvXvDRef31JlTbP6GlMLHKusaZbPH/aoGwAASWlUJI3vuOOOYC+KJE2ZMkX79u1TVVWVFi9eLKfTKUlqa2uTy+UKvq6trU3Tpk2TJDmdTrW3t4e87/Hjx9XR0RF8/ckyMzOVmZkZSalGqm/p6NVzciJLUqu3S/UtHSqeOD5+hQEAYJiIelA+/fRTpaWFviQ9PV1+v1+SVFBQIKfTqc2bNwef7+zs1Pbt21VcXCxJKi4u1qFDh9TQ0BBss2XLFvn9fs2cOTPqExkJ2g/3H06iaQcAQLKKqAelrKxMP/rRj3TGGWfo3HPP1TvvvKOHH35Y1157rSTJZrNp+fLluvfee/XlL39ZBQUFuuuuu5Sfn6958+ZJkiZPnqzS0lLdcMMNWrNmjY4dO6Zly5ZpwYIFYc3gGclys+wxbQcAQLKKKKA89thjuuuuu/T9739f7e3tys/P14033qi777472ObOO+/UkSNHtGTJEh06dEiXXHKJamtrZbf/7Uf3ueee07Jly3TFFVcoLS1N8+fP16OPPhq7szJUUUGOXA67PN6uPu9DsUlyOnqmHAMAkMoiWgfFFCN5HZTALB5JISElsALK6kUzVFro6vU6AABGumFbBwVDV1ro0upFM+R0hA7jOB12wgkAAJ+JaIgHsVFa6NJst5OVZAEA6AcBJUHS02xMJQYAoB8M8QAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxRiW6AMSXz2+pvqVD7Ye7lJtlV1FBjtLTbIkuCwCAEASUFFLb1KrKmma1eruCx1wOuyrK3CotdCWwMgAAQjHEkyJqm1pVXt0YEk4kyePtUnl1o2qbWhNUGQAAvRFQUoDPb6mypllWH88FjlXWNMvn76sFAADxR0BJAfUtHb16Tk5kSWr1dqm+pSN+RQEAMAACSgpoP9x/OImmHQAAw42AkgJys+wxbQcAwHAjoKSAooIcuRx29TeZ2Kae2TxFBTnxLAsAgH4RUFJAeppNFWVuSeoVUgL/ryhzsx4KAMAYBJQUUVro0upFM+R0hA7jOB12rV40g3VQAABGYaG2FFJa6NJst5OVZAEAxiOgpJj0NJuKJ45PdBkAAAyIIR4AAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBxWkg2Tz2+xRDwAAHFCQAlDbVOrKmua1ertCh5zOeyqKHOzyR4AAMOAIZ5B1Da1qry6MSScSJLH26Xy6kbVNrUmqDIAAJIXAWUAPr+lyppmWX08FzhWWdMsn7+vFgAAIFoElAHUt3T06jk5kSWp1dul+paO+BUFAEAKIKAMoP1w/+EkmnYAACA8BJQB5GbZY9oOAACEh4AygKKCHLkcdvU3mdimntk8RQU58SwLAICkR0AZQHqaTRVlbknqFVIC/68oc7MeCgAAMUZAGURpoUurF82Q0xE6jON02LV60QzWQQEAYBiwUFsYSgtdmu12spIsAABxQkAJU3qaTcUTxye6DAAAUgJDPAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYJ+KA8uGHH2rRokUaP368xowZoylTpuiPf/xj8HnLsnT33XfL5XJpzJgxKikp0XvvvRfyHh0dHVq4cKGys7M1btw4XXfddfrkk0+GfjYAACApRBRQ/vrXv2rWrFkaPXq0XnnlFTU3N+uhhx7S5z73uWCbBx54QI8++qjWrFmj7du3a+zYsZozZ466urqCbRYuXKhdu3Zp06ZN2rhxo7Zt26YlS5bE7qwAAMCIZrMsywq38YoVK/TGG2/o97//fZ/PW5al/Px83Xbbbbr99tslSV6vV3l5eVq3bp0WLFig3bt3y+126+2339YFF1wgSaqtrdVVV12lDz74QPn5+YPW0dnZKYfDIa/Xq+zs7HDLBwAACRTJ73dEPSj/9V//pQsuuEDf/va3lZubq+nTp+upp54KPt/S0iKPx6OSkpLgMYfDoZkzZ6qurk6SVFdXp3HjxgXDiSSVlJQoLS1N27dv7/Pvdnd3q7OzM+QxEvj8lur2fqyXdnyour0fy+cPOwsCAJDSRkXS+P3339fq1at166236p/+6Z/09ttv6x//8R+VkZGhxYsXy+PxSJLy8vJCXpeXlxd8zuPxKDc3N7SIUaOUk5MTbHOyqqoqVVZWRlJqwtU2taqyplmt3r8NbbkcdlWUuVVa6EpgZQAAmC+iHhS/368ZM2bovvvu0/Tp07VkyRLdcMMNWrNmzXDVJ0lauXKlvF5v8HHgwIFh/XtDVdvUqvLqxpBwIkkeb5fKqxtV29SaoMoAABgZIgooLpdLbrc75NjkyZO1f/9+SZLT6ZQktbW1hbRpa2sLPud0OtXe3h7y/PHjx9XR0RFsc7LMzExlZ2eHPEzl81uqrGlWX4M5gWOVNc0M9wAAMICIAsqsWbO0Z8+ekGPvvvuuzjzzTElSQUGBnE6nNm/eHHy+s7NT27dvV3FxsSSpuLhYhw4dUkNDQ7DNli1b5Pf7NXPmzKhPxBT1LR29ek5OZElq9XapvqUjfkUBADDCRHQPyi233KKLL75Y9913n77zne+ovr5eP/3pT/XTn/5UkmSz2bR8+XLde++9+vKXv6yCggLdddddys/P17x58yT19LiUlpYGh4aOHTumZcuWacGCBWHN4DFd++H+w0k07QAASEURBZQLL7xQGzZs0MqVK/XDH/5QBQUF+vGPf6yFCxcG29x55506cuSIlixZokOHDumSSy5RbW2t7HZ7sM1zzz2nZcuW6YorrlBaWprmz5+vRx99NHZnlUC5WfbBG0XQDgCAVBTROiimMHkdFJ/f0iWrtsjj7erzPhSbJKfDrj/84GtKT7PFuzwAABJm2NZBweDS02yqKOu5kfjk+BH4f0WZm3ACAMAACCjDoLTQpdWLZsjpCB3GcTrsWr1oBuugAAAwiIjuQUH4Sgtdmu12qr6lQ+2Hu5SbZVdRQQ49JwAAhIGAMozS02wqnjg+0WUAADDiMMQDAACMQ0ABAADGIaAAAADjEFAAAIBxuEnWAD6/xWwfAABOQEBJsNqmVlXWNIdsMOhy2FVR5ma9FABAymKIJ4Fqm1pVXt3Ya/djj7dL5dWNqm1qTVBlAAAkFgElQXx+S5U1zX3u1xM4VlnTLJ9/xG2VBADAkBFQEqS+paNXz8mJLEmt3i7Vt3TErygAAAxBQEmQ9sP9h5No2gEAkEwIKAmSm2UfvFEE7QAASCYElAQpKsiRy2FXf5OJbeqZzVNUkBPPsgAAMAIBJUHS02yqKHNLUq+QEvh/RZmb9VAAACmJgJJApYUurV40Q05H6DCO02HX6kUzWAcFAJCyWKgtwUoLXZrtdrKSLAAAJyCgGCA9zabiieMTXQYAAMZgiAcAABiHgAIAAIxDQAEAAMbhHpQ++PwWN60CAJBABJST1Da1qrKmOWSfHJfDrooyN9N+AQCIE4Z4TlDb1Kry6sZem/h5vF0qr25UbVNrgioDACC1EFA+4/NbqqxpltXHc4FjlTXN8vn7agEAAGKJgPKZ+paOXj0nJ7IktXq7VN/SEb+iAABIUQSUz7Qf7j+cRNMOAABEj4Dymdws++CNImgHAACixyyezxQV5MjlsMvj7erzPhSbejbxKyrIiXdpA2JKNAAgGRFQPpOeZlNFmVvl1Y2ySSEhJfBzX1HmNurHnynRAIBkxRDPCUoLXVq9aIacjtBhHKfDrtWLZhj1o8+UaABAMqMH5SSlhS7NdjuNHjYZbEq0TT1Tome7nUbVDQBAuAgofUhPs6l44vhEl9GvSKZEm3weAAD0h4ASAVNuSGVKNAAg2RFQwmTSDalMiQYAJDtukg2DaTekBqZE99d3Y1NPeDJtSjQAAOEioAzCxD16AlOiJfUKKSdOiZakur0f66UdH6pu78fsIwQAGDEY4hmEqTekBqZEnzzs5Pxs2EmSLlm1xYghKQAAIkVAGYTJN6T2NyV6U7NH5dWNvXp9AkNSpq3pAgDAyQgogzDhhtSBZg+dPCWaNVIAAMmAgDKIRO/RE+nsIVOHpAAAiAQ3yQ4i3BtSh6M3IprZQyYPSQEAEC4CShgSsUdPtLOHTBiSAgBgqBjiCVO89+iJdqgm0UNSAADEAgElAvHcoyfaoZrAkFR5daNsUkhIGe4hKQAAYoUhnhjw+a2YL4g2lKGaRAxJAQAQS/SgDNFw7dEz1KGaeA9JAQAQS/SgDMFw7tETi9lDgSGpb077goonjiecAABGDAJKlOKxRw9DNQCAVMUQT5TitSAaQzUAgFREQIlSPBdEi+fsIQAATMAQT5RYEA0AgOFDQIlSYJZNfwMtNvXM5mFBNAAAIkdAiVIi9+gBACDZEVCGgFk2AAAMD26SHSJm2QAAEHsElBhglg0AALHFEA8AADAOAQUAABiHgAIAAIzDPSgG8fktbrYFAEAEFGPUNrWqsqY5ZH8fl8OuijI305UBACmHIZ4h8vkt1e39WC/t+FB1ez+Oavfi2qZWlVc39tp80OPtUnl1o2qbWmNVLgAAIwI9KEMQi14Pn99SZU2z+oo1lnpWpa2sadZst5PhHgBAyhhSD8r9998vm82m5cuXB491dXVp6dKlGj9+vE499VTNnz9fbW1tIa/bv3+/5s6dq1NOOUW5ubm64447dPz48aGUEnex6vWob+no9R4nsiS1ertU39IxlHIBABhRog4ob7/9tp588kmdd955IcdvueUW1dTU6IUXXtDWrVt18OBBXX311cHnfT6f5s6dq6NHj+rNN9/UM888o3Xr1unuu++O/izibLBeD6mn16O/4Z4Th4Xe+PNfwvqb7Yf7DzEAACSbqIZ4PvnkEy1cuFBPPfWU7r333uBxr9erp59+Ws8//7y+9rWvSZLWrl2ryZMn66233tJFF12k1157Tc3Nzfrtb3+rvLw8TZs2Tffcc49+8IMf6F//9V+VkZERmzMbRpH0epy8wmxfw0LhyM2yD94IAIAkEVUPytKlSzV37lyVlJSEHG9oaNCxY8dCjp9zzjk644wzVFdXJ0mqq6vTlClTlJeXF2wzZ84cdXZ2ateuXX3+ve7ubnV2doY8Einc3oyT2/U3LDQQm3ruaykqyImkRAAARrSIe1DWr1+vxsZGvf32272e83g8ysjI0Lhx40KO5+XlyePxBNucGE4Czwee60tVVZUqKysjLXXYhNubcWK7gYaF+hO4JbaizM0NsgCAlBJRD8qBAwd0880367nnnpPdHr8hh5UrV8rr9QYfBw4ciNvf7ktRQY5cDrv6iwx99XoMNizUF6fDrtWLZrAOCgAg5UTUg9LQ0KD29nbNmDEjeMzn82nbtm16/PHH9eqrr+ro0aM6dOhQSC9KW1ubnE6nJMnpdKq+vj7kfQOzfAJtTpaZmanMzMxISh1W6Wk2VZS5VV7dKJsU0ivSX69HuMNCyy6fqC/nZbGSLAAgpUXUg3LFFVdo586d2rFjR/BxwQUXaOHChcF/jx49Wps3bw6+Zs+ePdq/f7+Ki4slScXFxdq5c6fa29uDbTZt2qTs7Gy53e4YndbwKy10afWiGXI6QnuS+uv1CHdYaNaXPq9vTvuCiieOJ5wAAFJWRD0oWVlZKiwsDDk2duxYjR8/Pnj8uuuu06233qqcnBxlZ2frpptuUnFxsS666CJJ0pVXXim3261rrrlGDzzwgDwej/7lX/5FS5cuNaqXJBylhS7NdjvD2j8nMCzk8Xb1eR+KTT3hhpthAQAYhpVkH3nkEaWlpWn+/Pnq7u7WnDlz9JOf/CT4fHp6ujZu3Kjy8nIVFxdr7NixWrx4sX74wx/GupS4SE+z9ZpK3F+7SIeFAABIVTbLsiLfPCbBOjs75XA45PV6lZ2dnehyImLipoDsogwAiIdIfr/ZiyfOIhkWigcTAxMAAPSgpLDAwnEnfwACUYkpzgCAWIrk93tImwVi5BrqfkIAAAwnAkqKYhdlAIDJCCgpKtr9hAAAiAcCSoqKZj8hAADihYCSoqLZTwgAgHghoMSAz2+pbu/HemnHh6rb+/GIuLE0sHCcpF4hhYXjAACJxjooQzSS1xEJ7Cd0cv3OEVI/ACB5sQ7KECTLOiKsJAsAiAdWko2DwdYRsalnHZHZbqfxP/bh7icEAEC8cA9KlFhHBACA4UMPSpRYRyQ2GF4CAPSFgBIl1hEZupF8gzEAYHgxxBMl1hEZmsANxicPk3m8XSqvblRtU2uCKgMAmICAEqXhXEdkJK6rEgk2KgQADIYhniEYjnVEUmHYI5IbjJldBACpiYAyRKWFLs12O2Nyo2d/66oEhj1Gyroqg+EGYwDAYAgoMRCLdUSSaV2VwXCDMQBgMNyDYohUWleFG4wBAIMhoBgilYY92KgQADAYAoohUm3YI3CDsdMRej5Ohz1p7rUBAESPe1AMERj28Hi7+rwPxaaeH+9kGvaI5Q3GAIDkQkAxRGDYo7y6UTYpJKQk87AHGxUCAPrCEI9BGPYAAKAHPSiGYdgDAAACipEY9gAApDqGeAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGGZXoApKVz2+pvqVD7Ye7lJtlV1FBjtLTbIkuCwCAEYGAMgxqm1pVWdOsVm9X8JjLYVdFmVulha4EVgYAwMjAEE+M1Ta1qry6MSScSJLH26Xy6kbVNrUmqDIAAEYOAkoM+fyWKmuaZfXxXOBYZU2zfP6+WgAAgAACSgzVt3T06jk5kSWp1dul+paO+BUFAMAIRECJofbD/YeTaNoBAJCqCCgxlJtlj2k7AABSFbN4YqioIEcuh10eb1ef96HYJDkdPVOOB8IUZQBAqiOgxFB6mk0VZW6VVzfKJoWElEC8qChzDxg2mKIMAABDPDFXWujS6kUz5HSEDuM4HXatXjRjwJDBFGUAAHrQgzIMSgtdmu12RjRMM9gUZZt6pijPdjsZ7gEAJD0CyjBJT7OpeOL4sNtHMkU5kvcFAGAkYojHEExRBgDgbwgohmCKMgAAf0NAMURginJ/d5fY1DObZ7ApygAAJAMCiiECU5Ql9Qop4U5RBgAgWRBQDDKUKcoAACQTZvEYJpopygAAJBsCioEinaIMAECyYYgHAAAYhx6UGGBzPwAAYouAMkRs7gcAQOwxxDME4W7u5/Nbqtv7sV7a8aHq9n4sn7+vHXcAAEAAPShRCndzP79fuuc39LAAABCJiHpQqqqqdOGFFyorK0u5ubmaN2+e9uzZE9Kmq6tLS5cu1fjx43Xqqadq/vz5amtrC2mzf/9+zZ07V6eccopyc3N1xx136Pjx40M/mzgKd3O/7z8/eA8LAAAIFVFA2bp1q5YuXaq33npLmzZt0rFjx3TllVfqyJEjwTa33HKLampq9MILL2jr1q06ePCgrr766uDzPp9Pc+fO1dGjR/Xmm2/qmWee0bp163T33XfH7qziYCib9gV6XSprmhnuAQCgDzbLsqL+hfzoo4+Um5urrVu36itf+Yq8Xq8+//nP6/nnn9e3vvUtSdL//M//aPLkyaqrq9NFF12kV155Rd/4xjd08OBB5eXlSZLWrFmjH/zgB/roo4+UkZEx6N/t7OyUw+GQ1+tVdnZ2tOUPSd3ej/Xdp94a8vv84oaLWPMEAJASIvn9HtJNsl6vV5KUk9OzgV1DQ4OOHTumkpKSYJtzzjlHZ5xxhurq6iRJdXV1mjJlSjCcSNKcOXPU2dmpXbt29fl3uru71dnZGfJItME29wvXUHpiAABIVlEHFL/fr+XLl2vWrFkqLCyUJHk8HmVkZGjcuHEhbfPy8uTxeIJtTgwngecDz/WlqqpKDocj+JgwYUK0ZcdMOJv7hSM3yz54IwAAUkzUAWXp0qVqamrS+vXrY1lPn1auXCmv1xt8HDhwYNj/ZjgG2tzvJ/9v+oA9LDb1zOYpKsgZ9joBABhpoppmvGzZMm3cuFHbtm3T6aefHjzudDp19OhRHTp0KKQXpa2tTU6nM9imvr4+5P0Cs3wCbU6WmZmpzMzMaEoddgNt7peWZlN5daNsUsh05EBoqShzs+IsAAB9iKgHxbIsLVu2TBs2bNCWLVtUUFAQ8vz555+v0aNHa/PmzcFje/bs0f79+1VcXCxJKi4u1s6dO9Xe3h5ss2nTJmVnZ8vtdg/lXBImsLnfN6d9QcUTxwdDx0A9LKsXzWAdFAAA+hHRLJ7vf//7ev755/XSSy9p0qRJweMOh0NjxoyRJJWXl+vll1/WunXrlJ2drZtuukmS9Oabb0rqmWY8bdo05efn64EHHpDH49E111yj66+/Xvfdd19YdZgwiycS7NUDAEBkv98RBRSbre8f1bVr1+rv//7vJfUs1HbbbbfpF7/4hbq7uzVnzhz95Cc/CRm+2bdvn8rLy/X6669r7NixWrx4se6//36NGhXeiNNICyj9IbgAAFLJsAUUUyRDQGGTQQBAqonbOiiITribDAIAkKoIKBGIxa7Eg20yKLEEPgAA7GYcplgNyYS7yWB9SwdL4AMAUhY9KGGI5ZBMuEvbswQ+ACCVEVAGEeshmXCXtmcJfABAKiOgDCKSIZlwDLbJIEvgAwBAQBlUrIdkwtlkkCXwAQCpjoAyiOEYkmEJfAAABsYsnkEEhmQ83q4+70OxqSdYRDokM9AmgwAApDoCyiACQzLDsStxYJNBAAAQiiGeMDAkAwBAfNGDEiaGZAAAiB8CSgQYkgEAID4Y4gEAAMahByVJ+PwWw08AgKRBQEkCsdrIEAAAUzDEM8LFciNDAABMQUAZwWK9kSEAAKYgoIxgsd7IEAAAUxBQRrBYb2QIAIApCCgj2HBsZAgAgAkIKCNYYCPD/iYT29QzmyfSjQwBAEg0AsoIFtjIUFKvkDLUjQwBAEgkAsoIx0aGAIBkxEJtSYCNDAEAyYaAkiTYyBAAkEwY4gEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYZ1SiCzDJ0eN+/bzuf7Wv41OdmXOKrik+Sxmjkj/D+fyW6ls61H64S7lZdhUV5Cg9zZbosmJSl6nnBgCmMuV7M6EB5YknntC//du/yePxaOrUqXrsscdUVFSUkFqqXm7WU79vkd/627EfvbxbN1xaoJVXuRNSUzzUNrWqsqZZrd6u4DGXw66KMrdKC10jui5Tzw0ATGXS92bCugd++ctf6tZbb1VFRYUaGxs1depUzZkzR+3t7XGvperlZj25LTScSJLfkp7c1qKql5vjXlM81Da1qry6MeSDKEkeb5fKqxtV29Q6Yusy9dwAwFSmfW8mLKA8/PDDuuGGG/S9731Pbrdba9as0SmnnKKf/exnca3j6HG/nvp9y4Btnvp9i44e98epovjw+S1V1jTL6uO5wLHKmmb5Tk5twywWdZl6bgBgKhO/NxMSUI4ePaqGhgaVlJT8rZC0NJWUlKiurq5X++7ubnV2doY8YuXndf/bq+fkZH6rp10yqW/p6JWST2RJavV2qb6lI35FKTZ1mXpuAGAqE783ExJQ/vKXv8jn8ykvLy/keF5enjweT6/2VVVVcjgcwceECRNiVsu+jk9j2m6kaD/c/wcxmnaxEou6TD03ADCVid+bI2KKysqVK+X1eoOPAwcOxOy9z8w5JabtRorcLHtM28VKLOoy9dwAwFQmfm8mJKCcdtppSk9PV1tbW8jxtrY2OZ3OXu0zMzOVnZ0d8oiVa4rP0mCzp9JsPe2SSVFBjlwOu/o7dZt67twuKsiJZ1kxqcvUcwMAU5n4vZmQgJKRkaHzzz9fmzdvDh7z+/3avHmziouL41vLqDTdcGnBgG1uuLQg6dZDSU+zqaKsZ/r0yR/IwP8rytxxn/sei7pMPTcAMJWJ35sJ+9W99dZb9dRTT+mZZ57R7t27VV5eriNHjuh73/te3GtZeZVbN36loFdPSppNuvErybsOSmmhS6sXzZDTEdpl53TYtXrRjIStFRKLukw9NwAwlWnfmzbLshI21/Lxxx8PLtQ2bdo0Pfroo5o5c+agr+vs7JTD4ZDX643pcA8ryZq12ioryQJA/A3n92Ykv98JDSjRGq6AAgAAhk8kv9/J3z0AAABGHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGCcUYkuIBqBxW87OzsTXAkAAAhX4Hc7nEXsR2RAOXz4sCRpwoQJCa4EAABE6vDhw3I4HAO2GZF78fj9fh08eFBZWVmy2Ya+gVFnZ6cmTJigAwcOsLfPSbg2feO69I9r0zeuS/+4Nv1LtmtjWZYOHz6s/Px8paUNfJfJiOxBSUtL0+mnnx7z983Ozk6KD8Bw4Nr0jevSP65N37gu/ePa9C+Zrs1gPScB3CQLAACMQ0ABAADGIaBIyszMVEVFhTIzMxNdinG4Nn3juvSPa9M3rkv/uDb9S+VrMyJvkgUAAMmNHhQAAGAcAgoAADAOAQUAABiHgAIAAIyTMgHliSee0FlnnSW73a6ZM2eqvr5+wPYvvPCCzjnnHNntdk2ZMkUvv/xynCqNv0iuzbp162Sz2UIedrs9jtXGx7Zt21RWVqb8/HzZbDa9+OKLg77m9ddf14wZM5SZmakvfelLWrdu3bDXGW+RXpfXX3+91+fFZrPJ4/HEp+A4qqqq0oUXXqisrCzl5uZq3rx52rNnz6CvS/bvmmiuS6p8z6xevVrnnXdecBG24uJivfLKKwO+Jtk/LydKiYDyy1/+UrfeeqsqKirU2NioqVOnas6cOWpvb++z/Ztvvqnvfve7uu666/TOO+9o3rx5mjdvnpqamuJc+fCL9NpIPSsatra2Bh/79u2LY8XxceTIEU2dOlVPPPFEWO1bWlo0d+5cXX755dqxY4eWL1+u66+/Xq+++uowVxpfkV6XgD179oR8ZnJzc4epwsTZunWrli5dqrfeekubNm3SsWPHdOWVV+rIkSP9viYVvmuiuS5SanzPnH766br//vvV0NCgP/7xj/ra176mb37zm9q1a1ef7VPh8xLCSgFFRUXW0qVLg//3+XxWfn6+VVVV1Wf773znO9bcuXNDjs2cOdO68cYbh7XORIj02qxdu9ZyOBxxqs4MkqwNGzYM2ObOO++0zj333JBjf/d3f2fNmTNnGCtLrHCuy+9+9ztLkvXXv/41LjWZpL293ZJkbd26td82qfRdExDOdUnF75mAz33uc9Z//Md/9Plcqn1ekr4H5ejRo2poaFBJSUnwWFpamkpKSlRXV9fna+rq6kLaS9KcOXP6bT9SRXNtJOmTTz7RmWeeqQkTJgyY9lNJqnxmojVt2jS5XC7Nnj1bb7zxRqLLiQuv1ytJysnJ6bdNKn5uwrkuUup9z/h8Pq1fv15HjhxRcXFxn21S7fOS9AHlL3/5i3w+n/Ly8kKO5+Xl9TsO7vF4Imo/UkVzbSZNmqSf/exneumll1RdXS2/36+LL75YH3zwQTxKNlZ/n5nOzk793//9X4KqSjyXy6U1a9bo17/+tX79619rwoQJuuyyy9TY2Jjo0oaV3+/X8uXLNWvWLBUWFvbbLlW+awLCvS6p9D2zc+dOnXrqqcrMzNQ//MM/aMOGDXK73X22TbXPy4jczRiJU1xcHJLuL774Yk2ePFlPPvmk7rnnngRWBhNNmjRJkyZNCv7/4osv1t69e/XII4/o5z//eQIrG15Lly5VU1OT/vCHPyS6FKOEe11S6Xtm0qRJ2rFjh7xer/7zP/9Tixcv1tatW/sNKakk6XtQTjvtNKWnp6utrS3keFtbm5xOZ5+vcTqdEbUfqaK5NicbPXq0pk+frj//+c/DUeKI0d9nJjs7W2PGjElQVWYqKipK6s/LsmXLtHHjRv3ud7/T6aefPmDbVPmukSK7LidL5u+ZjIwMfelLX9L555+vqqoqTZ06Vf/+7//eZ9tU+rxIKRBQMjIydP7552vz5s3BY36/X5s3b+53nK+4uDikvSRt2rSp3/YjVTTX5mQ+n087d+6Uy+UarjJHhFT5zMTCjh07kvLzYlmWli1bpg0bNmjLli0qKCgY9DWp8LmJ5rqcLJW+Z/x+v7q7u/t8LhU+LyESfZduPKxfv97KzMy01q1bZzU3N1tLliyxxo0bZ3k8HsuyLOuaa66xVqxYEWz/xhtvWKNGjbIefPBBa/fu3VZFRYU1evRoa+fOnYk6hWET6bWprKy0Xn31VWvv3r1WQ0ODtWDBAstut1u7du1K1CkMi8OHD1vvvPOO9c4771iSrIcffth65513rH379lmWZVkrVqywrrnmmmD7999/3zrllFOsO+64w9q9e7f1xBNPWOnp6VZtbW2iTmFYRHpdHnnkEevFF1+03nvvPWvnzp3WzTffbKWlpVm//e1vE3UKw6a8vNxyOBzW66+/brW2tgYfn376abBNKn7XRHNdUuV7ZsWKFdbWrVutlpYW609/+pO1YsUKy2azWa+99pplWan5eTlRSgQUy7Ksxx57zDrjjDOsjIwMq6ioyHrrrbeCz331q1+1Fi9eHNL+V7/6lXX22WdbGRkZ1rnnnmv95je/iXPF8RPJtVm+fHmwbV5ennXVVVdZjY2NCah6eAWmx578CFyLxYsXW1/96ld7vWbatGlWRkaG9cUvftFau3Zt3OsebpFel1WrVlkTJ0607Ha7lZOTY1122WXWli1bElP8MOvrukgK+Ryk4ndNNNclVb5nrr32WuvMM8+0MjIyrM9//vPWFVdcEQwnlpWan5cT2SzLsuLXXwMAADC4pL8HBQAAjDwEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAJGnbtm0qKytTfn6+bDabXnzxxYjfw7IsPfjggzr77LOVmZmpL3zhC/rRj34U8fuwmzEAAJAkHTlyRFOnTtW1116rq6++Oqr3uPnmm/Xaa6/pwQcf1JQpU9TR0aGOjo6I34eVZAEAQC82m00bNmzQvHnzgse6u7v1z//8z/rFL36hQ4cOqbCwUKtWrdJll10mSdq9e7fOO+88NTU1adKkSUP6+wzxAACAsCxbtkx1dXVav369/vSnP+nb3/62SktL9d5770mSampq9MUvflEbN25UQUGBzjrrLF1//fVR9aAQUAAAwKD279+vtWvX6oUXXtCll16qiRMn6vbbb9cll1yitWvXSpLef/997du3Ty+88IKeffZZrVu3Tg0NDfrWt74V8d/jHhQAADConTt3yufz6eyzzw453t3drfHjx0uS/H6/uru79eyzzwbbPf300zr//PO1Z8+eiIZ9CCgAAGBQn3zyidLT09XQ0KD09PSQ50499VRJksvl0qhRo0JCzOTJkyX19MAQUAAAQExNnz5dPp9P7e3tuvTSS/tsM2vWLB0/flx79+7VxIkTJUnvvvuuJOnMM8+M6O8xiwcAAEjq6SX585//LKknkDz88MO6/PLLlZOTozPOOEOLFi3SG2+8oYceekjTp0/XRx99pM2bN+u8887T3Llz5ff7deGFF+rUU0/Vj3/8Y/n9fi1dulTZ2dl67bXXIqqFgAIAACRJr7/+ui6//PJexxcvXqx169bp2LFjuvfee/Xss8/qww8/1GmnnaaLLrpIlZWVmjJliiTp4MGDuummm/Taa69p7Nix+vrXv66HHnpIOTk5EdVCQAEAAMZhmjEAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxvn/RrOP3XuAKTEAAAAASUVORK5CYII=" }, "metadata": {}, "output_type": "display_data" } ], "execution_count": 11 } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.5" } }, "nbformat": 4, "nbformat_minor": 5 }