{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from datasets import load_dataset\n",
    "from transformers import DistilBertForSequenceClassification, DistilBertTokenizer\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "import os\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since we are operating on a Mac with M2 chip, CUDA is not available. However, we can get GPU acceleration like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CUDA?             False\n",
      "MPS available?    True\n",
      "MPS built?        True\n",
      "Device:           mps\n"
     ]
    }
   ],
   "source": [
    "print(\"CUDA?            \", torch.cuda.is_available())\n",
    "\n",
    "print(\"MPS available?   \", torch.backends.mps.is_available()) #the MacOS is higher than 12.3+\n",
    "print(\"MPS built?       \", torch.backends.mps.is_built()) #MPS is activated\n",
    "\n",
    "device = torch.device(\"mps\")\n",
    "print(\"Device:          \", device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EDA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, we will load the dataset: https://huggingface.co/datasets/dair-ai/emotion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No config specified, defaulting to: emotion/split\n",
      "Found cached dataset emotion (/Users/david/.cache/huggingface/datasets/emotion/split/1.0.0/cca5efe2dfeb58c1d098e0f9eeb200e9927d889b5a03c67097275dfb5fe463bd)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "861120cd48de4646996c7bc8e7daf92a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "emotions = load_dataset(\"emotion\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 16000\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 2000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 2000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emotions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': Value(dtype='string', id=None), 'label': ClassLabel(names=['sadness', 'joy', 'love', 'anger', 'fear', 'surprise'], id=None)}\n",
      "\n",
      "{'text': ['i didnt feel humiliated', 'i can go from feeling so hopeless to so damned hopeful just from being around someone who cares and is awake', 'im grabbing a minute to post i feel greedy wrong', 'i am ever feeling nostalgic about the fireplace i will know that it is still on the property', 'i am feeling grouchy'], 'label': [0, 0, 3, 2, 3]}\n"
     ]
    }
   ],
   "source": [
    "train_ds = emotions[\"train\"]\n",
    "val_ds = emotions[\"validation\"]\n",
    "test_ds = emotions[\"test\"]\n",
    "\n",
    "print(train_ds.features)\n",
    "print()\n",
    "print(train_ds[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we will buid a DataFrame to use for further data analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>label_names</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i didnt feel humiliated</td>\n",
       "      <td>0</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i can go from feeling so hopeless to so damned...</td>\n",
       "      <td>0</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>im grabbing a minute to post i feel greedy wrong</td>\n",
       "      <td>3</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i am ever feeling nostalgic about the fireplac...</td>\n",
       "      <td>2</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i am feeling grouchy</td>\n",
       "      <td>3</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15995</th>\n",
       "      <td>i just had a very brief time in the beanbag an...</td>\n",
       "      <td>0</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15996</th>\n",
       "      <td>i am now turning and i feel pathetic that i am...</td>\n",
       "      <td>0</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15997</th>\n",
       "      <td>i feel strong and good overall</td>\n",
       "      <td>1</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15998</th>\n",
       "      <td>i feel like this was such a rude comment and i...</td>\n",
       "      <td>3</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15999</th>\n",
       "      <td>i know a lot but i feel so stupid because i ca...</td>\n",
       "      <td>0</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  label label_names\n",
       "0                                i didnt feel humiliated      0     sadness\n",
       "1      i can go from feeling so hopeless to so damned...      0     sadness\n",
       "2       im grabbing a minute to post i feel greedy wrong      3       anger\n",
       "3      i am ever feeling nostalgic about the fireplac...      2        love\n",
       "4                                   i am feeling grouchy      3       anger\n",
       "...                                                  ...    ...         ...\n",
       "15995  i just had a very brief time in the beanbag an...      0     sadness\n",
       "15996  i am now turning and i feel pathetic that i am...      0     sadness\n",
       "15997                     i feel strong and good overall      1         joy\n",
       "15998  i feel like this was such a rude comment and i...      3       anger\n",
       "15999  i know a lot but i feel so stupid because i ca...      0     sadness\n",
       "\n",
       "[16000 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emotions.set_format(\"pandas\")\n",
    "df = emotions[\"train\"][:]\n",
    "\n",
    "label_dict = {\n",
    "    id:label for id, label in enumerate(['sadness', 'joy', 'love', 'anger', 'fear', 'surprise'])\n",
    "    }\n",
    "\n",
    "df[\"label_names\"] = df[\"label\"].map(lambda x: label_dict[x])\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we can check class imbalances (and resample if necessary) and also text lengths."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "joy         5362\n",
      "sadness     4666\n",
      "anger       2159\n",
      "fear        1937\n",
      "love        1304\n",
      "surprise     572\n",
      "Name: label_names, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='label_names', ylabel='Count'>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGxCAYAAACDV6ltAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA58ElEQVR4nO3deVxWdf7//+elrLJcCQjIiIojIm6N6YTYon1caHEbm7EZjPSTqWVpVFbj2IJNaWPlMtpUOiaNS9ZU9rUNpUXTXCPJjdxC0QIJQ1BDQHj//ujn+XSFC6Jygedxv93Orc45r3Ou1/t0oc/O9T4XDmOMEQAAgI01cHcDAAAA7kYgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtufh7gbqi8rKSn3//fcKCAiQw+FwdzsAAKAajDE6evSoIiIi1KDBme8DEYiq6fvvv1dkZKS72wAAADVw4MABNWvW7Iz7CUTVFBAQIOnnCxoYGOjmbgAAQHUUFxcrMjLS+nv8TAhE1XTqY7LAwEACEQAA9cy5prswqRoAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANieh7sbAADgcpWTk6OCggJ3t1EvhISEqHnz5m57fQIRAACXQE5Ojtq2jVVJyU/ubqVe8PVtpG++yXJbKCIQAQBwCRQUFKik5CfF3fmkApu2dHc7dVpx7j5teHWSCgoKCEQAAFyOApu2VFDzGHe3gXNgUjUAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9twailJQUORwOlyU8PNzab4xRSkqKIiIi5Ovrq549e2r79u0u5ygtLdXYsWMVEhIiPz8/DRgwQAcPHnSpKSwsVFJSkpxOp5xOp5KSknTkyJHaGCIAAKgH3H6HqH379srNzbWWrVu3WvumTp2qadOmafbs2dq0aZPCw8PVp08fHT161KpJTk7W0qVLtWTJEq1Zs0bHjh1Tv379VFFRYdUkJiYqMzNTaWlpSktLU2ZmppKSkmp1nAAAoO7ycHsDHh4ud4VOMcZoxowZmjhxogYPHixJeu211xQWFqbFixdr9OjRKioq0rx587RgwQL17t1bkrRw4UJFRkbq448/VkJCgrKyspSWlqb169crLi5OkjR37lzFx8dr586diomJqb3BAgCAOsntd4h2796tiIgIRUVF6c9//rO+/fZbSVJ2drby8vLUt29fq9bb21s9evTQ2rVrJUkZGRkqLy93qYmIiFCHDh2smnXr1snpdFphSJK6desmp9Np1ZxOaWmpiouLXRYAAHB5cmsgiouL03/+8x8tX75cc+fOVV5enrp3767Dhw8rLy9PkhQWFuZyTFhYmLUvLy9PXl5eaty48VlrQkNDq7x2aGioVXM6U6ZMseYcOZ1ORUZGXtBYAQBA3eXWQHTTTTfp1ltvVceOHdW7d2998MEHkn7+aOwUh8Phcowxpsq2X/t1zenqz3WeCRMmqKioyFoOHDhQrTEBAID6x+0fmf2Sn5+fOnbsqN27d1vzin59Fyc/P9+6axQeHq6ysjIVFhaetebQoUNVXuuHH36ocvfpl7y9vRUYGOiyAACAy1OdCkSlpaXKyspS06ZNFRUVpfDwcKWnp1v7y8rKtGrVKnXv3l2S1KVLF3l6errU5Obmatu2bVZNfHy8ioqKtHHjRqtmw4YNKioqsmoAAIC9ufUps/Hjx6t///5q3ry58vPz9fTTT6u4uFjDhg2Tw+FQcnKyJk+erOjoaEVHR2vy5Mlq1KiREhMTJUlOp1MjRozQQw89pODgYAUFBWn8+PHWR3CSFBsbqxtvvFEjR47UK6+8IkkaNWqU+vXrxxNmAABAkpsD0cGDB/WXv/xFBQUFatKkibp166b169erRYsWkqRHHnlEJSUlGjNmjAoLCxUXF6cVK1YoICDAOsf06dPl4eGhIUOGqKSkRL169VJqaqoaNmxo1SxatEjjxo2znkYbMGCAZs+eXbuDBQAAdZbDGGPc3UR9UFxcLKfTqaKiIuYTAQDO6auvvlKXLl3UZ+J8BTXnE4mz+TFnp9Kf+V9lZGToqquuuqjnru7f33VqDhEAAIA7EIgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtebi7AUg5OTkqKChwdxv1QkhIiJo3b+7uNgAAlxkCkZvl5OSobdtYlZT85O5W6gVf30b65pssQhEA4KIiELlZQUGBSkp+UtydTyqwaUt3t1OnFefu04ZXJ6mgoIBABAC4qAhEdURg05YKah7j7jYAALAlJlUDAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbqzOBaMqUKXI4HEpOTra2GWOUkpKiiIgI+fr6qmfPntq+fbvLcaWlpRo7dqxCQkLk5+enAQMG6ODBgy41hYWFSkpKktPplNPpVFJSko4cOVILowIAAPVBnQhEmzZt0pw5c9SpUyeX7VOnTtW0adM0e/Zsbdq0SeHh4erTp4+OHj1q1SQnJ2vp0qVasmSJ1qxZo2PHjqlfv36qqKiwahITE5WZmam0tDSlpaUpMzNTSUlJtTY+AABQt7k9EB07dkxDhw7V3Llz1bhxY2u7MUYzZszQxIkTNXjwYHXo0EGvvfaafvrpJy1evFiSVFRUpHnz5umFF15Q79691blzZy1cuFBbt27Vxx9/LEnKyspSWlqa/v3vfys+Pl7x8fGaO3eu3n//fe3cudMtYwYAAHWL2wPRvffeq1tuuUW9e/d22Z6dna28vDz17dvX2ubt7a0ePXpo7dq1kqSMjAyVl5e71ERERKhDhw5Wzbp16+R0OhUXF2fVdOvWTU6n06o5ndLSUhUXF7ssAADg8uThzhdfsmSJvvrqK23atKnKvry8PElSWFiYy/awsDDt37/fqvHy8nK5s3Sq5tTxeXl5Cg0NrXL+0NBQq+Z0pkyZokmTJp3fgAAAQL3ktjtEBw4c0P3336+FCxfKx8fnjHUOh8Nl3RhTZduv/brmdPXnOs+ECRNUVFRkLQcOHDjrawIAgPrLbYEoIyND+fn56tKlizw8POTh4aFVq1bpn//8pzw8PKw7Q7++i5Ofn2/tCw8PV1lZmQoLC89ac+jQoSqv/8MPP1S5+/RL3t7eCgwMdFkAAMDlyW2BqFevXtq6dasyMzOtpWvXrho6dKgyMzPVqlUrhYeHKz093TqmrKxMq1atUvfu3SVJXbp0kaenp0tNbm6utm3bZtXEx8erqKhIGzdutGo2bNigoqIiqwYAANib2+YQBQQEqEOHDi7b/Pz8FBwcbG1PTk7W5MmTFR0drejoaE2ePFmNGjVSYmKiJMnpdGrEiBF66KGHFBwcrKCgII0fP14dO3a0JmnHxsbqxhtv1MiRI/XKK69IkkaNGqV+/fopJiamFkcMAADqKrdOqj6XRx55RCUlJRozZowKCwsVFxenFStWKCAgwKqZPn26PDw8NGTIEJWUlKhXr15KTU1Vw4YNrZpFixZp3Lhx1tNoAwYM0OzZs2t9PAAAoG6qU4Fo5cqVLusOh0MpKSlKSUk54zE+Pj6aNWuWZs2adcaaoKAgLVy48CJ1CQAALjdu/x4iAAAAdyMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA26tRIGrVqpUOHz5cZfuRI0fUqlWrC24KAACgNtUoEO3bt08VFRVVtpeWluq777674KYAAABq03kFomXLlmnZsmWSpOXLl1vry5Yt09KlS/X3v/9dLVu2rPb5XnrpJXXq1EmBgYEKDAxUfHy8PvroI2u/MUYpKSmKiIiQr6+vevbsqe3bt7uco7S0VGPHjlVISIj8/Pw0YMAAHTx40KWmsLBQSUlJcjqdcjqdSkpK0pEjR85n6AAA4DLmcT7FgwYNkiQ5HA4NGzbMZZ+np6datmypF154odrna9asmZ599lm1bt1akvTaa69p4MCB2rx5s9q3b6+pU6dq2rRpSk1NVZs2bfT000+rT58+2rlzpwICAiRJycnJeu+997RkyRIFBwfroYceUr9+/ZSRkaGGDRtKkhITE3Xw4EGlpaVJkkaNGqWkpCS999575zN8AABwmTqvQFRZWSlJioqK0qZNmxQSEnJBL96/f3+X9WeeeUYvvfSS1q9fr3bt2mnGjBmaOHGiBg8eLOnnwBQWFqbFixdr9OjRKioq0rx587RgwQL17t1bkrRw4UJFRkbq448/VkJCgrKyspSWlqb169crLi5OkjR37lzFx8dr586diomJuaAxAACA+q9Gc4iys7MvOAz9WkVFhZYsWaLjx48rPj5e2dnZysvLU9++fa0ab29v9ejRQ2vXrpUkZWRkqLy83KUmIiJCHTp0sGrWrVsnp9NphSFJ6tatm5xOp1UDAADs7bzuEP3SJ598ok8++UT5+fnWnaNTXn311WqfZ+vWrYqPj9eJEyfk7++vpUuXql27dlZYCQsLc6kPCwvT/v37JUl5eXny8vJS48aNq9Tk5eVZNaGhoVVeNzQ01Ko5ndLSUpWWllrrxcXF1R4TAACoX2oUiCZNmqSnnnpKXbt2VdOmTeVwOGrcQExMjDIzM3XkyBG9/fbbGjZsmFatWmXt//W5jTHnfL1f15yu/lznmTJliiZNmlTdYQAAgHqsRoHo5ZdfVmpqqpKSki64AS8vL2tSddeuXbVp0ybNnDlTjz76qKSf7/A0bdrUqs/Pz7fuGoWHh6usrEyFhYUud4ny8/PVvXt3q+bQoUNVXveHH36ocvfplyZMmKAHH3zQWi8uLlZkZOQFjBQAANRVNZpDVFZWZgWOi80Yo9LSUkVFRSk8PFzp6ekur7tq1Srrtbt06SJPT0+XmtzcXG3bts2qiY+PV1FRkTZu3GjVbNiwQUVFRWcdg7e3t/V1AKcWAABwearRHaK77rpLixcv1uOPP35BL/63v/1NN910kyIjI3X06FEtWbJEK1euVFpamhwOh5KTkzV58mRFR0crOjpakydPVqNGjZSYmChJcjqdGjFihB566CEFBwcrKChI48ePV8eOHa2nzmJjY3XjjTdq5MiReuWVVyT9/Nh9v379eMIMAABIqmEgOnHihObMmaOPP/5YnTp1kqenp8v+adOmVes8hw4dUlJSknJzc+V0OtWpUyelpaWpT58+kqRHHnlEJSUlGjNmjAoLCxUXF6cVK1ZY30EkSdOnT5eHh4eGDBmikpIS9erVS6mpqdZ3EEnSokWLNG7cOOtptAEDBmj27Nk1GToAALgM1SgQbdmyRb/73e8kSdu2bXPZdz4TrOfNm3fW/Q6HQykpKUpJSTljjY+Pj2bNmqVZs2adsSYoKEgLFy6sdl8AAMBeahSIPvvss4vdBwAAgNvUaFI1AADA5aRGd4huuOGGs3409umnn9a4IQAAgNpWo0B0av7QKeXl5crMzNS2bduq/NJXAACAuq5GgWj69Omn3Z6SkqJjx45dUEMAAAC17aLOIbr99tvP6/eYAQAA1AUXNRCtW7dOPj4+F/OUAAAAl1yNPjIbPHiwy7oxRrm5ufryyy8v+NurAQAAaluNApHT6XRZb9CggWJiYvTUU09Z3wYNAABQX9QoEM2fP/9i9wEAAOA2NQpEp2RkZCgrK0sOh0Pt2rVT586dL1ZfAAAAtaZGgSg/P19//vOftXLlSl1xxRUyxqioqEg33HCDlixZoiZNmlzsPgEAAC6ZGj1lNnbsWBUXF2v79u368ccfVVhYqG3btqm4uFjjxo272D0CAABcUjW6Q5SWlqaPP/5YsbGx1rZ27drpxRdfZFI1AACod2p0h6iyslKenp5Vtnt6eqqysvKCmwIAAKhNNQpE//M//6P7779f33//vbXtu+++0wMPPKBevXpdtOYAAABqQ40C0ezZs3X06FG1bNlSv/3tb9W6dWtFRUXp6NGjmjVr1sXuEQAA4JKq0RyiyMhIffXVV0pPT9c333wjY4zatWun3r17X+z+AAAALrnzukP06aefql27diouLpYk9enTR2PHjtW4ceP0+9//Xu3bt9fq1asvSaMAAACXynkFohkzZmjkyJEKDAysss/pdGr06NGaNm3aRWsOAACgNpxXIPr666914403nnF/3759lZGRccFNAQAA1KbzCkSHDh067eP2p3h4eOiHH3644KYAAABq03kFot/85jfaunXrGfdv2bJFTZs2veCmAAAAatN5BaKbb75ZTzzxhE6cOFFlX0lJiZ588kn169fvojUHAABQG87rsfvHHntM77zzjtq0aaP77rtPMTExcjgcysrK0osvvqiKigpNnDjxUvUKAABwSZxXIAoLC9PatWt1zz33aMKECTLGSJIcDocSEhL0r3/9S2FhYZekUQAAgEvlvL+YsUWLFvrwww9VWFioPXv2yBij6OhoNW7c+FL0BwAAcMnV6JuqJalx48b6/e9/fzF7AQAAcIsa/S4zAACAywmBCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2J6HuxsAzldWVpa7W6g3QkJC1Lx5c3e3AQB1HoEI9UZJ0WFJDt1+++3ubqXe8PVtpG++ySIUAcA5EIhQb5T/dFSS0e8SH1WTqLbubqfOK87dpw2vTlJBQQGBCADOgUCEesc/tLmCmse4uw0AwGWESdUAAMD2CEQAAMD2CEQAAMD23BqIpkyZot///vcKCAhQaGioBg0apJ07d7rUGGOUkpKiiIgI+fr6qmfPntq+fbtLTWlpqcaOHauQkBD5+flpwIABOnjwoEtNYWGhkpKS5HQ65XQ6lZSUpCNHjlzqIQIAgHrArYFo1apVuvfee7V+/Xqlp6fr5MmT6tu3r44fP27VTJ06VdOmTdPs2bO1adMmhYeHq0+fPjp69KhVk5ycrKVLl2rJkiVas2aNjh07pn79+qmiosKqSUxMVGZmptLS0pSWlqbMzEwlJSXV6ngBAEDd5NanzNLS0lzW58+fr9DQUGVkZOj666+XMUYzZszQxIkTNXjwYEnSa6+9prCwMC1evFijR49WUVGR5s2bpwULFqh3796SpIULFyoyMlIff/yxEhISlJWVpbS0NK1fv15xcXGSpLlz5yo+Pl47d+5UTAxPLAEAYGd1ag5RUVGRJCkoKEiSlJ2drby8PPXt29eq8fb2Vo8ePbR27VpJUkZGhsrLy11qIiIi1KFDB6tm3bp1cjqdVhiSpG7dusnpdFo1v1ZaWqri4mKXBQAAXJ7qTCAyxujBBx/Utddeqw4dOkiS8vLyJElhYWEutWFhYda+vLw8eXl5qXHjxmetCQ0NrfKaoaGhVs2vTZkyxZpv5HQ6FRkZeWEDBAAAdVadCUT33XeftmzZotdff73KPofD4bJujKmy7dd+XXO6+rOdZ8KECSoqKrKWAwcOVGcYAACgHqoTgWjs2LFatmyZPvvsMzVr1szaHh4eLklV7uLk5+dbd43Cw8NVVlamwsLCs9YcOnSoyuv+8MMPVe4+neLt7a3AwECXBQAAXJ7cOqnaGKOxY8dq6dKlWrlypaKiolz2R0VFKTw8XOnp6ercubMkqaysTKtWrdI//vEPSVKXLl3k6emp9PR0DRkyRJKUm5urbdu2aerUqZKk+Ph4FRUVaePGjbr66qslSRs2bFBRUZG6d+9eW8MFgMtCTk6OCgoK3N1GnZeVleXuFnAe3BqI7r33Xi1evFj/7//9PwUEBFh3gpxOp3x9feVwOJScnKzJkycrOjpa0dHRmjx5sho1aqTExESrdsSIEXrooYcUHBysoKAgjR8/Xh07drSeOouNjdWNN96okSNH6pVXXpEkjRo1Sv369eMJMwA4Dzk5OWrbNlYlJT+5u5V6o7y0zN0toBrcGoheeuklSVLPnj1dts+fP1/Dhw+XJD3yyCMqKSnRmDFjVFhYqLi4OK1YsUIBAQFW/fTp0+Xh4aEhQ4aopKREvXr1Umpqqho2bGjVLFq0SOPGjbOeRhswYIBmz559aQcIAJeZgoIClZT8pLg7n1Rg05bubqdOy926TtuWzdHJkyfd3Qqqwe0fmZ2Lw+FQSkqKUlJSzljj4+OjWbNmadasWWesCQoK0sKFC2vSJgDgVwKbtlRQc+6wn01x7j53t4DzUCcmVQMAALgTgQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANieh7sbAHBpZWVlubuFeiEkJETNmzd3dxsA3IRABFymSooOS3Lo9ttvd3cr9YKvbyN9800WoQiwKQIRcJkq/+moJKPfJT6qJlFt3d1OnVacu08bXp2kgoICAhFgUwQi4DLnH9pcQc1j3N0GANRpTKoGAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC259ZA9Pnnn6t///6KiIiQw+HQu+++67LfGKOUlBRFRETI19dXPXv21Pbt211qSktLNXbsWIWEhMjPz08DBgzQwYMHXWoKCwuVlJQkp9Mpp9OppKQkHTly5BKPDgAA1BduDUTHjx/XlVdeqdmzZ592/9SpUzVt2jTNnj1bmzZtUnh4uPr06aOjR49aNcnJyVq6dKmWLFmiNWvW6NixY+rXr58qKiqsmsTERGVmZiotLU1paWnKzMxUUlLSJR8fAACoHzzc+eI33XSTbrrpptPuM8ZoxowZmjhxogYPHixJeu211xQWFqbFixdr9OjRKioq0rx587RgwQL17t1bkrRw4UJFRkbq448/VkJCgrKyspSWlqb169crLi5OkjR37lzFx8dr586diomJqZ3BAgCAOqvOziHKzs5WXl6e+vbta23z9vZWjx49tHbtWklSRkaGysvLXWoiIiLUoUMHq2bdunVyOp1WGJKkbt26yel0WjUAAMDe3HqH6Gzy8vIkSWFhYS7bw8LCtH//fqvGy8tLjRs3rlJz6vi8vDyFhoZWOX9oaKhVczqlpaUqLS211ouLi2s2EAAAUOfV2TtEpzgcDpd1Y0yVbb/265rT1Z/rPFOmTLEmYTudTkVGRp5n5wAAoL6os4EoPDxckqrcxcnPz7fuGoWHh6usrEyFhYVnrTl06FCV8//www9V7j790oQJE1RUVGQtBw4cuKDxAACAuqvOBqKoqCiFh4crPT3d2lZWVqZVq1ape/fukqQuXbrI09PTpSY3N1fbtm2zauLj41VUVKSNGzdaNRs2bFBRUZFVczre3t4KDAx0WQAAwOXJrXOIjh07pj179ljr2dnZyszMVFBQkJo3b67k5GRNnjxZ0dHRio6O1uTJk9WoUSMlJiZKkpxOp0aMGKGHHnpIwcHBCgoK0vjx49WxY0frqbPY2FjdeOONGjlypF555RVJ0qhRo9SvXz+eMAMAAJLcHIi+/PJL3XDDDdb6gw8+KEkaNmyYUlNT9cgjj6ikpERjxoxRYWGh4uLitGLFCgUEBFjHTJ8+XR4eHhoyZIhKSkrUq1cvpaamqmHDhlbNokWLNG7cOOtptAEDBpzxu48AAID9uDUQ9ezZU8aYM+53OBxKSUlRSkrKGWt8fHw0a9YszZo164w1QUFBWrhw4YW0CgAALmN1dg4RAABAbSEQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2/NwdwMAUFdkZWW5u4U6j2uEyxWBCIDtlRQdluTQ7bff7u5W6o3y0jJ3twBcVAQiALZX/tNRSUa/S3xUTaLaurudOi136zptWzZHJ0+edHcrwEVFIAKA/59/aHMFNY9xdxt1WnHuPne3AFwSTKoGAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2Z6tA9K9//UtRUVHy8fFRly5dtHr1ane3BAAA6gDbBKI33nhDycnJmjhxojZv3qzrrrtON910k3JyctzdGgAAcDPbBKJp06ZpxIgRuuuuuxQbG6sZM2YoMjJSL730krtbAwAAbmaLQFRWVqaMjAz17dvXZXvfvn21du1aN3UFAADqCg93N1AbCgoKVFFRobCwMJftYWFhysvLO+0xpaWlKi0ttdaLiookScXFxRe1t2PHjkmSfty/UydLSy7quS83xbn7JUlF3+2Wp4fDzd3UfVyv6uNaVR/Xqvq4VtVXnPfz9JVjx45d9L9nT53PGHP2QmMD3333nZFk1q5d67L96aefNjExMac95sknnzSSWFhYWFhYWC6D5cCBA2fNCra4QxQSEqKGDRtWuRuUn59f5a7RKRMmTNCDDz5orVdWVurHH39UcHCwHI7LP+kXFxcrMjJSBw4cUGBgoLvbqdO4VtXHtao+rtX54XpVn92ulTFGR48eVURExFnrbBGIvLy81KVLF6Wnp+sPf/iDtT09PV0DBw487THe3t7y9vZ22XbFFVdcyjbrpMDAQFv8wFwMXKvq41pVH9fq/HC9qs9O18rpdJ6zxhaBSJIefPBBJSUlqWvXroqPj9ecOXOUk5Oju+++292tAQAAN7NNILrtttt0+PBhPfXUU8rNzVWHDh304YcfqkWLFu5uDQAAuJltApEkjRkzRmPGjHF3G/WCt7e3nnzyySofG6IqrlX1ca2qj2t1frhe1ce1Oj2HMed6Dg0AAODyZosvZgQAADgbAhEAALA9AhHkcDj07rvvursN1EM9e/ZUcnKyu9uwtZSUFP3ud79zdxsXnTFGo0aNUlBQkBwOhzIzM93dUr0xfPhwDRo0yN1t1Du2mlQNAJeb8ePHa+zYse5u46JLS0tTamqqVq5cqVatWikkJMTdLdUbM2fOPPevqUAVBCKgDiovL5enp6e720AtKCsrk5eX13kfZ4xRRUWF/P395e/vfwk6c6+9e/eqadOm6t69+yV7jZpe+7quOl9CiKr4yKweeuutt9SxY0f5+voqODhYvXv31vHjx7Vp0yb16dNHISEhcjqd6tGjh7766iuXY3fv3q3rr79ePj4+ateundLT013279u3Tw6HQ++8845uuOEGNWrUSFdeeaXWrVvnUrd27Vpdf/318vX1VWRkpMaNG6fjx49b+//1r38pOjpaPj4+CgsL0x//+Mdz9u8OaWlpuvbaa3XFFVcoODhY/fr10969eyVV/1rMnTtXkZGRatSokf7whz9o2rRpVb7V/L333lOXLl3k4+OjVq1aadKkSTp58qS13+Fw6OWXX9bAgQPl5+enp59++pKP/WIrLCzUHXfcocaNG6tRo0a66aabtHv3bkk//3JkX19fpaWluRzzzjvvyM/Pz/olx999951uu+02NW7cWMHBwRo4cKD27dtX20M5pzO9h0/3EeKgQYM0fPhwa71ly5Z6+umnNXz4cDmdTo0cOdJ6ry1ZskTdu3eXj4+P2rdvr5UrV1rHrVy5Ug6HQ8uXL1fXrl3l7e2t1atXV/nIbOXKlbr66qvl5+enK664Qtdcc432799v7T/Xe7EuGD58uMaOHaucnBw5HA61bNlSxhhNnTpVrVq1kq+vr6688kq99dZb1jEVFRUaMWKEoqKi5Ovrq5iYGM2cObPKeQcNGqQpU6YoIiJCbdq0qe2h1YpffmRWWlqqcePGKTQ0VD4+Prr22mu1adMmST+H6tatW+v55593OX7btm1q0KCB9WehbVzwb05Frfr++++Nh4eHmTZtmsnOzjZbtmwxL774ojl69Kj55JNPzIIFC8yOHTvMjh07zIgRI0xYWJgpLi42xhhTUVFhOnToYHr27Gk2b95sVq1aZTp37mwkmaVLlxpjjMnOzjaSTNu2bc37779vdu7caf74xz+aFi1amPLycmOMMVu2bDH+/v5m+vTpZteuXeaLL74wnTt3NsOHDzfGGLNp0ybTsGFDs3jxYrNv3z7z1VdfmZkzZ56zf3d46623zNtvv2127dplNm/ebPr37286duxoKioqqnUt1qxZYxo0aGCee+45s3PnTvPiiy+aoKAg43Q6rddIS0szgYGBJjU11ezdu9esWLHCtGzZ0qSkpFg1kkxoaKiZN2+e2bt3r9m3b19tX4oa6dGjh7n//vuNMcYMGDDAxMbGms8//9xkZmaahIQE07p1a1NWVmaMMebWW281t99+u8vxt956q/nLX/5ijDHm+PHjJjo62tx5551my5YtZseOHSYxMdHExMSY0tLSWh3X2ZztPfzL63HKwIEDzbBhw6z1Fi1amMDAQPPcc8+Z3bt3m927d1vvtWbNmpm33nrL7Nixw9x1110mICDAFBQUGGOM+eyzz4wk06lTJ7NixQqzZ88eU1BQYJ588klz5ZVXGmOMKS8vN06n04wfP97s2bPH7Nixw6Smppr9+/cbY6r3XqwLjhw5Yp566inTrFkzk5uba/Lz883f/vY307ZtW5OWlmb27t1r5s+fb7y9vc3KlSuNMcaUlZWZJ554wmzcuNF8++23ZuHChaZRo0bmjTfesM47bNgw4+/vb5KSksy2bdvM1q1b3TXES2rYsGFm4MCBxhhjxo0bZyIiIsyHH35otm/fboYNG2YaN25sDh8+bIwx5plnnjHt2rVzOf6BBx4w119/fW237XYEonomIyPDSKrWX5gnT540AQEB5r333jPGGLN8+XLTsGFDl9/4+9FHH502EP373/+2arZv324kmaysLGOMMUlJSWbUqFEur7V69WrToEEDU1JSYt5++20TGBhoBbGa9u8O+fn5RpLZunVrta7FbbfdZm655RaXcwwdOtQlEF133XVm8uTJLjULFiwwTZs2tdYlmeTk5EswokvrVADYtWuXkWS++OILa19BQYHx9fU1b775pjHGmHfeecf4+/ub48ePG2OMKSoqMj4+PuaDDz4wxhgzb948ExMTYyorK61zlJaWGl9fX7N8+fJaHNXZne09XN1ANGjQIJeaU++1Z5991tpWXl5umjVrZv7xj38YY/4vEL377rsux/4yEB0+fNhIskLCr1XnvVhXTJ8+3bRo0cIYY8yxY8eMj4+PWbt2rUvNiBEjrEB9OmPGjDG33nqrtT5s2DATFhZWpwL2pXAqEB07dsx4enqaRYsWWfvKyspMRESEmTp1qjHm54DfsGFDs2HDBmt/kyZNTGpqqlt6dyc+MqtnrrzySvXq1UsdO3bUn/70J82dO1eFhYWSpPz8fN19991q06aNnE6nnE6njh07ppycHElSVlaWmjdvrmbNmlnni4+PP+3rdOrUyfr3pk2bWueXpIyMDKWmplpzF/z9/ZWQkKDKykplZ2erT58+atGihVq1aqWkpCQtWrRIP/300zn7d4e9e/cqMTFRrVq1UmBgoKKioiTJumbS2a/Fzp07dfXVV7uc89frGRkZeuqpp1yu18iRI5Wbm2tdF0nq2rXrxR1cLcrKypKHh4fi4uKsbcHBwYqJiVFWVpYk6ZZbbpGHh4eWLVsmSXr77bcVEBCgvn37Svr5Ou3Zs0cBAQHWdQoKCtKJEyfq1K37i/EePtN/61/+PHp4eKhr167W9TvXsZIUFBSk4cOHKyEhQf3799fMmTOVm5tr7a/ue7Gu2bFjh06cOKE+ffq49P6f//zH5b3x8ssvq2vXrmrSpIn8/f01d+5cl59lSerYseNlOW/odPbu3avy8nJdc8011jZPT09dffXV1vuqadOmuuWWW/Tqq69Kkt5//32dOHFCf/rTn9zSszsRiOqZhg0bKj09XR999JHatWunWbNmKSYmRtnZ2Ro+fLgyMjI0Y8YMrV27VpmZmQoODlZZWZkknfapA4fDcdrX+eWE3lM1lZWV1j9Hjx6tzMxMa/n666+1e/du/fa3v1VAQIC++uorvf7662ratKmeeOIJXXnllTpy5MhZ+3eH/v376/Dhw5o7d642bNigDRs2SJJ1zaSzXwtjTJVr+OvrXFlZqUmTJrlcr61bt2r37t3y8fGx6vz8/C7u4GrR6d5bp7afuj5eXl764x//qMWLF0uSFi9erNtuu00eHj8/21FZWakuXbq4XKfMzEzt2rVLiYmJtTOQajjbe7hBgwZVrkV5eXmVc5zPf+tfv7/Odez8+fO1bt06de/eXW+88YbatGmj9evXS6r+e7GuOfXz9sEHH7j0vmPHDmse0ZtvvqkHHnhAd955p1asWKHMzEz97//+r8vPslS/f87O16n34un+jPrltrvuuktLlixRSUmJ5s+fr9tuu02NGjWq1V7rAgJRPeRwOHTNNddo0qRJ2rx5s7y8vLR06VKtXr1a48aN080336z27dvL29tbBQUF1nHt2rVTTk6Ovv/+e2vbrycIV8dVV12l7du3q3Xr1lWWU//n5eHhod69e2vq1KnasmWL9u3bp08//fSs/de2w4cPKysrS4899ph69eql2NjY8/4//bZt22rjxo0u27788kuX9auuuko7d+487fVq0ODy+BFs166dTp48aQVK6efru2vXLsXGxlrbhg4dqrS0NG3fvl2fffaZhg4dau276qqrtHv3boWGhla5TnXtqZkzvYebNGnickemoqJC27Ztq/Z5TwUXSTp58qQyMjLUtm3b8+6vc+fOmjBhgtauXasOHTpYIbS+vhfbtWsnb29v5eTkVOk7MjJSkrR69Wp1795dY8aMUefOndW6des6dWfRHU79mbxmzRprW3l5ub788kuXn8ubb75Zfn5+eumll/TRRx/pzjvvdEe7bsdj9/XMhg0b9Mknn6hv374KDQ3Vhg0b9MMPPyg2NlatW7fWggUL1LVrVxUXF+vhhx+Wr6+vdWzv3r0VExOjO+64Qy+88IKKi4s1ceLE8+7h0UcfVbdu3XTvvfdq5MiR8vPzU1ZWltLT0zVr1iy9//77+vbbb3X99dercePG+vDDD1VZWamYmJiz9l/bTj3JNGfOHDVt2lQ5OTn661//el7nGDt2rK6//npNmzZN/fv316effqqPPvrI5f++nnjiCfXr10+RkZH605/+pAYNGmjLli3aunVrvXya7HSio6M1cOBAjRw5Uq+88ooCAgL017/+Vb/5zW80cOBAq65Hjx4KCwvT0KFD1bJlS3Xr1s3aN3ToUD333HMaOHCgnnrqKTVr1kw5OTl655139PDDD7t81OtOZ3sP+/n56cEHH9QHH3yg3/72t5o+fbqOHDlS7XO/+OKLio6OVmxsrKZPn67CwsLz+sspOztbc+bM0YABAxQREaGdO3dq165duuOOOyTV3/diQECAxo8frwceeECVlZW69tprVVxcrLVr18rf31/Dhg1T69at9Z///EfLly9XVFSUFixYoE2bNlkfg9uRn5+f7rnnHj388MMKCgpS8+bNNXXqVP30008aMWKEVdewYUMNHz5cEyZMUOvWrc84leKy577pS6iJHTt2mISEBNOkSRPj7e1t2rRpY2bNmmWMMearr74yXbt2Nd7e3iY6Otr897//NS1atDDTp0+3jt+5c6e59tprjZeXl2nTpo1JS0s77aTqzZs3W8cUFhYaSeazzz6ztm3cuNH06dPH+Pv7Gz8/P9OpUyfzzDPPGGN+nmDdo0cP07hxY+Pr62s6depkPelxtv7dIT093cTGxhpvb2/TqVMns3LlSut6VPdazJkzx/zmN78xvr6+ZtCgQebpp5824eHhLq+TlpZmunfvbnx9fU1gYKC5+uqrzZw5c6z9v/xvUJ/8chLxjz/+aJKSkozT6TS+vr4mISHB7Nq1q8oxDz/8sJFknnjiiSr7cnNzzR133GFCQkKMt7e3adWqlRk5cqQpKiq61EOptrO9h8vKysw999xjgoKCTGhoqJkyZcppJ1X/8mfSmP/7uVu8eLGJi4szXl5eJjY21nzyySdWzalJ1YWFhS7H/nJSdV5enhk0aJBp2rSp8fLyMi1atDBPPPGEqaiosOrP9V6sK345qdoYYyorK83MmTNNTEyM8fT0NE2aNDEJCQlm1apVxhhjTpw4YYYPH26cTqe54oorzD333GP++te/WtfGGNenry5nvxxnSUmJGTt2rPUzdc0115iNGzdWOWbv3r1GkjXZ2o74bffARTZy5Eh98803Wr16tbtbQT2xb98+RUVFafPmzZflr+FA7frLX/6ihg0bauHChdU+5osvvlDPnj118OBBhYWFXcLu6q66+6ExUE88//zz+vrrr7Vnzx7NmjVLr732moYNG+butgDYzMmTJ7Vjxw6tW7dO7du3r9YxpaWl2rNnjx5//HENGTLEtmFIIhABF2zjxo3q06ePOnbsqJdffln//Oc/ddddd7m7LQA2s23bNnXt2lXt27fX3XffXa1jXn/9dcXExKioqEhTp069xB3WbXxkBgAAbI87RAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAumZ49eyo5OblatStXrpTD4TivX3VxOi1bttSMGTMu6BwA7IdABAAAbI9ABAAAbI9ABKBWLFy4UF27dlVAQIDCw8OVmJio/Pz8KnVffPGFrrzySvn4+CguLk5bt2512b927Vpdf/318vX1VWRkpMaNG6fjx4/XqCeHw6F///vf+sMf/qBGjRopOjpay5Yts/ZXVFRoxIgRioqKkq+vr2JiYjRz5kyXcwwfPlyDBg3S5MmTFRYWpiuuuEKTJk3SyZMnrd8y3qxZM7366qsux3333Xe67bbb1LhxYwUHB2vgwIHat2+ftX/lypW6+uqr5efnpyuuuELXXHON9u/fX6NxAjg3AhGAWlFWVqa///3v+vrrr/Xuu+8qOztbw4cPr1L38MMP6/nnn9emTZsUGhqqAQMGqLy8XJK0detWJSQkaPDgwdqyZYveeOMNrVmzRvfdd1+N+5o0aZKGDBmiLVu26Oabb9bQoUP1448/SpIqKyvVrFkzvfnmm9qxY4eeeOIJ/e1vf9Obb77pco5PP/1U33//vT7//HNNmzZNKSkp6tevnxo3bqwNGzbo7rvv1t13360DBw5Ikn766SfdcMMN8vf31+eff641a9bI399fN954o8rKynTy5EkNGjRIPXr00JYtW7Ru3TqNGjVKDoejxuMEcA4GAC6RHj16mPvvv/+0+zZu3GgkmaNHjxpjjPnss8+MJLNkyRKr5vDhw8bX19e88cYbxhhjkpKSzKhRo1zOs3r1atOgQQNTUlJijDGmRYsWZvr06dXqT5J57LHHrPVjx44Zh8NhPvroozMeM2bMGHPrrbda68OGDTMtWrQwFRUV1raYmBhz3XXXWesnT540fn5+5vXXXzfGGDNv3jwTExNjKisrrZrS0lLj6+trli9fbg4fPmwkmZUrV1ZrHAAuHHeIANSKzZs3a+DAgWrRooUCAgLUs2dPSVJOTo5LXXx8vPXvQUFBiomJUVZWliQpIyNDqamp8vf3t5aEhARVVlYqOzu7Rn116tTJ+nc/Pz8FBAS4fJT38ssvq2vXrmrSpIn8/f01d+7cKj23b99eDRr83x+nYWFh6tixo7XesGFDBQcHW+fNyMjQnj17FBAQYI0jKChIJ06c0N69exUUFKThw4crISFB/fv318yZM5Wbm1uj8QGoHg93NwDg8nf8+HH17dtXffv21cKFC9WkSRPl5OQoISFBZWVl5zz+1EdFlZWVGj16tMaNG1elpnnz5jXqzdPTs8prVVZWSpLefPNNPfDAA3rhhRcUHx+vgIAAPffcc9qwYcM5z3G281ZWVqpLly5atGhRlX6aNGkiSZo/f77GjRuntLQ0vfHGG3rssceUnp6ubt261WicAM6OQATgkvvmm29UUFCgZ599VpGRkZKkL7/88rS169evt8JNYWGhdu3apbZt20qSrrrqKm3fvl2tW7eulb5Xr16t7t27a8yYMda2vXv3XvB5r7rqKr3xxhsKDQ1VYGDgGes6d+6szp07a8KECYqPj9fixYsJRMAlwkdmAC655s2by8vLS7NmzdK3336rZcuW6e9///tpa5966il98skn2rZtm4YPH66QkBANGjRIkvToo49q3bp1uvfee5WZmandu3dr2bJlGjt27CXpu3Xr1vryyy+1fPly7dq1S48//rg2bdp0wecdOnSoQkJCNHDgQK1evVrZ2dlatWqV7r//fh08eFDZ2dmaMGGC1q1bp/3792vFihXatWuXYmNjL8KoAJwOgQjAJdekSROlpqbqv//9r9q1a6dnn31Wzz///Glrn332Wd1///3q0qWLcnNztWzZMnl5eUn6eb7PqlWrtHv3bl133XXq3LmzHn/8cTVt2vSS9H333Xdr8ODBuu222xQXF6fDhw+73C2qqUaNGunzzz9X8+bNNXjwYMXGxurOO+9USUmJAgMD1ahRI33zzTe69dZb1aZNG40aNUr33XefRo8efRFGBeB0HMYY4+4mAAAA3Ik7RAAAwPYIRAAuS4sWLXJ5PP+XS/v27d3dHoA6ho/MAFyWjh49qkOHDp12n6enp1q0aFHLHQGoywhEAADA9vjIDAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2N7/B4Y7qDECf5WoAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(df.label_names.value_counts())\n",
    "sns.histplot(df.label_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We might consider upsampling for the minority classes if the performance is hindered!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: title={'center': 'word_count'}, xlabel='label_names'>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAHNCAYAAADWsJtQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAABzP0lEQVR4nO3deVxUZfs/8M+wDTsCyqIgoCCK4pKmpSWaW5oEmWWPlvpolpmZmpJLi/aYlppLi5lZaqWWmbmlJqUiJm6YifsSLiiI4sIq6/X7w9+cLwMIDM44HPm8Xy9eNefcc+aa23NmrrnPde6jEREBERERkUpYmDsAIiIiIkMweSEiIiJVYfJCREREqsLkhYiIiFSFyQsRERGpCpMXIiIiUhUmL0RERKQqTF6IiIhIVZi8EBERkaoweSGzWLp0KTQajd5fnTp10KlTJ2zcuNHc4Sn8/f0xePBgg5+XnZ2NKVOmYMeOHUaP6UHVqVMndOrUqcJ2Go0GI0eONGks586dg0ajwdKlSw1+7o4dO6DRaLB69WqjxaPbJvcnojuYvJBZLVmyBHFxcdi9ezcWLVoES0tLhIeHY8OGDeYO7Z5kZ2dj6tSp/LIhIjIBK3MHQDVbs2bN0KZNG+Xxk08+CVdXV6xcuRLh4eFmjExdsrOzYW9vb+4wiIjuC468ULVia2sLGxsbWFtb6y2/fv06RowYgXr16sHGxgYNGjTA5MmTkZubCwC4ffs2WrVqhcDAQNy6dUt5XkpKCry8vNCpUycUFhYCAAYPHgxHR0ccPXoUXbp0gYODA+rUqYORI0ciOzu7whgvXLiAF198ER4eHtBqtWjSpAk++eQTFBUVAbhzyqFOnToAgKlTpyqnxSo6/XT06FF0794d9vb2qFOnDl5//XX89ttvpU4XdOrUCc2aNcPOnTvRvn172NvbY8iQIZWKDbj7KYiyTpUY0lciggULFqBly5aws7ODq6sr+vbti3///bdUu5kzZ8LPzw+2trZ46KGHsHnz5gr7vaSvvvoKjRo1glarRUhICH788Ue992JlZYUZM2aUet7OnTuh0Wjw888/G/R6Z86cwX//+18EBQXB3t4e9erVQ3h4OBISEspsf/v2bYwdOxZeXl6ws7NDWFgY/v7771LtDhw4gKeffhpubm6wtbVFq1atsGrVKoNiK4vu33nlypWYPHky6tatC2dnZ3Tt2hUnT57UaxsdHY2IiAj4+PjA1tYWgYGBePXVV3Ht2jW9dlOmTIFGo8Hhw4fx3HPPwcXFBW5ubhg7diwKCgpw8uRJPPnkk3BycoK/vz9mzpxZKq709HSMGzcOAQEBsLGxQb169TB69GhkZWXptfv555/Rrl07uLi4wN7eHg0aNFD2cyIIkRksWbJEAMiePXskPz9f8vLy5OLFizJq1CixsLCQLVu2KG1zcnKkefPm4uDgILNnz5atW7fKu+++K1ZWVtKrVy+l3alTp8TJyUn69OkjIiKFhYXyxBNPiIeHh1y+fFlpN2jQILGxsZH69evLhx9+KFu3bpUpU6aIlZWV9O7dWy9OPz8/GTRokPI4NTVV6tWrJ3Xq1JGFCxfKli1bZOTIkQJAXnvtNRERuX37tmzZskUAyNChQyUuLk7i4uLkzJkzd+2Py5cvi7u7u9SvX1+WLl0qmzZtkpdeekn8/f0FgGzfvl1pGxYWJm5ubuLr6yufffaZbN++XWJiYioVm4jI9u3bS21TRCQxMVEAyJIlS6rUV8OGDRNra2t56623ZMuWLbJixQpp3LixeHp6SkpKitLu/fffV/pm8+bNsmjRIqlXr554eXlJWFjYXftIB4D4+vpKSEiIrFy5UtavXy9PPvmkAJCff/5ZaffMM89I/fr1paCgQO/5zz33nNStW1fy8/Pv+hpl9UVMTIy89dZbsnr1aomJiZFff/1VIiMjxc7OTk6cOFGqf319fSUiIkI2bNggP/zwgwQGBoqzs7OcPXtWabtt2zaxsbGRxx9/XH766SfZsmWLDB48uNRr3+3frDy65/j7+8uAAQPkt99+k5UrV0r9+vUlKChIr1++/PJLmTFjhqxfv15iYmJk2bJl0qJFCwkODpa8vDylne7fLjg4WP73v/9JdHS0REVFCQAZOXKkNG7cWD799FOJjo6W//73vwJAfvnlF+X5WVlZ0rJlS6ldu7bMmTNH/vjjD5k/f764uLjIE088IUVFRSIisnv3btFoNPLCCy/Ipk2bZNu2bbJkyRJ56aWXKv3+6cHG5IXMQpe8lPzTarWyYMECvbYLFy4UALJq1Sq95R9//LEAkK1btyrLfvrpJwEg8+bNk/fee08sLCz01ovc+UIGIPPnz9db/uGHHwoA2bVrl7KsZPIyYcIEASB79+7Ve+5rr70mGo1GTp48KSIiV69eFQDy/vvvV6o/xo8fLxqNRo4ePaq3vEePHmUmLwDkzz//1Gtb2dgMTV4q01dxcXECQD755BO9dhcvXhQ7OzuJiooSEZEbN26Ira2tPPPMM3rt/vrrLwFQ6eTFzs5OLyEqKCiQxo0bS2BgoLJM9z5//fVXZdmlS5fEyspKpk6dWu5rlNUXJRUUFEheXp4EBQXJmDFjSr3uQw89pHwZi4icO3dOrK2t5eWXX1aWNW7cWFq1alUqkerdu7d4e3tLYWGh3jarkrwUT/BFRFatWiUAJC4ursznFRUVSX5+vpw/f14AyLp165R1uuSl5L9zy5YtBYCsWbNGWZafny916tRRfkyIiMyYMUMsLCxk//79es9fvXq1AJBNmzaJiMjs2bMFgNy8ebPS75dqFp42IrP67rvvsH//fuzfvx+bN2/GoEGD8Prrr+Pzzz9X2mzbtg0ODg7o27ev3nN1p2H+/PNPZdnzzz+P1157DePHj8e0adMwadIkdOvWrczXHjBggN7j/v37AwC2b99+13i3bduGkJAQtG3btlQsIoJt27ZV/KbLEBMTg2bNmiEkJERv+X/+858y27u6uuKJJ564L7EBFffVxo0bodFo8OKLL6KgoED58/LyQosWLZRTVHFxcbh9+3ap7bVv3x5+fn6VjqdLly7w9PRUHltaWqJfv344c+YMkpKSANw5vdaiRQt88cUXSruFCxdCo9HglVdeqfyb//8KCgowffp0hISEwMbGBlZWVrCxscHp06dx/PjxUu379+8PjUajPPbz80P79u2VPjtz5gxOnDih9EXxfuvVqxeSk5NLnd6piqefflrvcfPmzQEA58+fV5alpqZi+PDh8PX1hZWVFaytrZV/j7LeW+/evfUeN2nSBBqNBj179lSWWVlZITAwUO91Nm7ciGbNmqFly5Z677dHjx56pzIffvhhAHeO51WrVuHSpUv30AP0IGLyQmbVpEkTtGnTBm3atMGTTz6Jr776Ct27d0dUVBRu3rwJAEhLS4OXl5feFwEAeHh4wMrKCmlpaXrLhwwZgvz8fFhZWWHUqFFlvq6VlRXc3d31lnl5eSmvdzdpaWnw9vYutbxu3boVPrc8aWlpel/GOmUtA1BmDKaKrTJ9deXKFYgIPD09YW1trfe3Z88epXZC1173/LK2WRnlPb/4+xw1ahT+/PNPnDx5Evn5+fj666/Rt29fg15LZ+zYsXj33XcRGRmJDRs2YO/evdi/fz9atGiBnJycSsdYvM8AYNy4caX6bMSIEQBQquakKkr+22m1WgBQYi4qKkL37t2xZs0aREVF4c8//8S+ffuwZ88evXbFubm56T22sbGBvb09bG1tSy2/ffu28vjKlSs4fPhwqffr5OQEEVHeb8eOHbF27VoUFBRg4MCB8PHxQbNmzbBy5cp77A16UPBqI6p2mjdvjt9//x2nTp1C27Zt4e7ujr1790JE9BKY1NRUFBQUoHbt2sqyrKwsvPTSS2jUqBGuXLmCl19+GevWrSv1GgUFBUhLS9P7YE9JSQFQ+sO+OHd3dyQnJ5dafvnyZQDQi8UQ7u7uypdZcbqYSiqZyBkSm+4LRlfsrHO3L8rK9FXt2rWh0WgQGxurfDkWp1uma1/W+0pJSYG/v3+ZMZTV9m7LisfZv39/vP322/jiiy/wyCOPICUlBa+//nqlXqOkH374AQMHDsT06dP1ll+7dg21atWqdIzF+wwAJk6ciD59+pT5msHBwVWK1RBHjhzBP//8g6VLl2LQoEHK8jNnzhj9tWrXrg07Ozt8++23d12vExERgYiICOTm5mLPnj2YMWMG+vfvD39/fzz66KNGj43UhSMvVO0cOnQIAJQrdrp06YLMzEysXbtWr913332nrNcZPnw4Lly4gDVr1uCbb77B+vXrMXfu3DJfZ/ny5XqPV6xYAQDlTpTWpUsXHDt2DAcPHiwVi0ajQefOnQGU/nVbkbCwMBw5cgTHjh3TW178CpqKVDY2XYJw+PBhvXbr16+/67Yr6qvevXtDRHDp0iVlJK34X2hoKADgkUcega2tbant7d69W+/0QkX+/PNPvWSvsLAQP/30Exo2bAgfHx9lua2tLV555RUsW7YMc+bMQcuWLdGhQ4dKv05xGo2mVGL222+/3fWUxsqVKyEiyuPz589j9+7dSp8FBwcjKCgI//zzT5l91qZNGzg5OVUpVkPoEuGS7+2rr74y+mv17t0bZ8+ehbu7e5nvt6zkVavVIiwsDB9//DEAlHnFFtU8HHkhszpy5AgKCgoA3BnuX7NmDaKjo/HMM88gICAAADBw4EB88cUXGDRoEM6dO4fQ0FDs2rUL06dPR69evdC1a1cAwOLFi/HDDz9gyZIlaNq0KZo2bYqRI0fi7bffRocOHfRqQWxsbPDJJ58gMzMTDz/8MHbv3o1p06ahZ8+eeOyxx+4a75gxY/Ddd9/hqaeewgcffAA/Pz/89ttvWLBgAV577TU0atQIAODk5AQ/Pz+sW7cOXbp0gZubG2rXrn3XkYXRo0fj22+/Rc+ePfHBBx/A09MTK1aswIkTJwAAFhYV/86obGxeXl7o2rUrZsyYAVdXV/j5+eHPP//EmjVrytxuZfqqQ4cOeOWVV/Df//4XBw4cQMeOHeHg4IDk5GTs2rULoaGheO211+Dq6opx48Zh2rRpePnll/Hcc8/h4sWLmDJlikGncmrXro0nnngC7777LhwcHLBgwQKcOHGizGRvxIgRmDlzJuLj47F48eJKv0ZJvXv3xtKlS9G4cWM0b94c8fHxmDVrll6yVFxqaiqeeeYZDBs2DLdu3cL7778PW1tbTJw4UWnz1VdfoWfPnujRowcGDx6MevXq4fr16zh+/DgOHjxo8OXcVdG4cWM0bNgQEyZMgIjAzc0NGzZsQHR0tNFfa/To0fjll1/QsWNHjBkzBs2bN0dRUREuXLiArVu34q233kK7du3w3nvvISkpCV26dIGPjw9u3ryJ+fPnw9raGmFhYUaPi1TIfLXCVJOVdbWRi4uLtGzZUubMmSO3b9/Wa5+WlibDhw8Xb29vsbKyEj8/P5k4caLS7vDhw2JnZ6d3ZZDIncuWW7duLf7+/nLjxg0RuXMFjYODgxw+fFg6deokdnZ24ubmJq+99ppkZmbqPb/k1UYiIufPn5f+/fuLu7u7WFtbS3BwsMyaNUu5MkTnjz/+kFatWolWqxUApbZT0pEjR6Rr165ia2srbm5uMnToUFm2bJkAkH/++UdpFxYWJk2bNi1zG5WNLTk5Wfr27Stubm7i4uIiL774ohw4cKDMq40q21ciIt9++620a9dOHBwcxM7OTho2bCgDBw6UAwcOKG2KiopkxowZ4uvrKzY2NtK8eXPZsGGDhIWFVfpqo9dff10WLFggDRs2FGtra2ncuLEsX778rs/p1KmTuLm5SXZ2doXbFyn7aqMbN27I0KFDxcPDQ+zt7eWxxx6T2NjYUnHrrvL5/vvvZdSoUVKnTh3RarXy+OOP6/WDzj///CPPP/+8eHh4iLW1tXh5eckTTzwhCxcuLLXNqlxtVPzy8bu9t2PHjkm3bt3EyclJXF1d5bnnnpMLFy6UumJOd7XR1atX9bap209KKmtfzczMlHfeeUeCg4PFxsZGXFxcJDQ0VMaMGaNcQbZx40bp2bOn1KtXT2xsbMTDw0N69eolsbGxlX7/9GDTiBQb1ySqAQYPHozVq1cjMzPT3KFU6JVXXsHKlSuRlpYGGxub+/76auqru0lNTYWfnx/eeOONMidNIyL14Wkjomrigw8+QN26ddGgQQNkZmZi48aNWLx4Md555x2zJC5ql5SUhH///RezZs2ChYUF3nzzTXOHRERGwuSFqJqwtrbGrFmzkJSUhIKCAgQFBWHOnDn80q2ixYsX44MPPoC/vz+WL1+OevXqmTskoxAR5VYXd2NpaVnmFWlEDwqeNiIiUpEdO3YoV47dzZIlSyq8lxaRmjF5ISJSkYyMjApn3g0ICCh3viIitWPyQkRERKrCSeqIyOTOnTsHjUaDpUuXmjsUo5o+fXqpyROJyPSYvBARVRGTFyLzYPJCREaRnZ1t7hCIqIZg8kL0ADp69Cg0Go3e9PLx8fHQaDRo2rSpXtunn34arVu3BnDnDsMzZ85E48aNodVq4eHhgYEDByIpKUnvOZ06dUKzZs2wc+dOtG/fHvb29hgyZAiAOzeCfP755+Hk5AQXFxf069fvrjeYrMjNmzfx1ltvoUGDBko8vXr1Um6bAADXr1/HiBEjUK9ePdjY2KBBgwaYPHmy3o0nyzttpdFoMGXKFOXxlClToNFocPToUfznP/+Bi4sLPD09MWTIENy6dUvveVlZWVi2bBk0Gg00Gk2598UiIuPhPC9ED6CmTZvC29sbf/zxB5577jkAwB9//AE7OzscO3YMly9fRt26dVFQUICYmBgMHz4cAPDaa69h0aJFGDlyJHr37o1z587h3XffxY4dO3Dw4EG9u/4mJyfjxRdfRFRUFKZPnw4LCwvk5OSga9euuHz5MmbMmIFGjRrht99+Q79+/Qx+DxkZGXjsscdw7tw5vP3222jXrh0yMzOxc+dOJCcno3Hjxrh9+zY6d+6Ms2fPYurUqWjevDliY2MxY8YMHDp0CL/99luV+/DZZ59Fv379MHToUCQkJCj3JNLdETkuLg5PPPEEOnfujHfffRcA4OzsXOXXIyIDmO3GBERkUi+++KI0aNBAedy1a1cZNmyYuLq6yrJly0RE5K+//hIAsnXrVjl+/LgAkBEjRuhtZ+/evQJAJk2apCwLCwsTAPLnn3/qtf3yyy8FgKxbt05v+bBhw0rdT6ciH3zwgQCQ6Ojou7ZZuHChAJBVq1bpLf/444+V9yVS9v18dHCX+/fMnDlTr92IESPE1tZWioqKlGUODg4V3rOKiIyPp42IHlBdunTBv//+i8TERNy+fRu7du3Ck08+ic6dOyt3DP7jjz+g1Wrx2GOPYfv27QBQanKztm3bokmTJvjzzz/1lru6uuKJJ57QW7Z9+3Y4OTnh6aef1lvev39/g+PfvHkzGjVqpNw1vCzbtm2Dg4MD+vbtq7dc9x5KxmyIku+hefPmuH37NlJTU6u8TSIyDiYvRA8o3Zf+H3/8gV27diE/Px9PPPEEunbtqnyp//HHH+jQoQPs7OyQlpYGAPD29i61rbp16yrrdcpql5aWBk9Pz1LLvby8DI7/6tWr8PHxKbdNWloavLy8Sk2F7+HhASsrq1IxG6LkJG9arRYAkJOTU+VtEpFxMHkhekD5+PigUaNG+OOPPxAdHY02bdqgVq1a6NKlC5KTk7F3717s2bNHSXJ0X9bJycmltnX58mW9ehcAZd47x93dHVeuXCm1vCoFu3Xq1ClVKHy315MSc22mpqaioKBAidnW1hYA9Ip4AdxTckNE5sPkhegB1rVrV2zbtg3R0dHo1q0bAKBRo0aoX78+3nvvPeTn5yvJi+4U0A8//KC3jf379+P48ePo0qVLha/XuXNnZGRkYP369XrLV6xYYXDsPXv2xKlTp7Bt27a7tunSpQsyMzNLzbXy3XffKesBwNPTE7a2tjh8+LBeu3Xr1hkcV3FarZYjMURmwKuNiB5gXbp0wYIFC3Dt2jXMmzdPb/mSJUvg6uqqXCYdHByMV155BZ999hksLCzQs2dP5WojX19fjBkzpsLXGzhwIObOnYuBAwfiww8/RFBQEDZt2oTff//d4NhHjx6Nn376CREREZgwYQLatm2LnJwcxMTEoHfv3ujcuTMGDhyIL774AoMGDcK5c+cQGhqKXbt2Yfr06ejVq5eSmGk0Grz44ov49ttv0bBhQ7Ro0QL79u2rUlJVXGhoKHbs2IENGzbA29sbTk5OCA4OvqdtElElmLtimIhM58aNG2JhYSEODg6Sl5enLF++fLkAkD59+ui1LywslI8//lgaNWok1tbWUrt2bXnxxRfl4sWLeu3CwsKkadOmZb5mUlKSPPvss+Lo6ChOTk7y7LPPyu7duw2+2kgX/5tvvin169cXa2tr8fDwkKeeekpOnDihtElLS5Phw4eLt7e3WFlZiZ+fn0ycOFFu376tt61bt27Jyy+/LJ6enuLg4CDh4eFy7ty5u15tdPXqVb3nL1myRABIYmKisuzQoUPSoUMHsbe3FwASFhZm0PsjoqrhjRmJiIhIVVjzQkRERKrCmhciuq9EBIWFheW2sbS0LPNqJiIigCMvRHSfxcTEwNrauty/ZcuWmTtMIqrGWPNCRPdVRkYGTp48WW6bgICAUpPEERHpMHkhIiIiVeFpIyIiIlKValewW1RUhMuXL8PJyYkFe0RERDWEiCAjIwN169aFhUX5YyvVLnm5fPkyfH19zR0GERERmcHFixcrvClrtUtenJycANwJ3tnZ2czRlC8/Px9bt25F9+7dYW1tbe5wHgjsU9Ngvxof+9T42KemoZZ+TU9Ph6+vr5IHlKfaJS+6U0XOzs6qSF7s7e3h7OxcrXcINWGfmgb71fjYp8bHPjUNtfVrZUpGWLBLREREqsLkhYiIiFSFyQsRERGpCpMXIiIiUhUmL0RERKQqTF6IiIhIVZi8EBERkaoweSEiIiJVYfJCRERmV1hYiJiYGOzcuRMxMTEoLCw0d0hUjTF5ISIis1qzZg0CAwPRrVs3zJkzB926dUNgYCDWrFlj7tCommLyQkREZrNmzRr07dsXoaGhiI2NxcqVKxEbG4vQ0FD07duXCQyVickLERGZRWFhId566y307t0ba9euRbt27WBnZ4d27dph7dq16N27N8aNG8dTSFQKkxciIjKL2NhYnDt3DpMmTYKFhf7XkYWFBSZOnIjExETExsaaKUKqrpi8EBGRWSQnJwMAmjVrVuZ63XJdOyIdJi9ERGQW3t7eAIAjR46UuV63XNeOSIfJCxERmcXjjz8Of39/TJ8+HUVFRXrrioqKMGPGDAQEBODxxx83U4RUXTF5ISIis7C0tMQnn3yCjRs3IjIyEnv27EFOTg727NmDyMhIbNy4EbNnz4alpaW5Q6VqxsrcARARUc3Vp08frF69Gm+99RY6duyoLA8ICMDq1avRp08fM0ZH1RWTF6IHXE5ODsaOHYs9e/Zgy5YtmDNnDuzs7MwdFpGiT58+6N27Nz777DNs27YNTzzxBN544w3Y2NiYOzSqpnjaiOgBFhkZCXt7eyxcuBCHDh3CwoULYW9vj8jISHOHRqRYs2YNgoODMW7cOGzatAnjxo1DcHAwJ6iju2LyQvSAioyMxLp162BjY4OoqCh8+eWXiIqKgo2NDdatW8cEhqoFzrBLVcHkhegBlJOToyQuGRkZmDZtGry9vTFt2jRkZGQoCUxOTo65Q6UajDPsUlUxeSF6AI0fPx4AMHbs2FJ1AzY2Nhg9erReOyJz4Ay7VFVMXogeQKdPnwYAvPzyy2WuHzp0qF47MkxhYSFiYmKwc+dOxMTEcGSgijjDLlUVkxeiB1BQUBAAYPHixWWu/+abb/TaUeWtWbMGgYGB6NatG+bMmYNu3bohMDCQtRlVwBl2qaqYvBA9gGbNmgUAmDNnDvLy8vTW5eXlYd68eXrtqHJYXGpcnGGXqorJC9EDyM7ODhEREcjLy4OTkxMmTZqES5cuYdKkSXByckJeXh4iIiI434sBWFxqfJxhl6qKk9QRPaDWrl2rXC49e/ZsvXURERFYu3ateQJTKV1x6cqVK2FhYaGXpOiKS9u3b4/Y2Fh06tTJfIGqDGfYpapg8lJFxQv2HBwc0LlzZ/46oGpn7dq1uHXrFnr27InTp08jKCgImzdvhouLi7lDU53ixaVlHf8sLq26Pn36ICIiAtu3b8fmzZvRs2dPfqZSuXjaqApYsEdqERUVhdq1ayMuLg7Xrl1DXFwcateujaioKHOHpjq6otHPP/+8zOP/888/12tHhrG0tERYWBg6duyIsLAwJi5ULiYvBmLBHqlFVFQUZs2aBXd3dyxcuBBLlizBwoUL4e7ujlmzZjGBMdDjjz+OOnXqYOLEiWjWrJne8d+sWTNMmjQJHh4eLC4lug+YvBiABXukFnl5eZg7dy48PT2RlJSEIUOGwNXVFUOGDEFSUhI8PT0xd+7cUlciUfk0Go3y/yKi918iun+YvBiAs0GSWixYsAAFBQWYNm0arKz0S9usrKzwwQcfoKCgAAsWLDBThOoTGxuL1NRUzJgxA0eOHEHHjh3xn//8Bx07dsTRo0cxffp0pKam8vgnug+YvBiAs0GSWpw9exYA0Lt37zLX65br2lHFdMf1yJEjcebMGURHR2Ps2LGIjo7G6dOnMXLkSL12RGQ6TF4MwNkgSS0aNmwIANi4cSPy8vLw6aefYtGiRfj000+Rl5eHjRs36rWjihU//ssqLuXxf2/K2k+J7kqqmVu3bgkAuXXrlrlDKaWgoED8/f0lPDxcCgsLJS8vT9auXSt5eXlSWFgo4eHhEhAQIAUFBeYOVbWK9ylVXW5urlhZWYm9vb1YWFgIAOXPwsJC7O3txcrKSnJzc80dqmrw+Ded8ePHi5WVld5+amVlJePHjzd3aA8EtXyuGvL9z5EXA3A2SFILGxsbtGrVCtnZ2RAR9O/fH3PmzEH//v0hIsjOzkarVq1K3XGa7o7Hv2nwqjiqEpOnUgaqziMvOr/88ov4+/vr/UoICAiQX375xdyhqZ5afiFUd8VHXiwtLUv9ouXIS9Xx+Dce3X7q6ekp+fn5esd/fn6+eHp6cj81ArV8rnLkxcT69OlTZsEep7Gm6kJ3tdH8+fORnZ2N2bNno1evXpg9ezaysrIwd+5cXm1URTz+jYdXxVFV8fYAVaQr2MvKyuJskEbCWy4YT/GrjSwtLdGiRQtcvnwZLVq0gKWlJa82ukc8/o2j+H5a1vHP/fTePaifqwaPvFy6dAkvvvgi3N3dYW9vj5YtWyI+Pl5ZLyKYMmUK6tatCzs7O3Tq1AlHjx41atD04OEtF4xLdxXRBx98UGa//u9//9NrR2QO3E9N64H+XDXkfNT169fFz89PBg8eLHv37pXExET5448/5MyZM0qbjz76SJycnOSXX36RhIQE6devn3h7e0t6errRz3mZm1rOI1Z3v/zyi2g0GgkPD5fY2FhZuXKlxMbGSnh4uGg0GtYSVEFubq5yldFTTz2l169PPfWUctURawmqjsf/veN+ajpq/Fw1Wc3Lxx9/DF9fXyxZsgRt27aFv78/unTpomTFIoJ58+Zh8uTJ6NOnD5o1a4Zly5YhOzsbK1asMHLaRQ8C3nLBNCwtLeHo6AgAOHDgABISEpCTk4OEhAQcOHAAAODo6PhADB+TenE/NY2a8LlqUM3L+vXr0aNHDzz33HOIiYlBvXr1MGLECAwbNgwAkJiYiJSUFHTv3l15jlarRVhYGHbv3o1XX3211DZzc3ORm5urPE5PTwcA5OfnIz8/v0pv6n7RxVfd46zOYmJicO7cOXz//fcoLCws1afjx49Hx44dsX37doSFhZkzVFWJiYlBeno6XnjhBaxevRojRoxQ1llZWeGFF17Ajz/+yH69Bzz+7x33U9NQ6+eqIceSQcnLv//+iy+//BJjx47FpEmTsG/fPowaNQparRYDBw5ESkoKAMDT01PveZ6enjh//nyZ25wxYwamTp1aavnWrVthb29vSHj31e3bt7Fs2TIkJyfjq6++wqBBg2Bra2vusFRn586dAICkpCQkJydjy5YtSElJwfr16/Hkk08qvww2b96MrKwsc4aqKrp+jYiIwJNPPonx48cjIyMDTk5OmDVrFrRaLX788Uf2qxFER0ebOwTVKr6f9unTRzn+vby8lOOf+6nhin+upqWlKct1+2pOTg6A6ve5mp2dXem2GpHK3xLVxsYGbdq0we7du5Vlo0aNwv79+xEXF4fdu3ejQ4cOuHz5st4U2cOGDcPFixexZcuWUtssa+TF19cX165dg7Ozc6XfyP307LPPYsOGDaWWh4eH45dffjFDROoVExODbt26Kb+8CgoKlHVWVlbo27cvfvzxR0RHR1erXwjVna5f69Wrh0uXLpVar1vOfq26/Px8REdHo1u3brC2tjZ3OKqk209jY2PRrl27Un26Z88edOzYkfupgdTar+np6ahduzZu3bpV8fe/IcU09evXl6FDh+otW7BggdStW1dERM6ePSsA5ODBg3ptnn76aRk4cGClXqO6F+xGREQIALGxsZGoqCj58ssvJSoqSmxsbASAREREmDtEVSkoKBAXFxcBIB4eHrJw4UJZsmSJLFy4UDw8PASAuLi4cMp1AxUUFOhNt96jRw/56KOPpEePHnqT1bFfq44Fu/eOt1wwDbX2q8kKdjt06ICTJ0/qLTt16hT8/PwAAAEBAfDy8tIbRs3Ly0NMTAzat29vyEtVSzk5OVi3bh1sbGyQkZGBadOmwdvbG9OmTUNGRgZsbGywbt06ZUiOKlZYWIiMjAwAQNu2bRESEgJbW1uEhISgbdu2AICMjAxVF5aZQ2ZmpjKK1bNnT0yePBl+fn6YPHkyevbsCQAoKChAZmamOcOkGo63XDCNGtGvhmRF+/btEysrK/nwww/l9OnTsnz5crG3t5cffvhBafPRRx+Ji4uLrFmzRhISEuQ///nPA3Op9Ouvvy4AZMKECSJS+pdXVFSUAJDXX3/dnGGqyty5cwWAvPbaa2VOuf7qq68KAJk7d665Q1WVDh06CABp1apVmf3asmVLASAdOnQwd6iqxZEX4+EtF0xDbf1qyPe/wfc22rBhgzRr1ky0Wq00btxYFi1apLe+qKhI3n//ffHy8hKtVisdO3aUhIQEkwR/v3Xv3l0AyJkzZ6SgoECio6Nl7NixEh0dLQUFBXLy5EkBIN27dzd3qKoxcuRIASDJycmSnZ0tw4cPl5YtW8rw4cMlOztbLl26JABk5MiR5g5VVXx9fQWAxMXFSUZGhoSHh4ufn5+Eh4dLRkaG7Ny5UwCIr6+vuUNVpbKOf7o3ZR3/dO/UtK+aNHkxteqcvOhGXp555plS2ay/v79SD8ORl8rTjbz06NFDr0YD/78mo1u3bhx5qQLdyEudOnX0+lT3p1vOkRfDlfVr1t/fv9r+mlWD8ePHl3n8jx8/3tyhPRDUMkpoyPe/QVcb3Q/p6elwcXGpXLXxfZaTk6Ncvv3UU09hwoQJSEpKgo+PDz766CP89ttvAO5c7mVnZ2fOUFUjLy8Ptra2EBF4enpi6tSp0Gq1yM3Nxfvvv48rV65Ao9Hg9u3bsLGxMXe4qnHr1i3UqlULwJ2rBEePHo2AgAAkJiZi3rx5yMvLAwDcvHkTLi4uZoxUXdasWYO+ffuid+/eiIqKUo7/mTNnYuPGjVi9ejVv0GigqKgozJo1667H//jx4zFz5kxzh6lq+fn52LRpE3r16lWtr4wz5Pufd5U2gI2NjZK8REdHY926dbh+/TrWrVunFCnb29vzS7aKioqKUFRUBBFR/p+qpvg+mJeXh7///hvXr1/H33//rSQuJdtR+WrCrKX3W15eHubOnQtPT08kJSVhyJAhcHV1xZAhQ5CUlARPT0/MnTtXb58lAmBYwe79UJ1PG23fvl0AyOOPP17mULxu+fbt280dqmrwtJFp6E5x1qpVq8x9VbecpzgrT3f8x8XFiUjpofjdu3fz+DeQ7vj/+uuvRaR0n3711Vc8/o3gQTxtxJEXAyQnJwMANm3ahOTkZHh6esLa2hqenp5ITk5WThvp2lHFdLe6X7p0KbKysjB79mz06tULs2fPRlZWFpYuXarXjirn9OnTAO7cL+b8+fNwdHSERqOBo6Mjzp8/j7179+q1o4rpjutmzZqVuV63nMd/5emO6969eyMnJwejRo3ClClTMGrUKOTk5KB379567chwhYWFiImJwc6dOxETE/PAjAwadHuAmk43a3Djxo31Zi29cuUKvL29Ua9ePb12VDHdTT03btyIl19+GaNGjUJgYKBybnbjxo167ahygoKCsHXrVrRp0wY3b95UlmdmZsLPz0+phwkKCjJPgCqkO66PHDmCRx55pNT6I0eO6LWjiumO6549e+LQoUPK8kOHDmHhwoVo3ry5XjsyzJo1a/DWW2/h3LlzAIA5c+bA398fn3zyieprs1iwa4DCwkLY2toqk3/16NEDnTt3xvbt2/H7778DuDOl/e3bt9U9+c99lJeXBwcHB7i7uyMpKQkiohSWaTQa+Pj4IC0tDVlZWazPMEDx4nKg7H0VYHG5IQoLCxEYGIjQ0FCsXbsWhYWFyr5qaWmJyMhIHDlyBKdPn+bxX0l5eXnQarUAyi8sz83N5fFvIDUWl7Ng10Q4a6nx2djYYMyYMbhy5Qp8fHywePFiXL9+HYsXL4aPjw+uXLmCMWPG8IPLQMULHK2trREaGgonJyeEhobqXW3AQsjKqxGzlt5nxU9hODs7w9/fH1qtFv7+/npfXg/KqY77pUYUl5u6AMdQ1blgl7OWmg7neTCuiuZ5qV27NvfVKlLbrKXVma6wvF27dmUe/w8//DALy6tArcXlLNg1kQsXLgAAFixYgH379sHPzw+2trbw8/PD3r178emnn+q1o8qbOXMmbty4gfDwcPj5+SE8PBw3btzg/A5VpNsH169fj9TUVL19NTU1FWvWrNFrR5XXp08fJCQk6O2rhw8frnZD8GqgKxhfvnw5zp07B1dXV1haWsLV1RXnzp3DDz/8oNeOKqd4cXlZBbsPRHH5fUimDKKGkRfdHaRL/umW89es4TjyYly6fdXZ2bnMfdXJyYn7ahXpZtIu+cc7yhtON/Ki1WrL/UzlyIthdCMvM2bMKHM26OnTp6t+5IUFuwYoPmspALRr1w69evXCpk2blEtPAc5aaijOsGl8JffVF198Ea1bt0Z8fLzyaxbgvmqoyMhI5c7yZRWXRkREYO3ateYOUzVKFpaHhIQgMjISa9euxbFjx5TlLCw3TGFhIby9vXH16lX07t0bb7/9tlKw+/HHH2Pjxo3w8PDA5cuXq1WNFgt2TaRkcZOzszOsrKxKdbKqi6DuM86waRolP5CuXLmC7OxsXLlypdx2dHc5OTlK4pKRkYFp06bB29sb06ZNQ0ZGBmxsbLBu3Trk5OSYO1TVKH5xg42NDXr16oV69eqhV69eekX6vAjCcBqNRvl/3RhFNRuruDcmHgUyWHU+bdSsWbNKDXE2a9bM3KGqBmfYNI3IyEgBII6OjmXuq7rlkZGR5g5VNXSnOCZMmCAipffVqKgonuIwkO6URkX7qb+/v7lDVZXyThsFBAQ8EKeNOPJigMuXLwO4M8PuzZs38eijj6J27dp49NFHcfPmTaxfv16vHVWMM2yahq6//vrrL5w+fVq5PNra2hqnT59GTEyMXjuqmK5o9OWXX0ZeXh4+/fRTLFq0CJ9++iny8vIwdOhQvXZUsatXrwIAfvzxR/z777+wtbUFANja2uLff//Fd999p9eOKkdXiDty5EicOXMG0dHRGDt2LKKjo3H69GmMHDlSr50acYZdA9StWxfXr1/H5MmTERcXh5iYGL07db733ntKO6qcimbYbNGihV47qpyGDRsiISEBrVu3VuYmAu7cXTYoKAhWVlZKO6oc3azFAwYMQHx8vNKvmzZtwoQJE/DQQw8p7ahy6tSpg6ysLOVHis7t27fRoEEDvXZUeSVngw4LC0NWVhbCwsJgaWn5QMwGzYJdA1y/fh3u7u4AgIyMDGi1WiV5yc3NhZOTEwAgLS0Nbm5u5gxVNTjDpmlkZmYq+yNw90LIjIwMODo6miNE1SleXFqnTh3873//U4rL3333XWV0gMWllXf16lV4eHgoj/39/dG3b1+sXr1amdIeAFJTU5nAGECts0GzYNdE3Nzc4OnpCQBwcnJChw4dcPDgQXTo0EH5ovD09GTiYoDixc0uLi56M2wWvwqGRdCGKVk02rJlS7i7u6Nly5bltqO7K/4hf+vWLZw5cwbZ2dk4c+YMbt26VWY7Kl/J49rOzg6Wlpalkj8e/4apEbNBm7wCx0DVuWBXx9PTs8ziMk9PT3OHpjqcYdM0dEV6tra2Ze6ruqJzFkJWnq64XDeTdsm/5s2bs7jcQLrPUo1GU2af6pbzs7Vq1DYbNAt2TSwlJQWHDx9WLkXTaDQ4fPgwUlJSzByZ+hSfYfPatWt6RdDXrl3jDJtVpDuFsXr1apw4cUL5hWVpaYkTJ05g5cqVeu2oYrri5s2bNyM7OxvDhw9Hy5YtMXz4cGRnZ2Pz5s167ahiujuer1q1qszPVN3xX/zO6FR5ffr0wcmTJzF79mz06tULs2fPxokTJx6M2aDvQzJlEDWMvJQ3wy4ZRjfyEhQUVGafNmzYkCMvVVDy19bd/jjyUnm8rN/47jaKXfKPIy9VU9bIi7+//wMx8sKCXQNptVqliNTLywvPP/88Vq1apYy62NjYIDc315whqkrxIsjyCnZZBGmYkoWQZe2rAAshDZGXlwcHBwe4u7sjKSkJIqIUQWo0Gvj4+CAtLQ1ZWVksLq+klJQUvSte7lawm5ycDC8vLzNEqF5r1qxB37590bt3b0RFRSkz7M6cORMbN27E6tWrq90IDAt2TeTSpUvKl2laWhouXLiAJ554AhcuXEBaWhqAOx9wly5dMmeYqiUiKCoqUv6qWV6tKiVnJHZ1dYWdnR1cXV3LbUd3Z2NjgzFjxuDKlSvw8fHB4sWLcf36dSxevBg+Pj64cuUKxowZw8TlHogICgoKeOzfo8LCQrz11lvo3bs31q5di3bt2sHOzg7t2rXD2rVr0bt3b4wbN07dhdCmHAKqiup82sjV1VUAiLe3t4iUHjb28vISAOLq6mrOMFWlotNGgYGBPG1UBbp9taI/7quG401EjYenjUxDN8NuXFyciJT+rtq9ezdn2K1JMjIyAAAff/wxMjMz8eyzz+LNN9/Es88+i8zMTEybNk2vHVVMV4i7efNmpKWlISQkBE5OTggJCUFaWhp+++03vXZUObp98LvvviuzEHLx4sV67ajyZs6ciStXrujtq1euXOHNQ6tAV4j7888/l7mfLl++XK8dVY5u5txmzZqVuV63nDPs1hBOTk64ceMG/vvf/2LgwIHK8vPnz8PJyUm5oqP45GBUPt2spT179tRLUI4dOwZ3d3dlBljOWmoY3b5afD8F7gzLN2/eXK8dGaZt27bYv3+/8li3rz788MPYt2+fGSNTn1q1auHKlSt4/vnn9U4V6fZTXTJT/A7pVLGSM+yWxBl2TaA6F+xeunQJPj4+yuMXX3wRrVu3Rnx8vHJJHwAkJSWhXr165ghRdYoX7FpbW2PMmDFKwe7cuXORn58PgAW7hiq5r96tEJL7qmF0iYtGo8GAAQOU43/58uUQESYwBipZsHu3maBZsGsYzrBLeorP+AoAf/zxB06fPo0//vij3HZUORqNRq9gt/gt3ckwJT+Qbt++jezsbNy+fbvcdnR3mZmZSuKSnZ2Nb7/9FgEBAfj222+RnZ0NjUaD/fv3IzMz09yhqkbJ/S8rKwvZ2dnIysoqtx2VjzPsmkF1LtiNjIys1GyQkZGR5g5VNViwaxoshDQ+3fH/0ksviUjpIsgBAwbw+DcQ5yMyLc6wSwD+b+bMQ4cOISkpCa6urrC0tISrqyuSkpJw4MABvXZUseIFu2XNWsqC3aopXghZ1r7KQkjD6Y7rcePGlbl+7Nixeu2oYroZnjdu3Ijk5GR4enrC2toanp6eSE5Oxpo1a/TakWH69OmDM2fOIDo6GmPHjkV0dDROnz5d7eZ3qQomLwbQFY/Onj0bLi4ueOyxx+Dj44PHHnsMLi4umDNnjl47qpiuEHfx4sUoLCzEpUuXcOPGDVy6dAmFhYX45ptv9NpR5egKHKOiomBpaQkbGxtYWFjAxsYGlpaWmDRpkl47qljx478sPP4Np5sg8e2338b169dx9epV5Ofn4+rVq7h+/TomT56s145IhwW7BsjMzKzU1RkZGRlwdHS8DxGpX/GC3fKwYNcwJQsh74aFkJWnO/51NS+WlpZKEWRhYSHs7e0hIjz+DVByJui74UzQVbNmzRq89dZbekX6/v7++OSTT6rl6AsLdk3E0dFR74vW398fY8eOhb+/v7LM3t6eH1wGsLOz0ytwDgkJwcSJExESEqIsc3FxYeJioJIJiYWFBXr27AkLC4ty29HdOTo64uGHH4aIwN7eHoMHD8bZs2cxePBgJXF5+OGHefwboKyEpG3btpVqR+XT3R4gNDQUsbGxWLlyJWJjYxEaGoq+ffsqp+TUiiMvBuDIi/GxT02DIy+mU3KeFx1eJm24Y8eOoWnTphW2O3r0qN4PGiofL5UmPS+99JLy34yMDISHh8PPzw/h4eHIyMjAgAED9NpRxYr3aVkFu+zTqmnZsiUAICAgoMxCSD8/P712VHn79u0r8/hn4mK40NBQAHfmeEpNTYWfnx9sbW3h5+eH1NRUWFtb67WjyomNjcW5c+cwadKkUqOtFhYWmDhxIhITExEbG2umCO8dkxcDFL/awNLSEvXq1YOrqyvq1asHS0tLXm1QBbyCwzR0VxHNnDkTeXl5yMrKQkFBAbKyspCXl4fp06frtSPDFBYW4tq1a8jKysK1a9fUfYM7MyoqKgIATJgwATk5OUhLS0Nubi7S0tKQk5ODMWPG6LWjyil+e4DCwkLExMRg586diImJQWFh4QNxewDO82IA3TwPPj4+Zc5FUK9ePc7zYCD2qWlwnhfTadiwYZl92bBhQ3OHpjoWFhaV2k8tLCzMHaqq6G7MOGPGjFLzvPj7+8v06dNVf2NG1rwYoHh9RnlT2bM+o/KK96mNjQ1Gjx6t9Om8efOQl5cHgH1qqJI1L25ubnj++eexatUqXL9+XVnOmhfDBAYGKqOAPXr0QOfOnbF9+3b8/vvvAO5cJn3mzBlzhqgqJWte7rafsubFMIWFhahbty5SU1PRu3dvvP3220hKSoKPjw8+/vhjbNy4ER4eHrh8+TJrXmqC4v/IBQUFSEpKQnZ2NpKSklBQUFBmOypf8b7Ky8vT61Nd4lKyHVWseN8Bd27AaGtrW6o4umQ7urtbt24piUtWVhY2bNiAxo0bY8OGDcp09mfPnsWtW7fMGaaqlPxBkpGRgeTk5FJ3O+cPF8NJiRtdllymeqYdBDJcdT5tpJvK3tvb+65D8OBU9gbR9amXl1eZfapbzj41jKOjY6WG4x0dHc0dqmp06NBBAMiTTz4pIqVvD9C9e3cBIB06dDBnmKrC/dQ0yjttFBAQ8ECcNuLIiwF0U9THxsaWebXBzp079dpRxXR9tWvXLqSlpSEkJAROTk4ICQlBWloaYmJi9NpR5WRnZwMAFi1apNy2QufAgQP4/PPP9dpRxS5cuAAAeP/993H16lUEBQXhhRdeQFBQEK5evYp33nlHrx1VrPh+umPHDr11O3bs4H5aRbpC3JEjR5Z5e4CRI0fqtVMlQ7Ki999/v9yCv6KiInn//ffF29tbbG1tJSwsTI4cOWKyzOt+040STJgwQURK//KKioriKIGBdH3atm1bsbKy0tu3rKyspG3btuzTKuAvWuPTjbxYWlqW2Ze65Rx5qTzup6ahG3mJi4sTkdLfVbt371b9yIvByUvTpk0lOTlZ+UtNTVXWf/TRR+Lk5CS//PKLJCQkSL9+/cTb21vS09NNEvz9lp2dLQDExsZGcnNz9XaI3NxcsbGxEQCSnZ1t7lBVQ9enAMTDw0MWLlwoS5YskYULF4qHh4eyjn1qmPPnz5c6/TZq1KhSp+fOnz9v7lBV4+bNm3p9FxISIpMmTZKQkBC95Tdv3jR3qKpRcj+9W59yPzVMQUGB+Pv7S3h4uBQWFup9VxUWFkp4eLgEBARIQUGBuUPVY9LTRlZWVvDy8lL+dNM2iwjmzZuHyZMno0+fPmjWrBmWLVuG7OxsrFixwtCXqZbs7OwQERGBvLw8ODk5YdKkSbh06RImTZoEJycn5OXlISIiglPZG8DS0hIajQYAcOPGDZw5cwbZ2dk4c+YMbty4AQDQaDQs2DVQyf7Ky8tDdnZ2qQJd9mvlley7evXqwcHBAfXq1Su3Hd1dyc/KlJQUpKamIiUlpdx2VD5LS0t88skn2LhxIyIjI7Fnzx7k5ORgz549iIyMxMaNGzF79mxVH/8GXSo9ZcoUzJo1Cy4uLtBqtWjXrh2mT5+OBg0a4N9//0XDhg1x8OBBtGrVSnlOREQEatWqhWXLlpW5zdzcXOTm5iqP09PT4evri2vXrpnlUuns7GycPHmy3Dbjx49X6luK69ixI2bNmlXuc4ODgyt1I8Ka4tNPP8W4cePQvHlzHD58uNT60NBQJCQkYPbs2Rg1apQZIlQnT09PJfkrj6urK65cuXIfIlK/oKAgnD9/HlZWVnpXF+rolvv5+bFGq5J0fVoR9mnV/Prrr3j77bf1bswYEBCAjz76CM8884z5AruL9PR01K5du1KXSlsZsuF27drhu+++Q6NGjXDlyhVMmzYN7du3x9GjR5VM2dPTU+85np6e5e6cM2bMwNSpU0st37p1q1m+5M+ePYu33nqrSs/duXMn2rVrV26bTz75BA0bNqzS9h9E27ZtAwCMGTMG+fn5GDVqFHJzc6HVavHpp5/CysoKQ4YMwbZt2xAYGGjmaNUjPT0dADBq1CjUrVsXEyZMUNZ99NFHuHDhAhYsWID09HRs2rTJXGGqii7Ji4qKQkhICKZNm4Zr166hdu3aeOedd5CQkICPP/4YV65cYZ9Wkq5PJ02aBGdn51L76Y0bN9in90Cr1eK9997DmDFjkJ2dDXt7e7z77rvQarXVsj8NKcy+p0nqsrKy0LBhQ0RFReGRRx5Bhw4dcPnyZb3JsYYNG4aLFy9iy5YtZW5DjSMvOqeSb2H8r8cw65kQNPJ2qdRzOPKiTzfyYmdnh5ycnFLrbW1tcfv2bY68GIgjL8anGyUICQnBoUOHkJ+fj+joaHTr1g3W1tZo0aIFjh8/zlECA3DkxbTq1KlT5rxDLi4uuHr1qhkiKp8hIy/3PM9L165dZfjw4XL27FkBIAcPHtRb//TTT8vAgQMrvb3qXLBb0t/nronf2xvl73PXzB2KauXm5laqYC83N9fcoapKUlJSpQp2k5KSzB2qaqSmpir9duvWLb0iSN3nFgC9ixiofMX7tLz9lH1qOBcXl3I/V11cXMwdYin3bZ6X3NxcHD9+HN7e3ggICICXlxeio6OV9Xl5eYiJiUH79u3v5WXoAVb8V0Ht2rUxcuRIBAYGYuTIkahdu3aZ7ahqyqrToMqrU6cOXFzujLC6uLigRYsWiIuLQ4sWLfSW6y5ioIqVLG52dXWFnZ0dXF1dy21H5bt69arymXnr1i0cOnQIbdu2xaFDh/SWV8fRl8oyqOZl3LhxCA8PR/369ZGamopp06YhPT0dgwYNgkajwejRozF9+nQEBQUhKCgI06dPh729Pfr372+q+Enl2rZtCwBwd3fHzZs3MWLECGWdlZUV3NzccP36dbRt2xaJiYnmClN1QkND9R6npKRgwYIFZbYrfg+Zmi47OxsnTpy46/pt27YhLCwMmZmZOH78OI4fP66sc3R0xLZt23Dw4MG7Pr9x48Y8bVxMyf20ZJ8Wb8f9tPJ0n6tNmzaFs7Ozct89AHB2dkaTJk1w/PhxdX+uGjKko5u3xdraWurWrSt9+vSRo0ePKut1k9R5eXmJVquVjh07SkJCgsmGjcyNp43unYODgwCQjRs3SkZGhoSHh4ufn5+Eh4dLRkaGrFmzRgCIg4ODuUNVFd2Ef999950kJSWJq6urWFpaiqurqyQlJcnixYuViQDp/8THx1dq0rSq/sXHx5v7LVYrxfdT3cRpur/du3dzP62i4p+rubm5Mnv2bOnVq5fMnj1bcnNzq+3nKu8qfZ8cOp+GyC/3YO1rj6Cln7u5w1GlgIAAnDt3Dm5ubkhPT9c7tWFlZQVnZ2dcv34d/v7+6v2FYAZubm64ceMGvL29cfnyZeTn52PTpk3o1asXrK2t4e3tjZSUFLi6uvIXbTEVjbwUdzL5Jsb+nIA5z4Ui2LtWpZ7DkRd9uv20ItxPDaP7XHV3d8etW7dU87lqyPe/QaeNiIxt37598PDwwPXr1+Hu7o4PP/wQWq0Wubm5mDx5MtLS0pR2VHkJCQnw8fFBcnIyrl+/rnc36evXrytTGyQkJJgrxGrJ3t4eDz30UKXaWpxPgzY2B02ateCPlyrS7ac69vb26Nu3L1avXq132Sz3U8PoPlfT0tJQu3ZtTJs2Tflcfeedd3Dt2jWlnVrxxoxkVrpCRwBIS0vD/PnzceLECcyfP19JXEq2o4rVq1cPNjY2AO7UE9WvXx9bt25F/fr14e5+54vWxsam1OywRPdTyekRnJ2d4ejoWOpXd1nTKNDdFf+8vHbtmt7nqi5xKdlObTjyQmalKyLVzfNSsmBPt3zBggUYPXq0maKsnio6xREXF4dHHnkE+fn5pQp2ra2tERcXd9fiUp7eoPshJCRE7/HdCstDQkJ4xZEBKvpc1c2fpebPVSYvZFZnz54FAPz777+wtLTEww8/jCtXrsDT0xP79+9Hfn4+6tWrp7Sj/3PixAm0bt26Ss/Nz88v97nx8fGVPn1CVFW6q2BmzZqFZs2aoWfPnsq6zZs3Iz4+Hu+8847e1TJUsZrwucrkhcxKd6uEjRs34uWXX8bp06f1CksXLVqk147+T+PGjREfH1+ptoYWlzZu3PgeoyOqmLW1NfLz8zF+/PhS64onMtbW1vczLNWrCZ+rvNroHvBqo3uXl5cHBwcHuLu7IykpCSKiHGQajQY+Pj5IS0tDVlaWUsNBhuO+anzs03t35swZBAUFKY8tLS0RHh6ODRs2oLCwUFl++vRp3tvMAGr9XDXk+58Fu2RWNjY2GDNmDK5cuQIfHx8sXrwY169fx+LFi+Hj44MrV65gzJgx1eoAIyLjKFmIq9VqYWdnB61WW247Kl9N+FzlaSO6L8orLn3hhRdw5coVLF++XG+GXUtLSwwcOBAvvPACZy0legC1aNFC73F2djZWrlxZZruioqL7FZYqVFSwfy+fq2r4TGXyQvdFVYpLCwsL8d133+G7774rtx2LS4nUSVe18PbbbyMgIADDhw9X1i1cuBDHjx/H/PnzUc2qG6qFqhbsV+ZzVQ2fqUxe6L6obHFpVWctJSL10Wg0EBF8/PHHpdYVT2Q0Gs39DEsVanrBPpMXui8qO3MpZy0lqjn++ecfNG/eXG9ZaGhoqRl1//nnn/sZlirU9NmgWbBLRERmceHCBb3HlpaW8PX1haWlZbntiDjyQkREZtG7d2+9x4WFhdi0aVOZ7Vj3QsVx5IWIiMyqW7duOHz4sFLbotFocPjwYXTs2NHMkVF1xeSFiIjMKjo6GqGhocjNzcXatWuRm5uL0NBQ7Ny509yhUTXF5IWIiMxi48aNyv+XnHOk+OPi7YgA1rwQEZGZPPXUU8r/6+Ysady4MSIjI+/ajghg8kJERCZW3myw8fHxepOtlWwXHx/PGbapFCYvRERkUlWdDRZAhc9Tw2ywZHxMXoiIyKQ4wzYZG5MXIiIyKc6wTcbGq42IiIhIVZi8EBERkaoweSEiIiJVYfJCREREqsLkhYiIiFSFyQsRERGpCpMXIiIiUhUmL0RERKQqTF6IiIhIVZi8EBERkaoweSEiIiJVYfJCREREqsLkhYiIiFSFyQsRERGpCpMXIiIiUhUmL0RERKQqTF6IiIhIVZi8EBERkaoweSEiIiJVuafkZcaMGdBoNBg9erSyTEQwZcoU1K1bF3Z2dujUqROOHj16r3ESERERAbiH5GX//v1YtGgRmjdvrrd85syZmDNnDj7//HPs378fXl5e6NatGzIyMu45WCIiIqIqJS+ZmZkYMGAAvv76a7i6uirLRQTz5s3D5MmT0adPHzRr1gzLli1DdnY2VqxYYbSgiYiIqOayqsqTXn/9dTz11FPo2rUrpk2bpixPTExESkoKunfvrizTarUICwvD7t278eqrr5baVm5uLnJzc5XH6enpAID8/Hzk5+dXJbz7pqCgQPlvdY9VLdinpsF+NT72qfGxT01DLf1qSGwGJy8//vgjDh48iP3795dal5KSAgDw9PTUW+7p6Ynz58+Xub0ZM2Zg6tSppZZv3boV9vb2hoZ3X13MBAAr7NmzB5eOmDuaBwP71DTYr8bHPjU+9qlpqKVfs7OzK93WoOTl4sWLePPNN7F161bY2tretZ1Go9F7LCKllulMnDgRY8eOVR6np6fD19cX3bt3h7OzsyHh3Xf/XLgOJBzAI488ghb13cwdzgOBfWoa7FfjY58aH/vUNNTSr7ozL5VhUPISHx+P1NRUtG7dWllWWFiInTt34vPPP8fJkycB3BmB8fb2VtqkpqaWGo3R0Wq10Gq1pZZbW1vD2trakPDuOysrK+W/1T1WtWCfmgb71fjYp8bHPjUNtfSrIbEZVLDbpUsXJCQk4NChQ8pfmzZtMGDAABw6dAgNGjSAl5cXoqOjlefk5eUhJiYG7du3N+SliIiIiMpk0MiLk5MTmjVrprfMwcEB7u7uyvLRo0dj+vTpCAoKQlBQEKZPnw57e3v079/feFETERFRjVWlq43KExUVhZycHIwYMQI3btxAu3btsHXrVjg5ORn7pYiIiKgGuufkZceOHXqPNRoNpkyZgilTptzrpomIiIhK4b2NiIiISFWYvBAREZGqMHkhIiIiVWHyQkRERKrC5IWIiIhUhckLERERqQqTFyIiIlIVJi9ERESkKkxeiIiISFWYvBAREZGqMHkhIiIiVWHyQkRERKrC5IWIiIhUhckLERERqQqTFyIiIlIVJi9ERESkKkxeiIiISFWYvBAREZGqMHkhIiIiVWHyQkRERKrC5IWIiIhUhckLERERqQqTFyIiIlIVJi9ERESkKkxeiIiISFWYvBAREZGqMHkhIiIiVWHyQkRERKrC5IWIiIhUhckLERERqQqTFyIiIlIVJi9ERESkKkxeiIiISFWYvBAREZGqMHkhIiIiVWHyQkRERKrC5IWIiIhUhckLERERqQqTFyIiIlIVJi9ERESkKgYlL19++SWaN28OZ2dnODs749FHH8XmzZuV9SKCKVOmoG7durCzs0OnTp1w9OhRowdNRERENZdByYuPjw8++ugjHDhwAAcOHMATTzyBiIgIJUGZOXMm5syZg88//xz79++Hl5cXunXrhoyMDJMET0RERDWPQclLeHg4evXqhUaNGqFRo0b48MMP4ejoiD179kBEMG/ePEyePBl9+vRBs2bNsGzZMmRnZ2PFihWmip+IiIhqGKuqPrGwsBA///wzsrKy8OijjyIxMREpKSno3r270kar1SIsLAy7d+/Gq6++WuZ2cnNzkZubqzxOT08HAOTn5yM/P7+q4d0XBQUFyn+re6xqwT41Dfar8bFPjY99ahpq6VdDYjM4eUlISMCjjz6K27dvw9HREb/++itCQkKwe/duAICnp6dee09PT5w/f/6u25sxYwamTp1aavnWrVthb29vaHj31cVMALDCnj17cOmIuaN5MLBPTYP9anzsU+Njn5qGWvo1Ozu70m0NTl6Cg4Nx6NAh3Lx5E7/88gsGDRqEmJgYZb1Go9FrLyKllhU3ceJEjB07Vnmcnp4OX19fdO/eHc7OzoaGd1/9c+E6kHAAjzzyCFrUdzN3OA8E9qlpsF+Nj31qfOxT01BLv+rOvFSGwcmLjY0NAgMDAQBt2rTB/v37MX/+fLz99tsAgJSUFHh7eyvtU1NTS43GFKfVaqHVakstt7a2hrW1taHh3VdWVlbKf6t7rGrBPjUN9qvxsU+Nj31qGmrpV0Niu+d5XkQEubm5CAgIgJeXF6Kjo5V1eXl5iImJQfv27e/1ZYiIiIgAGDjyMmnSJPTs2RO+vr7IyMjAjz/+iB07dmDLli3QaDQYPXo0pk+fjqCgIAQFBWH69Omwt7dH//79TRU/ERER1TAGJS9XrlzBSy+9hOTkZLi4uKB58+bYsmULunXrBgCIiopCTk4ORowYgRs3bqBdu3bYunUrnJycTBI8ERER1TwGJS/ffPNNues1Gg2mTJmCKVOm3EtMRERERHfFexsRERGRqjB5ISIiIlVh8kJERESqwuSFiIiIVIXJCxEREakKkxciIiJSFSYvREREpCoG39uIiEiNEq9lISu3wGjbO3s1S/mv7t4xxuCgtUJAbQejbY/oQcTkhYgeeInXstB59g6TbPut1QlG3+b2cZ2YwBCVg8kLET3wdCMu8/q1RKCHo3G2mZOLjTvi0LvTo3Cw0xplm2dSMzH6p0NGHSEiehAxeSGiGiPQwxHN6rkYZVv5+flIqQM85OcKa2tro2yTiCqHBbtERESkKjVq5IUFe6QGxt5PAe6rRGrB479yakzywoI9UgNT7qcA91Wi6ozHf+XVmOSFBXukBqbYTwHuq0RqwOO/8mpM8qLDgj1SA2PupwD3VSI14fFfMRbsEhERkaoweSEiIiJVqXGnjci41HIFF8ArY4iIHhRMXqjK1HYFF8ArY4iIHgRMXqjK1HIFF2D+yngiIjIeJi90z3gFFxER3U8s2CUiIiJV4cgLERFVCQv2yVyYvBARkcFYsE/mxOSFiIgMxoJ9MicmL0REVGUs2CdzYMEuERERqQqTFyIiIlIVJi9ERESkKkxeiIiISFWYvBAREZGqMHkhIiIiVWHyQkRERKrC5IWIiIhUhckLERERqQqTFyIiIlIVJi9ERESkKkxeiIiISFWYvBAREZGqGJS8zJgxAw8//DCcnJzg4eGByMhInDx5Uq+NiGDKlCmoW7cu7Ozs0KlTJxw9etSoQRMREVHNZVDyEhMTg9dffx179uxBdHQ0CgoK0L17d2RlZSltZs6ciTlz5uDzzz/H/v374eXlhW7duiEjI8PowRMREVHNY2VI4y1btug9XrJkCTw8PBAfH4+OHTtCRDBv3jxMnjwZffr0AQAsW7YMnp6eWLFiBV599VXjRU5EREQ1kkHJS0m3bt0CALi5uQEAEhMTkZKSgu7duytttFotwsLCsHv37jKTl9zcXOTm5iqP09PTAQD5+fnIz8+/l/D0FBQUKP811nZ126nucZqKWvoUUE+/mipO7qvq2FfZpzz+df+tice/IdupcvIiIhg7diwee+wxNGvWDACQkpICAPD09NRr6+npifPnz5e5nRkzZmDq1Kmllm/duhX29vZVDa+Ui5kAYIVdu3bhvKPRNgsAiI6ONtq2TBmnsamlTwH19Kup4+S+Wr33VfbpHTz+a+bxn52dXem2VU5eRo4cicOHD2PXrl2l1mk0Gr3HIlJqmc7EiRMxduxY5XF6ejp8fX3RvXt3ODs7VzW8Uo5eTsfshD147LHH0LSucbabn5+P6OhodOvWDdbW1kbZpiniNBW19Cmgnn41VZzcV9Wxr7JPefzX5ONfd+alMqqUvLzxxhtYv349du7cCR8fH2W5l5cXgDsjMN7e3sry1NTUUqMxOlqtFlqtttRya2tro+68VlZWyn+NuV3AuLGaMk5jU0ufAurpV1PHyX21eu+r7NM7ePzXzOPfkO0YdLWRiGDkyJFYs2YNtm3bhoCAAL31AQEB8PLy0huaysvLQ0xMDNq3b2/ISxERERGVyaCRl9dffx0rVqzAunXr4OTkpNS4uLi4wM7ODhqNBqNHj8b06dMRFBSEoKAgTJ8+Hfb29ujfv79J3gARUUVyC2/DwvYSEtNPwsLWOCfoCwoKcLngMo5fP678Cr1XiemZsLC9hNzC2wBcjLJNogeRQUfcl19+CQDo1KmT3vIlS5Zg8ODBAICoqCjk5ORgxIgRuHHjBtq1a4etW7fCycnJKAETERnqctZ5OAR8hkn7jL/tBVsWGHV7DgHA5ayWaI2yT7UTkYHJi4hU2Eaj0WDKlCmYMmVKVWMiIjKqug5+yEp8A/P7tURDD+ONvPy16y90eKyD0UZezqZm4s2fDqFuZz+jbI/oQWWcI46IqBrTWtqi6HY9BDgHI8TdOKdj8vPzkWiViCZuTYxWsFh0+xaKbl+F1tLWKNsjelDxxoxERESkKhx5IapGTFFYCrC4lEgNePxXHpMXomrElIWlAItLiaozHv+Vx+SFqBoxRWEpwOJSIjXg8V95TF6IqhFTFJYCLC4lUgMe/5XHgl0iIiJSFSYvREREpCo15rQRpwc3PrX0KaCufiUiovLVmOSF04Mbn5r6FFBPvxIRUflqTPLC6cGNTy19CqirX4mIqHw1Jnnh9ODGp5Y+BdTVr0REVD4W7BIREZGq1JiRFyIiMh4W7JM5MXkhIiKDsWCfzInJCxERGYwF+2ROTF6IiMhgLNgnc2LBLhEREakKkxciIiJSFSYvREREpCpMXoiIiEhVmLwQERGRqjB5ISIiIlVh8kJERESqwuSFiIiIVIXJCxEREakKkxciIiJSFSYvREREpCpMXoiIiEhVmLwQERGRqjB5ISIiIlVh8kJERESqwuSFiIiIVMXK3AEQEZlaTn4hAODIpVtG22ZWTi4OXAW8zt+Ag53WKNs8k5pplO2QOpliPwUezH2VyQsRPfDO/v8P2glrEoy8ZSt8f2a/kbcJOGj50VwTmW4/BR60fZVHCBE98Lo39QIANPRwhJ21pVG2eTL5Ft5anYBP+oYi2NvFKNsE7nwZBNR2MNr2SD1MsZ8CD+a+yuSFiB54bg42eKFtfaNus6CgAADQsI4DmtUz3hcC1Vym2E+BB3NfZcEuERERqQpHXoiqERbsERFVjMkLUTXCgj0ioorxU4eoGmHBHhFRxQxOXnbu3IlZs2YhPj4eycnJ+PXXXxEZGamsFxFMnToVixYtwo0bN9CuXTt88cUXaNq0qTHjJnogsWCPiKhiBhfsZmVloUWLFvj888/LXD9z5kzMmTMHn3/+Ofbv3w8vLy9069YNGRkZ9xwsERERkcEjLz179kTPnj3LXCcimDdvHiZPnow+ffoAAJYtWwZPT0+sWLECr776aqnn5ObmIjc3V3mcnp4OAMjPz0d+fr6h4d1VRs6d1/jnwnXlV+i9yrp9pwiy9r9X4WBrpCLIq1kA7vxSNub7NwW19Cmgrn41Bd2/T019/6ZQ0/uUx796qGVfNSQ2o9a8JCYmIiUlBd27d1eWabVahIWFYffu3WUmLzNmzMDUqVNLLd+6dSvs7e2NFlvcFQ0AS0xed8xo27zDCt+f+dvI2wT2x+3CeTujb9ao1NangDr61RQuZgKAFfbs2YNLR8wdzYOhpvcpj3/1UMu+mp2dXem2Rk1eUlJSAACenp56yz09PXH+/PkynzNx4kSMHTtWeZyeng5fX190794dzs7ORovtkaw8hB5PRYM6DkYrhDyVcgtRvx7HzGeaoJGXMYsgLeHvXv2LINXUp4B6+tUU/rlwHUg4gEceeQQt6ruZO5wHQk3vUx7/6qGWfVV35qUyTHK1kUaj0XssIqWW6Wi1Wmi1pYcHra2tYW1tbbSYPGtZY8CjAUbbXnGNvFzQ0s/dJNuuztin6mFlZaX815jHVU1W0/uUx796qGVfNSQ2o86w6+V15zJP3QiMTmpqaqnRGCIiIqKqMGryEhAQAC8vL0RHRyvL8vLyEBMTg/bt2xvzpYiIiKiGMvi0UWZmJs6cOaM8TkxMxKFDh+Dm5ob69etj9OjRmD59OoKCghAUFITp06fD3t4e/fv3N2rgREREVDMZnLwcOHAAnTt3Vh7rim0HDRqEpUuXIioqCjk5ORgxYoQySd3WrVvh5ORkvKiJiIioxjI4eenUqRNE5K7rNRoNpkyZgilTptxLXERERERlMmrNCxEREZGpMXkhIiIiVWHyQkRERKrC5IWIiIhUhckLERERqQqTFyIiIlIVJi9ERESkKkxeiIiISFWYvBAREZGqMHkhIiIiVWHyQkRERKrC5IWIiIhUhckLERERqQqTFyIiIlIVJi9ERESkKkxeiIiISFWYvBAREZGqMHkhIiIiVWHyQkRERKrC5IWIiIhUhckLERERqQqTFyIiIlIVJi9ERESkKkxeiIiISFWYvBAREZGqMHkhIiIiVWHyQkRERKpiZe4AiIiqi+zsbJw4caJSbU8m30RuyhkcP2KHorRalXpO48aNYW9vfw8REt1hyn1VDfspkxciov/vxIkTaN26tUHP6b+s8m3j4+Px0EMPGRgVUWmm3FfVsJ8yeSEi+v8aN26M+Pj4SrXNzMnFb9vj8FTnR+Fop6309omMwZT7qhr2UyYvRET/n729faV/cebn5+PGtVQ82rYNrK2tTRwZkb6avq+yYJeIiIhUhSMvJbBgzzQq26/s08qr6QV7pB48/snYmLyUwII90zC0X9mnFavpBXukHjz+ydiYvJTAgj3TqGy/sk8rr6YX7JF68PgnY2PyUkJNL4Iylcr2K/u08rivklrw+CdjY8EuERERqQqTFyIiIlIVJi9ERESkKkxeiIiISFVMlrwsWLAAAQEBsLW1RevWrREbG2uqlyIiIqIaxCTJy08//YTRo0dj8uTJ+Pvvv/H444+jZ8+euHDhgilejoiIiGoQk1wqPWfOHAwdOhQvv/wyAGDevHn4/fff8eWXX2LGjBl6bXNzc5Gbm6s8Tk9PB3Dnkrn8/HxThGc0uviqe5xqwj41Dfar8bFPjY99ahpq6VdD4tOIiBjzxfPy8mBvb4+ff/4ZzzzzjLL8zTffxKFDhxATE6PXfsqUKZg6dWqp7axYsYJTPhMREdUQ2dnZ6N+/P27dugVnZ+dy2xp95OXatWsoLCyEp6en3nJPT0+kpKSUaj9x4kSMHTtWeZyeng5fX1907969wuDNLT8/H9HR0ejWrRsnVDIS9qlpsF+Nj31qfOxT01BLv+rOvFSGyWbY1Wg0eo9FpNQyANBqtdBqS08DbW1tXa07uTg1xaoW7FPTYL8aH/vU+NinplHd+9WQ2IxesFu7dm1YWlqWGmVJTU0tNRpDREREZCijJy82NjZo3bo1oqOj9ZZHR0ejffv2xn45IiIiqmFMctpo7NixeOmll9CmTRs8+uijWLRoES5cuIDhw4eb4uWIiIioBjFJ8tKvXz+kpaXhgw8+QHJyMpo1a4ZNmzbBz8+vwufqLn4ypHDHXPLz85GdnY309PRqfR5RTdinpsF+NT72qfGxT01DLf2q+96vzEXQRr9U+l4lJSXB19fX3GEQERGRGVy8eBE+Pj7ltql2yUtRUREuX74MJyenMq9Oqk50l3VfvHix2l/WrRbsU9Ngvxof+9T42KemoZZ+FRFkZGSgbt26sLAovyTXZJdKV5WFhUWFGVd14+zsXK13CDVin5oG+9X42KfGxz41DTX0q4uLS6Xa8a7SREREpCpMXoiIiEhVmLzcA61Wi/fff7/MGYKpatinpsF+NT72qfGxT03jQezXalewS0RERFQejrwQERGRqjB5ISIiIlVh8kJERESqwuSFTE5E8Morr8DNzQ0ajQaHDh0yd0gPnMGDByMyMtLcYahap06dMHr0aHOHUWNoNBqsXbvW3GFQMVOmTEHLli3NHUalVLtJ6ujBs2XLFixduhQ7duxAgwYNULt2bXOH9MCZP39+pe4HQkR0N+PGjcMbb7xh7jAqhclLNZOfn1+tb5xVFWfPnoW3tzfat29vstfIy8uDjY2NybZf3VV2VkoienBV9XNQRFBYWAhHR0c4OjqaIDLjq7GnjbZs2YLHHnsMtWrVgru7O3r37o2zZ88CAM6dOweNRoM1a9agc+fOsLe3R4sWLRAXF6e3ja+//hq+vr6wt7fHM888gzlz5qBWrVp6bTZs2IDWrVvD1tYWDRo0wNSpU1FQUKCs12g0WLhwISIiIuDg4IBp06aZ/L3fT4MHD8Ybb7yBCxcuQKPRwN/fHyKCmTNnokGDBrCzs0OLFi2wevVq5TmFhYUYOnQoAgICYGdnh+DgYMyfP7/UdiMjIzFjxgzUrVsXjRo1ut9vrVopftooNzcXo0aNgoeHB2xtbfHYY49h//79AO58SAUGBmL27Nl6zz9y5AgsLCyUY6Cmu3HjBgYOHAhXV1fY29ujZ8+eOH36NADg1q1bsLOzw5YtW/Ses2bNGjg4OCAzMxMAcOnSJfTr1w+urq5wd3dHREQEzp07d7/fitGsXr0aoaGhsLOzg7u7O7p27YqsrCzs378f3bp1Q+3ateHi4oKwsDAcPHhQ77mnT59Gx44dYWtri5CQEERHR+utr+xn7u7du9GxY0fY2dnB19cXo0aNQlZWlrJ+wYIFCAoKgq2tLTw9PdG3b98K4ze3u8VV1mnMyMhIDB48WHns7++PadOmYfDgwXBxccGwYcOUvvzxxx/Rvn172NraomnTptixY4fyvB07dkCj0eD3339HmzZtoNVqERsbW+q00Y4dO9C2bVs4ODigVq1a6NChA86fP6+sr+j7zaSkhlq9erX88ssvcurUKfn7778lPDxcQkNDpbCwUBITEwWANG7cWDZu3CgnT56Uvn37ip+fn+Tn54uIyK5du8TCwkJmzZolJ0+elC+++ELc3NzExcVFeY0tW7aIs7OzLF26VM6ePStbt24Vf39/mTJlitIGgHh4eMg333wjZ8+elXPnzt3vrjCpmzdvygcffCA+Pj6SnJwsqampMmnSJGncuLFs2bJFzp49K0uWLBGtVis7duwQEZG8vDx57733ZN++ffLvv//KDz/8IPb29vLTTz8p2x00aJA4OjrKSy+9JEeOHJGEhARzvcVqYdCgQRIRESEiIqNGjZK6devKpk2b5OjRozJo0CBxdXWVtLQ0ERH58MMPJSQkRO/5Y8aMkY4dO97vsKuVsLAwefPNN0VE5Omnn5YmTZrIzp075dChQ9KjRw8JDAyUvLw8ERF59tln5cUXX9R7/rPPPiv/+c9/REQkKytLgoKCZMiQIXL48GE5duyY9O/fX4KDgyU3N/e+vi9juHz5slhZWcmcOXMkMTFRDh8+LF988YVkZGTIn3/+Kd9//70cO3ZMjh07JkOHDhVPT09JT08XEZHCwkJp1qyZdOrUSf7++2+JiYmRVq1aCQD59ddfRUQq9Zl7+PBhcXR0lLlz58qpU6fkr7/+klatWsngwYNFRGT//v1iaWkpK1askHPnzsnBgwdl/vz5FcZvTuXFVXx/1ImIiJBBgwYpj/38/MTZ2VlmzZolp0+fltOnTyt96ePjI6tXr5Zjx47Jyy+/LE5OTnLt2jUREdm+fbsAkObNm8vWrVvlzJkzcu3aNXn//felRYsWIiKSn58vLi4uMm7cODlz5owcO3ZMli5dKufPnxeRyn2/mVKNTV5KSk1NFQCSkJCg/OMvXrxYWX/06FEBIMePHxcRkX79+slTTz2lt40BAwboJS+PP/64TJ8+Xa/N999/L97e3spjADJ69GgTvKPqY+7cueLn5yciIpmZmWJrayu7d+/WazN06FDlg78sI0aMkGeffVZ5PGjQIPH09FTlF4Ep6JKXzMxMsba2luXLlyvr8vLypG7dujJz5kwRufOBaWlpKXv37lXW16lTR5YuXWqW2KsL3ZfFqVOnBID89ddfyrpr166JnZ2drFq1SkRE1qxZI46OjpKVlSUiIrdu3RJbW1v57bffRETkm2++keDgYCkqKlK2kZubK3Z2dvL777/fx3dlHPHx8QKgUj+uCgoKxMnJSTZs2CAiIr///rtYWlrKxYsXlTabN28uM3kp7zP3pZdekldeeUXvtWJjY8XCwkJycnLkl19+EWdnZyVpqmr891N5cVU2eYmMjNRro+vLjz76SFmWn58vPj4+8vHHH4vI/yUva9eu1Xtu8eQlLS1NACg/KkuqzPebKdXY00Znz55F//790aBBAzg7OyMgIAAAcOHCBaVN8+bNlf/39vYGAKSmpgIATp48ibZt2+pts+Tj+Ph4fPDBB8p5REdHRwwbNgzJycnIzs5W2rVp08a4b64aO3bsGG7fvo1u3brp9ct3332nd8pi4cKFaNOmDerUqQNHR0d8/fXXev82ABAaGlqj61zKcvbsWeTn56NDhw7KMmtra7Rt2xbHjx8HcGdffuqpp/Dtt98CADZu3Ijbt2/jueeeM0vM1c3x48dhZWWFdu3aKcvc3d0RHBys9OFTTz0FKysrrF+/HgDwyy+/wMnJCd27dwdw59g/c+YMnJyclH3czc0Nt2/fVuWpuRYtWqBLly4IDQ3Fc889h6+//ho3btwAcOczcfjw4WjUqBFcXFzg4uKCzMxM5Xg9fvw46tevDx8fH2V7jz76aJmvU95nbnx8PJYuXar3udGjRw8UFRUhMTER3bp1g5+fHxo0aICXXnoJy5cvVz5ny4vfnIwR192+P4r3sZWVFdq0aaPsvxU9FwDc3NwwePBg9OjRA+Hh4Zg/fz6Sk5OV9ZX9fjOVGpu8hIeHIy0tDV9//TX27t2LvXv3ArhT8KRTvHBWo9EAAIqKigDcqR3QLdOREld7FBUVYerUqTh06JDyl5CQgNOnT8PW1lZp5+DgYNw3V43p+u+3337T65djx44pdS+rVq3CmDFjMGTIEGzduhWHDh3Cf//7X71/G6Bm9Vtl6fbBsvbN4stefvll/Pjjj8jJycGSJUvQr18/2Nvb39dYq6uSx3Hx5bo+tLGxQd++fbFixQoAwIoVK9CvXz9YWd25BqKoqAitW7fW28cPHTqEU6dOoX///vfnjRiRpaUloqOjsXnzZoSEhOCzzz5DcHAwEhMTMXjwYMTHx2PevHnYvXs3Dh06BHd3d+V4Las/S+6fOuV95hYVFeHVV1/V689//vkHp0+fRsOGDeHk5ISDBw9i5cqV8Pb2xnvvvYcWLVrg5s2b5cZvTuXFZWFhUarv8vPzS23DkM/Bkv1e0XOXLFmCuLg4tG/fHj/99BMaNWqEPXv2AKj895up1MjkJS0tDcePH8c777yDLl26oEmTJgZnu40bN8a+ffv0lh04cEDv8UMPPYSTJ08iMDCw1J+FRY3seoSEhECr1eLChQul+sTX1xcAEBsbi/bt22PEiBFo1aoVAgMDVflr1RwCAwNhY2ODXbt2Kcvy8/Nx4MABNGnSRFnWq1cvODg44Msvv8TmzZsxZMgQc4RbLYWEhKCgoED5QQPc+cw4deqUXh8OGDAAW7ZswdGjR7F9+3YMGDBAWffQQw/h9OnT8PDwKLWfq/XKMI1Ggw4dOmDq1Kn4+++/YWNjg19//RWxsbEYNWoUevXqhaZNm0Kr1eLatWvK80JCQnDhwgVcvnxZWVayELcyHnroIRw9erTMz1PdCKyVlRW6du2KmTNn4vDhwzh37hy2bdtWbvzmdre46tSpozfSUVhYiCNHjlR6u7okAwAKCgoQHx+Pxo0bGxxfq1atMHHiROzevRvNmjVTEnZzf7/VyEulddX/ixYtgre3Ny5cuIAJEyYYtI033ngDHTt2xJw5cxAeHo5t27Zh8+bNepnte++9h969e8PX1xfPPfccLCwscPjwYSQkJDxwVxVVlpOTE8aNG4cxY8agqKgIjz32GNLT07F79244Ojpi0KBBCAwMxHfffYfff/8dAQEB+P7777F//37l1B7dnYODA1577TWMHz8ebm5uqF+/PmbOnIns7GwMHTpUaWdpaYnBgwdj4sSJCAwMvOswfk0UFBSEiIgIDBs2DF999RWcnJwwYcIE1KtXDxEREUq7sLAweHp6YsCAAfD398cjjzyirBswYABmzZqFiIgIfPDBB/Dx8cGFCxewZs0ajB8/Xu8Uihrs3bsXf/75J7p37w4PDw/s3bsXV69eRZMmTRAYGIjvv/8ebdq0QXp6OsaPHw87OzvluV27dkVwcDAGDhyITz75BOnp6Zg8ebLBMbz99tt45JFH8Prrr2PYsGFwcHDA8ePHER0djc8++wwbN27Ev//+i44dO8LV1RWbNm1CUVERgoODy43fnMqLy8HBAWPHjsVvv/2Ghg0bYu7cubh582alt/3FF18gKCgITZo0wdy5c3Hjxg2DfqQkJiZi0aJFePrpp1G3bl2cPHkSp06dwsCBAwFUg++3+1JZUw1FR0dLkyZNRKvVSvPmzWXHjh1KAZmu4Onvv/9W2t+4cUMAyPbt25VlixYtknr16omdnZ1ERkbKtGnTxMvLS+91tmzZIu3btxc7OztxdnaWtm3byqJFi5T1KFa09qAqXrArIlJUVCTz58+X4OBgsba2ljp16kiPHj0kJiZGRERu374tgwcPFhcXF6lVq5a89tprMmHCBKWQTET/6hrS74+cnBx54403pHbt2qLVaqVDhw6yb9++Us85e/asAFAKeWu64gWS169fl5deeklcXFzEzs5OevToIadOnSr1nPHjxwsAee+990qtS05OloEDByr/Dg0aNJBhw4bJrVu3TP1WjO7YsWPSo0cPqVOnjmi1WmnUqJF89tlnIiJy8OBBadOmjWi1WgkKCpKff/5Z/Pz8ZO7cucrzT548KY899pjY2NhIo0aNZMuWLWUW7Fb0mbtv3z7p1q2bODo6ioODgzRv3lw+/PBDEblTvBsWFiaurq5iZ2cnzZs3V65QLC9+cyovrry8PHnttdfEzc1NPDw8ZMaMGWUW7BbvZ5H/68sVK1ZIu3btxMbGRpo0aSJ//vmn0kZXsHvjxg295xYv2E1JSZHIyEjx9vYWGxsb8fPzk/fee08KCwuV9hV9v5mSRoTTchrLsGHDcOLECcTGxpo7FKph/vOf/8DS0hI//PBDpZ/z119/oVOnTkhKSoKnp6cJoyOi++XcuXMICAjA33//rZqp/quiZhZeGMns2bPxzz//4MyZM/jss8+wbNkyDBo0yNxhUQ1SUFCAY8eOIS4uDk2bNq3Uc3Jzc3HmzBm8++67eP7555m4EJHqMHm5B/v27UO3bt0QGhqKhQsX4tNPP8XLL79s7rCoBjly5AjatGmDpk2bYvjw4ZV6zsqVKxEcHIxbt25h5syZJo6QiMj4eNqIiIiIVIUjL0RERKQqTF6IiIhIVZi8EBERkaoweSEiIiJVYfJCREREqsLkhaiG6NSpE0aPHl2ptjt27IBGozFoOvKy+Pv7Y968efe0DSKikpi8EBERkaoweSEiIiJVYfJCVAP98MMPaNOmDZycnODl5YX+/fsjNTW1VLu//voLLVq0gK2tLdq1a4eEhAS99bt370bHjh1hZ2cHX19fjBo1CllZWVWKSaPRYPHixXjmmWdgb2+PoKAgrF+/XllfWFiIoUOHIiAgAHZ2dggODsb8+fP1tjF48GBERkZi+vTp8PT0RK1atTB16lQUFBQod9r28fHBt99+q/e8S5cuoV+/fsod5yMiInDu3Dll/Y4dO9C2bVs4ODigVq1a6NChA86fP1+l90lE947JC1ENlJeXh//973/4559/sHbtWiQmJmLw4MGl2o0fPx6zZ8/G/v374eHhgaeffhr5+fkAgISEBPTo0QN9+vTB4cOH8dNPP2HXrl0YOXJkleOaOnUqnn/+eRw+fBi9evXCgAEDcP36dQBAUVERfHx8sGrVKhw7dgzvvfceJk2ahFWrVultY9u2bbh8+TJ27tyJOXPmYMqUKejduzdcXV2xd+9eDB8+HMOHD8fFixcBANnZ2ejcuTMcHR2xc+dO7Nq1C46OjnjyySeRl5eHgoICREZGIiwsDIcPH0ZcXBxeeeUVaDSaKr9PIrpH9+Xe1URkdmFhYfLmm2+WuW7fvn0CQDIyMkREZPv27QJAfvzxR6VNWlqa2NnZyU8//SQiIi+99JK88soretuJjY0VCwsLycnJERERPz8/mTt3bqXiAyDvvPOO8jgzM1M0Go1s3rz5rs8ZMWKEPPvss8rjQYMGiZ+fnxQWFirLgoOD5fHHH1ceFxQUiIODg6xcuVJERL755hsJDg6WoqIipU1ubq7Y2dnJ77//LmlpaQJAduzYUan3QUSmx5EXohro77//RkREBPz8/ODk5IROnToBAC5cuKDX7tFHH1X+383NDcHBwTh+/DgAID4+HkuXLoWjo6Py16NHDxQVFSExMbFKcTVv3lz5fwcHBzg5Oemdzlq4cCHatGmDOnXqwNHREV9//XWpmJs2bQoLi//7aPP09ERoaKjy2NLSEu7u7sp24+PjcebMGTg5OSnvw83NDbdv38bZs2fh5uaGwYMHo0ePHggPD8f8+fORnJxcpfdHRMZhZe4AiOj+ysrKQvfu3dG9e3f88MMPqFOnDi5cuIAePXogLy+vwufrTpcUFRXh1VdfxahRo0q1qV+/fpVis7a2LvVaRUVFAIBVq1ZhzJgx+OSTT/Doo4/CyckJs2bNwt69eyvcRnnbLSoqQuvWrbF8+fJS8dSpUwcAsGTJEowaNQpbtmzBTz/9hHfeeQfR0dF45JFHqvQ+iejeMHkhqmFOnDiBa9eu4aOPPoKvry8A4MCBA2W23bNnj5KI3LhxA6dOnULjxo0BAA899BCOHj2KwMDA+xJ3bGws2rdvjxEjRijLzp49e8/bfeihh/DTTz/Bw8MDzs7Od23XqlUrtGrVChMnTsSjjz6KFStWMHkhMhOeNiKqYerXrw8bGxt89tln+Pfff7F+/Xr873//K7PtBx98gD///BNHjhzB4MGDUbt2bURGRgIA3n77bcTFxeH111/HoUOHcPr0aaxfvx5vvPGGSeIODAzEgQMH8Pvvv+PUqVN49913sX///nve7oABA1C7dm1EREQgNjYWiYmJiImJwZtvvomkpCQkJiZi4sSJiIuLw/nz57F161acOnUKTZo0McK7IqKqYPJCVMPUqVMHS5cuxc8//4yQkBB89NFHmD17dpltP/roI7z55pto3bo1kpOTsX79etjY2AC4U58SExOD06dP4/HHH0erVq3w7rvvwtvb2yRxDx8+HH369EG/fv3Qrl07pKWl6Y3CVJW9vT127tyJ+vXro0+fPmjSpAmGDBmCnJwcODs7w97eHidOnMCzzz6LRo0a4ZVXXsHIkSPx6quvGuFdEVFVaEREzB0EERERUWVx5IWIiIhUhckLEZnc8uXL9S6pLv7XtGlTc4dHRCrD00ZEZHIZGRm4cuVKmeusra3h5+d3nyMiIjVj8kJERESqwtNGREREpCpMXoiIiEhVmLwQERGRqjB5ISIiIlVh8kJERESqwuSFiIiIVIXJCxEREanK/wOQRm2RbqMpvgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df[\"word_count\"] = df.text.apply(lambda x: len(x.split()))\n",
    "\n",
    "df.boxplot(column=\"word_count\", by=\"label_names\", showfliers=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is no tweet longer than 70 words - this easily fits into the context size of our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "emotions.reset_format()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Pre-Trained Models\n",
    "\n",
    "https://huggingface.co/distilbert-base-uncased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias', 'vocab_layer_norm.weight', 'vocab_transform.bias']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'pre_classifier.weight', 'classifier.bias', 'pre_classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model_checkpoint = \"distilbert-base-uncased\"\n",
    "num_labels = 6\n",
    "\n",
    "tokenizer = DistilBertTokenizer.from_pretrained(model_checkpoint)\n",
    "model = DistilBertForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DistilBertTokenizer(name_or_path='distilbert-base-uncased', vocab_size=30522, model_max_length=512, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DistilBertForSequenceClassification(\n",
       "  (distilbert): DistilBertModel(\n",
       "    (embeddings): Embeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (transformer): Transformer(\n",
       "      (layer): ModuleList(\n",
       "        (0-5): 6 x TransformerBlock(\n",
       "          (attention): MultiHeadSelfAttention(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (ffn): FFN(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (activation): GELUActivation()\n",
       "          )\n",
       "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (pre_classifier): Linear(in_features=768, out_features=768, bias=True)\n",
       "  (classifier): Linear(in_features=768, out_features=6, bias=True)\n",
       "  (dropout): Dropout(p=0.2, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "transformers.models.distilbert.modeling_distilbert.DistilBertForSequenceClassification"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We place the model on the GPU:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mps:0\n"
     ]
    }
   ],
   "source": [
    "model.to(device)\n",
    "print(model.device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can tokenize and detokenize a single data point like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i didnt feel humiliated\n",
      "{'input_ids': [101, 1045, 2134, 2102, 2514, 26608, 102], 'attention_mask': [1, 1, 1, 1, 1, 1, 1]}\n",
      "['[CLS]', 'i', 'didn', '##t', 'feel', 'humiliated', '[SEP]']\n",
      "[CLS] i didnt feel humiliated [SEP]\n"
     ]
    }
   ],
   "source": [
    "ex_text_orig = train_ds[0]['text']\n",
    "print(ex_text_orig)\n",
    "\n",
    "ex_encoded_text = tokenizer(train_ds[0]['text'])\n",
    "print(ex_encoded_text)\n",
    "\n",
    "ex_tokens = tokenizer.convert_ids_to_tokens(ex_encoded_text.input_ids)\n",
    "print(ex_tokens)\n",
    "\n",
    "ex_string_recon = tokenizer.convert_tokens_to_string(ex_tokens)\n",
    "print(ex_string_recon)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running a forward pass through the model with a batch of tokenized data points:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[  101,  1045,  2134,  ...,     0,     0,     0],\n",
      "        [  101,  1045,  2064,  ...,     0,     0,     0],\n",
      "        [  101, 10047,  9775,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [  101,  1045,  8823,  ...,     0,     0,     0],\n",
      "        [  101,  1045,  2514,  ...,     0,     0,     0],\n",
      "        [  101,  1045,  2211,  ...,     0,     0,     0]], device='mps:0'), 'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0]], device='mps:0')}\n",
      "SequenceClassifierOutput(loss=None, logits=tensor([[ 1.4233e-01, -3.5799e-02, -3.9052e-02, -1.2486e-01,  1.1337e-02,\n",
      "          4.1456e-03],\n",
      "        [ 1.5962e-01, -1.8645e-03, -5.0531e-02, -1.1383e-01,  1.6687e-02,\n",
      "         -1.2262e-02],\n",
      "        [ 1.4096e-01, -2.8270e-02, -2.3139e-02, -1.4966e-01, -5.5413e-03,\n",
      "         -3.0725e-04],\n",
      "        [ 1.2116e-01, -2.1436e-02, -2.2654e-02, -1.1917e-01,  8.3146e-03,\n",
      "          1.2819e-02],\n",
      "        [ 1.2686e-01, -2.5140e-03, -2.4941e-02, -1.6054e-01,  1.3207e-02,\n",
      "         -2.0017e-03],\n",
      "        [ 1.3553e-01, -4.4948e-02, -1.3296e-02, -1.1402e-01, -1.5227e-02,\n",
      "          3.8793e-02],\n",
      "        [ 1.5724e-01,  6.9897e-02, -3.3859e-02, -1.2927e-01, -1.4081e-02,\n",
      "          2.3798e-02],\n",
      "        [ 1.2619e-01, -5.0330e-03, -4.7603e-02, -1.2175e-01, -7.5088e-03,\n",
      "          1.0555e-02],\n",
      "        [ 8.0060e-02, -3.2129e-02, -2.3077e-02, -1.6091e-01,  2.7138e-02,\n",
      "         -7.4920e-03],\n",
      "        [ 1.3315e-01, -4.5875e-02, -5.3141e-02, -1.4352e-01, -1.4877e-03,\n",
      "          1.2945e-02],\n",
      "        [ 1.4902e-01, -1.4755e-02, -3.5898e-02, -1.3867e-01, -1.7973e-02,\n",
      "         -1.0358e-02],\n",
      "        [ 1.2576e-01,  6.0104e-04, -4.7409e-02, -1.2226e-01,  4.5271e-02,\n",
      "         -1.5301e-02],\n",
      "        [ 1.4248e-01, -2.0736e-02, -7.4252e-02, -1.3979e-01,  2.3742e-02,\n",
      "          7.6523e-03],\n",
      "        [ 1.4122e-01,  1.4110e-02, -6.2735e-02, -1.4574e-01, -1.3873e-03,\n",
      "          2.6653e-03],\n",
      "        [ 1.1123e-01, -7.4643e-03, -4.2686e-02, -1.1611e-01,  4.3666e-04,\n",
      "         -9.3827e-03],\n",
      "        [ 1.3886e-01, -1.5303e-03, -2.6063e-02, -1.4838e-01,  1.1344e-02,\n",
      "          1.5452e-02],\n",
      "        [ 1.4855e-01, -5.7397e-02, -3.0159e-02, -1.3209e-01,  2.1942e-02,\n",
      "          9.1073e-03],\n",
      "        [ 1.3703e-01, -2.3681e-02, -5.6169e-02, -1.3673e-01,  6.2913e-03,\n",
      "          1.8533e-02],\n",
      "        [ 1.4937e-01, -1.9236e-02, -3.5975e-02, -9.6610e-02, -2.9679e-02,\n",
      "          8.2103e-02],\n",
      "        [ 1.0979e-01, -1.7023e-02, -3.4062e-02, -1.4152e-01, -1.6392e-02,\n",
      "         -2.3875e-02],\n",
      "        [ 1.5651e-01, -1.8269e-03, -2.9920e-02, -1.3488e-01, -1.1655e-02,\n",
      "          3.7166e-03],\n",
      "        [ 1.0090e-01, -1.8375e-03, -2.4937e-02, -1.2581e-01,  2.1587e-02,\n",
      "         -2.9232e-05],\n",
      "        [ 1.1796e-01, -3.7179e-02, -4.9161e-02, -1.3596e-01,  1.5601e-02,\n",
      "          1.0488e-02],\n",
      "        [ 9.5583e-02, -3.4543e-02, -8.2448e-03, -1.1330e-01, -9.7575e-04,\n",
      "         -1.9508e-02],\n",
      "        [ 1.7488e-01, -6.2767e-03, -5.8071e-02, -1.0974e-01,  1.7336e-02,\n",
      "         -1.2677e-02],\n",
      "        [ 1.3011e-01,  7.0154e-03, -4.6719e-02, -1.2390e-01,  1.4996e-02,\n",
      "          5.8112e-03],\n",
      "        [ 1.6171e-01,  2.8571e-02,  1.4520e-02, -1.1348e-01,  1.6719e-02,\n",
      "          4.6525e-02],\n",
      "        [ 1.2096e-01, -4.7465e-03, -5.3050e-02, -1.2936e-01,  2.9222e-03,\n",
      "          4.4149e-03],\n",
      "        [ 1.2663e-01,  2.1371e-02, -1.5378e-02, -1.4178e-01, -4.9094e-03,\n",
      "          2.5740e-02],\n",
      "        [ 1.4949e-01,  4.5682e-02, -4.6581e-02, -1.1382e-01,  3.8925e-02,\n",
      "          1.8925e-02],\n",
      "        [ 1.5930e-01, -2.6000e-02, -8.1725e-02, -1.2844e-01,  2.3078e-02,\n",
      "          1.7999e-02],\n",
      "        [ 1.1508e-01, -5.4361e-02, -1.2097e-02, -1.1560e-01, -4.3136e-03,\n",
      "          1.6119e-02],\n",
      "        [ 1.0650e-01, -2.9456e-02, -9.4272e-03, -1.4746e-01, -3.1861e-02,\n",
      "          1.9064e-02],\n",
      "        [ 1.0366e-01, -9.7240e-03, -1.0911e-02, -1.2217e-01, -2.4717e-03,\n",
      "          1.3728e-02],\n",
      "        [ 1.4861e-01, -4.1146e-02, -4.7642e-02, -1.2031e-01,  4.4020e-02,\n",
      "          1.0007e-03],\n",
      "        [ 1.4028e-01, -5.5046e-02, -4.0656e-02, -1.2569e-01, -1.4845e-02,\n",
      "         -9.3913e-03],\n",
      "        [ 9.7403e-02, -1.8497e-02, -4.1954e-02, -1.5652e-01, -1.8027e-02,\n",
      "         -1.0393e-02],\n",
      "        [ 1.1734e-01,  2.1484e-03, -3.6896e-02, -1.3824e-01,  2.9927e-03,\n",
      "         -9.7712e-03],\n",
      "        [ 1.3530e-01, -4.1382e-03, -6.6913e-02, -1.2150e-01,  5.1750e-02,\n",
      "          5.2888e-03],\n",
      "        [ 1.0634e-01,  1.9518e-02, -2.7008e-02, -1.1104e-01,  1.6907e-02,\n",
      "         -2.3599e-02],\n",
      "        [ 1.5376e-01,  7.4966e-03, -6.2959e-02, -1.0318e-01, -9.9983e-03,\n",
      "          3.7106e-02],\n",
      "        [ 1.3008e-01,  2.0290e-02, -8.0737e-02, -1.4428e-01,  1.4979e-02,\n",
      "          2.6109e-03],\n",
      "        [ 1.4071e-01, -1.4517e-02, -2.0958e-02, -1.1339e-01,  1.2747e-02,\n",
      "          9.3763e-03],\n",
      "        [ 1.4183e-01, -9.8491e-03, -8.4350e-02, -1.4291e-01,  4.4465e-02,\n",
      "          9.4275e-04],\n",
      "        [ 1.1117e-01,  2.8014e-02, -3.6789e-02, -1.2835e-01,  2.3733e-02,\n",
      "          2.4466e-02],\n",
      "        [ 1.0633e-01, -3.3774e-02, -4.5758e-02, -1.3254e-01, -4.0638e-02,\n",
      "          3.3512e-02],\n",
      "        [ 1.3389e-01,  1.6373e-02, -6.2875e-03, -1.2495e-01,  3.4706e-02,\n",
      "         -1.7554e-02],\n",
      "        [ 1.3963e-01, -2.7059e-02, -4.0488e-02, -1.0715e-01, -5.6545e-03,\n",
      "          2.7460e-03],\n",
      "        [ 1.4890e-01, -4.0927e-02, -5.2144e-02, -7.0175e-02,  1.4726e-02,\n",
      "          4.2442e-02],\n",
      "        [ 1.3726e-01, -3.9996e-02, -4.6873e-02, -1.4888e-01,  2.3211e-02,\n",
      "          3.4968e-02],\n",
      "        [ 1.5906e-01, -4.4825e-02, -4.0935e-02, -1.3097e-01,  1.0965e-02,\n",
      "         -1.5885e-02],\n",
      "        [ 1.3486e-01, -4.9403e-02, -2.6408e-02, -1.2229e-01,  6.2970e-04,\n",
      "          3.2391e-02],\n",
      "        [ 1.2863e-01, -5.5887e-03, -5.2600e-02, -1.3157e-01, -2.0005e-02,\n",
      "         -1.3253e-02],\n",
      "        [ 1.2188e-01, -6.8094e-02, -4.3069e-02, -1.0393e-01, -8.7743e-03,\n",
      "          7.4693e-03],\n",
      "        [ 1.3216e-01,  3.3894e-02, -1.4873e-02, -7.0674e-02,  5.6290e-02,\n",
      "         -8.4897e-03],\n",
      "        [ 1.5045e-01,  4.7135e-03, -6.9442e-02, -1.3989e-01,  3.7274e-02,\n",
      "         -2.3230e-02],\n",
      "        [ 1.4205e-01,  1.5642e-02, -2.2888e-02, -1.3344e-01,  3.8608e-02,\n",
      "          6.5644e-03],\n",
      "        [ 1.2683e-01, -2.0757e-02, -7.0946e-02, -1.3669e-01,  4.3046e-03,\n",
      "          1.5445e-02],\n",
      "        [ 1.1804e-01, -6.8366e-03, -3.0480e-02, -1.3402e-01,  4.3425e-02,\n",
      "         -3.6347e-03],\n",
      "        [ 1.2283e-01, -4.2327e-02, -6.3867e-02, -1.2461e-01, -1.0048e-02,\n",
      "          2.0214e-02],\n",
      "        [ 1.0351e-01, -3.1913e-03, -4.3201e-02, -1.5012e-01,  3.7961e-02,\n",
      "          1.6582e-02],\n",
      "        [ 9.2689e-02, -2.0694e-02,  2.8925e-03, -1.4506e-01, -2.1509e-02,\n",
      "          2.4144e-02],\n",
      "        [ 1.3375e-01,  6.9467e-03, -6.7183e-02, -1.3332e-01, -8.7235e-03,\n",
      "          3.4065e-02],\n",
      "        [ 1.1857e-01,  4.5228e-02, -7.0513e-03, -1.3732e-01, -1.3650e-02,\n",
      "          4.4032e-02]], device='mps:0', grad_fn=<LinearBackward0>), hidden_states=None, attentions=None)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/david/miniforge3/envs/hugging/lib/python3.9/site-packages/torch/_tensor_str.py:115: UserWarning: MPS: nonzero op is supported natively starting from macOS 13.0. Falling back on CPU. This may have performance implications. (Triggered internally at /Users/runner/work/_temp/anaconda/conda-bld/pytorch_1682343673238/work/aten/src/ATen/native/mps/operations/Indexing.mm:218.)\n",
      "  nonzero_finite_vals = torch.masked_select(\n"
     ]
    }
   ],
   "source": [
    "ex_model_inputs = tokenizer(train_ds[:64]['text'], return_tensors=\"pt\", padding=True).to(device)\n",
    "ex_model_outputs = model(**ex_model_inputs)\n",
    "\n",
    "print(ex_model_inputs)\n",
    "print(ex_model_outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keep in mind that the output are logits, not probabilities!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.0419, -0.0022, -0.0660, -0.0210, -0.0499, -0.0132,  0.0737, -0.0452,\n",
       "        -0.1164, -0.0979, -0.0686, -0.0133, -0.0609, -0.0519, -0.0640, -0.0103,\n",
       "        -0.0400, -0.0547,  0.0500, -0.1231, -0.0181, -0.0301, -0.0783, -0.0810,\n",
       "         0.0055, -0.0127,  0.1546, -0.0589,  0.0117,  0.0926, -0.0358, -0.0552,\n",
       "        -0.0926, -0.0279, -0.0155, -0.1053, -0.1480, -0.0624, -0.0002, -0.0189,\n",
       "         0.0222, -0.0571,  0.0140, -0.0499,  0.0222, -0.1129,  0.0362, -0.0380,\n",
       "         0.0428, -0.0403, -0.0626, -0.0302, -0.0944, -0.0945,  0.1283, -0.0401,\n",
       "         0.0465, -0.0818, -0.0135, -0.0978, -0.0385, -0.0675, -0.0345,  0.0498],\n",
       "       device='mps:0', grad_fn=<SumBackward1>)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sum(ex_model_outputs.logits, dim=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating DataLoaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TweetDataset(Dataset):\n",
    "    def __init__(self, split):\n",
    "        self.text = split[\"text\"]\n",
    "        self.labels = split[\"label\"]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.text)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        return self.text[index], self.labels[index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 16\n",
    "\n",
    "train_dataset = TweetDataset(train_ds)\n",
    "val_dataset = TweetDataset(val_ds)\n",
    "test_dataset = TweetDataset(test_ds)\n",
    "\n",
    "train_data_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "val_data_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "test_data_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(('i feel angered and firey',\n",
       "  'i feel a tender compassion glancing at her huge and heavy rucksack',\n",
       "  'ive been hanging around younger people and when i am with them i feel like im but when i see the photos of us together i am suddenly shaken to see just how old i look',\n",
       "  'i sometimes feel shy about my musical taste because some of it wanders towards what some might call techno slander',\n",
       "  'i woke up today feeling just as thankful',\n",
       "  'i feel a recipe is only a theme which an intelligent cook can play each time with a variation',\n",
       "  'i am feeling very anxious about going to therapy w',\n",
       "  'i no longer feel happy to score well',\n",
       "  'im sitting at the spare desk feeling totally disillusioned and frustrated with my working life in general',\n",
       "  'i am feeling peaceful yet determined as i listen to the slight humming noise of the ceiling fan',\n",
       "  'i see myself starting to feel the emotional dependence on my parents i stop and breathe',\n",
       "  'i don t mean this to be a serious recollection of feelings only a funny in a not funny sort of way story so let s get back to where the action begins',\n",
       "  'ive been feeling so anxious and nauseous and tired but also so elated that some nights its all i can do to crawl into bed',\n",
       "  'im feeling craving theres always a tender morsel of a song ready to appease my appetite',\n",
       "  'i am so sorry you are feeling so sad',\n",
       "  'i go up to her and i say feeling very impressed with myself youre naomi klein right'),\n",
       " tensor([3, 2, 4, 4, 1, 1, 4, 1, 0, 1, 0, 5, 4, 2, 0, 5]))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ex_batch_text, ex_batch_labels = next(iter(train_data_loader)) \n",
    "ex_batch_text, ex_batch_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3, 2, 4, 4, 1, 1, 4, 1, 0, 1, 0, 5, 4, 2, 0, 5]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ex_batch_labels.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Training Loop\n",
    "\n",
    "We could run a `Trainer` object from the `transformers` library but I want to implement the loop from scratch as practice. Furthermore, the logging of training metrics is implemented on a low level, instead of using `Tensorboard` and `SummaryWriter`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=2e-5, weight_decay=0.01)\n",
    "\n",
    "def compute_metrics(logit_outputs, labels):\n",
    "    preds = torch.max(F.softmax(logit_outputs, dim=-1), dim=-1)[1]\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    f1 = f1_score(labels, preds, average='weighted')\n",
    "    return acc, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 3\n",
    "\n",
    "def train(\n",
    "        model=model,\n",
    "        train_data_loader=train_data_loader,\n",
    "        val_data_loader=val_data_loader,\n",
    "        tokenizer=tokenizer,\n",
    "        optimizer=optimizer,\n",
    "        loss_fn=loss_fn,\n",
    "        epochs=EPOCHS):\n",
    "    \n",
    "    logs = {\n",
    "        'train_losses': [],\n",
    "        'train_metrics': [],\n",
    "        'val_losses': [],\n",
    "        'val_metrics': []\n",
    "    }\n",
    "\n",
    "    for epoch in range(EPOCHS):\n",
    "        print(f\"--------------- EPOCH {epoch+1} ----------------\")\n",
    "\n",
    "        # TRAIN\n",
    "        train_loss_epoch = 0.0\n",
    "        train_loss_iterations = []\n",
    "        train_metrics_iterations = {'acc': [], 'f1': []}\n",
    "        model.train()\n",
    "        for i, train_batch in enumerate(train_data_loader):\n",
    "            # algo \n",
    "            optimizer.zero_grad()\n",
    "            inputs, labels = train_batch\n",
    "            inputs = tokenizer(inputs, return_tensors='pt', padding=True, truncation=True)\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "            outputs = model(**inputs)\n",
    "            loss = loss_fn(outputs.logits, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            # logging\n",
    "            train_loss_epoch += loss * inputs.input_ids.size()[0]\n",
    "            train_loss_iterations.append(loss)\n",
    "            acc, f1 = compute_metrics(logit_outputs=outputs.logits.cpu(), labels=labels.cpu())\n",
    "            train_metrics_iterations['acc'].append(acc)\n",
    "            train_metrics_iterations['f1'].append(f1)\n",
    "            \n",
    "            # free GPU memory\n",
    "            inputs.input_ids.detach()\n",
    "            inputs.attention_mask.detach()\n",
    "            labels.detach()\n",
    "\n",
    "        train_loss_epoch /= len(train_data_loader.dataset)\n",
    "        train_acc_epoch = sum(train_metrics_iterations['acc']) / len(train_data_loader)\n",
    "        train_f1_epoch = sum(train_metrics_iterations['f1']) / len(train_data_loader)\n",
    "        print(f\"Average TRAIN loss: {train_loss_epoch}\")\n",
    "        print(f\"Average TRAIN acc: {train_acc_epoch}\")\n",
    "        print(f\"Average TRAIN F1: {train_f1_epoch}\")\n",
    "        print()\n",
    "        logs[\"train_losses\"].append(train_loss_iterations)\n",
    "        logs[\"train_metrics\"].append(train_metrics_iterations)\n",
    "\n",
    "\n",
    "        # VALIDATE\n",
    "        val_loss_epoch = 0.0 \n",
    "        val_loss_iterations = []\n",
    "        val_metrics_iterations = {'acc': [], 'f1': []}\n",
    "        model.eval()\n",
    "        with torch.no_grad():\n",
    "            for val_batch in val_data_loader:\n",
    "                inputs, labels = val_batch\n",
    "                inputs = tokenizer(inputs, return_tensors='pt', padding=True, truncation=True)\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "                outputs = model(**inputs)\n",
    "                loss = loss_fn(outputs.logits, labels)\n",
    "\n",
    "                val_loss_epoch += loss * inputs.input_ids.size()[0]\n",
    "                val_loss_iterations.append(loss)\n",
    "                acc, f1 = compute_metrics(logit_outputs=outputs.logits.cpu(), labels=labels.cpu())\n",
    "                val_metrics_iterations['acc'].append(acc)\n",
    "                val_metrics_iterations['f1'].append(f1)\n",
    "\n",
    "                inputs.input_ids.detach()\n",
    "                inputs.attention_mask.detach()\n",
    "                labels.detach()\n",
    "\n",
    "            val_loss_epoch /= len(val_data_loader.dataset)\n",
    "            val_acc_epoch = sum(val_metrics_iterations['acc']) / len(val_data_loader)\n",
    "            val_f1_epoch = sum(val_metrics_iterations['f1']) / len(val_data_loader)\n",
    "            print(f\"Average VALIDATION loss: {val_loss_epoch}\")\n",
    "            print(f\"Average VALIDATION acc: {val_acc_epoch}\")\n",
    "            print(f\"Average VALIDATION F1: {val_f1_epoch}\")\n",
    "            print()\n",
    "            logs[\"val_losses\"].append(val_loss_iterations)\n",
    "            logs[\"val_metrics\"].append(val_metrics_iterations)\n",
    "    \n",
    "        # CHECKPOINT\n",
    "        torch.save(model, f\"checkpoints/model_epoch_{epoch+1}\")\n",
    "        \n",
    "    return logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- EPOCH 1 ----------------\n",
      "Average TRAIN loss: 0.5427333116531372\n",
      "Average TRAIN acc: 0.810875\n",
      "Average TRAIN F1: 0.7868587813738663\n",
      "\n",
      "Average VALIDATION loss: 0.19164295494556427\n",
      "Average VALIDATION acc: 0.9295\n",
      "Average VALIDATION F1: 0.9345529768992618\n",
      "\n",
      "--------------- EPOCH 2 ----------------\n",
      "Average TRAIN loss: 0.17533010244369507\n",
      "Average TRAIN acc: 0.9329375\n",
      "Average TRAIN F1: 0.9323326799152223\n",
      "\n",
      "Average VALIDATION loss: 0.15689700841903687\n",
      "Average VALIDATION acc: 0.9355\n",
      "Average VALIDATION F1: 0.9365527271447162\n",
      "\n",
      "--------------- EPOCH 3 ----------------\n",
      "Average TRAIN loss: 0.16286173462867737\n",
      "Average TRAIN acc: 0.9343125\n",
      "Average TRAIN F1: 0.9334694767618423\n",
      "\n",
      "Average VALIDATION loss: 0.17313864827156067\n",
      "Average VALIDATION acc: 0.93\n",
      "Average VALIDATION F1: 0.9266658026613909\n",
      "\n"
     ]
    }
   ],
   "source": [
    "history = train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It looks like we start to overfit within epoch 3. For further evaluation on the test set, we will load the checkpoint after epoch 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAABgbElEQVR4nO3dd3wUZf4H8M8mpFCSRUoahID0IkhPgoCAlBygniLYIirqIRYUPTX2cj/AO5UiNhSNHFI8Q5OiBCkBCSCQ0EuQkhASQoBkSSB15/dHzLKbbTO7szuzu5/367VKdp+ZeWZ2duY7T9UIgiCAiIiISMX8lM4AERERkT0MWIiIiEj1GLAQERGR6jFgISIiItVjwEJERESqx4CFiIiIVI8BCxEREakeAxYiIiJSvXpKZ0Auer0e58+fR0hICDQajdLZISIiIhEEQcDVq1cRFRUFPz/r5SheE7CcP38e0dHRSmeDiIiIHJCTk4OWLVta/dxrApaQkBAANTscGhqqcG6IiIhIDJ1Oh+joaMN93BqvCVhqq4FCQ0MZsBAREXkYe8052OiWiIiIVI8BCxEREakeAxYiIiJSPQYsREREpHoMWIiIiEj1GLAQERGR6jFgISIiItVjwEJERESqx4CFiIiIVI8BCxEREakeAxYiIiJSPQYsREREpHoMWFzol0P5+OVQntLZICIi8nheM1uz2pSWV2Hyor0AgMPvjUTDIB5qIiIiR7GExUXKKqst/puIiIikY8BCREREqseAhYiIiFSPAQsRERGpHgMWFxGUzgAREZEXYcBCREREqseAxUU0SmeAiIjIizBgcRFWCREREcmHAQsRERGpHgMWIiIiUj0GLERERKR6DFiIiIhI9RiwEBERkeoxYHEDjYadnImIiJzBgMWOmeuP4cmFe5CZU+TwOgSBnZyJiIicwYDFjl2nLyH1yAVc0JUpnRUiIiKfxYDFjgC/mkNUrXe8lIRVQkRERM5hwGKHv19NsFFZrVc4J0RERL6LAYsd9fxrAhZnSliIiIjIOQxY7LhWUQ0AOHflOvKKr+PrtFPQlVUqnCsiIiLfIjlgSUtLw9ixYxEVFQWNRoOVK1faTP/oo49Co9GYvbp27WpIk5ycbDFNWZnyDV33nr0CAPgk9QTGfZGO/1t3FG+uOGR3OXYMIiIiko/kgKW0tBQ9evTAvHnzRKWfM2cO8vLyDK+cnBw0adIE9913n0m60NBQk3R5eXkIDg6Wmj2Xyi26DgDYeuKiyful5VXYfLwAFVVs50JEROQK9aQukJCQgISEBNHptVottFqt4e+VK1fiypUreOyxx0zSaTQaRERESM2OKkxetBfbsgrx+IA2eHtsF6WzQ0RE5HXc3oZlwYIFuOOOOxATE2PyfklJCWJiYtCyZUuMGTMGGRkZNtdTXl4OnU5n8lLKtqxCAMCS3dmK5YGIiMibuTVgycvLw/r16/HEE0+YvN+pUyckJydj9erVWLJkCYKDgzFgwABkZWVZXdeMGTMMpTdarRbR0dEuyXNgPbZLJiIiUppb78bJyclo3Lgx7r77bpP3Y2Nj8fDDD6NHjx4YOHAgfvzxR3To0AGffvqp1XUlJSWhuLjY8MrJyXFJnt8c3dnsPUEQUHStAnM2ZiH70jXD+xwfjoiIyDUkt2FxlCAI+Pbbb5GYmIjAwECbaf38/NC3b1+bJSxBQUEICgqSO5vmebEQhQgAkpYfxPpD+fj299OG9xmvEBERuYbbSli2bt2KkydPYtKkSXbTCoKAzMxMREZGuiFnttWOdFvXzlOXAADF1zkmCxERkatJLmEpKSnByZMnDX+fPn0amZmZaNKkCVq1aoWkpCTk5uZi4cKFJsstWLAA/fv3R7du3czW+d577yE2Nhbt27eHTqfD3LlzkZmZic8++8yBXZKXv5V6HkslL6V/DTJHRERE8pIcsOzZswdDhgwx/D1t2jQAwMSJE5GcnIy8vDxkZ5v2likuLkZKSgrmzJljcZ1FRUV46qmnkJ+fD61Wi549eyItLQ39+vWTmj3ZlVuZQ+hSaYXodQgcRY6IiMgpkgOW22+/3eYNODk52ew9rVaLa9eumSf+y6xZszBr1iypWXGLSkuDwTH+ICIiciv22bWjwkIJy9XyKgVyQkRE5LsYsNjh6CzNglExDAtkiIiInMOAxY5R3TxzugAiIiJvwoDFjrbNGymdBSIiIp/HgMVVjOqB2EmIiIjIOQxYRIhuUl/pLBAREfk0BiwiRGmlBywsVCEiIpIPAxYRLI1qK4VgJ3w5nn8Vj323GwfPFTu1HSIiIm/FgEUEV8/C/PCCXdh8/CLGztvu2g0RERF5KAYsLiKloe3Fq+WuywgREZEXYMAigrNVQmzQQkRE5BwGLERERKR6DFhEcGTuIHsNbYmIiEg8BiwiDO7Q3KnlGboQERE5hwGLCFNubys6bUVVzezOHN2WiIhIPgxYRAgO8BedtsOb6zF93VEX5oaIiMj3MGBxgflpp0z+ZmkLERGRcxiwuAhjFCIiIvkwYCEiIiLVY8Ai0vND20lKLxjVA7GLMxERkXMYsIg0bURHpbNARETksxiwSNCvTRPRadnQloiISD4MWCSI0gaLTvv2qkOGfzN4ISIicg4DFhfZfPyi4d93fLIVpy6WKJgbIiIiz8aARYL4ds0cWu5aRTVeW35Q5twQERH5DgYsEozr1dLhZcsrq2XMCRERkW9hwCKBn59G6SwQERH5JAYs7qJhsENEROQoBiwSbXhxEFo3bSB5OYYrREREjmPAIlGH8BBs+ecQycuxgIWIiMhxDFjcJCO7CAW6MqWzQURE5JEYsLhRv+m/Ie3ERfsJiYiIyAQDFjd75NvdSmeBiIjI4zBgISIiItWTHLCkpaVh7NixiIqKgkajwcqVK22m37JlCzQajdnr2LFjJulSUlLQpUsXBAUFoUuXLlixYoXUrBEREZGXkhywlJaWokePHpg3b56k5Y4fP468vDzDq3379obP0tPTMWHCBCQmJmL//v1ITEzE+PHjsWvXLqnZIyIiIi9UT+oCCQkJSEhIkLyhsLAwNG7c2OJns2fPxvDhw5GUlAQASEpKwtatWzF79mwsWbJE8raIiIjIu7itDUvPnj0RGRmJYcOGYfPmzSafpaenY8SIESbvjRw5Ejt27LC6vvLycuh0OpMXEREReSeXByyRkZGYP38+UlJSsHz5cnTs2BHDhg1DWlqaIU1+fj7Cw8NNlgsPD0d+fr7V9c6YMQNardbwio6Odtk+EBERkbIkVwlJ1bFjR3Ts2NHwd1xcHHJycvDRRx9h0KBBhvc1dYaCFQTB7D1jSUlJmDZtmuFvnU7n1qBlaKcwbDpW4LbtERER+TJFujXHxsYiKyvL8HdERIRZaUpBQYFZqYuxoKAghIaGmrzc6fOHerl1e0RERL5MkYAlIyMDkZGRhr/j4uKQmppqkmbDhg2Ij493d9ZECw7wd2p5QRDw/JIMvL7ioEw5IiIi8l6Sq4RKSkpw8uRJw9+nT59GZmYmmjRpglatWiEpKQm5ublYuHAhgJoeQK1bt0bXrl1RUVGBRYsWISUlBSkpKYZ1TJ06FYMGDcKHH36Iu+66C6tWrcLGjRuxfft2GXZRnc5duY7V+88rnQ0iIiKPIDlg2bNnD4YMuTFbcW07kokTJyI5ORl5eXnIzs42fF5RUYGXX34Zubm5qF+/Prp27Yq1a9fib3/7myFNfHw8li5dijfffBNvvfUW2rZti2XLlqF///7O7JuqVVbrlc4CERGRx9AIgiAonQk56HQ6aLVaFBcXu609S+vX1jq03JmZo3HqYgmGfrzV4mdERES+Quz9m3MJKcQrokQiIiI3YcBCREREqseAhYiIiFSPAYtCvKPlEBERkXswYCEiIiLVY8BCREREqseARTGsEyIiIhKLAYtC2IaFiIhIPAYsREREpHqSh+Yn56X/eQk/H+A8QkRERGKxhMUJn4zv4dByD3y9E4t3ZdtPSERERAAYsDjlnl4tlc4CERGRT2DAQkRERKrHgIWIiIhUjwELERERqR4DFiIiIlI9BixERESkegxYiIiISPUYsBAREZHqMWAhIiIi1WPAQkRERKrHgIWIiIhUjwELERERqR4DFiIiIlI9Biwya9IwUOksEBEReR0GLE5K++cQjO4eafj7vj6cwZmIiEhuDFic1KppAyR0i7jxhqBcXoiIiLwVAxaZ6QVGLERERHJjwCIzxitERETyY8AiAw00hn8zXiEiIpIfAxYZaG7EK7KXsBRfr8Tes5chsOiGiIh8GAMWmQkyl7H8bc423PtFOtYezJN1vURERJ6EAYvM5C4IyS26DgBYx4CFiIh8GAMWGWjsJ3Eaa4SIiMiXSQ5Y0tLSMHbsWERFRUGj0WDlypU20y9fvhzDhw9H8+bNERoairi4OPz6668maZKTk6HRaMxeZWVlUrOnOFe1NWHAQkREvkxywFJaWooePXpg3rx5otKnpaVh+PDhWLduHfbu3YshQ4Zg7NixyMjIMEkXGhqKvLw8k1dwcLDU7CmOcQUREZH86kldICEhAQkJCaLTz5492+Tv6dOnY9WqVfj555/Rs2dPw/sajQYRERHwdM6WhGw4nI/Nxy/i3Tu7IKievzyZIiIi8nBub8Oi1+tx9epVNGnSxOT9kpISxMTEoGXLlhgzZoxZCUxd5eXl0Ol0Ji+lmHRrdrKM5an/7sWS3dn4b/pZk/fl7n1ERETkSdwesHz88ccoLS3F+PHjDe916tQJycnJWL16NZYsWYLg4GAMGDAAWVlZVtczY8YMaLVawys6Otod2bfiRsRyW7vmsqzxgs60/Q7bsBARkS9za8CyZMkSvPvuu1i2bBnCwsIM78fGxuLhhx9Gjx49MHDgQPz444/o0KEDPv30U6vrSkpKQnFxseGVk5Pjjl2wa2TXcHz/eD/0iblJ6awQERF5DbcFLMuWLcOkSZPw448/4o477rCZ1s/PD3379rVZwhIUFITQ0FCTlxpoNBoM7tAc4aHONRjecOSCTDkiIiLyfG4JWJYsWYJHH30UixcvxujRo+2mFwQBmZmZiIyMdEPunKexMBCLs21Ozl66Vmd9REREvktyL6GSkhKcPHnS8Pfp06eRmZmJJk2aoFWrVkhKSkJubi4WLlwIoCZYeeSRRzBnzhzExsYiPz8fAFC/fn1otVoAwHvvvYfY2Fi0b98eOp0Oc+fORWZmJj777DM59pGIiIg8nOQSlj179qBnz56GLsnTpk1Dz5498fbbbwMA8vLykJ2dbUj/1VdfoaqqCs888wwiIyMNr6lTpxrSFBUV4amnnkLnzp0xYsQI5ObmIi0tDf369XN2/xTDRrJERETykVzCcvvtt9sczTU5Odnk7y1btthd56xZszBr1iypWVGNuLZNAQCdIkJctg0GQERE5MskByxkLjQ4AEffH4XAeq5sEsSIhYiIfBcDFpnUD+SotERERK7C2ZqJiIhI9RiwuIjcbU7YhoWIiHwZAxYXkXvuH8YrRETkyxiwEBERkeoxYCEiIiLVY8DiIWyNfUNEROTtGLAQERGR6jFgcREWiBAREcmHAYuLNA8JknV9jH+IiMiXMWBxkZdHdJR1fSyxISIiX8aAxUVuahiIR+JilM4GERGRV2DA4kIsFSEiIpIHAxYXuq9PS9nWpWf0Q0REPowBiwt1b9kYHcNDZFnXtqxCWdZDRETkiRiwuFhYqLy9hYiIiHwRAxYiIiJSPQYsLuZM25PyqmoZc0JEROS5GLC4mF7v+LI930+VLyNEREQejAGLiwlOjFF7rcKxEpaTBSV4etFeHDmvc3jbREREasKAxcWU6I2cuGAX1h/Kx98//939GyciInIBBiwupkTAkldcBgAor3KiPoqIiEhFGLC4mDNVQkRERFSDAYuLcYBaIiIi5zFgcTEOqU9EROQ8BiwultAtUuksEBEReTwGLC722IDWWDCxj9LZICIi8mgMWFysnr8fhnUOVzobREREHo0Bi5fbe/aK0lkgIiJyGgMWD3altAIv/bgfO09dsprm3i92uDFHRERErsGAxYN9sOYIUvadw/3zdyqdFdWqrNZje1YhrlVUKZ0VIiJyAgMWD3b28jWls6B6szeewMMLduHJhXuUzgoRETlBcsCSlpaGsWPHIioqChqNBitXrrS7zNatW9G7d28EBwfj5ptvxpdffmmWJiUlBV26dEFQUBC6dOmCFStWSM0akZkfdmUDAH4/ab3ajIiI1E9ywFJaWooePXpg3rx5otKfPn0af/vb3zBw4EBkZGTg9ddfx/PPP4+UlBRDmvT0dEyYMAGJiYnYv38/EhMTMX78eOzatUtq9oiIiMgL1ZO6QEJCAhISEkSn//LLL9GqVSvMnj0bANC5c2fs2bMHH330Ee69914AwOzZszF8+HAkJSUBAJKSkrB161bMnj0bS5YskZpFnyFwFF0iIvIRLm/Dkp6ejhEjRpi8N3LkSOzZsweVlZU20+zYYb2HS3l5OXQ6ncmLiIiIvJPLA5b8/HyEh5sOnBYeHo6qqioUFhbaTJOfn291vTNmzIBWqzW8oqOj5c+8B9mXzfFWiIjIe7mll5BGozH5u7Yqw/h9S2nqvmcsKSkJxcXFhldOTo6MOfY893zO8VaIiMh7SW7DIlVERIRZSUlBQQHq1auHpk2b2kxTt9TFWFBQEIKCguTPMBEREamOy0tY4uLikJqaavLehg0b0KdPHwQEBNhMEx8f7+rseZSB/96EI+fZVoeIiHyP5IClpKQEmZmZyMzMBFDTbTkzMxPZ2TXjXSQlJeGRRx4xpJ88eTLOnj2LadOm4ejRo/j222+xYMECvPzyy4Y0U6dOxYYNG/Dhhx/i2LFj+PDDD7Fx40a88MILzu2dl8m5fB3PLt5n9fOKKj2+2vqnG3NERETkHpIDlj179qBnz57o2bMnAGDatGno2bMn3n77bQBAXl6eIXgBgDZt2mDdunXYsmULbr31VnzwwQeYO3euoUszAMTHx2Pp0qX47rvv0L17dyQnJ2PZsmXo37+/s/vnda5VVFv9LHnHacxYf8yNuSEiInIPyW1Ybr/9dpvjfyQnJ5u9N3jwYOzbZ71kAADGjRuHcePGSc2Oz9HbOPaHWV1EREReinMJeRhbAYv1PlVERESejQGLh9FzcFsiIvJBDFg8jM0SFhvj1oixMP0MHvl2N8oqrbeTISIiUgIDFg+jt1HEYitcybl8DduzCm2u++1Vh5F24iIW7TzrYO68X1W1HtUs5iIicjsGLB7G0fkOB/57Mx5esAt7zly2m9ZWTyRfVlWtx20fbsawj7dw4kkiIjdjwOJhqm3dKEXUCO09a3/OIW9qvCvnvuQWXUe+rgxnLl1DeZVexjUTEZE9DFg8jLPVETYDHi/kW3tLROS9GLB4GD8nG9b6WLxCREReggGLhzGOVxyJPWw12iUiIlIrBiwexlb5ikZEiw0xVUJOFuKoihftChGRT2PA4mFsjbUiJtDwxAKW8qpqh3vleODuEhGRBQxYPIyzpR+eViWUX1yGjm/+gmcXZyidFRNsC0RE5F4MWDyM7Soh+2yNlGtYj4rqhJb+UTPz99qDeQrnhIiIlMSAxcM4G0x4WAGL01wVeqkopiMi8gkMWDyMrRuluDYsnhWxOJtdV+2thx1GIiKPx4DFw1x3cth8zoPjODG9sIiIyDUYsHgYW0PC84bqWgL7HBERKYYBiwerWy0hV7sKb2qfwTYsRETegQGLB/vzYonkZTyt7YWasmtcguVpx5GIyNMxYPFgdW+abJ9ijkeEiMg7MGBxk5saBLh8G2sOeN9YJXLWvBzPv4rRc7fht6MXZFwrERG5AwMWN0l5Oh6PxreWbX25RddRUl5l8t71Svs9iMQ0HFVT411nS0iM9+TpRXtx+LwOk77f4+RaiYjI3RiwuMnNzRvh3Tu7yrKu7VmFGDBzk0PL+nLbC11ZpdJZICIiBzFg8UAPL9glKf2Gw/kOb+tKaQWW/ZGNqx56s/fh+IyIyKvUUzoD5HpP/XevpPTGXXafXLgHe85eweZjF/FlYm+Zc+a5OCYLEZF7sYSFbNpz9goA4BcnSmmU5I7WOKXlVfjXmiPY+9exIiIi+TFgIXLS7I0n8M3207j3ix1KZ4WIyGsxYCGv5o6Km5MF0gfwIyIiaRiwuJnSQ7oLIroJqadTM1TbrUml2SIi8loMWNxMVcGAD+DxJpJfZbUeJy5cFfUARCQXBixkRulSoFp6vYCzl68pnQ0iqmPKD/swYlYaFu/OVjor5EMYsLiZRuFowJOeh95ZfRirMs9LWqb4uuvGizH+6jzpOBLJLfVIzfQWC7afVjgn5EsYsLiZSgovPMJ/d56VlH7jkQvo8d4G/GvNEZfkh6XfRKb81FIcSz6BAQuZWZEhrVRDLaavOwoA+IZPfURu4cd4hdzIoYDl888/R5s2bRAcHIzevXtj27ZtVtM++uij0Gg0Zq+uXW/Mq5OcnGwxTVlZmSPZUzWlH0jElBIczdO5PiMeyKRKiMUt5KV2/Fko+hqgpolSyftJDliWLVuGF154AW+88QYyMjIwcOBAJCQkIDvbcuOrOXPmIC8vz/DKyclBkyZNcN9995mkCw0NNUmXl5eH4OBgx/ZKxfgDJyK1yrl8DQ9+vQsJc6w/hBpT+gGMfIvkgOWTTz7BpEmT8MQTT6Bz586YPXs2oqOj8cUXX1hMr9VqERERYXjt2bMHV65cwWOPPWaSTqPRmKSLiIhwbI/IpsPni5XOgseyVqiidENqIrlkS+yVx3Of3ElSwFJRUYG9e/dixIgRJu+PGDECO3aIG5Z8wYIFuOOOOxATE2PyfklJCWJiYtCyZUuMGTMGGRkZNtdTXl4OnU5n8vIICv++92UXKZsBL2Ecu7B6iHwVwxVyJ0kBS2FhIaqrqxEeHm7yfnh4OPLz7U+Ol5eXh/Xr1+OJJ54web9Tp05ITk7G6tWrsWTJEgQHB2PAgAHIysqyuq4ZM2ZAq9UaXtHR0VJ2RTH8gXsuPkySt5Mae/ux24YqFOjKUFmtVzobLufQ6Va3GFAQBFFFg8nJyWjcuDHuvvtuk/djY2Px8MMPo0ePHhg4cCB+/PFHdOjQAZ9++qnVdSUlJaG4uNjwysnJcWRX3M7aYXrvzq6WP5AgSut9bX7UhFVCRKbYrVl5h88Xo9/03/D3z39XOisuV09K4mbNmsHf39+sNKWgoMCs1KUuQRDw7bffIjExEYGBgTbT+vn5oW/fvjZLWIKCghAUFCQ+8yoRqa2P04WlZu+HhTi/L+eLva9XFRGpF4N15a3YlwsAOJTrIc0inCCphCUwMBC9e/dGamqqyfupqamIj4+3uezWrVtx8uRJTJo0ye52BEFAZmYmIiMjpWTPI3z9SB8MaNcUP/4jTumskESm3ZqVyweRWjBc8V5V1Xos3pWNPy+qZzZ6SSUsADBt2jQkJiaiT58+iIuLw/z585GdnY3JkycDqKmqyc3NxcKFC02WW7BgAfr3749u3bqZrfO9995DbGws2rdvD51Oh7lz5yIzMxOfffaZg7ulXu3CGuGHJ2LN3r+tfTMFcmOd2Go+IvJdHDjOey3aeRbv/lwzaviZmaMVzk0NyQHLhAkTcOnSJbz//vvIy8tDt27dsG7dOkOvn7y8PLMxWYqLi5GSkoI5c+ZYXGdRURGeeuop5OfnQ6vVomfPnkhLS0O/fv0c2CXP8+kDPRESHIBlT8ViwvydSmdHtMKScjRr5HnVckQkD7Zh8V57VdijVHLAAgBTpkzBlClTLH6WnJxs9p5Wq8W1a9b798+aNQuzZs1yJCteofZHH1jPc5rcf7b5JP7z63G8NaYLJt3WRunsWOWyUiJWCZGKvPLTfoQEB+CtMV3cul3GK+ROnnOHJLey10bjP78eBwB84KKJBjcduyB9IQsXT46RQnIp0JXhfNF1pbNhJvvSNfy45xwWbD+Nar17z3eO3K08X7rCMWBREXedeJ4wV9DjyXukL+RLv1xyq2q9gH7Tf0P8zE24XlGtdHZMVOqVG3+DJSzkTgxYfNDUpbZHEQZ477dH4BHyKfuyrxj+XVhSrmBOXEvqec2ARXm+9BUwYFGB2h99/QB/t2yvospLR0S08MtlTyeSw4Jtp5XOgiqx0a3yfOnRiQGLinSKCMGj8a3Rs1Vjl25HzE384w3HXZoHd2EbFpKDmu/LSp7iDFjInRiwqIhGo8G7d3bF5MFtXb6tjUcu4Owl8xF3a32+5U+X54GI5OXuAJ3xivJ86StgwKJCrr7mnC4sxRML92Dwf7a4dkMqVlWtx7gvduCNFQcdWt74O/KlCwZQ0wD19RUHsXzfOaWz4jYsqCO18qVTkwELeTVr1V/bTxZiz9kr+GFXtsXPpfClCwYArDlwHot3ZWPaj/uVzorbeEpJgrvPRbYRI3diwKJCarkGvPKTa29In/6WhY9+dW1bGdMi8hsHtqra18IM+RRdq1Q6C0T0F5XcLtzCoZFuyTf8uMd1Rf5lldX4OPUEAOCR+BiEhQS7bFs3CBb+5eyapF8wKqv18NNo4O+hE7GoJaB2J1YJkVr50qnJEhYVqHv994WLY5XRiJyVCpR2ZF+2PlWEVFJyX1mtR+z03zD04y0e24PJB+MVjwnSnD2lPPSUJB/BgEUFbmoYqNi2TxaUSLpx7j59GQVXy5zert5om64saDCtY7/xb1dNKWBP9uVruFRagbOXrsHNo6gTyc5D4jiv5kvfAauEFDTn/ltxsqAE/ds0USwPd3yyFa+O6oSnb7fflXp7ViEeXrBLlu2a9rJx109OvgjBOMjzpQsGAM8pbpCRyfnqe7tv1Y4/C1FaXoWGQbyVKMWXnntYwqKgu25tgZdGdFS8pf2s1BO4JGK48W1ZF2XbpuCmEhY1V7uoOW/kmdw9ZURltYAnFzow75dIgiDgmcX78NwS+9OJkPdjwEKABngs+Q+HFx/60RZ8tVXaQHN6kydW9TyyHs+/itavrcWt729wSUBhvKeeGq54aFthcpEdf15y2boLSyqw9kAeft5/HkXXKly2HU/mSz9HBiwEADhwrtjhZU8VlmLG+mOSljGpUnFbGxb7nv/rSa7oWiVOXCixms5Tgw05OFKF9/2OM3hm8T5UVcs7j5WurBLPLt6HjUcuyLpez2LU++2vfyb/flryQ4QaGV8nWCBpmS8dFgYsJPr2I+cPQ+4Gp/nFZTh10fpUA2Jdq6wy/NvVk0T60gX4ndWHsfZAHtYezJN1vbNTs7DmQB6ecGG1hKepqNLj3Z+PYMb6Y7I0kCdSCwYsKhTXtikAoGN4iMI5cR25n5zu/WKH3e2I4b4GwO5vbyAXZ0rESsqr7CeS4IKON+S6jHvglVdKC7pdfUbuOFmIpxbu8bnvrayyGtuzCl3+EOTt2LRbhbT1A3D0/VEIrOeHtq+vc/n2xN6A5GzTUW0csMhwmcwtui4ilf0ddXVzGuMqKk8tYZHjEFVV65F65AJ6t77JTYMGOsf4HFVTmyt71HaOPfhNTS/Dar2ABY/2VTg3rlWtF/B48h9oF9YIuVeu45fD+ZgYF4P37uqmdNY8FktYVKp+oL/HjoQqhjNVQtUSFja9udhfzmTUFhuHX203Ak+TvOMMnv5hH0bOSlM6K15AvuuEtTXJ3QA9r9h9JSwl5VX4354ctzfa3XX6EraeuIgF20/jl8P5AIDv08+6NQ/ehgELiWbvmnVBV4ZDueaNdwVBwL/WHMHC9DOG9/TGQYeEa+G/1hxBj/c2iCxRkc6Tnp6V4ifDMdp4tKaR7BXOSyQD69WrUr8qSz/Fub9lof/033Bext+cO39mr/50AP/86YBLu19bosQI3t6OAQvJ1m6j//TfMObT7fjzomnvmsycInyz/TTeXnXY8J7epEpIvG+2n0ZJeRW+3CKuB4S1yQ/VwGNLaWQ4jHLtu7vaAUnNr66sUvYeUUr5JPUECq6WY1bqCej1Aq6Uelb34tqG3n+cueLW7arrauMdGLCQ7DKzi0z+1pWZN7Q0KWBx241bWpWQ2dImhUKOZdq0gspTIxaypUBXhu7vbkDCnG1KZ0VWAoAnF+5Bzw9SXb4tvV5AoYjBLMm3sNEtubx41lL9t97JRrdil5FcxWMl+eXSCtz+0WZp67Lgt2MFhn97agmLM6dLbWmeN9e8bT5e8x1nFVgfx8dVBAgu6+l2+LwOR/N0Lll3XZO+/wObj1/EZw/2csv2yDOwhIVQVlktKp2s47Dorde7u47jF/IF209ZbSgs5ear1KSLcvLEdj6V1Xq8ufIgfjmU79DyHhpbykrOYMXeKbT5eM00IN/vOCPbNsnzMWAhl88abGn1rt5mZbUeZy/VHUjO8Y1W1c2wDFVannoT9LxwBVj6Rw4W7czG5EV7lc4K+QhXxPU5l6/hvzvPin7I9DasEiLRxN6YzZJZWM64SscVN+5Hv9uN30/KOMeJCzLpqZMfOnMhrv3eZWt0K3I9BT4yUNlzizPw6IDWhr+dOc56vYAf9+Q4nykL3DlAozcZPmsryir1OF90Ha+O6qR0dtyOJSwku7o3YkvtTUwasDpwVbW3iOVgRcTAcZJz4vgNXC3hyoFzRXh39WEUi+xi7IE1Qk4zPt/UvPu/HStA4oLdDi9v/Fv8+cB5vLb8oBzZIpmU/TVy8Y6ThQrnRBksYSGXsxdcqKmgwVpW1JBFvV5AtSAgwF/e54w75/0OACgtr8J/7ush67rrkrvRrdhzxxVBxtzfstCsURAe7N/KBWsXz9YxcOY4H3RiQlQ10WjUdY0hx7GEhURztBuuJ10snJ0IcvPxAvxx5rK4dUg8LuO/SkevD1JxrULe+XhqnRDZq0XJ4vyTBSUYMHMTFu/KlrSc3KfgyYISfJJ6Aq+vMC+B8KTzvS6TqSNcuh33LKMkV/5OPPgUcwoDFnI5Sz8u0yoht2XFLuMLtvEFUmy11WPf/YH7vkwXtzGJ+73n7BVcLaty+wBYavL6ioPILbpuMVBwLdMv62qZekbolfNGrrp2VSa/QadXoSrH8nW4c952bD1xUfKyavua3IUBC4nmcKNbCRZsP20yhL8rtiHW6LnbkZlT5LL1O15iZb7cwvQzuO3DTci+dM3ZbNklxw3S0QtupZtGjxUEweGZdd1dEmDrWKr1xib6EMmQf7V2w39y4R4cOFeMid863ubIXdQSyDJgIZezdLKb9hKq+felknJ8sOaIyRD+SnthaYbF9+X4/cp5DXh71WGcu3Id7/3s+mOnphuA6AEEJa536tJMdHxrPfKKpc+fo5JrO/1FPWerqaJSx0vp3D1K9phPt6tiqgkGLCSaoxMOGv+05qf9aTYAVe0F/roKxxZQ473HVp4qXT3AjQNyLru+1Eduq/efhyAAS3bXdOs16SWk1jugBcZ51avw3HAHpb4vTzpP7Dl8XufS0maxHApYPv/8c7Rp0wbBwcHo3bs3tm2zPmfGli1boNFozF7Hjh0zSZeSkoIuXbogKCgIXbp0wYoVKxzJGrlQ6pELotLVLbY3vthPX3cMCXO21ZmXR72C6/kDcM1Ts5r32xYp1+Hj+Vcx8N/OTWlwNE+HknLXNDT2drXn7fR1R9H3/zZ69Hg0jgYA3jjmixKleGaDZypAcsCybNkyvPDCC3jjjTeQkZGBgQMHIiEhAdnZtlvtHz9+HHl5eYZX+/btDZ+lp6djwoQJSExMxP79+5GYmIjx48dj165d0veIFDdznWkwWm3nRHe2flT8qI/SJz8MDrD8E5GjSNbh9hg22yy4/qIipTHyxqPiglxrtmcVImHONoyclebUehxmZf/UVC0mxvy0U7hUWoH5aadEL+PSU0ns8ZOh0a1S8YorN+uOTgt1819V7YEByyeffIJJkybhiSeeQOfOnTF79mxER0fjiy++sLlcWFgYIiIiDC9/f3/DZ7Nnz8bw4cORlJSETp06ISkpCcOGDcPs2bMl7xAp76rR0/APu87imcX7bKZ39mfQ518bnVwDUFGlR8rec8iv8wS6/1wxDpwrsplH3XXHnv7//ctxh5ZTepZn4ydWqRdLqff5tQfzAFivjnTXk6byl2of5eIDv+PPQny/44xqGpWqSd0jUqX3sDYsFRUV2Lt3L0aMGGHy/ogRI7Bjxw6by/bs2RORkZEYNmwYNm82LSJOT083W+fIkSNtrrO8vBw6nc7kReqx61TNSLNvrDjk8m2Jry6wfrf8YsufeOl/+3G1zHxdd877HQu2n7a43OnCUuy2MO6KmAtgyr5zdtPI5cc9OVgu0/ZM2kW4+EKvxoIMKbs8KfkPXBBZDVOtFxSfI0bNt23Hq4Sse/DrXXhn9WGkn5JxGg83MP6erB2X308W4oWlGbhSWiHLNj2uhKWwsBDV1dUIDw83eT88PBz5+ZZnQY2MjMT8+fORkpKC5cuXo2PHjhg2bBjS0m4U8ebn50taJwDMmDEDWq3W8IqOjpayK+Riu07bHjyt1CjIEHsDMJ5pV+775JYTBQ4tt8JKECAINWN1uKKho/G+l5ZXmQxUZ+m4FF2rwCs/HcC0H/fLckM0vj66+xLmcPzi4N3O2f377VgB3lopLmgfOTsN3d/dIOvAgHLM+6QoOcZhEXEMzl12rEOBU1w82vND3+zCyszzmL7uqEPrNasSUkEbFoeG5q9bfysIgtU63Y4dO6Jjx46Gv+Pi4pCTk4OPPvoIgwYNcmidAJCUlIRp06YZ/tbpdAxaVMReF7gJ83ca/SXg8y0nUT/A32p6AC6badeZuYz8/Cyfo+euXMeg/2xGvzZN8OM/4pzJnk0PfrML+41a71u6yZRW3AhS5L7oSD10kquQpCW3SK8XnC7yd2b5gqvlotKd/Guk4QPnihF7c1PR67eVM2eOtysLz9w6DosDZ5EgCFiZmYtbWjRGu7BGjm7YzkYcWy0g7XyUo3cnYL8tojtIKmFp1qwZ/P39zUo+CgoKzEpIbImNjUVWVpbh74iICMnrDAoKQmhoqMmL1ENK99o/zlzBv385jvd+PiJ5O3LUPc9KPYGM7CKHlq1nJWBZkZELANhtp6TJWfsldjV0NgA4mqfD4fM3ql+Vfgo33vpGC73Y9HoBCXO24dNNJx1bv4Xdk7rHUks5Fu0869Dop3Jw9Ns8KWJah21ZN/aprLIaU37YixUZ4qspHW5zK+L41z2Pfz6QhxeX7ccdn2x1cKuOOZavw4Nf78Tes+JHs3ZXtanHtWEJDAxE7969kZqaavJ+amoq4uPjRa8nIyMDkZGRhr/j4uLM1rlhwwZJ6yR1kTLIkNg6fkvkeAqc6+DNDLBewmJ8AXz4m10Oj5pqcd0KjWxaVa1HwpxtmLf5xvFydaNbKZ5YuMfsvfPF13H8wlVZt1O3mk/uBptrDuSJGv30m22nsHR3tio67b650v5UCcZVY8fyr2LdwXy8uGy/K7PlsEwHH2AksfDFJS7YjR1/XsK9X9huE2rMXb2EXN1eTQzJVULTpk1DYmIi+vTpg7i4OMyfPx/Z2dmYPHkygJqqmtzcXCxcuBBATQ+g1q1bo2vXrqioqMCiRYuQkpKClJQUwzqnTp2KQYMG4cMPP8Rdd92FVatWYePGjdi+fbtMu+kdptzeFp9v+VPpbIji+vrOmvUr9SOq3aq/lTuwcba2nyzEmgPncU+vltbXJwg4lKtDu7BGqB9ou2rMZr7sHI7lGbn4fPNJfDOxD7pGaU0+s3fjU6IO216AI/fXX1GlR2C9G89xtds33szAf2/GHZ3DTPJgK5+uCChyi67jX2tr2iakvjjIarrnluxD+/AQF+TAlJhTw6Gu4DIcPD8R21XBvRgAcFFk9aGU/Frb/Wq9gOuV1WgUZDkMUMkhMSG5W/OECRMwe/ZsvP/++7j11luRlpaGdevWISYmBgCQl5dnMiZLRUUFXn75ZXTv3h0DBw7E9u3bsXbtWtxzzz2GNPHx8Vi6dCm+++47dO/eHcnJyVi2bBn69+8vwy56h0/G98ArozopnQ3RpLQod2ZgJ6WrVf2tlbDUuaKUV+ltDvO+ev95jJ23HeO+tP9k5cwuv7XyEPKKy/DcEvMpBxxZrxqeuuSyKjMXHd5cj5S9N6oprO3exqPiG2m7YsyWEgu92SzZf64YP+11fW+03acvO9XA3HhZayVWjpZkiTn6ttb8y6E8HDxX7MB2nfveBUHA0TwdrlfYbijv6Ok15tPt6PbOr6KDJDVwqNHtlClTMGXKFIufJScnm/z9yiuv4JVXXrG7znHjxmHcuHGOZMcn3NJCaz+Rh/LEngyLd53FLS20Vp/e6ubqfNF1xM3YZHV9tTeV2vYhBVfLcCi3GLd3CLNa7SSWpQu9XF0UXR2vuHOU0qlLMwEAL/1PWjWFvUPg6j1QS8i4MjPXZimiNesO5uGf/9uPeQ/2wpBOYXh3tXrmEgOAyYtqxpE6M3O0W7f7y6F8PP3DPnSJDMW6qQNNPjOZi83BE6B2ipTNxwowvq95hxU1VDXWxbmEVO7314Zi+ZR4txTryklKEOLMD0OpB/zPNv+JyYv22ShhMf171ylpjW+H/GcLHk/eg+V/Nd41Xbf1nXZ3AGdva1KeirdlXcQzi/fh8l/jRhzKLcZ/d551InfyUXNBkqvyJrVEw7gxthRTftiH0opqPJb8BwDg+3SZv3MxjW5lOIZlldW4+7PfMXP9MfuJYT9btQ8xR/LMj6uc37m1a4YaT3kGLCrXonF99Gp1k9LZcClnfhhKV0mIaXRr6W97arsibz4mbXwYdx8OORucJi7YjbUH8vCvNTW9xcZ8KqYNm2t3WMz3psQoqZZmO1eavRuws0/stXu57mAe7vrsd9GTaspVUlB7Xlrz8/7zyMwpwpdbxbUzlOtbs/dwaK+UUs3BeF0MWMglKqr0eGOF/Z4DzlL6xyam0a3cpKzaansAC2uxe8OxkMAVu3neRlsfKe1BBEFA9iXXzxRtt0rIBWXrLptLxgNuXlN+2If9OUV4XeT1Rcw5Iybo+8bKaNe16jZKd2WPOOMtOfv911385/3n8a81R8yuHWqYRNKhNixE9iz9I0d0Wkd+BrW/JaVLWPythPyubAxss1uz0b//OHMZk5L/wKTbbnbZdgUXDM0g11c685dj+Gqr+Mn+bOXFmSzVXuhre4K1ad7Qas8MRygdtNeye4N28n5Xdz91IhseOxI4OFtq9fP+83an3pCSrV2nLqG/hMEEpah7XGsb5AfVU195BgMWDxYWEoRerW7CL4etT2Hg7ZTuJWS90a1CGTPa7D/+uxe6sirM2njCanIxA37Z3py0/RTVY0OmQ+dssCKW3fz+tdMbjlzAP/67FzFNG2DrP4c4vL0txwsMAxO6kvQB8lxRlGT9IyfboptuRoZzzjg7lnrhOeP79DMmAUtZZTVmrj8Gjcb5khxrv+FyGceOkgsDFg82pnsUXhje3uMDlo9Trd9Q7VF6llVrjW7rXgPU8hRcV84V56pMXLFfliaTlHP71XoBF3RliGpc325aOe/Bq/efBwCcdbKa6tHv/jD521XBsdzTKMgdzohdn/IVGY4xPvfqVsecu3Ld0FZmcIfmTm1HrdcmS9RX5kMkQrVeQHlVteI/Nqu9hJxY5+bjNxraSr0ZGae39QQq13GrrNabTGTpqu1Yct7BOVKe+WEf4mduwq8iAn1DlZATPbM89YYpmYt3tO5xFjMgHCC2DYvyDz+OsjdOiz2etNcMWDyYWnoHKOF/e8+hx3sbcF2G2YedYe2iWXcQLSlP6o/VeYI2J2DZH9lYc+C8+Scmm3XuDrLuYB7GfrodZwpLLay7Rr/pv6HrO7+ixEbQYpI/p3JkKn7mJmw7WSh5udoSyflprqkyqruPrp7rRS33WXuNMuWuMhIdsIhIs/v0ZfT9v41YeyDP4fy49Hu29fDh7K9KLSeQCAxYyGOVVeqR5WQbDGeJLWGR85pwvqgMr6YcxLOLzevJrxk9bTl7AZ3ywz4czC3GqykH7KY9cK7IuY05yNocTdcq7AdQzo6AakhjJ1HtjVwNJS2SephJbZ/kkiYs1vNQu70Z645i7m9ZdtPVZVwy8fP+8ygsqcAzi/fZzM/4L9NRKWGeNLF+3m/+8GHM1qG1e/7Zm97C9scGBVcdn/NNLgxYPJwaLoJKUroYV+zAcbasP5iHbVniSwqKr1eK+kyuRom1vTFs3jxEnonuOl//8+txN23JnLt/k+4YREwMOfbb5jDxdbKm0QA5l6/hq7RT+CT1BKoltMAvq6xG57d/kZy/3Wcu4zcJ0zKIZa+Rrq3SKWe/frFTKkxfJ25APFdiwOLh5Owe6YmULs20Og6LyMtIZbUeT/9g+6nOfN3iyDVugtJBoSO++/2M3TS1X52UmcUd4fIqIQ+pGhZzGJ74vk6DYpu9hDQor6o2SmstsfmWbfWOM16nJa4oYbEkM+fG/EUZ2VdcNuePZ5w9NRiweLCaWWI1aOjE7L7kHGs3I7H3eHtz+jgTK8h9o7SVF6e7VorcUTl3SQMN1h7IQ4c311ttuyAmW7tPX8arPx1A0bUK0dvWlVXi6UV78csh53v4uW5ofmnp5Tjf9tuYZLBudmrasNjfqNR82esO764bfGHJjQDl3JXr6Pt/G12yHU96HvHtx3PyeJ7ydCkrkVcYW40Sq6oFvLnyICqrzNclCAJmbbTeJsAe/V/T1jcMqmf12zF+/8+LJRj/ZbrD23NGbZsFe20XbB3yR77dDaBmEMP/3NfDaqNb42L9uRuzsP5QPtYfyseZmaOdKsVSyy/AfqNbmbdXZ33iy1ds259ThFZNG1j9XK4SR7kaITubH7WcP2IwYPECnnTCyU3Kb9X4icXV6o7Au+fsFYfW46oSlnxdGRbtzLb42faThRYbMYrNyiPf7sb2k4XY/qq4wdHeXX0Yl0rFlU5ckVCKYZeI+4WUe4q18VUs3cgvynguKj3asyvZ2rNTF0tN0xolfmfVIWRfvoYFE/u6raRRqaHrnf32PanKlwELebSjFmYydSdrv3Wx1wBXlhA5eqEu0Em/mRqPxbL9r67GqzKt93ywl7W3Vx2y+P6fdW5SzpB91F0rKxTzPThzz3BZlZDE9M5OwmeP2dD81ytNtmn8W6qd8Xn/uSLJ27W3H2oLEK+KnKLAG7ANixew1vDTF3y0wfFRcuVg7dIl9qL2719s92b55XA+NtQZ4Ezs5VLsOBV1WeuSbetJbNL3e2wOIGdvG3Ut/OuGoxZiAktHf4XOP+F6xki3cquo1pts01J+q/WCxQDEmUumhLa9buH09Brqir9sYsDiBez9+N74W2e8ObqzezLjY6zdbH7YZbm6pa7kHWfspnnqv3ulZMnAkevn+aLrePl/+x3a3turDptuXyMuD44GVu5QG6iUlNsfoNDR3fh000mnQg7V3HBsHIDcous4fuGqrJur0guumb/IDrkOt9juxM6yd4w8qR0gAxYPVnuz9LMz4MaTg25Gu7BG7siSz1Hip/71Nuu9GIxLdhwJBN5aaV4VI/aCtu6gaU+b6mrLS9YdFVepeCWvWNxAWMv+yMb+nCK76axVPdTeMKzt5idOzKUFSD8H95y5jDkbs2Tvzm3ra5yyyLGg2/iBQIBg8ndwndmErbctkZdcVUJXJZRIyu3spRtVq8a7I+Y8VxIDFg/z/eP9zN4Tc2NS4knEFxzOtd4NU04Fuhs317JK6zca44tPpV76DclWo1Z7l+m60yRYmtTyteUH0e2dX3E8X96nbUdkXxY3CeGrKQdFpbP2ExPXVsb86C7elY2k5QfsPolLvX+O+zIdszaewJLd9koB5Rvp9miePN+38b6O7xttWiVkJb/Sr32ODQ3rSVfY309eMvzbeHfu+ux392dGAgYsHsZ4Zs7aE03MD+WCyKdJkmbuppNu2c6S3Tmi0tU+hR7KLUa5jcDG+vIW3pNw39p9WtxMy6Yj8nrSpd4x/9uTg7Ssi1Y/t3SIX19xEEt252DTsRsjq1oa68XRNjCWGjA71SvNxpVIjmoHQTA9TvUDTMefkquGxV4PrldSDuCTDcqNpCw31VQpisCAxQtYe4L44qFe+P21oQCA88WOzWxL6lAt8qqiF4BFu7Ix5tPtKHBgZExbmxGThf/8Kn34bm8JV6wFXicLSvDPnw6g6Jr1KRVsuVp+Y7nbP9pi9rla7je24k5Hb4rmDcBN3zHpJWRlIxYb3doKrgTBbn7d9aDiKsYBpCe1YWG3Zi/gbyXsTLgl0vBvN7XvIhdJO2H96dyYIAj47vfTDm/H1mlyvsg1Qa/YApak5fYnYZSdhN+Ntf3IFXHcxN7QLQU9jrY7sFcyI+eTt1ztPozXInbguLoysq/Yrvr0gWul8T560v6yhMULiCpS96SzksxkirwpOd9D1sLIt3/9P2HONsmrE3ejEhexiK0W8zYFunIs2nnW6gzUM9YrPykdYGdGYRnWL8D89DQuKblr3u9ms4YLML0+Hs3T4e+f78DjyXtsbMe7rpXeUoIJsITFo9X+eMUELCxh8Q3OPslabsMifp11k84WMcT/qYvOjSOhFlIad9YdVM/WTbI2IDmW7/wgiQu23yh9k/uSYLz7V0oroK0fYOjB6HCVUJ3lbB2n04WluHPe7/j3vd2t5utjEeM2ScmrXi/Y7aWpRsa76Ekj3bKExUd421MDWVbtgshUyhrFtrUxdqpQvtFr5fZVmu2J8Iy5+ra15bi4akGx7H1VUkdQrQ3YjpzXoecHqYY5luQk5vR6JcV61eHGoxdk2QYA/HIoH93f24BNx2rWKUdPTLe1P3cwSLFWyucuDFg8WG0Q4ifiW2QJi28QDP9xcHkry4oNhDKyixzfuA9T40Pu2oOWZ7C2p7a7dO0UDXKxVBJg7wa/KjNXciApAMgX0aty8qK9KCmvwuPJe1CgK0PuFefaeGVkX3FqeSlMS1jEL5d2Qt7vVCoGLF6gT0wTAICtkkk1XhBJfs5+zxZL4gQgYU6acyv2AZ5WMyB3qWtt8ODK0lzj81vMHEGLdmajslpafnKvXMMvdabDsKff9N8wa6P4AQAPWRi/6e+f75C0TbEsBXXGx9HSeEnWBPgre5IzYPEC797ZFc8PbYcNLw62msaT6inJcU63YbGy+IkL3tHOxJU8bXDGRTuz8fSivaajycowDoucl5q6JQHGwdClUnHd9qVe+3RumEzwoW92uXwbrlDPWpdUN2HA4gW09QMwbURHm8Pvq22GUXKNuoNrSVoWLIlTiiOD/Mlh/aF8pyfPq3WjhMV1jM9P8YMpqosgCCYDJ5p+5r48GBv8n83YeMR++54AhYsRGbB4MCknN29EvsGp4ngrJwlPHXGcuZb3eH+DfBmRqErNDdzqnJOO5FRt5V4vLMtUOgtmx/HspWt4YqH1rt61AuqxhIUcFKkNFp12eJdwF+aE1MLZe48cXWd9lwaXSys87uFArvy6IzCoWzKw/pBjDYOVVLdLu6vV/V6q9QJmOjh2j7tmmLaGAYsHWjCxDyb0icYTA28WvUyP6MauyxCphjNtlQRYDnjY/kmcjUcvoNcHqfjfXtcMcOeqJjJyNZI1VAm58HSpu+rp69QxaJ5afLzhOH49nI+yOhORGkvZdw7lVY5VQU6YvxOVMs/yLQUHjvNAwzqHY1hnlpiQOUFwPMBgXCIPX+3aLabXjlQXdDca1q7KzMVHIgZ+q+u8D038+ulfcxyN693Sahpnp9jILy5DdJMGTq3DUQ6VsHz++edo06YNgoOD0bt3b2zbZn3I7uXLl2P48OFo3rw5QkNDERcXh19//dUkTXJyMjQajdmrrMx3TjRX84UZccmxgdtqWXvSZhyjDq4KKGWrEtLUzGtUOw4LAKuNS8XK1924BzgSrPiqn/aeM/n72+2n8dh3u22WvHgCyQHLsmXL8MILL+CNN95ARkYGBg4ciISEBGRnZ1tMn5aWhuHDh2PdunXYu3cvhgwZgrFjxyIjI8MkXWhoKPLy8kxewcHi22iQbYxXfIMzN59DuZbbr7DkRR3OOTkwmRjOftV3ffa7yd893tuANQfc22aDzL2/5gg2H7+IFRm5Tq9LyXuJ5CqhTz75BJMmTcITTzwBAJg9ezZ+/fVXfPHFF5gxY4ZZ+tmzZ5v8PX36dKxatQo///wzevbsaXhfo9EgIiJCanZIJJaw+A654wtXDPdP3sfaODTPLs6w+D65X4WDbVeMKTnekKQSloqKCuzduxcjRowweX/EiBHYsUPcKH16vR5Xr15FkyZNTN4vKSlBTEwMWrZsiTFjxpiVwNRVXl4OnU5n8iLrGK6QoziGj3czHT3WcbzGqJNxgBFYz8/pElMlv2dJAUthYSGqq6sRHm7a4DM8PBz5+eKGMv74449RWlqK8ePHG97r1KkTkpOTsXr1aixZsgTBwcEYMGAAsrKsz/Q6Y8YMaLVawys6OlrKrnitt8d0AQD8e5z1GUuJpGDA4t2M2y7xm/ZuAQqPVOssh3oJ1S0SEgRBVDHRkiVL8O6772LVqlUICwszvB8bG4vY2FjD3wMGDECvXr3w6aefYu7cuRbXlZSUhGnTphn+1ul0DFoAPH5bG4zvG41GQaZfracNG07qYdxTg7yPnI1uSd0C/DVOB6Ue04alWbNm8Pf3NytNKSgoMCt1qWvZsmWYNGkS/ve//+GOO+6wmdbPzw99+/a1WcISFBSEoKAg8Zn3IXWDFfItLBAhJTBeUSfjcVOC6vk7vT5XdF8XS1L5UGBgIHr37o3U1FST91NTUxEfH291uSVLluDRRx/F4sWLMXr0aLvbEQQBmZmZiIyMlJI9IiKSqLRCnsn+WIqrTqaTOQrIyL7i1Po8poQFAKZNm4bExET06dMHcXFxmD9/PrKzszF58mQANVU1ubm5WLhwIYCaYOWRRx7BnDlzEBsbayidqV+/PrRaLQDgvffeQ2xsLNq3bw+dToe5c+ciMzMTn332mVz7SeQznB37gnzLg1/vwoF3RyA0OMCp9TBeUaerRteDyYv2Ob0+Jb9myQHLhAkTcOnSJbz//vvIy8tDt27dsG7dOsTExAAA8vLyTMZk+eqrr1BVVYVnnnkGzzzzjOH9iRMnIjk5GQBQVFSEp556Cvn5+dBqtejZsyfS0tLQr18/J3ePyPcwYCGp9p69giEdwzgNgxeqkHkofSXPEIcaO0yZMgVTpkyx+FltEFJry5Ytdtc3a9YszJo1y5GsEBEZ/H6yEAPaNVM6Gx6ndlI7Z4bcYQGLOsk9YaGSMa1n93EiIhO+Xiz/wZojSmfBI9Xe05wqYfH1k0+l5B73Ua7JMh3BgMWHPD+0Hcb2iMJzQ9sBAJo1ClQ4RyQ3Xy/R9/X9d1TtaMYbjlxweB1H8zh4pxo5M7+YJUr+xtj/1YdMG9ERAFBeVY02zRpiQLtm6D/9N8PnD8e2wgP9WmH03O1KZZHIKUo+/XkyQRBQoCtzaq6ZxbsszydHypK9SkjWtUnDgMUHBdXzxz29zKcf/9fdtyiQGyJS2pVrlehn9PBC3kPukaqVbJjNKiEi8hqsEnLM+2sOK50FkpFxcyJvmruUAQsReY2sghK0fm2t0tnwOGWV8nZ9JWUZB+7yl7DIujpJGLAQERF5KW8qdWTAQkRE5KVKyuWZeqEWS1iIiIhI9TgOC6nSK6M62k3z0vAO6B1zkxtyQ0RESmMJCynm3+O6m/zfmLa+/cnQnhvWHrd3aC57voiISH04DgspZnyfaNzZIwrBAf4Or8OL2nQREZENHIeFFFU3WOkYHgIAGNYpXInsEBERmWEJC5lZ+/xtuFZZjdBg+1VCRETkO5QsUWcJC5mp5+9nNVgZ19t8SH8iIvINbHRLHuOj+3oonQUiIlIM27CQyg1s3wwbpw22+Jk3jaRIRETWsYSFVG94l3C0C2sk2/ru6Bwm27qIiMg92IaFPJojIx860426e0utw8sSEZHjWMJCqtesUZCs69MYz38uwqujOhn+XXf20eYh8uaNiIgsU3JofnZrJps+e7AX9mVfwaiuEYrmwzi+qdabfnZTgwBcvFru3gwREZFbsYSFbBrdPRJvjekCPz/rJSKOFBEar+0/47rj9Iy/Gf5u0bi+zWXrjrRYVe26iH9c75bow7mSyIamDQOVzgKR27BKiHzO33u1AAC0D2uE+/pEm1QRaTTA80PbmaSP1AYb/l33BxNlJ8DpEhnqcD4/uq8HFj8Z6/Dy5P38bQTzRN6GAQt5lKVP1dzAb2lR0/jVkfN3SMcwbHhxEFY/e5vZZ34aDV64o4Ph7x4ttQgJvlF7WbcNy4S+0Xg0vrXVbb05prMDObyBNyQiohpsw0IeJfbmptj2yhCEh9aUeljq7nz3rVHoFXMT3l512PDeymcG4ExhKeLbNQUAdPhrzqK6/P00JlVQ0U0aQGNUiVQ3YBnSKQxje0QheccZi+u7qYFzRfbOxCv/HNkR//n1uFPbJ/WJ0gbjfHEZAE7+Sb6FJSzkcaKbNEBgvZrTZ8wtNe1cjEtBZt/fE4/EtTZZ5tboxri7ZwuEhQTDlohQ08/r/j70Rm8kP9YXjYLM4+43R3fGM0PaYtrwDujsRJUQIL1Hk7GRXTmBpFT1PKBEq5lRzzRPyC+pT+bbw5XOgsdhCQs5zc9Pg0m3tcHYHpGYlXoCD/WPMUsj5sb930n98M220/i/v3cz+6xrixtBh3EJy+0dbwxAF1jPDxVVNV2IQoLr4YmBN9vd5hO3tUGVXsDFq+VYezDPbnqpfHkU4IHtm2FbVqHk5fz9NKjSq/vAGQexfk4EtJ4mLCQIBSrokVc/wB/XK6uVzoZTGjtZ8qsUlrCQVwgLCcaMe7qjWwvzgd26Rdkf7G1g++b4/vF+aHlTAwDAbe2aAQASY2MQFhKMba8MQcZbw82qhGqtemaA4d8aiLuJ3N2zBd69syvu6GI+8u53j/W1uMxrCZ0svm9JtSDg1ujGiNJaLlXq16aJqPX8NDnO8O8hHZuL3r6S2jRr6NByH9xtHrCqTezNN763ev7yBSytmzaQbV2uYKmBe90SUWf9c2RHu2mUbEchlieXvAUHWA8NlDz2DFjIpb5+pA/G9W4pqrSjru8f74ddrw9D7M01bV6imzTATQ0DoddbTm9c9WMtqDH28X09DMHVnT1a4B+Db8arozrhnp4tkPbPIRjS0TyIeS2hEyYPbit6HxoG1sPyp+OR9soQi5/Pe6CnqPUYjwz83WP9RG/fE43tHiX7OkffEinr+gZ3aI7kx/ri99eGOtwoe4mF3mebX74d/VrXBEMD2zdzKo+uYGlKjajG8gYsYkqs/ibz9ymXacM7YNNLg/Hn9L/h2AejlM6Ow8Z0j8Kk29rgy4d7K50VE6wSIpca3iUcw7s41o7D309jaNhr7OWRHfDisv24v2+01WWt1Sj8Y/DNSEroDEEQTIr1/f00SEqw35tIWz/A5O/4tk2x489LZumm//0WFF+vRHSTmidmPyslPmEyP50qYcvLt2P3mcto27wR7v1ih+F9e+PpWOOKGpbPHuqFta+tlXWdtdWRjj5J92ltPr6PRqPBV4m9seZgHsbcEomeH6Q6lUe5pCcNxa5TlzGqWwQ+2nBC6ezgg7u6Yfm+XJP3XhreAR+nKps3Pw1wc/PaTgjOn8gn/y8B7d5Y7/R6pPLXaPDWmC4WP2OVEJEEf+/ZEjuThmHGPbeYffbB3d0wsH0z3HWrlaf0v35sjjakrVty89zQ9mZppv/9FjzYvxWevt20JOb9u7oiPDQIPVs1BnDjCdqRG7tck0faKnn4+pE+WPPcbVgwsY/NdbRu1hDj+0QjqJ7p5WRifGs81L+V5Dx5QpMQ41KAmKaOVX35azS42ajarFNETa+5mxoGIjE2Bo0b3AiOHx/Qxup6VkyJd2j7UkRq6+Puni0QHOCP7x7ri68Sbzx5O9Mo3RG9Y25CQwsN7Z8b1t7kmMltcAfpVbE/PNHfqW3W8/dD3F8lzGIZd34wVvd6ZOy+3i1N/u5tY7BMTn5IJFGENtjihTIxNgb/ndTf4gUNcP7Hpq9TdFO3S/eh90biQSs36UfiWmPX63fgx3/EYeHj/QwX/d9eGow1z5mPR2PLZw/1svrZ4if6I75tU6ttQWp7Vf3+2lCb67mjcxi6tdCiQaD5sQwPDcKMe27BxmmDDe8F1glYggP88X9/Nw8q7RHT/ii+re2L+O+vDTV7z9ERi9+28KRpnMP/+3s3k8BDLD8/DVrcdCNYrTsmkUajQVJCJzwzpC2im1gPanu2usmt1Q9DOoZhpNFUHca/NUs3xRaN66OJhNGA7RVYfTuxpm1Z6ouDzD5LdrK69I7O1kuDH+hnvUS3Vt1r0oB2zlfrzZpwq6T0I7pYnkbF1lhV/x7XHdONfqv3/DWwpyV1Rxp3JwYs5FOc/bHVxiu7Xh+G314ajOYhQfjbLTUXiB7RjS12sa4rwN8Pgzo0NwQCwQH+iL7JvLHl14/0wQP9WuFzC0FFUD3rs13XD/TH4idjkRhr3lurNu/pSUPtluzUXnwt3UB+eCIWD/RrZRKwBfrfuJz8Y7D9NkuWHswD/f0sbm/pU7H4ZHwPADXTJSx+MtbQKLvWjHtuQbcWofjX3d0s7tt0CyVytbq1CMXqZweYvf/m6M7o29q8YbRxEBcWEoxl/4gzS1PX2udvQ3KdhtzGp2PdgA8A/jG4Lf45spPVYvjaRrrBAf6GkjtbOoQ3wleJvWVpKHv7X42/nx/aDnPuvxWfPtDTZJLSWoM6NMObo8UP3mhtfKZa2r9KUWqrW43dGt3Y6rk378Ge+Pe93c3ef+K2Nlj4eD88EheDp283X3bzy7cj9cVB6BHdWETu5RehDTYpvXx3rOWqGmMNA82vD2EhQVYb7Gs0GpOSGVvtspQsYWEbFvIJLRrXR27RdYzq5lhjvQf6RWPL8YuGKQXCQ4NR+yz24b3dMbB9c5OnTqm0DQLw5cO9EODvh0nf7wEAhAbXM1R7FZaYdyXd9NJgbDl+Ee+vOQLgRpfTjhG2L/gNg+qZPBXf3zcaS//IsZr+lpbmPbwsDRZofMOdMrid2efGRnWNwCcTeqDL27+avF/PX2Ox5KxDeAhib26K0d0jDcHavAd7YlXmeYzsGoGS8kq0CwvBA/1ulG69eEcHzNp4wmQdda2fOhCHz+twb68WJtv99tE+6B3TBNr6ATiWrzNbrlsL07F97DW8bRfWCF2jtDiaZ7quMJEzjde9Sax7fiCW7zuHiUZPzcufjseWExfx2Hd/WF3PhhdrSsQGd2iOkvIq9PnXRlHbt2TBxL64XFqB5iFBsFVpqNFoTM6NnybHoVNkKLq986tZ2oRuEbi9Y3O8M7YLOkWE4oGvd1pdr7W2Q22bm5+bQE1DUgB4JeWAyftv/lWCNqhDc7PvB7De2218n5b4cc85k/ek1o59cFdXzN92Ck8NvBkf/nIcJeVVAIC4m5vi2aGWf0OPDmiDA+eKsTwj1+LnGk3Nb7G0orrO+xp8/Ugfq21ixDQev79vNJo1FHfOuoJDJSyff/452rRpg+DgYPTu3Rvbtm2zmX7r1q3o3bs3goODcfPNN+PLL780S5OSkoIuXbogKCgIXbp0wYoVKxzJGpFFqdMGYdNLg23Wzdoy457u+P3VoQgNNq8jDwkOwAP9Wkkq9rZkVLdIDOscjolxMRjYvhn6GD3ZN2sUhJSn47Hu+YGG925u3giPxMWgTbOG6BHdGNtfHYoj7480efpv27zmYjuiSzjuvjXK4hPw+3d1w5InY/HenV0N740zqtNuEFgPR98fhd2vD0PPVo2tFi2HhwajVZMGaNu8oekgghNuxdRhN9r63NE5HHMeuBUNAuthdHfTAPKVkR0tVgjVrs+4ZKlxg0BMjG+NCG0w2oWZByOP3dYat7TQmnRDNy7qDgsJQufIUIzr3dIsSPL38zM0sDYuOQJq2prUTd+kYSC6GwV29/eNRqsmDTChTzTWPn+bofSmc2Qo/jmyo6G06PXRnXFH53B8+6jtdkIBRl2nb41ujC5RoXhzTBeTUgaNRoMhHcMMvYxqffdoX/SOuQlv/O1GKUdwgD+aNQqyWHVWl7WqEH8/DZrbCbiahwTh+aHtMaJLBOLbNsWzQ9qhT+smVksiWzVpAI1Gg8cGtEFc26Z4dVQntDcKjp8ceKMtj7Ub7L29WuKfIzuaDAUwbbjpVB/WWCq1seaDu7th2ytD8HDsjSA5REQJq7HEuNbY9spQJMa1xv53RuDW6MYYfUskljwVa1Kd9N2jfdG4QQDmPVjTq3Dmvd2xfEo85j7QEzFNG5hUKbdu2gDfTOyDZo0CzUpaLD0MJP31+zA+npbSdQwPwcx7u6OVkl3vBYmWLl0qBAQECF9//bVw5MgRYerUqULDhg2Fs2fPWkx/6tQpoUGDBsLUqVOFI0eOCF9//bUQEBAg/PTTT4Y0O3bsEPz9/YXp06cLR48eFaZPny7Uq1dP2Llzp+h8FRcXCwCE4uJiqbtE5NGqqvVCdbXe6ud5Rddtfl6rsqpa+HLLSWHv2ctO5aWyqtriZ99uPyXc/1W6UFpeafL+H6cvCfM2ZQmnLpYIen1NPvfnXBF2nbok5BdfFy4UX3c4P3VVVlULh3OLhWvlVUKFhXxOX3tESFywS6gyOl56vV546OudwqTk3cJvR/OFotIKq+vPuqATrpVXyZbfWqcvlghtXlsjDJj5m0neLCm+XiGsO3BeGPjhJuGhr+1fQ6ur9cKpiyXCi0szhJhX1wgxr64RDp4rEv4suGr4PqQYMPM3IebVNcLKjHM2l+/wxjoh5tU1wuvLDxi2u+nYBYtpX/1pvzAp+Q+z9T0wP12IeXWNMOGrHVb3raTM9Hwrq6wSjuYVC1MW7RWW/ZFttkz2pVJh8n/3CDGvrhHu/yrd5LPtWReFYR9vEf44fcnk/e93nBYmJe8WrleYf/dZF3TC4l1nhXNXrglnCkuEeZuyhF8P5VnMry32vov0PwuFD34+bMiDXq8Xci6XCjGvrhEG/XuT4b3aY137qr02FF+vEO74eIvwwc+HTdb77fZTQqc31wsZ2Vck51kssfdvjSBIq9Tv378/evXqhS+++MLwXufOnXH33XdjxowZZulfffVVrF69GkePHjW8N3nyZOzfvx/p6ekAgAkTJkCn02H9+htFVaNGjcJNN92EJUuWiMqXTqeDVqtFcXExQkOdG4qdiEhtLpdWICS4HgL8xRWM6/UCNBppvXgqqvQoKa9yqrSw+FoljuXr0K9NE5vbPlNYil8O5yMxNgZXy6qQVXAVt7VrJrnX0bWKKgTX8zeZf8xZer2AU4UlaNu8kdt7QcntUkk5QoIDDNVyH6w5ggXbTwOoaWw7vo/9xsTVesGlk8CKvX9LqhKqqKjA3r17MWLECJP3R4wYgR07dlhcJj093Sz9yJEjsWfPHlRWVtpMY22dAFBeXg6dTmfyIiLyVk0aBooOVoCaXkhSb7aB9fycrtrUNghA/5ub2t1262YNMXlwWzQMqocIbTAGtm/uUHDQILCerMEKUHPs2oWZV/15oqaNgkzaEL01pgtSXxyEmffcgnG9WtpY8ga1zFgvKWApLCxEdXU1wsNNu36Fh4cjPz/f4jL5+fkW01dVVaGwsNBmGmvrBIAZM2ZAq9UaXtHR9qNEIiIiX9c+PAT392sle6Dnag41uq0bdQp1Rg0Vk77u+1LXmZSUhOLiYsMrJ8d6LwciIiLybJKaNDdr1gz+/v5mJR8FBQVmJSS1IiIiLKavV68emjZtajONtXUCQFBQEIKClOteRURERO4jqYQlMDAQvXv3Rmqq6fwWqampiI+3PDx0XFycWfoNGzagT58+CAgIsJnG2jqJiIjIt0geOG7atGlITExEnz59EBcXh/nz5yM7OxuTJ08GUFNVk5ubi4ULFwKo6RE0b948TJs2DU8++STS09OxYMECk94/U6dOxaBBg/Dhhx/irrvuwqpVq7Bx40Zs375dpt0kIiIiTyY5YJkwYQIuXbqE999/H3l5eejWrRvWrVuHmJiaYcDz8vKQnZ1tSN+mTRusW7cOL774Ij777DNERUVh7ty5uPfeew1p4uPjsXTpUrz55pt466230LZtWyxbtgz9+zs3cRQRERF5B8njsKgVx2EhIiLyPC4Zh4WIiIhICQxYiIiISPUYsBAREZHqMWAhIiIi1WPAQkRERKrHgIWIiIhUjwELERERqZ7kgePUqnY4GZ1Op3BOiIiISKza+7a9YeG8JmC5evUqACA6OlrhnBAREZFUV69ehVartfq514x0q9frcf78eYSEhECj0ci2Xp1Oh+joaOTk5HAEXTt4rKTh8RKPx0o8HivxeKzEc+WxEgQBV69eRVRUFPz8rLdU8ZoSFj8/P7Rs2dJl6w8NDeUJLRKPlTQ8XuLxWInHYyUej5V4rjpWtkpWarHRLREREakeAxYiIiJSPQYsdgQFBeGdd95BUFCQ0llRPR4raXi8xOOxEo/HSjweK/HUcKy8ptEtEREReS+WsBAREZHqMWAhIiIi1WPAQkRERKrHgIWIiIhUjwGLHZ9//jnatGmD4OBg9O7dG9u2bVM6S2717rvvQqPRmLwiIiIMnwuCgHfffRdRUVGoX78+br/9dhw+fNhkHeXl5XjuuefQrFkzNGzYEHfeeSfOnTvn7l2RXVpaGsaOHYuoqChoNBqsXLnS5HO5js2VK1eQmJgIrVYLrVaLxMREFBUVuXjv5GfveD366KNm51psbKxJGl84XjNmzEDfvn0REhKCsLAw3H333Th+/LhJGp5bNcQcK55XN3zxxRfo3r27YfC3uLg4rF+/3vC56s8rgaxaunSpEBAQIHz99dfCkSNHhKlTpwoNGzYUzp49q3TW3Oadd94RunbtKuTl5RleBQUFhs9nzpwphISECCkpKcLBgweFCRMmCJGRkYJOpzOkmTx5stCiRQshNTVV2LdvnzBkyBChR48eQlVVlRK7JJt169YJb7zxhpCSkiIAEFasWGHyuVzHZtSoUUK3bt2EHTt2CDt27BC6desmjBkzxl27KRt7x2vixInCqFGjTM61S5cumaTxheM1cuRI4bvvvhMOHTokZGZmCqNHjxZatWollJSUGNLw3Koh5ljxvLph9erVwtq1a4Xjx48Lx48fF15//XUhICBAOHTokCAI6j+vGLDY0K9fP2Hy5Mkm73Xq1El47bXXFMqR+73zzjtCjx49LH6m1+uFiIgIYebMmYb3ysrKBK1WK3z55ZeCIAhCUVGREBAQICxdutSQJjc3V/Dz8xN++eUXl+bdneregOU6NkeOHBEACDt37jSkSU9PFwAIx44dc/FeuY61gOWuu+6yuoyvHq+CggIBgLB161ZBEHhu2VL3WAkCzyt7brrpJuGbb77xiPOKVUJWVFRUYO/evRgxYoTJ+yNGjMCOHTsUypUysrKyEBUVhTZt2uD+++/HqVOnAACnT59Gfn6+yTEKCgrC4MGDDcdo7969qKysNEkTFRWFbt26efVxlOvYpKenQ6vVon///oY0sbGx0Gq1Xnn8tmzZgrCwMHTo0AFPPvkkCgoKDJ/56vEqLi4GADRp0gQAzy1b6h6rWjyvzFVXV2Pp0qUoLS1FXFycR5xXDFisKCwsRHV1NcLDw03eDw8PR35+vkK5cr/+/ftj4cKF+PXXX/H1118jPz8f8fHxuHTpkuE42DpG+fn5CAwMxE033WQ1jTeS69jk5+cjLCzMbP1hYWFed/wSEhLwww8/YNOmTfj444/xxx9/YOjQoSgvLwfgm8dLEARMmzYNt912G7p16waA55Y1lo4VwPOqroMHD6JRo0YICgrC5MmTsWLFCnTp0sUjziuvma3ZVTQajcnfgiCYvefNEhISDP++5ZZbEBcXh7Zt2+L77783NFxz5Bj5ynGU49hYSu+Nx2/ChAmGf3fr1g19+vRBTEwM1q5di3vuucfqct58vJ599lkcOHAA27dvN/uM55Ypa8eK55Wpjh07IjMzE0VFRUhJScHEiROxdetWw+dqPq9YwmJFs2bN4O/vbxYRFhQUmEWgvqRhw4a45ZZbkJWVZegtZOsYRUREoKKiAleuXLGaxhvJdWwiIiJw4cIFs/VfvHjRq48fAERGRiImJgZZWVkAfO94Pffcc1i9ejU2b96Mli1bGt7nuWXO2rGyxNfPq8DAQLRr1w59+vTBjBkz0KNHD8yZM8cjzisGLFYEBgaid+/eSE1NNXk/NTUV8fHxCuVKeeXl5Th69CgiIyPRpk0bREREmByjiooKbN261XCMevfujYCAAJM0eXl5OHTokFcfR7mOTVxcHIqLi7F7925Dml27dqG4uNirjx8AXLp0CTk5OYiMjATgO8dLEAQ8++yzWL58OTZt2oQ2bdqYfM5z6wZ7x8oSXz2vrBEEAeXl5Z5xXjnVZNfL1XZrXrBggXDkyBHhhRdeEBo2bCicOXNG6ay5zUsvvSRs2bJFOHXqlLBz505hzJgxQkhIiOEYzJw5U9BqtcLy5cuFgwcPCg888IDFbnAtW7YUNm7cKOzbt08YOnSoV3Rrvnr1qpCRkSFkZGQIAIRPPvlEyMjIMHR7l+vYjBo1SujevbuQnp4upKenC7fccovHdacUBNvH6+rVq8JLL70k7NixQzh9+rSwefNmIS4uTmjRooXPHa+nn35a0Gq1wpYtW0y64l67ds2QhudWDXvHiueVqaSkJCEtLU04ffq0cODAAeH1118X/Pz8hA0bNgiCoP7zigGLHZ999pkQExMjBAYGCr169TLpLucLavvhBwQECFFRUcI999wjHD582PC5Xq8X3nnnHSEiIkIICgoSBg0aJBw8eNBkHdevXxeeffZZoUmTJkL9+vWFMWPGCNnZ2e7eFdlt3rxZAGD2mjhxoiAI8h2bS5cuCQ899JAQEhIihISECA899JBw5coVN+2lfGwdr2vXrgkjRowQmjdvLgQEBAitWrUSJk6caHYsfOF4WTpGAITvvvvOkIbnVg17x4rnlanHH3/ccD9r3ry5MGzYMEOwIgjqP680giAIzpXREBEREbkW27AQERGR6jFgISIiItVjwEJERESqx4CFiIiIVI8BCxEREakeAxYiIiJSPQYsREREpHoMWIiIiEj1GLAQERGR6jFgISIiItVjwEJERESqx4CFiIiIVO//AWIDM57tlvAoAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "losses_by_iteration = []\n",
    "for epoch in history[\"train_losses\"]:\n",
    "    for iteration in epoch:\n",
    "        losses_by_iteration.append(iteration.data.item())\n",
    "\n",
    "pd.Series(losses_by_iteration).plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation & Error Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "trained_model = torch.load(\"checkpoints/model_epoch_2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='mps', index=0)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trained_model.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_on_test():\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        test_loss = 0\n",
    "        for batch in test_data_loader:\n",
    "            inputs, labels = batch\n",
    "            inputs = tokenizer(inputs, return_tensors='pt', padding=True, truncation=True)\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "            outputs = trained_model(**inputs)\n",
    "            loss = loss_fn(outputs.logits, labels)\n",
    "\n",
    "            test_loss += loss * inputs.input_ids.size()[0]\n",
    "            acc, f1 = compute_metrics(outputs.logits.cpu(), labels.cpu())\n",
    "\n",
    "        test_loss /= len(test_data_loader.dataset)\n",
    "        print(\"Loss on TEST: \", test_loss)\n",
    "        print(\"Acc on TEST: \", acc)\n",
    "        print(\"F1 on TEST: \", f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss on TEST:  tensor(0.1681, device='mps:0')\n",
      "Acc on TEST:  0.9375\n",
      "F1 on TEST:  0.9444444444444444\n"
     ]
    }
   ],
   "source": [
    "eval_on_test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**93,75%** accuracy on unseen data. Let's get a better understanding of the errors the model makes on the different categories - this way we can check if the class imbalance of the dataset had a negative impact."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(trained_model, \"../models/model\")\n",
    "torch.save(tokenizer, \"../models/tokenizer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"../models/label_dict\", 'wb') as file:\n",
    "    pickle.dump(label_dict, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('ive been too deep down in the swamps swimming in muddy waters tortured by fear feeling lonely and lost',\n",
       "  'i am feeling a little apprehensive about the whole thing',\n",
       "  'i lift different now because it hurt so bad the day it happened that i can t get it out of my mind and i feel myself being a bit timid',\n",
       "  'im pretty sure and its been about a week and a half so although im feeling kind of betrayed and disillusioned by men at the moment everythings okay',\n",
       "  'i asked her what she meant by shes gonna feel jealous having loada of girls over me and then she said maybee i do like you a bitt',\n",
       "  'i was so uncomfortable and feeling weird feelings but wasn t sure if they were contractions since i never really felt contractions with jared until they jacked me up with pitocin',\n",
       "  'im not sure if im more at peace with our situation or if im just not feeling as bitter about it but in the past five months something has changed within me',\n",
       "  'i feel a bit calm now',\n",
       "  'i see all my friends posting pics and status updates of where they are going or what they are doing and i feel a bit jealous knowing it s not something i can get out and enjoy',\n",
       "  'i am feeling overwhelmed by trying to do it all that i think on the women before me',\n",
       "  'i dont know if i feel thrilled at finally getting to go camping again with people i like and know first time where thats happened',\n",
       "  'i feel like we may be coming to the point in the tv series where the show is incredibly popular but sadly the writers are coming to the end of their story lines and soon there will be nothing left to keep the plot a float',\n",
       "  'im sad if some people are unhappy about the flag for religious reasons but i know many religious people who do not feel it goes against their faith and they are very supportive',\n",
       "  'i carried my phone in my pocket and didn t feel the pull to get lost in it',\n",
       "  'i feel this strange sort of liberation',\n",
       "  'i feel inside this life is like a game sometimes then you came around me the walls just dissapeared nothing to surround me keep me from my fears im unprotected see how ive opened up youve made me trust coz ive never felt like this before im naked around you does it show'),\n",
       " tensor([0, 4, 4, 0, 3, 5, 3, 1, 3, 5, 1, 1, 2, 0, 5, 0])]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(iter(test_data_loader)) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hugging",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}