{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"5\"" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "XTREME has 183 configurations\n" ] } ], "source": [ "from datasets import get_dataset_config_names\n", "\n", "xtreme_subsets = get_dataset_config_names(\"xtreme\")\n", "print(f\"XTREME has {len(xtreme_subsets)} configurations\")" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['PAN-X.af', 'PAN-X.ar', 'PAN-X.bg']" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "panx_subsets = [s for s in xtreme_subsets if s.startswith(\"PAN\")]\n", "panx_subsets[:3]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "from datasets import load_dataset\n", "\n", "dataset = load_dataset(\"xtreme\", name=\"PAN-X.te\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "from collections import defaultdict\n", "from datasets import DatasetDict \n", "\n", "langs = [\"hi\", \"te\", \"ta\", \"en\"]\n", "fracs = [0.5709, 0.0777, 0.6360, 0.1067]\n", "fracs = [frac / sum(fracs) for frac in fracs]\n", "# return a DatasetDict if a key does not exist\n", "panx_ch = defaultdict(DatasetDict)\n", "\n", "for lang, frac in zip(langs, fracs):\n", " # load multilingual corpus \n", " ds = load_dataset(\"xtreme\", name=f\"PAN-X.{lang}\")\n", " # shuffle and downsample each split according to spoken proportion\n", " for split in ds:\n", " panx_ch[lang][split] = (\n", " ds[split]\n", " .shuffle(seed=42)\n", " .select(range(int(frac * ds[split].num_rows)))\n", " )" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
hitetaen
Number of training examples20515568561533
\n", "
" ], "text/plain": [ " hi te ta en\n", "Number of training examples 2051 55 6856 1533" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd \n", "\n", "pd.DataFrame({lang: [panx_ch[lang][\"train\"].num_rows] for lang in langs}, \n", " index=[\"Number of training examples\"])" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tokens: ['प्रेम', 'चोपड़ा', '-', 'गिरिधारीलाल']\n", "ner_tags: [1, 2, 0, 0]\n", "langs: ['hi', 'hi', 'hi', 'hi']\n" ] } ], "source": [ "element = panx_ch[\"hi\"][\"train\"][0]\n", "for key, value in element.items():\n", " print(f\"{key}: {value}\")" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tokens: Sequence(feature=Value(dtype='string', id=None), length=-1, id=None)\n", "ner_tags: Sequence(feature=ClassLabel(names=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC'], id=None), length=-1, id=None)\n", "langs: Sequence(feature=Value(dtype='string', id=None), length=-1, id=None)\n" ] } ], "source": [ "for key, value in panx_ch[\"hi\"][\"train\"].features.items():\n", " print(f\"{key}: {value}\")" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ClassLabel(names=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC'], id=None)\n" ] } ], "source": [ "tags = panx_ch[\"hi\"][\"train\"].features[\"ner_tags\"].feature\n", "print(tags)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def create_tag_names(batch):\n", " return {\"ner_tags_str\": [tags.int2str(idx) for idx in batch['ner_tags']]}\n", "\n", "panx_hi = panx_ch[\"hi\"].map(create_tag_names)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0123
Tokensप्रेमचोपड़ा-गिरिधारीलाल
TagsB-PERI-PEROO
\n", "
" ], "text/plain": [ " 0 1 2 3\n", "Tokens प्रेम चोपड़ा - गिरिधारीलाल\n", "Tags B-PER I-PER O O" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "hi_example = panx_hi[\"train\"][0]\n", "pd.DataFrame([hi_example[\"tokens\"], hi_example[\"ner_tags_str\"]], ['Tokens', 'Tags'])" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
PERORGLOC
train945753790
validation177140185
test158151176
\n", "
" ], "text/plain": [ " PER ORG LOC\n", "train 945 753 790\n", "validation 177 140 185\n", "test 158 151 176" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from collections import Counter \n", "\n", "split2freqs = defaultdict(Counter)\n", "for split, dataset in panx_hi.items():\n", " for row in dataset[\"ner_tags_str\"]:\n", " for tag in row:\n", " if tag.startswith(\"B\"):\n", " tag_type = tag.split(\"-\")[1]\n", " split2freqs[split][tag_type] += 1 \n", "\n", "pd.DataFrame.from_dict(split2freqs, orient=\"index\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Multilingual transformers" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "from transformers import AutoTokenizer \n", "\n", "bert_model_name = \"bert-base-cased\"\n", "xlmr_model_name = \"xlm-roberta-base\"\n", "bert_tokenizer = AutoTokenizer.from_pretrained(bert_model_name)\n", "xlmr_tokenizer = AutoTokenizer.from_pretrained(xlmr_model_name)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "text = \"Jack Sparrow loves New York!\"\n", "bert_tokens = bert_tokenizer(text).tokens()\n", "xlmr_tokens = xlmr_tokenizer(text).tokens()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['[CLS]', 'Jack', 'Spa', '##rrow', 'loves', 'New', 'York', '!', '[SEP]']" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bert_tokens" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['', '▁Jack', '▁Spar', 'row', '▁love', 's', '▁New', '▁York', '!', '']" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "xlmr_tokens" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "' Jack Sparrow loves New York!'" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# whitespace is preserved by using xlmr which uses the SentencePiece tokenizer\n", "\"\".join(xlmr_tokens).replace(u\"\\u2581\", \" \")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Transformers for Named Entity Recognition" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "import torch.nn as nn \n", "from transformers import XLMRobertaConfig\n", "from transformers.modeling_outputs import TokenClassifierOutput\n", "from transformers.models.roberta.modeling_roberta import RobertaModel, RobertaPreTrainedModel" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "class XLMRobertaForTokenClassification(RobertaPreTrainedModel):\n", " config_class = XLMRobertaConfig\n", "\n", " def __init__(self, config):\n", " super(XLMRobertaForTokenClassification, self).__init__(config)\n", " self.num_labels = config.num_labels\n", " # load model body\n", " self.roberta = RobertaModel(config, add_pooling_layer=False)\n", " # set up token classification head \n", " self.dropout = nn.Dropout(config.hidden_dropout_prob)\n", " self.classifier = nn.Linear(config.hidden_size, config.num_labels)\n", " # load and initialize weights \n", " self.init_weights()\n", "\n", " def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, labels=None, **kwargs):\n", " # use model body to get encoder representations\n", " outputs = self.roberta(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, **kwargs)\n", " # apply classifier to encoder representation\n", " sequence_output = self.dropout(outputs[0])\n", " logits = self.classifier(sequence_output)\n", " # calculate losses \n", " loss = None \n", " if labels is not None:\n", " loss_fct = nn.CrossEntropyLoss()\n", " loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n", " # return model output object \n", " return TokenClassifierOutput(\n", " loss=loss,\n", " logits=logits,\n", " hidden_states=outputs.hidden_states,\n", " attentions=outputs.attentions\n", " )\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Loading a Custom Model" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "index2tag = {idx: tag for idx, tag in enumerate(tags.names)}\n", "tag2index = {tag: idx for idx, tag in enumerate(tags.names)}" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "ClassLabel(names=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC'], id=None)" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tags" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "from transformers import AutoConfig\n", "\n", "xlmr_config = AutoConfig.from_pretrained(xlmr_model_name, num_labels=tags.num_classes, id2label=index2tag, label2id=tag2index)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] } ], "source": [ "import torch \n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "xlmr_model = (XLMRobertaForTokenClassification.from_pretrained(xlmr_model_name, config=xlmr_config).to(device))" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0123456789
Tokens<s>▁Jack▁Sparrow▁loves▁New▁York!</s>
Input IDs02176337456155555161723565753382
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7 8 9\n", "Tokens ▁Jack ▁Spar row ▁love s ▁New ▁York ! \n", "Input IDs 0 21763 37456 15555 5161 7 2356 5753 38 2" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# test on a small sequence of known entities \n", "input_ids = xlmr_tokenizer.encode(text, return_tensors=\"pt\")\n", "pd.DataFrame([xlmr_tokens, input_ids[0].numpy()], index=[\"Tokens\", \"Input IDs\"])" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of tokens in sequence: 10\n", "Shape of outputs: torch.Size([1, 10, 7])\n" ] } ], "source": [ "outputs = xlmr_model(input_ids.to(device)).logits \n", "predictions = torch.argmax(outputs, dim=-1)\n", "print(f\"Number of tokens in sequence: {len(xlmr_tokens)}\")\n", "print(F\"Shape of outputs: {outputs.shape}\")" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0123456789
Tokens<s>▁Jack▁Sparrow▁loves▁New▁York!</s>
TagsI-ORGI-ORGI-ORGI-ORGI-ORGI-ORGI-ORGI-ORGI-ORGI-ORG
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7 8 9\n", "Tokens ▁Jack ▁Spar row ▁love s ▁New ▁York ! \n", "Tags I-ORG I-ORG I-ORG I-ORG I-ORG I-ORG I-ORG I-ORG I-ORG I-ORG" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "preds = [tags.names[p] for p in predictions[0].cpu().numpy()]\n", "pd.DataFrame([xlmr_tokens, preds], index=[\"Tokens\", \"Tags\"])" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "def tag_text(text, tags, model, tokenizer):\n", " # get tokens with special characters\n", " tokens = tokenizer(text).tokens()\n", " # encode the sequence into IDs\n", " input_ids = xlmr_tokenizer(text, return_tensors=\"pt\").input_ids.to(device)\n", " # get predictions as a distribution over 7 possible classes \n", " outputs = model(input_ids)[0]\n", " # take argmax to get most likely class per token \n", " predictions = torch.argmax(outputs, dim=2)\n", " # convert to dataframe\n", " preds = [tags.names[p] for p in predictions[0].cpu().numpy()]\n", " return pd.DataFrame([tokens, preds], index=[\"Tokens\", \"Tags\"])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Tokenizing Texts for NER" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "words, labels = hi_example[\"tokens\"], hi_example[\"ner_tags\"]" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
01234567
Tokens<s>▁प्रेम▁चोपड़ा▁-▁गिरिधारीलाल</s>
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7\n", "Tokens ▁प्रेम ▁चोपड़ा ▁- ▁गिरि धारी लाल " ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenized_input = xlmr_tokenizer(hi_example[\"tokens\"], is_split_into_words=True)\n", "tokens = xlmr_tokenizer.convert_ids_to_tokens(tokenized_input[\"input_ids\"])\n", "pd.DataFrame([tokens], index=[\"Tokens\"])" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
01234567
Tokens<s>▁प्रेम▁चोपड़ा▁-▁गिरिधारीलाल</s>
Word IDsNone012333None
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7\n", "Tokens ▁प्रेम ▁चोपड़ा ▁- ▁गिरि धारी लाल \n", "Word IDs None 0 1 2 3 3 3 None" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# need to mask the subwords after the first subword\n", "word_ids = tokenized_input.word_ids() \n", "pd.DataFrame([tokens, word_ids], index=[\"Tokens\", \"Word IDs\"])" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
01234567
Tokens<s>▁प्रेम▁चोपड़ा▁-▁गिरिधारीलाल</s>
Word IDsNone012333None
Label IDs-1001200-100-100-100
LabelsB-PERI-PEROONoneNoneNoneNone
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7\n", "Tokens ▁प्रेम ▁चोपड़ा ▁- ▁गिरि धारी लाल \n", "Word IDs None 0 1 2 3 3 3 None\n", "Label IDs -100 1 2 0 0 -100 -100 -100\n", "Labels B-PER I-PER O O None None None None" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "previous_word_idx = None \n", "label_ids = []\n", "\n", "for word_idx in word_ids:\n", " if word_idx is None or word_idx == previous_word_idx:\n", " label_ids.append(-100)\n", " elif word_idx != previous_word_idx:\n", " label_ids.append(labels[word_idx])\n", " previous_word_idx = word_idx\n", "\n", "labels = [index2tag[l] if l != -100 else \"IGN\" for l in labels]\n", "index = [\"Tokens\", \"Word IDs\", \"Label IDs\", \"Labels\"]\n", "\n", "pd.DataFrame([tokens, word_ids, label_ids, labels], index=index)" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [], "source": [ "def tokenize_and_align_labels(examples):\n", " tokenized_inputs = xlmr_tokenizer(examples[\"tokens\"], truncation=True, is_split_into_words=True)\n", " labels = [] \n", " for idx, label in enumerate(examples[\"ner_tags\"]):\n", " word_ids = tokenized_inputs.word_ids(batch_index=idx)\n", " previous_word_idx = None \n", " label_ids = [] \n", " for word_idx in word_ids:\n", " if word_idx is None or word_idx == previous_word_idx:\n", " label_ids.append(-100)\n", " elif word_idx != previous_word_idx:\n", " label_ids.append(label[word_idx])\n", " previous_word_idx = word_idx\n", " labels.append(label_ids)\n", "\n", " tokenized_inputs[\"labels\"] = labels \n", " return tokenized_inputs" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "cab75f81ac8e410f8b2f18a734f39187", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/410 [00:00 \n", " \n", " \n", " [258/258 00:25, Epoch 3/3]\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
10.8404000.4126990.718690
20.3513000.3379110.772507
30.2341000.2847810.824338

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Adding files tracked by Git LFS: ['tokenizer.json']. This may take a bit of time if the files are large.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "1fcdd1559a68434eaada8b69e921f467", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Upload file pytorch_model.bin: 0%| | 1.00/1.03G [00:00 main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "To https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-hi\n", " 09586e2..5ce7b8e main -> main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "text/plain": [ "'https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-hi/commit/09586e26dd7362ff4a1437703e88e2e4edd41162'" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ "trainer.train()\n", "trainer.push_to_hub(commit_message=\"Training completed!\")" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "

\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
012345678910111213
Tokens<s>▁तेजी▁बच्चन▁से▁अमिताभ▁तथा▁अजिताभ▁दो▁पुत्र▁हुए▁।</s>
TagsOB-PERI-PEROB-PEROB-PERI-PERI-PEROOOOO
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7 8 9 \\\n", "Tokens ▁तेजी ▁बच्चन ▁से ▁अमिताभ ▁तथा ▁अज िता भ ▁दो \n", "Tags O B-PER I-PER O B-PER O B-PER I-PER I-PER O \n", "\n", " 10 11 12 13 \n", "Tokens ▁पुत्र ▁हुए ▁। \n", "Tags O O O O " ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "text_hi = \"तेजी बच्चन से अमिताभ तथा अजिताभ दो पुत्र हुए ।\"\n", "tag_text(text_hi, tags, trainer.model, xlmr_tokenizer)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Error Analysis" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "tags": [] }, "outputs": [], "source": [ "from torch.nn.functional import cross_entropy\n", "\n", "def forward_pass_with_label(batch):\n", " # convert dict of lists of list of dicts suitable for data collator\n", " features = [dict(zip(batch, t)) for t in zip(*batch.values())]\n", " # pad inputs and labels and put all tensors on device\n", " batch = data_collator(features)\n", " input_ids = batch[\"input_ids\"].to(device)\n", " attention_mask = batch[\"attention_mask\"].to(device)\n", " labels = batch[\"labels\"].to(device)\n", " with torch.no_grad():\n", " # pass data through model\n", " output = trainer.model(input_ids, attention_mask)\n", " # logit.size [batch_size, sequence_length, classes]\n", " # predict class with largest logit value on classes axis \n", " predicted_label = torch.argmax(output.logits, axis=-1).cpu().numpy() \n", " \n", " # calculate loss per token after flattening batch dimension with view \n", " loss = cross_entropy(output.logits.view(-1, 7), labels.view(-1), reduction=\"none\") \n", " # unflatten batch dimension and convert to numpy array \n", " loss = loss.view(len(input_ids), -1).cpu().numpy() \n", " \n", " return {'loss': loss, 'predicted_label': predicted_label}" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "tags": [] }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f97950abdb1640e19ab6d23070c6390d", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/410 [00:00\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
input_idsattention_masklabelslosspredicted_labelinput_tokens
0[0, 2218, 14136, 5988, 67691, 460, 2][1, 1, 1, 1, 1, 1, 1][IGN, B-ORG, IGN, I-ORG, I-ORG, I-ORG, IGN][0.0, 0.031384714, 0.0, 0.03725087, 0.03313421...[I-LOC, B-ORG, I-ORG, I-ORG, I-ORG, I-ORG, I-LOC][<s>, ▁स, जन, ▁घर, ▁जाना, ▁है, </s>]
1[0, 93019, 7475, 976, 156711, 41612, 3558, 967...[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1][IGN, O, IGN, IGN, IGN, B-PER, IGN, IGN, IGN, ...[0.0, 0.002951078, 0.0, 0.0, 0.0, 0.010400972,...[O, O, O, O, O, B-PER, B-PER, I-PER, B-PER, I-...[<s>, ▁पुनर्, प्र, े, षित, ▁फ़, ि, रो, ज़, ▁शा...
2[0, 11026, 3849, 8389, 1471, 871, 76302, 659, ...[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1][IGN, B-LOC, IGN, IGN, IGN, I-LOC, I-LOC, IGN,...[0.0, 0.01155906, 0.0, 0.0, 0.0, 0.01743069, 0...[I-LOC, B-LOC, I-LOC, I-LOC, I-LOC, I-LOC, I-L...[<s>, ▁ला, ह, ौ, ल, ▁और, ▁स्प, ी, ति, ▁जिला, <...
3[0, 20571, 3282, 6, 150685, 20, 3813, 1187, 11...[1, 1, 1, 1, 1, 1, 1, 1, 1, 1][IGN, B-PER, IGN, I-PER, IGN, O, O, IGN, IGN, ...[0.0, 0.010598798, 0.0, 0.012605397, 0.0, 0.00...[O, B-PER, I-PER, I-PER, I-PER, O, O, O, O, O][<s>, ▁अस, ित, ▁, सेन, ▁-, ▁था, ने, दार, </s>]
4[0, 9163, 2629, 76183, 1472, 6, 4, 46005, 1187...[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1][IGN, B-LOC, IGN, IGN, IGN, I-LOC, IGN, I-LOC,...[0.0, 0.071242474, 0.0, 0.0, 0.0, 0.5964037, 0...[I-LOC, B-LOC, I-LOC, I-LOC, I-LOC, I-LOC, I-L...[<s>, ▁मे, म्, फि, स, ▁, ,, ▁टे, ने, सी, </s>]
\n", "" ], "text/plain": [ " input_ids \\\n", "0 [0, 2218, 14136, 5988, 67691, 460, 2] \n", "1 [0, 93019, 7475, 976, 156711, 41612, 3558, 967... \n", "2 [0, 11026, 3849, 8389, 1471, 871, 76302, 659, ... \n", "3 [0, 20571, 3282, 6, 150685, 20, 3813, 1187, 11... \n", "4 [0, 9163, 2629, 76183, 1472, 6, 4, 46005, 1187... \n", "\n", " attention_mask \\\n", "0 [1, 1, 1, 1, 1, 1, 1] \n", "1 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] \n", "2 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] \n", "3 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] \n", "4 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] \n", "\n", " labels \\\n", "0 [IGN, B-ORG, IGN, I-ORG, I-ORG, I-ORG, IGN] \n", "1 [IGN, O, IGN, IGN, IGN, B-PER, IGN, IGN, IGN, ... \n", "2 [IGN, B-LOC, IGN, IGN, IGN, I-LOC, I-LOC, IGN,... \n", "3 [IGN, B-PER, IGN, I-PER, IGN, O, O, IGN, IGN, ... \n", "4 [IGN, B-LOC, IGN, IGN, IGN, I-LOC, IGN, I-LOC,... \n", "\n", " loss \\\n", "0 [0.0, 0.031384714, 0.0, 0.03725087, 0.03313421... \n", "1 [0.0, 0.002951078, 0.0, 0.0, 0.0, 0.010400972,... \n", "2 [0.0, 0.01155906, 0.0, 0.0, 0.0, 0.01743069, 0... \n", "3 [0.0, 0.010598798, 0.0, 0.012605397, 0.0, 0.00... \n", "4 [0.0, 0.071242474, 0.0, 0.0, 0.0, 0.5964037, 0... \n", "\n", " predicted_label \\\n", "0 [I-LOC, B-ORG, I-ORG, I-ORG, I-ORG, I-ORG, I-LOC] \n", "1 [O, O, O, O, O, B-PER, B-PER, I-PER, B-PER, I-... \n", "2 [I-LOC, B-LOC, I-LOC, I-LOC, I-LOC, I-LOC, I-L... \n", "3 [O, B-PER, I-PER, I-PER, I-PER, O, O, O, O, O] \n", "4 [I-LOC, B-LOC, I-LOC, I-LOC, I-LOC, I-LOC, I-L... \n", "\n", " input_tokens \n", "0 [, ▁स, जन, ▁घर, ▁जाना, ▁है, ] \n", "1 [, ▁पुनर्, प्र, े, षित, ▁फ़, ि, रो, ज़, ▁शा... \n", "2 [, ▁ला, ह, ौ, ल, ▁और, ▁स्प, ी, ति, ▁जिला, <... \n", "3 [, ▁अस, ित, ▁, सेन, ▁-, ▁था, ने, दार, ] \n", "4 [, ▁मे, म्, फि, स, ▁, ,, ▁टे, ने, सी, ] " ] }, "execution_count": 46, "metadata": {}, "output_type": "execute_result" } ], "source": [ "index2tag[-100] = \"IGN\"\n", "df[\"input_tokens\"] = df[\"input_ids\"].apply(\n", " lambda x: xlmr_tokenizer.convert_ids_to_tokens(x))\n", "df[\"predicted_label\"] = df[\"predicted_label\"].apply(\n", " lambda x: [index2tag[i] for i in x])\n", "df[\"labels\"] = df[\"labels\"].apply(\n", " lambda x: [index2tag[i] for i in x]) \n", "df[\"loss\"] = df.apply(\n", " lambda x: x[\"loss\"][:len(x[\"input_ids\"])], axis=1)\n", "df[\"predicted_label\"] = df.apply(\n", " lambda x: x[\"predicted_label\"][:len(x[\"input_ids\"])], axis=1)\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
input_idsattention_masklabelslosspredicted_labelinput_tokens
022181B-ORG0.03B-ORG▁स
059881I-ORG0.04I-ORG▁घर
0676911I-ORG0.03I-ORG▁जाना
04601I-ORG0.03I-ORG▁है
1930191O0.00O▁पुनर्
1416121B-PER0.01B-PER▁फ़
1517571I-PER0.02I-PER▁शाह
\n", "
" ], "text/plain": [ " input_ids attention_mask labels loss predicted_label input_tokens\n", "0 2218 1 B-ORG 0.03 B-ORG ▁स\n", "0 5988 1 I-ORG 0.04 I-ORG ▁घर\n", "0 67691 1 I-ORG 0.03 I-ORG ▁जाना\n", "0 460 1 I-ORG 0.03 I-ORG ▁है\n", "1 93019 1 O 0.00 O ▁पुनर्\n", "1 41612 1 B-PER 0.01 B-PER ▁फ़\n", "1 51757 1 I-PER 0.02 I-PER ▁शाह" ] }, "execution_count": 47, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_tokens = df.apply(pd.Series.explode) \n", "df_tokens = df_tokens.query(\"labels != 'IGN'\")\n", "df_tokens[\"loss\"] = df_tokens[\"loss\"].astype(float).round(2) \n", "df_tokens.head(7)" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0123456789
input_tokens▁▁)▁(▁का▁सी▁राज्य▁N▁डी▁र▁क्रिकेट
count2358080397916134
mean0.340.470.420.371.490.917.911.280.591.83
sum79.3637.833.3814.4110.448.167.917.687.637.32
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7 8 \\\n", "input_tokens ▁ ▁) ▁( ▁का ▁सी ▁राज्य ▁N ▁डी ▁र \n", "count 235 80 80 39 7 9 1 6 13 \n", "mean 0.34 0.47 0.42 0.37 1.49 0.91 7.91 1.28 0.59 \n", "sum 79.36 37.8 33.38 14.41 10.44 8.16 7.91 7.68 7.63 \n", "\n", " 9 \n", "input_tokens ▁क्रिकेट \n", "count 4 \n", "mean 1.83 \n", "sum 7.32 " ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(df_tokens.groupby(\"input_tokens\")[[\"loss\"]]\n", " .agg([\"count\", \"mean\", \"sum\"])\n", " .droplevel(level=0, axis=1)\n", " .sort_values(by=\"sum\", ascending=False)\n", " .reset_index()\n", " .round(2)\n", " .head(10)\n", " .T\n", ")" ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0123456
labelsI-ORGI-LOCOB-LOCB-ORGI-PERB-PER
count407163976185140259177
mean0.380.710.120.410.520.280.26
sum153.78116.0113.9176.572.7571.8945.95
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6\n", "labels I-ORG I-LOC O B-LOC B-ORG I-PER B-PER\n", "count 407 163 976 185 140 259 177\n", "mean 0.38 0.71 0.12 0.41 0.52 0.28 0.26\n", "sum 153.78 116.0 113.91 76.5 72.75 71.89 45.95" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(\n", " df_tokens.groupby(\"labels\")[[\"loss\"]]\n", " .agg([\"count\", \"mean\", \"sum\"])\n", " .droplevel(level=0, axis=1)\n", " .sort_values(by=\"sum\", ascending=False)\n", " .reset_index()\n", " .round(2)\n", " .T\n", ")" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "tags": [] }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAAIjCAYAAADP4ysCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAACkJ0lEQVR4nOzdd1gTWRcG8DcBKUpRmiAgRaRjpamrYG/Y1t4b9rWu2FfFruvqFhUVFDv2imUtIOq6q1iwixVFRUGqKFLz/REJBhIFl5b93t/z5FEmZyb3Hu7MnNzMBIFIJBKBiIiIqJwTlnUDiIiIiAqDRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUUL0f8hT09PeHp6Sn6OioqCQCDApk2bSrUdgwYNgrm5eam+ZlGkpqbC29sbhoaGEAgEmDBhQrG/hrm5OQYNGlTs21V05X1sUNlg0UIkw6ZNmyAQCKCmpoaXL18WeN7T0xOOjo5l0DIqTYsWLcKmTZswatQobN26Ff379y/rJimcDx8+YO7cuTh79mxZN4X+A5TLugFE5Vl6ejqWLFmCP/74o6ybUqLMzMyQlpaGChUqlHVTypWQkBC4u7tjzpw5JfYakZGREAr/u+8fP3z4AF9fXwCQmt37Gn9/f+Tk5JRQq0hR/Xf3FKJiUKdOHfj7++PVq1cl9hoikQhpaWkltv3CyJ1VUlJSKtN2lDexsbGoXLlyib6Gqqoqi8XPvH//HgBQoUIFqKqqlnFrqLxh0UL0BTNmzEB2djaWLFny1disrCzMnz8fNWrUgKqqKszNzTFjxgykp6dLxZmbm8PLywt//vknnJ2doa6ujnXr1uHs2bMQCATYvXs3fH19YWxsDE1NTXTr1g3JyclIT0/HhAkTYGBgAA0NDQwePLjAtgMDA9GsWTMYGBhAVVUV9vb28PPz+2rb81/TktsWWY/81xkcP34cjRs3RqVKlaCpqYn27dvjzp07BV7j4MGDcHR0hJqaGhwdHXHgwIGvtiv/63h4eEBTUxNaWlpwcXHBjh07pGL27NmD+vXrQ11dHXp6eujXr1+Bj/cGDRoEDQ0NvHz5Ep07d4aGhgb09fUxefJkZGdnS/X/6dOnOHr0qKTvUVFRko8Oo6KipLabu87nH4M8fPgQXbt2haGhIdTU1GBiYoJevXohOTlZEiPrmpYnT56ge/fu0NHRQcWKFeHu7o6jR4/KfL3du3dj4cKFMDExgZqaGpo3b45Hjx59NZ9z586FQCDAgwcP0K9fP2hra0NfXx8//fQTRCIRoqOj0alTJ2hpacHQ0BC//PKL1PoZGRmYPXs26tevD21tbVSqVAmNGzdGaGioJCYqKgr6+voAAF9fX0ke586dK/W7ePz4Mdq1awdNTU307dtX8tznY23OnDkQCoU4c+aMVDuGDx8OFRUV3Lhx46t9JsXHj4eIvsDCwgIDBgyAv78/pk2bhmrVqsmN9fb2xubNm9GtWzf8+OOPuHTpEhYvXox79+4VOEFHRkaid+/eGDFiBIYNGwYbGxvJc4sXL4a6ujqmTZuGR48e4Y8//kCFChUgFAqRmJiIuXPn4p9//sGmTZtgYWGB2bNnS9b18/ODg4MDOnbsCGVlZRw5cgSjR49GTk4OxowZU+h+29nZYevWrVLLkpKSMGnSJBgYGEiWbd26FQMHDkTr1q2xdOlSfPjwAX5+fvjuu+9w/fp1yUnn5MmT6Nq1K+zt7bF48WLEx8dj8ODBMDExKVR7Nm3ahCFDhsDBwQHTp09H5cqVcf36dZw4cQJ9+vSRxAwePBguLi5YvHgx3rx5g99++w1//fUXrl+/LjVjkp2djdatW8PNzQ3Lly/H6dOn8csvv6BGjRoYNWqUpP8TJ06EiYkJfvzxRwCQnIALIyMjA61bt0Z6ejrGjh0LQ0NDvHz5EsHBwUhKSoK2trbM9d68eYOGDRviw4cPGDduHHR1dbF582Z07NgRe/fuRZcuXaTilyxZAqFQiMmTJyM5ORnLli1D3759cenSpUK1s2fPnrCzs8OSJUtw9OhRLFiwADo6Oli3bh2aNWuGpUuXYvv27Zg8eTJcXFzQpEkTAEBKSgoCAgLQu3dvDBs2DO/evcOGDRvQunVrXL58GXXq1IG+vj78/PwwatQodOnSBd9//z0AoFatWpLXz8rKQuvWrfHdd99h+fLlqFixosx2zpo1C0eOHMHQoUNx69YtaGpq4s8//4S/vz/mz5+P2rVrF6q/pOBERFRAYGCgCIAoPDxc9PjxY5GysrJo3Lhxkuc9PDxEDg4Okp8jIiJEAETe3t5S25k8ebIIgCgkJESyzMzMTARAdOLECanY0NBQEQCRo6OjKCMjQ7K8d+/eIoFAIGrbtq1UfIMGDURmZmZSyz58+FCgL61btxZZWlpKLfPw8BB5eHhIfn769KkIgCgwMFBmPnJyckReXl4iDQ0N0Z07d0QikUj07t07UeXKlUXDhg2Tin39+rVIW1tbanmdOnVERkZGoqSkJMmykydPigAU6EN+SUlJIk1NTZGbm5soLS2tQLtEIpEoIyNDZGBgIHJ0dJSKCQ4OFgEQzZ49W7Js4MCBIgCiefPmSW2rbt26ovr160stMzMzE7Vv315qWe7YePr0qdTy3N9faGioSCQSia5fvy4CINqzZ88X+2dmZiYaOHCg5OcJEyaIAIjOnz8vWfbu3TuRhYWFyNzcXJSdnS31enZ2dqL09HRJ7G+//SYCILp169YXX3fOnDkiAKLhw4dLlmVlZYlMTExEAoFAtGTJEsnyxMREkbq6ulQ7s7KypF43N65q1aqiIUOGSJbFxcWJAIjmzJlToA25v4tp06bJfC7/2Lh165ZIRUVF5O3tLUpMTBQZGxuLnJ2dRZmZmV/sK/138OMhoq+wtLRE//79sX79esTExMiMOXbsGABg0qRJUstz36Hnn9q3sLBA69atZW5rwIABUtc4uLm5QSQSYciQIVJxbm5uiI6ORlZWlmSZurq65P/Jycl4+/YtPDw88OTJE6mPJIpq/vz5CA4OxqZNm2Bvbw8AOHXqFJKSktC7d2+8fftW8lBSUoKbm5vkY4KYmBhERERg4MCBUrMLLVu2lGzrS06dOoV3795h2rRpUFNTk3pOIBAAAK5cuYLY2FiMHj1aKqZ9+/awtbUtkH8AGDlypNTPjRs3xpMnTwqZka/L7euff/6JDx8+FHq9Y8eOwdXVFd99951kmYaGBoYPH46oqCjcvXtXKn7w4MFQUVGR/Ny4cWMAKHRfvL29Jf9XUlKCs7MzRCIRhg4dKlleuXJl2NjYSG1TSUlJ8ro5OTlISEhAVlYWnJ2dce3atUL3FwBGjRpVqDhHR0f4+voiICAArVu3xtu3b7F582YoK/NDg/8XLFqICmHWrFnIysqSe23Ls2fPIBQKYWVlJbXc0NAQlStXxrNnz6SWW1hYyH2t6tWrS/2ce/IzNTUtsDwnJ0eqGPnrr7/QokULVKpUCZUrV4a+vj5mzJgBAN9ctJw4cQK+vr6YPn06unbtKln+8OFDAECzZs2gr68v9Th58iRiY2MBQNL3mjVrFtj25x+LyfP48WMA+OIt5rmvIWt7tra2BfKvpqZW4KOeKlWqIDEx8avtKSwLCwtMmjQJAQEB0NPTQ+vWrbF69eqv/h6ePXsmsx92dnaS5z+Xf7xUqVIFAArdF1njTU1NDXp6egWW59/m5s2bUatWLaipqUFXVxf6+vo4evRokcaasrJyoT8mBAAfHx/Url0bly9fxpw5cwpV+NJ/B8tTokKwtLREv379sH79ekybNk1uXO47/6/5fEYkP3l38MhbLhKJAIhP7s2bN4etrS1WrFgBU1NTqKio4NixY1i5cuU33T769OlT9O3bFy1btsSCBQuknsvd3tatW2FoaFhg3fL87vff3CUl73ecexHv53755RcMGjQIhw4dwsmTJzFu3DgsXrwY//zzT5FO1F/ytXHxLesXZpvbtm3DoEGD0LlzZ/j4+MDAwABKSkpYvHixpNAsDFVV1SLd8v3kyRNJwXzr1q1Cr0f/DeX3qEJUzsyaNQvbtm3D0qVLCzxnZmaGnJwcPHz4UPKOGBBfVJmUlAQzM7MSb9+RI0eQnp6Ow4cPS717/vxujqJIS0vD999/j8qVKyMoKKjAiaVGjRoAAAMDA7Ro0ULudnL7nnui+VxkZORX25H7Ordv3y4wk5X/NSIjI9GsWbMCr1Gc+c+dyUhKSpJann8GJJeTkxOcnJwwa9YsXLx4EY0aNcLatWsLFIG5zMzMZObl/v37kufLg71798LS0hL79++XKuTyf6dNYQv5wsjJycGgQYOgpaWFCRMmYNGiRejWrZvkAl/67+PHQ0SFVKNGDfTr1w/r1q3D69evpZ5r164dAODXX3+VWr5ixQoA4msrSlruu+PP3w0nJycjMDDwm7Y3cuRIPHjwAAcOHJCcqD/XunVraGlpYdGiRcjMzCzwfFxcHADAyMgIderUwebNm6U+Njh16lSB6zNkadWqFTQ1NbF48WJ8/PhR6rncvjo7O8PAwABr166Vug38+PHjuHfvXrHmP7eIOnfunGRZdnY21q9fLxWXkpIidb0RIC5ghEJhgVvVP9euXTtcvnwZf//9t2TZ+/fvsX79epibm5ebj0NkjbdLly5JtRuA5G6g/EXet1ixYgUuXryI9evXY/78+WjYsCFGjRqFt2/f/uttk2LgTAtREcycORNbt25FZGQkHBwcJMtr166NgQMHYv369UhKSoKHhwcuX76MzZs3o3PnzmjatGmJt61Vq1ZQUVFBhw4dMGLECKSmpsLf3x8GBgZyLyCW5+jRo9iyZQu6du2Kmzdv4ubNm5LnNDQ00LlzZ2hpacHPzw/9+/dHvXr10KtXL+jr6+P58+c4evQoGjVqhFWrVgEQ38bdvn17fPfddxgyZAgSEhLwxx9/wMHBAampqV9si5aWFlauXAlvb2+4uLigT58+qFKlCm7cuIEPHz5g8+bNqFChApYuXYrBgwfDw8MDvXv3ltzybG5ujokTJxY9oXI4ODjA3d0d06dPR0JCAnR0dLBz584CBUpISAh++OEHdO/eHdbW1sjKysLWrVuhpKQkdW1QftOmTUNQUBDatm2LcePGQUdHB5s3b8bTp0+xb9++cvPtuV5eXti/fz+6dOmC9u3b4+nTp1i7di3s7e2lfqfq6uqwt7fHrl27YG1tDR0dHTg6Ohb5z2Dcu3cPP/30EwYNGoQOHToAEN/mXqdOHYwePRq7d+8u1v5ROVV2Ny4RlV+f3/KcX+5tmp/f8iwSiUSZmZkiX19fkYWFhahChQoiU1NT0fTp00UfP36UipN1G61IlHcLa/5bZOW1JfeW1bi4OMmyw4cPi2rVqiVSU1MTmZubi5YuXSrauHFjgVt0v3bLc+5rynrkvw01NDRU1Lp1a5G2trZITU1NVKNGDdGgQYNEV65ckYrbt2+fyM7OTqSqqiqyt7cX7d+/X+ZtrfIcPnxY1LBhQ5G6urpIS0tL5OrqKgoKCpKK2bVrl6hu3boiVVVVkY6Ojqhv376iFy9eSMUMHDhQVKlSpQLbz83n5+T9rh4/fixq0aKFSFVVVVS1alXRjBkzRKdOnZK65fnJkyeiIUOGiGrUqCFSU1MT6ejoiJo2bSo6ffp0gdf4/Fbi3O1369ZNVLlyZZGamprI1dVVFBwcLBUjb7x87fb1/P39fPyIRPLzk/82/5ycHNGiRYtEZmZmIlVVVVHdunVFwcHBMn+nFy9eFNWvX1+koqIidfuzvNfKfS53O1lZWSIXFxeRiYmJ1G3zIlHeLd67du36Yn/pv0EgEhXyai0iIiKiMlQ+5hmJiIiIvoJFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQR+uVwxy8nJwatXr6CpqVmsX19NRET0XyQSifDu3TtUq1btq1+eyKKlmL169arAX+MlIiKiL4uOjv7qHxJl0VLMNDU1AQAqjWdCoKxWxq0pWw+DxpR1E8oFJSFn3CiPshI/lSdpH9Kzvh70H/buXQpq2VhIzp9fwqKlmOV+JCRQVvu/L1q0tLTKugnlAosW+hyLFspP6f+8aMlVmEsquPcQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQlMu6AVQ43l51MLarCwyqVMLtp3GY6ncG1x68lhs/slM9DGlfByb6mkhIScOhCw8wb9N5pGdmAwCEQgGm9W2IHk3tYVClIl4nvMeO07exPOif0urSN9m47zzWbA9BXEIK7K2MsXBSV9SzN5MbfzjkOpatP4bo1wmwMNHHrNEd0KKhg+T5nwOO49Dpa3gZmwSVCkqoZWOK6SPao56DeSn05ttt2HsOq7eFIDYhBQ5Wxlj8YzfUc5Cfh0NnrmPJ+qOIjkmApak+fhrTES0/5SEzKxuL1wbj9N938exlPDQ11ODhYoOfRneEob52aXXpmzAPYv67w/DHtjOIjU+BY01jLPXpjvpfGMMHT1/DorVH8TwmHpam+pg7tjNaNcrbL0QiERavO4otBy8iOTUNbrUs8cu0nqhR3aAUevPtmAexTfvOY21QCOIS3sGuRjXMn9gVdb9wnAwOicDPAcfw4nUCzE30MWNUBzRvYC95fuLC7dhzPFxqHQ9XW2xfMbLE+iAPZ1oUQJcmNlgwzBNLd/wNz7FbcftJLPbN7wY97Yoy47t52mLO4CZYtuMi3EYEYuyvf6JLE1v8NKixJGZCN1cMaVcbU/zOwG1EIOZuPIdxXV0xvGPd0upWkR08fQ1zfz+AH4e0xslAHzhYVUPviX6IS3gnMz781lOMmrMFvTu449QmH7Rt4oTB0zbg3uNXkpga1fWx6MduOLt1Kg75jYepkQ56TvDD28TU0upWkR04dQ2zfzuAyd5tcGazDxxqGqPHhDVy83D55hOMmL0ZfTs0QMjmKWjbpBYGTgmQ5CHtYwZuRr7ApMGtcWazDzYtGYpHz2LRz2d9aXaryJgHsf0nr2LWrwcw1bstzm6dCseaxug6drXcPFy68QTeszahX6cGCNs2De09aqPf5PW4+yhvv/hty2ms2xWGFdN74VTgZFRUV0HXsavxMT2ztLpVZMyD2OEz1zBv1UFMHNwGxzdMhr2VMfpNWou3ibLzcOXWU4zx3YJeXu44sXEy2jR2gvf0Dbj/JEYqztPNFtcOzZM8Vs8dUBrdKYBFSz7R0dEYMmQIqlWrBhUVFZiZmWH8+PGIj48vszaN7uKMLSduYcep24iMjsekVafwIT0T/Vo5yox3tTPGpbsvsffsfUTHpiD0+jPsC7uP+taGeTH21XDsn8c4Gf4E0bEpOPzXA4Rej0J9a6PS6laRrdt5Fn07NkRvL3fYWBhi2ZQeUFdVwc5g2bND/rvD0NTNFmP6Noe1uSGmDm8PJxsTBO47L4n5vpUzmrjYwMxYD7aWRvAd1wXv3n/EvccvS6tbRbY2KBT9OjVEHy932FgYYfnUHlBXU8EOOXlYvysMzdzt8EO/5rC2MMT0Ee1Ry8YEG/aK86CloY69f4xB5xb1YGVWFc6OFlgyuRtu3I/Gi9cJpdm1ImEexNbsCMGAzg3Rt2MD2FoaYcX0XqiopoJth/+WGb9u51k0b2CHcf1bwMbCEDNHeaG2rSn894QBEM8urA0KxeQhrdHOoxYcaxrDz3cAXr9NxtGwG6XZtSJhHsTW7zyL3h0aoGd7N1hbGGKJT3eoqalgZ/AlmfEb9oTB080Wo/o0Q01zQ/gMawdHaxNs+uw4CQCqKsow0NWSPCpryX7TXNJYtHzmyZMncHZ2xsOHDxEUFIRHjx5h7dq1OHPmDBo0aICEhNI/cFVQFqKOVVWcjXgmWSYSAWERz+FiW03mOpfvvUQdq6qo96lIMTPURktnC5wKf5oXc/cVPOpURw3jKgAARwt9uNsb4/SVpzK3WdYyMrNwMzIaTZytJcuEQiEau1jjyu0ometcvf0UTVxspJZ5utnKjc/IzMLWQxehpaEOeyvj4mp6scrIzMKNyGh4fNYvoVCIJi42uHJL9u/uyu0oNHGxllrW1N1ObjwApKR+hEAggLamevE0vJgxD2IZmVmIuB8NT1fpPHi42iBcTr8u33oKTxdbqWXN3O0QfisKAPDsZTzexKfA0zUvRltDHfUdzBF+M6rY+1AcmAexjMws3HrwAo3zHyedrXHtTpTMda7ejpKKBwAPN1tczXec/Pv6I9T2moUmvRdi+vLdSEx+X9zNLxRe0/KZMWPGQEVFBSdPnoS6uvggVb16ddStWxc1atTAzJkz4efnV6pt0tVSh7KSEHGJ0gMkLuk9aprqyFxn79n70NFSx/Gfe0MgACooK2Hj0Qis2J1Xaa/ccwmaFVVwed0QZOfkQEkoxIIt57Hn7L0S7c+3Skh6j+zsHOjraEot19fRxKNnsTLXiY1/B/0q+eKraCI2PkVq2cm/bmPk7M1I+5iJqrpa2PXrKOhW1ijeDhQTeXkwqKKJR1FvZK4TG58CAx0tqWXiPMieLv6Ynol5qw/h+5b1oFmpfJ6smQex+KRUOfuFFh5+IQ/6ugX3o9z94s2nf/PHGOgW3HfKC+ZBLCFZ9n6hp6OJR89k5yEu4R30ZBwn4xLy+ujpZoe2HrVhaqSDZy/fYun6o+g3eR0Or50AJaXSnftg0fJJQkIC/vzzTyxcuFBSsOQyNDRE3759sWvXLqxZswYCgUDyXHp6OtLT0yU/p6SU/WBu5GSKST3cMXnNaVyNjIGFUWUsGdEMk3u7Sy607dLYBt2b2mHYsmDcfx4PJ0sDLBreFDHx77HzzJ0y7kHpalSvJs5snoKEpPfYdvgihv+0Ccf8JxXY8f8fZGZlw3tmIEQi4OepPcq6OWWGeSDK06lFPcn/7WpUg12NamjUcwH+vv4I3+WbpSlp/Hjok4cPH0IkEsHOzk7m83Z2dkhMTERcXJzU8sWLF0NbW1vyMDU1LdZ2xaekISs7B/pVKkkt169cCbEJsqfnZvZvhN0hd7H1z1u4G/UWR/9+hPmbz2Nidzfk1lvzhnrg1z2Xsf9cJO5GvcWukLtYc/AqJvZwLdb2FxedypWgpCQscFFdXMI7GMgpLgx0NRGX7+KzuMR3MNCVfrddSV0VFib6qO9ojpUz+kBZSYggOddFlDV5eYhNfAcDXXl50EJsgnQxHScjPvdE/eJ1Avb+Mabczi4AzEMu3coacvaLlALjPJeBrhbi4mXsR5/iq376N39MbHzBfae8YB7EdLRl7xdvE+S3WV9Hs8BFunGJ76CvI7+PZsZ60KlcCVEv4uTGlBQWLfmIRKIixU+fPh3JycmSR3R0dLG2JzMrBxGP3sCjdnXJMoEAaFKnOsLvv5K5jrqqMnLy9SM7R/RpXcGnmArIyZGOycnJgVAoQHmkUkEZtWxMcf7qA8mynJwcXLjyAM6O5jLXqe9ogfNXHkgtO3c5Um583nZFSM/I+rdNLhEqFZRR28YU58Kl83A+PBLOThYy13F2NMf5cOk8hF2+LxWfe6J+Eh2HvX+MgY52pfybKVeYBzGVCsqoY2uKsPBIybKcnBycC38AFzl5cHWykIoHgNBL9+HiZA4AMDPWRVVdLamYlNQ0XL0TBZda5sXeh+LAPIipVFCGk7UJLlx9KFmWk5ODC1cfyP0ah/qO5rhw5aHUsvPhkaj/hePkq9gkJCZ/gIFe6X8VAIuWT6ysrCAQCHDvnuxrOu7du4cqVapAX19farmqqiq0tLSkHsVtzYErGNCmFno1d4C1qQ5WjGmJSqoVsP3UbQCA349tMfuz25lPXH6Cwe1r4/smNqheVRuedc0wo38jnLj8WFKonLj0GJN6uaOViyVMDbTQvoEVRndxxtGLj4q9/cVlRC9PbD/8N3Ydu4wHUa8x9ec9+PAxA7283AAAP8zbhoV+RyTxw3p4IPSfe/DbEYKHUW/wc8Bx3LgfjcFdxbl6n5aORWuP4OrtKETHJODG/WhMWLgDr98mo0OzOmXRxUIZ2bspth2+iJ1HL+HB09fwWbYbHz5moHd7cR7G+G7F/DWHJfHDe3og5J97WLNdnIdl/scQcS8aQ7uJ85CZlY0h0zcg4t5z+PkOQHaOCG/iU/AmPgUZmeWzeAOYh1yj+zTDloMXERT8DyKfvsakJbvwPi0dfTu4AwBGztkC31WHJPEjennizN93sWrbGTyIeo0l648i4t5zDOvuAUD8xmZk76ZYvvEEjoXdxJ1HLzFq7lYY6mmjvUftMuljYTAPYsN7eSLoyN/Yc/wyHka9xvTle5CWloGen/aL8fO3YfHavOPk0O4eOHvpHtYFheLRszf4ZcNx3LwfjUG5x8kP6Zi/+tCn42Q8Llx5gKHTAmBurAcPV1uZbShJvKblE11dXbRs2RJr1qzBxIkTpa5ref36NbZv344BAwZIXc9SWg6ci4SeVkXM6N8IBlUq4taTOHSbvRdxSR8AACb6WlKzJsuD/oZIJMLMAd/BSFcD8clpOHH5MeZvviCJmbr2DGb0/w7Lx7SAnrY6Xie8x6bjN7Bsh+zbA8uDzi3qIT4pFcv8jyEuIQUONU0QtGKkZBrz5ZtEqZkiFycLrPEdgKXrj2HxumBYmOgjcMlQ2NUQ33WlJBTi0bNY7D62EQnJqaiiXQl1bKvj4JpxsLUsv7d+d2kpzsNS/2OfvkTLBLtWjpJM/754nSg1Tl1rWWLtvIFYvO4oFq49AktTA2xe5i3JQ0xsEk6cFxfATfsvlXqtg6vHolH9mqXUs6JhHsS+b1Ufb5NSsWjdUcTGv4OTtTH2/j7mszwkQPhZHtxqW8J/wSAs9AvG/DVHYGmqj23Lh8PeKu9uxPEDWuBDWjomLgpCcmoa3GvXwN7fR0NNtUKp96+wmAexjs3rIT7pPZYHHJd8CefWX0ZIrtHLf5x0drLAqjkDsMz/KJauFx8nAxYPlRwDhUoC3H/8CnuPhyMlNQ1V9bTQxMUWPsPaQVWl9EsIgaion4f8hz18+BANGzaEnZ0dFixYAAsLC9y5cwc+Pj5IT0/HP//8Ax0d2Xfs5EpJSYG2tjZUm86HQFmtlFpePr0+NLGsm1AuKJXTj9yobCiX8t0WVP69Ty+/M3ml4V1KCiyq6SI5Ofmrn1Zw7/lMzZo1ceXKFVhaWqJHjx6oUaMGhg8fjqZNm+Lvv//+asFCREREJYcfD+VjZmaGTZs2lXUziIiIKB/OtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCUC7rBvxXPQgaAy0trbJuRpkybLuorJtQLrw6NqOsm1AuqKsolXUTiMoltQr/3/tGRhH6z5kWIiIiUggsWoiIiEghsGghIiIihcCihYiIiBQCixYiIiJSCCxaiIiISCGwaCEiIiKFwKKFiIiIFAKLFiIiIlIILFqIiIhIIbBoISIiIoXAooWIiIgUAosWIiIiUggsWoiIiEghsGghIiIihcCihYiIiBQCixYiIiJSCCxaiIiISCGwaCEiIiKFwKKFiIiIFAKLFiIiIlIILFqIiIhIIbBoISIiIoXAooWIiIgUAosWIiIiUggsWoiIiEghsGghIiIihcCiRUEE7jsPl+99Ye75I9p5r8D1u8++GH8k5Dq+67UQ5p4/omm/JThz8Y7c2CnLdsGo4Xis33W2mFtd/Lw7OuPGtrGIOTYdp/4Ygno21eTGKisJ4dOvMa5tGYOYY9Nxft1wNHep8a+2WV4E7jsP166+sGj6I9oPK9x4aNx7ISya/ohm/b88HqYu24VqjcbDXwHGg//uMNTqOBuGjSagxaCfcfVO1BfjD56+Btdu82HYaAIa9lqIk39J50EkEmHR2mDYtpkBo+8movPoP/D4eWwJ9qB4MA9izIPYhj3nULfzHBg3nohWQ5bj2lfycOjMdbj3mA/jxhPRuM8inMqXh+DQCHQbuxo1W06FnttY3HrwogRb/2UsWhTAodPXMPf3A/hxSGv8GegDe6tq6D3RD28T3smMD7/1FKPmbEGfDu44uckHbZo4YfC0Dbj/+FWB2GNhN3DtzjMY6mmXdDf+tS6e9lgwsiWWbj0Hz5H+uP3kDfYt6QO9yhVlxs8a3BSDvOph6qo/4T7UD4HBV7F1bnc4WRl+8zbLg0Onr8H3jwOYNKQ1/twoHg99JvnhbaL88TB67hb09nLHyUAftGnshCHTN+D+k4Lj4XjYDVxVkPGw/+RVzPr1AKZ6t8XZrVPhWNMYXceuRpyc/eLSjSfwnrUJ/To1QNi2aWjvURv9Jq/H3Ud5efhty2ms2xWGFdN74VTgZFRUV0HXsavxMT2ztLpVZMyDGPMgduDUVfz02wH4DG2LkM1T4GBljO7j18jNw+WbTzD8p03o26EBQrdMRbsmtTBgij/ufXa++JCWAbfalpj9Q6fS6oZc5a5oGTRoEAQCgeShq6uLNm3a4ObNm3LXiYqKKrBOq1atcP36dUmMp6enVEzuY+TIkZKYz5draWnBxcUFhw4dKtH+Fsa6nWfRt2ND9PJyh42FIZZN6QF1VRUEBf8jMz5gdxiautlidN/msDY3xNTh7eFkY4KN+85LxcXEJWHWin1YPac/lJWVSqMr/8roru7Ycuw6dvx5A5HP32LSr0fxIT0T/drUkRnfo4UTVu74C6cuP8KzmCRsPHIVpy4/wg/d3L95m+XB+l1n0adDQ/Rq7w5rC0Ms9Sn8eKhpbogpw9vDydoEgXtljIeVijMe1uwIwYDODdG3YwPYWhphxfReqKimgm2H/5YZv27nWTRvYIdx/VvAxsIQM0d5obatKfz3hAEQv6teGxSKyUNao51HLTjWNIaf7wC8fpuMo2E3SrNrRcI8iDEPYn5BoejfqQH6dHCHjaURfpnWE+pqKthxRE4edp1FM3c7jO3fAtYWhpg+0gu1bEwRsOecJKZHO1f4eLeFh4tNaXVDrnJXtABAmzZtEBMTg5iYGJw5cwbKysrw8vL66nqnT59GTEwM/vzzT6SmpqJt27ZISkqSPD9s2DDJdnMfy5Ytk9pGYGAgYmJicOXKFTRq1AjdunXDrVu3iruLhZaRmYWbkdFo7GwtWSYUCtHYxRpXb0fJXOfK7adonG9webrZSsXn5ORgrO82jOrTDDaWRiXR9GJVQVmIOtZGOHvtqWSZSASEXXsKF3sTmeuoqijhY0aW1LKP6VlwdzT95m2WNcl4cMk3Hpzlj4erd56isbP0ePBws5WaOs/JycG4eYozHjIysxBxPxqernn9EgqF8HC1QfitpzLXuXzrKTxdbKWWNXO3Q/itKADAs5fxeBOfAk/XvBhtDXXUdzBH+M2oYu9DcWAexJgHsYzMLNy4Hw2P/HlwsZH0K78rt6IKFCNN3W1xRU7eylq5LFpUVVVhaGgIQ0ND1KlTB9OmTUN0dDTi4uK+uJ6uri4MDQ3h7OyM5cuX482bN7h06ZLk+YoVK0q2m/vQ0tKS2kblypVhaGgIa2trzJ8/H1lZWQgNDS2RfhZGQtJ7ZGfnQF9HU2q5vo4mYuVM98XFv4N+lXzxVTQRG58i+XnVtjNQUhLCu4dH8Te6BOhqV4SykhBxialSy+MS38OgiobMdUKuPMHobu6wNNaBQAB41rOA13e2qKqj8c3bLGvyxoOejqbc6d+4+HfQkzV+PhsPqz+Nh6HdFWM8xCelytkvtKT69bnY+BTo68rPw5tP/+aPMdDVlLvNssY8iDEPYvGS44P0eU18vvhCHvLlzUBHE7Hxso8nZU25rBvwNampqdi2bRusrKygq6tb6PXU1dUBABkZGd/0ullZWdiwYQMAQEVFRW5ceno60tPTJT+npJTPwfy5G/ejEbA7DCcDfSAQCMq6OSVm2uo/8dskL1zeOAoiAE9fJWLHnxHoW44/+ikLN+9HI2BPGP7c+N8eD0Sk+Mpl0RIcHAwNDfE73ffv38PIyAjBwcEQCgs3MZSUlIT58+dDQ0MDrq6ukuVr1qxBQECAVOy6devQt29fyc+9e/eGkpIS0tLSkJOTA3Nzc/To0UPuay1evBi+vr5F6V6R6FSuBCUlYYF30XEJ72CQrzrOpa+ribh8F2XGJb6Dga64+r504zHeJqbC+fu5kuezs3Pg+8dB+O8KQ/j+OcXah+IQn/wBWdk50M83A6JfpRJi882UfL5Ovzm7oVpBCTpaFRET/w5zvZsjKibpm7dZ1uSNh7cJ7wq8W8qlr6tZ4KLtuISC48Gl61zJ89nZOfBddRD+u8NweV/5Gw+6lTXk7Bcpkn7lZ6Crhbh4+Xmo+unfuPh3Uhcix8a/g5N1+fy4kHkQYx7EdCXHB+k3z+LzxRfykC9vsQnvYKAr+3hS1srlx0NNmzZFREQEIiIicPnyZbRu3Rpt27bFs2fP0LZtW2hoaEBDQwMODg5S6zVs2BAaGhqoUqUKbty4gV27dqFq1aqS5/v27SvZbu6jY8eOUttYuXIlIiIicPz4cdjb2yMgIAA6Ojpy2zp9+nQkJydLHtHR0cWaC5UKyqhlY4oLVx9IluXk5ODClQeo72gucx1nRwtcuPJAatm5y5GS+G5tXBCyZQpOb/KRPAz1tDG6TzMErRwpY4tlLzMrBxEPYuBRz1yyTCAAmtS1QPjdL99+l56ZjZj4d1BWEqJDY1scvxj5r7dZViTj4Uq+8XBV/nio72CB81fzjYfwSNR3EMd3beOCM1um4NQmH8nDUE8bo/o0w44V5XM8qFRQRh1bU4SFR0qW5eTk4Fz4A7g4Wchcx9XJQioeAEIv3YeLkzkAwMxYF1V1taRiUlLTcPVOFFxqmRd7H4oD8yDGPIipVFBGbVtTnAuXPj6I82Aucx1nJ3Ocy3e+CLscCWc5eStr5XKmpVKlSrCyspL8HBAQAG1tbfj7+yMgIABpaWkAgAoVKkitt2vXLtjb20NXVxeVK1cusF1tbW2p7cpiaGgIKysrWFlZITAwEO3atcPdu3dhYGAgM15VVRWqqqpF7GHRjOjlifELtqO2bXXUsa8O/11h+PAxA7283AAAY+dtg6G+NmaO6gAA8O7hge9H/461O0LQvKEDDp2+hhv3o/Hz1J4AAB3tStDRriT1GsrKStDX1YKVWVWUV2v2/YM1UzrhemQMrkW+wqjvXVFJrQK2nxBfye83tRNi3r7DvA0hAID6ttVgpKeFW49fo5quJqYO8IBQKMBvuy4Wepvl0fCenpiwUDwe6tpXh//uT+OhvXg8jJu/DYZ62pjx2XjoOuZ3rA3KGw83CzEeDHTK93gY3acZRvtuRV276qjnYA6/oFC8T0tH3w7iu8NGztkCI31tzPl0m+aIXp7wGvErVm07g1bfOWD/yauIuPccv87oDUB89+DI3k2xfOMJWJrqw8xYF4vWHoWhnjbae9Qus35+DfMgxjyIjerdFD/M24Y6dtVRz94Ma3eexYeP6ejtJc7D6LlbYKRfGT+NEb9hH9HTEx1H/obV28+gVSMH7D91DRH3nmPF9F6SbSYmv8eLN4l4HZcMAHj07A0A8SxNVTkzWSWlXBYt+QkEAgiFQqSlpcHY2FhunKmpKWrUKPjlYd/K1dUV9evXx8KFC/Hbb78V23aLqlOLeohPSsUy/2OIS0iBQ00T7FgxUnKx1cs3iRAK865FcHGywBrfAVi6/hgWrwuGhYk+ApcMhW2N8v+laV9y4Oxd6GlXxIxBHjCoooFbj9+g2/QdiEt6DwAwMdBCTo5IEq+qooyZgz1hblQF79MycOryI4xcehAp79MLvc3yKHc8/ByQNx62/5JvPAikx8PqueLxsOTTeNi4eChsLRV7PHzfqj7eJqVi0bqjn6bsjbH39zGS6f0XrxOk8uBW2xL+CwZhoV8w5q85AktTfWxbPhz2Vnl5GD+gBT6kpWPioiAkp6bBvXYN7P19NNRUKxR4/fKCeRBjHsS6tKyP+KRULFkvzoOjtTF2/zo6Lw/5zheutSyxbv4gLFobjIV+wbA01ceWZcNg99n54sT5Wxg7f7vk52GzNgEAfLzbYuqwdqXTsU8EIpFI9PWw0jNo0CC8efMGgYGBAIDExESsWrUKfn5+CAkJgaenZ4F1oqKiYGFhgevXr6NOnToyt+vp6Qlra2vMmzdParmqqiqqVKkCQFwcHThwAJ07d5Y8f/z4cXTp0gWPHz/+YsGUKyUlBdra2nj2OqHAnUn/b4zaLirrJpQLr47NKOsmlAvqKuX/u1+IykJ2Trk6DZe6lJQUVNOvjOTk5K+eN8vlNS0nTpyAkZERjIyM4ObmhvDwcOzZs0dmwVIU/v7+ku3mPnr37v3Fddq0aQMLCwssXLjwX702ERER/TvlbqZF0XGmJQ9nWsQ40yLGmRYi2TjTouAzLURERET5sWghIiIihcCihYiIiBQCixYiIiJSCCxaiIiISCGwaCEiIiKFwKKFiIiIFAKLFiIiIlIILFqIiIhIIbBoISIiIoXAooWIiIgUAosWIiIiUggsWoiIiEghsGghIiIihcCihYiIiBQCixYiIiJSCCxaiIiISCGwaCEiIiKFwKKFiIiIFAKLFiIiIlIILFqIiIhIIbBoISIiIoXAooWIiIgUAosWIiIiUggsWoiIiEghKJd1A/6rlIUCKAsFZd2MMvXq2IyybkK5UK21b1k3oVxIDJ1X1k0gKpdEIlFZN6FMFaX/nGkhIiIihcCihYiIiBQCixYiIiJSCCxaiIiISCGwaCEiIiKFwKKFiIiIFAKLFiIiIlIILFqIiIhIIbBoISIiIoXAooWIiIgUAosWIiIiUggsWoiIiEghsGghIiIihcCihYiIiBQCixYiIiJSCCxaiIiISCGwaCEiIiKFwKKFiIiIFAKLFiIiIlIILFqIiIhIIbBoISIiIoXAooWIiIgUAosWIiIiUggsWoiIiEghsGghIiIihcCihYiIiBQCixYiIiJSCMpl3QAqnA17z2H1thDEJqTAwcoYi3/shnoOZnLjD525jiXrjyI6JgGWpvr4aUxHtGzoAADIzMrG4rXBOP33XTx7GQ9NDTV4uNjgp9EdYaivXVpd+iaB+87Db0cI4hJSYG9ljAUTu6Kuvfw8HAm5jmX+x/DidQIsTPQxc1QHNP+Uh/ymLtuFrYcuwndcFwzr6VlCPSge3p1dMbZnIxjoaOD24zeY+vtRXLv/UmasspIQE/s2Qe9WdWCkr4lH0fGYu+4kzoQ/ksRM7NMYXo3tUbO6Hj6mZ+LynWjMXX8Sj6LjS6tL38R/dxj+2HYGsfEpcKxpjKU+3VHfwVxu/MHT17Bo7VE8j4mHpak+5o7tjFaN8saDSCTC4nVHseXgRSSnpsGtliV+mdYTNaoblEJvvh3zIMY8iP2XzxecaVEAB05dw+zfDmCydxuc2ewDh5rG6DFhDeIS3smMv3zzCUbM3oy+HRogZPMUtG1SCwOnBODe41cAgLSPGbgZ+QKTBrfGmc0+2LRkKB49i0U/n/Wl2a0iO3T6Gnz/OIBJQ1rjz40+sLeqhj6T/PA2UXYewm89xei5W9Dbyx0nA33QprEThkzfgPtPXhWIPR52A1fvPIOhXvku2gCgS1NHLBjVBks3n4Xn8LW4/fg19i0bAL3KlWTGzxraHIO8nDH1j6NwH7QKgYfDsXV+bzhZGUpiGtY2R8DBS2g1Zj2+99mMCspK2L9sICqqVSitbhXZ/pNXMevXA5jq3RZnt06FY01jdB27Wu5+cenGE3jP2oR+nRogbNs0tPeojX6T1+Puo7zx8NuW01i3KwwrpvfCqcDJqKiugq5jV+NjemZpdavImAcx5kHsv36+UIiiZdCgQejcubPc5z09PSEQCCAQCKCmpgZ7e3usWbNG8vymTZskz3/+UFNTk3qN3OUVKlSAhYUFpkyZgo8fP5Zk1wplbVAo+nVqiD5e7rCxMMLyqT2grqaCHcH/yIxfvysMzdzt8EO/5rC2MMT0Ee1Ry8YEG/aeBwBoaahj7x9j0LlFPViZVYWzowWWTO6GG/ej8eJ1Qml2rUjW7zqLPh0aold7d1hbGGKpTw+oq6ogSE4eAnaHoambLUb3bY6a5oaYMrw9nKxNEPgpD7li4pIwa+U+rJ7TH8rKSqXRlX9ldPeG2HL0KnacuI7IZ3GYtOIIPnzMRL+29WTG92hZGyt3nMOpSw/xLCYRGw+H49SlB/ihRyNJTPepWxH0ZwTuR8Xh9uM3GL1kP0wNK6OOdbXS6laRrdkRggGdG6JvxwawtTTCium9UFFNBdsO/y0zft3Os2jewA7j+reAjYUhZo7yQm1bU/jvCQMgfle9NigUk4e0RjuPWnCsaQw/3wF4/TYZR8NulGbXioR5EGMexP7r5wuFKFoKY9iwYYiJicHdu3fRo0cPjBkzBkFBQZLntbS0EBMTI/V49uyZ1DbatGmDmJgYPHnyBCtXrsS6deswZ86c0u6KlIzMLNyIjIaHi41kmVAoRBMXG1y59VTmOlduR6GJi7XUsqbudnLjASAl9SMEAgG0NdWLp+HFLCMzCzcjo9H4s34JhUI0drbG1dtRMte5eucpGjvbSC3zcLPF1Tt58Tk5ORg3bxtG9WkGG0ujkmh6saqgrIQ61kY4e/WxZJlIJELYtcdwcTCRuY5qBWV8zMiSWvYxPQvuTtXlvo5WJXFBn5iSVgytLn4ZmVmIuB8NT1fp/cLD1Qbhcsb55VtP4eliK7Wsmbsdwm9FAQCevYzHm/gUeLrmxWhrqKO+gznCb0YVex+KA/MgxjyI/T+cL/4zRUvFihVhaGgIS0tLzJ07FzVr1sThw4clzwsEAhgaGko9qlatKrUNVVVVGBoawtTUFJ07d0aLFi1w6tSp0u6KlISk98jOzoG+jqbUcoMqmoiNlz3dFxufAgMdLall+l+I/5ieiXmrD+H7lvWgWal8Fi3y8qCnoyl32jMu/h308sXr62giNj5F8vPqbWegpCTE0O4exd/oEqCrXRHKSkqIS3wvtTwu8T0M8vU1V8iVRxjdvSEsjXUgEAjgWb8GvBrboaqceIFAgMU/tMU/t57hXlRssfehOMQnpcocD/o6WlK/38/FxqdAX1f+eHjz6d/8MQa6mnK3WdaYBzHmQez/4Xzxn70QV11dHRkZGd+8/u3bt3Hx4kWYmcm/eAkA0tPTkZ6eLvk5JaV8DmZ5MrOy4T0zECIR8PPUHmXdnFJ18340AvaE4c+NPhAIBGXdnBIz7Y9j+G1yJ1zePA4iiPD0ZSJ2nLiOvnI+Tlo+vj3sLAzQduyGUm4pEZVn5eF88Z8rWrKzsxEUFISbN29i+PDhkuXJycnQ0NCQim3cuDGOHz8u+Tk4OBgaGhrIyspCeno6hEIhVq1a9cXXW7x4MXx9fYu3E5/RqVwJSkrCArMJsYnvYKAr+52yga4WYhOki6c4GfG5A/DF6wTsXz223M6yAPLz8DbhXYF3Fbn0dTXxNl98XMI7GOiK31VcuvEYbxNT4dJ1ruT57Owc+K46CP/dYbi8r2w/GpQlPvkDsrKzoV9F+qJb/SqVECtnxik++QP6/RQE1QrK0NFWR8zbd5g7vCWiYhILxC4b1x6tG9ig3fgNePW2/BbgupU1ZI6HuIQUye83PwNdLcTFyx8PVT/9Gxf/TuqC7Nj4d3Cylv3RW1ljHsSYB7H/h/OFQn08tH37dmhoaEge58/nXVC5Zs0aaGhoQF1dHcOGDcPEiRMxatQoyfOampqIiIiQegQEBEhtv2nTpoiIiMClS5cwcOBADB48GF27dv1im6ZPn47k5GTJIzo6ulj7rFJBGbVtTHEu/IFkWU5ODs6HR8LZyULmOs6O5jj/WTwAhF2+LxWfOwCfRMdh7x9joKMt+86T8kKlgjJq2ZjiwhXpPFy4+gD1Hc1lrlPfwQLnr0rn4Vx4pOQWyK5tXHBmyxSc2uQjeRjqaWNUn2bYsWJkSXXlX8nMykbEgxh41LOULBMIBGhSzxLhd158cd30zCzEvH0HZSUhOjSxx/G/7ks9v2xce7T/zg4dJwXi+eukkmh+sVGpoIw6tqYIC4+ULMvJycG58AdwkbNfuDpZSMUDQOil+3BxMgcAmBnroqqullRMSmoart6Jgkst82LvQ3FgHsSYB7H/h/OFQs20dOzYEW5ubpKfjY2NJf/v27cvZs6cCXV1dRgZGUEolK7HhEIhrKysvrj9SpUqSWI2btyI2rVrY8OGDRg6dKjcdVRVVaGqqvot3Sm0kb2bYuz8bahjZ4p69mZYt+ssPnzMQO/24lyM8d0KQ31t/DS6IwBgeE8PdBr1O9ZsD0HLRg44cOoqIu5F45dpvQCIB+CQ6RtwM/IFtv8yAtk5Isnnt1W0KkKlQvkcFsN7emLCwu2obVsdde2rw393GD58zECvT3kYN38bDPW0MWNUBwCAdw8PdB3zO9YGhaB5QwccOn0NN+9H4+epPQEAOtqVCux8yspKMNDRgpWZ9PVO5cmaPRexZloXXH/wCtfuvcCobg1QSU0F209cAwD4Tf8eMXEpmBdwGgBQ384ERnqauPXoNarpaWHqoKYQCgT4LeiCZJvLJ3ihW3Mn9JkVhNQPGTCoIp6VTHn/scBFvOXF6D7NMNp3K+raVUc9B3P4BYXifVo6+nZwBwCMnLMFRvramPNDJwDAiF6e8BrxK1ZtO4NW3zlg/8mriLj3HL/O6A1AXPyN7N0UyzeegKWpPsyMdbFo7VEY6mmjvUftMuvn1zAPYsyD2H/9fFE+z05yaGpqQlNT9hSXtrb2V4uSohAKhZgxYwYmTZqEPn36QF297D466dKyHuKTUrHU/9inL00ywa6VoyTTmC9eJ0pdk+FayxJr5w3E4nVHsXDtEViaGmDzMm/Y1RDfvhoTm4QT528DAJr2Xyr1WgdXj0Wj+jVLqWdF06mFOA8/BxxDXEIKHGqaYPsvI6H/6SKyl28SIfwsDy5OFlg9dwCWrj+GJeuCYWGij42Lh8LWsvzexlsYB0JvQ0+7ImYMagYDHQ3cevwa3aZulVyca2KgjZwckSReVUUZM4c0h3m1KnifloFTlx5i5KJ9SHmfdzv/0E6uAICjvw6Req3RS/Yj6M+Iku/UN/i+VX28TUrFonVHP03ZG2Pv72M+2y8SpMaDW21L+C8YhIV+wZi/5ggsTfWxbflw2FvljYfxA1rgQ1o6Ji4KQnJqGtxr18De30dDTbX8fl8N8yDGPIj9188XApFIJPp6WNkaNGgQkpKScPDgQZnPe3p6ok6dOvj1119lPr9p0yaMHz8ekZGRBZ4zMDCAUCiU+RpZWVkwNzfHhAkTMHny5EK1NSUlBdra2ngZmwgtLdmfpf6/yMwu90OrVFRrXXLXPCmSxNB5Zd0EonIpKzunrJtQplJSUmBsUAXJyclfPW8q1DUt/0ZKSgqMjIwKPGJj5d/SqaysjB9++AHLli3D+/fv5cYRERFRyVOImRZFwpmWPJxpEeNMixhnWohk40wLZ1qIiIjoP4ZFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECkG5rBvwX5WVLUJWtqism1Gm1FWUyroJ5UJi6LyybkK5UKXnhrJuQrnwbNOAsm5CuVBJlaefXBlZOWXdhDJVlP5zpoWIiIgUQqFK3cOHDxd6gx07dvzmxhARERHJU6iipXPnzoXamEAgQHZ29r9pDxEREZFMhSpacnL+vz9vIyIiorL3r65p+fjxY3G1g4iIiOiLily0ZGdnY/78+TA2NoaGhgaePHkCAPjpp5+wYQPvDiAiIqKSUeSiZeHChdi0aROWLVsGFRUVyXJHR0cEBAQUa+OIiIiIchW5aNmyZQvWr1+Pvn37Qkkp73s4ateujfv37xdr44iIiIhyFbloefnyJaysrAosz8nJQWZmZrE0ioiIiCi/Ihct9vb2OH/+fIHle/fuRd26dYulUURERET5Ffl7lGfPno2BAwfi5cuXyMnJwf79+xEZGYktW7YgODi4JNpIREREVPSZlk6dOuHIkSM4ffo0KlWqhNmzZ+PevXs4cuQIWrZsWRJtJCIiIvq2P5jYuHFjnDp1qrjbQkRERCTXN/+ZzStXruDevXsAxNe51K9fv9gaRURERJRfkYuWFy9eoHfv3vjrr79QuXJlAEBSUhIaNmyInTt3wsTEpLjbSERERFT0a1q8vb2RmZmJe/fuISEhAQkJCbh37x5ycnLg7e1dEm0kIiIiKvpMS1hYGC5evAgbGxvJMhsbG/zxxx9o3LhxsTaOiIiIKFeRZ1pMTU1lfolcdnY2qlWrViyNIiIiIsqvyEXLzz//jLFjx+LKlSuSZVeuXMH48eOxfPnyYm0cERERUa5CfTxUpUoVCAQCyc/v37+Hm5sblJXFq2dlZUFZWRlDhgxB586dS6ShRERE9P+tUEXLr7/+WsLNICIiIvqyQhUtAwcOLOl2EBEREX3RN3+5HAB8/PgRGRkZUsu0tLT+VYOIiIiIZCnyhbjv37/HDz/8AAMDA1SqVAlVqlSRehARERGVhCIXLVOmTEFISAj8/PygqqqKgIAA+Pr6olq1atiyZUtJtJGIiIio6B8PHTlyBFu2bIGnpycGDx6Mxo0bw8rKCmZmZti+fTv69u1bEu0kIiKi/3NFnmlJSEiApaUlAPH1KwkJCQCA7777DufOnSve1hERERF9UuSZFktLSzx9+hTVq1eHra0tdu/eDVdXVxw5ckTyBxSp+AXuO481O0IQl5ACeytjLJzYFXXtzeTGHwm5jqX+x/DidQIsTPQxa1QHNG/oIDN2yrJd2HroInzHdcHwnp4l1IPi4b87DH9sO4PY+BQ41jTGUp/uqO9gLjf+4OlrWLT2KJ7HxMPSVB9zx3ZGq0Z5eRCJRFi87ii2HLyI5NQ0uNWyxC/TeqJGdYNS6M23Yx7EvFvZYWwHJxhUVsftZwmYGvg3rj1+Kzd+ZDsHDGlpCxM9DSSkfMShS1GYF3QF6ZnZBWIndKqFOX1c4HfsNmZsvlSS3fjXNu+/gPU7QxCX8A52NarBd/z3qPOF48PR0Aj8suE4XrxOgLmxPqaN9EKzBvaS582aTJS53vRRHTCyd7Nib39x2bDnHFZtF+8XDjWNseTHbqj3hf3i0JnrWLwuGNExCbA01cfsMZ3Q8rP9Ijg0Apv2/4Ub958jMeUDQrdOhZN1+f+jwJv2nYdfkHg82NeohvlfPV9E4OeAvPPFjFEd0Pyz8TBh4XbsOR4utY6nqy22rxhZYn2Qp8gzLYMHD8aNGzcAANOmTcPq1auhpqaGiRMnwsfHp9gbSMCh09cw948D+HFIa/y50Qf2VtXQe5If3ia+kxkffuspRs3dgj5e7jgZ6IM2jZ0wePoG3H/yqkDssbAbuHbnGQz1tEu6G//a/pNXMevXA5jq3RZnt06FY01jdB27GnEJsvNw6cYTeM/ahH6dGiBs2zS096iNfpPX4+6jvDz8tuU01u0Kw4rpvXAqcDIqqqug69jV+Jhe8E9VlBfMg1iXBhZYMMANS/ddh+e0Q7j9LAH7ZrSBnpaazPhujSwxp7czlu29DrdJ+zB23QV0aWCBn3o5F4itW0MPg1rY4vaz+JLuxr925Mx1LFh9EOMHtUZwwI+ws6qG/pPXyT0+XLn1FGPnbUWP9m44GjAZrRo7YvjMjYh8EiOJCT/gK/X4eVovCAQCtPOoVVrdKrIDp67ip98OwGdoW4RsngIHK2N0H79G7n5x+eYTDP9pE/p2aIDQLVPRrkktDJjij3uP8/aLD2kZcKttidk/dCqtbvxrh85cg++qg5g0uA1ObJgMeytj9J209ovnizG+W9Dbyx1/bpyM1o2dMHT6Btz/bDwAQFM3W1w/NE/yWD13QGl0p4AiFy0TJ07EuHHjAAAtWrTA/fv3sWPHDly/fh3jx48v0rYGDRoEgUAgeejq6qJNmza4efPmV9e9c+cOevToAX19faiqqsLa2hqzZ8/Ghw8fpOLMzc0l269YsSKcnJwQEBBQYHsikQj+/v5o0KABtLS0oKGhAQcHB4wfPx6PHj0qUr+K27pdZ9G3Q0P0au8OGwtDLPPpAXVVFQQF/yMzPmB3GJq62WJ03+awNjfE1OHt4WRtgo17z0vFxcQlYdbKfVg9pz+UlZVKoyv/ypodIRjQuSH6dmwAW0sjrJjeCxXVVLDt8N8y49ftPIvmDewwrn8L2FgYYuYoL9S2NYX/njAA4t/52qBQTB7SGu08asGxpjH8fAfg9dtkHA27UZpdKxLmQWx0e0dsOROJHWcfIvJlEiYF/IUPGVno19RaZryrdVVciozF3r+eIDouFaE3X2LfxSeob6UnFVdJVRnrf/DE+PUXkJSaIXNb5UnA7rPo5dUAPdq5wdrcEIt+7A51NRXsPip7dihw7zl4uNpiZO9mqGleFZO928HR2gSb9+cdHwx0taQepy7cRoO6VqheTU/mNssDv6BQ9O/UAH06uMPG0gi/TOsJdTUV7DgiZ7/YdRbN3O0wtn8LWFsYYvpIL9SyMUXAnrzLHHq0c4WPd1t4uNjI3EZ55L/zLPp0aICe7d1gbWGIJT7i8bAzWPZ42LAnDJ5uthjVpxlqmhtiyjDxeAjcJ32+UFFRlhoTlbUqlkZ3Cihy0ZKfmZkZvv/+e9Sq9W0VeJs2bRATE4OYmBicOXMGysrK8PLy+uI6//zzD9zc3JCRkYGjR4/iwYMHWLhwITZt2oSWLVsW+O6YefPmISYmBrdv30a/fv0wbNgwHD9+XPK8SCRCnz59MG7cOLRr1w4nT57E3bt3sWHDBqipqWHBggXf1LfikJGZhZuR0WjskncgFgqFaOxsjau3o2Suc+XOUzR2lt7JPN1scfVOXnxOTg7GztuGUX2awcbSqCSaXqwyMrMQcT8anq55/RIKhfBwtUH4racy17l86yk8XWylljVzt0P4rSgAwLOX8XgTnwJP17wYbQ111HcwR/jNqGLvQ3FgHsQqKAlRx1IPZ2/lvSsWiYCwW6/gUlP2R1qXH7xBHUtd1KshPvGaGWiiZV1TnLr+Qiru56ENcfJ6NMJuFZyZLG8yMrNw68ELfOcsfXz4rn5NXLvzTOY61+5E4bv60oVdE1cbufFxCe8Q8vdd9GzvVnwNL2YZmVm4cT8aHvn3CxcbyTjP78qtqALFSFN3W1yRsx8pgozMLNx88AKN848HZ2up4//nrt6OkooHPp0v8p1f/r7+CLW8ZqFx74WYtnw3EpLfF3fzC6VQ17T8/vvvhd5g7ixMYamqqsLQ0BAAYGhoiGnTpqFx48aIi4uDvr5+gXiRSIShQ4fCzs4O+/fvh1AorrvMzMxgbW2NunXrYuXKlZg6dapkHU1NTclrTJ06FcuWLcOpU6fQtm1bAMCuXbuwc+dOHDp0CB07dpSsV716dbi7u0MkEhWpT8UpIek9srNzoK+jKbVcX0cTj57HylwnLv6dzPjY+BTJz6u2nYGSkhDe3T2Kv9ElID4pVU4etPAw6o3MdWLjU6CvKz8Pbz79mz/GQFc6V+UJ8yCmq6UGZSUh4pLTpJbHJaehZjXZH3Xu/esJdDTVcHyeFwQQoIKyEBtP3sOKg3mzSd83tERtC100m3G4RNtfXBKTxccHvSrSvzs9HU08lnd8SHgHvXzjR6+KJuISZP+u9524jEoV1dCmSfn9aChecpyU/nJTfR1NPHz2hf0iXx4MdDQRGy/7YxRFkJA7HmQc/x/LyUNcwjvo5x8/+cZDUzc7tPOoDVMjHTx7+RZL1h9F/8nrcHjtBCgp/eu5jyIpVNGycuXKQm1MIBAUuWj5XGpqKrZt2wYrKyvo6urKjImIiMDdu3exY8cOScGSq3bt2mjRogWCgoKkipZcOTk5OHDgABITE6GioiJZHhQUBBsbG6mCJX+/5ElPT0d6errk55SU8nmQ/9yN+9EI2BOGkxt9vtg3ov+SRvaGmNSlNiZvuIirD+NgYaiFJYPcMTmxDpbvj4CxbiUsHuiO7xcel3lh7v+r3ccuo3PLelBTrVDWTaEy0qlFPcn/7WpUg12NamjYcwEuXn9UYJampBWqaHn6tOSmy4KDg6GhoQFA/G27RkZGCA4OLlCQ5Hrw4AEAwM7OTubzdnZ2uHDhgtSyqVOnYtasWUhPT0dWVhZ0dHTg7e0ttU0bG+lpwgkTJkiufalcuTJevJCeQs61ePFi+Pr6FqKn30anciUoKQkLXEwWl/AOBvmq6Vz6upqy43XF70Iu3XiMt4mpcO46V/J8dnYOfFcdhP/uMITvm1O8nSgGupU15OQhRdKv/Ax0tRAXLz8PVT/9Gxf/TupC5Nj4d+X2DgHmQSw+5SOysnOgr60utVxfWx2xSWky15nZoz52n3uErSHiY8jd6ERUUlXGyuHf4ZcDEahtoQeDyuo4u6SzZB1lJSEa2hliWGt7VO27CTllOOsqSxVt8fEh/0WWbxPeFZh1yKWvo4m3+cbP20TZ8ZdvPMbj57FYVUYXXRaWruQ4Kf2mUXyc/MJ+kS8PsQnvYKAr+7iqCHRyx4OM47++nOODvo4m4vKPHznjIZeZsR50KldC1Iu4Ui9aSndeR4amTZsiIiICERERuHz5Mlq3bo22bdvi2bNnaNu2LTQ0NCQXxX6uKB/Z+Pj4ICIiAiEhIXBzc8PKlSthZWX1xXVmzpyJiIgIzJ49G6mpqXLjpk+fjuTkZMkjOjq60O0qDJUKyqhlY4oLVx5IluXk5ODC1Qeo72gucx1nBwtcuPpAatm58EjJLbHd2rggZMsUnN7kI3kY6mljdJ9mCCqDW9gKQ6WCMurYmiIsPFKyLCcnB+fCH8DFyULmOq5OFlLxABB66T5cnMwBAGbGuqiqqyUVk5Kahqt3ouBSy7zY+1AcmAexzOwcRDx5Cw+nvOuxBAKgiWM1hD+U/bGIuqoycvIdNrI/LRBAgHO3X6Hh5P1oMvWg5HHtcRz2XHiMJlMPlruCBRCPBydrE/x1Vfr48Ne1h6jnIPsW13oO5vjrmvTx4Xz4A5nxu45egpONCeytjIu34cVMpYIyatua4ly4dB7E+4W5zHWcncxx7op0HsIuR8JZzn6kCFQqKKOWtQkuXH0oWSY5X8i59bu+ozkuXHkotexceKTc8wsAvIpNQmLyB1Qtg7tO/9UfTCwOlSpVkiogAgICoK2tDX9/fwQEBCAtTfyuqUIF8dSktbW4qrt37x7q1q1bYHv37t2TxOTS09ODlZUVrKyssGfPHjg5OcHZ2Rn29uL70GvWrInISOmDur6+PvT19WFg8OXvqVBVVYWqqmoRe100I3p6YvzC7ahtWx117KvDf3cYPnzMQK9PF8aNnb8NhnramDmqAwDAu4cHvh/zO9YGhaB5QwccOn0NN+5H4+epPQGIq3Ed7UpSr6GsrAR9HS1YmVUt0b78G6P7NMNo362oa1cd9RzM4RcUivdp6ejbwR0AMHLOFhjpa2POp9sTR/TyhNeIX7Fq2xm0+s4B+09eRcS95/h1Rm8A4o/9RvZuiuUbT8DSVB9mxrpYtPYoDPW00d6jdpn182uYB7E1R29jzegmuP74La49jsOodo6opKqM7WfFJyK/MU0Qk/AB84KuAABOXH2O0e0dcTMqHlcexsLSUAszetbHiavPkSMSIfVjJu5FJ0q9xoePWUhI/VhgeXni3cMTPy7egVo2pqhtZ4aNe8LwIS0D3duJjw8TF26HoZ42po4Q3+AwuFsT9By3Cut3hqJZA3scOXMdtyKjscSnh9R2373/iKNnb2DWGNkfm5c3o3o3xQ/ztqGOXXXUszfD2p1n8eFjOnp7ifeL0XO3wEi/Mn761J8RPT3RceRvWL39DFo1csD+U9cQce85VkzvJdlmYvJ7vHiTiNdxyQCAR5+uCzHQ1ZLMUJY3w3p5YuLCHahla4q6duLzRVpahuRC6nHzt8FIXxvTR4rPF0O7e6DbD39gbVAoWjS0x6HT13DzfjSWTRGfL95/SMeKwBNo51EbBrqaiHoZj4VrDsPcWA8errZy21FSyrxoyU8gEEAoFCItLQ3GxgWr+zp16sDW1hYrV65Er169pD5GunHjBk6fPo3FixfL3b6pqSl69uyJ6dOn49ChQwCA3r17o0+fPjh06BA6dSp/9+N3alEP8UmpWBZwDHEJKXCoaYIdv4yUTN+9fJMI4WfXprg4WWDN3AFYuv4YFq8LhoWJPgIXD4WtZbWy6kKx+L5VfbxNSsWidUc/fXRhjL2/j5F8zPHidYJUHtxqW8J/wSAs9AvG/DVHYGmqj23Lh8PeKi8P4we0wIe0dExcFITk1DS4166Bvb+PLtef3zMPYgf+fgo9LTXM6FEfBpXVcSsqHt0W/4m45I8AABNdDeR8NrWyfH8ERABm9qwPI52KiE/5iBNXn2P+zqtl1IPi0aF5XcQnpWLFxhOSL5/csnyE5CLTV/mOD85OFvh9dn8sDziGn/2PwtxEH+sXDilwF+GRM9cgEonQsXk9KIIuLesjPikVS9aL9wtHa2Ps/nV03n7xJhFCYV4eXGtZYt38QVi0NhgL/YJhaaqPLcuGwa5G3n5x4vwtjJ2/XfLzsFmbAAA+3m0xdVi70ulYEXVqXg8JSe+xPOC4+HxhZYxtv+QbD0Lp88WqOQOwzP8olq4Xny82LB4K20/jQagkwL3Hr7DneDhSUtNQVU8LHi628BnWDqoqpV9CCERleGvMoEGD8ObNGwQGBgIAEhMTsWrVKvj5+SEkJASenp4y17t48SJatmyJVq1aYfr06TA0NMSlS5fw448/wtTUFCEhIZLZD3Nzc0yYMAETJkyQrH/37l04Ojri8uXLcHZ2hkgkQo8ePRAcHIzp06ejdevWqFq1Kp49e4YlS5bg8uXLiI8v3JdMpaSkQFtbG89iEqClVT4r8dKiplL+v/uFSk+VnhvKugnlwrNN5fv6kNJSSbXcvWcuM//vF36npKTAopoukpOTv3reLPNrWk6cOAEjIyMYGRnBzc0N4eHh2LNnj9yCBQAaNmyIf/75B0pKSmjbti2srKwwffp0DBw4EKdOnfrqxzX29vZo1aoVZs+eDUA8u7Nr1y78+uuvOHbsGJo3bw4bGxsMGTIEpqamBS7sJSIiotL3TTMt58+fx7p16/D48WPs3bsXxsbG2Lp1KywsLPDdd9+VRDsVBmda8nCmhT7HmRYxzrSIcaYlD2daSnCmZd++fWjdujXU1dVx/fp1yXeUJCcnY9GiRd/WYiIiIqKvKHLRsmDBAqxduxb+/v6SO3oAoFGjRrh27VqxNo6IiIgoV5GLlsjISDRp0qTAcm1tbSQlJRVHm4iIiIgKKHLRYmhoKPOvHl+4cAGWlpbF0igiIiKi/IpctAwbNgzjx4/HpUuXIBAI8OrVK2zfvh2TJ0/GqFGjSqKNREREREX/crlp06YhJycHzZs3x4cPH9CkSROoqqpi8uTJGDt2bEm0kYiIiKjoRYtAIMDMmTPh4+ODR48eITU1Ffb29pI/ekhERERUEr75RnkVFRXJ3+4hIiIiKmlFLlqaNm0KwWd/xyK/kJCQf9UgIiIiIlmKXLTUqVNH6ufMzExERETg9u3bGDhwYHG1i4iIiEhKkYuWlStXylw+d+5cpKam/usGEREREclSbH8wsV+/fti4cWNxbY6IiIhISrEVLX///TfU1NSKa3NEREREUor88dD3338v9bNIJEJMTAyuXLmCn376qdgaRkRERPS5Ihct2traUj8LhULY2Nhg3rx5aNWqVbE1jIiIiOhzRSpasrOzMXjwYDg5OaFKlSol1SYiIiKiAop0TYuSkhJatWrFv+ZMREREpa7IF+I6OjriyZMnJdEWIiIiIrmKXLQsWLAAkydPRnBwMGJiYpCSkiL1ICIiIioJhb6mZd68efjxxx/Rrl07AEDHjh2lvs5fJBJBIBAgOzu7+FtJRERE//cKXbT4+vpi5MiRCA0NLcn2EBEREclU6KJFJBIBADw8PEqsMURERETyFOmali/9dWciIiKiklSk72mxtrb+auGSkJDwrxpEREREJEuRihZfX98C34hLREREVBoEotyLVb5CKBTi9evXMDAwKOk2KbSUlBRoa2vjVVwStLS0yro5ZUpJyI8TKc/HDN5ZCABGHZaVdRPKhdfBU8q6CeWG8P/80ouUlBSYVK2C5OTkr543C31NC69nISIiorJU6KKlkBMyRERERCWi0Ne05OTklGQ7iIiIiL6oyF/jT0RERFQWWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQTlsm4AFc6GPeewavsZxManwKGmMZb82A31HMzlxh86cx2L1wUjOiYBlqb6mD2mE1o2cpA8HxwagU37/8KN+8+RmPIBoVunwsnapBR68u/47w7DH9vEeXCsaYylPt1R/wt5OHj6GhatPYrnMfGwNNXH3LGd0eqzPIhEIixedxRbDl5Ecmoa3GpZ4pdpPVGjukEp9ObbMQ9igfvOY82OEMQlpMDeyhgLJ3ZFXXszufFHQq5jqf8xvHidAAsTfcwa1QHNGzrIjJ2ybBe2HroI33FdMLynZwn1oHh4d6iHsd3cYKCjgdtPYjF1zUlci4yRGz+yiwuGtK8LEwMtJKSk4dD5+5i38SzSM7MBABrqKpgxsAm8GlpDr3JF3Hr8BtP8TuP6A/nbLA827juPNds/Gw+TuqLeF8bD4ZDrWLb+GKJzx8PoDmjx2Xj4OeA4Dp2+hpexSVCpoIRaNqaYPqL9F4+95cGGveewZnsIYhNS4GBljEWTuqGewxfycOY6lqw/iujXCbA00cdPYzpK8pCZlY3F64Jx5uJdPHsVD00NNTRxtsFPozvCUF+7tLokwZkWBXDg1FX89NsB+Axti5DNU+BgZYzu49cgLuGdzPjLN59g+E+b0LdDA4RumYp2TWphwBR/3Hv8ShLzIS0DbrUtMfuHTqXVjX9t/8mrmPXrAUz1bouzW6fCsaYxuo5dLTcPl248gfesTejXqQHCtk1De4/a6Dd5Pe4+ysvDb1tOY92uMKyY3gunAiejoroKuo5djY/pmaXVrSJjHsQOnb6GuX8cwI9DWuPPjT6wt6qG3pP88DZRdh7Cbz3FqLlb0MfLHScDfdCmsRMGT9+A+09eFYg9FnYD1+48g6Fe6R+Ui6qLhx0WDG+OpdsvwHPMRtx+8gb7FvaEnnZFmfHdmtpjzhBPLNt+AW7D/DF2xTF08bDDT4M9JTG/TWwLz3rmGLnsCBqN3ICQq09xcEkvGOlqlE6nvsHB09cw93fxeDgZ6AMHq2roPdFP7n4RfuspRs3Zgt4d3HFqkw/aNnHC4GkbpI6TNarrY9GP3XB261Qc8hsPUyMd9Jzgh7eJqaXVrSI7ePoa5vx+AJOHtsHpTT5wqGmMnhO/fL4YMWcz+nRogDObp6Btk1oYODVAkoe0jxm4GfkCkwa3xulNPghcPBSPn8ei/5T1pdktiXJZtAwaNAidO3f+YkxaWhrmzJkDa2trqKqqQk9PD927d8edO3ek4ubOnQuBQACBQAAlJSWYmppi+PDhSEhIKLDN69evo2fPnjAyMoKqqirMzMzg5eWFI0eOQCQSFWcXi8QvKBT9OzVAnw7usLE0wi/TekJdTQU7jvwtM37drrNo5m6Hsf1bwNrCENNHeqGWjSkC9pyTxPRo5wof77bwcLEprW78a2t2hGBA54bo27EBbC2NsGJ6L1RUU8G2w3LysPMsmjeww7j+LWBjYYiZo7xQ29YU/nvCAIhnF9YGhWLykNZo51ELjjWN4ec7AK/fJuNo2I3S7FqRMA9i63adRd8ODdGrvTtsLAyxzKcH1FVVEBT8j8z4gN1haOpmi9F9m8Pa3BBTh7eHk7UJNu49LxUXE5eEWSv3YfWc/lBWViqNrvwro793xZYTN7Dj5C1EPo/HpN9P4EN6Fvq1riUz3tXeBJfuvMDe0LuIfpOM0GtPse/sXdS3MQIAqKkoo+N3tpgbEIqLt6Px9FUilm67gCevEjHEq15pdq1I1u08i74dG6K316fxMEU8HnbKGQ/+n8bDmM/Hg40JAvfljYfvWzmjiYsNzIz1YGtpBN9xXfDu/Ufce/yytLpVZGuDQtFPkgcj/Dzly/uF/+4wNHOzww/9xHmYNqI9atmYYMOn/UJLQx17fx+DTi3qwcqsKpwdLbD4x264cT8aL14XPI+WtHJZtHxNeno6WrRogY0bN2LBggV48OABjh07hqysLLi5ueGff6R/OQ4ODoiJicHz588RGBiIEydOYNSoUVIxhw4dgru7O1JTU7F582bcu3cPJ06cQJcuXTBr1iwkJyeXZhclMjKzcON+NDxc84oLoVAIDxcbhN+KkrnOlVtRBYqRpu62uHLraUk2tURlZGYh4n40PPPnwdUG4XL6dfnWU3i62Eota+ZuJ8nbs5fxeBOfAk/XvBhtDXXUdzBH+M2oYu9DcWAexDIys3AzMhqNXawly4RCIRo7W+Pq7SiZ61y58xSNnaX3C083W1y9kxefk5ODsfO2YVSfZrCxNCqJpherCspC1KlpiLPX8n73IhEQdj0KLvbGMte5fPcF6tQ0RL1PRYqZYWW0dKmBU+GPAQDKSkIoKwnxMSNLar2P6VlwdyifHyHnjocmzvnGg4s1rsgZD1dvP0UTl4LjQV58RmYWth66CC0Nddhbyc5tWcvIzMKNyGipfgmFQjRxscGV27KPD1duR6HJZ/sRAHi62cmNB4CU1I8QCATQ1lQvnoYXgUJe0/Lrr7/i77//xvXr11G7dm0AgJmZGfbt2wc3NzcMHToUt2/fhkAgAAAoKyvD0NAQAGBsbIzu3bsjMDBQsr33799j6NChaN++Pfbv3y/1WnZ2dhg6dGiZzbTEJ71HdnYO9HW0pJbr62ji4bM3MteJjU+Bvo6m1DIDHU3ExsueHlQE8Umpn/Ig3S99HS08jPpCHnTzx2siNj4FAPDm07/5Ywx082LKG+ZBLEGyXxTs16PnsTLXiYt/JzP+8z6u2nYGSkpCeHf3KP5GlwBdrYpQVhIiLumD1PK4xPeoaaorc529oXeho1URx3/pD4EAqKCshI3B17Bip3imLjUtA5fvvoBPn0Z48DwesUnv0c3THi52xnjyKrHE+/QtvjgenskeD7Hx76BfJV98lYJj/uRftzFy9makfcxEVV0t7Pp1FHQrl8+Pyb6chy+dLwqeX+SdLz6mZ2L+mkPo0rIeNCuVftGikDMtO3bsQMuWLSUFSy6hUIiJEyfi7t27uHFD9rR2VFQU/vzzT6ioqEiWnTx5EvHx8ZgyZYrc18wtgPJLT09HSkqK1IOIFM+N+9EI2BOG32b2lbu//xc0qlUdk3o1wORVf8JzTCD6+e5DK9camNynkSRmxLIjEAgEuBc0Fm+Cp2B4Z2fsO3sXOWX4MXlZaVSvJs5snoLgdRPQ1N0Ww3/aJPf6kP+6zKxsDJsVCJEI+HlKjzJpg0IWLQ8ePICdnZ3M53KXP3jwQLLs1q1b0NDQgLq6OiwsLHDnzh1MnTpVansAYGOTN6UWHh4ODQ0NySM4OFjm6y1evBja2tqSh6mp6b/u3+d0K1eCkpIQcQnSxVBcwjsY5KuOcxnoahXYqWIT3sEg3ztpRaJbWeNTHqT7FZeQAgPdL+QhPn/8O0l81U//5o+JjX8nd5tljXkQ05HsFzL6pSN7nOvrasqO/9THSzce421iKpy7zoVJk4kwaTIRL14nwHfVQbh09S2ZjvxL8SkfkJWdA/3K0hfd6lephFg5F4vOHNgEu8/cxtYTN3A3Kg5HLz7A/MAwTOzZALm1WlRMErx8tsO443I49luFFuM2Q1lZiGcxSSXco2/zLePBQFcTcfku2o5LLDjmK6mrwsJEH/UdzbFyRh8oKwnlXh9S1r6YBznHf/H5Qsb5JV98ZlY2vGcGIvp1Avb8PqZMZlmAcl60bN++XapwOH8+7wKponxcY2Njg4iICISHh2Pq1Klo3bo1xo4d+8V1atWqhYiICEREROD9+/fIysqSGTd9+nQkJydLHtHR0YVuV2GoVFBGbVtTnAvPK8JycnJwLvwBXJzMZa7j7GSOc1ceSC0LuxwJZyeLYm1baVKpoIw6tqYIC4+ULMvLg+x+uTpZSMUDQOil+5K8mRnroqqullRMSmoart6Jgkst82LvQ3FgHsRUKiijlo0pLlyR3i8uXH2A+o7mMtdxdrDAhavS+8W58EjJreLd2rggZMsUnN7kI3kY6mljdJ9mCFoxsqS68q9kZuUg4uFreNQ1lywTCIAmdcwQflf2xaLqqsoFZkyyc0Sf1pWeYfqQnok3Ce+hraGG5vUtcezvh8XbgWKSOx7OX803Hq48gLOc8VDf0QLn8x0nz12OlBuft10R0jNknw/KmkoFZdS2MZXqV05ODs5fiYSzo+zjg7OjeYE8hF2+LxWfW7A8fRGHvb+PgY52pZLpQCGU66KlY8eOksIhIiICzs7OAABra2vcu3dP5jq5y62t8y4sUlFRgZWVFRwdHbFkyRIoKSnB1zfvnVPNmjUBAJGReQdtVVVVWFlZwcrK6ottVFVVhZaWltSjuI3q3RRbD13EzqOX8ODpa0xeuhsfPqajt5c7AGD03C2Yv/qwJH5ET0+E/H0Xq7efwcOo11jqfwwR957Du3sTSUxi8nvcevACkU9fAwAePXuDWw9eSK5vKI9G92mGLQcvIij4H0Q+fY1JS3bhfVo6+nYQ52HknC3wXXVIEj+ilyfO/H0Xq7adwYOo11iy/igi7j3HsE/XKwgEAozs3RTLN57AsbCbuPPoJUbN3QpDPW2096gtsw3lAfMgNqKnJ7Yf+Ru7j13Gg6jXmLp8Dz58zECv9m4AgLHzt2Gh3xFJvHcPD4T+cw9rg0Lw8NkbLN9wHDfuR2NIt8YAAB3tSrC1rCb1UFZWgr6OFqzMqpZJHwtjzf7LGNC2Dnq1cIK1qS5WjG2DSmoVsP3kTQCAn48XZg/Ou0bnxD+PMLh9PXzvYYfqVbXhWc8cMwY2wYlLD5HzqXhpVt8CzZ0tJc8fWdYHD6LjJdssj0b08sT2w39jV+54+PnTePASj4cf5kmPh2GfxoPfjhA8jHqDnwPE42FwV/F4eJ+WjkVrj+Dq7ShExyTgxv1oTFi4A6/fJqNDszpl0cVCGdm7KbYd/nS+iHoNn2W7pfIwxncrFqzJO18M6+GBkH/uYc2nPCwLOIYb96Mx9NN+kZmVjaEzNuDG/edYM3cAsnNEeBOfgjfxKcjILP3irVxfiKupqQlNzYJTWr169cLMmTNx48YNqetacnJysHLlStjb2xe43uVzs2bNQrNmzTBq1ChUq1YNrVq1go6ODpYuXYoDBw6USF/+jS4t6yM+KRVL1h9FbPw7OFobY/evoyXTmC/eJEIozHuH5FrLEuvmD8KitcFY6BcMS1N9bFk2DHY1qkliTpy/hbHzt0t+HjZrEwDAx7stpg5rVzodK6LvW9XH26RULFonzoOTtTH2/j4mLw+vEyD87J2iW21L+C8YhIV+wZi/5ggsTfWxbflw2Fvl5WH8gBb4kJaOiYuCkJyaBvfaNbD399FQU61Q6v0rLOZBrFOLeohPSsWygGOIS0iBQ00T7PhlpOSiwpdvEqXy4OJkgTVzB2Dp+mNYvC4YFib6CFw8FLaW1eS9hEI4EHYPetoVMWNAYxhUqYRbT2LRbeZuycW5JvpakmIEAJbv+AsiETBzkAeMdDUQn/wBJ/55hPmbwiQxWpVUMXuwJ6rpaSLx3Ucc+SsSCwLDkJWdU9rdK7TOuePBP288BK3INx6E+caDb77xsGSo5DipJBTi0bNY7D62EQnJqaiiXQl1bKvj4JpxsC3Hd5Z1blEP8Yni/UL85ZMm2LlylORygvx5cK1libW+A7F4/VEsWnsElqYG2LzUW5KHmLgknDh/GwDQbMBSqdc6sHosGtWrWUo9ExOIyvILSOQYNGgQkpKScPDgQZnPf/z4EZ6ennj16hV++eUXuLm54c2bN1i0aBFOnTqF06dPw91d/K5z7ty5OHjwICIiIqS24ebmBhcXF6xatQoAcODAAfTs2RMtW7bEuHHjULNmTaSmpuLEiROYOnUqDh8+jA4dOny17SkpKdDW1saruKQSmXVRJErC/+7FjFR0HzOyy7oJ5YJRh2Vl3YRy4XWw/Bsf/t8I/8MXfhdGSkoKTKpWQXJy8lfPm+X64yF51NTUEBISggEDBmDGjBmwsrJCmzZtoKSkhH/++UdSsHzJxIkTERAQILkGpUuXLrh48SIqVqyIAQMGwMbGBs2aNUNISAh27twJLy+vku4WERERfUG5nGlRZJxpycOZFvocZ1rEONMixpmWPJxp+Y/PtBAREdH/HxYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBCUy7oB/1VKQgGUhIKybgZRuSHk/gAAeHHIp6ybUC4Yei0r6yaUG2+PTyvrJpSpohwbONNCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkE5bJuABWO/+4w/LHtDGLjU+BY0xhLfbqjvoO53PiDp69h0dqjeB4TD0tTfcwd2xmtGjlInheJRFi87ii2HLyI5NQ0uNWyxC/TeqJGdYNS6M23Yx7EmAexjXvPYc32EMQmpMDeyhiLJnVDPQczufGHz1zH0vVHEf06ARYm+vhpTEe0aJiXh58DjuHgqWt4GZsElQpKqGVjiukjvb6Y2/Jg0/7zWBsUgriEd7CrUQ3zJ3RFXXv5eQgOjcDPAcfw4nUCzE30MWNkBzRvYC8V8zDqNRatPYJ/Ih4jKzsH1uZVsX7BEBhXrVLS3flm3h3qYWw3NxjoaOD2k1hMXXMS1yJj5MaP7OKCIe3rwsRACwkpaTh0/j7mbTyL9MxsAICGugpmDGwCr4bW0KtcEbcev8E0v9O4/kD+NsuDDXvOYdV28fHBoaYxlvzYDfW+MIYPnbmOxeuCER2TAEtTfcwe0wktPx0fMrOysWhtME5fvINnL+OhqaEGDxcb/DSmE4z0tUupR3k406IA9p+8ilm/HsBU77Y4u3UqHGsao+vY1YhLeCcz/tKNJ/CetQn9OjVA2LZpaO9RG/0mr8fdR68kMb9tOY11u8KwYnovnAqcjIrqKug6djU+pmeWVreKjHkQYx7EDp6+hjm/H8CPQ9vg1CYfONQ0Rq+Ja+TmIfzmE4ycsxl9OjTA6c1T0LZJLQyaGoB7j/PyYGlqgEU/dsfZbdNweO0EmBrpoOf4NXibKHub5cHhM9cwb9VBTBzUBscDJsPeyhj9flwrt81Xbj3FGN8t6NXeHSc2TEabxk7wnrEB95/knYijXr5FlzG/o0b1qtjz+w84tWkKxg9sDVWV8vs+t4uHHRYMb46l2y/Ac8xG3H7yBvsW9oSedkWZ8d2a2mPOEE8s234BbsP8MXbFMXTxsMNPgz0lMb9NbAvPeuYYuewIGo3cgJCrT3FwSS8Y6WqUTqe+wYFTV/HTbwfgM7QtQjZPgYOVMbqPl79fXL75BMN/2oS+HRogdMtUtGtSCwOm+Ev2i7SPGbgZGY0fh7TBmS1TsHmJNx49j0W/yetKs1sS5apoGTRoEAQCgeShq6uLNm3a4ObNm3LXiYqKgkAgQEREhNyYixcvol27dqhSpQrU1NTg5OSEFStWIDs7u0BsaGgo2rVrB11dXVSsWBH29vb48ccf8fLly+Lo4jdZsyMEAzo3RN+ODWBraYQV03uhopoKth3+W2b8up1n0byBHcb1bwEbC0PMHOWF2ram8N8TBkD8rnptUCgmD2mNdh614FjTGH6+A/D6bTKOht0oza4VCfMgxjyIrQ0KRb+ODdHbyx02Fkb4eUoPqKuqICj4H5nx63eHoambHcb0aw5rc0NMG9EeTjYm2Lj3vCSma2tneLjawNxYD7aWRpg3vgvevf8oVeCVN+t3nUXvDg3Qs70brC0MsWRyd6ipqWDn0Usy4zfsDYOnqy1G9WmGmuaG8PFuB0drE2zan5eHZeuPopm7PWaN7ghHaxOYG+uh1XeO0KuiWVrdKrLR37tiy4kb2HHyFiKfx2PS7yfwIT0L/VrXkhnvam+CS3deYG/oXUS/SUbotafYd/Yu6tsYAQDUVJTR8TtbzA0IxcXb0Xj6KhFLt13Ak1eJGOJVrzS7ViR+QaHo36kB+nRwh42lEX6Z1hPqairYcUTO8WHXWTRzt8PY/i1gbWGI6SO9UMvGFAF7zgEAtDTUse+PH9C5RT3UNKsKZycLLJncHTfuR+PF64TS7BqAcla0AECbNm0QExODmJgYnDlzBsrKyvDy8vrm7R04cAAeHh4wMTFBaGgo7t+/j/Hjx2PBggXo1asXRCKRJHbdunVo0aIFDA0NsW/fPty9exdr165FcnIyfvnll+LoXpFlZGYh4n40PF1tJMuEQiE8XG0QfuupzHUu33oKTxdbqWXN3O0QfisKAPDsZTzexKfA0zUvRltDHfUdzBF+M6rY+1AcmAcx5kEsIzMLNyOj0dhFOg9NXGxw5bbsPFy9HYUmLtZSy5q62cmNz8jMwtaDF6GloQ6HmsbF1/hilJGZhVsPXqBx/bx+CYVCNHa2xrU7UTLXuXo7Co2dpfPg4WqLq7fF8Tk5OTjz911Ymuqj7yQ/1O4wC17DV+DEOflvHstaBWUh6tQ0xNlreb9LkQgIux4FF3vZv7vLd1+gTk1D1PtUpJgZVkZLlxo4Ff4YAKCsJISykhAfM7Kk1vuYngV3B5MS6sm/k5GZhRv3o+GR//jgYiPZ3/O7cisKHp/tRwDQ1N0WV+QcTwDgXWoaBAIBtDXUi6XdRVHu5vpUVVVhaGgIADA0NMS0adPQuHFjxMXFQV9fv0jbev/+PYYNG4aOHTti/fr1kuXe3t6oWrUqOnbsiN27d6Nnz5548eIFxo0bh3HjxmHlypWSWHNzczRp0gRJSUnF0r+iik9KRXZ2DvR1pN/h6Oto4WHUG5nrxManQF83f7wmYuNTAABvPv2bP8ZANy+mvGEexJgHsYSk93LyoImHz76QBx2tAvGx8dLT5icv3MaI2ZuQ9jETVXW1sPu30dCtXD4/DkhIlp0HvSqaeCQnD3EJ76AnI29xCeLf9dvEVLxPS8fq7WcwxbsdZozqgNBL9zFsViB2/zYGDepalUxn/gVdrYpQVhIiLumD1PK4xPeoaaorc529oXeho1URx3/pD4EAqKCshI3B17Bip3hGIjUtA5fvvoBPn0Z48DwesUnv0c3THi52xnjyKrHE+/Qt4iX7RcFx/uX9It++L2O/yPUxPRO+qw7j+1b1oVkGRUu5m2n5XGpqKrZt2wYrKyvo6soeeF9y8uRJxMfHY/LkyQWe69ChA6ytrREUFAQA2LNnDzIyMjBlyhSZ26pcubLM5enp6UhJSZF6EJHialS/JkI2T0Xw+glo6m6HYbMC5V4P8F+U82n2udV3jhjW0xMONU3wQ78WaNHQHtsO/VXGrSs+jWpVx6ReDTB51Z/wHBOIfr770Mq1Bib3aSSJGbHsCAQCAe4FjcWb4CkY3tkZ+87eleTo/01mVjaGztwIEURYPqVHmbSh3BUtwcHB0NDQgIaGBjQ1NXH48GHs2rULQmHRm/rgwQMAgJ2dncznbW1tJTEPHz6ElpYWjIyMivQaixcvhra2tuRhampa5HZ+iW5lDSgpCQscNOMSUmCgqyVzHQNdLcTF549/J4mv+unf/DGx8e/kbrOsMQ9izIOYTuVKcvLwDga6sq+7MNDVkswmfCm+kroqLEz14exogV9n9oGykpLc6wHKmo627Dy8TZT/u9PX0cRbGXnLfXeuo10JykpCWJsbSsVYmVXFyzdJxdf4YhSf8gFZ2TnQryx90a1+lUqITUyVuc7MgU2w+8xtbD1xA3ej4nD04gPMDwzDxJ4NIBCIY6JikuDlsx3GHZfDsd8qtBi3GcrKQjyLSSrhHn0bXcl+IWOc63zh+JBvPMTK2C8ys7IxdMZGvIhJwL4/fiiTWRagHBYtTZs2RUREBCIiInD58mW0bt0abdu2xbNnz9C2bVtJQePg4PD1jX0iKkRVLBKJIMgdqUUwffp0JCcnSx7R0dFF3saXqFRQRh1bU4SFR0qW5eTk4Fz4A7g4Wchcx9XJQioeAEIv3YeLkzkAwMxYF1V1taRiUlLTcPVOFFxqmRdr+4sL8yDGPIipVFBGLRtTnL/yQLIsJycH569EwtlRdh7qO5pLxQNA2OX7cuMl2xXlICMz64sxZUWlgjKcrE1w4epDybKcnBxcuPpA7i2u9R3NpeIB4PyVSNR3NJdss7ZddTx+HisV8yQ6DsaG5fN258ysHEQ8fA2PuuaSZQIB0KSOGcLvyr6JQl1VucCMSXaO6NO60ueCD+mZeJPwHtoaamhe3xLH/pbOX3mhUkEZtW1NcS5cer8QHx/MZa7j7GSOcwX2i0g4f3Y8yS1YnkTHYd+qH6CjXalE2l8Y5e6alkqVKsHKKu8z04CAAGhra8Pf3x8BAQFIS0sDAFSoUOGr27K2Fl9sdu/ePTRs2LDA8/fu3YO9vb0kNjk5GTExMUWabVFVVYWqqmqh47/F6D7NMNp3K+raVUc9B3P4BYXifVo6+nZwBwCMnLMFRvramPNDJwDAiF6e8BrxK1ZtO4NW3zlg/8mriLj3HL/O6A1AvEOO7N0UyzeegKWpPsyMdbFo7VEY6mmjvUftEu3Lv8E8iDEPYiN7N8W4+dtQx9YUdR3MsH7nWXz4mIFeXm4AgB98t8JQXxuzRncEAAzv4YHOo3+H344QtGjogIOnr+LG/Wgsn9YLAPA+LR2/bjqJ1o0dUVVXGwnJqdi49zxexyWjQ7O6ZdbPrxne0xMTF+1AbVtT1LGrjoA9YUhLy0DPduI8jF+wDYZ62pg+sgMAYGg3D3Qb+wfW7QxF8wb2OHTmGm7ej8ZSn56SbY7s3Qyj52yGW+0aaFjPCmcv3cfpi3ew5/cfyqSPhbFm/2WsmeyF6w9e41rkK4zq4oJKahWw/aT4AmI/Hy/EvH2HeYHiu+ZO/PMIo793xc1Hb3Dl/itYGlfBjIFNcOLSQ+R8Kl6a1beAQCDAw+h4WBpXwTzvZngQHS/ZZnk0qndT/DBvG+rYVUc9ezOs3XkWHz6mo7eX+Pgweu4WGOlXxk9jxPvFiJ6e6DjyN6zefgatGjlg/6lriLj3HCumi/eLzKxsDJ62ATcjo7HjlxHIzhFJroOrolURKhVKt4wod0VLfgKBAEKhEGlpaTA2LtoV/K1atYKOjg5++eWXAkXL4cOH8fDhQ8yfPx8A0K1bN0ybNg3Lli2TuhA3V1JSktzrWkra963q421SKhatO4rY+HdwsjbG3t/HSKZ/X7xOgPCzdwZutS3hv2AQFvoFY/6aI7A01ce25cNhb1VNEjN+QAt8SEvHxEVBSE5Ng3vtGtj7+2ioqX69GCwrzIMY8yDWuUU9xCemYlnAsU9fomWCoJWjJNPgL98kQijMy4NLLUv4+Q7EkvVHsWjtEViYGmDTUm/Y1RDnQUkoxKNnb7D72GUkJKeiinYl1LGrjkN+42FrWbSPjUtTx+b1EJ/0Hss3HEfcpy/Z27p8hOTiypdvEqXGg7OTBVbNGYBl/kexdH0wLEz0EbBoqFQf2zaphcWTu2PVttOY/dt+1Kiuj/XzB8O1lmWp96+wDoTdg552RcwY0BgGVSrh1pNYdJu5W3Jxrom+lqQYAYDlO/6CSATMHOQBI10NxCd/wIl/HmH+pjBJjFYlVcwe7IlqeppIfPcRR/6KxILAMGRl55R29wqtS8v6iE9KxZL14uODo7Uxdv86Ou/4kG+/cK1liXXzB2HR2mAs9AuGpak+tiwbJtkvYmKTcOL8LQCAZ/+lUq91cM04fFe/Zin1TEwgKsxnJ6Vk0KBBePPmDQIDAwEAiYmJWLVqFfz8/BASEgJPT88C60RFRcHCwgI7d+6EjY30bVsODg44dOgQevXqhSFDhuCHH36AlpYWzpw5Ax8fHzRv3hy7d++WTAWuWbMGP/zwAwYPHowBAwbA3NwcL168wJYtW6ChoVGo255TUlKgra2NN/HJ0NIqn9cDEJWFjKzye6AvTZnMAwDApNPPZd2EcuPt8Wll3YQylZKSgmr6lZGc/PXzZrmbaTlx4oTk4xlNTU3Y2tpiz549MguWz/Xq1avAsujoaHTr1g2hoaFYuHAhGjdujI8fP6JmzZqYOXMmJkyYIPXZ5ejRo2FtbY3ly5ejS5cuSEtLg7m5Oby8vDBp0qRi7ScREREVTbmaafkv4EwLkWycaRHjTIsYZ1rycKal8DMt5e7uISIiIiJZWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQlAu6wb8V+XkiJCTIyrrZpQpgaCsW1A+CJgIAICykHkAgAqqSmXdhHIh4cS0sm5CuaHjOrasm1CmRNkZhY7lTAsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtBAREZFCYNFCRERECoFFCxERESkEFi1ERESkEFi0EBERkUJg0UJEREQKgUULERERKQQWLURERKQQWLQQERGRQmDRQkRERAqBRQsREREpBBYtREREpBBYtCiIgD3nUKfzHFRrPBEthyzH1TtRX4w/dOY63HrMR7XGE/Fdn0U49dcdqeePhEag69jVsGo5FbpuY3HrwYsSbH3xCdhzDrU7zYHRdxPRYvDX83Dw9HW4dZ8Po+8molFv2Xn4fuxq1GgxFTquipMH/91hqNVxNgwbTUCLQT8XIg/X4NptPgwbTUDDXgtxMl8eRCIRFq0Nhm2bGTD6biI6j/4Dj5/HlmAPigf3CzHuF2LMg5h39ya4ccgXMRdW4lTgZNSzN5Mbq6wkhI93G1w7MAcxF1bi/PZpaN7ArkCckb421s0bgMenluLV+RX4K2gG6thVL8luyMSiRQEcOHUVP/12AD5D2yJk8xQ4Whmj+/g1iEt4JzP+8s0nGPbTJvTr0AChW6aiXZNa6D/FH/cev5LEfEjLgHttS8z5oVNpdeNf23/qKmb9egBTvNsidMsUONY0Rrdx8vNw6VMe+nZsgLNbp6KdRy308/HHXUXPw0lxHqZ6t8XZrVPhWNMYXceulp+HG0/gPWsT+nVqgLBt09Deozb6TV6Pu4/y8vDbltNYtysMK6b3wqnAyaioroKuY1fjY3pmaXWryLhfiHG/EGMexLq0rIcFE7pgacBxePZfitsPX2LfH2OgV0VDZvysUR0wqMt3mPrzHrj3XIDA/RewddkwOFmbSGK0NdVxImASMrNy0H38Grj3XIhZv+5HUsqH0uqWRLkvWgYNGoTOnTvLfd7T0xMTJkyQ+3xCQgImTJgAMzMzqKiooFq1ahgyZAieP39eIPb169cYO3YsLC0toaqqClNTU3To0AFnzpwphp58uzVBoejfqQH6dnCHraURfpnWE+pqKth+5G+Z8et2nUVzdzuM7d8CNhaGmDHSC7VsTBGw55wkpmc7V/h4t4WHi01pdeNfW7MjFAM65+VhxbSeqPilPOwU52HcpzzMHOmFWramCNgtnYcp3m3h6apIeQjBgM4N0bdjA3EepvdCRTUVbDv8hTw0+CwPo7xQ29YU/nvCAIhnWdYGhWLykNZo51ELjjWN4ec7AK/fJuNo2I3S7FqRcL8Q434hxjyIje7TDFsOXsSOI/8g8ulrTFq8Ex8+ZqBfxwYy43u0c8XKTSdx6uJdPHsZj437LuDUxbv4oV8zScyEgS3x8k0ifpi3DdfuPsPzV/EIvXQfUS/flla3JMp90fJvJCQkwN3dHadPn8batWvx6NEj7Ny5E48ePYKLiwuePHkiiY2KikL9+vUREhKCn3/+Gbdu3cKJEyfQtGlTjBkzpsz6kJGZhRv3o+Hx2U4jFArh4WKD8FtRMtcJvxVV4KDbzN0W4beelmRTS5QkDy5FzIPrfy8PEfejpQ6iQqEQHq42cvt1+dZTeLrYSi1r5m4nyduzl/F4E58CT9e8GG0NddR3MEf4zahi70Nx4H4hxv1CjHkQq6CshDq2pjh7OVKyTCQSIexyJFycLGSuo1pBucCM6sf0DLjXriH5uU1jJ1y/9xyBi4fgwZ+LEbZtKgZ0blgynfgK5TJ51VIyc+ZMvHr1Co8ePYKhoSEAoHr16vjzzz9Rs2ZNjBkzBsePHwcAjB49GgKBAJcvX0alSpUk23BwcMCQIUPKpP0AEJ/0HtnZOTDQ0ZJabqCjiYfP3shcJzY+Bfo6mlLL9HU0ERsve5pUEeTmQT9fHvR1NPHgC3kwyJcHAx1NxMqZLlYE8Umpn/KQ//erhYdRXxgPurLGQwoA4M2nf/PHGOjmxZQ33C/EuF+IMQ9iupU1oKysVOAjsbiEFNQ0rypznZB/7mF032a4eP0Rnr54Cw8XG3g1rQMloUASY26shyFdG2PNjhCsCDyJeg5mWPJjN2RkZmPn0Usl2qf8/rNFS05ODnbu/F979x7VxLX9AfwbXiFAeMpbRBERsIjWB9L+1lW6xIAtWpWilvbCrVIVqogvan2LitfWcvVWhAsIfaBQa31hhaq1ytVq7dUolYgiIlhDpSJoFEHI+f2RZiSEtwjE7s9aWcvMnDlz9vZksjOZIRkIDg7mChYlgUCA8PBwLF++HBUVFQCA7OxsrF+/XqVgUTI1NW12PzU1NaipqeGe37/fMw/yhBBCSGMfbv4GW5ZNx8+7V4Axhhu//YGdB88gOGAU10ZLiwexpAQx8QcBAHlXb8HNyRb/mPx/VLR0lvLyclRWVsLNTf0qaABwc3MDYwyFhYUAFKfQXF1dm2zbktjYWKxZs+aZxtoSC1NDaGtr4U6FajF0p+KB2qdMJSsL4yYq7QewavRJWpMo81DeKA/lFQ9gbdF8Hhp/alLkTZPzYPRnHtQ/SVm1kIfyu03NB0V7Zf7K7z6ATS8Trs2duw9ULsbrSeh1oUCvCwXKg8LdShnq6uqbPBPb3FnTu5UyvLM4CXw9HZibGEJaXoXVH0xE8e27XJvf/7iPK0VlKttdLS5DwGtDOj2G1mjMNS3p6ekwMjLiHrm5uW3ajjHWKW2as3TpUlRVVXGP0tLSDvfVFD1dHXi6OuDkuavcMrlcjpPnrmKER98mtxnh0Rcnf7mqsuzHFr7T1ATN5eHEL63k4VyjPJzV/DwMcXXAiXNPv7N+Oh+ajmukRz+V9gBw/OwVLm+O9hawtjBWaXNfVo3/XS7GiMF9Oz2GzkCvCwV6XShQHhSe1NVD3OjaHh6Ph7+NcGn1Wp2a2jpIy6ugo62FgNeG4PCJS9y6sxeLMMDRSqV9/z5WuFVW0bkBtIHGnGmZMGECvLy8uOf29vYttre0tISpqSkkEkmT6yUSCXg8HpydnQEo/mOvXLnS7nHx+Xzw+fx2b9ce4dN9ELH2Kwxx64OX3R2RmPEjHj2uwdtvKE7fzVn9BWwtTbEyYgIAYNbUMQiYvQXb0o/B99VB2HvkPMSSEsQtncb1ea/qIW79fg9l5VUAgMI/v/e1sjBu9pNJdwt/2wcRa/7MwyBHJGT8iEfVDfKw6gvYWjXIw7QxCJi1BZ+lH8O4Vwfh2+//zMNHzedBeT2ElbkxrHv11Dy8hvA1X2KoWx+8PKgvtu86jofVNdzp3NmrvoCtpQl3m+asaWPwxqx/4bOvjmHc/w3Ct9//D2JJCf710XQAirk/e7oPPtmRDScHSzjaW2BDwiHY9DLB66M9uy3O1tDrQoFeFwqUB4X4nT8gftW7uCApwfnLxZgz3QeGAj7SD54BAGxf/S6k5VVYu+0AAGDYIEfYWpki7+ot2FmaIvr98dDS4mHLF0ef9rnrB+SkLMSC0HHYe/Q8hg3qi5BJryJqw64uj09jihahUAihsO2n7bS0tBAUFIT09HSsXbtW5bqW6upqxMfHQyQSwdzcHAAgEomwbds2zJs3T+26lsrKyhava3neJvkOwx+VMmz8zyHcufsAL7nY4+t/hXOn93/7/R60Glw0NXKwE/4TE4r1CVlYtz0LTg6W+HJTGNz623FtDufmYW5MOvd85vI0AMCSmf6IDhvfNYG102TfYbh7T4bYBnnYveVpHm41yoPXn3nYkJCFdfGKPHz1cRjcG+Xhg7UN8rAsDYAiDx++30PzME4xHzYkHvrzKxx7fLM14mkeyiqgxWuQB08nJK0LxfrtWYiJP6jIwyfvw935aR4i/z4Wj6prELVhF6pk1Rjl2R/fbA2HPl+3y+NrK3pdKNDrQoHyoLD3yHn0MjXCR7Neh5WFEHlXf0PgvKd/x6m3jTnkDb5d4PN1sWz2G+hr3wsPq2tw5NRlzF75Be7Lqrk2F/JL8O7iJKyMmIDFM/1x8/ZdfPTpHuzO/qXL4+OxZ/lupAuEhoaisrIS+/bta3L9mDFjYG9vj8WLF6sst7W1hY6ODry8vCAQCLBp0ya89NJLuHHjBpYvX46CggL89NNPcHJyAgAUFRXh1Vdfhbm5OdauXYvBgwejrq4OR44cwfbt25s9Y9PY/fv3YWJiAml5JYyNe2Yl3lUavG/+pfEoEQAAubxHH2q6DE0H0pj5yLndPYRuxeprUZOXhKqqqlbfNzXmmpaW7Ny5E0OHDlV5JCUlwcLCAmfOnIGPjw9mzZqF/v37IygoCP3798e5c+e4ggUAnJyccP78efj4+GDhwoV46aWX4Ovri2PHjmH79u3dGB0hhBBCAA0406Jp6EzLU/SJUoHOtCjQmRYFmg6kMTrT8hc700IIIYSQFx8VLYQQQgjRCFS0EEIIIUQjUNFCCCGEEI1ARQshhBBCNAIVLYQQQgjRCFS0EEIIIUQjUNFCCCGEEI1ARQshhBBCNAIVLYQQQgjRCFS0EEIIIUQjUNFCCCGEEI1ARQshhBBCNAIVLYQQQgjRCFS0EEIIIUQjUNFCCCGEEI1ARQshhBBCNAIVLYQQQgjRCFS0EEIIIUQjUNFCCCGEEI1ARQshhBBCNAIVLYQQQgjRCFS0EEIIIUQjUNFCCCGEEI1ARQshhBBCNIJOdw/gRcMYAwA8eHC/m0fS/Xi87h5Bz8CjRAAA5HLW3UPoEWg6kMZYfW13D6FbKeNXvn+2hIqWTvbgwQMAgItTn24eCSGEEKI5Hjx4ABMTkxbb8FhbShvSZnK5HLdv34ZQKOy2T9j379+Hg4MDSktLYWxs3C1j6AkoDwqUBwXKgwLlQYHyoNAT8sAYw4MHD2BnZwctrZavWqEzLZ1MS0sLvXv37u5hAACMjY3/0i9GJcqDAuVBgfKgQHlQoDwodHceWjvDokQX4hJCCCFEI1DRQgghhBCNQEXLC4jP52PVqlXg8/ndPZRuRXlQoDwoUB4UKA8KlAcFTcsDXYhLCCGEEI1AZ1oIIYQQohGoaCGEEEKIRqCihRBCCCEagYoWQgghhGgEKlpeMKWlpXjvvfdgZ2cHPT09ODo6IjIyEnfv3u3uobVZaGgoeDwe97CwsICfnx8uXbrU7DbFxcVq24wbNw4XLlzg2owZM0aljfIxe/Zsrk3D5cbGxhgxYgT279//XONti9DQULz55pvNrm8Ym76+Ptzd3REfH8+tT0tLazJ2fX19lX0ol+vq6qJfv35YsmQJHj9+/DxDa1ZH5oHS5cuXERQUBEtLS/D5fLi4uGDlypV49OiRSru+ffty/RsYGMDDwwPJyclq/THGkJSUBG9vbxgbG8PIyAiDBg1CZGQkCgsLOy3m1rQ2DwCguroaq1atgouLC/h8Pnr16oW33noLly9fVmm3evVqLnZtbW04ODjg/fffR0VFhVqfFy5cwNSpU2Fraws+nw9HR0e88cYbOHjwYJt+L6YzPcvxQSwWN9vm9OnTGD9+PMzMzKCvrw8PDw98+umnqK+vV2t7/PhxjB8/HhYWFjAwMIC7uzsWLlyI3377rTNCbJe2HBvmz5/f7PqKigrMnz8fjo6O0NPTg52dHd577z2UlJSotS0rK8PcuXPh5OQEPp8PBwcHBAQE4NixY50QSdtQ0fICKSoqwvDhw3Ht2jXs2rULhYWFSEhIwLFjx+Dt7d3kwain8vPzg1QqhVQqxbFjx6Cjo4M33nij1e2OHj0KqVSKnJwcyGQy+Pv7o7KyklsfFhbG9at8bNq0SaWP1NRUSKVS/PLLL3j11VcRGBiIvLy8zg6x0yljy8/PR1BQECIiIrBr1y5uvbGxsVrsN2/eVOlDmfeioiLExcUhMTERq1at6upQ1MbTnnlw5swZeHl5oba2FocOHcLVq1exfv16pKWlwdfXF7W1qj9Ot3btWkilUvz666945513EBYWhsOHD3PrGWN4++23MW/ePIwfPx7ff/898vPzkZKSAn19faxbt+65xN4RNTU1GDt2LHbs2IF169bh6tWr+O6771BXVwcvLy+cOXNGpf2gQYMglUpRUlKC1NRUZGdnY86cOSpt9u/fj1GjRkEmk+Hzzz+HRCJBdnY2Jk2ahOXLl6OqqqorQwTQ8eNDc/bu3YvRo0ejd+/eOH78OK5cuYLIyEisW7cO06ZNUynMEhMTMXbsWNjY2GDPnj3Iz89HQkICqqqqsHnz5s4Ir8tUVFRg1KhROHr0KBISElBYWIiMjAwUFhZixIgRKCoq4toWFxdj2LBh+OGHH/Dxxx8jLy8P2dnZ8PHxQURERNcNmpEXhp+fH+vduzd79OiRynKpVMoMDAzY7Nmzu2lk7RMSEsImTpyosiw3N5cBYHfu3Glymxs3bjAA7MKFC9yyU6dOMQAsOzubMcbY6NGjWWRkZIv7BsD27t3LPb9//z4DwLZs2dKRUDpNUzlpqKnYBgwYwKZNm8YYYyw1NZWZmJi0ex+TJ09mQ4cO7cCIn11H5oFcLmfu7u5s+PDhrL6+XmWdWCxmPB6Pbdy4kVvm6OjI4uLiVNqZm5uzqKgo7vmuXbsYALZ///5m99lVWpsHGzduZDwej4nFYpXl9fX1bPjw4czd3Z0b76pVq5inp6dKuwULFjAzMzPuuUwmYxYWFmzSpEnN7rMr42es844PSsoYJ0+erLbuwIEDDADLyMhgjDFWWlrK9PT02Pz585vcz71799oVS2foyLFBafbs2czQ0JBJpVKV5Y8ePWL29vbMz8+PW+bv78/s7e2ZTCZT66cr46YzLS+IiooK5OTkIDw8HAKBQGWdjY0NgoODkZmZ2eWncjuDTCbDV199BWdnZ1hYWLR5O2UeGn+ybqu6ujqkpKQAAPT09DrUR3cSCAQdjh0Afv31V5w+fbrHxN6WeSAWi5Gfn48FCxao/fCap6cnxo4dq3L2qSG5XI49e/bg3r17KjHv2rULAwcOxIQJE5rcrrt+GLUpO3fuhK+vLzw9PVWWa2lpISoqCvn5+bh48WKT2xYXFyMnJ0cl9u+//x53797FkiVLmt1nd8ff0eODkjLGRYsWqa0LCAiAi4sLN2d2796N2traZvNhamra7v13F7lcjoyMDAQHB8PGxkZlnUAgQHh4OHJyclBRUYGKigpkZ2cjIiIChoaGan11ZdxUtLwgrl27BsYY3Nzcmlzv5uaGe/fuoby8vItH1jFZWVkwMjKCkZERhEIhDhw4gMzMzFZ/AVSpsrISMTExMDIywsiRI7nl8fHxXL/KR3p6usq206dPh5GREfh8PqKiotC3b18EBQV1anzPU319Pb766itcunQJr732Gre8qqpKLXZ/f3+VbZV5V36nf+fOHSxevLirQ1AbT1vnwdWrVwGgxdeBso1SdHQ09/8dGBgIMzMzzJw5U6XPgQMHqmwzf/58blw95QdSAcVYW4pd2UYpLy8PRkZGEAgE6NevHy5fvozo6GiV/gCoxH/u3DmVOZSVlfU8QmnRsx4fGmptzri6unJtrl27BmNjY9ja2nZ88D1EeXk5KisrW5wvjDEUFhaisLAQjDG4urp28SjVUdHygtHEMylN8fHxgVgshlgsxs8//wyRSAR/f3/cvHkT/v7+3AFr0KBBKtu98sorMDIygpmZGS5evIjMzExYW1tz64ODg7l+lY/Gn6Dj4uIgFotx+PBhuLu7Izk5Gebm5l0Sd2vS09NV3jByc3O5dcqCTCAQICwsDFFRUSrXJwiFQrXYG190qsz72bNnERISgn/84x+YMmVKl8XXWEfnQXteB4sXL4ZYLMYPP/wALy8vxMXFwdnZucVtli1bBrFYjJUrV0Imk3UotmfR0jxoT+wDBw6EWCzGuXPnEB0dDZFIhLlz57a4zeDBg7n/k4cPH6Kurq7DcXRUR+dFS9qSN8ZYt59Zak5Lc6IlbY27p9Dp7gGQzuHs7AwejweJRIJJkyaprZdIJDAzM4OlpWU3jK79DA0NVd44kpOTYWJigqSkJCQnJ6O6uhoAoKurq7JdZmYm3N3dYWFh0eQpSxMTk1bfkGxsbODs7AxnZ2ekpqZi/PjxyM/Ph5WV1bMH9owmTJgALy8v7rm9vT337+DgYCxbtgwCgQC2trZqnzq1tLRajb1h3nfs2AFPT0+kpKRgxowZnRhF27V3Hri4uABQzPehQ4eq9SeRSLg2Sr169eL+v3fv3g0PDw8MHz4c7u7uAIABAwagoKBAZRtLS0tYWlp225xobh64uLhAIpE0uY1yecP49fT0uPxu3LgRr7/+OtasWYOYmBgAitgBoKCgAKNGjQKg+K2a1ubR89bR40NTGs6ZV155RW29RCLh5oKLiwuqqqoglUp73NmWlo4NTbG0tISpqWmL84XH43F55vF4uHLlSucNuIPoTMsLwsLCAr6+voiPj+desEplZWVIT0/H1KlTe+ynhNbweDxoaWmhuroa9vb23JuMo6OjSjsHBwf079+/075jHTlyJIYNG4b169d3Sn/PSigUcrE7OzurXL+kLMjs7e07dJq8MS0tLXz00UdYvny52pzqLq3NgyFDhsDV1RVxcXGQy+Uq2168eBFHjx7F9OnTm+3fwcEBU6dOxdKlS7ll06dPR0FBQY+49V2puXkwbdo0HD16VO26Fblcjri4OLi7u6td79LQ8uXL8cknn+D27dsAgHHjxsHc3Bz//Oc/n18wnaCtx4emKGNs6s6fAwcO4Nq1a9ycCQwMhJ6entodh0oN71Tsai0dG5qipaWFoKAg7Ny5E2VlZSrrqqurER8fD5FIBHNzc5ibm0MkEmHbtm14+PChWl9dGTcVLS+Qzz77DDU1NRCJRDh58iRKS0uRnZ0NX19f2Nvb95g33raoqalBWVkZysrKIJFIMHfuXMhkMgQEBDxTv48ePeL6VT7u3bvX4jbz589HYmJit/wNhs7EGFOLvaysTO3NvaG33noL2tra2LZtWxeO9Kn2zgMej4eUlBTk5+djypQp+Pnnn1FSUoLdu3cjICAA3t7eLf7NCgCIjIzEwYMH8csvvwBQFAKBgYGYNm0a1q5di7Nnz6K4uBgnTpxAZmYmtLW1OzvsDouKisLIkSMREBCA3bt3o6SkBOfOncOUKVMgkUiQkpLS4gcXb29vDB48GBs2bAAAGBkZITk5GYcOHcLrr7+OnJwcFBUV4dKlS9wbd3fE39HjQ0FBgdpXpHp6ekhMTMT+/fvx/vvv49KlSyguLkZKSgpCQ0MRGBjIXdPm4OCAuLg4bNmyBTNmzMCJEydw8+ZNnDp1CrNmzeLOUPU05eXlanH//vvv2LBhA2xsbODr64vDhw+jtLQUJ0+ehEgkwpMnT1Re99u2bUN9fT1GjhyJPXv24Nq1a5BIJNi6dSu8vb27Lpguu0+JdIni4mIWEhLCrK2tma6uLnNwcGBz585lf/zxR3cPrc1CQkIYAO4hFArZiBEj2DfffNPsNi3d0qg0evRolX6VD5FIxLVBo1ueGVPc0unq6srmzJnzrKF12LPc1siY4pbnpmIHwN3u2Nw+YmNjmaWlZZO3Oj5PHZkHSpcuXWJTpkxh5ubmTFdXl/Xv358tX76cPXz4UKVdU7c8M8aYSCRi/v7+3PP6+nqWkJDAvLy8mKGhIdPT02NOTk4sLCyM5efnP3OsbdXaPGCMsYcPH7Jly5YxZ2dnpqury8zNzdmUKVNYXl6eSrumbnlmTHGLN5/PZyUlJdyyc+fOscDAQGZlZcV0dHSYhYUFE4lELCMjo1tuee7o8aGpR2lpKWOMsZMnTzKRSMSMjY2Znp4eGzRoEPvkk09YXV2dWn9HjhxhIpGImZmZMX19febq6soWLVrEbt++/dzibk5bjg1NxR0TE8MYY6y8vJzNnTuXOTg4MF1dXWZtbc1CQ0PZzZs31fq6ffs2i4iIYI6OjkxPT4/Z29uzCRMmsOPHjz+n6NTxGOtBV9gQQgghhDSDvh4ihBBCiEagooUQQgghGoGKFkIIIYRoBCpaCCGEEKIRqGghhBBCiEagooUQQgghGoGKFkIIIYRoBCpaCCGEEKIRqGghhPQooaGhePPNN7nnY8aMafVP7z8PP/74I3g8Xou/q8Lj8bBv374297l69WoMGTLkmcZVXFwMHo8HsVj8TP0QoomoaCGEtCo0NBQ8Hg88Ho/7ZeC1a9eirq7uue/722+/bfNvurSl0CCEaC6d7h4AIUQz+Pn5ITU1FTU1Nfjuu+8QEREBXV1dlV9EVqqtrYWenl6n7Nfc3LxT+iGEaD4600IIaRM+nw8bGxs4Ojpizpw5GDt2LA4cOADg6Vc669evh52dHQYOHAgAKC0tRVBQEExNTWFubo6JEyeiuLiY67O+vh4LFiyAqakpLCwssGTJEjT+ObTGXw/V1NQgOjoaDg4O4PP5cHZ2RkpKCoqLi+Hj4wMAMDMzA4/HQ2hoKABALpcjNjYW/fr1g0AggKenJ7755huV/Xz33XdwcXGBQCCAj4+PyjjbKjo6Gi4uLjAwMICTkxNWrFiBJ0+eqLVLTEyEg4MDDAwMEBQUhKqqKpX1ycnJcHNzg76+PlxdXREfH9/usRDyIqKihRDSIQKBALW1tdzzY8eOoaCgAEeOHEFWVhaePHkCkUgEoVCI3NxcnDp1CkZGRvDz8+O227x5M9LS0rBjxw7897//RUVFBfbu3dvifv/+979j165d2Lp1KyQSCRITE2FkZAQHBwfs2bMHAFBQUACpVIotW7YAAGJjY/HFF18gISEBly9fRlRUFN555x2cOHECgKK4mjx5MgICAiAWizFz5kx8+OGH7c6JUChEWloa8vPzsWXLFiQlJSEuLk6lTWFhIb7++mscPHgQ2dnZuHDhAsLDw7n16enpWLlyJdavXw+JRIINGzZgxYoV+Pzzz9s9HkJeOF32e9KEEI0VEhLCJk6cyBhjTC6XsyNHjjA+n88WLVrErbe2tmY1NTXcNl9++SUbOHAgk8vl3LKamhomEAhYTk4OY4wxW1tbtmnTJm79kydPWO/evbl9McbY6NGjWWRkJGOMsYKCAgaAHTlypMlxHj9+nAFg9+7d45Y9fvyYGRgYsNOnT6u0nTFjBps+fTpjjLGlS5cyd3d3lfXR0dFqfTUGgO3du7fZ9R9//DEbNmwY93zVqlVMW1ub3bp1i1t2+PBhpqWlxaRSKWOMsf79+7OdO3eq9BMTE8O8vb0ZY4zduHGDAWAXLlxodr+EvKjomhZCSJtkZWXByMgIT548gVwux9tvv43Vq1dz6z08PFSuY7l48SIKCwshFApV+nn8+DGuX7+OqqoqSKVSeHl5cet0dHQwfPhwta+IlMRiMbS1tTF69Og2j7uwsBCPHj2Cr6+vyvLa2loMHToUACCRSFTGAQDe3t5t3odSZmYmtm7diuvXr0Mmk6Gurg7GxsYqbfr06QN7e3uV/cjlchQUFEAoFOL69euYMWMGwsLCuDZ1dXUwMTFp93gIedFQ0UIIaRMfHx9s374denp6sLOzg46O6uHD0NBQ5blMJsOwYcOQnp6u1pelpWWHxiAQCNq9jUwmAwAcOnRIpVgAFNfpdJaffvoJwcHBWLNmDUQiEUxMTJCRkYHNmze3e6xJSUlqRZS2tnanjZUQTUVFCyGkTQwNDeHs7Nzm9i+//DIyMzNhZWWldrZBydbWFmfPnsXf/vY3AIozCv/73//w8ssvN9new8MDcrkcJ06cwNixY9XWK8/01NfXc8vc3d3B5/NRUlLS7BkaNzc37qJipTNnzrQeZAOnT5+Go6Mjli1bxi27efOmWruSkhLcvn0bdnZ23H60tLQwcOBAWFtbw87ODkVFRQgODm7X/gn5K6ALcQkhz0VwcDB69eqFiRMnIjc3Fzdu3MCPP/6IefPm4datWwCAyMhIbNy4Efv27cOVK1cQHh7e4t9Y6du3L0JCQvDee+9h3759XJ9ff/01AMDR0RE8Hg9ZWVkoLy+HTCaDUCjEokWLEBUVhc8//xzXr1/H+fPn8e9//5u7uHX27Nm4du0aFi9ejIKCAuzcuRNpaWntinfAgAEoKSlBRkYGrl+/jq1btzZ5UbG+vj5CQkJw8eJF5ObmYt68eQgKCoKNjQ0AYM2aNYiNjcXWrVtx9epV5OXlITU1FZ9++mm7xkPIi4iKFkLIc2FgYICTJ0+iT58+mDx5Mtzc3DBjxgw8fvyYO/OycOFCvPvuuwgJCYG3tzeEQiEmTZrUYr/bt29HYGAgwsPD4erqirCwMDx8+BAAYG9vjzVr1uDDDz+EtbU1PvjgAwBATEwMVqxYgdjYWLi5ucHPzw+HDh1Cv379ACiuM9mzZw/27dsHT09PJCQkYMOGDe2Kd8KECYiKisIHH3yAIUOG4PTp01ixYoVaO2dnZ0yePBnjx4/HuHHjMHjwYJVbmmfOnInk5GSkpqbCw8MDo0ePRlpaGjdWQv7KeKy5K94IIYQQQnoQOtNCCCGEEI1ARQshhBBCNAIVLYQQQgjRCFS0EEIIIUQjUNFCCCGEEI1ARQshhBBCNAIVLYQQQgjRCFS0EEIIIUQjUNFCCCGEEI1ARQshhBBCNAIVLYQQQgjRCP8PQYuXIZRLcLUAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix \n", "\n", "def plot_confusion_matrix(y_preds, y_true, labels):\n", " cm = confusion_matrix(y_true, y_preds, normalize=\"true\")\n", " fig, ax = plt.subplots(figsize=(6, 6))\n", " disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)\n", " disp.plot(cmap=\"Blues\", values_format=\".2f\", ax=ax, colorbar=False)\n", " plt.title(\"Normalized confusion matrix\")\n", " plt.show() \n", " \n", "plot_confusion_matrix(df_tokens[\"labels\"], df_tokens[\"predicted_label\"], tags.names)" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0123456
tokens▁चिन▁राज्य▁(▁प्राचीन▁चीन▁)</s>
labelsB-ORGI-ORGI-ORGI-ORGI-ORGI-ORGIGN
predsB-LOCI-LOCI-LOCI-LOCI-LOCI-LOCI-LOC
losses3.975.405.004.905.364.880.00
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6\n", "tokens ▁चिन ▁राज्य ▁( ▁प्राचीन ▁चीन ▁) \n", "labels B-ORG I-ORG I-ORG I-ORG I-ORG I-ORG IGN\n", "preds B-LOC I-LOC I-LOC I-LOC I-LOC I-LOC I-LOC\n", "losses 3.97 5.40 5.00 4.90 5.36 4.88 0.00" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
01234567
tokens▁एनएसई▁(▁NSE▁)</s>
labelsB-ORGIGNIGNI-ORGI-ORGIGNI-ORGIGN
predsOOOOOOOO
losses1.660.000.006.987.910.007.100.00
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7\n", "tokens ▁एन एस ई ▁( ▁N SE ▁) \n", "labels B-ORG IGN IGN I-ORG I-ORG IGN I-ORG IGN\n", "preds O O O O O O O O\n", "losses 1.66 0.00 0.00 6.98 7.91 0.00 7.10 0.00" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
012345678910
tokens▁मुग़ल▁बादशाहों▁की▁सूची</s>
labelsB-PERIGNIGNIGNI-PERIGNIGNIGNI-PERI-PERIGN
predsB-ORGI-ORGI-ORGI-ORGI-ORGI-ORGI-ORGI-ORGI-ORGI-ORGI-ORG
losses5.800.000.000.004.820.000.000.005.315.510.00
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7 8 9 \\\n", "tokens ▁मु ग ़ ल ▁बाद शा ह ों ▁की ▁सूची \n", "labels B-PER IGN IGN IGN I-PER IGN IGN IGN I-PER I-PER \n", "preds B-ORG I-ORG I-ORG I-ORG I-ORG I-ORG I-ORG I-ORG I-ORG I-ORG \n", "losses 5.80 0.00 0.00 0.00 4.82 0.00 0.00 0.00 5.31 5.51 \n", "\n", " 10 \n", "tokens \n", "labels IGN \n", "preds I-ORG \n", "losses 0.00 " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def get_samples(df):\n", " for _, row in df.iterrows():\n", " labels, preds, tokens, losses = [], [], [], [] \n", " for i, mask in enumerate(row[\"attention_mask\"]):\n", " if i not in {0, len(row[\"attention_mask\"])}:\n", " labels.append(row[\"labels\"][i])\n", " preds.append(row[\"predicted_label\"][i])\n", " tokens.append(row[\"input_tokens\"][i])\n", " losses.append(f\"{row['loss'][i]:.2f}\")\n", " df_tmp = pd.DataFrame({\"tokens\": tokens, \"labels\":labels, \"preds\": preds, \"losses\": losses}).T\n", "\n", " yield df_tmp\n", "\n", "df[\"total_loss\"] = df[\"loss\"].apply(sum)\n", "df_tmp = df.sort_values(by=\"total_loss\", ascending=False).head(3)\n", "\n", "for sample in get_samples(df_tmp):\n", " display(sample)" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
01234567891011
tokens▁मिलोराड▁मज़िक▁(▁सर्बिया▁)</s>
labelsOIGNIGNIGNOIGNIGNOB-LOCIGNOIGN
predsB-LOCI-LOCI-LOCI-LOCI-LOCI-LOCI-LOCOB-LOCI-LOCOO
losses1.380.000.000.001.780.000.000.030.050.000.120.00
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7 8 9 \\\n", "tokens ▁मिल ो रा ड ▁म ज़ िक ▁( ▁सर् बिया \n", "labels O IGN IGN IGN O IGN IGN O B-LOC IGN \n", "preds B-LOC I-LOC I-LOC I-LOC I-LOC I-LOC I-LOC O B-LOC I-LOC \n", "losses 1.38 0.00 0.00 0.00 1.78 0.00 0.00 0.03 0.05 0.00 \n", "\n", " 10 11 \n", "tokens ▁) \n", "labels O IGN \n", "preds O O \n", "losses 0.12 0.00 " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0123456789
tokens▁राजेन्द्र▁सिंह▁(▁रज्जू▁भैया▁)</s>
labelsB-PERI-PERI-PERI-PERIGNIGNI-PERIGNI-PERIGN
predsB-PERI-PEROOOI-PERI-PERI-PEROO
losses0.020.012.003.400.000.000.270.001.300.00
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7 8 \\\n", "tokens ▁राजेन्द्र ▁सिंह ▁( ▁र ज् जू ▁भै या ▁) \n", "labels B-PER I-PER I-PER I-PER IGN IGN I-PER IGN I-PER \n", "preds B-PER I-PER O O O I-PER I-PER I-PER O \n", "losses 0.02 0.01 2.00 3.40 0.00 0.00 0.27 0.00 1.30 \n", "\n", " 9 \n", "tokens \n", "labels IGN \n", "preds O \n", "losses 0.00 " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "df_tmp = df.loc[df[\"input_tokens\"].apply(lambda x: u\"\\u2581(\" in x)].head(2)\n", "for sample in get_samples(df_tmp):\n", " display(sample) " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Cross Lingual Transfer" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "tags": [] }, "outputs": [], "source": [ "def get_f1_score(trainer, dataset):\n", " return trainer.predict(dataset).metrics[\"test_f1\"]" ] }, { "cell_type": "code", "execution_count": 54, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "F1-score of [hi] model on [hi] dataset: 0.798\n" ] } ], "source": [ "f1_scores = defaultdict(dict)\n", "f1_scores[\"hi\"][\"hi\"] = get_f1_score(trainer, panx_hi_encoded[\"test\"])\n", "print(f\"F1-score of [hi] model on [hi] dataset: {f1_scores['hi']['hi']:.3f}\")" ] }, { "cell_type": "code", "execution_count": 55, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
012345678910
Tokens<s>▁నా▁అమ్మ▁కళ్లకు▁నీళ్ళు▁పోయాయి</s>
TagsOB-PERI-PERI-ORGI-ORGOI-ORGI-ORGOOO
\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7 8 9 10\n", "Tokens ▁నా ▁అమ్మ ▁క ళ్ల కు ▁నీ ళ్ళు ▁ పోయాయి \n", "Tags O B-PER I-PER I-ORG I-ORG O I-ORG I-ORG O O O" ] }, "execution_count": 55, "metadata": {}, "output_type": "execute_result" } ], "source": [ "text_te = \"నా అమ్మ కళ్లకు నీళ్ళు పోయాయి\"\n", "tag_text(text_te, tags, trainer.model, xlmr_tokenizer)" ] }, { "cell_type": "code", "execution_count": 56, "metadata": { "tags": [] }, "outputs": [], "source": [ "def evaluate_lang_performance(lang, trainer):\n", " panx_ds = encode_panx_dataset(panx_ch[lang])\n", " return get_f1_score(trainer, panx_ds[\"test\"])" ] }, { "cell_type": "code", "execution_count": 57, "metadata": { "tags": [] }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f6541b0db0ff48c581bb593d1a0918a2", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/55 [00:00" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "6e28b43098b64415862bff66128ccb16", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/6856 [00:00" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ec43bbb1914d4dfbafca85b309e77b03", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/1533 [00:00" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "F1-score of [hi] model pn [te] dataset: 0.525\n", "F1-score of [hi] model pn [ta] dataset: 0.552\n", "F1-score of [hi] model pn [en] dataset: 0.529\n" ] } ], "source": [ "f1_scores[\"hi\"][\"te\"] = evaluate_lang_performance(\"te\", trainer)\n", "f1_scores[\"hi\"][\"ta\"] = evaluate_lang_performance(\"ta\", trainer)\n", "f1_scores[\"hi\"][\"en\"] = evaluate_lang_performance(\"en\", trainer)\n", "print(f\"F1-score of [hi] model pn [te] dataset: {f1_scores['hi']['te']:.3f}\")\n", "print(f\"F1-score of [hi] model pn [ta] dataset: {f1_scores['hi']['ta']:.3f}\")\n", "print(f\"F1-score of [hi] model pn [en] dataset: {f1_scores['hi']['en']:.3f}\")" ] }, { "cell_type": "code", "execution_count": 58, "metadata": { "tags": [] }, "outputs": [], "source": [ "def train_on_subset(dataset, num_samples):\n", " train_ds = dataset[\"train\"].shuffle(seed=42).select(range(num_samples)) \n", " valid_ds = dataset[\"validation\"]\n", " test_ds = dataset[\"test\"]\n", " training_args.logging_steps = len(train_ds) // batch_size \n", " \n", " trainer = Trainer(model_init=model_init, args=training_args,\n", " data_collator=data_collator, compute_metrics=compute_metrics,\n", " train_dataset=train_ds, eval_dataset=valid_ds, tokenizer=xlmr_tokenizer)\n", " trainer.train() \n", " if training_args.push_to_hub:\n", " trainer.push_to_hub(commit_message=\"Training completed!\") \n", " \n", " f1_score = get_f1_score(trainer, test_ds)\n", " return pd.DataFrame.from_dict(\n", " {\"num_samples\": [len(train_ds)], \"f1_score\": [f1_score]})" ] }, { "cell_type": "code", "execution_count": 59, "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/data/ssd2/abhiroop/misc/nlp-with-transformers/env/lib/python3.8/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [33/33 00:04, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
11.7380001.1433640.000000
21.1895000.9400890.109989
31.0587000.8537410.205519

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "

\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
num_samplesf1_score
02500.182665
\n", "
" ], "text/plain": [ " num_samples f1_score\n", "0 250 0.182665" ] }, "execution_count": 59, "metadata": {}, "output_type": "execute_result" } ], "source": [ "panx_ta_encoded = encode_panx_dataset(panx_ch[\"ta\"])\n", "training_args.push_to_hub = False \n", "metrics_df = train_on_subset(panx_ta_encoded, 250)\n", "metrics_df" ] }, { "cell_type": "code", "execution_count": 60, "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/data/ssd2/abhiroop/misc/nlp-with-transformers/env/lib/python3.8/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [63/63 00:07, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
11.4587000.8958340.156504
20.8011000.6399080.489311
30.6169000.5603480.525695

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [126/126 00:14, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
11.1495000.6057090.445705
20.4765000.4139040.620573
30.3330000.3805780.683262

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [252/252 00:28, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
10.8233000.3789210.660228
20.3351000.3009350.726016
30.2269000.2990770.750000

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [501/501 00:55, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
10.5809000.2971010.725979
20.2641000.2390290.749110
30.1773000.2407490.776014

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "for num_samples in [500, 1000, 2000, 4000]:\n", " result = train_on_subset(panx_ta_encoded, num_samples)\n", " metrics_df = pd.concat([metrics_df, result], ignore_index=True)" ] }, { "cell_type": "code", "execution_count": 61, "metadata": { "tags": [] }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAG2CAYAAACDLKdOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABS10lEQVR4nO3deVxU5f4H8M+wzMCwDCgwgCKoEOKGC4poqRWJ2vVq3X6aWqKZZVpqpLncEs0S7VbXFsvSQitLW8xMja6RWCru4i4JLpAKiMqwyTbz/P4wjowsggycYfi8X695xZzzzMz3YUbm0/M85xyFEEKAiIiIyEJYyV0AERERkSkx3BAREZFFYbghIiIii8JwQ0RERBaF4YaIiIgsCsMNERERWRSGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFkTXc/P777xg2bBi8vb2hUCiwcePGOz4mISEBPXr0gEqlgr+/P1avXt3gdRIREVHTIWu4KSgoQHBwMJYvX16r9ufOncPDDz+M+++/H0lJSZgxYwaefvpp/PLLLw1cKRERETUVCnO5cKZCocAPP/yAESNGVNtm9uzZ2LJlC44fPy5te/zxx5GTk4O4uLhGqJKIiIjMnY3cBdRFYmIiwsPDjbZFRERgxowZ1T6muLgYxcXF0n2DwYBr166hZcuWUCgUDVUqERERmZAQAnl5efD29oaVVc0TT00q3GRkZECr1Rpt02q1yM3NxY0bN2Bvb1/pMTExMVi4cGFjlUhEREQNKD09Ha1bt66xTZMKN3dj7ty5iIqKku7rdDq0adMG6enpcHZ2lrEyIiIiqq3c3Fz4+PjAycnpjm2bVLjx9PREZmam0bbMzEw4OztXOWoDACqVCiqVqtJ2Z2dnhhsiIqImpjZLSprUeW7CwsIQHx9vtG3btm0ICwuTqSIiIiIyN7KGm/z8fCQlJSEpKQnAzUO9k5KSkJaWBuDmlNK4ceOk9pMnT8bZs2fx8ssv4/Tp0/jwww/xzTff4MUXX5SjfCIiIjJDsoabAwcOoHv37ujevTsAICoqCt27d8f8+fMBAJcvX5aCDgC0bdsWW7ZswbZt2xAcHIy3334bq1atQkREhCz1ExERkfkxm/PcNJbc3FxoNBrodDquuSEiImoi6vL93aTW3BARERHdCcMNERERWRSGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFYbghIiIii8JwQ0RERBaF4YaIiIgsCsMNERERWRSGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFYbghIiIii8JwQ0RERBaF4YaIiIgsCsMNERERWRSGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFYbghIiIii8JwQ0RERBaF4YaIiIgsCsMNERERWRSGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFYbghIiIii8JwQ0RERBaF4YaIiIgsCsMNERERWRSGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFYbghIiIii8JwQ0RERBaF4YaIiIgsCsMNERERWRSGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFYbghIiIii8JwQ0RERBaF4YaIiIgsCsMNERERWRSGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFYbghIiIii8JwQ0RERBaF4YaIiIgsCsMNERERWRSGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFYbghIiIiiyJ7uFm+fDn8/PxgZ2eH0NBQ7Nu3r8b2y5YtQ2BgIOzt7eHj44MXX3wRRUVFjVQtERERmTtZw8369esRFRWF6OhoHDp0CMHBwYiIiEBWVlaV7b/66ivMmTMH0dHROHXqFD799FOsX78e8+bNa+TKiYiIyFzJGm7eeecdTJo0CRMmTEDHjh2xYsUKqNVqfPbZZ1W23717N/r164cxY8bAz88PgwYNwujRo+842kNERETNh2zhpqSkBAcPHkR4ePitYqysEB4ejsTExCof07dvXxw8eFAKM2fPnsXWrVsxdOjQal+nuLgYubm5RjciIiKyXDZyvXB2djb0ej20Wq3Rdq1Wi9OnT1f5mDFjxiA7Oxv33nsvhBAoKyvD5MmTa5yWiomJwcKFC01aOxEREZkv2RcU10VCQgIWL16MDz/8EIcOHcKGDRuwZcsWLFq0qNrHzJ07FzqdTrqlp6c3YsVERETU2GQbuXFzc4O1tTUyMzONtmdmZsLT07PKx7z66qt48skn8fTTTwMAunTpgoKCAjzzzDP497//DSuryllNpVJBpVKZvgNERERklmQbuVEqlejZsyfi4+OlbQaDAfHx8QgLC6vyMYWFhZUCjLW1NQBACNFwxRIREVGTIdvIDQBERUUhMjISISEh6N27N5YtW4aCggJMmDABADBu3Di0atUKMTExAIBhw4bhnXfeQffu3REaGoqUlBS8+uqrGDZsmBRyiIiIqHmTNdyMGjUKV65cwfz585GRkYFu3bohLi5OWmSclpZmNFLzyiuvQKFQ4JVXXsHFixfh7u6OYcOG4Y033pCrC0RERGRmFKKZzefk5uZCo9FAp9PB2dlZ7nKIiIioFury/d2kjpYiIiIiuhOGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFYbghIiIii8JwQ0RERBaF4YaIiIgsCsMNERERWRSGGyIiIrIoDDdERERkUWS9cCYRERE1PWV6A3Q3SqG7UYqcv/+be6MUOYU3f3Z3UmF07zay1cdwQ0RE1AwZDAJ5xWXQFZaHlJJbgaXwZljRVQgsFW/5xWU1PnePNi4MN0RERFR3QggUluhvCyHGIeX2YCKFl6JSCFG/13dS2cDZ3hYualtoKvy3vbujaTp4lxhuiIiIZFZUqr81UnKjFLrCW9M9uhul0BWWVDsNVGaoX0Kxt7WGxv5mKNH8HU409rZwsb8VWJyln5XSfmc7G9hYm+fSXYYbIiIiE6i4DqU8hNw+tVP+39zbpoGKSg31em1bawU09kpo7G0qhZCKt4ojLOWBRWVjbaLfgPlguCEiIvrb7etQbl+Loiu8+3Uod2KlwM0pnvKRESmk/B1Y7JUVthsHFntbaygUChP9Fpo+hhsiIrIoVa9DufNalJzCUuQVlaKeszzSOpTqRkrKQ0rF/c72tnBS2cDKigHFFBhuiIjILFW1DqXympOSKqeBSvX1Syh2tlZGIaTSWhT17dM9SrNfh9KcMNwQEVGDKdMbkFtUJoWQqtahVDr0+O8RFtOsQ6kmhFSY/qk4glIeWixxHUpzwnBDREQ1Kl+HUvEkbVWtQ6lquqe+61AUCtS4MLZ8mqeqaSCuQ2m+GG6IiJqBiutQ6npOlNwb9V+H4qiyqTqc3BZSKu7nOhS6Www3RERNSHGZvtqRkvIpn4Zch3Kno3aqmgbiOhRqbAw3RESN7PZ1KEa3Cidva4h1KDZWCmlUxOX2EHLbyds0auPDku1suQ6FmgaGGyKiu1DVOpTarkUxxToUZ7vqRkoq3q98yLFayXUoZPkYboio2apuHcrtZ4+tai1KQ6xDqc1aFI2a61CI7oThhoiavOIyfbVnjzU+9LjyNJCp1qFUd9ROxcWxFaeBnO1tYct1KEQNguGGiMxGqd6ArLxiZOUWVbsOpdJ0z40Sk6xDqThaUjGEaCpco8fltjZch0JknhhuiKhRFJXqkaErwmVdETJyb+CyrgiZ0v2b/83OL4a4y4GU8nUo1R+1U8ValL8XzHIdCpFlYbghonrLKyqtEFyKbv2s+zvE5BbhemFprZ7L1loBd0cVNGplpTPIVjr0uMJaFCc7rkMhopsYboioWkIIXC+8GVzKR1tuBZdbQaa2R//Y2VrBW2MPrbMdvDR28NSU/9cens4377d0UDKkEFG9MNwQNVN6g8DV/GJcrjjKklt5qqikrHbrWZzsbKSg4uV8K7ho//6vl7M9nO1tOP1DRA2O4YbIApWUGZCVd2uUJTO3qMKoyw1k6IqQlVeMsloey9zSQVlhlMXu71EWe6P7Dir+OSEi88C/RkRNzI0SvTQdVN1UUW0X5lopAA+nCqMsRtNFN8OLh7OKV0gmoiaF4YbIjBgtzK1wZFHFxbo5dViYW2mUxdl4qsjdUcVr/hCRxWG4IWoE5Qtzy6eEqpsqKijR1+r57G2tb00J3bYot3x7CzUX5hJR88RwQ1RPeoNA9t8LczPKw0tu5ami2i7MdbazgZfGvtqpIk+NHZztuDCXiKg6DDdENSgpMyAz9/ZRlltTRRm6ImTmFUNfy4W5bo7Kv6eK7OGpUd0MKxXCi6fGDmol/1kSEdUH/4pSs1W+MLf6qaKbC3Nrw0oBaJ1vH2UxXuvChblERI2D4YYsjhACecVlRmfJzdAVVzqySHejdgtzldZW0GpU8HK2r+Jw6JtTRW6OSi7MJSIyEww31OQUleqRkpVf41RRbRfmqpXWtwKLs/2tI4kqnISuhYOS61uIiJoQhhtqUs5lF2Dsyj24pCu6Y1uNvW0Voyy3poq0zlyYS0RkiRhuqMlIv1aIMSv34LKuCE52NvBtqZZGW6qaLuLCXCKi5ol//alJuKy7gTGrbgYbfw9HrHumD9wcVXKXRUREZogrIMnsZeUVYezKvUi/dgO+LdVY+3Qogw0REVWL4YbM2rWCEjyxai/OZheglYs9vprUB1pnO7nLIiIiM8ZwQ2ZLV1iKJ1btxZ+Z+dA6q/DVpFC0crGXuywiIjJzDDdklvKKSjEudh9OXs6Fm6MSa5/uA9+WDnKXRURETQDDDZmdwpIyPLV6P46k58BFbYsvnw6Fv4ej3GUREVETwXBDZqWoVI9Jnx/A/vPX4WRngy8nhqKDp7PcZRERURPCcENmo6TMgOe+PIhdKVfhoLTGmqd6o3MrjdxlERFRE8NwQ2ahVG/AC18fwvbkK7CztcJn43uhRxtXucsiIqImiOGGZKc3CER9cwS/nMiE0sYKK8eFILRdS7nLIiKiJorhhmRlMAjM/v4ofjpyCTZWCnw0tgfuC3CXuywiImrCGG5INkIIzN90HN8d/AvWVgq8P7o7HgzSyl0WERE1cQw3JAshBF7fcgpf7kmDQgG8MzIYQ7p4yV0WERFZAIYbksVb/0vGpzvPAQCWPtoVw7u1krkiIiKyFAw31Ojejz+D5dtTAQCvDe+Ekb18ZK6IiIgsCcMNNaqVv5/F29v+BAD8e2gQxoX5yVsQERFZHIYbajSfJ57HG1tPAQBeeugeTOrfTuaKiIjIEjHcUKNYvz8N8388AQCYen97vPBggMwVERGRpWK4oQa38fBFzNlwDAAw8d62mDkoUOaKiIjIkjHcUIP6+dhlvPTtEQgBPNGnDV55OAgKhULusoiIyILJHm6WL18OPz8/2NnZITQ0FPv27auxfU5ODqZOnQovLy+oVCrcc8892Lp1ayNVS3URfyoTL3x9GHqDwP/1bI3X/tmZwYaIiBqcjZwvvn79ekRFRWHFihUIDQ3FsmXLEBERgeTkZHh4eFRqX1JSgoceeggeHh747rvv0KpVK1y4cAEuLi6NXzzV6I8zV/Dcl4dQZhD4Z7A3lvyrK6ysGGyIiKjhKYQQQq4XDw0NRa9evfDBBx8AAAwGA3x8fPDCCy9gzpw5ldqvWLEC//nPf3D69GnY2tre1Wvm5uZCo9FAp9PB2dm5XvVT1facvYrxsftQVGpARCctPhjTA7bWsg8SEhFRE1aX72/ZvnFKSkpw8OBBhIeH3yrGygrh4eFITEys8jGbNm1CWFgYpk6dCq1Wi86dO2Px4sXQ6/XVvk5xcTFyc3ONbtRwDl64jomr96Oo1ID7A93x/mgGGyIialyyfetkZ2dDr9dDqzW+UKJWq0VGRkaVjzl79iy+++476PV6bN26Fa+++irefvttvP7669W+TkxMDDQajXTz8eHZcBvK8Ys6jI/dh4ISPfr5t8RHT/SE0obBhoiIGleT+uYxGAzw8PDAJ598gp49e2LUqFH497//jRUrVlT7mLlz50Kn00m39PT0Rqy4+TidkYsnPt2LvKIy9PZrgZXjQmBnay13WURE1AzJtqDYzc0N1tbWyMzMNNqemZkJT0/PKh/j5eUFW1tbWFvf+tIMCgpCRkYGSkpKoFQqKz1GpVJBpVKZtngykpKVjydW7UVOYSm6+bjg0/EhUCtlXatORETNmGwjN0qlEj179kR8fLy0zWAwID4+HmFhYVU+pl+/fkhJSYHBYJC2/fnnn/Dy8qoy2FDDu3C1AGNX7UF2fgk6ejljzYTecLK7u8XeREREpiDrtFRUVBRWrlyJNWvW4NSpU3juuedQUFCACRMmAADGjRuHuXPnSu2fe+45XLt2DdOnT8eff/6JLVu2YPHixZg6dapcXWjWLubcwJiVe5GZW4x7tI748ulQaNQMNkREJC9Z5w5GjRqFK1euYP78+cjIyEC3bt0QFxcnLTJOS0uDldWt/OXj44NffvkFL774Irp27YpWrVph+vTpmD17tlxdaLYyc4swZuUeXMy5gbZuDvjy6VC0cODoGRERyU/W89zIgee5qb/s/GKM+jgRqVcK0NrVHt88GwZvF3u5yyIiIgvWJM5zQ01TTmEJnli1F6lXCuClscPXk/ow2BARkVm5q3BTVlaGX3/9FR9//DHy8vIAAJcuXUJ+fr5JiyPzkltUiic/3YfTGXlwd1Jh7dOh8GmhlrssIiIiI3Vec3PhwgUMHjwYaWlpKC4uxkMPPQQnJycsXboUxcXFNZ5zhpquguIyTIjdj2MXdWjhoMTap0PRzt1R7rKIiIgqqfPIzfTp0xESEoLr16/D3v7WdMQjjzxidFg3WY4bJXpMXLMfBy9ch7OdDb6Y2Bv3aJ3kLouIiKhKdR65+eOPP7B79+5K55Xx8/PDxYsXTVYYmYfiMj2e+eIA9py9BkeVDT6fGIpO3hq5yyIiIqpWnUduDAZDlReq/Ouvv+DkxP+btySlegOmrj2MP85kw97WGrETeqGbj4vcZREREdWozuFm0KBBWLZsmXRfoVAgPz8f0dHRGDp0qClrIxmV6Q2YsS4Jv57KhNLGCqsiQ9DLr4XcZREREd1Rnc9zk56ejsGDB0MIgTNnziAkJARnzpyBm5sbfv/9d3h4eDRUrSbB89zcmcEgMPPbI9hw+CJsrRX4ZFwI7g807/eViIgsW12+v+u85sbHxwdHjhzB+vXrceTIEeTn52PixIkYO3as0QJjapqEEPj3xmPYcPgirK0UeH90DwYbIiJqUuo0clNaWooOHTpg8+bNCAoKasi6GoyU/C5dqjr5WVsDdna37hcUVP9kVlZAxUBXl7aFhUB1v3qFAlCr767tjRtAhQuLVuLgUG1bIQQW/pKC1fsvwkoBLHu8O/4Z7H1zZ1ERUMVaqyqf905t1eqbdQNAcTFQVmaatvb2N3/PAFBSApSWmqatnd3Nz0Vd25aW3mxfHZUKsLGpe9uyspu/i+oolYCtbd3b6vU337vq2NrebF/XtgbDzc+aKdra2Nz8XQA3/00UFpqmbV3+3TfjvxE1tuXfiLq35d+Imz/X8m9EnWZeRB15e3uLkydP1vVhZkOn0wkAQnfzT0Hl29Chxg9Qq6tuBwgxYIBxWze36tuGhBi39fWtvm3HjsZtO3asvq2vr3HbkJDq27q5GbcdMEDaZwDE4gHjhe/szcJ39mbxbY8hxm2HDq3+eW//GD32WM1t8/NvtY2MrLltVtattlOm1Nz23LlbbWfOrLnt8eO32kZH19x2375bbd98s+a227ffavvBBzW33bz5VtvY2JrbfvPNrbbffFNz29jYW203b6657Qcf3Gq7fXvNbd9881bbfftqbhsdfavt8eM1t50581bbc+dqbjtlyq22WVk1t42MvNU2P7/mto89JozU1LaZ/o2odFOrjdvyb8RN/BtxUwP8jZC+v3U6cSd1XlA8depULF26FGU1pWNqcpb1G4OP+zwGAHj9l+V47PQOmSsiIiK6O3VeUFx+sj5HR0d06dIFDhWHGgFs2LDBpAWaGqelKg85f7QrDUt/OwsAeHVQe0wM9anclkPOdW/LIeebP3Na6u7amtHfiFq15d+Iurfl34ibPzfAtFSdw82ECRNq3B8bG1uXp2t0PFrK2Gc7z+G1zScBALMiAjH1fn+ZKyIiIqqsQY+WMvfwQrX31d40KdhMe8CfwYaIiCxCncNNuStXriA5ORkAEBgYCHd3d5MVRQ3v+4N/4d8bjwEAnunfDi8+dI/MFREREZlGnRcUFxQU4KmnnoKXlxf69++P/v37w9vbGxMnTkRhTXPaZDZ+OnIJs747AiGAyDBfzB3SAYryOWsiIqImrs7hJioqCjt27MBPP/2EnJwc5OTk4Mcff8SOHTvw0ksvNUSNZEL/O5GBGeuTYBDA4718ED2sE4MNERFZlDovKHZzc8N3332HgQMHGm3fvn07Ro4ciStXrpiyPpNrzguKE5Kz8MznB1GiN2BEN2+8PbIbrK0YbIiIyPzV5fu7ziM3hYWF0Gq1lbZ7eHhwWsqM7U7JxrNf3Aw2Q7t44q3/C2awISIii1TncBMWFobo6GgUVTh+/caNG1i4cCHCwsJMWhyZxoHz1zBxzQEUlxkQHuSBZaO6w8a6zm89ERFRk1Dno6XeffddREREoHXr1ggODgYAHDlyBHZ2dvjll19MXiDVz5H0HIyP3Y8bpXrcF+CGD8b0gNKGwYaIiCxXncNN586dcebMGaxduxanT58GAIwePZpXBTdDJy/lYtxn+5BfXIbQti3wyZMhsLO1lrssIiKiBnVX57lRq9WYNGmSqWshEzqTmYcnPt0L3Y1S9Gjjgk/H94K9ksGGiIgsX53nJ2JiYvDZZ59V2v7ZZ59h6dKlJimK6udcdgHGrNqLawUl6NzKGbETesNRddfnayQiImpS6hxuPv74Y3To0KHS9k6dOmHFihUmKYruXvq1QoxduQdX8orRwdMJXzwVCo29rdxlERERNZo6h5uMjAx4eXlV2u7u7o7Lly+bpCi6O5d1NzBm1R5c0hWhvbsDvpgYClcHpdxlERERNao6hxsfHx/s2rWr0vZdu3bB29vbJEVR3WXlFWHsyr1Iv3YDbVqosfbpPnB3UsldFhERUaOr80KMSZMmYcaMGSgtLcUDDzwAAIiPj8fLL7/Myy/I5FpBCZ5YtRdnswvQysUeX00KhafGTu6yiIiIZFHncDNr1ixcvXoVU6ZMQUlJCQDAzs4Os2fPxty5c01eINWsVG9A5Gf78GdmPjycVFj7dChau6rlLouIiEg2db62VLn8/HycOnUK9vb2CAgIgErVNKZALO3aUjvPZOOJT/fC2c4GG6b0hb+Hk9wlERERmVyDXluqnKOjI3r16gUnJyekpqbCYDDc7VNRPSSezQYAhAdpGWyIiIhQh3Dz2Wef4Z133jHa9swzz6Bdu3bo0qULOnfujPT0dJMXSDXbnXoVABDWvqXMlRAREZmHWoebTz75BK6urtL9uLg4xMbG4vPPP8f+/fvh4uKChQsXNkiRVLW8olIc/UsHgOGGiIioXK0XFJ85cwYhISHS/R9//BHDhw/H2LFjAQCLFy/GhAkTTF8hVWv/+WvQGwR8W6q5iJiIiOhvtR65uXHjhtECnt27d6N///7S/Xbt2iEjI8O01VGNEv+ekurLURsiIiJJrcONr68vDh48CADIzs7GiRMn0K9fP2l/RkYGNBqN6SukapWvt+nTjuGGiIioXK2npSIjIzF16lScOHECv/32Gzp06ICePXtK+3fv3o3OnTs3SJFU2fWCEpy8nAuA622IiIgqqnW4efnll1FYWIgNGzbA09MT3377rdH+Xbt2YfTo0SYvkKq299xVCAEEeDjCw4lnIyYiIip31yfxa6os5SR+8388js8TLyAyzBcLh3PEjIiILFujnMSP5JUond/GTeZKiIiIzAvDTROUlVeEM1n5UCiAPu1ayF0OERGRWWG4aYLKR206ejnDRa2UuRoiIiLzwnDTBPH8NkRERNVjuGmCdkvhhuttiIiIbmeycJOeno6nnnrKVE9H1fjreiHSrhXC2kqBXm253oaIiOh2Jgs3165dw5o1a0z1dFSN8imprq01cFTV+jRFREREzUatvx03bdpU4/6zZ8/Wuxi6M663ISIiqlmtw82IESOgUChQ0zn/FAqFSYqiqgkhuN6GiIjoDmo9LeXl5YUNGzbAYDBUeTt06FBD1kkAzl8tREZuEZTWVujp6yp3OURERGap1uGmZ8+e0lXBq3KnUR2qv92p2QCA7m1cYGdrLXM1RERE5qnW01KzZs1CQUFBtfv9/f2xfft2kxRFVeOUFBER0Z3VOtzcd999Ne53cHDAgAED6l0QVc1gENhTHm78uZiYiIioOrWeljp79iynnWT0Z1YerhaUwN7WGsGtXeQuh4iIyGzVOtwEBATgypUr0v1Ro0YhMzOzQYqiysoPAQ/xc4XShieWJiIiqk6tvyVvH7XZunVrjWtwyLS43oaIiKh2OATQBOgNAnvO8uR9REREtVHrcKNQKCqdpI8n7WscJy7pkFdUBic7G3Tydpa7HCIiIrNW66OlhBAYP348VCoVAKCoqAiTJ0+Gg4ODUbsNGzaYtkKSpqRC27aEjTUH24iIiGpS63ATGRlpdP+JJ54weTFUtfLFxGGckiIiIrqjWoeb2NjYhqyDqlFSZsD+89cAcL0NERFRbXCOw8wd/SsHhSV6tHBQIlDrJHc5REREZo/hxsyVr7cJa9cSVlZcwE1ERHQnZhFuli9fDj8/P9jZ2SE0NBT79u2r1ePWrVsHhUKBESNGNGyBMiq/WGYfTkkRERHViuzhZv369YiKikJ0dDQOHTqE4OBgREREICsrq8bHnT9/HjNnzrzjNa+asqJSPQ6l5QDgehsiIqLakj3cvPPOO5g0aRImTJiAjh07YsWKFVCr1fjss8+qfYxer8fYsWOxcOFCtGvXrhGrbVyHLlxHSZkBWmcV2rk53PkBREREJG+4KSkpwcGDBxEeHi5ts7KyQnh4OBITE6t93GuvvQYPDw9MnDjxjq9RXFyM3Nxco1tTUfGSCzxhIhERUe3IGm6ys7Oh1+uh1WqNtmu1WmRkZFT5mJ07d+LTTz/FypUra/UaMTEx0Gg00s3Hx6fedTeW8vU2PL8NERFR7ck+LVUXeXl5ePLJJ7Fy5Uq4udXuApJz586FTqeTbunp6Q1cpWnkF5fh6F86ADePlCIiIqLaqfVJ/BqCm5sbrK2tkZmZabQ9MzMTnp6eldqnpqbi/PnzGDZsmLTNYDAAAGxsbJCcnIz27dsbPUalUkmXjGhK9p+/hjKDgE8Le/i0UMtdDhERUZMh68iNUqlEz549ER8fL20zGAyIj49HWFhYpfYdOnTAsWPHkJSUJN3++c9/4v7770dSUlKTmnK6k/JLLvRtV7sRKiIiIrpJ1pEbAIiKikJkZCRCQkLQu3dvLFu2DAUFBZgwYQIAYNy4cWjVqhViYmJgZ2eHzp07Gz3excUFACptb+rK19v09eeUFBERUV3IHm5GjRqFK1euYP78+cjIyEC3bt0QFxcnLTJOS0uDlVWTWhpUbzmFJThx6eZRXVxvQ0REVDcKIYSQu4jGlJubC41GA51OB2dnZ7nLqdIvJzLw7BcH0d7dAfEvDZS7HCIiItnV5fu7eQ2JNBGJFc5vQ0RERHXDcGOGpPU2PL8NERFRnTHcmJkrecX4MzMfANCH622IiIjqjOHGzCSevTklFeTlDFcHpczVEBERNT0MN2bm1nobjtoQERHdDYYbM5PI9TZERET1wnBjRi7m3MD5q4WwtlKgd9sWcpdDRETUJDHcmJHyKakurTRwsrOVuRoiIqKmieHGjJSHmzBOSREREd01hhszIYTgehsiIiITYLgxExeuFuKSrgi21gqE+HK9DRER0d1iuDETu/+ekurexhX2SmuZqyEiImq6GG7MRPklF3gVcCIiovphuDEDQgjsOcuT9xEREZkCw40ZOJOVj+z8EtjZWqFbGxe5yyEiImrSGG7MwO6Um1NSvfxaQGXD9TZERET1wXBjBnbz/DZEREQmw3AjM73h1nobLiYmIiKqP4YbmZ26nIvcojI4qmzQpZVG7nKIiIiaPIYbmZUfAh7atgVsrPl2EBER1Re/TWXG9TZERESmxXAjo1K9AfvOXQMA9G3vJnM1REREloHhRkZH/9KhsEQPV7UtOng6yV0OERGRRWC4kVH5VcD7tGsJKyuFzNUQERFZBoYbGZWvt+ElF4iIiEyH4UYmRaV6HLhwHQAQxvU2REREJsNwI5NDaddRUmaAu5MK7d0d5C6HiIjIYjDcyGRPhSkphYLrbYiIiEyF4UYmXG9DRETUMBhuZFBQXIak9BwAPL8NERGRqTHcyGD/+WsoMwi0drWHTwu13OUQERFZFIYbGSSm8irgREREDYXhRgaJZ/9eb+PPcENERGRqDDeNTFdYiuMXdQCAsHZcb0NERGRqDDeNbO+5qzAIoJ27Azw1dnKXQ0REZHEYbhrZbq63ISIialAMN41sT/l6Gx4CTkRE1CAYbhpRdn4xTmfkAQD6tGshczVERESWieGmEZWP2nTwdEJLR5XM1RAREVkmhptGdOuSC5ySIiIiaigMN41IOnkfrydFRETUYBhuGsll3Q2cyy6AlQLo3ZbrbYiIiBoKw00jKR+16dJKA429rczVEBERWS6Gm0Yind+G622IiIgaFMNNIxBCcL0NERFRI2G4aQRp1wpxMecGbKwU6OXnKnc5REREFo3hphGUj9p0b+MCtdJG5mqIiIgsG8NNI+B6GyIiosbDcNPAhBAVTt7H9TZEREQNjeGmgaVk5SM7vxgqGyt0b+MidzlEREQWj+GmgSX+fT2pED9XqGysZa6GiIjI8jHcNLDdKbyeFBERUWNiuGlABoOQRm54fhsiIqLGwXDTgE5ezoXuRikcVTbo2kojdzlERETNAsNNAyo/v00vP1fYWPNXTURE1Bj4jduAyqekuN6GiIio8TDcNJBSvQF7ud6GiIio0THcNJBjF3UoKNFDY2+Ljl7OcpdDRETUbDDcNJDy9TZ92rWAlZVC5mqIiIiaD4abBrI7NRsA19sQERE1NoabBlBcpseB89cB8HpSREREjY3hpgEcTstBcZkBbo4q+Hs4yl0OERFRs8Jw0wAqXgVcoeB6GyIiosZkFuFm+fLl8PPzg52dHUJDQ7Fv375q265cuRL33XcfXF1d4erqivDw8BrbyyHx7/U2PASciIio8ckebtavX4+oqChER0fj0KFDCA4ORkREBLKysqpsn5CQgNGjR2P79u1ITEyEj48PBg0ahIsXLzZy5VUrLClDUnoOAK63ISIikoNCCCHkLCA0NBS9evXCBx98AAAwGAzw8fHBCy+8gDlz5tzx8Xq9Hq6urvjggw8wbty4O7bPzc2FRqOBTqeDs7Ppzz/z+59XMO6zfWjlYo+ds+/ntBQREZEJ1OX7W9aRm5KSEhw8eBDh4eHSNisrK4SHhyMxMbFWz1FYWIjS0lK0aNGiyv3FxcXIzc01ujWk8vU2YVxvQ0REJAtZw012djb0ej20Wq3Rdq1Wi4yMjFo9x+zZs+Ht7W0UkCqKiYmBRqORbj4+PvWuuybSept2nJIiIiKSg+xrbupjyZIlWLduHX744QfY2dlV2Wbu3LnQ6XTSLT09vcHqyS0qxbGLOgBcTExERCQXGzlf3M3NDdbW1sjMzDTanpmZCU9Pzxof+9Zbb2HJkiX49ddf0bVr12rbqVQqqFQqk9R7J/vOXoNBAG3dHODtYt8or0lERETGZB25USqV6NmzJ+Lj46VtBoMB8fHxCAsLq/Zxb775JhYtWoS4uDiEhIQ0Rqm1UnG9DREREclD1pEbAIiKikJkZCRCQkLQu3dvLFu2DAUFBZgwYQIAYNy4cWjVqhViYmIAAEuXLsX8+fPx1Vdfwc/PT1qb4+joCEdHec8GfOt6Ugw3REREcpE93IwaNQpXrlzB/PnzkZGRgW7duiEuLk5aZJyWlgYrq1sDTB999BFKSkrw2GOPGT1PdHQ0FixY0JilG7maX4zTGXkAgD5cTExERCQb2c9z09ga6jw3W49dxpS1hxCodcIvL/Y32fMSERFR3b6/ZR+5sRQRnTzx0/P3IreoVO5SiIiImjWGGxOxtlKgS2uN3GUQERE1e036PDdEREREt2O4ISIiIovCcENEREQWheGGiIiILArDDREREVkUhhsiIiKyKAw3REREZFEYboiIiMiiMNwQERGRRWG4ISIiIovCcENEREQWhdeWIiJqRvR6PUpLeYFfMk9KpRJWVvUfd2G4ISJqBoQQyMjIQE5OjtylEFXLysoKbdu2hVKprNfzMNwQETUD5cHGw8MDarUaCoVC7pKIjBgMBly6dAmXL19GmzZt6vUZZbghIrJwer1eCjYtW7aUuxyiarm7u+PSpUsoKyuDra3tXT8PFxQTEVm48jU2arVa5kqIalY+HaXX6+v1PAw3RETNBKeiyNyZ6jPKcENEREQWheGGiIjIRBYsWIBu3bo12utt3LgR/v7+sLa2xowZMxrtdatTm/4PHDiwwWtluCEiIrOUkJAAhUJR7e3++++Xu8QGUd7v2hy2/+yzz+Kxxx5Deno6Fi1a1PDFmcCGDRsavFYeLUVERGapb9++uHz5cqXtmzZtwuTJkzFlypS7fu6SkpJ6n0tFbvn5+cjKykJERAS8vb2rbKPX66FQKExyYjxTadGiRYO/hvn0loiIGl9BQfW3oqLat71xo3Zt60CpVMLT09Podv36dcycORPz5s3D//3f/0ltjx8/jiFDhsDR0RFarRZPPvkksrOzpf0DBw7E888/jxkzZsDNzQ0REREAgB07dqB3795QqVTw8vLCnDlzUFZWVmNdCQkJ6N27NxwcHODi4oJ+/frhwoULRm2++OIL+Pn5QaPR4PHHH0deXp60r7i4GNOmTYOHhwfs7Oxw7733Yv/+/QCA8+fPSyNSrq6uUCgUGD9+fJU1ODk5AQAeeOABKBQKJCQkYPXq1XBxccGmTZvQsWNHqFQqpKWl4fr16xg3bhxcXV2hVqsxZMgQnDlzRnq+8sdt3rwZgYGBUKvVeOyxx1BYWIg1a9bAz88Prq6umDZtWq2OZKqp/5yWIiKihuXoWP3tX/8ybuvhUX3bIUOM2/r5Vd2uHnJycjB8+HAMHDjQaFojJycHDzzwALp3744DBw4gLi4OmZmZGDlypNHj16xZA6VSiV27dmHFihW4ePEihg4dil69euHIkSP46KOP8Omnn+L111+vtoaysjKMGDECAwYMwNGjR5GYmIhnnnnG6Cif1NRUbNy4EZs3b8bmzZuxY8cOLFmyRNr/8ssv4/vvv8eaNWtw6NAh+Pv7IyIiAteuXYOPjw++//57AEBycjIuX76Md999t1Idffv2RXJyMgDg+++/x+XLl9G3b18AQGFhIZYuXYpVq1bhxIkT8PDwwPjx43HgwAFs2rQJiYmJEEJg6NChRpfiKCwsxHvvvYd169YhLi4OCQkJeOSRR7B161Zs3boVX3zxBT7++GN89913Nb5Pd+p/oxDNjE6nEwCETqeTuxQiokZx48YNcfLkSXHjxo3KO4Hqb0OHGrdVq6tvO2CAcVs3t6rb3SW9Xi+GDBkigoKCRG5urtG+RYsWiUGDBhltS09PFwBEcnKyEEKIAQMGiO7duxu1mTdvnggMDBQGg0Hatnz5cuHo6Cj0en2VdVy9elUAEAkJCVXuj46OFmq12qjGWbNmidDQUCGEEPn5+cLW1lasXbtW2l9SUiK8vb3Fm2++KYQQYvv27QKAuH79ek2/EnH9+nUBQGzfvl3aFhsbKwCIpKQkaduff/4pAIhdu3ZJ27Kzs4W9vb345ptvjB6XkpIitXn22WeFWq0WeXl50raIiAjx7LPPVlvTnfovxM33Yvr06VU+vqbPal2+v7nmhoioOcvPr36ftbXx/ays6tvevqbj/Pm7Lqkq8+bNQ2JiIvbt2ydNx5Q7cuQItm/fDscqRoZSU1Nxzz33AAB69uxptO/UqVMICwszGnXp168f8vPz8ddffwEAOnbsaFTDvHnzMH78eEREROChhx5CeHg4Ro4cCS8vL6mdn5+fUY1eXl7I+vt3l5qaitLSUvTr10/ab2tri969e+PUqVN1/r1URalUomvXrkb9tLGxQWhoqLStZcuWCAwMNHpNtVqN9u3bS/e1Wi38/PyMfq9arVbqS3Vq6n9jYbghImrOHBzkb3sH69atw1tvvYUtW7YgICCg0v78/HwMGzYMS5curbSvYuhwqGNN3t7eSEpKku6XL4SNjY3FtGnTEBcXh/Xr1+OVV17Btm3b0KdPHwCodNkAhUIBg8FQp9euD3t7+7s6GV5Vdd9NX+TuP8A1N0REZMaSkpIwceJELFmyRFoEfLsePXrgxIkT8PPzg7+/v9GtpkATFBQkrT8pt2vXLjg5OaF169awsbExeq6KR/l0794dc+fOxe7du9G5c2d89dVXtepP+/btpXU/5UpLS7F//35plMhUlyAoFxQUhLKyMuzdu1fadvXqVSQnJxuNTFkShhsiIjJL2dnZGDFiBAYOHIgnnngCGRkZRrcrV64AAKZOnYpr165h9OjR2L9/P1JTU/HLL79gwoQJNQaEKVOmID09HS+88AJOnz6NH3/8EdHR0YiKiqr20Olz585h7ty5SExMxIULF/C///0PZ86cQVBQUK365ODggOeeew6zZs1CXFwcTp48iUmTJqGwsBATJ04EAPj6+kKhUGDz5s24cuUK8muaOqyFgIAADB8+HJMmTcLOnTtx5MgRPPHEE2jVqhWGDx9er+c2V5yWIiIis7RlyxZcuHABFy5cMJpeKufr64vz58/D29sbu3btwuzZszFo0CAUFxfD19cXgwcPrvH8Lq1atcLWrVsxa9YsBAcHo0WLFpg4cSJeeeWVah+jVqtx+vRprFmzBlevXoWXlxemTp2KZ599ttb9WrJkCQwGA5588knk5eUhJCQEv/zyC1xdXaW6Fi5ciDlz5mDChAkYN24cVq9eXevnr0psbCymT5+Of/zjHygpKUH//v2xdevWel1525wpRMXxuGYgNzcXGo0GOp0Ozs7OcpdDRNTgioqKcO7cObRt2xZ2dnZyl0NUrZo+q3X5/ua0FBEREVkUhhsiIiKyKAw3REREZFEYboiIiMiiMNwQERGRRWG4ISIiIovCcENEREQWheGGiIiILArDDREREVkUhhsiImqSBg4ciBkzZshdRqNLSEiAQqFATk6O3KWYLYYbIiIyW+PHj4dCoah0S0lJwYYNG7Bo0aIGr6G5hqjbrV69Gi4uLnKXUSu8cCYREZm1wYMHIzY21mibu7s7rK2tZaqIzB1HboiImiEhBApLymS51fV6zSqVCp6enkY3a2vrSiMqfn5+WLx4MZ566ik4OTmhTZs2+OSTT4yeKz09HSNHjoSLiwtatGiB4cOH4/z589W+9vjx47Fjxw68++670qjR+fPnqxzF2LhxIxQKhXR/wYIF6NatG7744gv4+flBo9Hg8ccfR15entTGYDAgJiYGbdu2hb29PYKDg/Hdd98ZPe/WrVtxzz33wN7eHvfff3+N9ZZLS0vD8OHD4ejoCGdnZ4wcORKZmZl1qq2ihIQETJgwATqdTvo9LFiwAADwxRdfICQkBE5OTvD09MSYMWOQlZV1xxobEkduiIiaoRulenSc/4ssr33ytQiolQ3z9fP2229j0aJFmDdvHr777js899xzGDBgAAIDA1FaWoqIiAiEhYXhjz/+gI2NDV5//XUMHjwYR48ehVKprPR87777Lv7880907twZr732GoCbo0a1lZqaio0bN2Lz5s24fv06Ro4ciSVLluCNN94AAMTExODLL7/EihUrEBAQgN9//x1PPPEE3N3dMWDAAKSnp+PRRx/F1KlT8cwzz+DAgQN46aWXanxNg8EgBZsdO3agrKwMU6dOxahRo5CQkFDr2irq27cvli1bhvnz5yM5ORkA4OjoCAAoLS3FokWLEBgYiKysLERFRWH8+PHYunVrrX9PpsZwQ0REZm3z5s3SFykADBkyBN9++22VbYcOHYopU6YAAGbPno3//ve/2L59OwIDA7F+/XoYDAasWrVKGmGJjY2Fi4sLEhISMGjQoErPp9FooFQqoVar4enpWefaDQYDVq9eDScnJwDAk08+ifj4eLzxxhsoLi7G4sWL8euvvyIsLAwA0K5dO+zcuRMff/wxBgwYgI8++gjt27fH22+/DQAIDAzEsWPHsHTp0mpfMz4+HseOHcO5c+fg4+MDAPj888/RqVMn7N+/H7169bpjbbdTKpXQaDRQKBSVfg9PPfWU9HO7du3w3nvvoVevXsjPzzd63xoTww0RUTNkb2uNk69FyPbadXH//ffjo48+ku47ODhU27Zr167Sz+VfxOVTJEeOHEFKSor0ZV6uqKgIqamp+OOPPzBkyBBp+8cff4yxY8fWqdbb+fn5Gb2el5eXVE9KSgoKCwvx0EMPGT2mpKQE3bt3BwCcOnUKoaGhRvvLg1B1Tp06BR8fHynYAEDHjh3h4uKCU6dOSeGmptrq4uDBg1iwYAGOHDmC69evw2AwALg5NdaxY8c6P58pMNwQETVDCoWiwaaGTM3BwQH+/v61amtra2t0X6FQSF+2+fn56NmzJ9auXVvpce7u7lAqlUhKSpK2abXaal/Hysqq0tqh0tLSOtcDAFu2bEGrVq2M2qlUqmpf21Rqqq22CgoKEBERgYiICKxduxbu7u5IS0tDREQESkpKTFlunTSNTzYREVE99ejRA+vXr4eHhwecnZ2rbFNViFIqldDr9Ubb3N3dkZeXh4KCAmkkqWIwqo2OHTtCpVIhLS0NAwYMqLJNUFAQNm3aZLRtz549NT5vUFAQ0tPTkZ6eLo3enDx5Ejk5OfUaSanq93D69GlcvXoVS5YskV7rwIEDd/0apsKjpYiIqFkYO3Ys3NzcMHz4cPzxxx84d+4cEhISMG3aNPz111/VPs7Pzw979+7F+fPnkZ2dDYPBgNDQUKjVasybNw+pqan46quvsHr16jrV4+TkhJkzZ+LFF1/EmjVrkJqaikOHDuH999/HmjVrAACTJ0/GmTNnMGvWLCQnJ9fqdcLDw9GlSxeMHTsWhw4dwr59+zBu3DgMGDAAISEhdaqxIj8/P+Tn5yM+Ph7Z2dkoLCxEmzZtoFQq8f777+Ps2bPYtGlTo5x76E4YboiIqFlQq9X4/fff0aZNGzz66KMICgrCxIkTUVRUVO1IDgDMnDkT1tbW6NixozTt0qJFC3z55ZfYunUrunTpgq+//lo6NLouFi1ahFdffRUxMTEICgrC4MGDsWXLFrRt2xYA0KZNG3z//ffYuHEjgoODsWLFCixevLjG51QoFPjxxx/h6uqK/v37Izw8HO3atcP69evrXF9Fffv2xeTJkzFq1Ci4u7vjzTffhLu7O1avXo1vv/0WHTt2xJIlS/DWW2/V63VMQSHqesKBJi43NxcajQY6na7GDzMRkaUoKirCuXPn0LZtW9jZ2cldDlG1avqs1uX7myM3REREZFEYboiIiMiiMNwQERGRRWG4ISIiIovCcENE1Ew0s+NHqAky1WeU4YaIyMKVn4m2sLBQ5kqIalZ+VmNr67pdouN2PEMxEZGFs7a2houLi3TdILVaLV04kshcGAwGXLlyBWq1GjY29YsnDDdERM1A+ZWc7+bCiESNxcrKCm3atKl3+Ga4ISJqBhQKBby8vODh4VHlBR6JzIFSqYSVVf1XzDDcEBE1I9bW1vVez0Bk7sxiQfHy5cvh5+cHOzs7hIaGYt++fTW2//bbb9GhQwfY2dmhS5cu2Lp1ayNVSkREROZO9nCzfv16REVFITo6GocOHUJwcDAiIiKqnRfevXs3Ro8ejYkTJ+Lw4cMYMWIERowYgePHjzdy5URERGSOZL9wZmhoKHr16oUPPvgAwM3V0j4+PnjhhRcwZ86cSu1HjRqFgoICbN68WdrWp08fdOvWDStWrLjj6/HCmURERE1PXb6/ZV1zU1JSgoMHD2Lu3LnSNisrK4SHhyMxMbHKxyQmJiIqKspoW0REBDZu3Fhl++LiYhQXF0v3dTodgJu/JCIiImoayr+3azMmI2u4yc7Ohl6vh1arNdqu1Wpx+vTpKh+TkZFRZfuMjIwq28fExGDhwoWVtvv4+Nxl1URERCSXvLw8aDSaGttY/NFSc+fONRrpMRgMuHbtGlq2bGkxJ7HKzc2Fj48P0tPTm8VUG/tr2dhfy9bc+gs0vz43VH+FEMjLy4O3t/cd28oabtzc3GBtbY3MzEyj7ZmZmdIJp27n6elZp/YqlQoqlcpom4uLy90XbcacnZ2bxT+ccuyvZWN/LVtz6y/Q/PrcEP2904hNOVmPllIqlejZsyfi4+OlbQaDAfHx8QgLC6vyMWFhYUbtAWDbtm3VticiIqLmRfZpqaioKERGRiIkJAS9e/fGsmXLUFBQgAkTJgAAxo0bh1atWiEmJgYAMH36dAwYMABvv/02Hn74Yaxbtw4HDhzAJ598Imc3iIiIyEzIHm5GjRqFK1euYP78+cjIyEC3bt0QFxcnLRpOS0szOhVz37598dVXX+GVV17BvHnzEBAQgI0bN6Jz585ydUF2KpUK0dHRlabfLBX7a9nYX8vW3PoLNL8+m0N/ZT/PDREREZEpyX6GYiIiIiJTYrghIiIii8JwQ0RERBaF4YaIiIgsCsONmVqwYAEUCoXRrUOHDtL+oqIiTJ06FS1btoSjoyP+9a9/VTq5YVpaGh5++GGo1Wp4eHhg1qxZKCsra+yuVOn333/HsGHD4O3tDYVCUenaYEIIzJ8/H15eXrC3t0d4eDjOnDlj1ObatWsYO3YsnJ2d4eLigokTJyI/P9+ozdGjR3HffffBzs4OPj4+ePPNNxu6a1W6U3/Hjx9f6f0ePHiwUZum1N+YmBj06tULTk5O8PDwwIgRI5CcnGzUxlSf4YSEBPTo0QMqlQr+/v5YvXp1Q3evktr0d+DAgZXe48mTJxu1aSr9/eijj9C1a1fpJG1hYWH4+eefpf2W9N4Cd+6vJb23VVmyZAkUCgVmzJghbTP791iQWYqOjhadOnUSly9flm5XrlyR9k+ePFn4+PiI+Ph4ceDAAdGnTx/Rt29faX9ZWZno3LmzCA8PF4cPHxZbt24Vbm5uYu7cuXJ0p5KtW7eKf//732LDhg0CgPjhhx+M9i9ZskRoNBqxceNGceTIEfHPf/5TtG3bVty4cUNqM3jwYBEcHCz27Nkj/vjjD+Hv7y9Gjx4t7dfpdEKr1YqxY8eK48ePi6+//lrY29uLjz/+uLG6KblTfyMjI8XgwYON3u9r164ZtWlK/Y2IiBCxsbHi+PHjIikpSQwdOlS0adNG5OfnS21M8Rk+e/asUKvVIioqSpw8eVK8//77wtraWsTFxZldfwcMGCAmTZpk9B7rdLom2d9NmzaJLVu2iD///FMkJyeLefPmCVtbW3H8+HEhhGW9t7XpryW9t7fbt2+f8PPzE127dhXTp0+Xtpv7e8xwY6aio6NFcHBwlftycnKEra2t+Pbbb6Vtp06dEgBEYmKiEOLml6mVlZXIyMiQ2nz00UfC2dlZFBcXN2jtdXX7l73BYBCenp7iP//5j7QtJydHqFQq8fXXXwshhDh58qQAIPbv3y+1+fnnn4VCoRAXL14UQgjx4YcfCldXV6P+zp49WwQGBjZwj2pWXbgZPnx4tY9pyv0VQoisrCwBQOzYsUMIYbrP8Msvvyw6depk9FqjRo0SERERDd2lGt3eXyFufgFW/HK4XVPurxBCuLq6ilWrVln8e1uuvL9CWO57m5eXJwICAsS2bduM+tgU3mNOS5mxM2fOwNvbG+3atcPYsWORlpYGADh48CBKS0sRHh4ute3QoQPatGmDxMREAEBiYiK6dOlidAX1iIgI5Obm4sSJE43bkTo6d+4cMjIyjPqn0WgQGhpq1D8XFxeEhIRIbcLDw2FlZYW9e/dKbfr37w+lUim1iYiIQHJyMq5fv95Ivam9hIQEeHh4IDAwEM899xyuXr0q7Wvq/dXpdACAFi1aADDdZzgxMdHoOcrblD+HXG7vb7m1a9fCzc0NnTt3xty5c1FYWCjta6r91ev1WLduHQoKChAWFmbx7+3t/S1nie/t1KlT8fDDD1eqqym8x7KfoZiqFhoaitWrVyMwMBCXL1/GwoULcd999+H48ePIyMiAUqmsdAFQrVaLjIwMAEBGRobRh6p8f/k+c1ZeX1X1V+yfh4eH0X4bGxu0aNHCqE3btm0rPUf5PldX1wap/24MHjwYjz76KNq2bYvU1FTMmzcPQ4YMQWJiIqytrZt0fw0GA2bMmIF+/fpJZxI31We4uja5ubm4ceMG7O3tG6JLNaqqvwAwZswY+Pr6wtvbG0ePHsXs2bORnJyMDRs2AGh6/T127BjCwsJQVFQER0dH/PDDD+jYsSOSkpIs8r2trr+A5b23ALBu3TocOnQI+/fvr7SvKfz7ZbgxU0OGDJF+7tq1K0JDQ+Hr64tvvvlGlj/Y1LAef/xx6ecuXbqga9euaN++PRISEvDggw/KWFn9TZ06FcePH8fOnTvlLqVRVNffZ555Rvq5S5cu8PLywoMPPojU1FS0b9++scust8DAQCQlJUGn0+G7775DZGQkduzYIXdZDaa6/nbs2NHi3tv09HRMnz4d27Ztg52dndzl3BVOSzURLi4uuOeee5CSkgJPT0+UlJQgJyfHqE1mZiY8PT0BAJ6enpVWrpffL29jrsrrq6r+iv3Lysoy2l9WVoZr165ZxO+gXbt2cHNzQ0pKCoCm29/nn38emzdvxvbt29G6dWtpu6k+w9W1cXZ2luV/Aqrrb1VCQ0MBwOg9bkr9VSqV8Pf3R8+ePRETE4Pg4GC8++67FvveVtffqjT19/bgwYPIyspCjx49YGNjAxsbG+zYsQPvvfcebGxsoNVqzf49ZrhpIvLz85GamgovLy/07NkTtra2iI+Pl/YnJycjLS1NmgMOCwvDsWPHjL4Qt23bBmdnZ2ko1Vy1bdsWnp6eRv3Lzc3F3r17jfqXk5ODgwcPSm1+++03GAwG6Q9LWFgYfv/9d5SWlkpttm3bhsDAQLOakqrKX3/9hatXr8LLywtA0+uvEALPP/88fvjhB/z222+VpstM9RkOCwszeo7yNhXXQjSGO/W3KklJSQBg9B43lf5WxWAwoLi42OLe2+qU97cqTf29ffDBB3Hs2DEkJSVJt5CQEIwdO1b62ezf43ovSaYG8dJLL4mEhARx7tw5sWvXLhEeHi7c3NxEVlaWEOLmYXht2rQRv/32mzhw4IAICwsTYWFh0uPLD8MbNGiQSEpKEnFxccLd3d1sDgXPy8sThw8fFocPHxYAxDvvvCMOHz4sLly4IIS4eSi4i4uL+PHHH8XRo0fF8OHDqzwUvHv37mLv3r1i586dIiAgwOjQ6JycHKHVasWTTz4pjh8/LtatWyfUarUsh0bX1N+8vDwxc+ZMkZiYKM6dOyd+/fVX0aNHDxEQECCKioqaZH+fe+45odFoREJCgtHhsYWFhVIbU3yGyw8lnTVrljh16pRYvny5LIfP3qm/KSkp4rXXXhMHDhwQ586dEz/++KNo166d6N+/f5Ps75w5c8SOHTvEuXPnxNGjR8WcOXOEQqEQ//vf/4QQlvXe3qm/lvbeVuf2I8LM/T1muDFTo0aNEl5eXkKpVIpWrVqJUaNGiZSUFGn/jRs3xJQpU4Srq6tQq9XikUceEZcvXzZ6jvPnz4shQ4YIe3t74ebmJl566SVRWlra2F2p0vbt2wWASrfIyEghxM3DwV999VWh1WqFSqUSDz74oEhOTjZ6jqtXr4rRo0cLR0dH4ezsLCZMmCDy8vKM2hw5ckTce++9QqVSiVatWoklS5Y0VheN1NTfwsJCMWjQIOHu7i5sbW2Fr6+vmDRpktEhlEI0rf5W1VcAIjY2Vmpjqs/w9u3bRbdu3YRSqRTt2rUzeo3Gcqf+pqWlif79+4sWLVoIlUol/P39xaxZs4zOhSJE0+nvU089JXx9fYVSqRTu7u7iwQcflIKNEJb13gpRc38t7b2tzu3hxtzfY4UQQtR//IeIiIjIPHDNDREREVkUhhsiIiKyKAw3REREZFEYboiIiMiiMNwQERGRRWG4ISIiIovCcENEREQWheGGyIKcP38eCoVCOv27OTh9+jT69OkDOzs7dOvWrdFe18/PD8uWLat1+4SEBCgUikrXy2luxo8fjxEjRshdBlG9MNwQmdD48eOhUCiwZMkSo+0bN26EQqGQqSp5RUdHw8HBAcnJyZWuIwMACoWixtuCBQvu6nX3799vdLXmO+nbty8uX74MjUZzV69XFytXrkRwcDAcHR3h4uKC7t27IyYmpsFfl6i5sJG7ACJLY2dnh6VLl+LZZ581+wt01lZJSQmUSuVdPTY1NRUPP/wwfH19q9x/+fJl6ef169dj/vz5SE5OlrY5OjpKPwshoNfrYWNz5z9d7u7udapTqVQ2ytXTP/vsM8yYMQPvvfceBgwYgOLiYhw9ehTHjx9v8Ncmai44ckNkYuHh4fD09Kzx/8QXLFhQaYpm2bJl8PPzk+6XTw8sXrwYWq0WLi4ueO2111BWVoZZs2ahRYsWaN26NWJjYys9/+nTp9G3b1/Y2dmhc+fO2LFjh9H+48ePY8iQIXB0dIRWq8WTTz6J7Oxsaf/AgQPx/PPPY8aMGXBzc0NERESV/TAYDHjttdfQunVrqFQqdOvWDXFxcdJ+hUKBgwcP4rXXXqt2FMbT01O6aTQaKBQK6f7p06fh5OSEn3/+GT179oRKpcLOnTuRmpqK4cOHQ6vVwtHREb169cKvv/5q9Ly3T0spFAqsWrUKjzzyCNRqNQICArBp0yZp/+3TUqtXr4aLiwt++eUXBAUFwdHREYMHDzYKY2VlZZg2bRpcXFzQsmVLzJ49G5GRkTVO62zatAkjR47ExIkT4e/vj06dOmH06NF44403pDb79+/HQw89BDc3N2g0GgwYMACHDh0yeh6FQoGPP/4Y//jHP6BWqxEUFITExESkpKRg4MCBcHBwQN++fZGamio9pvxz9/HHH8PHxwdqtRojR46ETqertl6DwYCYmBi0bdsW9vb2CA4OxnfffSftv379OsaOHQt3d3fY29sjICCgys8kUWNiuCEyMWtrayxevBjvv/8+/vrrr3o912+//YZLly7h999/xzvvvIPo6Gj84x//gKurK/bu3YvJkyfj2WefrfQ6s2bNwksvvYTDhw8jLCwMw4YNw9WrVwEAOTk5eOCBB9C9e3ccOHAAcXFxyMzMxMiRI42eY82aNVAqldi1axdWrFhRZX3vvvsu3n77bbz11ls4evQoIiIi8M9//hNnzpwBcHNUplOnTnjppZdw+fJlzJw5865+D3PmzMGSJUtw6tQpdO3aFfn5+Rg6dCji4+Nx+PBhDB48GMOGDUNaWlqNz7Nw4UKMHDkSR48exdChQzF27Fhcu3at2vaFhYV466238MUXX+D3339HWlqaUR+WLl2KtWvXIjY2Frt27UJubi42btxYYw2enp7Ys2cPLly4UG2bvLw8REZGYufOndizZw8CAgIwdOhQ5OXlGbVbtGgRxo0bh6SkJHTo0AFjxozBs88+i7lz5+LAgQMQQuD55583ekxKSgq++eYb/PTTT4iLi8Phw4cxZcqUamuJiYnB559/jhUrVuDEiRN48cUX8cQTT0iB+dVXX8XJkyfx888/49SpU/joo4/g5uZW4++AqMGZ5PKbRCSEECIyMlIMHz5cCCFEnz59xFNPPSWEEOKHH34QFf+5RUdHi+DgYKPH/ve//xW+vr5Gz+Xr6yv0er20LTAwUNx3333S/bKyMuHg4CC+/vprIYQQ586dEwCMrgZeWloqWrduLZYuXSqEEGLRokVi0KBBRq+dnp4uAEhXXh8wYIDo3r37Hfvr7e0t3njjDaNtvXr1ElOmTJHuBwcHi+jo6Ds+lxBCxMbGCo1GI90vv5r6xo0b7/jYTp06iffff1+67+vrK/773/9K9wGIV155Rbqfn58vAIiff/7Z6LWuX78u1QJApKSkSI9Zvny50Gq10n2tViv+85//SPfLyspEmzZtpM9AVS5duiT69OkjAIh77rlHREZGivXr1xu9z7fT6/XCyclJ/PTTT9X2JzExUQAQn376qbTt66+/FnZ2dtL96OhoYW1tLf766y9p288//yysrKykKzpX/AwXFRUJtVotdu/ebVTPxIkTxejRo4UQQgwbNkxMmDCh2tqJ5MCRG6IGsnTpUqxZswanTp266+fo1KkTrKxu/TPVarXo0qWLdN/a2hotW7ZEVlaW0ePCwsKkn21sbBASEiLVceTIEWzfvh2Ojo7SrUOHDgBgNIXRs2fPGmvLzc3FpUuX0K9fP6Pt/fr1q1efqxISEmJ0Pz8/HzNnzkRQUBBcXFzg6OiIU6dO3XHkpmvXrtLPDg4OcHZ2rvS7q0itVqN9+/bSfS8vL6m9TqdDZmYmevfuLe23tra+4+/Ny8sLiYmJOHbsGKZPn46ysjJERkZi8ODBMBgMAIDMzExMmjQJAQEB0Gg0cHZ2Rn5+fqX+VeyPVqsFAKPPh1arRVFREXJzc6Vtbdq0QatWraT7YWFhMBgMRuucyqWkpKCwsBAPPfSQ0efl888/lz4rzz33HNatW4du3brh5Zdfxu7du2vsP1Fj4IJiogbSv39/REREYO7cuRg/frzRPisrKwghjLaVlpZWeg5bW1uj+wqFospt5V+KtZGfn49hw4Zh6dKllfZ5eXlJPzs4ONT6ORva7bXMnDkT27Ztw1tvvQV/f3/Y29vjscceQ0lJSY3PU9ffXVXtb3/f7lbnzp3RuXNnTJkyBZMnT8Z9992HHTt24P7770dkZCSuXr2Kd999F76+vlCpVAgLC6vUv4r1lR+NV9W2unw+KsrPzwcAbNmyxSgQAYBKpQIADBkyBBcuXMDWrVuxbds2PPjgg5g6dSreeuutu3pNIlPgyA1RA1qyZAl++uknJCYmGm13d3dHRkaG0RelKc9Ns2fPHunnsrIyHDx4EEFBQQCAHj164MSJE/Dz84O/v7/RrS6BxtnZGd7e3ti1a5fR9l27dqFjx46m6Ug1du3ahfHjx+ORRx5Bly5d4OnpifPnzzfoa95Oo9FAq9Vi//790ja9Xl9p4W9tlP++CgoKANzs37Rp0zB06FB06tQJKpXKaMF3faSlpeHSpUvS/T179sDKygqBgYFV1qVSqZCWllbps+Lj4yO1c3d3R2RkJL788kssW7YMn3zyiUlqJbpbHLkhakBdunTB2LFj8d577xltHzhwIK5cuYI333wTjz32GOLi4vDzzz/D2dnZJK+7fPlyBAQEICgoCP/9739x/fp1PPXUUwCAqVOnYuXKlRg9ejRefvlltGjRAikpKVi3bh1WrVoFa2vrWr/OrFmzEB0djfbt26Nbt26IjY1FUlIS1q5da5J+VCcgIAAbNmzAsGHDoFAo8Oqrr9716ER9vPDCC4iJiYG/vz86dOiA999/H9evX6/xnEbPPfccvL298cADD6B169a4fPkyXn/9dbi7u0vTiQEBAfjiiy8QEhKC3NxczJo1C/b29iap2c7ODpGRkXjrrbeQm5uLadOmYeTIkVUeBu/k5ISZM2fixRdfhMFgwL333gudToddu3bB2dkZkZGRmD9/Pnr27IlOnTqhuLgYmzdvloI0kVw4ckPUwF577bVKX7xBQUH48MMPsXz5cgQHB2Pfvn13fSRRVZYsWYIlS5YgODgYO3fuxKZNm6QjWMpHW/R6PQYNGoQuXbpgxowZcHFxMVrfUxvTpk1DVFQUXnrpJXTp0gVxcXHYtGkTAgICTNaXqrzzzjtwdXVF3759MWzYMERERKBHjx4N+ppVmT17NkaPHo1x48YhLCwMjo6OiIiIgJ2dXbWPCQ8Px549e/B///d/uOeee/Cvf/0LdnZ2iI+PR8uWLQEAn376Ka5fv44ePXrgySefxLRp0+Dh4WGSmv39/fHoo49i6NChGDRoELp27YoPP/yw2vaLFi3Cq6++ipiYGAQFBWHw4MHYsmUL2rZtC+Dm+YHmzp2Lrl27on///rC2tsa6detMUivR3VIIU00gExE1cwaDAUFBQRg5ciQWLVokdzmVLFiwABs3bjSry3MQNQROSxER3aULFy7gf//7n3Sm4Q8++ADnzp3DmDFj5C6NqFnjtBQR0V2ysrLC6tWr0atXL/Tr1w/Hjh3Dr7/+yjUnRDLjtBQRERFZFI7cEBERkUVhuCEiIiKLwnBDREREFoXhhoiIiCwKww0RERFZFIYbIiIisigMN0RERGRRGG6IiIjIojDcEBERkUX5f6aWdA/tTUAYAAAAAElFTkSuQmCC", "text/plain": [ "

" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "fig, ax = plt.subplots()\n", "ax.axhline(f1_scores[\"hi\"][\"ta\"], ls=\"--\", color=\"r\")\n", "metrics_df.set_index(\"num_samples\").plot(ax=ax) \n", "plt.legend([\"Zero-shot from hi\", \"Fine-tuned on ta\"], loc=\"lower right\")\n", "plt.ylim((0, 1))\n", "plt.xlabel(\"Number of Training Samples\")\n", "plt.ylabel(\"F1 Score\")\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Fine-Tuning on Multiple Languages at Once" ] }, { "cell_type": "code", "execution_count": 62, "metadata": { "tags": [] }, "outputs": [], "source": [ "from datasets import concatenate_datasets\n", "\n", "def concatenate_splits(corpora):\n", " multi_corpus = DatasetDict()\n", " for split in corpora[0].keys():\n", " multi_corpus[split] = concatenate_datasets([corpus[split] for corpus in corpora]).shuffle(seed=42)\n", " return multi_corpus " ] }, { "cell_type": "code", "execution_count": 63, "metadata": { "tags": [] }, "outputs": [], "source": [ "panx_hi_ta_encoded = concatenate_splits([panx_hi_encoded, panx_ta_encoded])" ] }, { "cell_type": "code", "execution_count": 64, "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Cloning https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-hi-ta into local empty directory.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/data/ssd2/abhiroop/misc/nlp-with-transformers/env/lib/python3.8/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [1116/1116 02:00, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
10.4863000.2869280.764870
20.2297000.2283810.814884
30.1527000.2197330.841515

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Adding files tracked by Git LFS: ['tokenizer.json']. This may take a bit of time if the files are large.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f8b705fe0ca04ad5bc3b2f93cb96ebc3", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Upload file pytorch_model.bin: 0%| | 1.00/1.03G [00:00 main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "To https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-hi-ta\n", " ee5064e..18f9a46 main -> main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "text/plain": [ "'https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-hi-ta/commit/ee5064e5c1507fc85048d794b3dbba4d2a5f0895'" ] }, "execution_count": 64, "metadata": {}, "output_type": "execute_result" } ], "source": [ "training_args.logging_steps = len(panx_hi_ta_encoded[\"train\"]) // batch_size \n", "training_args.push_to_hub = True \n", "training_args.output_dir = \"xlm-roberta-base-finetuned-panx-hi-ta\"\n", "\n", "trainer = Trainer(model_init=model_init, args=training_args, \n", " data_collator=data_collator, compute_metrics=compute_metrics, \n", " tokenizer=xlmr_tokenizer, train_dataset=panx_hi_ta_encoded[\"train\"],\n", " eval_dataset=panx_hi_ta_encoded[\"validation\"])\n", "\n", "trainer.train()\n", "trainer.push_to_hub(commit_message=\"Training completed!\")" ] }, { "cell_type": "code", "execution_count": 65, "metadata": { "tags": [] }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "619f9075ad024d4384544673facf26f5", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/2051 [00:00" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "F1-score of [hi-fa] model on [hi] dataset: 0.823\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "d7dda0a00a5040dda9437f82ff4c9340", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/55 [00:00" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "F1-score of [hi-fa] model on [te] dataset: 0.543\n" ] }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "F1-score of [hi-fa] model on [ta] dataset: 0.820\n" ] }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "F1-score of [hi-fa] model on [en] dataset: 0.444\n" ] } ], "source": [ "for lang in langs:\n", " f1 = evaluate_lang_performance(lang, trainer)\n", " print(f\"F1-score of [hi-fa] model on [{lang}] dataset: {f1:.3f}\")" ] }, { "cell_type": "code", "execution_count": 66, "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Cloning https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-te into local empty directory.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/data/ssd2/abhiroop/misc/nlp-with-transformers/env/lib/python3.8/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [9/9 00:01, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
12.3928001.8361610.066667
21.3728001.0591160.000000
31.2081001.0058620.000000

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Adding files tracked by Git LFS: ['tokenizer.json']. This may take a bit of time if the files are large.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "baa19893e35f4eecab9cc5e0c7ccb852", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Upload file pytorch_model.bin: 0%| | 1.00/1.03G [00:00 main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "To https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-te\n", " cf8016b..99b3b19 main -> main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Cloning https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-ta into local empty directory.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/data/ssd2/abhiroop/misc/nlp-with-transformers/env/lib/python3.8/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [858/858 01:33, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
10.4843000.2531470.765018
20.2271000.2267590.786340
30.1524000.2320350.803556

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Adding files tracked by Git LFS: ['tokenizer.json']. This may take a bit of time if the files are large.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "530b8eb5e9c448f2b9b69b0e473b4d4a", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Upload file pytorch_model.bin: 0%| | 1.00/1.03G [00:00 main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "To https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-ta\n", " e0ad97a..3315405 main -> main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Cloning https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-en into local empty directory.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/data/ssd2/abhiroop/misc/nlp-with-transformers/env/lib/python3.8/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [192/192 00:21, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
10.9675000.4574140.593826
20.4302000.4075170.662712
30.3090000.3812270.700170

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Adding files tracked by Git LFS: ['tokenizer.json']. This may take a bit of time if the files are large.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "5d286f769fa2452286503c255d881d57", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Upload file pytorch_model.bin: 0%| | 1.00/1.03G [00:00 main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "To https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-en\n", " 773c984..7b5883b main -> main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "corpora = [panx_hi_encoded] \n", "\n", "# exclude hindi from iteration\n", "for lang in langs[1:]:\n", " training_args.output_dir = f\"xlm-roberta-base-finetuned-panx-{lang}\"\n", " # fine-tune on monolingual corpus \n", " ds_encoded = encode_panx_dataset(panx_ch[lang])\n", " metrics = train_on_subset(ds_encoded, ds_encoded[\"train\"].num_rows)\n", " # collect F1-scores in common dict \n", " f1_scores[lang][lang] = metrics[\"f1_score\"][0]\n", " # add monolingual corpus to list of corpora to concatenate\n", " corpora.append(ds_encoded)" ] }, { "cell_type": "code", "execution_count": 67, "metadata": { "tags": [] }, "outputs": [], "source": [ "corpora_encoded = concatenate_splits(corpora)" ] }, { "cell_type": "code", "execution_count": 68, "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Cloning https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-all into local empty directory.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/data/ssd2/abhiroop/misc/nlp-with-transformers/env/lib/python3.8/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [1314/1314 02:23, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossF1
10.4864000.3817090.698385
20.2419000.3050410.765427
30.1606000.3023100.794586

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Adding files tracked by Git LFS: ['tokenizer.json']. This may take a bit of time if the files are large.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "33a0d5910a584c2b8ad9f591ea95fe24", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Upload file pytorch_model.bin: 0%| | 1.00/1.03G [00:00 main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "To https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-all\n", " ab4985f..e6d5947 main -> main\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "data": { "text/plain": [ "'https://huggingface.co/the-neural-networker/xlm-roberta-base-finetuned-panx-all/commit/ab4985f7081e59d04b156e81f515f2ed40a854fd'" ] }, "execution_count": 68, "metadata": {}, "output_type": "execute_result" } ], "source": [ "training_args.logging_steps = len(corpora_encoded[\"train\"]) // batch_size \n", "training_args.output_dir = \"xlm-roberta-base-finetuned-panx-all\"\n", "\n", "trainer = Trainer(model_init=model_init, args=training_args,\n", " data_collator=data_collator, compute_metrics=compute_metrics,\n", " tokenizer=xlmr_tokenizer, train_dataset=corpora_encoded[\"train\"],\n", " eval_dataset=corpora_encoded[\"validation\"])\n", "\n", "trainer.train()\n", "trainer.push_to_hub(commit_message=\"Training completed!\")" ] }, { "cell_type": "code", "execution_count": 69, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "

\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Evaluated onhitetaen
Fine-tune on
hi0.79840.52480.55170.5289
each0.79840.00000.80690.7154
all0.81760.60430.80000.7467
\n", "
" ], "text/plain": [ "Evaluated on hi te ta en\n", "Fine-tune on \n", "hi 0.7984 0.5248 0.5517 0.5289\n", "each 0.7984 0.0000 0.8069 0.7154\n", "all 0.8176 0.6043 0.8000 0.7467" ] }, "execution_count": 69, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for idx, lang in enumerate(langs):\n", " f1_scores[\"all\"][lang] = get_f1_score(trainer, corpora[idx][\"test\"])\n", "scores_data = {\"hi\": f1_scores[\"hi\"],\n", " \"each\": {lang: f1_scores[lang][lang] for lang in langs},\n", " \"all\": f1_scores[\"all\"]}\n", "f1_scores_df = pd.DataFrame(scores_data).T.round(4)\n", "f1_scores_df.rename_axis(index=\"Fine-tune on\", columns=\"Evaluated on\", \n", " inplace=True)\n", "f1_scores_df" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 4 }