diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..af4b9bb798e8c859f6427828b91cc6336193f90c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,21 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +results/mac-results.csv filter=lfs diff=lfs merge=lfs -text +results/mac-results_lf.csv filter=lfs diff=lfs merge=lfs -text +results/mac-results_lf-r3.csv filter=lfs diff=lfs merge=lfs -text +results/experiment-1-results.csv filter=lfs diff=lfs merge=lfs -text +results/mac-results-no-flash-attn.csv filter=lfs diff=lfs merge=lfs -text +results/mac-results_lf-r2.csv filter=lfs diff=lfs merge=lfs -text +results/model_training_evaluation_times.csv filter=lfs diff=lfs merge=lfs -text +results/experiment-3-results.csv filter=lfs diff=lfs merge=lfs -text +results/mac-results_final.csv filter=lfs diff=lfs merge=lfs -text +results/mac-results_py3.11.csv filter=lfs diff=lfs merge=lfs -text +results/experiment-2-results.csv filter=lfs diff=lfs merge=lfs -text +results/mac-results-with-flash-attn.csv filter=lfs diff=lfs merge=lfs -text +results/mac-results_v3.csv filter=lfs diff=lfs merge=lfs -text +llama-factory/data/alpaca_mac.json filter=lfs diff=lfs merge=lfs -text +llama-factory/data/dataset_info.json filter=lfs diff=lfs merge=lfs -text +datasets/mac/mac-test.tsv filter=lfs diff=lfs merge=lfs -text +datasets/mac/mac-train.tsv filter=lfs diff=lfs merge=lfs -text +datasets/mac/mac.tsv filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..45b9bb7a4c08e7ba8ea529291dddd6b22f90a1a6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,152 @@ +*.out +*.log +*/outputs/ +*/models/ +*/wandb/ +*/cs605-nlp-assignment-2*/ +*/augmented_data/ +*/inflaton/ +*/llama.cpp/ +wandb/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +# *.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# JetBrains +.idea + +*.db + +.DS_Store +/outputs +/models +/llama.cpp +/llama-factory/saves +/llama-factory/saves-1 diff --git a/datasets/mac/mac-test.tsv b/datasets/mac/mac-test.tsv new file mode 100644 index 0000000000000000000000000000000000000000..86786c4006a5b4dff54f99b40f87226acbc6ca83 --- /dev/null +++ b/datasets/mac/mac-test.tsv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5663c7521eaf9942a9fea40b2950a46e37b761b22cc698eb6fe6b57bf70d0c4 +size 253194 diff --git a/datasets/mac/mac-train.tsv b/datasets/mac/mac-train.tsv new file mode 100644 index 0000000000000000000000000000000000000000..39a102e21beb0129a44e44564c1f3e12ba569dcf --- /dev/null +++ b/datasets/mac/mac-train.tsv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:424f0adcb2727eec322acef12294f4efb10412fc0b0529887d28dddc5171af05 +size 1031685 diff --git a/datasets/mac/mac.tsv b/datasets/mac/mac.tsv new file mode 100644 index 0000000000000000000000000000000000000000..dbecb0d8bf1d3d72827fac4d8988980f55c85434 --- /dev/null +++ b/datasets/mac/mac.tsv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93f3ab2ba07b67b0a3f9ff05291c1b6748851999cda050bc165f8dd259daa2aa +size 1289106 diff --git a/eval_modules/calc_repetitions.py b/eval_modules/calc_repetitions.py new file mode 100644 index 0000000000000000000000000000000000000000..76aefbe1c2369617db5256f303e713203e037a51 --- /dev/null +++ b/eval_modules/calc_repetitions.py @@ -0,0 +1,79 @@ +import os +import re +import math +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.ticker as mtick +import seaborn as sns +import nltk +import evaluate + +meteor = evaluate.load("meteor") + +print(f"loading: {__file__}") + +# final version +pattern_excessive_whitespaces = re.compile(r"\s{5,}") +pattern_text_repetitions = re.compile(r"(.{5}.*)\s*((\1)\s*)+", re.M | re.DOTALL) + + +def del_excessive_whitespaces(text, debug=False): + count = 0 + + if isinstance(text, str): + if debug: + print("----detect excessive whitespaces----") + count = len(text) + text = pattern_excessive_whitespaces.sub("", text) + count -= len(text) + if debug and count: + print(f"removed excessive whitespaces: {count}") + return text, count + + +# final version for repetition detection +def detect_text_repetitions(text, debug=False): + count = 0 + + if isinstance(text, str): + if debug: + print("----detect text repetitions----") + matches = pattern_text_repetitions.finditer(text) + for match in matches: + if debug: + print(match) + for groupNum in range(0, len(match.groups())): + groupNum = groupNum + 1 + print( + "Group {groupNum} found at {start}-{end}: `{group}`".format( + groupNum=groupNum, + start=match.start(groupNum), + end=match.end(groupNum), + group=match.group(groupNum), + ) + ) + + start, end = match.span() + count += end - start + + return count + + +def detect_repetitions(text, debug=False): + text, count_excessive_whitespaces = del_excessive_whitespaces(text, debug=debug) + count_text_repetitions = detect_text_repetitions(text, debug=debug) + total_repetitions = count_excessive_whitespaces + count_text_repetitions + + result = (count_excessive_whitespaces, count_text_repetitions, total_repetitions) + + if debug: + print(result) + return result + + +def detect_scores(text, debug=False): + newline_score, repetition_score, total_repetitions = detect_repetitions( + text, debug=debug + ) + return pd.Series([newline_score, repetition_score, total_repetitions]) diff --git a/llama-factory/config/llama3_8b_lora_sft.yaml b/llama-factory/config/llama3_8b_lora_sft.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b0568a636519830636eac0d628ea1ba1c4e3791 --- /dev/null +++ b/llama-factory/config/llama3_8b_lora_sft.yaml @@ -0,0 +1,46 @@ +### model +model_name_or_path: gradientai/Llama-3-8B-Instruct-Gradient-1048k + +### method +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +quantization_bit: 4 # use 4-bit QLoRA +loraplus_lr_ratio: 16.0 # use LoRA+ with lambda=16.0 +# use_unsloth: true # use UnslothAI's LoRA optimization for 2x faster training + +### dataset +dataset: alpaca_mac +template: llama3 +cutoff_len: 1024 +max_samples: 4528 +overwrite_cache: true +preprocessing_num_workers: 16 + +### output +# output_dir: saves/llama3-8b/lora/sft +output_dir: /Workspace/Users/donghao.huang@mastercard.com/lf-saves/llama3-8b/lora/sft/ +logging_steps: 10 +save_steps: 560 +plot_loss: true +overwrite_output_dir: true +# resume_from_checkpoint: true + +### train +per_device_train_batch_size: 1 +gradient_accumulation_steps: 8 +learning_rate: 1.0e-4 +num_train_epochs: 6.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +### eval +val_size: 0.01 +per_device_eval_batch_size: 1 +eval_strategy: steps +eval_steps: 560 + +report_to: none diff --git a/llama-factory/config/qwen2_0.5b_lora_sft.yaml b/llama-factory/config/qwen2_0.5b_lora_sft.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56d4603b741dbab5179e890d7d7f9c1d8aaeee69 --- /dev/null +++ b/llama-factory/config/qwen2_0.5b_lora_sft.yaml @@ -0,0 +1,42 @@ +### model +model_name_or_path: Qwen/Qwen2-0.5B-Instruct + +### method +stage: sft +do_train: true +finetuning_type: lora +lora_target: all + +### dataset +dataset: alpaca_mac +template: chatml +cutoff_len: 1024 +max_samples: 4528 +overwrite_cache: true +preprocessing_num_workers: 16 + +### output +output_dir: saves/qwen2-0.5b/lora/sft +logging_steps: 10 +save_steps: 560 +plot_loss: true +overwrite_output_dir: true + +### train +per_device_train_batch_size: 1 +gradient_accumulation_steps: 8 +learning_rate: 1.0e-4 +num_train_epochs: 6.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +### eval +val_size: 0.01 +per_device_eval_batch_size: 1 +eval_strategy: steps +eval_steps: 560 + +report_to: wandb +run_name: qwen2_0.5b_lora_sft # optional \ No newline at end of file diff --git a/llama-factory/config/qwen2_0.5b_lora_sft_unsloth.yaml b/llama-factory/config/qwen2_0.5b_lora_sft_unsloth.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7868676abf040a63f297fc2b5cf3d5ce6fb5f5e3 --- /dev/null +++ b/llama-factory/config/qwen2_0.5b_lora_sft_unsloth.yaml @@ -0,0 +1,45 @@ +### model +model_name_or_path: Qwen/Qwen2-0.5B-Instruct + +### method +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +quantization_bit: 4 # use 4-bit QLoRA +loraplus_lr_ratio: 16.0 # use LoRA+ with lambda=16.0 +use_unsloth: true # use UnslothAI's LoRA optimization for 2x faster training + +### dataset +dataset: alpaca_mac +template: chatml +cutoff_len: 1024 +max_samples: 4528 +overwrite_cache: true +preprocessing_num_workers: 16 + +### output +output_dir: saves/qwen2-0.5b/lora/sft +logging_steps: 10 +save_steps: 560 +plot_loss: true +overwrite_output_dir: true + +### train +per_device_train_batch_size: 1 +gradient_accumulation_steps: 8 +learning_rate: 1.0e-4 +num_train_epochs: 6.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +### eval +val_size: 0.01 +per_device_eval_batch_size: 1 +eval_strategy: steps +eval_steps: 560 + +report_to: wandb +run_name: qwen2_0.5b_lora_sft # optional \ No newline at end of file diff --git a/llama-factory/config/qwen2_1.5b_lora_sft.yaml b/llama-factory/config/qwen2_1.5b_lora_sft.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26a77bb6bdc0caab8b4ee5435bec8a93339fb86d --- /dev/null +++ b/llama-factory/config/qwen2_1.5b_lora_sft.yaml @@ -0,0 +1,42 @@ +### model +model_name_or_path: Qwen/Qwen2-1.5B-Instruct + +### method +stage: sft +do_train: true +finetuning_type: lora +lora_target: all + +### dataset +dataset: alpaca_mac +template: chatml +cutoff_len: 1024 +max_samples: 4528 +overwrite_cache: true +preprocessing_num_workers: 16 + +### output +output_dir: saves/qwen2-1.5b/lora/sft +logging_steps: 10 +save_steps: 560 +plot_loss: true +overwrite_output_dir: true + +### train +per_device_train_batch_size: 1 +gradient_accumulation_steps: 8 +learning_rate: 1.0e-4 +num_train_epochs: 6.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +### eval +val_size: 0.01 +per_device_eval_batch_size: 1 +eval_strategy: steps +eval_steps: 560 + +report_to: wandb +run_name: qwen2_1.5b_lora_sft # optional \ No newline at end of file diff --git a/llama-factory/config/qwen2_1.5b_lora_sft_unsloth.yaml b/llama-factory/config/qwen2_1.5b_lora_sft_unsloth.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1722908cbd01dadc8c5e37f0caa1d1d316b3a2f --- /dev/null +++ b/llama-factory/config/qwen2_1.5b_lora_sft_unsloth.yaml @@ -0,0 +1,45 @@ +### model +model_name_or_path: Qwen/Qwen2-1.5B-Instruct + +### method +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +quantization_bit: 4 # use 4-bit QLoRA +loraplus_lr_ratio: 16.0 # use LoRA+ with lambda=16.0 +use_unsloth: true # use UnslothAI's LoRA optimization for 2x faster training + +### dataset +dataset: alpaca_mac +template: chatml +cutoff_len: 1024 +max_samples: 4528 +overwrite_cache: true +preprocessing_num_workers: 16 + +### output +output_dir: saves/qwen2-1.5b/lora/sft +logging_steps: 10 +save_steps: 560 +plot_loss: true +overwrite_output_dir: true + +### train +per_device_train_batch_size: 1 +gradient_accumulation_steps: 8 +learning_rate: 1.0e-4 +num_train_epochs: 6.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +### eval +val_size: 0.01 +per_device_eval_batch_size: 1 +eval_strategy: steps +eval_steps: 560 + +report_to: wandb +run_name: qwen2_1.5b_lora_sft # optional \ No newline at end of file diff --git a/llama-factory/config/qwen2_7b_lora_sft.yaml b/llama-factory/config/qwen2_7b_lora_sft.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c22b8ad6fca5810c231320417dde1f8226900173 --- /dev/null +++ b/llama-factory/config/qwen2_7b_lora_sft.yaml @@ -0,0 +1,45 @@ +### model +model_name_or_path: Qwen/Qwen2-7B-Instruct + +### method +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +quantization_bit: 4 # use 4-bit QLoRA +loraplus_lr_ratio: 16.0 # use LoRA+ with lambda=16.0 +# use_unsloth: true # use UnslothAI's LoRA optimization for 2x faster training + +### dataset +dataset: alpaca_mac +template: chatml +cutoff_len: 1024 +max_samples: 4528 +overwrite_cache: true +preprocessing_num_workers: 16 + +### output +output_dir: saves/qwen2-7b/lora/sft +logging_steps: 10 +save_steps: 560 +plot_loss: true +overwrite_output_dir: true + +### train +per_device_train_batch_size: 1 +gradient_accumulation_steps: 8 +learning_rate: 1.0e-4 +num_train_epochs: 6.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +### eval +val_size: 0.01 +per_device_eval_batch_size: 1 +eval_strategy: steps +eval_steps: 560 + +report_to: wandb +run_name: qwen2_7b_lora_sft # optional \ No newline at end of file diff --git a/llama-factory/config/qwen2_7b_lora_sft_unsloth.yaml b/llama-factory/config/qwen2_7b_lora_sft_unsloth.yaml new file mode 100644 index 0000000000000000000000000000000000000000..513b937f111b73ef96b272c4c69def7b0bb2f39d --- /dev/null +++ b/llama-factory/config/qwen2_7b_lora_sft_unsloth.yaml @@ -0,0 +1,45 @@ +### model +model_name_or_path: Qwen/Qwen2-7B-Instruct + +### method +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +quantization_bit: 4 # use 4-bit QLoRA +loraplus_lr_ratio: 16.0 # use LoRA+ with lambda=16.0 +use_unsloth: true # use UnslothAI's LoRA optimization for 2x faster training + +### dataset +dataset: alpaca_mac +template: chatml +cutoff_len: 1024 +max_samples: 4528 +overwrite_cache: true +preprocessing_num_workers: 16 + +### output +output_dir: saves/qwen2-7b/lora/sft +logging_steps: 10 +save_steps: 560 +plot_loss: true +overwrite_output_dir: true + +### train +per_device_train_batch_size: 1 +gradient_accumulation_steps: 8 +learning_rate: 1.0e-4 +num_train_epochs: 6.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +### eval +val_size: 0.01 +per_device_eval_batch_size: 1 +eval_strategy: steps +eval_steps: 560 + +report_to: wandb +run_name: qwen2_7b_lora_sft # optional \ No newline at end of file diff --git a/llama-factory/data/alpaca_mac.json b/llama-factory/data/alpaca_mac.json new file mode 100644 index 0000000000000000000000000000000000000000..33dc8c3ae5b5ff6aea9481ae954e6fcee6cb435d --- /dev/null +++ b/llama-factory/data/alpaca_mac.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f03e62eb461c2204bbaef55f2de28ec115b1a5834b81f03b10f157551d5fe9f +size 2240344 diff --git a/llama-factory/data/dataset_info.json b/llama-factory/data/dataset_info.json new file mode 100644 index 0000000000000000000000000000000000000000..ad80357073bdcf6960b285519c4c0e6df9771b9a --- /dev/null +++ b/llama-factory/data/dataset_info.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84bce610296ed7e729647e85d25576b6226d20ddf0bca4982fb1deb02de35911 +size 13560 diff --git a/llama-factory/inference/qwen2_1.5b_lora_sft.yaml b/llama-factory/inference/qwen2_1.5b_lora_sft.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b505d93712224ea41f84de140fec81d12e9692a0 --- /dev/null +++ b/llama-factory/inference/qwen2_1.5b_lora_sft.yaml @@ -0,0 +1,4 @@ +model_name_or_path: Qwen/Qwen2-1.5B-Instruct +adapter_name_or_path: saves/qwen2-1.5b/lora/sft/checkpoint-1680 +template: chatml +finetuning_type: lora diff --git a/llm_toolkit/chat.py b/llm_toolkit/chat.py new file mode 100644 index 0000000000000000000000000000000000000000..39cbac337093bc069bcf431226022b9a317da5fe --- /dev/null +++ b/llm_toolkit/chat.py @@ -0,0 +1,88 @@ +import os +import sys +from llamafactory.chat import ChatModel +from llamafactory.extras.misc import torch_gc + +from dotenv import find_dotenv, load_dotenv + +found_dotenv = find_dotenv(".env") + +if len(found_dotenv) == 0: + found_dotenv = find_dotenv(".env.example") +print(f"loading env vars from: {found_dotenv}") +load_dotenv(found_dotenv, override=False) + +path = os.path.dirname(found_dotenv) +print(f"Adding {path} to sys.path") +sys.path.append(path) + +from llm_toolkit.translation_engine import * +from llm_toolkit.translation_utils import * + +model_name = os.getenv("MODEL_NAME") +load_in_4bit = os.getenv("LOAD_IN_4BIT") == "true" +eval_base_model = os.getenv("EVAL_BASE_MODEL") == "true" +eval_fine_tuned = os.getenv("EVAL_FINE_TUNED") == "true" +save_fine_tuned_model = os.getenv("SAVE_FINE_TUNED") == "true" +num_train_epochs = int(os.getenv("NUM_TRAIN_EPOCHS") or 0) +data_path = os.getenv("DATA_PATH") +results_path = os.getenv("RESULTS_PATH") + +max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally! +dtype = ( + None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ +) + +print( + model_name, + load_in_4bit, + max_seq_length, + num_train_epochs, + dtype, + data_path, + results_path, + eval_base_model, + eval_fine_tuned, + save_fine_tuned_model, +) + +adapter_name_or_path = ( + sys.argv[1] + if len(sys.argv) > 1 + else "llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-560" +) + +args = dict( + model_name_or_path=model_name, # use bnb-4bit-quantized Llama-3-8B-Instruct model + adapter_name_or_path=adapter_name_or_path, # load the saved LoRA adapters + template="chatml", # same to the one in training + finetuning_type="lora", # same to the one in training + quantization_bit=4, # load 4-bit quantized model +) +chat_model = ChatModel(args) + +messages = [] +print( + "Welcome to the CLI application, use `clear` to remove the history, use `exit` to exit the application." +) +while True: + query = input("\nUser: ") + if query.strip() == "exit": + break + if query.strip() == "clear": + messages = [] + torch_gc() + print("History has been removed.") + continue + + messages.append({"role": "user", "content": query}) + print("Assistant: ", end="", flush=True) + + response = "" + for new_text in chat_model.stream_chat(messages): + print(new_text, end="", flush=True) + response += new_text + print() + messages.append({"role": "assistant", "content": response}) + +torch_gc() diff --git a/llm_toolkit/eval.py b/llm_toolkit/eval.py new file mode 100644 index 0000000000000000000000000000000000000000..feddd5fa1952cf2e08dd24754007a0195711bc3d --- /dev/null +++ b/llm_toolkit/eval.py @@ -0,0 +1,67 @@ +import os +import sys +import torch +from dotenv import find_dotenv, load_dotenv + +found_dotenv = find_dotenv(".env") + +if len(found_dotenv) == 0: + found_dotenv = find_dotenv(".env.example") +print(f"loading env vars from: {found_dotenv}") +load_dotenv(found_dotenv, override=False) + +path = os.path.dirname(found_dotenv) +print(f"Adding {path} to sys.path") +sys.path.append(path) + +from llm_toolkit.translation_engine import * +from llm_toolkit.translation_utils import * + +model_name = os.getenv("MODEL_NAME") +adapter_name_or_path = os.getenv("ADAPTER_NAME_OR_PATH") +load_in_4bit = os.getenv("LOAD_IN_4BIT") == "true" +data_path = os.getenv("DATA_PATH") +results_path = os.getenv("RESULTS_PATH") + +print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path) + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(1) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + +model, tokenizer = load_model( + model_name, load_in_4bit=load_in_4bit, adapter_name_or_path=adapter_name_or_path +) + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(2) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + +datasets = load_translation_dataset(data_path, tokenizer) + +print("Evaluating model: " + model_name) +predictions = eval_model(model, tokenizer, datasets["test"]) + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(3) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + +if adapter_name_or_path is not None: + model_name += "_" + adapter_name_or_path.split("/")[-1] + +save_results( + model_name, + results_path, + datasets["test"], + predictions, + debug=True, +) + +metrics = calc_metrics(datasets["test"]["english"], predictions, debug=True) +print(metrics) diff --git a/llm_toolkit/eval_lf.py b/llm_toolkit/eval_lf.py new file mode 100644 index 0000000000000000000000000000000000000000..ef670714ffdc2ff2948cb1cf49079ce4a50c2b10 --- /dev/null +++ b/llm_toolkit/eval_lf.py @@ -0,0 +1,110 @@ +import os +import sys +import torch +from dotenv import find_dotenv, load_dotenv +from llamafactory.chat import ChatModel +from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig + +found_dotenv = find_dotenv(".env") + +if len(found_dotenv) == 0: + found_dotenv = find_dotenv(".env.example") +print(f"loading env vars from: {found_dotenv}") +load_dotenv(found_dotenv, override=False) + +path = os.path.dirname(found_dotenv) +print(f"Adding {path} to sys.path") +sys.path.append(path) + +from llm_toolkit.translation_utils import * + +model_name = os.getenv("MODEL_NAME") +adapter_name_or_path = os.getenv("ADAPTER_NAME_OR_PATH") +load_in_4bit = os.getenv("LOAD_IN_4BIT") == "true" +data_path = os.getenv("DATA_PATH") +results_path = os.getenv("RESULTS_PATH") + +print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path) + + +def load_model( + model_name, + max_seq_length=2048, + dtype=torch.bfloat16, + load_in_4bit=False, + adapter_name_or_path=None, +): + print(f"loading model: {model_name}") + + if adapter_name_or_path: + template = "llama3" if "llama-3" in model_name.lower() else "chatml" + + args = dict( + model_name_or_path=model_name, + adapter_name_or_path=adapter_name_or_path, # load the saved LoRA adapters + template=template, # same to the one in training + finetuning_type="lora", # same to the one in training + quantization_bit=4 if load_in_4bit else None, # load 4-bit quantized model + ) + chat_model = ChatModel(args) + return chat_model.engine.model, chat_model.engine.tokenizer + + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + bnb_config = BitsAndBytesConfig( + load_in_4bit=load_in_4bit, + bnb_4bit_quant_type="nf4", + bnb_4bit_use_double_quant=False, + bnb_4bit_compute_dtype=dtype, + ) + + model = AutoModelForCausalLM.from_pretrained( + model_name, + quantization_config=bnb_config, + torch_dtype=dtype, + trust_remote_code=True, + device_map="auto", + ) + + return model, tokenizer + + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(1) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + +model, tokenizer = load_model( + model_name, load_in_4bit=load_in_4bit, adapter_name_or_path=adapter_name_or_path +) + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(2) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + +datasets = load_translation_dataset(data_path, tokenizer) + +print("Evaluating model: " + model_name) +predictions = eval_model(model, tokenizer, datasets["test"]) + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(3) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + +if adapter_name_or_path is not None: + model_name += "_" + adapter_name_or_path.split("/")[-1] + +save_results( + model_name, + results_path, + datasets["test"], + predictions, + debug=True, +) + +metrics = calc_metrics(datasets["test"]["english"], predictions, debug=True) +print(metrics) diff --git a/llm_toolkit/llm_utils.py b/llm_toolkit/llm_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..15578840d8afd662fdd74ec08caa4e9e14672f87 --- /dev/null +++ b/llm_toolkit/llm_utils.py @@ -0,0 +1,165 @@ +import os +import re +import sys +import torch +from llamafactory.chat import ChatModel +from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TextStreamer + + +def load_model( + model_name, + max_seq_length=2048, + dtype=torch.bfloat16, + load_in_4bit=False, + adapter_name_or_path=None, +): + print(f"loading model: {model_name}") + + if adapter_name_or_path: + template = "llama3" if "llama-3" in model_name.lower() else "chatml" + + args = dict( + model_name_or_path=model_name, + adapter_name_or_path=adapter_name_or_path, # load the saved LoRA adapters + template=template, # same to the one in training + finetuning_type="lora", # same to the one in training + quantization_bit=4 if load_in_4bit else None, # load 4-bit quantized model + ) + chat_model = ChatModel(args) + return chat_model.engine.model, chat_model.engine.tokenizer + + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + bnb_config = BitsAndBytesConfig( + load_in_4bit=load_in_4bit, + bnb_4bit_quant_type="nf4", + bnb_4bit_use_double_quant=False, + bnb_4bit_compute_dtype=dtype, + ) + + model = AutoModelForCausalLM.from_pretrained( + model_name, + quantization_config=bnb_config, + torch_dtype=dtype, + trust_remote_code=True, + device_map="auto", + ) if load_in_4bit else AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype=dtype, + trust_remote_code=True, + device_map="auto", + ) + + return model, tokenizer + +def test_model(model, tokenizer, prompt): + inputs = tokenizer( + [prompt], + return_tensors="pt", + ).to("cuda") + + text_streamer = TextStreamer(tokenizer) + + _ = model.generate( + **inputs, max_new_tokens=2048, streamer=text_streamer, use_cache=True + ) + + +def extract_answer(text, debug=False): + if text: + # Remove the begin and end tokens + text = re.sub( + r".*?(assistant|\[/INST\]).+?\b", "", text, flags=re.DOTALL | re.MULTILINE + ) + if debug: + print("--------\nstep 1:", text) + + text = re.sub(r"<.+?>.*", "", text, flags=re.DOTALL | re.MULTILINE) + if debug: + print("--------\nstep 2:", text) + + text = re.sub( + r".*?end_header_id\|>\n\n", "", text, flags=re.DOTALL | re.MULTILINE + ) + if debug: + print("--------\nstep 3:", text) + + return text + +def eval_model(model, tokenizer, eval_dataset): + total = len(eval_dataset) + predictions = [] + for i in tqdm(range(total)): + inputs = tokenizer( + eval_dataset["prompt"][i : i + 1], + return_tensors="pt", + ).to("cuda") + + outputs = model.generate(**inputs, max_new_tokens=4096, use_cache=False) + decoded_output = tokenizer.batch_decode(outputs) + debug = i == 0 + decoded_output = [ + extract_answer(output, debug=debug) for output in decoded_output + ] + predictions.extend(decoded_output) + + return predictions + +def save_model( + model, + tokenizer, + include_gguf=True, + include_merged=True, + publish=True, +): + try: + token = os.getenv("HF_TOKEN") or None + model_name = os.getenv("MODEL_NAME") + + save_method = "lora" + quantization_method = "q5_k_m" + + model_names = get_model_names( + model_name, save_method=save_method, quantization_method=quantization_method + ) + + model.save_pretrained(model_names["local"]) + tokenizer.save_pretrained(model_names["local"]) + + if publish: + model.push_to_hub( + model_names["hub"], + token=token, + ) + tokenizer.push_to_hub( + model_names["hub"], + token=token, + ) + + if include_merged: + model.save_pretrained_merged( + model_names["local"] + "-merged", tokenizer, save_method=save_method + ) + if publish: + model.push_to_hub_merged( + model_names["hub"] + "-merged", + tokenizer, + save_method="lora", + token="", + ) + + if include_gguf: + model.save_pretrained_gguf( + model_names["local-gguf"], + tokenizer, + quantization_method=quantization_method, + ) + + if publish: + model.push_to_hub_gguf( + model_names["hub-gguf"], + tokenizer, + quantization_method=quantization_method, + token=token, + ) + except Exception as e: + print(e) diff --git a/llm_toolkit/translation_engine.py b/llm_toolkit/translation_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..b4743a4508f33c407fa88bb8250746cefc8ef905 --- /dev/null +++ b/llm_toolkit/translation_engine.py @@ -0,0 +1,130 @@ +import os +import pandas as pd +import torch +from unsloth import FastLanguageModel, is_bfloat16_supported +from trl import SFTTrainer +from transformers import TrainingArguments, TextStreamer +from llm_toolkit.translation_utils import * +from llamafactory.chat import ChatModel + +print(f"loading {__file__}") + + +def get_model_names( + model_name, save_method="merged_4bit_forced", quantization_method="q5_k_m" +): + hub_model = model_name.split("/")[-1] + "-MAC-" + local_model = "models/" + hub_model + + return { + "local": local_model + save_method, + "local-gguf": local_model + quantization_method, + "hub": hub_model + save_method, + "hub-gguf": hub_model + "gguf-" + quantization_method, + } + + +def load_model( + model_name, + max_seq_length=2048, + dtype=None, + load_in_4bit=False, + template="chatml", + adapter_name_or_path=None, +): + print(f"loading model: {model_name}") + + if adapter_name_or_path: + args = dict( + model_name_or_path=model_name, + adapter_name_or_path=adapter_name_or_path, # load the saved LoRA adapters + template=template, # same to the one in training + finetuning_type="lora", # same to the one in training + quantization_bit=4, # load 4-bit quantized model + ) + chat_model = ChatModel(args) + return chat_model.engine.model, chat_model.engine.tokenizer + + model, tokenizer = FastLanguageModel.from_pretrained( + model_name=model_name, # YOUR MODEL YOU USED FOR TRAINING + max_seq_length=max_seq_length, + dtype=dtype, + load_in_4bit=load_in_4bit, + trust_remote_code=True, + ) + FastLanguageModel.for_inference(model) + + return model, tokenizer + + +def test_model(model, tokenizer, prompt): + inputs = tokenizer( + [prompt], + return_tensors="pt", + ).to("cuda") + + text_streamer = TextStreamer(tokenizer) + + _ = model.generate( + **inputs, max_new_tokens=128, streamer=text_streamer, use_cache=True + ) + + +def load_trainer( + model, + tokenizer, + dataset, + num_train_epochs, + max_seq_length=2048, + fp16=False, + bf16=False, + output_dir="./outputs", +): + model = FastLanguageModel.get_peft_model( + model, + r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 + target_modules=[ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj", + ], + lora_alpha=16, + lora_dropout=0, # Supports any, but = 0 is optimized + bias="none", # Supports any, but = "none" is optimized + # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes! + use_gradient_checkpointing="unsloth", # True or "unsloth" for very long context + random_state=3407, + use_rslora=False, # We support rank stabilized LoRA + loftq_config=None, # And LoftQ + ) + + trainer = SFTTrainer( + model=model, + tokenizer=tokenizer, + train_dataset=dataset, + dataset_text_field="text", + max_seq_length=max_seq_length, + dataset_num_proc=2, + packing=False, # Can make training 5x faster for short sequences. + args=TrainingArguments( + per_device_train_batch_size=2, + gradient_accumulation_steps=4, + warmup_steps=5, + num_train_epochs=num_train_epochs, + learning_rate=2e-4, + fp16=not is_bfloat16_supported(), + bf16=is_bfloat16_supported(), + logging_steps=100, + optim="adamw_8bit", + weight_decay=0.01, + lr_scheduler_type="linear", + seed=3407, + output_dir=output_dir, + ), + ) + + return trainer diff --git a/llm_toolkit/translation_utils.py b/llm_toolkit/translation_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..03922e5aeeafd0a2683f46acb06f71138dec4ca4 --- /dev/null +++ b/llm_toolkit/translation_utils.py @@ -0,0 +1,420 @@ +import os +import re +import pandas as pd +import evaluate +import seaborn as sns +import matplotlib.pyplot as plt +from datasets import load_dataset +from langchain_openai import ChatOpenAI +from langchain_core.prompts import ChatPromptTemplate +from tqdm import tqdm + +print(f"loading {__file__}") + +bleu = evaluate.load("bleu") +rouge = evaluate.load("rouge") +meteor = evaluate.load("meteor") +accuracy = evaluate.load("accuracy") + + +def extract_answer(text, debug=False): + if text: + # Remove the begin and end tokens + text = re.sub( + r".*?(assistant|\[/INST\]).+?\b", "", text, flags=re.DOTALL | re.MULTILINE + ) + if debug: + print("--------\nstep 1:", text) + + text = re.sub(r"<.+?>.*", "", text, flags=re.DOTALL | re.MULTILINE) + if debug: + print("--------\nstep 2:", text) + + text = re.sub( + r".*?end_header_id\|>\n\n", "", text, flags=re.DOTALL | re.MULTILINE + ) + if debug: + print("--------\nstep 3:", text) + + return text + + +def calc_metrics(references, predictions, debug=False): + assert len(references) == len( + predictions + ), f"lengths are difference: {len(references)} != {len(predictions)}" + + predictions = [extract_answer(text) for text in predictions] + + correct = [1 if ref == pred else 0 for ref, pred in zip(references, predictions)] + accuracy = sum(correct) / len(references) + + results = {"accuracy": accuracy} + if debug: + correct_ids = [i for i, c in enumerate(correct) if c == 1] + results["correct_ids"] = correct_ids + + results["meteor"] = meteor.compute(predictions=predictions, references=references)[ + "meteor" + ] + + results["bleu_scores"] = bleu.compute( + predictions=predictions, references=references, max_order=4 + ) + results["rouge_scores"] = rouge.compute( + predictions=predictions, references=references + ) + return results + + +def save_results(model_name, results_path, dataset, predictions, debug=False): + if not os.path.exists(results_path): + # Get the directory part of the file path + dir_path = os.path.dirname(results_path) + + # Create all directories in the path (if they don't exist) + os.makedirs(dir_path, exist_ok=True) + df = dataset.to_pandas() + df.drop(columns=["text", "prompt"], inplace=True) + else: + df = pd.read_csv(results_path, on_bad_lines="warn") + + df[model_name] = predictions + + if debug: + print(df.head(1)) + + df.to_csv(results_path, index=False) + + +def load_translation_dataset(data_path, tokenizer=None): + train_data_file = data_path.replace(".tsv", "-train.tsv") + test_data_file = data_path.replace(".tsv", "-test.tsv") + + if not os.path.exists(train_data_file): + print("generating train/test data files") + dataset = load_dataset( + "csv", data_files=data_path, delimiter="\t", split="train" + ) + print(len(dataset)) + dataset = dataset.filter(lambda x: x["chinese"] and x["english"]) + + datasets = dataset.train_test_split(test_size=0.2) + print(len(dataset)) + + # Convert to pandas DataFrame + train_df = pd.DataFrame(datasets["train"]) + test_df = pd.DataFrame(datasets["test"]) + + # Save to TSV + train_df.to_csv(train_data_file, sep="\t", index=False) + test_df.to_csv(test_data_file, sep="\t", index=False) + + print("loading train/test data files") + datasets = load_dataset( + "csv", + data_files={"train": train_data_file, "test": test_data_file}, + delimiter="\t", + ) + + if tokenizer: + translation_prompt = "Please translate the following Chinese text into English and provide only the translated content, nothing else.\n{}" + + def formatting_prompts_func(examples): + inputs = examples["chinese"] + outputs = examples["english"] + + messages = [ + { + "role": "system", + "content": "You are an expert in translating Chinese to English.", + }, + None, + ] + + model_name = os.getenv("MODEL_NAME") + + if "mistral" in model_name.lower(): + messages = messages[1:] + + texts = [] + prompts = [] + for input, output in zip(inputs, outputs): + prompt = translation_prompt.format(input) + messages[-1] = {"role": "user", "content": prompt} + + prompt = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + prompts.append(prompt) + texts.append(prompt + output + tokenizer.eos_token) + return {"text": texts, "prompt": prompts} + + datasets = datasets.map( + formatting_prompts_func, + batched=True, + ) + + print(datasets) + return datasets + + +def eval_model(model, tokenizer, eval_dataset): + total = len(eval_dataset) + predictions = [] + for i in tqdm(range(total)): + inputs = tokenizer( + eval_dataset["prompt"][i : i + 1], + return_tensors="pt", + ).to("cuda") + + outputs = model.generate(**inputs, max_new_tokens=4096, use_cache=False) + decoded_output = tokenizer.batch_decode(outputs) + debug = i == 0 + decoded_output = [ + extract_answer(output, debug=debug) for output in decoded_output + ] + predictions.extend(decoded_output) + + return predictions + + +def save_model( + model, + tokenizer, + include_gguf=True, + include_merged=True, + publish=True, +): + try: + token = os.getenv("HF_TOKEN") or None + model_name = os.getenv("MODEL_NAME") + + save_method = "lora" + quantization_method = "q5_k_m" + + model_names = get_model_names( + model_name, save_method=save_method, quantization_method=quantization_method + ) + + model.save_pretrained(model_names["local"]) + tokenizer.save_pretrained(model_names["local"]) + + if publish: + model.push_to_hub( + model_names["hub"], + token=token, + ) + tokenizer.push_to_hub( + model_names["hub"], + token=token, + ) + + if include_merged: + model.save_pretrained_merged( + model_names["local"] + "-merged", tokenizer, save_method=save_method + ) + if publish: + model.push_to_hub_merged( + model_names["hub"] + "-merged", + tokenizer, + save_method="lora", + token="", + ) + + if include_gguf: + model.save_pretrained_gguf( + model_names["local-gguf"], + tokenizer, + quantization_method=quantization_method, + ) + + if publish: + model.push_to_hub_gguf( + model_names["hub-gguf"], + tokenizer, + quantization_method=quantization_method, + token=token, + ) + except Exception as e: + print(e) + + +def get_metrics(df): + metrics_df = pd.DataFrame(df.columns.T)[2:] + metrics_df.rename(columns={0: "model"}, inplace=True) + metrics_df["model"] = metrics_df["model"].apply(lambda x: x.split("/")[-1]) + metrics_df.reset_index(inplace=True) + metrics_df = metrics_df.drop(columns=["index"]) + + accuracy = [] + meteor = [] + bleu_1 = [] + rouge_l = [] + all_metrics = [] + for col in df.columns[2:]: + metrics = calc_metrics(df["english"], df[col], debug=True) + print(f"{col}: {metrics}") + + accuracy.append(metrics["accuracy"]) + meteor.append(metrics["meteor"]) + bleu_1.append(metrics["bleu_scores"]["bleu"]) + rouge_l.append(metrics["rouge_scores"]["rougeL"]) + all_metrics.append(metrics) + + metrics_df["accuracy"] = accuracy + metrics_df["meteor"] = meteor + metrics_df["bleu_1"] = bleu_1 + metrics_df["rouge_l"] = rouge_l + metrics_df["all_metrics"] = all_metrics + + return metrics_df + + +def plot_metrics(metrics_df, figsize=(14, 5), ylim=(0, 0.44)): + plt.figure(figsize=figsize) + df_melted = pd.melt( + metrics_df, id_vars="model", value_vars=["meteor", "bleu_1", "rouge_l"] + ) + + barplot = sns.barplot(x="variable", y="value", hue="model", data=df_melted) + + # Set different hatches for each model + hatches = ["/", "\\", "|", "-", "+", "x", "o", "O", ".", "*", "//", "\\\\"] + + # Create a dictionary to map models to hatches + model_hatches = { + model: hatches[i % len(hatches)] + for i, model in enumerate(metrics_df["model"].unique()) + } + + # Apply hatches based on the model + num_vars = len(df_melted["variable"].unique()) + for i, bar in enumerate(barplot.patches): + model = df_melted["model"].iloc[i // num_vars] + bar.set_hatch(model_hatches[model]) + + # Manually update legend to match the bar hatches + handles, labels = barplot.get_legend_handles_labels() + for handle, model in zip(handles, metrics_df["model"].unique()): + handle.set_hatch(model_hatches[model]) + + barplot.set_xticklabels(["METEOR", "BLEU-1", "ROUGE-L"]) + for p in barplot.patches: + if p.get_height() == 0: + continue + barplot.annotate( + f"{p.get_height():.2f}", + (p.get_x() + p.get_width() / 2.0, p.get_height()), + ha="center", + va="center", + xytext=(0, 10), + textcoords="offset points", + ) + + barplot.set(ylim=ylim, ylabel="Scores", xlabel="Metrics") + plt.legend(bbox_to_anchor=(0.5, -0.1), loc="upper center") + plt.show() + + +def plot_times(perf_df, ylim=0.421): + # Adjusted code to put "train-time" bars in red at the bottom + + fig, ax1 = plt.subplots(figsize=(12, 10)) + + color_train = "tab:red" + color_eval = "orange" + ax1.set_xlabel("Models") + ax1.set_ylabel("Time (mins)") + ax1.set_xticks(range(len(perf_df["model"]))) # Set x-ticks positions + ax1.set_xticklabels(perf_df["model"], rotation=90) + + # Plot "train-time" first so it's at the bottom + ax1.bar( + perf_df["model"], + perf_df["train-time(mins)"], + color=color_train, + label="train-time", + ) + + # Then, plot "eval-time" on top of "train-time" + ax1.bar( + perf_df["model"], + perf_df["eval-time(mins)"], + bottom=perf_df["train-time(mins)"], + color=color_eval, + label="eval-time", + ) + + ax1.tick_params(axis="y") + ax1.legend(loc="upper left") + + if "meteor" in perf_df.columns: + ax2 = ax1.twinx() + color_meteor = "tab:blue" + ax2.set_ylabel("METEOR", color=color_meteor) + ax2.plot( + perf_df["model"], + perf_df["meteor"], + color=color_meteor, + marker="o", + label="meteor", + ) + ax2.tick_params(axis="y", labelcolor=color_meteor) + ax2.legend(loc="upper right") + ax2.set_ylim(ax2.get_ylim()[0], ylim) + + # Show numbers in bars + for p in ax1.patches: + height = p.get_height() + if height == 0: # Skip bars with height 0 + continue + ax1.annotate( + f"{height:.2f}", + (p.get_x() + p.get_width() / 2.0, p.get_y() + height), + ha="center", + va="center", + xytext=(0, -10), + textcoords="offset points", + ) + + fig.tight_layout() + plt.show() + + +def translate_via_llm(text): + base_url = os.getenv("OPENAI_BASE_URL") or "http://localhost:8000/v1" + llm = ChatOpenAI( + model="gpt-4o", + temperature=0, + max_tokens=None, + timeout=None, + max_retries=2, + base_url=base_url, + ) + + prompt = ChatPromptTemplate.from_messages( + [ + ( + "human", + "Please translate the following Chinese text into English and provide only the translated content, nothing else.\n{input}", + ), + ] + ) + + chain = prompt | llm + response = chain.invoke( + { + "input": text, + } + ) + return response.content + + +def translate(text, cache_dict): + if text in cache_dict: + return cache_dict[text] + else: + translated_text = translate_via_llm(text) + cache_dict[text] = translated_text + return translated_text diff --git a/llm_toolkit/tune.py b/llm_toolkit/tune.py new file mode 100644 index 0000000000000000000000000000000000000000..def4156258c424091cded44e7c9bcc66615bb5f6 --- /dev/null +++ b/llm_toolkit/tune.py @@ -0,0 +1,143 @@ +import os +import sys +import torch +from dotenv import find_dotenv, load_dotenv + +found_dotenv = find_dotenv(".env") + +if len(found_dotenv) == 0: + found_dotenv = find_dotenv(".env.example") +print(f"loading env vars from: {found_dotenv}") +load_dotenv(found_dotenv, override=False) + +path = os.path.dirname(found_dotenv) +print(f"Adding {path} to sys.path") +sys.path.append(path) + +from llm_toolkit.translation_engine import * +from llm_toolkit.translation_utils import * + + +model_name = os.getenv("MODEL_NAME") +load_in_4bit = os.getenv("LOAD_IN_4BIT") == "true" +eval_base_model = os.getenv("EVAL_BASE_MODEL") == "true" +eval_fine_tuned = os.getenv("EVAL_FINE_TUNED") == "true" +save_fine_tuned_model = os.getenv("SAVE_FINE_TUNED") == "true" +num_train_epochs = int(os.getenv("NUM_TRAIN_EPOCHS") or 0) +data_path = os.getenv("DATA_PATH") +results_path = os.getenv("RESULTS_PATH") + +max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally! +dtype = ( + None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ +) + +print( + model_name, + load_in_4bit, + max_seq_length, + num_train_epochs, + dtype, + data_path, + results_path, + eval_base_model, + eval_fine_tuned, + save_fine_tuned_model, +) + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(1) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + +model, tokenizer = load_model(model_name, load_in_4bit=load_in_4bit) + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(2) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + +datasets = load_translation_dataset(data_path, tokenizer) + +if eval_base_model: + print("Evaluating base model: " + model_name) + predictions = eval_model(model, tokenizer, datasets["test"]) + + # calc_metrics(datasets["test"]["english"], predictions, debug=True) + + save_results( + model_name, + results_path, + datasets["test"], + predictions, + debug=True, + ) + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(3) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + + +def is_bfloat16_supported(): + return True + + +trainer = load_trainer( + model, + tokenizer, + datasets["train"], + num_train_epochs, + fp16=not is_bfloat16_supported(), + bf16=is_bfloat16_supported(), +) + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(4) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + +trainer_stats = trainer.train() + +# @title Show final memory and time stats +used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +used_memory_for_lora = round(used_memory - start_gpu_memory, 3) +used_percentage = round(used_memory / max_memory * 100, 3) +lora_percentage = round(used_memory_for_lora / max_memory * 100, 3) +print(f"(5) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.") +print( + f"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training." +) +print(f"Peak reserved memory = {used_memory} GB.") +print(f"Peak reserved memory for training = {used_memory_for_lora} GB.") +print(f"Peak reserved memory % of max memory = {used_percentage} %.") +print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.") + +if eval_fine_tuned: + print("Evaluating fine-tuned model: " + model_name) + FastLanguageModel.for_inference(model) # Enable native 2x faster inference + predictions = eval_model(model, tokenizer, datasets["test"]) + + # calc_metrics(datasets["test"]["english"], predictions, debug=True) + + save_results( + model_name + "(finetuned)", + results_path, + datasets["test"], + predictions, + debug=True, + ) + +gpu_stats = torch.cuda.get_device_properties(0) +start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) +max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) +print(f"(6) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") +print(f"{start_gpu_memory} GB of memory reserved.") + +if save_fine_tuned_model: + save_model(model, tokenizer) diff --git a/notebooks/00_Data_Analysis.ipynb b/notebooks/00_Data_Analysis.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..139a9953c1e22ea5d0bea371b7d65a2d41249c9d --- /dev/null +++ b/notebooks/00_Data_Analysis.ipynb @@ -0,0 +1,8465 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d4ad56f5-dd6b-47e2-8b75-bdc3cb0d5acd", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "288100c1-33d1-4e46-abaf-9a5ea4f7eca5", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /Users/inflaton/code/engd/papers/rapget-finetuning\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "396e9b1b-b8b6-4281-a574-e9decfd020f7", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: \n" + ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "a8313845-33ce-4fcf-8c79-9b34b4729352", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading /Users/inflaton/code/engd/papers/rapget-finetuning/llm_toolkit/translation_utils.py\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package wordnet to\n", + "[nltk_data] /Users/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /Users/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to\n", + "[nltk_data] /Users/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n" + ] + } + ], + "source": [ + "from llm_toolkit.translation_utils import *" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3f4860d2-8b3f-4ad9-9343-08fdce5076f9", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Data Processing" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "ba199929-4abf-4e8d-acfe-5f4ba45030b8", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chineseenglishunsloth/Qwen2-0.5B-Instructunsloth/Qwen2-0.5B-Instruct(finetuned)unsloth/Qwen2-1.5B-Instructunsloth/Qwen2-1.5B-Instruct(finetuned)unsloth/Qwen2-0.5B-Instruct-bnb-4bitunsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned)unsloth/Qwen2-1.5B-Instruct-bnb-4bitunsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned)unsloth/Qwen2-7B-Instructunsloth/Qwen2-7B-Instruct(finetuned)unsloth/Qwen2-7B-Instruct-bnb-4bitunsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned)
0老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞...Old Geng picked up his shotgun, squinted, and ...Old Tang held his gun, squinting his eyes with...Old Geng lifted his rifle and narrowed his eye...Old Geng took up his gun, squinted one of its ...Old Geng raised the rifle, squeezed one tiny t...Old Teng raised his gun and looked up at a pai...Old Geng raised his rifle, squinted his eyes, ...Old耿拿起枪，眯着眼睛一搂扳机就响了枪，金麻雀噼里啪啦的往下掉，铁砂子在柳枝间飞溅，发出“...Old Geng raised his pistol, squinted, and fire...Old Aigang raised his rifle, squinting one of ...Old Geng raised his rifle and squinted into th...Old Geng raised his gun, squinting one of his ...Old Geng raised his rifle and squinted into th...
1次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，...Next day Grannie Liu was up before dawn. As so...The next morning when it was still dark, Liu G...It was still not light when this little update...By the time the next day dawned, Liu Lao got u...Having been woken just before daybreak, Granni...The next day at dawn, Liu Geowon got up early ...Three or four hours before this, Grannie Liu h...At dawn the next day, Liu Langlang got up to b...But by some miracle of preparation, Grannie Li...The next morning, before dawn, Old Liu rose to...First thing next morning Grannie Liu rose befo...The next morning, before dawn, Old Liu rose to...First thing in the morning Grannie Liu rose an...
\n", + "
" + ], + "text/plain": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "1 次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "1 Next day Grannie Liu was up before dawn. As so... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct \\\n", + "0 Old Tang held his gun, squinting his eyes with... \n", + "1 The next morning when it was still dark, Liu G... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct(finetuned) \\\n", + "0 Old Geng lifted his rifle and narrowed his eye... \n", + "1 It was still not light when this little update... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct \\\n", + "0 Old Geng took up his gun, squinted one of its ... \n", + "1 By the time the next day dawned, Liu Lao got u... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct(finetuned) \\\n", + "0 Old Geng raised the rifle, squeezed one tiny t... \n", + "1 Having been woken just before daybreak, Granni... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct-bnb-4bit \\\n", + "0 Old Teng raised his gun and looked up at a pai... \n", + "1 The next day at dawn, Liu Geowon got up early ... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned) \\\n", + "0 Old Geng raised his rifle, squinted his eyes, ... \n", + "1 Three or four hours before this, Grannie Liu h... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct-bnb-4bit \\\n", + "0 Old耿拿起枪，眯着眼睛一搂扳机就响了枪，金麻雀噼里啪啦的往下掉，铁砂子在柳枝间飞溅，发出“... \n", + "1 At dawn the next day, Liu Langlang got up to b... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned) \\\n", + "0 Old Geng raised his pistol, squinted, and fire... \n", + "1 But by some miracle of preparation, Grannie Li... \n", + "\n", + " unsloth/Qwen2-7B-Instruct \\\n", + "0 Old Aigang raised his rifle, squinting one of ... \n", + "1 The next morning, before dawn, Old Liu rose to... \n", + "\n", + " unsloth/Qwen2-7B-Instruct(finetuned) \\\n", + "0 Old Geng raised his rifle and squinted into th... \n", + "1 First thing next morning Grannie Liu rose befo... \n", + "\n", + " unsloth/Qwen2-7B-Instruct-bnb-4bit \\\n", + "0 Old Geng raised his gun, squinting one of his ... \n", + "1 The next morning, before dawn, Old Liu rose to... \n", + "\n", + " unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned) \n", + "0 Old Geng raised his rifle and squinted into th... \n", + "1 First thing in the morning Grannie Liu rose an... " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "df = pd.read_csv(\"results/mac-results.csv\")\n", + "df.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "52730964-6323-48dc-a6db-2015938e7dff", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['chinese',\n", + " 'english',\n", + " 'unsloth/Qwen2-0.5B-Instruct',\n", + " 'unsloth/Qwen2-0.5B-Instruct(finetuned)',\n", + " 'unsloth/Qwen2-1.5B-Instruct',\n", + " 'unsloth/Qwen2-1.5B-Instruct(finetuned)',\n", + " 'unsloth/Qwen2-0.5B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned)',\n", + " 'unsloth/Qwen2-1.5B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned)',\n", + " 'unsloth/Qwen2-7B-Instruct',\n", + " 'unsloth/Qwen2-7B-Instruct(finetuned)',\n", + " 'unsloth/Qwen2-7B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned)']" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.columns.to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "0ea840e0-0e63-4590-a5a8-76c0dc97e30d", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "df = df[\n", + " [\n", + " \"chinese\",\n", + " \"english\",\n", + " \"unsloth/Qwen2-0.5B-Instruct\",\n", + " \"unsloth/Qwen2-0.5B-Instruct-bnb-4bit\",\n", + " \"unsloth/Qwen2-0.5B-Instruct(finetuned)\",\n", + " \"unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned)\",\n", + " \"unsloth/Qwen2-1.5B-Instruct\",\n", + " \"unsloth/Qwen2-1.5B-Instruct-bnb-4bit\",\n", + " \"unsloth/Qwen2-1.5B-Instruct(finetuned)\",\n", + " \"unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned)\",\n", + " \"unsloth/Qwen2-7B-Instruct\",\n", + " \"unsloth/Qwen2-7B-Instruct-bnb-4bit\",\n", + " \"unsloth/Qwen2-7B-Instruct(finetuned)\",\n", + " \"unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned)\",\n", + " ]\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "37543d79-ca2f-46c0-8ca5-67d7a9659431", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "df.to_csv(\"results/experiment-1-results.csv\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f75d024c-3abe-4695-8dd0-e9599efe8aec", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chineseenglishunsloth/Qwen2-0.5B-Instructunsloth/Qwen2-0.5B-Instruct(finetuned)unsloth/Qwen2-1.5B-Instructunsloth/Qwen2-1.5B-Instruct(finetuned)unsloth/Qwen2-7B-Instruct-bnb-4bitunsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned)gradientai/Llama-3-8B-Instruct-Gradient-1048kgradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned)unsloth/Qwen2-7B-Instructunsloth/Qwen2-72B-Instruct-bnb-4bitunsloth/Qwen2-7B-Instruct(finetuned)unsloth/mistral-7b-instruct-v0.3unsloth/mistral-7b-instruct-v0.3(finetuned)unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned)
0老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞...Old Geng picked up his shotgun, squinted, and ...Old Teng holds his gun up, his eyes narrowed a...Old Geng raised his rifle and tilted his head ...Old Jin raises his gun, squints one eye as he ...Old Geng raised his pistol, squinted through t...Old Geng raised his gun, squinting one of his ...Old Geng raised his rifle and squinted into on...The old man pulled out his gun, squinting one ...Old Geng raised his rifle, squinting through t...Old Geng raised his gun, squinted one of his t...Lao Geng raised his gun, narrowed one of his t...Old Geng raised his rifle and squinted into th...Geng Da initiates firing, squinting to form a ...Old Geng aimed and fired. A triangular slit op...Old Geng raised his gun, narrowed one of his t...
1次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，...Next day Grannie Liu was up before dawn. As so...The next morning, Liu Geo woke up at five o'cl...But not before noon did Grannie Liu rise up an...At dawn the next day, Liu Langlang got up earl...She got up about dawn with a purpose already e...The next morning, before dawn, Old Liu rose to...First thing next morning Grannie Liu rose earl...The next day, when the sun had not yet risen, ...Grannie Liu got up before daylight was even vi...The next morning, before the dawn had fully br...Before dawn next morning, Granny Liu got up to...First thing in the morning Grannie Liu rose to...The next day, when it was still dark, Liu Lao ...Before dawn next day Grannie Liu got up and bu...As soon as it was light, Grannie Liu got up an...
\n", + "
" + ], + "text/plain": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "1 次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "1 Next day Grannie Liu was up before dawn. As so... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct \\\n", + "0 Old Teng holds his gun up, his eyes narrowed a... \n", + "1 The next morning, Liu Geo woke up at five o'cl... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct(finetuned) \\\n", + "0 Old Geng raised his rifle and tilted his head ... \n", + "1 But not before noon did Grannie Liu rise up an... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct \\\n", + "0 Old Jin raises his gun, squints one eye as he ... \n", + "1 At dawn the next day, Liu Langlang got up earl... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct(finetuned) \\\n", + "0 Old Geng raised his pistol, squinted through t... \n", + "1 She got up about dawn with a purpose already e... \n", + "\n", + " unsloth/Qwen2-7B-Instruct-bnb-4bit \\\n", + "0 Old Geng raised his gun, squinting one of his ... \n", + "1 The next morning, before dawn, Old Liu rose to... \n", + "\n", + " unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned) \\\n", + "0 Old Geng raised his rifle and squinted into on... \n", + "1 First thing next morning Grannie Liu rose earl... \n", + "\n", + " gradientai/Llama-3-8B-Instruct-Gradient-1048k \\\n", + "0 The old man pulled out his gun, squinting one ... \n", + "1 The next day, when the sun had not yet risen, ... \n", + "\n", + " gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) \\\n", + "0 Old Geng raised his rifle, squinting through t... \n", + "1 Grannie Liu got up before daylight was even vi... \n", + "\n", + " unsloth/Qwen2-7B-Instruct \\\n", + "0 Old Geng raised his gun, squinted one of his t... \n", + "1 The next morning, before the dawn had fully br... \n", + "\n", + " unsloth/Qwen2-72B-Instruct-bnb-4bit \\\n", + "0 Lao Geng raised his gun, narrowed one of his t... \n", + "1 Before dawn next morning, Granny Liu got up to... \n", + "\n", + " unsloth/Qwen2-7B-Instruct(finetuned) \\\n", + "0 Old Geng raised his rifle and squinted into th... \n", + "1 First thing in the morning Grannie Liu rose to... \n", + "\n", + " unsloth/mistral-7b-instruct-v0.3 \\\n", + "0 Geng Da initiates firing, squinting to form a ... \n", + "1 The next day, when it was still dark, Liu Lao ... \n", + "\n", + " unsloth/mistral-7b-instruct-v0.3(finetuned) \\\n", + "0 Old Geng aimed and fired. A triangular slit op... \n", + "1 Before dawn next day Grannie Liu got up and bu... \n", + "\n", + " unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) \n", + "0 Old Geng raised his gun, narrowed one of his t... \n", + "1 As soon as it was light, Grannie Liu got up an... " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.read_csv(\"results/mac-results_final.csv\")\n", + "df.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "e1a0f0a6-912c-46b0-b50e-68eddb2de0ae", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['chinese',\n", + " 'english',\n", + " 'unsloth/Qwen2-0.5B-Instruct',\n", + " 'unsloth/Qwen2-0.5B-Instruct(finetuned)',\n", + " 'unsloth/Qwen2-1.5B-Instruct',\n", + " 'unsloth/Qwen2-1.5B-Instruct(finetuned)',\n", + " 'unsloth/Qwen2-7B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned)',\n", + " 'gradientai/Llama-3-8B-Instruct-Gradient-1048k',\n", + " 'gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned)',\n", + " 'unsloth/Qwen2-7B-Instruct',\n", + " 'unsloth/Qwen2-72B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-7B-Instruct(finetuned)',\n", + " 'unsloth/mistral-7b-instruct-v0.3',\n", + " 'unsloth/mistral-7b-instruct-v0.3(finetuned)',\n", + " 'unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned)']" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.columns.to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "22158e4e-eb2c-443a-81f5-9c9c4849a97a", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['unsloth/Qwen2-7B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned)']" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cols = [col for col in df.columns if \"7B-Instruct-bnb-4bit\" in col]\n", + "cols" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "22a25102-8e6d-4561-b9cf-502598cf0b39", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chineseenglishunsloth/Qwen2-0.5B-Instructunsloth/Qwen2-0.5B-Instruct(finetuned)unsloth/Qwen2-1.5B-Instructunsloth/Qwen2-1.5B-Instruct(finetuned)gradientai/Llama-3-8B-Instruct-Gradient-1048kgradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned)unsloth/Qwen2-7B-Instructunsloth/Qwen2-72B-Instruct-bnb-4bitunsloth/Qwen2-7B-Instruct(finetuned)unsloth/mistral-7b-instruct-v0.3unsloth/mistral-7b-instruct-v0.3(finetuned)unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned)
0老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞...Old Geng picked up his shotgun, squinted, and ...Old Teng holds his gun up, his eyes narrowed a...Old Geng raised his rifle and tilted his head ...Old Jin raises his gun, squints one eye as he ...Old Geng raised his pistol, squinted through t...The old man pulled out his gun, squinting one ...Old Geng raised his rifle, squinting through t...Old Geng raised his gun, squinted one of his t...Lao Geng raised his gun, narrowed one of his t...Old Geng raised his rifle and squinted into th...Geng Da initiates firing, squinting to form a ...Old Geng aimed and fired. A triangular slit op...Old Geng raised his gun, narrowed one of his t...
1次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，...Next day Grannie Liu was up before dawn. As so...The next morning, Liu Geo woke up at five o'cl...But not before noon did Grannie Liu rise up an...At dawn the next day, Liu Langlang got up earl...She got up about dawn with a purpose already e...The next day, when the sun had not yet risen, ...Grannie Liu got up before daylight was even vi...The next morning, before the dawn had fully br...Before dawn next morning, Granny Liu got up to...First thing in the morning Grannie Liu rose to...The next day, when it was still dark, Liu Lao ...Before dawn next day Grannie Liu got up and bu...As soon as it was light, Grannie Liu got up an...
2钱老板道：“是，是，多谢香主。” 在一张椅上坐了，续道：“属下将小郡主藏在猪肚里带进宫来，一...'Thank you, Master,' said Butcher Qian, seatin...Money Master said, 'Yes, yes, thank you for yo...If you like,' said Butcher Qian sitting at a t...Mr. Qian said, \"Yes, yes, thank you very much....Of course,' said Butcher Qian gratefully. 'Tha...The boss said, \"Yes, thank you, Master. I sat ...'No, no, thank you,'said Butcher Qian, sitting...Mr. Qian said, \"Yes, yes, thank you for your c...Mr. Qian said, \"Yes, yes, many thanks, Perfume...Yes, Goong-goong, ' said Butcher Qian, sitting...The boss said, \"Yes, yes, thank you, Madam. Si...Many thanks, Master,' said Butcher Qian, and h...Yes, yes, thank you, Master,' said Butcher Qia...
3但已经晚了，物理学家静静地躺在地上，半睁的双眼看着从他的头颅上流出的血迹，疯狂的会场瞬间陷入...But it was already too late. The physicist lay...But it was too late; physicist lay lifelessly ...But already too late: the physicist lay peacef...But it was too late. Physicists lay quietly on...But it was too late. The physicist was already...But it was too late. The physicist lay still o...But it was too late. The physicist lay quietly...But it was too late. The physicist lay quietly...But it was too late. The physicist lay quietly...But it was too late. The physicist lay on the ...The text is: \"But it's too late, the physicist...But it was already late. The physicist lay sti...But it was too late. The physicist lay on the ...
4但这时，绍琳却做出了一件出人意料的事，与一位受迫害的教育部高干结了婚，当时那名高干还在干校住...But then Shao did something that no one expect...But this time, Rong Ling did something out of ...However, Shen refused to make a surprising ann...But at this time, Shen Lin made a surprising d...But at that moment, Shao Lin did something une...But at that time, Shao Lin did something unexp...However, at that moment, Shao Lin took an unex...But then, in a surprise move, she married a hi...But, in a surprising move, she married a perse...But then, Shao Lin surprised everyone by marry...Shao Lin surprisingly married a high-ranking o...But then Shao Lin did something unexpected: sh...But at this time, Shao Lin did something unexp...
\n", + "
" + ], + "text/plain": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "1 次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，... \n", + "2 钱老板道：“是，是，多谢香主。” 在一张椅上坐了，续道：“属下将小郡主藏在猪肚里带进宫来，一... \n", + "3 但已经晚了，物理学家静静地躺在地上，半睁的双眼看着从他的头颅上流出的血迹，疯狂的会场瞬间陷入... \n", + "4 但这时，绍琳却做出了一件出人意料的事，与一位受迫害的教育部高干结了婚，当时那名高干还在干校住... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "1 Next day Grannie Liu was up before dawn. As so... \n", + "2 'Thank you, Master,' said Butcher Qian, seatin... \n", + "3 But it was already too late. The physicist lay... \n", + "4 But then Shao did something that no one expect... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct \\\n", + "0 Old Teng holds his gun up, his eyes narrowed a... \n", + "1 The next morning, Liu Geo woke up at five o'cl... \n", + "2 Money Master said, 'Yes, yes, thank you for yo... \n", + "3 But it was too late; physicist lay lifelessly ... \n", + "4 But this time, Rong Ling did something out of ... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct(finetuned) \\\n", + "0 Old Geng raised his rifle and tilted his head ... \n", + "1 But not before noon did Grannie Liu rise up an... \n", + "2 If you like,' said Butcher Qian sitting at a t... \n", + "3 But already too late: the physicist lay peacef... \n", + "4 However, Shen refused to make a surprising ann... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct \\\n", + "0 Old Jin raises his gun, squints one eye as he ... \n", + "1 At dawn the next day, Liu Langlang got up earl... \n", + "2 Mr. Qian said, \"Yes, yes, thank you very much.... \n", + "3 But it was too late. Physicists lay quietly on... \n", + "4 But at this time, Shen Lin made a surprising d... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct(finetuned) \\\n", + "0 Old Geng raised his pistol, squinted through t... \n", + "1 She got up about dawn with a purpose already e... \n", + "2 Of course,' said Butcher Qian gratefully. 'Tha... \n", + "3 But it was too late. The physicist was already... \n", + "4 But at that moment, Shao Lin did something une... \n", + "\n", + " gradientai/Llama-3-8B-Instruct-Gradient-1048k \\\n", + "0 The old man pulled out his gun, squinting one ... \n", + "1 The next day, when the sun had not yet risen, ... \n", + "2 The boss said, \"Yes, thank you, Master. I sat ... \n", + "3 But it was too late. The physicist lay still o... \n", + "4 But at that time, Shao Lin did something unexp... \n", + "\n", + " gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) \\\n", + "0 Old Geng raised his rifle, squinting through t... \n", + "1 Grannie Liu got up before daylight was even vi... \n", + "2 'No, no, thank you,'said Butcher Qian, sitting... \n", + "3 But it was too late. The physicist lay quietly... \n", + "4 However, at that moment, Shao Lin took an unex... \n", + "\n", + " unsloth/Qwen2-7B-Instruct \\\n", + "0 Old Geng raised his gun, squinted one of his t... \n", + "1 The next morning, before the dawn had fully br... \n", + "2 Mr. Qian said, \"Yes, yes, thank you for your c... \n", + "3 But it was too late. The physicist lay quietly... \n", + "4 But then, in a surprise move, she married a hi... \n", + "\n", + " unsloth/Qwen2-72B-Instruct-bnb-4bit \\\n", + "0 Lao Geng raised his gun, narrowed one of his t... \n", + "1 Before dawn next morning, Granny Liu got up to... \n", + "2 Mr. Qian said, \"Yes, yes, many thanks, Perfume... \n", + "3 But it was too late. The physicist lay quietly... \n", + "4 But, in a surprising move, she married a perse... \n", + "\n", + " unsloth/Qwen2-7B-Instruct(finetuned) \\\n", + "0 Old Geng raised his rifle and squinted into th... \n", + "1 First thing in the morning Grannie Liu rose to... \n", + "2 Yes, Goong-goong, ' said Butcher Qian, sitting... \n", + "3 But it was too late. The physicist lay on the ... \n", + "4 But then, Shao Lin surprised everyone by marry... \n", + "\n", + " unsloth/mistral-7b-instruct-v0.3 \\\n", + "0 Geng Da initiates firing, squinting to form a ... \n", + "1 The next day, when it was still dark, Liu Lao ... \n", + "2 The boss said, \"Yes, yes, thank you, Madam. Si... \n", + "3 The text is: \"But it's too late, the physicist... \n", + "4 Shao Lin surprisingly married a high-ranking o... \n", + "\n", + " unsloth/mistral-7b-instruct-v0.3(finetuned) \\\n", + "0 Old Geng aimed and fired. A triangular slit op... \n", + "1 Before dawn next day Grannie Liu got up and bu... \n", + "2 Many thanks, Master,' said Butcher Qian, and h... \n", + "3 But it was already late. The physicist lay sti... \n", + "4 But then Shao Lin did something unexpected: sh... \n", + "\n", + " unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) \n", + "0 Old Geng raised his gun, narrowed one of his t... \n", + "1 As soon as it was light, Grannie Liu got up an... \n", + "2 Yes, yes, thank you, Master,' said Butcher Qia... \n", + "3 But it was too late. The physicist lay on the ... \n", + "4 But at this time, Shao Lin did something unexp... " + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = df.drop(columns=cols, axis=1)\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9f230b6e-d666-4eb2-ac63-605acac4109b", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['chinese',\n", + " 'english',\n", + " 'unsloth/Qwen2-0.5B-Instruct',\n", + " 'unsloth/Qwen2-0.5B-Instruct(finetuned)',\n", + " 'unsloth/Qwen2-1.5B-Instruct',\n", + " 'unsloth/Qwen2-1.5B-Instruct(finetuned)',\n", + " 'gradientai/Llama-3-8B-Instruct-Gradient-1048k',\n", + " 'gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned)',\n", + " 'unsloth/Qwen2-7B-Instruct',\n", + " 'unsloth/Qwen2-72B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-7B-Instruct(finetuned)',\n", + " 'unsloth/mistral-7b-instruct-v0.3',\n", + " 'unsloth/mistral-7b-instruct-v0.3(finetuned)',\n", + " 'unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned)']" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.columns.to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "03e517fc-3ef3-4b94-8ec7-7b6a0e4490c2", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "df = df[\n", + " [\n", + " \"chinese\",\n", + " \"english\",\n", + " \"unsloth/Qwen2-0.5B-Instruct\",\n", + " \"unsloth/Qwen2-0.5B-Instruct(finetuned)\",\n", + " \"unsloth/Qwen2-1.5B-Instruct\",\n", + " \"unsloth/Qwen2-1.5B-Instruct(finetuned)\",\n", + " \"unsloth/Qwen2-7B-Instruct\",\n", + " \"unsloth/Qwen2-7B-Instruct(finetuned)\",\n", + " \"unsloth/mistral-7b-instruct-v0.3\",\n", + " \"unsloth/mistral-7b-instruct-v0.3(finetuned)\",\n", + " \"gradientai/Llama-3-8B-Instruct-Gradient-1048k\",\n", + " \"gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned)\",\n", + " \"unsloth/Qwen2-72B-Instruct-bnb-4bit\",\n", + " \"unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned)\",\n", + " ]\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "65559d9f-98cc-4f2b-80ab-78f58a269f39", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "df.to_csv(\"results/experiment-2-results.csv\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "d24728f3-13f6-48c7-b911-d8e4d5f4803e", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "df1 = pd.read_csv(\"results/mac-results-no-flash-attn.csv\")\n", + "df2 = pd.read_csv(\"results/mac-results-with-flash-attn.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "ab6f1aea-c86b-4d47-a673-454747c35af7", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['chinese',\n", + " 'english',\n", + " 'unsloth/Qwen2-0.5B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned)',\n", + " 'unsloth/Qwen2-1.5B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned)',\n", + " 'unsloth/Qwen2-0.5B-Instruct',\n", + " 'unsloth/Qwen2-0.5B-Instruct(finetuned)',\n", + " 'unsloth/Qwen2-1.5B-Instruct',\n", + " 'unsloth/Qwen2-1.5B-Instruct(finetuned)']" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df2.columns.to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "792f7863-c8e7-427e-8ea7-514eb8edcdb3", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'unsloth/Qwen2-0.5B-Instruct': 'Qwen2-0.5B(flash-attn:true)',\n", + " 'unsloth/Qwen2-0.5B-Instruct(finetuned)': 'Qwen2-0.5B(finetuned)(flash-attn:true)',\n", + " 'unsloth/Qwen2-1.5B-Instruct': 'Qwen2-1.5B(flash-attn:true)',\n", + " 'unsloth/Qwen2-1.5B-Instruct(finetuned)': 'Qwen2-1.5B(finetuned)(flash-attn:true)'}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "columns = df2.columns.to_list()\n", + "new_columns = {\n", + " col: col.replace(\"unsloth/\", \"\").replace(\"-Instruct\", \"\") + \"(flash-attn:true)\"\n", + " for col in columns[2:] if not \"4bit\" in col\n", + "}\n", + "new_columns" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "ffd47e31-c936-4f20-bafe-ccd852db5f32", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "df = df2[[\"chinese\", \"english\"] + list(new_columns.keys())]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3b2cb515-fe62-4247-9192-89cf3da47e04", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "df = df.rename(columns=new_columns)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "64f65440-3a8c-432b-9337-1bb05d5acd09", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chineseenglishQwen2-0.5B(flash-attn:true)Qwen2-0.5B(finetuned)(flash-attn:true)Qwen2-1.5B(flash-attn:true)Qwen2-1.5B(finetuned)(flash-attn:true)
0老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞...Old Geng picked up his shotgun, squinted, and ...Old耿举起枪，眯着眼睛，枪声轰鸣，子弹砰砰砰地落在地上，一颗颗冰雹般的大鸟扑棱棱地落在柳树...Old Geng raised his rifle and tilted his head,...Old Geer lifted his gun, squinted one of his e...Old Geng raised his gun, squinted, and emptied...
1次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，...Next day Grannie Liu was up before dawn. As so...The next day morning when the sun was still ri...First thing that morning the old lady did rise...The next morning, Liu Langlang got up early an...In the predawn light she arose, dressed, and b...
\n", + "
" + ], + "text/plain": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "1 次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "1 Next day Grannie Liu was up before dawn. As so... \n", + "\n", + " Qwen2-0.5B(flash-attn:true) \\\n", + "0 Old耿举起枪，眯着眼睛，枪声轰鸣，子弹砰砰砰地落在地上，一颗颗冰雹般的大鸟扑棱棱地落在柳树... \n", + "1 The next day morning when the sun was still ri... \n", + "\n", + " Qwen2-0.5B(finetuned)(flash-attn:true) \\\n", + "0 Old Geng raised his rifle and tilted his head,... \n", + "1 First thing that morning the old lady did rise... \n", + "\n", + " Qwen2-1.5B(flash-attn:true) \\\n", + "0 Old Geer lifted his gun, squinted one of his e... \n", + "1 The next morning, Liu Langlang got up early an... \n", + "\n", + " Qwen2-1.5B(finetuned)(flash-attn:true) \n", + "0 Old Geng raised his gun, squinted, and emptied... \n", + "1 In the predawn light she arose, dressed, and b... " + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "278acb35-c83f-4493-a873-f3e0398d4d42", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'unsloth/Qwen2-0.5B-Instruct': 'Qwen2-0.5B(flash-attn:false)',\n", + " 'unsloth/Qwen2-0.5B-Instruct(finetuned)': 'Qwen2-0.5B(finetuned)(flash-attn:false)',\n", + " 'unsloth/Qwen2-1.5B-Instruct': 'Qwen2-1.5B(flash-attn:false)',\n", + " 'unsloth/Qwen2-1.5B-Instruct(finetuned)': 'Qwen2-1.5B(finetuned)(flash-attn:false)'}" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "new_columns_2 = {key: new_columns[key].replace(\"true\", \"false\") for key in new_columns.keys()}\n", + "new_columns_2" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f6fb727e-922d-4e62-8b05-65b63d502234", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "for key, value in new_columns_2.items():\n", + " df[value] = df1[key]" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7987db8b-ed83-4104-8130-d1fd11206f29", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['chinese',\n", + " 'english',\n", + " 'Qwen2-0.5B(flash-attn:true)',\n", + " 'Qwen2-0.5B(finetuned)(flash-attn:true)',\n", + " 'Qwen2-1.5B(flash-attn:true)',\n", + " 'Qwen2-1.5B(finetuned)(flash-attn:true)',\n", + " 'Qwen2-0.5B(flash-attn:false)',\n", + " 'Qwen2-0.5B(finetuned)(flash-attn:false)',\n", + " 'Qwen2-1.5B(flash-attn:false)',\n", + " 'Qwen2-1.5B(finetuned)(flash-attn:false)']" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "columns = df.columns.to_list()\n", + "columns" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "72aa7754-7f3b-4bd0-b2f5-47affb56ed86", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['Qwen2-0.5B(flash-attn:false)',\n", + " 'Qwen2-0.5B(flash-attn:true)',\n", + " 'Qwen2-0.5B(finetuned)(flash-attn:false)',\n", + " 'Qwen2-0.5B(finetuned)(flash-attn:true)',\n", + " 'Qwen2-1.5B(flash-attn:false)',\n", + " 'Qwen2-1.5B(flash-attn:true)',\n", + " 'Qwen2-1.5B(finetuned)(flash-attn:false)',\n", + " 'Qwen2-1.5B(finetuned)(flash-attn:true)']" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sorted_columns = sorted(columns[2:], key=lambda x: x.replace(\"flash\", \"a\"))\n", + "sorted_columns" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "11143adc-621b-41e5-8d24-02d9e75f0f3d", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chineseenglishQwen2-0.5B(flash-attn:false)Qwen2-0.5B(flash-attn:true)Qwen2-0.5B(finetuned)(flash-attn:false)Qwen2-0.5B(finetuned)(flash-attn:true)Qwen2-1.5B(flash-attn:false)Qwen2-1.5B(flash-attn:true)Qwen2-1.5B(finetuned)(flash-attn:false)Qwen2-1.5B(finetuned)(flash-attn:true)
0老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞...Old Geng picked up his shotgun, squinted, and ...Old Teng raises his gun, closing his eyes with...Old耿举起枪，眯着眼睛，枪声轰鸣，子弹砰砰砰地落在地上，一颗颗冰雹般的大鸟扑棱棱地落在柳树...Old Geng raised his rifle and made a twist eye...Old Geng raised his rifle and tilted his head,...Old耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝...Old Geer lifted his gun, squinted one of his e...Old Geng raised his pistol, squinted through t...Old Geng raised his gun, squinted, and emptied...
1次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，...Next day Grannie Liu was up before dawn. As so...The next day, after dawn, Liu Geong woke up at...The next day morning when the sun was still ri...First thing that made him up and go out of his...First thing that morning the old lady did rise...At dawn on the second day, Liu Laolao got up a...The next morning, Liu Langlang got up early an...But she didn't begin to wake her again until t...In the predawn light she arose, dressed, and b...
\n", + "
" + ], + "text/plain": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "1 次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "1 Next day Grannie Liu was up before dawn. As so... \n", + "\n", + " Qwen2-0.5B(flash-attn:false) \\\n", + "0 Old Teng raises his gun, closing his eyes with... \n", + "1 The next day, after dawn, Liu Geong woke up at... \n", + "\n", + " Qwen2-0.5B(flash-attn:true) \\\n", + "0 Old耿举起枪，眯着眼睛，枪声轰鸣，子弹砰砰砰地落在地上，一颗颗冰雹般的大鸟扑棱棱地落在柳树... \n", + "1 The next day morning when the sun was still ri... \n", + "\n", + " Qwen2-0.5B(finetuned)(flash-attn:false) \\\n", + "0 Old Geng raised his rifle and made a twist eye... \n", + "1 First thing that made him up and go out of his... \n", + "\n", + " Qwen2-0.5B(finetuned)(flash-attn:true) \\\n", + "0 Old Geng raised his rifle and tilted his head,... \n", + "1 First thing that morning the old lady did rise... \n", + "\n", + " Qwen2-1.5B(flash-attn:false) \\\n", + "0 Old耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝... \n", + "1 At dawn on the second day, Liu Laolao got up a... \n", + "\n", + " Qwen2-1.5B(flash-attn:true) \\\n", + "0 Old Geer lifted his gun, squinted one of his e... \n", + "1 The next morning, Liu Langlang got up early an... \n", + "\n", + " Qwen2-1.5B(finetuned)(flash-attn:false) \\\n", + "0 Old Geng raised his pistol, squinted through t... \n", + "1 But she didn't begin to wake her again until t... \n", + "\n", + " Qwen2-1.5B(finetuned)(flash-attn:true) \n", + "0 Old Geng raised his gun, squinted, and emptied... \n", + "1 In the predawn light she arose, dressed, and b... " + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = df[[\"chinese\", \"english\"] + sorted_columns]\n", + "df.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "b643a059-56df-4ee6-a224-84e98d793ae8", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "df.to_csv(\"results/experiment-3-results.csv\", index=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f2a0da95-a3e8-476d-8063-dc38e8b001c0", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Experiment 1" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "a8f0a771-bba2-41a8-a9bf-09fa198e80e7", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chineseenglishunsloth/Qwen2-0.5B-Instructunsloth/Qwen2-0.5B-Instruct-bnb-4bitunsloth/Qwen2-0.5B-Instruct(finetuned)unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned)unsloth/Qwen2-1.5B-Instructunsloth/Qwen2-1.5B-Instruct-bnb-4bitunsloth/Qwen2-1.5B-Instruct(finetuned)unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned)unsloth/Qwen2-7B-Instructunsloth/Qwen2-7B-Instruct-bnb-4bitunsloth/Qwen2-7B-Instruct(finetuned)unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned)
0老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞...Old Geng picked up his shotgun, squinted, and ...Old Tang held his gun, squinting his eyes with...Old Teng raised his gun and looked up at a pai...Old Geng lifted his rifle and narrowed his eye...Old Geng raised his rifle, squinted his eyes, ...Old Geng took up his gun, squinted one of its ...Old耿拿起枪，眯着眼睛一搂扳机就响了枪，金麻雀噼里啪啦的往下掉，铁砂子在柳枝间飞溅，发出“...Old Geng raised the rifle, squeezed one tiny t...Old Geng raised his pistol, squinted, and fire...Old Aigang raised his rifle, squinting one of ...Old Geng raised his gun, squinting one of his ...Old Geng raised his rifle and squinted into th...Old Geng raised his rifle and squinted into th...
1次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，...Next day Grannie Liu was up before dawn. As so...The next morning when it was still dark, Liu G...The next day at dawn, Liu Geowon got up early ...It was still not light when this little update...Three or four hours before this, Grannie Liu h...By the time the next day dawned, Liu Lao got u...At dawn the next day, Liu Langlang got up to b...Having been woken just before daybreak, Granni...But by some miracle of preparation, Grannie Li...The next morning, before dawn, Old Liu rose to...The next morning, before dawn, Old Liu rose to...First thing next morning Grannie Liu rose befo...First thing in the morning Grannie Liu rose an...
\n", + "
" + ], + "text/plain": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "1 次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "1 Next day Grannie Liu was up before dawn. As so... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct \\\n", + "0 Old Tang held his gun, squinting his eyes with... \n", + "1 The next morning when it was still dark, Liu G... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct-bnb-4bit \\\n", + "0 Old Teng raised his gun and looked up at a pai... \n", + "1 The next day at dawn, Liu Geowon got up early ... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct(finetuned) \\\n", + "0 Old Geng lifted his rifle and narrowed his eye... \n", + "1 It was still not light when this little update... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned) \\\n", + "0 Old Geng raised his rifle, squinted his eyes, ... \n", + "1 Three or four hours before this, Grannie Liu h... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct \\\n", + "0 Old Geng took up his gun, squinted one of its ... \n", + "1 By the time the next day dawned, Liu Lao got u... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct-bnb-4bit \\\n", + "0 Old耿拿起枪，眯着眼睛一搂扳机就响了枪，金麻雀噼里啪啦的往下掉，铁砂子在柳枝间飞溅，发出“... \n", + "1 At dawn the next day, Liu Langlang got up to b... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct(finetuned) \\\n", + "0 Old Geng raised the rifle, squeezed one tiny t... \n", + "1 Having been woken just before daybreak, Granni... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned) \\\n", + "0 Old Geng raised his pistol, squinted, and fire... \n", + "1 But by some miracle of preparation, Grannie Li... \n", + "\n", + " unsloth/Qwen2-7B-Instruct \\\n", + "0 Old Aigang raised his rifle, squinting one of ... \n", + "1 The next morning, before dawn, Old Liu rose to... \n", + "\n", + " unsloth/Qwen2-7B-Instruct-bnb-4bit \\\n", + "0 Old Geng raised his gun, squinting one of his ... \n", + "1 The next morning, before dawn, Old Liu rose to... \n", + "\n", + " unsloth/Qwen2-7B-Instruct(finetuned) \\\n", + "0 Old Geng raised his rifle and squinted into th... \n", + "1 First thing next morning Grannie Liu rose befo... \n", + "\n", + " unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned) \n", + "0 Old Geng raised his rifle and squinted into th... \n", + "1 First thing in the morning Grannie Liu rose an... " + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "df = pd.read_csv(\"results/experiment-1-results.csv\")\n", + "df.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "ffea8930-7f7b-4713-abd7-05b616cb0451", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['unsloth/Qwen2-0.5B-Instruct',\n", + " 'unsloth/Qwen2-0.5B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-0.5B-Instruct(finetuned)',\n", + " 'unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned)',\n", + " 'unsloth/Qwen2-1.5B-Instruct',\n", + " 'unsloth/Qwen2-1.5B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-1.5B-Instruct(finetuned)',\n", + " 'unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned)',\n", + " 'unsloth/Qwen2-7B-Instruct',\n", + " 'unsloth/Qwen2-7B-Instruct-bnb-4bit',\n", + " 'unsloth/Qwen2-7B-Instruct(finetuned)',\n", + " 'unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned)']" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "columns = df.columns.to_list()\n", + "columns = columns[2:]\n", + "columns" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "5e73669f-c244-425c-9499-3d4ae3096a15", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "unsloth/Qwen2-0.5B-Instruct: {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.26682092609395136, 'bleu_scores': {'bleu': 0.050004191193532964, 'precisions': [0.32555012625848556, 0.07871253405994551, 0.025538396146217057, 0.009553670232386574], 'brevity_penalty': 1.0, 'length_ratio': 1.010036435905929, 'translation_length': 30493, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.320537152573054, 'rouge2': 0.09507415729829477, 'rougeL': 0.26440324942487026, 'rougeLsum': 0.26411350380714455}}\n", + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit: {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.2576132895489498, 'bleu_scores': {'bleu': 0.03850233688649031, 'precisions': [0.2645428602787839, 0.060745943190507204, 0.019074366625709438, 0.007169434612941283], 'brevity_penalty': 1.0, 'length_ratio': 1.2071546869824445, 'translation_length': 36444, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.30733722807033026, 'rouge2': 0.0839680099329593, 'rougeL': 0.251300692614542, 'rougeLsum': 0.2521096163974129}}\n", + "unsloth/Qwen2-0.5B-Instruct(finetuned): {'accuracy': 0.00088261253309797, 'correct_ids': [147], 'meteor': 0.29032409482315213, 'bleu_scores': {'bleu': 0.06508609399238363, 'precisions': [0.3407579117113485, 0.09377291935878182, 0.03598822203642444, 0.01652015762352228], 'brevity_penalty': 0.9858565320713017, 'length_ratio': 0.9859556144418682, 'translation_length': 29766, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.32302970292391686, 'rouge2': 0.11205363155520029, 'rougeL': 0.2670231142333206, 'rougeLsum': 0.2670773075347084}}\n", + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned): {'accuracy': 0.00264783759929391, 'correct_ids': [147, 199, 533], 'meteor': 0.2882153528907209, 'bleu_scores': {'bleu': 0.06212332936010444, 'precisions': [0.33329989299090423, 0.08908275694275486, 0.03363958619691818, 0.01549364798130207], 'brevity_penalty': 0.9904816510671316, 'length_ratio': 0.99052666445843, 'translation_length': 29904, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.31897252603731596, 'rouge2': 0.10751579569274039, 'rougeL': 0.2627116615131784, 'rougeLsum': 0.2626045388376806}}\n", + "unsloth/Qwen2-1.5B-Instruct: {'accuracy': 0.00176522506619594, 'correct_ids': [658, 659], 'meteor': 0.33552118264217856, 'bleu_scores': {'bleu': 0.08285381577653864, 'precisions': [0.40636974021865224, 0.12583290620194773, 0.051405438435685916, 0.02290685609386224], 'brevity_penalty': 0.9405675222192741, 'length_ratio': 0.9422656508777741, 'translation_length': 28447, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3880786820188693, 'rouge2': 0.141529446199457, 'rougeL': 0.3282669684173835, 'rougeLsum': 0.3280684791570985}}\n", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit: {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.31208334401862997, 'bleu_scores': {'bleu': 0.07153184431712963, 'precisions': [0.3459574058012913, 0.10217347823188212, 0.04077178883192462, 0.018166744764445086], 'brevity_penalty': 1.0, 'length_ratio': 1.0311692613448162, 'translation_length': 31131, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3685043743114341, 'rouge2': 0.12938566838112653, 'rougeL': 0.31268323506765083, 'rougeLsum': 0.3126763191874815}}\n", + "unsloth/Qwen2-1.5B-Instruct(finetuned): {'accuracy': 0.00264783759929391, 'correct_ids': [147, 170, 194], 'meteor': 0.35503843183028994, 'bleu_scores': {'bleu': 0.09734851870184895, 'precisions': [0.38486636126948554, 0.12903115371448134, 0.05879839025606325, 0.030757244091566802], 'brevity_penalty': 1.0, 'length_ratio': 1.0050679032792316, 'translation_length': 30343, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.380650624350626, 'rouge2': 0.15412228408294604, 'rougeL': 0.32336591903518175, 'rougeLsum': 0.3231128403604364}}\n", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned): {'accuracy': 0.00264783759929391, 'correct_ids': [147, 170, 533], 'meteor': 0.345364472322921, 'bleu_scores': {'bleu': 0.09480544437628514, 'precisions': [0.383379631179089, 0.12652195088012244, 0.05757323387768404, 0.03015777157092172], 'brevity_penalty': 0.9896453343069915, 'length_ratio': 0.9896985756873137, 'translation_length': 29879, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3745725185836431, 'rouge2': 0.14891614545218895, 'rougeL': 0.31780823584988394, 'rougeLsum': 0.3176397827357503}}\n", + "unsloth/Qwen2-7B-Instruct: {'accuracy': 0.00088261253309797, 'correct_ids': [240], 'meteor': 0.3700620963466071, 'bleu_scores': {'bleu': 0.10682777284063055, 'precisions': [0.41151630221291086, 0.14519802821689615, 0.06604907715154515, 0.033000626127951085], 'brevity_penalty': 1.0, 'length_ratio': 1.011858231202385, 'translation_length': 30548, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4169112258053055, 'rouge2': 0.16853252030860638, 'rougeL': 0.3593401448919635, 'rougeLsum': 0.359285473942102}}\n", + "unsloth/Qwen2-7B-Instruct-bnb-4bit: {'accuracy': 0.00264783759929391, 'correct_ids': [240, 364, 659], 'meteor': 0.3722333296692946, 'bleu_scores': {'bleu': 0.10974864974763812, 'precisions': [0.415292107511336, 0.14678679908535544, 0.0680204487361545, 0.03498779495524817], 'brevity_penalty': 1.0, 'length_ratio': 1.0080821464060947, 'translation_length': 30434, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.416464443302337, 'rouge2': 0.16839501653083946, 'rougeL': 0.35793445682567243, 'rougeLsum': 0.35784562978464673}}\n", + "unsloth/Qwen2-7B-Instruct(finetuned): {'accuracy': 0.00617828773168579, 'correct_ids': [147, 199, 202, 309, 350, 531, 658], 'meteor': 0.4112873282163906, 'bleu_scores': {'bleu': 0.13823813520357786, 'precisions': [0.4426776533079097, 0.18058506870661567, 0.09432987789131958, 0.05415704387990762], 'brevity_penalty': 0.9724324694495059, 'length_ratio': 0.9728055647565419, 'translation_length': 29369, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4318960223962858, 'rouge2': 0.1990647305648936, 'rougeL': 0.3797262701650865, 'rougeLsum': 0.3796463709477957}}\n", + "unsloth/Qwen2-7B-Instruct-bnb-4bit(finetuned): {'accuracy': 0.0088261253309797, 'correct_ids': [12, 147, 199, 309, 350, 419, 531, 534, 658, 659], 'meteor': 0.40972424327766704, 'bleu_scores': {'bleu': 0.13629523030680843, 'precisions': [0.43929173693086004, 0.1763509485570011, 0.09143024062511411, 0.05240108153395027], 'brevity_penalty': 0.9819523660115357, 'length_ratio': 0.9821132825438887, 'translation_length': 29650, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4392490392686588, 'rouge2': 0.20198397087010342, 'rougeL': 0.38046002968318615, 'rougeLsum': 0.38080840575676933}}\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
accuracymeteorbleu_1rouge_l
count12.00000012.00000012.00000012.000000
mean0.0024270.3345240.0877800.317085
std0.0026650.0521120.0319130.046850
min0.0000000.2576130.0385020.251301
25%0.0006620.2897970.0643450.266368
50%0.0022070.3404430.0888300.320587
75%0.0026480.3706050.1075580.358286
max0.0088260.4112870.1382380.380460
\n", + "
" + ], + "text/plain": [ + " accuracy meteor bleu_1 rouge_l\n", + "count 12.000000 12.000000 12.000000 12.000000\n", + "mean 0.002427 0.334524 0.087780 0.317085\n", + "std 0.002665 0.052112 0.031913 0.046850\n", + "min 0.000000 0.257613 0.038502 0.251301\n", + "25% 0.000662 0.289797 0.064345 0.266368\n", + "50% 0.002207 0.340443 0.088830 0.320587\n", + "75% 0.002648 0.370605 0.107558 0.358286\n", + "max 0.008826 0.411287 0.138238 0.380460" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "metrics_df = get_metrics(df)\n", + "metrics_df.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "e9b2f2b4-39bd-448a-80ee-1746dc582cb9", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
modelmeteorbleu_1rouge_l
0Qwen2-0.5B-Instruct0.2668210.0500040.264403
1Qwen2-0.5B-Instruct-bnb-4bit0.2576130.0385020.251301
2Qwen2-0.5B-Instruct(finetuned)0.2903240.0650860.267023
3Qwen2-0.5B-Instruct-bnb-4bit(finetuned)0.2882150.0621230.262712
4Qwen2-1.5B-Instruct0.3355210.0828540.328267
5Qwen2-1.5B-Instruct-bnb-4bit0.3120830.0715320.312683
6Qwen2-1.5B-Instruct(finetuned)0.3550380.0973490.323366
7Qwen2-1.5B-Instruct-bnb-4bit(finetuned)0.3453640.0948050.317808
8Qwen2-7B-Instruct0.3700620.1068280.359340
9Qwen2-7B-Instruct-bnb-4bit0.3722330.1097490.357934
10Qwen2-7B-Instruct(finetuned)0.4112870.1382380.379726
11Qwen2-7B-Instruct-bnb-4bit(finetuned)0.4097240.1362950.380460
\n", + "
" + ], + "text/plain": [ + " model meteor bleu_1 rouge_l\n", + "0 Qwen2-0.5B-Instruct 0.266821 0.050004 0.264403\n", + "1 Qwen2-0.5B-Instruct-bnb-4bit 0.257613 0.038502 0.251301\n", + "2 Qwen2-0.5B-Instruct(finetuned) 0.290324 0.065086 0.267023\n", + "3 Qwen2-0.5B-Instruct-bnb-4bit(finetuned) 0.288215 0.062123 0.262712\n", + "4 Qwen2-1.5B-Instruct 0.335521 0.082854 0.328267\n", + "5 Qwen2-1.5B-Instruct-bnb-4bit 0.312083 0.071532 0.312683\n", + "6 Qwen2-1.5B-Instruct(finetuned) 0.355038 0.097349 0.323366\n", + "7 Qwen2-1.5B-Instruct-bnb-4bit(finetuned) 0.345364 0.094805 0.317808\n", + "8 Qwen2-7B-Instruct 0.370062 0.106828 0.359340\n", + "9 Qwen2-7B-Instruct-bnb-4bit 0.372233 0.109749 0.357934\n", + "10 Qwen2-7B-Instruct(finetuned) 0.411287 0.138238 0.379726\n", + "11 Qwen2-7B-Instruct-bnb-4bit(finetuned) 0.409724 0.136295 0.380460" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "metrics_df.drop(columns=[\"all_metrics\", \"accuracy\"], inplace=True, errors=\"ignore\")\n", + "metrics_df" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "d83b0ebd-4626-42ca-a278-47c9f2ac998f", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "perf_df = metrics_df.copy()\n", + "perf_df.drop(columns=[\"bleu_1\", \"rouge_l\"], inplace=True)\n", + "\n", + "perf_df[\"train-time(mins)\"] = [\n", + " 62.99,\n", + " 85.05,\n", + " 0,\n", + " 0,\n", + " 92.74,\n", + " 139.92,\n", + " 0,\n", + " 0,\n", + " 97.77,\n", + " 103.4,\n", + " 0,\n", + " 0,\n", + "]\n", + "perf_df[\"eval-time(mins)\"] = [\n", + " 22.53,\n", + " 41.88,\n", + " 26.47,\n", + " 36.87,\n", + " 30.02,\n", + " 59.6,\n", + " 34.15,\n", + " 50.73,\n", + " 37.58,\n", + " 39.87,\n", + " 37.05,\n", + " 36.82,\n", + "]\n", + "perf_df[\"GPU\"] = [\"RTX 4080\"] * 8 + [\"L40\"] * 4" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f47724aa-1f6a-41f3-9741-b6eac27081f2", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
modelmeteortrain-time(mins)eval-time(mins)GPU
0Qwen2-0.5B-Instruct0.26682162.9922.53RTX 4080
1Qwen2-0.5B-Instruct-bnb-4bit0.25761385.0541.88RTX 4080
2Qwen2-0.5B-Instruct(finetuned)0.2903240.0026.47RTX 4080
3Qwen2-0.5B-Instruct-bnb-4bit(finetuned)0.2882150.0036.87RTX 4080
4Qwen2-1.5B-Instruct0.33552192.7430.02RTX 4080
5Qwen2-1.5B-Instruct-bnb-4bit0.312083139.9259.60RTX 4080
6Qwen2-1.5B-Instruct(finetuned)0.3550380.0034.15RTX 4080
7Qwen2-1.5B-Instruct-bnb-4bit(finetuned)0.3453640.0050.73RTX 4080
8Qwen2-7B-Instruct0.37006297.7737.58L40
9Qwen2-7B-Instruct-bnb-4bit0.372233103.4039.87L40
10Qwen2-7B-Instruct(finetuned)0.4112870.0037.05L40
11Qwen2-7B-Instruct-bnb-4bit(finetuned)0.4097240.0036.82L40
\n", + "
" + ], + "text/plain": [ + " model meteor train-time(mins) \\\n", + "0 Qwen2-0.5B-Instruct 0.266821 62.99 \n", + "1 Qwen2-0.5B-Instruct-bnb-4bit 0.257613 85.05 \n", + "2 Qwen2-0.5B-Instruct(finetuned) 0.290324 0.00 \n", + "3 Qwen2-0.5B-Instruct-bnb-4bit(finetuned) 0.288215 0.00 \n", + "4 Qwen2-1.5B-Instruct 0.335521 92.74 \n", + "5 Qwen2-1.5B-Instruct-bnb-4bit 0.312083 139.92 \n", + "6 Qwen2-1.5B-Instruct(finetuned) 0.355038 0.00 \n", + "7 Qwen2-1.5B-Instruct-bnb-4bit(finetuned) 0.345364 0.00 \n", + "8 Qwen2-7B-Instruct 0.370062 97.77 \n", + "9 Qwen2-7B-Instruct-bnb-4bit 0.372233 103.40 \n", + "10 Qwen2-7B-Instruct(finetuned) 0.411287 0.00 \n", + "11 Qwen2-7B-Instruct-bnb-4bit(finetuned) 0.409724 0.00 \n", + "\n", + " eval-time(mins) GPU \n", + "0 22.53 RTX 4080 \n", + "1 41.88 RTX 4080 \n", + "2 26.47 RTX 4080 \n", + "3 36.87 RTX 4080 \n", + "4 30.02 RTX 4080 \n", + "5 59.60 RTX 4080 \n", + "6 34.15 RTX 4080 \n", + "7 50.73 RTX 4080 \n", + "8 37.58 L40 \n", + "9 39.87 L40 \n", + "10 37.05 L40 \n", + "11 36.82 L40 " + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "perf_df" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "8ad373fe-73bf-4ca9-842b-3dcc94bfb3b0", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAPeCAYAAADd/6nHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3gU5frG8e9ueu+VBBISWuhdqoAg4BEL9kITRVHxiEdF/KmADexdbChyPNh7A5EqRXqRTkKAAAlJCOl1y++PhUAgQAhJNuX+XNdeZmbeefcZopi988w7BqvVakVERERERERERKQGGe1dgIiIiIiIiIiINDwKpUREREREREREpMYplBIRERERERERkRqnUEpERERERERERGqcQikREREREREREalxCqVERERERERERKTGKZQSEREREREREZEap1BKRERERERERERqnEIpERERERERERGpcY72LqA2MJlMbNy4kZCQEIxG5XQiIiIiIiIiUnUsFgtHjhyhY8eOODoqijlBfxLAxo0b6datm73LEBEREREREZF6bM2aNXTt2rXC4+es2sf7S/eSlltEqzBvpl3Vmg6Rvuc976fNh3ng840Migvhw5FdACgxW3j5j10s2ZnGgYx8vFwd6R0byKShLQnxdq3sJV0UhVJASEgIYPuXIywszM7ViIiIiIiIiEh9kpycTLdu3Urzh4r4efNhnv1lB89e24aOkb58vCKRkbNWs+jhfgR6upz1vKSMfJ7/dQfdovzL7C8oMbPtUDYTLoulVZg3WQUlTPt5O3d+uo6fJ/Su9LVdDIVSUHrLXlhYGBEREXauRkRERERERETqowtZMuij5Ync3C2SG7tEAvDcNW1ZtDOVr9YlcW+/2HLPMVusPPjlJiYOasaaxGNkF5aUHvN2deKzO7uXGf/0Va25+p0VHMosoJGvWyWu6OJoASURERERERERkRqQk5NDdnZ26auoqKjcccUmC1sPZdErNrB0n9FooFdsIBv2Z551/jcW7iHAw5mbujauWD2FJgwG8Ha1T8+SQikRERERERERkRoQFxeHj49P6Wv69OnljjuWX4zZYj3jNr0gTxfScssPstbuy+CrtUnMuK5dhWopLDEzY94Ormofjper04VdSBXR7XsiIiIiIiIiIjVg+/btNGrUqHTbxeXsa0NdiNwiExO/3MT069ri7+F83vElZgv3z92A1QrPXtOmSmqoDIVSFWSxWCguLrZ3GVJFnJ2dL+heXhERERERkdrMbDZTUlJy/oFSLZycnHBwcDjvOC8vL7y9vc87zs/dGQejgfTTuqLScosIKmeR8/1H8zh4rIA7P11Xus9itQIQ8/hvLPrPpTQJ8ABsgdR9/9vAwWMFfH7XJXbrkgKFUhVSXFxMYmIiFovF3qVIFTEajURHR+PsfP4EWUREREREpLayWq2kpKSQmZlp71IaPF9fX0JDQzEYDBc9l7OjkTaNfFgZn87g1qEAWCxWVsYfZWTPJmeMjwnyZP6Dfcvse/mPXeQVmZgyrDVhPrZFzE8EUvuO5vH5XZfgV4GuquqkUOo8rFYrycnJODg4EBkZqe6aesBisXD48GGSk5Np3LhxlfyFISIiIiIiYg8nAqng4GDc3d31+cYOrFYr+fn5pKamAhAWFlYl897ZO5r/fL2ZthG+dIj0YdbyfeQXm7ihs+1pfA99uYkQH1cmDWmJq5MDLUK9ypzvfbwD6sT+ErOF8Z9tYNvhLGaN6orZaiU1pxAAXzdnnB1rPu9QKHUeJpOJ/Px8wsPDcXd3t3c5UkWCgoI4fPgwJpMJJyf7tSqKiIiIiIhUltlsLg2kAgIC7F1Og+bmZutESk1NJTg4uEK38p3PsPbhZOQV89qC3aTlFNEq3JtP7+hGkJft9r1DmQUXFEKmZBXy544jAFzx5l9ljn1+1yX0iKn5f4cUSp2H2WwG0G1e9cyJ76fZbFYoJSIiIiIiddKJNaTUQFE7nPg+lJSUVEkoBTCqZxSjekaVe+zLu3uc89xXbmxfZjvS3519M/5VJXVVFd2LVkFqgaxf9P0UEREREZH6Qp9vagd9Hy6cQikREREREREREalxCqXkvKKionj99dfr3NwiIiIiIiIiUntpTalK2tGyVY2+X6udOy5ofL9+/ejQoUOVBD5r167Fw8PjouaYPXs2Dz744BmPKa2KuUVERERERKTyzBYraxIzSM0pJNjLlW7R/jgY69ataFX5GVhqjkKpBspqtWI2m3F0PP+/AkFBQdVWR3XOLSIiIiIiIuc2b2sy037eTnJWYem+MB9XpgyLY0ibMDtWVjtdyGdpOT/dvlcPjR49mqVLl/LGG29gMBgwGAzMnj0bg8HA77//TufOnXFxcWH58uUkJCRw9dVXExISgqenJ127duXPP/8sM9/pt9gZDAY++ugjrr32Wtzd3WnWrBk//fTTWetZsmQJY8aMISsrq7SeqVOnnnXu999/nyuvvBJ3d3datWrFqlWriI+Pp1+/fnh4eNCzZ08SEhLKvMePP/5Ip06dcHV1pWnTpkybNg2TyXTRf5YiIiIiIiL11bytyYz/bEOZQAogJauQ8Z9tYN7W5Gp53379+jFhwgQefPBB/Pz8CAkJ4cMPPyQvL48xY8bg5eVFbGwsv//+e+k5W7duZejQoXh6ehISEsKIESNIT08Hyv8MvG/fvvOeB1BUVMQDDzxAcHAwrq6u9O7dm7Vr15YeX7JkSbmfpaVqKJSqh9544w169OjBXXfdRXJyMsnJyURGRgLw2GOPMWPGDHbs2EG7du3Izc3liiuuYOHChWzcuJEhQ4YwbNgwDhw4cM73mDZtGjfeeCNbtmzhiiuu4LbbbiMjI6PcsT179uT111/H29u7tJ6HH374rHM/88wzjBw5kk2bNtGyZUtuvfVW7r77biZPnsy6deuwWq3cf//9peP/+usvRo4cyb///W+2b9/O+++/z+zZs3nuuecq8acnIiIiIiJSN1mtVvKLTRV65RSWMOWnbVjLm+f4P6f+tJ2cwpIKzWe1ljfT2X366acEBgayZs0aJkyYwPjx47nhhhvo2bMnGzZs4PLLL2fEiBHk5+eTmZnJgAED6NixI+vWrWPevHkcOXKEG2+8ETj7Z+DznQfw6KOP8u233/Lpp5+yYcMGYmNjGTx48Bmfb0//LC1VQ/1m9ZCPjw/Ozs64u7sTGhoKwM6dOwF4+umnGTRoUOlYf39/2rdvX7r9zDPP8P333/PTTz+VCX5ON3r0aG655RYAnn/+ed58803WrFnDkCFDzhjr7OyMj48PBoOhtJ5zGTNmTOlfEpMmTaJHjx48+eSTDB48GIB///vfjBkzpnT8tGnTeOyxxxg1ahQATZs25ZlnnuHRRx9lypQp530/ERERERGR+qCgxEzcU/OrZC4rkJJdSNupf1Ro/PanB+PuXPGIoX379jzxxBMATJ48mRkzZhAYGMhdd90FwFNPPcXMmTPZsmULf/75Jx07duT5558vPf/jjz8mMjKS3bt307x58zM+AwO8/fbb5zyvUaNGzJw5k9mzZzN06FAAPvzwQxYsWMCsWbN45JFHSs87/bO0VA2FUg1Mly5dymzn5uYydepUfv31V5KTkzGZTBQUFJy3U+rUZNjDwwNvb29SU1MBaN26Nfv37wegT58+ZVouK+LUuUNCQgBo27ZtmX2FhYVkZ2fj7e3N5s2bWbFiRZnOKLPZTGFhIfn5+bi7u1/Q+4uIiIiIiEj1OvVzn4ODAwEBAWd87gNITU1l8+bNLF68GE9PzzPmSUhIoHnz5uW+x/nOKywspKSkhF69epXud3Jyolu3buzYUfZhY6d/lpaqoVCqgTn9SXcPP/wwCxYs4OWXXyY2NhY3Nzeuv/56iouLzzmPk5NTmW2DwYDFYgHgt99+o6SkBAA3N7cLrvHUuQ0Gw1n3nXi/3Nxcpk2bxvDhw8+Yy9XV9YLfX0REREREpC5yc3Jg+9ODKzR2TWIGoz9Ze95xs8d0pVu0f4Xe+0KU95nybJ/7cnNzGTZsGC+88MIZ84SFnX0x9vOdd/paxeeip8ZXD4VS9ZSzszNms/m841asWMHo0aO59tprAdt/tCcWhKusJk2aVLqeyujUqRO7du0iNja2WuYXERERERGpCwwGQ4VvoevTLIgwH1dSsgrLXVfKAIT6uNKnWRAORkOV1nmhOnXqxLfffktUVNRZn3pX3mfO850XExODs7MzK1asKP0cW1JSwtq1a3nwwQer/DrkTFrovJ6Kiopi9erV7Nu3j/T09NKuotM1a9aM7777jk2bNrF582ZuvfXWs4692Hpyc3NZuHAh6enp5OfnV9ncTz31FHPmzGHatGls27aNHTt28MUXX5TenywiIiIiIiJlORgNTBkWB9gCqFOd2J4yLM7ugRTAfffdR0ZGBrfccgtr164lISGB+fPnM2bMmNIgqrzPwOc7z8PDg/Hjx/PII48wb948tm/fzl133UV+fj5jx46181U3DOqUqqRWO3ecf5AdPfzww4waNYq4uDgKCgr45JNPyh336quvcscdd9CzZ08CAwOZNGkS2dnZVV5Pz549ueeee7jppps4evQoU6ZMYerUqVUy9+DBg/nll194+umneeGFF3BycqJly5bceeedVTK/iIiIiIhIfTSkTRgzb+/EtJ+3k5xVWLo/1MeVKcPiGNLm7LfG1aTw8HBWrFjBpEmTuPzyyykqKqJJkyYMGTIEo9HWa3P6Z+DExESioqLOe96MGTOwWCyMGDGCnJwcunTpwvz58/Hz87PnJTcYBuuFPrexHjp48CCRkZEkJSURERFR5lhhYSGJiYlER0drfaJ6RN9XERERERGp66rqc43ZYmVNYgapOYUEe7nSLdq/VnRI1TXn+n6cK3doyNQpJSIiIiIiItKAORgN9IgJsHcZ0gDZdU2p6dOn07VrV7y8vAgODuaaa65h165dZcYUFhZy3333ERAQgKenJ9dddx1HjhwpM+bAgQP861//wt3dneDgYB555BFMJlNNXoqIiIiIiIiIiFwAu4ZSS5cu5b777uPvv/9mwYIFlJSUcPnll5OXl1c6ZuLEifz88898/fXXLF26lMOHDzN8+PDS42azmX/9618UFxezcuVKPv30U2bPns1TTz1lj0sSEREREREREZEKsOvte/PmzSuzPXv2bIKDg1m/fj19+/YlKyuLWbNmMXfuXAYMGADAJ598QqtWrfj777+55JJL+OOPP9i+fTt//vknISEhdOjQgWeeeYZJkyYxdepUnJ2d7XFpIiIiIiIiIiJyDnbtlDpdVlYWAP7+/gCsX7+ekpISBg4cWDqmZcuWNG7cmFWrVgGwatUq2rZtS0hISOmYwYMHk52dzbZt28p9n6KiIrKzs0tfOTk51XVJIiIiIiIiItVKzy+rHfR9uHC1JpSyWCw8+OCD9OrVizZt2gCQkpKCs7Mzvr6+ZcaGhISQkpJSOubUQOrE8RPHyjN9+nR8fHxKX3FxcVV8NSIiIiIiIiLVy8nJCYD8/Hw7VyJw8vtw4vsi51drnr533333sXXrVpYvX17t7zV58mQeeuih0u1Dhw4pmBIRETnVXD0Gmlv1204REandHBwc8PX1JTU1FQB3d3cMBv0/vKZZrVby8/NJTU3F19cXBwcHe5dUZ9SKUOr+++/nl19+YdmyZURERJTuDw0Npbi4mMzMzDLdUkeOHCE0NLR0zJo1a8rMd+LpfCfGnM7FxQUXF5fS7ezs7Kq6FBEREREREaklzBYraxIzSM0pJNjLlW7R/jgY61doc+Jz74lgSuzH19f3rDmElM+uoZTVamXChAl8//33LFmyhOjo6DLHO3fujJOTEwsXLuS6664DYNeuXRw4cIAePXoA0KNHD5577jlSU1MJDg4GYMGCBXh7e6v7SUREREREpIGatzWZaT9vJzmrsHRfmI8rU4bFMaRNmB0rq1oGg4GwsDCCg4MpKSmxdzkNlpOTkzqkKsGuodR9993H3Llz+fHHH/Hy8ipdA8rHxwc3Nzd8fHwYO3YsDz30EP7+/nh7ezNhwgR69OjBJZdcAsDll19OXFwcI0aM4MUXXyQlJYUnnniC++67r0w3lFS92bNn8+CDD5KZmXlB5+3bt4/o6Gg2btxIhw4dqqU2ERERERFpuOZtTWb8Zxs4/UbslKxCxn+2gZm3d6pXwRTYbuVTKCJ1jV1DqZkzZwLQr1+/Mvs/+eQTRo8eDcBrr72G0Wjkuuuuo6ioiMGDB/Puu++WjnVwcOCXX35h/Pjx9OjRAw8PD0aNGsXTTz9dvcXX9FobdXRdi9GjR5OZmckPP/xQui8yMpLk5GQCAwPtV5iIiIiIiNRLZouVaT9vPyOQArACBmDaz9sZFBda727lE6lr7H773vm4urryzjvv8M4775x1TJMmTfjtt9+qsjSpRg4ODrrPVkREREREqsWaxIwyt+ydzgokZxUy/bcddG7ih6+7M34eTvi5O+Pr7oSLo7qNRGqK0d4FSPWxWCxMnz6d6Oho3NzcaN++Pd988w0Wi4WIiIjSTrUTNm7ciNFoZP/+/QC8+uqrtG3bFg8PDyIjI7n33nvJzc2t8PtPnTqVTz/9lB9//BGDwYDBYGDJkiXs27cPg8HApk2bAFiyZAkGg4H58+fTsWNH3NzcGDBgAKmpqfz++++0atUKb29vbr311jKPOj3b9YmIiIiISMOVmnP2QOpUHy1PZPz/NnDLh38z5PW/6P78Qlo8MY+4p+bRa8Yi/vXmX9z+0Wrun7uBJ374h1f+2MWs5Yl8t+Egi3emsvHAMfal55GVX4LFUjfvbBGxt1rx9D2pHtOnT+ezzz7jvffeo1mzZixbtozbb7+d+fPnc8sttzB37lzGjx9fOv5///sfvXr1okmTJgAYjUbefPNNoqOj2bt3L/feey+PPvpomdsnz+Xhhx9mx44dZGdn88knnwDg7+/P4cOHyx0/depU3n77bdzd3bnxxhu58cYbcXFxYe7cueTm5nLttdfy1ltvMWnSpHNeX1BQEJdeeunF/NGJiIiIiEgdFezlWqFxnRr7YjAYOJZfTGZ+CZn5xViskF9sJr+4gEOZBRV+T6MBfI93Wvm5O+Pr5mTrwHJ3ws/jlP3H/3nia1enutGV1RCeYij2oVCqnioqKuL555/nzz//LH1SYdOmTVm+fDnvv/8+jz76KK+88goHDhygcePGWCwWvvjiC5544onSOR588MHSr6Oionj22We55557KhxKeXp64ubmRlFRUYVu13v22Wfp1asXAGPHjmXy5MkkJCTQtGlTAK6//noWL17MpEmTznt9CqVERERERBqmbtH+hPm4nvUWPgMQ6uPK1/f0LBOsWCxWcgpNHMsvLg2qTg2sjp2yfeo/84vNWKyQkVdMRl4xkFfhWt2cHPB1PyXAci8nwPI4cdw2xtvVCWMNBkIN5SmGYh8Kpeqp+Ph48vPzGTRoUJn9xcXFdOzYkQ4dOtCqVSvmzp3LY489xtKlS0lNTeWGG24oHfvnn38yffp0du7cSXZ2NiaTicLCQvLz83F3dy8z74EDB4iLiyvdfvzxx3n88ccvqOZ27dqVfh0SEoK7u3tpIHVi35o1ayp0fSIiUvOmfgvTviu7r0UY7HzZ9nXCEXh4LizfBUUlMKQ9vDUKQnzOPe+hDJj0Bfy+GfKLIDYEPrkbuhz/X4TVClO+hQ8XQ2Ye9GoOM++AZlq+UESkQXIwGpgyLI57PttwxrETUc6UYXFndPoYjQZ83J3wcXciCo8Kv1+RyUxWfskpodVpAVaebTszv5jMgpMBl9lipaDETEGW+ZxrYJ1xDQbwcTtLgOVeNsDyOaUzy835wruyGuJTDKVmKZSqp06s/fTrr7/SqFGjMsdcXFwAuO2220pDqblz5zJkyBACAgIA2LdvH1deeSXjx4/nueeew9/fn+XLlzN27FiKi4vPCKXCw8NL14gC2216F8rJyan0a4PBUGb7xD6LxVLh6xMRkZrXOgL+nHxy+8RasXmFcPkMaN8YFh3/ncWT38Cwl+HvaWA8yyqXx/Kg1zToHwe/PwpBXrAnBfxO+azw4i/w5nz49G6IDoYnv4bBM2D7i+DqXD3XKSIitVtMkCcGOCNMCa2GDh8XRweCvR0I9q7YbYNge+hXTpGJzLySMzqzjuWXkHWWzqzcIhNWK8e7t0ousE7jWTqwTgRbJ0It235vVyem/qSnGEr1UihVT8XFxeHi4sKBAwfOeivbrbfeyhNPPMH69ev55ptveO+990qPrV+/HovFwiuvvILx+CeFr7766qzv5+joSGxs7Bn7nZ2dMZvNF3k1Z6rI9YmISM1zNEKo75n7V+yGfWmw8TnwPv57jU/vAb9xsGg7DGxT/nwv/AyRAbbOqBOig09+bbXC6/PgiWvg6i62fXPGQ8i98MN6uLlHVVyViIjUNS/M24UVuDwumDG9mta6tZAMBgPerrZb8RoHuJ//hOOKTRYyC052YJ3aeXUsv7g05Do14MrML8ZksVJkspCSXUhKdsW7ss7lxFMM1yRm0CMmoErmlIZHoVQ95eXlxcMPP8zEiROxWCz07t2brKwsVqxYgbe3N6NGjSIqKoqePXsyduxYzGYzV111Ven5sbGxlJSU8NZbbzFs2DBWrFhRJrSqqKioKObPn8+uXbsICAjAx+c892hU4fWJiEjN23MEwu8DVyfo0Qym3wSNA6HIZLvdwOWUJlhXJ9vCsMt3nT2U+mk9DG4HN7wBS3dCIz+4dyDcNcB2PDENUjJhYOuT5/i4Q/cYWLVHoZSISEO0dl8Gf+44goPRwKNDWhEb7GnvkqqMs6ORYC/XCi/mDraurLxisy3Eyj+zM+tsHVo5haYKzV/Rpx2KlEehVGXdWvsf+fnMM88QFBTE9OnT2bt3L76+vnTq1KnMWk+33XYb9957LyNHjsTNza10f/v27Xn11Vd54YUXmDx5Mn379mX69OmMHDnygmq46667WLJkCV26dCE3N5fFixcTFRVVY9cnIiI1p3sMzL7bto5UcqZtfak+T8PWF+CSWPBwsa0N9fyNtg6nx74Es8U29mz2psHMhfDQUHj8ali7Fx6YA86OMKqvLZCCM9elCvE5eUxERBoOq9XK87/tAODGLpH1KpCqLIPBgKeLI54ujkRewCory/ekcfusNecddyEBmcjpDFartfanK9Xs4MGDREZGkpSURERERJljhYWFJCYmEh0djaur/mOrL/R9FRE5j7kXf2tDZh40+Te8ejuM7Qd/bIHxn9i6m4wGuKUHbD8E3WJsC5OXx3mkbUHzlVNP7nvgU1s4tWoarNxtW3Pq8NsQ5ndyzI1v2ta6+PKBi7iAOvALKBERKWve1mTu+WwDbk4OLH2k3wWt8yRlmS1Wer+wiJSswnLXlTrxFMPlkwbUilsia7tz5Q4N2VmWFRURERG5OL4e0DwM4lNs25e3g4TXIHUmpL8H/70XDh2DpsFnnyPMF+LKPs+CVo3gwFHb1yfWrzqSVXbMkazy17YSEZH6y2S28OK8XQDc2SdagdRFOvEUQzj51MITzvUUQ5ELoVBKREREqkVuISQcsQVLpwr0sgVWi7ZBajZc1ensc/RqDruSy+7bnQxNAm1fRwfZwqeF204ez86H1Qm2Na1ERKTh+HJdEnvT8/D3cGZc36b2LqdeGNImjJm3dyLUp2zAF+rjyszbO1XpUwylYdKaUiIiIlIlHv4fDOtkC4wOH4Mp34KDEW7paTv+yVJoFQ5B3rZFyP/9X5g4BFqEn5zjsufh2i5w/+W27YlDoec0eP5HuLE7rEmADxbDB2Ntxw0GeHAIPPsDNAu1hVRPfgPhvnBN55q8ehERsaf8YhOv/7kHgAkDYvFydTrPGVJRQ9qEMSgulDWJGbXuKYZS9ymUEhERkSpxMANueRuO5kKQF/RuAX9Ps4VQYOt4mvwlZORCVBD839W20OlUCUcgPefkdtcY+P5B23lPf28LnV6/HW7rdXLMo1dCXhGMmwWZ+dC7OcybBK7O1X7JIiJSS3z0VyJpOUU09nfntu5N7F1OveNgNNAjJsDeZUg9pFCqgrQefP2i76eISNX7YsK5j8+42fY6l31vnLnvyk6219kYDPD09baXiIg0PEdzi3h/aQIADw9ugbOjVqkRqSv0X+t5ODg4AFBcXGznSqQqnfh+nvj+ioiIiIhI3fTWonjyis20beTDlW21xpFIXaJOqfNwdHTE3d2dtLQ0nJycMBqV49V1FouFtLQ03N3dcXTUfwIiIiIiInXV/qN5/G/1fgAeG9oSo9Y5EqlT9In8PAwGA2FhYSQmJrJ//357lyNVxGg00rhxYwwG/U9LRERERKSuemn+LkrMVvo2D6JXbKC9yxGRC6RQqgKcnZ1p1qyZbuGrR5ydndX1JiIiIiJSh205mMkvW5IxGOCxIS3tXY6IVIJCqQoyGo24urrauwwREREREZEGz2q1MuP3nQBc06ERceHedq5IRCpDrSIiIiIiIiJSpyzdncbKhKM4Oxh5aFBze5cjIpWkUEpERERERETqDIvlZJfUyB5NiPR3t3NFIlJZCqVERERERESkzvhh0yF2puTg5erIff1j7V2OiFwEhVIiIiIiIiJSJxSWmHnlj90AjO8Xg5+Hs50rEpGLoVBKRERERERE6oT/rtrPocwCQr1duaNXtL3LEZGLpFBKREREREREar2sghLeXhwPwEODmuPq5GDnikTkYimUEhERERERkVpv5pIEsgpKaB7iyXWdI+xdjohUAYVSIiIiIiIiUqsdzizgkxWJADw6uCUORoOdKxKRqqBQSkRERERERGq11xbspshkoVuUP5e1CrZ3OSJSRRRKiYiIiIiISK21KyWHbzccBOCxK1piMKhLSqS+UCglIiIiIiIitdaL83ZiscLQNqF0auxn73JEpAoplBIREREREZFaafXeoyzcmYqD0cDDg1vYuxwRqWIKpURERERERKTWsVqtTP99JwA3d40kJsjTzhWJSFVTKCUiIiIiIiK1zrytKWxKysTd2YF/D2xm73JEpBoolBIREREREZFapcRs4cX5uwC4s09Tgr1c7VyRiFQHhVIiIiIiIiJSq3yxNonE9DwCPJwZ17epvcsRkWqiUEpERERERERqjbwiE2/8uQeABy5rhqeLo50rEpHqolBKREREREREao2P/kokPbeIJgHu3NKtsb3LEZFqpFBKREREREREaoX03CI+WJYAwCODW+DsqI+sIvWZ/gsXERERERGRWuHNhXvIKzbTLsKHK9qE2bscEalmCqVERERERETE7hLT85i7+gAAjw1tidFosHNFIlLdtGKciIiIiIiI2N3Lf+zCZLHSr0UQPWMC7V2OSK0wZ9U+3l+6l7TcIlqFeTPtqtZ0iPQ973k/bT7MA59vZFBcCB+O7FK632q18tqC3Xy+NonsghK6RPnx7DVtiQ70qMarODt1SomIiIiIiIhdbU7K5NctyRgMMGlIS3uXI1Ir/Lz5MM/+soN/D2zGrxN6ExfmxchZq0nPLTrneUkZ+Tz/6w66Rfmfcey9pXv5ZOU+nrumDT/c1ws3J0dGfryawhJzdV3GOSmUEhEREREREbuxWq1M/30HANd2bESrMG87VyRSO3y0PJGbu0VyY5dImoV48dw1bXFzduCrdUlnPcdssfLgl5uYOKgZkf7uZY5ZrVY+XpHIhAGxXN46lFZh3rx6U3uOZBfxx/Yj1X055VIoJSIiIiIiInazZFcaf+/NwNnRyH8ub2HvckSqVU5ODtnZ2aWvoqLyu56KTRa2HsqiV+zJW1mNRgO9YgPZsD/zrPO/sXAPAR7O3NS18RnHkjIKSMspKjOnt6sTHSJ92bD/WOUv6iIolBIRERERERG7MFusvDBvJwCje0bRyNfNzhWJVK+4uDh8fHxKX9OnTy933LH8YswWK4GeLmX2B3m6kHaW2/fW7svgq7VJzLiuXbnH03ILS+eo6JzVTQudi4iIiIiIiF18v/EQO1Ny8HZ15N5+MfYuR6Tabd++nUaNGpVuu7i4nGN0xeUWmZj45SamX9cWfw/nKpmzJiiUEhERERERkRpXWGLm1T92AXBv/1h83evOB2mRyvLy8sLb+/zrpvm5O+NgNJyxqHlabtEZnU4A+4/mcfBYAXd+uq50n8VqBSDm8d9Y9J9LCfJ0LZ0j2Nu1zJxxdlrLTaGUiIiIiIiI1Lg5q/ZxOKuQMB9XRveMsnc5IrWKs6ORNo18WBmfzuDWoQBYLFZWxh9lZM8mZ4yPCfJk/oN9y+x7+Y9d5BWZmDKsNWE+bjg5GAjycmFl/FFah/sAkFNYwqakTG6/5Mw5a4JCKREREREREalRWfklvLM4AYCHBjXH1cnBzhWJ1D539o7mP19vpm2ELx0ifZi1fB/5xSZu6BwJwENfbiLEx5VJQ1ri6uRAi1CvMud7uzoBlNl/R69o3lq0h6hADyL93Xjlj92EeLtweVxIzV3YKRRKiYiIiIiISI16d0k8WQUltAjxYninCHuXI1IrDWsfTkZeMa8t2E1aThGtwr359I5uBHnZbt87lFmAwWC4oDnvubQpBcUmJn/3D9mFJXSN8uPTMd3sFgwbrNbjNxk2YAcPHiQyMpKkpCQiIvQXooiICHMv7AeceunWBv8jkohItTiUWUD/l5dQbLLw8eguDGhpnw4NkZqk3KF8RnsXICIiIiIiIg3Hawt2U2yy0D3an/4tgu1djojYkUIpERERERERqRE7U7L5dsNBACZf0eqCbz0SkfpFoZSIiIiIiIjUiBd+34nVCv9qG0aHSF97lyMidqZQSkRERERERKrdqoSjLN6VhqPRwMODW9i7HBGpBRRKiYiIiIiISLWyWq3M+H0HALd0a0x0oIedKxKR2kChlIiIiIiIiFSr3/5JYfPBLDycHXjgsmb2LkdEagmFUiIiIiIiIlJtSswWXpq/E4C7+jYlyMvFzhWJSG2hUEpERERERESqzedrDrDvaD6Bns7c1aepvcsRkVpEoZSIiIiIiIhUi9wiE28u3APAvy9rhoeLo50rEpHaRKGUiIiIiIiIVIsPl+0lPbeY6EAPbu7W2N7liEgto1BKREREREREqlxqTiEf/rUXgEcGt8DJQR8/RaQs/a0gIiIiIiIiVe7NhXvILzbTPtKXoW1C7V2OiNRCCqVERERERESkSu1Ny+XzNUkATB7aEoPBYOeKRKQ2UiglIiIiIiIiVerlP3ZhtlgZ0DKYS5oG2LscEamlFEqJiIiIiIhIldl44Bi//ZOC0QCThrS0dzkiUosplBIREREREZEqYbVamf77TgCu6xRBi1AvO1ckIrWZXUOpZcuWMWzYMMLDwzEYDPzwww9ljhsMhnJfL730UumYqKioM47PmDGjhq9EREREREREFu9KZU1iBi6ORiYOam7vckSklrNrKJWXl0f79u155513yj2enJxc5vXxxx9jMBi47rrryox7+umny4ybMGFCTZQvIiIiIiIix5ktVl74fRcAo3tFEe7rZueKRKS2c7Tnmw8dOpShQ4ee9XhoaNnHhv7444/079+fpk2bltnv5eV1xlgRERERERGpOd9uOMiuIzn4uDlx76Wx9i5HROqAOrOm1JEjR/j1118ZO3bsGcdmzJhBQEAAHTt25KWXXsJkMp1zrqKiIrKzs0tfOTk51VW2iIiIiIhIvVdYYua1BbsBuL9/LD7uTnauSETqArt2Sl2ITz/9FC8vL4YPH15m/wMPPECnTp3w9/dn5cqVTJ48meTkZF599dWzzjV9+nSmTZtW3SWLiIiIiIg0CLNX7iM5q5BGvm6M6NHE3uWISB1RZ0Kpjz/+mNtuuw1XV9cy+x966KHSr9u1a4ezszN3330306dPx8XFpdy5Jk+eXOa8Q4cOERcXVz2Fi4iIiIiI1GOZ+cW8uzgegIcGNcfVycHOFYlIXVEnQqm//vqLXbt28eWXX553bPfu3TGZTOzbt48WLVqUO8bFxaVMYJWdnV1ltYqIiIiIiDQk7yyOJ7vQRMtQL67p2Mje5YhIHVIn1pSaNWsWnTt3pn379ucdu2nTJoxGI8HBwTVQmYiIiIiISMN18Fg+n67cD8BjQ1viYDTYuSIRqUvs2imVm5tLfHx86XZiYiKbNm3C39+fxo0bA7Yupq+//ppXXnnljPNXrVrF6tWr6d+/P15eXqxatYqJEydy++234+fnV2PXISIiIiIi0hC9umA3xWYLPZoGcGnzIHuXIyJ1jF1DqXXr1tG/f//S7RPrPI0aNYrZs2cD8MUXX2C1WrnlllvOON/FxYUvvviCqVOnUlRURHR0NBMnTiyzXpSIiIiIiIhUve2Hs/l+4yEAJl/REoNBXVIicmHsGkr169cPq9V6zjHjxo1j3Lhx5R7r1KkTf//9d3WUJiIiIiIiIufwwrydWK1wZbsw2kX42rscEamD6sSaUiIiIiIiIlJ7rIxPZ+nuNByNBh4ZXP4DpkREzkehlIiIiIiIiFSYxWJlxrydANzWvTFNAjzsXJGI1FUKpURERERERKTCfv0nmS0Hs/BwdmDCZc3sXY6I1GEKpURERERERKRCik0WXpq/C4C7L40h0NPFzhWJSF2mUEpEREREREQq5PM1BziQkU+gpwt39om2dzkiUscplBIREREREZHzyiks4c2FewB4cGAz3J3t+jB3EakHFEqJiIiIiIjIeX24bC9H84ppGujBTV0j7V2OiNQDCqVERERERETknFKzC/nwr0QAHh3SAicHfZQUkYunv0lERERERETknN5YuIeCEjMdG/syuHWovcsRkXpCoZSIiIiIiIicVUJaLl+sTQJg8tBWGAwGO1ckIvWFQikRERERERE5q5fm7cJssTKwVTDdov3tXY6I1CMKpURERERERKRc6/cfY962FIwGmDSkpb3LEZF6RqGUiIiIiIiInMFqtfLC7zsBuKFzJM1CvOxckYjUNwqlRERERERE5AwLd6SyZl8GLo5GHhzUzN7liEg9pFBKREREREREyjCZLbwwz9YldUfvaMJ83OxckYjURwqlREREREREpIxvNxxkT2ouvu5O3HNpjL3LEZF6SqGUiIiIiIiIlCooNvPagj0A3N8/Fh83JztXJCL1lUIpERERERERKfXJykRSsgtp5OvGiB5N7F2OiNRjCqVEREREREQEgGN5xcxckgDAw4Ob4+LoYOeKRKQ+UyglIiIiIiIiALy9OJ6cQhNxYd5c3b6RvcsRkXpOoZSIiIiIiIiQlJHPf1ftB+CxoS0xGg12rkhE6juFUiIiIiIiIsKrC3ZTbLbQKzaAPs0C7V2OiDQACqVEREREREQauG2Hs/hh0yEAHhvSCoNBXVIiUv0USomIiIiIiDRwL8zbhdUKV7UPp22Ej73LEZEGQqGUiIiIiIhIA7YiPp1lu9NwcjDw8OUt7F2OiDQgCqVEREREREQaKIvFyvTfdwBwW/cmNA5wt3NFItKQKJQSERERERFpoH7ecpith7LxdHFkwoBYe5cjIg2MQikREREREZEGqNhk4eU/dgFwz6VNCfB0sXNFItLQKJQSERERERFpgP63ej9JGQUEe7lwR+9oe5cjIg2QQikREREREZEGJqewhLcWxQPw4MDmuDs72rkiEWmIFEqJiIiIiIg0MO8v3UtGXjExQR7c2CXC3uWISAOlUEpERERERKQBSc0u5KPlewF4dEhLHB30sVBE7EN/+4iIiIiIiDQgr/25h8ISC52b+HF5XIi9yxGRBkyhlIiIiIiISAMRn5rLV+uSAJg8tCUGg8HOFYlIQ6ZQSkREREREpIF4cd5OzBYrg+JC6BLlb+9yRKSBUyglIiIiIiLSAKzfn8Ef249gNMCkIS3sXY6IiEIpERERERGR+s5qtTL9t50A3NQ1kthgLztXJCKiUEpERERERKTeW7D9COv2H8PVyciDA5vbuxwREQAc7V2AiIiIiIiIVB+T2cIL82xdUmN7RxPi7WrnikSkouas2sf7S/eSlltEqzBvpl3Vmg6RvuWOnbc1mXcWJ7DvaB4ms5WoQA/u6hPN8E4RpWPyiky8MG8nf2w7wrH8YiL93RndM4rbL2lSQ1dUlkIpERERERGReuyb9QdJSMvDz92Juy+NsXc5IlJBP28+zLO/7ODZa9vQMdKXj1ckMnLWahY93I9AT5czxvu4OXNf/1higz1wcjCycEcqj3yzhQBPFy5tHgTAs79uZ2XCUV67qQMRfm78tSedJ3/cSoi3K4PiQmr6EnX7noiIiIiISH1VUGzmtT93AzBhQDO8XZ3sXJGIVNRHyxO5uVskN3aJpFmIF89d0xY3Zwe+WpdU7vgeMQEMaRNKbLAXTQI8uKN3NC1DvVi3L6N0zPr9x7iuUwQ9YgKI9Hfn1u6NaRXmxeakzBq6qrIUSomIiIiIiNRTH69I5Eh2ERF+btx2SWN7lyPS4OXk5JCdnV36KioqKndcscnC1kNZ9IoNLN1nNBroFRvIhv2Z530fq9XKivh09qbl0S3av3R/5yZ+/LnjCClZhVitVlYmpJOYlkefZoHnmK366PY9ERERERGReigjr5j3liQA8MjgFrg4Oti5IhGJi4srsz1lyhSmTp16xrhj+cWYLdYzbtML8nQhIS3vrPNnF5ZwyfMLKTZZMBoNPHt1G/o0Cyo9PvWq1kz+7h8umb4QR6MBo8HA9OFt6d404OIurJIUSomIiIiIiNRDby+KJ6fIRJtG3gxrF27vckQE2L59O40aNSrddnE5c22oi+Hp7MhvD/Qhr9jEyvijPPPrdiL93ekRYwudPl25j00HMvloZBca+bmxJjGDp46vKdXbDt1SCqVERERERETqmaSMfP779z4AHhvSCqPRYN+CRAQALy8vvL29zzvOz90ZB6OB9Nyyt/el5RYRVM4i5ycYjQaiAj0AaB3uQ3xqLu8uiadHTACFJWZemr+L90d0ZkBL26LmrcK82X44mw/+2muXUEprSomIiIiIiNQzL/+xixKzlT7NAu3yQVNELo6zo5E2jXxYGZ9eus9isbIy/iidmvhWeB6L1UqxyQJAidlCidmKwVA2pDYaDVit1iqp+0KpU0pERERERKQe2Xooix83HQZg0pCWdq5GRCrrzt7R/OfrzbSN8KVDpA+zlu8jv9jEDZ0jAXjoy02E+LiW/nf+zuJ42kX40MTfg2KzmcU70/h+4yGevaYNAF6uTnSP9mf6bztwdXQgws+Nv/ce5bsNB3niyriz1lGdFEqJiIiIiIjUIy/M2wnANR3CadPIx87ViEhlDWsfTkZeMa8t2E1aThGtwr359I5uBHnZbt87lFlQpuupoNjMkz9sJTmrEFcnB2KCPHjtpg4Ma39yTbm3bu3Ii/N28eCXG8nML6GRnxuPDG7B7d3t83ROg9VePVq1yMGDB4mMjCQpKYmIiAh7lyMiImJ/c7X2CLc2+B+RRKQO+mtPGiNmrcHZwcjC/1xKpL+7vUsSEZQ7nI3WlBIREREREakHLBYrM363dUndfkkTBVIiUusplBIREREREakHft5ymG2Hs/FyceT+AbH2LkdE5LwUSomIiIiIiNRxRSbbo94B7ukXg7+Hs50rEhE5P4VSIiIiIiIiddxnfx/g4LECQrxduKNXtL3LERGpEIVSIiIiIiIidVh2YQlvL9oDwMSBzXFzdrBzRSIiFaNQSkREREREpA57f2kCx/JLiA325PrOeqqXiNQdCqVERERERETqqJSsQmYtTwRg0pCWODroI56I1B36G0tERERERKSOev3P3RSWWOga5cfAVsH2LkdE5IIolBIREREREamD9hzJ4at1SQA8NrQlBoPBzhWJiFwYhVIiIiIiIiJ10Ivzd2GxwuDWIXRu4m/vckRELphCKRERERERkTpm3b4MFmw/goPRwKNDWtq7HBGRSlEoJSIiIiIiUodYrVae/20HADd1jSQmyNPOFYmIVI5CKRERERERkTpk/rYjbDiQiZuTAw9e1sze5YiIVJpCKRERERERkTrCZLbw4vydANzZJ5pgb1c7VyQiUnkKpUREREREROqIr9YdZG9aHv4ezozr29Te5YiIXBSFUiIiIiIiInVAfrGJ1/7cDcADA2LxcnWyc0UiIhdHoZSIiIiIiEgdMOuvRNJyimjs786t3ZvYuxwRkYumUEpERERERKSWO5pbxPvL9gLw8OAWODvqo5yI1H2O9i5AREREREREzmS2WFmTmEFqTiG//ZNMbpGJto18uLJtmL1LExGpEnaN15ctW8awYcMIDw/HYDDwww8/lDk+evRoDAZDmdeQIUPKjMnIyOC2227D29sbX19fxo4dS25ubg1ehYiIiIiISNWatzWZ3i8s4pYP/+bfX2xi/rYjAFzWMhij0WDn6kREqoZdQ6m8vDzat2/PO++8c9YxQ4YMITk5ufT1+eeflzl+2223sW3bNhYsWMAvv/zCsmXLGDduXHWXLiIiIiIiUi3mbU1m/GcbSM4qPOPYGwv3MG9rsh2qEhGpena9fW/o0KEMHTr0nGNcXFwIDQ0t99iOHTuYN28ea9eupUuXLgC89dZbXHHFFbz88suEh4eXe15RURFFRUWl2zk5OZW8AhERERERkapjtliZ9vN2rOcYM+3n7QyKC8VBHVMiUsfV+tXxlixZQnBwMC1atGD8+PEcPXq09NiqVavw9fUtDaQABg4ciNFoZPXq1Wedc/r06fj4+JS+4uLiqvUaREREREREylNYYiYpI591+zL4Zcthpv60rdwOqROsQHJWIWsSM2quSBGRalKrFzofMmQIw4cPJzo6moSEBB5//HGGDh3KqlWrcHBwICUlheDg4DLnODo64u/vT0pKylnnnTx5Mg899FDp9qFDhxRMiYiIiIhIlTGZLaTnFnMku9D2yiniSNbJr1OzC0nJLiQzv6RS86fmnD24EhGpK2p1KHXzzTeXft22bVvatWtHTEwMS5Ys4bLLLqv0vC4uLri4uJRuZ2dnX1SdIiIiIiLSMFitVo7ll3DkeKiUml3Ikeyik+HT8a/Tc4uwnOsevFO4OBoJ8XYlxNsFB6OBv/eevwsq2Mv1Iq9ERMT+anUodbqmTZsSGBhIfHw8l112GaGhoaSmppYZYzKZyMjIOOs6VCIiIiIiIuXJKSzhSLati+lITiEpWbaAKTXHFjalZBWSllNEsdlSofkcjAaCvVwI9nYlxMuFEG9XQn1cCT7+dYi3K6Herni7OWIw2NaHMlus9H5hESlZheWuK2UAQn1c6RbtX3UXLiJiJ3UqlDp48CBHjx4lLCwMgB49epCZmcn69evp3LkzAIsWLcJisdC9e3d7lioiIiIi0qCZLVbWJGaQmlNIsJctRLHXwtyFJWbScopIOaWb6cTtc0eyC0k93t2UV2yu8JwBHs4Ee7sS6m0LmIKPdzqFHg+bgr1dCPBwueBrdjAamDIsjvGfbcAAZYKpEzNNGRanRc5FpF6wayiVm5tLfHx86XZiYiKbNm3C398ff39/pk2bxnXXXUdoaCgJCQk8+uijxMbGMnjwYABatWrFkCFDuOuuu3jvvfcoKSnh/vvv5+abbz7rk/dERERERKR6zduazLSft5dZsDvMx5Upw+IY0iasyt7njHWbTr2V7sQaTjkXtm6Tl4sjIT62gCnEy7Xc4CnYyxVnx+p7ZtSQNmHMvL3TGX+GodXwZygiYk8Gq9VawTudq96SJUvo37//GftHjRrFzJkzueaaa9i4cSOZmZmEh4dz+eWX88wzzxASElI6NiMjg/vvv5+ff/4Zo9HIddddx5tvvomnp2eF6zh48CCRkZEkJSURERFRJdcmIiJSp83Vb+C51W4/IonUafO2JjP+sw1n3Hp24m+Vmbd3Om+ocmLdppTjodKJdZtOX8Opsus2nbh1ruzXttvqPFxqz80ktanbTEQujnKH8tn1b9x+/fpxrkxs/vz5553D39+fuXPnVmVZIiIiIiJSCWaLlWk/by93LSQrtmBq6k/biA705GjuidvpTq7blJJl267udZvqCgejgR4xAfYuQ0Sk2tSeXwOIiIiIiEidtiYxo8ztZqezAinZRQx+fVmF5quudZtERKR2UCglIiIiIiJVIjXn7IHUqVwdjUT4u9t13SYREbE/hVIiIiIiIlIlgr1cKzTukzHddFuaiIigXz2IiIiIiEiV6Bbtj7+H81mPG7A9ha9btH/NFSUiIrWWQikREREREakSiem55Bebyj12YtWnKcPitAaUiIgACqVERERERKQKpOcWMfqTtRSWWIgJ8iDU26XM8VAfV2be3okhbcLsVKGIiNQ2WlNKREREREQuSmGJmbvmrOPgsQKaBLjz1d098HV3Zk1iBqk5hQR72W7ZU4eUiIicSqGUiIiIiIhUmsVi5T9fb2bjgUx83Jz4eHRXAjxtXVJazFxERM5Ft++JiIiIiEilvfzHLn7dkoyTg4H3bu9MTJCnvUsSEZE6QqGUiIiIiIhUyldrk3h3SQIA04e3U2eUiIhcEIVSIiIiIiJywVbEp/P49/8AMGFALNd3jrBzRSIiUtcolBIRERERkQsSn5rDPZ+tx2SxclX7cB4a1NzeJYmISB2kUEpERERERCosPbeIMbPXklNooksTP168vh0Gg56qJyIiF06hlIiIiIiIVEhhiZm75qwjKaOAxv7uvD+iM65ODvYuS0RE6ihHexcgIiIiIiK1n8Vi5T9fb2bjgUx83Jz4ZExXAjxd7F1WzZirTjButdq7AhGph9QpJSIiIiIi5/XyH7v4dUsyTg4G3ru9MzFBnvYuSURE6jiFUiIiIiIick5frUvi3SUJAEwf3o4eMQF2rkhEROoDhVIiIiIiInJWK+PTefy7fwCYMCCW6ztH2LkiERGpLxRKiYiIiIhIueJTc7j7s/WYLFaGtQ/noUHN7V2SiIjUIwqlRERERETkDOm5RYyZvZacQhOdm/jx0vXtMBi04LeIiFQdhVIiIiIiIlJGYYmZu+asIymjgMb+7nwwojOuTg72LktEROoZhVIiIiIiIlLKYrHyn683s/FAJj5uTnwypisBni72LktEROohR3sXICIiIiIitccrC3bx65ZknBwMvHd7Z2KCPO1dUp0080/ba1+abbt1BDx1LQztYNtOOAIPz4Xlu6CoBIa0h7dGQYjP2ec0W2Dqt/DZCkjJhHA/GN0XnrgGTtxZabit/HNfvAUeubKKLk5EpIoolBIREREREQC+WpfEO4sTAJg+vB09YgLsXFHdFeEPM26GZqFgtcKnf8HVr8LG5yEqEC6fAe0bw6LHbeOf/AaGvQx/TwPjWe5neeFnW9D16T22kGvdXhjzAfi4wQNDbGOS3yl7zu+bYeyHcF236rtWEZHKUiglIiIiZ9jxdEt7l2B3rW61dwUiNWtlfDqPf/cPABMGxHJ95wg7V1S3DetUdvu5G22B0t/xcCjD1kG18Tnwdrcd//Qe8BsHi7bDwDblz7lyN1zdGf7V0bYdFQSfr4I1e0+OCfUte86P66F/HDQNrpLLEhGpUlpTSkRERESkgYtPzeHuz9ZjslgZ1j6chwY1t3dJ9YrZAl+sgrwi6BELRSbb7XYuTifHuDqB0WC7ne9sejaHhdtgd7Jte/N+2/ih7csffyQLft0EYy+tsksREalS6pQSEREREWnA0nOLGDN7LTmFJjo38eOl69thOLFAkVyUfw5Aj6lQWAKervD9RIiLgCBv8HCBSV/A8zfabu977EtbeJWcefb5HhsG2QXQ8hFwMNrGP3cD3Nar/PGfLgMvVxjetTquTkTk4imUEhERERFpoApLzIybs46kjAIa+7vzwYjOuDo52LuseqNFOGx6HrIK4JvVMOo9WPqELZj6+gEY/wm8Od/WIXVLD+gUZfv6bL5aDf9bAXPvg9aNYNN+ePAz24Lno/qeOf7jpbbAytW52i5RROSiKJQSEREREWmALBYr//l6MxsOZOLt6sjHo7sS4Oli77LqFWdHiA21fd05GtbuhTfmw/tj4fJ2kPAapOeAoxF8PSD03nOv/fTIXFu31M09bNttG8P+dJj+05mh1F87YVcyfDmheq5NRKQqKJQSEREREWmAXlmwi1+3JOPkYOD9EV2IDfa0d0n1nsUKRSVl9wV62f65aBukZsNVnc4874T84jOfzOdgtM17ullLbEFY+yYXVbKISLVSKCUiIiIi0sB8tS6JdxYnADB9eDt6xATYuaL6Z/IXtgXIGwdCTgHMXQlLdsD8SbbjnyyFVuG29aVW7YF//xcmDrHd8nfCZc/DtV3g/stt28M6wnM/QOMAaB0BG/fBq7/DHactZJ6dD1+vgVf0FFERqeUUSomIiIiINCAr49N5/Lt/AJgwIJbrO0fYuaL6KTUbRr5nW7jcxx3aRdoCqUFtbcd3JcPkLyEjF6KC4P+uholDy86RcMR2e98Jb42CJ7+Bez+xzR/uB3cPgKeGlz3vi79ti6ff0rNaL1FE5KIZrFZrOc2eDcvBgweJjIwkKSmJiAj9T1lERGRHy1b2LsHuWu3cYe8SRKpcfGoOw99dSXahiWHtw3nz5g560l5FzNWfEbc2+I+NIhdFuUP51CklIiIilbYuP5+PM46yrbCINLOJN8MbMdDLq/T42+lp/J6TQ0pJCU4GA3Gurvw7MIj2bm6lY7YXFvJKWipbCwsxApd7efFocAgepy+ccop0k4lX01JZkZdPjsVMFzd3Hg8JIcrZ9oipTLOZt9PTWJmXT7KpBD8HBy7z9OKBwEC8HPRkMWmY0nOLGDN7LdmFJjo38eOl69spkBIREbs6+097IiIiIueRb7HQwsWVJ0NCyj0e5ezM/wWH8ENUNP9t3IRGTk7cdTCJDJMJgFRTCXckHaCxkzNfNGnCBxGRxBcV83/JyWd9T6vVyoRDB0kqKeHtRo34NiqKMCcnxiYdIN9iASDNZCLNZOKR4CB+jIrm+dAwlufl8mRKStX/IYjUAYUlZsbNWUdSRgGN/d35YERnXJ0U0IqIiH2pU0pEREQqra+nJ309z/7Eriu9fcpsTwoK5tusLHYVFdHD0ZEluXk4GQw8GRKC8XjHxpTQEK7Zt4/9xcU0Od75dKr9JSVsLizkx6homrnYHl8/JSSEvgk5/JadzfW+vjRzceGNRidb4xs7O/PvoCAmJSdjslpxVHeINCAWi5X/fL2ZDQcy8XZ15OPRXQnwdLF3WSIiIgqlREREpGYUW618lZWJl9FIy+NhUrHVgpPBUBpIAbgYbI3cGwryyw2liq2W4+NOnmM0GHA2GNhQkM/1vr7lvn+u2YKn0ahAShqcVxbs4tctyTg5GHh/RBdig88eJIuISO0yZ9U+3l+6l7TcIlqFeTPtqtZ0iPQtd+y8rcm8sziBfUfzMJmtRAV6cFefaIZ3KruGVXxqDjN+38nqvRmYLFaahXgy8/bONPJ1K3fe6qRQSkRERKrVktxc/nP4EIVWK0GOjnwUEYmfo+1HkO7uHryYmsqsjKOM8POnwGLhtbQ0ANJM5nLni3Z2IczRkdfS05gaEoqb0cicjAxSTKaznnPMZGLm0XRu8PGtlmsUqa2+WpfEO4sTAJg+vB09YgLsXJGIiFTUz5sP8+wvO3j22jZ0jPTl4xWJjJy1mkUP9yOwnI5XHzdn7usfS2ywB04ORhbuSOWRb7YQ4OnCpc2DANh/NI/r31vFTV0ieXBgc7xcHdl9JBcXR/us7qRQSkRERKpVN3d3vouKJtNs5uusTB5KPswXjZsQ4OhIMxcXng8L44XUVF5PS8NoMHC7rx8BDg5nXfjSyWDgzUYRPJGSTI/4PTgAPdw96OPhQXnPhso1m7nn0EFiXFy4LzCwGq9UpHZZGZ/O499sABy5P/gLrt91Jeyyd1V2oKfGiUgd9dHyRG7uFsmNXSIBeO6atizamcpX65K4t1/sGeNP/8XDHb2j+XbDQdbtyygNpV6av4v+LYKZfMXJJy03CfCoxqs4N4VSIiIiUq3cjUaaODvTBGjv5saQvQl8m5XFuADbD05XevtwpbcP6SYTbkYjBuDTYxlEODuddc7Wrq58HxVNjtlMidWKv6MjN+3fRxtX1zLj8ixmxh08iIfRyFvhjXDSrXvSQMSn5nDPZ+sx4cgwn6U8FPI/e5ckIiJATk4O2dnZpdsuLi64uJzZ9VRssrD1UBb39osp3Wc0GugVG8iG/ZnnfR+r1crKhKPsTcvjsaH+gG2NwcU7U7n70hhGzFrN9sPZRPi7c2+/GAa3Dr34i6sEPX1PREREapSVk+tCnSrQ0REPo5Hfc7JxMRjo6X7+39p5OTjg7+jIvuJithUWMsDTq/RYrtnMnUlJOBngnUYRuBj1Y480DOm5RYyZvZbsQhOd3bfzUuTrGA3qFhIRqQ3i4uLw8fEpfU2fPr3cccfyizFbrGfcphfk6UJabtFZ588uLCHuqXk0+7/fGTN7LdOuak2fZrYuqfS8IvKKzcxcksClzYOYM7Ybg1uHcM9n6/l779Gqu8gLoE4pERERqbQ8i4UDxcWl24dKSthRWIiPgwO+Dg68f/QoAzw9CXR0JNNsZu6xYxwxmRjs5V16zv+OHaOjmxvuRiMr8/J4OS2ViUFBeDucfFz9vxL3MjEwiIFettBpXk42/g4OhDk6sbuoiOmpR7jM05NeHrYgK9ds5s6DSRRarLzQKJxci4Vciy0I83dwwEEdU1JPFZaYGTdnHUkZBTT2d+eD0GdxNZbYuywRETlu+/btNGrUqHS7vC6pi+Hp7MhvD/Qhr9jEyvijPPPrdiL93ekRE4D1+O8nBsWFcGefpgC0Dvdhw/5j/G/1AS5pWvPrDiqUEhERkUrbVljA6KSk0u0X0lIBuMbbmykhoSQWF/Hvw1kcM5vxNTrQxs2V/0Y2ptkpP4D9U1jA2+lp5FutNHV2ZmpIKFf5+JR5n8TiYnIsJxcxTzOZeDE1lXSTiSBHR6728eGegJPrRW0vKmRLYSEAQxL3lplrQdOmNHI686l+InWdxWLl4a83s+FAJt6ujnw8uisBf2af/0QREakxXl5eeHt7n3ecn7szDkYD6ad1RaXlFhFUziLnJxiNBqICbb+kax3uQ3xqLu8uiadHTAB+7s44Gg00O+0prDHBnqzbd6wSV3PxFEqJiIhIpXVz92B7i5ZnPf5mo4izHjthRlj4ecec/h4j/PwZ4edf6bpE6qNXFuzily3JOBoNvDeiM7GnfegQEZG6w9nRSJtGPqyMTy9d78lisbIy/igjezap8DwWq5Vik6V0znYRPuxNzyszJjEtj0a+blVX/AXQ4goiIiIiInXcV+uSeGdxAgDTh7elZ4yeNCkiUtfd2Tuaz9cm8c36g8Sn5vB/P2wlv9jEDZ1tT+N76MtNvDBvZ+n4dxbH89eeNA4czSc+NYcPl+3l+42HuLbjydsFx/WN4Zcth/l8zQH2pefx6cp9LNyZyogeFQ+6qpI6pURERERE6rCV8ek8/t0/ANzfP5Ybjj86XERE6rZh7cPJyCvmtQW7ScspolW4N5/e0Y0gL9vte4cyCzCcsk5mQbGZJ3/YSnJWIa5ODsQEefDaTR0Y1v5kV/qQNqE8d01b3l0Sz9SfttE0yJOZt3Wia9TZO9Crk8FqtTb4R3EcPHiQyMhIkpKSiIg4/20GIiIi9d2Olq3sXYLdtdq5w94liJxXfGouw99dQXahiWHtw3njpg4Yjacs5D9Xi/pzaxV83NGfY9X8OYo0YModyqdOKRERERGROuhobhFjZq8hu9BE5yZ+vHR9u7KB1EWY+afttS/Ntt06Ap66FoZ2sO2LfrD88756AG7oXv6x0e/Bp3+V3Te4HcybdHJ7dzI8MhdW7IZiE7RrDM9cD/1bX+wViYhIbaRQSkRERESkjiksMXPXnHUkZRTQ2N+dD0Z0xtXJocrmj/CHGTdDs1CwWm1h0tWvwsbnoWU4JL9TdvwHi+ClX2Fo+3PPO6QdfHL3yW0Xp7LHr3zZ9p6L/g/cnOH13+HKVyDhVQj1rZJLExGRWkShlIiIiIhIHWKxWHn4681sOJCJt6sjH4/uSsA5Hg9eGcM6ld1+7kZb59Tf8bauqdMDou/XwY3dwdP13PO6OJ09XErPgT0pMOsuW4cU2IKxd/+ErQcVSomI1Ed6+p6IiIiISB3yyoJd/LIlGUejgfdGdCY22LNa389sgS9WQV4R9Ig98/j6RNi0H8b2O/9cS3ZA8Hho8TCM/xiO5pw8FuAJLcJgzl+QVwgmM7y/CIK9oXN0lV2OiIjUIuqUEhERERGpI75al8Q7ixMAmD68LT1jAqvtvf45AD2mQmGJrQPq+4kQV87avLOWQKtw6Nn83PMNaQ/Du0J0ECSkwuNfwtAXYdU0cDCCwQB/ToZrXgOvO8FosAVS8yaBn0d1XKGIiNibQikRERERkTpgZXw6j3/3DwD394/lhi6R1fp+LcJh0/OQVQDfrIZR78HSJ8oGUwXFMHclPHnN+ee7ucfJr9s2tt2iFzMRlmyHy9rY1q66b7YtiPrrSduaUh8tgWEvw9pnIMyvii9QRETsTrfviYiIiIjUcvGpudzz2XpMFitXtgvjoUHnaUuqAs6OEBtqu3Vu+s3QvjG8Mb/smG9WQ34RjOxz4fM3DYZAL4g/YttetA1+2Qhf3A+9WkCnaHh3jC2cOv2pfSIiUj+oU0pERESkHjFbrKxJzCA1p5BgL1e6RfvjYDTYuyy5CEdzixgzew3ZhSY6N/Hj5RvaY7TD99RihaKSsvtmLYWrOkGQ94XPd/AoHM2FMF/bdn6x7Z/G035tbjTa3ltEROofhVIiIiIi9cS8rclM+3k7yVmFpfvCfFyZMiyOIW3C7FiZVFZhiZm75qwjKaOAxv7ufDCiM65ODtX+vpO/gKHtoXEg5BTYbtFbsgPmTzo5Jj4Flu2E3x4pf46WD8P0m+DarpBbCNO+g+u62p6il3AEHv0cYkNgcDvb+B7NbGtHjXoPnrrW1iH14WJITIV/dajuKxYREXtQKCUiIiJSD8zbmsz4zzZwekNJSlYh4z/bwMzbOymYqmMsFisPf72ZDQcy8XZ15OPRXQnwdKmR907NhpHvQXIm+LhDu0hbIDWo7ckxHy+FCH+4vG35c+xKhqx829cORthywHYbXmYehPvZznvmBnBxso0J9LItav5/X8GA56HEBK0j4MeHoH2Tar1cERGxE4VSIiIiInWc2WJl2s/bzwikAKyAAZj283YGxYXqVr465NUFu/llSzKORgPvjehMbLBnjb33rHHnH/P8TbbX2Vj/d/JrN2eY/9j55+zStGLjRESkftBC5yIiIiJ13JrEjDK37J3OCiRnFbImMaPmipKL8tW6JN5eHA/A9OFt6RkTaOeKREREqp5CKREREZE6LjXn7IFUZcaJfa2MT+fx7/4B4P7+sdzQJdLOFYmIiFQPhVIiIiIidVywl2uFxn2yIpFVCUexWvUos9oqPjWXez5bj8li5cp2YTw0qLm9SxIREak2CqVERERE6rhu0f6E+bhyvtWiNiVlccuHf3PNOyv4ZcthTGZLjdQnFXM0t4gxs9eQXWiicxM/Xr6hPUatASYiIvWYQikRERGROs7BaGDKsLhyFzo3HH9Nu6o1t3VvjIujkc0Hs7h/7kb6v7KET1fuI7/YVMMVy+kKS8zcNWcdSRkFNPZ354MRnXF1crB3WSIiItVKT98TERERqQcGtAzB392ZjPziMvtDfVyZMiyOIW3CAJg4qDlzVu3nv6v2kZRRwJSftvHan7sZeUkTRvaMItDTxR7lN2gWi5WHv97MhgOZeLs68vHorgTo+yAiIg2AQikRERGReuD7jQfJyC8mwMOZV29qT2Z+CcFernSL9sfhlFvAAj1deGhQc8ZfGsPX65P46K9EDmTk8+aieN5ftpfrOkdwV5+mRAd62PFqGpZXF+zmly3JOBoNvDeiM7HBnvYuSUREpEYolBI5Ya7WbOBWLXwrIlIXlZgtvL04HoDx/WK4tHnwec9xc3ZgZI8obuvehHlbU/hgWQKbD2Yxd/UBPl9zgMvjQhjXN4bOTfyqu/wG7at1SaXfu+nD29IzJtDOFYmIiNQchVIiIiIiddyPmw6TlFFAgIczt3ZvfEHnOhgN/KtdGFe0DWV1YgYfLNvLop2pzN92hPnbjtCliR/j+jZlYKsQLbpdxVYmpPP4d/8AcH//WG7oEmnnikRERGqWXRc6X7ZsGcOGDSM8PByDwcAPP/xQeqykpIRJkybRtm1bPDw8CA8PZ+TIkRw+fLjMHFFRURgMhjKvGTNm1PCViIiIiNiHyWzhneOdNnf1bYq7c+V+52gwGLikaQAfj+7Kgol9uaFzBE4OBtbtP8a4/65n4GtL+XzNAQpLzFVZfoMVn5rLPf9dj8li5cp2YTw0qLm9SxIREalxdg2l8vLyaN++Pe+8884Zx/Lz89mwYQNPPvkkGzZs4LvvvmPXrl1cddVVZ4x9+umnSU5OLn1NmDChJsoXERERsbtftiSTmJ6Hn7sTIy5pUiVzNgvx4qUb2rN80gDuuTQGL1dH9qblMfm7f+j9wmLeXrSHzNMWVJeKO5pbxJjZa8guNNG5iR8v39BeXWgiItIg2fX2vaFDhzJ06NByj/n4+LBgwYIy+95++226devGgQMHaNz4ZGu6l5cXoaGh1VqriIiISG1jtlh5a9EeAO7s0xQPl6r90S7E25XHhrbk/gGxfLHmAB8vT+RwViEv/7Gbd5ckcGOXSMb2jibS371K37c+Kywxc9ecdSRlFNDY350PRnTG1cnB3mWJiIjYhV07pS5UVlYWBoMBX1/fMvtnzJhBQEAAHTt25KWXXsJkMp1znqKiIrKzs0tfOTk51Vi1iIiISPX49Z9kEtLy8HFzYmSPqumSKo+niyN39mnK0kf789pN7WkZ6kV+sZnZK/fR7+UlTPh8I1sPZVXb+9cXFouVh7/ezIYDmXi7OvLx6K4EeLrYuywRERG7qTMLnRcWFjJp0iRuueUWvL29S/c/8MADdOrUCX9/f1auXMnkyZNJTk7m1VdfPetc06dPZ9q0aTVRtoiIiEi1sFisvLXQ1iU1tnc0Xq5O1f6eTg5Gru0YwTUdGvHXnnQ+WLaX5fHp/Lz5MD9vPkzPmADG9W3Kpc2DMBh0O9rpXl2wm1+2JONoNPDeiM7EBnvauyQRERG7qhOhVElJCTfeeCNWq5WZM2eWOfbQQw+Vft2uXTucnZ25++67mT59Oi4u5f/mafLkyWXOO3ToEHFxcdVTvIiIiEg1mLcthT2puXi5OjKqZ1SNvrfBYKBv8yD6Ng9i66EsPvxrL79sSWZlwlFWJhylZagXd/VpyrD24Tg71qnG/Grz9bok3j6+IP304W3pGRNo54pERETsr9b/lHAikNq/fz8LFiwo0yVVnu7du2Mymdi3b99Zx7i4uODt7V368vLyquKqpaGa8RMYboMH/3ty3weLoN+z4D3Wdiwz7/zzmC3w5NcQ/SC4jYaYifDM92C1nhyTWwj3z4aI+21j4h6B9/6s2usREZHayWKx8ubxLqkxvaLxcav+LqmzadPIhzdu7sjSR/pxR69o3J0d2JmSw3++3kzfFxfzwbIEcgpL7FZfbbAyIZ3J3/0DwP39Y7mhS6SdKxIREakdanWn1IlAas+ePSxevJiAgIDznrNp0yaMRiPBwcE1UKHISWsT4P1F0K5x2f35RTCkne01+cuKzfXCzzDzT/j0HmgdAev2wpgPwMcNHhhiG/PQZ7BoO3x2L0QFwR//wL2fQLgfXNW5aq9NRERqlwU7jrAzJQdPF0fG9oq2dzkARPi589SwOP59WTM+W72f2Sv3kZJdyPO/7eSthfHc2r0xY3pFE+rjau9Sa1R8ai73/Hc9JouVK9uF8dCg5vYuSUREpNawayiVm5tLfHx86XZiYiKbNm3C39+fsLAwrr/+ejZs2MAvv/yC2WwmJSUFAH9/f5ydnVm1ahWrV6+mf//+eHl5sWrVKiZOnMjtt9+On5+fvS5LGqDcQrjtXfjwTnj2h7LHHjz+gMkl2ys+38rdcHVn+FdH23ZUEHy+CtbsPWXMHhjVB/odv/N03AB4fyGsSVAoJSJSn1mtJ7ukRveMwsfdfl1S5fFxd+K+/rHc2SeaHzYe4oNle0lIy+P9ZXv5eEUiV7VvxLi+TWkRWv871Y/mFjFm9hqyC010auzLyze0x2jUWlsiIiIn2PX2vXXr1tGxY0c6drR98n7ooYfo2LEjTz31FIcOHeKnn37i4MGDdOjQgbCwsNLXypUrAdtteF988QWXXnoprVu35rnnnmPixIl88MEH9rwsaYDumw3/6gAD21TNfD2bw8JtsDvZtr15PyzfBUPbnzKmGfy0AQ5l2G7rW7wNdqfA5W2rpgYREamdFu5IZdvhbNydHRjbu3Z0SZXHxdGBm7o2ZsHES/loZBe6RflTYrby7YaDDH59GaM/WcPKhHSsp96bXo8UlpgZ99/1JGUU0NjfnQ9HdsHVycHeZYmIiNQqdu2U6tev3zl/EDnfDymdOnXi77//ruqyRC7IF6tgQyKsfabq5nxsGGQXQMtHwMFoW2PquRvgtl4nx7w1CsbNgogJ4OgARoOtU6tvq6qrQ0REaher1cqbi2xdUiN7ROHn4Wznis7PaDQwMC6EgXEhbDxwjA+W7WXethSW7Epjya402kX4cFefpgxtE4qjQ61f7rRCLBYrD3+9mfX7j+Ht6sjHo7sS4Fn+A3hEREQaslq9ppRIbZd0FP49BxZMBtcq/Fzw1Wr43wqYex+0bgSb9sODn9nWixrV1zbmrT/g73j46T/QJBCW7bR1bIX7VV3HlkidNFe3xnBr/ew8EViyO40tB7Nwc3Lgzj61t0vqbDo29mPm7Z3Zl57HR8v38vW6g2w5mMWEzzcS6e/G2F7R3Ng1Enfnuv0j6qsLdvPLlmQcjQbeG9GZ2GBPe5ckIiJSK9Xt/+OL2Nn6REjNhk7/d3Kf2WILiN7+A4o+tXU6XahH5tq6pW7uYdtu2xj2p8P0n2yhVEExPP4lfD/x5LpT7RrbwquXf1UoJSJSH1mtVt7409YldfsljQmsw503UYEePHtNWyYObM6cVfuZs2ofSRkFTP15O68v3MOIS5owqmdUnbzGr9cl8fZi25qp04e3pWdMoJ0rEhERqb0USolchMtawz8zyu4b8wG0DINJwyoXSAHkF4PxtHMdjGA53vxQYoISs+2WvTPGWCr3niIiUrstj09nU1ImLo5G7urb1N7lVIkATxcmDmrOPZfG8M36JD78K5EDGfm8tSie95ft5bpOEdzVJ5qmQXWj02hlQjqTv/sHgPv6x3BDl0g7VyQiIlK7KZQSuQhebtDmtJ83PVwgwOvk/pRM2yv+iG37nyTwcoXGgeB//Gfsy56Ha7vA/Zfbtod1hOd+gMYB0DoCNu6DV3+HOy61Hfd2h0tbwSOfg5uz7fa9pTtgzl/w6u3Ve80iIlLzTu2Suq17E4K9XO1cUdVyc3ZgRI8obu3ehPnbUnh/2V42J2Xy+ZoDfLH2AINahXD3pU3p3MTf3qWeVXxqLvf8dz0mi5Ur24Xxn0Et7F2SiIhIradQSqSavbcQpn13crvv8QXRPxkHo4+HTAlHID3n5Ji3RsGT38C9n9huDwz3g7sHwFPDT4754n6Y/CXc9i5k5NqCqeduhHsuq/5rEhGRmrVq71HW7T+Gs6ORuy+tH11S5XEwGriibRhD24SyJjGDD5btZeHOVP7YfoQ/th+hcxM/xvVtyqBWIRhPbxe2o6O5Rdwxey3ZhSY6Nfbl5Rva16r6REREaiuFUiJVbMkTZbenXmd7ncu+N8pue7nB6yNsr7MJ9YVP7q5UiSIiUsec6JK6pWskId71q0uqPAaDge5NA+jeNIA9R3L48K+9/LDxMOv3H+Pu/66naaAHd/ZpyvBOjXB1crBrrYUlZsb9dz0HMvJp7O/OhyO72L0mERGRuqJ+PHdXREREpJ76e+9RVidm4Oxg5J5+MfYup8Y1C/Hixevbs3xSf8b3i8HL1ZG96Xk8/v0/9H5hEW8t3ENmfrFdarNYrDz89WbW7z+Gt6sjH4/uSkAdXJxdRETEXhRKiYiIiNRiby2ydUnd0CWCMB83O1djP8Herkwa0pJVky/jiX+1ItzHlfTcYl5ZsJse0xcx9adtJGXk12hNry7YzS9bknE0Gnjv9s7EBteNBdlFRERqC4VSIiIiwMw/od1j4D3W9uoxBX7fdPJ4YTHc9wkE3A2ed8B1r8ORrHPPabXCU99A2H3gNhoGPg97Uk4e35cGYz+A6Adtx2MmwpRvoNhU5ZcnddS6fRmsiD+Kk4OB8Q2wS6o8ni6O3NmnKUsf7c/rN3WgVZg3BSVmZq/cx6UvLeb+uRv45+B5/uOsAl+vS+LtxfEAPD+8LT1jA6v9PUVEROobhVIiIiJAhD/MuBnWPwfrnoUBreHqV2HbQdvxiZ/Bzxvh6wdg6ZNw+BgMf+3cc774C7w5H94bA6uftj2dc/AMW8AFsPMwWKzw/h2w7UV47XbbwxEe/7J6r1XqjjcX2UKP6ztHEOHnbudqahcnByPXdGzEbw/05r9ju9GnWSAWK/yyJZlhby/nlg/+ZvGuVKxWa5W/98qEdB7//h8A7usfw41dIs9zhoiIiJRHC52LiIgAwzqV3X7uRlv31N/xtsBq1hKYe58trALbgwZaPQJ/74FLmp05n9UKr8+DJ66Bq7vY9s0ZDyH3wg/r4eYeMKS97XVC02DYlWx735dvq46rlLpk44FjLNudhoPRwL39Yu1dTq1lMBjo0yyIPs2C2HY4iw+X7eXnLcms2nuUVXuP0iLEi7v6NuWq9uE4O17872PjU3O557/rKTFbubJdGP8Z1KIKrkJERKRhUqeUiIjIacwW+GIV5BVBj1hYnwglZhjY5uSYluHQOABWxZc/R2IapGTCwNYn9/m4Q/cYWLXn7O+dlQ/+WpZGgLeOd0kN79iISH91SVVE63AfXr+5I8se7c/Y3tF4ODuw60gOD3+9mb4vLub9pQlkF5ZUev6juUXcMXst2YUmOjX25eUb2mM0GqrwCkRERBoWhVIiIiLH/XPAtl6Uyyi452P4fiLERdjCJWdH8PUoOz7Ex3asPCf2h/hU/Jz4FHjrD7h7QOWvQeqHLQczWbQzFaMB7uuvLqkL1cjXjSevjGPlY5fx6JAWBHm5kJJdyPTfd9Jr+iKe/20HyVkFFzRnYYmZcf9dz4GMfCL93fhwZBdcnRyq6QpEREQaBt2+JyIiclyLcNj0PGQVwDerYdR7sPSJmnnvQxkw5EW4oTvcpVCqwXtzoa1L6poOjYgK9DjPaDkbH3cn7u0Xy9je0fy48TAf/LWX+NRcPli2l4+XJ3JVh3DG9W1Ky1DvM841W6ysScwgNaeQIE8X5q45wPr9x/B2deST0V0J8HSxwxWJiIjULwqlREREjnN2hNhQ29edo2HtXnhjPtx0ie2JeJl5ZbuljmRBqG/5c53YfyQLwvzKntOhSdmxh49B/+egZzP4YGxVXY3UVVsPZfHnjiMYDHDfAHVJVQUXRwdu7BrJ9Z0jWLwrlfeX7WVNYgbfbTjEdxsOcWnzIO7u25QeMQEYDAbmbU1m2s/bSc4qLDOP0QDv3d6Z2GAvO12JiIhI/aJQSkRE5CwsVigqsQVUTg6wcBtc1812bNdhOHDUtuZUeaKDbMHUwm3QIcq2LzsfVifA+IEnxx3KsAVSnaNti6cbdWN9g/f28bWkhrULJyZIC4xVJaPRwGWtQrisVQibkjL5YFkC87amsHR3Gkt3p9GmkTfdowP4eHki5T2zz2LlotakEhERkbIUSomIiACTv4Ch7aFxIOQUwNyVsGQHzJ9kW6B8bD946DPw9wBvd5jwKfRoVvbJey0fhuk3wbVdwWCAB4fAsz9As1BbSPXkNxDuC9d0to0/lAH9noUmgfDyrZCWfXKus3VgSf22MyWbedtSMBhggrqkqlWHSF/eva0z+4/m8dFfiXy9Pomth7LZeij7rOcYgGk/b2dQXCgOWuBcRETkolU6lDpw4AD79+8nPz+foKAgWrdujYuL7q0XEZG6KTUbRr4HyZm2EKpdpC2QGtTWdvy122237lz3BhSZYHBbeHdM2Tl2JduennfCo1fanuA3bhZk5kPv5jBvErg6244v+Afij9heERPKzmX9X7VdqtRiJ564d0XbMJqF6BaxmtAkwINnrmnDxEHNee7X7Xy74dBZx1qB5KxC1iRm0CMmoOaKFBERqacuKJTat28fM2fO5IsvvuDgwYNYrScbm52dnenTpw/jxo3juuuuw6j7D0REpA6ZNe7cx12d4Z0xttfZnB4kGQzw9PW2V3lGX2p7iQDsOZLDb/8kA+qSsgd/D2f6Ng86Zyh1QmpO4XnHiIiIyPlVODl64IEHaN++PYmJiTz77LNs376drKwsiouLSUlJ4bfffqN379489dRTtGvXjrVr11Zn3SIiIiL1yluL4rFaYUjr0HKfBifVL9jLtUrHiYiIyLlVuFPKw8ODvXv3EhBwZqtycHAwAwYMYMCAAUyZMoV58+aRlJRE165dq7RYERERkfooPjWXn7ccBmDCZeqSspdu0f6E+biSklVY7kLnBiDUx5Vu0f41XZqIiEi9VOFQavr06RWedMiQIZUqRkRERKQhenexrUtqYKsQWof72LucBsvBaGDKsDjGf7YBA5QJpk4saz5lWJwWORcREakilVr4qaCggPz8kyu57t+/n9dff5358+dXWWEiIiIiDcG+9Dx+2GRbx+jflzU7z2ipbkPahDHz9k6E+pS9RS/Ux5WZt3diSJswO1UmIiJS/1Tq6XtXX301w4cP55577iEzM5Pu3bvj5OREeno6r776KuPHj6/qOkVERETqpXcWx2OxwoCWwbSNUJdUbTCkTRiD4kJZk5hBak4hwV62W/bUISUiIlK1KtUptWHDBvr06QPAN998Q0hICPv372fOnDm8+eabVVqgiIiISH2VlJHPdxttXVJ64l7t4mA00CMmgKs7NKJHTIACKRERkWpQqVAqPz8fLy8vAP744w+GDx+O0WjkkksuYf/+/VVaoIiIiEh99e6SeMwWK32bB9GxsZ+9yxERERGpUZUKpWJjY/nhhx9ISkpi/vz5XH755QCkpqbi7a1HGIuIiIicz8Fj+Xy97iAA/9YT90RERKQBqlQo9dRTT/Hwww8TFRVF9+7d6dGjB2DrmurYsWOVFigiIiJSH81ckoDJYqVXbACdm/jbuxwRERGRGlephc6vv/56evfuTXJyMu3bty/df9lll3HttddWWXEiIiIi9dHhzAK+WpcEwAMD9MQ9ERERaZgqFUoBhIaGEhoaWmZft27dLrogERERkfru/aUJlJitXNLUn+5NA+xdjoiIiIhdVCqUysvLY8aMGSxcuJDU1FQsFkuZ43v37q2S4kRERETqmyPZhXy+9niX1GXqkhIREZGGq1Kh1J133snSpUsZMWIEYWFhGAx6RK6IiIhIRby/dC/FJgtdo/zooS4pERERqQfmbU3m9T/3MO/Bvhd0XqVCqd9//51ff/2VXr16VeZ0ERERkQYpNaeQ/63eD9i6pPSLPREREakr/rd6P8v3pOPkYGRMryg6NvZjZXw6z/66g8T0PIZ3anTBc1YqlPLz88PfX0+JEREREbkQHy7bS5HJQsfGvvSODbR3OSIiIiIV8u6SeF5bsJuWod4kpOWyYPsR7h8Qy+yV+xjTK4rbujXBx93pguc1VqaYZ555hqeeeor8/PzKnC4iIiLS4KTnFvHZ3wcAdUmJiIhI3fLNuoNMH96Onyf0ZvaYbhSazKzff4ylj/Tj3n6xlQqkoJKdUq+88goJCQmEhIQQFRWFk1PZN9+wYUOlihERERGprz76K5GCEjPtInzo1zzI3uWIiIhIHTBn1T7eX7qXtNwiWoV5M+2q1nSI9C137LytybyzOIF9R/Mwma1EBXpwV59ohneKKHf849//w9zVB3jyyjjG9o4+Zx2HMgvoGWNbC7NbtD9ORiMTBzbH3blSsVKpSp19zTXXXNSbioiIiDQkGXnFzFm1D4B/q0tKREREKuDnzYd59pcdPHttGzpG+vLxikRGzlrNoof7EejpcsZ4Hzdn7usfS2ywB04ORhbuSOWRb7YQ4OnCpaf9Qmze1hQ2HsgkxPvMecpTbLbg4njyZjsnBwO+leyOOlWlQqkpU6Zc9BuLiIiINBQfL08kv9hM63BvBrQMtnc5IiIiUgd8tDyRm7tFcmOXSACeu6Yti3am8tW6JO7tF3vG+B4xZZ/qe0fvaL7dcJB1+zLKhFIpWYVM/Wkbc8Z2Y8wnaytczysLduPm5ABAidnKW4v24OVaNph68sq4Cs8HlQylRERERKRisvJLmL1yH6C1pERERBq6nJwcsrOzS7ddXFxwcTmzW6nYZGHroSzu7RdTus9oNNArNpAN+zPP+z5Wq5WVCUfZm5bHY0NPPqjOYrEy8ctNjOvblOYhXhWuu1uUP3vTcku3OzXx5UBG2XXGDVz4zzgVDqX8/f3ZvXs3gYGB+Pn5nfMHqoyMjAsuRERERKQ+mrUikdwiEy1DvRjUKsTe5YiIiIgdxcWV7SSaMmUKU6dOPWPcsfxizBbrGbfpBXm6kJCWd9b5swtLuOT5hRSbLBiNBp69ug19mp3skpq5NAFHBwNjekVdUN1f3t3jgsZXVIVDqddeew0vL1uK9vrrr1dLMSIiIiL1SVZBCZ+sSARsXVJGo7qkREREGrLt27fTqFGj0u3yuqQuhqezI7890Ie8YhMr44/yzK/bifR3p0dMAP8czOKTFfv49YHeF925nZFXDIC/h/NFzVPhUGrUqFHlfi0iIiIi5ft05T5yCk00D/FkSOtQe5cjIiIidubl5YW3t/d5x/m5O+NgNJCeW1Rmf1puEUHlLHJ+gtFoICrQA4DW4T7Ep+by7pJ4esQEsGZfBkfziug5Y1HpeLPFynO/bufj5YmseGzAOWvKKijh5fm7+GXLYbIKSgDwcXNiWPtw/nN5C3zcLnzh84taUyo1NZXU1FQsFkuZ/e3atbuYaUVERETqvJzCEmYtt3VJTRigLikRERGpOGdHI20a+bAyPp3Bx3+xZbFYWRl/lJE9m1R4HovVSrHJltkM79iI3rGBZY6P/Hg113aM4IYuEeecJzO/mOHvriQlu5CrOzQiNtgTgPjUHL5Zf5AV8el8N74XPhf4RL5KhVLr169n1KhR7NixA6vVWuaYwWDAbDZXZloRERGRemPOqv1kFZQQE+TBFW3D7F2OiIiI1DF39o7mP19vpm2ELx0ifZi1fB/5xSZu6Gx7Gt9DX24ixMeVSUNaAvDO4njaRfjQxN+DYrOZxTvT+H7jIZ69pg0Afh7O+J12u52j0UiQlwsxQZ7nrOWNhXtwcjCy9JH+BHmV7dSaOKg5I2et4Y2Fe3hqWA08fe+OO+6gefPmzJo1i5CQED1FRkREROQUeUUmPvprL2DrknJQl5SIiIhcoGHtw8nIK+a1BbtJyymiVbg3n97RrTQUOpRZUCaPKSg28+QPW0nOKsTVyYGYIA9eu6kDw9qHX3Qtf2w7wvPD254RSAEEe7ny2NCW/N/3W2smlNq7dy/ffvstsbGxlTldREREpF777O/9HMsvITrQgyvbqUtKREREKmdUzyhG9Ywq99jpT8R7eHALHh7c4oLmP986Uiek5RTRPOTs3VQtQr1IO239q4owXvAZwGWXXcbmzZsrc6qIiIhIvZZfbOKDZbYuqfv6x+LoUKkft0RERERqDT8PJw4eKzjr8aSMAnxraqHzjz76iFGjRrF161batGmDk1PZN77qqqsqM62IiIhInTd39QGO5hXT2N+dqztcfLu8iIiIiL31bRbES/N38dnY7jg7lv2FW5HJzCt/7OLS5kEXPG+lQqlVq1axYsUKfv/99zOOaaFzERERaagKS8y8t/REl1QMTuqSEhERkXrgocubM+ytFfR/eQkjejQhJsgTq9VKfFoun63aT7HZwms3dbjgeSsVSk2YMIHbb7+dJ598kpCQkMpMISIiIlLvfL7mAOm5RTTydWN4p3M/WllERESkrgjzceP7e3vyxA9beXHeTqzH9xuA3s2CePqq1oT7ul3wvJUKpY4ePcrEiRMVSImIiIgcZ+uSSgBsa0mpS0pERETqk0h/dz69oxtZ+SUkHs0DICrAHV9350rPWalQavjw4SxevJiYmJhKv7GIiEh9Y7YaWZPXmtQSP4KdjtHNYxsOBou9y5Ia8vW6JI5kFxHu48p1nRvZuxwRERGRauHj7kQHd98y+6xWK0fzign0dLmguSoVSjVv3pzJkyezfPly2rZte8ZC5w888EBlphUREamz5mX1YNrhcSSXnFzgMcwpjSnhHzDEZ5UdK5OaUGQy8+4SW5fU+H4xuDg62LkiERERkarT8snfWTFpAAHHQ6cxn6zhhevaEeztCkB6bjHdn/+TvdP/dUHzVvrpe56enixdupSlS5eWOWYwGBRKiYhIgzIvqwfj9z9eem/9CSklAYzf/zgzmzyvYKqe+2b9QZKzCgnxduGGLpH2LkdERESkShWZLGV+1l2TmEFhSdk7Ak7/WbgiKhVKJSYmVuY0ERGResdsNTLt8Ljj/xM2lDlmxYgBC9MOj2OQ92rdyldPFZssvLvY1iV1z6UxuDqpS0pEREQaHsP5h5xBK3CKiIhchDV5rY/fslf+/4atGEkuCWJNXuuaLUxqzPcbD3Ios4AgLxdu6dbY3uWIiIiI1BkVDqVmzJhBQUFBhcauXr2aX3/9tdJFiYiI1BWpJX5VOk7qlhKzhbcXxwNwd9+m6pISERGReslA2V/BGgwGDJVpjTpNhW/f2759O40bN+aGG25g2LBhdOnShaAg22KuJpOJ7du3s3z5cj777DMOHz7MnDlzLr46ERGRWi7Y6ViVjpO65cdNh0nKKCDQ05nbujexdzkiIiIi1cIK9H95CYbjSVResYkr3vwL4/Ftq7UyK0pdQCg1Z84cNm/ezNtvv82tt95KdnY2Dg4OuLi4kJ+fD0DHjh258847GT16NK6urpUqSEREpC7p5rGNMKc0kksCKf8WPithTul089hW06VJNTOZLbxzvEvqrj5NcXNWl5SIiIjUTy9d375a5r2ghc7bt2/Phx9+yPvvv8+WLVvYv38/BQUFBAYG0qFDBwIDA6ulSBERkdrKwWBhSvgH3LP/8XKOWgED7d12aZHzeuiXLckkpufh5+7E7ZeoS0pERETqr0g/Nzo38cPRoWqXJq/U0/eMRiMdOnSgQ4cOVVqMiIhIXdTEORlbl5SVU7ulfBxyyTJ7MS+7N7PTr2R04C/2KlGqmNli5c1FewC4s09TPFwq9SOViNQRO55uae8S7K7VrfauQETs6ZYP/2bN/w0k0NOlSufVT1AiIiIXaVb61QBc4f0XIwJ/J7XEj2CnY3Tz2Ma7qTfwypERTDs8jkDHTK70XW7naqUq/PpPMnvT8vBxc2JkD3VJiYiISP1WuRWjzk+hlIiIyEVILfHjx8x+AIwL/oEO7rvLHL8/+EvSTH7MOXolDyX9B3/HbHp6brFDpVJVLBYrby083iXVOxovVyc7VyQiIiJS/argYXtnUCglIiJyEeYc/RclVie6uG87I5ACMBhgSvgHpJt8+S2rN+P2PcEXMY/Rxm2vHaqVqjBvWwp7UnPxcnVkVK8oe5cjIiIiUiMe/nozzo7nXlPq/RFdLmjOql2hSkREpAHJt7jw2dErALgz6IezjnMwWHg18hW6e/xDrsWd0YnTOFAUUkNVSlWyWKy8ebxL6o5e0XirS0pEREQaCA8XR7xcnc75ulAX1SkVHx9PQkICffv2xc3NDavVisFQHQ1dIiIitc+3GZeRafamifNhBnmvPudYV2MJH0Y9w00JM9hR2JSRic/wTewjBDpm1VC1UhUW7DjCzpQcPF0cuaNXtL3LEREREakxU69qXeULnVeqU+ro0aMMHDiQ5s2bc8UVV5CcnAzA2LFj+c9//lOlBYqIiNRGFquBWenXAHBH4E84GCznPcfbIZ9Po6cQ4ZTCvuJwxiROJdfsVs2VSlWxWk92SY3uGYWPu7qkREREpGGorvajSoVSEydOxNHRkQMHDuDu7l66/6abbmLevHkVnmfZsmUMGzaM8PBwDAYDP/zwQ5njVquVp556irCwMNzc3Bg4cCB79uwpMyYjI4PbbrsNb29vfH19GTt2LLm5uZW5LBERkQr7M7sb+4rD8XHI4Qb/BRU+L9jpGHOaPoW/Qxb/FDRj/P7JFFu0xGNdsHBHKtsOZ+Pu7MDY3uqSEhERkYajup6+V6lQ6o8//uCFF14gIiKizP5mzZqxf//+Cs+Tl5dH+/bteeedd8o9/uKLL/Lmm2/y3nvvsXr1ajw8PBg8eDCFhYWlY2677Ta2bdvGggUL+OWXX1i2bBnjxo2rzGWJiIhU2EfHu6Ru9Z+Hu7Hogs5t6nKYj6On4WYo5K/cTjxy8EEsVt3+XptZrVbeXGT7xdjIHlH4eTjbuSIRERGRmvP5XZfg61b1XeKVCqXy8vLKdEidkJGRgYtLxe8vHDp0KM8++yzXXnvtGcesViuvv/46TzzxBFdffTXt2rVjzpw5HD58uLSjaseOHcybN4+PPvqI7t2707t3b9566y2++OILDh8+XJlLExEROa8t+bGsyWuLk6GE0YE/V2qODu67mRn1PI6Y+DGzH88n31HFVUpVWrI7jS0Hs3BzcuCuPuqSEpHyrcvP596DSVwaH0/crp38mZNT5rjVauWt9DT6xu+h4+5d3JF0gH3FxWXG3HfwIAMS4umwexd94/cwKfkwqaaSCr2/1Wpl3MGkct/7cEkJ9xxMotPuXfSO38NLqamYrNXV+yAi9c17SxPILzGXbr+7JJ6sgpN/Nx3LK2bgq0sveN5KhVJ9+vRhzpw5pdsGgwGLxcKLL75I//79KzPlGRITE0lJSWHgwIGl+3x8fOjevTurVq0CYNWqVfj6+tKly8lHDg4cOBCj0cjq1WdfcLaoqIjs7OzSV85pf2GLiIicy4kuqWE+fxHilFHpefp5beDFyDeOz3ktH6Sd+UsasT+r1cobf9q6pEb0aEJAFS/wKSL1R77FQgsXV54MKf8Jq7MyMvjs2DGmhITyReMmuBmNjDuYRJHl5LqE3dzdeS28Eb9GN+WNRo1IKi7hwUMV+4X7nGPHyl33xWy1Mv7gQUqsVv7XuAnPh4bxQ3YWb6WnV+YyRaQBWrY7jWLTyb+r3l2cQFb+yVDKZLGyN+3Cl1Kq1CIWL774Ipdddhnr1q2juLiYRx99lG3btpGRkcGKFSsqM+UZUlJSAAg57S/0kJCQ0mMpKSkEBweXOe7o6Ii/v3/pmPJMnz6dadOmVUmdIiLSsBwqDuLXzD4AjA36/qLnG+63mHSTL88nj+X55LEEOmYy3G/xRc8rVWd5fDqbkjJxcTRyV5+m9i5HRGqxvp6e9PX0LPeY1WplzrEM7g4I4DIvLwBmhIbRJyGehbm5XOHtDcAof//Scxo5OXFngD8TDh2ixGrF6RxPOt9RWMjsYxl81SSKSxPiyxxbkZdHQnERsyJjCXR0pBUwITCQV9PSuC8wEGc9QV1EzuP0vkprFXVaVqpTqk2bNuzevZvevXtz9dVXk5eXx/Dhw9m4cSMxMTFVUlh1mjx5MllZWaWv7du327skERGpIz5NvxIzDvT03Exrt8QqmXNc0PfcGWgLuB5N+jdLcjpVybxy8U7tkrqtexOCvNQlJSKVc7CkhHSzmR7uHqX7vBwcaOfqyqaCgnLPyTSb+SU7m45ubucMpAosFh5JPswTwSEEOZ7Zd7C5sIBmLi4EnnKst7sHuRYL8UUXti6iiEhVqvTjfnx8fPi///u/qqyljNDQUACOHDlCWFhY6f4jR47QoUOH0jGpqallzjOZTGRkZJSeXx4XF5cya19lZ2dXYeUiIlJf5Zjd+DxjCAB3BV58l9SpHg/7mHSTLz9k9mf8vsf5POZxOrjvrtL3kAu3au9R1u0/hrOjkbsvVZeUiFReutkEUCYYAghwdCw9dsIraanMPXaMAquV9q6uzIyIPOfcM1JT6ejmVtqBdcZ7m0wEOpz5vieOiYicj+H4q8y+KmiyrHQoVVhYyJYtW0hNTcVyyj3QAFddddVFFxYdHU1oaCgLFy4sDaGys7NZvXo148ePB6BHjx5kZmayfv16OnfuDMCiRYuwWCx07979omsQERE51VcZg8ixeBDjksSlXuurdG6jwfr/7N13dFTV3sbx70x6L6SQkEBC6B1BOoiKYEMQFMtVUcRyRb3K1Yu+4hVsWK69YUFFBTugWFBBKdKk95YQekglvU55/xgIjEmoSU4meT5rzQrnzJ59njOamclv9t6HF2JeI9MSxJL88xiT/ATftHiY5l66cIeRjo2SuuH8WCIDvQ1OIyINxZiQUEYGBXOorIy3MzN4JOUQ7zSJwVTJX4C/5+exsrCAb+N0EQYRqTl24KGvN+Dp7phwV2Kx8X+zN+Hr6QbgtN7UmTirotS8efO45ZZbyKhkYTyTyYTVaq3kURXl5+eTmHh8vnNycjLr168nNDSUpk2b8sADD/D000/TsmVL4uPjefzxx4mOjmb48OEAtG3blksvvZQ77riDqVOnUlZWxr333sv1119PdHT02ZyaiIhIpSx2Mx9mDANgbNgczKbqv2KRp9nCO82mcOPuZ9hY1Ipbdj/JrBYPE+FxpNqPJae2YncmK5Oz8HQzc/fAur88gYjUbcdGKmVYLE5T7DItFtp4ORe9Q9zdCQHiPD1p7unJRbuT2FBcTBcfnwr9riwsZH9ZGb12OY+ufeDQQbr5+DC9aTPC3N3ZWFzsdH+mpfKRWyIilRnRNcZpZNTwrk0qtjkv5oz7PatXoPvuu49rr72W//73vxUWIj8Tq1evdrpa3/jx4wEYPXo0H3/8Mf/5z38oKCjgzjvvJDs7m379+jFv3jy8vY+/aM+YMYN7772Xiy++GLPZzMiRI3n99dfPOpOIiEhl5uX04WBZJI3csrm6Bhci93cr4sP4yVyT+CJ7SqO5JflJvkqYQKBbYY0dUyr3xu+OUVKjzo8hKqjiH4IiImcixsODMDc3VhQW0Pbo3zP5Visbi4u5PjikyscdG3tQaq98FMLY0EZcExTstG/YnmQmRERwoZ9j0fXO3j68m5lJpsVSPm1vWWEh/mYzLTw9z+3ERKRBeGlU5xrp96yKUqmpqYwfP/6cClIAAwcOPOmK7SaTiSeffJInn3yyyjahoaHMnDnznHKIiIicjN0O76dfDcBNjX7C21xao8cLc8/hk/jHGZH0P7YXx3PHnseZHv9fvM1lp36wVIstoXEsTczEw83EPwe2MDqOiLiIApuNfaXH3yMOlpWxrbiYIDc3oj08uCUklHczM2nm6UmMhwevZ2QQ4e7OxUev2LehqIjNxcWc5+NDoJsb+0tLeSMjg1gPD7p4O4rjqWVljDmwnymNo+jk40O4u3uli5tHuXsQc7Tg1NfPjwRPLx5JSeHf4eFkWC28npHODcEheJrP6tpXItLAPPz1hlO2MZnghWvOrHh1VkWpa665hoULF7rElfZERETO1ZrCtmwoao2nqZSbw36slWM29Url4/gnuD7pOVYWdOTB/Q/xZtPncTOd3Xx9OTOftx4EwDXdYmgSrFFS4rq2PdnG6AiGa3tj7R1rS3ERt+7fX779fLrjokzDAwN5Niqa20NDKbLbeOLwYfJsNs7z8eG9mFi8jhaGfMwm5ufn8WZGOkV2O+Hu7vTz9ePlRtHlxSMLdpJLSymuYuRUZdxMJt6OieHJ1MPcuG8vPmYzwwKDuC8srBrPXkTqs2/WHqBJsA/towM5ydiiM3ZWRak333yTa6+9liVLltCxY0c8PDyc7r///vurJZyIiEhd8EH6cABGhPxBmHtOrR23g89u3ot7mluTJ/NzTl8mHbqTJ6OnVsuVTqRq20OasiayDW5mE/dolJSInIEevn5sbV11IdBkMnFfWDj3hYVXen8rL28+im160mM08fA86TGASu9v4uHBu6e4ip+ISFVu6tmM7zccYn9WEdd2j+Hqrk0I9j336b9nVZT6/PPP+fXXX/H29mbhwoVOV4EwmUwqSomISL2xt6Qxv+T2BuD2sDm1fvw+/ht5OfYl7tv3Hz7NvJII9yPcF/llredoSGa2vgSAEV2bEBvqa3AaEREREeM9NbwDE69sy7zNh/l69QFemLeDi9pEMOr8WAa0DKv06qCn46yKUo899hiTJ0/mkUcewaw5yCIiUo99lHEVdswMDFhNS+/9p35ADbgy+E8yLMFMOnQ3L6XeTLj7Ea5v9KshWeq7ncExrGrcFrPNyrgLNUpKRERE5BgvdzeGdWnCsC5NOHCkkG/WHODxOZux2uz8+uAA/LzOvMR0VhWl0tJSrrvuOhWkRESkXsux+PHVEceomTvCZhua5dawHxgX4Rgh9X8Hx/FrTk9D89RXx9aSuvDAOuLC/AxOIyIiIlI3mU0mTJiwY8dqO/tFps6qqjR69Gi+/FJTB0REpH6bmXUphTYf2ngn08f/1FccqWkPRX7KqJBfseHGffv+w+qCtkZHqlcSg6JZEdUBs93G9TsXGB1HREREpE4psVj5bv1BbvpgJRf+byE7UnN58qoOLHvkorMaJQVnOX3ParXywgsv8Msvv9CpU6cKC52//PLLZxVGRESkrii1ufNxxlUA3BE+u04sLm4ywbMxb5JpCWZBXg/G7HmCbxL+QyvvfUZHqxe+ODpKasCB9cTkpxucRkRERKTumDhnE3M3pBAV5M2o7rG8fkNXQv0MWuh806ZNdO3aFYDNmzc73Xe2i1uJiIjUJT/k9CfV0ogI90yGBi02Ok45d5ONN5s9zz92P83awraMTp7MtwkPE+2ZYXQ0l5Yc2Jil0Z0waZSUiIiISAUzVu4jOsiHpqG+rEzOZGVyZqXt3r25+xn1e1ZFqT/++ONsHiYiIuIS7Hb4IH04AKPDfsDTbDE20N/4mEuYFvck1yY9T2JJU25JfpJvEv5DsHu+0dFc1hetHKOk+h/cSLO8VIPTiIiIiNQtI7rG1MjMgbOb9CciIlKPLS/oxNbiBHxMxfwj9Gej41QqxD2PT+L/y4ik/5FY0pQxe55gRvOJ+JhLjI7mcvYGRLKkSScArt853+A0InVHgc3K6xkZzM/LI8tqpa2XN49GRNDRxweAdju2V/q4f4eHc3too0rvG5SUyCFLxUL/DcHBPB7ZmINlpVyye3elj305OppLAwLP8mxERORcvDSqc430e9pFqREjRvDxxx8TGBjIiBEjTtp21qxZ5xxMRETEKMdGSV0bOr9Ojz6K9szgk/j/ck3SC6wtbMu9eyfwbtzT+sbpDH3R6mLsJjN9D20kPvew0XFE6ozHDx9mV0kJz0dFE+7uztzcHG4/sJ+5cfFEeniwKKGFU/slBfk8fvgwg/0Dquzzq2ZxWE/Y3lVSwtgD+xkS4HhMY/eK/X6dnc2HWVn09/OvtnMTEZG64bQ/twYFBZWvFxUUFFRjgURERIyUWBzD73k9MGFjTNh3Rsc5pVbe+5gW9yQ37X6KBXk9+L8D9/K83a41Hk/Tfv9wFsV0AeCGHRolJXJMsc3Gb3l5vNkkhu6+vgDcGxbOwvx8vsjO5l/h4YS7O/8p8Xt+Pj18fYn1rHrh29C/PeaDrExiPTw438dxDDeTqUK/8/PzuDQwAD/zWV04XERE6rDTLkp99NFHPPnkkzz00EN89NFHNZlJRETEMNMyhgFwSeBK4rxSDE5zes7328obTV/g7r3/x1dHBhPx604eGtLa6Fgu4cujo6R6p2wmIeeQ0XFE6gwrdqyA598K3N5mM2uLCiu0z7BYWJyfz7NRUad9jFK7nbm5uYwOCamykL6luJjtJSU8Hhl5RvlFRMQ1nNHXDZMnTyY/v+5OYxARETkXmZZAZh25CIA7wmcbnObMDA5ayTNN3gbgzT8Smb5sj7GBXMAhv0b8EXseADds/83gNCJ1i5/ZjS7ePkzNzCDNUobVbuf7nBzWFxWRbrFWaP9dTg6+ZjOXnGTq3t8tyMsjz2rl6pPMwvg2J5vmnp50PTqSSkRE6pczKkrZ7faayiEiImK4zzIvp8TuRWefnXT33Wp0nDN2Q6NfGB/5GQCT5m7hh40a+XMyX7S6GJvJTI/DW2mZc9DoOCJ1znNRUdiBgUlJdNm5gxnZR7g8IBBzJYOaZuXmcGVgIF5nMMVuVk4O/f38iHD3qPT+YpuNH3NzGamlQ0RE6q0znpitNSpERKQ+KrZ58GnmFQCMDZ9dI5e8rQ33RXzBzb2aYbfD+C83sCwpw+hIddJh31AWxHYDtJaUSFWaenrySdNmrG7Zit8TWvBlszgs2InxcC4irS4sJLm0lGuCgk+774NlZSwvLGDkSR7za14eRTYbwwJVlBIRqa/O+AI9rVq1OmVhKisr66wDiYiInIttT7Y5q8f90qwHGV1DiCg8QvwbmWyzn10/dcGkre3JyC/h582HufOTNXx5Vy/aR+uPuhN92eoibGY3uqVup82RfUbHEanTfM1mfM1mcqxWlhYU8O/wCKf7Z+Vk097Lmzbe3qfd5+ycbELd3LjAv+or6n2bk81F/gEVFkcXEZH644xf4SdPnqyr74mISL1iB2YlDABgWNIS3Ow2YwOdIzeziVeu60JWwV+sTM7i1o9WMeuffYgN1ZosAKk+IfzW9HwA/qG1pESq9GdBPnYg3sOTfWVlvJieRrynp9MaUPlWK7/k5fFwRESlfdy2fx+D/AP4R0hI+T6b3c7snByGBwXhXsWX3XtLS1ldVMTUmJhqPScREalbzrgodf311xNRxZuOiIiIK1od0YZ9gY3xKStmyN6/jI5TLbw93Hjvlu5c9+5yth/O4+ZpK/nmn30I8/cyOprhvmp1IVazG13TdtL2yF6j44jUWXlWG69mpHPYYiHIbGZwQAD/CgvH44RC0k95ediBKwICK+1jf2kpR6wWp33LCwtJsVgYcZKpe7Nycoh0d6evr191nIqIiNRRZ1SU0npSIiJSH81u4RglddnelfhZig1OU32CfDyYPqYHI95exp7MQsZ8vIrP7+iFn1fDnQqT7hPEr816AHDjDo2SEjmZywIDuSyw8mLTMaOCgxkVHFzl/fMTWlTY19fPj62tTz5F+sHwcB4MDz+tnCIi4rp09T0REWnQdgdGsS6iFWablauS/jQ6TrWLDPTmk9t7EOLrwcYDOdz92RpKLa49PfFcfN3yQixmdzqlJ9IhM9noOCIiIiIN2hkVpWw2m6buiYhIvXJslFS/QxuJLDpicJqakRDuz4e3no+PhxtLdmXwn282YLM1vC+aMr0DmdesJ6BRUiIiIiJ1wRkVpUREROqTTO9AFsZ0BWBE4mKD09Ssrk1DePum83A3m5iz/hBTft5mdKRa902LgZS5edA+YzedMpKMjiMiIiLS4KkoJSIiDdbc+D5YzO60z9hN6+z9RsepcRe2juD5kZ0AeH9JMu8tbjiFmSyvAH6K7w04RklplUwRERER4zXclU5FRKRBK3bz5Mf4PgCMSFoEQIHNyusZGczPyyPLaqWtlzePRkTQ0ceHMrud1zPSWZxfwIGyUvzNbvT282V8eDgR7h5VHmdQUiKHLJYK+28IDubxyMZO++x2O3cdPMCfBQW8Ht2EQQEB1XjGDiO7xZCRX8KUn7fz7E/bCfP3YsR59f+S69+2uIBSNw/aZu6ha/ouo+OIiIiICCpKiYhIAzW/aXfyPX2Jys+gZ8pWAB4/fJhdJSU8HxVNuLs7c3NzuP3AfubGxeNrNrO1uJi7GzWijbcXuVYbz6alMu7AQb6Oi6vyOF81i8N6wvaukhLGHtjPkEoKTp8cOVIrI3juHNCctLwSpv2ZzH++2UionycDW9ffNSOzPf3LC5AaJSUiIiJSd2j6noiINDhWTMxO6A/A1UmLccNOsc3Gb3l5PBQeQXdfX5p5enJvWDhNPTz4IjubADc3psU25bLAQOI9vejs48PEiEi2lBRzqKysymOFursTfsJtUUE+sR4enO/j69RuW3ExHx/J4unGUTV67gAmk4nHLm/LsC7RWGx27pmxlg37s2v8uEaZ1WIAJe6etDqyj25pO4yOIyIiIiJHqSglIiINzsqodhzyD8e/tJBL9q0GwIodK+Bpch5H4202s7aosNJ+8mw2TECg+fTeTkvtdubm5jIiKAjTCccpstl4OOUQEyMiCXevnUHMZrOJF6/pTP+WYRSWWrnt41XsTs+vlWPXphxPX+Y27wvAjds1SkpERESkLlFRSkREGpzZCRcAcEXycrytpQD4md3o4u3D1MwM0ixlWO12vs/JYX1REekWa4U+Smw2Xk5P4/KAQPzd3E7ruAvy8sizWrk6KMhp/3NpaXT18eHiGlhD6mQ83c28c1M3OjYJIquglFs+/Iu03OJazVDT5iQMoNjdi4TsA/RIbXhXHBQRERGpy1SUEhGRBmVHcCybw5rjbrMwNHmp033PRUVhBwYmJdFl5w5mZB/h8oBAzH8bXlNmtzP+0CHswBORkad97Fk5OfT383NaGP33/DxWFhbwSMTp91Od/L3c+ei284lr5MuBI0WM/mgVucVVT0d0JXkePnzXvB+gtaRERERE6iIVpUREpEGZ3WIAABccWE+j4lyn+5p6evJJ02asbtmK3xNa8GWzOCzYifE4XkRyFKQOcshSxrTYpqc9SupgWRnLCwsYGRTstH9lYSH7y8rotWsnHXdsp+OO7QA8cOggo/ftPYczPX1h/l58MqYnYf5ebEvJ5c5PVlNqdv1rocxJ6E+RhzfNcw7SO2WL0XFERERE5G9UlBIRkQYjzSeYJdGdABiRuKjKdr5mM+Hu7uRYrSwtKOAif8e0umMFqb2lpUyLiSX4NAtSALNzsgl1c+MCf3+n/WNDGzEnLp5ZJ9wAJkRE8EwtLHp+TNNGvnx82/n4e7mzYncWL3a7AasLjy3K9/Dmu6OL2d+wY74Ln4mIiIhI/aWilIiINBjfNe+HzexGl7SdNM9NqXD/nwX5LCnI50BpKcsKCrh1/z7iPT25OiiIMrudBw4dZEtxMS9ERWMF0i0W0i0WSu328j5u27+PGUeOOPVrs9uZnZPD8KAg3P+2kHq4uzstvbycbgBR7h7EeHpW/5NwEh2aBPHezd3wcDPxZ5POvNtpOPZTP6xO+r55Pwo8fGiWe5g+hzYbHUdEREREKuH6Y/NFREROQ4G7F/PiegIwInFxpW3yrDZezUjnsMVCkNnM4IAA/hUWjofJxMGyUv7Id1ydbsTePU6P+zg2lh6+fgDsLy3liNXidP/ywkJSLBZG/G3qXl3Up0UYL4/qwv0z1zC3eV9CinO5YecCo2OdkQJ3L2YnOKZp3rDjN8wuW1oTERERqd9UlBIRkQbh12Y9KPTwITY3lW5pOyptc1lgIJcFBlZ6XxMPT7a2bnPK48xPaFFhX18/v9N67DFn0rYmDO0czZZH/8vUTlfzSbvLCCnJ49K9fxma6Uz8EN+XfE9fYvNS6Xdwo9FxRERERM7aJ8v38O6i3aTnl9A2KpDJV7WnS2xwpW3nbU7hrT+S2JNZgMVqJy7Mjzv6xzPivBgAyqw2/vfrDhZuT2dfViEB3u70axHGhMvaEBnoXYtndZyKUiIiUu9ZTWbmHF1faETSIo2cOQ3Ddi/liFcgX7a+mDe6XENwST69Dm81OtYpFbl5MqvFBQBcv2M+bvpvLSIiIi5q7oZDPP3DNp6+ugNdY4P5cGkyt0xbye8PDSTM36tC+yAfT8Zd2IIWEX54uJlZsC2Nh7/ZSCN/Ly5oFU5RmZUtB3O57+IWtI0KJKeojMlztzJ2+mrm3tfPgDPUmlIiItIALI3qSJpvKEEl+Vy0f63RcVzG6G0/c8nev7CZzEw5/2a2hMYZHemUfozvQ66XH03y07ng4Aaj44iIiIictQ/+TOb6HrGM6h5Ly8gAnhneER9PN75avb/S9r0TGnFph8a0iAigWSM/xvSLp03jAFbvyQIg0NuDz8b25MpO0SSE+3Ne0xCevKo9mw7mcDC7qDZPrZyKUiIiUq/ZoXzkzNDdS/G0WU7+AClnAv61/ht6HN5KqZsHk3qNYW9ApNGxqlTs5sk3LQcCR0dJ2W3GBhIRERH5m7y8PHJzc8tvJSUllbYrtdjYfDCHvi3CyveZzSb6tghj7d7sUx7HbrezNDGD3ekF9IgPrTpPsQWTCQK9jZlIp+l79cVMXeyaGzVFQ0Qq2hIax47QpnhYy7giebnRcVyOm93Go6s+5dG+d7E9NI6Jve/g5SVvEF6UY3S0Cn6K60WOlz+NCzK48MA6o+OIiIiIVNCuXTun7SeeeIJJkyZVaHeksBSrzV5hml64vxdJ6QVV9p9bXEavZxdQarFhNpt4elgH+rcMr7RtcZmV5+Zt46rO0QR4e5z5yVQDFaVERKRem310lNTF+9cQXJpvcBrX5G0tY/LyD3lowDj2B0Qysfcd/G/JWwSUGTPMuzIlZvcTRkkt0CgpERERqZO2bt1KkyZNyre9vCquDXUu/D3d+en+/hSUWliWmMlTP24lNtSX3gmNnNqVWW3cO3Mtdjs8PbxDtWY4EypKiRy17Uljr3ZVF7S90egEItXrkF8jlke1B+DqxMUGp3FtgWWFPL3sfcYPuJd9gY2Z1GsMzyx7D29rmdHRAJgX14sj3oFEFGZx8f41RscRERERqVRAQACBVVzt+UQhvp64mU1k5DtP70vPLyG8kkXOjzGbTcSF+QHQPjqIxLR83l6Y6FSUKrPaGDdjLQeOFPH5Hb0MGyUFWlNKRETqsTkJ/bGbzJx/eBtN89OMjuPyIoqyeXrZB/iXFrK1UTzPdb8Jq8n4jxKlZne+bnkhANftWIC7RkmJiIiIi/N0N9OhSRDLEjPK99lsdpYlZnJes+DT7sdmt1NqOf7Z6FhBak9mATPG9iTEz7M6Y58x4z9JioiI1IA8Dx9+bdoDgBGJiwxOU3/E5R3miZUf4WEtY2VUe97oPBKjV/T7tVkPMn2CCC88wqD9qw1OIyIiIlI9xvaL5/NV+/lmzQES0/J4bM5mCkstXNstFoDxX67n+Xnby9u/9UciS3alsy+zkMS0PN5fvJvZ6w5ydVfHdMEyq41/fraWTQdzePW6rljtdtLyiknLK3YqXNUmTd8TEZF66ee4XpS4e9I85yCdMxKNjlOvdMhM5pHVn/FMj9H8EteT0JJcbtn2iyFZSs1ufNnyIgBG7foDT5vVkBwiIiIi1W1o52iyCkp55bedpOeV0DY6kOljehAe4Ji+dzC7CJPp+EXPikqtPD5nMyk5xXh7uJEQ7scr13VhaOdoAA7nFDN/WyoAl7++xOlYn9/Rq8K6U7VBRSkREal3ykxufN+8H+AYJaXrk1a/PilbGLf+W97oei2ft76EkOI8hiYvq/Uc85ueT4ZvMI2Kshm8d2WtH19ERESkJo3uE8foPnGV3vflXb2dth8a0pqHhrSusq/YUF/2PHdFdcY7Z5q+JyIi9c7imM5k+gTRqCiHAQc2GB2n3rp870puOjpC6p1Ow1kS3alWj19mcuPLVhcDcK1GSYmIiIi4HBWlRESkXrEDs1pcAMBVu//Ew65CRU26ccdvXJ68DLvJzAvdbmRDWEKtHfv3pt1I8w0hpDiXS/dolJSIiIiIq9H0vQZsyncwazVsPwQ+ntCnJTx/PbR2TDclKx+e+BZ+3QT7MiA8EIZ3g6euhSDfqvu9dSpMd56eypBOMG/C8e2rXoL1eyEtF0L8YFB7eP4GiA6p/vMUkYZlQ1gLdgc1wctSymV7Vhgdp94zAfdsmE2Olz9LozvxZM9beXHJ2zTPTanR41pMZr44YZSUl81So8cTERERkeqnkVIN2KLtMG4QrJgMvz0CZVYY/BwUFDvuP3TEcfvfjbD5efj4Lpi3EW5/79R9X9oJUt46fvv8Xuf7L2wHX90HO16Eb/8FSWlwzWvVf44i0vAcGyU1eN9fBJQVGZymYXDDzn9Wz6RDRhKFHj5M7HMHh31Da/SYf8Scx2G/RgQX56n4KCIiIuKiNFKqATtx5BI4ik4R/4Q1yTCgLXSIhW8fOH5/QiQ8MwpuehssVnB3q7pvLw9oHFz1/Q9edvzfzcLhkaEw/BUos4CHi/1fabXbeSszg7m5uWRYLES4uzM8MIi7GzUqvxLC/6UcYk5urtPj+vn68V5s7En7nnnkCB9mZZJhtdLay4vHIiLp5ONTfv/ofXtZVeT8R/eooGAmNW5cTWcn4loS0/JY1bgtJruN4UlLTv0AqTaeNgtPrPyIh/vdw56gaCb2uYP/LX6T4NKCaj+W1WTmy9aOUVIjExfhbS2r9mOIiIiISM1zsT//pSblFDp+hvqfvE2gz8kLUgALtzkKXCF+cFE7ePpaaBRQedusfJix1DF90NUKUgAfZGXyRXY2UxpH0cLLk83FxTyWchh/NzM3hxwfKdDPz49nGkeVb3uaTn49sJ9zc3k+PY0nIiPp5O3Dp0eyuPPAfn6Mb04j9+NP1LVBQdwbFl6+7XOKfkXqs2l/JgPQO2UL0QWZBqdpePzLinl6+QeMH3AvB/3DeaL37Tz351R8rKXVepxFTbpw0D+cwJJ8rjDgin8iIiIiUj00fU8AsNnggU+hbyvHCKnKZOTBU7PhzotO3telneGTu2HBo441qhZtg8teAKvNud2Ez8FvDDS6C/Zlwnfjq+dcatv6oiIu8vfnAn9/mnh4MiQgkL5+vmwqKnZq52kyEe7uXn4Lcjt5Ze/jI1lcGxTEiKBgWnh58URkY7zNZmbl5Di18zabnfr1P0W/IvVVRn4J3649CMCIxEUGp2m4GhXn8syy9wksKWBnSFOe6XELZabqe12yYuLz1oMAxyip6i54iYiIiEjtUVFKABj3MWw+AF/cW/n9uYVwxYvQrglMGnHyvq7vDVd1g45NYXh3+OFhWLUbFm51bvfwlbDuGfj1EXAzwy1TwW6vltOpVV18fFhRUMCeUscfRtuLi1lbVER/fz+ndqsKC+mXuIvLd+9m8uHDZFurviJYqd3O1uJievke78NsMtHb15f1xc7T9X7IzaVP4i6uSt7Ny+lpFNlsf+9OpEH4bMVeSi02Wmfto13WHqPjNGgx+elMXjENL0spayLb8Op5o7BRPaM4lzTpzIGACAJKC7hSo6REREREXJoLTpaS6nbvx/DDOlj8OMQ0qnh/XhFc+gIEeMPsB898il3zCAgLgMRUuLjD8f1hAY5bqyhoGw2x98OKROjd8pxOp9bdEdqIApuNK5J34wZYgX+FhTM0MKi8TT8/fwYFBBDj4cG+0jJezUjnrgP7mdm0GW6VTLfLtlqwAmHuzk92Izd3dpcWlm9fERhEtIc7Ee7u7Cgp4eX0dPaUlvJ6k5gaOluRuqm4zMqny/cCjlFSmsRqvDZH9vHYX58wqddt/B7bjZDiPMZu+eGc+rSdMErq6sTF+FpKqiOqiIiIiBhERakGzG6H+6bD7NWwcCLER1Rsk1sIQ553LFz+/b/B2/PMj3MgEzLzISq46ja2oyOkSlxwrdp5eXn8kJvLi1HRtPDyZHtxCVPSUh0Lngc5ClOXBwaWt2/l5U1rLy+GJO/mr8JCevv5VdX1KY0KDnbqN9zNnTEH9rOvtJSmnmfxH0vERc1Zd5DMglKaBPvQN2WT0XHkqPPTtvPAuq95udv1fNtyIKHFuYxIWnzW/S2N7si+wMb4lRVx1e6l1ZhURERERIyg6XsN2LiP4bOlMHOcYxTU4WzHrejo8hy5hTD4OSgogWl3QG7R8TYnrg/V5iGYvcrx7/xieHgmrNgFe9JhwWYY9jK0iIQhnRxtVibCm7/C+j2wNx1+3wI3vOm4up+rjZIC+F96GmNDG3F5YCCtvLy5KiiI0aGhvJ9V9SLLsZ6ehLi5sa+s8rVQgt3ccQMyLBan/ZlWS4XRUyc6dmW+qvoVqY9sNjsfHF3g/La+cbjZNYW1Lrlk/2rGbHaMkHq/41X8HtP1rPo5cZTU8KQl+FmKT/EIEREREanrNFKqAXtnvuPnwKed9390J9x6AazdAyuTHPta/G0R8uRXIe7oBd92pBy/cp+bGTbug+lLILsAokNgcEd46lrHaCsAX0+YtQqe+NZR8IoKhks7wcThx9u4kiKbrUJ114wJ20kWyDpcVka21Up4FQUmT5OJdt7erCgsYFCA47KFNrudFYWF3BgcUmW/24sdf6SFu+lXWxqORbvSSUzLJ8DLnevOj+WA0YGkgmsSF5LlHcicFgN4+bzrCSotoFvazjPqY0VUO5KDovEtK2JY0pIaSioiIiIitUl/uTZg9hknv39gu1O3+Xs/Pp7wyyMnb9+xKfz+2Kn7dRUX+vvzblYmUR4etPDyZFtxCdOPZDHi6NS9ApuNtzMyGBwQQJi7G/tKy3gpPY2mHh70O2Eh89v272OQfwD/CHEUnW4NCeXRwyl08Paho7c3nxw5QpHNxtVH+91XWsqPubkM8Pcn2M3MjpISnk9Lo7uPD629vWv/iRAxyAdLdgNwfY9YArxdsLLdAJiAOzbPJdvLn4Wx5/F0j9E89+dUWmfvP63H24GZrS8BYNjuPwkoKzr5A0RERETEJagoJXKOHouM5PWMDJ5MPUyW1UqEuzujgoL5Z1gYAG7AzpISvsvNIffo/X39/LgvLBxP8/ExVvtLSzliPT5d77LAQLKsVt7ISCfDaqWNlxfvxsSWT9/zMJlYXljAJ0eyKLLbaezuziX+AdzdqJLV6kXqqS2HcliamImb2cStfeONjiMnYcbO+LVfkuPlz7qIVvy39+28vPhNmhRknPKxKxu3Iyk4Bp+yYoYnapSUiIiISH2hopTIOfIzu/FoRCSPRkRWer+32cz7sbGn7Gd+QosK+/4RElI+curvojw8+KRpszMLK1LPTDu6ltTlHaNoEuxjcBo5FQ+7lYl/TWdCv3+SGBzDxD538NLiNwktyavyMSeOkhqavJTAssIq24qIiIiIa9FC5yIi4pJSc4uZu+EQAGP7aZSUq/C1lPDk8g+Iys/gsF8jHu8zlgL3qqccr45ow66QWLwspVydePZX7hMRERGRukdFKRERcUnTl+2hzGqnR1wonWODjY4jZyCkJJ+nl71PcHEeu4Oa8GTPWyk1u1VoZwdmtHGMkroyeRnBpQW1nFREREREapKKUiIi4nIKSy3MWLkPgLH9NUrKFUUXZvLU8g/wKStmY3gL/tftRqyYnNqsC2/JjtBmeFlKGZm40JigIiIiIlJjVJQSERGX882aA+QUlRHXyJeL21a+npvUfS1yDvL4X9Nxt1lY0qQz73YahgUTG8MS+KNJF97rcBUAl+9ZTkhJvsFpRURERKS6aaFzERFxKVabvXyB89v7xeNmNp3iEVKXdU3fxb/XfMHz59/E3Ob9WBDbjUKPExatt9tplnvYuIAiIiIiUmM0UkpERFzK/G2p7M0sJMjHg5HdYoyOI9Vg4MH1XLJnJYBzQeqoV7uOYmlUh9qOJSIiIiI1TEUpERFxKR8s2Q3ATb2a4uupAb/1gRUT6yJbg91e8U6TYyTcux2HVVhzSkRERERcm4pSIiLiMtbvz2bVniN4uJm4pXec0XGkmmwJa06GT3B5AaoCk4l03xC2hDWv1VwiIiIiUrPqfFEqLi4Ok8lU4TZu3DgABg4cWOG+u+++2+DUIiJSE46NkrqqcxMiA70NTiPVJcsroFrbiYiIiIhrqPPzHlatWoXVai3f3rx5M5dccgnXXntt+b477riDJ598snzb19e3VjOKiEjNO3CkkJ83Oxa8vr1fvMFppDqFluRVazsRERERcQ11vigVHh7utP3cc8+RkJDABRdcUL7P19eXxo0bn3afJSUllJSUlG/n5elDrohIXffx0j1YbXb6tQijXXSg0XGkGrXP2E1YUTYZ3kGVT+Gz2wkvyqZ9xu7aDyciIiIiNabOT987UWlpKZ999hljxozBdMKH1hkzZhAWFkaHDh149NFHKSwsPGk/U6ZMISgoqPzWrl27mo4uIiLnILe4jC9W7Qfg9v4aJVXfuGHn7o1zHBt/X+z86PZdm77DjUoWQhcRERERl+VSRak5c+aQnZ3NrbfeWr7vxhtv5LPPPuOPP/7g0Ucf5dNPP+Wmm246aT+PPvooOTk55betW7fWcHIRETkXX63aT36JhZYR/gxsFX7qB4jL6ZuymYl/TSesOMdpf3hRNhP/mk7flM0GJRMRERGRmlLnp++daNq0aVx22WVER0eX77vzzjvL/92xY0eioqK4+OKLSUpKIiEhodJ+vLy88PLyKt/Ozc2tudAiInJOLFYbHy3dA8DY/vFOI2WlfumbspleKVvYEtacLK8AQkvyaJ+xWyOkREREROoplylK7d27l/nz5zNr1qyTtuvZsycAiYmJVRalRETEdfy8+TAHs4sI8/dkWJcmRseRGuaGnU4ZSUbHEBEREZFa4DLT9z766CMiIiK44oorTtpu/fr1AERFRdVCKhERqUl2u50PljgWt765VxzeHm4GJxIRERERkeriEiOlbDYbH330EaNHj8bd/XjkpKQkZs6cyeWXX06jRo3YuHEjDz74IAMGDKBTp04GJhYRkeqweu8RNhzIwcvdzE29mhodR0REREREqpFLFKXmz5/Pvn37GDNmjNN+T09P5s+fz6uvvkpBQQGxsbGMHDmSiRMnGpRURESq0/uLHaOkRpwXQyN/r1O0FhERERERV+ISRanBgwdj//slooHY2FgWLVpkQCIREalpyRkF/LYtFYDb+8UbnEZERERERKqby6wpJSIiDctHS5Ox2+GiNhG0iPA3Oo6IiIiIiFQzFaVERKTOyS4s5evVBwAYq1FSIiIiIiL1kopSIiJS58xYuY+iMivtogLpndDI6DgiIiIiIlIDVJQSEZE6pdRiY/qyPQDcMSAek8lkbCAREREREakRKkqJiEidMnfDIdLySogM9OKKjtFGxxERERERkRqiopSIiNQZdrud95fsBuDWPvF4uuttSkRERESkvtKnfRERqTOWJWWy/XAevp5u3NijqdFxRERERESkBqkoJSIidcaxUVKjuscS5OthcBoREREREalJKkqJiEidsCs1j4U70jGZ4La+cUbHERERERGRGqailIiI1AnT/kwGYEi7xjRr5GdwGhERERERqWkqSomIiOHS80qYte4gAGP7xxucRkREREREaoOKUiIiYrjPVuyl1GKjS2ww3ZqFGB1HRERERERqgYpSIiJiqOIyK5+u2As4RkmZTCaDE4mIiIiISG1QUUpERAw1e91BsgpKaRLsw6XtGxsdR0REREREaomKUiIiYhibzc4HS3YDMKZfPO5uelsSEREREWko9OlfREQMs2hnOknpBQR4uTOqe4zRcUREREREpBapKCUiIoZ5/+goqRt6NiXA28PgNCIiIiIiUptUlBIREUNsOZTDsqRM3MwmRveJMzqOiIiIiIjUMhWlRETEENOWJANwRccomgT7GJxGRERERERqm4pSIiJS6w7nFPP9hkMAjO0fb3AaERERERExgopSIiJS66Yv34PFZqdHfCidYoKNjiMiIiIiIgZQUUpERGpVQYmFGSv2AjC2n0ZJiYiIiIg0VCpKiYhIrfpmzQFyiy3ENfJlUNtIo+OIiIiIiIhBVJQSEZFaY7XZmfanY4Hz2/vFYzabDE4kIiIiIiJGUVFKRERqzW9bU9mXVUiwrwfXdIs1Oo6IiIiIiBhIRSkREak1HyzZDcBNPZvh4+lmcBoRERERETGSilIiIlIr1u07wuq9R/B0M3NL72ZGxxEREREREYOpKCUiIrXig6NrSV3VJZqIQG+D04iIiIiIiNFUlBIRkRq3P6uQnzelAI4FzkVERERERFSUEhGRGvfxsj3Y7NC/ZRhtowKNjiMiIiIiInWAu9EBpHpse7KN0REM1/ZGoxOISGVyi8v4ctV+QKOkRERERETkOBWlRESkRn35137ySyy0jPDnglbhRscREREREXEZnyzfw7uLdpOeX0LbqEAmX9WeLrHBlbadtzmFt/5IYk9mARarnbgwP+7oH8+I82LK29jtdl75bSefr9pPblEZ3eNCeHp4R+LD/GrpjJxp+p6IiNSYMquNj5Y6Fjgf2z8ek8lkcCIREREREdcwd8Mhnv5hG/8a1JIf7+tHu6gAbpm2koz8kkrbB/l4Mu7CFsy+pw/zHujPtd1iePibjSzamV7eZuqi3Xy0bA/PDO/AnHF98fFw55YPV1JcZq2t03KiopSIiNSYnzcf5lBOMWH+ngzr0sToOCIiIiIiLuODP5O5vkcso7rH0jIygGeGd8TH042vVu+vtH3vhEZc2qExLSICaNbIjzH94mnTOIDVe7IAxyipD5cmc99FLRjcvjFtowJ5+brOpOaW8OvW1No8tXIqSomISI2w2+18sGQ3ADf3isPbw83gRCIiIiIixsrLyyM3N7f8VlJS+ainUouNzQdz6NsirHyf2Wyib4sw1u7NPuVx7HY7SxMz2J1eQI/4UAD2ZxWRnlfi1GegtwddYoNZu/fIuZ3YWdKaUiIiUiNW7TnCxgM5eLmbualXU6PjiIiIiIgYrl27dk7bTzzxBJMmTarQ7khhKVabnTB/L6f94f5eJKUXVNl/bnEZvZ5dQKnFhtls4ulhHejf0rGua3p+cXkff+8zvYopgTVNRSkREakR7x8dJTWyWwyN/vbGJyIiIiLSEG3dupUmTY4va+HlVb2fk/093fnp/v4UlFpYlpjJUz9uJTbUl94Jjar1ONVFRSkREal2yRkFzN/mmJc+pm+8wWlEREREROqGgIAAAgMDT9kuxNcTN7OpwqLm6fklFUY6nchsNhF39Ep67aODSEzL5+2FifROaES4v3d5HxGB3k59tos6daaaoDWlRESk2n34ZzJ2O1zcJoIWEf5GxxERERERcSme7mY6NAliWWJG+T6bzc6yxEzOaxZ82v3Y7HZKLTYAYkN9CA/wYlliZvn9ecVlrN+fzXnNQqot+5nQSKkGLrWsjJfS01lSkE+x3U5TD0+eiWpMB28fyux2Xs9IZ3F+AQfKSvE3u9Hbz5fx4eFEuHtU2WeBzcrrGRnMz8sjy2qlrZc3j0ZE0NHHp7xNhsXCy+lpLC0oJM9mpbuPL/8XGUmcp2dtnLaI1KAjBaV8vcZxRZDb+2uUlIiIiIjI2RjbL55/f72BjjHBdIkNYtqfeygstXBtt1gAxn+5nsggbyZc2gaAt/5IpFNMEM1C/Si1Wvljezqz1x3k6eEdADCZTIzpG88bv+8iLsyP2FAfXvp1J5GBXgxuF2nIOaoo1YDlWK38Y99eevj68W5MLKFubuwtKyPQ7LhCVrHNxtbiYu5u1Ig23l7kWm08m5bKuAMH+Toursp+Hz98mF0lJTwfFU24uztzc3O4/cB+5sbFE+nhgd1u576DB3A3mXizSRP83cx8nHWE2/fvY258c3zNGsAn4spm/rWP4jIb7aMD6d28bs5dFxERERGp64Z2jiaroJRXfttJel4JbaMDmT6mB+EBjul7B7OLMJlM5e2LSq08PmczKTnFeHu4kRDuxyvXdWFo5+jyNndf0JyiUguPztpEbnEZ58eFMP22HoZdKVtFqQZsWlYmjT08eDYqqnxfzAkjlQLc3JgW63zFrIkRkVy3by+HysqI9qg4WqrYZuO3vDzebBJDd19fAO4NC2dhfj5fZGfzr/Bw9paVsaG4mO/i4ml5dFG3JyIjGZCUx0+5uVwTHFwDZysitaHEYuXjZXsAGNs/3ulNUkREREREzszoPnGM7hNX6X1f3tXbafuhIa15aEjrk/ZnMpkYP7g14wefvF1t0ZCUBuz3/Hw6eHvzwMGD9EvcxYg9yXydnX3Sx+TZbJiAwCpGM1mxYwU8//aHqLfZzNqiQgBK7Y75rF4ntDGbTHiaTOVtRMQ1zd2QQnpeCY0DvbmiY/SpHyAiIiIiIg2WilIN2IGyMr7IzqaZpyfvxcRyfXAIz6alMicnp9L2JTYbL6encXlAIP5ulQ/t8zO70cXbh6mZGaRZyrDa7Xyfk8P6oiLSLVYA4j29iHJ355WMdHKsVkrtdj7IzOSwxVLeRkRcj91u54MluwHHNzqe7nqLERERERGRqmn6XgNms9vp4O3Dg+HhALTz9mZXSQlfZh9heFCQU9syu53xhw5hxzHV7mSei4pi4uEUBiYl4Xa038sDAtlaUgyAh8nE601imHg4hd6Ju3ADevv60d/PD3sNnKeI1I6liZlsP5yHr6cbN/ZoeuoHiIiIiIhIg6aiVAMW7u5Ogpfz1e4SPD35LT/PaZ+jIHWQQ5YyPoptWuUoqWOaenrySdNmFNpsFNhshLu7M/7QQWJOWIOqvbc3s+PiybNaKbPbCXV357q9e+jg7V19Jygiter9o6OkRnWPJci36it0ioiIiIiIgKbvNWjn+fiSXFrqtG9PWSnR7sf/mDxWkNpbWsq0mFiCT1GQOpGv2Uy4uzs5VitLCwq4yD+gQpsANzdC3d3ZU1rKluLiStuISN23MzWPRTvTMZtgTN94o+OIiIiIiIgLUFGqAbslJISNRUW8m5nB3tJSfsjN4evsbG4ICQYcBakHDh1kS3ExL0RFYwXSLRbSLRZK7ccn2t22fx8zjhwp3/6zIJ8lBfkcKC1lWUEBt+7fR7ynJ1efMCVwXl4ufxUWsL+0lAV5eYzdv4+L/f3p6+dXW6cvItVo2pJkAIa0b0zTRr4GpxEREREREVeg6XsNWEcfH15vEsMr6em8k5lJjIcHj0REMjTQUTxKs5TxR34+ACP27nF67MexsfTwdRSQ9peWcsRqKb8vz2rj1Yx0DlssBJnNDA4I4F9h4XiccLW9dIuFF9LSyLBYCHd3Z1hQEHc3CqvhMxaRmpCeV8Ls9QcBGNtfo6REREREROT0qCjVwA3092egv3+l9zXx8GRr6zan7GN+Qgun7csCA7ksMPCkj7k5JJSbQ0JPP6iI1FmfrthLqcVG16bBdGum32sRERERETk9mr4nIiJnrbjMymcr9gIwtl9zg9OIiIiIiIgrUVFKRETO2qy1B8kqKCUmxIch7SONjiMiIiIiIi5ERSkRETkrNpudD/7cDcBtfeNxd9NbioiIiIiInD79BSEiImdl4c40dqcXEODlznXnxxodR0REREREXIyKUiIiclbeX5wMwA09m+LvpetmiIiIiIjImVFRSkREztjmgzks352Ju9nErX3ijI4jIiIiIiIuSEUpERE5Y9P+dIySuqJTFNHBPganERERERERV6SilIiInJHDOcXM3XAIgLH9mhucRkREREREXJWKUiIickY+XrYHi81Oz/hQOsYEGR1HRERERERclIpSIiJy2gpKLMxcuReAsf01SkpERERERM6eilIiInLavl69n9xiC/FhflzcJsLoOCIiIiIi4sJUlBIRkdNitdn5cOkeAMb0i8dsNhkbSEREREREXJqKUiIiclp+23qYfVmFBPt6cM15MUbHERERERERF6eilIiInJb3lyQDcFPPZvh4uhmcRkREREREXJ2KUiIickpr9x1hzd4jeLqZuaVPM6PjiIiIiIhIPaCilIiInNK0o6OkruoSTUSAt8FpRERERESkPlBRSkRETmp/ViE/b04BYGz/eIPTiIiIiIhIfVGni1KTJk3CZDI53dq0aVN+f3FxMePGjaNRo0b4+/szcuRIUlNTDUwsIkax2uwsT8rku/UHWZ6UidVmNzpSvfHR0j3Y7NC/ZRhtGgcaHUdEREREROoJd6MDnEr79u2ZP39++ba7+/HIDz74ID/++CNff/01QUFB3HvvvYwYMYKlS5caEVVEDDJvcwqT524lJae4fF9UkDdPDG3HpR2iDEzm+nKLy/hy1T4AxvZvbnAaERERERGpT+p8Ucrd3Z3GjRtX2J+Tk8O0adOYOXMmF110EQAfffQRbdu2ZcWKFfTq1au2o4qIAeZtTuGfn63l7+OiDucU88/P1vLOTeepMHUOvvhrHwWlVlpF+jOgZZjRcUSkoZlpMjqB8W7UyF8REam/6vT0PYBdu3YRHR1N8+bN+cc//sG+fY5v7NesWUNZWRmDBg0qb9umTRuaNm3K8uXLT9pnSUkJubm55be8vLwaPQcRqRlWm53Jc7dWKEgB5fsmz92qqXxnqcxq46OlewAY2685JpP+OBQRERERkepTp4tSPXv25OOPP2bevHm88847JCcn079/f/Ly8jh8+DCenp4EBwc7PSYyMpLDhw+ftN8pU6YQFBRUfmvXrl0NnoWI1JS/krOcpuz9nR1IySlm5e7M2gtVj/y0KYWUnGLC/L0Y1jXa6DgiIiIiIlLP1Onpe5dddln5vzt16kTPnj1p1qwZX331FT4+Pmfd76OPPsr48ePLtw8ePKjClIgLSsuruiB1on9MW0lkgDcRgV5EHP15bDvyhH2N/LxwM2s0EIDdbueDJckA3NK7GV7ubgYnEhERERGR+qZOF6X+Ljg4mFatWpGYmMgll1xCaWkp2dnZTqOlUlNTK12D6kReXl54eXmVb+fm5tZUZBGpQREB3qfVzm6Hw7nFHM4tBnKqbGc2QXiAo0gVGehF+NGfEX/72ci//hev/krOYtPBHLzczdzUq5nRcUREREREpB5yqaJUfn4+SUlJ3HzzzXTr1g0PDw8WLFjAyJEjAdixYwf79u2jd+/eBicVkdrQIz6UiAAv0vJKKr3fBDQO8ubbf/YhM7+U1Nxi0vJKyn+mnbCdkV+CzQ6puSWk5paw6WDVxzWbIMzfy2nEVcURWN408vPE3a1Oz5Ku0vtHR0mN7BZDqJ+nwWlERERERKQ+qtNFqYceeoihQ4fSrFkzDh06xBNPPIGbmxs33HADQUFB3H777YwfP57Q0FACAwO577776N27t668J9JAmE3QOLDyotSxcUxPDG1HdLAP0cE+dCSoyr6sNjuZ+SVORavU3GJSc0tIz3P8TMsrJj3PUbxKy3O03UzVIy1Nx4pXAY4iVUSAFxFHf0ae8DPMv24Vr3an57NgeyoAt/eLNziNiIiIiIjUV3W6KHXgwAFuuOEGMjMzCQ8Pp1+/fqxYsYLw8HAAXnnlFcxmMyNHjqSkpIQhQ4bw9ttvG5xaRGrLDxtT2HgwFzczhPh6kpFfWn5f4yBvnhjajks7RJ1WX25mk6NgFOhNhyanKF4VlJB2tEiVmuv4d2pecfm+tNwS0vNLsNrspOeVkJ5XwpZDJy9eNfI7Vrw6Yfrg3wpY4QFeeNRg8cpqs/NXchZv/rELux0uah1OQrh/jR1PREREREQatjpdlPriiy9Oer+3tzdvvfUWb731Vi0lEpG64khBKZO+3wLAvRe25P6LW/JXchZpecVEBHjTIz60RtZ9cjObHFP1ArzhFCOvsgocUwbT/zb66sSpg2l5juJVRn4JGfklbE2p+tgmE4T6ep4w2qqS9a8CvQn398LT/cyKV/M2pzB57lanqxmuP5DDvM0pp13YExGpKyZ9C5NnOe9rHQXb/+f4d3Ep/HsGfLECSspgSCd4+zaIrPplHdM/Kt//wg3w8JWOf1/1EqzfC2m5EOIHg9rD8zdAdMi5n5OIiEh9VKeLUiIiVXnqh61kFpTSKtKfey5MwM1sondCI6NjlXMzmwg/OrrpZGw2O1mFpc7FqhNGXqXmlZB+9D6LzU5mQSmZBaVsO0nxCiDUz7N8umBkgFf5OlcnTiEMD/DCy92NeZtT+Odna7H/rY8jBaX887O1vHPTeSpMiYjLaR8D8x89vn3iRUQf/Ax+XA9f3w9BvnDvxzDiFVg6qer+Uv72HejPG+D292Fkj+P7LmwH/3cVRAXDwSPw0Ey45jVYdpJ+RUREGjIVpUTE5SzckcasdQcxmeD5kZ3wOvEvDRdjNpsI8/cizN+L9idpZ7PZOVJYWr62ldP0wbxja185/l1mdYzSyiooZfvhvJMeP9jHnfwSa4WCFIAdx9pck+du5ZJ2jev9FQdFpH5xN0Pj4Ir7cwph2kKYOQ4uOvrC+9Fd0PZhWLELerWsvL+/9/XdGkcRqnnE8X0PXnb8383C4ZGhMPwVKLOAhz51i4iIVKC3RxFxKfklFh6bvRmA2/rE07Vpw5gTYTabaOTvRSN/L9oRWGU7m81OdlGZ83TBClMHjxevsossJz2uHUjJKeav5Kw6NRJNRORUdqVC9Djw9oDeLWHKddA0DNYkQ5kVBnU43rZNNDRtBMsTqy5KnSg1xzHSavpdVbfJyocZS6FPSxWkREREqqK3SBFxKS/O287B7CJiQnx4aEgro+PUOWaziVA/T0L9PGl7khl3drudI4VlfLlqH8/P23HKftPyik/ZRkSkruiZAB/f5VhHKiXbsb5U/ydh8/NwOBs83SHYz/kxkUGO+07H9MUQ4A0jzq9434TP4c3foLAEerWAHx46x5MRERGpx+rONchFRE5h9Z4sPlmxF4DnRnTC11N19bNlMjmKV11iT2+kmWNhdxER13BZF7i2J3Rq6ljE/KeHIbsQvlpZPf1/uAj+0Re8PSve9/CVsO4Z+PURcDPDLVPBXtkcaREREVFRSkRcQ3GZlQnfbsRuh2u7xdCvZZjRkeqFHvGhRAV5U9VqUSYgKshxNUMREVcV7AetoiDxsGNtqFILZBc4t0nNqXwNqr9bsh12pMDYgZXfHxbgONYlHeGLe+Gn9bAi8dzyi4iI1FcqSomIS3jz90SS0gsID/Bi4hXtjI5Tb7iZTTwx1PF8/r0wdWz7iaHttMi5iLi0/GJISnVcFa9bPHi4wYItx+/fcQj2ZULvFqfua9pCRx+dm526re3oCKmSsrNJLSIiUv+pKCUidd7WQ7lMXZQEwFPD2hPk62Fwovrl0g5RvHPTeTQOcp6i1zjIm3duOo9LO5xkcSoRkTrooRmwaBvsSYdlO+HqVxxT6W7oA0G+cPtAGP8Z/LHFsfD5be85FkM/cZHzNg/B7FXO/eYWwtd/VT5KamUivPkrrN8De9Ph9y1ww5uQEOnoW0RERCrSgiwiUqdZrDYmfLsRi83Ope0bq0BSQy7tEMUl7RrzV3IWaXnFRAQ4puxphJSIuKIDWY6CUGY+hAdAv9awYjKEH7146Ss3gdkEI1+DEgsM6Qhv3+bcx44UyCl03vfFCsf6UDf0qXhMX0+YtQqe+BYKShyjsi7tBBOHg5e+SxEREamUilIiUqdN+zOZTQdzCPR258lh7Y2OU6+5mU30TmhkdAwRkXP2xX0nv9/bE966zXGrin1GxX13XuS4VaZjU/j9sdPPKCIiIpq+JyJ12J6MAl7+bScAE69oR0SgrgAnIiIiIiJSX6goJSJ1ks1mZ8K3Gymx2OjbohHXdo8xOpKIiIiIiIhUIxWlRKRO+mLVflYmZ+Hj4caUqzthMmltIxERERERkfpERSkRqXMO5xQz5adtAPx7cCuaNvI1OJGIiIiIiIhUNxWlRKROsdvtTJyzibwSC51jg7mtb7zRkURERERERKQGqCglInXKDxtTmL8tDQ83Ey+M7ISbWdP2RERERERE6iMVpUSkzjhSUMqk77cAcM/AFrRuHGBwIhEREREREakpKkqJSJ3x1A9bySwopVWkP/dcmGB0HBEREREREalBKkqJSJ2wcEcas9YdxGSC50d2wsvdzehIIiIiIiIiUoNUlBIRw+WXWHhs9mYAbusTT9emIQYnEhERERERkZqmopSIGO7Feds5mF1ETIgPDw1pZXQcERERERERqQUqSomIoVbvyeKTFXsBeG5EJ3w93Q1OJCIiIiIiIrVBRSkRMUxxmZUJ327Ebodru8XQr2WY0ZFERERERESklqgoJSKGefP3RJLSCwgP8GLiFe2MjiMiIiIiIiK1SEUpETHE1kO5TF2UBMBTw9oT5OthcCIRERERERGpTSpKiUits1htTPh2IxabnUvbN+bSDlFGRxIREREREZFapqKUiNS6aX8ms+lgDoHe7jw5rL3RcURERERERMQAKkqJSK1Kzijg5d92AjDxynZEBHobnEhERERERESMoKKUiNQam83OI99upMRio1+LMK7tFmN0JBERERERETGIilIiUmu+WLWflclZ+Hi48ezVHTGZTEZHEhEREREREYO4Gx1ARBqGwznFTPlpGwAPDWlN00a+BicSERERERGp2z5Zvod3F+0mPb+EtlGBTL6qPV1igytt+/lf+5i19gA7DucB0DEmiIeHtHFqX1Bi4fl52/l1SypHCkuJDfXl1j5x3NSrWS2cTUUaKSUiNc5utzNxzibySix0iQ3m1j5xRkcSERERERGp0+ZuOMTTP2zjX4Na8uN9/WgXFcAt01aSkV9SafsVuzO5qnM0n9/Zi1n39CUqyIebp63kcE5xeZunf9zKop3pvHJdF+aPv4AxfeN54vst/LY1tbZOy4mKUiJS437YmML8bWl4uJl4fmQn3MyaticiIiIiInIyH/yZzPU9YhnVPZaWkQE8M7wjPp5ufLV6f6XtX7u+Kzf3jqN9dBAtIvx5fmQn7HZYmphR3mbN3iOMPC+G3gmNiA315caeTWkbFcCG/dm1dFbONH1PRGrUkYJSJn2/BYBxF7agdeMAgxOJiIiISK2bqS8ludFudAKpA/Ly8sjNzS3f9vLywsvLq0K7UouNzQdzuGdgQvk+s9lE3xZhrN2bfVrHKiqzUma1EezrUb6vW7MQ5m9LZVT3WCIDvVi+O5Pk9AIevyLs7E/qHKgoJSI16qkftpJZUEqrSH/uGdjC6Dh1nz6w6QObiIiISGX0ObFefE5s166d0/YTTzzBpEmTKrQ7UliK1WYnzN+5YBXu70VSesFpHeu5n7cRGehN3xbHC06TrmrPo7M20WvKAtzNJswmE1NGdKRn80ZnfjLVQEUpEakxC3ekMWvdQUwmeH5kJzzdNWNYREREREQarq1bt9KkSZPy7cpGSVWHtxcmMndDCl/c2QtvD7fy/dOX7WH9vmw+uKU7TUJ8+Cs5i/9+t5nIQG/6taz90VIqSolIjcgvsfDY7M0AjOkbT9emIQYnEhERERERMVZAQACBgYGnbBfi64mb2VRhUfP0/BLC/U9eyHpvcRLvLExixtietI06fqziMisv/rKDd2/uxkVtIgFoGxXI1kO5vLdktyFFKQ1bEJEa8eK87RzMLiI21Id/D25ldBwRERERERGX4elupkOTIJadsEi5zWZnWWIm5zULrvJxUxcl8caCRKaP6UGnGOd2ZVYbZVY7JpPzVFCz2YTdbszUSI2UEpFqt3pPFp+s2AvAlKs74euplxoREREREZEzMbZfPP/+egMdY4LpEhvEtD/3UFhq4dpusQCM/3I9kUHeTLi0DQDvLEzild928tr1XYgJ8SEtrxgAP093/LzcCfD2oGd8KFN+2oa3uxsxIT6s2J3JrLUHmHhluypz1CT9pSgi1aq4zMqEbzdit8Oo7jGGDAEVERERERFxdUM7R5NVUMorv+0kPa+EttGBTB/Tg/AAx/S9g9lFTqOePluxl1KrjX/OWOvUz78ubsmDlzhmr7xxY1demLeDB75cR3ZhGU1CfHh4SGtu6tm09k7sBCpKiUi1evP3RJLSCwgP8OKxy42ptouIiIiIiNQHo/vEMbpPXKX3fXlXb6ftpY9cdMr+IgK8+d+1nasjWrXQmlIiUm22Hspl6qIkAJ4a1p4gXw+DE4mIiIiIiEhdpZFSIlItLFYbE77diMVm57IOjbm0Q5TRkRqUd+Y7bnvSHdvtY+C/V8NlXY63Wb4LHvsKViaBmwm6NINfHgEfz8r7tNpg0rfw2VI4nA3RIXDrAJg4HI6NEjb9o/LHvnADPHxlNZ2ciKuaaTp1m/ruRmMWTRWR+uNkn3H2pEP8A5U/7qv74dqeld9nt8MT38L7f0B2AfRtBe+MgZaNj7eJ+xfszXB+3JTr4JGrzvWMROREKkqJSLWY9mcymw7mEOjtzuRh7Y2O0+DEhMJz1zs+TNntMH0JDHsZ1j3r+PC2fBdc+jw8ehW8MRrczbBhH5hP8jfz83MdHwKn3+3oY/VuuO09CPKB+y91tEl5y/kxP2+A29+HkT1q7lxFRESk4TjZZ5w20RU/i7z3O7z4I1x2ktlJL/wAr/8C0++C+Ah4/GsY8hxsfQG8T/iy7slr4I4Lj28HeFfvudUWI768LLPAxK/hp/WwO93x+XFQB8d/y+iQmj1fcS0qSonIOUvOKODl33YCMPHKdkS46ju2Cxt6nvP2M6McHz5WJDo+eDz4Kdw/xPnbvdbRJ+9z2U4Y1g2u6OrYjguHz5fDX7uPt2kc7PyY79bAhe2gecRZn4qIiIhIuVN9xvn7Z5HZq2FUT/Cv4uOo3Q6vznMUT4Z1d+z75J8QeQ/MWQPXn7BET4B3xf5dkRFfXhaWwto98PjV0LkpHCmAf30KV70Eq5+utVMXF6A1pUTknNhsdh75diMlFhv9WoRxbbcYoyM1eFYbfLEcCkqgdwtIy3F86xURCH0mQeQ/4YKn4M8dJ++nTytYsAV2pji2N+x1PKaqbx5Tc+DH9XD7BdV5NiIiIiIOf/+M83drkmH9Xrh9YNV9JKc7RvYMOmFgf5Av9ExwFGdO9NxcaHQXdP0/ePEHsFir4yxq39Dz4PIujqJUqyhHYc/f21HYA+cvL9vHOL64HNULvE6yPOyJX17GhcM1PWFwx+NfXgb5wm+POvppHQ29WsKbox3/jfZlVN2vNDwaKSUi5+SLVftZmZyFj4cbU0Z0dLokqdSuTfug9yQoLnN80Jj9ILSLgRVHP2BNmgX/u9ExHPuTJXDxs7D5eef1E070yFDILYI2D4Ob2fFB8Jlr4R99K28/fbHjG8UR59fI6YmIiEgDVdVnnL+bthDaRju+WKvK4WzHz8gg5/2RQcfvA0eR5rw4CPV3FGAe/RJSsuHlm87lTIxntcHXKyt+efmPvo4vL5NSHdMinxkF/VpX3U+fVo6pkjtTHIWuY19enuz5ySlyTO0L9q320xIXpqKUiJy1wznFTPlpGwAPDWlNbKjeYYzUOhrWP+t4w/9mJYyeCosmgu3oOsN3XQS3HR3F1DXOMQrqw4Uw5frK+/tqJcxYCjPHQfsmjm8eH/jMsQ7A6AEV23+4yPGBxruKtQdEREREzkZVn3FOLEwVlcLMZfD48Oo55vjLj/+7U1PwdIe7PnQsdn6yEUR1ldFfXhaXwoTP4YbeEKg/GeQEKkqJyFmx2+1MnLOJvBILXWKDubVPnNGRGjxPd2hx9INDt3hYtRte+8XxoQGgXRPn9m2jYV9m1f09PNPx2GNrK3Rs6rgKzZTvKxallmyHHSnw5X3Vcy4iIiIix1T1Gefd24+3+WYlFJbALf1P3texNaJScyDqhAW3U3McBZmq9GzhmL63J/3U63LWRUZ+eVlmgVFvgB1457YaOkFxWVpTSkTOyg8bU5i/LQ0PNxMvXNMJt5OthCiGsNmhpMwxzz86xFE0OtHOw9AsrOrHF5aC+W/vEm7m4x9eTjRtoeNDYueTfJgTERERqQ7HPuOcaNoiuOo8CA88+WPjwx2FqQVbju/LLXRMYevdsurHrd/rWPg7IqjqNnXZscJet3hHoalzU0dhLyrYcf+5fHnZsSnc3B8evNTx5eWJjhWk9mbAb49olJRUpKKUiJyxIwWlTPre8U4+7sIWtIoMMDiRPPoFLN7m+PZu0z7H9sJtjiHUJhM8fIXj0sffrITEw45LH28/5LwQ6MXPwpu/Ht8e2hWemQM/rnP0O3sVvPwzXN3d+di5hfD1XzB2ICIiIiLV6mSfcY5JPAyLt8PYCyvvo81Djs8x4Phc9MCl8PQc+H6No89bpkJ0MAzv5mizfBe8+rNjnaTdaY4RQQ9+Bjf1gxC/mjzb2lMbX14eK0jtOgzzH4VG+pNBKqHpeyJyxp76YSuZBaW0ivTnnoGVXPpEal1aruMDVUq242onnWLhlwlwSUfH/Q9c5lhD4MHPIKvA8e3Yb49CQuTxPpJSISPv+PYbo+Hxb+Cejxz9R4c4hnb/d4Tzsb9Y4bi88A19avw0RRqEd+Y7bnvSHdvtY+C/V8NlXZzb2e1w+Qswb6NjbZDh3St0VW7WKpg6H9bsgax8WPcMdIlzbjPwaVi0zXnfXRfB1NsRETHMqT7jgGNdy5hQx9XfKrMjBXIKj2//50rHQt93ToPsQujXCuZNOL4uppe74yp/k2Y5Cjfx4Y5RQCeuM+VKHv3CcfXkpmGQV+RYe2vhNsfzeOzLyye+dXw+7NIMpi9xfHn5zb+O93Hxs44vJu8d7Ng+9uVl00aO96l1exxfXo45OgWwzALXvAZr98APDznWnDq2kHyov2PklgioKCUiZ2jhjjRmrTuIyQTPj+yEp7sGXNYF0+48dZtHrnLcqrLnNeftAB949WbH7WTuvMhxE5HqERMKz13vWFzWbnf8cTDsZVj3rOOD/zGvznP8MXE6CoodV1Ea1Qvu+KDqdndcCE9ec3zbVxcuEBGDnc5nnGevc9yqYp/hvG0yOV7rTny9O9F58bDiydPPWNcZ8eXlwSPw/VrHv7v8n3OePx6Dge1q7HTFxagoJSKnLb/EwmOzNwMwpm88XZuGnOIRIiJypoae57z9zCjHyKkViceLUuv3wEs/wuqnIWrcqfu8+ejCv8dGX1XF1+v4IsAiIlI/GPHlZVx4xWKgSGU0xEFETtuL87ZzMLuI2FAf/j24ldFxRETqPavNMYWkoAR6H50tXVgCN74Fb91a/QWkGUsh7C7oMMEx3aOwpHr7FxERETmRRkqJyGlZvSeLT1bsBWDK1Z3w1URwEZEas2kf9J7kmE7h7+1YM6rd0VFSD34GfVrBsJOsIXU2buzjWNQ2Ohg27ocJnzvWYZn1YPUeR0REROQY/VUpIqdUXGZlwrcbsdthVPcY+rU8yaU4RETknLWOhvXPQk6R46qZo6fCoomQmAq/b3GsL1XdTlwbrmNTx2XCL37WsY7IieuKiIiIiFQXFaVE5JTe/D2RpPQCwgO8eOxyrUooIlLTPN2hRWPHv7vFw6rd8Nov4OMJSWkQfIdz+5GvQv82sHBi9WXomeD4maiilIiIiNQQFaVE5KS2Hspl6qIkAJ4a1p4gXw+DE4mINDw2u+Oy5JNHwtiBzvd1fAReuaniAunnar1jxjZRwdXbr4iIiMgxKkqJSJUsVhsTvt2IxWbnsg6NubRDlNGRRETqvUe/gMs6Q9MwyCuCmctg4TbH5bsbB1e+uHnTMIiPOL7d5iGYch1cfb5jOysf9mXAoWzH9o4Ux89j/SWlOo5zeRdo5A8b9znWrhrQBjo1rakzFRERkYZORSkRqdK0P5PZdDCHQG93Jg9rb3QcEZEGIS0XbpkKKdkQ5AudYh0FqUs6nn4fO1Igp/D49vdr4Lb3jm9f/6bj5xMjYNJIx3TB+Zvh1XmOK/3FhsLI82Hi8Oo4IxEREZHKqSglIpVKzijg5d92AjDxynZEBHgbnEhEpGGYdueZtbfPOPW+Wy9w3KoS2wgWPX5mxxURERE5V2ajA4hI3WOz2Xnk242UWGz0axHGtd1ijI4kIiIiIiIi9YyKUiJSwRer9rMyOQsfDzemjOiIyWQyOpKIiIiIiIjUMypKiYiTwznFTPlpGwAPDWlNbKivwYlERERERESkPlJRSkTK2e12Js7ZRF6JhS6xwdzaJ87oSCIiIiIiIlJPqSglIuV+2JjC/G1peLiZeOGaTriZNW1PREREREREakadvvrelClTmDVrFtu3b8fHx4c+ffrw/PPP07p16/I2AwcOZNGiRU6Pu+uuu5g6dWptxxVxabkevkz6fgsA4y5sQavIgDPvZKaKWNxoNzqBiIiIiIiIS6jTI6UWLVrEuHHjWLFiBb/99htlZWUMHjyYgoICp3Z33HEHKSkp5bcXXnjBoMQiruu9jleRWVBK68gA7hnYwug4IiIiIiIiUs/V6ZFS8+bNc9r++OOPiYiIYM2aNQwYMKB8v6+vL40bN67teCL1xqqINixo2h2TCZ4b2RFP9zpdrxYREREREZF6wKX+8szJyQEgNDTUaf+MGTMICwujQ4cOPProoxQWFp60n5KSEnJzc8tveXl5NZZZpK4rdPfijS4jARjTN56uTUMMTiQiIiIiIiINQZ0eKXUim83GAw88QN++fenQoUP5/htvvJFmzZoRHR3Nxo0bmTBhAjt27GDWrFlV9jVlyhQmT55cG7FF6ryP211Gum8IjQsy+ffgIUbHERERERERkQbCZYpS48aNY/Pmzfz5559O+++8887yf3fs2JGoqCguvvhikpKSSEhIqLSvRx99lPHjx5dvHzx4kHbt2tVMcJE6bHNoHD/E9wHg/vXf4Ot5i8GJREREREREpKFwiaLUvffeyw8//MDixYuJiYk5aduePXsCkJiYWGVRysvLCy8vr/Lt3Nzc6gsr4iJKze681nUUdpOZwXtX0jV9l9GRREREREREpAGp00Upu93Offfdx+zZs1m4cCHx8fGnfMz69esBiIqKquF0Iq7t89aDOBAQQUhxLmM3/2B0HBEREREREWlg6nRRaty4ccycOZPvvvuOgIAADh8+DEBQUBA+Pj4kJSUxc+ZMLr/8cho1asTGjRt58MEHGTBgAJ06dTI4vUjdtTswiq9bXgjAuA2zCCgrMjTPlO9g1mrYfgh8PKFPS3j+emgd7dxu+S547CtYmQRuJujSDH55xPGYU3nue3j0S/jXpfDqzY59e9Ih/oHK2391P1zb85xOS0RERERERE6iThel3nnnHQAGDhzotP+jjz7i1ltvxdPTk/nz5/Pqq69SUFBAbGwsI0eOZOLEiQakFXENVpOZV7uOwmp2o+/BjfRN2Wx0JBZth3GD4PwEsFjh/76Cwc/B1hfAz9vRZvkuuPR5ePQqeGM0uJthwz4wm07d/6okePd36NTUeX9sI0h5y3nfe7/Diz/CZZ2r59xERERERESkcnW6KGW32096f2xsLIsWLaqlNCL1w+yEAewKicW/tJB7Ns42Og4A8yY4b398F0T8E9Ykw4C2jn0Pfgr3D4FHrjre7u8jqSqTXwz/eBveHwtPz3G+z80MjYOd981eDaN6gr/3mZ6FiIiIiIiInAmz0QFEpPYc9Avj07ZDALhj81xCS/IMTlS5nELHz1B/x8+0HMeUvYhA6DMJIv8JFzwFf+44dV/jPoYrusCgDqduuyYZ1u+F2weeXW4RERERERE5fSpKiTQQNky81vVaSt086Jq2k0v2rTI6UqVsNnjgU+jbCjrEOvbtTnP8nDQL7rjQMbLqvDi4+FnYdbjqvr5YDmuTYcp1p3fsaQuhbTT0aXUuZyAiIiIiIiKnQ0UpOSdWTGwMS2Bhky5sDEvAymks8COGmNesB5vCEvCylHL/+m/q7H+pcR/D5gPwxb3H99mOzuS96yK47QLoGgev3Ayto+DDhZX3sz8T/vUJzBgH3qexEHpRKcxcplFSIiIiIiIitaVOryklddvSqA5M7TScDJ/g8n1hRdncvXFOnVg8W47L8A5kWocrARi97WcaF2YZnKhy934MP6yDxY9DTKPj+6OCHT/bNXFu3zYa9mVW3teaZEjLhfMeO77PaoPF2+HNX6FkumNNqWO+WQmFJXBL/+o4ExERERERETkVFaXkrCyN6sDTPUZX2J/hHcTTPUYz8a/pKkzVEXbgzc4jKfTwoXXWXq5K+tPoSBXY7XDfdMci4wsnQnyE8/1x4RAdAjtSnPfvPFz1VfIubg+bnnPed9t70CYKJgx1LkgBTFsEV50H4YHndi4iIiIiIiJyejR9T86YFRNTOw13bJj+Ngns6Pa7HYdpKl8dsbhJZ1ZGtcfdZuHBdV/hxsmvammEcR/DZ0th5jgI8IbD2Y5bUanjfpMJHr4CXv/FMaIp8TA8/jVsP+Q83e7iZx2joAACfBxrUp148/OCRgHH16o6JvGwYwTV2Atr/lxFRERERETEQSOl5IxtCWvuNGWvApOJdN8QtoQ1p1NGUq3lkopyPXx5p9PVAFy/YwHN8lINTlS5d+Y7fg582nn/R3fCrRc4/v3AZVBcBg9+BlkF0Lkp/PYoJEQeb5+UChlncUHBDxdBTCgM7nh2+UVEREREROTMqSglZyzLK+C02iUFRqkoZbD3Ol5Fjpc/cTkpjNr5u9FxqmSfcXrtHrnKcavKntdO/viFEyvf/+x1jpuIiIiIiIjUHhWl5IyFlpzeUJT3Ol7FpvAWDEtaQqeMJE3mq2WrItqwoGl3zHYbD6z7Cg+71ehIIiIiIiIiIuVUlJIz1j5jN2FF2WR4B1VcUwrAbsfDZqHMzYPlUR1YHtWB+JxDDEtawoUH1uFps9R+6Aam0N2LN7qMBGBY0hJaZ+83OJGIiIiIiIiIMy10LmfMDTt3b5zj2LD/bdHso9sTVs/g3fkvcMXuZXhZSkkOiubV867j5iET+aTtEDK9dYmzmvRxu8tI9w2hcUEmt2z7xeg4IiIiIiIiIhWoKCVnpW/KZib+NZ2w4hyn/eFF2Uz8azp9UzbTND+NezfO4tNfnuL2zT8QUXiEXC9/Pm99CaMHP8YL3W5gR3BsFUeQs7U5NI4f4vsAcP+6r/G2lhqcSERERERERKQiTd+Ts9Y3ZTO9UrawJaw5WV4BhJbk0T5jN244j54KKCvimsSFXJ20mGVRHfiueT+2hDXnj9hu/BHbjbaZexietIQ+KZtwt9sMOpv6odTszmtdR2E3mRm8dyVdMxKNjiQiIiIiIiJSKRWl5Jy4YT/tK+y52W30P7SR/oc2siuoCd8l9GdRTBe2NYpjW6M4wgqzGZq8lEv3rCSwrLCGk9dPn7cexIGACEKKcxm7+Qej44iIiIiIiIhUSdP3xBAtcw7y0NovmP7LM9y4/VeCi/PI8A3mo/ZXcMuQibzeeSR7AyKNjulSkoKi+brlhQCM2zCLgLIigxOJiIiIiIiIVE0jpcRQoSV53Lz9V67buYBFTboyJ6E/u4Ob8HN8b36O703XtB0MT/qT7qnbMf9tWqAcZzWZebXrKKxmN/oe3EjflM1GRxIRERERERE5KRWlpE7wtFm5ZP9qBu1fzeZGzfkuoR/LozqwLqI16yJa0yQ/nauS/mTQ/tX4WkqMjlvnzE4YQGJwDP6lhdyzcbbRcUREREREREROSUUpqVNMQMfM3XTM3E2qbwjfx/fll7ieHPQP553OVzO93aVcuvcvhu5eSuPCLKPj1gkH/cL4tO0QAO7YPJfQkjyDE4mIiIiIiIicmopSUmdFFh7hji0/cNP2X5nftDvfNe/HwYAIZrW4gDkJ/emZsoXhSUvomLkbk9FhDWLDxGtdr6XUzYOuaTu5ZN8qoyOJiIiIiIiInBYVpaTO87GWMjR5GVckL2dNRGvmJPRnbWRrlkd3ZHl0R5rnHGRY0p/El1nx9nAzOm6tmtesB5vCEvCylHL/+m8abHFOREREREREXI+KUuIyzNg5P20756dtZ29AJN8378eC2G7sDmrCK+ddxyfP/c4/ejblpl7NiAj0NjpujUv3DmJahysBGL3tZ01nFBEREREREZdiNjqAyNlolpfKfRu+5dNfnmLM5h8ILzxCZkEpr/+eSN/nf+fBL9ez8UC20TFrjB14q/MICj18aJ21l6uS/jQ6koiIiIiIiMgZ0UgpcWkBZUVcm7iQEUmL2fvlfD5amszqvUeYve4gs9cdpHuzEG7rG8+Q9pG4u9WfGuziJp1ZGdUed5uFB9d9hRt2oyOJiIiIiIiInBEVpaRecLPbuKJTFFd0imLjgWw+WrqHHzYeYvXeI6zee4ToIG9u6RPH9efHEuzraXTcc5Lr4cs7na4G4PodC2iWl2pwIhEREREREZEzV3+Gjogc1SkmmFeu68LSCRdx/0UtaOTnyaGcYp77eTu9pizgsdmbSEzLMzrmWXuv41XkePkTl5PCqJ2/Gx1HRERERERE5KyoKCX1VkSgN+MHt2bpIxfx4jWdaBsVSHGZjRkr9zHo5cXcPG0lf+xIw2ZznalvqyLasKBpd8x2Gw+s+woPu9XoSCIiIiIiIiJnRdP3pN7z9nDj2u6xXNMthpXJWXz4ZzK/bUtlya4MluzKoHmYH7f2jaODmyc+1lKj41ap0N2LN7qMBGBY0hJaZ+83OJGIiIiIiIjI2VNRShoMk8lEr+aN6NW8EfuzCpm+bA9frtrP7owC/vvdFvwufZwhe1ZyVfJSIguPGB23go/bXUa6bwiNCzK5ZdsvRscREREREREROScqSkmDFBvqy8Qr2/HAJa34ds0BPl62h+QMmNVyIHNaDKB3ymaGJf1Jh8zdmIwOC2wOjeOH+D4A3L/ua7zr8IguERERERERkdOhopQ0aP5e7ozuE8fNvZrx6YUjmZPQj3URrVka3Yml0Z1IyD7AsKQ/ueDgOjxtxqzfVGp257Wuo7CbzAzeu5KuGYmG5BAREREREZHa9cnyPby7aDfp+SW0jQpk8lXt6RIbXGnbz//ax6y1B9hx2HFhr44xQTw8pE2F9olpeTz383ZW7s7CYrPTMtKfd27qRpNgnxo+m4pUlBIBzGYTPVK30SN1G3sDIvmueT9+j+1GUnAML3e7ng/bX8EVe5ZzefJyQktq98p9n7cexIGACEKKcxm7+YdaPbaIiIiIiIgYY+6GQzz9wzaevroDXWOD+XBpMrdMW8nvDw0kzN+rQvsVuzO5qnM0510Vgpe7G1MXJXHztJX89uAFNA7yBmBvZgHXTF3Odd1jeWBQKwK83dmZmo+XuzHXwVNRSuRvmuWlcv+Gb7l168/Mi+vJ3Pi+ZPgGM6PNYL5sdREXHFjPsKQltMw5WONZkoKi+brlhQCM2zCLgLKiGj+miIiIiIiIGO+DP5O5vkcso7rHAvDM8I78vj2Nr1bv556BLSq0f+36rk7bz4/sxLzNh1mamMHIbjEAvPjLDi5sHcGjl7ctb9eskV8NnsXJqSglUoXAskJG7fqDEYmLWBbVkTkJ/dnWKI4FTbuzoGl32mfsZvjuJfRO2YKb3Vbtx7eazLzadRRWsxt9D26kb8rmaj+GiIiIiIiI1J68vDxyc3PLt728vPDyqjjqqdRiY/PBHO4ZmFC+z2w20bdFGGv3Zp/WsYrKrJRZbQT7egBgs9n5Y3sad12QwM3TVrL1UC4xob7cMzCBIe0bn9uJnSVjxmeJuBB3u40Bhzbw8pI3eXXha1y4fw1uNitbwprzTI/RjLnkUb5pMZA8j+qdfzs7YQCJwTH4lxZyz8bZ1dq3iIiIiIiI1L527doRFBRUfpsyZUql7Y4UlmK12StM0wv39yI9v+S0jvXcz9uIDPSmb4swADIKSigotfLOwiQuaBXOJ7f3YEj7SO7+bA0rdmee24mdJY2UEjkDrbP38581n3P7lh/5Mb43P8b1Js03hGkdruSzNoMZtG81w3YvITY//ZyOc9AvjE/bDgHgzs3f1/o6ViIiIiIiIlL9tm7dSpMmTcq3KxslVR3eXpjI3A0pfHFnL7w93ACw2x33XdIukrH9mwPQPjqItXuPMGPlPno1b1QjWU5GRSmRs9CoOJdbtv3C9TsW8EdMV75L6E9yUDQ/Nu/Dj8370D11O8OSlnBe2k7M2M+obxsmXut6LaVuHnRN28Ggfatr6CxERERERESkNgUEBBAYGHjKdiG+nriZTWT8bVRUen4J4ZUscn6i9xYn8c7CJGaM7UnbqOPHCvH1xN1somWEv1P7hAh/Vu85cgZnUX1UlBI5B542C0P2rWLwvlVsDEvgu4T+rGjcjtWRbVgd2YaYvDSGJS1h0P41eFtLT6vPec16sCksAS9LKfev/xZTDZ+DiIiIiIiI1C2e7mY6NAliWWJG+XpPNpudZYmZ3NKnWZWPm7ooibd+T2T67T3oFBNcoc9OMUHszihw2p+cXkCT4OpdjuZ0qSglUg1MQOeMJDpnJJHiG8r3zfvxS7MeHAiI4K0uI5ne7jKG7P2LobuXElnkXIG2YmJLWHOyvAIw22180OFKAG7d+hONC7MMOBsREREREREx2th+8fz76w10jAmmS2wQ0/7cQ2GphWu7Oa7GN/7L9UQGeTPh0jYAvLMwiVd+28lr13chJsSHtLxiAPw83fHzcpR/7hyQwH2fr6VHfCi9mzdi0c50FmxP44s7exlyjipKiVSzqMIs7tr8PTdv/4Xfmnbn++b9OOQfzrctBzK7xQB6H9rM8KTFtM/aw7KoDkztNJwMn2CnPqLz0hi6e6kxJyAiIiIiIiKGG9o5mqyCUl75bSfpeSW0jQ5k+pgehAc4pu8dzC7CZDo+t+azFXsptdr454y1Tv386+KWPHhJKwAu7dCYZ4Z35O2FiUz6fgvNw/155x/ncX5caO2d2AlUlBKpIb6WEobtXsrQ3ctYFdmG7xL6sy6iFUubdGJpk040LsjgsG8lC8nZ7RzyD2dFVHv6pmyu/eAiIiIiIiJSJ4zuE8foPnGV3vflXb2dtpc+ctFp9Tnq/FhGnR97rtGqhYpSIjXMjJ2eqdvombqNPQGN+S6hHwtiu3HYL6zyB5hMYLfzbsdh9ErZgtsZLpQuIiIiIiIi4grMRgcQaUji8g7zr/Xf8MiqT0/e0GQi3TeELWHNayeYiIiIiIiISC1TUUrEAKVunqfVLssroIaTiIiIiIiIiBhDRSkRA4SW5FVrOxERERERERFXo6KUiAHaZ+wmrCgb7FWsF2W3E154hPYZu2s1l4iIiIiIiEhtUVFKxABu2Ll74xzHxt8LU0e379r0nRY5FxERERERkXpLRSkRg/RN2czEv6YTVpzjtD+8KJuJf02nb8pmg5KJiIiIiIiI1Dx3owOINGR9UzbTK2ULW8Kak+UVQGhJHu0zdmuElIiIiIiIiNR7KkqJGMwNO50ykoyOISIiIiIiIlKrNH1PRERERERERERqnYpSIiIiIiIiIiJS61SUEhERERERERGRWqeilIiIiIiIiIiI1DoVpUREREREREREpNapKCUiIiIiIiIiIrVORSkREREREREREal1KkqJiIiIiIiIiEitU1FKRERERERERERqnYpSIiIiIiIiIiJS61SUEhERERERERGRWqeilIiIiIiIiIiI1DoVpUREREREREREpNapKCUiIiIiIiIiIrVORSkREREREREREal1KkqJiIiIiIiIiEitU1FKRERERERERERqnYpSIiIiIiIiIiJS61SUEhERERERERGRWldvilJvvfUWcXFxeHt707NnT/766y+jI4mIiIiIiIiISBXqRVHqyy+/ZPz48TzxxBOsXbuWzp07M2TIENLS0oyOJiIiIiIiIiIilXA3OkB1ePnll7njjju47bbbAJg6dSo//vgjH374IY888kiF9iUlJZSUlJRv5+TkAJCSklI7gWvA4bJSoyMYLuDAgXN6vJ7Dc38OyayeHC5Nz+G50+/yOTvn32X0PIJeE6uFnsNzp9fEc6bXxOqh18RqoOfw3FXD77NRjtUbbDabwUnqFpPdbrcbHeJclJaW4uvryzfffMPw4cPL948ePZrs7Gy+++67Co+ZNGkSkydPrsWUIiIiIiIiItLQ/fXXX5x//vlGx6gzXH6kVEZGBlarlcjISKf9kZGRbN++vdLHPProo4wfP75822KxsG3bNmJjYzGb68WMxlqVl5dHu3bt2Lp1KwEBAUbHcUl6DquHnsdzp+fw3Ok5PHd6Ds+dnsPqoefx3Ok5PHd6Ds+dnsPqoefx3NhsNlJTU+natavRUeoUly9KnQ0vLy+8vLyc9vXt29egNK4vNzcXgCZNmhAYGGhwGtek57B66Hk8d3oOz52ew3On5/Dc6TmsHnoez52ew3On5/Dc6TmsHnoez13Tpk2NjlDnuPywoLCwMNzc3EhNTXXan5qaSuPGjQ1KJSIiIiIiIiIiJ+PyRSlPT0+6devGggULyvfZbDYWLFhA7969DUwmIiIiIiIiIiJVqRfT98aPH8/o0aPp3r07PXr04NVXX6WgoKD8anxSs7y8vHjiiScqTImU06fnsHroeTx3eg7PnZ7Dc6fn8NzpOaweeh7PnZ7Dc6fn8NzpOaweeh6lJrj81feOefPNN3nxxRc5fPgwXbp04fXXX6dnz55GxxIRERERERERkUrUm6KUiIiIiIiIiIi4DpdfU0pERERERERERFyPilIiIiIiIiIiIlLrVJQSEREREREREZFap6KUiIiIiIiIiIjUOhWl5Kzs27ePytbIt9vt7Nu3z4BEIiLGWbx4MRaLpcJ+i8XC4sWLDUgkIiKuTO8rItJQqCglZyU+Pp709PQK+7OysoiPjzcgkWv65JNPKCkpqbC/tLSUTz75xIBEInI2LrzwQrKysirsz8nJ4cILLzQgket58sknKSwsrLC/qKiIJ5980oBEIlT6Hi2n5ubmRlpaWoX9mZmZuLm5GZDI9eh9RUQaCpO9suEuIqdgNptJTU0lPDzcaf/evXtp164dBQUFBiVzLW5ubqSkpBAREeG0PzMzk4iICKxWq0HJXIPNZmPRokUsWbKEvXv3UlhYSHh4OF27dmXQoEHExsYaHbHO27ZtG1988UWlz+GQIUMYOXIkXl5eRses86p6Tdy5cyfdu3cnNzfXoGSuQ6+H5y47O5vZs2dX+fvcp08foyPWeT///HP5a+L+/fux2Wz4+fnRtWtXBg8ezG233UZ0dLTRMes8s9nM4cOHK/w+Hzp0iISEBIqKigxK5jr0vlI9kpOTK31N7N27N97e3kbHq/P0viK1wd3oAOJaxo8fD4DJZOLxxx/H19e3/D6r1crKlSvp0qWLQelcj91ux2QyVdh/4MABgoKCDEjkGoqKinjppZd45513yMrKokuXLkRHR+Pj40NiYiJz5szhjjvuYPDgwfz3v/+lV69eRkeuc9auXct//vMf/vzzT/r27UvPnj25+uqr8fHxISsri82bN/PYY49x33338Z///IcHHnhAxalKjBgxAnC8Jt56661Oz5HVamXjxo36wHaaqno93LBhA6GhoQYkch2HDh3iv//9LzNmzCA6OpoePXrQpUuX8t/nP/74g//97380a9aMJ554guuuu87oyHXO7NmzmTBhAnl5eVx++eVMmDCh/H3l2Gvi/Pnzeeqpp7j11lt56qmnKhQLBF5//XXA8Zr4wQcf4O/vX36f1Wpl8eLFtGnTxqh4LkHvK9VjxowZvPbaa6xevZrIyEin3+ekpCS8vb35xz/+wYQJE2jWrJnRcescva9IbVJRSs7IunXrAMcfD5s2bcLT07P8Pk9PTzp37sxDDz1kVDyX0bVrV0wmEyaTiYsvvhh39+O/ilarleTkZC699FIDE9ZtrVq1onfv3rz//vtccskleHh4VGizd+9eZs6cyfXXX89jjz3GHXfcYUDSumvkyJE8/PDDfPPNNwQHB1fZbvny5bz22mu89NJL/N///V/tBXQRx4rHdrudgIAAfHx8yu/z9PSkV69e+n/vFEJCQspfD1u1auVUmLJareTn53P33XcbmLDu69q1K6NHj2bNmjW0a9eu0jZFRUXMmTOHV199lf379+u9+m9eeOEFXnnlFS677DLM5oqrW4waNQqAgwcP8sYbb/DZZ5/x4IMP1nbMOu+VV14BHK+JU6dOdZqq5+npSVxcHFOnTjUqnkvQ+8q569q1K56entx66618++23FUbOl5SUsHz5cr744gu6d+/O22+/zbXXXmtQ2rpJ7ytSmzR9T87KbbfdxmuvvUZgYKDRUVzS5MmTy3/++9//dvom8diHtpEjRzoV/eS4bdu20bZt29NqW1ZWxr59+0hISKjhVK6lrKys0mJedbVvaCZPnszDDz/sNHpUTs/06dOx2+2MGTOGV1991WmU6LHXw969exuYsO7LzMykUaNGNdZe5ExdeOGFzJo1i5CQEKOjuKzJkyfz0EMP4efnZ3QUl/PLL78wZMiQ02qbmZnJnj176NatWw2nci16X5HapKKUnJWcnBysVmuFKRVZWVm4u7urWHWapk+fznXXXac57SIuLjk5GYvFQsuWLZ3279q1Cw8PD+Li4owJ5kIWLVpEnz59VPwUERERaUBUlJKzctlllzF06FDuuecep/1Tp07l+++/56effjIomTQEGzduPO22nTp1qsEkruvYuh+n4/7776/BJPXDBRdcwJgxYxg9erTT/s8++4wPPviAhQsXGhOsjsvNzS3/EuNUi/bqy46qff/996fd9qqrrqrBJK7r2JqZp+Pll1+uwST1w8iRI+nRowcTJkxw2v/CCy+watUqvv76a4OS1W3nnXceCxYsICQkpHyph6qsXbu2FpO5ljNZBF7vLZXT+4rUJhWl5KyEhoaydOnSClOotm/fTt++fcnMzDQoWd0XGhrKzp07CQsLK19LpSqVXQpYHFekMZlMVS6MfCJdsaty8fHxTtvp6ekUFhaWry+VnZ2Nr68vERER7N6924CEriUwMJC1a9fSokULp/2JiYl0796d7OxsY4LVcSdece/Y7/XfHfs91+9y1f6+BtKx18cTt4/R81i5Cy+80Gl77dq1WCwWWrduDTiueObm5ka3bt34/fffjYjoUsLDw/n999/p2LGj0/5NmzYxaNAgUlNTDUpWt504FfzYUg9VeeKJJ2opleup6v2kMnpNrJzeV6Q2aaFzOSslJSVYLJYK+8vKynSZ31N45ZVXCAgIAODVV181NoyLSk5OLv/3unXreOihh3j44YfL151Zvnw5L730Ei+88IJREeu8E5/DmTNn8vbbbzNt2rTyP8B27NjBHXfcwV133WVURJdiMpnIy8ursP/YVGep3O+//14+DfyPP/4wOI3rstls5f+eP38+EyZM4Nlnn3V6TZw4cSLPPvusURHrvBP//3v55ZcJCAhg+vTp5WsiHTlyhNtuu43+/fsbFdGl5OfnV7oupoeHxxmNYmloTiw0qeh09k78fd6zZw+PPPIIt956q9Nr4vTp05kyZYpREes8va9IbdJIKTkrF154IR06dOCNN95w2j9u3Dg2btzIkiVLDEomDU2PHj2YNGkSl19+udP+n376iccff5w1a9YYlMx1JCQk8M0339C1a1en/WvWrOGaa65xKmBJ5YYOHYqPjw+ff/55+dWmrFYr1113HQUFBfz8888GJ5SGokOHDkydOpV+/fo57V+yZAl33nkn27ZtMyiZ62jSpAm//vor7du3d9q/efNmBg8ezKFDhwxK5jp69OjBlVdeyX//+1+n/ZMmTWLu3Ll6bz4Dq1evLv+9bdeunRbkPkMXX3wxY8eO5YYbbnDaP3PmTN577z1Nrz8Nel+RmqaRUnJWnn76aQYNGsSGDRu4+OKLAViwYAGrVq3i119/NTida7FarcyePdvpA8ewYcNwd9ev5+nYtGlThalo4JietnXrVgMSuZ6UlJRKRz5arVZNsThNzz//PAMGDKB169blIymWLFlCbm6upvqcgSNHjjBt2jSn18PbbrutwkU1pGpJSUnl03BPFBQUxJ49e2o9jyvKzc0lPT29wv709PRKR0RKRY8//jgjRowgKSmJiy66CHB8Tvz888+1ntRpOnDgADfccANLly51mlrfp08fvvjiC2JiYowN6CKWL1/O1KlTK+zv3r07Y8eONSCR69H7itQ086mbiFTUt29fli9fTmxsLF999RVz586lRYsWbNy4UUPbz8CWLVto1aoVo0ePZvbs2cyePZvRo0fTsmVLNm/ebHQ8l9C2bVumTJlCaWlp+b7S0lKmTJlSYc0zqdzFF1/MXXfd5bRo6po1a/jnP//JoEGDDEzmOtq1a8fGjRsZNWoUaWlp5OXlccstt7B9+3Y6dOhgdDyXsHjxYuLi4nj99dc5cuQIR44c4fXXXyc+Pp7FixcbHc9lnH/++YwfP96poJyamsrDDz9Mjx49DEzmOq6++mpuu+02Zs2axYEDBzhw4ADffvstt99+OyNGjDA6nksYOnQoc+bMITExkXvuuYd///vfHDhwgPnz5zN8+HCj47mEsWPHUlZWxrZt28jKyiIrK4tt27Zhs9lUTDkDsbGxvP/++xX2f/DBB8TGxhqQyPXofUVqmqbviRiod+/ehIeHV1i34tZbbyU9PZ1ly5YZnLDu++uvvxg6dCh2u738SnsbN27EZDIxd+5cvVmehvT0dEaPHs28efPw8PAAwGKxMGTIED7++GMiIiIMTigNQceOHenduzfvvPOO0xTIe+65h2XLlrFp0yaDE7qGxMRErr76anbu3Fn+B9f+/ftp2bIlc+bMqbAYv1RUWFjIQw89xIcffkhZWRkA7u7u3H777bz44ov4+fkZnFAaAh8fH5YtW1bp1Pr+/ftTWFhoUDLX8tNPPzFy5EhatGhBz549Acdnx127dvHtt99WWP5BKtL7itQ0FaXkrOzbt++k9zdt2rSWkrg2Hx8fVq9eXem6Feeff74WjT9NBQUFzJgxg+3btwOO0VM33nij/nA4Qzt37ix/Dtu0aUOrVq0MTuQ6TjWSZ8CAAbWUxHX5+Piwfv368sX2j9mxYwddunTR6+EZsNvt/Pbbb06viYMGDTrtq1GJQ0FBAUlJSYBj7T29p0htatWqFZ999lmFL9f++usvbrzxRhITEw1K5nr279/PO++84/SaePfdd2uk1BnQ+4rUJC1aI2clLi7upC9CutrU6WnVqhWpqakVilJpaWn61uEM+Pn5ceeddxodw+XFxcVht9tJSEjQmmZnaODAgRX26XLJZ+a8885j27ZtFYpS27Zto3Pnzgalck0mk4nBgwczYMAAvLy89EfDWUpJSSElJYUBAwbg4+OD3W7Xc3mazGazPieeoxdffJH77ruPt956i+7duwOORc//9a9/8b///c/gdK4lNjZWV4k7R3pfkZqkvzrkrKxbt85pu6ysjHXr1vHyyy/zzDPPGJTKNZx4KeQpU6Zw//33M2nSJHr16gXAihUrePLJJ3n++eeNiuhyPv30U9599112797N8uXLadasGa+88grNmzdn2LBhRser8woLC7nvvvuYPn064Bgx1bx5c+677z6aNGnCI488YnDCuu/IkSNO28deEx9//HG9Jp7Exo0by/99//33869//YvExESn18O33nqL5557zqiILsdms/HMM88wdepUUlNTy3+fH3/8ceLi4rj99tuNjljnZWZmMmrUKP744w9MJhO7du2iefPm3H777YSEhPDSSy8ZHbHOmz17ttP2sdfE6dOnM3nyZINS1X0hISFOf+wXFBTQs2fP8i+KLBYL7u7ujBkzRmtznYElS5aUf078+uuvadKkCZ9++un/s3fnYTXm///An+eUUimVhEhpLE1aSVSohJRBmBkzQmoYjX0djCl1jGwzpMFgbDXGx5p9mWhXjLVCSKRCMUoSaT2/P/w6X8dpOZ2Wd3e9Htflupz7Pn88r3N1Xvd93vf7/XqjS5cuEjvKEUl0XSH1jZbvkTp1+vRprFu3jrZXrcKnTw/Lv4Llxz5+TU8Sq/fHH3/Ax8cHc+fOxS+//II7d+7AwMAAe/bsQVBQECIiIlhHbPTmzJmD2NhYBAQEYNiwYUhMTISBgQGOHz8OX19fiUFoIr2oqCjMnz+ftj+vRHk9rO5WhOqh9AQCAYKCgiAQCDB16lTcvn0bBgYGOHDgAAICAnDp0iXWERu9SZMm4cWLF9ixYwc+//xzJCQkwMDAAP/88w/mz5+PO3fusI7IWfv27cOBAwdw/Phx1lEapfKHQ9Jwd3evxyRNx5EjRzBx4kS4ubnhr7/+QlJSEgwMDLBp0yacOXMGZ86cYR2x0aPrCqlvNChF6lRKSgrMzMzw9u1b1lEaraioKKnfa2dnV49JmgYjIyP4+/vD1dUVqqqqoh8Pt2/fhr29PV6+fMk6YqOnp6eHAwcOoF+/fmKfYUpKCnr16iU2u4/UzL1792BpaYn8/HzWURqltLQ0qd+rp6dXj0majq5du2Lbtm1wdHQU+z7fu3cP1tbWErP6iKT27dvjn3/+gZmZmdhn+OjRI5iamtL3uRboMyQNzcLCAvPmzcOkSZPEvs83b96Es7MzsrKyWEds9Oi6QuobLd8jMvn0R6pQKERmZiZ8fX3RrVs3Rqm4gQaa6lZqaqrEzjQAoKioSIOjUvrvv/8q3GHv7du31DNASh8vQwP+ryauXr0a5ubmbEJxAA001b2nT59W2JOwrKxMtJMcqdrbt2+hrKwscTwnJweKiooMEjUNBQUFCAwMRMeOHVlH4SQPDw+sXLkSOjo6rKNwyv379yvcbKR169bIzc1t+EAcRNcVUt9oUIrIRF1dXeLHqlAohK6uLvbv388oFfcNGjQIu3fvph9qNdClSxfEx8dLfGbnzp3D559/zigVt1haWuL06dOYNWsWgP9bSrpjxw5YW1uzjMYZ5ubmFS5D69evH3bt2sUoFbeVL5eiBx01Y2RkhJiYGImaePjw4QoH8ImkAQMGIDg4GCtWrADwoSaWlZVh7dq1cHBwYJyOGz7tjSQUCvHmzRsoKytj7969DJM1fp8+5Cj3999/Y9SoUTAwMAAAmJqaNmQszmrfvj1SUlKgr68vdvzixYuiz5JUja4rpL7RoBSRyad9evh8Ptq2bYuuXbvSrl1SOHHiRIXHo6OjcerUKdEWtSNHjmzIWJw0f/58zJgxA+/fv4dQKMSVK1fwv//9D6tWrcKOHTtYx+MEf39/ODs7IykpCSUlJdi4cSOSkpIQFxdXo+WmzVlqaqrY6/Ka2LJlS0aJuCMwMLDC4+np6di9ezfat28P4EMjdFI9Hx8fuLu74+nTpygrK0NISAju37+P4OBgnDp1inU8Tli7di0cHR1x7do1FBUV4ccff8SdO3eQk5OD2NhY1vE4ISAgQOx1eU3s27cvNDQ02ITiiMoecgDA2LFjRbtAUp896UydOhVz5szBrl27wOPx8OzZM1y6dAkLFy6Et7c363icQNcVUt+opxSpseLiYkybNg3e3t7o0qUL6zicJE1zX7rhkN7ff/8NX19fPHz4EACgo6MDPz8/2g2kBh4+fIjVq1cjISEB+fn56NWrFxYvXgwTExPW0Rq94uJiDBs2DFu3bqVZPTLg8/no2LGjxAONtLQ06OjooEWLFuDxeHj06BGjhNwTExMDgUAg9n328fHB0KFDWUfjjNevX2PTpk1in+GMGTPQoUMH1tEavZKSEvj7+8PT0xOdOnViHYdzzM3N0alTJ/z6669QUlIC8GGmWbdu3XD27FnRdYZm1UtHKBTC398fq1atwrt37wB8aPGwcOFC0WxIUj26rpD6RINSRCatW7dGfHw8DUrJyNnZGXJycti1a5dYL58WLVogISEBRkZGDNNx17t375Cfn19hfyRC6lPbtm0RFxdHg1Iy8PLywr///ot9+/aJLbmlekgId6mqquLWrVsSS6ZI9cpn550/fx579+4VLY+imlg7RUVFSElJQX5+PoyMjNCqVSvWkQgh/x+tsyIycXV1xbFjxzBv3jzWUTjp7Nmz2LBhAywtLbFlyxZ88cUXrCM1CcrKyhU2pyXVKysrQ0pKCl68eIGysjKxcxU1CCXiJkyYgJ07d2L16tWso3DO1q1bcfToUTg5OeHHH3/EzJkzWUdqEoqKiir8Pnfu3JlRIm7Jzc3FlStXKvwMJ02axCgVdwwaNAhR20urCwAAl6xJREFUUVE0KCUDBQUFBAQE4OzZsxg5ciSmT5+OxYsXs47FeQoKCjSgV0t0XSH1hQaliEy6desGgUCA2NhY9O7dGyoqKmLnqfdH9ebNmwcHBwe4ubnh5MmT2LBhA+tInPT8+XMsXLgQYWFhePHihcSSSFoCWb3Lly9j/PjxSEtLk/j8aBmpdEpKSrBr1y5cuHChwpq4fv16Rsm4YfTo0bCyssKkSZNw+vRp7N69m3Ukznrw4AE8PT0RFxcndpz60Ejv5MmTcHNzQ35+PtTU1MQadvN4PBqUkoKzszOWLFmCW7duVVgTqWdm9ZydnXHt2jV4eHjg7NmzrONw1tu3b7F69WrRfeKnAyq0NLx6dF0h9Y2W7xGZVLVsj3p/1ExBQQHmzZuH8PBwPHr0CImJifQkpwacnZ2Rnp6OmTNnokOHDhK7Qo4aNYpRMu4wNzdH9+7d4efnV+Fn2Lp1a0bJuKO6Hbk+3RyCVEwoFGL16tUIDAzEf//9R/VQBra2tpCXl8eSJUsq/D6bmZkxSsYd3bt3h4uLC/z9/Wn2rYz4fH6l5+hHbM0FBgYiIiICv//+O/XpqqFvv/0WUVFRmDhxYoU1cc6cOYyScQddV0h9o0EpQhqJEydOICIiAkuXLqWeSDWgqqqKmJgYmJubs47CWSoqKkhISEDXrl1ZRyEEAHDjxg3ExMRg0qRJtFNXDamoqOD69eswNDRkHYWzVFRUcOvWLdounpAmQF1dHadPn4atrS3rKJxF1xVS3yp/jEFIFQQCgWgHi48VFBRAIBAwSMR9vXr1wq+//koDUjWkq6tb5S6GpHp9+/ZFSkoK6xic5unpiTdv3kgcf/v2LTw9PRkk4raCggJ4eXnRgJQMjIyM8PLlS9YxOM3JyQnXrl1jHYPTgoODUVhYKHG8qKgIwcHBDBJxm5qaGq1CkJGGhgY0NTVZx+A0uq6Q+kYzpYhM5OTkkJmZKTGAkp2dDW1tbZqWLQM1NTXEx8fTk9kaCg0NxW+//YZt27ZRQ1UZHT16FD///DMWLVoEExMTtGjRQuy8qakpo2TcUVlNfPnyJdq3b4+SkhJGybiJ6qHswsPD8fPPP8Pf37/C77OamhqjZNyxc+dOCAQCeHh4VPgZUj+k6tF9Yt1SVVVFQkIC1UQZ7N27F8ePH0dQUBAtx5URXVdIfaNBKSITPp+P58+fo23btmLHw8PDMW7cOPz333+MknEX3XDIRkNDA+/evUNJSQmUlZUlLpQ5OTmMknFHRb0/eDweNbCUQl5eHoRCITQ0NPDgwQOxmlhaWoqTJ09iyZIlePbsGcOU3EP1UHbl3+dPe37Q91l61A+p9iq7T0xISICDgwNdm2uIaqLsLCws8PDhQwiFQujr60vcJ964cYNRMu6g6wqpb7T7HqkRDQ0N8Hg88Hg8dO/eXaw4lZaWIj8/H15eXgwTkuYmICCAdQTOS01NZR2Bs9TV1cVq4qd4PB78/PwYJCPNFTXVr71Pd+ci0rOwsBDVREdHR8jL/99PjdLSUqSmpmLYsGEME3LThAkTaDaKjFxdXVlH4Dy6rpD6RjOlSI0EBQVBKBTC09MTAQEBYrtyKSgoQF9fH9bW1gwTcteqVavwww8/QF1dnXUUQoiUoqKiIBQKMWjQIBw5ckSsb4WCggL09PSgo6PDMCE37du3D6NGjZLYRp4Q0riVD8L7+flhwYIFaNWqlehc+X3i2LFjoaCgwCoiIYSQRoYGpYhMoqKiRNuDkrpR/lX8dGosqVp6enqV5zt37txASbiruqazkyZNaqAk3JWWlobOnTvT97eOZGRkAPiwkQGpmejo6CrPDxw4sIGScFd1G7b4+Pg0UBLuCgoKwjfffANFRUXWUTgtLCwMGzZswN27dwEAn3/+OebOnYvBgwczTkaaE7qukPpGg1JEJjdu3ECLFi1gYmICADh+/Dh2794NIyMj+Pr60hOwGti5cyc2bNiABw8eAAC6deuGuXPnYsqUKYyTcQOfz69yIIDWuVfv0x3OiouL8e7dOygoKEBZWZl6f0jh3LlzaNWqFfr37w8A2Lx5M/78808YGRlh8+bNtIucFEpKSuDn54fAwEDk5+cDAFq1aoVZs2Zh+fLlEn1ASMUq6xFXjmpi9SwsLMReFxcXIzU1FfLy8vjss8+oB40UMjIywOPx0KlTJwDAlStXsG/fPhgZGeH7779nnI4btmzZgjlz5uDLL78UrUK4fPkyDh8+jA0bNmDGjBmME3ID3SfWHl1XSH2jaS5EJtOmTcOSJUtgYmKCR48eYdy4cRgzZgwOHTqEd+/eUZ8fKfn4+GD9+vWYNWuW6Ibj0qVLmDdvHtLT06t9WkuAmzdvir0uLi7GzZs3sX79eqxcuZJRKm559eqVxLEHDx7ghx9+wKJFixgk4p5FixZhzZo1AIBbt25h/vz5WLBgASIiIjB//nzs3r2bccLGb9asWQgJCcHatWvF6qGvry+ys7Pxxx9/ME7IDZ9+n8trore3N9VEKX16XQE+bGowefJkjB49mkEi7hk/fjy+//57TJw4EVlZWRg8eDCMjY3x999/Iysri2abScHf3x8bNmzAzJkzRcdmz54NW1tb+Pv706CUlI4ePSr2urwmBgUFUc9HKdF1hdQ7ISEyUFNTE6akpAiFQqFw9erVwqFDhwqFQqHw4sWLwk6dOrGMxilaWlrCffv2SRzft2+fsE2bNgwSNR2nTp0S2tnZsY7BaVevXhX26NGDdQxOUFFREaampgqFQqFw+fLlwrFjxwqFQqHw+vXrwnbt2jFMxh1qamrCM2fOSBw/ffq0UE1NjUGipiUyMlLYq1cv1jE4LTExUainp8c6Bieoq6sL7927JxQKhcKNGzcKbWxshEKhUPjPP/8Iu3TpwjIaZ6ioqAgfPHggcTw5OVmooqLCIFHT8vfffwtHjhzJOgan0XWF1JXK97wlpApCoVC0O82FCxfg4uIC4EP/j5cvX7KMxinFxcWwtLSUON67d2+UlJQwSNR09OjRA1evXmUdg9Pk5eXx7Nkz1jE4QUFBAe/evQPwoSYOHToUAKCpqYm8vDyW0ThDUVER+vr6Ese7dOlCS8LrQLt27XD//n3WMTjt9evXeP36NesYnFBcXCzqJ3XhwgWMHDkSAGBoaIjMzEyW0Thj5MiRErN8gA8tM7744gsGiZqWfv36ISwsjHUMTqPrCqkrtHyPyMTS0hK//PILBg8ejKioKNGyitTUVLRr145xOu6YOHEi/vjjD6xfv17s+Pbt2+Hm5sYoFbd8+oNfKBQiMzMTvr6+6NatG6NU3HLixAmx1+Wf4aZNm2Bra8soFbf0798f8+fPh62tLa5cuYIDBw4AAJKTk0U9VUjVZs6ciRUrVmD37t2iH7OFhYVYuXKl2PIVUrXExESx1+Xf59WrV8Pc3JxNKI4JDAwUe13+Gf71119wdnZmlIpbevbsia1bt2L48OE4f/48VqxYAQB49uwZ2rRpwzhd4/Xx356RkRFWrlyJyMhIsZ5SsbGxWLBgAauITUJBQQECAwPRsWNH1lE4ga4rpL5Ro3Mik8TERLi5uSE9PR3z58/H8uXLAXzoCZKdnY19+/YxTth4zZ8/X/T/kpIS7NmzB507d0a/fv0AAP/++y/S09MxadIk/P7776xickZFDSyFQiF0dXWxf/9+0Y0cqdynDSx5PB7atm2LQYMG4bfffkOHDh0YJeOO9PR0TJ8+HRkZGZg9eza+++47AMC8efNQWloq8SOXfDBmzBix1xcuXICioiLMzMwAAAkJCSgqKoKjoyNCQkJYROSc8pr46e1dv379sGvXLhgaGjJKxh1dunQRe83n80U1cenSpVBVVWWUjDsiIyMxevRo5OXlwd3dHbt27QIA/PTTT7h37x59nyvx6d9eZXg8Hh49elTPaZoGDQ0NsftEoVCIN2/eQFlZGXv37hXN4iOVo+sKqW80KEXq1Pv37yEnJ0e7JFXBwcFBqvfxeDyEh4fXcxrui4qKEntd/uOha9eukJenyaCENGYeHh5Sv5eaxUsnLS1N7HV5TWzZsiWjRKS5Ki0tRV5entjuo48fP4aysjK0tbUZJiPNyZ49e8QGpcprYt++fWlnXCnRdYXUNxqUIrVSVFSEFy9eiPpLlevcuTOjRKS5iY6Oho2NjcQAVElJCeLi4jBw4EBGybhDIBBg4cKFUFZWFjteUFCAdevW0S5JUiorK0NKSkqFNZH+DklDCQ4Oxrhx40RLIMsVFRVh//79mDRpEqNk3OHp6YmNGzdKzIh6+/YtZs2aJZr1Qwhp/NLT06Grqysxq778HP1mqR5dV0h9o0EpIpPk5GR89913iIuLEzsuFArB4/FQWlrKKBlpbuTk5JCZmSnx1DU7Oxva2tr0tygF+gxr7/Llyxg/fjzS0tIkprdTTSQNib7PtVfZZ/jy5Uu0b9+eNiKRwvPnz7Fw4UKEhYXhxYsXEnWR/g6rV1paij179og+w08fdtBseulQTaw9+gxJfaO1LUQmHh4ekJeXx6lTp9ChQ4cKnz6Q6r19+xarV6+u9IaD+gVUr3wg9FPZ2dlQUVFhkIh7KvsMExISoKmpySAR93h5ecHS0hKnT5+mmigj+hFbNyr7Pj958gStW7dmkIg78vLyIBQKRT1nPl6aUlpaijNnztCyMylNnjwZ6enp8Pb2ppooozlz5mDPnj0YPnw4jI2N6TOUUWXzL/Lz82n5mZToukLqGw1KEZnEx8fj+vXr1NiulqZMmYKoqChMnDiRbtpqqLxBMo/Hw+TJk8WmFJeWliIxMRE2Njas4nFCefNPHo+H7t27i/39lZaWIj8/H15eXgwTcseDBw9w+PBhdO3alXUUzqIfsbVjYWEh+j47OjqKLWkuLS1Famoqhg0bxjBh46euri5WEz/F4/Hg5+fHIBn3XLx4ETExMbQzVy3s378fBw8ehIuLC+sonFS+sRCPx4OPj49Yi4LS0lL8+++/9PdZDbqukIZCg1JEJkZGRnj58iXrGJx39uxZnD59Gra2tqyjcE75kxmhUAhVVVUoKSmJzikoKKBfv36YOnUqq3icEBAQAKFQCE9PT/j5+Yk97VJQUIC+vj7tXiilvn37IiUlhQalaoF+xNaOq6srgA8PjZycnNCqVSvRufLv89ixYxml44aIiAgIhUIMGjQIR44cEZspqqCgAD09Pejo6DBMyB26urqVzlAh0lFQUKBrSi3cvHkTwIf7xFu3bkFBQUF0TkFBAWZmZli4cCGreJxA1xXSUKinFJFJeHg4fv75Z/j7+8PExERitz01NTVGybilS5cuOHPmDD7//HPWUTjLz88PCxcupKV6tRAVFQVbW1varbAWjh49ip9//hmLFi2qsCaampoySsYdRkZG+Pvvv2FhYcE6CqcFBQVh3LhxtCylFtLS0tC5c2earVcLoaGh+O2337Bt2zbo6+uzjsNJv/32Gx49eoRNmzbR32IteHh4YOPGjfTbpBboukLqGw1KEZnw+XwAkLhIUqPzmtm7dy+OHz+OoKAgiZ3PiPRKSkoQGRmJhw8fYvz48VBVVcWzZ8+gpqYm9lSHVO7hw4fYvXs3Hj58iI0bN0JbWxtnz55F586d0bNnT9bxGr3ymvgxHo9HNbEG6Eds3cnNzcXhw4fx8OFDLFq0CJqamrhx4wbatWuHjh07so7HCTExMdi2bRsePXqEQ4cOoWPHjvjrr7/QpUsX9O/fn3W8Rk9DQwPv3r1DSUkJlJWVJQbqc3JyGCXjjtGjRyMiIgKampro2bOnxGcYEhLCKBk3paSk4OHDhxg4cCCUlJQq7ZNEKkbXFVKf6LE4kUlERATrCE3Cb7/9hocPH6Jdu3bQ19eXuOG4ceMGo2TckZaWhmHDhiE9PR2FhYUYMmQIVFVVsWbNGhQWFmLr1q2sIzZ6UVFRcHZ2hq2tLaKjo7Fy5Upoa2sjISEBO3fuxOHDh1lHbPRSU1NZR+C8cePG4d27d/jss8/oR2wtJCYmYvDgwWjdujUeP36MqVOnQlNTEyEhIUhPT0dwcDDriI3ekSNHMHHiRLi5ueHGjRsoLCwEALx+/Rr+/v44c+YM44SNX0BAAOsInKeuro7Ro0ezjsF5OTk5+OqrrxAREQEej4cHDx7AwMAA3333HTQ0NPDbb7+xjtjo0XWF1DeaKUUIQ9U1TF2+fHkDJeEuV1dXqKqqYufOnWjTpg0SEhJgYGCAyMhITJ06FQ8ePGAdsdGztrbGV199hfnz50NVVVX0GV65cgVjxozBkydPWEckzUBQUFCV593d3RsoCbc5Ojqid+/eWLt2rdj3OS4uDuPHj8fjx49ZR2z0LCwsMG/ePEyaNEnsM7x58yacnZ2RlZXFOiIhREqTJk3CixcvsGPHDnz++eei7/M///yD+fPn486dO6wjNnp0XSH1jWZKkRpJTEyU6n3UP0U6NOhUezExMYiLixNrYAkA+vr6ePr0KaNU3HLr1i3s27dP4ri2tjZtaFCNEydOSPW+kSNH1nMS7qNBp7px7do1bN++XeJ4x44daTBFSvfv38fAgQMljrdu3Rq5ubkNH4hD8vLypHof9feR3osXL3D//n0AQI8ePaCtrc04EbeEhobin3/+QadOncSOd+vWDWlpaYxScQtdV0h9o0EpUiPm5uaiPimVof4pNXft2jXcvXsXwIdmv71792aciDvKysoq/Ht78uQJVFVVGSTiHnV1dWRmZqJLly5ix2/evEl9AqpRvjNNVagmSq+0tBRHjx4Vq4ejRo2iJvw1oKioWOHAQHJyMtq2bcsgEfe0b98eKSkpEr3NLl68CAMDAzahOEJdXb3KPj3UZ096eXl5mDFjBvbv3y/6vOTk5DBu3Dhs3rxZbMdcUrm3b99W2Lc1JycHioqKDBJxD11XSH2juzxSI9Q3pW49efIE3377LWJjY6Gurg7gQyNBGxsb7N+/X+KpDpE0dOhQBAQEiJ7g8Hg85OfnY/ny5XBxcWGcjhu++eYbLF68GIcOHQKPx0NZWRliY2OxcOFCTJo0iXW8Rq2srIx1hCbjzp07GDlyJLKystCjRw8AwJo1a9C2bVucPHkSxsbGjBNyw8iRIyEQCHDw4EEAH2pieno6Fi9eTFt3S2nq1KmYM2cOdu3aBR6Ph2fPnuHSpUtYuHAhvL29Wcdr1KjnaN2ZOnUqbt68iVOnTsHa2hoAcOnSJcyZMwfTpk3D/v37GSfkhgEDBiA4OBgrVqwAANF9ztq1a+Hg4MA4HTfQdYXUN+opRQhDw4YNQ25uLoKCgkQ/wu7fvw8PDw+oqanh3LlzjBM2fk+ePIGTkxOEQiEePHgAS0tLPHjwAFpaWoiOjqZp7lIoKirCjBkzsGfPHpSWlkJeXh6lpaUYP3489uzZAzk5OdYRSTNgbW2Ntm3bIigoCBoaGgCAV69eYfLkyfjvv/8QFxfHOCE3vH79Gl9++SWuXbuGN2/eQEdHB1lZWbC2tsaZM2egoqLCOmKjJxQK4e/vj1WrVuHdu3cAPswUWLhwoeiHLSH1TUVFBf/884/Ebo8xMTEYNmwY3r59yygZt9y+fRuOjo7o1asXwsPDMXLkSNy5cwc5OTmIjY3FZ599xjpio0fXFVLfaFCK1JqJiQnOnDkDXV1d1lE4R0lJCXFxcbCwsBA7fv36dQwYMEB0M0yqVlJSgv379yMxMRH5+fno1asX3NzcoKSkxDoap6Snp+P27dvIz8+HhYUFunXrxjoSJ6mpqSE+Pp6W+dSQkpISrl27hp49e4odv337Nvr06YOCggJGybjp4sWLYjVx8ODBrCNxTlFREVJSUpCfnw8jIyO0atWKdSROGj58OHbs2IEOHTqwjsIpnTt3xunTp2FiYiJ2PDExES4uLrQJSQ28fv0amzZtQkJCgqgmzpgxg/4ma4iuK6S+0PI9UmuPHz9GcXEx6xicpKurW+FnV1paCh0dHQaJuEleXh4TJkxgHYPzOnfujM6dO7OOwXn0rEc23bt3x/PnzyUGpV68eIGuXbsySsVd/fv3l5hhQWpGQUEBRkZGrGNwXnR0NA0qy+Dnn3/G/Pnz8ddff6F9+/YAgKysLCxatIiWkdZQ69atsWzZMtYxOI+uK6S+0KAUIQytW7cOs2bNwubNm2FpaQngQ9PzOXPm4Ndff2WcjjsePHiAiIgIvHjxQqLHj4+PD6NU3FFaWoo9e/YgLCysws8wPDycUTLS1H3cOHXVqlWYPXs2fH190a9fPwDA5cuXIRAIsGbNGlYROSksLKzS7/OuXbsYpeKOt2/fYvXq1ZV+ho8ePWKUjDR1FhYWYo3iHzx4IPbAKD09HYqKivjvv/8wbdo0VjE5Jzc3F1euXKnw+0y9M6VD1xVSn2hQitTagAEDaJlUDWhoaIjdcLx9+xZ9+/YV7S5VUlICeXl5eHp6SrWzV3P3559/4ocffoCWlhbat28v9tnyeDwalJLCnDlzsGfPHgwfPhzGxsZV7pxEqjdhwgTa7lxKn+7UJRQK8fXXX4uOlc86GzFiBO3WJSU/Pz8IBAJYWlqiQ4cO9H2WwZQpUxAVFYWJEyfSZ1gH9PT00KJFC9YxOIHu++reyZMn4ebmhvz8fKipqUncJ9KgVPXoukLqG/WUIqSBBQUFSf1ed3f3ekzSNOjp6WH69OlYvHgx6yicpaWlheDgYNqtkDS4qKgoqd9rZ2dXj0majg4dOmDt2rWYOHEi6yicpa6ujtOnT8PW1pZ1FEJILXXv3h0uLi7w9/eHsrIy6zicRNcVUt9ophSRSXZ2Ntq0aQMAyMjIwJ9//omCggKMHDkSAwYMYJyucaOBprr16tUrfPXVV6xjcJqCggL17KmFN2/eIDk5GT169ECrVq1w48YNBAQEoKCgAK6urnBzc2MdsdGigaa6V1RUBBsbG9YxOE1DQwOampqsYzQZb9++xYEDB1BQUIChQ4fSJhoymD59OgQCAbS0tFhH4ZynT59i9uzZNCBVC3RdIfWNzzoA4ZZbt25BX18f2traMDQ0RHx8PPr06YMNGzZg+/btcHBwwLFjx1jH5KThw4cjMzOTdQzO+eqrrxAaGso6BqctWLAAGzdupAbdMoiOjkbHjh3Rp08f6OnpITQ0FPb29rh69Sru3r2LSZMm4c8//2Qdk3NMTEyQkZHBOgYnTZkyBfv27WMdg9NWrFgBHx8f2gFXBunp6bCzs4OqqiqGDBmC9PR09OrVC1OmTMGsWbNgbm6O6Oho1jE5Z+/evWI9+Ij0nJyccO3aNdYxOI2uK6S+0fI9UiPOzs6Ql5fHkiVL8Ndff+HUqVNwcnIS/eiaNWsWrl+/jsuXLzNOyj2qqqpISEigbeRraNWqVVi/fj2GDx8OExMTib4Vs2fPZpSMO0aPHo2IiAhoamqiZ8+eEp9hSEgIo2SN38CBA9GtWzcIBALs2rUL69evxw8//AB/f38AwC+//ILDhw8jPj6ebVCOoXoouzlz5iA4OBimpqYwNTWV+D6vX7+eUTLusLCwwMOHDyEUCqGvry/xGd64cYNRssbv66+/RkZGBmbOnImDBw8iOTkZn332GXbu3Ak+n48ffvgBOTk5tIFGDVFNlN3OnTshEAjg4eFR4X3iyJEjGSXjDrqukPpGg1KkRrS0tBAeHg5TU1NRw8CrV6+id+/eAIB79+6hX79+yM3NZRuUg+iGQzZdunSp9ByPx6NdkqTg4eFR5fndu3c3UBLuUVdXx+XLl2FoaIiioiIoKSnhxo0bMDMzAwCkpKTAwsICb968YZyUW6geys7BwaHSczwejwYDpODn51fl+eXLlzdQEu5p3749Tpw4ASsrK+Tk5EBLSwuxsbGwtrYGACQkJMDR0REvX75knJRbqCbKjs+vfGEQj8ejTTSkQNcVUt+opxSpkZycHLRv3x4A0KpVK6ioqEBDQ0N0XkNDg358yYh2p5FNamoq6wicR4NOssvLyxP1nlFQUICysjJUVVVF51VVVWkJkAxoV1fZRUREsI7AeTToJLsXL15AT08PAKCpqQllZWW0a9dOdL59+/Z49eoVq3icRffWsisrK2MdgfPoukLqG/WUIjX26TagtC2o7NLT00V9fG7fvg1dXV0AH7ZBT09PZxmNECIFHo8nsb001UTZREdHo6SkBABw5swZdOjQAQBQUlJCPWgI4ZBPayKRjZycHF68eCFxPDs7G3JycgwSEUJI/aCZUqTGJk+eDEVFRQDA+/fv4eXlBRUVFQBAYWEhy2ic06VLF2RmZkJbW1vseE5ODrp06UJTiisxf/58rFixAioqKpg/f36V76V17hXr1asXwsLCoKGhAQsLiyp/OFD/lMoJhUI4OjpCXv7D5fTdu3cYMWIEFBQUAEA0yEKq5+DgUGE9fP36NRwcHKgeVmHMmDHYs2cP1NTUMGbMmCrfSz3iKqapqYnk5GRoaWlBQ0OjypqYk5PTgMm4x8fHR7TTWVFREVauXInWrVsDAM0crYHKOqwUFhaKrjGkYoGBgfj+++/RsmVLBAYGVvle6j1aMbqukIZEg1KkRtzd3cVeT5gwQeI9kyZNaqg4nCcUCiu88c3Pz0fLli0ZJOKGmzdvori4GMCHAZPKfjzQE9rKjRo1SjS47OrqyjYMh326zGfUqFES7xk7dmxDxeG0yuphdna26MEHqVjr1q1Fn52amhrVPhls2LBBtPQ2ICCAbRgOGzhwIO7fvy96bWNjI9HbceDAgQ0di1PKB1F4PB527NiBVq1aic6VlpYiOjoahoaGrOJxwoYNG+Dm5oaWLVtiw4YNlb6Px+PRoFQl6LpCGhI1OieEgfLZPRs3bsTUqVNFTxSBDzcc//77L+Tk5BAbG8sqYqOWmJgIY2PjKptXkqp9/BQxPT0dnTp1os+TMFH+BPb48eMYNmyYaLAU+FAPExMT0aNHD5w7d45VxEbvxIkTcHZ2pr6EtfDxDNzo6GjY2NiIZkAS0pDKN3BJS0tDp06dxJbqKSgoQF9fHwKBAH379mUVsdF7/fq1aHYekQ1dV0hDokEpQhgo38UiKioK1tbWYtOwy284Fi5ciG7durGK2KjJycmJlvkYGBjg6tWraNOmDetYnCIvL49nz55BW1tb7PMkpKGV7/4YFBSEr7/+WqzBeXk9nDp1KrS0tFhFbPTk5OSQlZWFtm3b0vdZRi1atMCTJ0/Qrl07+gxJo+Dg4ICQkBCxDYWIdD7+Dg8aNAghISFQV1dnHYtT6LpCGhI9AiI18uLFC7GCFB8fjw0bNiAlJQUdOnTAzJkzYW9vzy4gR5TvYuHh4YGNGzdCTU2NcSJuUVdXR2pqKrS1tfH48WPaWUUGOjo6OHLkCFxcXCAUCvHkyRO8f/++wvd27ty5gdNxx5UrV9C7d2/Rk+xTp05h3bp1opo4e/ZsWtJcjfLdH/X19bFo0SKxmaNEOm3btsXly5cxYsSISpdBkqrp6+sjMDAQQ4cOhVAoxKVLlyodDKDlZ5UbMWIEvv76a3z55Ze0g2Yt0Y5nsmvVqhWys7Ohra2NyMhIUcsHIj26rpCGRDOlSI18PFIeFxcHe3t72NjYwMrKCvHx8YiIiEBYWBjdsEnp9evXKC0tFW0pXy4nJwfy8vI0WFWJ77//HsHBwejQoYNo6VllO9F82suCfLB9+3bMmjWrykbc5Tch1GC6ch/XxJMnT8LV1RUTJkxA3759cfPmTezZswcHDx7E6NGjWUdt9FJTU1FSUiIxQ/TBgwdo0aIF9PX12QTjAF9fXwgEAql+NND3uWLHjh2Dl5cXXrx4AR6PV2mTaaqJVePz+ZCTk4OKigq+/fZbTJkyBb1792Ydi5PGjh0LKysrLF68WOz42rVrcfXqVRw6dIhRssZv7NixiI2Nxeeff46oqCjY2NhU2hw+PDy8gdNxA11XSEOiQSlSI3w+H1lZWdDW1sbQoUOhq6uLnTt3is7PnTsXt27dQlhYGMOU3OHs7IwRI0Zg+vTpYse3bt2KEydO4MyZM4ySNX7nzp1DSkoKZs+eDYFAIGpQ+6k5c+Y0cDLuePPmDdLS0mBqaooLFy5UugTSzMysgZNxx8c1ccCAAejfvz9WrVolOu/v74+TJ0/i0qVLDFNyg52dHTw9PSU21Ni7dy927NiByMhINsE44t69e0hJScHIkSOxe/fuSpeqVNSMn/yf/Px8qKmp4f79+5UuVaFeNZXj8/m4ffs2QkNDsWvXLty5cwcmJiaYMmUK3NzcaClaDbRt2xbh4eEwMTERO37r1i0MHjwYz58/Z5Ss8SsoKEBQUBAePnyI3377TaJ/68eqaoTe3NF1hTQUGpQiNfLxDzAdHR2EhISgX79+ovN37tyBvb09/vvvP4YpuUNTU1P0JOdj9+7dg62tLbKzsxkl4w4PDw8EBgZWOihFqhcUFIRvvvlGrME0kc7HNbFdu3Y4c+aM2KyA+/fvo1+/fnj16hXDlNygpqaGGzduoGvXrmLHU1JSYGlpidzcXDbBOMbPz4+WQdZSVFQUbG1tqdG5DD6uicCHJc47d+7EgQMHUFRUBFdXV0yZMgWDBg1inLTxU1JSQnx8PHr06CF2/N69e7CwsEBBQQGjZNzi4OCAo0ePUk+pWqDrCqlvtNUSqbE3b94gLy8PLVu2lPgR27JlS7x7945RMu4pLCyscPlUcXEx3WxIaffu3TQgJYOPn0e4u7vTgFQtJCUlITExEUpKShX2N6tqiST5PzweD2/evJE4Xr7MmUhn+fLl9MNBBm/fvhX9387OrtoBqY/fTypnZWWFbdu24dmzZ9iyZQsyMjIwZMgQ1rE4wcTEBAcOHJA4vn//fhgZGTFIxE0RERE0IFVLdF0h9Y0GpUiNde/eHRoaGnj8+DGuXbsmdu7OnTvQ0dFhlIx7rKyssH37donjW7dupR4MVfDy8sKTJ0+keu+BAwfw999/13Mi7unZsyf279+PoqKiKt/34MED/PDDD1i9enUDJeMeR0dHmJubIz09HbGxsWLnbt68SY3ipTRw4ECsWrVKbACqtLQUq1atQv/+/Rkma/yGDRuGy5cvV/u+N2/eYM2aNdi8eXMDpOKWrl27YvXq1cjMzKz0PUKhEOfPn4ezszMCAwMbMB33KSsrY/LkyYiJicHdu3dZx+EEb29vrFixAu7u7ggKCkJQUBAmTZqElStXwtvbm3W8Rm316tVSPyT/999/cfr06XpOxD10XSENieYlkxr5dCeQDh06iL1OTU3F999/35CROO2XX37B4MGDkZCQAEdHRwBAWFgYrl69itDQUMbpGq+2bduiZ8+esLW1xYgRI2BpaQkdHR20bNkSr169QlJSEi5evIj9+/dDR0enwoG/5u7333/H4sWLMX36dAwZMqTSz/DOnTuYOXMmfvjhB9aRG6XU1FSx161atRJ7XVRUJNGkllRszZo1GDhwIHr06IEBAwYAAGJiYpCXl0eNaKvx1VdfYezYsWjdunWVNfHMmTMYPnw41q1bxzpyoxMZGYmffvoJvr6+MDMzq/AzvHTpEuTl5bF06VJMmzaNdeRGyc7OrtKG0uW6d+/eQGm4bcSIETh27Bj8/f1x+PBhKCkpiXpA2tnZsY7XqCUlJUFPTw9fffWVqCa2bdsWwIfZy+U1ce/evXj27BmCg4MZJ2586LpCGhL1lCKEsfj4eKxbtw7x8fGiG46lS5dK7EBFxD1//hw7duzA/v37kZSUJHZOVVUVgwcPxpQpUzBs2DBGCbnh4sWLOHDgAGJiYpCWloaCggJoaWnBwsICTk5O1JiWNKhnz55h06ZNSEhIENXDmTNnSuxQSiQVFhbi0KFDOHDgAC5evIjXr18D+LAs0sjICE5OTvjuu+8kehgScenp6Th48CAuXrxYYU10dnaudLdXQkjjkZCQgE2bNuHw4cPIy8uDnJwcFBUVRTOoLCwsMGXKFEyePBktW7ZknLZxousKaSg0KEUI4bxXr14hPT1d9OPhs88+k2oLW0IIaapev36NgoICtGnTBi1atGAdhzRjq1evhpeXF/X1IUyUlZUhMTFRbJDZ3NwcWlparKNxDl1XSH2hQSlSp9zd3ZGRkUFLLaSUnp5e5XnqRVO99PR06OrqVjgIlZ6eTp+hFIKDgzFu3DiJZudFRUXYv38/Jk2axCgZ9w0ePBiPHj3Co0ePWEdp9KKjo6s8P3DgwAZKwm3R0dGwsbGRaNRdUlKCuLg4+hylIBAIsHDhQonGvgUFBVi3bh18fHwYJeMmNTU1xMfHw8DAgHUUTuHz+VU+YKMNIEhDoesKqW80KEXq1E8//YTMzEzs3r2bdRROoBuO2pOTk0NmZqZo++ly2dnZ0NbWps9QCvQZ1p/Nmzfj5cuXWL58OesojR6fL7n3ysf1kf4OpUPf59qjz7BuqaqqIiEhgQalauj48eNir4uLi3Hz5k0EBQXBz88P3333HaNk3ELf59qjz5DUN2p0TuqUv78/6wiccvPmTbHX5Tcc69evx8qVKxml4hahUFjhwF5+fj71CJBSZZ/hkydP0Lp1awaJmo4ZM2awjsAZr169EntdXg+9vb2pHtZAZd/n7OxsqKioMEjEPZV9hgkJCdTfjDSYUaNGSRz78ssv0bNnTxw4cIAGpaRU2fyLwsLCapvykw/oukLqGw1KEcKQmZmZxLHy3S3WrVuHMWPGMEjFDfPnzwfwYSaFt7e32DKL0tJS/PvvvzA3N2eUjhssLCzA4/HA4/Hg6OgoNi27tLQUqamp1CieNJiKBkCHDBkCBQUFzJ8/H9evX2eQijvKrxc8Hg+TJ08WW45bWlqKxMRE2NjYsIrHCRoaGqKa2L17d4mZevn5+fDy8mKYkJuSkpKgo6PDOkaT0a9fP9rpWgqBgYEAPtTEHTt2iO2OW1paiujoaBgaGrKKxwl0XSENhQalSI0lJSVh06ZNuHTpErKysgAA7du3h7W1NWbOnAkjIyPGCbmvR48euHr1KusYjVr5LDOhUIhbt26JPe1SUFCAmZkZFi5cyCoeJ7i6ugL4sAOkk5OT2A2bgoIC9PX1MXbsWEbpmoaHDx9i6tSp1GevFtq1a4f79++zjtHolQ/qCYVCqKqqQklJSXROQUEB/fr1w9SpU1nF44SAgAAIhUJ4enrCz89PbKC0vCZaW1szTMhNurq6rCM0GQUFBQgMDETHjh1ZR2n0NmzYAOBDTdy6davYrpnl3+etW7eyiscJdF0hDYV6SpEaOXv2LFxdXdGrVy84OTmhXbt2AIDnz5/j/PnzuH79Oo4fPw4nJyfGSbkhLy9P7LVQKERmZiZ8fX1x7949xMfHswnGIR4eHti4cSPU1NRYR+GsoKAgjBs3jpY71oOEhAT06tWL+i1IITExUex1eT1cvXo1SkpKcPHiRUbJuMXPzw+LFi2SaNJNpBcVFQUbGxvaXUpGW7ZsQUhICDQ1NTFt2jQ4OjqKzr18+RJWVla0+YMUymfulRMKhXjz5g2UlZWxd+9ejBw5kmE67nBwcEBISAg0NDRYR+EsPz8/LFy4kJbqkXpDg1KkRszMzDBq1CgIBIIKz/v6+iIkJETixwWpWEWNzoVCIXR1dbF//356IiuF169fo7S0VKLPR05ODuTl5WmwitSr8uUBlXn69Cl+/fVXGpSSQnk9/PS2pF+/fti1axcts5BSamoqSkpK0K1bN7HjDx48QIsWLaCvr88mWCOXl5cnul58+sDoU3RdqVxgYCCWLl0KDw8PvH79GgcPHoSvry+WLl0K4MNDTB0dHaqJUggKChJ7zefz0bZtW/Tt25cGWAghTQoNSpEaUVJSQnx8PHr06FHh+fv378Pc3BwFBQUNnIyboqKixF6X33B07dpVYttVUjFnZ2eMGDEC06dPFzu+detWnDhxAmfOnGGUrHHT1NREcnIytLS0JJ7GfionJ6cBk3ELn89Hhw4dKm2WWlRUhKysLPoBJoW0tDSx1+X1kGbw1YydnR08PT3h7u4udnzv3r3YsWMHIiMj2QRr5D7eXaqynXHLm/3S97lyPXv2xLJlyzB+/HgAQFxcHFxdXeHl5QWBQECDUlIqKSmBv78/PD090alTJ9ZxOG3s2LGwsrLC4sWLxY6vXbsWV69exaFDhxgla9x69eqFsLAwaGhoiHqQVubGjRsNmIw0RfSrl9SIvr4+Tp8+Xemg1OnTp6Gnp9fAqbipuLgYQUFB8Pb2RpcuXVjH4ax///0X69evlzhub2+PZcuWMUjEDRs2bICqqiqAD31UiGz09PSwZs0afP311xWej4+PR+/evRs4FfcUFxfD09MTW7dulZjhQ2rm5s2bsLW1lTjer18/zJw5k0EibggPDxfNuI2IiGCchrtSU1PFGh/b2NggPDwcgwcPRnFxMebOncsuHIfIy8tj3bp1mDRpEusonBcdHQ1fX1+J487Ozvjtt98aPhBHjBo1StTYvLwHKSH1hQalSI0IBAKMHz8ekZGRGDx4sFhPqbCwMJw7dw779u1jnJIbWrRogSNHjsDb25t1FE4rLCxESUmJxPHi4mKasVeFj2dRfDqjgkivd+/euH79eqWDUhUtRyOSWrRoQcu+6wiPx8ObN28kjpcvdSYVs7Ozq/D/pGa0tLSQkZEhtkzU2NgY4eHhGDRoEJ49e8YuHMcMGjQIUVFRtOS2lvLz8yuczdyiRYtql+o2Z8uXL6/w/4TUB1q+R2osLi4OgYGBFe6+N2fOHOqDVAPu7u4wNzfHvHnzWEfhLAcHBxgbG+P3338XOz5jxgwkJiYiJiaGUTJuKS0txdGjR3H37l0AgJGREUaNGkXLSKuRlJSEd+/ewdLSssLzxcXFePbsGc0glcK8efOgqKiI1atXs47CaSNGjICSkhL+97//iXabKi0txbhx4/D27VucPXuWcUJuePXqFXbu3ClWEz08PCT6FxJx48ePR7t27UQ7n33szp07cHBwQHZ2Ng2QSmHr1q3w8/ODm5sbevfuLdFkmhqdS8fKygpffPEFfHx8xI77+vri5MmTuH79OqNk3HPt2jWxmkgzwUldoUEpQhj65Zdf8Ntvv8HR0bHCG47Zs2czSsYdsbGxGDx4MPr06SPa4ScsLAxXr15FaGgoBgwYwDhh43fnzh2MHDkSWVlZoqW5ycnJaNu2LU6ePAljY2PGCUlzMGvWLAQHB6Nbt24V1sOKlukSSUlJSRg4cCDU1dVF9S8mJgZ5eXkIDw+n77MUoqOjMWLECLRu3Vo04Hz9+nXk5ubi5MmTGDhwIOOEjVdiYiKuX78ODw+PCs/fvn0bR44coZkXUuDz+ZWeo95m0jt58iTGjBmD8ePHY9CgQQA+3Cf+73//w6FDh2hpmhSePHmCb7/9FrGxsVBXVwcA5ObmwsbGBvv376e+Z6TWaFCK1Nrq1avh5eUlKlJEelX1kuLxeLRlspTi4+Oxbt06xMfHQ0lJCaampli6dCn1ppGStbU12rZti6CgINGOPq9evcLkyZPx33//IS4ujnFCbqGaKBsHB4cqz1OfH+k9e/YMmzZtQkJCgqgmzpw5k2b5SMnExATW1tb4448/xGabTZ8+HXFxcbh16xbjhISQmjh9+jT8/f3F7hOXL19OS3WlNGzYMOTm5iIoKEj08PL+/fvw8PCAmpoazp07xzgh4ToalCK1pqamhvj4eBgYGLCOQgiRgZKSEq5du4aePXuKHb99+zb69OlDvblqiGoiIdxW2U7DtMNwzeTm5uLKlSt48eIFysrKRMd5PB4mTpzIMBk3BAcHY9y4caJm0+WKioqwf/9+aoJOGoySkhLi4uJgYWEhdvz69esYMGAA3r17xygZaSqoWQipNRrXlJ1AIMDChQuhrKwsdrygoADr1q2TWP9OJKWnp1d5vnPnzg2UhLu6d++O58+fSwxKvXjxAl27dmWUiruoJsrG09MTGzduFO0KWe7t27eYNWsWdu3axSgZt0RHR1d5npaeVa9Xr164e/euxKDU3bt3YWZmxigVt5w8eRJubm7Iz8+Hmpqa2HbyNCglHQ8PDwwbNgza2tpix9+8eQMPDw8alCINRldXF8XFxRLHS0tLoaOjwyARaWpophSpNVVVVSQkJNCsABnIyckhMzNT4oYjOzsb2tra1C9ACnw+X+xm91P0GVbs4x1nLl68iB9//BG+vr7o168fAODy5csQCARYvXo1XFxcWMXkJKqJsqmsHr58+RLt27evcJdNIqmiPjQf10iqiRX7ePfHu3fv4scff8SsWbPEauLmzZuxevVqjBs3jlVMzujevTtcXFzg7+8v8eCNSIfP5+P58+do27at2PGEhAQ4ODggJyeHUTJuofvE2jt+/Dj8/f2xefNmUZ+9a9euYdasWVi8eDH15SK1RoNSpNYyMjKgo6Mj6rtApFfZDUd4eDjGjRuH//77j1Ey7khISBB7XVxcjJs3b2L9+vVYuXIlxowZwyhZ4/bpTVr5paD82Mev6YatZqgm1kxeXh6EQiE0NDTw4MEDsXpYWlqKkydPYsmSJbSVvJRev34t9rq8Jnp7e2PlypWiDSGIuPKaWN1tMdVE6aioqODWrVs0OC8DCwsL8Hg8JCQkoGfPnmK74JaWliI1NRXDhg3DwYMHGabkjuPHj4u9Lq+JQUFB8PPzw3fffccoWeOmoaEhdp/49u1blJSUiP4ey/+voqJCA6Sk1mj5Hqk1XV1d1hE4p7zQ83g8dO/eXeIpdn5+Pry8vBgm5I6KllJYWlpCR0cH69ato0GpSlDT6PpDNbFm1NXVxerhp3g8Hvz8/Bgk46bWrVtLHBsyZAgUFBQwf/582v68EqmpqawjNClOTk64du0aDUrJoHzWSXx8PJycnNCqVSvROQUFBejr62Ps2LGM0nHPqFGjJI59+eWX6NmzJw4cOECDUpUICAhgHYE0IzRTitTYli1bEBISAk1NTUybNk3sqevLly9hZWVFu8ZVIygoCEKhEJ6enggICBD7EVF+w2Ftbc0wIfelpKTAzMwMb9++ZR2FNHFUE2snKioKQqEQgwYNwpEjR8R2iFNQUICenh71rKgD9+7dg6WlJfLz81lHIc3Azp07IRAI4OHhARMTE7Ro0ULs/MiRIxkl446goCB88803Eo3OSd149OgRTE1NqSYS0gjQoBSpkcDAQCxduhQeHh54/fo1Dh48CF9fXyxduhQA8Pz5c+jo6NDUdilFRUXB1tZWbGo2qZmPeyMBH5adZWZmwtfXF/fu3UN8fDybYBw2aNAg7N69G3p6eqyjNHpUE+tOWloaOnfuXGXvD1K9j3sjAf9XE1evXo2SkhJcvHiRUTLuMjAwwD///INu3bqxjsIZFfU2K0dLIKWTkZEBHo+HTp06AQCuXLmCffv2wcjICN9//z3jdNxWUFCApUuX4uzZs7h//z7rOJzj4eGBlStX0gMjUmfolzCpkW3btuHPP//E+PHjAQA//PADXF1dUVBQAIFAwDgd96iqquLu3bswMTEB8GHd++7du2FkZARfX18oKCgwTtj4lS/9+ZhQKISuri7279/PKBU3nDhxosLj0dHROHXqlGgZGj3RrhzVxLpz9+5dZGRkoH///gCAzZs3488//4SRkRE2b94MDQ0Nxgm5wdzcvMLeSP369aMdDKsRGBhY4fH09HTs3r0b7du3BwDMnj27IWNxUllZGesInDd+/Hh8//33mDhxIrKysjB48GAYGxvj77//RlZWFu3QLKVPeyMJhUK8efMGysrK2Lt3L8Nkjd+nDznK/f333xg1apRoea6pqWlDxiJNEM2UIjWirKyMpKQk6Ovri47dvn0bgwcPhoeHB+bOnUuzAmqgT58+WLJkCcaOHYtHjx7ByMgIY8aMwdWrVzF8+HBazy2FqKgosdd8Ph9t27ZF165daQZaNaRp7EtPtKtGNbHumJiYYM2aNXBxccGtW7dgaWmJBQsWICIiAoaGhti9ezfriJyQlpYm9rq8JrZs2ZJRIu7g8/no2LGjxLUjLS0NOjo6aNGiBXg8Hi3HraH379/T358MNDQ0cPnyZfTo0QOBgYE4cOAAYmNjERoaCi8vL/o7lFJQUJDY6/Ka2LdvX3rYUY2q7hPLj9N9IqkL9IuN1IiWlhYyMjLEfoAZGxsjPDwcgwYNot2Raig5ORnm5uYAgEOHDsHOzg779u1DbGwsvvnmGxqUqkZxcTGCgoLg7e2NLl26sI7DOU5OTpCTk8OuXbugra0tOt6iRQskJCTAyMiIYTpuoJpYd1JTU0V/c0eOHMGIESPg7++PGzduwMXFhXE6biguLoanpye2bt1KS81k8P333+Pff//Fvn378Pnnn4uOt2jRAqGhoVQTa6C0tBT+/v7YunUrnj9/juTkZBgYGMDb2xv6+vrUXFoKxcXFon5SFy5cEM1aNjQ0RGZmJstonFFSUoK0tDR4enqKlkES6ZmamqJTp0749ddfoaSkBODDTLNu3brh7NmzdJ0hdabyBd+EVKB///4ICQmROG5kZISwsDCcPXuWQSruEgqFoinuFy5cEP3w0tXVxcuXL1lG44QWLVrgyJEjrGNw1tmzZ+Ho6AhLS0ucOnWKdRxOoppYdxQUFPDu3TsAH+rh0KFDAQCampoSveNIxVq0aFHpcgtSva1bt8LHxwdOTk7YtGkT6zictnLlSuzZswdr164Va0VgbGyMHTt2MEzGHT179sTWrVsRExOD8+fPY9iwYQCAZ8+eoU2bNozTcYO8vDzWrVuHkpIS1lE46cqVK+jatSvGjh2LnJwc6OnpiR7C6ejoQE9Pj/qPkjpBg1KkRpYsWVLpuuGePXsiPDyc1rjXgKWlJX755Rf89ddfiIqKwvDhwwF8mDHQrl07xum4wdXVFceOHWMdg7PmzZuHEydOYPHixZg2bZpoUIBIh2pi3enfvz/mz5+PFStW4MqVK6J6mJycTE+4a2DChAnYuXMn6xicNXr0aFy6dAlHjx6Fs7MzsrKyWEfipODgYGzfvh1ubm6Qk5MTHTczM8O9e/cYJuOONWvWYNu2bbC3t8e3334LMzMzAB/6QVpZWTFOxx2DBg2SaPVApKOgoICAgAD8+uuvGDlyJFatWkX94ki9oOV7pEZMTU2rbGZnbGwMY2PjBkzEbQEBAXBzc8OxY8ewbNkydO3aFQBw+PBh2NjYME7HDd26dYNAIEBsbCx69+4NFRUVsfPUkLZ65ubmuHbtGubNmwdzc/Mqe0wRcVQT686mTZswffp0HD58GH/88Qc6duwI4MOMvvIZAqR6JSUl2LVrFy5cuFBhTVy/fj2jZNzRsWNHXLhwAatXr4aFhQXVRBk8ffpUdE/zsbKyMhQXFzNIxD329vZ4+fIl8vLyxHofff/991BWVmaYjFucnZ2xZMkS3Lp1q8KaSJu5VM/Z2RnXrl2Dh4cHzQAn9YIanROZ5ebm4sqVK3jx4oXYqDmPx8PEiRMZJuO+9+/fQ05ODi1atGAdpdGrqpcUNaStuRMnTiAiIgJLly4V6zNFqkc1kTQGDg4OVZ6PiIhooCRNw40bNxATE4NJkyZRU+Qa6N27N+bNm4cJEyZAVVUVCQkJMDAwgEAgwPnz5xETE8M6Imkm+PzKFwZRk+6aCwwMREREBH7//XeaxUzqDA1KEZmcPHkSbm5uyM/Ph5qamthWqzweDzk5OQzTcU9RUZHED1kA6Ny5M6NEpLl68uQJOnToILbcglSPamLdKCsrQ0pKSoX1cODAgYxSkeYsNjYWlpaWoobTRDrHjx+Hu7s7li5dCoFAAD8/P9y/fx/BwcE4deoUhgwZwjpio/f8+XMsXLgQYWFhePHihcSMPRpMIYQ0FTQoRWTSvXt3uLi4wN/fn6YQ10JycjK+++47xMXFiR2nLValJxAIsHDhQom/w4KCAqxbt476+dSQmpoa4uPjYWBgwDoKp1BNrL3Lly9j/PjxSEtLk/jxRfVQep6enti4cSNUVVXFjr99+xazZs3Crl27GCXjJqqJsouJiYFAIEBCQgLy8/PRq1cv+Pj4iDYxIFVzdnZGeno6Zs6ciQ4dOog97ACAUaNGMUrGLcHBwRg3bpzEwHJRURH279+PSZMmMUrGTVQTSX2gQSkiExUVFdy6dYsKUi3Z2tpCXl4eS5YsqfCGo7ypJamcnJwcMjMzJZaaZWdnQ1tbm37I1tDHyyyI9Kgm1p65uTm6d+8OPz+/Cuth69atGSXjlspq4suXL9G+fXvahaqGqCYSVlRVVRETEwNzc3PWUTiN7hPrFtVEUh+o0TmRiZOTE65du0YFqZbi4+Nx/fp1GBoaso7CWeWzyj6VkJAATU1NBolIc0Q1sfYePHiAw4cPV9gcmVQvLy8PQqEQQqEQb968QcuWLUXnSktLcebMGeoTRxqMj48PHBwcYG1tLfa3SKSnq6tLTfbrQGX3iU+ePKGHHYQ0EjQoRWQyfPhwLFq0CElJSTAxMZFoyE07WUjHyMgIL1++ZB2DkzQ0NMDj8cDj8dC9e3exG47S0lLk5+fDy8uLYUJu+umnn2gwTwZUE2uvb9++SElJoUEpGamrq4vVxE/xeDz4+fkxSMZt27ZtQ7t27VjH4JxLly5h/fr1KCkpQZ8+fWBnZwd7e3vY2tpCSUmJdTxOCAgIwJIlS7Bt2zbo6+uzjsM5FhYWopro6OgIefn/+9lbWlqK1NRU2tlVBhMmTICamhrrGKSJoeV7RCa0k0XdCA8Px88//wx/f/8Kf8hS0a9cUFAQhEIhPD09ERAQIPa0S0FBAfr6+rC2tmaYkJvKLwkVPVUklaOaWHtHjx7Fzz//jEWLFlVYD01NTRkl44aoqCgIhUIMGjQIR44cERtcVlBQgJ6eHnR0dBgm5KaMjAwAH2atkJopKSnBv//+i+joaERFRSEuLg6FhYXo06cPLl68yDpeo6ehoYF3796hpKQEysrKEjWRNtCoWvkgvJ+fHxYsWIBWrVqJzpXfJ44dOxYKCgqsIhJC/j8alCKEofIfsp8OAFCjc+lFRUWJenMR2e3cuRMbNmzAgwcPAADdunXD3LlzMWXKFMbJSHNR0cAej8ejelhDaWlp6Ny5Mw0s10JJSQn8/PwQGBiI/Px8AECrVq0wa9YsLF++XGJwgFQtOTkZERERuHDhAo4dO4bWrVvTLHEpBAUFVXne3d29gZJwW1BQEL755hvaQbOWwsLCsGHDBty9excA8Pnnn2Pu3LkYPHgw42SkKaBBKVJr79+/p34BMoqKiqryvJ2dXQMl4a4bN26gRYsWMDExAfBhG+rdu3fDyMgIvr6+9ARMCj4+Pli/fj1mzZolml126dIlbNq0CfPmzYNAIGCckFuoJsomLS2tyvN6enoNlITbzp07h1atWqF///4AgM2bN+PPP/+EkZERNm/eDA0NDcYJG78ffvgBISEhEAgEYjXR19cXrq6u+OOPPxgnbPy2b9+OyMhIREVFobCwEAMGDIC9vT3s7e1hampKg6akwWRkZIDH46FTp04AgCtXrmDfvn0wMjLC999/zzgdN2zZsgVz5szBl19+KaqJly9fxuHDh7FhwwbMmDGDcULCeUJCZFBSUiIUCARCHR0doZycnPDhw4dCoVAo/Pnnn4U7duxgnI40J5aWlsLDhw8LhUKh8OHDh0JFRUXht99+K+zatatwzpw5bMNxhJaWlnDfvn0Sx/ft2yds06YNg0TcQzWRNBbGxsbC06dPC4VCoTAxMVGooKAgXLp0qbBfv37CyZMnM07HDWpqasIzZ85IHD99+rRQTU2NQSLu4fF4Qm1tbeGaNWuEb968YR2HU16/fi3VPyKd/v37C4ODg4VCoVCYmZkpVFVVFVpbWwu1tLSEfn5+jNNxQ8eOHYW///67xPFNmzYJdXR0GCQiTQ2tdyEyWblyJYKCgrB27VpMnTpVdNzY2BgBAQH47rvvGKZr/BITE6V6H/VQqV5ycrJou+RDhw7Bzs4O+/btQ2xsLL755hsEBAQwzccFxcXFsLS0lDjeu3dv2j5eSlQTZXfixAmp3kfN4qWTmpoKIyMjAMCRI0cwYsQI+Pv748aNG3BxcWGcjhsUFRUrbCzdpUsXmn0rpZCQEERHR2P//v1Yvnw5LCwsRDOl+vfvD2VlZdYRG63yTQsqI6QlzTVy+/ZtWFlZAQAOHjwIExMTxMbGIjQ0FF5eXvDx8WGcsPHLzc2tsCn80KFDsXjxYgaJSFNDg1JEJsHBwdi+fTscHR3FdjgzMzPDvXv3GCbjBnNzc1GvlMrQDYd0hEIhysrKAAAXLlzAF198AeBDU1rqWSGdiRMn4o8//sD69evFjm/fvh1ubm6MUnEL1UTZubq6VvseqofSU1BQwLt37wB8qImTJk0CAGhqaiIvL49lNM6YOXMmVqxYgd27d4v60BQWFmLlypWYOXMm43Tc4OrqKvpuv379GjExMTh06BC++OIL8Pl8vH//nm3ARiwiIoJ1hCaluLhY9D2+cOGC6AGHoaEhMjMzWUbjjJEjR+Lo0aNYtGiR2PHjx4+L7rsJqQ0alCIyefr0aYXbdpeVlaG4uJhBIm5JTU1lHaHJsLS0xC+//ILBgwcjKipK1OsjNTWVtvGuwvz580X/5/F42LFjB0JDQ9GvXz8AwL///ov09HTRD1pSNaqJsisfVCZ1o3///pg/fz5sbW1x5coVHDhwAMCHWaXlPVWIpDFjxoi9vnDhAjp16gQzMzMAQEJCAoqKiuDo6MgiHidlZ2cjKioKkZGRiIyMxJ07d6ChoYEBAwawjtaoUT/RutWzZ09s3boVw4cPx/nz57FixQoAwLNnz9CmTRvG6RqvwMBA0f+NjIywcuVKREZGivWUio2NxYIFC1hFJE0IDUoRmRgZGSEmJkai8ezhw4dhYWHBKBV3UMPeuhMQEAA3NzccO3YMy5YtEw0MHD58GDY2NozTNV43b94Ue927d28AwMOHDwEAWlpa0NLSwp07dxo8GxdRTSSNxaZNmzB9+nQcPnwYf/zxBzp27AgAOHv2bIXLL8gHrVu3Fns9duxYsde6uroNGYfzTExMkJSUBE1NTQwcOBBTp06FnZ0dtSWQ0fDhw7Fjxw506NCBdRTOWbNmDUaPHo1169bB3d1dNNB84sQJ0bI+ImnDhg1irzU0NJCUlISkpCTRMXV1dezatQs///xzQ8cjTQztvkdkcvz4cbi7u2Pp0qUQCATw8/PD/fv3ERwcjFOnTmHIkCGsI3KOiYkJzpw5Qze+deT9+/eQk5OjrbtJg6CaWLfU1NQQHx8PAwMD1lEIITLYvHkz7O3toa2tDUVFRaipqbGOxGmqqqpISEigmiij0tJS5OXlie0++vjxYygrK0NbW5thMkIIQINSpBZiYmIgEAiQkJCA/Px89OrVCz4+Phg6dCjraJxENxy1U1RUhBcvXkgsBercuTOjRKS5oZpYd6ge1k5ZWRlSUlIqrIkDBw5klIo0F7m5uVi2bBkOHDiAV69eAQDatm0LDw8PeHt7U5NzGVBNJIQ0ZTQoRUgjQTccsklOTsZ3332HuLg4seO0O4303r59i9WrVyMsLKzCH7GPHj1ilIw0V1QPZXf58mWMHz8eaWlpEptpUE2UzvPnz7Fw4UJRTfz0c6TPsHI5OTmwtrbG06dP4ebmhs8//xwAkJSUhH379sHQ0BAXL15EYmIiLl++jNmzZzNOzA3GxsY4e/YszaaXAX2fa6+0tBR79uyp9D4xPDycUTLSVFBPKSITHx8fODg4wNraGi1btmQdp0kYMGAAlJSUWMfgHA8PD8jLy+PUqVPo0KFDldsok4pNmTIFUVFRmDhxIn2GMqKaWLcmTJhAy31k5OXlBUtLS5w+fZq+zzKaPHky0tPT4e3tTZ9hDQkEAigoKODhw4cSm40IBAIMHToUEydORGhoqFgjZVK127dvs47AWfR9rr05c+Zgz549GD58OIyNjekzJHWOZkoRmQwZMgSXLl1CSUkJ+vTpAzs7O9jb28PW1pYGVkiDUlFRwfXr12FoaMg6Cmepq6vj9OnTsLW1ZR2Fs6gmksZCRUUFCQkJFe4GSaSjqqqKmJgYmJubs47COfr6+ti2bRucnJwqPH/u3Dm4uLhg+fLlWL58eQOn4663b9/iwIEDKCgowNChQ9GtWzfWkTiDvs+1p6WlheDgYLi4uLCOQpooPusAhJvOnz+P3NxchIWFwcXFBdeuXcOYMWOgrq6O/v37s47HGdnZ2aL/Z2RkwMfHB4sWLUJMTAzDVNxiZGSEly9fso7BaRoaGtDU1GQdg9OoJtbOmzdvcP36deTn5wMAbty4gUmTJuGrr77C33//zTgdt/Tt2xcpKSmsY3Carq6uxBIfIp3MzEz07Nmz0vPGxsbg8/k0IFWF9PR02NnZQVVVFUOGDEF6ejp69eqFKVOmYNasWTA3N0d0dDTrmJxB3+faU1BQoAcdpF7RoBSRmby8PGxtbTF27FiMHj0aTk5OKCsrw71791hHa/Ru3boFfX19aGtrw9DQEPHx8ejTpw82bNiA7du3w8HBAceOHWMdkxPWrFmDH3/8EZGRkcjOzkZeXp7YP1K9FStWwMfHB+/evWMdhdOoJsomOjoaHTt2RJ8+faCnp4fQ0FDY29vj6tWruHv3LiZNmoQ///yTdUzOmDVrFhYsWIA9e/bg+vXrSExMFPtHqhcQEIAlS5bg8ePHrKNwjpaWVpWfW2pqKu12Vo2FCxeiqKgIW7duhbKyMpycnNCtWzdkZmbi+fPncHZ2hq+vL+uYnEHf59pbsGABNm7cSIN7pN7Q8j0ik+3btyMyMhJRUVEoLCzEgAEDYG9vD3t7e5iamtJa42o4OztDXl4eS5YswV9//YVTp07ByclJ9MNr1qxZuH79Oi5fvsw4aePH538YW//0b44anUvPwsICDx8+hFAohL6+Plq0aCF2/saNG4yScQfVRNkNHDgQ3bp1g0AgwK5du7B+/Xr88MMP8Pf3BwD88ssvOHz4MOLj49kG5YjymvgxHo9HNbEGNDQ08O7dO5SUlEBZWVmiJubk5DBK1vh5enri4cOHOH/+PBQUFMTOFRYWwsnJCQYGBti1axejhI1f+/btceLECVhZWSEnJwdaWlqIjY2FtbU1ACAhIQGOjo40S1xK9H2uvdGjRyMiIgKampro2bOnxGcYEhLCKBlpKmhQisiEz+ejbdu2WLBgAaZPn45WrVqxjsQpWlpaCA8Ph6mpKfLz86GmpoarV6+id+/eAIB79+6hX79+yM3NZRuUA6Kioqo8b2dn10BJuMvPz6/K87TMonpUE2Wnrq6Oy5cvw9DQEEVFRVBSUsKNGzdgZmYGAEhJSYGFhQXevHnDOCk3pKWlVXleT0+vgZJwV1BQUJXn3d3dGygJ9zx58gSWlpZQVFTEjBkzYGhoCKFQiLt372LLli0oLCzE1atX0blzZ9ZRGy0+n4/MzExRo/hWrVohMTFRtBvp8+fPoaOjQwPMUqLvc+15eHhUeX737t0NlIQ0VTQoRWRy7NgxREdHIzIyEnfv3oWFhYVoVkD//v2hrKzMOmKjxufzkZWVJZrC/un253TDQQi3UE2UHdVDQpqW1NRUTJ8+HaGhoaLlPjweD0OGDMGmTZuoN001qCYSQpobedYBCDe5urrC1dUVAPD69WvExMTg0KFD+OKLL8Dn8/H+/Xu2ATng0+U8tLynZqTtjWJqalrPSZqOa9eu4e7duwA+NJAvn7lHqkc1UXY8Hk+s/n36mkjnxIkTUr1v5MiR9ZykaSgtLcXRo0fFauKoUaMgL0+3ztXp0qULzp49i1evXuHBgwcAgK5du9KGGjXg4+MjephRVFSElStXonXr1gBA/R+lJG1fUTU1tXpO0nS8ePEC9+/fBwD06NGD+sOROkMzpYjMsrOzERUVhcjISERGRuLOnTvQ0NDAgAEDcPToUdbxGjU+nw9nZ2coKioCAE6ePIlBgwZBRUUFwIe+C+fOnaOnYFXg8/miPimVof4p0nny5Am+/fZbxMbGQl1dHQCQm5sLGxsb7N+/H506dWIbkCOoJsqGz+fD2NhY9GM/MTERhoaGon40JSUluHPnDn2Xq1FRL6lPUU2Uzp07dzBy5EhkZWWhR48eAIDk5GS0bdsWJ0+ehLGxMeOEpCmzt7eXamA+IiKiAdJwV/l9YmWoz5708vLyMGPGDOzfv1/0ecnJyWHcuHHYvHmzaMCUEFnRoBSRiYmJCZKSkqCpqYmBAwfC3t4ednZ2NCtFStWtzS5Ha7QrV13flHLUP6V6w4YNQ25uLoKCgkQ/wO7fvw8PDw+oqanh3LlzjBM2flQTZVddT7Ny1NuMNBRra2u0bdsWQUFB0NDQAAC8evUKkydPxn///Ye4uDjGCQkh1amu52g56j1avXHjxuHmzZv4/fffRQ33L126hDlz5sDc3Bz79+9nnJBwHQ1KEZls3rwZ9vb20NbWhqKiIk19JYTDlJSUEBcXBwsLC7Hj169fx4ABA2ipgBSoJhLSdCgpKeHatWvo2bOn2PHbt2+jT58+KCgoYJSMEEIanoqKCv755x/0799f7HhMTAyGDRuGt2/fMkpGmorq53oT8onc3FwkJSXBzs4O7du3h4aGBtq3b4+lS5fSj1fClImJCTIyMljH4BxdXV0UFxdLHC8tLYWOjg6DRNxCNZE0Vmpqanj06BHrGJzTvXt3PH/+XOL4ixcvqEk3qXcjRozAX3/9RYOf9WD48OHIzMxkHYNz2rRpU+ESvdatW4tmkxJSGzQoRWokJycHffv2RVBQEMaOHYvffvsNv/32G0aOHInff/8dAwcOxPv373HlyhUEBgayjttovXjxQux1fHw83N3dYWtriy+//BKRkZFsgnHc48ePKxxcIVVbt24dZs2ahWvXromOXbt2DXPmzMGvv/7KMFnjRzWx9q5cuSLW0+PUqVOws7NDx44dYWlpieDgYIbpuI0mw0svLy9P9G/VqlWYPXs2Dh8+jCdPnuDJkyc4fPgw5s6dizVr1rCOSpq406dPw9PTEx06dMAPP/yA69evs47UZERHR9Ngnwx+/vlnzJ8/H1lZWaJjWVlZWLRoEby9vRkmI00FLd8jNTJ37lyEhYXhwoULaNeundi5rKwsDB06FD169EBoaCgCAwPh7u7OKGnjJicnh8zMTGhrayMuLg729vawsbGBlZUV4uPjERERgbCwMAwcOJB1VE75dNtkUjkNDQ2xBqBv375FSUmJqNl0+f9VVFSQk5PDKmajRzWx9j6uhydPnoSrqysmTJiAvn374ubNm9izZw8OHjyI0aNHs47KOVQTpfdpU+Ty2+PyYx+/psbIpD7x+Xzcvn0boaGh2LVrF+7cuQMTExNMmTIFbm5uNDOlFqgmSs/CwkKsJj548ACFhYXo3LkzACA9PR2Kioro1q0bbty4wSomaSJoX1tSI8eOHcO2bdskfnwBQPv27bF27Vq4uLhg+fLl9OOrCh+PBfv6+mLixInYuXOn6NjcuXPh5+eHsLAwFvE4a8CAAVBSUmIdgxMCAgJYR2gSqCbW3sf1cO3atfjxxx+xatUq0bEuXbpg7dq1NCglgwkTJlB/MynRTmakMdHS0sLcuXMxd+5cXLlyBTt37sTPP/+MH3/8Ea6urpgyZQoGDRrEOibn6OnpoUWLFqxjcIKrqyvrCKQZoZlSpEYUFRXx8OHDSreIf/LkCfT19VFSUtLAybiFz+cjKysL2tra0NHRQUhICPr16yc6f+fOHdjb2+O///5jmJIQUh2qibX3cT1s164dzpw5g969e4vO379/H/369cOrV68YpiSEkIbxcU382Lt373Dw4EHs3LkTcXFxNGOPENJkUE8pUiNaWlp4/PhxpedTU1MlLqKkYm/evEFeXh5atmwJRUVFsXMtW7akBslSys7OFv0/IyMDPj4+WLRoEWJiYhim4i5qAlozVBPrRlJSEhITE6GkpISysjKJ8zSoJ503b97g+vXryM/PBwDcuHEDkyZNwldffYW///6bcTpuog00SGOhrKyMyZMnIyYmBnfv3mUdh5Pevn2LXbt2YfPmzXjw4AHrOJw0ffp0vHz5knUM0sTQoBSpEScnJyxbtgxFRUUS5woLC+Ht7Y1hw4YxSMY93bt3h4aGBh4/fizWYBr4MFOKdj2r2q1bt6Cvrw9tbW0YGhoiPj4effr0wYYNG7B9+3Y4ODjg2LFjrGNyDjUBrRmqiXXD0dER5ubmSE9PR2xsrNi5mzdvinpYkMpFR0ejY8eO6NOnD/T09BAaGgp7e3tcvXoVd+/exaRJk/Dnn3+yjsk5tIEGaWh2dnZQUFCo8j3du3dvoDTclZ6eDjs7O6iqqmLIkCFIT09Hr169MGXKFMyaNQvm5uaIjo5mHZNz9u7di7y8PNYxSBNDy/dIjTx58gSWlpZQVFTEjBkzYGhoCKFQiLt372LLli0oLCzE1atX6QdENaKiosRed+jQQewGY+PGjSgqKsKiRYsaOhpnODs7Q15eHkuWLMFff/2FU6dOwcnJSfSja9asWbh+/TouX77MOCm3UBPQmqGaWHtpaWlir1u1aoU2bdqIXpfvvjdp0qQGzcU1AwcORLdu3SAQCLBr1y6sX78eP/zwA/z9/QEAv/zyCw4fPoz4+Hi2QTmGaiIh3PT1118jIyMDM2fOxMGDB5GcnIzPPvsMO3fuBJ/Pxw8//ICcnByEh4ezjsopVBNJfaBBKVJjqampmD59OkJDQ8V2oxkyZAg2bdqErl27Mk5ImgMtLS2Eh4fD1NQU+fn5UFNTw9WrV0W9aO7du4d+/fohNzeXbVCOMTY2xtmzZ6Grq8s6CmdQTSSNgbq6Oi5fvgxDQ0MUFRVBSUkJN27cgJmZGQAgJSUFFhYWePPmDeOk3OLi4oKdO3eiQ4cOrKOQZmz16tXw8vKCuro66yic0b59e5w4cQJWVlbIycmBlpYWYmNjYW1tDQBISEiAo6MjLUWrIRqUIvWBBqWIzF69eiVaj921a1doamoyTkSak08bgX56kXz+/Dl0dHSoEagU0tPToaurK7b1L/BhV7SMjAya5SMlqomEJaqJdSc6Oho2NjaQlxffpLqkpARxcXEYOHAgo2SkuVJTU0N8fDwNBNQAn89HZmamaHfcVq1aITExkWoiIY0Q9ZQiMtPQ0ICVlRWsrKzox1cdc3d3p61+pfDpIMqnr4l0unTpUuFOjzk5OejSpQuDRNxENbF+DB48mH6ISYHH44nVwE9fE+k5ODggJydH4vjr16/h4ODAIBFp7mgOgWw+rYlENnJycnjx4oXE8ezsbMjJyTFIRJoa+erfQghpaB07dgSfT2PG1Zk8ebJo58L379/Dy8sLKioqAD40mSbSEQqFFd6s5efno2XLlgwSEfJ/Ro8eTcsrpCAUCuHo6Cia3fPu3TuMGDFC1DCZdjCUXmU1MTs7W3SNIYQ0fj4+PlBWVgYAFBUVYeXKlWjdujUA0C7XNVDZoGhhYWG1TfkJkQYt3yOEcJKHh4dU79u9e3c9J+Gu+fPnA/jQWH/q1KmiGzcAKC0txb///gs5OTmJ3dAIIY2Pn5+fVO9bvnx5PSfhrjFjxgAAjh8/jmHDhokeegAfamJiYiJ69OiBc+fOsYpImqmMjAzo6OjQrJQasLe3l2p2VERERAOk4abAwEAAwLx587BixQq0atVKdK60tBTR0dF4/Pgxbt68ySoiaSJoUIoQQpqp8mUoUVFRsLa2FnvapaCgAH19fSxcuBDdunVjFZEQQhpM+cOOoKAgfP3111BSUhKdK6+JU6dOhZaWFquIhBDSYMpbOKSlpaFTp05ig6LlNVEgEKBv376sIpImggalCGEkKSkJmzZtwqVLl5CVlQXgw04h1tbWmDlzJoyMjBgnJM2Fh4cHNm7cCDU1NdZRCJHw8OFDTJ06lbbtJg3Gz88PixYtEps9SkhD2rJlC0JCQqCpqYlp06bB0dFRdO7ly5ewsrLCo0ePGCYkzYmDgwNCQkKgoaHBOgppoqhpDSEMnD17FhYWFrh58yZGjRoFHx8f+Pj4YNSoUUhISECvXr3wzz//sI7ZqH3acDE+Ph7u7u6wtbXFl19+icjISDbBOCggIKDCfjM5OTnIy8tjkIiQ/5Ofn4+oqCjWMRq9K1euiO0iderUKdjZ2aFjx46wtLREcHAww3TcMmnSJDx9+lTi+IMHD/D48eOGD0SalcDAQCxatAiGhoZQVFSEi4sLVq1aJTpfWlqKtLQ0hgm5YcSIEfjrr79QUFDAOgrnRURE0IAUqVc0U4oQBszMzDBq1CgIBIIKz/v6+iIkJASJiYkNnIw75OTkkJmZCW1tbcTFxcHe3h42NjawsrJCfHw8IiIiEBYWRlt3S8HZ2RkjRozA9OnTxY5v3boVJ06cwJkzZxglI81Bec+Kyjx9+hS//vorbdtdjY9r4smTJ+Hq6ooJEyagb9++uHnzJvbs2YODBw9i9OjRrKM2enZ2dvD09IS7u7vY8b1792LHjh300IPUq549e2LZsmUYP348ACAuLg6urq7w8vKCQCDA8+fPoaOjQzWxGnw+H3JyclBRUcG3336LKVOmoHfv3qxjcdLYsWNhZWWFxYsXix1fu3Ytrl69ikOHDjFKRpoKGpQihAElJSXEx8ejR48eFZ6/f/8+zM3N6elOFfh8PrKysqCtrY2hQ4dCV1cXO3fuFJ2fO3cubt26hbCwMIYpuUFTUxOxsbH4/PPPxY7fu3cPtra2yM7OZpSMNAd8Ph8dOnSodAefoqIiZGVl0Q+wanxcEwcMGID+/fuLza7w9/fHyZMncenSJYYpuUFNTQ03btxA165dxY6npKTA0tISubm5bIKRZkFZWRlJSUnQ19cXHbt9+zYGDx4MDw8PzJ07lwalpMDn83H79m2EhoZi165duHPnDkxMTDBlyhS4ubnRzJ8aaNu2LcLDw2FiYiJ2/NatWxg8eDCeP3/OKBlpKmj5HiEM6Ovr4/Tp05WeP336NPT09BowEbfdvn0bU6dOFTs2depUmmkmpcLCwgqX7xUXF9PAKKl3enp62LBhA1JTUyv8V1WtJBVLTk7Gl19+KXZs7NixuHfvHqNE3MLj8fDmzRuJ469fv6aBAFLvtLS0kJGRIXbM2NgY4eHh2L17N3788UdGybhHS0sLc+fORWJiIi5duoS+ffvi559/RseOHTF+/HjqVSil/Pz8Ch8ctWjRgto8kDpBg1KEMCAQCLB48WKMHDkSgYGBOHDgAA4cOIDAwECMGjUKS5cuxcqVK1nHbPTevHmDvLw8tGzZUmzrbgBo2bIl3r17xygZt1hZWWH79u0Sx7du3UpT3Um96927N65fv17peR6PB5rULZ2kpCQkJiZCSUkJZWVlEucrGnwmkgYOHIhVq1aJDUCVlpZi1apV6N+/P8NkpDno378/QkJCJI4bGRkhLCwMZ8+eZZCK+6ysrLBt2zY8e/YMW7ZsQUZGBoYMGcI6FieYmJjgwIEDEsf3799PGzOROiHPOgAhzdFXX32Fjh07IjAwEL/99pvE7nuRkZGwtrZmnLLx6969OwBAKBTi2rVrsLCwEJ27c+cOdHR0WEXjlF9++QWDBw9GQkKCaIefsLAwXL16FaGhoYzTkaZOIBBUOYBsZGSE1NTUBkzEXY6OjqIBvNjYWPTp00d07ubNm+jcuTOraJyyZs0aDBw4ED169MCAAQMAADExMcjLy6OZFaTeLVmypNKB+p49eyI8PBxHjhxp4FRNh7KyMiZPnozJkycjOTmZdRxO8Pb2xpgxY/Dw4UMMGjQIwIf7xP/973/UT4rUCeopRQjhpE934+rQoYNokAoANm7ciKKiIixatKiho3FSfHw81q1bh/j4eCgpKcHU1BRLly5Ft27dWEcjhEjh0924WrVqhTZt2ohel+++N2nSpAbNxVXPnj3Dpk2bkJCQIKqJM2fOhKamJutohBApODg44OjRo1BXV2cdpUk4ffo0/P39xe4Tly9fDjs7O9bRSBNAg1KENBKrV6+Gl5cXXTwJIc0e1UNCCAFyc3Nx5coVvHjxQmxJLo/Hw8SJExkmI4SQukODUoQ0EmpqaoiPj4eBgQHrKKSZSU9Pr/I8LfkhDY3qIWEpOjq6yvMDBw5soCSkOTt58iTc3NyQn58PNTU18Hg80Tkej4ecnByG6biJHngQ0jjRoBQhjYSqqioSEhLoR1gdcXd3R0ZGBvX/kAKfzxe72f0U7TZFGhrVw7o3ePBgPHr0CI8ePWIdpdHj8yX3Afq4RlJNJA2he/fucHFxgb+/P5SVlVnHaRLogYds6D6R1DdqdE4IaZI6duxY4Q8LIunmzZtir4uLi3Hz5k2sX7+edoEkpIkYPXo0Xr58yToGJ7x69UrsdXlN9Pb2pppIGszTp08xe/ZsGpCqQzQXQzZHjx4Ve11eE4OCguDn58coFWlKaKYUIY1ERkYGdHR0ICcnxzoKIQA+NLVct24dIiMjWUchzQzVQ9IYRUVFYf78+ZXujEZIXRozZgy++eYbfP3116yjNBk0C7du7du3DwcOHMDx48dZRyEcR4NShBBCKpSSkgIzMzO8ffuWdRRCCGHu3r17sLS0RH5+PusopBnYuXMnBAIBPDw8YGJighYtWoidHzlyJKNk3EUPPOrWo0ePYGpqSjWR1BoNShHCyJYtWxASEgJNTU1MmzYNjo6OonMvX76ElZUV9f6oRlJSEjZt2oRLly4hKysLANC+fXtYW1tj5syZMDIyYpyQG/Ly8sReC4VCZGZmwtfXF/fu3UN8fDybYKTZoHpY/x4+fIipU6dSnz0pJCYmir0ur4mrV69GSUkJLl68yCgZaU6qakHA4/Gojw9hqqCgAEuXLsXZs2dx//591nEIx1FPKUIYCAwMxNKlS+Hh4YHXr1/DxcUFvr6+WLp0KYAPDQPT0tIYp2zczp49C1dXV/Tq1QujRo1Cu3btAADPnz/H+fPn0atXLxw/fhxOTk6MkzZ+6urqEg0shUIhdHV1sX//fkapSHNB9bBh5OfnIyoqinUMTjA3NwePx5PoP9OvXz/s2rWLUSrS3JSVlbGOwHn0wKNuaGhoiN0nCoVCvHnzBsrKyti7dy/DZKSpoJlShDDQs2dPLFu2DOPHjwcAxMXFwdXVFV5eXhAIBHj+/Dl0dHToKVgVzMzMMGrUKAgEggrP+/r6IiQkROKJN5H06Q9VPp+Ptm3bomvXrpCXp2cXpH5RPawbgYGBVZ5/+vQpfv31V/ocpfDpIGh5TWzZsiWjRKS5e//+Pf391dCnDzwOHjwo9sCDri3SCwoKEntdXhP79u0LDQ0NRqlIU0KDUoQwoKysjKSkJOjr64uO3b59G4MHD4aHhwfmzp1LF8pqKCkpIT4+Hj169Kjw/P3792Fubo6CgoIGTsYtxcXFmDZtGry9vdGlSxfWcUgzRPWwbvD5fHTo0AEKCgoVni8qKkJWVhZ9jtUoLi7GsGHDsHXrVnTr1o11HNKMlZaWwt/fH1u3bsXz58+RnJwMAwMDeHt7Q19fH9999x3riI0aPfCoGyUlJfD394enpyc6derEOg5pomi/dEIY0NLSQkZGhtgxY2NjhIeHY/fu3fjxxx8ZJeMOfX19nD59utLzp0+fhp6eXgMm4qYWLVrgyJEjrGOQZozqYd3Q09PDhg0bkJqaWuG/quol+T8tWrSgGbakUVi5ciX27NmDtWvXig02GxsbY8eOHQyTcUNqaipsbGxEr21sbBAeHo7t27eLZkuR6snLy2PdunUoKSlhHYU0YTQoRQgD/fv3R0hIiMRxIyMjhIWF4ezZswxScYtAIMDixYsxcuRIBAYG4sCBAzhw4AACAwMxatQoLF26FCtXrmQdkxNcXV1x7Ngx1jFIM0X1sG707t0b169fr/R8RT2SSMUmTJiAnTt3so5Bmrng4GBs374dbm5uYrvFmZmZ4d69ewyTcQM98Kg7gwYNop6EpF5RsxBCGFiyZEmlPx569uyJ8PBwmr1Sja+++godO3ZEYGAgfvvtN4nd9yIjI2Ftbc04JTd069YNAoEAsbGx6N27N1RUVMTOz549m1Ey0hxQPawbAoEA7969q/S8kZERUlNTGzARd5WUlGDXrl24cOFChTVx/fr1jJKR5uTp06fo2rWrxPGysjIUFxczSMQt5Q88BgwYIHa8/IGHg4MDo2Tc4+zsjCVLluDWrVsV1sSRI0cySkaaCuopRQghzVxVvaR4PB7tTEMIaVaq+7EaERHRQElIc9a7d2/MmzcPEyZMgKqqKhISEmBgYACBQIDz588jJiaGdcRGLTExEdevX4eHh0eF52/fvo0jR45g+fLlDZyMe/j8yhdX8Xg86stFao1mShHCWG5uLq5cuYIXL16Ibf/L4/EwceJEhsm4Z/Xq1fDy8oK6ujrrKJxCsydIY0H1sG5RTZQNDTqRxsDHxwfu7u54+vQpysrKEBISgvv37yM4OBinTp1iHa/RMzU1hampaaXnjY2NYWxs3ICJuOvj6zEh9YFmShHC0MmTJ+Hm5ob8/HyoqamBx+OJzvF4POTk5DBMxz1qamqIj4+HgYEB6yicIhAIsHDhQigrK4sdLygowLp16+Dj48MoGWlOqB7WPaqJsvH09MTGjRuhqqoqdvzt27eYNWsWdu3axSgZaW5iYmIgEAiQkJCA/Px89OrVCz4+Phg6dCjraJxCDzxqJzg4GOPGjYOioqLY8aKiIuzfvx+TJk1ilIw0FTQoRQhD3bt3h4uLC/z9/SUGBEjNfTy9nUhPTk4OmZmZ0NbWFjuenZ0NbW1tmpZNGgTVw7pHNVE2ldXEly9fon379rQLFSEcQg88ao/uE0l9o933CGHo6dOnmD17Nv0AI0wJhUKxm7RyCQkJ0NTUZJCINEdUDwlreXl5eP36NYRCId68eYO8vDzRv1evXuHMmTMSP8oIqS8+Pj6IiIjA+/fvWUfhtAULFsDT0xP5+fnIzc3Fq1evRP9oQEo6ld0nPnnyBK1bt2aQiDQ11FOKEIacnJxw7do1eopdR5KSkqCjo8M6BmdoaGiAx+OBx+Ohe/fuYjccpaWlyM/Ph5eXF8OEpDmhelj3qCbWjLq6ulhN/BSPx4Ofnx+DZKQ5unTpEtavX4+SkhL06dMHdnZ2sLe3h62tLZSUlFjH4wx64CE7CwsLUU10dHSEvPz/DR2UlpYiNTUVw4YNY5iQNBU0KEUIQ8OHD8eiRYuQlJQEExMTtGjRQuw8bbFaM7q6uqwjcEpAQACEQiE8PT3h5+cn9rRLQUEB+vr6sLa2ZpiQNCdUD+se1cSaiYiIgFAoxKBBg3DkyBGxmaIKCgrQ09OjQT7SYM6fP4+SkhL8+++/iI6ORlRUFAIDA1FYWIg+ffrg4sWLrCNyAj3wkJ2rqysAID4+Hk5OTmjVqpXoXPl94tixYxmlI00J9ZQihCHaYrV2tmzZgpCQEGhqamLatGlwdHQUnXv58iWsrKzw6NEjhgm5ISoqCra2tmJPwAhpaFQPa49qYt1IS0tD586dK1yuQggLycnJiIiIwIULF3Ds2DG0bt0aL1++ZB2LE3bu3AmBQAAPDw964CGjoKAgfPPNNxKNzgmpK9RTihCGysrKKv1HP8CqFhgYiEWLFsHQ0BCKiopwcXHBqlWrROdLS0uRlpbGMCF3qKqq4u7du6LXx48fh6urK3766ScUFRUxTEaaE6qHtUM1se7cvXsXsbGxotebN2+Gubk5xo8fj1evXjFMRpqT7du3Y/z48ejYsSNsbGxw7tw59O/fH9euXcN///3HOh5nTJ06FRkZGRAIBPjqq6/g6uoq+jd69GjW8Thh0KBBYn9zV65cwdy5c7F9+3aGqUhTQoNShDQS1MiyZrZt24Y///wTmzZtwl9//YWIiAhs2LABPj4+rKNxzrRp05CcnAwAePToEcaNGwdlZWUcOnQIP/74I+N0pDmielhzVBPrzqJFi5CXlwcAuHXrFubPnw8XFxekpqZi/vz5jNOR5sLLywthYWGYM2cOHj9+jKNHj2LOnDkwMzOjWXw1QA88am/8+PGIiIgAAGRlZWHw4MG4cuUKli1bBoFAwDgdaQpoUIoQhkpLS7FixQp07NgRrVq1Ei2r8Pb2xs6dOxmna9xSU1NhY2Mjem1jY4Pw8HBs374dS5cuZZiMe5KTk2Fubg4AOHToEOzs7LBv3z7s2bMHR44cYRuONBtUD2uHamLdSU1NhZGREQDgyJEjGDFiBPz9/bF582acPXuWcTrSXISEhMDNzQ379+9H27ZtYWNjg59++gmhoaF49+4d63icRA88ZHP79m1YWVkBAA4ePAgTExPExcXh77//xp49e9iGI00CDUoRwtDKlSuxZ88erF27FgoKCqLjxsbG2LFjB8NkjZ+WlhYyMjLEjhkbGyM8PBy7d++mGT41IBQKUVZWBgC4cOECXFxcAHxokkw9K0hDoXpYO1QT646CgoLoR/+FCxcwdOhQAICmpqZoBhUh9c3V1RXr16/HjRs3kJWVhZ9++glPnz7FF198IdaEn1SNHnjUXnFxsaif1IULF0R9uAwNDZGZmckyGmkiaFCKEIaCg4Oxfft2uLm5QU5OTnTczMwM9+7dY5is8evfvz9CQkIkjhsZGSEsLIyeZteApaUlfvnlF/z111+IiorC8OHDAXyYLdCuXTvG6UhzQfWwdqgm1p3+/ftj/vz5WLFiBa5cuSKqicnJyejUqRPjdKQ5yc7ORkhICLy9vfHTTz9h7969UFNTg7OzM+tonEEPPGqvZ8+e2Lp1K2JiYnD+/HkMGzYMAPDs2TO0adOGcTrSFNCgFCEMPX36FF27dpU4XlZWhuLiYgaJuGPJkiUwNTWt8FzPnj0RHh5OvVSkFBAQgBs3bmDmzJlYtmyZ6G/y8OHDYsuBCKlPVA9rh2pi3dm0aRPk5eVx+PBh/PHHH+jYsSMA4OzZs6IfY4TUNxMTE2hra2PatGl4+vQppk6dips3b+Lly5c4evQo63icQQ88am/NmjXYtm0b7O3t8e2338LMzAwAcOLECdGyPkJqgycUCoWsQxDSXPXu3Rvz5s3DhAkToKqqioSEBBgYGEAgEOD8+fOIiYlhHZE0Y+/fv4ecnJzE9smE1Aeqh4QQ8n82b94Me3t7aGtrQ1FREWpqaqwjcZKSkhLu3bsHPT09sWtLUlISrKyskJ+fzzoiJ5SWliIvLw8aGhqiY48fP4aysjK0tbUZJiNNgTzrAIQ0Zz4+PnB3d8fTp09RVlaGkJAQ3L9/H8HBwTh16hTreJyRm5uLK1eu4MWLF6LeSADA4/EwceJEhsm4paioSOIzBIDOnTszSkSaE6qHdYdqYu2VlZUhJSWlwpo4cOBARqlIc5Gbm4ukpCQsX74cr169AgC0bdsWHh4e8Pb2hrKyMuOE3GFkZISYmBjo6emJHT98+DAsLCwYpeIeOTk5sQEpANDX12cThjQ5NFOKEMZiYmIgEAiQkJCA/Px89OrVCz4+PqLGqqRqJ0+ehJubG/Lz86Gmpia2TTKPx0NOTg7DdNyQnJyM7777DnFxcWLHhUIheDwebZlMGgzVw9qjmlh7ly9fxvjx45GWloZPb5OpJpL6lpOTA2trazx9+hRubm74/PPPAQBJSUnYt28fDA0NcfHiRSQmJuLy5cuYPXs248SN2/Hjx+Hu7o6lS5dCIBDAz89P7IHHkCFDWEds9J4/f46FCxciLCwML168kKiLVBNJbdGgFCGE07p37w4XFxf4+/vTk0MZ2draQl5eHkuWLEGHDh3EfsQCEPUOIIQ0flQTa8/c3Bzdu3eHn59fhTWxdevWjJKR5mDu3LkICwvDhQsXJDYbycrKwtChQ9GjRw+EhoYiMDAQ7u7ujJJyBz3wqB1nZ2ekp6dj5syZFdbEUaNGMUpGmgoalCKEIR8fHzg4OMDa2hotW7ZkHYeTVFRUcOvWLRgYGLCOwlkqKiq4fv06DA0NWUchzRjVw7pBNbH2VFRUkJCQUGHjfULqm76+PrZt2wYnJ6cKz587dw4uLi5Yvnw5li9f3sDpSHOkqqqKmJgYmJubs45CmijafY8Qhi5duoQRI0ZAXV0dAwYMwM8//4wLFy6goKCAdTTOcHJywrVr11jH4DQjIyO8fPmSdQzSzFE9rBtUE2uvb9++SElJYR2DNFOZmZno2bNnpeeNjY3B5/NpQEpKPj4+iIiIwPv371lH4SxdXV2JJXuE1CVqdE4IQ+fPn0dJSQn+/fdfREdHIyoqCoGBgSgsLESfPn1w8eJF1hEbveHDh2PRokVISkqCiYmJxE5xI0eOZJSMO9asWYMff/wR/v7+FX6GtOMPaQhUD+sG1cTamzVrFhYsWICsrKwKP0NTU1NGyUhzoKWlhcePH6NTp04Vnk9NTaXdzmrg0qVLWL9+PUpKStCnTx/Y2dnB3t4etra2UFJSYh2PEwICArBkyRJs27aNmpuTekHL9whpJJKTkxEREYELFy7g2LFjaN26Nc1ekQKfX/mET2pIK53yz/DTHgHU6JywQvVQdlQTa6+iz5DH41FNJA3C09MTDx8+xPnz56GgoCB2rrCwEE5OTjAwMMCuXbsYJeSeTx94xMXF0QOPGtDQ0MC7d+9QUlICZWVliYF62kCD1BYNShHC0Pbt2xEZGYmoqCgUFhZiwIABsLe3h729PUxNTSUGCQipD1FRUVWet7Oza6AkpDmjekgai7S0tCrPf7q1PCF16cmTJ7C0tISioiJmzJgBQ0NDCIVC3L17F1u2bEFhYSGuXr2Kzp07s47KOfTAQzZBQUFVnqdm+6S2aFCKEIb4fD7atm2LBQsWYPr06WjVqhXrSJz2/v17apBMCEdRPax7VBMJ4abU1FRMnz4doaGhol4+PB4PQ4YMwaZNm6gJfw3QAw9CGj8alCKEoWPHjiE6OhqRkZG4e/cuLCwsRBfK/v3703beUigtLYW/vz+2bt2K58+fIzk5GQYGBvD29oa+vj6+++471hEbrcTERKneR/1TSEOgelg3qCbK7sSJE1K9j/pykYby6tUrPHjwAADQtWtXaGpqMk7EPfTAQ3Z5eXlSvY96j5LaokEpQhqJ169fIyYmBocOHcL//vc/8Pl82ilECgKBAEFBQRAIBJg6dSpu374NAwMDHDhwAAEBAbh06RLriI0Wn88X9UmpDPVPISxQPZQd1UTZVdWPqxzVREK4hR54yK78PrEy1GeP1BXafY8QxrKzsxEVFYXIyEhERkbizp070NDQwIABA1hH44Tg4GBs374djo6O8PLyEh03MzPDvXv3GCZr/FJTU1lHIEQM1cPao5oou7KyMtYRCCF1zNXVFa6urgDEH3h88cUX9MCjGhEREawjkGaCBqUIYcjExARJSUnQ1NTEwIEDMXXqVNjZ2dFyqRp4+vRphb0VysrKUFxczCARd1CzXtKYUD2sG1QTCSFEHD3wkA1tdEMaCg1KEcKQl5cX7O3toa2tDUVFRVqTLQMjIyPExMRIDLAcPnwYFhYWjFJxl4mJCc6cOQNdXV3WUUgzQ/WwblBNrFtqamqIj4+HgYEB6yiEEBnQA4+6NXz4cOzYsQMdOnRgHYU0ITQoRQgjubm5SEpKwvLly/Hq1SsAQNu2beHh4QFvb29a4y4lHx8fuLu74+nTpygrK0NISAju37+P4OBgnDp1inU8znn8+DHNpiANjuph3aGaWLeo9Soh3EYPPOpWdHQ0CgoKWMcgTQw1OieEgZycHFhbW+Pp06dwc3PD559/DgBISkrCvn37YGhoiIsXLyIxMRGXL1/G7NmzGSdu3GJiYiAQCJCQkID8/Hz06tULPj4+GDp0KOtonKOqqoqEhASaFUAaDNXDukc1se5QTSSEu3Jzc7Fs2TIcOHCAHnjUEaqJpD7QTClCGBAIBFBQUMDDhw/Rrl07iXNDhw7FxIkTERoaisDAQEYpuWPAgAE4f/486xhNwoABA6CkpMQ6BmlGqB7WPaqJdWfChAk0s4IQDqrqgcfvv/+O8+fP0wMPGejp6aFFixasY5AmhmZKEcKAvr4+tm3bBicnpwrPnzt3Di4uLli+fDmWL1/ewOm4xcfHBw4ODrC2tkbLli1ZxyGE1BDVw7pFNZEQQoC5c+ciLCwMFy5ckHjgkZWVhaFDh6JHjx6iBx7u7u6MkhJCaFCKEAYUFRXx8OFDdOrUqcLzT548gb6+PkpKSho4GfcMGTIEly5dQklJCfr06QM7OzvY29vD1taWZvxIKTs7G23atAEAZGRk4M8//0RBQQFGjhxJO9OQekf1sG5RTaydN2/eIDk5GT169ECrVq1w48YNBAQEoKCgAK6urnBzc2MdkRAiBXrgUT/evn2LAwcOoKCgAEOHDkW3bt1YRyJNAJ91AEKaIy0tLTx+/LjS86mpqdDW1m64QBx2/vx55ObmIiwsDC4uLrh27RrGjBkDdXV19O/fn3W8Ru3WrVvQ19eHtrY2DA0NER8fjz59+mDDhg3Yvn07HBwccOzYMdYxSRNH9bBuUU2UXXR0NDp27Ig+ffpAT08PoaGhsLe3x9WrV3H37l1MmjQJf/75J+uYhBApZGZmomfPnpWeNzY2Bp/PpwGpKqSnp8POzg6qqqoYMmQI0tPT0atXL0yZMgWzZs2Cubk5oqOjWcckTQANShHCgJOTE5YtW4aioiKJc4WFhfD29sawYcMYJOMmeXl52NraYuzYsRg9ejScnJxQVlaGe/fusY7WqP34448wMTFBdHQ07O3t8cUXX2D48OF4/fo1Xr16hWnTpmH16tWsY5Imjuph3aOaKJuff/4ZX331FTIyMjB37lyMGzcOM2fOxN27d3H79m34+flh8+bNrGMSQqRADzxqb+HChSgqKsLWrVuhrKwMJycndOvWDZmZmXj+/DmcnZ3h6+vLOiZpAmj5HiEMPHnyBJaWllBUVMSMGTNgaGgIoVCIu3fvYsuWLSgsLMTVq1fRuXNn1lEbve3btyMyMhJRUVEoLCzEgAEDYG9vD3t7e5iamoLH47GO2GhpaWkhPDwcpqamyM/Ph5qaGq5evYrevXsDAO7du4d+/fohNzeXbVDSpFE9rFtUE2Wnrq6Oy5cvw9DQEEVFRVBSUsKNGzdgZmYGAEhJSYGFhQXevHnDOCkhpDqenp54+PAhzp8/DwUFBbFzhYWFcHJygoGBAXbt2sUoYePXvn17nDhxAlZWVsjJyYGWlhZiY2NhbW0NAEhISICjoyNevnzJOCnhOhqUIoSR1NRUTJ8+HaGhoSj/GvJ4PAwZMgSbNm1C165dGSfkBj6fj7Zt22LBggWYPn06WrVqxToSZ/D5fGRlZYmeFH66ze/z58+ho6OD0tJSljFJM0D1sO5QTZQd1URCmg564FF7fD4fmZmZokbxrVq1QmJiItVEUufkWQcgpLnq0qULzp49i1evXuHBgwcAgK5du0JTU5NxMm4JCQlBdHQ09u/fj+XLl8PCwkI0K6B///5QVlZmHbFR+3TWBM2iICxQPaw7VBNlx+PxxGrgp68JIdzRqVMnXLp0CdOnT8fSpUsrfOBBA1LV+7QmElIfaKYUIaTJeP36NWJiYnDo0CH873//A5/Px/v371nHarT4fD6cnZ2hqKgIADh58iQGDRoEFRUVAB+mt587d46egBHCUVQTa4bP58PY2Bjy8h+e2SYmJsLQ0FC09KekpAR37tyhmkgIx9ADD9nw+Xx8//33oocZmzdvxoQJE9C6dWsAwLt37/Dnn39STSS1RoNShBDOy87ORlRUFCIjIxEZGYk7d+5AQ0MDAwYMwNGjR1nHa7Q8PDyket/u3bvrOQkhpC5RTZSNn5+fVO+j3boIIc2Bvb29VLOjIiIiGiANacpoUIoQwmkmJiZISkqCpqYmBg4cCHt7e9jZ2cHU1JR1NEIIaXBUEwkhhBDCJdRTihDCaV5eXrC3t4e2tjYUFRWhpqbGOhIhhDBDNZEQQgghXMJnHYAQQmSVm5uLpKQk2NnZoX379tDQ0ED79u2xdOlSvHv3jnW8Ru/Fixdir+Pj4+Hu7g5bW1t8+eWXiIyMZBOMECITqom1c+XKFbHeKKdOnYKdnR06duwIS0tLBAcHM0xHCCENa8SIEfjrr79QUFDAOgpp4mj5HiGEk3JycmBtbY2nT5/Czc0Nn3/+OQAgKSkJ+/btg6GhIS5evIjExERcvnwZs2fPZpy48ZGTk0NmZia0tbURFxcHe3t72NjYwMrKCvHx8YiIiEBYWBgGDhzIOiohpBpUE2vv45p48uRJuLq6YsKECejbty9u3ryJPXv24ODBgxg9ejTrqIQQUu/4fD7k5OSgoqKCb7/9FlOmTEHv3r1ZxyJNEA1KEUI4ae7cuQgLC8OFCxfQrl07sXNZWVkYOnQoevTogdDQUAQGBsLd3Z1R0saLz+cjKysL2traGDp0KHR1dbFz507R+blz5+LWrVsICwtjmJIQIg2qibX3cU0cMGAA+vfvj1WrVonO+/v74+TJk7h06RLDlIQQ0jD4fD5u376N0NBQ7Nq1C3fu3IGJiQmmTJkCNzc3aGhosI5ImggalCKEcJK+vj62bdsGJyenCs+fO3cOLi4uWL58Oe2UVImPf4Dp6OggJCQE/fr1E52/c+cO7O3t8d9//zFMSQiRBtXE2vu4JrZr1w5nzpwRmxVw//599OvXD69evWKYkhBCGsbHNRH4sMR5586dOHDgAIqKiuDq6oopU6Zg0KBBjJMSrqOeUoQQTsrMzETPnj0rPW9sbAw+n08/vqrx5s0b5OXloWXLllBUVBQ717JlS+pDQwhHUE2sG0lJSUhMTISSkhLKysokzpeUlDBIRQgh7FlZWWHbtm149uwZtmzZgoyMDAwZMoR1LNIE0KAUIYSTtLS08Pjx40rPp6amip7skMp1794dGhoaePz4Ma5duyZ27s6dO9DR0WGUjBBSE1QT64ajoyPMzc2Rnp6O2NhYsXM3b95E586dGSUjhJDGQVlZGZMnT0ZMTAzu3r3LOg5pAuRZByCEEFk4OTlh2bJlOH/+PBQUFMTOFRYWwtvbG8OGDWOUjhsiIiLEXnfo0EHsdWpqKr7//vuGjEQIkRHVxNpLTU0Ve92qVSux10VFRVi8eHFDRiKEEGbs7Owkrief6t69ewOlIU0Z9ZQihHDSkydPYGlpCUVFRcyYMQOGhoYQCoW4e/cutmzZgsLCQly9epWeahNCmgWqiYQQQgjhIhqUIoRwVmpqKqZPn47Q0FCUlzIej4chQ4Zg06ZN6Nq1K+OEhBDScKgmEkIIqU+rV6+Gl5cX1NXVWUchTQgNShFCOO/Vq1d48OABAKBr167Q1NRknKhpcHd3R0ZGBsLDw1lHIYTUANXE+jF48GA8evQIjx49Yh2FEEKYUFNTQ3x8PAwMDFhHIU0I9ZQihHCehoYGrKysWMdocjp27Ag+n/bDIIRrqCbWj9GjR+Ply5esYxBCCDM0n4XUB5opRQghhBBCCCGEkCqpqqoiISGBZkqROkWDUoQQQgghhBBCCKlSRkYGdHR0ICcnxzoKaUJoXQYhhDRjSUlJmD59OiwsLNChQwd06NABFhYWmD59OpKSkljHI4SQRuPhw4cYNGgQ6xiEEMKMrq4uDUiROkc9pQghpJk6e/YsXF1d0atXL4waNQrt2rUDADx//hznz59Hr169cPz4cTg5OTFOSggh7OXn5yMqKop1DEIIaTBbtmxBSEgINDU1MW3aNDg6OorOvXz5ElZWVrT5A6k1Wr5HCCHNlJmZGUaNGgWBQFDheV9fX4SEhCAxMbGBkxFCSMMLDAys8vzTp0/x66+/orS0tIESEUIIO4GBgVi6dCk8PDzw+vVrHDx4EL6+vli6dCmADw8xdXR0qCaSWqNBKUIIaaaUlJQQHx+PHj16VHj+/v37MDc3R0FBQQMnI4SQhsfn89GhQwcoKChUeL6oqAhZWVn0A4wQ0iz07NkTy5Ytw/jx4wEAcXFxcHV1hZeXFwQCAQ1KkTpDy/cIIaSZ0tfXx+nTpysdlDp9+jT09PQaOBUhhLChp6eHNWvW4Ouvv67wfHx8PHr37t3AqQghhI3U1FTY2NiIXtvY2CA8PByDBw9GcXEx5s6dyy4caVJoUIoQQpopgUCA8ePHIzIyEoMHDxbrKRUWFoZz585h3759jFMSQkjD6N27N65fv17poBSPxwMtMCCENBdaWlrIyMiAvr6+6JixsTHCw8MxaNAgPHv2jF040qTQ8j1CCGnG4uLiEBgYiEuXLiErKwsA0L59e1hbW2POnDmwtrZmnJAQQhpGUlIS3r17B0tLywrPFxcX49mzZzSDlBDSLIwfPx7t2rXDhg0bJM7duXMHDg4OyM7OpuV7pNZoUIoQQgghhBBCCCEiiYmJuH79Ojw8PCo8f/v2bRw5cgTLly9v4GSkqaFBKUIIISKrV6+Gl5cX1NXVWUchhBDmqCYSQggh9YsGpQghhIioqakhPj4eBgYGrKMQQghzVBMJIQTIzc3FlStX8OLFC5SVlYmO83g8TJw4kWEy0hRQo3NCCCEi9JyCEEL+D9VEQkhzd/LkSbi5uSE/Px9qamrg8XiiczQoReoCn3UAQgghhBBCCCGEND4LFiyAp6cn8vP/X3v3GhPl1a5x/BpUEFFSpoIEBQlSq1QwEMUCIvhWHKvRNjX9IhrPEQ9Bk4ZW2zgaG22jsW2IVfFISaO2KpJoWouHguAR2xSNA2oRrdLEtgImqCCn/aH7HTeB7uzsV59H5/n/kvkw65aZa/hwG+6staZB9fX1qqurcz9qa2vNjgcPwPE9AIDb7du3FRISom7dupkdBQBMR08EYHV+fn66fPkyx5jxzLBTCgDgFhoayh9fAPDf6IkArM7hcOjixYtmx4AH404pALCwzZs3Kz8/X3a7XQsWLNAbb7zhrv3111+Kj4/XjRs3TEwIAMahJwJAR5MmTVJWVpZcLpeio6PVo0ePDvUpU6aYlAyeguN7AGBR2dnZWrFihWbPnq379+/r22+/1erVq7VixQpJ0t27dxUSEqLW1laTkwLAs0dPBIDOvLz++XCVzWajJ+I/xk4pALConJwcbd++XdOmTZMkLVy4UG+//bYePXqkNWvWmJwOAIxFTwSAztra2syOAA/HUAoALKq6ulqJiYnu54mJiTp58qTGjRun5uZmLVu2zLxwAGAweiIA/O8aGxvVs2dPs2PAwzCUAgCL6tu3r27fvq3w8HD32rBhw3Ty5En961//0u+//25eOAAwGD0RADprbW3VunXrtHXrVt29e1fXrl1TRESEVq5cqfDwcM2dO9fsiHjB8e17AGBRo0ePVn5+fqf1qKgonThxQt9//70JqQDAHPREAOhs7dq1ys3N1fr16+Xt7e1eHzZsmHbs2GFiMngKhlIAYFHLly9XTExMl7XXXntNJ0+elNPpNDgVAJiDnggAneXl5Wnbtm1KT09Xt27d3OvDhw9XZWWlicngKfj2PQAAAAAA0Imvr68qKys1cOBA9enTR+Xl5YqIiJDL5VJ8fLwaGhrMjogXHHdKAQBUX1+vCxcu6I8//ujwLSs2m00zZswwMRkAGI+eCAB/i4qKUklJiQYOHNhh/cCBA4qNjTUpFTwJQykAsLjDhw8rPT1dDQ0N8vf3l81mc9f4AwyA1dATAeAJp9OpmTNnqqamRm1tbcrPz9fVq1eVl5enI0eOmB0PHoDjewBgcYMHD9bEiRO1bt069erVy+w4AGAqeiIAdFRSUqI1a9aovLxcDQ0NiouLk9Pp1Pjx482OBg/AUAoALM7Pz0+XL19WRESE2VEAwHT0RAAAjMO37wGAxTkcDl28eNHsGADwXKAnAsATTqdTP/74oxobG82OAg/FnVIAYHGTJk1SVlaWXC6XoqOj1aNHjw71KVOmmJQMAIxHTwSAJ86ePavPPvtMLS0tGjlypFJSUpSamqqkpCT5+vqaHQ8egON7AGBxXl7/vGnWZrOptbXVwDQAYC56IgB01NLSovPnz+vUqVMqLi7WmTNn1NTUpJEjR6q0tNTseHjBsVMKACzuf37dOQBYHT0RADrq3r27kpKSFBgYKLvdrj59+qigoECVlZVmR4MH4E4pAIAb9wUAwBP0RABWt23bNk2bNk39+/dXYmKijh49qtGjR+vixYv6888/zY4HD8BQCgAsrrW1VR9//LH69++v3r1768aNG5KklStXaufOnSanAwBj0RMB4ImMjAydOHFCS5cu1c2bN3Xo0CEtXbpUw4cPl81mMzsePABDKQCwuLVr1yo3N1fr16+Xt7e3e33YsGHasWOHickAwHj0RAB4Ij8/X+np6dq3b58CAwOVmJioDz/8UIWFhXr48KHZ8eABuOgcACwuMjJSOTk5euONN9SnTx+Vl5crIiJClZWVSkhIUF1dndkRAcAw9EQA6Nr9+/dVUlKi/fv3a+/evfLy8uKYM/5jXHQOABZXU1OjyMjITuttbW1qbm42IREAmIeeCAAd3bt3T8XFxSoqKlJRUZGuXLmigIAAJScnmx0NHoChFABYXFRUlEpKSjRw4MAO6wcOHFBsbKxJqQDAHPREAHgiOjpaLpdLdrtdY8aM0fz585WSkqKYmBizo8FDMJQCAItzOp2aOXOmampq1NbWpvz8fF29elV5eXk6cuSI2fEAwFD0RAB4IiMjQ6mpqQoKCpKPj4/8/f3NjgQPw0XnAGBxb731lg4fPqzjx4/Lz89PTqdTFRUVOnz4sNLS0syOBwCGoicCwN/q6+vlcrmUkpKi4OBgBQQEKDg4WCtWrOCSczw1XHQOAAAAAADcamtrlZCQoJqaGqWnp2vo0KGSJJfLpT179mjIkCEqLS3VpUuXdO7cOWVmZpqcGC8qju8BgMU5nU6NHTtWCQkJ6tmzp9lxAMBU9EQAkNasWSNvb29VVVWpX79+nWrjx4/XjBkzVFhYqOzsbJNSwhOwUwoALC4tLU1nz55VS0uLRo4cqZSUFKWmpiopKUm+vr5mxwMAQ9ETAUAKDw9XTk6OHA5Hl/WjR49q4sSJWrVqlVatWmVwOngShlIAALW0tOj8+fM6deqUiouLdebMGTU1NWnkyJEqLS01Ox4AGIqeCMDqfHx8VFVVpQEDBnRZv3PnjsLDw9XS0mJwMngaju8BANS9e3clJSUpMDBQdrtdffr0UUFBgSorK82OBgCGoycCsLq+ffvq5s2b/ziUqq6uVlBQkMGp4InYKQUAFrdt2zYVFRWpuLhYTU1NSk5OVmpqqlJTUxUTEyObzWZ2RAAwDD0RAKQ5c+aoqqpKx44dk7e3d4daU1OTHA6HIiIitGvXLpMSwlMwlAIAi/Py8lJgYKDee+89LVq0SL179zY7EgCYhp4IAH8fzxsxYoR8fHy0ePFiDRkyRO3t7aqoqNDmzZvV1NSksrIyhYWFmR0VLziGUgBgcQUFBTp16pSKiopUUVGh2NhY966A0aNHq1evXmZHBADD0BMB4G/V1dVatGiRCgsL9e+xgc1mU1pamjZt2qTIyEiTE8ITMJQCALjdv39fJSUl2r9/v/bu3SsvLy81NjaaHQsATEFPBACprq5O169flyRFRkbKbrebnAiehIvOAQC6d++eiouLVVRUpKKiIl25ckUBAQFKTk42OxoAGI6eCABPBAQEKD4+3uwY8FDslAIAi4uOjpbL5ZLdbteYMWOUmpqqlJQUxcTEmB0NAAxHTwQAwDjslAIAi8vIyFBqaqqCgoLk4+Mjf39/syMBgGnoiQAAGMfL7AAAAPPU19fL5XIpJSVFwcHBCggIUHBwsFasWKGHDx+aHQ8ADEVPBADAWBzfAwCLqq2tVUJCgmpqapSenq6hQ4dKklwul/bs2aMhQ4aotLRUly5d0rlz55SZmWlyYgB4duiJAAAYj+N7AGBRa9askbe3t6qqqtSvX79OtfHjx2vGjBkqLCxUdna2SSkBwBj0RAAAjMdOKQCwqPDwcOXk5MjhcHRZP3r0qCZOnKhVq1Zp1apVBqcDAGPREwEAMB5DKQCwKB8fH1VVVWnAgAFd1u/cuaPw8HC1tLQYnAwAjEdPBADAeFx0DgAW1bdvX928efMf69XV1QoKCjIuEACYiJ4IAIDxGEoBgEU5HA599NFHevz4cadaU1OTVq5cqQkTJpiQDACMR08EAMB4HN8DAIu6c+eORowYIR8fHy1evFhDhgxRe3u7KioqtHnzZjU1NamsrExhYWFmRwWAZ46eCACA8RhKAYCFVVdXa9GiRSosLNS//zuw2WxKS0vTpk2bFBkZaXJCADAOPREAAGMxlAIAqK6uTtevX5ckRUZGym63m5wIAMxDTwQAwBgMpQAAAAAAAGA4LjoHAAAAAACA4RhKAQAAAAAAwHAMpQAAAAAAAGA4hlIAAAAAAAAwHEMpAACA/6eioiLZbDbV19f/n38mPDxcX3zxxTPLBAAA8KJgKAUAADzWrFmzZLPZlJGR0am2ePFi2Ww2zZo1y/hgAAAAYCgFAAA8W2hoqPbt26dHjx651xobG7Vnzx6FhYWZmAwAAMDaGEoBAACPFhcXp9DQUOXn57vX8vPzFRYWptjYWPdaU1OTMjMzFRQUpJ49e2r06NEqKyvr8FrfffedBg8eLF9fX40dO1Y3b97s9H6lpaVKTk6Wr6+vQkNDlZmZqQcPHnSZrb29XatXr1ZYWJh8fHwUEhKizMzMp/PBAQAAnnMMpQAAgMebM2eOdu/e7X6+a9cuzZ49u8O/ef/993Xw4EF99dVX+vnnnxUZGSmHw6Ha2lpJ0u3bt/XOO+9o8uTJ+uWXXzRv3jwtX768w2tUVVVpwoQJmjp1qi5duqRvvvlGpaWlWrJkSZe5Dh48qM8//1w5OTm6fv26CgoKFB0d/ZQ/PQAAwPOJoRQAAPB406dPV2lpqW7duqVbt27p9OnTmj59urv+4MEDbdmyRRs2bNCbb76pqKgobd++Xb6+vtq5c6ckacuWLRo0aJA2btyoV199Venp6Z3uo/rkk0+Unp6uZcuW6ZVXXlFiYqKys7OVl5enxsbGTrl+++03BQcHa9y4cQoLC1N8fLzmz5//TH8XAAAAzwuGUgAAwOMFBgZq0qRJys3N1e7duzVp0iT17dvXXa+qqlJzc7OSkpLcaz169FB8fLwqKiokSRUVFRo1alSH101ISOjwvLy8XLm5uerdu7f74XA41NbWpurq6k653n33XT169EgRERGaP3++Dh06pJaWlqf50QEAAJ5b3c0OAAAAYIQ5c+a4j9F9+eWXz+Q9GhoatGDBgi7vherqUvXQ0FBdvXpVx48f17Fjx7Ro0SJt2LBBxcXF6tGjxzPJCAAA8LxgpxQAALCECRMm6PHjx2pubpbD4ehQGzRokLy9vXX69Gn3WnNzs8rKyhQVFSVJGjp0qC5cuNDh586dO9fheVxcnFwulyIjIzs9vL29u8zl6+uryZMnKzs7W0VFRTp79qwuX778ND4yAADAc42dUgAAwBK6devmPorXrVu3DjU/Pz8tXLhQWVlZstvtCgsL0/r16/Xw4UPNnTtXkpSRkaGNGzcqKytL8+bN008//aTc3NwOr/PBBx/o9ddf15IlSzRv3jz5+fnJ5XLp2LFj2rRpU6dMubm5am1t1ahRo9SrVy99/fXX8vX11cCBA5/NLwEAAOA5wk4pAABgGf7+/vL39++y9umnn2rq1KmaMWOG4uLi9Ouvv+qHH35QQECApL+P3x08eFAFBQUaPny4tm7dqnXr1nV4jZiYGBUXF+vatWtKTk5WbGysnE6nQkJCunzPl156Sdu3b1dSUpJiYmJ0/PhxHT58WC+//PLT/eAAAADPIVt7e3u72SEAAAAAAABgLeyUAgAAAAAAgOEYSgEAAAAAAMBwDKUAAAAAAABgOIZSAAAAAAAAMBxDKQAAAAAAABiOoRQAAAAAAAAMx1AKAAAAAAAAhmMoBQAAAAAAAMMxlAIAAAAAAIDhGEoBAAAAAADAcAylAAAAAAAAYLj/AuDcijw2Rj/gAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from llm_toolkit.translation_utils import plot_times\n", + "\n", + "plot_times(perf_df)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6464f8dc-deb7-4a73-aac5-206553ce5036", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Experiment 2" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f9a892d0-3c3e-47da-8306-7909a11f9e9f", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chineseenglishunsloth/Qwen2-0.5B-Instructunsloth/Qwen2-0.5B-Instruct(finetuned)unsloth/Qwen2-1.5B-Instructunsloth/Qwen2-1.5B-Instruct(finetuned)unsloth/Qwen2-7B-Instructunsloth/Qwen2-7B-Instruct(finetuned)unsloth/mistral-7b-instruct-v0.3unsloth/mistral-7b-instruct-v0.3(finetuned)gradientai/Llama-3-8B-Instruct-Gradient-1048kgradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned)unsloth/Qwen2-72B-Instruct-bnb-4bitunsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned)
0老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞...Old Geng picked up his shotgun, squinted, and ...Old Teng holds his gun up, his eyes narrowed a...Old Geng raised his rifle and tilted his head ...Old Jin raises his gun, squints one eye as he ...Old Geng raised his pistol, squinted through t...Old Geng raised his gun, squinted one of his t...Old Geng raised his rifle and squinted into th...Geng Da initiates firing, squinting to form a ...Old Geng aimed and fired. A triangular slit op...The old man pulled out his gun, squinting one ...Old Geng raised his rifle, squinting through t...Lao Geng raised his gun, narrowed one of his t...Old Geng raised his gun, narrowed one of his t...
1次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，...Next day Grannie Liu was up before dawn. As so...The next morning, Liu Geo woke up at five o'cl...But not before noon did Grannie Liu rise up an...At dawn the next day, Liu Langlang got up earl...She got up about dawn with a purpose already e...The next morning, before the dawn had fully br...First thing in the morning Grannie Liu rose to...The next day, when it was still dark, Liu Lao ...Before dawn next day Grannie Liu got up and bu...The next day, when the sun had not yet risen, ...Grannie Liu got up before daylight was even vi...Before dawn next morning, Granny Liu got up to...As soon as it was light, Grannie Liu got up an...
\n", + "
" + ], + "text/plain": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "1 次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "1 Next day Grannie Liu was up before dawn. As so... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct \\\n", + "0 Old Teng holds his gun up, his eyes narrowed a... \n", + "1 The next morning, Liu Geo woke up at five o'cl... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct(finetuned) \\\n", + "0 Old Geng raised his rifle and tilted his head ... \n", + "1 But not before noon did Grannie Liu rise up an... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct \\\n", + "0 Old Jin raises his gun, squints one eye as he ... \n", + "1 At dawn the next day, Liu Langlang got up earl... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct(finetuned) \\\n", + "0 Old Geng raised his pistol, squinted through t... \n", + "1 She got up about dawn with a purpose already e... \n", + "\n", + " unsloth/Qwen2-7B-Instruct \\\n", + "0 Old Geng raised his gun, squinted one of his t... \n", + "1 The next morning, before the dawn had fully br... \n", + "\n", + " unsloth/Qwen2-7B-Instruct(finetuned) \\\n", + "0 Old Geng raised his rifle and squinted into th... \n", + "1 First thing in the morning Grannie Liu rose to... \n", + "\n", + " unsloth/mistral-7b-instruct-v0.3 \\\n", + "0 Geng Da initiates firing, squinting to form a ... \n", + "1 The next day, when it was still dark, Liu Lao ... \n", + "\n", + " unsloth/mistral-7b-instruct-v0.3(finetuned) \\\n", + "0 Old Geng aimed and fired. A triangular slit op... \n", + "1 Before dawn next day Grannie Liu got up and bu... \n", + "\n", + " gradientai/Llama-3-8B-Instruct-Gradient-1048k \\\n", + "0 The old man pulled out his gun, squinting one ... \n", + "1 The next day, when the sun had not yet risen, ... \n", + "\n", + " gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) \\\n", + "0 Old Geng raised his rifle, squinting through t... \n", + "1 Grannie Liu got up before daylight was even vi... \n", + "\n", + " unsloth/Qwen2-72B-Instruct-bnb-4bit \\\n", + "0 Lao Geng raised his gun, narrowed one of his t... \n", + "1 Before dawn next morning, Granny Liu got up to... \n", + "\n", + " unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) \n", + "0 Old Geng raised his gun, narrowed one of his t... \n", + "1 As soon as it was light, Grannie Liu got up an... " + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "df = pd.read_csv(\"results/experiment-2-results.csv\")\n", + "df.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "a1781c13-526e-4586-b14f-da5ba0d8a1b4", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "unsloth/Qwen2-0.5B-Instruct: {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.26453254295068257, 'bleu_scores': {'bleu': 0.04698039499333136, 'precisions': [0.309245348800355, 0.07347623117890723, 0.023966406063295892, 0.008945686900958467], 'brevity_penalty': 1.0, 'length_ratio': 1.0450811526995694, 'translation_length': 31551, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3105157084661859, 'rouge2': 0.08870671944085522, 'rougeL': 0.25592585227840225, 'rougeLsum': 0.2559676791644768}}\n", + "unsloth/Qwen2-0.5B-Instruct(finetuned): {'accuracy': 0.00176522506619594, 'correct_ids': [147, 533], 'meteor': 0.28664792904792147, 'bleu_scores': {'bleu': 0.0633353272697663, 'precisions': [0.33089419978517726, 0.08866324714749294, 0.0345015434901035, 0.016769504485747815], 'brevity_penalty': 0.9867295481943301, 'length_ratio': 0.9868168267638291, 'translation_length': 29792, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3173288345459394, 'rouge2': 0.10618140518074348, 'rougeL': 0.26121163488973825, 'rougeLsum': 0.26105762971788327}}\n", + "unsloth/Qwen2-1.5B-Instruct: {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.3108076173265163, 'bleu_scores': {'bleu': 0.07171893503391259, 'precisions': [0.39917895129688374, 0.11838516093835243, 0.0465865090686774, 0.01994873985476292], 'brevity_penalty': 0.8809955157186649, 'length_ratio': 0.8875455448824114, 'translation_length': 26795, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3666102879704475, 'rouge2': 0.1266675803266789, 'rougeL': 0.31056277362475493, 'rougeLsum': 0.31064508228763266}}\n", + "unsloth/Qwen2-1.5B-Instruct(finetuned): {'accuracy': 0.00441306266548985, 'correct_ids': [147, 170, 309, 526, 533], 'meteor': 0.3412870633079792, 'bleu_scores': {'bleu': 0.09439554395746873, 'precisions': [0.3842746400885936, 0.1271192353310542, 0.05777680938373824, 0.029647860658841348], 'brevity_penalty': 0.9869644625887559, 'length_ratio': 0.9870486916197416, 'translation_length': 29799, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.37271457574577416, 'rouge2': 0.1461825645475855, 'rougeL': 0.31782980248661996, 'rougeLsum': 0.3176719516563038}}\n", + "unsloth/Qwen2-7B-Instruct: {'accuracy': 0.00088261253309797, 'correct_ids': [77], 'meteor': 0.3706126714366309, 'bleu_scores': {'bleu': 0.10881761334734624, 'precisions': [0.41661179878851723, 0.14690695209109872, 0.06769832799715404, 0.033841135698135585], 'brevity_penalty': 1.0, 'length_ratio': 1.006160980457105, 'translation_length': 30376, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.41631654006456126, 'rouge2': 0.16673054869332232, 'rougeL': 0.35965715874226606, 'rougeLsum': 0.3597263235968114}}\n", + "unsloth/Qwen2-7B-Instruct(finetuned): {'accuracy': 0.00617828773168579, 'correct_ids': [147, 199, 309, 526, 531, 658, 935], 'meteor': 0.4016299621577931, 'bleu_scores': {'bleu': 0.1340342690832733, 'precisions': [0.44661102067675684, 0.1784426820475847, 0.0918075911311537, 0.05294140732310349], 'brevity_penalty': 0.9554111104454712, 'length_ratio': 0.9563762835375952, 'translation_length': 28873, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.43259257689704633, 'rouge2': 0.1981555489001432, 'rougeL': 0.3801845230388961, 'rougeLsum': 0.37954828954862707}}\n", + "unsloth/mistral-7b-instruct-v0.3: {'accuracy': 0.00088261253309797, 'correct_ids': [77], 'meteor': 0.3221588341261717, 'bleu_scores': {'bleu': 0.08500847701637192, 'precisions': [0.3839848248763634, 0.12152594314699355, 0.05129145876137364, 0.023885167464114832], 'brevity_penalty': 0.9776268817156146, 'length_ratio': 0.9778734680357735, 'translation_length': 29522, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.36677374147027, 'rouge2': 0.132281273499625, 'rougeL': 0.31779356328547126, 'rougeLsum': 0.3178345285457188}}\n", + "unsloth/mistral-7b-instruct-v0.3(finetuned): {'accuracy': 0.00529567519858782, 'correct_ids': [77, 147, 199, 240, 309, 364], 'meteor': 0.38725013933546865, 'bleu_scores': {'bleu': 0.1255052835038013, 'precisions': [0.4230690780527604, 0.16504614079128804, 0.083050972304066, 0.04753304641869044], 'brevity_penalty': 0.9740309389176179, 'length_ratio': 0.9743623716462405, 'translation_length': 29416, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4141364853072854, 'rouge2': 0.18555040788669064, 'rougeL': 0.3582465278807073, 'rougeLsum': 0.35834634945673904}}\n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k: {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.31732985329639374, 'bleu_scores': {'bleu': 0.05328901091080452, 'precisions': [0.23537940003837707, 0.07314835044789163, 0.03125210027555615, 0.014986438652139935], 'brevity_penalty': 1.0, 'length_ratio': 1.5535939052666445, 'translation_length': 46903, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.36349616085155206, 'rouge2': 0.12989126572749554, 'rougeL': 0.3115960462482712, 'rougeLsum': 0.3117420637997418}}\n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned): {'accuracy': 0.01235657546337158, 'correct_ids': [41, 77, 133, 147, 199, 301, 348, 364, 413, 526, 533, 778, 893, 1011], 'meteor': 0.3921204969248105, 'bleu_scores': {'bleu': 0.12623244689421761, 'precisions': [0.43614474551573396, 0.16861682918020945, 0.08602838748541095, 0.048517732169536844], 'brevity_penalty': 0.9536797061184072, 'length_ratio': 0.9547201059953627, 'translation_length': 28823, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.42640620693017617, 'rouge2': 0.19258710119333206, 'rougeL': 0.36806549573170777, 'rougeLsum': 0.3679460597175971}}\n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit: {'accuracy': 0.00088261253309797, 'correct_ids': [533], 'meteor': 0.39327797374995854, 'bleu_scores': {'bleu': 0.12559679608930588, 'precisions': [0.43200657894736844, 0.1652031298048997, 0.08068242402701262, 0.04321421958896501], 'brevity_penalty': 1.0, 'length_ratio': 1.0069559456773767, 'translation_length': 30400, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.44060491048814143, 'rouge2': 0.18992148221555435, 'rougeL': 0.38307187208859017, 'rougeLsum': 0.38321351591078995}}\n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned): {'accuracy': 0.00794351279788173, 'correct_ids': [147, 199, 240, 272, 364, 419, 658, 659, 820], 'meteor': 0.45716060031914707, 'bleu_scores': {'bleu': 0.17001816364266625, 'precisions': [0.48491918577128557, 0.2166942090643789, 0.12017135952546593, 0.07191963944694829], 'brevity_penalty': 0.9793863208805533, 'length_ratio': 0.9795958926796953, 'translation_length': 29574, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4843039193858598, 'rouge2': 0.2423067746227518, 'rougeL': 0.424815398022727, 'rougeLsum': 0.4247048351560071}}\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
accuracymeteorbleu_1rouge_l
count12.00000012.00000012.00000012.000000
mean0.0033830.3537350.1004110.337413
std0.0039260.0557710.0375670.050474
min0.0000000.2645330.0469800.255926
25%0.0006620.3156990.0696230.311338
50%0.0013240.3559500.1016070.338038
75%0.0055160.3924100.1257560.371095
max0.0123570.4571610.1700180.424815
\n", + "
" + ], + "text/plain": [ + " accuracy meteor bleu_1 rouge_l\n", + "count 12.000000 12.000000 12.000000 12.000000\n", + "mean 0.003383 0.353735 0.100411 0.337413\n", + "std 0.003926 0.055771 0.037567 0.050474\n", + "min 0.000000 0.264533 0.046980 0.255926\n", + "25% 0.000662 0.315699 0.069623 0.311338\n", + "50% 0.001324 0.355950 0.101607 0.338038\n", + "75% 0.005516 0.392410 0.125756 0.371095\n", + "max 0.012357 0.457161 0.170018 0.424815" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "metrics_df = get_metrics(df)\n", + "metrics_df.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "01218949-56d7-47f3-b741-14457e5e855a", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
modelaccuracymeteorbleu_1rouge_lall_metrics
0Qwen2-0.5B-Instruct0.0000000.2645330.0469800.255926{'accuracy': 0.0, 'correct_ids': [], 'meteor':...
1Qwen2-0.5B-Instruct(finetuned)0.0017650.2866480.0633350.261212{'accuracy': 0.00176522506619594, 'correct_ids...
2Qwen2-1.5B-Instruct0.0000000.3108080.0717190.310563{'accuracy': 0.0, 'correct_ids': [], 'meteor':...
3Qwen2-1.5B-Instruct(finetuned)0.0044130.3412870.0943960.317830{'accuracy': 0.00441306266548985, 'correct_ids...
4Qwen2-7B-Instruct0.0008830.3706130.1088180.359657{'accuracy': 0.00088261253309797, 'correct_ids...
5Qwen2-7B-Instruct(finetuned)0.0061780.4016300.1340340.380185{'accuracy': 0.00617828773168579, 'correct_ids...
6mistral-7b-instruct-v0.30.0008830.3221590.0850080.317794{'accuracy': 0.00088261253309797, 'correct_ids...
7mistral-7b-instruct-v0.3(finetuned)0.0052960.3872500.1255050.358247{'accuracy': 0.00529567519858782, 'correct_ids...
8Llama-3-8B-Instruct-Gradient-1048k0.0000000.3173300.0532890.311596{'accuracy': 0.0, 'correct_ids': [], 'meteor':...
9Llama-3-8B-Instruct-Gradient-1048k(finetuned)0.0123570.3921200.1262320.368065{'accuracy': 0.01235657546337158, 'correct_ids...
10Qwen2-72B-Instruct-bnb-4bit0.0008830.3932780.1255970.383072{'accuracy': 0.00088261253309797, 'correct_ids...
11Qwen2-72B-Instruct-bnb-4bit(finetuned)0.0079440.4571610.1700180.424815{'accuracy': 0.00794351279788173, 'correct_ids...
\n", + "
" + ], + "text/plain": [ + " model accuracy meteor \\\n", + "0 Qwen2-0.5B-Instruct 0.000000 0.264533 \n", + "1 Qwen2-0.5B-Instruct(finetuned) 0.001765 0.286648 \n", + "2 Qwen2-1.5B-Instruct 0.000000 0.310808 \n", + "3 Qwen2-1.5B-Instruct(finetuned) 0.004413 0.341287 \n", + "4 Qwen2-7B-Instruct 0.000883 0.370613 \n", + "5 Qwen2-7B-Instruct(finetuned) 0.006178 0.401630 \n", + "6 mistral-7b-instruct-v0.3 0.000883 0.322159 \n", + "7 mistral-7b-instruct-v0.3(finetuned) 0.005296 0.387250 \n", + "8 Llama-3-8B-Instruct-Gradient-1048k 0.000000 0.317330 \n", + "9 Llama-3-8B-Instruct-Gradient-1048k(finetuned) 0.012357 0.392120 \n", + "10 Qwen2-72B-Instruct-bnb-4bit 0.000883 0.393278 \n", + "11 Qwen2-72B-Instruct-bnb-4bit(finetuned) 0.007944 0.457161 \n", + "\n", + " bleu_1 rouge_l all_metrics \n", + "0 0.046980 0.255926 {'accuracy': 0.0, 'correct_ids': [], 'meteor':... \n", + "1 0.063335 0.261212 {'accuracy': 0.00176522506619594, 'correct_ids... \n", + "2 0.071719 0.310563 {'accuracy': 0.0, 'correct_ids': [], 'meteor':... \n", + "3 0.094396 0.317830 {'accuracy': 0.00441306266548985, 'correct_ids... \n", + "4 0.108818 0.359657 {'accuracy': 0.00088261253309797, 'correct_ids... \n", + "5 0.134034 0.380185 {'accuracy': 0.00617828773168579, 'correct_ids... \n", + "6 0.085008 0.317794 {'accuracy': 0.00088261253309797, 'correct_ids... \n", + "7 0.125505 0.358247 {'accuracy': 0.00529567519858782, 'correct_ids... \n", + "8 0.053289 0.311596 {'accuracy': 0.0, 'correct_ids': [], 'meteor':... \n", + "9 0.126232 0.368065 {'accuracy': 0.01235657546337158, 'correct_ids... \n", + "10 0.125597 0.383072 {'accuracy': 0.00088261253309797, 'correct_ids... \n", + "11 0.170018 0.424815 {'accuracy': 0.00794351279788173, 'correct_ids... " + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "metrics_df" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9b63fdc6-40ab-4a04-bf57-19f66cd4509f", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "unsloth/Qwen2-0.5B-Instruct: {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.26453254295068257, 'bleu_scores': {'bleu': 0.04698039499333136, 'precisions': [0.309245348800355, 0.07347623117890723, 0.023966406063295892, 0.008945686900958467], 'brevity_penalty': 1.0, 'length_ratio': 1.0450811526995694, 'translation_length': 31551, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3105157084661859, 'rouge2': 0.08870671944085522, 'rougeL': 0.25592585227840225, 'rougeLsum': 0.2559676791644768}}\n", + "unsloth/Qwen2-0.5B-Instruct(finetuned): {'accuracy': 0.00176522506619594, 'correct_ids': [147, 533], 'meteor': 0.28664792904792147, 'bleu_scores': {'bleu': 0.0633353272697663, 'precisions': [0.33089419978517726, 0.08866324714749294, 0.0345015434901035, 0.016769504485747815], 'brevity_penalty': 0.9867295481943301, 'length_ratio': 0.9868168267638291, 'translation_length': 29792, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3173288345459394, 'rouge2': 0.10618140518074348, 'rougeL': 0.26121163488973825, 'rougeLsum': 0.26105762971788327}}\n", + "unsloth/Qwen2-1.5B-Instruct: {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.3108076173265163, 'bleu_scores': {'bleu': 0.07171893503391259, 'precisions': [0.39917895129688374, 0.11838516093835243, 0.0465865090686774, 0.01994873985476292], 'brevity_penalty': 0.8809955157186649, 'length_ratio': 0.8875455448824114, 'translation_length': 26795, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3666102879704475, 'rouge2': 0.1266675803266789, 'rougeL': 0.31056277362475493, 'rougeLsum': 0.31064508228763266}}\n", + "unsloth/Qwen2-1.5B-Instruct(finetuned): {'accuracy': 0.00441306266548985, 'correct_ids': [147, 170, 309, 526, 533], 'meteor': 0.3412870633079792, 'bleu_scores': {'bleu': 0.09439554395746873, 'precisions': [0.3842746400885936, 0.1271192353310542, 0.05777680938373824, 0.029647860658841348], 'brevity_penalty': 0.9869644625887559, 'length_ratio': 0.9870486916197416, 'translation_length': 29799, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.37271457574577416, 'rouge2': 0.1461825645475855, 'rougeL': 0.31782980248661996, 'rougeLsum': 0.3176719516563038}}\n", + "unsloth/Qwen2-7B-Instruct: {'accuracy': 0.00088261253309797, 'correct_ids': [77], 'meteor': 0.3706126714366309, 'bleu_scores': {'bleu': 0.10881761334734624, 'precisions': [0.41661179878851723, 0.14690695209109872, 0.06769832799715404, 0.033841135698135585], 'brevity_penalty': 1.0, 'length_ratio': 1.006160980457105, 'translation_length': 30376, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.41631654006456126, 'rouge2': 0.16673054869332232, 'rougeL': 0.35965715874226606, 'rougeLsum': 0.3597263235968114}}\n", + "unsloth/Qwen2-7B-Instruct(finetuned): {'accuracy': 0.00617828773168579, 'correct_ids': [147, 199, 309, 526, 531, 658, 935], 'meteor': 0.4016299621577931, 'bleu_scores': {'bleu': 0.1340342690832733, 'precisions': [0.44661102067675684, 0.1784426820475847, 0.0918075911311537, 0.05294140732310349], 'brevity_penalty': 0.9554111104454712, 'length_ratio': 0.9563762835375952, 'translation_length': 28873, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.43259257689704633, 'rouge2': 0.1981555489001432, 'rougeL': 0.3801845230388961, 'rougeLsum': 0.37954828954862707}}\n", + "unsloth/mistral-7b-instruct-v0.3: {'accuracy': 0.00088261253309797, 'correct_ids': [77], 'meteor': 0.3221588341261717, 'bleu_scores': {'bleu': 0.08500847701637192, 'precisions': [0.3839848248763634, 0.12152594314699355, 0.05129145876137364, 0.023885167464114832], 'brevity_penalty': 0.9776268817156146, 'length_ratio': 0.9778734680357735, 'translation_length': 29522, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.36677374147027, 'rouge2': 0.132281273499625, 'rougeL': 0.31779356328547126, 'rougeLsum': 0.3178345285457188}}\n", + "unsloth/mistral-7b-instruct-v0.3(finetuned): {'accuracy': 0.00529567519858782, 'correct_ids': [77, 147, 199, 240, 309, 364], 'meteor': 0.38725013933546865, 'bleu_scores': {'bleu': 0.1255052835038013, 'precisions': [0.4230690780527604, 0.16504614079128804, 0.083050972304066, 0.04753304641869044], 'brevity_penalty': 0.9740309389176179, 'length_ratio': 0.9743623716462405, 'translation_length': 29416, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4141364853072854, 'rouge2': 0.18555040788669064, 'rougeL': 0.3582465278807073, 'rougeLsum': 0.35834634945673904}}\n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k: {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.31732985329639374, 'bleu_scores': {'bleu': 0.05328901091080452, 'precisions': [0.23537940003837707, 0.07314835044789163, 0.03125210027555615, 0.014986438652139935], 'brevity_penalty': 1.0, 'length_ratio': 1.5535939052666445, 'translation_length': 46903, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.36349616085155206, 'rouge2': 0.12989126572749554, 'rougeL': 0.3115960462482712, 'rougeLsum': 0.3117420637997418}}\n", + "gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned): {'accuracy': 0.01235657546337158, 'correct_ids': [41, 77, 133, 147, 199, 301, 348, 364, 413, 526, 533, 778, 893, 1011], 'meteor': 0.3921204969248105, 'bleu_scores': {'bleu': 0.12623244689421761, 'precisions': [0.43614474551573396, 0.16861682918020945, 0.08602838748541095, 0.048517732169536844], 'brevity_penalty': 0.9536797061184072, 'length_ratio': 0.9547201059953627, 'translation_length': 28823, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.42640620693017617, 'rouge2': 0.19258710119333206, 'rougeL': 0.36806549573170777, 'rougeLsum': 0.3679460597175971}}\n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit: {'accuracy': 0.00088261253309797, 'correct_ids': [533], 'meteor': 0.39327797374995854, 'bleu_scores': {'bleu': 0.12559679608930588, 'precisions': [0.43200657894736844, 0.1652031298048997, 0.08068242402701262, 0.04321421958896501], 'brevity_penalty': 1.0, 'length_ratio': 1.0069559456773767, 'translation_length': 30400, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.44060491048814143, 'rouge2': 0.18992148221555435, 'rougeL': 0.38307187208859017, 'rougeLsum': 0.38321351591078995}}\n", + "unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned): {'accuracy': 0.00794351279788173, 'correct_ids': [147, 199, 240, 272, 364, 419, 658, 659, 820], 'meteor': 0.45716060031914707, 'bleu_scores': {'bleu': 0.17001816364266625, 'precisions': [0.48491918577128557, 0.2166942090643789, 0.12017135952546593, 0.07191963944694829], 'brevity_penalty': 0.9793863208805533, 'length_ratio': 0.9795958926796953, 'translation_length': 29574, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4843039193858598, 'rouge2': 0.2423067746227518, 'rougeL': 0.424815398022727, 'rougeLsum': 0.4247048351560071}}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_utils.py:302: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n", + " barplot.set_xticklabels([\"METEOR\", \"BLEU-1\", \"ROUGE-L\"])\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABboAAALHCAYAAABIevSFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdeVhU1eMG8HdmANkEFwQEURQV3CkQNPfEL/azTK0k91Bb3BXT1EorF9xzIy3NJbUkE802rHDfDSWXEFcEWUUEBJVlZn5/jDMyzgyzMDADvJ/n4XmGe8859wyRzHnvuecIpFKpFEREREREREREREREVZTQ1B0gIiIiIiIiIiIiIioPBt1EREREREREREREVKUx6CYiIiIiIiIiIiKiKo1BNxERERERERERERFVaQy6iYiIiIiIiIiIiKhKY9BNRERERERERERERFUag24iIiIiIiIiIiIiqtIYdBMRERERERERERFRlcagm4iIiIiIiIiIiIiqNAbdRERERERERERERFSlmUXQHRERAU9PT1hbWyMwMBBnz57VWHbr1q0QCARKX9bW1pXYWyIiIiIiIiIiIiIyJyYPuiMjIxEWFoZ58+bh/Pnz6NChA4KDg5GZmamxjoODA9LS0hRfd+7cqcQeExEREREREREREZE5MXnQvXLlSrz77rsIDQ1F69atsWHDBtja2mLz5s0a6wgEAri6uiq+XFxcKrHHRERERERERERERGROLEx58aKiIsTGxmL27NmKY0KhEEFBQTh16pTGevn5+WjSpAkkEglefPFFLFq0CG3atFFbtrCwEIWFhYrvJRIJsrOzUb9+fQgEAuO9GSIiIiIiIiIiIiIyGqlUiocPH8LNzQ1CYdlztk0adGdlZUEsFqvMyHZxccHVq1fV1vH29sbmzZvRvn175ObmYvny5XjppZdw5coVNGrUSKV8eHg4Pv/88wrpPxERERERERERERFVrOTkZLXZb2kmDboN0blzZ3Tu3Fnx/UsvvYRWrVrh66+/xvz581XKz549G2FhYYrvc3Nz0bhxYyQnJ8PBwaFS+kxERERERERERERE+snLy4OHhwdq166ttaxJg24nJyeIRCJkZGQoHc/IyICrq6tObVhaWuKFF17AjRs31J6vVasWatWqpXLcwcGBQTcRERERERERERGRmdNlCWqTbkZpZWUFPz8/xMTEKI5JJBLExMQozdoui1gsxqVLl9CwYcOK6iYRERERERERERERmTGTL10SFhaGUaNGwd/fHwEBAVi1ahUKCgoQGhoKABg5ciTc3d0RHh4OAPjiiy/QqVMnNG/eHDk5OVi2bBnu3LmDsWPHmvJtEBEREREREREREZGJmDzoDgkJwb179zB37lykp6fD19cX0dHRig0qk5KSlHbUfPDgAd59912kp6ejbt268PPzw8mTJ9G6dWtTvQUiIiIiIiIiIiIiMiGBVCqVmroTlSkvLw+Ojo7Izc3lGt1EREREREREREREZkqfLNeka3QTEREREREREREREZUXg24iIiIiIiIiIiIiqtIYdBMRERERERERERFRlcagm4iIiIiIiIiIiIiqNAbdRERERERERERERFSlMegmIiIiIiIiIiIioiqNQTcRERERERERERERVWkMuomIiABERETA09MT1tbWCAwMxNmzZ3Wqt2vXLggEAgwYMEDlXHx8PPr37w9HR0fY2dmhY8eOSEpKMnLPiYiIiIiIiIhBNxER1XiRkZEICwvDvHnzcP78eXTo0AHBwcHIzMwss15iYiI+/PBDdOvWTeXczZs30bVrV/j4+ODw4cO4ePEiPv30U1hbW1fU2yAiIiIiIiKqsQRSqVRq6k5Upry8PDg6OiI3NxcODg6m7g4REZmBwMBAdOzYEevWrQMASCQSeHh4YNKkSZg1a5baOmKxGN27d8fo0aNx7Ngx5OTkYN++fYrzb7/9NiwtLbF9+/bKeAtERERERERE1Y4+WS5ndBMRUY1WVFSE2NhYBAUFKY4JhUIEBQXh1KlTGut98cUXcHZ2xpgxY1TOSSQS/Pbbb2jZsiWCg4Ph7OyMwMBApSCciIiIiIiIiIyHQTcREdVoWVlZEIvFcHFxUTru4uKC9PR0tXWOHz+Ob7/9Fhs3blR7PjMzE/n5+Vi8eDH69u2LP//8EwMHDsSgQYNw5MgRo78HIiIiIiIioprOwtQdICIiqkoePnyIESNGYOPGjXByclJbRiKRAABef/11TJs2DQDg6+uLkydPYsOGDejRo0el9ZeIiIiIiIioJmDQTURENZqTkxNEIhEyMjKUjmdkZMDV1VWl/M2bN5GYmIjXXntNcUwebFtYWCAhIQEeHh6wsLBA69atleq2atUKx48fr4B3QURERERERFSzcekSIiKq0aysrODn54eYmBjFMYlEgpiYGHTu3FmlvI+PDy5duoS4uDjFV//+/dGrVy/ExcXBw8MDVlZW6NixIxISEpTqXrt2DU2aNKnw90RERERERERU03BGNxER1XhhYWEYNWoU/P39ERAQgFWrVqGgoAChoaEAgJEjR8Ld3R3h4eGwtrZG27ZtlerXqVMHAJSOz5gxAyEhIejevTt69eqF6Oho/PLLLzh8+HBlvS0iIiIiIiKiGoNBNxER1XghISG4d+8e5s6di/T0dPj6+iI6OlqxQWVSUhKEQv0egho4cCA2bNiA8PBwTJ48Gd7e3tizZw+6du1aEW+BiIiIiIiIqEYTSKVSqak7UZny8vLg6OiI3NxcODg4mLo7RERERERERERERKSGPlku1+gmIiIiIiIiIiIioiqNQTcRERERERERERERVWkMuomIiIiIiIiIiIioSmPQTURERERERERERERVGoNuIiIiIiIiIiIiIqrSGHQTERERERERERERUZXGoJuIiIiIiIiIiIiIqjQG3URERERERERERERUpVmYugNERESmMrPHaEzpMlzvevmFjzA8chYSshLx/dtL8YKbj95tuC96We86RERERERERKQeZ3QTEVGNtezoFqw+sUPveva1bLEjZDG8nTwxdNdMXEi9WgG9IyIiIiIiIiJdMegmIqIaa0b3UIbdRERERERERNUAg24iIqqxpnQZzrCbiIiIiIiIqBpg0E1ERDUaw24iIiIiIiKiqo9BNxER1XgMu4mIiIiIiIiqNgbdREREYNhNRERERERElSsiIgKenp6wtrZGYGAgzp49q1O9Xbt2QSAQYMCAAYpjxcXF+Oijj9CuXTvY2dnBzc0NI0eORGpqagX13vww6CYiInqKYTcRERERERFVhsjISISFhWHevHk4f/48OnTogODgYGRmZpZZLzExER9++CG6deumdPzRo0c4f/48Pv30U5w/fx5RUVFISEhA//79K/JtmBWBVCqVmroTlSkvLw+Ojo7Izc2Fg4ODqbtDREQmlDLnoNrjq0/swLKjWzCjeyimdBmud7v5hY8wPHIWErIS8f3bS/GCm49KGfdFL+vdLhEREREREVUPgYGB6NixI9atWwcAkEgk8PDwwKRJkzBr1iy1dcRiMbp3747Ro0fj2LFjyMnJwb59+zRe49y5cwgICMCdO3fQuHHjingbFU6fLJczuomIiJ7Dmd1ERERERERUUYqKihAbG4ugoCDFMaFQiKCgIJw6dUpjvS+++ALOzs4YM2aMTtfJzc2FQCBAnTp1ytvlKoFBNxERkRoMu4mIiIiIiKgiZGVlQSwWw8XFRem4i4sL0tPT1dY5fvw4vv32W2zcuFGnazx58gQfffQRhgwZUmNWtWDQTUREpAHDbiIiIiIiIjK1hw8fYsSIEdi4cSOcnJy0li8uLsbgwYMhlUqxfv36SuihebAwdQeIiIjMmXyN7mVHtyh9ryt52D08chaG7pqpcc1uIiIiIiIiqhmcnJwgEomQkZGhdDwjIwOurq4q5W/evInExES89tprimMSiQQAYGFhgYSEBHh5eQF4FnLfuXMHBw8erDGzuQHO6CYiItKKM7uJiIiIiIjIWKysrODn54eYmBjFMYlEgpiYGHTu3FmlvI+PDy5duoS4uDjFV//+/dGrVy/ExcXBw8MDwLOQ+/r16/j7779Rv379SntP5oAzuomIiHRg7JndeVvHG72PREREREREVDWEhYVh1KhR8Pf3R0BAAFatWoWCggKEhoYCAEaOHAl3d3eEh4fD2toabdu2Vaov32BSfry4uBhvvvkmzp8/j19//RVisVix3ne9evVgZWVVeW/ORBh0ExER6ciYYTcRERERERHVXCEhIbh37x7mzp2L9PR0+Pr6Ijo6WrFBZVJSEoRC3RfjSElJwf79+wEAvr6+SucOHTqEnj17GqvrZksglUqlpu5EZcrLy4OjoyNyc3Nr1Bo1RESkKmH6r7CvZat3vdUndmDZ0S2Y0T1U77AbAPILH8F7xat61yMiIiIiIiKqSfTJcrlGNxER1VjDI2chv/CR3vWMsWY3ERERERERERkPg24iIqqxErISTRZ2ExEREREREZHxMOgmIqIa6/u3lzLsJiIiIiIiIqoGGHQTEVGN9YKbD8NuIiIiIiIiomqAQTcREdVoDLuJiIiIiIiIqj4G3UREVOMx7CYiIiIiIiKq2hh0E5FWERER8PT0hLW1NQIDA3H27Fmd6u3atQsCgQADBgxQOi6VSjF37lw0bNgQNjY2CAoKwvXr1yug50S6Y9hNREREREREVHUx6CaiMkVGRiIsLAzz5s3D+fPn0aFDBwQHByMzM7PMeomJifjwww/RrVs3lXNLly7FmjVrsGHDBpw5cwZ2dnYIDg7GkydPKuptEOmEYTcRERERERFR1SSQSqVSU3eiMuXl5cHR0RG5ublwcHAwdXeIzF5gYCA6duyIdevWAQAkEgk8PDwwadIkzJo1S20dsViM7t27Y/To0Th27BhycnKwb98+ALLZ3G5ubpg+fTo+/PBDAEBubi5cXFywdetWvP3225XyvogAIGXOQbXHL6RexdBdM+Ht5IkdIYthX8tW77ZXn9iBZUe3YEb3UEzpMlzlvPuil/Vuk4iIiIiIiKo2TePQ0vILH2F45CwkZCXi+7eX4gU3H72vo25cWxXHofpkuZzRTUQaFRUVITY2FkFBQYpjQqEQQUFBOHXqlMZ6X3zxBZydnTFmzBiVc7dv30Z6erpSm46OjggMDCyzTaLKxJndREREREREZCr2tWyxI2QxvJ08MXTXTFxIvap3G8YY11Y1DLqJSKOsrCyIxWK4uLgoHXdxcUF6erraOsePH8e3336LjRs3qj0vr6dPm0SmwLCbiIiIiIiITKUiwu7qjkE3ERnNw4cPMWLECGzcuBFOTk6m7g5RuTHsJiIiIiIiIlMxdthd3THoJiKNnJycIBKJkJGRoXQ8IyMDrq6uKuVv3ryJxMREvPbaa7CwsICFhQW+++477N+/HxYWFrh586ainq5tEpkaw24iIiIiIiIyFWOG3dUdg24i0sjKygp+fn6IiYlRHJNIJIiJiUHnzp1Vyvv4+ODSpUuIi4tTfPXv3x+9evVCXFwcPDw80LRpU7i6uiq1mZeXhzNnzqhtk2qWiIgIeHp6wtraGoGBgTh79qzGslFRUfD390edOnVgZ2cHX19fbN++XalMRkYG3nnnHbi5ucHW1hZ9+/bF9evX9e4Xw24iIiIiIiIyBkPGg8YKu6s7Bt1EVKawsDBs3LgR27ZtQ3x8PMaNG4eCggKEhoYCAEaOHInZs2cDAKytrdG2bVulrzp16qB27dpo27YtrKysIBAIMHXqVCxYsAD79+/HpUuXMHLkSLi5uWHAgAEmfKckZ6qwOTIyEmFhYZg3bx7Onz+PDh06IDg4GJmZmWqvXa9ePXz88cc4deoULl68iNDQUISGhuLAgQMAAKlUigEDBuDWrVv4+eefceHCBTRp0gRBQUEoKCjQ++di7LCbiIiIiIiIah5DJz8ZI+yu7hh0E1GZQkJCsHz5csydOxe+vr6Ii4tDdHS0YjPJpKQkpKWl6dXmzJkzMWnSJLz33nvo2LEj8vPzER0dDWtr64p4C6QHU4bNK1euxLvvvovQ0FC0bt0aGzZsgK2tLTZv3qz22j179sTAgQPRqlUreHl5YcqUKWjfvj2OHz8OALh+/TpOnz6N9evXo2PHjvD29sb69evx+PFj/PDDDwb9fIwZdhMREREREVHNU54nfRl2l00glUqlpu5EZcrLy4OjoyNyc3Ph4OBg6u4QEZmVwMBAdOzYEevWrQMgW6rGw8MDkyZNwqxZuu3Q/OKLL6Jfv36YP38+rl27Bm9vb1y+fBlt2rRRtOnq6opFixZh7NixAICioiLY2trip59+UprZP2rUKOTk5ODnn38u85pSqRQHDx5E//79sW/fPvTp0weXLl1C+/btcePGDXh5eSnKenh4oHfv3ti6dSt+fecrgx7fupB6FUN3zYS3kyd2hCyGfS1bvdtwX/Sy3nWIiIiIiIioakuZcxCrT+zAsqNbMKN7KKZ0Ga53G/mFjzA8chYSshLx/dtLdR7XVsVxqD5ZLmd0ExFpYK7rRVeUoqIixMbGIigoSHFMKBQiKCgIp06d0lpfKpUiJiYGCQkJ6N69OwCgsLAQAJRm6wuFQtSqVUsx8xoAsrKyIBaLFU8KyLm4uCA9PV3jNXNzc2Fvbw8rKyv069cPa9euRZ8+fQDI1oxv3LgxZs+ejQcPHqCoqAhLlizB3bt3FU8hGGPXakNndhMREREREVHNVN49nDizWz0G3UREapj7etEVwVRhc3nUrl0bcXFxOHfuHBYuXIiwsDAcPnwYAGBpaYmoqChcu3YN9erVg62tLQ4dOoRXXnkFQqHsz58xdq1m2E1ERERERET6YthtfAy6iYjUqArrRZuL8obNAODk5ASRSISMjAyltjMyMuDq6qrx2kKhEM2bN4evry+mT5+ON998E+Hh4Yrzfn5+iIuLQ05ODtLS0hAdHY379++jWbNmAGCUXasZdhMREREREZEhGHYbF4NuIqLnmHIJD1MyVdgMAFZWVvDz80NMTIzimEQiQUxMDDp37qzze5BIJIqfdWmOjo5o0KABrl+/jn/++Qevv/46AON8KGDYTURERERERIZi2G08DLqJiJ5TFZfwMAZThc1yYWFh2LhxI7Zt24b4+HiMGzcOBQUFCA0NBQCMHDkSs2fPVpQPDw/HX3/9hVu3biE+Ph4rVqzA9u3bMXz4s408du/ejcOHDyuWjOnTpw8GDBiA//3vf4oyDLuJiIiIiIhqLmPvz5Wfn4+JEyeiUaNGsLGxUTwlXhaG3cZhYeoOEBFVF/IlPPLz8xETE4OwsDA0a9YMPXv2VCzhMWbMGNSrVw8ikQhBQUF45ZVXIJVKTd11hbCwMIwaNQr+/v4ICAjAqlWrVMJmd3d3xYzt8PBw+Pv7w8vLC4WFhfj999+xfft2rF+/XtHm7t270aBBAzRu3BiXLl3ClClTVMJmAAgJCcG9e/cwd+5cpKenw9fXF9HR0YobDklJSUrLnRQUFGD8+PG4e/cubGxs4OPjgx07diAkJERRJi0tDWFhYcjIyEDDhg0xcuRIfPrppyrvW/6hYHjkLAzdNVOvXavl5GH30F0zMTxyFnaELIZ9LVu92iAiIiIiIqLKI9+fa8OGDQgMDMSqVasQHByMhIQEODs7q5SX78/l4+MDKysr/PrrrwgNDYWzszOCg4MByMbVBw8exI4dO+Dp6Yk///wT48ePh5ubG/r376+xL1O6yCZtLTu6Rel7XRljXFvVmcWMbn3unJS2a9cuCAQCDBgwoGI7SEQ1iimX8DC1kJAQLF++HHPnzoWvry/i4uJUwubSM9DlYXObNm3QpUsX7NmzBzt27MDYsWMVZdLS0jBixAj4+Phg8uTJGDFihMZ1ySdOnIg7d+6gsLAQZ86cQWBgoOLc4cOHsXXrVsX3CxYswPXr1/H48WNkZ2fj5MmTSiE3AEyePBnJyckoKirCnTt3MH/+fFhZWam9Nmd2ExERERGZD3OYZUvVn7H35wKAkydPYtSoUejZsyc8PT3x3nvvoUOHDjrlnZzZXT4mD7rld07mzZuH8+fPo0OHDggODkZmZmaZ9RITE/Hhhx+iW7duldRTIqopTL2Eh6mZMmw2NYbdRERERESmp29WJJ9le+rUKVy8eBGhoaEIDQ3FgQMHFGXCwsIQHR2NHTt2ID4+HlOnTsXEiROxf//+ynpbZGYqYn8uAHjppZewf/9+pKSkQCqV4tChQ7h27ZrKU82aMOw2nMmDbn3vnACAWCzGsGHD8Pnnn5vVTEgiqj5MtV40mR7DbiIiIiIi0zK3WbZUPVXE/lwAsHbtWrRu3RqNGjWClZUV+vbti4iICKUwXBuG3YYx6Rrd8jsnpcMiXe6cfPHFF3B2dsaYMWNw7NixMq9RWFioNKMyLy+v/B0nqgGCtn2mU7mSx4WIW7ET+SmZeGHGCDg2c9f7Wrf3H8WtqENoNqgXmvbvjr9H6XbtimTK9aLJ9LhmNxERERGRaRiaFclJpVIcPHgQCQkJWLJkieK4fJbt6NGj4ebmhsOHD+PatWv48ssvK+R9UPVV1v5cgCzoPn36NPbv348mTZrg6NGjmDBhAtzc3JRmj2tTEWt2u+NlvdqoakwadJd15+TqVfV3Go4fP45vv/0WcXFxOl0jPDwcn3/+eXm7SkRqGCPkBoCm/WV3NW9FHZIdGGWsHpbPxIkTMXHiRLXnDh8+rPT9ggULsGDBgjLbmzx5MiZPnmys7lEFY9hNRERERFT5DMmKANksW3d3dxQWFkIkEuGrr75SmWX73nvvoVGjRrCwsIBQKMTGjRv1mmVL1Ut59+cCAF9fX8THxyM8PBw9e/bE48ePMWfOHOzduxf9+vUDALRv3x5xcXFYvny5XkE3YPywO2/reL3qVzUmX7pEHw8fPsSIESOwceNGODk56VRn9uzZyM3NVXwlJydXcC+JagZjhdxyTft3R7NBvZ6F3URmgMuYEBERERFVDfJZtufOncPChQsRFhamNEGp9Czb2NhYrFixAhMmTMDff/9tuk6TSVXE/lzFxcUoLi5WegIcAEQiESQSiUH9NOYyJtWdSWd063vn5ObNm0hMTMRrr72mOCb/JbGwsEBCQgK8vLyU6tSqVQu1atWqgN4T1VzGDrnl5DO7icxJRc3sJiIiIiIiVVVhli1VH2FhYRg1ahT8/f0REBCAVatWqezP5e7ujvDwcACylSP8/f3h5eWFwsJC/P7779i+fTvWr18PAHBwcECPHj0wY8YM2NjYoEmTJjhy5Ai+++47rFy5EgCQX/hI7yd9jTWzu7oz6Yxufe+c+Pj44NKlS4iLi1N89e/fH7169UJcXBw8PDwqs/tElSoiIgKenp6wtrZGYGBgmRtmREVFwd/fH3Xq1IGdnR18fX2xfft2pTICgUDt17JlywAAubdS1LZdUSG3HMNuMkcVMbObiIiIiIhUVZVZtlQ9hISEYPny5Zg7dy58fX0RFxensj9XWlqaorx8f642bdqgS5cu2LNnD3bs2IGxY8cqyuzatQsdO3bEsGHD0Lp1ayxevBgLFy7EBx98AAAGP+lrjJnd1Z1AKpVKTdmByMhIjBo1Cl9//bXizsmPP/6Iq1evwsXFReXOyfPeeecd5OTkYN++fTpdLy8vD46OjsjNzYWDg4MR3wlRxYmMjMTIkSOxYcMGBAYGYtWqVdi9ezcSEhLg7OysUv7w4cN48OABfHx8YGVlhV9//RXTp0/Hb7/9huDgYABQ2UH4jz/+wJgxY3Djxg00a9YMFrbWKkF2RYfccuawGSXVDClzDupVPr/wEYZHzkJCVqJBM7sB4ELqVdnaaE/y9a5LRERERFQT6JsVqZtlO2vWLKxfv14RQPbs2RNZWVlYt26dYpbtuHHjsHLlSowbN86Ub5dqGAdre3g7eRq8h9PqEzuw7OgWzOgeqvfMbvdFVW8zSn2yXJMuXQLI7pzcu3cPc+fORXp6Onx9fVXunDx/x42oplm5ciXeffddxaMzGzZswG+//YbNmzdj1izVmaHynX7lpkyZgm3btuH48eOKoPv5R75+/vln9OrVC82aNQMA2Ls748Ky7YpAu7JCbqp5Xmzkj+tZ17AlZCfau/nqXf9iahxCI4ehhVNLbBq8Hfa17HWuayfQ71rGXMaEiIiIiIjU0zcrks+yvXv3LmxsbODj44MdO3YgJCREUWbXrl2YPXs2hg0bhuzsbDRp0kRpli2ZVkREBJYtW4b09HR06NABa9euRUBAgNqyUVFRWLRoEW7cuIHi4mK0aNEC06dPx4gRIxRlBAL1g72lS5dixowZFfIedPX8spaVvYxJdWbyGd2VjTO6qaopKiqCra0tfvrpJwwYMEBxfNSoUcjJycHPP/9cZn2pVIqDBw+if//+2Ldvn9Ku03IZGRlo1KgRtm3bhqFDhwIAem6YrQi2208ajFtRhyst5OaM7sonX7rmvWFLEdTt2R/Ju2nX8PnKt5CTlwmBQABNfzLk5+o4OOOz6T/B3bWF4txfR7fjm50zNV7bvlZtk4TcALDp5DyDPhQYY2Z3VbyTTkREREREZGymeIrdlFLmHFQ86VvZM7ur4jhUnyyXU6WJzFxWVhbEYrHizrWci4uLyj/cpeXm5sLe3h5WVlbo168f1q5dqzbkBoBt27ahdu3aGDRokOKYhU0t+E4fBhunOriwdDtybyRzJnc1JhAIMG7kl0ohNwA0atgSX3y4F3UcnCEUiDTWFwpEqOPgjC8+3KsUcgNAn+4jMH7kl4ow/XmmCrkBGGXXakPX7CYiIiIiIiLlp9hbt26NDRs2wNbWFps3b1ZbvmfPnhg4cCBatWoFLy8vTJkyBe3bt8fx48cVZVxdXZW+nn+K3dSe38PJFGt2V0cMuomqqdq1ayMuLg7nzp3DwoULERYWhsOHD6stu3nzZgwbNgzW1taV20kyG5NC16LXSyFqzzV0aYYFM/fD0aEBhELVsFsoFMHRoQEWfPQLGrqo/9DQq8vbmBS6Vu05U4XcX51YU+6NPBh2ExERERERGa6oqAixsbEICgpSHBMKhQgKCsKpU6e01pdKpYiJiUFCQgK6d++utkxGRgZ+++03jBkzxmj9NgaG3cbHoJvIzDk5OUEkEiEjI0PpeEZGhso626UJhUI0b94cvr6+mD59Ot588021m7oeO3YMCQkJSjsEA882nnyclYMXZo6AY3MPXFi2Hbm3UozzxsisdAt8o8zzLg2a4JVeo9Wek0ql+L+Xx8DFqbHWazR2b2VwH+WMFXKvPrbcKLtWM+wmIiIiIiIyjKmeYjcXDLuNy+SbURJR2aysrODn54eYmBjFGt0SiQQxMTGYOHGizu1IJBIUFhaqHP/222/h5+eHDh06KB1/fuNJh6buiFuxU2mDyurolS17AQCnFs5C7s1r8OgVjDYj3ldbVlJcjL8njoCkuAh+Uz9Bg/YvKs6Ji4vw1/tvAwA6zvgc9Vu1U9tGfupdHP9kMgBoXP+6MkgkEq0b/95NuwYBZEuPyGd2SyRiCAVCJKdd0+kaqek3Fd8LhSJIJGK9+mnUkLvbhwDKv5GHMTaoJCIiIiIiIt3Jn2LPz89HTEwMwsLC0KxZM/Ts2VOlrLk/xS4Pu7lBZflxRjdRFRAWFoaNGzdi27ZtiI+Px7hx41BQUIDQ0FAAwMiRIzF79mxF+fDwcPz111+4desW4uPjsWLFCmzfvh3Dhyv/Q5eXl4fdu3erzOYGoLLxpHzNbnt35xoxs1sebicfOoA7Mb+rnC/MzcHxT6dAUlwEAHBq66t0XmRphWb9ZLOkzy2bh/zUuypt3P/voiLk9ugVbMzu6+3e/WStZRLvXoFYUgIAcHPxgpuLFwBALClBYvIVrfUz7yehuKRQsU53h9Y9YF3LTuc+GjvkHt9lsuI4Z3YTERERERFVPlM9xW5uOLPbOBh0E1UBISEhWL58OebOnQtfX1/ExcUhOjpa8WhPUlIS0tLSFOULCgowfvx4tGnTBl26dMGePXuwY8cOlX/Yd+3aBalUiiFDhqhcU92s7ZoUdjs0boq2oRMAAPE7NyF69CAk/LQDtw/sx9XIrTg0bTQeZcoeo+q++CsI1MyGbj7gbdRt2RoAcPyTyTj28STc+i0Ktw/sx9ll83Bu+WcAgLotW6PVUNP+0U1OTSjzvFgiRmrGLTSo3wiTx0RgxdxDWDH3ECaPiUCD+o2QlnETYi2zs+XX8G3TC8s++RtzJu3E+vB/dOpfRYbccgy7iYiIiIiURUREwNPTE9bW1ggMDMTZs2c1lo2KioK/vz/q1KkDOzs7+Pr6Yvv27Srl4uPj0b9/fzg6OsLOzg4dO3ZEUlJSRb4NMmOln2KXkz/F3rlzZ53b0fcpdnPEsLv8BFJTPitvAnl5eXB0dERubi4cHBxM3R0isxW07TON5+Trdz8/69sYbu8/ipt7DhqtPX3Jly6Ry4z7B+fXLFJb1rFpC/iO/xA29RtobE8iFiP++01IPnRA7flm/d5A8wFvQygS4Y/QgYZ3vJw+mfIDfNv00ng+Ny8LZ+N+R68uQ2AhslQ6VyIuxqETPyDA9//g6OCksY24K4dgXcsOPs0DlI63v11cZt8qMuS2E1xXKbv6xA4sO7oFM7qHGvS4V37hIwyPnIWErMQylzFxX/Sy3m0TEREREVWWyMhIjBw5Ehs2bEBgYCBWrVqF3bt3IyEhAc7OzirlDx8+jAcPHsDHxwdWVlb49ddfMX36dPz2228IDpY9wXrz5k0EBARgzJgxGDJkCBwcHHDlyhV06tRJbZs1UUREBJYtW4b09HR06NABa9euRUBAgNqyUVFRWLRoEW7cuIHi4mK0aNEC06dPx4gRI5TKxcfH46OPPsKRI0dQUlKC1q1bY8+ePWjcuOx9lipLZGQkRo0aha+//hoBAQFYtWoVfvzxR1y9ehUuLi4YOXIk3N3dFTO2w8PD4e/vDy8vLxQWFuL333/HrFmzsH79eqUJfnl5eWjYsCFWrFiBDz74QOma12Zrf6pZLr8wH2N/HIHrWdewJWQn2rv56v0eS49rfxm5tMyyF1KvYuiumfB28jRoGRNA87i2Ko5D9clyGXQTkVplBd1AxYTdt/cfxa2oQyZdq/r5oBuQrZ2dczMB2VcvQ1JUBMvajnD17wTruvV1bldcXISM2DN4lJEKALB1cYOLXyBEllaKMqYMun/6RvMmHxWtrKC7omdyqwu6gcoJu6viBwwiIiIiqjkCAwPRsWNHrFu3DoBsxqyHhwcmTZqEWbNm6dTGiy++iH79+mH+/PkAgLfffhuWlpZqZ3pTzb65sG7dOkXA7+vrizVr1iAwMBAA0LNnT3h6emLr1q0AgE8++QSRkZG4e/cubGxs4OPjgylTpiAkJESpzW+++QZTp05FWloaHB0dlc7pGnQbO+TeNHg7XKzTtNapqLC7Ko5DGXSXgUE3kW60Bd2AccNuecjdbFAvs5rRXZkYdCurjOVKNAXdQMWH3VXxAwYRERER1QxFRUWwtbXFTz/9hAEDBiiOjxo1Cjk5Ofj555/LrC+VSnHw4EH0798f+/btQ58+fSCRSODo6IiZM2fi+PHjuHDhApo2bYrZs2crruH1xsto2r+7Xn0tPZbUt25pf4/6zOC6xsKbC5VHl6C7IkJu+1r2ZY5DS6uIsLsqjkP1yXK5RjcRGcxYa3Yb64MJVR+VEXJrwzW7iYiIiKimysrKglgsVuwLJefi4oL0dM2TZHJzc2Fvbw8rKyv069cPa9euRZ8+fQAAmZmZyM/Px+LFi9G3b1/8+eefGDhwIAYNGoQjR44AgEHjwab9u6PZoF64FXUIt/cf1bs+ALPYf6qoqAixsbEICgpSHBMKhQgKCsKpU6e01pdKpYiJiUFCQgK6d5f9HCUSCX777Te0bNkSwcHBcHZ2RmBgIPbt21dRb6PaqKiQWx9cs1t/DLqJqFzKG3Yz5KbnmUPILcewm4iIiIhId7Vr10ZcXBzOnTuHhQsXIiwsDIcPHwYgC10B4PXXX8e0adPg6+uLWbNm4dVXX8WGDRvKdd3yhN25t1JwYZnpZzub6uYCqTKHkFvO2GF3dcegm4jKzdCw2xxD7ujRg3BqwUe4dzEW0qcfxLQpflSAm7/+hOjRgxA9ehAOvDcYlzZHID9N99Bfn7IVLb8gB2+93xDHzkYZ3Maxs1F46/2GyC/I0aueOYXccgy7iYiIiKimcXJygkgkQkZGhtLxjIwMuLq6aqwnFArRvHlz+Pr6Yvr06XjzzTcVGwg6OTnBwsICrVu3VqrTqlUrJCUllbvPhoTd8pDb3t181qrWl6luLlRX5hRyyxkz7K7uGHQTkVHoG3abY8gtl3vrOmJXLcSBsW+iIKPsTSJSTx9FzMQRuB71veKYtKQEKcdjcPzjSYhdtRCSEs2bLUpKihG3YSWOfzzJaP0vr0tXjwMAvt+7yOA2dkYtBABcTjiucx1zDLnlGHYTERERUU1iZWUFPz8/xMTEKI5JJBLExMSgc+fOOrcjkUhQWFioaLNjx45ISEhQKnPt2jU0adLEKP3WJ+wuHXL7Th9mlOuXR1W8uVDdVFbIbch40Fhhd3XHoJuIjEbXsNucQ+7Oc5fBd/wMCC2tAADHZk/A4/v31JZNPXUEF79ZBQCo59MWXReuQffFXyFw9kL4DBkNALh3MRbn14SrnR0ulYhxbvnnSD+rexhc0aRSKfb+sRoAcP9BKtIyb+vdRlrGLWTnyG4QRP2xBrrseWzOIbecscNuIiIiIiJzFhYWho0bN2Lbtm2Ij4/HuHHjUFBQgNBQ2azQkSNHYvbs2Yry4eHh+Ouvv3Dr1i3Ex8djxYoV2L59O4YPfxauzZgxA5GRkdi4cSNu3LiBdevW4ZdffsH48eON1m9dwu7nQ24Lm1pGu76hqurNheqiMmdyGzr5yRhhd3XHoJuIjEpb2G3OITcAOHp6wdW/M4LWbYets+yuedxXy1TKFT8qwMWNskDYe/BIdJzxOewbNoKtsyvqtmgFzz6vossXXwIAsi7HIf3cSZU2Uk8fw4Nr/wEAOn+6pKLeks4KHuXi8KkfcTv5MgBAKBBh189LcP9BquKRt7JIJBLcf5CKH35eAqFABAC4nXQJh0/9iIJHuRrrVYWQW86YYTcRERERkTkLCQnB8uXLMXfuXPj6+iIuLg7R0dGKNaSTkpKQlvbsCdiCggKMHz8ebdq0QZcuXbBnzx7s2LEDY8eOVZQZOHAgNmzYgKVLl6Jdu3bYtGkT9uzZg65duxq172WF3eYYcstV1ZsLVV1lL1dSnid9GXaXTSDVZapdNZKXlwdHR0fk5ubCwcHB1N0hMltB2z4rV/2Sx4WIW7ET+SmZeGHGCDg2c9c55P57VPmuXR6vbNmreF2Ym4ND02Qzs3uu2AjruvUV5xL//AVXd21Bw07d0OG9aRrby4g9jQsRSwEAfTcrr3kdPXoQAKDTJ0tQp1kL/BE60GjvQ1/2dnUUYbRQKIJEIlY6b2lRC85OHvBw88E7gz9H/bpuAICs7BRs3T0Pd1MTkJmVjOKSQqV6pduys3VEQ+dmaNPyJQwdOAdCoexea+jkliYLuaVF/8K+lq3e11x9YgeWHd2CGd1DDXr8K7/wEbxXvKp3PSIiIiKi6qy849DnPT8GLSvkNuU4tLR169Zh2bJlSE9Ph6+vL9asWYPAwEAAQM+ePeHp6YmtW7cCAD755BNERkbi7t27sLGxgY+PD6ZMmYKQkBClNjdv3ozw8HDcvXsX3t7e+Pzzz/H6669X9lszK9dmJwMwzZrc0qJ/MTxyFhKyEvH920vxgpuP3te8kHoVQ3fNhLeTJ3aELNZ5XOu+6GW9r2Vq+mS5FpXUJyIywJDvyv4HSFwswe6xR9DpvVbw7KJ5za6yJJ5Ix+lv4vHWph4QWZZ+yKN8s63lM7vjVuzEhWXbUdfHE1kXEsx2Jrc6tRzrwDO4PxIP7Ef6P6fh2aef4tztP/YBANy79CqzDRe/TorXT3KyYV2nnuz1g2zFccemzY3Ya8OUnnH9fMgNAMUlhUjNuInX+oxThNwA4FTPHS+06YWzF35Xu0RJ6bYKHuXi3v1k9OrytiLkBmDSmdzDI2fp9aFATh5uy3et1jfsNiRcJyIiIiIi/cjHnreiDiHvdioeXE00y5ncpU2cOBETJ05Ue06+yaTcggULsGDBAq1tjh49GqNHjzZG96oVU208KX/Sd3jkLAzdNdOgsFs+s3vorpkGj2urIy5dQkQVRh52ix8XIutCApxe8K4yIbecpX1tAEDxQ+WlNwpzHwAArOvUV6mjSXH+w1Kv8xSvBQJBebpoFHUdXSASqr/3KRAIIBAIMOGd1ejddajK+d5dh2HCqFWKcuqIhBao6+iC+TP3w91VOdg3VcidX5hvlF2rDV3GhIiIiIiIKl7T/t3h9II3si4kQPy40KxDbqo8pgq55Z7fw4nLmBgHZ3QTUYVK/uuM4vWDq4nIvZUCx2buJuyRfgqfzrwWWlmpPS+ysdG5LfkGl2W1ZyrzZ+7H7PBX8DA/W+WcVCrFxNA16NHpLY31e3QeDAgEWLdFfeBsZ+eIBTN/gbOTh8o5U4XcY38cUe474OWd2U1ERERERBUr91YKHlxNVHyf/NeZKjcBq7r66Zt0xWuxuAQrv3kPZ+P+UFs2wPcVhL33DUQizVFmRlaSxnEtAKVx7SIThtxynNltfJzRTaSHiIgIeHp6wtraGoGBgTh79qzGslFRUfD390edOnVgZ2cHX19fbN++XWP5Dz74AAKBAKtWraqAnptG6fXQeqyfpXGDSnOWdFD2R7aeTzul4w06+AMA0s8eL7N+Ud6zmeC2DZwVr22cXBSvH2Vllruf5eXi1BjWtew0nvds1EZrG03cW2s8Z13LTm3IbQhjhdzXs64Z5Q44Z3YTEREREZmn0mty91g/S+MGlWR6IpEFGjVsCZHIUs05S3i4eZcZcgP6jWtNHXLLcWa3cTHoJtJRZGQkwsLCMG/ePJw/fx4dOnRAcHAwMjPVh5T16tXDxx9/jFOnTuHixYsIDQ1FaGgoDhw4oFJ27969OH36NNzc3NS0VDU9v+mHfBmTqhR234n5XfG6jldLpXPeg0cBABJ+/A45t66rrV9S+ARnl38GAGjS51UIhCLFOaFIhKavDAAAnPpiBsSFhWpaqDyFRY+RlS37byIQyP40lF7KJEmHP7bJpcrI6wqfvud79++iqPhJuftpzJB7S8hOAMb5UMCwm4iIiIjIvKjbeLJp/+4Mu81YcupVSCQlAGTjUvnYVCIp0WlMqs+41hxCbjmG3cbDoJtIRytXrsS7776L0NBQtG7dGhs2bICtrS02b96stnzPnj0xcOBAtGrVCl5eXpgyZQrat2+P48eVZwCnpKRg0qRJ2LlzJywtVe9cVkXPh9xyVSXslhQX48p3XyN+5yYAwIuT56isPW3f0B0NA7sBAE4v+AjZ1/6DtNTGi4+yMnFkxvvIv3sHAOD1muqyH037DgAgW7v71MJZFfFWdJaafgNSqQQA4OrcFONHfgl/32DF+eQU7X9oS39o6OjbF+NHfgmXBp4AAKlUgpT0G+Xqo7FD7tIfahh2ExERERFVH+pCbjmG3ebrdvJlSKVS1LKywVuvTsdbr05HLSsbSKVSJCZf1lpfn3GtqULur06sUXucYbdxcI1uIh0UFRUhNjYWs2fPVhwTCoUICgrCqVOntNaXSqU4ePAgEhISsGTJEsVxiUSCESNGYMaMGWjTRvvSEFWBppBbTh52x63YiQvLtuOFGSPMas3u6NGDlL5vGzoBzr7+asu2GzMRTx7cx4Nr/+Hs4k80ttl14VpYPd3UsjSr2g7ounAtjn88SRGIm0pS6lU0qO+BkNc+RNfANyASitCry9u4eedf/LAvHEmpCVrbSE5JQIfWPTFkwCx4NekAAOje+S0cP7MHkb8sR3LKVTT1aGtQ/yoy5JYzxtpmXLObiIiIiMi0ygq55eRj1VtRh5S+J9N5UvgID/Oz8VqfcRjYdyJq29cDAAT3HIV90RH488hWFBY9Qi0rzWM0Y4xrNTFWyL362HLM6PqK2vNcs7v8GHQT6SArKwtisRguLi5Kx11cXHD1qua7bLm5uXB3d0dhYSFEIhG++uor9OnTR3F+yZIlsLCwwOTJ+gd35khbyC1n7mE3ANRp7o3Ww9+DQ+OmGssILSwRMPMLpJ05houb1gJP7xzLNX1lIJq+MkBtyC1n39AdL6/Zhlu/Rxmt74Zo2cwfq784DksL5U0yvZp0wCdTdiFZhw8EwwZ9DA83b6VjIqEIPToPxkv+r+N+TppBfauMkFuOYTcRERERUdWlS8gt93zYjVGV0UPSRCwpweovTqB+3YZKxx3s62Pkm3PRr/e7KBGXQPN/UeOMa9UxZsg9pduHZZZj2F0+DLqJKlDt2rURFxeH/Px8xMTEICwsDM2aNUPPnj0RGxuL1atX4/z58yrLYlRFuobccuYadnedvxrW9erDwka3PwICoRBunXvArXMPPMnJRnH+QwgtrWDj5AyhSKS9AQBW9rXhM9i0n6oaOmsO9AGoBNj6lrG0rAXXp8uY6KMyQ245ht1EREREVJMN+e5lvcof/fIiAKD7tPYGXU9cLMHusUfQ6b1WsHtpsEFtAPqF3HIqYTeZjJ2NA+xsHDSefz4AV8cY49rnGTvklo1r1e/zJcew23AMuol04OTkBJFIhIyMDKXjGRkZcHV11VhPKBSiefPmAABfX1/Ex8cjPDwcPXv2xLFjx5CZmYnGjRsryovFYkyfPh2rVq1CYmKi4gODJpIS2Qzi09/EI+ms+k0xtUmNuw8AOLbqIoQWz5btbzVV90e39A255cwx7LZ39zC4rnWderCuU8+IvanZTBFyyzHsJiIiIqKaSts49HnyMaW+9eRKj2t7v2RQEwaF3HLmsGzJK1v2muzaf4QONNm1zV3FhNy6qaiwu7rjZpREOrCysoKfnx9iYmIUxyQSCWJiYtC5c2ed25FIJCgsLAQAjBgxAhcvXkRcXJziy83NDTNmzMCBAweM/h4qiqEht1xV2aCSKpcpQ245blBJRERERFS5DBkPlifkljOHsJvMiylDbrmK2KCyuuOMbiIdhYWFYdSoUfD390dAQABWrVqFgoIChIaGAgBGjhwJd3d3hIeHAwDCw8Ph7+8PLy8vFBYW4vfff8f27duxfv16AED9+vVRv359pWtYWlrC1dUV3t6yx2m0PfpV+hEvzy6aZ5aXJfFEOk5/E49uU9tDZPns3tc9qfa65Q255Z6f2Y3PNxrcVkWRiMV4nJUBSVERLO0dYF1X/5nbJY8f4Um2bLaDPsujmKOi4icAACtLa4PbKHicB1tr1fXLzSHkluPMbiIiIiKqafRdgsSYS5ec0/NJX2OE3FXJkwfZKM7Pg9DKCjZOLjovlylnjHGtMUmlUoOXcpVKpXj05GGZy50YyhxCbjljz+yu7hh0E+koJCQE9+7dw9y5c5Geng5fX19ER0crNqhMSkqCUPgsKC4oKMD48eNx9+5d2NjYwMfHBzt27EBISIip3oJRGSvklisddpuTood5uB29D7f/2Kdyrt3YyXDr1B0CYdkPx+Ql3caV7V8j9+Y1peMOTbzQNnR8mRtemqthE2V93v21YRtMpt9LxKRPOqNbwCBsenmV4rg5hdxyDLuJiIiIiCqH/ElfXcLumhJySyUSpJ09gYvffKlyrklQP3i99hasapcd9pY1rkWoDrPcKsjazRMxeUyEQXXXbJ6A42f3Yt2C03Bp0MRofTKnkFvOmGF3dcelS4j0MHHiRNy5cweFhYU4c+YMAgMDFecOHz6MrVu3Kr5fsGABrl+/jsePHyM7OxsnT57UGnInJiZi6tSpFdR7/ZQ8LtR4ztght5w87DYX+Wl3cXDKO+o/DAC4tGkNzi6dC0lJscY27h6LwcnPpquE3ACQd+cmTn42HXePxaipab5SM24qXqdl3DKojdOxvwIALsYfVRwzx5BbjsuYEBERERFVPF2XtawpIbekpBixqxaoDbkB4M7fv+HglHeQn6b5Z6VtXGtKF+OPQirVP2iXSqWKseTp878arz+VFHIbMh401jIm1R2DbiJSK27FTrVhd0WF3HLm8gGl6GEejn8s+6Nkae+A7ks3oO/mKPTdHIXgb/eg0ydLAAAPrv2HS9+uU9tGZtw5XN4iuzvt0SsY//s6UtHG/76JhPfgUQCAy1sikBn3TyW8K+OI+mMNhEIRhEIRov5Yo3f9J4WPsO+A7OeS+zALgHmH3HLGDruJiIiIiEiZLns4VceQW9N48MK6pci6HAcAeGHCTMV4su/mKLy8agvsG8lmMh//eBKK8h+q1Nd1XGsquQ+zcOHyQb3rnb8cg7yHsmVB90avwxMDxmbPq8yZ3IZOfjJG2F3dMegmIrXyUzJVwu6KDrnNyc1fdgMA7Bs1Qc9lX8PWyVlxTiAQoE6zFooPBWlnjqEgPVWpvlQqxfk1svXaWw0bizYj3ofQ0lJxXmhhiaZ9X0ezfoMAAOfXLKrQ96PNlYSTuP8gTe3d9ILHebiZGIfjZ/fix1+W4+jpnyCRiCGRiHHk9G78+MtyHD+7Fzfv/ItHj1U/XEmlUtx/kIYrCSfx97EdiNg6BY8e5wIAhEJRlQi55YwZdhMRERERkaqywu7qGHIDsvHg82Ox/LQU3LsYCwDo8sWXcPHrpHTeysERnT4Oh6W9bN8jdTO2dRnXmpJQKMLW3fPw699fI/biX0hJv4HikiKVcsUlRUhJv4HYi3/h17+/xrbdn0EolK1P/uhxLiK2TsHfx3boNa4trbKXKynPk74Mu8vGNbqJSK0XZozAhWXbEbdiJ3ynD0PyX2dqTMgtEYtx5+/fAAABH34GUS31H57qNGsB77dGImH3d7j641b4TZ6jOJdzM0Hxuknv/9N4rRYDh+LWb1FG6rnhPlv5BgCgs19/TB27XrHe/IlzP2PVpg8U5URC1T8bUb+vhlhSAkD2YWnq2A14yb8/AEAikWDVpnE4Fbv/WRsiS8UHD4lEbNKQ+0Lq1XJt5FHeNbuJiIiIiEhV6T2c5Gt2A6iWIbdczs1rqNvcW/G9PLj2GTIatRupX4PaopY1Os9dhqMzP8DtP/ai5RtDIXgaAOs6rjUliUSMzHt38N1PX0AqlQAA6td1w4KZ++FUT7ZGe1Z2Cj5e8hqyc2T7QwkEQggFQkgkYgCyiVXn/j2gtISJLuPaBW2nADDNmtzl3cPJGGt2V1ec0U1Eajk2c8cLM0YgPyUTR8YtrjEhNwA8zspQvLZycCyzrGtgVwDAveceNcu+ehkA0LiMkBsABEIhvN8aaUg3ja5jh2BMGr1WaVPVLh1fx7iRKyEQCCAQCCCWlCg+gACAVCqBWFKiOD9uxEpFyA0AQqEQk0LXwL9DsOKYWKy8prkpZ3KXZ22z8s7sJiIiIiIizUrP7P7ni03454tN1TbkBoDsq5eUvk89fQQA4OjpVWa90rO0H93LVLzWZ1xrSvIxplAogqNDA8yd9qMi5AYAp3rumDvtRzjWdoJQKFKMQZXaKDXG1HVcC5h248ny7uHEmd3qMegmIo0cm7mjro+n4nuPPoGaC1eQiIgIeHp6wtraGoGBgTh79qzGslFRUfD390edOnVgZ2cHX19fbN++XXG+uLgYH330Edq1awc7Ozu4ublh5MiRSE1VXnZEUqT6qJQmJY/VB5zip23UqlNXexuFT3S+XkXp7Pcawt7fCEsLK5VzL3cZgonvyNbiFggEKuflxya+swa9urytct7Sshamv78RnV58VW19U4XcF1Pjyr2RB8NuIiIiIqKKY2FTC80G9VR832xQz2oZcgOq41BpiSyMreWofUypaKP4WRv6jGtNTSS0gKNDAyyYuR9uLqrBvrtrcyz46Bc4OjRQ+5QxIBuX6jOuNWXILcew2/gYdBORRrf3H0XWhQQ4veANkU0tjRtUVpTIyEiEhYVh3rx5OH/+PDp06IDg4GBkZmaqLV+vXj18/PHHOHXqFC5evIjQ0FCEhobiwIEDAIBHjx7h/Pnz+PTTT3H+/HlERUUhISEB/fv3V2rH0t5B5z4WPsgGANSqU0/puFVtWRvFajYEeV5BuuYdsivLlDFfwUJkqfF8905vYmBfzX+4B70yBd07vanxvIXIElPHrkdD52bl6idgvJA7NHKYUXatZthNRERERFQxcm+l4OLaH2Hv4QJ7DxdcXPuj2g0qqwPL2upnXUvEJWqPy5Vej1q+Xrfste7jWlMTS0rw8aSdcG3gqbGMawNPfDxpp8psbjk3Fy+9xrWmDrnlGHYbF4NuIlKr9MaTHaa8rVjGpDLD7pUrV+Ldd99FaGgoWrdujQ0bNsDW1habN29WW75nz54YOHAgWrVqBS8vL0yZMgXt27fH8ePHAQCOjo7466+/MHjwYHh7e6NTp05Yt24dYmNjkZSUpGjHuu6z0Doj9nSZfUw5cQgA0LTv60rHXZ9uFJJ4YD8Kc3M01pcUFyP97Ikyr1EZRCLtWzY8fpIPoZq750KhBR490R7oi0QWeJifbVD/5IwZcrdwammUDwUMu4mIiIiIjK/0xpN+c0LhNydU7QaV1YWrv/Jmky3eGAYA+G/7N2o3V5TLvXVd8dq61AQsfca15kCXMeWjx3kaz+XlZ+s1rjWHkFuOYbfxMOgmIrWeX5O79JrdlRF2FxUVITY2FkFBQYpjQqEQQUFBOHXqlNb6UqkUMTExSEhIQPfumtcVz83Nle02XaeO0vH2700DAFyIWIqHd++obf/2H/uQduYYAKBRt95K563rOcGxaXMAwJnwOZAUF6u0UfyoAKfmz1SUL62yl2zJy7+vsX25Oyn/QSwuhkhooVjbTCS0gFhcjKS78Vrr5z7MwsOCBwCebiDydJMUXRk75N40WPYzYthNRERERObMFMs5mlrpkFu+JnfpNbsrOuyu7J+5dT0nWNetr9Ru4159Acj2f0qM/lnttXNuXcfphbMAAO3fnaJyXpdxrakJBLJoMjlF+zgsKTVBqY5QKFK8fpifrde41lQhd35hvtrjDLuNQ/utDqIabsVXvUx3cXvV9Ywri7qNJ+Vh94Vl2xG3YmeFbgKSlZUFsVgMFxcXpeMuLi64elXzP9i5ublwd3dHYWEhRCIRvvrqK/Tp00dt2SdPnuCjjz7CkCFD4OCg/FhXw4AuSD15CFmX43Bi7jS0GDQUDdr7wcq+NkoKn+DKd1/jQcIVAED796bCwsZWpX3f8TNwZMb7eJSZjr8mDEO70Amo4+UNoYUFCjLScG7ZPEXZwNkLFa/lS7Zs2LABgYGBWLVqFYKDg5GQkABnZ2eV68iXbPHx8YGVlRV+/fVXhIaGwtnZGcHBwUpLtnTo0AEPHjzAlClT0L9/f/zzj2wTzeTUBLRp+ZLGnysAJD394PFiu94YMmA2AOCHfeE49+8BJKVoD7rvpl4DAFhZWuO1/43Da0Hv42L8Ua31gIoJuUt/qDHGrtXysHvorpkYHjkLO0IWw76W6u8FEREREZGuTDE2MDV1IbecPOyOW7ETF5ZtxwszRsCxmXsZrenPFD/z0uNBOUtbO7R/byoufrMKCbu/Q86ta/DoGQy7hrL3m37mOBJ2fwcAqNuyNRoGdlNpQ5dxLUYPMuaPTy9h732DnVELkZF1RxFil0Uehjs7NcawgXPQvlV3/PLXBvzy1wYUFT/Ra1xrqpB77I8j8MvIpWrPT+kyHACw7OgWpe91ZYxxbVUnkJrD7ZtKlJeXB0dHR+Tm5qoEW0TqmDLo/kdL0C0ulmD32CPo9F4reHZxNegaiSfScfqbeLy1qQdEls8e8rgnLWMWdBkfPMor91YKzn2+EampqXB3d8fJkyfRuXNnxfmZM2fiyJEjOHPmjNr6EokEt27dQn5+PmJiYjB//nzs27cPPXv2VCpXXFyMN954A3fv3sXhw4cV/x68smXvs7ZKinEhYhnu/av5A1/796bCrZPmn1VBRhqOzZ6g8bzQwhJdF6yGrbMr/ggdCAAIDAxEx44dsW7dOsV78vDwwKRJkzBr1iyNbZX24osvol+/fpg/f77a8+fOnUNAQADu3LmDxo0bY9zIlejddajG9nLzsrD62/EYMmA2WjR9Qenc9dvn8f3ecEwdux6ODk4aWgBiju9EcmoCBvadrFSu/W3V2e6lVWTIbSe4rlQuv/ARhkfOQkJWosEfCi6kXsXQXTPh7eRZZtjtvuhlvdsmIiIioprFFGODId/p9zn16JcXAQDdp7XXq55c6XFtiWsXncaaJY8LEbdiJ/JTMo0Wdv896jMApvmZlx6HPi/11BFc3Lha4/mGnbqh3eiJEFqoX5ta27jWlLHgT9+kQywuweFTP+K/66cwKXRtmeXXbJ6Itt5d0KPTW0rLlOTmZSHqj9Vo7N5K53HtG2irV1+NFXJfz7qGq2H7yiy7+sQOLDu6BTO6h+oddsuupXlcWxXHofpkuVy6hIj0VlHLmMgDdABwcnKCSCRCRkaGUpmMjAy4umoO9YVCIZo3bw5fX19Mnz4db775JsLDw5XKFBcXY/Dgwbhz5w7++usvjf9QCi0s4TdlDrouXAuPnsFK51q+MRxBETvKDLkBwM6lIYI3/QS/aZ+gjpe34riVQx34Tf0EfTb8AFvnZ+/HVEu29HwppMx27e3rYu60H1VCbgBo0fRFzAvbDXv7sncD7/nS23hn8BdlhuHPq+iZ3M/jMiZEREREZC5MvZyjKeg6oaqiljExx5+5W+ce6L1uO1oNGwsLWzvF8Qa+/ui2aB06vDdNY8gNaB/XmppIZIHeXYdi/KhVWstOeGc1Xu4yRGUtbkcHJ4SGzC/XuLYsxgy5t4Ts1Fqey5gYjkuXEGlRUiwx2bXFxWXP6JaUyPomLpFAbGA/xU/bkLeloOVfB2MvY1J6ljgAWFlZwc/PDzExMRgwYICsjxIJYmJiMHHiRJ3blUgkKCx8FsTLQ+7r16/j0KFDqF+/fhm1ZewbuqPNyPfRZuT7+r2ppwRCIRq0exEN2r2otayplmwRaVkvW9t5Y7VRWmWH3HJcxoSIiIiIzIGpxgb6ji0V41IDx6Slx7X6PDVcEcuYmHoJTU0sbe3QpPf/oUnv/9P9zTynvOPailYZY059x6SA8UNu2bj2urZqXMbEQAy6qcqJiIjAsmXLkJ6ejg4dOmDt2rUICAhQWzYqKgqLFi3CjRs3UFxcjBYtWmD69OkYMWKEUpkNGzYgNjYW2dnZuHDhAnx9fRXnZ03VbQ1hUzq3OQHnNmtfz6osez44pvR97609tdYxVtj9/FIocmFhYRg1ahT8/f0REBCAVatWoaCgAKGhoQCAkSNHwt3dXTFjOzw8HP7+/vDy8kJhYSF+//13bN++HevXrwcgC7nffPNNnD9/Hr/++ivEYjHS09MByNZVs7Ky0rvv5qJ27dqIi4tTLNkSFhaGZs2aqV2yZfDgwZBKpYqfizkyVcgtx7CbiIiIiEqr7HFoeZR3bLB77BGDrmtoPblzmxPQY/0svcaUxgq7y/uUcnUZj1Wl3/PKUDEht+4qIux2R9VbukQfDLqpSjH2phAAUFBQgK5du2Lw4MF49913VdoYMsp0d7xu19KyRneJBOc2J8C1XT14dnYps6wmiacykH4pGx1He0Nkof9qRuUNu8ta7zskJAT37t3D3LlzkZ6eDl9fX0RHRyvuriclJUEofNbngoICjB8/Hnfv3oWNjQ18fHywY8cOhITIHl9KSUnB/v37AUDlj+uhQ4dUPoSYQnmXbAFk7y0+Ph7h4eFK76n0ki0HDx40230KTB1yyzHsJiIiIiLANONQwHRjg07vtdL5ZwMAp7+JN6ieXOlxrSETp8obdsvX+8YH4TV6PGaq33NzZeqQW87YYXfe1vEG9aOq4GaUVKVU5KYQiYmJaNq0qcodRm5GqRtDNqjUVEe+CYgpNH89BHaubnDx6wSRpe4zvJ9kZyE99jSKHuZBZGWFej5tUcfLGwJB2f8N5aRSKaKf7nYdGBiIgIAArF0r24hDIpGgcePGmDhxos6/56NHj8atW7dw+PBhAKpLtjRo0ECp/E/fpCv15YefF+PVoPfgYK99aRd18vLv49e/v8HQAbO1li29GWVlh9zPb0apTkVtUFkVNwEhIiIiqolMMQ4tfe3KHhuYcjNKu5cGG9QGYNgGlaXrlDx6AsA0P/O+m6OQc+MqshOuQFxUBKvaDnD16wTrerrvbyQuLkJG7GkUpKcCgM7j2j9CByred2X/npcehz7vn3//hEAggF979UvAaKNtXFt6HPq8ig65dRmHPs9YG1Seu3tZ77qmpk+WyxndVGXIN4WYPftZcKbvphAHDx5EQkIClixZUpFdrZH0ndltSDBeGW7u/1Hx2qNXMFoNHQuhSPM6Xo/v30PcV8uQe/uG2vMvTp4DZ1//Mq+ZGfcPzq9ZBIyW3Xc09ZItV2+cwd4/1uDkP/uxboH2/7fUmRPeDxlZd/BC25fRqnmgTnXMZSb38ypqZjcRERERmT9Tj0NNPTaoSvSd2f18MC5nip/5gTFvqPTv6g+bYV23PgLnLIJN/QYq5+UkYjFu7NuFW7/tUXtel3GtqX/P1bW35KtRAIAfN6TqPIGsNEPHtZUxkzu/8JHeT/oaa2Z3dcegm6qMytgUgspH17DbXENuAGj6ykDc/+9f5N25heRDB5CfkoyOMz5X+6GgICMNx2ZPUHzvGdwflva1UZiTjaSYPwAA59csQtvQCWjUrbfa6909FoPLWyKUjpl6yZaY498DAHLyMvH4SQFsrO2gj8dPCpCTl6loS5eg21xDbrmKCLvPrXjVaP0jIiIiooph6nGoqccGVY2uYXdZs79N+TP3fmskSgqfoCA9BelnT+DJg/s4MuN9dAuPgJ1LQ5X3IRGLcW7ZPDy49h8AoH6r9nBq54uihw91HtcCpv89f975S38/e305Bn7tgvRuw5BxbWUtV2LospbGCLurOwbdVO3puikEGYe2sNucQ24A8H5rBIARyE+7i+MfT8aDa//hxr5daPnGMKVyUolEEXLbOjdE4OyFqOVYR3G+9bB3cSfmd8Tv3ITLWyLg0KQZHBo3VWojL+m2IuRu9uqbSucmTpyIiRMnqu2j/PE3uQULFmDBggUa35Onpyd0XaUqLfM2jp6RzQQoKnqMP49sw+vB+q3h9eeRrSgqlj32d/T0T3iz3zS4NvDUWN7cQ245Y4fdRERERFR9GXMcaqqxQVWlLezWZYmTyv6ZN3v1TbQY8DYEpQJ0yZjJODV/Jh7evYMTn05Fnw0/KJ0HgPjvNylC7o4zPkf9Vu1KndVtXFseFZG3SKVSRO5fCqFQFspH7l+KF9v21mtWtyHj2spckzshK9FkYXd1x6CbqoyK3BSCjEtT2G3uIXdp9g0boeOMz3Fu2Tzc+m0PvPq/pbS2WdblCwAAoaUVus5fBaGlpUobTXr/H/JTkpF8+AD+2/ENOs0JVzp/ectXAADvkHfQNLh/Bb4b7TZ+Pwup6Tdx5+4VCAVCiKUSxZpmp2L3w71hCzR0boqGzs3Q1rsrHB1ka8Xl5mXhcsJxpGXeQlrmbaSkXUfi3f8UH+SEAiHmhP8fmjRqAzdXL1kbLl54oc3LilkQVSHkljNm2E1ERERE5o/j0KpJU9htyDrelaHloKEqx4SWlgiYtQAxE0dAUlKMrMtxaND+RcV5cVEhkg8dAAB0XbgG9g0bqbSh27j2DZP+nicmX8HNO3FIzbiFtMxbSE5NQHrmbcX520mXMGVuVzRya4mGzs3g5tIMXk184enRBoBsLfELVw4iLeMm0jJv6zyube/UT3GNyt548vllLRl2G49QexEi82BlZQU/Pz/ExMQojkkkEsTExKBz5846tyORSFBYWFgRXaRS5GF3fkom4lbsRPZ/t6pMyC1Xv1U71Ht6Rzwj9ozSuRtP1/Ju/+4UtSG3XKshowEAOTcSUPL4keJ4yeNHyLtzEwDQpPcrRu23If48sg2XE47jYcEDiCUliuNicTFu3vkXx87sQeT+ZTh2di9sbWorztva1Maxs3tl587swc07/0Isfraph1hSgocFD3A54Tj+PLIN3/30OR7kpCs96mfKkHv1iR16X1Medns7eWLorpm4kKr5UT5NDNnQkoiIiIgqH8ehVZc87LZ3d8aFZduR/d8tswy5y2Jpa4eOMz4HAFzavFbpXMZ52RjVoUkztSG3nC7jWlP+nltYWGLXz0ux/8+v8M+/B5RCbrm0zFv4598D2P/nV9j181JYlgrrhUIhsh+kYdvuz/Qa18pVdsgNPJv8JJ/ZnV/4SGud503pMhwzuodi2dEtBo1rqysG3VSlhIWFYePGjdi2bRvi4+Mxbtw4lU0hSm+eEB4ejr/++gu3bt1CfHw8VqxYge3bt2P48Gd3u7KzsxEXF4f//pM97pOQkIC4uDjF5hBkOHnYnXsjGReWboeNU50qE3LLNWj3AgDgUUaq0vHcW7Jdkm2cnMusXzoEf5J9X+1roYXmoLyyCAQCrY+CBfi+gunvb4Sl5bP/fpaWtTD9/Y0I8C07rJe3P27klwjqpny32ZQzuQ39UGCMsJuIiIiIqgaOQ6suedht41QHF5ZuR+6N5CoTcsvZOstmVBfl5SodL0iXjVHrt+6gtQ1t41rAdL/njRq2xBcf7kUdB2cIBZo3zBQKRKjj4IwvPtwLd9cWSuf6dB+B8SO/1GtcC5gm5JZj2F0xuHQJVSnG3hQCAPbv36/4hxsA3n77bQDAvHnz8Nlnn1XOGyOzVfQwr8zzVrUdKqknFWtS6Fqs3TJJ7TmBQIDOfv0xafRaWIhUQ3lLCytMe+9rrPl2Ik6f/0Xj+nOTQteiW6DqbuKmCrm/OrFG8aEAMHzX6vIsY0JERERE5o/jUDKl5wPu51nVrl3meUD7uBYw7e95Q5dmWDBzP+YuH4icvExIJGKlvgmFIjg6NMAXM/bBxamx2v736vI2LCwsdR7XfnVihclCbrnSezhxGRPj4IxuqnImTpyIO3fuoLCwEGfOnEFgYKDi3OHDh7F161bF9wsWLMD169fx+PFjZGdn4+TJk0r/6ALAO++8A6lUqvLFDxflJ1+T27G5B16YOQKPs3IQt2InSh5XnUf27l/5FwBg6+KmdNzKwREA8Ciz7BkXkpJny3hY16uv9rWkuBim1i3wDTR2b6X2nFQqxdCBc9SG3HIWIksMGzhHY8jd2L212pDbEMYKuVcfW17uO+Cc2U1ERERUM3AcWjXJ1+R+nJWDF2aOgGNzD1xYth25t1JM3TWdPb6fCQBw9GqpdNzOVTZGzboUp7UNbeNaOVP+nrs0aIJXeo1W23+pVIr/e3mMxpBbTp9xralDbjnO7DYuzugmogqhbuNJdRtUqiMQCND76zVlrn0NyALimPfV/1Fq+FIg2o59R2s/8xLv4MwXixXf990cpXidn3oXeUmy9cFc/AKV6rUbPRGxqxbi3LJ56L1uOyxt7dS2f+fv3wEAjk2bw8Lm2Z1ZCxtbOHq1RO7Na4jftRltRryvta8VSSKRIDX9puJ7oVCkdBf9bto1rR8q7qZdU/q+dBupGTchkUiUZgAYwqghd7cPAZT/DjhndhMRERERmR91G086NHVX2aDSnEmKi3Fx4xoAQPPXBiudc/HrBAC4H38R9+Mvof7Tdbifp21ca07upl2DALKlR4RC2TImEokYQoEQyc+NN9XRZ1xrDiG3HGd2Gw9ndBOR0akLuQHVDSo1zezuFfGl1pAbkK1/3StiJezdle9Ktxg8CG3GjNKprw6eTRDw8UyV4/f/u4jjn8j+6Hn0ClbamRoAnNq+AIGF7F7h2cWfqMzKlkokuLZnBxJ+3AYAaDNqnMo15OF28qEDuBPzu079rSiZ95NQXFKoWM+sQ+sesK5lB4FA9mciOUX7TOWkp7OZBQIhbKzt0b5V96ffC1Bc/AT37ieXq4/GDrlLf6jhzG4iIiIioupDXcgNqG5QaS4zu9WNBwtzc3D80ymQFBcBAJza+iqdF1laoVk/2VOz55bNQ37qXZU2dBnXmpPEu1cUm0i6uXjBzcULACCWlCAx+YrW+vqMa00Vcl9MjVN7nDO7jYMzuonIqDSF3HLysLusmd0WNtY6X8/Cxgad538qewRKIoFQpHnzCk0cvZqiz+b1kEokuH1gP+5djEV2/CUAQN2WrdFq6FiVOgKhEF3nr8ax2RPw8O4d/Pl+CFwDusDRszkglSJh93eKsm1DJ8ChcVOVNhwaN0Xb0Am4vCUC8Ts3ATs26t13Y0lOTQAA+LbphaED5sDTow3yC3Lw858R+O3vjYoQu8w2Uq7C0tIar/Z+F/3/Nx72dnVwO/kyftgXjguXDyI5NQEuDZoY1L+KDLnlOLObiIiIiKjq0xRyy8nDbnOa2R2/cxPid25C0/8bBKvaDijMyUbigf2K890XfwWBmqdjmw94Gw+ux+PBtf9w/JPJsGvoDveXekFgYaHzuNZciCVipGbcQoP6jTBkwGx08R8AADjxzz78sC8caRk3IZaIIRJqHvMbY1yribFC7tDIYbgatk/tec7sLj8G3URkNNpCbjldwm59CQQCCAwIuZXaEAqRELlV8X2zfm+g+YC3NYbndi4N0WPZ1zizaA6ePLiP9LMnkH72hFKZFyfPgbOvv8ZrNurWG1a1HXF+zaJy9b28LC2sMH/Gz/BpHqA4Zm9XB8MGfoz/e3kszl74Q2sb3l4dMeLNeajr6Kw41tSjLeZM2on4G2dQaMAdaaByQm45ht1EREREZA5WfNXLdBe3F5ju2uWkLeSWUxd2/2/Lhkrurarbv0cpfe/YtAV8x38Im/oN1JYXikToOONzxH+/CcmHDqAgLQXX9ijP5NU2rjUX+fkPEDr4c/TqMkRpf6huAYPQ2e81HDrxA/LzH8DRwUljG8YY16rtmxFD7hZOLcssx7C7fARSTTuHVVN5eXlwdHREbm4uHBwcTN0dqgJM+QHjHy0fMMTFEuweewSd3msFzy6uBl0j8UQ6Tn8Tj7c29YDI8tnd3HvS7nq1o2vIrUsdocSw92IMt38+DFsXN7j4Bao81lWWJw/uI/2f0yh+mAuhlRXq+bRDHa+WikemtJFKpbATqt80ozIMKaprsmu3v615M86KDrntBNfV1ll9YgeWHd2CGd1DDfpQkF/4CMMjZyEhK1Fj2O2+6GW92yUiIiKimsGcx6HPO/rlRQBA92ntDbpe6XGt3UuDtVfQQNeQW1OdlyNWGXzt8hJKnZFzMwHZVy9DUlQEy9qOcPXvBOu69XVuQ1xchIzYM3iUkQoAOo9r/wgdWK6+l8dP36Sb7NpljUNLM3bIvWnwdrhYp2mtcyH1KobumglvJ0+Dwm5A/bi2Ko5D9clyOaObSIvYs6b7hzexlpagu0QiK3cqw/BrPK2beDIdIotnQbfdS7q3YUjIDVTMzO7yav56iPZCaljXrQ/PPv0Mvq6ugXhNUpkzuZ/Hmd1EREREZErmPA59XmrcfVm9E4b1ufS4to0e49DSDAm5AeWZ3aYkEAhQt7kP6jY3fMwgsrSCW6duRuwVVUTIreu4ljO7DcMZ3URa1NQAsvfWeTqVMzTkLqsNq1qGreNsDCK4mOzatgLTBaHmNqO7skJuTTO65SpyZndVvJNORERERJWjpo5De6yfpfeY0tCQ+/k2auo4lDO61avIkFvbOLQ0Y8/sXnpks971TY0zuomMaPEq/ZbwMKZYLY+MSUok2PPBMXQc7Q3PlwxcuuRkOs5tTsAbG7pBWGpGd7YOdY0RcgOqM7sD5swxqB2qHkw5k/t5nNlNRERERKZgzuPQ5x1bJVu6pNtUw5YuKT2u1fdJX2OE3IBsZjckBlWlasiUM7mfZ+yZ3UtR9YJufTDoJoNERERg2bJlSE9PR4cOHbB27VoEBASoLbtx40Z89913uHz5MgDAz88PixYtUiqfkZGBjz76CH/++SdycnLQvXt3rF27Fi1atKiU91MWC0vT7UIsstTtA4bIQqi0vrZe13gabgufb0PLsx7GCrnlSofd5iY/LQUJkVtx72IsAKCWY100fWUA3Lu+DEtbO631pRIJsi5fwKXN61CUlwsAcGzWAs37D4ZT2xfMaqdrXX20MBgCgQCL50QbVD+/IAehYa0weUwE2jd4TXHcnEJuOYbdRERERFTZqsI4VE4+YcrQManiuhZC5Kdk6hx2GyvkrgqKHxUg6eAfuB71PQBAYGEBt0490PSVAbBvqNv71jSufcOETxY3OnsI3QIGGVT32NkorPl2ArasjIe9XR2j9cmcQm45Y4bd1R2DbtJbZGQkwsLCsGHDBgQGBmLVqlUIDg5GQkICnJ2dVcofPnwYQ4YMwUsvvQRra2ssWbIE//vf/3DlyhW4u7tDKpViwIABsLS0xM8//wwHBwesXLkSQUFB+O+//2BnZ4fdq4KU2lyy6V8AwEdjOxj0HopLJBg24xAmDmuN7v4Nyy6s+36ENYaxQ245edhtLiQlxbi4aS3Szx5XOl6Y+wBXd23B1V1b0P7dKXDr3ENjGwUZaTj+6RRIS0qUjufeuo7YVQsBAN3CI2DnouX30Iw8fpKPW0kXn74ugI219rD/eZeuyn6m3+9dhAnvyYJucwy55Soi7HYHly4hIiIiMmfPj0NLO/pPGtbt/A87l/WCpYVhAW+Z49oaOg7VdQ+nmhRyp54+iovfrFI6Ji0pQcrxGKQcj0GD9n54YeJMCC0s1dbXNq5tHTyjorqu1fd7FxkcdO+Mko2nLyccR6cXXzVKfyor5L6QelXvyU8Mu3VT9aYRksmtXLkS7777LkJDQ9G6dWts2LABtra22LxZ/eMPO3fuxPjx4+Hr6wsfHx9s2rQJEokEMTExAIDr16/j9OnTWL9+PTp27Ahvb2+sX78ejx8/xg8//FCZb41Kub3/qNrjFRVyy6n7gFKcX4C0U2eRfvYfFBcUGNTuw7spSD1+CvevxEMq0f5MmlQixrnlnys+DHgPHokey79B1/mr4T99Lho+3eTj4sbVSD2t/mf1+P49HJs9QRFyd5zxOXos+xqd5y6D7/gZED7d/frY7Al4fP+eQe/LFP48sk3ta11JpVLs/WM1AOD+A9mO4OYccstN6TIcM7qHYtnRLVh9Yofe9eVht7eTJ4bummn0/hERERERVXXyyU/ymd0ljwtVypgi5JZKJMi6dAWpJ04hPyXNoDbKGtdqGg+mnjqiCLnr+bRF14Vr0H3xVwicvRA+Q0YDAO5djMX5NeFqx7m6jmtN5f6DVKRl3ta7XlrGLWTnyP47RP2xBsbYfrAyZ3IP3TUTF1Kv6t2+POxOyErE8MhZyC98pHcb1R1ndJNeioqKEBsbi9mzZyuOCYVCBAUF4dSpUzq18ejRIxQXF6NevXoAgMJC2R8ua2trpTZr1aqF48ePY+zYsUZ8B6SrW1GHAABN+z9bG66iQ+7nSaVS/LdlO1KPK/9uNXwpEG3GjNJpg5biggIcnvShynG/GVNRr5W3xnqpp4/hwbX/AACdP10Cx6bPltGxd/eAUxtfOHg0RcLu73Dxm1Vw7uAPCxvlu6lnFsnWGq/dqAkCZi1QLHNiU78BHD294NzBH8c/nYJHmemI+2oZes817x2yi4sLkX4vEXuj1ymO7TuwDi+26w3XBp6wtNT++1DwKBdn46JxO1m2lJFQIKoSIbecMWd2ExERERGRquf3cCo99jRFyJ0dn4DYZatUjvdcuxyWdjosZanDuDbuq2Xo/OlSpfPFjwpwcaNsgpD34JHwDH5dMQa2dXZF3RatUL9VO5yYOw1Zl+OQfu4kGgZ2VWpDl3GtKQkFIuz6eQlGvjkXdR1dIdSyrKdEIsGD3HT88PMSCAUiiKUluJ10CYdP/YgA376ws3XUek35uLY9mimOVfZyJfLJT4Ysa2mMmd3VGYNu0ktWVhbEYjFcXJR3BHZxccHVq7rdjfroo4/g5uaGoCDZY2A+Pj5o3LgxZs+eja+//hp2dnb48ssvcffuXaSlGXanlMqv2aBeSmF3ZYfcAHBz7y+KDwNNX30FUrEYiX/8ibSTZ2Dv7gbPV/6ntY3SIbfPsBDc+/cS7l/+D7HLVqHbinBY162jtt6lTWsAAJ0+Uf4wUJpn39eReTEWDxKuIOlgNJr1e/bI1ZPsLDx5cB8A0PnTpRBaqj5GJrS0RODsRTg0bTRyb9/Q+l6kUimSTuxD+sWjsHVyh89r42Bpo9+yIcWPC3D1l/V4lJUC1/bd0bjLAAgEArwzzQcNnZuhTcuXMHTgHMUHjDMXfsefR7bhbtp1PMjNgFQqUbrBUPAoB2Gf94RAIES9Oq5wd22O//UYhcAX/g+A7IPI93sX4cq1k0jLvIWCR7I1yoVCESQSMcSSEpOG3PmFj8q1kUfp73UlD7uJiIiIiEg9dWE3gEoPuZ88yFGE3PXbtoZTuzZI+GE3ANlYs8/m9Vrb0GVcm3v7Bp48uA/ruvUV9VKOHwQANOzUDU37DlDbdu1GTfDChJm4ELEU/369UiXo1mVcq6/c5ARcj5aNhVr0DYWjh+YJZOqUHteKJSU4+c/POPnPz7C0qAVnJw94uPngncGfo35dNwBAVnYKtu6eh7upCcjMSkZxifIsf6FQhK+2TcVX2wA7W0edx7X9ZiUBMM2a3OXdw4lht2ZmsXRJREQEPD09YW1tjcDAQJw9e1Zj2aioKPj7+6NOnTqws7ODr68vtm83v83rSL3Fixdj165d2Lt3r2IGt6WlJaKionDt2jXUq1cPtra2OHToEF555RWtd/Oo4jTt310Rdv+7elelh9wAcPvXPwAAPVYtQfNB/dHirYHo/qUsILy+e6/W+vkpqYrXQd9+BY/ePfFi2CR4D3kLAPDfFvX/djx5kK147di0ucb2BQIB2ox8HwBwbY/yUhbpsacBAK4BXdWG3HK1HOvAM7i/lncik3RiH2K//Rgp//yF69FbcXb9NJ3qlXbmq2m4Hr0VKf/8hdhvP0bSiX0AZDOt791PRq8ubyv9f/di296wtLRGdk4apFLZo3ClHwuTv5ZKJbj/IBVWVjZ4sW1vxXmhUIheXUJw736yIuQGAIlErHhtypnchj7uZYxlTIiIiIiISLPSy5jELtqC2EVbKn1N7oTvIwEAPiOG4MWwSWjc52UEffuV4vzDuyla29B1XJv+z2nlen/sAwC4d+lVZvsufp0Ur5/kPBvL6jqu1cej7HQcWTQMyad/Q/Lp33Bk0XA8yk7Xq43S49rSiksKkZpxE75teilCbgBwqueOF9r0QmrGTZWQG1AeW+ozrgVMt/Hk88tachkT4zF5iijf2HDevHk4f/48OnTogODgYGRmZqotX69ePXz88cc4deoULl68iNDQUISGhuLAgQOV3POaycnJCSKRCBkZGUrHMzIy4OrqWmbd5cuXY/Hixfjzzz/Rvn17pXN+fn6Ii4tDTk4O0tLSEB0djfv376NZs2YaWqPK0LR/dzi94I2sCwkQPy6s1JBbIn72x8rKwUHxupajo9oy6uQl3gEAeA95S2kWsntP2RIh9y//p7ZecX6e4rW25VGEIvUPxhQ9lLVh5+qm9nxplva1tZYBgPSLRwGBEJBKAKkEGZdPQCop+2dQmlQiRuaVE4r6EAhlbQKo6+iC+TP3w91V+QOQpWUtTH9/IwJf+L8yfxYCgQCBL/bD9Pc3qixh4u7aAvNn7kddRxeIhKo/L1OF3PmF+eX6UFDesJuIiIiIiMrm2Mwd7ScNRn5yBvKTM9B+0uBK3XgyMzYOANCwc4DimEAggM+wEADAwztJZdbXZ1xb/PDZxCBAtlkkAFjXqQ9dFec/LPVa93GtrrJvxqHkySNIJWJIJWKUPClA9s04vdpQGteW6p9AIMCEd1ajd9ehKnV6dx2GCaNWKcqpIxJa6DWuNVXILcewu2KYPOjWd2PDnj17YuDAgWjVqhW8vLwwZcoUtG/fHsePH1dbnozLysoKfn5+io0kASg2luzcubPGekuXLsX8+fMRHR0Nf39/jeUcHR3RoEEDXL9+Hf/88w9ef/11o/af9JN7KwUPriYqvk/+60ylXVtQ6u5r6U01Ss8mFmiZ8S//IJF16YrS8UdpsjvONs4N1NYTWum+xXlhjuzDx/M7XIuetmFRy1qljkobpe60l8XWqdQHOoEQNnVdIBCKdOwpIBCKYFPHRfah4ik7p0YAgAUzf0FD56Zq61laWGHau1+jY4e+Gtvu2KEvpo3dAAuR+tnrDZ2bYv7M/bC1dVA5Z6qQe+yPI8r9oYBhNxERERFRxSl5XIhbUYcV39+KOqx2g8qKYmkvWyrycWaW0vF7cRcBALXq1Cmzvj7jWk3jUJGNjc79FVo+a0Ofca2ubOur3mRQd6zMNpxUy0ulUkx4ZzV6dHpLY70enQdjwjurNW48aWfnqNe41pQhtxzDbuMzadAt39hQvlYzoN/GhlKpFDExMUhISED37t3VliksLEReXp7SF5VPWFgYNm7ciG3btiE+Ph7jxo1DQUEBQkNDAQAjR45U2qxyyZIl+PTTT7F582Z4enoiPT0d6enpyM/PV5TZvXs3Dh8+jFu3buHnn39Gnz59MGDAAPzvf9rXYKaKUXpN7h7rZymWMbm9/2ilXF8gEMCxuWxG/5n5i1GYk4ui/HycnrsAAGDv7qb1rnT9Nq0AyGZuJx86ipInT/AwKRmnP1sEAGj9jvq1lW2cnq1B/yhL/dMlcrmJsrW13V7qoXS8nk9bAEDC7u/U7n5dWtLBP8o8L+fz2ji4tH0JEAhgU9cZnSau0aleaYETV8OmrjMgEMKl7Uvw6f8BAMDZyaPMeiKRBRo1bAmRmiBbJLKEh5s3RBpmt8u5ODWGdS391hRXx1gh9/Wsa0b5UMCwm4iIiIjI+EpvPOk/dyz8545Ffkom4lbsrLSwu8ME2VKVpz9biIdJyRAXFyPpr4O4fyUeAFCvVdnrU+szrq3n006pboMOskmC6WfLnthZlPdsJrhtA2fFa33Gtbqq16wd2oXMgNDCEkILS7QLmYF6zdppr1hK6XFtaZ6N2mit28S9tcZz1rXs9BrXmjrklmPYbVwmDbrL2tgwPV3zGj+5ubmwt7eHlZUV+vXrh7Vr16JPnz5qy4aHh8PR0VHx5eFR9i89aRcSEoLly5dj7ty58PX1RVxcHKKjoxX/HZOSkpQ2kVy/fj2Kiorw5ptvomHDhoqv5cuXK8qkpaVhxIgR8PHxweTJkzFixAj88MMPlf7e6Jnn1+QuvWZ3ZYXdL06bCAB4eCcZR8Nm4cjkGYp1tzvO+bCsqgBkd8b9ZkwFAFzd/gMOjZ+mCLldAvxQz6el2npCkQhNXxkAADj1xQyIC9V/iHp49w6u7pJtwvH85iB1vJ594Lm+V/Pv8p2Y37W+DzlLGzt0CfsGAzddxCsrDqJu07Y615Wr16wdXllxEAM3/YsuYd/AwloWPBcVP9FaNzn1KiSSEgCAQCCE4OnMcImkBEk6/DEuLHqMrOwURX0AapcyKYsxQ+4tITsBGOdDAcNuIiIiIiLjKR1yy9fkLr1md2WF3XW9W8AlwA8AcPqzRTj4/mTFZpR+M6dpfcoY0H1cW8dLeXzqPXgUACDhx++Qc+u62rZLCp/g7PLPAABN+ryq9MSvruNafbXoG4rXv76A17++gBZ9Q/WuX3pcW3o8qMuYMrlUGXld4dP3fO/+Xb3GteYQcssx7DYeky9dYojatWsjLi4O586dw8KFCxEWFobDhw+rLTt79mzk5uYqvpKTkyu3s9XUxIkTcefOHRQWFuLMmTMIDAxUnDt8+DC2bt2q+D4xMRFSqVTl67PPPlOUmTx5MpKTk1FUVIQ7d+5g/vz5sKqAx2xId+o2nqzssNvCxga9v16DliFvQCCS/XPV/M0B6P31Gljo+PhWvVbe6LYiHM7+LwIALO3t4f9RGNp/MLbMevLgujj/IU4tnIXH9+8pnc+IPY0Tc2WbQTbo4K+yFrdAIMCLk+cAAG79tge3/9iHkiePFeclxcW48t3XiN+5CQAUZXWhz3IluraRkn5Da53byZchlUpRy8oGb706HW+9Oh21rGwglUqRmHxZa/3U9BuKjT9cnZti/Mgv4e8brHOfjR1yl/5Qw7CbiIiIiMg8qAu55UwRdrf/YCz8Zk5TLH1Zv21rdFsRrnHi1PN0Gde+OHmOyhPL9g3d0TBQtr/U6QUfIfvaf0p7ND3KysSRGe8j/65sbyqv11SX/dBlXGsIgVCoU8hfdhsipfFgcor2cLd0GN7Rty/Gj/wSLg08AQBSqUSvca2pQu6vTqh/Mptht3HoN5XOyAzd2FAoFKJ5c9nC8r6+voiPj0d4eDh69uypUrZWrVqoVatyNs8jqk40bTzZtL9smaBbUYeUvjeWkseFsCp1WaGlJZoEB6FJcJDmSlpY162DDuPf1auOVW0HdF24Fsc/noT8u3dwZMb7ass5tfXFCxNmqD3n7OuPtqETcHlLBBJ2f4eE3d+pLdc2dAKcfTWvXV8ZklOuoqmH5hniTwof4WF+Nl7rMw4D+05Ebft6AIDgnqOwLzoCfx7ZisKiR6hlZauxjaTUq2hQ3wMhr32IroFvQCQUoVeXt4HbxVr7V5Eht5z8Q8HQXTMxPHIWdoQshn0tze9HnSldZMvhLDu6Rel7IiIiIiLSrqyQW04edl9Yth1xK3ZqHLsaUz2flui6+AuD62sb12oaD7YbMxFPHtzHg2v/4eziTzS233XhWljZ11Y5rsu4tumWd3R7ExXgw/c34eadf/HDvnAkpSZoLZ+ckoAOrXtiyIBZ8GrSAQDQvfNbOH5mDyJ/Wa7XuFZfxgq5Vx9bjhldX1F7Xh52D4+chaG7ZuL7t5fiBTcfva5hjHFtVWbSGd2Gbmz4PIlEgkIjPYJBRDJlfVCoqJnd8g815sK+oTteXrMNnn3Vb4ra4f0w+E39RGUjytIadeuNlz5bgbotVdcSq9PcGy99tgKNuvU2Wp8N1dKr7KBdLCnB6i9OYOSbcxUhNwA42NfHyDfnYvUXJ1AiLin7Gs38sfqL4+jReTBEesxKr4yQW44zu4mIiIiITEdbyC1nipndpiC0sETAzC/Q/t0pgEA1wmv6ykC8vGYb7Btq/llpG9eamleTDvhkyi4MG6j9Kedhgz7GJ1N+UITcACASitCj82Cs/vyYXuNafRgz5J7SrexlWDmzu3xMOqMbkG1sOGrUKPj7+yMgIACrVq1S2djQ3d0d4eHhAGRrbvv7+8PLywuFhYX4/fffsX37dqxfv96Ub4OoxjH2zO7Sd+7NiZV9bfgMHoWWbwzH46xMSIqLYGlfG9Z16mmv/JRD46YInLUAJY8f4Un2fQCAdb36sLAxn7uqrk8f99LEzsYBdjYOGs/Xr9tQ6zU07X5dlsoMueU4s5uIiIiIyDR0CbnlTDGz2xQEQiHcOveAW+ceeJKTjeL8hxBaWsHGyRlCkW4TiIwxrq1oHm5lb+yprYylZa1yj2vVMXbILRvXql9zXY4zuw1n8qA7JCQE9+7dw9y5c5Geng5fX1+VjQ2Fpdb9KSgowPjx43H37l3Y2NjAx8cHO3bsQEhIiKneAlVzmzdcMtm1c7T8zZKUyNY7Pv1NPJLOGhYQp8bJgtdjqy5CaPHs/7VWU7UH18YKu59/PM0cCUUi2LloD3PLYmFjC3v36v+HxVhMEXLLMewmIiIiIlMy53Ho8+RjyqNfXjToeqXHtf5zx+oUcsvVlLBbzrpOvXKF08YY19YkFRNy66aiwu7qziw2o9RnY8MFCxbg+vXrePz4MbKzs3Hy5EmG3EQmVN5lTHRZg41qHlOG3HJcxoSIiIiIqHIZMh40xjImxlySk6oHU4bcchWxjEl1Z/IZ3UTmbvQH7Ux27X/sBWWeFxdLsHvsEXR6rxU8u2jewLUsiSfScfqbeHSb2h4iy2f3vu5JdW/D0Jnd5hhy56ckG7ysyJMH2SjOz4PQygo2Ti46P0Zmjgoe58HWurbKzt+6kkqlePTkod6PhQHmEXLLcWY3EREREZmCOY9Dnyefyd19WnuDrld6XGuo8szsvr3/KG5FHYLXq4MNvr4xSMRiPM7KgKSoCJb2DrCuq//MbXNeLlNfRcVPAABWltYGt2HouNYcQm45Y8/sru7MYkY3GSYiIgKenp6wtrZGYGAgzp49q7Hsxo0b0a1bN9StWxd169ZFUFCQ2vLx8fHo378/HB0dYWdnh44dOyIpKaki3wZVE/rO7DbHkBsAjn86BX9PGI6Tn89AXtJtreWlEglSTx9D9OhBODx9LE7MC8Ox2RPx57tvIf77b1H0ME9rG0UP85Cw+ztjdN8o0u8l4p2p3li7eaLBbazZPAHvTPVGxr07etUzp5BbjjO7iYiIiIjMnyEzu+Uhd7NBvSqhh5ol7P4Of777Fo7NnogT88JwePpYRI8ehJSThyGVSLTWz0u6jVMLZ+HvCcNx/NMpeo9rzdGwiU0xbKL++zzJGTquNaeQW86YM7urOwbdVVRkZCTCwsIwb948nD9/Hh06dEBwcDAyM9Wv03z48GEMGTIEhw4dwqlTp+Dh4YH//e9/SElJUZS5efMmunbtCh8fHxw+fBgXL17Ep59+Cmtrw++eUc2ia9htriF3aXl3buLkZ9Nx91iMxjKSkmLErlqAi998qfb8nb9/w8Ep7yA/LUXteQDIT7uLg1Pewe0/9pW3y0ZzOvZXAMDF+KOQSvWY2v+UVCrFxXjZf//T53/VuZ45htxyDLuJiIiIiMyfPmF36ZDb0P2mjEXTePDSpjU4u3QuJCXFGuvePRaDk59NR+7NayrndBnXmqPUjJuK12kZtwxqw5BxbWWF3IaMB40Vdld3XLqkilq5ciXeffddhIaGAgA2bNiA3377DZs3b8asWapr7uzcuVPp+02bNmHPnj2IiYnByJEjAQAff/wx/u///g9Llz67w+Pl5aV4/e+i/1Npt6CwBOO3nsPNjHysD+2Idh51jPH2VFXdp21qHG3LmFREyP0g4Tr+jfgaxfkFcPbzhffQEFjXraNzfUlxMe78eRA39uxD381RkJQU487fvyPhx224vCUCVrUd4ezrr1LvwrqlyLocBwB4YcJMuPh1UpwrysvF2eWfIf/uHRz/eBJeXrMNVva1leoXPczD8Y9lf/gs7fVf4qMiPCl8hH0HIgAAuQ+zcOHyQbzYrrdebZy/HIO8h7LH5fZGr0Nwz1BYa1nuw5xDbjljL2OyFJuN3kciIiKi6iwiIgLLli1Deno6OnTogLVr1yIgIEBt2Y0bN+K7777D5cuXAQB+fn5YtGiRSvn4+Hh89NFHOHLkCEpKStC6dWvs2bMHjRs3rvD3QxVDl2VM1IXcf40eB4FQiOZvDkDj3j0htLTU+ZpPHuTgv83f4f6VeNg4N0DrUcNQr5W3zvUt7R3Qee5S2Do5A5BNHsq9fQOnF3yEB9f+w6Vv16HD+9NU6mXGncPlLbLxm0evYLR6e7Si37qOa81R1B9rIBSKFK8nvLNKr/qGjGsrcya3octaGmMZk+qOM7qroKKiIsTGxiIoKEhxTCgUIigoCKdOndKpjUePHqG4uBj16snWfJJIJPjtt9/QsmVLBAcHw9nZGYGBgdi3b1+Z7djVssBX73SEl4s9xm05h0vJOYa+LapGNM3s1jXkvrhhk87Xurh+E/5ZshLF+QUAgMzYOBybPhvZ8Qk61S95/Bgx70/GjT37FMeEFpZo2vd1NOs3CABwfs0ilTvA+WkpuHcxFgDQ5YsvlUJuALBycESnj8Nh+TTcVneH/uYvuwEA9o2aoOeyrxXHpRKxTn3XRCqR6PR4W2lXEk7i72M7ELF1Ch49zgUACIUibN09D7/+/TViL/6FlPQbKC4pUqlbXFKElPQbiL34F379+2ts2/2Z4kPJo8e5iNg6BX8f24ErCSdx/0Gays+yKoTccsac2U1EREREuuNTzaSPsmZ2lzWTWyqR4PqPUYh5fzJKHj/W6VrZ8Qk4Nn027l+JBwA8zryH2GWr9BrX9lz2tSLkBgCBQIA6zVqg0ydLAABpZ46hID1Vua9SKc6vCQcAtBo2Fm1GvK8Uzus6rlWnvGNSfdsoeJyHm4lxOH52L378ZTmOnv4JEokYEokYR07vxo+/LMfxs3tx886/ePT4oeq1pFLcf5Bm8Li2spcrKc+TvsaY2V2dcUZ3FZSVlQWxWAwXFxel4y4uLrh6Vbdf8I8++ghubm6KsDwzMxP5+flYvHgxFixYgCVLliA6OhqDBg3CoUOH0KNHD41tycPu8VvPYdyWcxU7s5uqjOdndnv0CdR5JnfG2Vg86NUDdb1blHmN7PgEZJyThc2dPvsYNs5OSDt1Fle3/4DYZasQtCkCAmHZ9/POLloGALB3d4PfR8p3yFsMHIpbv0UBAHJuXkPd5s/uyMuDa58ho1G7URO1bVvUskbnuctwdOYHuP3HXrR8YygETwNgiViMO3//BgAI+PAziGrVQvatSzi9bjKe5GTCuU0XdJrwJSys7crs//Ou/bEZ/0WtBgC0eWMqWvTVLVD9bOUbAACRyFLx4UciESPz3h1899MXkEplwXn9um5YMHM/nOrJ/vtlZafg4yWvITsnDQAgEAghFAghefqhRiqV4ty/B5SWMOns1x9Tx66H8Ol/G1OG3BdSr5ZrI4/yzuwmIiIiIt2Y4qlmqtrUzexO/uuMxpC7x5plkIrFuLAqAg/vJOP8l+sQMGdGmdeQSiSIXbYKAOA9dDDce3TFo7QMnP5soV7j2gat1C+dUqdZC3i/NRIJu7/D1R+3wm/yHMW5nJvPJnc16a36BL6ctnGt85hnM52LHxfgzFfTkHnlBGzquCBw4mrUa6bfxqj6jmtPnPsZqzZ9oPheJFSNKqN+Xw2xpASA7CbA1LEb8JJ/fwCyiZurNo3Dqdj9z9rQYVx78v3TAEyzJnfpJ31Lf68rzuzWjDO6a6DFixdj165d2Lt3r+JOteTp7M/XX38d06ZNg6+vL2bNmoVXX30VGzZs0NomZ3aTOqVndh8Zt1iv5Ur+jfhGa5n/tsk+vHb67GPUbtwIFtbW8OjVHfXbtgYAxR11TaRSKQpSZAFtpy8+gZW98h80gVAI77dkH4Kzr15SOpd6+ggAwNGz7A/Cpe/KP7r3bLbJ46wMxWsrB0cAwOl1k1GYmwVIpci8chJX92v/f6+07FsXcfnH5ZCUFENSUoxLkcuQfeuS9oqliMXKa7+JJSWQSiUQCkVwdGiAudN+VITcAOBUzx1zp/0Ix9pOEApFkEolig8g6trs2CEYk0avVYTcAEw6k7u8G3mUZ2Y3EREREenGnJ5qpqql9MzuI+MWl7kmt5W9PWo5OiLwU9mNk9wbt7Su7Sx/krh+29ZoHNQLIktL1G7cCJ0+kwXS+oxrNXEN7AoAuBf3j/K1r8qW5WlcRsgN6Daulbv6y3pkXjkJSKV4nJOJMxFTtPb/efqOa7t0fB3jRq6EQCCAQCBQjEHl5GNM+flxI1YqQm5A9m/BpNA18O8QrDimy7gWMO3Gk+Xdw4kzu9Vj0F0FOTk5QSQSISMjQ+l4RkYGXF1dy6y7fPlyLF68GH/++Sfat2+v1KaFhQVat26tVL5Vq1ZISkrSqV8Mu0kdjz6Bitd1fTx1XpO7OD9fa5nHmfcAALYNlZ9ukAfdRXl5ZdYvvbyHQCBQW6ak8AkAQFKk/GiTtEQW5tZyrKu1n3KS4mdtPN8eADzJyVR6vKsg667ObQPAo/upao5p3giztE4vvqrxZyASWsDRoQEWzNwPNxfVYN/dtTkWfPQLHB0aqL37Dsh+vp39XkPY+xthaWGldM5UIffF1Dij7FrNsJuIiIioYpX1VHN6erpObZT1VHPfvn3x559/YuDAgRg0aBCOHDmipTWqShybuaOuj6fi+9JjVHVKPxWsbUnIwgc5AIAGHZRnPds4NwCg37hWk5LH6scZ4qdjylp1tI9JtY1r5R5llRo/SiV4/CBDryVIpBKxQePal7sMwcR31gBQPzaXH5v4zhr06vK2ynlLy1qY/v5Gvca1pgy55Rh2Gx+D7irIysoKfn5+iIl5tmuuRCJBTEwMOnfurLHe0qVLMX/+fERHR8PfX3kDAisrK3Ts2BEJCcrrGl+7dg1NmqhflkEdht1UmnxNbpFNLTi94I2sCwlKa3aXxdn/Ra1l6rdpBQBIOXJccUwqleLarp8AAA6enmXWF4pEiteFublqyxSky/7QW9Z2VHteIi5Re7x0f+QsS21GqW7zSec2XQCBUPYllcC1vX47f9fz8oWFtR0EQhEEQhEsrO1Qz8tXp7pTx65HQ+dmas+JJSX4eNJOuDbw1FjftYEnPp60U2U2t5ybixemjPkKFiLdN3TRxFghd2jkMKPsWs2wm4iIiGqSiIgIeHp6wtraGoGBgTh79qzGshs3bkS3bt1Qt25d1K1bF0FBQWrLx8fHo3///nB0dISdnR06duyo84QrXVTEU81UddzefxRZFxLg9II3RDa1VNbsfl7pCVOlx4zq1PaUbVp6dWek0tgv7dQZAPqNazUpfJANAKhVp57ScavasjFlcb7qmtXP0zaulXNt3x2QShTjUpe2XRTLb+pCIBQZPK7t3ulNDOyrOSwe9MoUdO/0psbzFiJLvca1pg655Rh2GxeD7ioqLCwMGzduxLZt2xAfH49x48ahoKBAsV7ZyJEjMXv2bEX5JUuW4NNPP8XmzZvh6emJ9PR0pKenI7/U3cUZM2YgMjISGzduxI0bN7Bu3Tr88ssvGD9+vF59Y9hNgOrGkx2mvK12g0pNvIe8pbVM69Gyx68Svv8R51esQXLMYfw95tnvq717Q61ttHhrIADg6LRZuPZjlNI5SXEx0s+eAAC4+itvNtnijWEAgP+2f1Pm42y5t64rXluX+mBiXffZ64xY2dpgnSZ8iZZ9Q+Hu3wd+YxaicZcBWvtfmm09V/SYswMenfrBo1M/9JizA7b1yn7KQ04kssDD/GyN5x890f7h6dFjzTPo8/KzIRKVf1sIY4bcLZxaGuVDAcNuIiIiqilMtSGkuT7VTOav9MaTHaa8rXGDSgC49mMUbkT9jCNTPwIANH2t7CVBAKB2o2dPLP89Zjzu/BmD8yvX4ur2XQD0G9dqknJCtu9V076vKx139ZONURMP7Edhbo7G+rqMa+UadxkAvzEL4e7fBy37hiJw/Jda+/+88oxrHz/Jh1DNU8JCoYVOY1J9xrXmEHLLMew2HgbdVVRISAiWL1+OuXPnwtfXF3FxcYiOjlY8ypWUlIS0tDRF+fXr16OoqAhvvvkmGjZsqPhavny5oszAgQOxYcMGLF26FO3atVNs1tG1a1e9+8ewu2Z7PuSWL1dSes3ussJuvxlTYV23jtbrWNetA78ZUwHI1uO+ujNSca7n2uUaailr0rcPGr4ke3TtTvRfiuPFjwpwav5M2XXqOcG6bn2leo179QUgWxctMfpntW3n3LqO0wtl67u1f1d1bbP278k2v7wQsRQP796BhbUd2g6ejsDxX6JJ14EaH7kqi6OHN/zfXQz/dxfD0cNbe4Wnch9m4WHBAwBPN5R8etdeIJD9mUhO0f6HMik1QamOUChSvH6Yn428/Pu6vxE1jB1ybxq8HYBxPhQw7CYiIqKaoPSGkK1bt8aGDRtga2uLzZs3qy2/c+dOjB8/Hr6+vvDx8cGmTZsUTyPLld4Q8oUXXoCXlxf69+8PZ+dne92Y81PNZL5Kh9zyNblLr9n9fNh9J/ov3P41GgDg1vUleA14VafrlB57Xtv1E+5f/g+AfuPah3fvqByXSqW4/cc+pJ05BgBo1K23cr16TnBs2hwAcCZ8DiTFxSpt6DqulRMIBGjSdSACx3+JtoOnl7mJpCblGdfeSfkPYnExREILxZrcIqEFxOJiJN0tew8uQL9xralC7vxC9cvZMOw2jvJPrwOQl5eHgwcPwtvbG61alf3IBRnPxIkTMXHiRLXnDh8+rPR9YmKiTm2OHj0ao0ePLmfPZORh9/it5zBuyzmsD+2Idh51jNI2mS9NIbec/APGrahDSt+XVq+V7gFtvVbeCNoUgez4BBTm5MChSWPYN9JtHXBA9oe87dh34D3kLdy/Eo8n2VkoyEjDuWXzFGUCZy9UqWdpa4f2703FxW9WIWH3d8i5dQ0ePYNh11B27fQzx5Gw+zsAQN2WrdEwsJtKGw0DuiD15CFkXY7DibnTMGjLfzr329jupl4DAFhZWuO1/43Da0Hv42L8UeyMWoiMrDuKELss8g8Nzk6NMWzgHLRv1R2//LUBv/y1AUXFT5CcmoA2LV8yqH8VEXKX/lBjjF2r5WH30F0zMTxyFnaELIZ9LVu9+0lERERkjuQbQpZ+cthYG0LOnDkTwcHBuHDhApo2bYrZs2djwIABSnXDwsIwatQo+Pv7IyAgAKtWrVJ5qtnd3R3h4eEAZE81z507F99//73iqWYAsLe3h/3TTehnzJiBkJAQdO/eHb169UJ0dDR++eUXlfE0VT3qQm45edh9Ydl2xK3YCd/pw2BhUwvtPhgDqUQCp3ZtYGmne8BraWeHPpvXIz8lDXmJibBycED9Nq2U1vrW5sTcaWgxaCgatPeDlX1tlBQ+wZXvvsaDhCsAgPbvTYWFjerYwnf8DByZ8T4eZabjrwnD0C50Aup4eUNoYaHzuNacJD0dU77YrjeGDJD9W/PDvnCc+/cAklK0B936jGtNFXKP/XEEfhm5VO35KV2GAwCWHd2i9L2ujDGureoMCroHDx6M7t27Y+LEiXj8+DH8/f2RmJgIqVSKXbt24Y033jB2P6mKYthds2gLueV0Cbv1IRAKta5rpo2lnR1cA/zx1+j3FMeEFpboumA1bOo3UFvHrVN3QCrFxY2rkRF7WrEESWkNO3VDu9ET1X7IEQiFeHHybFyIWIZ7//6jcr4ypd+7jX6938XAvpPh6OAEAOjs9xoCfF/B4VM/4r/r2gcvj548xLiRK9Gj01uKZUrefv0jvNJrDKL+WI30zESDgu6KDrnlGHYTERERaVbWhpBXr+o2c7CsDSEXLFiAJUuWIDo6GoMGDcKhQ4fQo0cPRd2QkBDcu3cPc+fORXp6Onx9fVWeahaW+sxd+qnm0ubNm4fPPvsMwLOnmsPDwzF58mR4e3sb/FQzmY+yQm45dWG3a4C/2rK6sndvqNPymZpcj/oe16O+Vzne/r2psrGnGjb1G6BbeASOzZ4AaUkJLm5crVJG27jWXOTmZaGpR1sMGTAbLZq+oDg+c/xWXL99Ht/vDUduXpZivKqOMca1mhgr5L6eda3Mcgy7y8egoPvo0aP4+OOPAQB79+6FVCpFTk4Otm3bhgULFjDoJiUMu2sGXUNuOWOH3cZk5VAH7UZPhFNbX6134d0690CDDv5IPXUE1/f+gJJHBQCABr7+8Bn8Duxc3cqsL7SwhN+UOchPSymzXEXr+dLbEKnZZEQkskDvrkPR86UQrW1MeGe12jYcHZwQGjIfYj1265arrJBbjmE3ERERUcWQbwh5+PBhjRtCAoCvry9OnjyJDRs2KAXdgPk/1Uymp0vILfd82B0wZ04l9VJV14VrceevX5F8+IDiWMs3hqPxy33VzuQuzc6lIYI3/YSsK3G4uX83cm7KZi3rM641B/b2dTF32o9qz7Vo+iLmhe3WOqY0xrhWHWOG3FtCdmotz7DbcAYF3bm5uYpHjaKjo/HGG2/A1tYW/fr1w4wZM4zaQaoeqnLYXVIsMdm1xcVlr2UlKZH1TVwigdjAfoqftiFvS0GPfx30DbnlzDHs7rs5Snuh51ja2qFJ7/9Dk97aNyvRxL6h7sutVAR1Hwb0OW+sNkqr7JBbjmE3ERERkSpjbAj5999/67wh5PHjx43X+SrInMehz1OMSw3sc+lxbXnoE3LLlQ67Tcm+oTvajHwfbUa+b1B9gVCIBu1eRIN2Lxq5Z5WnMsac+o5JAeOH3LJx7XWt9Rh2G8agoNvDwwOnTp1CvXr1EB0djV27ZLvJPnjwQGlnZKLSqmrYPWuq5k0TzcW5zQk4t1n7Gspl2fPBMaXve2/tqVM9Q0NuOXMMu8n0TBVyyzHsJiIiIlJWekNI+frZ8g0hNc2yBmQbQi5cuBAHDhzghpB6qArj0OftHnukXPXPbU5AbwOHg4aE3HLysJvoeRUTcuuuIsJud7ysVxtVjUFB99SpUzFs2DDY29ujcePG6NmzJwDZkibt2rUzZv+omqmKYfeQUaa743W7Vtl30sUlEpzbnADXdvXg2dmlzLKaJJ7KQPqlbHQc7Q2RhX6PM5U35JZ7Puz2enWwQe1Q9WDqkFuOYTcRERGRMm4IWXnMeRz6vNPfyDYJ7PSeYfsmlR7X3t5/VO+gujwht5xjM3fAdJPoyQyZOuSWM3bYnbd1vEH9qCoMCrrHjx+PgIAAJCcno0+fPooNH5o1a4YFCxYYtYNU/VS1sNsvoOzH8CqS1F5L0F0s+0Dg2dkFnl0M72f6pWx4vuQKkeWzoPuetOw6xgq55UqH3eYWdEulUuTcuIrshCsQFxXBqrYDXP06wbqe5k0wnicuLkJG7GkUpKcCAOxc3eDi1wmi/2fvzMNjuv4H/M5MJmSREJGIWJLY9yX2tbaiK9XSVmkpVUtrK0VbXS1FW7RoS+leqkX12x+K2msNKYLYgsgqQsiezMzvj3HHTDJ7JplJct7n8Twz99xz7pkgcz/vPefzUboX17SLleP//Y1MJiO8RV+7+ms0Gn75YwGP9XkF8NEddxXJLSFkt0AgEAgEAsEDREHIksOV49CCXD+aDGB3TKof1x7+2radvo6Q3KWF7NQUEiMOk3vvLgp3d/waNaNy3YbIZNb9fZmKaz2rFvPEzXA3/RY+3vZN4G76Lf6382ueHzjLwbNyHckt4UjZXdaxS3QDtG3blhYtWhATE0PdunVxc3Pj0UcfdeTcBGWY0ia7BYY4WnJLuOKNSXLkcU4sm1fo+Plf1lCxSlU6zJ5ntnq1WqXi0uZ1XPnrd6PttXr2o/Hzo5ErbM8V5iw0Gg0fr3gRgF+/jLf6xkqf85eOsGnrMv49voV9L2u3Zbqa5JYoLtktEAgEAoFAUBoRBSEFxUnYUz2tTmtZXiR31q2bRK5YRFrMJaPtbV6fTUCrtkbbJMzFtU+tPeuQedrD7AWP8cVHh+zrO/9RklKu0bpZLxrX6+CwOZWE5E7PybR58ZOjZHdZx66yq5mZmbz88st4enrStGlTrl+/DsBrr73GggVl/4cmcAyS7K4b6M24tcc4HXvH2VMS6JF2Jc7o8eKS3BKmblA0ajUaddH2kqlVKjQaC0vV9bixf5fBzUDDZ0ZQ94khVG/fBYDs27fYO30sGUkJJq93bNG7OsldtXELGg4ZQeiAQfjUCQMgdvd2ji16F7XKfPVoV+LE6Z0PXp/ZZdcYuw78DMCdu9rVH64quSWkm4KG/iE8v24GJ+PP2zyGJLujU66WiyfpAoFAIBAIBAKBrYQ+0V0nu2O2mM5T7gzJrdFoihy3mYprTY2bkZTA3uljdZI7pN8T1B88jNq9B+jOObFsHjf2m47LLMW1zuTO3WSysjNs7peVnaGLJaXY0hGU1EruF9bPJD0n0+bxJ3V5gendR7Jo31qWHvzR5v7lYWexXSu6Z82axX///ceePXvo37+/7nifPn147733mDlTBPDFRcIHX5TYtU7euMqz3y+nYUAQPw8fD16Ov4ZY2e26nFz0QyGRXdyS2xhpMdc4+qHhA7T2b83At26oVf01Gg3Xtu3g4oZNumNewUG0nz0dNw8Pk/3uXo/hzNrlAIQ99jT1Bz6LTG8bpvrl1zn04Qzu3bjGwXcm0/fLXwzaAc79vJrbF7RPx9tNf5+qjfVrGAwnPeEGB956ndsXznJp8zpaPf2hVZ/JmWg0GtZvWYj8frXq9VsW0qZZb5tWdSckx7DviFb+5+ZmubzklnD0ym6BQCAQCAQCgUBQmII1nAqK7JKW3PlZWZz49AvSLl95MMfHBlB30ONWx0GW4tpLm9fRYPAwg3aNWs3+WRMA8AwIosOsuVTwraxrbzJsDNd2/R/nflrNmbXL8akThk9twzjZmrjWmeTmZvH33u94sp9teaP/3vstuXnZAOw7/BtPPzqF6tVCrO5vbAFcSaYrkRY/2ZPWsqgru8s6donuzZs3s379ejp27Gjwn7pp06ZcvnzZYZMTOI+Cktu7QkVsf9ZkHUJ2uybewQEGstspkvtyDEfnLix0/OjchXSYMxOfEMuV2KO++Y6Ef48YHMuIS2D3hKn0/moZcqXSaL8za1cA0HDoS4T2e6JQu1yppP3Mj9g1cTjq/DxSzkRSrUUbXbsqN4fY3dsB6Dp3Gd5BNQuN4R1Uk3bT39et+rZGdGemJpJ6ORLPqsH4hdlX/Df1ymkyb8XhV7cVnn7aPHqrfp5JUEAoQYF1ad20ly634tXYKC5fiyQ+6QoJyVeIjY8mMTlGN1bM9dNMmtOVmjUaEBQQRo3AMOrWaUVIraYAqNVqTkb9Q0LSZRKSY4hPvMy1G1HIZXJUGjUajaZUSG4JR8pugUAgEAgEAoFAYBxTsrukJbc6L4/dE6YWOh7zv63k3Emj6ajhFsewJq698tfv1H3iGYMaTilnTgIgV7rT9cMlRmPXOr0fIT0ultg92zn749d0nD3foN2auNZWVHm5JEcdBCCgaRe76k5Jca1Uu+lQxBaCg+prY9KAMJo17Iqvj7YmVtrdFM5EHyAh+QoJyTHEJVzk6o2zOlktl8mZPf8R6tRsSo3qda2Oa/eO2qubT0nn5C5qDSchu01jl+i+efMmAQEBhY5nZGTYlatV4FoYk9zFjZDdrkeracOI/OQnTi76gRavDeHKxj0lKrkB3c1Ai/FjCGyrlchJx09wasUqjnywgL5rVprtn5eRoZPc3T9bQAVfXzQaDYfnfER6XDzXd+4mZMDDhfrlZ2Vy95r2oV0dvS1hBVF6eulE9ek1n9NryVpdW9IJ7XV96oQZldwSVRs3x69xc1LPnTb7WQDSYqPZO28Y+dnax07Nh06nfv+RFvvpc2HrGs78uhgAt4pe9Jj9I761GvL33u+QyWS8MmyhQQEhNzcl6/5YyJ27ychkMqNPvhOSr5B4MwaNRkNlnwDem/abrk0ul5N6O4HvNrxnck7OlNxLD/5Y5KrV9spugUAgEAgEAoFAYJqCslt6XZLpSq7v2gNApTq16fDOm8jkcnLv3mXv5DeJP/AvDYY+hdLL/PZ3a+PapIgj1OjYTdfv0pZftf3GTDIrpBs/N4rYPdu5cyma/KxM3Dy00tTauNYWVHm57F84ktRLWgnvV6813d9ci9zNetldMK5VqfK4fO0/rlw/hUajIbzFw7Rv9WC+nh6V2H90ExGn/jYak6rU+dzLuM2Z6AOciT5gc1zrjMKTxmo4CdntGOwS3W3btuWvv/7itddeA9DJ7dWrV9OpUyfHzU5Q4jhDcksI2e1auHlUoNW0YUTMW8vJhT8A0HbO6BKT3Pp5y6SbgYKvNWp1oXQh+tyKOgdAyICHqeDrC2h/X4W/OYW9r0/n4oZNRkV3duot3Wu5m/kn3J4B2hXRuXfTDI5nJMYDULVJS7P9Aao1b22V6L64bS2q3Bzd+6jfl1Dv4RfN/gz00ajVnN24VPdelZvNxW1raTtmATKZjHEjPqNn56EGfWoGNeCDNzYxZ/Eg7qWnotLkGx1bLlNQqZIfH7yxiaDAMIO2vt2H46ZQsvIH7UqIgjcmzlzJ7Yiq1fbKboFAIBAIBAKBZf6b90iJX3PV7kss33mRXz/rXeLXFhhSUHaXdOHJS79tBqD15PG6uMvdx4fQx/oT879tpJyOIqhje5P9rY1rATKT4g36pl25CICHf+GFpvroS/Ds1Ft4B3vqXuvOsRDXWkty1EGd5AZIvXSSpDMHCWrV0+oxCsa1+rRvNYDJY75EqSfOlcoKTBu7iiWrXuXYf9tMjiu5SVviWmdIbgkhu4sHu4pRzps3j9mzZzNu3Djy8/NZunQpDz/8MGvXrmXu3LmOnqOghHCm5JYQBSoFjkSj0t4wyBQKw+P5qvvH7foVaEBBwV0Q90qVLI9x726R51FUXhv5eaGbAYmgwDA+mrEFX59qutzc+sjlCnx9qvHRm38WktwSPbs8y2sjPzfa5izJveLgsiIX8ihqgUqBQCAQCAQCgesxpmc9JvSp7+xpCFwASUIXjCnVUkxp5aKjouBeyafYr+FsZDIZncKfYMorXxlIbgmlmztTXvmKjm3M50W3Ja51puSWkGS3lLPbGQUqyxp2/Y/s2rUr//33H/n5+TRv3py///6bgIAADh06RHh4uKPnKCgBXEFySwjZ7RpIObmzUu7QesZwfOvV4uSiH0i7Elci19e/YUg6fsLoa0s3Ff4ttHmiY/63ldy7WpmsUas5uVRbjKP+04OM9qvoV1X3Wp2XZ/YaWbe0lZ596zYwOO5VvQYAKacjzfYHuBX1n8VzAOr3H4nC/cH/zaaDJ9t0YyWTy2k6eLLuvcK9oi71SbcOg832DaxWhwE9Rxlt02g0PNLrZQL9a5sdo1uHwdQObmz1fE3hKMm9dP9ih1StFrJbIBAIBAKBoPhwVjw4pmc9p1xXYIh+Tu6wp3pyZeNuYrbsK7Hr1xs8EICIjz/T7UzNSUvj2rYdAFRtaj6+sSWu9QysYdDX3Ue7KzkzOdHsNdT5D2JW/VjWlrjWWgKadsGvXmvde796rQls1sWmMQrGtaCNKZ8fNBs3hemV524KJcMGzTaaThOgdnATm+JaZ0tuCSG7HYvNqUvy8vIYO3Ys77zzDqtWrSqOOQlKGFeS3BIF05isWdDD2VMqdxQsPOkTGqzL2V1Sebo7zJnJkQ8WcGpF4d817d+aYbG/0suLGl07EX/gEHsnv1movVYv4/+u3Dw88a3bgLTLFzi3bg1Nh481ep46L49Tq5YBUO/xIQZtgeEdAbh17hS3zp2mamPjhSPT429w93qM0baC+NZqSJ+5fxapGGX9/iOp2qBtoWKUarXaIIeZMW4kXECG9um5tLJbrVYhl8mJTbhg8dpqtZr4xAcFi+VyBWq1yqb5O1Ryd3sDKPp2L5HGRCAQCAQCgaD4EGktyy+mCk8WLFBZnNR5uBeXft9Melw8O18eb9AW1LmDxfzcYH1cGxjewaCt+aiJRCyZy7FF79L7ix9Qehq/1rWd/weAb2g9XX5usD6utQWF0p3ub64l6Yy2GGVgsy425ecGw7j26IoHhT5vJFywuHjqRoG4Uz+mjE+6bFNc6wqSW0KkMXEcNotupVLJ77//zjvvvFMc8xGUMK4ouSX0Zbeg5ClYeFLK2V2SstsnpA7t35pRqEK1VJnaGpqMHE6FKlWI+fP/dMcq16tL6ykTzRb0aDp8LP++N43Y3dvxrlGLOr0NcwPmpN3hyPzZqPNyAfBv1sqgXaF0J+zRwVz563eOLXqXrh8tw7uGYVHKW2dPcWzxewDU6tnPqs/j6VcdT7/+Vp1rCr+w5oUk+c1bsQRWM/8zvXojCpVam8usRmBdQHuToFLnczU2yuJ1k29dJy8/R1f8o2WTHpy7eMTqeTtacmtvarR574TsFggEAoFAIDBNwgdfFNvY6TnZPP/DCqKTE1g3YgKta4YYtEs7fYXsLl+YktwFc3YXt+yWK5X0XP4px+YtJj3uQQ7t+kOeok6/PlaNYU1cW6tnPxRKQ2Hs36w1Mjc3NPn5HF3wNp3eWWgQw2rUai5u+pkrf20EoOmL4wpd25q41rOyVR9Dh9zN3aac3MaQ4tpTFd4hJzcLjUZNbNx5wpub/5lev7+DViaTU7GCJw3rtiMyajcymYy8vGyb4lpnSe5T8ZF0Ci780ELIbsdgVzHKgQMHsnnzZqZMmeLo+QhKEFsld0ZOPl4V7PonYzeS7I5DbflkgUMxJrKdIbt964bSd83KB7nRbMyBJpPJqDfoceoNehy1SoVMLjeb00vCp3YozUZO4Mza5Zz7aTXnflpN6CNP4V7Jh5w7qVzdvkV3bvcFK4zOq97AZ7l98Ry3L5zlwNuv4xUUTHDnnsjc3Lh5KkJXgLJKgyY0fn60TZ/L0cTGR5u9IVCpVcQnXaFa1Zo8N3AWXdoOBODg8c38snk+CUmXUalVKIzk8Na/BkCrpj15fuBsQmo1JT3jDiRbnl/xSG5DhOwWCAQCgUAgKFksSW7AYKevkN3lA1OSW6KkZbebhwedPnwHjUaDRq1GrjAd85jCUlxrLB6UyeV0/XAp+2dN4N6Na/w9dijV23fBN6QeaDREb/hed26zkRPwqR1aaAxr4tqn1p61+fM4ipXzj/PH38v5a+cqncQ2R2zceZTKijzWewxPPDweb6/KxMSe4ZfN8zl55h+b4lpbcZTkHrl+GOenbjbaLmR30bHLWtavX58PPviAgwcPEh4ejleBrRqvv277UxFByWLPSu7x3x5jxUvtnCK7IbdErynApMAubtkds2UfdR8bUui4I4p82HpDUrNbb9wr+XJi2Tzt3P5vo0G7b2h9Wo1/A4+q1Uxer9309zn382pid28nIyGOC78b5ssKe3Qw9QY+a9fNkiNxs1CFOz39NiOHvE/PLs8Z5E3r1v4pOoU/zu6Dv5CefhtfH3+TYyjd3Plw+h80qvegKrm3V2XAfL64kpDcEkJ2CwQCgUAgEJQM1khuKJzWUsjuso0lyS1R0rIbtIuoChaltHkME3GtqXjQKzCIHou+4si82WTfvkXi0YMkHj1ocE6b12cT0KqtyWtaimudibdXZYYNeotHeo3m6MmtFs9vWLcdw59+lyq+AbpjobWaMfu1nzh36Qg5FvJb68e1XLd+no6U3PX9G5g9T8juomGXsfzmm2+oXLkyERERREREGLTJZDIhul0ce9OVXE5Kd5rsFrgWxSW7pZsaY6LbWQS0aku/b37nzuVoUs+fQZ2bi7KSL9XbdqRilaoW+8sVCpoOH0ujZ0eSFHGEzCTtdjfPwBoEhncotD3NWbRqan7rma+PP327jzDa5qZQmmyz5RrGKEnJLSFkt0AgEAgEAkHxYq3klhCyu/xgjeSWcIbsdgYeVavx0CertKL7+GHy7qUhd3fHr1FzKtdtYNWOZXNxrStQxTeQfg+9ZPE8c+c0rtfBZJuEYVxrXYFOR0vu1UN+ABLMni9kt/3YZStjYqwrnCZwPYqSk3vlyHaMW3us3MnuiKPmKxwXJ1crmP/CUuVrtz1dPZRk/zXu9736byIKtwdPl706m+/naNmt/+Te1ZDJZFSp14gq9eyXlQqlOzU6dnPgrMo+zpDcEkJ2CwQCgUAgEBQPtkpuifImu105Di1IfOQtbb+D9s1ZP661VnJLlBfZDVCxSlVC+j5qd39HxLXljeKQ3NbGtUJ220eRTaVGowGw6gmSwLkUtfBk81qVy6Xs/uU7y3minE3i6VQST6cWaYxja6IN3ve2ILrBcbK70PY0kZK93ONMyS0hZLdAIBAIBAKBY7FXckuUJ9ldGuLQghz++lyR+ieeTqX3NNtFtSNkd9qVOKqEVLe5n6Ds4kzJLVEcsnshvWzqX9qw21J+//33LFq0iIsXLwLQoEEDpk+fzvDhwx02OYHjKKrkliiPsnvBEuc9EY7wNv8ASZ2v5vdX99NuVENCOtv3pXz130SOrYlm8JfdkOut6LZWmxdVdlubg01QfnAFyS0hZLdAIBAIBAKBYyiq5JYoL7LblePQguxfcgqAbpNb2HU9/bjWXooiu9OuxHFy0Q/0Wh5u9/UFZQtXkNwSjpbdC1lj1zxKC3YZyk8//ZR33nmHiRMn0qVLFwAOHDjAq6++SkpKClOmTHHoJAVFw1GSW6K8yW43ZdGLINqLQmndDYbCTY7CznlK6UrkBcfQWD+GvbLbFSX3tlFP4RtWn3pPDMG/WWurimDmZWZw/Z+tXNz4MwAyNzdqdOxB6ICBeAdZJ/3TE+LwdBEBmp5xh5FTG/P6y8vp1v4pu8bYf3Qjy76ZwNpPz90vOGkdriS5JYTsFggEAoFAICgajpLcEuVBdpeGOFRCWjBlb0yqu65b0frbI7slye0dHGDx3OIkPSGO6PXfcvOUtg5eBd8qhA4YSHDXXig9vSz216jVpJw5yek1X5B7Nw3A5rjW1Xhzbj9kMhkLZm+zq7+9ca0rSW4JR8ruso5d/9I///xzVq5cyccff8wTTzzBE088wcKFC1mxYgXLli1z9BwFRcDRkltCkt1SgcqMnHyHjCsovUiy2zs4gJOLfiDtSpzZ811RckukXblIxJK5bB/9NBlJ5otExB/ex66Jw3WSG0CTn0/cgV0ceOs1IpbMRZ1vusiFOj+PyC8/5cBbrzls/kXl9PkDAPy8aZ7dY/y0cS4AZ6IPWN3HFSW3xKQuLzC9+0gW7VvL0oM/2txfkt0N/UN4ft0MTsaXvq2oAoFAIBAIBPbgaMktIcnuuoHejFt7jNOxdxwyrqB0E/pEd8Ke6smVjbuJ2bLP7Ln6krvVtGElNEPjHHjrNZ3kBshJu835dWvZNXE48Yf2mu2bkZTA368+S8SSuTrJDbbFta5GVnY6V66f4vK1/8jKzrBrDHvi2pKS3PbEg5Lsjk65ygvrZ5Kek2nzGOVBdtsluhMSEujcuXAC386dO5OQULr+85RliktySwjZLSiItbLb0ZJbo1aTcjqK+IOHSI+z73dQXnoGCYeO0mnOIlqNn45c6Q7A/lkTyLp102if+EN7OfX1EgD8GjWj69xldF+wgg6z5tLouVEA3DwVwYll89GoCyce16hVHFv8PolHrZfBxY1Go2HT1qUA3LodT0Ky7cWHE5KukHpH+/ewcesyXS0Hc7iy5JZwtOwWCAQCgUAgKOsUl+SWELJbYAxrZHdBye3mUYGEQ0dJPHqcvAz7xOq9G3HEHzjErahzRuM/SzQcMoIei7+m64dLaTttDkEduwFwatVS4g8b/xxZt26yf9YENPlaH9Nu+vv0WPSVTXGtK/L33u+MvrYWe+LaklzJbe/iJ0fI7rKOXaK7Xr16/Prrr4WOr1+/nvr16xd5UoKiU9ySW0LIbkFBLMluayR36rloo8dNnbtz9AROfvYFUd98z6F3PmDHqHFW35xoNBqi1nzPntff4MyqtfiG1KV62070+eIHPAO0ec8jVywq1C8vM4NTq7RfnA2HjKDd9PfxDqqJZ0B1qtRvTEjfx+jywWcApJyJJPHYv4XGiD+8n9sXzgLQ6Z2Prf7MxUVGZhp7Dv1KTOwZAOQyBev++Jhbt+NRW3GjplaruXU7nl/++Bi5TAFAzPXT7Dn0KxmZaSb7lQbJLeFI2S0QCAQCgUBQliluyS0hZLfAGOZktzHJDXBm1VpOf/kNe157gzOrv7VqwQ5AXkYGO0aN4/Ccj4ha8z0nPlnGztETbIprO73zMaH9B+Lh5493cC38m7ai5StTaPjMCABOfb2E/KzCUvPIvNkAVKpZh95f/EDVxs3xqFrNprjWlcjLyyE2PppN277QHdu8/Qti46PJy8uxagx74tqSTldSlJ2+Qnabx67Eyu+//z5Dhw5l3759uhzdBw8eZNeuXUYFuKBkKSnJLVHecnYLLGMqZ7e1K7kjFi2h2yfzqVilstnrZN++Q8SiJQBUbdYE/+ZNif5lAwB7XnuDvmtWWpzr5U1/En/gEAChjw3QHZcrlXSYNY/dU0aRFnOJ7Nu3qFilqq497sA/AAR17EZo/4FGx65Usw6tJ8zg5PKF/PfVpwR16GrQfnq1NtVTx7c/xje0PnlZGZz/cyWZKXFUb9Gd2l0GIpPZlh8vLTaai9u0uaTr9x+Jby3rCrq8NKWRTkbL5QrUahUqdT7/Hv+Df4//gdKtAgH+tahVoxEvDXmfqlVqAJCSGse3G97lRnw0ySmx5OUb3nzI5QpWfDeZFd+Bl6cvQQFhNG3QmecHzUZ+P0+cMyV3ek5mkatW25uzWyAQCAQCgaCsUlKSW6I85OwW2I6xnN2mJDdo40GNSsXVrX+T8O8RvINrEDLgYYvX2fPaG7rXjYYN5eZ/p7l15qxNca1vqPG4LaT/kySfiuB2dBTX/9lG2KMPck1np6aQffsWAJ3eWYhcqSzU35q41rffgyKcGo2G6wc3k3hqH57+wTR6fBxKD8s5wvWxNa49cvL/+Hvvd9xIuMjttCQ0GrXB+RmZd5j6/kPIZHL8KlcnuHo9Hu7xIh1aPwJoF1z9vGkeURf+JSH5itVx7XcPrwCck5O7qDWcHJGzu6xi14ruwYMHc+TIEfz9/dm8eTObN2/G39+fo0ePMmjQIEfPUWADJS25JcTKbkFBCq7s/m/pOpvSlZxd+4PFc6J/Xg9Ao+HP0Wbqa9Tu24s+36zQtd+7YT5POEDM/7YC0GPJx9R76gmDtgq+lQnppz2WePywYb+tmwEI7tLT7PiB4R11r7PvpD54ffvBa9/QegAcWTGFi9u+Je74DiK+eYvrBzdbnL8+mamJ7J03jNjDfxF7+C/2znuBzNREq/rqr7hWq1WF2vPyc4hPukyrpj11khvA3y+Y1k17Ep90uZDkLjhWRmYaN2/F0rPLszrJDTh1JXdRcpsVdWW3QCAQCAQCQVmkpCW3hFjZLTCG/sru/5auMym5Aeo99QT1nxlE98+0i1Iubthkcfz0uHjd6z7frKBW74doM/U1Gj73DGBbXGsMmUxG0xFjAbjwu2HckRihjVGrt+9qVHJLWBPXSlw/uJmIb94i7vgOLm77lqMrp1icf0FsjWvbNOuNUlmR1DsJaDTaFdf6q+ml1xqNdgexu7sHbZr11rXL5XJ6dhnKzVuxNsW14LzCk46o4SRWdhvH7rKr4eHh/Pjjj0RERBAREcGPP/5I69atHTk3gY04S3JLCNktKIgku1VZOaScjMa/dUOrc3LfOnPW4jnJEZEABHVqrzsmk8loNGwoAPeuXTfbX6168MXn7uNj9ByldyUA8u4Zpt7ISbsNQMXKVQv1MUVe+j2913cN5gyQHHUQNGrtH5mcxFPmi6cUJPVyJPnZmWjUKjRqFfnZGaRejrSqbxXfQBRy4zsxZDIZMpmMCS8tpXfX5wu19+46jAkvLtGdZwyF3I0qvoF8OGMLwdXrGbQ5S3Kn56QXuZBHUWS3QCAQCAQCQVnDWZJbQshugTFCn+iOf+uGpJyMRpWVY1Ry61PB11f3Wj9mNMbdq9cAaPjcMwaxUPBD2vzatsS1ppArjMdpufe0MaVX9RpG2/WxFNdKJJ7aBzK5Li5NOnMQjRFhbAqNWmVzXKtUVmDa2FV0aP2I2ZXfMpmMDm0eZdrYVSiVhn9/wdXr8+GMLTbFtc6S3BJCdhcPdonu//u//2P79u2Fjm/fvp2tW7cWeVIC23G25JYQsltQkNgdR3Svb5+/arJAZUE8AqpZPEfprd1ClZWcYnD8ZuQpACpUrmy2v0xvVbGpYiE591dey93djbYrPDwszlNCKgRiajyPyoHam4r7ePnXtHpsAM+qwVYdM8aHM7bg6Wlc9ms0Gia8tJQeHZ8x2b9HpyFMeGmpyTx2Xl6+fDTjT4ICQgu1OUtyj/51uEOqVgvZLRAIBAKBQOB8yS0hZLegIGlX4rh9/qruvX6Magz9mEY/ZjSGtGAq5XSUwfHMBO3OWlviWlPk3NHKaLmb4aptxf2Y0s0K/2MprpXw9NeLH2VyPKoEIpMrLI7/oIvCrrhW6ebOlDFf0a5lf5PntGvZnymjv8RNYXz1elBAqE1xrTMlt4SQ3Y7HLtE9c+ZMVEaeamk0GmbOnFnkSQlsw1Ukt4SQ3QIJ/ZzcPVbONFmg0hhNXrKc97jlBO0WrsPvzeXe9VhUeXlc3/EPt6LOAeDX2Hx+aplMhm+9MACOfLiAnDuFCyZe/0f78M6vUXOD49VatgUg8egBs9fIvftgTM9qAbrXHv6ButeZKckAdJi4FI8qASCTE9isM42eeNXs2AXxC2tO86HTkbspkbspaT50On5hzS13BAL9a1OxgukbrJCaTS2OUSe4icm2ihW8CPCvZdVcLOEoyX0x5YJDbgqE7BYIBAKBQFDecRXJLSFkt0BCPyd3j5UzTRaoBMi5k0ZuejqH53wEgHdwDYs1k6o2bQxoV27H7t5HfnY2967Hcvi9eYBtca3Jz3D1EgA1OvcwOO7XqBkA0Ru+N7lwS8JSXCvR6PFxBDbrDDIZHlUC6DhxmcX5F8TeuFahcKNmUAMURkS2QqGkVo2GKEysbpewJa51tuSWELLbsdglui9evEiTJoWFRqNGjbh06VKRJyWwHleT3BJCdgsKFp4smLPbnOwObB+OX6MGFq9RpWF9AttrC2ccfm8e/4x9XVeMMnzGFItP3wHaTJkIwL1rseybavig7tqu/9O9rlzXcD4Nh7wIQPSv33PnykWjY+fnZHN08XsA1On7mMGTcLlCQeiAgQAc+mA6qpwc/MKaM+CTfxi0+j+6TP0at4q2Ff0AbQHKJ786yZNfnaR+/5FW98vJzSIlVft3Irv/9F1/y9d1K75sY/XOkfrK73/mm7dukJuXbfV8TOFIyb126E+AY24KhOwWCAQCgUBQXnE1yS0hZLfAWOFJ/ZzdBWX3vqkz2fv6dF3e7Xaz3zA2rAEyuZzw6ZMBOP/DL+weP0UnuW2Ja1U5hesdAdy7cY3z69YCENp/oEFb5boPFnZd3PSLyfGtiWsllB5edJn6NYNWn2LAJ/9QJbSZxfkXpChxbWz8edRqrT+SyeS62FStzrcqJrUlrnUFyS0hZLfjsEt0+/r6cuXKlULHL126hJeX7WJGYB+uKrklhOwuvxSU3BLWyu4Wr462+lotXh1N+Iwpui1hVZs1odsn8626odDOyYPeXy2jwdDByBT3v0Tz8oj6/ivO/bQagDavzy70JN87KJigDtq8a4c/epPUC2cNcpdlpiSzd/pY0m9oc7bVfbxw2g/pRiUv/R6H5j6Q7LZsDTOGTC63SvLrE594SVf4o3pAKONHfEbbVv107bFxlr9o9W8a2rXqz/gRnxFYLQTQFg6JSyzag1BHS279mxohuwUCgUAgEAhsx1Ult4SQ3eUXY5JbwpTsluLBek8PpPdXy3CzMk2lX+OGdPtkPgFt2wCg9Pam7ZtTbYprD82dSdatmwbHkiIOc3COthhktZZtC+XilslktHl9NgBX/vqdmK2byc/O0rVbG9cao6gxqb1jxMSeQaPRUMHdg2cem8Yzj02jgrsHGo2Gq7FnLPa3Ja51luRecdD4Knkhux2D+TX/JnjyySeZPHkymzZtom7duoBWck+bNo0nnnjCoRMUGMfVJbeEJLvHrT3G+G+PseKldnhVsOufnaCUYEpyS0iyO/KTnzi56AdaTx+Ob5h1eaRN4deoAV0XfGB3f7lSSZ1+fajTrw/bRj1l0NZs5AQCWrU12q/5yxPJvn2L2xfOcnTB2ybH7zr3c9zvF//Qx72SD13nfs6Bt17TCXFncT3+PNWq1mLo42/QtcNgFHIFPbs8y+Vr//HL5vlcj4+2OEZsXDQtmzzEcwNnUrdOSwC6d3qGA0d+Z/2fi4mNO09oLdtXBEDxSm4J6abg+XUzeGH9TH4cugDvCp42XWdSF+3WxEX71hq8FwgEAoFAIChruLrklpBk9/hvjzFu7TFWjmxH81qVnT0tQTFiTnJLSLHqlY27de/7rFpu9zUrVqlMy/Fj7O6ffuMae6cbT2Hi36wVrSdMN9oW0KotzUZO4Mza5URv+J7oDd8bPc9cXOsqZOdkci89lcf7jmNQ/4lU8vYDoN9DL7J523L+3vstObmZVHA3HaM5Iq41haMk99L9i5nedYDRdkl2v7B+Js+vm8HPzy6kdY1GNl3DEXFtacauFd0LFy7Ey8uLRo0aERoaSmhoKI0aNaJq1aosXrzY0XMUFKC0SG4JsbK7/GBJckvYksbEWVSu15DO731CzW69TZ4jd1PSfsYHtBgzyaDYhkTogEH0WvadyafkoH2C3mvZd4T0f9Ih87aXBmFtWfrBAXp0GoJC78l73ToteXvSOoYNmm1xjGFPvcXbk37RSW4AhVxBj05DWPr+fhrUte/GqiQkt4RY2S0QCAQCgUBgmdIiuSXEyu7ygzWSW8JcGpOSxlQ82HLsVMInv12oEKU+Nbv1pvN7n1ClQeEUw9bEta6CSp3P0g8OMuLpOTrJDeDjXZURT89h6QcHyVeZ90mOiGuN4UjJPamb+ZQ4YmV30bBraa2vry///vsvO3bs4L///sPDw4OWLVvSrVs3R89PUIDSJrklxMruso+1kluiOFZ2O4KuHy6lol9V3Dyse+Ipk8up0akHNTr1IPtOKnnp95Ar3fHwD0CusG6rlrt3JRoVyI1W0gQFhJptr1XDfGFPS+colRWofj+NiS2UpOSWECu7BQKBQCAQCExT2iS3hFjZXfaxRXJL6K/srvvYkOKeokkaDXmRBoNfICslGXVeLkrvSlSs7Ge54318aofSYeZH5Gdlkp16C8CmuNYV8PLwwcvDx2R71SpBFsdwRFxbEEdLbm1ca7zOl4RY2W0/NpnGQ4cOcevWLR577DFkMhkPP/wwCQkJvPvuu2RmZjJw4EA+//xzKlSw/MtEYB+lUXJLlFbZvebL00679h0LnlSdr809dfjrc1w/mmzXNeIjtV+C+5ecQu72YFVy48mWZbWErZJbwhVlt3dwLbv7VqzsZ9PNiMA8zpDcEkJ2CwQCgUAgEBSmtEpuidIku105Di2IFFPu++yUXdfTj2t7d7ZrCLskt4QtMWxxIlco8Aq0LHPN4ebhiXdw2ZeZJUXxSG7rKC7ZXdaxKXXJBx98QFRUlO796dOnGTNmDH379mXmzJn8+eefzJ8/3+GTFDygtEpuCZHGpOxhr+SWKA1pTAQljzMlt4RIYyIQCAQCgUDwgNIuuSVEGhPXxp54sCiSW8JVZLfAdXCm5JYojjQmZR2bltNGRkby4Ycf6t6vW7eO9u3bs2rVKgBq1arFu+++y3vvvefQSQoe4CzJnZ6TbWeim8KUtpXdo15t7rRrH/eWmW1X5anZMHovHV9pTEiX6nZd4+rBRA5/fY5uk1ugUD549nVTY7lvUSW3RMGV3b2Wh9s9VnGhVqnISklCnZuL0tuHilVsX7ldmreRFSQ3LxsAd6X9v48ysu7iWbFwkU5XkNwSYmW3QCAQCAQCQdmR3BKlYWW3K8ehBZFWcnef0sKu6+nHtcds3OnrCMldmsi+nUpe+l3k7u54+AdanS5TwhFxrSPRaDTIZLb9e9Pvm5l9z2y6E3txBckt4eiV3WUdm+zi7du3CQwM1L3fu3cvAwY8qBTarl07YmNjHTc7QSGcJbmf/2EFq1+17T+SOUqb7BYUxlGSW0JfdrsSuffuErNtMzFbNxdqaz76dWp07I5Mbn5zzN3rMUT98BVply8YHPepU5dmI8fjU9t8HjFXZNhE7Zw3fJVgV//Em1d57e1OdGv/FKt7LdEddyXJLSFkt0AgEAgEgvJOWZLcEqVBdpdHpJ2+1sju8iK5NWo1CUcPcurrzwq11enzKHUffwb3SuZlr7m49qm1Zx01VZv5fM1EXn95uV19l62ZwIGjm/jio8MEVqvjsDm5kuSWcKTsLuvYlLokMDCQmJgYAHJzczlx4gQdO3bUtd+7dw+l0nQlWEHpQ//JvaMRaUxcm/ysHJNtjpbcEpLsdhXSE27wz6SXjN4MAJxevYyjC+egzs8zOcaN/bv4971phSQ3wN1rl/n3vWnc2L/LUVMuEeKTLuteJyRdsWuMwxH/A+DUuQfVzV1RckuINCYCgUAgEAjKM2VNckuINCauh7VpLcuL5Fbn5xGx5COjkhvg2s6/+GfSS6QnmP5ZWYprncmpc/vQaKzYTl4AjUajiyUPn/if4+ZTQpLbnnjQUWlMyjo2ie5HHnmEmTNnsn//fmbNmoWnpyfdunXTtZ86dYq6des6fJIC51Bwe1pxIGS36xL5yU9GZXdxSW6Jgjco6rw8Yv7azo5R49g5egJXt+1AnWdaLBsj+/YdTnyyjB2jxnFg5hxSz0Vb7JN77y4H3tJ+KSm9fei+8Ev6r9lI/zUb6ffN73R8+2MAbl84y+lvvjA6RnLkMc6s1T6drtWzHw9/tV43xsNfr6fhkBcBOLN2OcmRx236TM5k49ZlyOUK5HIFG7cus7l/dk4mm7drfy5p91IA15bcEo6W3QKBQCAQCASlhbIouSWE7HYtrKnh5AzJfTv6Intef4Mdo8bx3/KvyL59x6b+luJaU/HgyS8WknImEoDWE2bo4sn+azbSa8lavGtqVzIfeOs1ctPvFepvbVzrLNLupXDyzD829ztxZhd372nTgm7a9gXZdsRmBSnJldz2Ln5yhOwu69gkuj/88EPc3Nzo0aMHq1atYtWqVbi7u+va16xZw8MPP+zwSQpKnpLMwSZkt2uSHpdcSHYXt+QuSH5WFrvGvs6l3zcD2i1bF3/dyK6xr5OflWXVGKnnotk/bRa3os4BkJV8k4hFSzj15Wqz/S7/uQEA75p1eGjRV3j6B+jaZDIZlcPq624KEo7sJyMx3qC/RqPhxDJtcd7Gw0bTdPhY5Ho7XuRuSkL7P0nYo08BcGLZPKs+D4BGrbL6XGvHiIr+l1u3E4w+Tc/Iusvlq5EcOLqJX/9czL7Dv6FWq1CrVew9vIFf/1zMgaObuHztPzKzCt9caTQabt1OICr6X3bu/5Hl304iMysNALlcUSokt4QjZbdAIBAIBAJBacFZknvJ3m0lch0hu10Lc7LbGZL71MrVHP/4U/LSMwBIjohk/7RZVi2gAuvi2hPL5hWKxdIT4rh5KgKALh98RmB4R4N2dx9fOr41H6W3tu6RsRXb1sS19qBRq9Go1Xb1fTCGCrlcwbcb3uV/O78i4tQO4hIvkZefW+jcvPxc4hIvEXFqB//b+RXfbXgPuVybnzwzK43l305i5/4fbYpr9SnpdCVF2ekrZLd5bEqK7O/vz759+0hLS8Pb2xtFgaT3GzZswNvb9n8MAtfCGYVGRM5u16P19OGcXPQDkZ/8RKtpw4jdcaREJTfA0XmLAPAOrkH4m1PQqFScXLKce9diOfHZF7SfPd1sf41aTcSiJQA0fH4IwT26kpmQxOH35pJ0NILbPXtQpWHhL3a1SsW1nX8B0P6N91BUMH7zVDmsPg2fGUH0hu85/+u3hL8+W9d25/KDm546vR8xOcf6g57nyl8bzX4OibysDI6smEJy1EE8KgfSYeJS/MJsK1KTeuU0h794new7yQQ07ULHCZ/hVtGL9z4dDECn8CeYPHol8vt5xw8e+4Mlq1/V9VfIC/+/3Ph/S1GptQ+oZDIZk0d/See2TwCgVqtZsnochyK2PBhDodTdeKjVKqdK7pPx54tUyKOoObsFAoFAIBAIBMZZsncbC//5i+d7D7B8sgMwlrO7QsPSWzy+tKNfw0nK2Q2UuOROPRdN0jGtbO743lt4BPiTcOgo53/4hYhFS+izernFmk3WxrV3Ll+gSr2Gun6SuG703Cgq1TSeg9qtQkU6zVnEvhmvErN1Ew0GP4/svgC2Nq61lQtb13B241IAmg6eTP3+ti3i0Y9r0WhIvnmN73/7AI1GK86rVqnBRzO24O+nzdGekhrHWx8/TuodbUpdmUyOXCZHfX/xlkaj4dh/2w1SmFgT137UbBLgnJzcRa3h5Iic3WUVm1Z0S/j6+haS3AB+fn4GK7wFpQ9nVtMWK7tdC9+wYFpPH056XDJ7xy0occmt0WjIiNN+kXX84G3cvb2p4OtLh3dmApB26YrFXF7SE/aqzZpQu09PFEollWrXpON7WiH93/KvjfbLSknSvXb38TV7jeodugJws8BWs9TzZwCobUZyA8jkcho+M8LsORLn/1xJctS/oNGQdSeZI8snWdVPn8NfvE5OWor2hiLqX85v+VLX1q5lP14b9bnuZgCgS7snGTfiU2QyGTKZDJU6X3cDAqDRqFGp83Xt44Z/qpPcAHK5nNdGLqNty366YyqVYeoZZ67kLkpus6Ku7BYIBAKBQCAQGEeS3DN6PVqi1y24slvgXPRXdh//YDXHP1hd4jm5z373E6CV3JVq18StYkVq9exO1WZNAHQ7h01hS1ybev60Qd/4w3sB8A0xnyJYf5V25s1k3Wtb4lprSb1yijO/Lkadn4c6P4/T6xeReuW05Y566Me1gC7GlMsV+PpUY86UX3WSG8DfL5g5U37Ft5I/crlCF4Pqox9jWhvXgnMLTxa1hpNY2W0cu0S3oGziTMktIWS3a+EbFkyVRiG697X6diixa+tvg5LJZA9e631ZWdoqlXM/b1q1loarnj0CqgGQl55utJ86t/BWKVPkZxkXnKr7Y1SoXMXyGDnZVl0rM0UvP51GTdbtJJvSmGjUKrLvJBv0yUi5AUCn8MeZOnYVSrfCDyt7dXmOiS9pc3Hr/11ISMcmvrSMnl2eLdSuVFZg2thVdGzzmNH+zpLcp+Iji1zIQ8hugUAgEAgEAseiL7kn9+hf4tfXl90C5+PmUYGwpx7SvQ976qESLTyZlXwTAM+gQIPjkujOvXvXbH9b4tqCcagmX+tDKvhajikl1HkPxrAlrrWWzFvxRo6ZLoRpdIyUwucr5G74+lTjoxlbqBFYWOwHV6/HR2/+ia9PNaO7jEH787UlrnWm5JYQstvxCNEtAFxDcksI2e06xGzZR8rJaPxbN0ThUcFkgcriQK63ayQnLU33Wv9GQm5kZ4k+lUJqA3D+p/UGq78TDh0BIKBtG6P9lN4+Vs8z53YqABUq+xkcd6+kHSPPSEGQgmQkWndjUL1Fd9CoQSYHmZzAZl1029KsQSZXENC0i64/GrV2TGDSyytwUyhN9u3e8WkG9Tf9xf3UgEl07/i0yXY3hZLJo1cSFBBm9XxN4SjJPXL9MIdUrRayWyAQCAQCgcAxGJPczogHJdktcD5pV+I49fmveNcKxLtWIKc+/9VogcriomrTxgDE7T2gO6bRaLiw7jcAfEJCzPa3Ja5VVjK+6lqtMv9/QD/WlfJ1a19bH9dai1/dVrhV9EImVyCTK3Cr6IVf3VY2jWEQ195Hpc7nrdd+onq1ENP9qoXw1ms/FVrNLVEjsK5Nca2zJbeEkN2ORYhugUtJbomCsltQ8ugXnmw56VldGpOSlN31nxkEwL4pM7nw60YubfyDvZPfBCD0cfMpQQAq1Xyw3Wnny+O59vcuTnz6Oed/WAdAw+eeMdqvYpUH0jop4rDZa8Qd3K2dT/8nDY5Xv18o5Or2LeSk3THZX52XR+LRg2avIVG7y0DCX55LcNu+NOg/kg7jP7Oqnz4dJ3xGg/4jCW7bl/CX51K7y0AAFArLOfGzstORG3l6Lpe7kZltWegrFG7cS0+1ec76OFJy1/dv4JCbAiG7BQKBQCAQCIqOqZXczlr8JGpGOR/9wpPhs0cSPnuk0QKVxUmTUdo0k9E//8qJT5YRu2sPO18er2v3Dg6yOIa1cW31tobFJusPHgbA2R++Npu2M+3KRd3rinoLsGyJa63F0686PWb/SK2Oj1Kr46P0mP0jnn7VbRpDP67Vx5qYMjPL9Ar6u+mpNsW1riC5JYTsdhxCdJdzXFFyS+jLbkHJUzAnt37O7pKS3XX69yWoszZdyrVtO4j5n7bqeo2unak78DGrxnjo8wfVlC+s+41bZ84CED59MhWrVDbZr8UrUwA4uXwh925cK9Su0WiI2bqZhCP7AajZrbdBe0U/f3xD6wFwZP5s1Hl5hcbIy8zg0IczdOdbQiaTUafrIDqM/4xmQ6bhVtHLYp+CuFX0otmQaXQY/xl1ug7SbZ+7m37LYt9rcWdRqfJQyN10uc0UcjdUqjyu3zCfmw4g7V4K9zJu3/8scl2VbGtxtORePeQHwDE3BUJ2CwQCgUAgENiPuXQlYqdv+URfcks5ufVzdpeU7K5YpTLh0ycD2nzc539ar2vTjzXNYU1cW9HPn4pVqhr0q91T+38h9fwZrm77w+jYd65c5PBcbb7vFmMK13CyJq61Fd9aDWk7ZgFtxyzAt1ZDyx0KoB/Xat9r1WRsnOU47Hp8tEEfuVyhe30vPdWmuNZZkjs9x7jjErLbMQjRXY5xZcktIcluQcljrPBkSctumUxGs9Ev8dDni2n+6ss0e2UkD32+mKajhhvN9WwMpZcXfdespNOHc2j68ghaT5lIn9XL8Wts/gs5qH0X/Ju1AuDgnClc/t9v3L0eQ3ZqCukJNzi6cA7RG74HoMUrk3HzKFyRvdV4bfXszOREdkwYRvyhvWQmJ5KdmsKtc6fZNXG47majw6y51v5YioXY+zcM5rh+/8ajTfPefDJnN5/M2U2b5r3vt1kW3TfiLwDgrqzIU49MYs0nUUx9xXhB0IIUh+TWv6kRslsgEAgEAoHAOVjKyS3SWpY/jEluCWfIbr/GDemzejltpr1O05dH0OmDt+m7ZiVKL+sWHlkT1xqLB5WeXrR4ZTIA0Ru+5+TyhaRE/UdWagpZqSnEbN3M4Y+0K8OrNGhCUIduhcawJq51JlNf+ZqAqrWABxLbHJIMD/CvzdRXvmbNJ1E8NeB13JUVte02xLXOktyjfx1usl3I7qIj9uKUU+yR3Kt2X2JMz3rFP7kCNK9VmQs4voiCwDwFJbeEJLtPLvqByE9+cnjF67QrcVQJMdz6pPTyonr7tkUa1zs4yKptZRIyuZw2r8/i5PJF3PzvOBc3/szFjT8XOq/FK5Op0dH4z8qjajW6zV/O/lkT0OTnc2rV0kLnyN2UdP1oKR5Vq1n/YYqBxOSrNG3Q2WR72t0UQms147mBs6gf2lp3fMb4b7kYc4KfN80n7W4Kvj6mV6Yn3ozh0d5jGNT/dd15ncIfh5jCq931KW7JLSHdFLywfibPr5vBz88upHWNRjZdR5Ldz6+bwQvrZ/Lj0AV4Vyj8EEQgEAgEAoFAYF3hSWnx07i1xxj/7TFWvNROpBUpw5iT3BKS7I785CdOLvqB1tOH4xsWbGQ0xyGTy3X5uu3FXFxrKh6s0bE7aDScWrWUpIjDRlOQBHXsRvNREw0KXOrP25q41ll0Cn+c9q0GsOfQr5y9eMji+ZnZ9xg34lN6dHxGl6bk2SffZEDPl9m4dalNca2tOEpyX0y5YPa8SV1eAGDRvrUG763FEXFtaUas6C6H2LuSe/nOi6zafal4JycoFRTXym7ppsZVkLspCZ80m65zP6fWQ/0M2hoMfoE+y380KbklvAKD6Lf6N8KnvE3lug9Wkbv7VCZ88tv0/fIXPANsy2lWHDzUeajZdm/vKsyZ8quB5JaoH9qGd6duwNvbfDXwhzo/y0tDPjArwwtSUpJbQqzsFggEAoFAICgZrJHcEgVrOImV3WUXS5Jbwhkru51FjU496P3FDzQeNho3zweryKu1aku3eV/Q8pUpyN1MF2C0FNc6G4XCjd5dn2f8i0ssnjvhpaX06vJcoVzcvj7+jBz6YZHiWnM4UnKvHfqTxfPFym77EY9ByxlFSVcyoU99lu/UFjlwxspuZ5Gfp3batVV55tNzqPO1c1Plq1HZOU/V/TGksXRY+O3g6JXd+k/uXQ3voGCajhhL0xFj7eovk8up1rwN1Zq3cfDMHIfCQr5sS+2OGkOfkpbcEmJlt0AgEAgEAkHxYovklihvK7tdOQ4tiC4utXPO+nGtNZJbwhkru52F0tOLOr0foU7vRyyfbIKixrXFTUnEnLbGpOB4ya2Nay9a6iZWdttJ2f1WEBSiqDm5Jbld3mT3zMn7nD0FixxbE82xNZZzUZnj91f3G7zv/e1DFvs4SnYX3J4mEDhLcksI2S0QCAQCgUBQPNgjuSXKk+wuDXFoQTaM3luk/sfWRNNj5UybYkpHye78rBzcHZeRU1BGKB7JbT3FIbuD6WXTGKUNl/hGWL58OYsWLSIxMZGWLVvy+eef0759e6Pnrlq1iu+//54zZ84AEB4ezrx580yeL9DiqMKT5VF2P/ei8554xVQw/yRdla/m2Jpoqjf3I6RToF3XuHooicTTqbQb1RCFm+3ZjIoqu43mYHPe4gWBC+BsyS0hZLdAIBAIBAKBYymK5JYoL7LblePQghz+WluYvuMr9uWw1o9r7Vk4VVTZnZ+VQ+QnP9F+9mybry0ouzhbcks4Wnbf/Xa8XfMoLTj922D9+vVMnTqVL7/8kg4dOrBkyRL69etHdHQ0AQGFUxjs2bOH5557js6dO1OxYkU+/vhjHn74YaKioggOLptbVIqKoyS3RHmT3eHtnZc/WeNtQXTnaW8IQjoFEtLF/nkmnk4lpHN1FMoHovumxvr+9spuawqNlDQXN6/Dq3oNAsM7olC6W90vOzWFxIjD5N67i8LdHb9Gzahct6GuirYlNBoN2HY/WWxoNBp++WMBj/V5BR/vqnaNcTf9Fv/b+TXP21jkw1Ukt4SQ3QKBQCAQCASOwRGSW6I8yG5XjkMLcv1oMoDdMal+XGsv9spuSXKnxyXbfW1HoNFouHPpPKnRUahyc3Gv5EP18I5U9LO+vpEqL5ekiMNkJMYD2BXXuhLH//sbmUxGeIu+dvUvSlzrKpJbwpGyu6zj9G+CTz/9lDFjxjBy5EgAvvzyS/766y/WrFnDzJmF/wJ++skwafvq1av5/fff2bVrFyNGjCiROZcmHC25Jcqb7BZYxlbZ7YqSG+Dyll91r2v17Efj50cjV5jO45V16yaRKxaRFmO8UGub12cT0Mp4ZW2J5MjjnFg2j6fWnrVv0g7m/KUjbNq6jH+Pb+GLjyxXvjbG7PmPkpRyjdbNetG4Xger+ria5JYoLtktEAgEAoFAUF5wpOSWKA+yW2AbtspufcndevrwEpxpYba/PLjQsfO/rKFilap0mD0Pj6rVTPZVq1Rc2ryOK3/9brTdmrjW1dBoNHy84kUAfv0y3uoFZPrYG9eWhOROz8m0efGTo2R3Wcf2PAUOJDc3l4iICPr06aM7JpfL6dOnD4cOWfePMDMzk7y8PPz8/Iy25+TkcPfuXYM/5YXiktwSY3rW0xWoXLXbuOQTlC8k2Z0el0zkJz+Rn5Vj9LzikNwajQa1SlW0MdRqQgcMwqdOGACxu7dzbNG7JsfNSEpg7/SxOskd0u8J6g8eRu3eA3TnnFg2jxv7d5m85o39uzixbF6R5u1odh34GYA7d5PJys6wuX9WdgZ37iYbjGUJV5XcEo6oWi3J7uiUq+XiSbpAIBAIBAIBFI/klpBk9+WkdMZ/e4yMnHyHji8ofUiy2zs4gJOLfiDtSpzR8wpKbt+wYDRqNRp10fJoqlUq7W5dO2n4zAjqPjGE6u27AJB9+xZ7p48lIynB5PWOLXpXJ7mrNm5BwyEjbIprXZETp3c+eH3GdDxtDnvi2pJayf3C+pmk52TaPP6kLi8wvftIFu1by9KDP9rcvzzsLHaq6E5JSUGlUhEYaLg9JTAwkMTERKvGePPNN6lRo4aBLNdn/vz5+Pr66v7UqlWryPMuDRS35JYQsltQEEuy2xrJnZ+VZfX18rOyODp3ETtfHs+uMRPZMWoclzZusenmIi3mGjtGjWPn6Ak0fGY4nd9dTNe5ywC4feEslzavK9RHo1azf9YEADwDguj52RoaDX2Juo8OpsmwMfRfs5HGw0YDcGbtcu5ejyk0xt3rMZxZuxyAsMeetnq+xUlCcgz7jmhvknJzs/h773c2j/H33m/JzcsGYN/h30i8edXs+a4uuSUcLbsFAoFAIBAIyjrFKbklhOwWFMSS7DYmuQF2jp7AztET2DFqHGmXC8dvptBoNFzd+jc7Ro1j15iJ7Hx5PP++84FNcW3YY0/Tb/VvhA4YSP2Bz9Lq1Wk8/NV6KtWsA8DBdyYblfDnfl7N7QvancHtpr9Pu+nvEdp/oE1xrauh0WhYv2UhcrkCuVzB+i0LbX54YE9cW5LpSqTFT86Q3WWdUr2vZ8GCBaxbt449e/ZQsWJFo+fMmjWLqVOn6t7fvXu3zMvukpLcEiKNiaAgptKYWLuSe/eEqfT+ahlypdLsddR5eeyeMLXQ8Zj/bSXnThpNR1nefpZ2OYajcxcWOu4dVJN209/XPR2v+8QzBrnNUs6cBECudKfrh0uMzrVO70dIj4slds92zv74NR1nzzdoP7N2BQANh75EaL8nAEi9cprMW3H41W2Fp5/tOe5UebkkRx0EIKBpF6vzsa36eSbxiZe5diMKuUyOSqPW5TQ7FLGF4KD6BAWEEhQQRrOGXfH10eaKS7ubwpnoAyQkXyEhOYa4hItcvXFWdyMil8mZPf8R6tRsSo3qdbVjBNalddNeyOXaZ62lQXJLODKNiUAgEAgEAkFZpiQkt4RIYyIoiKk0JqYkd0GOzl1Ihzkz8QmpY/FaUd98R8K/RwyOZcQl2BTXNnjq+ULH5Uol7Wd+xK6Jw1Hn55FyJpJqLdro2lW5OcTu3g5A17nL8A6qWWgMa+Jar5ZNDPpkpiaSejkSz6rB+IU1t/j5jWFLXHs1NorL1yKJT7pCQvIVYuOjSUx+8KAh5vppJs3pSs0aDQgKCKNGYBh167QipFZTANRqNSej/iEh6TIJyTFWx7Ut/B/VXaOkc3IXtYZTUdOYlGWc+pvf398fhUJBUlKSwfGkpCSqVzf/H2Hx4sUsWLCAnTt30qJFC5PnVahQgQoVXCP3b0lQ0pJbQshuQUEKyu6wpx7i1Oe/Wp2u5PrO3YQMeNj8Obv2AFCpTm06vPMmMrmc3Lt32Tv5TeIP/EuDoU+h9PIyO4YkuVuMH0Ng2zYGbVUbN8evcXNSz50mKeIINTp207Vdup/Lu8WYSWZvXBo/N4rYPdu5cyma/KxM3Dy0X2D5WZncvXYZgDr3U51c2LqGM78uBsCtohc9Zv+Ib62GZuevjyovl/0LR5J6SSvh/eq1pvuba5G7WZbdpp5wq1R5XL72H1eun0Kj0RDe4mHat3qQmsXToxL7j24i4pS2UEjBJ+0qdT73Mm5zJvoAZ6IPIJPJeGXYQp3kBpwquZce/LHIVavtld0CgUAgEAgEZZWSlNwSQnYLClJQdrd4bQhXNu4xKbn7rlkJQNLxE5xasYojHyzQHTNFXkaGTnJ3/2wBFXx90Wg0HJ7zEelx8VbHtXX7G49JlJ5eOlF9es3n9FqyVteWdEJ7XZ86YUYlt4SluLZOy2G692mx0eydN4z8bO1K4+ZDp1O//0iz8y+IrXGtm5uSdX8s5M7dZKMxJUBC8hUSb8ag0Wio7BPAe9N+07XJ5XJSbyfw3Yb3TF7DWFw77hGt6HZG4UljNZyE7HYMTk1d4u7uTnh4OLt2Pci3o1ar2bVrF506dTLZb+HChXz44Yds27aNtm3NF3krTzhLckuINCaCgkiyO+1SLCcX/oCHf2Wrc3Jf3LDJ4jmXftsMQOvJ45HdF6fuPj6EPqa9mU45HWW2v/42rYKSW6Ja89YAZCbFGxxPu6J9qOPhH2D2GvoSPDv1ltHXcjftOWc3LtUdU+Vmc3Hbg5sYa0iOOqiT3ACpl06SdOagVX1lMpnFAh/tWw1g2thVKJUP/v6UygpMG7vKQH6bG3/ciM/o083wC9iZK7mLktusqGlMBAKBQCAQCMoizpDcEiKNiaAgkuz28K/MyYU/kHYp1mKRSv3Y0FJqj1tR5wAIGfAwFXx9AW3sE/7mFMC2uNYUngHahaC5d9MMjmckamPUqk1aWryGpbhW4uK2tahyH6Qfjfp9iU3pTTRqtc1xbc2gBnzwxiYq+wQgl5kumCmXKajsE8AHb2wiuHp9g7a+3YczfsRnNsW14BzJLVGwhpNIY+IYnCq6AaZOncqqVav47rvvOHfuHOPGjSMjI4ORI7VPjEaMGMGsWbN053/88ce88847rFmzhpCQEBITE0lMTCQ9Pd1ZH8ElcLbklhCyW+AoZArLv56kL1xZgerR6nxtkQ2ZvOi/4nLvmS9g617Jp8jXcAVeG/m5yTaZTEan8CeY8spXKI2sDle6uTPlla/o2OZxszcVr438nJ6dhxY67izJveLgsiIX8hCyWyAQCAQCWL58OSEhIVSsWJEOHTpw9OhRk+dGRUUxePBgQkJCkMlkLFmypNA5UlvBPxMmTCjGTyFwBM6U3BJCdgtKEo3KeEyqkWJSG+JaUxQU3AVxr1TJ4jUsxbXOJigwjI9mbMHXpxpyeWHZLZcr8PWpxkdv/klQYJjRMXp2edamuNaZkltCyG7H43TRPXToUBYvXsycOXNo1aoVkZGRbNu2TVeg8vr16yQkPKguu3LlSnJzc3n66acJCgrS/Vm8eLGzPoLTcRXJLSFkt0BCysntW68WrWcMJyvljtEClcao//Qgi+fUGzwQgIiPP9Ntb8pJS+Path0AVG3a2Gx/fRGedPyE0XNuRf0HgGdgDYPj7j7ap/WZyeYL56rz83SvK/pVNfpanac9p+ngybpjCveKNm8RC2jaBb96rXXv/eq1JrBZF6v6duswmNrBxn9eGo2G5wfNxk1hOkWLm0LJsEGzTRYJqR3chG4dBls1F0s4SnIv3b/YIVWrhewWCAQCQXlm/fr1TJ06lXfffZcTJ07QsmVL+vXrR3JystHzMzMzCQsLY8GCBSbTVR47doyEhATdnx07tPd2zzzzTLF9DkHRcQXJLSFkt0BCysmdlXKH1jOG41uvltEClfrox4aWFk/5t9DmiY7531Zy72plskat5uTS5YBtca0psm5pf5/61m1gcNyrujZGTTkdafEaluJaifr9R6Jwf1ADr+ngyTYtIJPJ5XbHtYHV6jCg5yijbRqNhkd6vUygf22zY9gS1zpbcksI2e1YXCJh1cSJE5k4caLRtj179hi8v3r1avFPqBThapJbQuTsFhgrPGmsQKUpavXqYfEadR7uxaXfN5MeF8/Ol8cbtAV17mAxPzdAhzkzOfLBAk6t0G5d6r9mo64tPf4Gd69ri2AEhncw6Nd81EQilszl2KJ36f3FDyg9jV/r2s7/A8A3tJ4uPzeAm4cnvnUbkHb5AufWraHp8LHU7z+Sqg3a2l2MUqF0p/uba3XpSgKbdbEqPzdo00bFJ17WvZfLFajVKt37GwkXLN5U3Ei4YPBef4z4pMuo1WqD3Nz24FDJ3e0NoOi5zRyRs1sgEAgEgtLKp59+ypgxY3Q7cr/88kv++usv1qxZw8yZMwud365dO9q1awdgtB2gWrVqBu8XLFhA3bp16dHD8v2hwDm4kuSWEDm7BcYKT/qEBhcqUCmxY9Q4g/7t35ph8RpKLy9qdO1E/IFD7J38ZqF2a+NaU6jz8ji1ahkA9R4fYtAWGN4RgFvnTnHr3GmqNjZeONJSXKuPb62G9Jn7Z5GKURYlrr2RcAEZ2l3C0sputVqFXCYntkC8aQxb4lpXkNwSIme343D6im6B/biq5JYQK7vLL8YkNzzI2Z0el2x2ZXfP5Z9ZrEwN2vzXPZd/inew4VPp+kOeounLL1o1V5+QOkZvYG6dPcWBt7VferV69jOoTA3g36w1MjftjfLRBW/rVmVLaNRqLvz+I9G/aos8Nn3R8KYJoOnwsQDE7t7OtV1aIe4X1pya7frbLLkl5G7uBLXqSVCrnlZLboDkW9fJy8/RpR5p2aQHFSt4IZNpvyZi4yyvVL5+fzWzTCbHo6I3LRp3v/9eRl5eNjdvxdr6cQxwtOTWv6kRK7sFAoFAILCd3NxcIiIi6NOnj+6YXC6nT58+HDp0yGHX+PHHHxk1apTFvKsC5+CKkltCrOwuvxiT3PAgZ7d3cIDZld0d5szEt26oVddqMnI4oY8/YnCscr26NsW1UjyoT07aHQ68Mwl1Xi4A/s1aGbQrlO6EPardNXts0bukx98oNIY1cW1BPP2qU7Ndf7skt4S9ce3VG1Go1Nr/pzUC61IjsC4AKnU+V2PN1+AC2+JaZ0nuU/GRRo+Lld2OQTzOLKW4uuSWECu7yx+mJLeEJLvNrex286iItbh5eNDpw3fQaDRo1GrkCtPFK0zhWzeUvmtWolGridm+hZunIkg9dxqAKg2a0Pj50YX6yORyun64lP2zJnDvxjX+HjuU6u274BtSDzQaojd8rzu32cgJ+NQufJPkUzuUZiMncGbtcs79tJrGfabaPHdHERsfDUCrpj15fuBsQmo1JT3jDn/8vZy/dq7SSWyzY8SdR6msyGO9x/DEw+Px9qpMTOwZftk8n5Nn/iE2PprAanXsml9xSm4JsbJbIBAIBALbSElJQaVS6dJOSgQGBnL+vGMe+m7evJk7d+7w0ksvOWQ8gWNxZcktIVZ2lz9MSW4JSXYXXNndZ7U23YittZ5kMhn1Bj1OvUGPo1apkMnlNj+YO/fTas79tJrQR57CvZIPOXdSubp9i669+4IVRudVb+Cz3L54jtsXznLg7dfxCgomuHNPZG5uVse1roJKrSI+6QrVqtbkuYGz6NJ2IAAHj2/ml83zSUi6jEqtQmEkh7eEI+JaUzhKco9cP4zzUzcbbRcru4uO+O1eCiktkltCyO7ygyXJLWGN7LYVmUxWqACIzWPI5USv/1b3PuzRwdQb+KxJee4VGESPRV9xZN5ssm/fIvHoQRKPHjQ4p83rswlo1dbkNWt26417JV9OLJtXpLkXFaWbOx9O/4NG9drrjnl7VWbYoLd4pNdojp7canGMhnXbMfzpd6niG6A7FlqrGbNf+4lzl46QY8cTaSgZyS0hZLdAIBAIBK7FN998w4ABA6hRo4blkwUlSmmQ3BJCdpcfLEluCWOyu0qIfbtq9bFn4ZU+Mf+30eC9b2h9Wo1/A4+q1YyeL1coaDf9fc79vJrY3dvJSIjjwu+GK3ktxbWuQnr6bUYOeZ+eXZ4zqA/Vrf1TdAp/nN0HfyE9/Ta+Pv4mx3BEXGt0bg6U3PX9G5g9T8juoiF+s5cySpvklhCyu+xjreSWKA7Z7QjqPTkUz8AaBIZ3KLStyxgeVavx0CertKL7+GHy7qUhd3fHr1FzKtdtYNWT/IBWben3ze+OmL7dtGra02RbFd9A+j30ksUxzJ3TuF4Hk23mKEnJLSFkt0AgEAgE1uHv749CoSApKcngeFJSkslCk7Zw7do1du7cycaNGy2fLChRSpPklhCyu+xjreSWKCi7ey0PL6GZFqbfN79z53I0qefPoM7NRVnJl+ptO1KxSlWLfeUKBU2Hj6XRsyNJijhCZlI8gE1xrSvg6+NP3+4jjLa5KZQm2/RxRFxbEEdL7tVDfgASzJ4vZLf9iN/qpYzSKLklSqvsjjia6LRrX61gXpKq8tXa8w4lmT3P7DXu9736byIKtwfblrw6Wz+GrZJbwhVld70nh9rVr2KVqoT0fdTu64qck4VxhuSWELJbIBAIBALLuLu7Ex4ezq5duxg4cCCgLQS2a9cuJk6caL6zFaxdu5aAgAAefdT+eyyB4ymNkluiNMluV45DCxIfeUvb76B9c9aPa5vaEIfqY6vkltCX3c5EJpNRpV4jqtSzP2ZQKN2p0bGbA2clKA7JbW1cK2S3fbjmb3SBSUqr5JYojbL7l+9cv6hc4ulUEk+nFmmMY2uiDd73tvIGw17JLWFMdrs7f2G3wMk4U3JLCNktEAgEAoFlpk6dyosvvkjbtm1p3749S5YsISMjg5EjRwIwYsQIgoODmT9/PqAtLnn27Fnd67i4OCIjI/H29qZevQexgVqtZu3atbz44ou4uYmw1VUozZJborTI7tIQhxbk8NfnitQ/8XQqDbNybI4p7ZXcEpLsFgj0cabkligO2b2QXjb1L2243m9zgVlKs+SWKG2ye8GS7k67doS3+Sfp6nw1v7+6n3ajGhLS2b7toVf/TeTYmmgGf9kNud6Kbmu0eVElt0RB2d1+9my7xhGUDVxBcksI2S0QCAQCgXmGDh3KzZs3mTNnDomJibRq1Ypt27bpClRev34duV6xs/j4eFq3bq17v3jxYhYvXkyPHj3Ys2eP7vjOnTu5fv06o0aNKrHPIjBPWZDcEqVBdrtyHFqQ/UtOAdBtcgu7rqcf19q607eoklvCzaMCqO3qKiiDuILklnC07F7IGrvmUVpwrd/kAos4S3Iv2buN53s7brzSJLvdlM6rQqxQWneDoXCTo7BznlK6EnnBMTTm+zlKckvoy25XIz0hjuj133LzVAQAFXyrEDpgIMFde6H09LLYX6NWk3LmJKfXfEHu3TQAfMPqU++JIfg3a+1Sla6t5c25/ZDJZCyYvc2u/ukZdxg5tTGvv7ycFtUe1x13JcktIWS3QCAQCATmmThxoslUJfryGiAkJASNxsKNJvDwww9bdZ6gZChLklvC1WV3aYhDJaQFU/bGpLrruslJj0u2WnY7SnKXBvIyM7j+z1YubvwZAJmbGzU69iB0wEC8g6z73Kbi2ib9nBeb7D+6kW7tn7K777JvJrD203N4e1V22JxcSXJLOFJ2l3VKn10RlDjSTY2jGdOzHhP61Gf5zous2n3J4eMLig9HS24JSXa7Cur8PCK//JQDb72muxkAyEm7zfl1a9k1cTjxh/aaHSMjKYG/X32WiCVzdZIbIO3KRSKWzGX76KfJSDJfiMLVyMpO58r1U1y+9h9Z2Rl2jXH6/AEAft40T3fMFSW3xKQuLzC9+0gW7VvL0oM/Wu5QAEl2N/QP4fl1MzgZX/q2ogoEAoFAICi/lDXJLSHJ7stJ6Yz/9hgZOfnOnlK5p/X04TrZnZ+VY/K88iS54w/vY9fE4TrJDaDJzyfuwC4OvPUaEUvmos7PM9nfUlzrTPTjQVv5aeNcAM5EH3DUdEpMctsTD0qyOzrlKi+sn0l6TqbNY5QH2S1Et8As+k/uiwMhu12XmC37jB4vLsktYewGJS89g4RDR0k8epy8DPvE6r0bccQfOMStqHNo1Jb3pGnUKo4tfp/Eo9ovzYZDRtBj8dd0/XApbafNIeh+kY9Tq5YSf9j4zyrr1k32z5qAJl97w9xu+vv0WPQVneYsotX46cjvV7/eP2sCWbdu2vW5nMHfe78z+tpaNBoNm7YuBeDWbW1FcFeW3BKOlt0CgUAgEAgEpYWyKLklhOx2LaTFT+ZktzMkt0atJuV0FPEHD5EeZ99CJXNxral4MP7QXk59vQQAv0bN6Dp3Gd0XrKDDrLk0ek6b2unmqQhOLJtvNM61Nq51Frdux5OQHGNzv4SkK6Te0f49bNy6zCE7gEpyJbe9i58cIbvLOq6zL0fgchTcnpbM/xXLdUpTGpPyxJWNuwEIfeJBbrjiltwF0Wg0nF37A/EHDhkcD+rcgaYvv4hMZnlLXV5GBntee6PQ8fDpk/Fr3NBkv/jD+7l9QVsoqdM7H+MbWl/X5h1cC/+mrfCpFUr0hu859fUSAlq2xc3DcOvQkXnaXOOVatah/cyPdGlOPKpWwzekLgEt23LgnUlkJicSuWIRvee4doXsvLwcEm9eZdO2L3THNm//gjbNe1O9WghKpeV/DxmZaRyN3EZM7BkA5DJFqZDcEo5MYyIQCAQCgUBQWnCW5D554yrBIcV/HVdPY1LeKFjDST/2dIbkTj0XTcSiJYWOP/T5YpReVqSytCKujVyxiE7vLDRoz8vM4NQq7QKhhkNGENLvSV0M7BlQnSr1G1O1cXMOzplCyplIEo/9S1CHrgZjWBPXOhO5TMG6Pz5mxNNzqOJb3aCmgzHUajW30xL55Y+PkcsUqDT5xFw/zZ5Dv9K+VX+8PH0tXlOKa1sQpjtW0ulKpMVP9qS1dEQak7KM+M0tMEpJ52ATstv1CHuqp4HsLmnJDXB505+6m4HQxwagUam4uvVvEv49gndwDUIGPGxxDH3J3WjYUG7+d5pbZ84SsWgJ3T6ZT8UqlY32O716GQAd3za8GdAnpP+TJJ+K4HZ0FNf/2UbYow9yi2WnppB9+xYAnd5ZiFypLNRfrlTSYdY8dk8ZRVqM5R0NGo2G6wc3k3hqH57+wTR6fBxKD8s3VvrkZWVw/s+VZKbEUb1Fd2p3GYhMJuOlKY0ICgijaYPOPD9otu4G48jJ/+Pvvd9xI+Eit9OS0GjUBg8YMjLvMPX9h5DJ5PhVrk5w9Xo83ONFOrR+BNDeiPy8aR5RF/4lIfkKGZna9C1yuQK1WoVKne9UyZ2ek1nkqtX2ym6BQCAQCAQCgWlO3rjKs98vZ/+cniVyPWOyG+GOnIYx2Q2UuOTOvn1HJ7mrNmuCf/OmRP+yAdDGmn3XrLQ4hjVxbVrMJbJv36Jilaq6fnEH/gEgqGM3QvsPNDp2pZp1aD1hBieXL+S/rz4tJLqtiWttJS02movbtLFQ/f4j8a1legGZMfTjWpU6n3+P/8G/x/9A6VaBAP9a1KrRiJeGvE/VKjUASEmN49sN73IjPprklFjy8g1X+cvlClZ8N5kV34GXp6/Vce2jM68DzsnJXdQaTkJ2m0akLhEUwlmFRkQaE9ci9InuOtn939J1JS65AWL+txWAHks+pt5TT1D/mUF0/0wrCC9u2GSxf3pcvO51n29WUKv3Q7SZ+hoNn3sGgLNrjRe+zL6dqnvtG2r6oYtMJqPpiLEAXPjdMJVFYsRhAKq372pUcktU8K1MSL8nLHwSLdcPbibim7eIO76Di9u+5ejKKVb10+fIiilc3PYtccd3EPHNW1w/uBnQrrS+eSuWnl2eNXiK3qZZb5TKiqTeSUCj0W6F098WJr3WaNTcuh2Pu7sHbZo9qFwrl8vp2WUoN2/F6iQ3gFqt0r125kruouQ2K2oaE4FAIBAIBAKBcSTJ3TAgqESvWzCNicC56KcxiZi3loh5a0s8J3f0z+sBaDT8OdpMfY3afXvR55sVuvZ7N+IsjmFtXJt4/LBhv62bAQjuYv5hT2B4R93r7DsPYllr41pbyExNZO+8YcQe/ovYw3+xd94LZKYm2jSGflyrT15+DvFJl2nVtKdOcgP4+wXTumlP4pMuF5LcYBhb2hLXgvMKTzqihpNIY2IcIboFBji7mraQ3a5F6BPd8W/dkJST0aiyckpUcqtVD76s3H18dK8r+PoaPccYd69eA6Dhc88YrEIOfkibIuTWmbNG++Wl39W9tpQeRa4wvjEm9552DK/qNYy266P0rmTxHIDEU/tAJgeNGjRqks4cRKM2/zPQR6NWkRx1UNcfmVw7JlDFN5APZ2whuLrhDZBSWYFpY1fRofUjZn8WMpmMDm0eZdrYVYVSmARXr8+HM7ZQxTcQhbzwz8tZkjs9J73IhTyKIrsFAoFAIBAIBIXRl9w/Dx9f4tfXl90C5+MbFkyL14aQHptEemwSLV4bUqKFJ5MjIgEI6tRed0wmk9Fo2FAA7l27bra/LXFt3r0HC4NAWywSoGLlqlhLXvo9vdfWx7XWkno5kvzsTDRqFRq1ivzsDFIvR9o0hkFcqzc/mUzGhJeW0rvr84X69O46jAkvLtGdZwyF3M2muNZZkltCyO7iQYhugQ5nS24JIbtdh7Qrcdw+f1X3PnbHkRK7tkzv6at+UQ391cQyC/m7pBuJlNNRBsczE7RPnD0CqhntJ3d3t3qeOXe0Nx9yN8NV24r7Y7hVqGh5DL0n7ebw9Ne7oZPJ8agSiEyusHKmIJMr8KgcqL2puI+Xf00APprxJ0EBoUb7Kd3cmTLmK9q1NP17oV3L/kwZ/SVuCuOr14MCQvlwxhY8PX0KtTlLco/+dbhDqlYL2S0QCAQCgUDgGApKbm8r7qWLA0l2C5xPflYOVzbu0b2/snGP0QKVxYXSW5sqMis5xeD4zchTAFSoXNlsf1viWlNxqMLDw+r5ypUPxrAlrrUWz6qFHzIYO2Z2DP/C52s0Gia8tJQeHZ8x2a9HpyFMeGmpycKTXl6+NsW1zpTcEkJ2Ox4hugWA60huCSG7nY9+Tu4eK2fq0pjEbNlXIteXyWT41tMWhzjy4QJy7qSRm57O4TkfAeAdXMPiU+mqTRsD2pXbsbv3kZ+dzb3rsRx+bx4ATV4ynlvZwz9Q9zozJdnsNdKuav991ujcw+C4X6NmAERv+N5o9Wt9rv+z1Wy7RKPHxxHYrDPIZHhUCaDjxGVW9dOnw8SleFQJAJmcwGadafTEqwAE+Ncy20+hcKNmUAMURkS2QqGkVo2GKEysbpcI9K9NxQq25RQ3hqMk98WUCw65KRCyWyAQCAQCgaDomJLczooHm9eq7JTrCh6gX3iy7ZzRtJ0zmvS4ZCI/+anEZHfLCdpUlYffm8u967Go8vK4vuMfbkWdA8Cvsfn81LbEtX6Nmhv0rdayLQCJRw+YvUbu3QcrwT2rBehe2xLXWotfWHOaD52O3E2J3E1J86HT8QtrbrmjHvpxrT4hNZta7FsnuInJtooVvGyKa50tuSWE7HYsohilwOUkt4R+gcoej9Z28mzKHwVzcoc+0R3AoEBlcdNmykR2T5jKvWux7Js606Ct3ew3TPR6gEwuJ3z6ZCIWLeH8D79w/odfdG2B7cPxa9TAaD+5QkHogIHEbN3MoQ+m89Cir1FUKJyy5d6Na5xfpy3CUbA4SOW6D254Lm76hQaDhxm91rVd/2fxc0goPbzoMvVrNGqVTSu59fELa86AT/4pNEZuXjbuSvMrZmLjz6NW5wMgu78qXKNRo1bnc92KL+Oc3CxSUuN0/TUatdFUJuZwpOReO/QnwDGFPIpaoFIgEAgEgtJG3Ox/DN4vPfgji/atZXr3kXZ9D6bnZPLC+plEp1y1WJgreF4vm8cXuDbmVnIv33kReBAfCsoH+pJbPyd3wQKVxZ1as0rD+gS2DyfpaIRuwZRE+IwpFncZg/VxbeW6hvFpwyEvcvO/40T/+j1VGjSlcljhYpL5OdkcXfweAHX6PmYQ41kb19pK/f4jqffwi4DlXdbG0I9rt4xpjep+jHk9/jx1apoW2aCNSSUUcjdU6nzkcgVqtYqbt27YFNe6guSWkGS3KFBZdMSK7nKOq0puCWllt6DkMVZ4Ur9AZUms7Hbz8KD3V8toMHQwMoX211W9pwfS+6tluFm5fcuvcUO6fTKfgLZtAFB6e9P2zam0eHW02X6SuM5Lv8ehuTPJunXToD0p4jAH52iLQVZr2bZQLm6ZTEab12cDcOWv34nZupn87Cxduzovj6jvv+LcT6sBdOdag72S29wYcYmWV8rExJ5Bo9FQwd2DZx6bxjOPTaOCuwcajYarsWcs9o9PvKQr/FE9IJTxIz6jbat+Vs/Z0ZJb/6ZGrOwWCAQCgaBoOKJQc1FXtAlKJ5bSlYidvuUPU5IbDAtUltTK7havjiZ8xhRd6suqzZrQ7ZP5JhdOFcSauLbN67ML7Vj2DgomqIO2vtThj94k9cJZgxpNmSnJ7J0+lvQb2tpUdR8vnPbDmrjWHmRyuV2S23AMhUE8GBtn+fe+/gKrdq36M37EZwRWCwG0i7BsiWudJblXHDS+M1us7HYMYkV3OcZWyX069o5Ttm+N6VmPC+SW+HXLO6aejhf3yu78rBzc9S4rVyqp068Pdfr1sXvMilUq03L8GJv6uFfyoevczznw1muk37jG3uljjZ7n36wVrSdMN9oW0KotzUZO4Mza5URv+J7oDd8bPa/ZyAkEtGpr0/wcTWzceUJrNTPZnp2Tyb30VB7vO45B/SdSydsPgH4Pvcjmbcv5e++35ORmUsHd9BPj6/HnqVa1FkMff4OuHQajkCvo2eVZiMmzOL/ilNwSYmW3QCAQCARFo6jfg45Y0SYoXViTk1t/p6/+e0HZxJzklpBkd0mu7PZr1ICuCz6wu7+luNZUPNj85Ylk377F7QtnObrgbZPjd537Oe7elQodtyauDV37knUfohh4Y+xqLl/7j182z+d6fLTF82PjomnZ5CGeGziTunVaAtC90zMcOPI76/9cbFNcayuOktxL9y9metcBRtvFyu6iI1Z0l1PsWck9bu0xTsfeKd6JCVwGczcKxbWyW7qpcRW8g4Lptew7Qvo/abS95diphE9+u1AhSn1qdutN5/c+oUqDwluwKtdrSOf3PqFmt94Om7O9NKhrXrSr1Pks/eAgI56eo5PcAD7eVRnx9ByWfnCQfFW++WuEtWXpBwfo0WkIChtWpZeE5JYQK7sFAoFAICgaYmW3wFpsKTwpajiVHyxJbglnrOx2BnI3Je1nfECLMZNAVljhhQ4YRK9l3+EdZPpnZSmudTZ167Tk7UnrGDbI8i7nYU+9xduTftFJbgCFXEGPTkNY+v5+m+JaW3Ck5J7UzXwaVrGyu2iIFd3lEHvTldQN9Gbc2mOsHNlOFOYQOHxlt/6Te1fC3bsSjYa8SIPBL5CVkow6LxeldyUqVvaz3Pk+PrVD6TDzI/KzMslOvQVARb+quHm4zlPV6ve3e5nCy8MHLw8fk+1VqwRZvIap6tfmKEnJLSFWdgsEAoFAUDTEym6BJWyR3BJiZXf5wBrJLeGMld3OQCaXU6NTD2p06kH2nVTy0u8hV7rj4R+AXGHdAiJHxLXFTa0a5gt7WjpHqaxQ5LjWGI6W3Nq49qLZ88XKbvsRorucUZSc3Cteasf4b4+VO9m95svTTrv2HQvfWep8bb7jw1+f4/pR+wRxfKRWvO5fcgq524MnxI0nWxbXjpLdBbenuSJyhQKvQMsy1xxuHp54B5f9LxZH4QzJLSFkt0AgEAgERUPIboEp7JHcEuVJdrtyHFoQKabc99kpu66nH9e2nTPaKsktUV5kt0TFyn5FktOOiGvLE8Ujua2juGR3WUekLilHFLXwpFcFN1a81E63slukMRFA0dOYWJODTVD+cKbklhBpTAQCgUAgKBoijYmgIEWR3BIijUnZxp540BFpTByZklNQNnCm5JYojjQmZR2xorucUFTJLSHJ7vK0snvUq82ddu3j3jKz7ao8NRtG76XjK40J6VLdrmtcPZjI4a/P0W1yCxTKB8++bmqsH8Peld2uKLnT42LtTiuSfTuVvPS7yN3d8fAPtHobmSuSkXUXz4qVClX+thaNRkNm9j2bt4WBa0huCbGyWyAQCASColEcK7uD6eXweQqKH0dIbonysLLblePQgkgrubtPaWHX9fTjWnspysrumC37uLJxN3UfG2L39R2BWqUiKyUJdW4uSm8fKlaxfeW2K6fLtJXcvGwA3JX2/66wN651Bckt4eiV3WUdsaK7HOAoyS0hVnYLjGHrym5XlNwAB96ZxM4JL/Dv+9O5ez3G4vkatZr4w/vZNuop9kwbzcF3p7J/1kT+HvMM537+htx7dy2OkXvvLtEbvnfE9B1C4s2rvDS5IZ+vmWj3GMvWTOClyQ1JunnNpn6uJLklxMpugUBQFJYvX05ISAgVK1akQ4cOHD161OS5UVFRDB48mJCQEGQyGUuWLCl0zsqVK2nRogU+Pj74+PjQqVMntm7dWoyfQCAoOo5e2S0ofThSckuIld2CgtizsluS3GFP9SyBGZomesP3/D3mGfbPmsjBd6eyZ9poto16irh/96BRqy32v3s9hkNzZ7JzwgsceGeSzXGtKzJsYijDJtpe50nC3rjWlSS3hCNXdpd1hOgu4zhacksI2S0whrWy21Ultz53r13m3/emcWP/LpPnqPPziFjyEae+/sxo+7Wdf/HPpJdIT4gzOUZ6wg3+mfQSMVs3F3XKDuNwxP8AOHVuHxqNDUv776PRaDh1Tvv3f/jE/6zu54qSW0LIboFAYA/r169n6tSpvPvuu5w4cYKWLVvSr18/kpON19XIzMwkLCyMBQsWUL268Z1aNWvWZMGCBURERHD8+HF69erFk08+SVRUVHF+FIGgyDhSdgtKF8UhuSWE7BYUxBbZrS+57a035ShMxYOnVy/j6MI5qPPzTPa9sX8X/743jbTLFwq1WRPXuiLxSZd1rxOSrtg1hj1xbUlJbmel8yoPtS6E6C7DFJfklhCyW2AMS7K7OCT37eiL7Hn9DXaMGsd/y78i+/Ydm/qr8/KI+Ws7O0aNo/+ajTz89XoaDnkRgDNrl5Mcedxov5NfLCTlTCQArSfMoP+ajbo/vZasxbtmHQAOvPUauen3CvXPvXeXA29pv/iU3ran+CgOsnMy2bx9OQBp91I4eeYfm8c4cWYXd+9pt8tt2vYF2VZIYVeW3BKOlt0CgaDs8+mnnzJmzBhGjhxJkyZN+PLLL/H09GTNmjVGz2/Xrh2LFi3i2WefpUIF49utH3/8cR555BHq169PgwYNmDt3Lt7e3hw+fLg4P4pAUAhnPPSVgnxB6aE4JbeEkN2Cglgju41J7h2jxrFz9ASubtuBOs+0WDZG9u07nPhkGTtGjePAzDmknou2qb/S24fuC7/UxZP9vvmdjm9/DMDtC2c5/c0XRvslRx7jzFpt/FarZz8e/mq9bgxr41pXZOPWZcjlCuRyBRu3LrO5vz1xbUmu5Ba1K4oPIbrLKMUtuSWE7BYYw5TstlZyn/pytdXXOrVyNcc//pS89AwAkiMi2T9tltU3FvlZWewa+zqXft+sOyZ3UxLa/0nCHn0KgBPL5hV6ApyeEMfNUxEAdPngMwLDOxq0u/v40vGt+Si9KwHGn9Bf/nMDAN416/DQoq90xzVqlVVzN4VGrbZqe5s+UdH/snP/jyz/dhKZWWkAyOUKvt3wLv/b+RURp3YQl3iJvPzcQn3z8nOJS7xExKkd/G/nV3y34T3kcm1+8sysNJZ/O4md+38kKvpfbt1OKPSzLA2SW8KRslsgEJRtcnNziYiIoE+fPrpjcrmcPn36cOjQIYdcQ6VSsW7dOjIyMujUqZNDxhQIrMVZO5xsrZUhcB4lIbklhOwWFMSc7Da3klujVnPx143sGvs6+VlZVl0r9Vw0+6fN4lbUOQCykm8SsWiJTXHtQ4u+wtM/QPdeJpNROay+TnYnHNlPRmK84Vw1Gk4smw9A42GjaTp8LHKlUtdubVxrjKLGpLaOkZF1l8tXIzlwdBO//rmYfYd/Q61WoVar2Ht4A7/+uZgDRzdx+dp/ZGYVXkSm0Wi4dTvB7ri2pNOViELNxYcoRlkGKSnJLVEeC1QKLFOwQGWtvh2sXsmddDSC2z17UKVhfbPXSD0XTdIxrWzu+N5beAT4k3DoKOd/+IWIRUvos3o5Mrn553lH5y0CwDu4BuFvTjFoqz/oea78tRGAO5cvUKVeQ12bJK4bPTeKSvdXbhfErUJFOs1ZxL4ZrxKzdRMNBj+P7L4AVqtUXNv5FwDt33gPRYUKpF45zeEvXif7TjIBTbvQccJnuFX0Mjv/glzYuoazG5cC0HTwZOr3t06ovvfpYAAUCqXu5ketVpF88xrf//YBGo1WnFetUoOPZmzB30/795eSGsdbHz9O6p0EAGQyOXKZHPX9mxqNRsOx/7YbpDDpFP4Ek0evRH7/78aZkvtk/PkiFfIoaoFKgUBQdklJSUGlUhEYGGhwPDAwkPPnixaQnD59mk6dOpGdnY23tzebNm2iSZMmRRpTILAV6aGvKNQsMEZJSm6J8lCgUmAbxgpUxu44YlJy91i2CI1Kxckly7l3LZYTn31B+9nTzV5Do1YTsWgJAA2fH0Jwj65kJiRx+L25NsW11RobT51SOaw+DZ8ZQfSG7zn/67eEvz5b13bn8oPFXXV6P2JyfEtxbcDLvXXv87IyOLJiCslRB/GoHEiHiUvxC7OtMKqtce3BY3+wZPWruvcKeWFVufH/lqJS5wPahwCTR39J57ZPAKBWq1myehyHIrY8GMOKuPbfsdrdcM7IyV0chZrLQ1oSaxArussYJS25JcTKboEx9Fd27x23wKZ0Jf8t/9riOWe/+wnQSu5KtWviVrEitXp2p2ozbbAvi5VAvwABAABJREFUPVE3hUajISNOK2g7fvA27t6GX2gyuZyGz4wAIPX8aYO2+MN7AfANqWv2GvpP5TNvPsjJmpWSpHvt7uMLwOEvXicnLQU0GpKj/uX8li/Njl2Q1CunOPPrYtT5eajz8zi9fhGpV05b7qiHSmW4RU+lzkejUSOXK/D1qcacKb/qJDeAv18wc6b8im8lf+RyBRqNWncDYmzMdi378dqoz3WSG3DqSu6iFvIoyspugUAgsJeGDRsSGRnJkSNHGDduHC+++CJnz5519rQE5QxRu0JgCmdIbgmxsltQEP2V3XvHLTCbk9vd25sKvr50eGcmAGmXrljM7SztJK7arAm1+/REoVRSqXZNOr6nFdK2xLWmqN6hKwA3C6QeST1/BoDaZiQ3WBfXSpz/cyXJUf+CRkPWnWSOLJ9kcf4FsTWu7dLuScaN+BSZTIZMJtPFoBJSjCm1jxv+qU5yg3bH3Gsjl9G2ZT/dMWviWnBu4UlHF2oWK7u1CNFdhnCW5JYQsltgjFp9O+heV2kUYnVO7rz0dIvnZCXfBMAzyHC1nCS6c+/eNdtfP72HTCYzek5+TjYA6lzDrU2afK3MreBbxeI8JdR5D8YoOB5A9p1kg+1dGSk3rB4bIPNWvJFjpgth6tOxzWMmfwYKuRu+PtX4aMYWagQWFvvB1evx0Zt/4utTzejTd9D+fDuFP87UsatQurkbtDlLcp+Kj3RI1WohuwUCgTH8/f1RKBQkJSUZHE9KSjJZaNJa3N3dqVevHuHh4cyfP5+WLVuydOnSIo0pcAzLly8nJCSEihUr0qFDB44ePWry3KioKAYPHkxISAgymYwlS5YUOmflypW0aNECHx8ffHx86NSpE1u3bi3GT2A9olCzwBjOlNwSQnYLCuIbFkyVRiG69/oxqjH0dwVbSgmZc78+VLWWhquePQKqAbbFtabIzzL++1V1P6asUNlyTGoprpXITNGLHzVqsm4n2ZSCRKNW2RXX9uryHBNf0ubiNhaXSscmvrSMnl2eLdSuVFZg2thVNsW1zpTcEkJ2Ox4hussIzpbcEkJ2C/SRcnIrPCrg37ohKSejjRaoNEZA2zYWz6natDEAcXsP6I5pNBourPsNAJ+QELP95QqF7nVOWprRczIStV/0ykq+RtvVqnyjx/XnIyHl69a+Llx8MqBpF5DJtX80aqq3sK3yt1/dVrhV9EImVyCTK3Cr6IVf3VZW9Z08eiVBAWFG21TqfN567SeqVwsx2b96tRDeeu2nQqu5JWoE1mXSyytwUyiNttuCoyT3yPXDHFK1WshugUBgDHd3d8LDw9m1a5fumFqtZteuXQ7Pp61Wq8nJKVxoy5mUJ+ErsX79eqZOncq7777LiRMnaNmyJf369SM5Odno+ZmZmYSFhbFgwQKTDz9q1qzJggULiIiI4Pjx4/Tq1Ysnn3ySqKioQuc642cuZLdAH1eQ3BJCdgv0idmyj5ST0fi3bojCo4LJApUS+gum9GNGY1QKqQ3A+Z/WG8R+CYeOALbFtabIuZ0KQIXKfgbH3StpY8q89MI5qwtiKa6VqN6iO2jUurg0sFkXXfpNa5DJFXbHtd07Ps2g/qZl8VMDJtG949Mm290USpviWmdLbgkhux2LEN1lAFeR3BJCdgugcOHJlpOeNVqg0hQNn3vG4jlNRmm3X0X//CsnPllG7K497Hx5vK7dOzjI4hj1nxkEwL4pM7nw60aDNnVeHolHDwJQva1hscn6g4cBcPaHr81uZ0u7clH3uqLejUnFKg9eJ0Voc4N1nPAZDfqPJLhtX8JfnkvtLgMtzl8fT7/q9Jj9I7U6Pkqtjo/SY/aPePpZt2pQoXDjXnqqyfbMbMs3T5lZplfQ301PRaEoelkIR0ru+v4NHHJTIGS3QCAwxdSpU1m1ahXfffcd586dY9y4cWRkZDBypLZ+wogRI5g1a5bu/NzcXCIjI4mMjCQ3N5e4uDgiIyO5dOmBpJk1axb79u3j6tWrnD59mlmzZrFnzx6GDRtW4p/PFM4Wvs7i008/ZcyYMYwcOZImTZrw5Zdf4unpyZo1a4ye365dOxYtWsSzzz5LhQoVjJ7z+OOP88gjj1C/fn0aNGjA3Llz8fb25vDhwwbnOfNnLmS3AFxLcksI2S0Aw8KTLSc9a7JAJcCFXzdyaeMf7J38JgChj5tPCQJQqeaDHcs7Xx7Ptb93ceLTzzn/wzrAtrjWFHEHtXWvQvs/aXC8erg2Rr26fQs5aXdM9rcmrpWo3WUg4S/PJbhtXxr0H0mH8Z9ZnH9BihLXZmWnIzeyS1gud7MqJrUlrnUFyS0hZLfjEKK7lONqkltCyO7yTUHJLaUr0c/ZbU52h0+fTMUqlS1ep2KVyoRPnwxo83Gf/2m9ru2hzxdbNdc6/fsS1Fm7de3ath2643mZGRz6cIb2On7+VKxS1aBf7Z7a/2+p589wddsfRse+c+Uih+dq87u1GFM4t1mLV7TFL08uX8i9G9dwq+hFsyHT6DD+M+p0HWRyy5U5fGs1pO2YBbQdswDfWg0td7hP2r0U7mXcBu4XlLz/1F4m035NxMZZ/qK8Hh9t0EcuV+he30tP5W76Les/iBEcLblXD/kBcMxNgZDdAoHAGEOHDmXx4sXMmTOHVq1aERkZybZt23QFKq9fv05CQoLu/Pj4eFq3bk3r1q1JSEhg8eLFtG7dmtGjR+vOSU5OZsSIETRs2JDevXtz7Ngxtm/fTt++fUv885nCmcLXWeTm5hIREUGfPn10x+RyOX369OHQoUMOuYZKpWLdunVkZGQU2hXg7J+5kN3lG1eU3BJCdpdv9CW3lJNbP2d3Qdl9bdsOYv63DYAaXTtTd+BjVl1HP/a8sO43bp3R1s2wJa69d+NaoeMajYaYrZtJOLIfgJrdehv28/PHN1RbdPXI/Nmo8/IKjWFtXCshk8mo03UQHcZ/RrMh08wWkTRFUeLaa3FnUanyUMjddDm5FXI3VKo8rt8wX4MLbItrnSW503OMp7MRstsxFH15ncBpuKrklpBk9/hvjzFu7TFWjmxH81qVnT0tQTFjSnJLSDcYVzbuNnivj19j6wWtX+OG9Fm9nNRz0eTcuYNPndp417QuDzhov8ibjX6Jhs89w62oc2SnppCRlMCxRe/qzukwa26hfkpPL1q8MplTXy8hesP33LlygVoP9cMrSHvtxCMHiN7wPQBVGjQhqEO3QmMEte9C/L+7STkTycE5U3hqrfMKid2IvwCAu7Iijz88jsf7jOXUuX38tHEuSSnXdBLbHNJNQ4B/bYYNmk2Lxt35c8eX/LnjS3LzsomNj6Zpg852za84JLf+TY0jqlZLQf7z62bwwvqZ/Dh0Ad4VPG2ep0AgKFtMnDiRiRMnGm3bs2ePwfuQkBCLRa+++eYbR02tWJCEr/5K9eIQvhs2bDAqfJ1FSkoKKpVK9xBDIjAwkPPnixZonj59mk6dOpGdnY23tzebNm2iSZMmunZX+Zk74ntwUpcXAFi0b63Be4Hr4sqSW2JMT60IXL7zosF7QdnGmOSWkGT3yUU/EPnJT7SaNgw3jwo0f/VlNGo1/s2bovSyXvAqvbzou2Yl6XEJ3L16FXcfH6o2bWyQ69sSB+dMof5Tz1OtRTju3pXIz8km6vuvuB2t3UXT4pXJuHkU/p3aavx09k4fS2ZyIjsmDKP5yAlUrtsQuZub1XGtK3H9fkzZpnlvnhuo/V77ZfN8jv23netxlkW3LXGtsyT36F+H8+eIhUbbi/o96Ii4trQjRHcpxdUlt4SQ3eULS5JbwhrZbQsyudxiXjNLKL28qN6+LTtGvaI7JndT0vWjpXhUrWa0T42O3UGj4dSqpSRFHNalINEnqGM3mo+aaPQmRyaX0+b1WZxcvoib/x0v1F6SJN6M4dHeYxjU/3V8ffwB6BT+OO1bDWDPoV85e9FyoJyZfY9xIz6lR8dndGlKnn3yTQb0fJmNW5eSmHzVLtFd3JJbQshugUAgKDrOFL5llYYNGxIZGUlaWhq//fYbL774Inv37tV9dlf6mQvZXb4oDZJbQsju8oU5yS1hTHZXb9+2SNf1Dg6yKn2mKS5u/JmLG38udLzFK5O1sacRPKpWo9v85eyfNQFNfj6nVhUuTm0prnUV0u6mEFqrGc8NnEX90Na64zPGf8vFmBP8vGk+aXdTdPGqMRwR15rCUZL7YsoFs+cJ2V00hOguhZQWyS0hZHf5wFrJLeFo2e1I3H0q03zURPybtbL4FL5Gpx5Ua9mW+EN7ubjpF/IzMwCo1qotjYa8hFf1Gmb7y92UhE+aTXpCnNnzipuHOj+LwkiREYXCjd5dn+ehzkMtjjHhpaVGx/D18Wfk0A9R2VCtW6KkJLeEkN0CgUDgulgSvs7E398fhUJBUlKSwfGkpCSTObCtxd3dnXr1tFIuPDycY8eOsXTpUr766qsijWsN9vzMhewuH5QmyS0hZHf5wBrJLVFQdrefPbuEZlmYrnM/59qO/xG7Z7vuWIPBL1C7V3+jK7n18QoMot/q30iJiuTylg3cuaxdtWxLXOsKeHtXYc6UX4221Q9tw7tTN1iMKR0R1xrDkZJ77dCfLJ4vZLf9CNFdyihtkluiNMvu/Dy1066tyjOfy0qdr52bKl+Nys55qu6PIY2lw4bfDrZKbglXlN3912y0fFIBlJ5e1On9CHV6Wy5WYgrvIOvTrRQHxm4GbGl31Bj6lLTklhCyWyAQCOynrApfS7i7uxMeHs6uXbsYOHAgAGq1ml27dplMXWMvarWanJwHOWVd8WcuZHfZpjRKbonSJLtdOQ4tiC4utXPO+nFtUbBFckvoy25n4h0UTNMRY2k6Yqxd/WVyOdWat6Fa8zYOnlnJURIxp60xKThecmvj2osW+wnZbR9CdJcySqPkliitsnvmZNNFE12FY2uiObbGcg5lc/z+6n6D972/fciqfvZKbglXlN0C5+MsyS0hZLdAIBDYhzOFr7OZOnUqL774Im3btqV9+/YsWbKEjIwMRo4cCcCIESMIDg5m/vz5gDa39tmzZ3Wv4+LiiIyMxNvbWyeXZ82axYABA6hduzb37t3j559/Zs+ePWzf/mDFn6v+zIXsLpuUZsktUVpkd2mIQwuyYfTeIvU/tiaa3naGg/ZIbglJdgsEBSkeyW09xSG7g+ll0xilDSG6SxmlVXJLlEbZ/dyLznviFVPB/JN0Vb6aY2uiqd7cj5BOgWbPNcXVQ0kknk6l3aiGKNxs285UVMktUVB2131siF3jCMoGzpbcEkJ2CwQCgX04S/g6m6FDh3Lz5k3mzJlDYmIirVq1Ytu2bbrc2devX0eut3U8Pj6e1q0f5CBdvHgxixcvpkePHrpCpcnJyYwYMYKEhAR8fX1p0aIF27dvp2/fvgbXdtWfuZDdZYuyILklSoPsduU4tCCHv9YWCez4in11k/Tj2pgt+2wW1UWR3BK+YcHgvEX0AhfE2ZJbwtGy++634+2aR2lBiO5ShrMk98kbVwkOccxYpU12h7cv2pbPoqDxtiC687Q3BCGdAgnpYv88E0+nEtK5Ogrlg+DrpsZ8H0dJbgl92e1qoluj0XDn0nlSo6NQ5ebiXsmH6uEdqehnughGQVR5uSRFHCYjMR4Ar+o1CAzviELpXlzTLlaO//c3MpmM8BZ9LZ9sBI1Gwy9/LOCxPq8APrrjriK5JYTsFggEAttxpvB1NhMnTjS5ilr6LBIhISFoNOZvuL755hurruvKP/PikN0Ly/hqNFekLEluCVeX3a4chxbk+tFkALtjUv249vDXtu30dYTkLi1kp6aQGHGY3Ht3Ubi749eoGZXrNkQms+7vy1Rc61m1mCduhrvpt/Dxtm8Cd9Nv8b+dX/P8wFkOnpXrSG4JR8ruso4Q3QKLSDc1++f0dNiYpU12CwxxtOSWcMUbk+TI45xYNq/Q8fO/rKFilap0mD3PbPVqtUrFpc3ruPLX70bba/XsR+PnRyNX2J4rzFloNBo+XvEiAL9+GW/1jZU+5y8dYdPWZfx7fAv7XtZuy3Q1yS1RXLJbIBAIyjLOEr7lGVf+mTtadi9kjcPmJrCOsia5JVxddpdHwp7qaXVay/IiubNu3SRyxSLSYi4ZbW/z+mwCWrU1O4a5uPaptWcdMk97mL3gMb746JB9fec/SlLKNVo360Xjeh0cNqeSkNzpOZklvsNJimvLOq5fdlXgVPSf3DsaSXbXDfRm3NpjnI694/BrCOwn7Uqc0ePFJbklTN2gaNRqNOqi7SVTq1QWgzp9buzfZXAz0PCZEdR9YgjV23cBIPv2LfZOH0tGUoLJ6x1b9K5Ocldt3IKGQ0YQOmAQPnXCAIjdvZ1ji95FrTJfPdqVOHF654PXZ3bZNcauAz8DcOeudvWHq0puCemmoKF/CM+vm8HJ+PM2jyEF+dEpV8vFk3SBQCAQCPQp+D2YnpNp8xiTurzA9O4ji2F2AkuURcktMaZnPSb0qc/ynRdZtdu4SBSUHKFPdNfJ7pgtpvOUO0NyazSaIsdtpuJaU+NmJCWwd/pYneQO6fcE9QcPo3bvAbpzTiybx439puMyS3GtM7lzN5ms7Ayb+2VlZ+hiSSm2dAQltZK7qN+Di/atZenBH23uXx52FosV3QKTFNyelsk/Dr+GWNntupxc9EMhkV3cktsYaTHXOPqh4VPH9m/NwLduqFX9NRoN17bt4OKGTbpjXsFBtJ89HTcPD5P97l6P4cza5QCEPfY09Qc+i0xvy6/65dc59OEM7t24xsF3JtP3y18M2gHO/bya2xe0T8fbTX+fqo2b67UOJz3hBgfeep3bF85yafM6Wj39oVWfyZloNBrWb1mI/H616vVbFtKmWW+bVnUnJMew74hW/ufmZrm85JZw9MpugUAgEAhKMyfjzzslnZfI0e0cnCW503OyS8RaiJXdrkXBGk4FRXZJS+78rCxOfPoFaZevPJjjYwOoO+hxq+MgS3Htpc3raDB4mEG7Rq1m/6wJAHgGBNFh1lwq+FbWtTcZNoZru/6Pcz+t5sza5fjUCcOntmGcbE1c60xyc7P4e+93PNnPtrzRf+/9lty8bAD2Hf6Npx+dQvVqIVb3N7YAriTTlUgPfUXtCscjRLfAKMZysNn+rMk6hOx2TbyDAwxkt1Mk9+UYjs5dWOj40bkL6TBnJj4hdSyOEfXNdyT8e8TgWEZcArsnTKX3V8uQK5VG+51ZuwKAhkNfIrTfE4Xa5Uol7Wd+xK6Jw1Hn55FyJpJqLdro2lW5OcTu1hZr6jp3Gd5BNQuN4R1Uk3bT39et+rZGdGemJpJ6ORLPqsH4hTW3eL4xUq+cJvNWHH51W+Hpp82jt+rnmQQFhBIUWJfWTXvp8nhejY3i8rVI4pOukJB8hdj4aBKTY3RjxVw/zaQ5XalZowFBAWHUCAyjbp1WhNRqCoBareZk1D8kJF0mITmG+MTLXLsRhVwmR6VRo9FoSoXklnCk7BYIBCXH8uXLWbRoEYmJibRs2ZLPP/+c9u3bGz03KiqKOXPmEBERwbVr1/jss8+YPHmywTn79u1j0aJFREREkJCQwKZNmxg4cKCuvc9375F2JY6Ti37AOziAVtOG4eZRwegxR7PzxfccPqZAYAxRu6J84SzJ/fwPK1j9askUZRSy27UwJbtLWnKr8/LYPWFqoeMx/9tKzp00mo4abnEMa+LaK3/9Tt0nnjGo4ZRy5iQAcqU7XT9cYjR2rdP7EdLjYonds52zP35Nx9nzDdqtiWttRZWXS3LUQQACmnaxq+6UFNdKtZsORWwhOKi+NiYNCKNZw674+mhrYqXdTeFM9AESkq+QkBxDXMJFrt44q5PVcpmc2fMfoU7NptSoXtfquHbvqL26+ZR0Tm5RqLn4EKlLBIVwRqERkcbE9Wg1bZhOdqeevVLikhvQ3Qy0GD+GvmtW0nfNSlqMHwPAkQ8s55bKy8jQSe7uny2g75qV9PlmBd7BNQC4vnO30X75WZncvXYZgDp6W8IKovT0ot309wE4veZzg7akE9rr+tQJMyq5Jao2bo5fY+uEdVpsNDvfeoyjK6ay58OhXNy21qp++lzYuoY9Hw7l6Iqp7HzrcdJiowH4e+93fP/b+9y+k2hQrMrNTcm6Pxay5e8VHP9vu4HklkhIvsLx/7az5e8VrPtjIUq9Gx25XE7q7QS+2/Aef+/9jjPRB7iXcRuVOl93jjMlt73bvRyRxkQgEJQM69evZ+rUqbz77rucOHGCli1b0q9fP5KTk42en5mZSVhYGAsWLKB6deNFtTIyMmjZsiXLly83eV3fsGBaTx9OelwykZ/8ROrZK8UuuQWCksSR6bzs2b4tKNtIkjs62XiKwOJCpDFxLQqmMXFGupLru/YAUKlObfqsXk7fNSvpseRjAOIP/EtehuW0G9bGtUkRhgu0Lm35VdtvzCSzQrrxc6MAuHMpmvysB79PrY1rbUGVl8v+hSM5tHQCh5ZOYP/Ckajzc20aQz+uBVCp8rh87T/2H/md9VsWsf/oJjw9KunO9/SoxP6jm7RtR37n8rX/UKnyHsxJnc+9jNuciT5gV1zrjMKTjkznZW8ak7KKEN0CA5xZTVvIbtfCzaMCraYNw8O/MicX/kDapdgSldz6ecsC27Yx+tpSzu5bUecACBnwMBV8fQGQyWSEvzkFwCCdiT7Zqbd0r+Vu5p9wewZoJUju3TSD4xmJ8QBUbdLSbH+Aas1bWzwH4OK2tahyc3Tvo35fYlPeco1azdmNS3XvVbnZOlkuk8kYN+Iz+nQzfBJcM6gBH7yxico+AchlpgtmymUKKvsE8MEbmwiuXt+grW/34Ywf8Rkymczo1j5nruQuSm6zospugUBQMnz66aeMGTOGkSNH0qRJE7788ks8PT1Zs8Z4Mbt27dqxaNEinn32WSpUMC6jBwwYwEcffcSgQYPMXluS3WmXYjm58Ac8/CsLyS0oMzi6doWQ3QIJfcm9bsSEEr++vuwWOB992e2MwpOXftsMQOvJ43UpP9x9fAh9rD8AKaejzPa3Ja7NTIo36Jt2Rftv0MM/wOw19CW4fixrS1xrLclRB0m9dFL3PvXSSZLOHLRpjIJxrT7tWw1g2thVKJUP7pWUygpMG7uK9q3My3op3rQlrnWG5JYQsrt4EKJboMOZkltCyG6BI9GotDcMMoWhoNXkq+4fL/qvwIKCuyDulSqZbQfIvXe3yPMoKq+N/JyenYcabQsKDOOjGVvw9ammy82tj1yuwNenGh+9+SdBgWFGx+jZ5VleG/m50TZnSe4VB5cVuZCHkN0CgWuTm5tLREQEffr00R2Ty+X06dOHQ4cOOXFmAkHppzgKNQvZLSgouVvXDHHKPCTZLRBIErpgTKmWYkp58Ws190o+xX4NZyOTyegU/gRTXvkKpVvhVChKN3emvPIVHduYz4tuS1zrTMktIWS34xGiWwC4huSWELLbNZBycmel3KH1jOH41qvFyUU/kHYlrkSur3/DkHT8hNHXlm4q/Fto80TH/G8ruXe1MlmjVnNyqXaref2nja/Eq+hXVfdanZdn9ByJrFvare++dRsYHPeqrk2PknI60mx/gFtR/1k8B6B+/5Eo3B/832w6eLJNN1YyuZymgyfr3ivcK1K//0gAunUYbLZvYLU6DOg5ymibRqPhkV4vE+hf2+wY3ToMpnZwY6vnawpHSe6l+xc7pGq1kN0CgeuSkpKCSqUiMDDQ4HhgYCCJiYnFfn0pJ7dvvVq0njGcrJQ7RH7yE/lZxlcxCQSlDSG7BY7ElOR2VjwocnS7BvrpSvTTmJQU9QYPBCDi4890OaFz0tK4tm0HAFWbmo9vbIlrPQNrGPR199HuSs5MNn/Pos5/ELPqx7K2xLXWEtC0C371HuxI9qvXmsBmXWwao2BcC9qY8vlBs3FTmF557qZQMmzQbKOFJAFqBzexKa51tuSWELLbsQjRLXApyS1RUHYLSh79nNx+TcIMcnaXlOzuMGcmAKdWrGLHqHHsGDWOUytWAdoK1ZZQenlRo2snAPZOfpMdo8axc/QE7l2LBaBWrx5G+7l5eOrE9bl1xre2g/Zm4dSqZQDUe3yIQVtgeEcAbp07xa1zp02OkR5/g7vXC+e9NoZvrYb0mfsn7cd/ykPvrNdJaluo338kD72znvbjP6XP3D/xrdVQ+1msSIFyI+ECMrRPz+VyhW51t1wmJzbhgsX+arWa+MTLuvfGVodbwqGSu9sbQNFvCoTsFggExihYeNKvSZhBzm4huwVlBSG7BY7A3Epusfip/FIwJ3fBnN0lQZ2HewGQHhfPzpfHs2PUOPZN0capQZ07oPTysjiGtXFtYHgHg37NR00E4Niid8nLNJ0L/NrO/wPAN7Qebh4PihpaG9fagkLpTvc319Jp0nI6TVpO9zfXIjeyAtsc+nGtPjesiCkLnqMfU8YnXbYprnUFyS0hZLfjcHP2BATOxRUlt4Qku8d/K0S3MyhYeFLK2R35yU+cXPRDieTr9gmpQ/u3ZhSqUC1VpraGJiOHU6FKFWL+/D/dscr16tJ6ykSzBT2aDh/Lv+9NI3b3drxr1KJO70cM2nPS7nBk/mzUedrCG/7NWhm0K5TuhD06mCt//c6xRe/S9aNleNcwLEp56+wpji1+D4BaPftZ9Xk8/arj6dffqnNN4RfWHL8wwwKYN2/FEljN/M/06o0oXRHJGoF1Ae1Ngkqdz9VY87npAJJvXScvPweZTIZGo6Flkx6cu3jEYj8JR0tu7U2NNu9dUatWS0H+C+tn8vy6Gfz87EJRcFIgcAH8/f1RKBQkJSUZHE9KSjJZaNIRFJTcUk5uKWf3yUU/EPnJT2UuX/eAtYa1Ly78/hNX/vodwKrvwabDxxq0a9Rq/n71WTT5+VSqWYdO7yw0+O7WqNVc3PQzV/7aaHJ1V0kgk8moU7Mpi9/Zafa8xJtXee3tTrrvwdbNenHu4hFycrPQaNQMG/QWA/tPNDvGpm2f8/OmechkcipW8CRy8lmb5urIbdrnp27WHXfE96AU5D+/bgYvrJ/Jj0MX4F3B03JHQanHUroSafHTypHtaF6rslPmKCh5TBWelF5f2bjb4H1xIVcq6bn8U47NW0x63IMc2vWHPEWdfn3M9HyANXFtrZ79UCgNhbF/s9bI3NzQ5OdzdMHbZr8HAZq+OK7Qta2Jaz0rW/UxdMjd3Alq1dO2TgWQ4tpTFd7RfQ/Gxp0nvLn5n+n1+w9Tpe/BhnXbERm1G5lMRl5etk1xrbMk96n4SDoFF35A4ojvwaLGtWUBIbrLMbZK7oycfLwqlOw/GUl2x2F9wT2BYzAmsp0hu33rhtJ3zcoHudFszIEmk8moN+hx6g16HLVKhUwuN5vTS8KndijNRk7gzNrlnPtpNed+Wk3oI0/hXsmHnDupXN2+RXdu9wUrjM6r3sBnuX3xHLcvnOXA26/jFRRMcOeeyNzcuHkqgtT7K72rNGhC4+dH2/S5HE1sfLTZGwKVWkV80hWqVa3JcwNn0aXtQAAOHt/ML5vnk5B0GZVahcLMKu3Y+GgAWjXtyfMDZxNSqynpGXcg2fL8ikdyGyJkt0BQ9nB3dyc8PJxdu3YxcOBAQLu7ZNeuXUycaF4oFgVjkluirMtufYr6PSiTy+n64VL2z5rAvRvX+HvsUKq374JvSD3QaIje8H1JfySjVKtas0jfg3/8vZy/dq7SBe/miI07j1JZkcd6j+GJh8db9R0q4ehcpAURsltgD9bk5JYWPwnZXX4wJbklSlp2u3l40OnDd9BoNGjUauQK23emWopri/o92GzkBHxqhxYaw5q49qm1tj00dSQr5x8v0vegt1dlYmLP8Mvm+Zw8849Nca2tOEpyj1w/zOBhsT5CdhcdIbrLKfas5B7/7TFWvNTOKbIbckv0mgJMCuzilt0xW/ZR97EhhY47osiHrTckNbv1xr2SLyeWzdPO7f82GrT7htan1fg38KhazeT12k1/n3M/ryZ293YyEuK48LvhFqKwRwdTb+Czdt0sORI3C1W409NvM3LI+/Ts8pxB3rRu7Z+iU/jj7D74C+npt/H18Tc5htLNnQ+n/0Gjeu11x7y9KgPm88WVhOSWELJbICh7TJ06lRdffJG2bdvSvn17lixZQkZGBiNHatM/jRgxguDgYObPnw9oC1iePXtW9zouLo7IyEi8vb2pV0+brzU9PZ1Lly7prhETE0NkZCR+fn7Url3bpOSWKC+y2xHfg16BQfRY9BVH5s0m+/YtEo8eJPHoQYNz2rw+u9g+gzUs+/DfIn0PDhv0Fo/0Gs3Rk1stXqth3XYMf/pdqvgG3D9iXc7V4im4dbHQeUJ2C2zB2sKT+jt9hewu+1iS3BIlLbtBu4iqYFFKm8cwEdcW9XswoFVbk9e0FNc6k6J/D0JorWbMfu0nzl06Qo6FlB/6cS3XrZ+nIyV3ff8GZs8TsrtoCNFdDrE3XcnlpHSnyW6Ba1Fcslu6qTEmup1FQKu29Pvmd+5cjib1/BnUubkoK/lSvW1HKlaparG/XKGg6fCxNHp2JEkRR8hM0m538wysQWB4h0Lb05xFq6bmt575+vjTt/sIo21uCqXJNluuYYySlNwSQnYLBP/P3nmHR1G9bfjeTYeQUEISCCX0XgOEJkXwBygqoICIIlWlSBUEVCyIKE1AmoiADUEUsX2iiPROINIDUkMqIRBISN/9/lhm2V4n2U1y7uviunZnzjlzJkCeeZ85532LFwMHDuTmzZvMmjWLxMREmjdvzrZt27QFKq9fv45SJ+iMj4+nRYuHRZYWLFjAggUL6Ny5M7t27QLg2LFjdO368Hfa5MmTAXjppZdYv369TcZ1QZjd7pj3Ww4d9KtQkS4LP9cE+McOkXsvDaW3N+XrN6Fsrbo27dQqSOTQwXKBIfToMtTqGLa0MaRgTG7zCLNbYAu2mtwSwuwuOdhicku4wux2BXLooKW41h2QQwcb1I40e05CP6617WWx3Cb3mgFfAwkW2wuz23GEW1nCcCYn98phrRm97miJM7ujjliucFyQXPWxLFj5eZptT1cPJllsZ/EaD/pePZCIh+fDQL90e8v95Da7dd/cuxsKhYJytetTrrbjZqWHlzeV2z4i46yKP64wuSWE2S0QFC/GjRtnNlWJZF5LhIeHW8333KVLF4ttbDWs5TS78zKziV74Lbw616H+BY0cOuhbrgLhjz0h04xKBoVtcksIs1tgCXtNbomSZna7cxxqSHz0LU2//Y7NWTeutdXkligpZjc4r4NyxLUljYIwuW2Na4XZ7Rglw6kUAM4XnmxStWyJNLu/+9L+yvGFTeKpVBJPpTo1xtG1MXrfu1kxukE+s9toe5pIyV7icaXJLSHMboFAXpYvX878+fNJTEykWbNmfPrpp7Rp08Zs+82bN/P2229z9epV6tSpw8cff8zjjz8soJSUlMQbb7zBX3/9xZ07d+jUqROffvopderUKYzbkQ05zG7J5E6PsyNhs6DY4yqTW0KY3QJTOGpyS5Qks7soxKGGHFp9zqn+iadS6TbFfqNaDrM77XIc5cILrlC1oOjhSpNboiDM7nk8alf/okbxdykFgPMmt0RJNLs/Wuy6N8JR/pbfpKvyVPz46l5aD69HeHvHRPnqgUSOro3hmVWPoNRZ0W2rbe6s2W1rDjZBycEdTG4JYXYLBPKwadMmJk+ezKpVq4iMjGTx4sX06NGDmJgYgoODjdofOHCAQYMGMXfuXHr37s2GDRvo06cPx48fp3HjxqjVavr06YOXlxc///wzAQEBLFq0iO7du3P27FlKlzauZO/OOGN265rcLaa+WMAzFRQVXG1ySxSU2S0omjhrckuUFLPbneNQQ/YuPgnAIxObOnQ93bjWUZwxu9Mux3Fi/tc8ujzC4esLihfuYHJLyG12z2OtQ/MoKjhf3U3g9shlcktIZreUszsjO0+mmbonnl5Kl/3xsPJHMqY9PK23NfvnwRhKgzHs+hk9MLv9w4I5Mf9r0i7H2dTPHU3ubcP7cfCDN7h5MkpbEdsaufczuPTbD2wb3o9tw/vx58sDOLV2OekJtv0cALvaFjTpGXfo/0ol9h5xvEjJ3iNb6P9KJdIz7tjVz51MbokJHV5gaqdhzN+zjiX7v7HewQApyK8XFM7zG6dxwoZq4gJBcWPRokWMGjWKYcOG0bBhQ1atWkWpUqVYu9b0g/aSJUvo2bMnU6dOpUGDBsyePZuWLVuybNkyAC5evMihQ4dYuXIlrVu3pl69eqxcuZLMzEy+++67wrw12ZDM7vS4ZKIXfmtTvm1Dk1vO4tCO4KwORi2eox1j56QRXP3rV3LvZxTgjAsOR3VQlzfm9GD6hz3t7ucuJreEHDooBfkxKVd5YdN0p+YjcA1ymdwSktldK8Sf0euOcir2jizzdCfcOQ41FZcaxpP29gf0Umk6Qo2nOlGzX1cub9nJlV/22NRHMrn9w4xfvBcmzuqgWqXi5sko/pk4TDuGvXGtu+GoDko4Gte6k8ktYaiD6VaKbppCimuLO8LoLubIbXJLlDSzW2Ade81udzS5JdIuXyRq8Rz+HPksGUmWi0TEH9rDjnEvcnHLBu0xdV4ecft2sO/N14haPAdVnvkiF6q8XKJXLWLfm6/JNn9nOXV+HwAbfvrQ4TG+3TIHgNMx+2zu444mt4QwuwUCx8nJySEqKoru3btrjymVSrp3787BgwdN9jl48KBee4AePXpo22dna0xgX9+HzzVKpRIfHx/27bP99467YY/Z7W4mty6O6uDNk1Ha49lptzm/cR07xr1I/MHdFq9nTatdgSM6qEtmVjqXr5/k0rV/ycyy3ex3N5NbQm6zW1C0kNvkligJZrfAfuwxu3VN7uZTBhfSDE3jrA7+9epzRC2eQ87dNO1xe+Jad8NRHdTFkbi2sExuOV76Omp2F3eE0V2MKSiTW0KY3QJDbDW75Ta51SoVKafOEL//IOlxjgl4bnoGCQeP0G7WfJqPmYrSyxuAvTPGknnrpsk+8Qd3c3L1YgDK129MxzlL6fTRCiJnzKH+oOEA3DwZxfGlc02+RVer8jm64D0Sj7iPKaNWq/npjyUA3LodT0LyFbvHSEi6TOodzd/Dlj+WWi3qBu5tckvIbXYLBCWFlJQU8vPzCQkJ0TseEhJCYqLpolWJiYkW29evX59q1aoxY8YMbt++TU5ODh9//DE3btwgIaFoBXKG2GJ2u6vJ7awO1hswhM4LVtNx9hJaTZlFpQfFK09+voT4Q6bNisxbN9k7Y2wB3ZFjOKKDhvy1+0uTny3hria3hJxmt6DoUFAmt4QwuwWmsMXsNjS5Pf18SDh4hMQjx8jNcMxYvXcjjvh9B7l15pxDq6id0UF1nsaPaT31PTrP/8yuuNYdcUQHdXEkri3Mldxy7XByxOwu7giju5hS0Ca3hDC7BYZYM7ttMblTz8WYPG6u7d8jx3Lik2Wc+eIrDr79PtuHj7b54UStVnNm7VfsGv86pz9fR2B4LUJbtaP7sq8pFazJex69Yr5Rv9z7GZz8XCOc9QYMofXU9/CvVIVSwaGUq9OA8Md60+H9TwBIOR1N4tEDRmPEH9rL7QtnAWj39sc233NBkXE/jV0Hv+dK7GkAlAoPNv78Mbdux6Oy4UFNpVJx63Y83/38MUqFBwBXrp9i18HvybifZrZfUTC5JeQ0uwUCgeN4eXmxZcsWLly4QPny5SlVqhQ7d+6kV69eKJVF//HWktntriY34LQO1ujZB7/yQfiHVSWoUXOavTyJev2HAHBy9WLyMo2DucMfzizAO7IPR3VQl9zcbGLjY/hp2zLtsa1/LiM2PobcXPMr/F1hcrtqh5OodVF0KGiTW0KY3QJTWDK7TZncAKc/X8epVV+w67XXOb1mvc0vKnMzMtg+fDSHZn3AmbVfcXzhUv4eOdauuNZZHSxTpTrdln1NhQZN8KtQ0a641p1wVAd1cSSuLex0JXKm8xJmtz7Fu4pgCaWwTG6JkligUmAZcwUqbV3JHTV/MY8snItvubIWr5N1+w5R8xcDUKFxQ4KaNCLmu80A7HrtdR5bu9LqXC/99Cvx+zRb4Wv07qU9rvTyInLGh+ycNJy0K/+RdfsWvuUqaM/H7fsHgEptH6FGzz4mxy5TpTotxk7jxPJ5/PvZIipFdtQ7f2rNUgDavvUxgTXqkJuZwflfV3I/JY7Qpp2o1qEPCoV9hWDSYmO4uE1TOLFOz2EEVrWtoMvQSfW1QbhS6YFKlU++Ko8Dx37mwLGf8fL0ITioKlUr12fogPeoUK4yACmpcazf/A434mNIToklN0//4UOp9GDFlxNZ8SWULhVIpeCaNKrbnuf7ztSaUa40udOz7ztdtdrRApUCQUkhKCgIDw8PkpKS9I4nJSURGmq6kHJoaKjV9hEREURHR5OWlkZOTg4VK1YkMjKSVq1ayX8TLsBUgUrAbU1uU9irg6YI7/k0ySejuB1zhuv/bKPmE/2057JSU8i6favgbsBGFnw20ikdPHzi//hr95fcSLjI7bQk1GqVnv5n3L/D5Pe6oFAoKV82lLDQ2vyv80tEtngccN1KblGoWWCJwjK5JUpKgUqBfZgqUGnO5AZNPKjOz+fqH3+RcOAw/mGVCe/1P6vX2fXa69rP9QcP5Oa/p7h1+qxdcW1gDdNxm6062O7teSi9vIz62xLXBvZ4WIRTrVZzff9WEk/uoVRQGPWfHI2Xn31Fvu2Na53VQZVKxYafPuTMhQMkJF+2Oa798n8rANfk5C6IQs32xrXFlaK/5EWgR2Gb3BJiZbfAEMOV3f8u2WhXupKz67622iZmwyYA6r84iJaTX6PaY4/S/YsV2vP3blgvgnXltz8A6Lz4Y2r3e0rvnE9gWcJ7aI4lHjuk3++PrQCEdehqcfyQiLbaz1l3Uh9+vv3wc2CN2gAcXjGJi9vWE3dsO1FfvMn1/Vutzl+X+6mJ7P5wMLGHfif20O/s/vAF7qeaTgtgiO5KM5Uq3+h8bl428UmXaN6oq9bkBggqH0aLRl2JT7pkFNwbjpVxP42bt2Lp2uE5vRWXrlzJ7WwhD2dWdgsEJQVvb28iIiLYsWOH9phKpWLHjh20a9fOZJ927drptQfYvn27yfaBgYFUrFiRixcvcuzYMZ5++ml5b8CF6K7sjvpwHVEfrrPL5F6+fDnh4eH4+voSGRnJkSNHLLbfvHkz9evXx9fXlyZNmvB///d/eueTkpIYOnQolStXplSpUvTs2ZOLFy9aHNMeHTSFQqGg0ZBXALjwo/7v28QojTaHtulo1K8wcVYHWzbuhpeXL6l3ElCrNSvNdFcRSp/Vas2KcW9vP1o27ga4Nl2JqF0hMEdhm9wSYmW3wBS6K7v/XbLRrMkNULvfU9Tp35dOn2gWpVzc/JPV8dPj4rWfu3+xgqrdutBy8mvUG9QfsC+uNYWtOmjK5JawJa6VuL5/K1FfvEncse1c3LaeIysnWZ2/IfbGtc7oIGjqtHTtMJCbt2LtimvBdYUnC6JQs1jZrUEsuy1GuMrklhAruwWGSGb37tEfkXIihqAW9WzOyX3r9FmrbZKjogGo1K6N9phCoaD+4IGc/3YT965dp0wV80aAKv+h8HkHBJhs4+VfBoDce/pbjrPTbgPgW7aCUR9z5Kbfw7ds+Qef7+rNGSD5zH6QBF2hJPHkHqp37Gvz+KmXosnLeihueVkZpF6KplR565WqywWGcPfeLfJVxi+ppPmNHbqEzm37G53v1nEwnh5eLP9yIoDJLX4eSk8CylTgvdd/olJwDb1zrjK507PTtQ8FjrwBd3Zlt0BQkpg8eTIvvfQSrVq1ok2bNixevJiMjAyGDdNUfh8yZAhhYWHMnTsXgAkTJtC5c2cWLlzIE088wcaNGzl27BirV6/Wjrl582YqVqxItWrVOHXqFBMmTKBPnz7873+alVeDvnrUprld3Z/IodXn6L+mMx5ejq0B2fPJSQA6TWr64Ih8RZYDa4bR9LUBnJinCZRbTLPN5N60aROTJ09m1apVREZGsnjxYnr06EFMTAzBwcFG7Q8cOMCgQYOYO3cuvXv3ZsOGDfTp04fjx4/TuHFj1Go1ffr0wcvLi59//pmAgAAWLVpE9+7dOXv2LKVLW1/tZU0HzaH0MP08mXNPM0bp0MomzxcWzuqgl5cPU175nCVrRnMk+g+zW+UVCgVtWjzOxJEr8fTQGBquzMkt1w4nsbK7eOEqk1tCrOwWmKLGU524eyWelBOaVCKmTG5dfAIDtZ9V+fkoPTzMtr179RoA9Qb119OzsC6PEPPdZrviWnPIoYPW4lqJxJN7QKGEB4Zz0un9qFX5KJTmfwa6qFX5dse1zuigRFhoHWZP+4V3FvS1Oa5NP3/bJSa3hBw6KFZ2GyNWdBcTXG1yS4iV3QJDYrcf1n6+ff6q2QKVhvgFV7TaxstfE1RnJqfoHb8ZrTEcfMqWtdhfobOaylyxkOwHK86U3t4mz3v4+Vmdp4RUCMTceH5lQzQPFQ8oHVTF5rEBSlUwNj5MHTPF7Gm/UKqUabNfrVabNbklOrcbwNihS8w+lJQuHcgH0341Cu4Bl5ncI79/UZaq1c6saBMISgoDBw5kwYIFzJo1i+bNmxMdHc22bdu0BSevX7+uV0Syffv2bNiwgdWrV9OsWTN++OEHtm7dSuPGjbVtEhISePHFF6lfvz7jx4/nxRdf5Lvvviv0eyto8jKzubxll/b75S27TBaoNGTRokWMGjWKYcOG0bBhQ1atWkWpUqVYu3atyfZLliyhZ8+eTJ06lQYNGjB79mxatmzJsmWaHJkXL17k0KFDrFy5ktatW1OvXj1WrlxJZmamzT93azpojuw7miBc6akf1Ho8GMPTRc+9ujijgwBent5MGvUZrZuZfzndullPJo1cpRfcu7rwpNyFmsXK7qKNq01uCbGyW2BI2uU4bp+/qv2uG6OaQvd3ucJK7Q9pwVTKqTN6x+8naHbW2hPXmkMOHbQW10qUCtKJHxVK/MqF2Gxya7p4OBTXOqqDulQKrmFXXOtKk1tCrOyWH2F0FwPcxeSWEGa3QEI3J3fnldPNFqg0RcOh1lcFNRur2cJ16N053LseS35uLte3/8OtM+cAKN/Acn5qhUJBYO2aABye/RHZd4wLRV3/R5PapHz9JnrHKzbT5IBNPLLP4jVy7j4cs1TFhyvo/IJCtJ/vpyQDEDluCX7lgkGhJKRxe+o/9arFsQ0pX7MJTQZORenphdLTiyYDp1K+ZhPrHYGQoGr4+ph/wAqv0sjqGNXDGpo95+tTmuCgqjbNxRpybtOW46FAmN0CgW2MGzeOa9eukZ2dzeHDh4mMjNSe27VrF+vXr9dr379/f2JiYsjOzub06dM8/vjjeufHjx9PbGwsOTk5XLt2jdmzZ+Nth3laFNAtPNlq1khazRppskClITk5OURFRdG9e3ftMaVSSffu3Tl48KDJPgcPHtRrD9CjRw9t++xszfV8fR8+ZyqVSnx8fNi3z7wW2qOD5ki7+h8Aldt31jtevr7mxUfM5q8s9i8snNVBDw9PqlSqi4eJAN7Dw4uqlevhYbCqz1Um98n4aO1nYXYLwH1MbglhdgskdHNyd1453WyBSoDsO2nkpKdzaNYHAPiHVba666hCowaAZkdy7M495GVlce96LIfe/RCwL641ew826KC5hVsS1uJaifpPjiakcXtQKPArF0zbcUutzt8QR+NaR3TQEHviWleb3BLC7JYXlxvd9uQOPHPmDM888wzh4eEoFAoWL15ceBN1U9zN5JYQZrfAsPCkYc5uS2Z3SJsIyteva/Ua5erVIaSNpnDGoXc/5J9XxmuLUUZMm2T17TtAy0njALh3LZY9k6frnbu242Fu0rK19OdTb8BLAMR8/xV3LpvOT5qXncWRBe8CUP2x3npvwpUeHtTo1QeAg+9PJT87m/I1m9Br4T/0XfMvHSavxtPXvqIfoClA+fRnJ3j6sxPU6TnM5n7ZOZmkpGr+ThQP3r57KB8+RFy3QWxjddpIfZUP7vnmrRvk5GbZPB9zyJ2LFOR5KBBmt0AgkBtdk1vKya2bs9uS2Z2SkkJ+fr52xbxESEgIiYmmazckJiZabF+/fn2qVavGjBkzuH37Njk5OXz88cfcuHFDbzW+3j3YqYOmuHfjGuc3alJjGBZ/LlvLtoLLhYUcOhgbfx7Vg+3WCoVSq8kqVZ5NWmwNOXOR6iLM7pKNu5ncEsLsFpgqPKmbs9vQ7N4zeTq7x0/V5t1uPfN1U8PqoVAqiZg6EYDzX3/HzjGTtCa3PXGtszp48Sfzu6tsiWslvPxK02HyavquOUmvhf9QrkZj7MWZuNZZHbQnrnUHk1tCmN3y4VKjW8od+M4773D8+HGaNWtGjx49SE42varj/v371KxZk48++ojQ0NBCnq374a4mt4Qwu0suhia3hK1md9NXR9p8raavjiRi2iTtlrAKjRvyyMK5Nj1QaObkR7fPllJ34DMoPB6IaG4uZ776jHPfrgGg5fiZRm/y/SuFUSnyEQAOffAGqRfOotYpdnE/JZndU18h/YYmZ1utJ43TfkgPKrnp9zg456HJbs/WMFMolEqbTH5d4hP/0xb+CA2uwZghn9CqeQ/t+dg460Kr+9DQunlPxgz5hJCK4YCmcEhc4n92zcmQgiy4JcxugUDgTpgyuSVsNbvlxsvLiy1btnDhwgXKly9PqVKl2LlzJ7169dIWVnRWBzNv3dQ7nxR1iP2zNEWwKjZrZZSDVKFQ0HL8TNnu0Vnk0MErsadRq9X4ePvRv/cU+veego+3H2q1mquxp52an9wFtwwRZnfJxF1NbglhdpdcTJncEubMbikerP1sH7p9thRPG9NUlm9Qj0cWziW4VUsAvPz9afXGZLviWmd18PLvP3Llj63kZWVqz9sa15rC2ZjU0TGc1UF74lpXmdwr9pteJS/MbnlwaaVA3dyBAKtWreL3339n7dq1TJ8+3ah969atad26NYDJ8yUJdze5JUSBypKHOZNbQjK7oxd+y4n5XxsF8I5Qvn5dOn70vsP9lV5eVO/Rneo9urNteD+9c42HjSW4eSuT/ZqMGEfW7VvcvnCWIx+9ZXb8jnM+xftB8Q9dvMsE0HHOp+x78zWtEeAqrsefp2KFqgx88nU6Rj6Dh9KDrh2e49K1f/lu61yux8dYHSM2LoZmDbswqM90alVvBkCndv3Zd/hHNv26gNi489Soav+KAChYk1tCjkIeokClQCBwFksmt4Rkdp+Y/zXRC781CuCDgoLw8PAgKSlJr19SUpLZxSKhoaFW20dERBAdHU1aWho5OTlUrFiRyMhIWrXS6OSfI03XcrBVB3dPNb11O6hxc1qMnWryXHDzVjQeNtbkucLGWR3Myr7PvfRUnnxsNH17jqOMv6ZwZ48uL7F123L+2r2e7Jz7+HjbX2RKbpN7zYCvAeOV/KJAZcnC3U1uCVGgsuRhyeSWkGLVy1t2ar93/3y5w9f0LVeWZmNGOdzfWR08vW45MZu/MpvOy1Jc6y7IoYNyxLXmkMvkXrJ3AVM79jJ5XhSodB6Xreh2JHegI2RnZ3P37l29P0WdomJyS4iV3SUHaya3hD1pTFxF2dr1aP/uQqo80s1sG6WnF22mvU/TURP0im1I1OjVl0eXfmn2LTlo3qA/uvRLwns+Lcu8HaVuzVYseX8fndsNwEPnzXut6s14a8JGBve1vmJucL83eWvCd9rgHsBD6UHndgNY8t5e6tZy7MGqMExuCbGyWyAQuBJbTG4JSyu7vb29iYiIYMeOHdpjKpWKHTt20K5dO5PjtWvXTq89wPbt2022DwwMpGLFily8eJFjx47x9NOmNUwOHWz2ymQiJr5lVIBLF0taXZg4q4P5qjyWvL+fIc/O0gb3AAH+FRjy7CyWvL+fvHz7n6MLwuS2FNyLld0lg6JickuIld0lB1tMbglLaUwKG2d1sP27CylX17hWhC1xrbsghw7KEdeaQk6Te8IjllPiiJXdzuGypbWWcgeePy/fw8zcuXN57733ZBvP1RQ1k1tCrOwu/thqcksUxMpuOeg4ewm+5Svg6WfbG0+FUknldp2p3K4zWXdSyU2/h9LLG7+gYJQetm3V8vYvQ32D3GiFTaXgGhbPV61sPQ+qpTZeXj6EPti+bQ+FaXJLiJXdAoHAFdhjckuYWtktMXnyZF566SVatWpFmzZtWLx4MRkZGdqdlEOGDCEsLIy5c+cCMGHCBDp37szChQt54okn2LhxI8eOHWP16tXaMTdv3kzFihWpVq0ap06dYsKECfTp04f//e9/AHRZtMYpHaz7zAtkpiSjys3By78MvmXLW+/sRjirg6X9AijtF2D2fIVyleyeU2Gb3BJiZXfxpqiZ3BJiZXfxxx6TW0J3ZXet3gMKeopmcVYHA6rVIHL6B+Rl3icr9RaAXXGtOyCHDsoR1xoit8mtiWtN1/mSECu7HafYO40zZsxg8uTJ2u93796lalXLFc/dmaJocksUVbN77apTLrv2HSvxoSpPk3vq0OpzXD9iOre9NeKjNSK4d/FJlJ4PVyU3mGjdrJaw1+SWcEez2z/M8d8PvmXLF7mg3J1xhcktIcxugaDos+eTkza1M6eD9iCNIV3THg0Fx0xuCUOzm1c1xvXAgQO5efMms2bNIjExkebNm7Nt2zbtIpPr169rc2sDtG/fng0bNvDWW28xc+ZM6tSpw9atW2nc+GGqjYSEBCZPnkxSUhKVKlViyJAhvP3229rzzuqg0sOD0iH2m7kC07jK5JYQZnfxpKia3BJFyex25zjUEEMdtBfduLZbe4eGcMjklrAnhi1I5NBBT79S+IcVfzOzsCgYk9s2CsrsLu64LHWJI7kDHcHHx4eAgAC9P0WZompyS4g0JsUPR01uiaKQxkRQ+LjS5JYQaUwExZHly5cTHh6Or68vkZGRHDlyxGL7zZs3U79+fXx9fWnSpAn/93//p3deoVCY/DN//vyCvI1ihTMmt4RuGhNdxo0bx7Vr18jOzubw4cNERkZqz+3atYv169frte/fvz8xMTFkZ2dz+vRpHn/8cb3z48ePJzY2lpycHK5du8bs2bPx9va2e76CgsfVJreESGNSvCjqJreESGPi3jgSDzpjcku4i9ktcB9caXJLFEQak+KOy5bT6uYO7NOnD/Awd+C4ceNcNS23x1Umd3p2lmz/Worayu7hrzZx2bWP+Sssns/PVbF55G7avtyA8A6OvSC6uj+RQ6vP8cjEpnh4PXz3dVNtva+zJreE4cruR5dHODxWQaHKzyczJQlVTg5e/gH4lrN/xVpR3kZmSE5uFgDeXo7/PsrIvEspX+PiZO5gckuIld2C4sSmTZuYPHkyq1atIjIyksWLF9OjRw9iYmIIDg42an/gwAEGDRrE3Llz6d27Nxs2bKBPnz4cP35cu8o3IUG/GN0ff/zBiBEjeOaZZwrlnizRaVJTm9qZ00F7kFawSde0RUNBHpNbQjK73Rk5dDDrdiq56XdRenvjFxRic3oUd0TSQYXC8vOeOdRq0//Q3MXkliiIld1hPOrwfASOUVxMbomisLLbneNQQwx10F5049qjdu70lcPkLko4q4NyxLVyolarndLB+1n3LKY7cRR3MLkl5F7ZXdxxqbtob+7AnJwczp49q/0cFxdHdHQ0/v7+1K5d22X3UZi4yuR+/usVrHlVvq2CRc3sFhgjl8ktoWt2uxM59+5yZdtWrvyx1ehck5Hjqdy2EwqlZWPk7vUrnPn6M9IuXdA7HlC9Fo2HjSGgmuU8Yu7I4HGaOW/+LMFKS9Mk3rzKa2+145E2/Vjz6GLtcXcyuSWE2S0oLixatIhRo0Zpn7NWrVrF77//ztq1a5k+3Xh1x5IlS+jZsydTp04FYPbs2Wzfvp1ly5axatUqAKNdeD///DNdu3alZs2aACxc0bUgb8kydgbpcnLllz1WtVFOk1vC1em/zOGsDqpVKhKO7Ofk6k+MzlXv/gS1nuwv63wLA10dHD9iuUNjLF07li8eXaJ3zN1Mbgm5ze6768c4PSeBfRQnk1uiKJjdJRFpp68t2lhSTG5bdNC7jGWz11Jc22/dWbmmajefrh3nlA7uO/ITyz44REjF6rLNyZ1Mbgk5ze7ijstSl4Amd+CCBQuYNWsWzZs3Jzo62ih3oO5Kofj4eFq0aEGLFi1ISEhgwYIFtGjRgpEjR7rqFoo9um/u5UakMXFv8jKzzZ6T2+SWkMxudyE94Qb/TBhq8mEA4NSapRyZNwtVXq7ZMW7s3cGBd6cYBfcAd69d4sC7U7ixd4dcUy4U4pMuaT8nJF12aIxDUb8BcPLcw+rm7mhyS4g0JoKiTk5ODlFRUXTv3l17TKlU0r17dw4ePGiyz8GDB/XaA/To0cNs+6SkJH7//XdGjBgh38SLKJe37OTKL3vMni8Ik9tdcVYHVXm5RC3+wGRwD3Dt79/5Z8JQuaZbaOjqoLmV2ZZQq9V6Ggrua3JLyJnGRFD4FDeTW0KkMXE/bE1rWVJMblt1MD3B/M/KWlzrSuTQwUPHf5NvPoVkcrsqnVdJqHXh8iW048aNM5uqZNeuXXrfw8PDHfoPIHAMw+1pIP9bPrGy232JXvityQeGgjK5JTz9fED18LsqN5drf/3Dfz9uRaFUUvvZPlTr1gWll5fNY2bdvsPZtV9x68w5/IIr0vClwZRvYLnacs69u+x7UyNKXv4BtJs1j1JBmq39arWatCv/ceiDN7h94SynvlhGs1cmGY2RHH2U0+s0b6erdu1Bg+eGa+etysvl2t//R8z3X3J63XK8ywQS3qJoiM6WP5aiVHpoP48dutiu/lnZ99n6p+bnknYvBXBvk1tC7pXd81gr+xwFAnOkpKSQn5+vXUwgERISwvnzph+SExMTTbZPTEw02f7LL7+kTJky9OvXT55JF2Fq9uvK5S07AeOcnyXJ5LZXB4ObtzIa48SyeaScjgagxdhphES01Z7LuZvGkQXvkn7jWsHfjIwY6uCJ0//Qskk3u8Y4fnoHd+/d0n53hcmdnn2/0Hc4SUG+oPApjia3hFjZ7V4YprU0pZWuMLlvx1zk3+WfkZueQXBEc+o9PxDfcmVt7m8trk2OPuaUDu578zUeXfol3v76qSFtiWtdiRw6+NO2ZfToMgxfOzXJkMJcyS0KNRccLl3RLXBfCjMHm1jZ7Z6kxyUTvfBbvZXdBW1yG5KXmcmOV8bz349bAc2WrYvfb2HHK+PJy8y0aYzUczHsnTKDW2fOAZCZfJOo+Ys5uWqNxX6Xft0MgH+V6nSZ/5n2YQA0xdfK1qxD27c+BiDh8F4yEuP1+qvVao4v1aRdajB4JI1efEXPnFd6elGj59PUfEJjCB1f+qFN9wOgVuXb3NbWMc7EHODW7QSTLxMzMu9y6Wo0+478xPe/LmDPoR9QqfJRqfLZfWgz3/+6gH1HfuLStX+5n3nP+FpqNbduJ3Am5gB/7/2G5esncD8zDQCl0qNImNwScq7sFgiKG2vXrmXw4MH4+hbNgtlyUuOpTlqzW3dld0kyuR3RQUMNSk+I4+bJKAA6vP+JXnAP4B0QSNs35+Llb1zvoTBxVgfXb36H3/7+jKiT24lL/I/cvByjMXLzcohL/I+ok9v57e/P+HLzu9qXzq5aye2qHU72musCeXCVyb1497ZCuY5Y2e1eSGa3qZXdrjC5T65cw7GPF5GbngFAclQ0e6fMIPVcjE39bYlr5dBBUyu2bYlrHUGtUqFWqaw3tDhGviw6eD8zjeXrJ/D33m/simt1Kex0JaJQc8Ehls4KjHBFoRGxstv9aDH1RU7M/1q7sjt2++FCNbkBjnw4HwD/sMpEvDEJdX4+JxYv5961WI5/sow2M6da7K9WqYiavxiAes8PIKxzR+4nJHHo3TkkHYnidtfOlKtnLOyq/Hyu/f07AG1efxcPH9MPT2Vr1qFe/yHEbP6K89+vJ2L8TO25O5cePvRU7/a42TnW6fs8l3/fYvE+JHIzMzi8YhLJZ/bjVzaEyHFLKF/TviI1qZdPcWjZeLLuJBPcqANtx36Cp29p3l2kKRzXLuIpJo5cifJB3vH9R39m8ZpXtf09lMb/L7f83xLyVZoXVAqFgokjV9G+1VOApsjw4jWjORj1y8MxPLy0Dx4qVb5LTe4T8eedKuTh7MpugaCwCAoKwsPDg6SkJL3jSUlJRnm2JUJDQ21uv3fvXmJiYti0aZN8ky7iSFopreyu+lhkiTG5wTEdvHPpAuVqP9xxJQXs9QcNp0wV07k3PX18aTdrvgwzdhxndTD55jW++uF91GqNYVChXGU+mPYLQeU1/0ZSUuN48+MnSb2T8OAaSpQKJSpVvkvTlUgvfUXtCkFBsXj3Nub98zvPd+tVKNcztbLbp554seIqTK3sBgrd5E49F0PSUY3Z3PbdN/ELDiLh4BHOf/0dUfMX033Ncqs1m2yNa53RwT3TXuXKHz9R95nnUTwwgG2Na+3lwh9rObtFUyOi0TMTqdPTvkU8unEtarVTOgiaF8pH//1TL4WJLXHtB40nAK7JyV0QhZrFym4NYkW3QA9XVtMWK7vdi8CaYbSY+iLpccnsHv1RoZvcarWajDiNkLV9/y28/f3xCQwk8m1NwbS0/y5bTWUkvWGv0Lgh1bp3xcPLizLVqtD2XY0h/e/y1Sb7ZaY8NHa8AwItXiM0siMAN6OP6V/7/GkAqlkI7gEUSiX1+g+x2Ebi/K8rST5zANRqMu8kc3j5BJv66XJo2Xiy01I0DxRnDnD+l1Xac62b9eC14Z9qHwYAOrR+mtFDFqFQKFAoFOSr8rQPIABqtYp8VZ72/OgXF2mDe9DkAH5t2FJaNeuhPZafr5/T3JUruZ3Jbebsym5ByWX58uWEh4fj6+tLZGQkR44csdh+8+bN1K9fH19fX5o0acL//d//GbU5d+4cTz31FIGBgZQuXZrWrVtz/fp17Xlvb28iIiLYseNhLmSVSsWOHTto166dyeu2a9dOrz3A9u3bTbb/4osviIiIoFmzZhbvpaShu7J79+iPSozJDY7pYOr5U3rn4g/tBiAwvJbFMXRXp7kCZ3VQ0lal0oPAgIrMmvS9NrgHCCofxqxJ3xNYJgil0kOrvYBLc3KL2hWCgkQyuac9+kShXtdwZbfAteiu7D72/hqOvb+m0HNyn/3yW0BjcpepVgVPX1+qdu1EhcYNAbQ7h81hT1wrhw7ev5ms/WxPXGsrqZdPcvr7BajyclHl5XJq03xSL5+y3lEH3bgWnNNBCV1ttTWuBdcWnpSzdoVY2f0QYXQLtLjS5JYQZrd7EVgzjHL1w7Xfqz4WWWjX1t0GpVAoHn7WEStrW6Wyb98BoGIz/VXPfsEVAchNTzfZT5VjvFXKHHmZpgO7/Adj+JQtZ32M7CybrnU/RafAiFpF5u0ku9KYqFX5ZN1J1uuTkXIDgHYRTzL5lc/x8vQ26vdoh0GMG7oU0P+7kJCOjRu6lK4dnjM67+Xlw5RXPqdty94m+7vK5D4ZH+10IQ9hdgvsZdOmTUyePJl33nmH48eP06xZM3r06EFycrLJ9gcOHGDQoEGMGDGCEydO0KdPH/r06cPp06e1bS5dukTHjh2pX78+u3bt4uTJk7z99ttGKUQmT57M559/zpdffsm5c+cYPXo0GRkZDBumWYUzZMgQZsyYoW0/YcIEtm3bxsKFCzl//jzvvvsux44dM6qtcvfuXTZv3iyKg5tBVzvL1Q8vESY3OKaDhvqrztM8B/oEWh/DlTirg6BZWRYYUJEPpv1C5RBjQyMstDYfvPErgQEV9XZXubLwpCjULCgodE3uiZ17Fvr1dc1ugevx9POhZr8u2u81+3Up1MKTmck3AShVSb9uiWR059y9a7G/PXGtHDqoyn04hj1xra3cvxVv4pj5Qpgmx0gxbu+oDuqiUCjsimtdaXJLCLNbfoTRLQDcw+SWEGa3+3Dllz2knIghqEU9PPx8jHJ2FyRKDw/t5+y0NO1n3QcJ3TamKBNeDYDz327SW/2dcPAwAMGtWprs5+UfYPM8s2+nAuBTtrzece8ymjFy041zdRqSkWjbg0Fo006gVoFCCQolIY07aLel2YJC6UFwow7a/qhVmjGBCSNW4OlhvsBnp7bP0reneeHu12sCndo+a/a8p4cXE0eupFJwTZvnaw45t2nLUbVamN0Ce1i0aBGjRo1i2LBhNGzYkFWrVlGqVCnWrjVdnHTJkiX07NmTqVOn0qBBA2bPnk3Lli1ZtmyZts2bb77J448/zrx582jRogW1atXiqaeeIjhYf5XrwIEDWbBgAbNmzaJ58+ZER0ezbds2bcHJ69evk5CQoG3fvn17NmzYwOrVq2nWrBk//PADW7dupXHjxnrjbty4EbVazaBBg+T6MRUbpJzcHn4+BLWoR8qJGL2c3cUZR3TQq4zp1WaqfMvPgu5QrN5ZHcxX5fHma98SWjHc7BihFcN587Vv9VaxucrklhBmt0BuTJncrogHJbNb4HrSLsdx8tPv8a8agn/VEE5++r1ezu6CpkKjBgDE7d6nPaZWq7mw8QcAAsLDLfa3J66VQwd161bYE9faSvlazfH0LY1C6YFC6YGnb2nK12pu1xh6ce0DHNVBXSqH1LIrrnW1yS0hzG55EUa3wK1MbglDs1tQ+OgWnmw24TltGpPCNLvr9O8LwJ5J07nw/Rb+2/Izuye+AUCNJy1vhQYoU+Xhqrm/R4zh2l87OL7oU85/vRGAeoP6m+znW+6haZ0UdcjiNeL2a3Kv1uj5tN7x0AeFQq7++QvZaXfM9lfl5pJ4ZL/Fa0hU69CHiBFzCGv1GHV7DiNyzCc29dOl7dhPqNtzGGGtHiNixByqdegDgIeH9Zz4mVnpKE28PVcqPbmfZd3I8PDw5F56qt1z1kXuXKRyPBQIs1tgKzk5OURFRdG9e3ftMaVSSffu3Tl48KDJPgcPHtRrD9CjRw9te5VKxe+//07dunXp0aMHwcHBREZGsnXrVpPjjRs3jmvXrpGdnc3hw4eJjHy42njXrl2sX79er33//v2JiYkhOzub06dP8/jjxr97X375Ze7fv09goDxbYosLhoUnm014zmSBSrn537pVen+kLboKhYJ6g/oT1KSR9nunRR8ZtTf802nRR9r2QY0b0uCF5/TG1G2riyM6GNpKv8hWnWcGA3D269UWzey0yxcd+VHJhlw6aMsY9zP1Vw66yuResX+p9rMwuwVyYW4lt6sWP4maUa5Ht/BkxMxhRMwcZrJAZUHScLgmvVbMhu85vnApsTt28feIMdrz/mGVrI5ha1wrhw766izAsieutZVS5UPpPPMbqrZ9gqptn6DzzG8oVd50vRdz6Ma1ujiig7rcTU+1K651B5NbQpjd8iGM7hKOO5rcErpmt6DwMczJrZuzu7DM7uo9H6NSe40Jc23bdq78pqm6Xrlje2r16W3TGF0+fVhN+cLGH7h1+iwAEVMn4luurNl+TV+eBMCJ5fO4d+Oa0Xm1Ws2VP7aScHgvAFUe6aZ33rd8EIE1agNweO5MVLm5RmPk3s/g4Oxp2vbWUCgUVO/Yl8gxn9B4wBQ8fUtb7WOIp29pGg+YQuSYT6jesa92+9zd9FtW+16LO0t+fi4eSk+tweGh9CQ/P5frNyznpgNIu5fCvYzbD+5Fqa2SbSsFUXAL5HkoEGa3wBZSUlLIz8/XrqCWCAkJITEx0WSfxMREi+2Tk5NJT0/no48+omfPnvz111/07duXfv36sXv37oK5EYFVDE1uKV2Jbs7uwlrZ7agOSviWK0vE1ImAJg/p+W8fFhzVHduonwM66Fuugt75al01Rlfq+dNc3fazyevcuXyRQ3OmW72PgsRZHVQ8WNEWG2ddf67Hx+j1sRc5t2nrIsxugbNYSlcidvqWTHRNbiknt27O7sIyux3VQV1siWvl0MGmo4xrONkS19pLYNV6tBr1Ea1GfURg1XrWOxigG9dqvjuug0qlh/bzvfRUu+JaV5nc6dmmPS5hdsuDeEVZgnFnk1tCMrsFhY+pwpOS2X1i/tdEL/y2wIuAKBQKGo8cSr1B/bl15hxqlYqgJo3wKm27wetVujSPrV1JelwCd69exTsggAqNGlitjF2pTQfiD+wk5XQ0+2dNok6/56nYNAJv/zLkZWdx5qvPuB1zBoCmL0/E08+4InvzMVPZPfUV7icnsn3sYJoMG0vZWvVQenqSkZTA0fnvaNtGzphj8z0VBLHxMTSq295im+sPHjxaNunGoD6aPL7fbZ3L0X//5Hqc9QD/RvwFALy9fHnyf6N5svsrnDxnm9FTECa37kONHFWrpSD/+Y3TeGHTdL4Z+BH+Psb/LgQCOVE9yOn49NNPM2mSJpBp3rw5Bw4cYNWqVXTu3NmV0yuxmDK5JSRtvbxlp973gsJRHdSlfIN6dF+znNRzMWTfuUNA9Wr4V7Gea9xZHfQqVZqmL0/k5OrFxGz+ijuXL1C1Sw9KV9JcO/HwPmI2f2XzfRQkzujgt1vmkJRyTRu8W0IyAYKDqjG470y75ih3LlJD5NDBCR1eAGD+nnV63wXFG2s5uVcOa83odUcZs/4oK4a2FiutSwCmTG4JyeyOXvgtJ+Z/XShFnh3VQQlb4lpndbBc3YZUinzEaAxb4tqu0zfb+yORjckvr3ZKB5s26MSv21fx6/ZV5ORm2RXXusrkHvn9i/w6ZJ7J887qoBxxbVFHKEQJxRGT+/Od/zGqa+2Cn5wBTaqW5QLyF1EQWMZc4F3QZnfa5TjKhetvffIqXZrQNq2cGtc/rJJN28okFEolLcfP4MTy+dz89xgXt2zg4pYNRu2avjyRym1N/6z8KlTkkbnL2TtjLOq8PE5+vsSojdLTi44fLMGvQkXbb6YASEy+avGBIO1uCjWqNmZQnxnUqdFCe3zamPVcvHKcDT/NJe1uCoEB5lemJ968whPdRtG353htu3YRT8IV41V+uhS0yS0hzG5BQRMUFISHhwdJSUl6x5OSkggNNb3lMzQ01GL7oKAgPD09adiwoV6bBg0asG/fPgSuwZzJLVHYZjfYr4OGKJRKbZ5SW5FDByu37QRqNSc/X0JS1CGTW68rtTUO7AuTD6f/7pQOtmnei10Hv+fsRdMpjHS5n3WP0UMW0bltf832bCsaKlEwBbeMU8YIs1tgL7YUnpQWPwmzu2RgyeSWcIXZ7YgOGmIprnVWB5sMH2fyJbatca2rcFoHgeeefoNeXUew5Y8ldsW19iKXyX0x5YLFdsLsdg6RuqQE4uhK7uV/X+Tznf8V7OQERYKCSmMiPdS4C0pPLyImzKTjnE+p2qWH3rm6z7xA9+XfmDW5JUqHVKLHmh+ImPQWZWs93NblHVCWiIlv8diq7ygVbF9Os4KgS/uBFs/7+5dj1qTv9UxuiTo1WvLO5M34+1uuBt6l/XMMHfC+RRPAkMIyuSVEGhNBQeLt7U1ERAQ7duzQHlOpVOzYsYN27dqZ7NOuXTu99gDbt2/Xtvf29qZ169bExOivgLlw4QLVq1eX+Q4EtmJLwO2KNCauQA4drNyuM92WfU2DwSPxLPVw9VvF5q145MNlNHuwLdtVOKuDHh6edOv4PGNeWmz1WmOHLuHRDoNsykEqUTAmt3lEGhOBrdhicksY1nASaUyKL9ZMbglXpDFxFbbooNLTfAFGa3Gtq5FDBwMDghg2cLZTca0l5DS51w381mp7kcbEccRr0BKGM+lKxnavw/K/NSs3XLGy21Xk5apcdu38XIXF86o8zdzy81TkOzjP/AdjSGNpsfLbQe6V3bpv7t0N/0phNBryCo2GvOJQf4VSScUmLanYpKXMM5MPDyv5sq2dl2sMXQrb5JYQK7sFBcnkyZN56aWXaNWqFW3atGHx4sVkZGQwbNgwAIYMGUJYWBhz584FYMKECXTu3JmFCxfyxBNPsHHjRo4dO8bq1au1Y06dOpWBAwfSqVMnunbtyrZt2/j111/ZtWuXK25RD3fWUG07czpoB1o9fnC/tq4qk3tl95Vf9lCr9wCnxigI5NBBr1Klqd7tcap3s16Mevny5cyfP5/ExESaNWvGp59+Sps2bcy237x5M2+//TZXr16lTp06fPzxx3qFV4cOHcqXX36p16dHjx5s27ZN75izOiiH1hpS2Ca3hFjZLbCGPSa3RElb2V0UNFTCUAftRTeutcXklnDFym5XYY8OmsPZuLagKYyY014dBflNbk1ca72QtljZ7RjFVxUERjibk1syt0ua2T19ovuvsjq6Noaja63ns7LEj6/u1fvebX0Xq33kMrsNt6cJBK4yuSWE2S0oKAYOHMjNmzeZNWsWiYmJNG/enG3btmkLTl6/fh2lzrbT9u3bs2HDBt566y1mzpxJnTp12Lp1K40bN9a26du3L6tWrWLu3LmMHz+eevXq8eOPP9KxY8dCvz9DioKGShjqoCNsHqkpAGqLhkrIZXZf+WUPl7fsdEujuzDZtGkTkydPZtWqVURGRrJ48WJ69OhBTEwMwcHGL9MPHDjAoEGDmDt3Lr1792bDhg306dOH48eP6/0/69mzJ+vWrdN+9/EpuBolcuEqk1tCmN0CczhickuUJLO7KGmohKSDjnJ0bQydV063K6aUy+zOy8zG2/1/tQsKmYIxuW2nIMzuMB61a4yiRvFUBIERchWeLIlm96CXXPfG64qP5Tfp+Xkqjq6NIbRJecLbhTh0jasHk0g8lUrr4fXw8LQ/m5GzZrfJHGyuW7wgcANcbXJLCLNbUFCMGzeOcePGmTxnahV2//796d+/v8Uxhw8fzvDhw+WYnqy4s4ZKOKuDAIdWawoRtn3Zsdydzprdkslds19Xh65fnFi0aBGjRo3S7pJYtWoVv//+O2vXrmX69OlG7ZcsWULPnj2ZOnUqALNnz2b79u0sW7aMVatWadv5+PiYzaXvjrja5JYQZrfAEGdMbomSYnYXBQ2VcFYHdeNaRxZOOWt252VmE73wW9rMtK/Ir6B442qTW0Jus/vu+jEOzaOoUPzUQGCEXCa3REkzuyPauC6oUftbMbpzNQ8E4e1CCO/g+DwTT6US3j4UD6+HAf5Nte39HTW7bSk0Uthc3LqR0qGVCYloi4eXt839slJTSIw6RM69u3h4e1O+fmPK1qqHQmHbQ6JarQb7nicLDLVazXc/f0Tv7i8T4F/BoTHupt/it79X87ydRT7cxeSWEGa3QOAc7qyhupjSQXu4fiQZQKvF9miohKNmt67JXeOpTi59WXx1+29O6eCd/86TGnOG/JwcvMsEEBrRFt/yttd1yMnJISoqihkzHmqPUqmke/fuHDxousDVwYMHmTx5st6xHj16sHXrVr1ju3btIjg4mHLlyvHoo4/ywQcfUKGCeY10VAd1OfbvXygUCiKaPmZXP3cxuSUKwuyeV8xXoxVX5DC5JUqC2V1UNBSMddBedONaR3HU7JZM7vS4ZIevLQdy6GB+bg5JUYfISIwHcCiudScc1UEJZ+JadzG5JeQ0u4s7xUsJBEbIbXJLlDSzW2Ade81udzS5AS798r32c9WuPWjw/EiUHubzeGXeukn0ivmkXTFdqLXl+JkENzddWVsiOfoYx5d+SL91Zx2btMyc/+8wP/2xlAPHfmHZB9YrX5ti5twnSEq5RovGj9KgdqRNfdzN5JYoKLNb4FpclcNXILCEvWa3kcntYs5/t9bomD06aGo833IViJz5IX4VKprtr8rPByAlJYX8/HxtKiCJkJAQzp83XYQpMTHRZPvExETt9549e9KvXz9q1KjBpUuXmDlzJr169eLgwYN4mHlGcEQHdVGr1Xy84iUAvl8Vb/MLA3czuSXkNrvnYfxvTeDeyGlyS5QEs1tgH/aa3bomd4upLxbiTI35c8QzRsfs0cH/tm7k8u8/mjxvS1zrbjiqg7o4GtcWhsmdnn2/0Hc4SXFtcUeoQDGmoExuCWF2Cwyx1ewuCJNbrVajVqmcEm+1SkWNXn25dfZf7l67TOzOP0mPi6X11PdMjpuRlMDeGWO138N7PIWXfxmy76RyfccfABxf+iGNh42lyiPdTF7zxt4dnF633OE5FwQ79m0A4M7dZDKzMvDzLW2lhz6ZWRncuZusHcuWAN9dTW6JgjC7jy7sLdv8BPZR0nL4bl7c3ehYbp6KwVN3Mm5wQzq1quTQuHuOJbDs27N8O78rXuZSfhTNBUQuxVazuyBMbrVKsxxcoXRsZbscOliv/xDysrPISIwj8ch+sm7fYvfUV3hk7nJKhxj/W1Xl53N0/jsw8lmH5mwLzz33nPZzkyZNaNq0KbVq1WLXrl1062Z8X47ooCHHT/398PPpHUQ0Mf5/bIi7mtwScprdgqJFQZjcEsLsFhhiq9ltaHIH1gxDneecDoJGlxRKpUPGLDiug7cvaBZNVWjQlKAmzcm5d8/muNYdcUQHDXEkri2sldyuSudVEnYWO/6/V+DWFLTJLTGqa23Gdq/D8r8v8vlO0ytaBSULyexOj0smeuG35GVm6523xeTOy8y0+Xp5mZkcmTOfv0eMYceocWwfPpr/tvyiSQViI2lXrrF9+Gj+HjmWev1fpP07C+g4ZykAty+c5b+tG436qFUqrcldKrgSXT9ZS/2BQ6n1xDM0HDyKnmu30GDwSABOr1vO3etXjMa4e/2KNriv2bvgAnR7SEi+wp7DmpUAOTmZ/LX7Sys9jPlr93pycrMA2HPoBxJvXrXY3t1NbgnJ7K4XFM7zG6dxIt706kBLSEF+TMpV2ecnsB3dHL4NGzZk1apVlCpVirVrTa8Q1M3h26BBA2bPnk3Lli1ZtmyZXjsph6/0p1y5coVxO4JiSI2nOlGzX1cub9nJlV+Mi5FZMrmd1cG/R45l+/DRpF0y1i1zqNVqrv7xl9M62GPND9To1Yc6fZ6j+atT+N9nmyhTpToA+9+eqDXidTm3YY02uA8KCsLDw4OkpCS9NklJSWbza4eGhtrVHqBmzZoEBQXx33+mn33t1UFD1Go1m36Zh1LpgVLpwaZf5ln9+3SFye2sDr6waTrp2fftHkOY3UWLgjS5JSSz+1JSOmPWHyUjO69AriMoOkhmt39YMCfmf03a5Ti986ZMbsBpHdw+fDQ7Ro3j7xFjOPD2+3bFtXLoYOup79F66rvU6NnHrrjW3XBEBw1xJK4tzHQlzurg1E7DmL9nHUv2f2N3/+KOeNVZDCksk1tCrOwWGGJuZbetK7l3jp1Mt8+WovTysngdVW4uO8dONjp+5bc/yL6TRqPh1refpV26wpE584yO+1eqQuup73F0/jtc/v1Haj3VXy+3WcrpEwAovbzpOHuxyblW7/Y46XGxxO76k7PfrKbtzLl650+vWwFAvYFDqdHjKQBSL5/i/q04ytdqTqny9ue4y8/NIfnMfgCCG3WwOR/b5xumE594iWs3zqBUKMlXq7Q5zQ5G/UJYpTpUCq5BpeCaNK7XkcAATa64tLspnI7ZR0LyZRKSrxCXcJGrN85qH0SUCiUz5z5O9SqNqBxaSzNGSC1aNHoU5YOVEkXB5JaQc2W3wDW4Uw5fgcAS5lZ227KSWw4dPDJnHpGzphMQXt3qGGe++JKEA4ep1Uv/eo7ooC5KLy/aTP+AHeNeRJWXS8rpaCo2bak9n5+TTezOP7Xfvb29iYiIYMeOHfTp0wcAlUrFjh07zBaAbdeuHTt27GDixInaY9u3b6ddu3Zm7/fGjRvcunWLSpU0K+v2H93qlA5ejT3DpWvRxCddJiH5MrHxMSQmPzRYrlw/xYRZHalSuS6VgmtSOaQmtao3J7xqI8B1K7lF7QqBNQrD5JYQK7sFhphb2W3O5DbEER3UJSMuwa64tm6/542O26ODHecsxb9SFaMxbIlrSzdrqNfnfmoiqZeiKVUhjPI1m1i9f1PYE9c6q4MqlYoTZ/4hIekSCclXbI5rmwY9ob1GYefkFoWaCw6xoruYUdgmt4RY2S0wxHBld+rZy3alK7n+906r17i+YxcAZapXo/ua5Ty2diWdF38MQPy+A+RmZFgdQwrum44ZxWNrV+qdq9CgCeUbaIQ9KUr/weW/B7m8m46aYPHBpcGg4QDc+S+GvMyHb2vzMu9z99olAKp36wXAhT/Wsmv2QI6smMzfbz5JWmyM1fnrkp+bw955wzi4ZCwHl4xl77xhqPJybOr71+4vOR2zj3sZt8lXPVwFk5+fy6Vr/7L38I9s+mU+e4/8RCm/MtrzpfzKsPfIT5pzh3/k0rV/yc/Pfdhflce9jNucjtnHX7u/5Ksf3uP2nURtcA+41OR25A24XCu7Ba7BUg5f3Zy8utiaw/err75ix44dfPzxx+zevZtevXqR/yB3sEDgCIYru20xuZ3VwcfWrqTpmFEAHH7feh7H3IwMo+BeF3t00BRepUrTeup7AJxa+6neuaTjmusGVK+pPTZ58mQ+//xzvvzyS86dO8fo0aPJyMhg2LBhAAwZMkTvRdeECRPYtm0bCxcu5Pz587z77rscO3ZMa4ynp6czdepUDh06xNWrV9mxYwdPP/00tWvXpkePHgBO66Cnpxcbf57HL3+t4Ni/f+oF9xIJyZc59u+f/PLXCjb+PA+vByaFK9OVyLXDydEVbQL3pjBNbgmxsltgiOHK7tSzly2a3M7qYKdPPuKxtSvp/sUK/MMqA/bFtaawVQdNmdwStsS1EmmxMfz9Zm+OrJjMrtkDubhtHfZib1zrjA6CZtFK6u0Evtz8rl1xrYQrCk/KtcNJrOw2RhjdxQhXmdwSwuwWGCKZ3Wn/xXJi3tf4BZW1OSf3xc0/WW3z3w9bAWgxcYw2j5p3QAA1emseplNOnbHYX3ebVkirlibbVGzSAoD7SfF6x9Mua3Yw+AUZ5/PVRdcEz0q9ZfKz0lPT5uyWJdpj+TlZdj9UJJ/ZT+p/J7TfU/87QdLp/Tb1VSgUVvPItWneiymvfI6X18O/Py8vH6a88jltmps3KXTHHz3kE7o/ov+22ZUruR19KJDD7BYUL5577jmeeuopmjRpQp8+ffjtt984evQou3btcvXUBEUcXbPblpzccuig7mdrW5pvnTkHQHiv/5k8b48OmqNUsGYlWM7dNL3jGYkaba7QsJn22MCBA1mwYAGzZs2iefPmREdHs23bNu3LquvXr5OQkKBt3759ezZs2MDq1atp1qwZP/zwA1u3btXm3/fw8ODkyZM89dRT1K1blxEjRhAREcHevXu1efid1cEqlery/us/UTYgGKXCfO5UpcKDsgHBvP/6T4SF1gFwaU5uOdN5CbO7eOEKk1tCmN0CQySz2y+oLCfmfU3af7FWi1Q6qoM+gYGA5nd+xBuTAPviWnPYo4PmsBbXSlzcto78nIfpR8/8uNiu9CZqlcruuNYZHZR4rNOLjBnyiV1xLbjG5JYQZnfBIIzuYoKrTW4JYXYL5ELhYf3Xk7ZolkFBDVWeZgWlM0VEJHLu3bV43rtMgNPXcAdeG/ap2XMKhYJ2EU8x6eXP8PI0ToXi5enNpJc/o23LJy0+VLw27FO6th9odNxVJveK/UudeigQZnfRxF1y+AoEBYmcOmgNdb5pLZYTw8DeEO8yZfS+jxs3jmvXrpGdnc3hw4eJjHxYDHLXrl2sX79er33//v2JiYkhOzub06dP8/jjj2vP+fn58eeff5KcnExOTg5Xr15l9erVers8nNVBgEohNflg2i8EBlREqTT+WSqVHgQGVOSDN36lUsjDFeyuLDwpd+0KYXYXD1xpcksIs1tQmJjTQbWkxXbEteawVwdNjmElrnU1juqgLl07PGdXXOtKk1tCmN3yI4zuYoC7mNwSwuwWSEg5uQNrV6XFtBfJTLljskClKeo829dqm9rP9AEg6uNPtLkws9PSuLZtOwAVGjWw2F/XAEg6dtxkm1tn/gWgVEhlvePeAZq39feTTac6kFDlPdy+7Fu+gsnPqlxNm0bPTNQe8/D2pU7PYRbHNiS4UQfK126h/V6+dgtCGnewqe8jkc9QLcz0z0utVvN835l4ephfcefp4cXgvjPNFgmpFtaQRyKfsWku1pB1m7aTDwXC7C566ObwlZBy+JrLySvl8NXF3hy+/374uF1/DrzzP5pXL0cZXy++Gd3e7v66fwRFG910JZYKVErIoYO6n62Z5UFNNfkxr/z2h8nz9uigOTJvJQMQWKuu3vHSoRptTjkVbbF/YSCHDoZUrE6vrsNNnlOr1Tz+6AhCgqrpHXeVyZ2enQ4Is1ugjzuY3BLC7BZISDm5M1Pu0GLaiwTWrmqyQKUujupgzl2NmaxWqTixRFNs2Z641hxy6KC1uFaiTs9heHj7ar83emaiXS/OFUqlw3GtIzpoiD1xratNbglhdsuLMLqLOO5mcksIs1tgWHiyfMOaejm7rZndVR/tbPUa1f/3KADpcfH8PWIM24ePZs+k6QBUah+JV+nSVseInKVpf3LF52wfPlrvXHr8De5e1+QHC4mI1DvXZLgmd+fR+e+Qe998DtRrf/8fAIE1auPp97DAhKdfKe2DyrmNawHNQ0WXtzfRZswius/5lcCq9azOXxcPL286vbGOdhOW027Ccjq9sQ6liRXYplCpVMQnXtJ+N3yLfiPhgtUxDNvojhGfdAmVDBW95c5FCs4/FAizu+jhihy+9lLax5MVQ1tTK8Sf0euOcir2jtP3LSh6GObkNszZbQpndXD78NGcXKHZztvmzWlW+3uVLk3ljuZf+tijg6ZQ5eZy8vOlANR+coDeuZCItgDcOnfS6jwLAzl08EbCBRQotP2lMZQKJbEmtNhVJvfI7x8WHhVmtwDcy+SWEGa3wLDwZPmGNfVydhua3c7q4O6Jb7B9+Gj+HjmWe9diAfviWlPYqoO3zp0yO4Ytca1EYNV6dJ/zK23GLKLL25vsXnwFzsW19uqgIfbEte5gcksIs1s+hNFdhHFXk1tCmN0lF0OTW8rJbVig0pzZ3XX5J1YrU4Mm72fX5Yu0hT4k6gzoR6MRL9k014Dw6iYfYG6dPcm+tzSiV7VrD73K1ABBjVug8NRUcj/y0VtGq9HUKhUXfvyGmO+/BKDRS/omOkCjF18BIHbnn1zboTECytdsQpXWPa1WpjaH0tObSs27Uql5V5tNboDkW9fJzcvWbrlu1rAzvj6lUSg0MhEbZz1ovf4gsFUolPj5+tO0QacH3xXk5mZx81asvbejR0EW3BJmd8nCFTl8HUGY3SUbc4UnbTG75dDByFnTCaxVw6YxGg57kRpP6u8ecFQHdclOu8O+tyegytUUVg5q3FzvvIeXNzWfkGe3kBzIoYNXb5zRFs+qHFKLyiG1AE1Ry6uxlnOu24Kc27R1EWZ3ycYdTW4JYXaXXAxNbiknt2GBSnMru53VwbK1a9kV1zqrg0fnv0N6/A2jMWyJaw0pVT6UKq17Ur5mE6tzN4ejca2zOmhPXOsqk/tkfLTJ48LslgdPV09A4BjubnJLjOpaG4Dlf1/U+y4ovpgzuSUks/vE/K+JXvityTaefr7YiqefH+1mv41arUatUqF0IEdoYK0aPLZ2JWqViit//sLNk1GkPngjXq5uQxo8P9Koj0KppOPsJeydMZZ7N67x1ysDCW3TgcDw2qBWE7P5K23bxsPGElDN+CEpoFoNGg8by+l1yzn37RoadJ9s99zlIjZeUwm7eaOuPN9nJuFVG5GecYef/1rO739/rg3eLY4Rdx4vL196dxvFU/8bg3/pslyJPc13W+dy4vQ/xMbHEFKxukPzK0iTW2JCB01xsPl71ul9txUpyH9h03Se3ziNDc/No0Xl+nbPU1A4jBs3Trsi2xBTBST79+9P//79TbaXcvgWBJLZPWb9UUavO8rKYa1pUrVsgVxL4D6YM7klpGOXt+zU+w7w2NqVdl9PVwfB/tzeCoWC2n2fJHrVQqd08Ny3a6jxeD+8ywSQfSeVq3/+om3b6aMVJudVu89z3L54zq75FhTO6mC+Kp/4pMtUrFCFQX1m0KFVHwD2H9vKd1vnkpB0iXxVPh4mcpfagty5SA2RQwelIP/5jdN4YdN0vhn4Ef4+pax3FLgMdza5JSSze/S6o4xZf5QVQ1tT2kdYIcUZcya3hGR2Ry/8lhPzv9a26b5Gk27EUR2s3fdJVPn5KJRKqwURDXFWB29fOMu+t8ZTulIYYe27ovD0tDmudRfk0EE54lpzyGVyD9s0mPOTt5o8L4cOOhvXFnXEb/ciSFExuSWE2V1ysGZyS9hidtuLQqFwuhCWQqkkZtN67feaTzxD7T7PmTXPS4dUovP8zzj84Uyybt8i8ch+Eo/s12vTcvxMgpu3MnvNKo90w7tMIMeXfujU3J3Fy9Ob2VN/pn7tNtpj/qXLMrjvmzz+6EiOnDCde1WXerVa8+Kz71AuMFh7rEbVxsx87VvO/XeYbAdXZhWGyS0hzG6BOyLM7pKFNZNbwpLZ7SjOBrty6OCV/9uidz6wRh2aj3kdvwoVTfZXenjQeup7Ts1bLpzVwfT02wwb8B5dOwzSq4vxSJt+tIt4kp37vyM9/TaBAUF2z61gCm5dNGonzO6SRVEwuSWE2V1ysGZyS5gyu8uFO7arVhdHFl7p4qgOntuwhtidf5KREMeFH/VX8lqLa90FOXRQjrjW5NxkNLnrBNW12E6Y3c4hfrMXMYqayS0hzO7ij60mt0RBmN1yUPvpgZQKqUxIRKTRti5T+FWoSJeFn2uM7mOHyL2XhtLbm/L1m1C2Vl2b3uQHN29Fjy9+lGP6DtO8UVez58oFhtCjy1CrY1hq06B2pNlzlihMk1tCmN0Cd0SY3SUDW01uiYIwu52h/qARTungnUsxpJ4/jSonB68ygYS2aotvuQpW+7tL4O6sDgYGBPFYpyEmz3l6eJk9Z42CMbnNI8zukkFRMrklhNld/LHV5JYwNLsfXR5RSDM1xlkdbPTiK9R/bhhJUYe5nxQPYFdc6w7IoYNyxLWGyG1yrxnwNZBgsb0wux1H/FYvYhRFk1uiqJrdUUcSXXbtqz6Wg8P8PM0W46sHkxy/xoO+Vw8k4uH5cCVX6fa2j2GvyS3hjmZ37acHOtTPt1wFwh97wuHr2ru1rSTgCpNbQpjdAnekKJrd7qyh2nZmdNAe4qNvacbYr7lfezRUwl6TW8LQ7K7Ve4Cl5gWKszpYrnZ9ytUWvyvlpLBNbglhdhdviqLJLVGUzO6ioKEShjpoL7pxbSMHNBTsN7kldM1uVyKHDnp4eVO57SMyzkpQECa3rXGtMLsdwz1/owvMUlRNbomiaHZ/96X7F5VLPJVK4qlUp8Y4ujZG73s3Gx8wHDW5JUyZ3d6uX9gtcDGuNLklhNktcEeKmtldFDRUwlAHHeHQak2uaFs1VMJRk1tC1+x2pdEtcC9cZXJLCLO7eFKUTW6JomJ2FyUNlZB00FEST6VSLzPb7pjSUZNbQjK7BQJdXGlySxSE2T2PR+3qX9Rwv9/mAosUZZNboqiZ3R8tdt024Ch/y2/SVXkqfnx1L62H1yO8vWP5xK4eSOTo2hieWfUISp2VbLbY5s6a3BKGZnebmTMdGkdQPHAHk1tCmN0Cd6Qomd3urKES5nTQHvYuPgnAIxObArZpqISzJreEq9OWuJpn1ruuIOUgyrns2uZwtcktUVBmt8A1FAeTW6IomN1FQUMlDHXQXnTjWnt3+jprckt4+vmAyqGugmKIO5jcEnKb3fNY69A8igru9ZtcYBVXmdyLd2/j+W7yjVeUzG5PL9dVIfbwsu0Bw8NTiYeD85S2aSsNx1Bb7ieXyS2ha3a7G+kJccRsWs/Nk1EA+ASWo0avPoR1fBSvUqWt9lerVKScPsGptcvIuZsGQGDNOtR+agBBjVu4VaVrW3ljTg8UCgUfzdzmUP/0jDsMm9yA8SOW07Tik9rj7mRySwizW+COFBWzu6hoKJjQQTuQDHKpf9qlOJsCbblMbokaT3VyyyBdDh3MvZ/B9X/+4OKWDQAoPD2p3LYzNXr1wb+SY6aGK9HVwUfa9HNojL1HtuhpqIS7mNwSBWF2H13Y26k5CeynOJncEu5udhcFDZUw1EGHr+upJD0u2WazWy6Tuygghw6ai2sb9nBdbLL3yBandHDpF2NZt+gc/qXLyjYndzK5JeQ0u4s7Rc9dERQ60kON3IzqWpux3euw/O+LfL7zP9nHFxQccpvcEpLZ7S6o8nKJXrWIfW++pn0YAMhOu835jevYMe5F4g/utjhGRlICf736HFGL52iDe4C0yxeJWjyHP0c+S0aS5UIU7kZmVjqXr5/k0rV/yczKcGiMU+f3AbDhpw+1x9zR5JaY0OEFpnYaxvw961iy/xvrHQyQgvx6QeE8v3EaJ+KL3lZUgfshmd21QvwZve4op2LvuHpKggecmP81aZfjLLaR2+R2V+TQwfhDe9gx7kVtcA+gzssjbt8O9r35GlGL5xTY/AsKUzpoL99uMb5vdzO5JeTQQSnIj0m5KsucBPZR3ExuCcnsvpSUzpj1R8nIznP1lEo8Laa+qDW78zKzzbYrSSa3LTqoyss1299aXOtK5NDB0zH75JpOoZnczurgC5umk5593+4xSoLZLYxugUV039wXBMLsdl+u/LLH5PGCMrklTD2g5KZnkHDwCIlHjpGb4Zixeu9GHPH7DnLrzDnUKuvL3dSqfI4ueI/EIxrRrDdgCJ0XrKbj7CW0mjKLSg+KfJz8fAnxh0z/rDJv3WTvjLGo8zQPzK2nvkfn+Z/RbtZ8mo+ZivJB9eu9M8aSeeumQ/flCv7a/aXJz7aiVqv56Y8lANy6rakI7s4mt4TcZrdAIAfC7HZP/MOCLZrdrjC57dVBQ9QqFSmnzhC//yDpcba9oJVDB+MP7ubk6sUAlK/fmI5zltLpoxVEzphD/UHDAfSC9qKAoQ4mJF+xe4yEpMuk3tH/e3BXk1tCTrNbUPgUR5NbQpjd7oW0+MmS2e0Kk9sRHTTEUlzrrA4eXzrXpL7bGte6Cjl0cMsfS1GrrWxJt4HCXMktx0tfR83u4o777MsRuB2G29OS+b8CuU5RSmNSkri8ZSegn/OzoE1uQ9RqNWfXfU38voN6xyu1j6TRiJdQKKxvqcvNyGDXa68bHY+YOpHyDeqZ7Rd/aC+3L5wFoN3bHxNYo472nH9YVYIaNSegag1iNn/FydWLCW7WCk8//a1Dhz/U5BovU6U6baZ/oE1z4lehIoHhtQhu1op9b0/gfnIi0Svm022We1fIzs3NJvHmVX7atkx7bOufy2jZpBuhFcPx8rL+7yHjfhpHordxJfY0AEqFR5EwuSXkTGMiEMhFUUljUpJoPmUw0Qu/5cT8r40C8MI2uR3VQV1Sz8UQNX+x0fEuny7Aq7T5FF726mC7t/UNzNz7GZz8XGMI1xswhPAeT2u1v1RwKOXqNKBCgybsnzXJpvsoKGLjY5zSwY0/f8yQZ2dRLjAUpZU0LiqVittpiXz388coFR7a464wuZfs/8Yl6bxE+i/X4CqT+8SNq4SFF/x13D2NSUnDsIaTbuzpCpPbUR2UsCWudVYHU05Hk3j0AJUiO+qNYUtc60qc1cF8dR5Xrp9i18HvadO8J6VLBVq9phTXNqWm9lhhpyuRXvqKQs3yI35zC0xS2DnYhNntftTs11XP7C5skxvg0k+/ah8GavTuhTo/n6t//EXCgcP4h1UmvNf/rI6hG9zXHzyQm/+e4tbps0TNX8wjC+fiW66syX6n1iwFoO1b+g8DuoT3fJrkk1HcjjnD9X+2UfOJh7nFslJTyLp9C4B2b89D6eVl1F/p5UXkjA/ZOWk4aVes72hQq9Vc37+VxJN7KBUURv0nR+PlZ/3BSpfczAzO/7qS+ylxhDbtRLUOfVAoFAydVJ9KwTVpVLc9z/edqX3AOHzi//hr95fcSLjI7bQk1GqV3guGjPt3mPxeFxQKJeXLhhIWWpv/dX6JyBaPA5oHkQ0/fciZCwdISL5Mxn3NtnWl0gOVKp98VZ5LTe707PtOV612NMgXFD4J7y+z3kgm0rOzeP7rFcQkJ7BxyNgCD9KF2e1eePr5mDS7XbGS21EdlMi6fUcb3Fdo3JCgJo2I+W6zduzH1q403c8BHcy6fQvfchW05+P2/QNApbaPUKNnH5PXKVOlOi3G2r5LxpwO2kNabAwXt2k0oE7PYUx+r69TOnjg2M8cOPYzXp4+BAdVpWrl+gwd8B4VylUGICU1jvWb3+FGfAzJKbHk5umvbnTVSm5Ru0JQ0Jy4cZXnvlrO3lldC+V6psxuhHfkMkyZ3UChm9yO6qAutsS1zurgieXz+PezRUZGty1xrb0Y6mBgVdtenEvoxrVy6KBS6cGKLyey4ksoXSrQ5rj2ienXAdfk5C6IQs3C7NYgUpcIjHBVoRGRxsS9qPFUJ63Z/e+SjYVucgNc+e0PADov/pja/Z6iTv++dPpEYxBe3PyT1f7pcfHaz92/WEHVbl1oOfk16g3qD8DZdaYLX2bdTtV+Dqxh/qWLQqGg0ZBXALjwo34qi8SoQwCEtuloMriX8AksS3iPp6zciYbr+7cS9cWbxB3bzsVt6zmy0v4VbIdXTOLitvXEHdtO1Bdvcn3/VkCzwuzmrVi6dnhO7y16y8bd8PLyJfVOAmq1Ziuc7rYw6bNareLW7Xi8vf1o2fhh5VqlUknXDgO5eStWG9wDqFT52s+uXMntTG4zZ9OYCIovhiZ3YRWSFmlM3AvJ7JbSmPy7ZGOhm9yO6qAuMRs2AVD/xUG0nPwa1R57lO5frNCev3fDdHoWR3Qw8dghvXNX/tgKQFgHyyZXSERbyzehgzkdtJX7qYns/nAwsYd+J/bQ7+z+UGPyOqODErl52cQnXaJ5o67a4B4gqHwYLRp1JT7pktuY3ICoXSEoUCSTu15wpUK9rmEaE4Fr0U1jEvXhOqI+XFfoObkd1UFdbI1r5dDBrDsPY1lb41p7MKWD91MT7RpDN67VxREdBH1NtSeuBdcVnpS7doVIY/IQYXQL9HB1NW1hdrsXNZ7qRFCLeqSciCE/M7tQTW5V/kOx8g4I0H72CQw02cYUd69eA6DeoP56q7XCumhShNw6fdZkv9z0u9rP1lZ5KT1Mb4zJuacZo3RoZZPndfHyL2O1DUDiyT2gUIJaBWoVSaf3ozYRKJtDrcon+cx+bX8USs2YQLnAEGZP+4WwUP0HIC8vH6a88jmRLR63+LNQKBREtnyCKa98brR1Oyy0DrOn/UK5wBA8lMY/L1eZ3OnZ6U4X8nAmyBcUT1xlcksIs9u9kMzu/MxsUk7EENSiXqEWnnRUB3VJjooGoFK7NtpjCoWC+oMHAnDv2nWT/RzRwdx7aXrHs9NuA+BbtoJRH0ewpIO2knopmrys+6hV+ahV+eQ9KMrsjA5K/RUKBWOHLqFbx+eNznfrOJixLy3WtpNwZU5uUahZUFDomtwbXhxT6NfXNbsFriewZhhNXxtAemwS6bFJNH1tQKEWnnRUByXsiWvl0MHc9Hs6n22Pa23FlA6mXoq2awy9uFZnfo7ooC4eSk+74lpXmdwSwuwuGITRLdDiapNbQpjd7kPa5Thun7+q/R67/XChXVuh8/ZVt6iG7mpihZX8XdKDRMqpM3rH7ydo3jj7BVc02U/p7W3zPLPvaB4+lJ76q9U8Hozh6eNrfQydN+2WKBWk80CnUOJXLgSF0sN8BwMUSg/8yoZoHioeUDqoCgAfTPuVSsE1TPbz8vRm0qjPaN3M/O+F1s16MmnkKjw9TK/aqxRcg9nTfqFUqQCjc64yuUd+/6IsVauF2S2QcLXJLWFodgtci6523j5/1WyByoLAUR3UxctfkyIrMzlF7/jN6JMA+JQta7KfIzpoTn89/PysjmELlnTQVkpVMG2uOKODoHm+GTt0CZ3b9jd77c7tBjB26BK9ZyFXF54UZrdAbgxNbn8bfocUBJLZLXA9eZnZXN6yS/v98pZdJgtUFhSO6qCEPXGtHDooFXq2NJ4zmNJBc9podowg4/aO6qAupUsH2hXXutLklhBmt/wIo1sAuI/JLSHMbtejm5O788rp2jQmV36xb+WToygUCgJra4pDHJ79Edl30shJT+fQrA8A8A+rbPWtdIVGDQDNirXYnXvIy8ri3vVYDr37IQANh5rOKekXFKL9fD8l2eI10q5q/n1Wbt9Z73j5+o0BiNn8lcnq17pc/+cPi+cl6j85mpDG7UGhwK9cMG3HLbWpny6R45bgVy4YFEpCGren/lOvAhAcVNViPw8PT6pUqouHiQDew8OLqpXr4WFmdbtESFA1fH3syyluCjm3acvxUCDMbgG4j8ktoWt2C1yHbk7uziuna9OYFJbZ7agO6tJsrCZF16F353Dveiz5ublc3/4Pt86cAzBb0NIRHSxfv4ne8YrNWgGQeGSfxf45d9MsntfFnA7aSvmaTWgycCpKTy+Unl40GThVNh0Mr9LI6vWrhzXU++4qk/tkfLT2szC7BXJhzuR2VTwoal24Ht3Ck61mjaTVrJGkxyUTvfDbQjO7HdVBCXviWjl0sFTFYO1ne+JaWzGlg+VrNrHeUQfduFYXR3RQF1+f0nbFta42uSWE2S0vwugWuJ3JLaFrdgsKH8Oc3Lo5uwvL7G45aRwA967FsmfydHaPn6rNN9p65uuWugKaN+MRUycCcP7r79g5ZpI2uA9pE0H5+nVN9lN6eFCjVx8ADr4/lfxs0w9R925c4/xGTREOw+IgZWs9fOC5+NN3Zud4bcf/Wb0PCS+/0nSYvJq+a07Sa+E/lKvR2Oa+EuVrNqHXwn/ou+ZfOkxejaevJuDOyc2y2jc2/jwqVR4ACoUSxYMVcSpVHtdtEOPsnExSUuO0/QGzW7jNIXcuUpDnoUCY3SUbW0zujOy8Qp+XZHYLXINh4UnDnN2FYXY7qoO6lKtXh5A2EQAcevdD/nllvLYIV8S0SWZ3Vzmig2Vr6c+n3oCXAIj5/ivuXDb9PJiXncWRBe9avQ8JczpoD3V6DuPpz07w9GcnqNNzmGw6aMsYsTpt7NVQkHebti7C7BY4i6WV3GLxU8lE1+SWcnLr5uwuLLPbUR3Uxda41lkdrP5Yb70dv7bGtfZiqIP2ohvXyqGDygf3fPPWDbviWncwuSWE2S0fwugu4biryS0hmd2CwsdU4cnCNrs9/fzo9tlS6g58BoWH5tdV7Wf70O2zpXjauH2rfIN6PLJwLsGtWgLg5e9Pqzcm0/TVkRb7ScZ1bvo9Ds6ZTuatm3rnk6IOsX+WphhkxWatjHKQKhQKWo6fCcDl33/kyh9bycvK1J5X5eZy5qvPOPftGgBtW1uwJ12JrWPEJVoPHq7EnkatVuPj7Uf/3lPo33sKPt5+qNVqrsaetto/PvE/beGP0OAajBnyCa2a97B5zgVZcEuY3QJHsXUl95j1R11mdgsKH0OTW8IVZrejOqhL01dHEjFtkjbVSYXGDXlk4VyLRrkjOmi4U8u/UhiVIjX5xA998AapF87q1aa4n5LM7qmvkH7jms33op2fk1qqUCq15oZcOhgbZz2o1TUBWje379ld7lykhgizW+Ao1tKViJ2+JQ9TJreEK8xuR3RQF1viWjl0sNaTxmk/bIlrHUFXBx1FofRwWgfHDPmEkIrhgKYwtD1xratM7hX7Te/MFma3PIjopwRjr8l9KvaOS7ZvjepamwvkFPp1SzrmCk9KAfvlLTv1vstFXmY23jqXVXp5Ub1Hd6r36O7wmL7lytJszCi7+niXCaDjnE/Z9+ZrpN+4xu6pr5hsF9S4OS3GTjV5Lrh5KxoPG8vpdcuJ2fwVMZu/Mtmu8bCxBDdvZdf85CY27jw1qppfIZ6VfZ976ak8+dho+vYcRxn/8gD06PISW7ct56/d68nOuY+PdymzY1yPP0/FClUZ+OTrdIx8Bg+lB107PAdXcq3OryBNbgnpoeD5jdN4YdN0vhn4Ef4+5u/HFBM6aNIAzN+zTu+7oHhiT7qSS0npjFl/lBVDWwvzuQRgyuSWkMzu6IXfcmL+10YBfEHgiA4aUr5+XTp+9L5dfeTQwSYjxpF1+xa3L5zlyEdv2T3vwuDJx0Y7pYOXrv3Ld1vncj0+xuq1YuNiaNawC4P6TKdW9WY2aSgUTMEtSDBq46wOSkH+C5um8/zGaWx4bh4tKte3e66CooMtOblHddUUlZN2+krfBcUTSya3hGR2n5j/NdELvzUbu8qJIzqoi7W41lkd7DjnU7wfFHfWxZa4tsa6obbdRAHw+itrnNNBoFO7/uw7/CObfl1gV1xrL3KZ3Ev2LmBqx14mz8uhg3LEtUUZsaK7hOLISu7R645yKvZOwU5M4DZYelAoqJXd0kONu+BfKYxHl35JeM+nTZ5v9spkIia+ZVSIUpcqj3Sj/bsLKVfXOJdY2dr1aP/uQqo80k22OTtK3VqWjfZ8VR5L3t/PkGdnaU1ugAD/Cgx5dhZL3t9PXr7l1ap1a7Ziyfv76NxuAB52rKQrDJNbQqzsFtiKvTm5Vw5rrTW7XbGyW1C4mDO5JVyxsttVOKuDSk8v2kx7n6ajJugVkZSo0asvjy79UtY524uzOlirejPemrCRwX2t7+4a3O9N3prwnTa4t4WCMLktBfdiZbfAVuwpPClqOJUcrJncEq5Y2e0KbNVB/0rmf1bW4lpX46wOeig96NxuAEve22tXXGsPcprcEx6xnIZVrOx2DrGkqATiaLqSWiH+jF53lJXDWovCHALZV3brvrl3J7z9y1B/wEvUfeYFMlOSUeXm4OVfBt+y5a13fkBAtRpETv+AvMz7ZKXeAsC3fAU8/dznrWrog+1e5ijtF0BpvwCz5yuUq2T1GuaqX1uiME1uCbGyW2ANRwpPNqlalpXDWjN63VGxsrsEYIsmFsTK7rTLcZQLD3VqjILAWR1UKJVUbteZyu06k3Unldz0eyi9vPELCkbp4Xw6LzmQQwerVrZc0MzWNroUtsktIVZ2C6xhj8ktIVZ2lwxsMbklXLGy2xXIoYNyxLUFjbM66OXl43Rcawq5TW5NXGu5Fp1Y2e04IsIqYTiTk3vF0NaMWX+0xJnda1edctm171jRLFWeJs/jodXnuH7EMYM4PloTcO5dfBKl58M3xA0mWg/S5TK7DbenuSNKDw9Kh1gPYi3h6VcK/7DiLyxy4QqTW0KY3QJzOGJyS5REs9udNVTCnA7agzTGnk9OArZpKMhrdqddjuPE/K95dHmEQ/0LAzl00LdsebcKyt0ZV5ncEsLsFpjDEZNboiSZ3UVBQyUMddBedOPaVrNG2qWFJcXslnBWB+WIa0sSBWNy20ZBmd3FHZG6pAThbOHJ0j6erBjaWruyW6QxEYDzaUxsycEmKHm40uSWEGlMBIY4Y3JLSGa3SGMikJAjjYlkcvuHBRfADAVFEVeb3BIijYnAEGdMbgmRxqR440g8KEcaEzlTcgqKB640uSUKIo1Jcad4LyMSaHHW5JaQzO6StLJ7+KtNXHbtY/4Ki+fzc1VsHrmbti83ILyDY1uVr+5P5NDqczwysSkeXg/ffd1U2z6Goyu73dHkTo+LdTitSNbtVHLT76L09sYvKMRttlM7QkbmXUr5ljGq/G0rarWa+1n37N4WBu5hckuIld0CCTlMbomStLLbnTVUwpwO2oO0gq3TpKaAfRoKzq3s1jW5m08ZbN+FZcZZHVTl55OZkoQqJwcv/wB8yxXdldvO6KBETm4WAN5e9hmB7mJySxTEyu4wHnV4PgLXIYfJLVESVnYXBQ2VMNRBe9GNax3FmZXdV37Zw+UtO6nVe4DD15cDOXTQndNl2oujOqiLo3GtO5jcEnKv7C7uFM+oSqCHXCa3REk0uwXWsdfsdkeTG2Df2xMACKhei8bDxhBQzXI+TbVKRcKR/Zxc/YnRuerdn6DWk/3xLmM5yM25d5cr27bSYsCHjk9cRhJvXuW1t9rxSJt+jB+x3KExlq4dy74jP7Hsg0OEVKxucz93MrklhNktkNPklihJZrfANhwxuw1Nbk8/H1AV0oRNsGvKSL3v9urglT+2Gp1rMnI8ldt2QqEsWhtRHdVBXQaP0zyDbP4sweY+7mZyS8htdt9dP8bpOQkKFzlNbomSYHYL7MMRs1syuWv261pIszRNzOavnNLBu9evcObrz0i7dEHvuK1xrTviiA7q4mhc604mt4ScZndxp2g9MQrsRm6TW0KkMRGYwtY0Ju5qcuty99olDrw7hRt7d5hto8rLJWrxByZNboBrf//OPxOGkp5gfht6esIN/pkw1ORDjas4FPUbACfP7UGttnNZIppVbCfPaf7+Dx3/zeZ+7mhyS4g0JiWXgjC5JUQaE4Eh9qQxMWlyuxly6OCpNUs5Mm8Wqrxcs2NY0mpX4KgO6hKfdEn7OSHpsk193NXklpAzjYmgaFEQJreESGMiMMSeNCa6Jrej9abkwlkdPPDuFCOTG2yLa90RR3TQEEfi2sIyuV2Vzqsk1LoQRncxpqBMbglhdgtMYc3sLgiT+3bMRXaNf53tw0fz7/LPyLp9x67+qtxcrvz+J9uHj6bn2i38b/Um6g14CYDT65aTHH3MZL8Ty+aRcjoagBZjp9Fz7Rbtn0cXr8O/imYF1743XyMn/Z5R/5x7d9n3pkb4vPwd39osJ1nZ99n6p+Ztd9q9FE6c/sfuMY6f3sHde5rtcj9tW0aWDaawO5vcEnKb3QL3pyBNbglhdgsMscXstmRyO6uDf48cy9Vt21Hlmg+oTZF1+47TOthp3ipt/x5f/Ejbtz4G4PaFs5z6YpnJ6yZHH+X0Osd2HxUUjuigIVv+WIpS6YFS6cGWP5Zabe8Kk9sVL32lIF9QdChIk1tCmN0CQ2wxu02Z3M7q4PGFS9k+fDT7ps8i9VyMXf3l0MGqXXvwv882acewNa51R+zVQUMciWsLcyW3qF1RcAiju5hS0Ca3hDC7BaYwZ3bbanKfXLXG5mudXLmGYx8vIjc9A4DkqGj2Tplh84NFXmYmO14Zz38/btUeU3p6UaPn09R8oh8Ax5d+aPQGOD0hjpsnowDo8P4nhES01TvvHRBI2zfn4uVfBjD9hv7Sr5sB8K9SnS7zP9MeV6vybZq7OdQqFWqVfXvXz8Qc4O+937B8/QTuZ6YBoFR6sH7zO/z292dEndxOXOJ/5OblGPXNzcshLvE/ok5u57e/P+PLze+iVGryst7PTGP5+gn8vfcbzsQc4NbtBKOfZVEwuSXkNLsF7k1hmNwSwuwWGGLJ7La2kttZHVSrVFz8fgs7XhlPXmamTWOknoth75QZescc0cFSQQ+LaSoUCsrWrKMN8hMO7yUjMV6vv1qt5vjSuUbjOqKDxmPYrsXO6iBocoheuhrNviM/8f2vC9hz6AdUqnxUqnx2H9rM978uYN+Rn7h07V/uZ+q/NHDVSm5X7XCyN32YwHUUhsktIcxugSGWzG5LK7md0cFbZ84BkJl8k6j5i+2Ka53VwQaDR9LoxVdQenlpz9sa15rC2ZjU3jGc0UHQ/Cxu3U5wOK4t7HQlolBzwSGSQRZDCsvklhA5uwWmMMzZXfWxSJtXcicdieJ2186Uq1fH4jVSz8WQdFRjNrd99038goNIOHiE819/R9T8xXRfs9xqLrMjH84HwD+sMhFvTNI7V6fv81z+fQsAdy5doFztetpzUsBef9BwylQxnXvT08eXdrPms2faq1z54yfqPvM8igeBryo/n2t//w5Am9ffxcPHh9TLpzi0bDxZd5IJbtSBtmM/wdO3tMX5G3Lhj7Wc3bIEgEbPTKROT9sM1XcXPQOAh4eX9uFHpcon+eY1vvrhfdRqjWFQoVxlPpj2C0HlNX9/KalxvPnxk6Te0eRNUyiUKBVKVA8eatRqNUf//VNv63a7iKeYOHIlygd/N640uU/En3eqkIezObsF7klhmtwSIme3wBBTObsBq+lK6r84yCkdVOfnc2Lxcu5di+X4J8toM3Oqxf5qlYqo+YtN34OdOmiKsjXrUK//EGI2f8X579cTMX6m9tydS8ZmvqM6KJGbmcHhFZNIPrMfv7IhRI5bQvmalgvCjXuzrVM6uP/ozyxe86r2vIfS+P/+lv9bQr4q78E1FEwcuYr2rZ5yaboS6aWvqF0hMEVhmtwSIme3wBBTObtjtx82a3J3XjrfKR2s9/wAwjp35H5CEofenWNXXFuxgenUKbbqYPVuj5sd31pcGzyim/a7IzpodD92xrXO6CCASqVi8ZrRHIz65eEYNsS1B145BLgmJ3dBFGouCWlJbEGs6C5mFLbJLSFWdgtMobuye/foj+xKV/Lv8tVW25z98ltAY3KXqVYFT19fqnbtRIXGDQG0b9TNoVaryYjTBKZt338Lb399QVMoldTrPwSA1POn9M7FH9oNQGB4LYvX0H0rf/9msvZzZkqS9rN3QCAAh5aNJzstBdRqks8c4PwvqyyObUjq5ZOc/n4BqrxcVHm5nNo0n9TLp6x31CE/X3+LXr4qD7VahVLpQWBARWZN+l5rcgMElQ9j1qTvCSwThFLpgVqt0j6AmBqzdbMevDb8U21wD7h0Jbczuc2cXdktcE9cYXJLiJXdAkN0V3Yfe38Nx95fYzUnt7M66BMYSOTb0wFI+++y1ZyW0spx6ZqG2KOD5giN7AjATYMt16nnTwNQ7UFwL4cOnv91JclnDoBaTeadZA4vn2C1j7M62KH104wesgiFQoFCodBqr4Q0pnR+9IuLtMG9K3Nyi9oVAnO4wuSWECu7BYboruzePfojizm5ndXBat274uHlRZlqVWj7rsaQtieuNYetOmgOW+JaCUd00BB741pndBBAqVTy2rCltGrWQ3vMlrgWXFt4Us7aFWJl90OE0V2McJXJLSHMboEpqj4Wqf1crn64zTm5c9PTrbbJTL4JQKlKIXrHpWA75+5di/11tzUrFAqTbfKyswBQ5ehvbVLnaYJYn8ByVucpocp9OIbheABZd5L1tndlpNyweWyA+7fiTRwzXwBMl7Yte5v9GXgoPQkMqMgH036hcoixsR8WWpsP3viVwICKJt++g+bn2y7iSSa/8jlent5651xlcp+Mj3a6kIcwu4sXrjS5JYTZLTDE08+Hmv26aL/X7NfFauFJZ3VQdxW4tRQg2Q/ygVdsZn21lzUdNEdepunfr/kPxvApq9FiZ3RQ2z5Fp71aRebtJKtbr53VQYBHOwxi3NCl2ram+gOMG7qUrh2e0x53ZeFJUahZYApXmtwSwuwWGBJYM4xy9cO133VjVFPIoYN+wRUB++Jac9iqgxbHsBLXSjiig3rjqfIdimsd1UEJLy8fprzyuV1xrStNbglhdsuPMLqLCa42uSWE2S3QRcrJ7eHnQ1CLeqSciDFZoNIUwa1aWm1ToVEDAOJ279MeU6vVXNj4AwAB4eEW+ys9PLSfs9PSTLbJSNQIvVcZ06vNVPmWTSjdFQBSnlLNZ+Pik8GNOoBCqfmjVhHa1L7K3+VrNcfTtzQKpQcKpQeevqUpX6u5TX0njlxJpeCaJs/lq/J487VvCa0YbrZ/aMVw3nztW6NVbBKVQ2oxYcQKPD28TJ63Bzm3actRtVqY3cUDdzC5JYTZLdAl7XIcJz/9Hv+qIfhXDeHkp9+bLFApIYcO6hrkum1MUSa8GgDnv91kdj4S1nTQHNm3UwHwKVte77h3Gc0YuQ8KXTqjgxKhTTuBWqXV45DGHbTpViz2k0EHO7V9lr49zQfJ/XpNoFPbZ/WOucrklhBmt0AXdzC5JYTZLdDlyi97SDkRQ1CLenj4+ZgtUCnhqA7qal7CwcOAfXGtOWzVQUtYi2slHNVBCYXSw+G41hEd1MXTw8uuuNbVJreEMLvlRRjdxQB3MbklhNktAOPCk80mPGeyQKU56g3qb7VNw+Ga7VcxG77n+MKlxO7Yxd8jxmjP+4dVsjpGnf59AdgzaToXvt+id06Vm0vikf0AhLbSLzZZ55nBAJz9erXF7Wxply9qP/vqPJj4lnv4OSlKkxus7dhPqNtzGGGtHiNixByqdehjdf66lCofSueZ31C17RNUbfsEnWd+Q6nyoTb19fDw5F56qtnz97OsPzzdzzS/cvBueioeHs7nG5Y7F6kcDwXC7C76uJPJLSHMbgHoF56MmDmMiJnDTBao1MVZHfxvy8/snvgGADWetLwVGqBMFcs7tezRQXPE7dfU+6jR82m946EPCkFf/VOTk9MZHZSo1qEPESPmENbqMer2HEbkmE9s7iuHDmZmpaM0sSpcqfQ0qcWuMrlX7F+q/SzMbgG4l8ktIcxuAegXnmw24TmzBSoBp3Xw7xFjuPbXDo4v+pTzX28E7ItrzWGLDman3THb35a4VsIZHZRwJq61VwcNsSeudQeTW0KY3fIhjO4ijruZ3BLC7C7ZGJrcUroS3ZzdlszuiKkT8S1X1up1fMuVJWLqRECTh1R3NVmXTxfYNNfqPR+jUnvN1rVr27Zrj+fez+Dg7Gma65QPwrdcBb1+1bpq/r+lnj/N1W0/mxz7zuWLHJqjye/WdJRxbrOmL2uKX55YPo97N67h6VuaxgOmEDnmE6p37Gt2y5UlAqvWo9Woj2g16iMCq9az3uEBafdSuJdxG3hQSOvBW3uFQiMTsXHWhfJ6fIxeH6XSQ/v5Xnoqd9Nv2X4jJiiIglsgz0OBMLuLLu5ocksIs7tko2tySzm5dXN2WzK7wXEdvPLbNgAqd2xPrT69bRrD3LXs1UFD1Go1V/7YSsLhvQBUeaSb3nnf8kEE1tAvNueoDkooFAqqd+xL5JhPaDxgil1FoeXQwWtxZ8nPz8VD6anNReqh9CQ/P5frNyznXLcFObdp6yLM7pKNO5rcEsLsLtnomtxSTm7dnN2GZrccOnhh4w/cOn0WsC+udVYHD8+diSo312gMW+NaCWd0UMKZuNZZHbQnrnWVyZ2ebTqdjTC75UEY3UUYdzW5JYTZXTIxZ3JL2GJ2l29ge2BavkE9uq9ZTssp42k0Ygjt3n+Lx9auxKu0bYKsUChoPHIoXT5dQJNXR5CVmsKtc6fYMe5F7cNG5Iw5Rv28SpWm6csTAYjZ/BUnls8j5cy/ZKamkJmawpU/tnLoA81KgHJ1G1Ip8hGjMSq16UBQ4+YA7J81yeZ7LghuxF8AwNvLl36PT2DtwjNMfnk1wRWqAg+Dd0tIDw3BQdWY/PJq1i48Q79e4/H20gQ7sTaMYY6CMLl1H2qE2V0ycWeTW0KY3SUTUya3hDWz21kdbPzyMLp8uoBGw1+0OTD1Kl2ax9audFoHL/32A3evXyErNYX0hBscmTeLmM1fAdD05Yl4+pUyGqP5mKk2zbEwkEMHrz8Yo2WTbiyctZOFs3bSskm3B+ecM7rlzkVqiDC7SybubHJLCLO7ZGLK5JYwZ3Y7q4PtZs+i0YghtJg0ju5rltsV1zqrg/eTE9k+djDxB3dzPznRrrjWnXBWB+2Ja11lco/8/kWz54XZ7TzO7yMXuAR3N7klJLN7zPqjjF53lJXDWtOkallXT0tQQFgzuSWkB43LW3bqfXcUhVJpNa+ZNbxKlya0TSu2D39Ze0zp6UXHD5bgV6GiyT6V23YCtZqTny8hKeqQya3Xldo+QpPh4/QKmujOu+X4GZxYPp+b/x4zOl+YJN68whPdRtG353gCA4IAaBfxJG2a92LXwe85e/Gg1THuZ91j9JBFdG7bX7s9+7mn36BX1xFs+WMJiclXaVS3vd1zK2iTW0J6KHhh03Se3ziNDc/No0Xl+nZdRwryn984jRc2TeebgR/h72P8QCpwPUXB5JaQzO7R644yZv1RVgxtTWkf8QhXXLFkcktIZnf0wm85Mf9rPc31t5JKxBySDjrDsYXvGR2zRwcvbtnAxS0bjNo1fXmiRnNN4FehIo/MXe7UvOXCWR1Mu5tCjaqNGdRnBnVqtNAenzZmPRevHGfDT3NJu5ui1Wl7KJiCWxeN2sihgxM6vADA/D3r9L4L3I+iYHJLjOqqWfW6/O+Let8FxRNLJreEZHafmP810Qu/pfmUwU7roH9YJZvShpnDGR3cO2Ms6rw8Tn6+xKiNtbjWXZBDB+WIa80hl8l9MeWCxXbO6qAccW1RRkRJRZCiYnJLCLO7ZGCryS0ht9ktJ94BZWkyfBxBjZubDMx1qdyuMxWbtSL+4G4u/vQdefczAKjYvBX1BwyldGhli/2Vnl5ETJhJeoL57eeFQZf2z+FhosiIh4cn3To+T5f2A62OMXboEpNjBAYEMWzgbPLtqNYtUVgmt4Qwu0sGRcnklhBmd8nAFpNbwpLZ7So8S5V2Sgevbf+N2F1/as/VfeYFqj3a0+QKNl1KhzhuKsiJszro71+OWZO+N3muTo2WvDN5s0NaWjAmt3mE2V0yKEomt4Qwu0sGtpjcEoZmd5uZMwtplsZ0nPOpUzrYY80PpJyJ5tIvm7lzSbNq2Z641h2QQwfliGtNIafJvW7gt1bbC7PbcUSEVMQoaia3RFE2u/NyVS67dn6u5a1SqjzN3PLzVOQ7OM/8B2NIY2mx47eDvSa3hDua3T3XbrHeyACvUqWp3u1xqnezXqzEHP6VXGtOmHoYsOe8XGPoUtgmt4Qwu4s3RdHkliiKZrc7a6i2nTkdtAOtHkv368Bfiz0mt4Qps7tcuH3FF+Wk+7KvHe7rXymMRkNeodGQV2ScUeHirA7KobWGFLbJLSHM7uJNUTS5JYqS2V0UNFTCSAftRDeudQZ7TG4JXbPblTirgwqlkopNWlKxSUuZZ1Z4FEbMaa+OgvwmtyauNd4VZYgwux3DvaMjgRFF0eSWKKpm9/SJ5osmugtH18ZwdK3j+Y8Bfnx1r973buu72NTPUZNbwh3NboHrcZXJLSHM7uJJUTa5JYqa2V0UNFTCUAcdYfPI3YDtGirhiMktYWh2P7o8wq5rC4ovrjK5JYTZXTwpyia3RFExu4uShkpIOugoR9fG0M3BcNARk1tCMrsFAkMKxuS2nYIwu8N41K4xihruGxkJTFJUTW6Jomh2D3rJdW+8rvhYfpOen6fi6NoYQpuUJ7xdiEPXuHowicRTqbQeXg8PT/u2MzlrcksYmt21eg9waBxB8cDVJreEMLuLH0Xd5JYoSma3O2uohDM6KHFotaY4UtuX7a8X4YzJLaFrdgsE4HqTW0KY3cWL4mBySxQFs7soaKiEMzoI+nHtlV/22G1UO2NySwTWDAPXLaIXuCGuNrkl5Da7764f49A8igruGRUJzOIqk/vEjauEhcszVlEzuyPauG4LsNrfitGdq3kgCG8XQngHx+eZeCqV8PaheHg9DPBvqi33kcvkltA1u93N6Far1dz57zypMWfIz8nBu0wAoRFt8S1vezGo/NwckqIOkZEYD0Dp0MqERLTFw8u7oKZdoBz79y8UCgURTR9zqL9area7nz+id/eXgQDtcXcxuSWE2V28KA4mt0RRMbvdWUN1MaWD9nD9SDKAVosT7mfbZFjLYXJLSGa3uyKHDmalppAYdYice3fx8PamfP3GlK1VD4XCPkPGHdDVwQD/Cg6NcTf9FroaKuEuJrdEQZjd84r5ajR3pDiZ3BLubnYXFQ0FYx20F9249tBq+3b6ymFyFxWc1UFzcW0px2RIFu6m33JKB3/7ezXP95kh86zcx+SWkNPsLu64X0QkcDukh5q9s7rKNmZRM7sF+shtcku444NJcvQxji/90Oj4+e/W4luuApEzP7RYvVqVn89/Wzdy+fcfTZ6v2rUHDZ4fidLD/lxhrkKtVvPxipcA+H5VvEMGw/n/DvPTH0s5cOwX9ozQbMt0N5NboqDMbkHhU1xMbomiYnaXRKIXfmvVuJbT5Jbw9PNxu9Vocuhg5q2bRK+YT9qV/0yebzl+JuEtilbOSV0dXPbBQYfGmDn3CfaM1E+5424mt4TcZvc81soyL4HtFDeTW8Ldze6SSM1+XW1Oa1lSTG5bdDC4eSuLY1iKa/utOyvLPB1h5ke9ndLBpJRrtGj8KA1qR8o2p8IwudOz7xf6Dicpri3uuH/ZVYFL0X1zLzeS2V0rxJ/R645yKvaO7NcQOE7a5TiTxwvK5JYw94CiVqlQq5yL3lX5+ajVVpaq63Bj7w69h4F6/YdQ66kBhLbpAEDW7VvsnvoKGUkJZq93dP472uC+QoOm1BswhBq9+hJQvSYAsTv/5Oj8d1DlW64e7U4cP/X3w8+ndzg0xo59GwC4c1ez+sNdTW4J6aGgXlA4z2+cxon483aPIQX5MSlXS8SbdHfEVSb34t3bCmxsyey+lJTOmPVHycjOK7BrCWwnPS6Z6IXfkpeZbfJ8QZjctmCvDhqiVqvt0is5dDAjKYHdU1/RBvfhPZ6izjODqdatl7aNqcDd3dHVwcysDLv7Z2ZlaDVUwl1NbglDHUzPvm/3GBM6vMDUTsNknZfANoqjyS0xqmttxnavw/K/L/L5TtNGoqDwqPFUJ63ZfeUX83nKXWFy26uDJscwE9c6q4M39pqPy6zFta5EDh2UNFUOCmslt7M6OH/POpbs/8bu/iVhZ7FbLPtZvnw58+fPJzExkWbNmvHpp5/Spk0bs+03b97M22+/zdWrV6lTpw4ff/wxjz/+eCHOuGRguD3tPv/Ifg2xstt9OTH/ayMju6BNblOkXbnGkdn6bx3bvDmNwFo1bOqvVqu5tm07Fzf/pD1WOqwSbWZOxdPPz2y/u9evcHrdcgBq9n6WOn2eQ6F8+G5QNWI8B2dP496Na+x/eyKPrfpO7zzAuQ1ruH1B83a89dT3qNCgic7ZF0lPuMG+N8dz+8JZ/tu6kebPzrbpnlyJWq1m0y/zUD6oVr3pl3m0bNzNrlXdCclX2HNYY3rk5GS6vcktIffKbkHJYPHubcz753ee1wlG5Eas7HY/Wkx9kRPzvza5sruwTW5HdVCXvMxMji9aRtqly9pjNXr3olbfJy3+/rdXB+s+o596Ra1SsXfGWABKBVcicsYcfALLas83HDyKazv+j3PfrrHpPgoKtVrtlA7+tftLnu5hX77Mv3avJyc3S/vdFSb3ifjzLknnJXJ0uwZXmdzp2VmF4lqIld3uhWENJ0Mju7BNbkd1UBdrca2zOnh63XICqtckoJp+nGxLXOtK5NDBPYd+4NknJhFaMdzm/qZe/BdmuhLppa+oXSE/Lo+CNm3axOTJk1m1ahWRkZEsXryYHj16EBMTQ3BwsFH7AwcOMGjQIObOnUvv3r3ZsGEDffr04fjx4zRu3NgFd1A8MZWDzf53TbYhzG73xD8sWM/sdonJfekKR+bMMzp+ZM48ImdNJyC8utUxznzxJQkHDusdy4hLYOfYyXT7bClKLy+T/U6vWwFAvYFDqdHjKaPzSi8v2kz/gB3jXkSVl0vK6WgqNm2pPZ+fk03szj8B6DhnKf6VqhiN4V+pCq2nvqdd7WaL0X0/NZHUS9GUqhBG+ZpNrLY3RerlU9y/FUf5Ws0pVV6TR+/zDdOpFFyDSiG1aNHoUZQPHn6uxp7h0rVo4pMuk5B8mdj4GBKTr2jHunL9FBNmdaRK5bpUCq5J5ZCa1KrenPCqjQBQqVScOPMPCUmXSEi+QnziJa7dOINSoSRfrUKtVhcJk1tCTrNbUPyRTO5pjz5BQeeTEGa3exFYM8yk2e2KldyO6qCEKjeXnWMnGx2/8tsfZN9Jo9HwF032c0QHaz3VXy9nd8rpEwAovbzpOHuxyblW7/Y46XGxFu/BEFM6aA/5uTkkn9kPQHCjDk7r4Hc/f8TBqF8Iq1RHo8XBNWlcryOBAZpaIGl3Uzgds4+E5MskJF8hLuEiV2+c1QbprlrJLWpXlCxcZXI///UK1rxaOKmJhNntXpgzuwvb5HZUB3WxJa51Vgdjd/3J2W9W03bmXL3ztsS19mKog47UnZLiWjl0UKlQMnPu41Sv0ojKobVsjmt3D9+tnU9h5+QWhZoLDpdHQIsWLWLUqFEMG6bZgrZq1Sp+//131q5dy/Tpxlu7lyxZQs+ePZk6dSoAs2fPZvv27SxbtoxVq1YV6tyLK64oNCLMbvej+ZTBRC/8lhPzv6bpawO4vGVXoZrcgPZhoOmYUYS00pjISceOc3LF5xx+/yMeW7vSYv/cjAxtcN/pk4/wCQxErVZzaNYHpMfFc/3vnYT3+p9Rv7zM+9y9dgmA6hZWYXqVKq0N0E+t/ZRHF6/Tnks6rrluQPWaJoN7iQoNmlC+QRNSz52yeC8AabEx7P5wMHlZmtdOTQZOpU5P+7bvXvhjLae/XwCAp29pOs/8hsCq9fhr95coFApeHjxP+zAA4Onpxcaf53HnbjIKhcLkm++E5Msk3ryCWq2mbEAw7075QXtOqVSSejuBLze/a3ZOrjS5l+z/xumq1Y4G+YLija7JPbFzT5L5vwK/pjC73QtDs7tmvy6c/PT7QjW5HdVBXa7v2AVAmerViHz7DRRKJTl377J74hvE7ztA3YH98Cpd2qifIzqYFHWYym0f0Z7775fvAWg6aoLFQLzBoOEW70EXczpoK/m5OeydN4zU/zTmQ/naLUh1Ugfz83O5dO1fLl8/iVqtJqLp/2jT/OHzRym/Muw98hNRJ/8y0mJXpiuR0nkJs1tQEEgmd0xyAlB4z02GZnfnJ6oV2rUFxhia3dLnwkxX4qgO6mJrXOuMDsbu+pM7/8WQl3kfTz/N71Nb41p7MKWDnd5Yh9LTdrPbMK51RgcB8lV53Mu4zemYfZyO2Wd3XOuKwpMFUahZmN0aXBr95OTkEBUVxYwZDyukKpVKunfvzsGDppPRHzx4kMmT9d+m9ejRg61bt5psn52dTXb2w/yIaWlpANy9e9fJ2buGe1mZBTr+yfjrDPvuc+pUDGX1gOGo1WrtNdM9cwv02gDzB7Vg8rfHeXXtET55oSUNwwIBuK/Uzzman69ZGXc/y7FcpLl5mv45uflWx8hyYe7kXA/L26DyczX3kZeTT26mYz+LvBzN/eVm5qHK00+90XjMs0Qv/IYT874GoMUbQyhdKchszlG5UKoy9fKWVWjUgLzMTO1nidyMDKN0Ibokn/gXgGrdH8XD21s7RrPxo9n/xttc3PwTVbo8otdHzX3S4x/mJ1fl5qLKNf9v3ztA8280524aeZkP9z3ci70GQLk6DfWOm6J8/UaknjtFbma6xXYxv60mL/vhNuXTP3xC9Ueesfgz0EWtUnHmx8Xa73nZmcT8tpoWQ98DYMRzc2nf6mnuZ957OLeylZgx7hs+/PR57qXfRo3p/w8KlASUKceMcd9QLjBUb4wOrfugys9nzUbT1bCX9/2cmhVqkZ59z+R5c5xOOMWYLSOpVaE2i59eCajtHmP+nnVk5+Uwuu1Au/oBrOozi1Fb3mHQd1P54pn3aVqprl39i6oOFWUKWkMlVuz/myV7/mJCp/8xIrIz97IyC0VDAWpULM2iwS2Y9M1xXll7hEWDW3Jfabya3B4dNEdOrub3wf2sPLw8Tf8ecmcNlbCkg7aievBcImlxnlqjk6UrBdF0/EBOfPwVJ+Z9TemwijQe86ymTQFpqVL18N+5IzpoiJTypMmrw8l/8Eyt9PKieo/uXPvzb5KiThDaOgLQaKiEIzqYfuMaeZkR2uNplzVGk1eZAKtj5OZb1lCwroO2kHhyjza4B7SfndFB7fzUaiKadGf0kEXk5uWQm5ejPTd6yCJWfjmRKJ06GQBDNz7vlA5+fmgVqw5+yqvtXmNIq2F29XdWB2tXqMqafu8x4sdZDPpuKp/3e4/SPral0xEaWvgUloYCZORkM2LjGi7eTGTdoFGkZ9lfG8UZBrWrTk5ePsv/vkjrbpXNtrNFB61hKa4tChoqYaiD9qIb1xrqY9XHIlHl5mnN7vAnH6HqY5EFoqO6Giphjw6awta4FnBKByXS4+Pwrxym/SxhKa7NVVjXUAlTOhgX9TehTW1/8WAY10o4ooOmsDWuTc++55QOAqRnZ/DaTy9z6dZ/rOi3xua4VqXIcEoHJYa36kt2Xo5dcW1R1FBpzrbUmlGonalI4yTx8fGEhYVx4MAB2rVrpz0+bdo0du/ezeHDh436eHt78+WXXzJo0CDtsRUrVvDee++RlJRk1P7dd9/lvfdsf3gVCAQCgUAgEAgEAoFAIBAIBAKB+xAbG0uVKuZ3CoIbpC4paGbMmKG3AlylUpGamkqFChXsKhojEAgKh7t371K1alViY2MJCAhw9XQEAoFAICgyCA0VCAQCgcAxhIYKBO6LWq3m3r17VK5sfpeNhEuN7qCgIDw8PIxWYiclJREaarowTGhoqF3tfXx88PHRz4FYtmxZxyctEAgKhYCAAPGAIRAIBAKBAwgNFQgEAoHAMYSGCgTuSWBgoE3tHEskJRPe3t5ERESwY8cO7TGVSsWOHTv0Upno0q5dO732ANu3bzfbXiAQCAQCgUAgEAgEAoFAIBAIBMUbl6cumTx5Mi+99BKtWrWiTZs2LF68mIyMDIYNGwbAkCFDCAsLY+7cuQBMmDCBzp07s3DhQp544gk2btzIsWPHWL16tStvQyAQCAQCgUAgEAgEAoFAIBAIBC7C5Ub3wIEDuXnzJrNmzSIxMZHmzZuzbds2QkJCALh+/TpK5cOF5+3bt2fDhg289dZbzJw5kzp16rB161YaN27sqlsQCAQy4uPjwzvvvGOUckggEAgEAoFlhIYKBAKBQOAYQkMFguKBQq1Wq109CYFAIBAIBAKBQCAQCAQCgUAgEAgcxaU5ugUCgUAgEAgEAoFAIBAIBAKBQCBwFmF0CwQCgUAgEAgEAoFAIBAIBAKBoEgjjG6BQCAQCAQCgUAgEAgEAoFAIBAUaYTRLRAIBAKBQCAQCAQCgUAgEAgEgiKNMLoFAoFVhg4dikKh4NVXXzU6N3bsWBQKBUOHDtVra/inZ8+e7Nq1y+Q53T+7du1i/fr1Js/5+vrqXTs2Npbhw4dTuXJlvL29qV69OhMmTODWrVt67bp06aI3Rt26dZk7dy6iFq9AIBAIChtDnaxQoQI9e/bk5MmT2jYKhYKtW7ea7G9JSxMTE7XX6NOnj9m+d+7cMTu/M2fO8MwzzxAeHo5CoWDx4sVO3K1AIBAIBPajq5VeXl7UqFGDadOmkZWVpdfut99+o3PnzpQpU4ZSpUrRunVr1q9fr9fGkvaFh4cb6dzOnTvp3bs3FStWxNfXl1q1ajFw4ED27NljNKYlLTZ3X6b0WSAQyIcwugUCgU1UrVqVjRs3kpmZqT2WlZXFhg0bqFatml7bnj17kpCQoPfnu+++o3379nrHBgwYYNS2ffv2AAQEBBiNce3aNe01Ll++TKtWrbh48SLfffcd//33H6tWrWLHjh20a9eO1NRUvTmNGjWKhIQEYmJimDFjBrNmzWLVqlUF+BMTCAQCgcA0utq3Y8cOPD096d27t11jxMTEGOlkcHCw03O7f/8+NWvW5KOPPiI0NNTp8QQCgUAgcARJKy9fvswnn3zCZ599xjvvvKM9/+mnn/L000/ToUMHDh8+zMmTJ3nuued49dVXef311x265ooVK+jWrRsVKlRg06ZNxMTE8NNPP9G+fXsmTZpk1L6gtFggEDiOp6snIBAIigYtW7bk0qVLbNmyhcGDBwOwZcsWqlWrRo0aNfTa+vj4mA2OdY/7+fmRnZ1tsq1CobAYYI8dOxZvb2/++usv/Pz8AKhWrRotWrSgVq1avPnmm6xcuVLbvlSpUtrxhg0bxrJly9i+fTujR4+28ScgEAgEAoE86OpkaGgo06dP55FHHuHmzZtUrFjRpjGCg4MpW7as7HNr3bo1rVu3BmD69Omyjy8QCAQCgS3oamXVqlXp3r0727dv5+OPPyY2NpYpU6YwceJEPvzwQ22fKVOm4O3tzfjx4+nfvz+RkZE2X+/69etMnDiRiRMnsmjRIr1zTZs2Zfz48UZ9CkqLBQKB44gV3QKBwGaGDx/OunXrtN/Xrl3LsGHDCn0eqamp/Pnnn4wZM0ZrckuEhoYyePBgNm3aZDI1iVqtZu/evZw/fx5vb+/CmrJAIBAIBCZJT0/nm2++oXbt2lSoUMHV0xEIBAKBwO04ffo0Bw4c0MZvP/zwA7m5uSZXbr/yyiv4+/vz3Xff2XWNH3/8kdzcXKZNm2byvEKhsH/iAoGg0BFGt0AgsJkXXniBffv2ce3aNa5du8b+/ft54YUXjNr99ttv+Pv76/3RfdNuC2lpaUZj9OrVC4CLFy+iVqtp0KCByb4NGjTg9u3b3Lx5U3tsxYoV+Pv74+PjQ6dOnVCpVCbfygsEAoFAUNDo6mSZMmX45Zdf2LRpE0ql7Y/mVapU0dPIRo0aFeCMBQKBQCAoXCSt9PX1pUmTJiQnJzN16lQALly4QGBgIJUqVTLq5+3tTc2aNblw4YJd17tw4QIBAQF6u4p//PFHPa09deqUXh+hxQKB+yFSlwgEApupWLEiTzzxBOvXr0etVvPEE08QFBRk1K5r1656aUMAypcvb9e1ypQpw/Hjx/WOGa7etqeY5ODBg3nzzTe5ffs277zzDu3bt9fmAxcIBAKBoDDR1cnbt2+zYsUKevXqxZEjR6hevbpNY+zdu5cyZcpov3t5edk1h+vXr9OwYUPt95kzZzJz5ky7xhAIBAKBoKCQtDIjI4NPPvkET09PnnnmmQK9puGq7R49ehAdHU1cXBxdunQhPz9f77w5Ld67d692kRbAZ599pk3/KRAIChZhdAsEArsYPnw448aNA2D58uUm25QuXZratWs7dR2lUml2jNq1a6NQKDh37hx9+/Y1On/u3DnKlSunl+c0MDBQO973339P7dq1adu2Ld27d3dqngKBQCAQ2IuhTq5Zs4bAwEA+//xzPvjgA5vGqFGjhtm8oAEBAXoFnCXu3LmDh4cHpUuXxt/fn+joaO05e19ICwQCgUBQkOhq5dq1a2nWrBlffPEFI0aMoG7duqSlpREfH0/lypX1+uXk5HDp0iW6du0KaDQRNDuGDXXzzp07BAYGAlCnTh3S0tJITEzUrur29/endu3aeHqats7MaXGrVq30NDYkJMTu+xcIBI4hUpcIBAK76NmzJzk5OeTm5tKjRw+XzKFChQo89thjrFixgszMTL1ziYmJfPvttwwcONBsHjV/f38mTJjA66+/bteqcIFAIBAICgKFQoFSqTTSNEepV68eZ86cITs7W+/48ePHqVGjBl5eXnh6elK7dm3tH2F0CwQCgcBdUSqVzJw5k7feeovMzEyeeeYZvLy8WLhwoVHbVatWkZGRwaBBgwCNga1UKomKitJrd/nyZdLS0qhbty4Azz77LF5eXnz88cdOz9fPz09PY3VXfQsEgoJFrOgWCAR24eHhwblz57SfTZGdnU1iYqLeMU9PT5NpTsyhVquNxgBNZWulUsmyZcto3749PXr04IMPPqBGjRqcOXOGqVOnEhYWxpw5cyyO/8orrzB79mx+/PFHnn32WZvnJRAIBAKBs+jq5O3bt1m2bBnp6ek8+eST2jZXrlzRWw0GmmBdIjk5maysLL3zFSpUwMvLi8GDB/P+++8zZMgQpk2bRmBgIHv27GHx4sXMmzfP4txycnI4e/as9nNcXBzR0dHaVW0CgUAgELiC/v37M3XqVJYvX87rr7/OvHnzmDJlCr6+vrz44ot4eXnx888/M3PmTKZMmUJkZCSgSYk5cuRIpkyZgqenJ02aNCE2NpY33niDtm3batNZVqtWjYULFzJhwgRSU1MZOnQoNWrUIDU1lW+++QYwjn8tabE50tLSjPS9QoUKVK1a1dkfkUAgQBjdAoHAAaTtX+bYtm2bUWGQevXqcf78eZuvcffuXZPFRRISEggNDaVOnTocO3aMd955hwEDBpCamkpoaCh9+vThnXfesboyrXz58gwZMoR3332Xfv362VUATCAQCAQCZ9DVyTJlylC/fn02b95Mly5dtG0mT55s1G/v3r3az/Xq1TM6f/DgQdq2bUvZsmXZu3cv06dP56mnniItLY3atWuzaNEiRowYYXFu8fHxtGjRQvt9wYIFLFiwgM6dO7Nr1y4771QgEAgEAnnw9PRk3LhxzJs3j9GjRzNx4kRq1qzJggULWLJkCfn5+TRq1IiVK1cybNgwvb5Llizho48+4o033uDatWuEhoby2GOPMWfOHL1dwK+99hoNGjRg0aJFPPvss9y9e5cKFSrQrl07tm3bRpMmTfTGtaTF5ti1a5eezgKMGDGCNWvWOPJjEQgEBijUYt++QCAQCAQCgUAgEAgEAoFAIBAIijBiCaNAIBAIBAKBQCAQCAQCgUAgEAiKNMLoFggEAoFAIBAIBAKBQCAQCAQCQZFGGN0CgUAgEAgEAoFAIBAIBAKBQCAo0gijWyAQCAQCgUAgEAgEAoFAIBAIBEUaYXQLBAKBQCAQCAQCgUAgEAgEAoGgSCOMboHg/9m78+gaz/X/4+8tEZkREUHThEYINYcS1SQ1hKoWLaXUVBxFK8ZDqSSmoqa0itYULUpbSr+iplRiqLliSCI1BXViHk5jiJD9+yO/PMcWklAtqc9rrb1W9nPfzz3tnCzn2levR0RERERERERERPI1BbpFREREREREREREJF9ToFtERERERERERERE8jUFukVEREREngImk4nly5c/7mWIiIiIiPwlFOgWEREREfmbdO7cGZPJRM+ePbO19e7dG5PJROfOnfM0VkxMDCaTicuXL+epf0pKCk2bNn2A1YqIiIiI5B8KdIuIiIiI/I08PDxYvHgx169fN67duHGDRYsW8eyzzz7y+W7evAmAu7s7hQoVeuTji4iIiIg8CRToFhERERH5G9WoUQMPDw+WLVtmXFu2bBnPPvss1atXN65lZGTw8ccfU6ZMGezs7KhatSrff/89AMnJyQQFBQFQtGhRi0zwwMBA+vTpQ0hICK6urgQHBwPZS5f8/vvvtGvXDhcXFxwcHPDz82P79u0A7N27l6CgIJycnHB2dqZmzZrs2rXrrzwWEREREZE/xfpxL0BERERE5GnTtWtX5s2bR/v27QGYO3cuXbp0ISYmxujz8ccfs2DBAmbOnEm5cuXYuHEjHTp0oHjx4rz44ossXbqUN954g6SkJJydnbGzszPunT9/Pu+99x5btmy55/ypqakEBARQunRpfvzxR9zd3fn111/JyMgAoH379lSvXp0ZM2ZgZWVFXFwcBQsW/OsORERERETkT1KgW0RERETkb9ahQweGDh3K8ePHAdiyZQuLFy82At1paWmMHTuW9evXU7duXQDKli3L5s2b+eKLLwgICMDFxQUANzc3ihQpYjF+uXLlmDBhwn3nX7RoEefOnWPnzp3GON7e3kb7iRMnGDRoEBUqVDDGExERERF5kinQLSIiIiLyNytevDjNmjUjMjISs9lMs2bNcHV1NdoPHz7MtWvXaNSokcV9N2/etChvcj81a9bMsT0uLo7q1asbQe679e/fn27duvH111/TsGFDWrduzXPPPZeHnYmIiIiIPB4KdIuIiIiIPAZdu3alT58+AHz++ecWbampqQBERUVRunRpi7a8PFDSwcEhx/Y7y5zcS1hYGG+//TZRUVH89NNPhIaGsnjxYlq2bJnr3CIiIiIij4MeRikiIiIi8hg0adKEmzdvkp6ebjwwMkvFihUpVKgQJ06cwNvb2+Ll4eEBgI2NDQC3b99+4LmrVKlCXFwcFy9evG8fHx8f+vXrx9q1a2nVqhXz5s174HlERERERP4uCnSLiIiIiDwGVlZWJCYmkpCQgJWVlUWbk5MTAwcOpF+/fsyfP58jR47w66+/8tlnnzF//nwAPD09MZlMrFy5knPnzhlZ4HnRrl073N3dadGiBVu2bOHo0aMsXbqUrVu3cv36dfr06UNMTAzHjx9ny5Yt7Ny5E19f30e6fxERERGRR0mBbhERERGRx8TZ2RlnZ+d7to0aNYqPPvqIjz/+GF9fX5o0aUJUVBRlypQBoHTp0oSHhzNkyBBKlChhlEHJCxsbG9auXYubmxuvvPIKlStXZty4cVhZWWFlZcWFCxfo2LEjPj4+tGnThqZNmxIeHv5I9iwiIiIi8lcwmc1m8+NehIiIiIiIiIiIiIjIw1JGt4iIiIiIiIiIiIjkawp0i4iIiIiIiIiIiEi+pkC3iIiIiIiIiIiIiORr1o97ASIiIiJ/FbPZzK1bt7h9+/bjXoqIiDxCVlZWWFtbYzKZHvdSRERE5AmhQLeIiIj8I928eZOUlBSuXbv2uJciIiJ/AXt7e0qWLImNjc3jXoqIiIg8AUxms9n8uBchIiIi8ihlZGRw6NAhrKysKF68ODY2Nsr6ExH5hzCbzdy8eZNz585x+/ZtypUrR4ECqsopIiLytFNGt4iIiPzj3Lx5k4yMDDw8PLC3t3/cyxERkUfMzs6OggULcvz4cW7evImtre3jXpKIiIg8ZvraW0RERP6xlOEnIvLPpb/xIiIicif9y0BERERERERERERE8jUFukVEREREREREREQkX1ONbhEREXmqpIycZvF+z+/JtP3qc8q7lWTRO71wLPRo67yWHNHnkY4nIiIiIiIi2SmjW0RERJ5af3WQ+884efIkXbt2pVSpUtjY2ODp6Unfvn25cOHCY1vT3r17adeuHR4eHtjZ2eHr60tERESe7v3uu++oUKECtra2VK5cmVWrVuXYPyYmBpPJlO11+vRpo0/nzp0t2ooVK0aTJk3Yt29fjmMnJydjMpmIi4vL09pzExkZSZEiRR7JWHkRGBhISEjI3zafiIiIiEh+oEC3iIiIPJWe5CD30aNH8fPz49ChQ3zzzTccPnyYmTNnEh0dTd26dbl48eJjWdfu3btxc3NjwYIFxMfHM2zYMIYOHcq0adNyvO+XX36hXbt2vPvuu+zZs4cWLVrQokULDhw4kOucSUlJpKSkGC83NzeL9iZNmhht0dHRWFtb8+qrr/6pff5Vbt68+biXICIiIiLyj6VAt4iIiDx1nuQgN0Dv3r2xsbFh7dq1BAQE8Oyzz9K0aVPWr1/PqVOnGDZsGNOmTeP555837lm+fDkmk4mZM2ca1xo2bMjw4cON9ytWrKBGjRrY2tpStmxZwsPDuXXrltFuMpmYPXs2LVu2xN7ennLlyvHjjz8a7V27diUiIoKAgADKli1Lhw4d6NKlC8uWLctxPxERETRp0oRBgwbh6+vLqFGjqFGjRq4BcgA3Nzfc3d2NV4EClv98LVSokNFWrVo1hgwZwsmTJzl37lyuY2fJyh6Pjo7Gz88Pe3t7/P39SUpKMvrs3buXoKAgnJyccHZ2pmbNmuzatYuYmBi6dOnClStXjMzysLAwALy8vBg1ahQdO3bE2dmZHj16GHNdvnzZGDsuLg6TyURycrJxbcuWLQQGBmJvb0/RokUJDg7m0qVLdO7cmdjYWCIiIoz57rxPRERERORppUC3iIiIPFWe9CD3xYsXWbNmDb169cLOzs6izd3dnfbt27NkyRICAgJISEgwArqxsbG4uroSExMDQHp6Olu3biUwMBCATZs20bFjR/r27UtCQgJffPEFkZGRjBkzxmKO8PBw2rRpw759+3jllVdo3759jhnkV65cwcXFJcc9bd26lYYNG1pcCw4OZuvWrbmeR7Vq1ShZsiSNGjViy5YtOfZNTU1lwYIFeHt7U6xYsVzHvtuwYcOYNGkSu3btwtramq5duxpt7du355lnnmHnzp3s3r2bIUOGULBgQfz9/Zk6dSrOzs5GZvnAgQON+yZOnEjVqlXZs2cPH330UZ7WERcXR4MGDahYsSJbt25l8+bNNG/enNu3bxMREUHdunXp3r27MZ+Hh8cD71VERERE5J9GD6MUERGRp8qTHOQGOHToEGazGV9f33u2+/r6cunSJdzc3HBxcSE2NpY333yTmJgYBgwYYNTM3rFjB+np6fj7+wOZAewhQ4bQqVMnAMqWLcuoUaMYPHgwoaGhxvidO3emXbt2AIwdO5ZPP/2UHTt20KRJk2xr+eWXX1iyZAlRUVE57un06dOUKFHC4lqJEiUs6m3frWTJksycORM/Pz/S0tKYPXs2gYGBbN++nRo1ahj9Vq5ciaOjIwBXr16lZMmSrFy5Mlvmd16MGTOGgIAAAIYMGUKzZs24ceMGtra2nDhxgkGDBlGhQgUAypUrZ9xXuHBhTCYT7u7u2cZ8+eWXGTBggPH+5MmTua5jwoQJ+Pn5MX36dONapUqVjJ9tbGywt7e/53wiIiIiIk8rZXSLiIjIU+VJDnLfyWw259heqFAhXnrpJWJiYrh8+TIJCQn06tWLtLQ0Dh48SGxsLLVq1cLe3h7ILL0xcuRIHB0djVdWVvC1a9eMcatUqWL87ODggLOzM2fPns02/4EDB3j99dcJDQ2lcePGAJw4ccJi/LFjxz70/suXL8+//vUvatasib+/P3PnzsXf358pU6ZY9AsKCiIuLo64uDh27NhBcHAwTZs25fjx4wA0bdrUWM+dweJ7uXPvJUuWBDD23r9/f7p160bDhg0ZN24cR44cydM+/Pz88rznLFkZ3SIiIiIiknfK6BYREZGnypMe5Pb29sZkMpGYmEjLli2ztScmJlK8eHGKFClCYGAgX375JZs2baJ69eo4Ozsbwe/Y2FgjOxkyy3qEh4fTqlWrbGPa2v7vPAoWLGjRZjKZyMjIsLiWkJBAgwYN6NGjh0UN8FKlShEXF2e8zypp4u7uzpkzZyzGOHPmzANnJNeuXZvNmzdbXHNwcMDb29t4P3v2bAoXLsysWbMYPXo0s2fP5vr16/fc293ubDeZTADG3sPCwnj77beJiorip59+IjQ0lMWLF9/zM7p7fXfKyjS/84uM9PR0iz53l6wREREREZHcKaNbREREnipPcpAboFixYjRq1Ijp06cbAdosp0+fZuHChXTu3BnAqNP93XffGbW4AwMDWb9+vfEwwyw1atQgKSkJb2/vbK8HKfMRHx9PUFAQnTp1ylbf29ra2mLcrEB33bp1iY6Otui7bt066tatm+d5ITPTOSvT+n5MJhMFChQwzq506dLGejw9PR9ovrv5+PjQr18/1q5dS6tWrZg3bx6QWUrk9u3beRqjePHiAKSkpBjX7vxyADIzy+8+rzs9yHwiIiIiIk8LBbpFREREnjDTpk0jLS2N4OBgNm7cyMmTJ1m9ejWNGjXCx8eHESNGAJkB0aJFi7Jo0SKLQPfy5ctJS0ujXr16xpgjRozgq6++Ijw8nPj4eBITE1m8eLFFRnZuDhw4QFBQEI0bN6Z///6cPn2a06dPGw/EvJ++ffuyevVqJk2axMGDBwkLC2PXrl306dPH6DN06FA6duxovJ86dSorVqzg8OHDHDhwgJCQEH7++Wd69+5tMXZaWpqxjsTERN5//31SU1Np3rx5nveVm+vXr9OnTx9iYmI4fvw4W7ZsYefOnUYddS8vL1JTU4mOjub8+fMWpWDu5u3tjYeHB2FhYRw6dIioqCgmTZpk0Wfo0KHs3LmTXr16sW/fPg4ePMiMGTM4f/68Md/27dtJTk7m/Pnz2TLuRURERESeRipdIiIiIk+VkiP65N7pMStXrhw7d+4kLCyMNm3acPbsWcxmM61ateLrr7826m6bTCbq169PVFQUL774IpAZ/HZ2dqZ8+fIWZTOCg4NZuXIlI0eOZPz48RQsWJAKFSrQrVu3PK/r+++/59y5cyxYsIAFCxYY1z09PUlOTr7vff7+/ixatIjhw4fz4YcfUq5cOZYvX87zzz9v9ElJSeHEiRPG+5s3bzJgwABOnTqFvb09VapUYf369QQFBVmMvXr1aiPL28nJiQoVKlhkuD8KVlZWXLhwgY4dO3LmzBlcXV1p1aoV4eHhxv569uzJW2+9xYULFwgNDSUsLOyeYxUsWJBvvvmG9957jypVqlCrVi1Gjx5N69atjT4+Pj6sXbuWDz/8kNq1a2NnZ8cLL7xgPCR04MCBdOrUiYoVK3L9+nWOHTuGl5fXI9uviIiIiEh+ZDLn9qQjERERkXzmxo0bHDt2jDJlyljUn87PQkNDmTx5MuvWraNOnTqPezkiIo/dP/FvvYiIiDw8ZXSLiIiI5APh4eF4eXmxbds2ateu/UB1tUVERERERP7pFOgWERERySe6dOnyuJcgIiIiIiLyRFIqkIiIiIiIiIiIiIjkawp0i4iIiIiIiIiIiEi+pkC3iIiIiIiIiIiIiORrCnSLiIiIiIiIiIiISL6mQLeIiIiIiIiIiIiI5GsKdIuIiIiIiIiIiIhIvqZAt4iIiIiIiIiIiIjka9aPewEiIiIif6e9Y1/5W+er+uGqv3U+ebp99NFHnDlzhi+//BIAs9nMv/71L77//nsuXbrEnj17CAkJoVq1akydOvXxLvYvlJycTJkyZdizZw/VqlUjISGBxo0bk5SUhIODw+NenoiIiIj8BZTRLSIiIk+VWRsOP+4l5MnJkyfp2rUrpUqVwsbGBk9PT/r27cuFCxce25r27t1Lu3bt8PDwwM7ODl9fXyIiIvJ073fffUeFChWwtbWlcuXKrFqV8xcAMTExmEymbK/Tp08bfTp37mzRVqxYMZo0acK+fftyHDs5ORmTyURcXFye1p6byMhIihQp8kjGyovAwEBCQkKyXT99+jQREREMGzbMuLZ69WoiIyNZuXIlKSkpPP/88yxbtoxRo0Y90jV17tyZFi1aPNIxH6WKFStSp04dJk+e/LiXIiIiIiJ/EQW6RURE5Kny+fpDT3yw++jRo/j5+XHo0CG++eYbDh8+zMyZM4mOjqZu3bpcvHjxsaxr9+7duLm5sWDBAuLj4xk2bBhDhw5l2rRpOd73yy+/0K5dO95991327NlDixYtaNGiBQcOHMh1zqSkJFJSUoyXm5ubRXuTJk2MtujoaKytrXn11Vf/1D7/Kjdv3vxLx589ezb+/v54enoa144cOULJkiXx9/fH3d0da2trXFxccHJy+kvX8iTq0qULM2bM4NatW497KSIiIiLyF1CgW0RERJ4qvRuWe+KD3b1798bGxoa1a9cSEBDAs88+S9OmTVm/fj2nTp1i2LBhTJs2jeeff964Z/ny5ZhMJmbOnGlca9iwIcOHDzfer1ixgho1amBra0vZsmUJDw+3CPqZTCZmz55Ny5Ytsbe3p1y5cvz4449Ge9euXYmIiCAgIICyZcvSoUMHunTpwrJly3LcT0REBE2aNGHQoEH4+voyatQoatSokWuAHMDNzQ13d3fjVaCA5T9fCxUqZLRVq1aNIUOGcPLkSc6dO5fr2Fmyssejo6Px8/PD3t4ef39/kpKSjD579+4lKCgIJycnnJ2dqVmzJrt27SImJoYuXbpw5coVI7M8LCwMAC8vL0aNGkXHjh1xdnamR48exlyXL182xo6Li8NkMpGcnGxc27JlC4GBgdjb21O0aFGCg4O5dOkSnTt3JjY2loiICGO+rPsWL15M8+bNjTE6d+7M+++/z4kTJzCZTHh5eQHZM8K9vLwYO3YsXbt2xcnJiWeffdYofZLl5MmTtGnThiJFiuDi4sLrr79uzBsWFsb8+fNZsWKFsaaYmJg87TUrG37NmjX4+vri6OhofHlxp9mzZ+Pr64utrS0VKlRg+vTpFu07duygevXq2Nra4ufnx549e7J9zo0aNeLixYvExsZmaxMRERGR/E+BbhEREXmqdA/yfqKD3RcvXmTNmjX06tULOzs7izZ3d3fat2/PkiVLCAgIICEhwQjoxsbG4urqSkxMDADp6els3bqVwMBAADZt2kTHjh3p27cvCQkJfPHFF0RGRjJmzBiLOcLDw2nTpg379u3jlVdeoX379jlmkF+5cgUXF5cc97R161YaNmxocS04OJitW7fmeh7VqlWjZMmSNGrUiC1btuTYNzU1lQULFuDt7U2xYsVyHftuw4YNY9KkSezatQtra2u6du1qtLVv355nnnmGnTt3snv3boYMGULBggXx9/dn6tSpODs7G5nlAwcONO6bOHEiVatWZc+ePXz00Ud5WkdcXBwNGjSgYsWKbN26lc2bN9O8eXNu375NREQEdevWpXv37sZ8Hh4eXLx4kYSEBPz8/IxxIiIiGDlyJM888wwpKSns3LnzvnNOmjTJCBD36tWL9957zwj0p6enExwcjJOTE5s2bWLLli1GQPrmzZsMHDiQNm3aWGTX+/v75/ncr127xsSJE/n666/ZuHEjJ06csDjDhQsXMmLECMaMGUNiYiJjx47lo48+Yv78+UDm5/7qq69SsWJFdu/eTVhYmMX9WWxsbKhWrRqbNm3K89pEREREJP/QwyhFRETkqdM9yBvILGNy5/snwaFDhzCbzfj6+t6z3dfXl0uXLuHm5oaLiwuxsbG8+eabxMTEMGDAAKNm9o4dO0hPTzcCjuHh4QwZMoROnToBULZsWUaNGsXgwYMJDQ01xu/cuTPt2rUDYOzYsXz66afs2LGDJk2aZFvLL7/8wpIlS4iKispxT6dPn6ZEiRIW10qUKGFRb/tuJUuWZObMmfj5+ZGWlsbs2bMJDAxk+/bt1KhRw+i3cuVKHB0dAbh69SolS5Zk5cqV2TK/82LMmDEEBAQAMGTIEJo1a8aNGzewtbXlxIkTDBo0iAoVKgBQrlw5477ChQtjMplwd3fPNubLL7/MgAEDjPcnT57MdR0TJkzAz8/PImu5UqVKxs82NjbY29tbzHfixAnMZjOlSpWyWJeTkxNWVlb3XNudXnnlFXr16gXAv//9b6ZMmcKGDRsoX748S5YsISMjg9mzZ2MymQCYN28eRYoUISYmhsaNG2NnZ0daWlqu89xLeno6M2fO5LnnngOgT58+jBw50mgPDQ1l0qRJtGrVCoAyZcoYX9Z06tSJRYsWkZGRwZw5c7C1taVSpUr8/vvvvPfee9nmKlWqFMePH3/gNYqIiIjIk08Z3SIiIvJUetIzu81mc47thQoV4qWXXiImJobLly+TkJBAr169SEtL4+DBg8TGxlKrVi3s7e2BzNIbI0eOxNHR0XhlZQVfu3bNGLdKlSrGzw4ODjg7O3P27Nls8x84cIDXX3+d0NBQGjduDGQGW+8cf+zYsQ+9//Lly/Ovf/2LmjVr4u/vz9y5c/H392fKlCkW/YKCgoiLiyMuLo4dO3YQHBxM06ZNjWBm06ZNjfXcGSy+lzv3XrJkSQBj7/3796dbt240bNiQcePGceTIkTzt484M67zKyuh+ENevXwfA1tb2gecDy71nBe2z9r53714OHz6Mk5OTcZYuLi7cuHEjz+eQE3t7eyPIDZlnnzX31atXOXLkCO+++67F79bo0aONuRMTE6lSpYrF3uvWrXvPuezs7Cx+30VERETkn0MZ3SIiIvLUehIzu729vTGZTCQmJtKyZcts7YmJiRQvXpwiRYoQGBjIl19+yaZNm6hevTrOzs5G8Ds2NtbITobM8g7h4eFGVuyd7gwQFixY0KLNZDKRkZFhcS0hIYEGDRrQo0cPixrgpUqVIi4uznifVdLE3d2dM2fOWIxx5syZB87+rV27Nps3b7a45uDggLf3/z632bNnU7hwYWbNmsXo0aOZPXu2EQS+e293u7M9K3M5a+9hYWG8/fbbREVF8dNPPxEaGsrixYvv+Rndvb47ZWWa3/lFRnp6ukWfu0vW5IWrqysAly5donjx4g98f06fe2pqKjVr1mThwoXZ7stprrzs9X5zZ92TmpoKwKxZs3jhhRcs+llZWd137vu5ePGiRVBdRERERP45lNEtIiIiT7UnLbO7WLFiNGrUiOnTpxsB2iynT59m4cKFdO7cGcCo0/3dd98ZtbgDAwNZv3698TDDLDVq1CApKQlvb+9srwcp8xEfH09QUBCdOnXKVt/b2traYtysQHfdunWJjo626Ltu3br7Zt3eT1xcnJFpfT8mk4kCBQoYZ1e6dGljPZ6eng803918fHzo168fa9eupVWrVsybNw/ILCVy+/btPI2RFRi+82GLd345AJnZ1Xef153uNd9zzz2Hs7MzCQkJeVrHg6hRowaHDh3Czc0t2+9O4cKF77umvOw1NyVKlKBUqVIcPXo029xlypQBMsv57Nu3jxs3bhj3bdu27Z7jHThwgOrVqz/QGkREREQkf1CgW0RERJ56T1qwe9q0aaSlpREcHMzGjRs5efIkq1evplGjRvj4+DBixAggMyBatGhRFi1aZBHoXr58OWlpadSrV88Yc8SIEXz11VeEh4cTHx9PYmIiixcvtsjIzs2BAwcICgqicePG9O/fn9OnT3P69GnjgZj307dvX1avXs2kSZM4ePAgYWFh7Nq1iz59+hh9hg4dSseOHY33U6dOZcWKFRw+fJgDBw4QEhLCzz//TO/evS3GTktLM9aRmJjI+++/T2pqKs2bN8/zvnJz/fp1+vTpQ0xMDMePH2fLli3s3LnTqKPu5eVFamoq0dHRnD9/PsfSGN7e3nh4eBAWFsahQ4eIiopi0qRJFn2GDh3Kzp076dWrF/v27ePgwYPMmDGD8+fPG/Nt376d5ORkzp8/T0ZGBgUKFKBhw4bZMt4fhfbt2+Pq6srrr7/Opk2bOHbsGDExMXzwwQf8/vvvxpr27dtHUlIS58+fJz09PU97zYvw8HA+/vhjPv30U3777Tf279/PvHnzmDx5MgBvv/02JpOJ7t27k5CQwKpVq5g4cWK2cZKTkzl16lS2B6OKiIiIyD+EWUREROQf5vr16+aEhATz9evXH/dSHtqxY8fMnTp1MpcoUcJsMpnMgLlVq1bmq1evWvR7/fXXzdbW1uY//vjDbDabzbdv3zYXLVrUXKdOnWxjrl692uzv72+2s7MzOzs7m2vXrm3+8ssvjXbA/MMPP1jcU7hwYfO8efPMZrPZHBoaagayvTw9PXPdz7fffmv28fEx29jYmCtVqmSOioqyaO/UqZM5ICDAeD9+/Hjzc889Z7a1tTW7uLiYAwMDzT///HO2e+5ch5OTk7lWrVrm77//Pse1HDt2zAyY9+zZYzabzeYNGzaYAfOlS5eMPnv27DED5mPHjpnT0tLMbdu2NXt4eJhtbGzMpUqVMvfp08fi96tnz57mYsWKmQFzaGio2Ww2mz09Pc1TpkzJNv/mzZvNlStXNtva2prr169v/u6774y5ssTExJj9/f3NhQoVMhcpUsQcHBxsrC8pKclcp04ds52dncV9q1atMpcuXdp8+/ZtY5wpU6Zk+3wCAgLMffv2Nd7fa51Vq1Y19mE2m80pKSnmjh07ml1dXc2FChUyly1b1ty9e3fzlStXzGaz2Xz27Flzo0aNzI6OjmbAvGHDhjztdd68eebChQtbzP3DDz+Y7/6/KQsXLjRXq1bNbGNjYy5atKj5pZdeMi9btsxo37p1q7lq1apmGxsbc7Vq1cxLly61+IzNZrN57Nix5uDg4Gyfh+Rf/4S/9SIiIvLomMzmXJ50JCIiIpLP3Lhxg2PHjlGmTJmHfjjfkyY0NJTJkyezbt066tSp87iXI08gs9nMCy+8QL9+/WjXrt3jXs4T5ebNm5QrV45FixZZ/JcOkr/9E//Wi4iIyMNT6RIRERGRfCA8PJxPP/2Ubdu2ZXs4pAhk1if/8ssvuXXr1uNeyhPnxIkTfPjhhwpyi4iIiPyDKaNbRERE/nGU5Sci8s+nv/UiIiJyJ2V0i4iIiIiIiIiIiEi+pkC3iIiIiIiIiIiIiORrCnSLiIiIiIiIiIiISL6mQLeIiIiIiIiIiIiI5GsKdIuIiIiIiIiIiIhIvqZAt4iIiIiIiIiIiIjkawp0i4iIiIiIiIiIiEi+Zv24FyAiIiLyd/J7vjj/7lb1oe5Nv5VB+0Eb6NO+Ii/5lczTPa1D1j/UXCIiIiIiIpJ3yugWEREReQKdPHmSrl27UqpUKWxsbPD09KRv375cuHDhsa1p7969tGvXDg8PD+zs7PD19SUiIiLX++Lj43njjTfw8vLCZDIxderUXO9JTk7GZDJle23bts3oExYWZtFWuHBh6tevT2xsbK7jm0wmli9fnmu/vIiJicFkMnH58uVHMl5uOnfuTIsWLf6WuURERERE8gsFukVERESeMEePHsXPz49Dhw7xzTffcPjwYWbOnEl0dDR169bl4sWLj2Vdu3fvxs3NjQULFhAfH8+wYcMYOnQo06ZNy/G+a9euUbZsWcaNG4e7u/sDzbl+/XpSUlKMV82aNS3aK1WqZLRt3bqVcuXK8eqrr3LlypUH3t9f7ebNm497CSIiIiIi/1gKdIuIiIg8YXr37o2NjQ1r164lICCAZ599lqZNm7J+/XpOnTrFsGHDmDZtGs8//7xxz/LlyzGZTMycOdO41rBhQ4YPH268X7FiBTVq1MDW1payZcsSHh7OrVu3jHaTycTs2bNp2bIl9vb2lCtXjh9//NFo79q1KxEREQQEBFC2bFk6dOhAly5dWLZsWY77qVWrFp988glt27alUKFCD3QWxYoVw93d3XgVLFjQot3a2tpoq1ixIiNHjiQ1NZXffvstz3NkZY8vW7aMoKAg7O3tqVq1Klu3bjX6HD9+nObNm1O0aFEcHByoVKkSq1atIjk5maCgIACKFi2KyWSic+fOAAQGBtKnTx9CQkJwdXUlODjYmCsuLs4Y+/Lly5hMJmJiYoxr8fHxvPrqqzg7O+Pk5ET9+vU5cuQIYWFhzJ8/nxUrVhiZ7HfeJyIiIiLytFKgW0REROQJcvHiRdasWUOvXr2ws7OzaHN3d6d9+/YsWbKEgIAAEhISOHfuHACxsbG4uroaQc/09HS2bt1KYGAgAJs2baJjx4707duXhIQEvvjiCyIjIxkzZozFHOHh4bRp04Z9+/bxyiuv0L59+xwzyK9cuYKLi8ujO4C7vPbaa7i5ufHiiy9aBN3vJS0tjXnz5lGkSBHKly//wHMNGzaMgQMHEhcXh4+PD+3atTO+COjduzdpaWls3LiR/fv3M378eBwdHfHw8GDp0qUAJCUlkZKSYlHOZf78+djY2LBlyxaLLyFycurUKV566SUKFSrEzz//zO7du+natSu3bt1i4MCBtGnThiZNmhiZ7P7+/g+8VxERERGRfxo9jFJERETkCXLo0CHMZjO+vr73bPf19eXSpUu4ubnh4uJCbGwsb775JjExMQwYMMAIsu7YsYP09HQjCBoeHs6QIUPo1KkTAGXLlmXUqFEMHjyY0NBQY/zOnTvTrl07AMaOHcunn37Kjh07aNKkSba1/PLLLyxZsoSoqKhHegYAjo6OTJo0iXr16lGgQAGWLl1KixYtWL58Oa+99prRb//+/Tg6OgKZJVKcnJxYsmQJzs7ODzznwIEDadasGZB5XpUqVeLw4cNUqFCBEydO8MYbb1C5cmUg8/yyZAX63dzcKFKkiMWY5cqVY8KECcb75OTkXNfx+eefU7hwYRYvXmxksPv4+BjtdnZ2pKWlPXAZGBERERGRfzJldIuIiIg8gcxmc47thQoV4qWXXiImJobLly+TkJBAr169SEtL4+DBg8TGxlKrVi3s7e2BzAdJjhw5EkdHR+PVvXt3UlJSuHbtmjFulSpVjJ8dHBxwdnbm7Nmz2eY/cOAAr7/+OqGhoTRu3BiAEydOWIw/duzYh96/q6sr/fv354UXXqBWrVqMGzeODh068Mknn1j0K1++PHFxccTFxbF7927ee+89Wrduza5duwDo2bOnxZpycufeS5YsCWDs/YMPPmD06NHUq1eP0NBQ9u3bl6d93F1TPC/i4uKoX79+tjItIiIiIiJyfwp0i4iIiDxBvL29MZlMJCYm3rM9MTGR4sWLU6RIEQIDA4mJiWHTpk1Ur14dZ2dnI/gdGxtLQECAcV9qairh4eFGUDguLo79+/dz6NAhbG1tjX53B1dNJhMZGRkW1xISEmjQoAE9evSwqAFeqlQpi/F79uz5KI7E8MILL3D48GGLazY2Nnh7e+Pt7U316tUZN24cpUuXZurUqQCMHDnSYk05uXPvJpMJwNh7t27dOHr0KO+88w779+/Hz8+Pzz77LNc1Ozg4WLwvUCDzn993fpGRnp5u0efukjUiIiIiIpI7BbpFREREniDFihWjUaNGTJ8+nevXr1u0nT59moULFxoPO8yq0/3dd98ZtbgDAwNZv349W7ZsMa4B1KhRg6SkJCMofOcrK/iaF/Hx8QQFBdGpU6ds9b2tra0txn3Utbvj4uKMTOucWFlZGWfn5uZmsaY/w8PDg549e7Js2TIGDBjArFmzgMxgO8Dt27dzHaN48eIApKSkGNfuDsBXqVKFTZs2ZQuAZ7GxscnTXCIiIiIiTxMFukVERESeMNOmTSMtLY3g4GA2btzIyZMnWb16NY0aNcLHx4cRI0YAmQHRokWLsmjRIotA9/Lly0lLS6NevXrGmCNGjOCrr74iPDyc+Ph4EhMTWbx4sUVGdm4OHDhAUFAQjRs3pn///pw+fZrTp08bD8S8n5s3bxoZ1Tdv3uTUqVPExcVZZGdPmzaNBg0aGO/nz5/PN998w8GDBzl48CBjx45l7ty5vP/++xZj37p1y1jHoUOHGD16NAkJCbz++ut53ldehISEsGbNGo4dO8avv/7Khg0bjDrqnp6emEwmVq5cyblz50hNTb3vOHZ2dtSpU4dx48aRmJhIbGxsts+gT58+/Pe//6Vt27bs2rWLQ4cO8fXXX5OUlASAl5cX+/btIykpifPnz983IC4iIiIi8jTRwyhFRETkqbLrQM5B2dy8PfARLSQH5cqVY+fOnYSFhdGmTRvOnj2L2WymVatWfP3110bdbZPJRP369YmKiuLFF18EMoPfzs7OlC9f3qJsRnBwMCtXrmTkyJGMHz+eggULUqFCBbp165bndX3//fecO3eOBQsWsGDBAuO6p6dnjg9Z/M9//kP16tWN9xMnTmTixIkEBAQQExMDwPnz5zly5IjFfaNGjeL48eNYW1tToUIFlixZwptvvmnRJz4+3sjytre357nnnmPGjBl07Ngxz/vKi9u3b9O7d29+//13nJ2dadKkCVOmTAGgdOnSxsM+u3TpQseOHYmMjLzvWHPnzuXdd9+lZs2alC9fngkTJhh1ziEzq//nn39m0KBBBAQEYGVlRbVq1YwvLrp3705MTAx+fn6kpqayYcMGi+x9EREREZGnkcmc25OORERERPKZGzducOzYMcqUKWNRfzo/Cw0NZfLkyaxbt446deo87uWIiDx2/8S/9SIiIvLwlNEtIiIikg+Eh4fj5eXFtm3bqF279gPV1RYREREREfmnU6BbREREJJ/o0qXL416CiIiIiIjIE0mpQCIiIiIiIiIiIiKSrynQLSIiIiIiIiIiIiL5mgLdIiIiIiIiIiIiIpKvKdAtIiIiIiIiIiIiIvmaAt0iIiIiIiIiIiIikq8p0C0iIiIiIiIiIiIi+ZoC3SIiIiIiIiIiIiKSr1k/7gWIiIiI/J0mTQ/6W+cb0GvD3zqfPN3mzJnDkiVLWLt2rXEtLCyMGTNmcPbsWX744QeWL1/O5cuXWb58+eNb6N/AZDLxww8/0KJFC86fP0/FihX59ddfeeaZZx730kRERETkL6CMbhEREZEn0MmTJ+natSulSpXCxsYGT09P+vbty4ULFx7bmvbu3Uu7du3w8PDAzs4OX19fIiIicr0vPj6eN954Ay8vL0wmE1OnTs31nuTkZEwmU7bXtm3bjD5hYWEWbYULF6Z+/frExsbmOr7JZHpkgd6YmBhMJhOXL19+JOPlpnPnzrRo0SLb9Rs3bvDRRx8RGhpqXEtMTCQ8PJwvvviClJQUmjZtSkREBJGRkY90TWFhYVSrVu2Rjvkoubq60rFjR4uzEREREZF/FgW6RURERJ4wR48exc/Pj0OHDvHNN99w+PBhZs6cSXR0NHXr1uXixYuPZV27d+/Gzc2NBQsWEB8fz7Bhwxg6dCjTpk3L8b5r165RtmxZxo0bh7u7+wPNuX79elJSUoxXzZo1LdorVapktG3dupVy5crx6quvcuXKlQfe31/t5s2bf+n433//Pc7OztSrV8+4duTIEQBef/113N3dKVSoEIULF6ZIkSJ/6VqeRF26dGHhwoWP7X8/IiIiIvLXUqBbRERE5AnTu3dvbGxsWLt2LQEBATz77LM0bdqU9evXc+rUKYYNG8a0adN4/vnnjXuWL1+OyWRi5syZxrWGDRsyfPhw4/2KFSuoUaMGtra2lC1blvDwcG7dumW0m0wmZs+eTcuWLbG3t6dcuXL8+OOPRnvXrl2JiIggICCAsmXL0qFDB7p06cKyZcty3E+tWrX45JNPaNu2LYUKFXqgsyhWrBju7u7Gq2DBghbt1tbWRlvFihUZOXIkqamp/Pbbb3meIyt7fNmyZQQFBWFvb0/VqlXZunWr0ef48eM0b96cokWL4uDgQKVKlVi1ahXJyckEBWWWwylatCgmk4nOnTsDEBgYSJ8+fQgJCcHV1ZXg4GBjrri4OGPsy5cvYzKZiImJMa7Fx8fz6quv4uzsjJOTE/Xr1+fIkSOEhYUxf/58VqxYYWSyZ923ePFimjdvbowRFhZmvC9QoAAmkwnInhEeGBjIBx98wODBg3FxccHd3Z2wsDCLM7p8+TLdunWjePHiODs78/LLL7N3714AIiMjCQ8PZ+/evcaaIiMj87TXrGz46Oho/Pz8sLe3x9/fn6SkJIv5c/vdPXToEC+99BK2trZUrFiRdevWZfucK1WqRKlSpfjhhx+ytYmIiIhI/qdAt4iIiMgT5OLFi6xZs4ZevXphZ2dn0ebu7k779u1ZsmQJAQEBJCQkcO7cOQBiY2NxdXU1Aojp6els3bqVwMBAADZt2kTHjh3p27cvCQkJfPHFF0RGRjJmzBiLOcLDw2nTpg379u3jlVdeoX379jlmwF65cgUXF5dHdwB3ee2113Bzc+PFF1+0CLrfS1paGvPmzaNIkSKUL1/+gecaNmwYAwcOJC4uDh8fH9q1a2cEU3v37k1aWhobN25k//79jB8/HkdHRzw8PFi6dCkASUlJpKSkWJRzmT9/PjY2NmzZssXiS4icnDp1ipdeeolChQrx888/s3v3brp27cqtW7cYOHAgbdq0oUmTJkYmu7+/PwCbN2/Gz8/PGGfgwIHMmzcPwOh7P/Pnz8fBwYHt27czYcIERo4caREsbt26NWfPnuWnn35i9+7d1KhRgwYNGnDx4kXeeustBgwYYJFd/9Zbb+Xx1DMNGzaMSZMmsWvXLqytrenatavRltvvbkZGBq1atcLGxobt27czc+ZM/v3vf99zntq1a7Np06YHWpuIiIiI5A96GKWIiIjIE+TQoUOYzWZ8fX3v2e7r68ulS5dwc3PDxcWF2NhY3nzzTWJiYhgwYIARZN2xYwfp6elGEDQ8PJwhQ4bQqVMnAMqWLcuoUaMYPHiwRd3izp07065dOwDGjh3Lp59+yo4dO2jSpEm2tfzyyy8sWbKEqKioR3oGAI6OjkyaNIl69epRoEABli5dSosWLVi+fDmvvfaa0W///v04OjoCmSVSnJycWLJkCc7Ozg8858CBA2nWrBmQeV6VKlXi8OHDVKhQgRMnTvDGG29QuXJlIPP8smQF+t3c3LKVBClXrhwTJkww3icnJ+e6js8//5zChQuzePFiI4Pdx8fHaLezsyMtLc2iDMzly5e5cuUKpUqVMq45Ojoa68mtZEyVKlWM34Ny5coxbdo0oqOjadSoEZs3b2bHjh2cPXvWyMifOHEiy5cv5/vvv6dHjx44Ojoa2fUPY8yYMQQEBAAwZMgQmjVrxo0bN7C1tc31d3f9+vUcPHiQNWvWGPsfO3YsTZs2zTZPqVKl2LNnz0OtUURERESebMroFhEREXkCmc3mHNsLFSrESy+9RExMDJcvXyYhIYFevXqRlpbGwYMHiY2NpVatWtjb2wOZD5IcOXIkjo6Oxqt79+6kpKRw7do1Y9wqVaoYPzs4OODs7MzZs2ezzX/gwAFef/11QkNDady4MQAnTpywGH/s2LEPvX9XV1f69+/PCy+8QK1atRg3bhwdOnTgk08+sehXvnx54uLiiIuLY/fu3bz33nu0bt2aXbt2AdCzZ0+LNeXkzr2XLFkSwNj7Bx98wOjRo6lXrx6hoaHs27cvT/u4u6Z4XsTFxVG/fv1sZVpycv36dQBsbW0feD6w3Dtk7j9r73v37iU1NZVixYpZnOWxY8eMGuB/Vk5nn9vvbmJiIh4eHhZB/rp1695zHjs7O4vfdxERERH551BGt4iIiMgTxNvbG5PJRGJiIi1btszWnpiYSPHixSlSpAiBgYF8+eWXbNq0ierVq+Ps7GwEv2NjY40MWYDU1FTCw8Np1apVtjHvDI7eHVw1mUxkZGRYXEtISKBBgwb06NHDogZ4qVKlLOoxP+qSJi+88EK22ss2NjZ4e3sb76tXr87y5cuZOnUqCxYsYOTIkQwcODBP49+596x61ll779atG8HBwURFRbF27Vo+/vhjJk2axPvvv5/jmA4ODhbvCxTIzDO584uM9PR0iz53l6zJi2LFimEymbh06dID3ws5f+6pqamULFnSooZ4lpweapmXvd5r/rvPPq+/u3lx8eJFihcv/kD3iIiIiEj+oIxuERERkSdIsWLFaNSoEdOnTzeydLOcPn2ahQsXGg87zKrT/d133xm1uAMDA1m/fj1btmwxrgHUqFGDpKQkvL29s72yApJ5ER8fT1BQEJ06dcpW39va2tpi3Ecd6I6LizOyfXNiZWVlnJ2bm5vFmv4MDw8PevbsybJlyxgwYACzZs0CMoPtALdv3851jKwg6531su/8cgAys5s3bdp036CwjY1NtrlsbGyoWLEiCQkJed5PXtWoUYPTp09n+3y9vb1xdXW975ryste8zp/T766vry8nT560mGfbtm33HOvAgQNUr179gdcgIiIiIk8+ZXSLiIjIU2XuzP1/63wDej34PdOmTcPf35/g4GBGjx5NmTJliI+PZ9CgQfj4+DBixAggMyBatGhRFi1axMqVK4HMQPfAgQMxmUzUq1fPGHPEiBG8+uqrPPvss7z55psUKFCAvXv3cuDAAUaPHp2ndR04cICXX36Z4OBg+vfvz+nTp4HMwHJOWbI3b940ArA3b97k1KlTxMXF4ejoaASfp02bxg8//EB0dDTwv4c4ZgUlly1bxty5c5k9e7bF2Ldu3TLW8ccff7BkyRISEhLu+zDChxUSEkLTpk3x8fHh0qVLbNiwwaij7unpiclkYuXKlbzyyivY2dndt0yKnZ0dderUYdy4cZQpU4azZ89aZMUD9OnTh88++4y2bdsydOhQChcuzLZt26hduzbly5fHy8uLNWvWkJSURLFixShcuDAFCxYkODiYzZs3ExIS8kj33rBhQ+rWrUuLFi2YMGECPj4+/Oc//yEqKoqWLVvi5+eHl5cXx44dIy4ujmeeeQYnJ6c87TUvcvvdbdiwIT4+PnTq1IlPPvmE//73vwwbNizbONeuXWP37t1/qqSOiIiIiDy5FOgWERGRp0r8vvOPewm5KleuHDt37iQsLIw2bdpw9uxZzGYzrVq14uuvvzbqbptMJurXr09UVBQvvvgikBn8dnZ2pnz58hZlM4KDg1m5ciUjR45k/PjxFCxYkAoVKtCtW7c8r+v777/n3LlzLFiwgAULFhjXPT09c3zI4n/+8x+LLNqJEycyceJEAgICjHIY58+fz1bvedSoURw/fhxra2sqVKjAkiVLePPNNy36xMfHG1ne9vb2PPfcc8yYMYOOHTvmeV95cfv2bXr37s3vv/+Os7MzTZo0YcqUKQCULl3aeGBily5d6NixI5GRkfcda+7cubz77rvUrFmT8uXLM2HCBKPOOWRm9f/8888MGjSIgIAArKysqFatmvHFRffu3YmJicHPz4/U1FQ2bNhAYGAg7777Ln5+fly5coXChQs/sr2bTCZWrVrFsGHD6NKlC+fOncPd3Z2XXnqJEiVKAPDGG2+wbNkygoKCuHz5MvPmzaNz58657jUvcvvdLVCgAD/88APvvvsutWvXxsvLi08//TTbA1RXrFjBs88+S/369R/NwYiIiIjIE8Vkzu1JRyIiIiL5zI0bNzh27BhlypR56IfzPWlCQ0OZPHky69ato06dOo97OfKEat26NTVq1GDo0KGPeylPnDp16vDBBx/w9ttvP+6lyCPyT/xbLyIiIg9PNbpFRERE8oHw8HA+/fRTtm3blu3hkCJZPvnkk/uWTXmanT9/nlatWtGuXbvHvRQRERER+Ysoo1tERET+cZTlJyLyz6e/9SIiInInZXSLiIiIiIiIiIiISL6mQLeIiIiIiIiIiIiI5GsKdIuIiIiIiIiIiIhIvqZAt4iIiIiIiIiIiIjkawp0i4iIiIiIiIiIiEi+pkC3iIiIiIiIiIiIiORrCnSLiIiIiIiIiIiISL5m/bgXICIiIvJ3Kl3dlZf6VXmoe2+nZ/Bdt1jq9PDFq557nu75puPPDzWXiIiIiIiI5J0yukVERESeQCdPnqRr166UKlUKGxsbPD096du3LxcuXHhsa9q7dy/t2rXDw8MDOzs7fH19iYiIyPW+wMBATCZTtlezZs3u26dEiRK0bt2a48eP5zh2TEwMJpOJy5cv/9ntARAWFka1atUeyVh54eXlxdSpU/+2+URERERE/qkU6BYRERF5whw9ehQ/Pz8OHTrEN998w+HDh5k5cybR0dHUrVuXixcvPpZ17d69Gzc3NxYsWEB8fDzDhg1j6NChTJs2Lcf7li1bRkpKivE6cOAAVlZWtG7d2qJf9+7dSUlJ4T//+Q8rVqzg5MmTdOjQ4a/c0kNLT09/3EsQEREREZE7KNAtIiIi8oTp3bs3NjY2rF27loCAAJ599lmaNm3K+vXrOXXqFMOGDWPatGk8//zzxj3Lly/HZDIxc+ZM41rDhg0ZPny48X7FihXUqFEDW1tbypYtS3h4OLdu3TLaTSYTs2fPpmXLltjb21OuXDl+/PFHo71r165EREQQEBBA2bJl6dChA126dGHZsmU57sfFxQV3d3fjtW7dOuzt7bMFuu3t7XF3d6dkyZLUqVOHPn368Ouvvz7Q2UVGRlKkSBHWrFmDr68vjo6ONGnShJSUFKNPTEwMtWvXxsHBgSJFilCvXj2OHz9OZGQk4eHh7N2718gsj4yMNM5mxowZvPbaazg4ODBmzBhjrjtlfQ53+r//+z9q1aqFra0trq6utGzZEsjMYj9+/Dj9+vUz5hMRERERkYejQLeIiIg8VTJuZXA7/eFeGbcyALj9AGM8qIsXL7JmzRp69eqFnZ2dRZu7uzvt27dnyZIlBAQEkJCQwLlz5wCIjY3F1dWVmJgYIDPjeOvWrQQGBgKwadMmOnbsSN++fUlISOCLL74gMjKSMWPGWMwRHh5OmzZt2LdvH6+88grt27fPMYP8ypUruLi4PNAe58yZQ9u2bXFwcMjxHL799lteeOGFBxob4Nq1a0ycOJGvv/6ajRs3cuLECQYOHAjArVu3aNGiBQEBAezbt4+tW7fSo0cPTCYTb731FgMGDKBSpUpG9vlbb71ljBsWFkbLli3Zv38/Xbt2zdNaoqKiaNmyJa+88gp79uwhOjqa2rVrA5mZ7s888wwjR4405hMRERERkYejh1GKiIjIU+X0gUt81y32T42xc24SO+cm5a3zuw829qFDhzCbzfGEGwMAAH3rSURBVPj6+t6z3dfXl0uXLuHm5oaLiwuxsbG8+eabxMTEMGDAAKNm9o4dO0hPT8ff3x/IDGAPGTKETp06AVC2bFlGjRrF4MGDCQ0NNcbv3Lkz7dq1A2Ds2LF8+umn7NixgyZNmmRbyy+//MKSJUuIiorK8/527NjBgQMHmDNnTra26dOnM3v2bMxmM9euXcPHx4c1a9bkeews6enpzJw5k+eeew6APn36MHLkSAD++9//cuXKFV599VWj/c6zdnR0xNraGnf37A8bffvtt+nSpcsDrWXMmDG0bduW8PBw41rVqlWBzEx3KysrnJyc7jmfiIiIiIjknQLdIiIi8tSp0+PeQeTc3L6Vwc65SbhXdsGrbolHvCpLZrM5x/ZChQrx0ksvERMTQ8OGDUlISKBXr15MmDCBgwcPEhsbS61atbC3twcyHyS5ZcsWiwzu27dvc+PGDa5du2b0q1KlitHu4OCAs7MzZ8+ezTb/gQMHeP311wkNDaVx48YAnDhxgooVKxp9PvzwQz788EOL++bMmUPlypWNrOY7tW/fnmHDhgFw5swZxo4dS+PGjdm9ezdOTk5UqlTJeDhl/fr1+emnn+55Nvb29kYQG6BkyZLGHlxcXOjcuTPBwcE0atSIhg0b0qZNG0qWLHm/ozb4+fnl2uducXFxdO/e/YHvExERERGRB6NAt4iIiDxVSlUrhle9h8uevZ2eGej2qlviocfIjbe3NyaTicTERKOW850SExMpXrw4RYoUITAwkC+//JJNmzZRvXp1nJ2djeB3bGwsAQEBxn2pqamEh4fTqlWrbGPa2toaPxcsWNCizWQykZFhWYIlISGBBg0a0KNHD4sa4KVKlSIuLs54f3dJk6tXr7J48WIju/puhQsXxtvb2ziHOXPmULJkSZYsWUK3bt1YtWqV8RDIu8u63Olee7jzi4N58+bxwQcfsHr1apYsWcLw4cNZt24dderUue+YQLZSKwUKFMj2hcTdD6nMaZ0iIiIiIvLoqEa3iIiIyBOkWLFiNGrUiOnTp3P9+nWLttOnT7Nw4UI6d+4MYNTp/u6774xa3IGBgaxfv54tW7YY1wBq1KhBUlIS3t7e2V4FCuT9n4Tx8fEEBQXRqVOnbPW9ra2tLca9O9D93XffkZaWRocOHfI0l5WVFYBxDp6ensbYpUuXzvOa76V69eoMHTqUX375heeff55FixYBYGNjw+3bt/M0RvHixfnjjz+4evWqce3OQD9kZshHR0ffd4wHmU9ERERERO5PgW4RERGRJ8y0adNIS0sjODiYjRs3cvLkSVavXk2jRo3w8fFhxIgRQGYQtWjRoixatMgi0L18+XLS0tKoV6+eMeaIESP46quvCA8PJz4+nsTERBYvXmyRkZ2bAwcOEBQUROPGjenfvz+nT5/m9OnTxgMxczNnzhxatGhBsWLF7tl+7do1Y8y9e/fy3nvvYWtra5RGeRSOHTvG0KFD2bp1K8ePH2ft2rUcOnTIqNPt5eXFsWPHiIuL4/z586Slpd13rBdeeAF7e3s+/PBDjhw5wqJFi4iMjLToExoayjfffENoaCiJiYns37+f8ePHG+1eXl5s3LiRU6dOcf78+Ue2TxERERGRp41Kl4iIiMhT5dSePxlMfMCHSz6McuXKsXPnTsLCwmjTpg1nz57FbDbTqlUrvv76a6Oetslkon79+kRFRfHiiy8CmcFvZ2dnypcvb1FqIzg4mJUrVzJy5EjGjx9PwYIFqVChAt26dcvzur7//nvOnTvHggULWLBggXHd09OT5OTkHO9NSkpi8+bNrF279r59Zs2axaxZswAoWrQoVapUYdWqVZQvXz7Pa8yNvb09Bw8eZP78+Vy4cIGSJUvSu3dv/vWvfwHwxhtvsGzZMoKCgrh8+TLz5s0zMujv5uLiwoIFCxg0aBCzZs2iQYMGhIWF0aNHD6NPYGAg3333HaNGjWLcuHFGeZksI0eO5F//+hfPPfccaWlpudZmFxERERGRezOZ9a9pERER+Ye5ceMGx44do0yZMhb1p/Oz0NBQJk+enKda0iIiT4N/4t96EREReXjK6BYRERHJB8LDw/Hy8mLbtm3Url37gepqi4iIiIiI/NMp0C0iIiKST3Tp0uVxL0FEREREROSJpFQgEREREREREREREcnXFOgWERERERERERERkXxNgW4RERERERERERERydcU6BYRERERERERERGRfE2BbhERERERERERERHJ1xToFhEREREREREREZF8TYFuEREREREREREREcnXrB/3AkRERET+Ttb2tjiWdqPagPZY2xXK831Xjp5izydfP/C96zuFPeRK5Wn3zjvv4Ovry4cffgjAtWvXeOedd1i3bh1//PEHly5dolq1aoSEhBASEvJ4F/sXiomJISgoiEuXLlGkSBFWr17NkCFD+PXXXylQQHk7IiIiIpJJ/zIUERGRp8rDBLkBCpctTfVB75B66ixxkxZy63raX7TCTCdPnqRr166UKlUKGxsbPD096du3LxcuXPhL583J3r17adeuHR4eHtjZ2eHr60tERESu9wUGBmIymbK9mjVrdt8+JUqUoHXr1hw/fjzHsWNiYjCZTFy+fPnPbg+AsLAwqlWr9kjGygsvLy+mTp2a7frevXtZtWoVH3zwgXFt/vz5bNq0iV9++YWUlBQKFy7Mzp076dGjxyNdU2Bg4BMdOG/SpAkFCxZk4cKFj3spIiIiIvIEUaBbREREnioPE+TO8ncFu48ePYqfnx+HDh3im2++4fDhw8ycOZPo6Gjq1q3LxYsX/5J5c7N7927c3NxYsGAB8fHxDBs2jKFDhzJt2rQc71u2bBkpKSnG68CBA1hZWdG6dWuLft27dyclJYX//Oc/rFixgpMnT9KhQ4e/cksPLT09/S8d/7PPPqN169Y4Ojoa144cOYKvry/PP/887u7umEwmihcvjr29/V+6lidR586d+fTTTx/3MkRERETkCaJAt4iIiDxVHjbIneXvCHb37t0bGxsb1q5dS0BAAM8++yxNmzZl/fr1nDp1imHDhjFt2jSef/55457ly5djMpmYOXOmca1hw4YMHz7ceL9ixQpq1KiBra0tZcuWJTw8nFu3bhntJpOJ2bNn07JlS+zt7SlXrhw//vij0d61a1ciIiIICAigbNmydOjQgS5durBs2bIc9+Pi4oK7u7vxWrduHfb29tkC3fb29ri7u1OyZEnq1KlDnz59+PXXXx/o7CIjIylSpAhr1qzB19cXR0dHmjRpQkpKitEnJiaG2rVr4+DgQJEiRahXrx7Hjx8nMjKS8PBw9u7da2SWR0ZGGmczY8YMXnvtNRwcHBgzZowx152yPoc7/d///R+1atXC1tYWV1dXWrZsCWRmTh8/fpx+/foZ8wHcvn2b77//nubNmxtjBAYGMmnSJDZu3IjJZCIwMBDInhGe22cIcODAAZo2bYqjoyMlSpTgnXfe4fz580BmADk2NpaIiAhjTcnJyXnaa1Y2/Ndff42XlxeFCxembdu2/PHHH0afjIwMPv74Y8qUKYOdnR1Vq1bl+++/txh31apV+Pj4YGdnR1BQEMnJydk+5+bNm7Nr1y6OHDmSrU1EREREnk4KdIuIiIg8oL8y2H3x4kXWrFlDr169sLOzs2hzd3enffv2LFmyhICAABISEjh37hwAsbGxuLq6EhMTA2RmHG/dutUIiG7atImOHTvSt29fEhIS+OKLL4iMjGTMmDEWc4SHh9OmTRv27dvHK6+8Qvv27XPMIL9y5QouLi4PtMc5c+bQtm1bHBwccjyHb7/9lhdeeOGBxobMWtYTJ07k66+/ZuPGjZw4cYKBAwcCcOvWLVq0aEFAQAD79u1j69at9OjRA5PJxFtvvcWAAQOoVKmSkX3+1ltvGeOGhYXRsmVL9u/fT9euXfO0lqioKFq2bMkrr7zCnj17iI6Opnbt2kBmpvszzzzDyJEjjfkA9u3bx5UrV/Dz8zPGWbZsGd27d6du3bqkpKTk+OVCTp/h5cuXefnll6levTq7du1i9erVnDlzhjZt2gAQERFB3bp1jez6lJQUPDw88nz2R44cYfny5axcuZKVK1cSGxvLuHHjjPaPP/6Yr776ipkzZxIfH0+/fv3o0KEDsbGxQGbJnlatWtG8eXPi4uLo1q0bQ4YMyTbPs88+S4kSJdi0aVOe1yYiIiIi/2x6GKWIiIjIQ8gKdu/55GviJi28Z0mUhwmAHzp0CLPZjK+v7z3bfX19uXTpEm5ubri4uBAbG8ubb75JTEwMAwYMMGpm79ixg/T0dPz9/YHM4OeQIUPo1KkTAGXLlmXUqFEMHjyY0NBQY/zOnTvTrl07AMaOHcunn37Kjh07aNKkSba1/PLLLyxZsoSoqKg872/Hjh0cOHCAOXPmZGubPn06s2fPxmw2c+3aNXx8fFizZk2ex86Snp7OzJkzee655wDo06cPI0eOBOC///0vV65c4dVXXzXa7zxrR0dHrK2tcXd3zzbu22+/TZcuXR5oLWPGjKFt27aEh4cb16pWrQpkZrpbWVnh5ORkMd/x48exsrLCzc3NuObi4oK9vT02Njb3XNudcvoMp02bRvXq1Rk7dqzRf+7cuXh4ePDbb7/h4+ODjY2NkV3/oDIyMoiMjMTJyQnIfKBmdHQ0Y8aMIS0tjbFjx7J+/Xrq1q0LZP4ebt68mS+++IKAgABmzJjBc889x6RJkwAoX748+/fvZ/z48dnmKlWqVK413EVERETk6aGMbhEREZGHlFNm963racRNeviH5ZnN5hzbCxUqxEsvvURMTAyXL18mISGBXr16kZaWxsGDB4mNjaVWrVpG/ea9e/cycuRIHB0djVdW1u61a9eMcatUqWL87ODggLOzM2fPns02/4EDB3j99dcJDQ2lcePGAJw4ccJi/DuDqVnmzJlD5cqVjazmO7Vv3564uDj27t3L5s2b8fb2pnHjxkbpi0qVKhljN23a9L5nY29vbwSxAUqWLGnswcXFhc6dOxMcHEzz5s2JiIiwKGuSkzszrPMqLi6OBg0aPNA9169fp1ChQtlKoORVTp/h3r172bBhg8XnVKFCBYBHUgbEy8vLCHKD5dkfPnyYa9eu0ahRI4v5v/rqK2PuxMTEbFn8WUHxu9nZ2Vn87oqIiIjI000Z3SIiIiJ/wr0yuwHiJi0k9VT2AHFuvL29MZlMJCYmGrWc75SYmEjx4sUpUqQIgYGBfPnll2zatInq1avj7OxsBL9jY2MJCAgw7ktNTSU8PJxWrVplG9PW1tb4uWDBghZtJpOJjIwMi2sJCQk0aNCAHj16WNQAL1WqFHFxccb7u0uaXL16lcWLFxvZ1XcrXLgw3t7exjnMmTOHkiVLsmTJErp168aqVauMh0DeXdblTvfaw51fHMybN48PPviA1atXs2TJEoYPH866deuoU6fOfccEspVaKVCgQLYvJO5+SGVO67wfV1dXrl27xs2bN7GxsXng+3P6DFNTU2nevPk9M6RLlix53zHzste8zA2Z5VxKly5t0a9QoQevnX/x4kWKFy/+wPeJiIiIyD+TAt0iIiIif9Kdwe7dY+cBcP38ZaoPeueBxypWrBiNGjVi+vTp9OvXzyJQevr0aRYuXEjv3r0BCAgIICQkhO+++86oxR0YGMj69evZsmULAwYMMO6tUaMGSUlJRiD5YcXHx/Pyyy/TqVOnbPW9ra2tcxz/u+++Iy0tjQ4dOuRpLisrKyAzwxnA09PzIVedXfXq1alevTpDhw6lbt26LFq0iDp16mBjY8Pt27fzNEbx4sX5448/uHr1qhEEvzPQD5nZ1dHR0fcteXKv+apVqwZkfqGQ9fOjUqNGDZYuXYqXlxfW1vf+vwL3WlNe9pqbihUrUqhQIU6cOGHxJcydfH19sz08c9u2bdn63bhxgyNHjlC9evUHWoOIiIiI/HOpdImIiIjII1C4bGmqvN+G1JNnSD15hirvt6Fw2dK533gP06ZNIy0tjeDgYDZu3MjJkydZvXo1jRo1wsfHhxEjRgCZQdSiRYuyaNEii0D38uXLSUtLo169esaYI0aM4KuvviI8PJz4+HgSExNZvHixRUZ2bg4cOEBQUBCNGzemf//+nD59mtOnTxsPxMzNnDlzaNGiBcWKFbtn+7Vr14wx9+7dy3vvvYetra1RGuVROHbsGEOHDmXr1q0cP36ctWvXcujQIaNOt5eXF8eOHSMuLo7z58+Tlnb/OusvvPAC9vb2fPjhhxw5coRFixYRGRlp0Sc0NJRvvvmG0NBQEhMTs9Wb9vLyYuPGjZw6dYrz588DmUHlGjVqsHnz5ke27yy9e/fm4sWLtGvXjp07d3LkyBHWrFlDly5djOC2l5cX27dvJzk5mfPnz5ORkZGnvebGycmJgQMH0q9fP+bPn8+RI0f49ddf+eyzz5g/fz4APXv25NChQwwaNIikpKT7zrNt2zYKFSp037ImIiIiIvIUMouIiIj8w1y/ft2ckJBgvn79+uNeykM7duyYuVOnTuYSJUqYTSaTGTC3atXKfPXqVYt+r7/+utna2tr8xx9/mM1ms/n27dvmokWLmuvUqZNtzNWrV5v9/f3NdnZ2ZmdnZ3Pt2rXNX375pdEOmH/44QeLewoXLmyeN2+e2Ww2m0NDQ81Atpenp2eu+zl48KAZMK9du/ae7QEBARZjFi1a1BwQEGD++eefcxx3w4YNZsB86dIls9lsNs+bN89cuHBhiz4//PCDOeufvadPnza3aNHCXLJkSbONjY3Z09PTPGLECPPt27fNZrPZfOPGDfMbb7xhLlKkiBkw9n6vs8ka29vb22xnZ2d+9dVXzV9++aX57n9iL1261FytWjWzjY2N2dXV1dyqVSujbevWreYqVaqYCxUqZHHf9OnTs32Gffv2NQcEBFhc8/T0NE+ZMsV4n9tnaDabzb/99pu5ZcuW5iJFipjt7OzMFSpUMIeEhJgzMjLMZrPZnJSUZK5Tp47Zzs7ODJiPHTuWp72Ghoaaq1atajH3lClTLH4/MjIyzFOnTjWXL1/eXLBgQXPx4sXNwcHB5tjYWKPP//3f/5m9vb3NhQoVMtevX988d+5ci8/YbDabe/ToYf7Xv/5llqfbP+FvvYiIiDw6JrM5lycdiYiIiOQzN27c4NixY5QpU8ai/nR+FhoayuTJk/NUS1ryv+vXr1O+fHmWLFmirOW7nD9/nvLly7Nr1y7KlCnzuJcjj9E/8W+9iIiIPDzV6BYRERHJB8LDw/Hy8mLbtm3Url2bAgVUge6fzM7Ojq+++sooZyL/k5yczPTp0xXkFhERERELyugWERGRfxxl+YmI/PPpb72IiIjcSalAIiIiIiIiIiIiIpKvKdAtIiIiIiIiIiIiIvmaAt0iIiIiIiIiIiIikq8p0C0iIiIiIiIiIiIi+ZoC3SIiIiIiIiIiIiKSrynQLSIiIiIiIiIiIiL5mgLdIiIiIvlYTEwMJpOJy5cv/+1zR0ZGUqRIkT89jpeXF1OnTn2ge5KTkzGZTMTFxf3p+UVEREREJP+zftwLEBEREfk7NZ4382+db22Xnn/p+P7+/qSkpFC4cOFc+8bExBAUFMSlS5ceSYA6LwIDA4mNjc12/ZVXXiEqKuqhx/Xw8CAlJQVXV9c/szxDZGQkISEhf9sXBoGBgVSrVu2BA/z3EhMTQ//+/YmPj8fDw4Phw4fTuXPn+/ZPSkqiZ8+eJCQkcOXKFUqVKsXbb79NaGgoBQsW/NPrERERERF5HBToFhERkafK+m698X6zBc82CKTAAwT1bly6TMLcr7gQn4idW3EqdmqPi2/5v3CleWNjY4O7u/sjHfPmzZvY2Ng8krGWLVvGzZs3jfcXLlygatWqtG7d+k+Na2Vl9cj3nReP8mwehWPHjtGsWTN69uzJwoULiY6Oplu3bpQsWZLg4OB73lOwYEE6duxIjRo1KFKkCHv37qV79+5kZGQwduzYv3kHIiIiIiKPhkqXiIiIyFPFnJHBoW+XEf2vD7h1/Xqe7rmYmMSmAUO5EJ8IwPWz59j9yVT2zZz9yNcXGBjI+++/T0hICEWLFqVEiRLMmjWLq1ev0qVLF5ycnPD29uann34CspcuOX78OM2bN6do0aI4ODhQqVIlVq1aRXJyMkFBQQAULVoUk8lkZP0GBgbSp08fQkJCcHV1NQKkkydPpnLlyjg4OODh4UGvXr1ITU19oP24uLjg7u5uvNatW4e9vX22QPcff/xBu3btcHBwoHTp0nz++ec5jnt36ZKsc4iOjsbPzw97e3v8/f1JSkoy7tm7dy9BQUE4OTnh7OxMzZo12bVrFzExMXTp0oUrV65gMpkwmUyEhYUBmWVVRo0aRceOHXF2dqZHjx73LBcTFxeHyWQiOTnZuLZlyxYCAwOxt7enaNGiBAcHc+nSJTp37kxsbCwRERHGfHfel+XLL7+kVKlSZGRkWFx//fXX6dq1KwAzZ86kTJkyTJo0CV9fX/r06cObb77JlClT7nt2ZcuWpUuXLlStWhVPT09ee+012rdvz6ZNm3I8cxERERGRJ5kC3SIiIvJUeWnKOJw8PQD4dcq0XPubMzLY/clUAMq/3YaXv/iUOmHDADizYzeXkg498jXOnz8fV1dXduzYwfvvv897771H69at8ff359dff6Vx48a88847XLt2Ldu9vXv3Ji0tjY0bN7J//37Gjx+Po6MjHh4eLF26FMgsXZGSkkJERITFnDY2NmzZsoWZMzPLuxQoUIBPP/2U+Ph45s+fz88//8zgwYP/1N7mzJlD27ZtcXBwsLj+ySefULVqVfbs2cOQIUPo27cv69ate+Dxhw0bxqRJk9i1axfW1tZGQBigffv2PPPMM+zcuZPdu3czZMgQChYsiL+/P1OnTsXZ2ZmUlBRSUlIYOHCgcd/EiRONtX300Ud5WkdcXBwNGjSgYsWKbN26lc2bN9O8eXNu375NREQEdevWpXv37sZ8Hh4e2cZo3bo1Fy5cYMOGDca1ixcvsnr1atq3bw/A1q1badiwocV9wcHBbN26Nc9ndvjwYVavXk1AQECe7xERERERedKodImIiIg8VQoVLswLHw1hfbfeXDl8FLPZjMlkum//i4mZGcHFnq/Isw0zM6Kdnn2GOmEfsi1sLHs//5LATz95pGusWrUqw4cPB2Do0KGMGzcOV1dXunfvDsCIESOYMWMG+/bty3bviRMneOONN6hcuTKQmb2bxcXFBQA3N7dsNbrLlSvHhAkTLK6FhIQYP3t5eTF69Gh69uzJ9OnTH2pfO3bs4MCBA8yZMydbW7169RgyZAgAPj4+bNmyhSlTptCoUaMHmmPMmDFGwHbIkCE0a9aMGzduYGtry4kTJxg0aBAVKlQAMvecpXDhwphMpnuWQ3n55ZcZMGCA8f7kyZO5rmPChAn4+flZnFWlSpWMn21sbLC3t8+x/ErRokVp2rQpixYtokGDBgB8//33uLq6Gtn5p0+fpkSJEhb3lShRgv/+979cv34dOzu7+46f9cVJWloaPXr0YOTIkbnuS0RERETkSaWMbhEREXnqmAr8759A5rvKQtwt7dJlAIpXrWxx3c6tOADpD1jKIy+qVKli/GxlZUWxYsWMwDVgBDbPnj2b7d4PPviA0aNHU69ePUJDQ+8ZDL+XmjVrZru2fv16GjRoQOnSpXFycuKdd97hwoUL98wkP3HiBI6OjsbrXrWe58yZQ+XKlaldu3a2trp162Z7n5iYWSqmZ8+eFmPn5M6zK1myJPC/c+rfvz/dunWjYcOGjBs3jiNHjuQ4VhY/P7889btTVkb3g6hUqZKxx6ZNmwKZWehLly4lLS0NgIULF9K2bVsKFPjz/4xfsmQJv/76K4sWLSIqKoqJEyf+6TFFRERERB4XBbpFRETkqXPzv/81fi5gZZVjXyevZwE4uHAJZrPZuJ6ydTsAbn41Hvn6Ct71kEyTyWRxLSsD/e7azQDdunXj6NGjvPPOO+zfvx8/Pz8+++yzXOe8u5RIcnIyr776KlWqVGHp0qXs3r3bqJt958Mls5QqVYq4uDjj1bNnT4v2q1evsnjxYt59991c13K3kSNHWoydk5zOKSwsjPj4eJo1a8bPP/9MxYoV+eGHH3Kd/+6zyQoy3/n7kJ6ebtEnp0zq+1m1apWxx9mzM+u/N2/eHLPZTFRUFCdPnmTTpk1G2RIAd3d3zpw5YzHOmTNncHZ2znUNHh4eVKxYkXbt2jFu3DjCwsK4ffv2A69bRERERORJoNIlIiIi8lQ5vGwFx1auBqBM81dy7e/0TGnj5/Xv9sKn7ZtcOJDAhQMJAJRv1/p+tz42Hh4e9OzZk549ezJ06FBmzZrF+++/j42NDUCegpm7d+8mIyODSZMmGYHdb7/99r79ra2t8fb2vm/7d999R1paGh06dLhn+7Zt27K99/X1BTJLrbi5ueW65rzw8fHBx8eHfv360a5dO+bNm0fLli2xsbHJc5C3ePHMbP6UlBSKFi0KkC0AX6VKFaKjowkPD7/nGPeaz9PTM1s/W1tbWrVqxcKFCzl8+DDly5enRo3/fblSt25dVq1aZXHPunXrsmXI5yYjI4P09HQyMjKwyuXLHxERERGRJ5EyukVEROSpkhXkLvWiP8+1eDVP9wR+9r+SDr8t/t4IctccFIJt0SKPfI1/RkhICGvWrOHYsWP8+uuvbNiwwQgYe3p6YjKZWLlyJefOnSM1h7Ir3t7epKen89lnn3H06FG+/vpr4yGVD2POnDm0aNGCYsWK3bN9y5YtTJgwgd9++43PP/+c7777jr59+z70fHe7fv06ffr0ISYmhuPHj7NlyxZ27txpnI2XlxepqalER0dz/vz5e5ZnyeLt7Y2HhwdhYWEcOnSIqKgoJk2aZNFn6NCh7Ny5k169erFv3z4OHjzIjBkzOH/+vDHf9u3bSU5O5vz58/fMzs/Svn17oqKimDt3rkU2N2SWdTl69CiDBw/m4MGDTJ8+nW+//ZZ+/foZfaZNm2ZRRmXhwoV8++23JCYmcvToUb799luGDh3KW2+9le2/JhARERERyS8U6BYREZGnyvM9uhD42UQqdX0nx4dQ3qmggwON5s6g7qgRVHq3I9X79aHh7M9x8S3/F6/2wd2+fZvevXvj6+tLkyZN8PHxMR6IWLp0acLDwxkyZAglSpSgT58+9x2natWqTJ48mfHjx/P888+zcOFCPv7444daU1JSEps3b86xbMmAAQPYtWsX1atXZ/To0UyePJng4OCHmu9erKysuHDhAh07dsTHx4c2bdrQtGlTI+Pa39+fnj178tZbb1G8ePFsD+a8U8GCBfnmm284ePAgVapUYfz48YwePdqij4+PD2vXrmXv3r3Url2bunXrsmLFCqytM/+DyoEDB2JlZUXFihUpXrw4J06cuO98L7/8Mi4uLiQlJfH2229btJUpU4aoqCjWrVtH1apVmTRpErNnz7Y4u/Pnz1vUI7e2tmb8+PHUrl2bKlWqEB4eTp8+fYxyKSIiIiIi+ZHJfGdxQREREZF/gBs3bnDs2DHKlCmDra3t416OiIj8BfS3XkRERO6kjG4RERERERERERERydcU6BYRERERERERERGRfE2BbhERERERERERERHJ1xToFhEREREREREREZF8TYFuEREREREREREREcnXFOgWERERERERERERkXxNgW4RERERERERERERydcU6BYRERERERERERGRfE2BbhERERERERERERHJ1xToFhEREREREREREZF8zfpxL0BERETk79R03g+knjrJ5o/6AtBk7rIc+1+/cI7YQf/K1vfQ8sUc+fFbyjRtSfnW79z3/p+6tHwEq76/mJgYgoKCuHTpEkWKFPlL57pbZGQkISEhXL58+U+N4+XlRUhICCEhIXm+Jzk5mTJlyrBnzx6qVav2p+aXe7t58yYVK1bkq6++wt/fH4CDBw/SuXNn4uLiqFChAsuXL38qPoewsDCWL19OXFwcAEOGDOHq1at89tlnj3dhIiIiImJQRreIiIhIDm7+90qO7TZOTn/TSu7N39+flJQUChcunGvfmJgYTCbTnw5MP4jAwEBMJlO2V7Nmzf7UuB4eHqSkpPD8888/knVGRkb+rV8UBAYGPlBgPycxMTHUqFGDQoUK4e3tTWRkZI79k5KSCAoKokSJEtja2lK2bFmGDx9Oenq6Rb+ZM2dSpkwZI8gNEBoaioODA0lJSURHRz/yzyGLyWRi+fLlj3TMR2ngwIHMnz+fo0ePPu6liIiIiMj/p0C3iIiIPHVsXYoZP2fcFdy72/ULZwEo/JyPxXUH91IAnN8f92gX94BsbGxwd3fHZDI9sjFv3rz5yMZatmwZKSkpxuvAgQNYWVnRunXrPzWulZUV7u7uWFv/vf+B4qM8m0fh2LFjNGvWjKCgIOLi4ggJCaFbt26sWbPmvvcULFiQjh07snbtWpKSkpg6dSqzZs0iNDTU6GM2m5k2bRrvvvuuxb1HjhzhxRdfxNPTk2LFij22z+Fxc3V1JTg4mBkzZjzupYiIiIjI/6dAt4iIiDx1rO3sjcB14uK59+2XkZ7OvlmfAuDdvI1FW4madQC4kLiPC4n7H9naAgMDef/99wkJCaFo0aKUKFGCWbNmcfXqVbp06YKTkxPe3t789NNPQPYs7ePHj9O8eXOKFi2Kg4MDlSpVYtWqVSQnJxMUFARA0aJFMZlMdO7c2ZizT58+hISEGAE8gMmTJ1O5cmUcHBzw8PCgV69epKamPtB+XFxccHd3N17r1q3D3t4+W6D7jz/+oF27djg4OFC6dGk+//zzHMdNTk7GZDIZpSSyziE6Oho/Pz/s7e3x9/cnKSnJuGfv3r0EBQXh5OSEs7MzNWvWZNeuXcTExNClSxeuXLliZJyHhYUBmWVVRo0aRceOHXF2dqZHjx73zIyPi4vDZDKRnJxsXNuyZQuBgYHY29tTtGhRgoODuXTpEp07dyY2NpaIiAhjvjvvy/Lll19SqlQpMjIyLK6//vrrdO3aFfhf1vWkSZPw9fWlT58+vPnmm0yZMuW+Z1e2bFm6dOlC1apV8fT05LXXXqN9+/Zs2rTJ6LN7926OHDlikXlvMpnYvXs3I0eONM7oYT4HgBUrVlCjRg0jozw8PJxbt24ZZw7QsmVLTCaT8b5z5860aNHCYpyQkBACAwON94GBgXzwwQcMHjzY+N3L+iyzXL58mW7dulG8eHGcnZ15+eWX2bt3r0WfcePGUaJECZycnHj33Xe5ceNGtnNs3rw5ixcvvu85i4iIiMjfS4FuEREReSpVeiez7vbJDWs4Hr0qW3valcts/qgvGemZGbyuz1ezaLcqaEPZZm8AsPOTUFL/83u2MS4k7Huotc2fPx9XV1d27NjB+++/z3vvvUfr1q3x9/fn119/pXHjxrzzzjtcu3Yt2729e/cmLS2NjRs3sn//fsaPH4+joyMeHh4sXboUyCxdkZKSQkREhMWcNjY2bNmyhZkzZwJQoEABPv30U+Lj45k/fz4///wzgwcPfqg9ZZkzZw5t27bFwcHB4vonn3xC1apV2bNnD0OGDKFv376sW7fugccfNmwYkyZNYteuXVhbWxsBYYD27dvzzDPPsHPnTnbv3s2QIUMoWLAg/v7+TJ06FWdnZyPzfODAgcZ9EydONNb20Ucf5WkdcXFxNGjQgIoVK7J161Y2b95M8+bNuX37NhEREdStW5fu3bsb83l4eGQbo3Xr1ly4cIENGzYY1y5evMjq1atp3749AFu3bqVhw4YW9wUHB7N169Y8n9nhw4dZvXo1AQEBxrVNmzbh4+OD0x2leVJSUqhUqRIDBgzIdkZ3y+lz2LRpEx07dqRv374kJCTwxRdfEBkZyZgxYwDYuXMnAPPmzSMlJcV4n1fz58/HwcGB7du3M2HCBEaOHGnxu9S6dWvOnj3LTz/9xO7du6lRowYNGjTg4sWLAHz77beEhYUxduxYdu3aRcmSJZk+fXq2eWrXrs3vv/9+zy8pREREROTv93T9N4YiIiIi/5/zs2V4vktvDsz7nMSFs0lcOJsyr7TCxsmZtMsXSV7zo9H3pXHTMRXInh/g3aItlw4lcum3BDYP/wCHkqUp7R+Eydqac/t2czFxP3wSmu2+3FStWpXhw4cDMHToUMaNG4erqyvdu3cHYMSIEcyYMYN9+7IH0k+cOMEbb7xB5cqVgczs3SwuLi4AuLm5ZatHXa5cOSZMmGBx7c4a0l5eXowePZqePXveM+iXFzt27ODAgQPMmTMnW1u9evUYMmQIAD4+PmzZsoUpU6bQqFGjB5pjzJgxRsB2yJAhNGvWjBs3bmBra8uJEycYNGgQFSpUADL3nKVw4cKYTCbc3d2zjfnyyy8zYMAA4/3JkydzXceECRPw8/OzOKtKlSoZP9vY2GBvb3/P+bIULVqUpk2bsmjRIho0aADA999/j6urq5Gdf/r0aUqUKGFxX4kSJfjvf//L9evXsbOzu+/4WV+cpKWl0aNHD0aOHGm0HT9+nFKlSln0zypR4ujoaKz7/Pnz9xw7p88hPDycIUOG0KlTJyDzd3TUqFEMHjyY0NBQihcvDkCRIkVyPJ/7qVKlilGGpVy5ckybNo3o6GgaNWrE5s2b2bFjB2fPnqVQoUJA5hcZy5cv5/vvv6dHjx5MnTqVd9991yjbMnr0aNavX58tqzvrfI4fP25knYuIiIjI46OMbhEREXlqPVO/ATU++NB4f2zVMpKWRBpB7sJlyhHwyRfYu9072FbAyopag8LxCMos9XE15RS/LV1A0pLIzCD3Q6pSpYrxs5WVFcWKFTMC14AR2Dx79my2ez/44ANGjx5NvXr1CA0NvWcw/F5q1qyZ7dr69etp0KABpUuXxsnJiXfeeYcLFy7cM5P8xIkTODo6Gq+xY8dm6zNnzhwqV65M7dq1s7XVrVs32/vExEQAevbsaTF2Tu48u5IlSwL/O6f+/fvTrVs3GjZsyLhx4zhy5EiOY2Xx8/PLU787ZWV0P4hKlSoZe2zatCmQmYW+dOlS0tLSAFi4cCFt27alwD2+eHlQS5Ys4ddff2XRokVERUUxceJEo+369evY2to+9Ng5fQ579+5l5MiRFp9pVnb7vX63/szcWfPfOXdqairFihWzmP/YsWPG70NiYiIvvPCCxRh3/34CxpcIj2LNIiIiIvLnKaNbREREnmpu1fwInrOUy0eSuHjwABk3b1LQqTDufnWwLVos1/sLWFlR6Z1/UaFtF87s3s61M/8BwL5EKUrUfCGXu++tYMGCFu9NJpPFtawHT95duxmgW7duBAcHExUVxdq1a/n444+ZNGkS77//fo5z3l1KJDk5mVdffZX33nuPMWPG4OLiwubNm3n33Xe5efMm9vb2Fv1LlSpl1GmG/2WPZ7l69SqLFy+2yBrOq5EjR+ZYJuNOOZ1TWFgYb7/9NlFRUfz000+EhoayePFiWrZsmeOYd59NVpDZbDYb19LveqhpTpnU97Nq1SpjnKz7mzdvjtlsJioqilq1arFp0yaL+tvu7u6cOXPGYpwzZ87g7Oyc6xqyyqVUrFiR27dv06NHDwYMGICVlRWurq7s3//wX9bk9DmkpqYSHh5Oq1atst2XU3C9QIECFmcO2c/97rmz5r9z7pIlSxITE5Ptvrv/K4fcZJU6ycpAFxEREZHHS4FuEREReeqZTCaKelegqHeFhx7DqqANperUf4SrengeHh707NmTnj17MnToUGbNmsX777+PjY0NALdv3851jN27d5ORkcGkSZOMwO6333573/7W1tZ4e3vft/27774jLS2NDh063LN927Zt2d77+voCmaVW3Nzccl1zXvj4+ODj40O/fv1o164d8+bNo2XLltjY2OTpXOB/gc2UlBSKFi0KYBHkh8ys4ujoaMLDw+85xr3m8/T0zNbP1taWVq1asXDhQg4fPkz58uWpUaOG0V63bl1WrbKsMb9u3bp7ZiDnJCMjg/T0dDIyMrCysqJ69erMmDEDs9lsBKoflRo1apCUlJTj70vBggWznU/x4sU5cOCAxbW4uLhsge3c5j59+jTW1tb3LTfi6+vL9u3b6dixo3Ht7t9PgAMHDlCwYEGLkjQiIiIi8viodImIiIjIP0hISAhr1qzh2LFj/Prrr2zYsMEIGHt6emIymVi5ciXnzp0jNTX1vuN4e3uTnp7OZ599xtGjR/n666+Nh1Q+jDlz5tCiRQuKFbt3lvyWLVuYMGECv/32G59//jnfffcdffv2fej57nb9+nX69OlDTEwMx48fZ8uWLezcudM4Gy8vL1JTU4mOjub8+fM5lqPw9vbGw8ODsLAwDh06RFRUFJMmTbLoM3ToUHbu3EmvXr3Yt28fBw8eZMaMGUZNay8vL7Zv305ycjLnz5+/Z3Z+lvbt2xMVFcXcuXONh1Bm6dmzJ0ePHmXw4MEcPHiQ6dOn8+2339KvXz+jz7Rp0yzKqCxcuJBvv/2WxMREjh49yrfffsvQoUN56623jKBxUFAQqampxMfH5/GE827EiBF89dVXhIeHEx8fT2JiIosXLzbq0kPm+URHR3P69GkuXboEZNZK37VrF1999RWHDh0iNDQ0W+A7Nw0bNqRu3bq0aNGCtWvXkpyczC+//MKwYcPYtWsXAH379mXu3LnMmzeP3377jdDQ0Huew6ZNm6hfv/5DZe+LiIiIyKOnjG4RERF5qvzUJecyFfnd7du36d27N7///jvOzs40adLEKHVRunRp40GAXbp0oWPHjkRGRt5znKpVqzJ58mTGjx/P0KFDeemll/j4448tslzzKikpic2bN7N27dr79hkwYAC7du0iPDwcZ2dnJk+eTHBw8APPdT9WVlZcuHCBjh07cubMGVxdXWnVqpWRce3v70/Pnj156623uHDhAqGhoYSFhd1zrIIFC/LNN9/w3nvvUaVKFWrVqsXo0aNp3bq10cfHx4e1a9fy4YcfUrt2bezs7HjhhRdo164dAAMHDqRTp05UrFiR69evc+zYsftmGL/88su4uLiQlJTE22+/bdFWpkwZoqKi6NevHxERETzzzDPMnj3b4uzOnz9vUY/c2tqa8ePH89tvv2E2m/H09KRPnz4WwfFixYrRsmVLFi5cyMcff/xAZ52b4OBgVq5cyciRIxk/fjwFCxakQoUKdOvWzegzadIk+vfvz6xZsyhdujTJyckEBwfz0UcfMXjwYG7cuEHXrl3p2LHjA5VYMZlMrFq1imHDhtGlSxfOnTuHu7s7L730klH7/q233uLIkSPGPG+88Qbvvfcea9assRhr8eLF9/0dEREREZG/n8l8d6E7ERERkXzuxo0bHDt2jDJlyvypB+qJPM327dtHo0aNOHLkSK4PAX3a/PTTTwwYMIB9+/Zhba3cocdFf+tFRETkTipdIiIiIiIi2VSpUoXx48dz7Nixx72UJ87Vq1eZN2+egtwiIiIiTxBldIuIiMg/jrL8RET++fS3XkRERO6kjG4RERERERERERERydcU6BYRERERERERERGRfE2BbhERERERERERERHJ1xToFhEREREREREREZF8TYFuEREREREREREREcnXFOgWERERERERERERkXxNgW4RERGRfMJkMrF8+fLHvQwRw52/k8nJyZhMJuLi4h7rmp4EkZGRFClS5HEvQ0REROSpYv24FyAiIiLyd3ojMvFvnW9pZ98H6t+5c2cuX76c7wLamzdv5t///jcHDx7k2rVreHp68q9//Yt+/frleF9qaipDhgxh+fLlXLhwgTJlyvDBBx/Qs2dPo4+XlxfHjx8HoECBApQoUYKmTZsyceJEihYtet+xIyMjCQkJ4fLly49kj3/3Z2Mymfjhhx9o0aJFnvpv2LCBSZMmsX37dv744w9Kly6Nn58fvXv35qWXXvprFwt4eHiQkpKCq6vrIx03r+cQHx/PiBEj2L17N8ePH2fKlCmEhIRk6/f555/zySefcPr0aapWrcpnn31G7dq1s/Uzm8288sorrF69Otv8O3fuZMiQIezevRuTyUTt2rWZMGECVatW/ZO7FREREZGHpYxuEREREfnTHBwc6NOnDxs3biQxMZHhw4czfPhwvvzyyxzv69+/P6tXr2bBggUkJiYSEhJCnz59+PHHHy36jRw5kpSUFE6cOMHChQvZuHEjH3zwwV+5pYeWnp7+t885ffp0GjRoQLFixViyZAlJSUn88MMP+Pv75/hlw+3bt8nIyHgka7CyssLd3R1r68eTS3Pt2jXKli3LuHHjcHd3v2efJUuW0L9/f0JDQ/n111+pWrUqwcHBnD17NlvfqVOnYjKZsl1PTU2lSZMmPPvss2zfvp3Nmzfj5OREcHDwY/nsRURERCSTAt0iIiIi+dS///1vfHx8sLe3p2zZsnz00UcWgbawsDCqVavG3LlzefbZZ3F0dKRXr17cvn2bCRMm4O7ujpubG2PGjLEYd/LkyVSuXBkHBwc8PDzo1asXqampOa6levXqtGvXjkqVKuHl5UWHDh0IDg5m06ZNOd73yy+/0KlTJwIDA/Hy8qJHjx5UrVqVHTt2WPRzcnLC3d2d0qVLExQURKdOnfj1118f6LyyzuPrr7/Gy8uLwoUL07ZtW/744w+jz/fff0/lypWxs7OjWLFiNGzYkKtXrxIWFsb8+fNZsWIFJpMJk8lETEyMUa5jyZIlBAQEYGtry8KFC4257jR16lS8vLwsrs2dO5dKlSpRqFAhSpYsSZ8+fQCMfi1btsRkMmW7704nTpwgJCSEkJAQ5s+fz8svv4ynpydVqlShb9++7Nq1y+ibVVLjxx9/pGLFihQqVIgTJ06wc+dOGjVqhKurK4ULFyYgICDb+R46dIiXXnoJW1tbKlasyLp16yza71W65MCBAzRt2hRHR0dKlCjBO++8w/nz5432wMBAPvjgAwYPHoyLiwvu7u6EhYUZ7Q9yDrVq1eKTTz6hbdu2FCpU6J59Jk+eTPfu3enSpQsVK1Zk5syZ2NvbM3fuXIt+cXFxTJo0Kdt1gIMHD3Lx4kVGjhxJ+fLlqVSpEqGhoZw5c8b4Lw/udu7cOfz8/GjZsiVpaWn33YOIiIiIPDwFukVERETyKScnJyIjI0lISCAiIoJZs2YxZcoUiz5Hjhzhp59+YvXq1XzzzTfMmTOHZs2a8fvvvxMbG8v48eMZPnw427dvN+4pUKAAn376KfHx8cyfP5+ff/6ZwYMHP9Da9uzZwy+//EJAQECO/fz9/fnxxx85deoUZrOZDRs28Ntvv9G4ceP73nPq1Cn+7//+jxdeeOGB1gSZ57F8+XJWrlzJypUriY2NZdy4cQCkpKTQrl07unbtSmJiIjExMbRq1Qqz2czAgQNp06YNTZo0ISUlhZSUFPz9/Y1xhwwZQt++fUlMTCQ4ODhPa5kxYwa9e/emR48e7N+/nx9//BFvb28gszQGwLx580hJSTHe38vSpUtJT0+/72d0d1bytWvXGD9+PLNnzyY+Ph43Nzf++OMPOnXqxObNm9m2bRvlypXjlVdeMb4EyMjIoFWrVtjY2LB9+3ZmzpzJv//97xz3d/nyZV5++WWqV6/Orl27WL16NWfOnKFNmzYW/ebPn4+DgwPbt29nwoQJjBw50giiP8g55ObmzZvs3r2bhg0bGtcKFChAw4YN2bp1q8X5vP3223z++ef3zAwvX748xYoVY86cOdy8eZPr168zZ84cfH197xmIP3nyJPXr1+f555/n+++/v28QXkRERET+HNXoFhEREcmnhg8fbvzs5eXFwIEDWbx4sUXAMyMjg7lz5+Lk5ETFihUJCgoiKSmJVatWUaBAAcqXL8/48ePZsGGDETi+s66xl5cXo0ePpmfPnkyfPj3XNT3zzDOcO3eOW7duERYWRrdu3XLs/9lnn9GjRw+eeeYZrK2tKVCgALNmzcpWU/rf//43w4cP5/bt29y4cYMXXniByZMn5+WYLGRkZBAZGYmTkxMA77zzDtHR0YwZM4aUlBRu3bpFq1at8PT0BKBy5crGvXZ2dqSlpd0z+BkSEkKrVq0eaC2jR49mwIAB9O3b17hWq1YtAIoXLw5AkSJF7luGI8tvv/2Gs7OzRb+lS5fSqVMn4/3WrVuNvaSnpzN9+nSLetIvv/yyxZhffvklRYoUITY2lldffZX169dz8OBB1qxZQ6lSpQAYO3YsTZs2ve+6pk2bRvXq1Rk7dqxxbe7cuXh4ePDbb7/h4+MDQJUqVQgNDQWgXLlyTJs2jejoaBo1avRA55Cb8+fPc/v2bUqUKGFxvUSJEhw8eNB4369fP/z9/Xn99dfvOY6TkxMxMTG0aNGCUaNGGetes2ZNtrItSUlJNGrUiJYtW963FIqIiIiIPBrK6BYRERHJp5YsWUK9evVwd3fH0dGR4cOHc+LECYs+Xl5eRlAXMoN6FStWpECBAhbX7qxRvH79eho0aEDp0qVxcnLinXfe4cKFC1y7dg0AR0dH43XnQyMBNm3axK5du5g5cyZTp07lm2++AWDhwoUW92WVNPnss8/Ytm0bP/74I7t372bSpEn07t2b9evXW4w7aNAg4uLi2LdvH9HR0QA0a9aM27dv57qmnM6jZMmSxt6rVq1KgwYNqFy5Mq1bt2bWrFlcunQpp4/A4Ofnl6d+Wc6ePct//vMfGjRokOd7Tpw4YbHPOwPIdwdQg4ODiYuLIyoqiqtXrxrnBGBjY0OVKlUs+p85c4bu3btTrlw5ChcujLOzM6mpqcbvU2JiIh4eHkaQG6Bu3bo5rnfv3r1s2LDBYs0VKlQAMjPrs9y9ljs/kwc9hz/rxx9/5Oeff2bq1Kn37XP9+nXeffdd6tWrx7Zt29iyZQvPP/88zZo14/r16xb96tevT6tWrYiIiFCQW0REROQvpoxuERERkXxo69attG/fnvDwcIKDgylcuDCLFy9m0qRJFv0KFixo8d5kMt3zWtYDCZOTk3n11Vd57733GDNmDC4uLmzevJl3332XmzdvYm9vb1GD2dnZ2WKsMmXKAJmZ0GfOnCEsLIx27drx2muvWZQaKV26NNevX+fDDz/khx9+oFmzZkBm0DMuLo6JEydalJhwdXU1ynqUK1eOqVOnUrduXTZs2EDDhg1zXFNu55G1dysrK9atW8cvv/zC2rVr+eyzzxg2bBjbt2839nU/Dg4OFu8LFCiA2Wy2uHZn/XQ7O7scx7uXUqVKWezTxcUFyDyPK1eucPr0aSPr2dHREW9v73s+GNLOzi5b0LVTp05cuHCBiIgIPD09KVSoEHXr1uXmzZsPvM4sqampNG/enPHjx2drK1mypPFzTp/JvdzvHHLj6uqKlZUVZ86csbh+5swZ49x+/vlnjhw5QpEiRSz6vPHGG9SvX5+YmBgWLVpEcnIyW7duNb4wWrRoEUWLFmXFihW0bdsWgEKFCtGwYUNWrlzJoEGDKF26dJ7WKSIiIiIPR4FuERERkXzol19+wdPTk2HDhhnX7vcgvAexe/duMjIymDRpkhHE+/bbby36ZAWcc5ORkWE8eM/Jyckikxrgv//9L+np6RbZ5ZAZcM4p0JnVBzAyaPO6ptyYTCbq1atHvXr1GDFiBJ6envzwww/0798fGxsbi8zonBQvXpzTp09jNpuNoPKdwVknJye8vLyIjo4mKCjonmMULFjQYj5ra+t77vPNN99kyJAhjB8/PluN9rzasmUL06dP55VXXgEy60rf+dBIX19fTp48SUpKihGk3rZtW45j1qhRg6VLl+Ll5XXPgHte5fUccmNjY0PNmjWJjo6mRYsWQObvaHR0tPEQ0CFDhmQrt1O5cmWmTJlC8+bNgcwa3gUKFLD4siDr/Z2/twUKFODrr7/m7bffJigoiJiYGIuMeBERERF5tBToFhEREXnCXLlyxSIoClCsWDGL9+XKlePEiRMsXryYWrVqERUVxQ8//PCn5/b29iY9PZ3PPvuM5s2bs2XLFmbOnJnrfZ9//jnPPvusUZpi48aNTJw4kQ8++OC+9zg7OxMQEMCgQYOws7PD09OT2NhYvvrqq2z1t//44w8jcHzy5EkGDx5M8eLFLR4I+Wdt376d6OhoGjdujJubG9u3b+fcuXP4+voCmWVP1qxZQ1JSEsWKFaNw4cL3HSswMJBz584xYcIE3nzzTVavXs1PP/1kkW0eFhZGz549cXNzo2nTpvzxxx9s2bKF/9fevcf1eP//A3+8K510kkqFepeSFJXFxKilNMkcZqZhOc8UcmY6j5QJOcQQhcR3FmtmOaRyzCmxlCSSUc5MSqX6/dGv69NbZ7NZ87jfbu/brfd1va7X63W9rku3m+f17HlNmzZNGC8+Ph69e/eGnJwcWrVqVetYenp6CAkJwYwZM/D48WOMHTsWBgYGePz4MXbs2AHgfw8G6mJsbIzt27fD2toaf/75p3BNqjg4OKBjx45wc3PD999/jz///FPiIUtt3N3dsWnTJri6umLevHlQV1fH9evXsWvXLmzevLnBOVVp7DqUlJQgPT1d+PnOnTtITU0VstsBYNasWXBzc4O1tTV69OiBVatW4cWLFxg3bhwAQFtbu9Za4Hp6ekJWv6OjI+bOnQt3d3dMmzYN5eXlCAoKgoyMTI2HFtLS0oiKioKrqyvs7e2RmJj4l2uNExEREVHtWKObiIiI6F8mMTERVlZWEh9/f3+JNp9++ilmzpwJDw8PWFpa4tSpU/D29v7LY1tYWGDFihUIDg6Gubk5oqKisHTp0gaPKy8vx8KFC2FpaQlra2usW7cOwcHBCAgIqPe4qkD9qFGj0LlzZwQFBWHJkiU16mz7+PhAR0cHurq6cHFxQcuWLXHo0KEaDwD+ChUVFRw7dgzOzs7o2LEjvLy8EBISIrxwcdKkSTAxMYG1tTU0NTVx8uTJOvsyNTVFWFgY1q1bBwsLC5w9exZz5syRaOPm5oZVq1YhLCwMZmZmcHFxQVZWlrA/JCQEhw8fRvv27WFlZVXv3KdNm4ZDhw7hwYMHGD58OIyNjeHs7IybN28iLi5O4qWatQkPD8eTJ0/QrVs3jBkzBtOnT4eWlpawX0pKCnv37kVRURF69OiBiRMnYsmSJfX2qauri5MnT6KsrAz9+/dHly5d4OnpCTU1tRpZ/PVp7DrcvXtX+PeSl5eH5cuXw8rKSiJD+4svvsDy5cvh4+MDS0tLpKamIi4ursYLKuvTqVMn/PLLL7h8+TJsbGzQp08f3L17F3FxcRIlWarIyMggOjoaZmZmsLe3r7f+OBERERG9OVHF68UDiYiIiJq5ly9f4ubNmzAwMIC8vPy7ng4REf0N+LueiIiIqmNGNxERERERERERERE1awx0ExEREREREREREVGzxkA3ERERERERERERETVrDHQTERERERERERERUbPGQDcRERERERERERERNWsMdBMRERERERERERFRs8ZANxERERERERERERE1awx0ExEREREREREREVGzxkA3ERERERERERERETVrDHQTERERNRMikQj79u1719MgElS/J3NyciASiZCamvpO5/RvEBERATU1tQbbeXt7Y/LkycL3iooKTJ48Gerq6sJa2tnZwdPT8++b7L/A6/dOeno62rVrhxcvXrzbiREREVGzIvOuJ0BERET0T9qzMf8fHW/4ZO0mtR87diyePn3a7ALaJ06cwPz583H16lUUFhZCX18fX3/9NWbOnFnvcQUFBViwYAH27duHR48ewcDAANOnT8eUKVOENmKxGLdu3QIASElJoU2bNhgwYACWL1+OVq1a1dl3REQEPD098fTp07dyjv/0tRGJRNi7dy+GDBnSqPYJCQkICQnBmTNn8Pz5c7Rt2xbW1tZwd3dH3759/97JAmjfvj3y8vKgoaHxVvtt7DpcuXIFPj4+uHDhAm7duoWVK1fWGiBet24dvv/+e+Tn58PCwgJr1qxBjx49arSrqKiAs7Mz4uLiaox/7tw5LFiwABcuXIBIJEKPHj2wbNkyWFhYNPq88vPzERoait9//13YFhcXh4iICCQmJsLQ0BAaGhqIiYlBixYtGt1vY/zbf8907twZPXv2xIoVK+Dt7f2up0NERETNBDO6iYiIiOgva9myJTw8PHDs2DFkZGTAy8sLXl5e2LhxY73HzZo1C3FxcdixYwcyMjLg6ekJDw8PxMbGSrQLCAhAXl4ecnNzERUVhWPHjmH69Ol/5ym9sdLS0n98zLCwMPTr1w+tW7fG7t27kZmZib1796JXr171PmwoKytDeXn5W5mDtLQ0tLW1ISPzbnJpCgsLYWhoiKCgIGhr1/6Aaffu3Zg1axZ8fX2RkpICCwsLODk54f79+zXarlq1CiKRqMb2goICfPLJJ9DT08OZM2dw4sQJKCsrw8nJqUnXfvPmzejVqxf09fWFbdnZ2dDR0UGvXr2EtVRXV4eysnKj+/2vGDduHNavX49Xr16966kQERFRM8FANxEREb1XLlw+/MbHVlRUYOe+pfiz4NFbnNGbmz9/Pjp27AhFRUUYGhrC29tbItDm5+cHS0tLbNmyBXp6elBSUsLUqVNRVlaGZcuWQVtbG1paWliyZIlEvytWrECXLl3QsmVLtG/fHlOnTkVBQUG9c7GysoKrqyvMzMwgFosxevRoODk54fjx4/Ued+rUKbi5ucHOzg5isRiTJ0+GhYUFzp49K9FOWVkZ2traaNu2LT7++GO4ubkhJSWlSetVtR7bt2+HWCyGqqoqRo4ciefPnwtt9uzZgy5dukBBQQGtW7eGg4MDXrx4AT8/P0RGRuLnn3+GSCSCSCRCYmKiUHJh9+7dsLW1hby8PKKiooSxqlu1ahXEYrHEti1btsDMzAxycnLQ0dGBh4cHAAjthg4dCpFIVOO46nJzc+Hp6QlPT09ERkbC3t4e+vr66Nq1K2bMmIHz588LbatKasTGxqJz586Qk5NDbm4uzp07B0dHR2hoaEBVVRW2trY11jcrKwt9+/aFvLw8OnfujMOHJf8t1Va6JC0tDQMGDICSkhLatGmDMWPG4OHDh8J+Ozs7TJ8+HfPmzYO6ujq0tbXh5+cn7G/KOnTv3h3ff/89Ro4cCTk5uVrbrFixApMmTcK4cePQuXNnbNiwAYqKitiyZYtEu9TUVISEhNTYDgBXr17F48ePERAQABMTE5iZmcHX1xf37t0T/vLgdQ8ePIC1tTWGDh2K4uJiAMCuXbswaNAgoc3YsWMxbdo05ObmSpzr66VLxGIxAgMDMX78eCgrK0NPT6/GA6Xbt29jxIgRUFNTg7q6OgYPHoycnBwAqPNeTkxMhEgkkvgLiNTUVIhEIuHYqvvn4MGDMDU1hZKSEj755BPk5eVJjL9582aYmppCXl4enTp1QlhYmMT+s2fPwsrKCvLy8rC2tsbFixdrrJmjoyMeP36MpKSkWteUiIiI6HUMdBMREdF7JWjdV6ioqHijY69eP4O9v63Gt0Eub3lWb0ZZWRkRERFIT09HaGgoNm3ahJUrV0q0yc7Oxm+//Ya4uDhER0cjPDwcAwcOxB9//IGkpCQEBwfDy8sLZ86cEY6RkpLC6tWrceXKFURGRuLo0aOYN29ek+Z28eJFnDp1Cra2tvW269WrF2JjY3Hnzh1UVFQgISEB165dQ//+/es85s6dO/jll1/w4YcfNmlOQOV67Nu3D/v378f+/fuRlJSEoKAgAEBeXh5cXV0xfvx4ZGRkIDExEcOGDUNFRQXmzJmDESNGCEG9vLw89OrVS+h3wYIFmDFjBjIyMuDk5NSouaxfvx7u7u6YPHkyfv/9d8TGxsLIyAhAZWkMANi6dSvy8vKE77X56aefUFpaWuc1ej0rubCwEMHBwdi8eTOuXLkCLS0tPH/+HG5ubjhx4gSSk5NhbGwMZ2dn4SFAeXk5hg0bBllZWZw5cwYbNmzA/Pnz6z2/p0+fwt7eHlZWVjh//jzi4uJw7949jBgxQqJdZGQkWrZsiTNnzmDZsmUICAgQguhNWYeGlJSU4MKFC3BwcBC2SUlJwcHBAadPn5ZYny+//BLr1q2rNTPcxMQErVu3Rnh4OEpKSlBUVITw8HCYmprWGoi/ffs2+vTpA3Nzc+zZswdycnJ4/Pgx0tPTYW1tLbQLDQ1FQEAA2rVr1+C5hoSECAHiqVOn4ptvvkFmZiaAyr8ocHJygrKyMo4fP46TJ08KAemSkpIG7+WGFBYWYvny5di+fTuOHTuG3NxczJkzR9gfFRUFHx8fLFmyBBkZGQgMDIS3tzciIyMBVGbEu7i4oHPnzrhw4QL8/Pwkjq8iKysLS0vLBh+WEREREVVhjW4iIiJ676SkxeODLg4NN3xN/ImdAICnf95H0csXUJBv+ban1iReXl7Cz2KxGHPmzMGuXbskAp7l5eXYsmULlJWV0blzZ3z88cfIzMzEgQMHICUlBRMTEwQHByMhIUEIHL+ePbp48WJMmTKlRlZmbdq1a4cHDx7g1atX8PPzw8SJE+ttv2bNGkyePBnt2rWDjIwMpKSksGnTpho1pefPnw8vLy+UlZXh5cuX+PDDD7FixYrGLJOE8vJyRERECKUgxowZg/j4eCxZsgR5eXl49eoVhg0bJpST6NKli3CsgoICiouLaw1+enp6YtiwYU2ay+LFizF79mzMmDFD2Na9e3cAgKamJgBATU2tzjIcVa5duwYVFRWJdj/99BPc3NyE76dPnxbOpbS0FGFhYRL1pO3t7SX63LhxI9TU1JCUlAQXFxccOXIEV69excGDB6GrqwsACAwMxIABA+qc19q1a2FlZYXAwEBh25YtW9C+fXtcu3YNHTt2BAB07doVvr6+AABjY2OsXbsW8fHxcHR0bNI6NOThw4coKytDmzZtJLa3adMGV69eFb7PnDkTvXr1wuDBg2vtR1lZGYmJiRgyZAi+++47Yd4HDx6sUbYlMzMTjo6OGDp0qEQplNzcXFRUVAhrCQCqqqpQVlYWSsDUx9nZGVOnTgVQ+W9j5cqVSEhIgImJCXbv3o3y8nJs3rxZGG/r1q1QU1NDYmIi+vfvX++93JDS0lJs2LABHTp0AAB4eHggICBA2O/r64uQkBDh34OBgQHS09Pxww8/wM3NDTt37kR5eTnCw8MhLy8PMzMz/PHHH/jmm29qjKWrq1tnljwRERHR65jRTURERO8VKSlp7I5d1uSs7rz7N3HszE8AgJKSIhxKivw7ptcku3fvRu/evaGtrQ0lJSV4eXkhNzdXoo1YLJao79umTRt07twZUlJSEtuq1yg+cuQI+vXrh7Zt20JZWRljxozBo0ePUFhYCABQUlISPtVfGgkAx48fx/nz57FhwwasWrUK0dHRACqzPKsfV5WluWbNGiQnJyM2NhYXLlxASEgI3N3dceTIEYl+586di9TUVFy+fBnx8fEAgIEDB6KsrKzBOdW3Hjo6OsK5W1hYoF+/fujSpQs+//xzbNq0CU+ePKnvEgiqZ+Y2xv3793H37l3069ev0cfk5uZKnGf1APLrWdtOTk5ITU3Fr7/+ihcvXgjrBFRmynbt2lWi/b179zBp0iQYGxtDVVUVKioqKCgoEO6njIwMtG/fXiIwa2NjU+98L126hISEBIk5d+rUCUBlZn2V1+dS/Zo0dR3+qtjYWBw9ehSrVq2qs01RUREmTJiA3r17Izk5GSdPnoS5uTkGDhyIoqIiiXZ9+vTBsGHDEBoaKnGNqtrJy8u/0Tyrr5lIJIK2trawZpcuXcL169ehrKwsrJG6ujpevnwpse5vSlFRUQhyA5LX68WLF8jOzsaECRMkrtHixYuFsTMyMtC1a1eJc6/rXlJQUBB+7xARERE1hBndRERE9F4pLy/DzdzfMcPnI7TT7QgdLUPotjFEB31LiNub/f825bh45Sjy7mUj7/5N3M3Pxq0/rkBKJIWyinJUVFQg+ucgnL4Qi7Y6xtDRMoCOliHMTT6CqooGAODZnw+RlnkCwyfXHXT9K06fPo1Ro0bB398fTk5OUFVVxa5duxASEiLRrkWLFhLfRSJRrduqXkiYk5MDFxcXfPPNN1iyZAnU1dVx4sQJTJgwASUlJVBUVJSowayioiLRl4GBAYDKTOh79+7Bz88Prq6u+PTTTyVKjbRt2xZFRUX49ttvsXfvXgwcOBBAZQAvNTUVy5cvlygxoaGhIZT1MDY2xqpVq2BjY4OEhAQ4ODjUO6eG1qPq3KWlpXH48GGcOnUKhw4dwpo1a7Bo0SKcOXNGOK+6tGwpmd0vJSVV42FK9frpCgoK9fZXG11dXYnzVFdXB1C5Hs+ePUN+fr6QoaukpAQjI6NaXwypoKBQIzDu5uaGR48eITQ0FPr6+pCTk4ONjQ1KSkqaPM8qBQUFGDRoEIKDg2vs09HREX6u75rUpq51aIiGhgakpaVx7949ie337t0T1u3o0aPIzs6GmpqaRJvPPvsMffr0QWJiInbu3ImcnBycPn1aeGC0c+dOtGrVCj///DNGjhwJAJCTk4ODgwP279+PuXPnom3bthJzAYAnT54IWetNUd+aFRQU4IMPPkBUVFSN4+obq+pcqt+3tb1cs7axq46pquW/adOmGqWFpKWl6xy7Lo8fP5YIqhMRERHVh4FuIiIiei/l3b+B/Ac3UVFRATUVLfjN3iPsk5KSwuMneYj80a/O48vKSpF96xJu5F5GRUUFPujaHz0s/1fGQVFBGcfP7gXw9wS6T506BX19fSxatEjY9jb+xP/ChQsoLy9HSEiIEPj6v//7P4k2VQHnhpSXlwsv3lNWVpbIpAaAP//8E6WlpRLZ5UBlQKy+QGdVG+B/mbGNnVNDRCIRevfujd69e8PHxwf6+vrYu3cvZs2aBVlZWYnM6PpoamoiPz8fFRUVQlC5enBWWVkZYrEY8fHx+Pjjj2vto0WLFhLjycjI1Hqew4cPx4IFCxAcHFyjRntjnTx5EmFhYXB2dgZQWVe6+ksjTU1Ncfv2beTl5QlB6uTk5Hr77NatG3766SeIxeJaA+6N1dh1aIisrCw++OADxMfHY8iQIQAq79H4+HjhJaALFiyoUW6nS5cuWLlypfDiyMLCQkhJSUk8LKj6Xv2+lZKSwvbt2/Hll1/i448/RmJiopAR36FDB6ioqCA9PV0o4fK2dOvWDbt374aWlladD31qu5erguB5eXlo1aoVAMl7tjHatGkDXV1d3LhxA6NGjaq1jampKbZv346XL18KWd113UtpaWkYPnx4k+ZARERE7y+WLiEiIqL3lpRIGmoqWgiYsxdttY0l9jn2HYOpX62ESCSqkf36uh6WAzD7601o0UJO2NaihRxmf73pjeb17NkzpKamSnxu374t0cbY2Bi5ubnYtWsXsrOzsXr1auzdu/eNxqvOyMgIpaWlWLNmDW7cuIHt27djw4YNDR63bt06/PLLL8jKykJWVhbCw8OxfPlyjB49us5jVFRUYGtri7lz5yIxMRE3b95EREQEtm3bhqFDh0q0ff78OfLz85GXl4ezZ89i7ty50NTUbNJL9Bpy5swZBAYG4vz588jNzUVMTAwePHgAU1NTAJVlTy5fvozMzEw8fPiw1mzXKnZ2dnjw4AGWLVuG7OxsrFu3Dr/99ptEGz8/P4SEhGD16tXIyspCSkoK1qxZI+yvCoTn5+fXW0JFT08PISEhCA0NhZubGxISEpCTk4OUlBSsXr0aQMPZtMbGxti+fTsyMjJw5swZjBo1SiLr3MHBAR07doSbmxsuXbqE48ePSzxkqY27uzseP34MV1dXnDt3DtnZ2Th48CDGjRvX6AcGTVmHkpIS4d9LSUkJ7ty5g9TUVFy/fl1oM2vWLGzatAmRkZHIyMjAN998gxcvXmDcuHEAAG1tbZibm0t8gMo1rsrqd3R0xJMnT+Du7o6MjAxcuXIF48aNg4yMTI2HFtLS0oiKioKFhQXs7e2Rn58P4H8vwTxx4kSj16GxRo0aBQ0NDQwePBjHjx/HzZs3kZiYiOnTp+OPP/4AUPu9bGRkhPbt28PPzw9ZWVn49ddfa/yFSGP4+/tj6dKlWL16Na5du4bff/8dW7duFWrqf/nllxCJRJg0aRLS09Nx4MABLF++vEY/OTk5uHPnjsRfdhARERHVh4FuIiIiei9JSUlDVUUTi+f/Ap02hrW2+bj3SEwbt6bWfUBl9q/NB59i5uQf0EJGtsb+2rY1RmJiIqysrCQ+/v7+Em0+/fRTzJw5Ex4eHrC0tMSpU6fg7e39RuNVZ2FhgRUrViA4OBjm5uaIiorC0qVLGzyuvLwcCxcuhKWlJaytrbFu3ToEBwdLvKSuNrt27UL37t0xatQodO7cGUFBQViyZEmNOts+Pj7Q0dGBrq4uXFxc0LJlSxw6dAitW7f+S+dbnYqKCo4dOwZnZ2d07NgRXl5eCAkJEV64OGnSJJiYmMDa2hqampo4efJknX2ZmpoiLCwM69atg4WFBc6ePYs5c+ZItHFzc8OqVasQFhYGMzMzuLi4ICsrS9gfEhKCw4cPo3379rCysqp37tOmTcOhQ4fw4MEDDB8+HMbGxnB2dsbNmzcRFxcn8VLN2oSHh+PJkyfo1q0bxowZg+nTp0NLS0vYLyUlhb1796KoqAg9evTAxIkTsWTJknr71NXVxcmTJ1FWVob+/fujS5cu8PT0hJqaWo0s/vo0dh3u3r0r/HvJy8vD8uXLYWVlJZGh/cUXX2D58uXw8fGBpaUlUlNTERcXV+MFlfXp1KkTfvnlF1y+fBk2Njbo06cP7t69i7i4OImSLFVkZGQQHR0NMzMz2NvbC/WsJ06ciF27djX41wtNpaioiGPHjkFPTw/Dhg2DqakpJkyYgJcvXwoZ3rXdyy1atEB0dDSuXr2Krl27Ijg4GIsXL27y+BMnTsTmzZuxdetWdOnSBba2toiIiBAeFCgpKeGXX37B77//DisrKyxatKjW8jbR0dHo37+/8GJYIiIiooaIKpr6JiYiIiKif7mXL1/i5s2bMDAweOOXvRER/Z0qKirw4YcfYubMmXB1dX3X0/lXKSkpgbGxMXbu3InevXvX2Y6/64mIiKg6ZnQTERERERH9w0QiETZu3IhXr16966n86+Tm5uLbb7+tN8hNRERE9DpmdBMREdF/DrP8iIj++/i7noiIiKpjRjcRERERERERERERNWsMdBMRERERERERERFRs8ZANxERERERERERERE1awx0ExEREREREREREVGzxkA3ERERERERERERETVrDHQTERERERERERERUbPGQDcRERERERERERERNWsMdBMRERER/QfY2dnB09PzL/WRk5MDkUiE1NTUtzKn6saOHYshQ4bU2+ZtnAMRERERvZ9k3vUEiIiIiP5J1xbebnTbsJOrEXp8OWb0mYOpvac3eayC4gJ0W2Ha5OMA4Pbt2/D19UVcXBwePnwIHR0dDBkyBD4+PmjduvUb9flXXbp0CUFBQThx4gQePnwIsViMKVOmYMaMGfUeZ2dnh6SkpBrbnZ2d8euvv6K0tBReXl44cOAAbty4AVVVVTg4OCAoKAi6urpCe5FIJPwsLS0NXV1dDB8+HEuXLoWcnFyd4/v5+WHfvn1vLXhrZ2cHS0tLrFq16q30V5+cnBwYGBjg4sWLsLS0/NvHexuuX78OKysrSEtL4+nTp006NiYmBi1atBC+i8VieHp6MvhNRERERA1iRjcRERFRLd5GkHvi/415o7Fv3LgBa2trZGVlITo6GtevX8eGDRsQHx8PGxsbPH78+I36/asuXLgALS0t7NixA1euXMGiRYuwcOFCrF27tt7jYmJikJeXJ3zS0tIgLS2Nzz//HABQWFiIlJQUeHt7IyUlBTExMcjMzMSnn35ao6+tW7ciLy8PN2/eRFhYGLZv347Fixf/Lef7V1RUVODVq1fvehr/uNLSUri6uqJPnz5vdLy6ujqUlZXf8qyIiIiI6H3AQDcRERHRa95WkDvr4bU3Gt/d3R2ysrI4dOgQbG1toaenhwEDBuDIkSO4c+cOFi1ahLVr18Lc3Fw4Zt++fRCJRNiwYYOwzcHBAV5eXsL3n3/+Gd26dYO8vDwMDQ3h7+8vEYwViUTYvHkzhg4dCkVFRRgbGyM2NlbYP378eISGhsLW1haGhoYYPXo0xo0bh5iYmHrPR11dHdra2sLn8OHDUFRUFALdqqqqOHz4MEaMGAETExP07NkTa9euxYULF5CbmyvRl5qaGrS1tdG+fXu4uLhg8ODBSElJadL6VpXQWL58OXR0dNC6dWu4u7ujtLRUaBMWFgZjY2PIy8ujTZs2GD58uHBsUlISQkNDIRKJIBKJkJOTg8TERIhEIvz222/44IMPICcnhxMnTtRarsPT0xN2dnbC9/LycixbtgxGRkaQk5ODnp4elixZAgAwMDAAAFhZWUEkEkkcV5tXr17Bw8MDqqqq0NDQgLe3NyoqKoT9YrEYgYGBGD9+PJSVlaGnp4eNGzfW6Ofq1avo1asX5OXlYW5uXmtGfm28vLzQqVMnjBgxos42/v7+0NTUhIqKCqZMmYKSkhJhX/XSJXZ2drh16xZmzpwprDURERERUV0Y6CYiIiKq5m0Gubd+EdXk4x8/foyDBw9i6tSpUFBQkNinra2NUaNGYffu3bC1tUV6ejoePHgAAEhKSoKGhgYSExMBVGbWnj59WgiMHj9+HF999RVmzJiB9PR0/PDDD4iIiBACqlX8/f0xYsQIXL58Gc7Ozhg1alS9GeTPnj2Durp6k84xPDwcI0eORMuWLevtVyQSQU1Nrc42165dw9GjR/Hhhx82aXwASEhIQHZ2NhISEhAZGYmIiAhEREQAAM6fP4/p06cjICAAmZmZiIuLQ9++fQEAoaGhsLGxwaRJk4QM9fbt2wv9LliwAEFBQcjIyEDXrl0bNZeFCxciKCgI3t7eSE9Px86dO9GmTRsAwNmzZwEAR44cQV5eXoMPFSIjIyEjI4OzZ88iNDQUK1aswObNmyXahISEwNraGhcvXsTUqVPxzTffIDMzU6LN3LlzMXv2bFy8eBE2NjYYNGgQHj16VO/YR48exY8//oh169bV2SY+Ph4ZGRlITExEdHQ0YmJi4O/vX2vbmJgYtGvXDgEBAcJaExERERHVhYFuIiIiov/vbQe5u+paNrmPrKwsVFRUwNS09trepqamePLkCbS0tKCuri5k2iYmJmL27NnC97Nnz6K0tBS9evUCUBnAXrBgAdzc3GBoaAhHR0d89913+OGHHyT6Hzt2LFxdXWFkZITAwEAUFBQIwdbXnTp1Crt378bkyZMbfX5nz55FWloaJk6cWGebly9fYv78+XB1dYWKiorEPldXVygpKUFeXh4mJiYwMzPDwoULGz1+lVatWmHt2rXo1KkTXFxcMHDgQMTHxwMAcnNz0bJlS7i4uEBfXx9WVlaYPr3yflBVVYWsrCwUFRWFDHVpaWmh34CAADg6OqJDhw6NegDw/PlzhIaGYtmyZXBzc0OHDh3w0UcfCeujqakJAGjdujW0tbUb7LN9+/ZYuXIlTExMMGrUKEybNg0rV66UaOPs7IypU6fCyMgI8+fPh4aGBhISEiTaeHh44LPPPoOpqSnWr18PVVVVhIeH1znuo0ePMHbsWERERNS4ZtXJyspiy5YtMDMzw8CBAxEQEIDVq1ejvLy8Rlt1dXVIS0tDWVlZWGsiIiIiorow0E1ERESEf0eQu7rq5SZqIycnh759+yIxMRFPnz5Feno6pk6diuLiYly9ehVJSUno3r07FBUVAVS+SDIgIABKSkrCpyorubCwUOi3ehZyy5YtoaKigvv379cYPy0tDYMHD4avry/69+8PoDJAXL3/wMDAGseFh4ejS5cu6NGjR63nVVpaihEjRqCiogLr16+vsX/lypVITU3FpUuXsH//fly7dg1jxoxp9PhVzMzMJALUOjo6wnk6OjpCX18fhoaGGDNmDKKioiTWqD7W1taNalclIyMDxcXF6NevX6OPOX78uMR5RkX97y8HevbsKVHiw8bGBllZWSgrKxO2Vb/GIpEI2traNa6xjY2N8LOMjAysra2RkZEBoHLtqsYeMGAAAGDSpEn48ssvhcz3ulhYWAj3ZNU4BQUFuH278S+JJSIiIiKqjcy7ngARERHRu/ZvCnIbGRlBJBIhIyMDQ4cOrbE/IyMDmpqaUFNTg52dHTZu3Ijjx4/DysoKKioqQvA7KSkJtra2/5tjQQH8/f0xbNiwGn3Ky8sLP7do0UJin0gkqpFtm56ejn79+mHy5MkSNcB1dXWRmpoqfH89+/jFixfYtWsXAgICaj33qiD3rVu3cPTo0Vozg7W1tWFkZAQAMDExwfPnz+Hq6orFixdDLBbXO3519Z2nsrIyUlJSkJiYiEOHDsHHxwd+fn44d+5cvaVUANQoxyIlJVXjoUX1WuCvl6dpDGtra4nzrCpz0liNucb1OXDggHAOVfM/evQoYmNjsXz5cgCVD2rKy8shIyODjRs3Yvz48U2aIxERERFRUzHQTURERO+1f1OQG6gsUeHo6IiwsDDMnDlTIhCan5+PqKgouLu7AwBsbW3h6emJH3/8UajFbWdnhyNHjuDkyZOYPXu2cGy3bt2QmZkpBInf1JUrV2Bvbw83N7ca9b1lZGTq7f/HH39EcXExRo8eXWNfVZA7KysLCQkJaN26daPmU5WVXVRU1OD4TSEjIwMHBwc4ODjA19cXampqOHr0KIYNGwZZWVmJDOn6aGpqIi0tTWJbamqqEGw2NjaGgoIC4uPjay3nIisrCwAS4ykoKNR5nmfOnJH4npycDGNjY4ns9cZITk4WsrNfvXqFCxcuwMPDAwCgr69fo/3p06cl5vjzzz8jODgYp06dQtu2bYXtly5dQlFRkXBfJycnQ0lJSaLOeXVNWWsiIiIier8x0E1ERETvrX9bkLvK2rVr0atXLzg5OWHx4sUwMDDAlStXMHfuXHTs2BE+Pj4AKktQtGrVCjt37sT+/fsBVAa658yZA5FIhN69ewt9+vj4wMXFBXp6ehg+fDikpKRw6dIlpKWlYfHixY2aV1paGuzt7eHk5IRZs2YhPz8fQGWwuaqWdH3Cw8MxZMiQGkHs0tJSDB8+HCkpKdi/fz/KysqEvtXV1YVgLwA8ffoU+fn5KC8vR1ZWFgICAtCxY8c6a5q/if379+PGjRvo27cvWrVqhQMHDqC8vBwmJiYAALFYjDNnziAnJwdKSkr1Zo7b29vj+++/x7Zt22BjY4MdO3YgLS0NVlZWACqz6efPn4958+ZBVlYWvXv3xoMHD3DlyhVMmDABWlpaUFBQQFxcHNq1awd5eXmoqqrWOV5ubi5mzZqFr7/+GikpKVizZg1CQkKavAbr1q2DsbExTE1NsXLlSjx58qTerOzX1//8+fOQkpKCubm5xPaSkhJMmDABXl5eyMnJga+vLzw8PCAlVXtFRbFYjGPHjmHkyJGQk5ODhoZGk8+FiIiIiN4PrNFNRERE76V/a5AbqMzyPXfuHAwNDTFixAjo6+tjwIAB6NixI06ePAklJSUAlSUn+vTpA5FIhI8++ghAZfBbRUUF1tbWEmU0nJycsH//fhw6dAjdu3dHz549sXLlylqzc+uyZ88ePHjwADt27ICOjo7w6d69e4PHZmZm4sSJE5gwYUKNfXfu3EFsbCz++OMPWFpaSvR96tQpibbjxo2Djo4O2rVrB1dXV5iZmeG3336DjMzby99QU1NDTEwM7O3tYWpqig0bNiA6OhpmZmYAgDlz5kBaWhqdO3eGpqYmcnNz6+zLyckJ3t7emDdvHrp3747nz5/jq6++kmjj7e2N2bNnw8fHB6ampvjiiy+EmtkyMjJYvXo1fvjhB+jq6mLw4MH1zv2rr75CUVERevToAXd3d8yYMaNJLwutEhQUhKCgIFhYWODEiROIjY19K0Hmfv36wdjYGH379sUXX3yBTz/9FH5+fnW2DwgIQE5ODjp06NCohylERERE9P4SVTT0piMiIiKiZubly5e4efMmDAwMJOpPN2e+vr5YsWIFDh8+jJ49e77r6RARvXP/xd/1RERE9OZYuoSIiIioGfD394dYLEZycjJ69OhRZ6kHIiIiIiKi9xED3URERETNxLhx4971FIiIiIiIiP6VmApERERERERERERERM0aA91ERERERERERERE1Kwx0E1EREREREREREREzRoD3URERERERERERETUrDHQTURERERERERERETNGgPdRERERERERERERNSsMdBNRERERERERERERM0aA91ERERERP8BdnZ28PT0/Et95OTkQCQSITU19a3MqbqxY8diyJAh9bZpyjmMGTMGgYGBwvfCwkJ89tlnUFFRgUgkwtOnTyEWi7Fq1ao3n3QzkJiYKJwvAMTFxcHS0hLl5eXvdmJERERE/zCZdz0BIiIion/SnW+P1rkv9OQOfH9sK+b2HYcZvUc3ue+C4kKM3r0AmQ9zsHPkMljpdkLbQPs3muft27fh6+uLuLg4PHz4EDo6OhgyZAh8fHzQunXrN+rzr7p06RKCgoJw4sQJPHz4EGKxGFOmTMGMGTPqPc7Ozg5JSUk1tjs7O+PXX39FaWkpvLy8cODAAdy4cQOqqqpwcHBAUFAQdHV1hfYikUj4WVpaGrq6uhg+fDiWLl0KOTm5Osf38/PDvn373lrw1s7ODpaWlv9IADUnJwcGBga4ePEiLC0t//bx3obr16/DysoK0tLSQvC1sWJiYtCiRQvhu1gshqenZ43g96VLl3DgwAGsX79e2BYZGYnjx4/j1KlT0NDQgKqqKs6dO4eWLVv+ldOp4Z+8/m/ik08+gbe3N6KiojBmzJh3PR0iIiKifwwzuomIiIj+vxm9R2Nu33H4/thWhJ7c0eTjleQUseOLIJhoiPHlrnm4ePfqG83jxo0bsLa2RlZWFqKjo3H9+nVs2LAB8fHxsLGxwePHj9+o37/qwoUL0NLSwo4dO3DlyhUsWrQICxcuxNq1a+s9LiYmBnl5ecInLS0N0tLS+PzzzwFUZuKmpKTA29sbKSkpiImJQWZmJj799NMafW3duhV5eXm4efMmwsLCsH37dixevPhvOd+/oqKiAq9evXrX0/jHlZaWwtXVFX369Hmj49XV1aGsrNxguzVr1uDzzz+HkpKSsC07OxumpqYwNzeHtrY2RCIRNDU1oaio+EZzac7Gjh2L1atXv+tpEBEREf2jGOgmIiIiquZtB7vfhLu7O2RlZXHo0CHY2tpCT08PAwYMwJEjR3Dnzh0sWrQIa9euhbm5uXDMvn37IBKJsGHDBmGbg4MDvLy8hO8///wzunXrBnl5eRgaGsLf318iGCsSibB582YMHToUioqKMDY2RmxsrLB//PjxCA0Nha2tLQwNDTF69GiMGzcOMTEx9Z6Puro6tLW1hc/hw4ehqKgoBLpVVVVx+PBhjBgxAiYmJujZsyfWrl2LCxcuIDc3V6IvNTU1aGtro3379nBxccHgwYORkpLSpPWtKqGxfPly6OjooHXr1nB3d0dpaanQJiwsDMbGxpCXl0ebNm0wfPhw4dikpCSEhoZCJBJBJBIhJydHKB/x22+/4YMPPoCcnBxOnDhRa7kOT09P2NnZCd/Ly8uxbNkyGBkZQU5ODnp6eliyZAkAwMDAAABgZWUFkUgkcVxtXr16BQ8PD6iqqkJDQwPe3t6oqKgQ9ovFYgQGBmL8+PFQVlaGnp4eNm7cWKOfq1evolevXpCXl4e5uXmtGfm18fLyQqdOnTBixIg62/j7+0NTUxMqKiqYMmUKSkpKhH3VS5fY2dnh1q1bmDlzprDWAFBWVoY9e/Zg0KBBEseFhITg2LFjEuv0eumShu5xAEhLS8OAAQOgpKSENm3aYMyYMXj48CGAuq9/REQE1NTUJPqp+jdZxc/PD5aWlti+fTvEYjFUVVUxcuRIPH/+XGhTXl6OpUuXwsDAAAoKCrCwsMCePXsk+j1w4AA6duwIBQUFfPzxx8jJyamxxoMGDcL58+eRnZ1d53UgIiIi+q9hoJuIiIjoNW8z2N1Ujx8/xsGDBzF16lQoKChI7NPW1saoUaOwe/du2NraIj09HQ8ePAAAJCUlQUNDA4mJiQAqM2tPnz4tBPyOHz+Or776CjNmzEB6ejp++OEHRERECAHVKv7+/hgxYgQuX74MZ2dnjBo1qt4M8mfPnkFdXb1J5xgeHo6RI0fWW1Li2bNnEIlENYKH1V27dg1Hjx7Fhx9+2KTxASAhIQHZ2dlISEhAZGQkIiIiEBERAQA4f/48pk+fjoCAAGRmZiIuLg59+/YFAISGhsLGxgaTJk0SMtTbt28v9LtgwQIEBQUhIyMDXbt2bdRcFi5ciKCgIHh7eyM9PR07d+5EmzZtAABnz54FABw5cgR5eXkNPlSIjIyEjIwMzp49i9DQUKxYsQKbN2+WaBMSEgJra2tcvHgRU6dOxTfffIPMzEyJNnPnzsXs2bNx8eJF2NjYYNCgQXj06FG9Yx89ehQ//vgj1q1bV2eb+Ph4ZGRkIDExEdHR0YiJiYG/v3+tbWNiYtCuXTsEBAQIaw0Aly9fxrNnz2BtbS3RdtKkSbCxsWlwneq7x58+fQp7e3tYWVnh/PnziIuLw71794TAfUPXvyHZ2dnYt28f9u/fj/379yMpKQlBQUHC/qVLl2Lbtm3YsGEDrly5gpkzZ2L06NHCg4bbt29j2LBhGDRoEFJTUzFx4kQsWLCgxjh6enpo06YNjh8/3ui5ERERETV3DHQTERER1eJtBbubKisrCxUVFTA1Na11v6mpKZ48eQItLS2oq6sLAbDExETMnj1b+H727FmUlpaiV69eACqDewsWLICbmxsMDQ3h6OiI7777Dj/88INE/2PHjoWrqyuMjIwQGBiIgoICIdj6ulOnTmH37t2YPHlyo8/v7NmzSEtLw8SJE+ts8/LlS8yfPx+urq5QUVGR2Ofq6golJSXIy8vDxMQEZmZmWLhwYaPHr9KqVSusXbsWnTp1gouLCwYOHIj4+HgAQG5uLlq2bAkXFxfo6+vDysoK06dPB1CZfS4rKwtFRUUhQ11aWlroNyAgAI6OjujQoUOjHgA8f/4coaGhWLZsGdzc3NChQwd89NFHwvpoamoCAFq3bg1tbe0G+2zfvj1WrlwJExMTjBo1CtOmTcPKlSsl2jg7O2Pq1KkwMjLC/PnzoaGhgYSEBIk2Hh4e+Oyzz2Bqaor169dDVVUV4eHhdY776NEjjB07FhERETWuWXWysrLYsmULzMzMMHDgQAQEBGD16tW1vjhRXV0d0tLSUFZWFtYaAG7dugVpaWloaWlJtFVUVISsrGyD61TfPb527VpYWVkhMDAQnTp1gpWVFbZs2YKEhARcu3atwevfkPLyckRERMDc3Bx9+vTBmDFjhPuuuLgYgYGB2LJlC5ycnGBoaIixY8di9OjRwr/T9evXo0OHDggJCRGu8dixY2sdS1dXF7du3Wr03IiIiIiaOwa6iYiIiOrwNoLdb6p6uYnayMnJoW/fvkhMTMTTp0+Rnp6OqVOnori4GFevXkVSUhK6d+8u1Ce+dOkSAgICoKSkJHyqslILCwuFfqtnIbds2RIqKiq4f/9+jfHT0tIwePBg+Pr6on///gAqA8TV+w8MDKxxXHh4OLp06YIePXrUel6lpaUYMWIEKioqJF40WGXlypVITU3FpUuXsH//fly7dk144V5jxq9iZmYmEaDU0dERztPR0RH6+vowNDTEmDFjEBUVJbFG9ameZdwYGRkZKC4uRr9+/Rp9zPHjxyXOMyoqStjXs2dPiXIZNjY2yMrKQllZmbCt+jUWiUTQ1taucY1tbGyEn2VkZGBtbY2MjAwAlWtXNfaAAQMAAJMmTcKXX34pZL7XxcLCQqJmto2NDQoKCnD79u1Gn39RURHk5OQkzrMp6rvHL126hISEBIn17dSpEwC8lTIgYrFYogZ59fvu+vXrKCwshKOjo8T427ZtE8bOyMio8RcM1a9VdQoKCo2+b4mIiIj+C2Te9QSIiIiI/s1m9B4NAPj+2FaJ738XIyMjiEQiZGRkYOjQoTX2Z2RkQFNTE2pqarCzs8PGjRtx/PhxWFlZQUVFRQh+JyUlwdbWVjiuoKAA/v7+GDZsWI0+5eXlhZ9btGghsU8kEtXItk1PT0e/fv0wefJkiRrgurq6SE1NFb6/nlX74sUL7Nq1CwEBAbWee1WQ+9atWzh69GitmcHa2towMjICAJiYmOD58+dwdXXF4sWLIRaL6x2/uvrOU1lZGSkpKUhMTMShQ4fg4+MDPz8/nDt3rt5SKgBqlGORkpKq8dCiei3w18vTNIa1tbXEeVaVOWmsxlzj+hw4cEA4h6r5Hz16FLGxsVi+fDmAygc15eXlkJGRwcaNGzF+/PgmzbE+GhoaKCwsRElJCWRlZZt8fH3nX1BQgEGDBiE4OLjGcTo6OnX22dB1buzYAPDrr7+ibdu2Eu3k5OTqHLsujx8/Fv4igIiIiOh9wEA3ERERvVcKigubnGn9Twa7W7duDUdHR4SFhWHmzJkSgdD8/HxERUXB3d0dAGBrawtPT0/8+OOPQi1uOzs7HDlyBCdPnsTs2bOFY7t164bMzEwhSPymrly5Ant7e7i5udWo7y0jI1Nv/z/++COKi4sxenTN9asKcmdlZSEhIQGtW7du1HyqsrKLiooaHL8pZGRk4ODgAAcHB/j6+kJNTQ1Hjx7FsGHDICsrK5EhXR9NTU2kpaVJbEtNTRUCnsbGxlBQUEB8fHyt5VyqArnVx1NQUKjzPM+cOSPxPTk5GcbGxk0qr1F1XFV29qtXr3DhwgV4eHgAAPT19Wu0P336tMQcf/75ZwQHB+PUqVMSQdtLly6hqKhIuK+Tk5OhpKRUZ53r2tba0tISQOUDl6qf35Zu3brhp59+glgshoxM7f9Vqm1OmpqaeP78OV68eCE87Kj+MKIxOnfuDDk5OeTm5ko8pKrO1NS0xsszk5OTa7R7+fIlsrOzYWVl1aQ5EBERETVnLF1CRERE75XRuxegoLjpf87/V8uYNMXatWtRXFwMJycnHDt2DLdv30ZcXBwcHR3RsWNH+Pj4AKgswdCqVSvs3LlTItC9b98+FBcXo3fv3kKfPj4+2LZtG/z9/XHlyhVkZGRg165dEhnZDUlLS8PHH3+M/v37Y9asWcjPz0d+fr7wQsyGhIeHY8iQITWC2KWlpRg+fDjOnz+PqKgolJWVCX2XlJRItH369Cny8/Nx9+5dJCUlISAgAB07dqyzpvmb2L9/P1avXo3U1FTcunUL27ZtQ3l5OUxMTABUlp84c+YMcnJy8PDhw3qzoe3t7XH+/Hls27YNWVlZ8PX1lQh8y8vLY/78+Zg3b55QoiI5OVmoh62lpQUFBQXhpYjPnj2rd+65ubmYNWsWMjMzER0djTVr1mDGjBlNXoN169Zh7969uHr1Ktzd3fHkyZN6s7JNTU1hbm4ufNq2bQspKSmYm5ujVatWQruSkhJMmDAB6enpOHDgAHx9feHh4QEpqdr/WyIWi3Hs2DHcuXMHDx8+BFAZVO7WrRtOnDjR5PNqiLu7Ox4/fgxXV1ecO3cO2dnZOHjwIMaNGycEt2u7/h9++CEUFRXx7bffIjs7Gzt37hRebtpYysrKmDNnDmbOnInIyEhkZ2cjJSUFa9asQWRkJABgypQpyMrKwty5c5GZmVnnOMnJyZCTk6uzrAkRERHRfxEzuomIiOi9cu6PtIYb1WEZ7LEMW97ibGpnbGyMc+fOwc/PDyNGjMD9+/dRUVGBYcOGYfv27UKNY5FIhD59+uDXX3/FRx99BKAy+K2iogITExOJMhpOTk7Yv38/AgICEBwcjBYtWqBTp071vhTydXv27MGDBw+wY8cO7Njxv2C/vr4+cnJy6j02MzMTJ06cwKFDh2rsu3PnjpCl+nqGbkJCghDEB4Bx48YJ566trY2+ffsiMDCwzuzbN6GmpoaYmBj4+fnh5cuXMDY2RnR0NMzMzAAAc+bMgZubGzp37oyioiLcvHmzzr6cnJzg7e2NefPm4eXLlxg/fjy++uor/P7770Ibb29vyMjIwMfHB3fv3oWOjg6mTJkCoDKzfPXq1QgICICPjw/69OmDxMTEOsf76quvUFRUhB49ekBaWhozZsxo0stCqwQFBSEoKAipqakwMjJCbGwsNDQ0mtzP6/r16wdjY2P07dsXxcXFcHV1hZ+fX53tAwIC8PXXX6NDhw4oLi4WyoNMnDgR27ZtE7LM3xZdXV2cPHkS8+fPR//+/VFcXAx9fX188sknQjC+tusvFouxY8cOzJ07F5s2bUK/fv3g5+fX5LX/7rvvoKmpiaVLl+LGjRtQU1NDt27d8O233wIA9PT08NNPP2HmzJlYs2YNevTogcDAwBoPIaKjozFq1CiJeuhERERE/3WiiobedERERETUzLx8+RI3b96EgYGBRP3p5szX1xcrVqzA4cOH0bNnz3c9HaJ3qqioCCYmJti9ezezll/z8OFDmJiY4Pz58zAwMHjX0/lb/Rd/1xMREdGbY0Y3ERERUTPg7+8PsViM5ORk9OjRo85SD0TvAwUFBWzbtk0oZ0L/k5OTg7CwsP98kJuIiIjodczoJiIiov8cZvkREf338Xc9ERERVcdUICIiIiIiIiIiIiJq1hjoJiIiIiIiIiIiIqJmjYFuIiIi+s9ihTYiov8u/o4nIiKi6hjoJiIiov+cFi1aAAAKCwvf8UyIiOjvUvU7vup3PhEREb3fZN71BIiIiIjeNmlpaaipqeH+/fsAAEVFRYhEonc8KyIiehsqKipQWFiI+/fvQ01NDdLS0u96SkRERPQvIKrg33sRERHRf1BFRQXy8/Px9OnTdz0VIiL6G6ipqUFbW5sPMomIiAgAA91ERET0H1dWVobS0tJ3PQ0iInqLWrRowUxuIiIiksBANxERERERERERERE1a3wZJRERERERERERERE1awx0ExEREREREREREVGzxkA3ERERERERERERETVrDHQTERERERERERERUbPGQDcRERERERERERERNWsMdBMRERERERERERFRs8ZANxERERERERERERE1a/8Px5SuJdJaSOEAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "df = pd.read_csv(\"results/experiment-2-results.csv\")\n", + "metrics_df = get_metrics(df)\n", + "metrics_df.describe()\n", + "plot_metrics(metrics_df, figsize=(18, 5), ylim=(0, 0.5))" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "954eda90-de91-4771-b731-919328aba5b5", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_utils.py:302: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n", + " barplot.set_xticklabels([\"METEOR\", \"BLEU-1\", \"ROUGE-L\"])\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABboAAALHCAYAAABIevSFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdeVhU1eMG8HdmANkEFwQEURQV3CkQNPfEL/azTK0k91Bb3BXT1EorF9xzIy3NJbUkE802rHDfDSWXEFcEWUUEBJVlZn5/jDMyzgyzMDADvJ/n4XmGe8859wyRzHnvuecIpFKpFEREREREREREREREVZTQ1B0gIiIiIiIiIiIiIioPBt1EREREREREREREVKUx6CYiIiIiIiIiIiKiKo1BNxERERERERERERFVaQy6iYiIiIiIiIiIiKhKY9BNRERERERERERERFUag24iIiIiIiIiIiIiqtIYdBMRERERERERERFRlcagm4iIiIiIiIiIiIiqNAbdRERERERERERERFSlmUXQHRERAU9PT1hbWyMwMBBnz57VWHbr1q0QCARKX9bW1pXYWyIiIiIiIiIiIiIyJyYPuiMjIxEWFoZ58+bh/Pnz6NChA4KDg5GZmamxjoODA9LS0hRfd+7cqcQeExEREREREREREZE5MXnQvXLlSrz77rsIDQ1F69atsWHDBtja2mLz5s0a6wgEAri6uiq+XFxcKrHHRERERERERERERGROLEx58aKiIsTGxmL27NmKY0KhEEFBQTh16pTGevn5+WjSpAkkEglefPFFLFq0CG3atFFbtrCwEIWFhYrvJRIJsrOzUb9+fQgEAuO9GSIiIiIiIiIiIiIyGqlUiocPH8LNzQ1CYdlztk0adGdlZUEsFqvMyHZxccHVq1fV1vH29sbmzZvRvn175ObmYvny5XjppZdw5coVNGrUSKV8eHg4Pv/88wrpPxERERERERERERFVrOTkZLXZb2kmDboN0blzZ3Tu3Fnx/UsvvYRWrVrh66+/xvz581XKz549G2FhYYrvc3Nz0bhxYyQnJ8PBwaFS+kxERERERERERERE+snLy4OHhwdq166ttaxJg24nJyeIRCJkZGQoHc/IyICrq6tObVhaWuKFF17AjRs31J6vVasWatWqpXLcwcGBQTcRERERERERERGRmdNlCWqTbkZpZWUFPz8/xMTEKI5JJBLExMQozdoui1gsxqVLl9CwYcOK6iYRERERERERERERmTGTL10SFhaGUaNGwd/fHwEBAVi1ahUKCgoQGhoKABg5ciTc3d0RHh4OAPjiiy/QqVMnNG/eHDk5OVi2bBnu3LmDsWPHmvJtEBEREREREREREZGJmDzoDgkJwb179zB37lykp6fD19cX0dHRig0qk5KSlHbUfPDgAd59912kp6ejbt268PPzw8mTJ9G6dWtTvQUiIiIiIiIiIiIiMiGBVCqVmroTlSkvLw+Ojo7Izc3lGt1EREREREREREREZkqfLNeka3QTEREREREREREREZUXg24iIiIiIiIiIiIiqtIYdBMRERERERERERFRlcagm4iIiIiIiIiIiIiqNAbdRERERERERERERFSlMegmIiIiIiIiIiIioiqNQTcRERERERERERERVWkMuomIiABERETA09MT1tbWCAwMxNmzZ3Wqt2vXLggEAgwYMEDlXHx8PPr37w9HR0fY2dmhY8eOSEpKMnLPiYiIiIiIiIhBNxER1XiRkZEICwvDvHnzcP78eXTo0AHBwcHIzMwss15iYiI+/PBDdOvWTeXczZs30bVrV/j4+ODw4cO4ePEiPv30U1hbW1fU2yAiIiIiIiKqsQRSqVRq6k5Upry8PDg6OiI3NxcODg6m7g4REZmBwMBAdOzYEevWrQMASCQSeHh4YNKkSZg1a5baOmKxGN27d8fo0aNx7Ngx5OTkYN++fYrzb7/9NiwtLbF9+/bKeAtERERERERE1Y4+WS5ndBMRUY1WVFSE2NhYBAUFKY4JhUIEBQXh1KlTGut98cUXcHZ2xpgxY1TOSSQS/Pbbb2jZsiWCg4Ph7OyMwMBApSCciIiIiIiIiIyHQTcREdVoWVlZEIvFcHFxUTru4uKC9PR0tXWOHz+Ob7/9Fhs3blR7PjMzE/n5+Vi8eDH69u2LP//8EwMHDsSgQYNw5MgRo78HIiIiIiIioprOwtQdICIiqkoePnyIESNGYOPGjXByclJbRiKRAABef/11TJs2DQDg6+uLkydPYsOGDejRo0el9ZeIiIiIiIioJmDQTURENZqTkxNEIhEyMjKUjmdkZMDV1VWl/M2bN5GYmIjXXntNcUwebFtYWCAhIQEeHh6wsLBA69atleq2atUKx48fr4B3QURERERERFSzcekSIiKq0aysrODn54eYmBjFMYlEgpiYGHTu3FmlvI+PDy5duoS4uDjFV//+/dGrVy/ExcXBw8MDVlZW6NixIxISEpTqXrt2DU2aNKnw90RERERERERU03BGNxER1XhhYWEYNWoU/P39ERAQgFWrVqGgoAChoaEAgJEjR8Ld3R3h4eGwtrZG27ZtlerXqVMHAJSOz5gxAyEhIejevTt69eqF6Oho/PLLLzh8+HBlvS0iIiIiIiKiGoNBNxER1XghISG4d+8e5s6di/T0dPj6+iI6OlqxQWVSUhKEQv0egho4cCA2bNiA8PBwTJ48Gd7e3tizZw+6du1aEW+BiIiIiIiIqEYTSKVSqak7UZny8vLg6OiI3NxcODg4mLo7RERERERERERERKSGPlku1+gmIiIiIiIiIiIioiqNQTcRERERERERERERVWkMuomIiIiIiIiIiIioSmPQTURERERERERERERVGoNuIiIiIiIiIiIiIqrSGHQTERERERERERERUZXGoJuIiIiIiIiIiIiIqjQG3URERERERERERERUpVmYugNERESmMrPHaEzpMlzvevmFjzA8chYSshLx/dtL8YKbj95tuC96We86RERERERERKQeZ3QTEVGNtezoFqw+sUPveva1bLEjZDG8nTwxdNdMXEi9WgG9IyIiIiIiIiJdMegmIqIaa0b3UIbdRERERERERNUAg24iIqqxpnQZzrCbiIiIiIiIqBpg0E1ERDUaw24iIiIiIiKiqo9BNxER1XgMu4mIiIiIiIiqNgbdREREYNhNRERERERElSsiIgKenp6wtrZGYGAgzp49q1O9Xbt2QSAQYMCAAYpjxcXF+Oijj9CuXTvY2dnBzc0NI0eORGpqagX13vww6CYiInqKYTcRERERERFVhsjISISFhWHevHk4f/48OnTogODgYGRmZpZZLzExER9++CG6deumdPzRo0c4f/48Pv30U5w/fx5RUVFISEhA//79K/JtmBWBVCqVmroTlSkvLw+Ojo7Izc2Fg4ODqbtDREQmlDLnoNrjq0/swLKjWzCjeyimdBmud7v5hY8wPHIWErIS8f3bS/GCm49KGfdFL+vdLhEREREREVUPgYGB6NixI9atWwcAkEgk8PDwwKRJkzBr1iy1dcRiMbp3747Ro0fj2LFjyMnJwb59+zRe49y5cwgICMCdO3fQuHHjingbFU6fLJczuomIiJ7Dmd1ERERERERUUYqKihAbG4ugoCDFMaFQiKCgIJw6dUpjvS+++ALOzs4YM2aMTtfJzc2FQCBAnTp1ytvlKoFBNxERkRoMu4mIiIiIiKgiZGVlQSwWw8XFRem4i4sL0tPT1dY5fvw4vv32W2zcuFGnazx58gQfffQRhgwZUmNWtWDQTUREpAHDbiIiIiIiIjK1hw8fYsSIEdi4cSOcnJy0li8uLsbgwYMhlUqxfv36SuihebAwdQeIiIjMmXyN7mVHtyh9ryt52D08chaG7pqpcc1uIiIiIiIiqhmcnJwgEomQkZGhdDwjIwOurq4q5W/evInExES89tprimMSiQQAYGFhgYSEBHh5eQF4FnLfuXMHBw8erDGzuQHO6CYiItKKM7uJiIiIiIjIWKysrODn54eYmBjFMYlEgpiYGHTu3FmlvI+PDy5duoS4uDjFV//+/dGrVy/ExcXBw8MDwLOQ+/r16/j7779Rv379SntP5oAzuomIiHRg7JndeVvHG72PREREREREVDWEhYVh1KhR8Pf3R0BAAFatWoWCggKEhoYCAEaOHAl3d3eEh4fD2toabdu2Vaov32BSfry4uBhvvvkmzp8/j19//RVisVix3ne9evVgZWVVeW/ORBh0ExER6ciYYTcRERERERHVXCEhIbh37x7mzp2L9PR0+Pr6Ijo6WrFBZVJSEoRC3RfjSElJwf79+wEAvr6+SucOHTqEnj17GqvrZksglUqlpu5EZcrLy4OjoyNyc3Nr1Bo1RESkKmH6r7CvZat3vdUndmDZ0S2Y0T1U77AbAPILH8F7xat61yMiIiIiIiKqSfTJcrlGNxER1VjDI2chv/CR3vWMsWY3ERERERERERkPg24iIqqxErISTRZ2ExEREREREZHxMOgmIqIa6/u3lzLsJiIiIiIiIqoGGHQTEVGN9YKbD8NuIiIiIiIiomqAQTcREdVoDLuJiIiIiIiIqj4G3UREVOMx7CYiIiIiIiKq2hh0E5FWERER8PT0hLW1NQIDA3H27Fmd6u3atQsCgQADBgxQOi6VSjF37lw0bNgQNjY2CAoKwvXr1yug50S6Y9hNREREREREVHUx6CaiMkVGRiIsLAzz5s3D+fPn0aFDBwQHByMzM7PMeomJifjwww/RrVs3lXNLly7FmjVrsGHDBpw5cwZ2dnYIDg7GkydPKuptEOmEYTcRERERERFR1SSQSqVSU3eiMuXl5cHR0RG5ublwcHAwdXeIzF5gYCA6duyIdevWAQAkEgk8PDwwadIkzJo1S20dsViM7t27Y/To0Th27BhycnKwb98+ALLZ3G5ubpg+fTo+/PBDAEBubi5cXFywdetWvP3225XyvogAIGXOQbXHL6RexdBdM+Ht5IkdIYthX8tW77ZXn9iBZUe3YEb3UEzpMlzlvPuil/Vuk4iIiIiIiKo2TePQ0vILH2F45CwkZCXi+7eX4gU3H72vo25cWxXHofpkuZzRTUQaFRUVITY2FkFBQYpjQqEQQUFBOHXqlMZ6X3zxBZydnTFmzBiVc7dv30Z6erpSm46OjggMDCyzTaLKxJndREREREREZCr2tWyxI2QxvJ08MXTXTFxIvap3G8YY11Y1DLqJSKOsrCyIxWK4uLgoHXdxcUF6erraOsePH8e3336LjRs3qj0vr6dPm0SmwLCbiIiIiIiITKUiwu7qjkE3ERnNw4cPMWLECGzcuBFOTk6m7g5RuTHsJiIiIiIiIlMxdthd3THoJiKNnJycIBKJkJGRoXQ8IyMDrq6uKuVv3ryJxMREvPbaa7CwsICFhQW+++477N+/HxYWFrh586ainq5tEpkaw24iIiIiIiIyFWOG3dUdg24i0sjKygp+fn6IiYlRHJNIJIiJiUHnzp1Vyvv4+ODSpUuIi4tTfPXv3x+9evVCXFwcPDw80LRpU7i6uiq1mZeXhzNnzqhtk2qWiIgIeHp6wtraGoGBgTh79qzGslFRUfD390edOnVgZ2cHX19fbN++XalMRkYG3nnnHbi5ucHW1hZ9+/bF9evX9e4Xw24iIiIiIiIyBkPGg8YKu6s7Bt1EVKawsDBs3LgR27ZtQ3x8PMaNG4eCggKEhoYCAEaOHInZs2cDAKytrdG2bVulrzp16qB27dpo27YtrKysIBAIMHXqVCxYsAD79+/HpUuXMHLkSLi5uWHAgAEmfKckZ6qwOTIyEmFhYZg3bx7Onz+PDh06IDg4GJmZmWqvXa9ePXz88cc4deoULl68iNDQUISGhuLAgQMAAKlUigEDBuDWrVv4+eefceHCBTRp0gRBQUEoKCjQ++di7LCbiIiIiIiIah5DJz8ZI+yu7hh0E1GZQkJCsHz5csydOxe+vr6Ii4tDdHS0YjPJpKQkpKWl6dXmzJkzMWnSJLz33nvo2LEj8vPzER0dDWtr64p4C6QHU4bNK1euxLvvvovQ0FC0bt0aGzZsgK2tLTZv3qz22j179sTAgQPRqlUreHl5YcqUKWjfvj2OHz8OALh+/TpOnz6N9evXo2PHjvD29sb69evx+PFj/PDDDwb9fIwZdhMREREREVHNU54nfRl2l00glUqlpu5EZcrLy4OjoyNyc3Ph4OBg6u4QEZmVwMBAdOzYEevWrQMgW6rGw8MDkyZNwqxZuu3Q/OKLL6Jfv36YP38+rl27Bm9vb1y+fBlt2rRRtOnq6opFixZh7NixAICioiLY2trip59+UprZP2rUKOTk5ODnn38u85pSqRQHDx5E//79sW/fPvTp0weXLl1C+/btcePGDXh5eSnKenh4oHfv3ti6dSt+fecrgx7fupB6FUN3zYS3kyd2hCyGfS1bvdtwX/Sy3nWIiIiIiIioakuZcxCrT+zAsqNbMKN7KKZ0Ga53G/mFjzA8chYSshLx/dtLdR7XVsVxqD5ZLmd0ExFpYK7rRVeUoqIixMbGIigoSHFMKBQiKCgIp06d0lpfKpUiJiYGCQkJ6N69OwCgsLAQAJRm6wuFQtSqVUsx8xoAsrKyIBaLFU8KyLm4uCA9PV3jNXNzc2Fvbw8rKyv069cPa9euRZ8+fQDI1oxv3LgxZs+ejQcPHqCoqAhLlizB3bt3FU8hGGPXakNndhMREREREVHNVN49nDizWz0G3UREapj7etEVwVRhc3nUrl0bcXFxOHfuHBYuXIiwsDAcPnwYAGBpaYmoqChcu3YN9erVg62tLQ4dOoRXXnkFQqHsz58xdq1m2E1ERERERET6YthtfAy6iYjUqArrRZuL8obNAODk5ASRSISMjAyltjMyMuDq6qrx2kKhEM2bN4evry+mT5+ON998E+Hh4Yrzfn5+iIuLQ05ODtLS0hAdHY379++jWbNmAGCUXasZdhMREREREZEhGHYbF4NuIqLnmHIJD1MyVdgMAFZWVvDz80NMTIzimEQiQUxMDDp37qzze5BIJIqfdWmOjo5o0KABrl+/jn/++Qevv/46AON8KGDYTURERERERIZi2G08DLqJiJ5TFZfwMAZThc1yYWFh2LhxI7Zt24b4+HiMGzcOBQUFCA0NBQCMHDkSs2fPVpQPDw/HX3/9hVu3biE+Ph4rVqzA9u3bMXz4s408du/ejcOHDyuWjOnTpw8GDBiA//3vf4oyDLuJiIiIiIhqLmPvz5Wfn4+JEyeiUaNGsLGxUTwlXhaG3cZhYeoOEBFVF/IlPPLz8xETE4OwsDA0a9YMPXv2VCzhMWbMGNSrVw8ikQhBQUF45ZVXIJVKTd11hbCwMIwaNQr+/v4ICAjAqlWrVMJmd3d3xYzt8PBw+Pv7w8vLC4WFhfj999+xfft2rF+/XtHm7t270aBBAzRu3BiXLl3ClClTVMJmAAgJCcG9e/cwd+5cpKenw9fXF9HR0YobDklJSUrLnRQUFGD8+PG4e/cubGxs4OPjgx07diAkJERRJi0tDWFhYcjIyEDDhg0xcuRIfPrppyrvW/6hYHjkLAzdNVOvXavl5GH30F0zMTxyFnaELIZ9LVu92iAiIiIiIqLKI9+fa8OGDQgMDMSqVasQHByMhIQEODs7q5SX78/l4+MDKysr/PrrrwgNDYWzszOCg4MByMbVBw8exI4dO+Dp6Yk///wT48ePh5ubG/r376+xL1O6yCZtLTu6Rel7XRljXFvVmcWMbn3unJS2a9cuCAQCDBgwoGI7SEQ1iimX8DC1kJAQLF++HHPnzoWvry/i4uJUwubSM9DlYXObNm3QpUsX7NmzBzt27MDYsWMVZdLS0jBixAj4+Phg8uTJGDFihMZ1ySdOnIg7d+6gsLAQZ86cQWBgoOLc4cOHsXXrVsX3CxYswPXr1/H48WNkZ2fj5MmTSiE3AEyePBnJyckoKirCnTt3MH/+fFhZWam9Nmd2ExERERGZD3OYZUvVn7H35wKAkydPYtSoUejZsyc8PT3x3nvvoUOHDjrlnZzZXT4mD7rld07mzZuH8+fPo0OHDggODkZmZmaZ9RITE/Hhhx+iW7duldRTIqopTL2Eh6mZMmw2NYbdRERERESmp29WJJ9le+rUKVy8eBGhoaEIDQ3FgQMHFGXCwsIQHR2NHTt2ID4+HlOnTsXEiROxf//+ynpbZGYqYn8uAHjppZewf/9+pKSkQCqV4tChQ7h27ZrKU82aMOw2nMmDbn3vnACAWCzGsGHD8Pnnn5vVTEgiqj5MtV40mR7DbiIiIiIi0zK3WbZUPVXE/lwAsHbtWrRu3RqNGjWClZUV+vbti4iICKUwXBuG3YYx6Rrd8jsnpcMiXe6cfPHFF3B2dsaYMWNw7NixMq9RWFioNKMyLy+v/B0nqgGCtn2mU7mSx4WIW7ET+SmZeGHGCDg2c9f7Wrf3H8WtqENoNqgXmvbvjr9H6XbtimTK9aLJ9LhmNxERERGRaRiaFclJpVIcPHgQCQkJWLJkieK4fJbt6NGj4ebmhsOHD+PatWv48ssvK+R9UPVV1v5cgCzoPn36NPbv348mTZrg6NGjmDBhAtzc3JRmj2tTEWt2u+NlvdqoakwadJd15+TqVfV3Go4fP45vv/0WcXFxOl0jPDwcn3/+eXm7SkRqGCPkBoCm/WV3NW9FHZIdGGWsHpbPxIkTMXHiRLXnDh8+rPT9ggULsGDBgjLbmzx5MiZPnmys7lEFY9hNRERERFT5DMmKANksW3d3dxQWFkIkEuGrr75SmWX73nvvoVGjRrCwsIBQKMTGjRv1mmVL1Ut59+cCAF9fX8THxyM8PBw9e/bE48ePMWfOHOzduxf9+vUDALRv3x5xcXFYvny5XkE3YPywO2/reL3qVzUmX7pEHw8fPsSIESOwceNGODk56VRn9uzZyM3NVXwlJydXcC+JagZjhdxyTft3R7NBvZ6F3URmgMuYEBERERFVDfJZtufOncPChQsRFhamNEGp9Czb2NhYrFixAhMmTMDff/9tuk6TSVXE/lzFxcUoLi5WegIcAEQiESQSiUH9NOYyJtWdSWd063vn5ObNm0hMTMRrr72mOCb/JbGwsEBCQgK8vLyU6tSqVQu1atWqgN4T1VzGDrnl5DO7icxJRc3sJiIiIiIiVVVhli1VH2FhYRg1ahT8/f0REBCAVatWqezP5e7ujvDwcACylSP8/f3h5eWFwsJC/P7779i+fTvWr18PAHBwcECPHj0wY8YM2NjYoEmTJjhy5Ai+++47rFy5EgCQX/hI7yd9jTWzu7oz6Yxufe+c+Pj44NKlS4iLi1N89e/fH7169UJcXBw8PDwqs/tElSoiIgKenp6wtrZGYGBgmRtmREVFwd/fH3Xq1IGdnR18fX2xfft2pTICgUDt17JlywAAubdS1LZdUSG3HMNuMkcVMbObiIiIiIhUVZVZtlQ9hISEYPny5Zg7dy58fX0RFxensj9XWlqaorx8f642bdqgS5cu2LNnD3bs2IGxY8cqyuzatQsdO3bEsGHD0Lp1ayxevBgLFy7EBx98AAAGP+lrjJnd1Z1AKpVKTdmByMhIjBo1Cl9//bXizsmPP/6Iq1evwsXFReXOyfPeeecd5OTkYN++fTpdLy8vD46OjsjNzYWDg4MR3wlRxYmMjMTIkSOxYcMGBAYGYtWqVdi9ezcSEhLg7OysUv7w4cN48OABfHx8YGVlhV9//RXTp0/Hb7/9huDgYABQ2UH4jz/+wJgxY3Djxg00a9YMFrbWKkF2RYfccuawGSXVDClzDupVPr/wEYZHzkJCVqJBM7sB4ELqVdnaaE/y9a5LRERERFQT6JsVqZtlO2vWLKxfv14RQPbs2RNZWVlYt26dYpbtuHHjsHLlSowbN86Ub5dqGAdre3g7eRq8h9PqEzuw7OgWzOgeqvfMbvdFVW8zSn2yXJMuXQLI7pzcu3cPc+fORXp6Onx9fVXunDx/x42oplm5ciXeffddxaMzGzZswG+//YbNmzdj1izVmaHynX7lpkyZgm3btuH48eOKoPv5R75+/vln9OrVC82aNQMA2Ls748Ky7YpAu7JCbqp5Xmzkj+tZ17AlZCfau/nqXf9iahxCI4ehhVNLbBq8Hfa17HWuayfQ71rGXMaEiIiIiIjU0zcrks+yvXv3LmxsbODj44MdO3YgJCREUWbXrl2YPXs2hg0bhuzsbDRp0kRpli2ZVkREBJYtW4b09HR06NABa9euRUBAgNqyUVFRWLRoEW7cuIHi4mK0aNEC06dPx4gRIxRlBAL1g72lS5dixowZFfIedPX8spaVvYxJdWbyGd2VjTO6qaopKiqCra0tfvrpJwwYMEBxfNSoUcjJycHPP/9cZn2pVIqDBw+if//+2Ldvn9Ku03IZGRlo1KgRtm3bhqFDhwIAem6YrQi2208ajFtRhyst5OaM7sonX7rmvWFLEdTt2R/Ju2nX8PnKt5CTlwmBQABNfzLk5+o4OOOz6T/B3bWF4txfR7fjm50zNV7bvlZtk4TcALDp5DyDPhQYY2Z3VbyTTkREREREZGymeIrdlFLmHFQ86VvZM7ur4jhUnyyXU6WJzFxWVhbEYrHizrWci4uLyj/cpeXm5sLe3h5WVlbo168f1q5dqzbkBoBt27ahdu3aGDRokOKYhU0t+E4fBhunOriwdDtybyRzJnc1JhAIMG7kl0ohNwA0atgSX3y4F3UcnCEUiDTWFwpEqOPgjC8+3KsUcgNAn+4jMH7kl4ow/XmmCrkBGGXXakPX7CYiIiIiIiLlp9hbt26NDRs2wNbWFps3b1ZbvmfPnhg4cCBatWoFLy8vTJkyBe3bt8fx48cVZVxdXZW+nn+K3dSe38PJFGt2V0cMuomqqdq1ayMuLg7nzp3DwoULERYWhsOHD6stu3nzZgwbNgzW1taV20kyG5NC16LXSyFqzzV0aYYFM/fD0aEBhELVsFsoFMHRoQEWfPQLGrqo/9DQq8vbmBS6Vu05U4XcX51YU+6NPBh2ExERERERGa6oqAixsbEICgpSHBMKhQgKCsKpU6e01pdKpYiJiUFCQgK6d++utkxGRgZ+++03jBkzxmj9NgaG3cbHoJvIzDk5OUEkEiEjI0PpeEZGhso626UJhUI0b94cvr6+mD59Ot588021m7oeO3YMCQkJSjsEA882nnyclYMXZo6AY3MPXFi2Hbm3UozzxsisdAt8o8zzLg2a4JVeo9Wek0ql+L+Xx8DFqbHWazR2b2VwH+WMFXKvPrbcKLtWM+wmIiIiIiIyjKmeYjcXDLuNy+SbURJR2aysrODn54eYmBjFGt0SiQQxMTGYOHGizu1IJBIUFhaqHP/222/h5+eHDh06KB1/fuNJh6buiFuxU2mDyurolS17AQCnFs5C7s1r8OgVjDYj3ldbVlJcjL8njoCkuAh+Uz9Bg/YvKs6Ji4vw1/tvAwA6zvgc9Vu1U9tGfupdHP9kMgBoXP+6MkgkEq0b/95NuwYBZEuPyGd2SyRiCAVCJKdd0+kaqek3Fd8LhSJIJGK9+mnUkLvbhwDKv5GHMTaoJCIiIiIiIt3Jn2LPz89HTEwMwsLC0KxZM/Ts2VOlrLk/xS4Pu7lBZflxRjdRFRAWFoaNGzdi27ZtiI+Px7hx41BQUIDQ0FAAwMiRIzF79mxF+fDwcPz111+4desW4uPjsWLFCmzfvh3Dhyv/Q5eXl4fdu3erzOYGoLLxpHzNbnt35xoxs1sebicfOoA7Mb+rnC/MzcHxT6dAUlwEAHBq66t0XmRphWb9ZLOkzy2bh/zUuypt3P/voiLk9ugVbMzu6+3e/WStZRLvXoFYUgIAcHPxgpuLFwBALClBYvIVrfUz7yehuKRQsU53h9Y9YF3LTuc+GjvkHt9lsuI4Z3YTERERERFVPlM9xW5uOLPbOBh0E1UBISEhWL58OebOnQtfX1/ExcUhOjpa8WhPUlIS0tLSFOULCgowfvx4tGnTBl26dMGePXuwY8cOlX/Yd+3aBalUiiFDhqhcU92s7ZoUdjs0boq2oRMAAPE7NyF69CAk/LQDtw/sx9XIrTg0bTQeZcoeo+q++CsI1MyGbj7gbdRt2RoAcPyTyTj28STc+i0Ktw/sx9ll83Bu+WcAgLotW6PVUNP+0U1OTSjzvFgiRmrGLTSo3wiTx0RgxdxDWDH3ECaPiUCD+o2QlnETYi2zs+XX8G3TC8s++RtzJu3E+vB/dOpfRYbccgy7iYiIiIiURUREwNPTE9bW1ggMDMTZs2c1lo2KioK/vz/q1KkDOzs7+Pr6Yvv27Srl4uPj0b9/fzg6OsLOzg4dO3ZEUlJSRb4NMmOln2KXkz/F3rlzZ53b0fcpdnPEsLv8BFJTPitvAnl5eXB0dERubi4cHBxM3R0isxW07TON5+Trdz8/69sYbu8/ipt7DhqtPX3Jly6Ry4z7B+fXLFJb1rFpC/iO/xA29RtobE8iFiP++01IPnRA7flm/d5A8wFvQygS4Y/QgYZ3vJw+mfIDfNv00ng+Ny8LZ+N+R68uQ2AhslQ6VyIuxqETPyDA9//g6OCksY24K4dgXcsOPs0DlI63v11cZt8qMuS2E1xXKbv6xA4sO7oFM7qHGvS4V37hIwyPnIWErMQylzFxX/Sy3m0TEREREVWWyMhIjBw5Ehs2bEBgYCBWrVqF3bt3IyEhAc7OzirlDx8+jAcPHsDHxwdWVlb49ddfMX36dPz2228IDpY9wXrz5k0EBARgzJgxGDJkCBwcHHDlyhV06tRJbZs1UUREBJYtW4b09HR06NABa9euRUBAgNqyUVFRWLRoEW7cuIHi4mK0aNEC06dPx4gRI5TKxcfH46OPPsKRI0dQUlKC1q1bY8+ePWjcuOx9lipLZGQkRo0aha+//hoBAQFYtWoVfvzxR1y9ehUuLi4YOXIk3N3dFTO2w8PD4e/vDy8vLxQWFuL333/HrFmzsH79eqUJfnl5eWjYsCFWrFiBDz74QOma12Zrf6pZLr8wH2N/HIHrWdewJWQn2rv56v0eS49rfxm5tMyyF1KvYuiumfB28jRoGRNA87i2Ko5D9clyGXQTkVplBd1AxYTdt/cfxa2oQyZdq/r5oBuQrZ2dczMB2VcvQ1JUBMvajnD17wTruvV1bldcXISM2DN4lJEKALB1cYOLXyBEllaKMqYMun/6RvMmHxWtrKC7omdyqwu6gcoJu6viBwwiIiIiqjkCAwPRsWNHrFu3DoBsxqyHhwcmTZqEWbNm6dTGiy++iH79+mH+/PkAgLfffhuWlpZqZ3pTzb65sG7dOkXA7+vrizVr1iAwMBAA0LNnT3h6emLr1q0AgE8++QSRkZG4e/cubGxs4OPjgylTpiAkJESpzW+++QZTp05FWloaHB0dlc7pGnQbO+TeNHg7XKzTtNapqLC7Ko5DGXSXgUE3kW60Bd2AccNuecjdbFAvs5rRXZkYdCurjOVKNAXdQMWH3VXxAwYRERER1QxFRUWwtbXFTz/9hAEDBiiOjxo1Cjk5Ofj555/LrC+VSnHw4EH0798f+/btQ58+fSCRSODo6IiZM2fi+PHjuHDhApo2bYrZs2crruH1xsto2r+7Xn0tPZbUt25pf4/6zOC6xsKbC5VHl6C7IkJu+1r2ZY5DS6uIsLsqjkP1yXK5RjcRGcxYa3Yb64MJVR+VEXJrwzW7iYiIiKimysrKglgsVuwLJefi4oL0dM2TZHJzc2Fvbw8rKyv069cPa9euRZ8+fQAAmZmZyM/Px+LFi9G3b1/8+eefGDhwIAYNGoQjR44AgEHjwab9u6PZoF64FXUIt/cf1bs+ALPYf6qoqAixsbEICgpSHBMKhQgKCsKpU6e01pdKpYiJiUFCQgK6d5f9HCUSCX777Te0bNkSwcHBcHZ2RmBgIPbt21dRb6PaqKiQWx9cs1t/DLqJqFzKG3Yz5KbnmUPILcewm4iIiIhId7Vr10ZcXBzOnTuHhQsXIiwsDIcPHwYgC10B4PXXX8e0adPg6+uLWbNm4dVXX8WGDRvKdd3yhN25t1JwYZnpZzub6uYCqTKHkFvO2GF3dcegm4jKzdCw2xxD7ujRg3BqwUe4dzEW0qcfxLQpflSAm7/+hOjRgxA9ehAOvDcYlzZHID9N99Bfn7IVLb8gB2+93xDHzkYZ3Maxs1F46/2GyC/I0aueOYXccgy7iYiIiKimcXJygkgkQkZGhtLxjIwMuLq6aqwnFArRvHlz+Pr6Yvr06XjzzTcVGwg6OTnBwsICrVu3VqrTqlUrJCUllbvPhoTd8pDb3t181qrWl6luLlRX5hRyyxkz7K7uGHQTkVHoG3abY8gtl3vrOmJXLcSBsW+iIKPsTSJSTx9FzMQRuB71veKYtKQEKcdjcPzjSYhdtRCSEs2bLUpKihG3YSWOfzzJaP0vr0tXjwMAvt+7yOA2dkYtBABcTjiucx1zDLnlGHYTERERUU1iZWUFPz8/xMTEKI5JJBLExMSgc+fOOrcjkUhQWFioaLNjx45ISEhQKnPt2jU0adLEKP3WJ+wuHXL7Th9mlOuXR1W8uVDdVFbIbch40Fhhd3XHoJuIjEbXsNucQ+7Oc5fBd/wMCC2tAADHZk/A4/v31JZNPXUEF79ZBQCo59MWXReuQffFXyFw9kL4DBkNALh3MRbn14SrnR0ulYhxbvnnSD+rexhc0aRSKfb+sRoAcP9BKtIyb+vdRlrGLWTnyG4QRP2xBrrseWzOIbecscNuIiIiIiJzFhYWho0bN2Lbtm2Ij4/HuHHjUFBQgNBQ2azQkSNHYvbs2Yry4eHh+Ouvv3Dr1i3Ex8djxYoV2L59O4YPfxauzZgxA5GRkdi4cSNu3LiBdevW4ZdffsH48eON1m9dwu7nQ24Lm1pGu76hqurNheqiMmdyGzr5yRhhd3XHoJuIjEpb2G3OITcAOHp6wdW/M4LWbYets+yuedxXy1TKFT8qwMWNskDYe/BIdJzxOewbNoKtsyvqtmgFzz6vossXXwIAsi7HIf3cSZU2Uk8fw4Nr/wEAOn+6pKLeks4KHuXi8KkfcTv5MgBAKBBh189LcP9BquKRt7JIJBLcf5CKH35eAqFABAC4nXQJh0/9iIJHuRrrVYWQW86YYTcRERERkTkLCQnB8uXLMXfuXPj6+iIuLg7R0dGKNaSTkpKQlvbsCdiCggKMHz8ebdq0QZcuXbBnzx7s2LEDY8eOVZQZOHAgNmzYgKVLl6Jdu3bYtGkT9uzZg65duxq172WF3eYYcstV1ZsLVV1lL1dSnid9GXaXTSDVZapdNZKXlwdHR0fk5ubCwcHB1N0hMltB2z4rV/2Sx4WIW7ET+SmZeGHGCDg2c9c55P57VPmuXR6vbNmreF2Ym4ND02Qzs3uu2AjruvUV5xL//AVXd21Bw07d0OG9aRrby4g9jQsRSwEAfTcrr3kdPXoQAKDTJ0tQp1kL/BE60GjvQ1/2dnUUYbRQKIJEIlY6b2lRC85OHvBw88E7gz9H/bpuAICs7BRs3T0Pd1MTkJmVjOKSQqV6pduys3VEQ+dmaNPyJQwdOAdCoexea+jkliYLuaVF/8K+lq3e11x9YgeWHd2CGd1DDXr8K7/wEbxXvKp3PSIiIiKi6qy849DnPT8GLSvkNuU4tLR169Zh2bJlSE9Ph6+vL9asWYPAwEAAQM+ePeHp6YmtW7cCAD755BNERkbi7t27sLGxgY+PD6ZMmYKQkBClNjdv3ozw8HDcvXsX3t7e+Pzzz/H6669X9lszK9dmJwMwzZrc0qJ/MTxyFhKyEvH920vxgpuP3te8kHoVQ3fNhLeTJ3aELNZ5XOu+6GW9r2Vq+mS5FpXUJyIywJDvyv4HSFwswe6xR9DpvVbw7KJ5za6yJJ5Ix+lv4vHWph4QWZZ+yKN8s63lM7vjVuzEhWXbUdfHE1kXEsx2Jrc6tRzrwDO4PxIP7Ef6P6fh2aef4tztP/YBANy79CqzDRe/TorXT3KyYV2nnuz1g2zFccemzY3Ya8OUnnH9fMgNAMUlhUjNuInX+oxThNwA4FTPHS+06YWzF35Xu0RJ6bYKHuXi3v1k9OrytiLkBmDSmdzDI2fp9aFATh5uy3et1jfsNiRcJyIiIiIi/cjHnreiDiHvdioeXE00y5ncpU2cOBETJ05Ue06+yaTcggULsGDBAq1tjh49GqNHjzZG96oVU208KX/Sd3jkLAzdNdOgsFs+s3vorpkGj2urIy5dQkQVRh52ix8XIutCApxe8K4yIbecpX1tAEDxQ+WlNwpzHwAArOvUV6mjSXH+w1Kv8xSvBQJBebpoFHUdXSASqr/3KRAIIBAIMOGd1ejddajK+d5dh2HCqFWKcuqIhBao6+iC+TP3w91VOdg3VcidX5hvlF2rDV3GhIiIiIiIKl7T/t3h9II3si4kQPy40KxDbqo8pgq55Z7fw4nLmBgHZ3QTUYVK/uuM4vWDq4nIvZUCx2buJuyRfgqfzrwWWlmpPS+ysdG5LfkGl2W1ZyrzZ+7H7PBX8DA/W+WcVCrFxNA16NHpLY31e3QeDAgEWLdFfeBsZ+eIBTN/gbOTh8o5U4XcY38cUe474OWd2U1ERERERBUr91YKHlxNVHyf/NeZKjcBq7r66Zt0xWuxuAQrv3kPZ+P+UFs2wPcVhL33DUQizVFmRlaSxnEtAKVx7SIThtxynNltfJzRTaSHiIgIeHp6wtraGoGBgTh79qzGslFRUfD390edOnVgZ2cHX19fbN++XWP5Dz74AAKBAKtWraqAnptG6fXQeqyfpXGDSnOWdFD2R7aeTzul4w06+AMA0s8eL7N+Ud6zmeC2DZwVr22cXBSvH2Vllruf5eXi1BjWtew0nvds1EZrG03cW2s8Z13LTm3IbQhjhdzXs64Z5Q44Z3YTEREREZmn0mty91g/S+MGlWR6IpEFGjVsCZHIUs05S3i4eZcZcgP6jWtNHXLLcWa3cTHoJtJRZGQkwsLCMG/ePJw/fx4dOnRAcHAwMjPVh5T16tXDxx9/jFOnTuHixYsIDQ1FaGgoDhw4oFJ27969OH36NNzc3NS0VDU9v+mHfBmTqhR234n5XfG6jldLpXPeg0cBABJ+/A45t66rrV9S+ARnl38GAGjS51UIhCLFOaFIhKavDAAAnPpiBsSFhWpaqDyFRY+RlS37byIQyP40lF7KJEmHP7bJpcrI6wqfvud79++iqPhJuftpzJB7S8hOAMb5UMCwm4iIiIjIvKjbeLJp/+4Mu81YcupVSCQlAGTjUvnYVCIp0WlMqs+41hxCbjmG3cbDoJtIRytXrsS7776L0NBQtG7dGhs2bICtrS02b96stnzPnj0xcOBAtGrVCl5eXpgyZQrat2+P48eVZwCnpKRg0qRJ2LlzJywtVe9cVkXPh9xyVSXslhQX48p3XyN+5yYAwIuT56isPW3f0B0NA7sBAE4v+AjZ1/6DtNTGi4+yMnFkxvvIv3sHAOD1muqyH037DgAgW7v71MJZFfFWdJaafgNSqQQA4OrcFONHfgl/32DF+eQU7X9oS39o6OjbF+NHfgmXBp4AAKlUgpT0G+Xqo7FD7tIfahh2ExERERFVH+pCbjmG3ebrdvJlSKVS1LKywVuvTsdbr05HLSsbSKVSJCZf1lpfn3GtqULur06sUXucYbdxcI1uIh0UFRUhNjYWs2fPVhwTCoUICgrCqVOntNaXSqU4ePAgEhISsGTJEsVxiUSCESNGYMaMGWjTRvvSEFWBppBbTh52x63YiQvLtuOFGSPMas3u6NGDlL5vGzoBzr7+asu2GzMRTx7cx4Nr/+Hs4k80ttl14VpYPd3UsjSr2g7ounAtjn88SRGIm0pS6lU0qO+BkNc+RNfANyASitCry9u4eedf/LAvHEmpCVrbSE5JQIfWPTFkwCx4NekAAOje+S0cP7MHkb8sR3LKVTT1aGtQ/yoy5JYzxtpmXLObiIiIiMi0ygq55eRj1VtRh5S+J9N5UvgID/Oz8VqfcRjYdyJq29cDAAT3HIV90RH488hWFBY9Qi0rzWM0Y4xrNTFWyL362HLM6PqK2vNcs7v8GHQT6SArKwtisRguLi5Kx11cXHD1qua7bLm5uXB3d0dhYSFEIhG++uor9OnTR3F+yZIlsLCwwOTJ+gd35khbyC1n7mE3ANRp7o3Ww9+DQ+OmGssILSwRMPMLpJ05houb1gJP7xzLNX1lIJq+MkBtyC1n39AdL6/Zhlu/Rxmt74Zo2cwfq784DksL5U0yvZp0wCdTdiFZhw8EwwZ9DA83b6VjIqEIPToPxkv+r+N+TppBfauMkFuOYTcRERERUdWlS8gt93zYjVGV0UPSRCwpweovTqB+3YZKxx3s62Pkm3PRr/e7KBGXQPN/UeOMa9UxZsg9pduHZZZj2F0+DLqJKlDt2rURFxeH/Px8xMTEICwsDM2aNUPPnj0RGxuL1atX4/z58yrLYlRFuobccuYadnedvxrW9erDwka3PwICoRBunXvArXMPPMnJRnH+QwgtrWDj5AyhSKS9AQBW9rXhM9i0n6oaOmsO9AGoBNj6lrG0rAXXp8uY6KMyQ245ht1EREREVJMN+e5lvcof/fIiAKD7tPYGXU9cLMHusUfQ6b1WsHtpsEFtAPqF3HIqYTeZjJ2NA+xsHDSefz4AV8cY49rnGTvklo1r1e/zJcew23AMuol04OTkBJFIhIyMDKXjGRkZcHV11VhPKBSiefPmAABfX1/Ex8cjPDwcPXv2xLFjx5CZmYnGjRsryovFYkyfPh2rVq1CYmKi4gODJpIS2Qzi09/EI+ms+k0xtUmNuw8AOLbqIoQWz5btbzVV90e39A255cwx7LZ39zC4rnWderCuU8+IvanZTBFyyzHsJiIiIqKaSts49HnyMaW+9eRKj2t7v2RQEwaF3HLmsGzJK1v2muzaf4QONNm1zV3FhNy6qaiwu7rjZpREOrCysoKfnx9iYmIUxyQSCWJiYtC5c2ed25FIJCgsLAQAjBgxAhcvXkRcXJziy83NDTNmzMCBAweM/h4qiqEht1xV2aCSKpcpQ245blBJRERERFS5DBkPlifkljOHsJvMiylDbrmK2KCyuuOMbiIdhYWFYdSoUfD390dAQABWrVqFgoIChIaGAgBGjhwJd3d3hIeHAwDCw8Ph7+8PLy8vFBYW4vfff8f27duxfv16AED9+vVRv359pWtYWlrC1dUV3t6yx2m0PfpV+hEvzy6aZ5aXJfFEOk5/E49uU9tDZPns3tc9qfa65Q255Z6f2Y3PNxrcVkWRiMV4nJUBSVERLO0dYF1X/5nbJY8f4Um2bLaDPsujmKOi4icAACtLa4PbKHicB1tr1fXLzSHkluPMbiIiIiKqafRdgsSYS5ec0/NJX2OE3FXJkwfZKM7Pg9DKCjZOLjovlylnjHGtMUmlUoOXcpVKpXj05GGZy50YyhxCbjljz+yu7hh0E+koJCQE9+7dw9y5c5Geng5fX19ER0crNqhMSkqCUPgsKC4oKMD48eNx9+5d2NjYwMfHBzt27EBISIip3oJRGSvklisddpuTood5uB29D7f/2Kdyrt3YyXDr1B0CYdkPx+Ql3caV7V8j9+Y1peMOTbzQNnR8mRtemqthE2V93v21YRtMpt9LxKRPOqNbwCBsenmV4rg5hdxyDLuJiIiIiCqH/ElfXcLumhJySyUSpJ09gYvffKlyrklQP3i99hasapcd9pY1rkWoDrPcKsjazRMxeUyEQXXXbJ6A42f3Yt2C03Bp0MRofTKnkFvOmGF3dcelS4j0MHHiRNy5cweFhYU4c+YMAgMDFecOHz6MrVu3Kr5fsGABrl+/jsePHyM7OxsnT57UGnInJiZi6tSpFdR7/ZQ8LtR4ztght5w87DYX+Wl3cXDKO+o/DAC4tGkNzi6dC0lJscY27h6LwcnPpquE3ACQd+cmTn42HXePxaipab5SM24qXqdl3DKojdOxvwIALsYfVRwzx5BbjsuYEBERERFVPF2XtawpIbekpBixqxaoDbkB4M7fv+HglHeQn6b5Z6VtXGtKF+OPQirVP2iXSqWKseTp878arz+VFHIbMh401jIm1R2DbiJSK27FTrVhd0WF3HLm8gGl6GEejn8s+6Nkae+A7ks3oO/mKPTdHIXgb/eg0ydLAAAPrv2HS9+uU9tGZtw5XN4iuzvt0SsY//s6UtHG/76JhPfgUQCAy1sikBn3TyW8K+OI+mMNhEIRhEIRov5Yo3f9J4WPsO+A7OeS+zALgHmH3HLGDruJiIiIiEiZLns4VceQW9N48MK6pci6HAcAeGHCTMV4su/mKLy8agvsG8lmMh//eBKK8h+q1Nd1XGsquQ+zcOHyQb3rnb8cg7yHsmVB90avwxMDxmbPq8yZ3IZOfjJG2F3dMegmIrXyUzJVwu6KDrnNyc1fdgMA7Bs1Qc9lX8PWyVlxTiAQoE6zFooPBWlnjqEgPVWpvlQqxfk1svXaWw0bizYj3ofQ0lJxXmhhiaZ9X0ezfoMAAOfXLKrQ96PNlYSTuP8gTe3d9ILHebiZGIfjZ/fix1+W4+jpnyCRiCGRiHHk9G78+MtyHD+7Fzfv/ItHj1U/XEmlUtx/kIYrCSfx97EdiNg6BY8e5wIAhEJRlQi55YwZdhMRERERkaqywu7qGHIDsvHg82Ox/LQU3LsYCwDo8sWXcPHrpHTeysERnT4Oh6W9bN8jdTO2dRnXmpJQKMLW3fPw699fI/biX0hJv4HikiKVcsUlRUhJv4HYi3/h17+/xrbdn0EolK1P/uhxLiK2TsHfx3boNa4trbKXKynPk74Mu8vGNbqJSK0XZozAhWXbEbdiJ3ynD0PyX2dqTMgtEYtx5+/fAAABH34GUS31H57qNGsB77dGImH3d7j641b4TZ6jOJdzM0Hxuknv/9N4rRYDh+LWb1FG6rnhPlv5BgCgs19/TB27XrHe/IlzP2PVpg8U5URC1T8bUb+vhlhSAkD2YWnq2A14yb8/AEAikWDVpnE4Fbv/WRsiS8UHD4lEbNKQ+0Lq1XJt5FHeNbuJiIiIiEhV6T2c5Gt2A6iWIbdczs1rqNvcW/G9PLj2GTIatRupX4PaopY1Os9dhqMzP8DtP/ai5RtDIXgaAOs6rjUliUSMzHt38N1PX0AqlQAA6td1w4KZ++FUT7ZGe1Z2Cj5e8hqyc2T7QwkEQggFQkgkYgCyiVXn/j2gtISJLuPaBW2nADDNmtzl3cPJGGt2V1ec0U1Eajk2c8cLM0YgPyUTR8YtrjEhNwA8zspQvLZycCyzrGtgVwDAveceNcu+ehkA0LiMkBsABEIhvN8aaUg3ja5jh2BMGr1WaVPVLh1fx7iRKyEQCCAQCCCWlCg+gACAVCqBWFKiOD9uxEpFyA0AQqEQk0LXwL9DsOKYWKy8prkpZ3KXZ22z8s7sJiIiIiIizUrP7P7ni03454tN1TbkBoDsq5eUvk89fQQA4OjpVWa90rO0H93LVLzWZ1xrSvIxplAogqNDA8yd9qMi5AYAp3rumDvtRzjWdoJQKFKMQZXaKDXG1HVcC5h248ny7uHEmd3qMegmIo0cm7mjro+n4nuPPoGaC1eQiIgIeHp6wtraGoGBgTh79qzGslFRUfD390edOnVgZ2cHX19fbN++XXG+uLgYH330Edq1awc7Ozu4ublh5MiRSE1VXnZEUqT6qJQmJY/VB5zip23UqlNXexuFT3S+XkXp7Pcawt7fCEsLK5VzL3cZgonvyNbiFggEKuflxya+swa9urytct7Sshamv78RnV58VW19U4XcF1Pjyr2RB8NuIiIiIqKKY2FTC80G9VR832xQz2oZcgOq41BpiSyMreWofUypaKP4WRv6jGtNTSS0gKNDAyyYuR9uLqrBvrtrcyz46Bc4OjRQ+5QxIBuX6jOuNWXILcew2/gYdBORRrf3H0XWhQQ4veANkU0tjRtUVpTIyEiEhYVh3rx5OH/+PDp06IDg4GBkZmaqLV+vXj18/PHHOHXqFC5evIjQ0FCEhobiwIEDAIBHjx7h/Pnz+PTTT3H+/HlERUUhISEB/fv3V2rH0t5B5z4WPsgGANSqU0/puFVtWRvFajYEeV5BuuYdsivLlDFfwUJkqfF8905vYmBfzX+4B70yBd07vanxvIXIElPHrkdD52bl6idgvJA7NHKYUXatZthNRERERFQxcm+l4OLaH2Hv4QJ7DxdcXPuj2g0qqwPL2upnXUvEJWqPy5Vej1q+Xrfste7jWlMTS0rw8aSdcG3gqbGMawNPfDxpp8psbjk3Fy+9xrWmDrnlGHYbF4NuIlKr9MaTHaa8rVjGpDLD7pUrV+Ldd99FaGgoWrdujQ0bNsDW1habN29WW75nz54YOHAgWrVqBS8vL0yZMgXt27fH8ePHAQCOjo7466+/MHjwYHh7e6NTp05Yt24dYmNjkZSUpGjHuu6z0Doj9nSZfUw5cQgA0LTv60rHXZ9uFJJ4YD8Kc3M01pcUFyP97Ikyr1EZRCLtWzY8fpIPoZq750KhBR490R7oi0QWeJifbVD/5IwZcrdwammUDwUMu4mIiIiIjK/0xpN+c0LhNydU7QaV1YWrv/Jmky3eGAYA+G/7N2o3V5TLvXVd8dq61AQsfca15kCXMeWjx3kaz+XlZ+s1rjWHkFuOYbfxMOgmIrWeX5O79JrdlRF2FxUVITY2FkFBQYpjQqEQQUFBOHXqlNb6UqkUMTExSEhIQPfumtcVz83Nle02XaeO0vH2700DAFyIWIqHd++obf/2H/uQduYYAKBRt95K563rOcGxaXMAwJnwOZAUF6u0UfyoAKfmz1SUL62yl2zJy7+vsX25Oyn/QSwuhkhooVjbTCS0gFhcjKS78Vrr5z7MwsOCBwCebiDydJMUXRk75N40WPYzYthNRERERObMFMs5mlrpkFu+JnfpNbsrOuyu7J+5dT0nWNetr9Ru4159Acj2f0qM/lnttXNuXcfphbMAAO3fnaJyXpdxrakJBLJoMjlF+zgsKTVBqY5QKFK8fpifrde41lQhd35hvtrjDLuNQ/utDqIabsVXvUx3cXvV9Ywri7qNJ+Vh94Vl2xG3YmeFbgKSlZUFsVgMFxcXpeMuLi64elXzP9i5ublwd3dHYWEhRCIRvvrqK/Tp00dt2SdPnuCjjz7CkCFD4OCg/FhXw4AuSD15CFmX43Bi7jS0GDQUDdr7wcq+NkoKn+DKd1/jQcIVAED796bCwsZWpX3f8TNwZMb7eJSZjr8mDEO70Amo4+UNoYUFCjLScG7ZPEXZwNkLFa/lS7Zs2LABgYGBWLVqFYKDg5GQkABnZ2eV68iXbPHx8YGVlRV+/fVXhIaGwtnZGcHBwUpLtnTo0AEPHjzAlClT0L9/f/zzj2wTzeTUBLRp+ZLGnysAJD394PFiu94YMmA2AOCHfeE49+8BJKVoD7rvpl4DAFhZWuO1/43Da0Hv42L8Ua31gIoJuUt/qDHGrtXysHvorpkYHjkLO0IWw76W6u8FEREREZGuTDE2MDV1IbecPOyOW7ETF5ZtxwszRsCxmXsZrenPFD/z0uNBOUtbO7R/byoufrMKCbu/Q86ta/DoGQy7hrL3m37mOBJ2fwcAqNuyNRoGdlNpQ5dxLUYPMuaPTy9h732DnVELkZF1RxFil0Uehjs7NcawgXPQvlV3/PLXBvzy1wYUFT/Ra1xrqpB77I8j8MvIpWrPT+kyHACw7OgWpe91ZYxxbVUnkJrD7ZtKlJeXB0dHR+Tm5qoEW0TqmDLo/kdL0C0ulmD32CPo9F4reHZxNegaiSfScfqbeLy1qQdEls8e8rgnLWMWdBkfPMor91YKzn2+EampqXB3d8fJkyfRuXNnxfmZM2fiyJEjOHPmjNr6EokEt27dQn5+PmJiYjB//nzs27cPPXv2VCpXXFyMN954A3fv3sXhw4cV/x68smXvs7ZKinEhYhnu/av5A1/796bCrZPmn1VBRhqOzZ6g8bzQwhJdF6yGrbMr/ggdCAAIDAxEx44dsW7dOsV78vDwwKRJkzBr1iyNbZX24osvol+/fpg/f77a8+fOnUNAQADu3LmDxo0bY9zIlejddajG9nLzsrD62/EYMmA2WjR9Qenc9dvn8f3ecEwdux6ODk4aWgBiju9EcmoCBvadrFSu/W3V2e6lVWTIbSe4rlQuv/ARhkfOQkJWosEfCi6kXsXQXTPh7eRZZtjtvuhlvdsmIiIioprFFGODId/p9zn16JcXAQDdp7XXq55c6XFtiWsXncaaJY8LEbdiJ/JTMo0Wdv896jMApvmZlx6HPi/11BFc3Lha4/mGnbqh3eiJEFqoX5ta27jWlLHgT9+kQywuweFTP+K/66cwKXRtmeXXbJ6Itt5d0KPTW0rLlOTmZSHqj9Vo7N5K53HtG2irV1+NFXJfz7qGq2H7yiy7+sQOLDu6BTO6h+oddsuupXlcWxXHofpkuVy6hIj0VlHLmMgDdABwcnKCSCRCRkaGUpmMjAy4umoO9YVCIZo3bw5fX19Mnz4db775JsLDw5XKFBcXY/Dgwbhz5w7++usvjf9QCi0s4TdlDrouXAuPnsFK51q+MRxBETvKDLkBwM6lIYI3/QS/aZ+gjpe34riVQx34Tf0EfTb8AFvnZ+/HVEu29HwppMx27e3rYu60H1VCbgBo0fRFzAvbDXv7sncD7/nS23hn8BdlhuHPq+iZ3M/jMiZEREREZC5MvZyjKeg6oaqiljExx5+5W+ce6L1uO1oNGwsLWzvF8Qa+/ui2aB06vDdNY8gNaB/XmppIZIHeXYdi/KhVWstOeGc1Xu4yRGUtbkcHJ4SGzC/XuLYsxgy5t4Ts1Fqey5gYjkuXEGlRUiwx2bXFxWXP6JaUyPomLpFAbGA/xU/bkLeloOVfB2MvY1J6ljgAWFlZwc/PDzExMRgwYICsjxIJYmJiMHHiRJ3blUgkKCx8FsTLQ+7r16/j0KFDqF+/fhm1ZewbuqPNyPfRZuT7+r2ppwRCIRq0exEN2r2otayplmwRaVkvW9t5Y7VRWmWH3HJcxoSIiIiIzIGpxgb6ji0V41IDx6Slx7X6PDVcEcuYmHoJTU0sbe3QpPf/oUnv/9P9zTynvOPailYZY059x6SA8UNu2bj2urZqXMbEQAy6qcqJiIjAsmXLkJ6ejg4dOmDt2rUICAhQWzYqKgqLFi3CjRs3UFxcjBYtWmD69OkYMWKEUpkNGzYgNjYW2dnZuHDhAnx9fRXnZ03VbQ1hUzq3OQHnNmtfz6osez44pvR97609tdYxVtj9/FIocmFhYRg1ahT8/f0REBCAVatWoaCgAKGhoQCAkSNHwt3dXTFjOzw8HP7+/vDy8kJhYSF+//13bN++HevXrwcgC7nffPNNnD9/Hr/++ivEYjHS09MByNZVs7Ky0rvv5qJ27dqIi4tTLNkSFhaGZs2aqV2yZfDgwZBKpYqfizkyVcgtx7CbiIiIiEqr7HFoeZR3bLB77BGDrmtoPblzmxPQY/0svcaUxgq7y/uUcnUZj1Wl3/PKUDEht+4qIux2R9VbukQfDLqpSjH2phAAUFBQgK5du2Lw4MF49913VdoYMsp0d7xu19KyRneJBOc2J8C1XT14dnYps6wmiacykH4pGx1He0Nkof9qRuUNu8ta7zskJAT37t3D3LlzkZ6eDl9fX0RHRyvuriclJUEofNbngoICjB8/Hnfv3oWNjQ18fHywY8cOhITIHl9KSUnB/v37AUDlj+uhQ4dUPoSYQnmXbAFk7y0+Ph7h4eFK76n0ki0HDx40230KTB1yyzHsJiIiIiLANONQwHRjg07vtdL5ZwMAp7+JN6ieXOlxrSETp8obdsvX+8YH4TV6PGaq33NzZeqQW87YYXfe1vEG9aOq4GaUVKVU5KYQiYmJaNq0qcodRm5GqRtDNqjUVEe+CYgpNH89BHaubnDx6wSRpe4zvJ9kZyE99jSKHuZBZGWFej5tUcfLGwJB2f8N5aRSKaKf7nYdGBiIgIAArF0r24hDIpGgcePGmDhxos6/56NHj8atW7dw+PBhAKpLtjRo0ECp/E/fpCv15YefF+PVoPfgYK99aRd18vLv49e/v8HQAbO1li29GWVlh9zPb0apTkVtUFkVNwEhIiIiqolMMQ4tfe3KHhuYcjNKu5cGG9QGYNgGlaXrlDx6AsA0P/O+m6OQc+MqshOuQFxUBKvaDnD16wTrerrvbyQuLkJG7GkUpKcCgM7j2j9CByred2X/npcehz7vn3//hEAggF979UvAaKNtXFt6HPq8ig65dRmHPs9YG1Seu3tZ77qmpk+WyxndVGXIN4WYPftZcKbvphAHDx5EQkIClixZUpFdrZH0ndltSDBeGW7u/1Hx2qNXMFoNHQuhSPM6Xo/v30PcV8uQe/uG2vMvTp4DZ1//Mq+ZGfcPzq9ZBIyW3Xc09ZItV2+cwd4/1uDkP/uxboH2/7fUmRPeDxlZd/BC25fRqnmgTnXMZSb38ypqZjcRERERmT9Tj0NNPTaoSvSd2f18MC5nip/5gTFvqPTv6g+bYV23PgLnLIJN/QYq5+UkYjFu7NuFW7/tUXtel3GtqX/P1bW35KtRAIAfN6TqPIGsNEPHtZUxkzu/8JHeT/oaa2Z3dcegm6qMytgUgspH17DbXENuAGj6ykDc/+9f5N25heRDB5CfkoyOMz5X+6GgICMNx2ZPUHzvGdwflva1UZiTjaSYPwAA59csQtvQCWjUrbfa6909FoPLWyKUjpl6yZaY498DAHLyMvH4SQFsrO2gj8dPCpCTl6loS5eg21xDbrmKCLvPrXjVaP0jIiIiooph6nGoqccGVY2uYXdZs79N+TP3fmskSgqfoCA9BelnT+DJg/s4MuN9dAuPgJ1LQ5X3IRGLcW7ZPDy49h8AoH6r9nBq54uihw91HtcCpv89f975S38/e305Bn7tgvRuw5BxbWUtV2LospbGCLurOwbdVO3puikEGYe2sNucQ24A8H5rBIARyE+7i+MfT8aDa//hxr5daPnGMKVyUolEEXLbOjdE4OyFqOVYR3G+9bB3cSfmd8Tv3ITLWyLg0KQZHBo3VWojL+m2IuRu9uqbSucmTpyIiRMnqu2j/PE3uQULFmDBggUa35Onpyd0XaUqLfM2jp6RzQQoKnqMP49sw+vB+q3h9eeRrSgqlj32d/T0T3iz3zS4NvDUWN7cQ245Y4fdRERERFR9GXMcaqqxQVWlLezWZYmTyv6ZN3v1TbQY8DYEpQJ0yZjJODV/Jh7evYMTn05Fnw0/KJ0HgPjvNylC7o4zPkf9Vu1KndVtXFseFZG3SKVSRO5fCqFQFspH7l+KF9v21mtWtyHj2spckzshK9FkYXd1x6CbqoyK3BSCjEtT2G3uIXdp9g0boeOMz3Fu2Tzc+m0PvPq/pbS2WdblCwAAoaUVus5fBaGlpUobTXr/H/JTkpF8+AD+2/ENOs0JVzp/ectXAADvkHfQNLh/Bb4b7TZ+Pwup6Tdx5+4VCAVCiKUSxZpmp2L3w71hCzR0boqGzs3Q1rsrHB1ka8Xl5mXhcsJxpGXeQlrmbaSkXUfi3f8UH+SEAiHmhP8fmjRqAzdXL1kbLl54oc3LilkQVSHkljNm2E1ERERE5o/j0KpJU9htyDrelaHloKEqx4SWlgiYtQAxE0dAUlKMrMtxaND+RcV5cVEhkg8dAAB0XbgG9g0bqbSh27j2DZP+nicmX8HNO3FIzbiFtMxbSE5NQHrmbcX520mXMGVuVzRya4mGzs3g5tIMXk184enRBoBsLfELVw4iLeMm0jJv6zyube/UT3GNyt548vllLRl2G49QexEi82BlZQU/Pz/ExMQojkkkEsTExKBz5846tyORSFBYWFgRXaRS5GF3fkom4lbsRPZ/t6pMyC1Xv1U71Ht6Rzwj9ozSuRtP1/Ju/+4UtSG3XKshowEAOTcSUPL4keJ4yeNHyLtzEwDQpPcrRu23If48sg2XE47jYcEDiCUliuNicTFu3vkXx87sQeT+ZTh2di9sbWorztva1Maxs3tl587swc07/0Isfraph1hSgocFD3A54Tj+PLIN3/30OR7kpCs96mfKkHv1iR16X1Medns7eWLorpm4kKr5UT5NDNnQkoiIiIgqH8ehVZc87LZ3d8aFZduR/d8tswy5y2Jpa4eOMz4HAFzavFbpXMZ52RjVoUkztSG3nC7jWlP+nltYWGLXz0ux/8+v8M+/B5RCbrm0zFv4598D2P/nV9j181JYlgrrhUIhsh+kYdvuz/Qa18pVdsgNPJv8JJ/ZnV/4SGud503pMhwzuodi2dEtBo1rqysG3VSlhIWFYePGjdi2bRvi4+Mxbtw4lU0hSm+eEB4ejr/++gu3bt1CfHw8VqxYge3bt2P48Gd3u7KzsxEXF4f//pM97pOQkIC4uDjF5hBkOHnYnXsjGReWboeNU50qE3LLNWj3AgDgUUaq0vHcW7Jdkm2cnMusXzoEf5J9X+1roYXmoLyyCAQCrY+CBfi+gunvb4Sl5bP/fpaWtTD9/Y0I8C07rJe3P27klwjqpny32ZQzuQ39UGCMsJuIiIiIqgaOQ6suedht41QHF5ZuR+6N5CoTcsvZOstmVBfl5SodL0iXjVHrt+6gtQ1t41rAdL/njRq2xBcf7kUdB2cIBZo3zBQKRKjj4IwvPtwLd9cWSuf6dB+B8SO/1GtcC5gm5JZj2F0xuHQJVSnG3hQCAPbv36/4hxsA3n77bQDAvHnz8Nlnn1XOGyOzVfQwr8zzVrUdKqknFWtS6Fqs3TJJ7TmBQIDOfv0xafRaWIhUQ3lLCytMe+9rrPl2Ik6f/0Xj+nOTQteiW6DqbuKmCrm/OrFG8aEAMHzX6vIsY0JERERE5o/jUDKl5wPu51nVrl3meUD7uBYw7e95Q5dmWDBzP+YuH4icvExIJGKlvgmFIjg6NMAXM/bBxamx2v736vI2LCwsdR7XfnVihclCbrnSezhxGRPj4IxuqnImTpyIO3fuoLCwEGfOnEFgYKDi3OHDh7F161bF9wsWLMD169fx+PFjZGdn4+TJk0r/6ALAO++8A6lUqvLFDxflJ1+T27G5B16YOQKPs3IQt2InSh5XnUf27l/5FwBg6+KmdNzKwREA8Ciz7BkXkpJny3hY16uv9rWkuBim1i3wDTR2b6X2nFQqxdCBc9SG3HIWIksMGzhHY8jd2L212pDbEMYKuVcfW17uO+Cc2U1ERERUM3AcWjXJ1+R+nJWDF2aOgGNzD1xYth25t1JM3TWdPb6fCQBw9GqpdNzOVTZGzboUp7UNbeNaOVP+nrs0aIJXeo1W23+pVIr/e3mMxpBbTp9xralDbjnO7DYuzugmogqhbuNJdRtUqiMQCND76zVlrn0NyALimPfV/1Fq+FIg2o59R2s/8xLv4MwXixXf990cpXidn3oXeUmy9cFc/AKV6rUbPRGxqxbi3LJ56L1uOyxt7dS2f+fv3wEAjk2bw8Lm2Z1ZCxtbOHq1RO7Na4jftRltRryvta8VSSKRIDX9puJ7oVCkdBf9bto1rR8q7qZdU/q+dBupGTchkUiUZgAYwqghd7cPAZT/DjhndhMRERERmR91G086NHVX2aDSnEmKi3Fx4xoAQPPXBiudc/HrBAC4H38R9+Mvof7Tdbifp21ca07upl2DALKlR4RC2TImEokYQoEQyc+NN9XRZ1xrDiG3HGd2Gw9ndBOR0akLuQHVDSo1zezuFfGl1pAbkK1/3StiJezdle9Ktxg8CG3GjNKprw6eTRDw8UyV4/f/u4jjn8j+6Hn0ClbamRoAnNq+AIGF7F7h2cWfqMzKlkokuLZnBxJ+3AYAaDNqnMo15OF28qEDuBPzu079rSiZ95NQXFKoWM+sQ+sesK5lB4FA9mciOUX7TOWkp7OZBQIhbKzt0b5V96ffC1Bc/AT37ieXq4/GDrlLf6jhzG4iIiIioupDXcgNqG5QaS4zu9WNBwtzc3D80ymQFBcBAJza+iqdF1laoVk/2VOz55bNQ37qXZU2dBnXmpPEu1cUm0i6uXjBzcULACCWlCAx+YrW+vqMa00Vcl9MjVN7nDO7jYMzuonIqDSF3HLysLusmd0WNtY6X8/Cxgad538qewRKIoFQpHnzCk0cvZqiz+b1kEokuH1gP+5djEV2/CUAQN2WrdFq6FiVOgKhEF3nr8ax2RPw8O4d/Pl+CFwDusDRszkglSJh93eKsm1DJ8ChcVOVNhwaN0Xb0Am4vCUC8Ts3ATs26t13Y0lOTQAA+LbphaED5sDTow3yC3Lw858R+O3vjYoQu8w2Uq7C0tIar/Z+F/3/Nx72dnVwO/kyftgXjguXDyI5NQEuDZoY1L+KDLnlOLObiIiIiKjq0xRyy8nDbnOa2R2/cxPid25C0/8bBKvaDijMyUbigf2K890XfwWBmqdjmw94Gw+ux+PBtf9w/JPJsGvoDveXekFgYaHzuNZciCVipGbcQoP6jTBkwGx08R8AADjxzz78sC8caRk3IZaIIRJqHvMbY1yribFC7tDIYbgatk/tec7sLj8G3URkNNpCbjldwm59CQQCCAwIuZXaEAqRELlV8X2zfm+g+YC3NYbndi4N0WPZ1zizaA6ePLiP9LMnkH72hFKZFyfPgbOvv8ZrNurWG1a1HXF+zaJy9b28LC2sMH/Gz/BpHqA4Zm9XB8MGfoz/e3kszl74Q2sb3l4dMeLNeajr6Kw41tSjLeZM2on4G2dQaMAdaaByQm45ht1EREREZA5WfNXLdBe3F5ju2uWkLeSWUxd2/2/Lhkrurarbv0cpfe/YtAV8x38Im/oN1JYXikToOONzxH+/CcmHDqAgLQXX9ijP5NU2rjUX+fkPEDr4c/TqMkRpf6huAYPQ2e81HDrxA/LzH8DRwUljG8YY16rtmxFD7hZOLcssx7C7fARSTTuHVVN5eXlwdHREbm4uHBwcTN0dqgJM+QHjHy0fMMTFEuweewSd3msFzy6uBl0j8UQ6Tn8Tj7c29YDI8tnd3HvS7nq1o2vIrUsdocSw92IMt38+DFsXN7j4Bao81lWWJw/uI/2f0yh+mAuhlRXq+bRDHa+WikemtJFKpbATqt80ozIMKaprsmu3v615M86KDrntBNfV1ll9YgeWHd2CGd1DDfpQkF/4CMMjZyEhK1Fj2O2+6GW92yUiIiKimsGcx6HPO/rlRQBA92ntDbpe6XGt3UuDtVfQQNeQW1OdlyNWGXzt8hJKnZFzMwHZVy9DUlQEy9qOcPXvBOu69XVuQ1xchIzYM3iUkQoAOo9r/wgdWK6+l8dP36Sb7NpljUNLM3bIvWnwdrhYp2mtcyH1KobumglvJ0+Dwm5A/bi2Ko5D9clyOaObSIvYs6b7hzexlpagu0QiK3cqw/BrPK2beDIdIotnQbfdS7q3YUjIDVTMzO7yav56iPZCaljXrQ/PPv0Mvq6ugXhNUpkzuZ/Hmd1EREREZErmPA59XmrcfVm9E4b1ufS4to0e49DSDAm5AeWZ3aYkEAhQt7kP6jY3fMwgsrSCW6duRuwVVUTIreu4ljO7DcMZ3URa1NQAsvfWeTqVMzTkLqsNq1qGreNsDCK4mOzatgLTBaHmNqO7skJuTTO65SpyZndVvJNORERERJWjpo5De6yfpfeY0tCQ+/k2auo4lDO61avIkFvbOLQ0Y8/sXnpks971TY0zuomMaPEq/ZbwMKZYLY+MSUok2PPBMXQc7Q3PlwxcuuRkOs5tTsAbG7pBWGpGd7YOdY0RcgOqM7sD5swxqB2qHkw5k/t5nNlNRERERKZgzuPQ5x1bJVu6pNtUw5YuKT2u1fdJX2OE3IBsZjckBlWlasiUM7mfZ+yZ3UtR9YJufTDoJoNERERg2bJlSE9PR4cOHbB27VoEBASoLbtx40Z89913uHz5MgDAz88PixYtUiqfkZGBjz76CH/++SdycnLQvXt3rF27Fi1atKiU91MWC0vT7UIsstTtA4bIQqi0vrZe13gabgufb0PLsx7GCrnlSofd5iY/LQUJkVtx72IsAKCWY100fWUA3Lu+DEtbO631pRIJsi5fwKXN61CUlwsAcGzWAs37D4ZT2xfMaqdrXX20MBgCgQCL50QbVD+/IAehYa0weUwE2jd4TXHcnEJuOYbdRERERFTZqsI4VE4+YcrQManiuhZC5Kdk6hx2GyvkrgqKHxUg6eAfuB71PQBAYGEBt0490PSVAbBvqNv71jSufcOETxY3OnsI3QIGGVT32NkorPl2ArasjIe9XR2j9cmcQm45Y4bd1R2DbtJbZGQkwsLCsGHDBgQGBmLVqlUIDg5GQkICnJ2dVcofPnwYQ4YMwUsvvQRra2ssWbIE//vf/3DlyhW4u7tDKpViwIABsLS0xM8//wwHBwesXLkSQUFB+O+//2BnZ4fdq4KU2lyy6V8AwEdjOxj0HopLJBg24xAmDmuN7v4Nyy6s+36ENYaxQ245edhtLiQlxbi4aS3Szx5XOl6Y+wBXd23B1V1b0P7dKXDr3ENjGwUZaTj+6RRIS0qUjufeuo7YVQsBAN3CI2DnouX30Iw8fpKPW0kXn74ugI219rD/eZeuyn6m3+9dhAnvyYJucwy55Soi7HYHly4hIiIiMmfPj0NLO/pPGtbt/A87l/WCpYVhAW+Z49oaOg7VdQ+nmhRyp54+iovfrFI6Ji0pQcrxGKQcj0GD9n54YeJMCC0s1dbXNq5tHTyjorqu1fd7FxkcdO+Mko2nLyccR6cXXzVKfyor5L6QelXvyU8Mu3VT9aYRksmtXLkS7777LkJDQ9G6dWts2LABtra22LxZ/eMPO3fuxPjx4+Hr6wsfHx9s2rQJEokEMTExAIDr16/j9OnTWL9+PTp27Ahvb2+sX78ejx8/xg8//FCZb41Kub3/qNrjFRVyy6n7gFKcX4C0U2eRfvYfFBcUGNTuw7spSD1+CvevxEMq0f5MmlQixrnlnys+DHgPHokey79B1/mr4T99Lho+3eTj4sbVSD2t/mf1+P49HJs9QRFyd5zxOXos+xqd5y6D7/gZED7d/frY7Al4fP+eQe/LFP48sk3ta11JpVLs/WM1AOD+A9mO4OYccstN6TIcM7qHYtnRLVh9Yofe9eVht7eTJ4bummn0/hERERERVXXyyU/ymd0ljwtVypgi5JZKJMi6dAWpJ04hPyXNoDbKGtdqGg+mnjqiCLnr+bRF14Vr0H3xVwicvRA+Q0YDAO5djMX5NeFqx7m6jmtN5f6DVKRl3ta7XlrGLWTnyP47RP2xBsbYfrAyZ3IP3TUTF1Kv6t2+POxOyErE8MhZyC98pHcb1R1ndJNeioqKEBsbi9mzZyuOCYVCBAUF4dSpUzq18ejRIxQXF6NevXoAgMJC2R8ua2trpTZr1aqF48ePY+zYsUZ8B6SrW1GHAABN+z9bG66iQ+7nSaVS/LdlO1KPK/9uNXwpEG3GjNJpg5biggIcnvShynG/GVNRr5W3xnqpp4/hwbX/AACdP10Cx6bPltGxd/eAUxtfOHg0RcLu73Dxm1Vw7uAPCxvlu6lnFsnWGq/dqAkCZi1QLHNiU78BHD294NzBH8c/nYJHmemI+2oZes817x2yi4sLkX4vEXuj1ymO7TuwDi+26w3XBp6wtNT++1DwKBdn46JxO1m2lJFQIKoSIbecMWd2ExERERGRquf3cCo99jRFyJ0dn4DYZatUjvdcuxyWdjosZanDuDbuq2Xo/OlSpfPFjwpwcaNsgpD34JHwDH5dMQa2dXZF3RatUL9VO5yYOw1Zl+OQfu4kGgZ2VWpDl3GtKQkFIuz6eQlGvjkXdR1dIdSyrKdEIsGD3HT88PMSCAUiiKUluJ10CYdP/YgA376ws3XUek35uLY9mimOVfZyJfLJT4Ysa2mMmd3VGYNu0ktWVhbEYjFcXJR3BHZxccHVq7rdjfroo4/g5uaGoCDZY2A+Pj5o3LgxZs+eja+//hp2dnb48ssvcffuXaSlGXanlMqv2aBeSmF3ZYfcAHBz7y+KDwNNX30FUrEYiX/8ibSTZ2Dv7gbPV/6ntY3SIbfPsBDc+/cS7l/+D7HLVqHbinBY162jtt6lTWsAAJ0+Uf4wUJpn39eReTEWDxKuIOlgNJr1e/bI1ZPsLDx5cB8A0PnTpRBaqj5GJrS0RODsRTg0bTRyb9/Q+l6kUimSTuxD+sWjsHVyh89r42Bpo9+yIcWPC3D1l/V4lJUC1/bd0bjLAAgEArwzzQcNnZuhTcuXMHTgHMUHjDMXfsefR7bhbtp1PMjNgFQqUbrBUPAoB2Gf94RAIES9Oq5wd22O//UYhcAX/g+A7IPI93sX4cq1k0jLvIWCR7I1yoVCESQSMcSSEpOG3PmFj8q1kUfp73UlD7uJiIiIiEg9dWE3gEoPuZ88yFGE3PXbtoZTuzZI+GE3ANlYs8/m9Vrb0GVcm3v7Bp48uA/ruvUV9VKOHwQANOzUDU37DlDbdu1GTfDChJm4ELEU/369UiXo1mVcq6/c5ARcj5aNhVr0DYWjh+YJZOqUHteKJSU4+c/POPnPz7C0qAVnJw94uPngncGfo35dNwBAVnYKtu6eh7upCcjMSkZxifIsf6FQhK+2TcVX2wA7W0edx7X9ZiUBMM2a3OXdw4lht2ZmsXRJREQEPD09YW1tjcDAQJw9e1Zj2aioKPj7+6NOnTqws7ODr68vtm83v83rSL3Fixdj165d2Lt3r2IGt6WlJaKionDt2jXUq1cPtra2OHToEF555RWtd/Oo4jTt310Rdv+7elelh9wAcPvXPwAAPVYtQfNB/dHirYHo/qUsILy+e6/W+vkpqYrXQd9+BY/ePfFi2CR4D3kLAPDfFvX/djx5kK147di0ucb2BQIB2ox8HwBwbY/yUhbpsacBAK4BXdWG3HK1HOvAM7i/lncik3RiH2K//Rgp//yF69FbcXb9NJ3qlXbmq2m4Hr0VKf/8hdhvP0bSiX0AZDOt791PRq8ubyv9f/di296wtLRGdk4apFLZo3ClHwuTv5ZKJbj/IBVWVjZ4sW1vxXmhUIheXUJw736yIuQGAIlErHhtypnchj7uZYxlTIiIiIiISLPSy5jELtqC2EVbKn1N7oTvIwEAPiOG4MWwSWjc52UEffuV4vzDuyla29B1XJv+z2nlen/sAwC4d+lVZvsufp0Ur5/kPBvL6jqu1cej7HQcWTQMyad/Q/Lp33Bk0XA8yk7Xq43S49rSiksKkZpxE75teilCbgBwqueOF9r0QmrGTZWQG1AeW+ozrgVMt/Hk88tachkT4zF5iijf2HDevHk4f/48OnTogODgYGRmZqotX69ePXz88cc4deoULl68iNDQUISGhuLAgQOV3POaycnJCSKRCBkZGUrHMzIy4OrqWmbd5cuXY/Hixfjzzz/Rvn17pXN+fn6Ii4tDTk4O0tLSEB0djfv376NZs2YaWqPK0LR/dzi94I2sCwkQPy6s1JBbIn72x8rKwUHxupajo9oy6uQl3gEAeA95S2kWsntP2RIh9y//p7ZecX6e4rW25VGEIvUPxhQ9lLVh5+qm9nxplva1tZYBgPSLRwGBEJBKAKkEGZdPQCop+2dQmlQiRuaVE4r6EAhlbQKo6+iC+TP3w91V+QOQpWUtTH9/IwJf+L8yfxYCgQCBL/bD9Pc3qixh4u7aAvNn7kddRxeIhKo/L1OF3PmF+eX6UFDesJuIiIiIiMrm2Mwd7ScNRn5yBvKTM9B+0uBK3XgyMzYOANCwc4DimEAggM+wEADAwztJZdbXZ1xb/PDZxCBAtlkkAFjXqQ9dFec/LPVa93GtrrJvxqHkySNIJWJIJWKUPClA9s04vdpQGteW6p9AIMCEd1ajd9ehKnV6dx2GCaNWKcqpIxJa6DWuNVXILcewu2KYPOjWd2PDnj17YuDAgWjVqhW8vLwwZcoUtG/fHsePH1dbnozLysoKfn5+io0kASg2luzcubPGekuXLsX8+fMRHR0Nf39/jeUcHR3RoEEDXL9+Hf/88w9ef/11o/af9JN7KwUPriYqvk/+60ylXVtQ6u5r6U01Ss8mFmiZ8S//IJF16YrS8UdpsjvONs4N1NYTWum+xXlhjuzDx/M7XIuetmFRy1qljkobpe60l8XWqdQHOoEQNnVdIBCKdOwpIBCKYFPHRfah4ik7p0YAgAUzf0FD56Zq61laWGHau1+jY4e+Gtvu2KEvpo3dAAuR+tnrDZ2bYv7M/bC1dVA5Z6qQe+yPI8r9oYBhNxERERFRxSl5XIhbUYcV39+KOqx2g8qKYmkvWyrycWaW0vF7cRcBALXq1Cmzvj7jWk3jUJGNjc79FVo+a0Ofca2ubOur3mRQd6zMNpxUy0ulUkx4ZzV6dHpLY70enQdjwjurNW48aWfnqNe41pQhtxzDbuMzadAt39hQvlYzoN/GhlKpFDExMUhISED37t3VliksLEReXp7SF5VPWFgYNm7ciG3btiE+Ph7jxo1DQUEBQkNDAQAjR45U2qxyyZIl+PTTT7F582Z4enoiPT0d6enpyM/PV5TZvXs3Dh8+jFu3buHnn39Gnz59MGDAAPzvf9rXYKaKUXpN7h7rZymWMbm9/2ilXF8gEMCxuWxG/5n5i1GYk4ui/HycnrsAAGDv7qb1rnT9Nq0AyGZuJx86ipInT/AwKRmnP1sEAGj9jvq1lW2cnq1B/yhL/dMlcrmJsrW13V7qoXS8nk9bAEDC7u/U7n5dWtLBP8o8L+fz2ji4tH0JEAhgU9cZnSau0aleaYETV8OmrjMgEMKl7Uvw6f8BAMDZyaPMeiKRBRo1bAmRmiBbJLKEh5s3RBpmt8u5ODWGdS391hRXx1gh9/Wsa0b5UMCwm4iIiIjI+EpvPOk/dyz8545Ffkom4lbsrLSwu8ME2VKVpz9biIdJyRAXFyPpr4O4fyUeAFCvVdnrU+szrq3n006pboMOskmC6WfLnthZlPdsJrhtA2fFa33Gtbqq16wd2oXMgNDCEkILS7QLmYF6zdppr1hK6XFtaZ6N2mit28S9tcZz1rXs9BrXmjrklmPYbVwmDbrL2tgwPV3zGj+5ubmwt7eHlZUV+vXrh7Vr16JPnz5qy4aHh8PR0VHx5eFR9i89aRcSEoLly5dj7ty58PX1RVxcHKKjoxX/HZOSkpQ2kVy/fj2Kiorw5ptvomHDhoqv5cuXK8qkpaVhxIgR8PHxweTJkzFixAj88MMPlf7e6Jnn1+QuvWZ3ZYXdL06bCAB4eCcZR8Nm4cjkGYp1tzvO+bCsqgBkd8b9ZkwFAFzd/gMOjZ+mCLldAvxQz6el2npCkQhNXxkAADj1xQyIC9V/iHp49w6u7pJtwvH85iB1vJ594Lm+V/Pv8p2Y37W+DzlLGzt0CfsGAzddxCsrDqJu07Y615Wr16wdXllxEAM3/YsuYd/AwloWPBcVP9FaNzn1KiSSEgCAQCCE4OnMcImkBEk6/DEuLHqMrOwURX0AapcyKYsxQ+4tITsBGOdDAcNuIiIiIiLjKR1yy9fkLr1md2WF3XW9W8AlwA8AcPqzRTj4/mTFZpR+M6dpfcoY0H1cW8dLeXzqPXgUACDhx++Qc+u62rZLCp/g7PLPAABN+ryq9MSvruNafbXoG4rXv76A17++gBZ9Q/WuX3pcW3o8qMuYMrlUGXld4dP3fO/+Xb3GteYQcssx7DYeky9dYojatWsjLi4O586dw8KFCxEWFobDhw+rLTt79mzk5uYqvpKTkyu3s9XUxIkTcefOHRQWFuLMmTMIDAxUnDt8+DC2bt2q+D4xMRFSqVTl67PPPlOUmTx5MpKTk1FUVIQ7d+5g/vz5sKqAx2xId+o2nqzssNvCxga9v16DliFvQCCS/XPV/M0B6P31Gljo+PhWvVbe6LYiHM7+LwIALO3t4f9RGNp/MLbMevLgujj/IU4tnIXH9+8pnc+IPY0Tc2WbQTbo4K+yFrdAIMCLk+cAAG79tge3/9iHkiePFeclxcW48t3XiN+5CQAUZXWhz3IluraRkn5Da53byZchlUpRy8oGb706HW+9Oh21rGwglUqRmHxZa/3U9BuKjT9cnZti/Mgv4e8brHOfjR1yl/5Qw7CbiIiIiMg8qAu55UwRdrf/YCz8Zk5TLH1Zv21rdFsRrnHi1PN0Gde+OHmOyhPL9g3d0TBQtr/U6QUfIfvaf0p7ND3KysSRGe8j/65sbyqv11SX/dBlXGsIgVCoU8hfdhsipfFgcor2cLd0GN7Rty/Gj/wSLg08AQBSqUSvca2pQu6vTqh/Mptht3HoN5XOyAzd2FAoFKJ5c9nC8r6+voiPj0d4eDh69uypUrZWrVqoVatyNs8jqk40bTzZtL9smaBbUYeUvjeWkseFsCp1WaGlJZoEB6FJcJDmSlpY162DDuPf1auOVW0HdF24Fsc/noT8u3dwZMb7ass5tfXFCxNmqD3n7OuPtqETcHlLBBJ2f4eE3d+pLdc2dAKcfTWvXV8ZklOuoqmH5hniTwof4WF+Nl7rMw4D+05Ebft6AIDgnqOwLzoCfx7ZisKiR6hlZauxjaTUq2hQ3wMhr32IroFvQCQUoVeXt4HbxVr7V5Eht5z8Q8HQXTMxPHIWdoQshn0tze9HnSldZMvhLDu6Rel7IiIiIiLSrqyQW04edl9Yth1xK3ZqHLsaUz2flui6+AuD62sb12oaD7YbMxFPHtzHg2v/4eziTzS233XhWljZ11Y5rsu4tumWd3R7ExXgw/c34eadf/HDvnAkpSZoLZ+ckoAOrXtiyIBZ8GrSAQDQvfNbOH5mDyJ/Wa7XuFZfxgq5Vx9bjhldX1F7Xh52D4+chaG7ZuL7t5fiBTcfva5hjHFtVWbSGd2Gbmz4PIlEgkIjPYJBRDJlfVCoqJnd8g815sK+oTteXrMNnn3Vb4ra4f0w+E39RGUjytIadeuNlz5bgbotVdcSq9PcGy99tgKNuvU2Wp8N1dKr7KBdLCnB6i9OYOSbcxUhNwA42NfHyDfnYvUXJ1AiLin7Gs38sfqL4+jReTBEesxKr4yQW44zu4mIiIiITEdbyC1nipndpiC0sETAzC/Q/t0pgEA1wmv6ykC8vGYb7Btq/llpG9eamleTDvhkyi4MG6j9Kedhgz7GJ1N+UITcACASitCj82Cs/vyYXuNafRgz5J7SrexlWDmzu3xMOqMbkG1sOGrUKPj7+yMgIACrVq1S2djQ3d0d4eHhAGRrbvv7+8PLywuFhYX4/fffsX37dqxfv96Ub4OoxjH2zO7Sd+7NiZV9bfgMHoWWbwzH46xMSIqLYGlfG9Z16mmv/JRD46YInLUAJY8f4Un2fQCAdb36sLAxn7uqrk8f99LEzsYBdjYOGs/Xr9tQ6zU07X5dlsoMueU4s5uIiIiIyDR0CbnlTDGz2xQEQiHcOveAW+ceeJKTjeL8hxBaWsHGyRlCkW4TiIwxrq1oHm5lb+yprYylZa1yj2vVMXbILRvXql9zXY4zuw1n8qA7JCQE9+7dw9y5c5Geng5fX1+VjQ2Fpdb9KSgowPjx43H37l3Y2NjAx8cHO3bsQEhIiKneAlVzmzdcMtm1c7T8zZKUyNY7Pv1NPJLOGhYQp8bJgtdjqy5CaPHs/7VWU7UH18YKu59/PM0cCUUi2LloD3PLYmFjC3v36v+HxVhMEXLLMewmIiIiIlMy53Ho8+RjyqNfXjToeqXHtf5zx+oUcsvVlLBbzrpOvXKF08YY19YkFRNy66aiwu7qziw2o9RnY8MFCxbg+vXrePz4MbKzs3Hy5EmG3EQmVN5lTHRZg41qHlOG3HJcxoSIiIiIqHIZMh40xjImxlySk6oHU4bcchWxjEl1Z/IZ3UTmbvQH7Ux27X/sBWWeFxdLsHvsEXR6rxU8u2jewLUsiSfScfqbeHSb2h4iy2f3vu5JdW/D0Jnd5hhy56ckG7ysyJMH2SjOz4PQygo2Ti46P0Zmjgoe58HWurbKzt+6kkqlePTkod6PhQHmEXLLcWY3EREREZmCOY9Dnyefyd19WnuDrld6XGuo8szsvr3/KG5FHYLXq4MNvr4xSMRiPM7KgKSoCJb2DrCuq//MbXNeLlNfRcVPAABWltYGt2HouNYcQm45Y8/sru7MYkY3GSYiIgKenp6wtrZGYGAgzp49q7Hsxo0b0a1bN9StWxd169ZFUFCQ2vLx8fHo378/HB0dYWdnh44dOyIpKaki3wZVE/rO7DbHkBsAjn86BX9PGI6Tn89AXtJtreWlEglSTx9D9OhBODx9LE7MC8Ox2RPx57tvIf77b1H0ME9rG0UP85Cw+ztjdN8o0u8l4p2p3li7eaLBbazZPAHvTPVGxr07etUzp5BbjjO7iYiIiIjMnyEzu+Uhd7NBvSqhh5ol7P4Of777Fo7NnogT88JwePpYRI8ehJSThyGVSLTWz0u6jVMLZ+HvCcNx/NMpeo9rzdGwiU0xbKL++zzJGTquNaeQW86YM7urOwbdVVRkZCTCwsIwb948nD9/Hh06dEBwcDAyM9Wv03z48GEMGTIEhw4dwqlTp+Dh4YH//e9/SElJUZS5efMmunbtCh8fHxw+fBgXL17Ep59+Cmtrw++eUc2ia9htriF3aXl3buLkZ9Nx91iMxjKSkmLErlqAi998qfb8nb9/w8Ep7yA/LUXteQDIT7uLg1Pewe0/9pW3y0ZzOvZXAMDF+KOQSvWY2v+UVCrFxXjZf//T53/VuZ45htxyDLuJiIiIiMyfPmF36ZDb0P2mjEXTePDSpjU4u3QuJCXFGuvePRaDk59NR+7NayrndBnXmqPUjJuK12kZtwxqw5BxbWWF3IaMB40Vdld3XLqkilq5ciXeffddhIaGAgA2bNiA3377DZs3b8asWapr7uzcuVPp+02bNmHPnj2IiYnByJEjAQAff/wx/u///g9Llz67w+Pl5aV4/e+i/1Npt6CwBOO3nsPNjHysD+2Idh51jPH2VFXdp21qHG3LmFREyP0g4Tr+jfgaxfkFcPbzhffQEFjXraNzfUlxMe78eRA39uxD381RkJQU487fvyPhx224vCUCVrUd4ezrr1LvwrqlyLocBwB4YcJMuPh1UpwrysvF2eWfIf/uHRz/eBJeXrMNVva1leoXPczD8Y9lf/gs7fVf4qMiPCl8hH0HIgAAuQ+zcOHyQbzYrrdebZy/HIO8h7LH5fZGr0Nwz1BYa1nuw5xDbjljL2OyFJuN3kciIiKi6iwiIgLLli1Deno6OnTogLVr1yIgIEBt2Y0bN+K7777D5cuXAQB+fn5YtGiRSvn4+Hh89NFHOHLkCEpKStC6dWvs2bMHjRs3rvD3QxVDl2VM1IXcf40eB4FQiOZvDkDj3j0htLTU+ZpPHuTgv83f4f6VeNg4N0DrUcNQr5W3zvUt7R3Qee5S2Do5A5BNHsq9fQOnF3yEB9f+w6Vv16HD+9NU6mXGncPlLbLxm0evYLR6e7Si37qOa81R1B9rIBSKFK8nvLNKr/qGjGsrcya3octaGmMZk+qOM7qroKKiIsTGxiIoKEhxTCgUIigoCKdOndKpjUePHqG4uBj16snWfJJIJPjtt9/QsmVLBAcHw9nZGYGBgdi3b1+Z7djVssBX73SEl4s9xm05h0vJOYa+LapGNM3s1jXkvrhhk87Xurh+E/5ZshLF+QUAgMzYOBybPhvZ8Qk61S95/Bgx70/GjT37FMeEFpZo2vd1NOs3CABwfs0ilTvA+WkpuHcxFgDQ5YsvlUJuALBycESnj8Nh+TTcVneH/uYvuwEA9o2aoOeyrxXHpRKxTn3XRCqR6PR4W2lXEk7i72M7ELF1Ch49zgUACIUibN09D7/+/TViL/6FlPQbKC4pUqlbXFKElPQbiL34F379+2ts2/2Z4kPJo8e5iNg6BX8f24ErCSdx/0Gays+yKoTccsac2U1EREREuuNTzaSPsmZ2lzWTWyqR4PqPUYh5fzJKHj/W6VrZ8Qk4Nn027l+JBwA8zryH2GWr9BrX9lz2tSLkBgCBQIA6zVqg0ydLAABpZ46hID1Vua9SKc6vCQcAtBo2Fm1GvK8Uzus6rlWnvGNSfdsoeJyHm4lxOH52L378ZTmOnv4JEokYEokYR07vxo+/LMfxs3tx886/ePT4oeq1pFLcf5Bm8Li2spcrKc+TvsaY2V2dcUZ3FZSVlQWxWAwXFxel4y4uLrh6Vbdf8I8++ghubm6KsDwzMxP5+flYvHgxFixYgCVLliA6OhqDBg3CoUOH0KNHD41tycPu8VvPYdyWcxU7s5uqjOdndnv0CdR5JnfG2Vg86NUDdb1blHmN7PgEZJyThc2dPvsYNs5OSDt1Fle3/4DYZasQtCkCAmHZ9/POLloGALB3d4PfR8p3yFsMHIpbv0UBAHJuXkPd5s/uyMuDa58ho1G7URO1bVvUskbnuctwdOYHuP3HXrR8YygETwNgiViMO3//BgAI+PAziGrVQvatSzi9bjKe5GTCuU0XdJrwJSys7crs//Ou/bEZ/0WtBgC0eWMqWvTVLVD9bOUbAACRyFLx4UciESPz3h1899MXkEplwXn9um5YMHM/nOrJ/vtlZafg4yWvITsnDQAgEAghFAghefqhRiqV4ty/B5SWMOns1x9Tx66H8Ol/G1OG3BdSr5ZrI4/yzuwmIiIiIt2Y4qlmqtrUzexO/uuMxpC7x5plkIrFuLAqAg/vJOP8l+sQMGdGmdeQSiSIXbYKAOA9dDDce3TFo7QMnP5soV7j2gat1C+dUqdZC3i/NRIJu7/D1R+3wm/yHMW5nJvPJnc16a36BL6ctnGt85hnM52LHxfgzFfTkHnlBGzquCBw4mrUa6bfxqj6jmtPnPsZqzZ9oPheJFSNKqN+Xw2xpASA7CbA1LEb8JJ/fwCyiZurNo3Dqdj9z9rQYVx78v3TAEyzJnfpJ31Lf68rzuzWjDO6a6DFixdj165d2Lt3r+JOteTp7M/XX38d06ZNg6+vL2bNmoVXX30VGzZs0NomZ3aTOqVndh8Zt1iv5Ur+jfhGa5n/tsk+vHb67GPUbtwIFtbW8OjVHfXbtgYAxR11TaRSKQpSZAFtpy8+gZW98h80gVAI77dkH4Kzr15SOpd6+ggAwNGz7A/Cpe/KP7r3bLbJ46wMxWsrB0cAwOl1k1GYmwVIpci8chJX92v/f6+07FsXcfnH5ZCUFENSUoxLkcuQfeuS9oqliMXKa7+JJSWQSiUQCkVwdGiAudN+VITcAOBUzx1zp/0Ix9pOEApFkEolig8g6trs2CEYk0avVYTcAEw6k7u8G3mUZ2Y3EREREenGnJ5qpqql9MzuI+MWl7kmt5W9PWo5OiLwU9mNk9wbt7Su7Sx/krh+29ZoHNQLIktL1G7cCJ0+kwXS+oxrNXEN7AoAuBf3j/K1r8qW5WlcRsgN6Daulbv6y3pkXjkJSKV4nJOJMxFTtPb/efqOa7t0fB3jRq6EQCCAQCBQjEHl5GNM+flxI1YqQm5A9m/BpNA18O8QrDimy7gWMO3Gk+Xdw4kzu9Vj0F0FOTk5QSQSISMjQ+l4RkYGXF1dy6y7fPlyLF68GH/++Sfat2+v1KaFhQVat26tVL5Vq1ZISkrSqV8Mu0kdjz6Bitd1fTx1XpO7OD9fa5nHmfcAALYNlZ9ukAfdRXl5ZdYvvbyHQCBQW6ak8AkAQFKk/GiTtEQW5tZyrKu1n3KS4mdtPN8eADzJyVR6vKsg667ObQPAo/upao5p3giztE4vvqrxZyASWsDRoQEWzNwPNxfVYN/dtTkWfPQLHB0aqL37Dsh+vp39XkPY+xthaWGldM5UIffF1Dij7FrNsJuIiIioYpX1VHN6erpObZT1VHPfvn3x559/YuDAgRg0aBCOHDmipTWqShybuaOuj6fi+9JjVHVKPxWsbUnIwgc5AIAGHZRnPds4NwCg37hWk5LH6scZ4qdjylp1tI9JtY1r5R5llRo/SiV4/CBDryVIpBKxQePal7sMwcR31gBQPzaXH5v4zhr06vK2ynlLy1qY/v5Gvca1pgy55Rh2Gx+D7irIysoKfn5+iIl5tmuuRCJBTEwMOnfurLHe0qVLMX/+fERHR8PfX3kDAisrK3Ts2BEJCcrrGl+7dg1NmqhflkEdht1UmnxNbpFNLTi94I2sCwlKa3aXxdn/Ra1l6rdpBQBIOXJccUwqleLarp8AAA6enmXWF4pEiteFublqyxSky/7QW9Z2VHteIi5Re7x0f+QsS21GqW7zSec2XQCBUPYllcC1vX47f9fz8oWFtR0EQhEEQhEsrO1Qz8tXp7pTx65HQ+dmas+JJSX4eNJOuDbw1FjftYEnPp60U2U2t5ybixemjPkKFiLdN3TRxFghd2jkMKPsWs2wm4iIiGqSiIgIeHp6wtraGoGBgTh79qzGshs3bkS3bt1Qt25d1K1bF0FBQWrLx8fHo3///nB0dISdnR06duyo84QrXVTEU81UddzefxRZFxLg9II3RDa1VNbsfl7pCVOlx4zq1PaUbVp6dWek0tgv7dQZAPqNazUpfJANAKhVp57ScavasjFlcb7qmtXP0zaulXNt3x2QShTjUpe2XRTLb+pCIBQZPK7t3ulNDOyrOSwe9MoUdO/0psbzFiJLvca1pg655Rh2GxeD7ioqLCwMGzduxLZt2xAfH49x48ahoKBAsV7ZyJEjMXv2bEX5JUuW4NNPP8XmzZvh6emJ9PR0pKenI7/U3cUZM2YgMjISGzduxI0bN7Bu3Tr88ssvGD9+vF59Y9hNgOrGkx2mvK12g0pNvIe8pbVM69Gyx68Svv8R51esQXLMYfw95tnvq717Q61ttHhrIADg6LRZuPZjlNI5SXEx0s+eAAC4+itvNtnijWEAgP+2f1Pm42y5t64rXluX+mBiXffZ64xY2dpgnSZ8iZZ9Q+Hu3wd+YxaicZcBWvtfmm09V/SYswMenfrBo1M/9JizA7b1yn7KQ04kssDD/GyN5x890f7h6dFjzTPo8/KzIRKVf1sIY4bcLZxaGuVDAcNuIiIiqilMtSGkuT7VTOav9MaTHaa8rXGDSgC49mMUbkT9jCNTPwIANH2t7CVBAKB2o2dPLP89Zjzu/BmD8yvX4ur2XQD0G9dqknJCtu9V076vKx139ZONURMP7Edhbo7G+rqMa+UadxkAvzEL4e7fBy37hiJw/Jda+/+88oxrHz/Jh1DNU8JCoYVOY1J9xrXmEHLLMew2HgbdVVRISAiWL1+OuXPnwtfXF3FxcYiOjlY8ypWUlIS0tDRF+fXr16OoqAhvvvkmGjZsqPhavny5oszAgQOxYcMGLF26FO3atVNs1tG1a1e9+8ewu2Z7PuSWL1dSes3ussJuvxlTYV23jtbrWNetA78ZUwHI1uO+ujNSca7n2uUaailr0rcPGr4ke3TtTvRfiuPFjwpwav5M2XXqOcG6bn2leo179QUgWxctMfpntW3n3LqO0wtl67u1f1d1bbP278k2v7wQsRQP796BhbUd2g6ejsDxX6JJ14EaH7kqi6OHN/zfXQz/dxfD0cNbe4Wnch9m4WHBAwBPN5R8etdeIJD9mUhO0f6HMik1QamOUChSvH6Yn428/Pu6vxE1jB1ybxq8HYBxPhQw7CYiIqKaoPSGkK1bt8aGDRtga2uLzZs3qy2/c+dOjB8/Hr6+vvDx8cGmTZsUTyPLld4Q8oUXXoCXlxf69+8PZ+dne92Y81PNZL5Kh9zyNblLr9n9fNh9J/ov3P41GgDg1vUleA14VafrlB57Xtv1E+5f/g+AfuPah3fvqByXSqW4/cc+pJ05BgBo1K23cr16TnBs2hwAcCZ8DiTFxSpt6DqulRMIBGjSdSACx3+JtoOnl7mJpCblGdfeSfkPYnExREILxZrcIqEFxOJiJN0tew8uQL9xralC7vxC9cvZMOw2jvJPrwOQl5eHgwcPwtvbG61alf3IBRnPxIkTMXHiRLXnDh8+rPR9YmKiTm2OHj0ao0ePLmfPZORh9/it5zBuyzmsD+2Idh51jNI2mS9NIbec/APGrahDSt+XVq+V7gFtvVbeCNoUgez4BBTm5MChSWPYN9JtHXBA9oe87dh34D3kLdy/Eo8n2VkoyEjDuWXzFGUCZy9UqWdpa4f2703FxW9WIWH3d8i5dQ0ePYNh11B27fQzx5Gw+zsAQN2WrdEwsJtKGw0DuiD15CFkXY7DibnTMGjLfzr329jupl4DAFhZWuO1/43Da0Hv42L8UeyMWoiMrDuKELss8g8Nzk6NMWzgHLRv1R2//LUBv/y1AUXFT5CcmoA2LV8yqH8VEXKX/lBjjF2r5WH30F0zMTxyFnaELIZ9LVu9+0lERERkjuQbQpZ+cthYG0LOnDkTwcHBuHDhApo2bYrZs2djwIABSnXDwsIwatQo+Pv7IyAgAKtWrVJ5qtnd3R3h4eEAZE81z507F99//73iqWYAsLe3h/3TTehnzJiBkJAQdO/eHb169UJ0dDR++eUXlfE0VT3qQm45edh9Ydl2xK3YCd/pw2BhUwvtPhgDqUQCp3ZtYGmne8BraWeHPpvXIz8lDXmJibBycED9Nq2U1vrW5sTcaWgxaCgatPeDlX1tlBQ+wZXvvsaDhCsAgPbvTYWFjerYwnf8DByZ8T4eZabjrwnD0C50Aup4eUNoYaHzuNacJD0dU77YrjeGDJD9W/PDvnCc+/cAklK0B936jGtNFXKP/XEEfhm5VO35KV2GAwCWHd2i9L2ujDGureoMCroHDx6M7t27Y+LEiXj8+DH8/f2RmJgIqVSKXbt24Y033jB2P6mKYthds2gLueV0Cbv1IRAKta5rpo2lnR1cA/zx1+j3FMeEFpboumA1bOo3UFvHrVN3QCrFxY2rkRF7WrEESWkNO3VDu9ET1X7IEQiFeHHybFyIWIZ7//6jcr4ypd+7jX6938XAvpPh6OAEAOjs9xoCfF/B4VM/4r/r2gcvj548xLiRK9Gj01uKZUrefv0jvNJrDKL+WI30zESDgu6KDrnlGHYTERERaVbWhpBXr+o2c7CsDSEXLFiAJUuWIDo6GoMGDcKhQ4fQo0cPRd2QkBDcu3cPc+fORXp6Onx9fVWeahaW+sxd+qnm0ubNm4fPPvsMwLOnmsPDwzF58mR4e3sb/FQzmY+yQm45dWG3a4C/2rK6sndvqNPymZpcj/oe16O+Vzne/r2psrGnGjb1G6BbeASOzZ4AaUkJLm5crVJG27jWXOTmZaGpR1sMGTAbLZq+oDg+c/xWXL99Ht/vDUduXpZivKqOMca1mhgr5L6eda3Mcgy7y8egoPvo0aP4+OOPAQB79+6FVCpFTk4Otm3bhgULFjDoJiUMu2sGXUNuOWOH3cZk5VAH7UZPhFNbX6134d0690CDDv5IPXUE1/f+gJJHBQCABr7+8Bn8Duxc3cqsL7SwhN+UOchPSymzXEXr+dLbEKnZZEQkskDvrkPR86UQrW1MeGe12jYcHZwQGjIfYj1265arrJBbjmE3ERERUcWQbwh5+PBhjRtCAoCvry9OnjyJDRs2KAXdgPk/1Uymp0vILfd82B0wZ04l9VJV14VrceevX5F8+IDiWMs3hqPxy33VzuQuzc6lIYI3/YSsK3G4uX83cm7KZi3rM641B/b2dTF32o9qz7Vo+iLmhe3WOqY0xrhWHWOG3FtCdmotz7DbcAYF3bm5uYpHjaKjo/HGG2/A1tYW/fr1w4wZM4zaQaoeqnLYXVIsMdm1xcVlr2UlKZH1TVwigdjAfoqftiFvS0GPfx30DbnlzDHs7rs5Snuh51ja2qFJ7/9Dk97aNyvRxL6h7sutVAR1Hwb0OW+sNkqr7JBbjmE3ERERkSpjbAj5999/67wh5PHjx43X+SrInMehz1OMSw3sc+lxbXnoE3LLlQ67Tcm+oTvajHwfbUa+b1B9gVCIBu1eRIN2Lxq5Z5WnMsac+o5JAeOH3LJx7XWt9Rh2G8agoNvDwwOnTp1CvXr1EB0djV27ZLvJPnjwQGlnZKLSqmrYPWuq5k0TzcW5zQk4t1n7Gspl2fPBMaXve2/tqVM9Q0NuOXMMu8n0TBVyyzHsJiIiIlJWekNI+frZ8g0hNc2yBmQbQi5cuBAHDhzghpB6qArj0OftHnukXPXPbU5AbwOHg4aE3HLysJvoeRUTcuuuIsJud7ysVxtVjUFB99SpUzFs2DDY29ujcePG6NmzJwDZkibt2rUzZv+omqmKYfeQUaa743W7Vtl30sUlEpzbnADXdvXg2dmlzLKaJJ7KQPqlbHQc7Q2RhX6PM5U35JZ7Puz2enWwQe1Q9WDqkFuOYTcRERGRMm4IWXnMeRz6vNPfyDYJ7PSeYfsmlR7X3t5/VO+gujwht5xjM3fAdJPoyQyZOuSWM3bYnbd1vEH9qCoMCrrHjx+PgIAAJCcno0+fPooNH5o1a4YFCxYYtYNU/VS1sNsvoOzH8CqS1F5L0F0s+0Dg2dkFnl0M72f6pWx4vuQKkeWzoPuetOw6xgq55UqH3eYWdEulUuTcuIrshCsQFxXBqrYDXP06wbqe5k0wnicuLkJG7GkUpKcCAOxc3eDi1wmi/2fvzMNjuv4H/M5MJmSREJGIWJLY9yX2tbaiK9XSVmkpVUtrK0VbXS1FW7RoS+leqkX12x+K2msNKYLYgsgqQsiezMzvj3HHTDJ7JplJct7n8Twz99xz7pkgcz/vPefzUboX17SLleP//Y1MJiO8RV+7+ms0Gn75YwGP9XkF8NEddxXJLSFkt0AgEAgEAsEDREHIksOV49CCXD+aDGB3TKof1x7+2radvo6Q3KWF7NQUEiMOk3vvLgp3d/waNaNy3YbIZNb9fZmKaz2rFvPEzXA3/RY+3vZN4G76Lf6382ueHzjLwbNyHckt4UjZXdaxS3QDtG3blhYtWhATE0PdunVxc3Pj0UcfdeTcBGWY0ia7BYY4WnJLuOKNSXLkcU4sm1fo+Plf1lCxSlU6zJ5ntnq1WqXi0uZ1XPnrd6PttXr2o/Hzo5ErbM8V5iw0Gg0fr3gRgF+/jLf6xkqf85eOsGnrMv49voV9L2u3Zbqa5JYoLtktEAgEAoFAUBoRBSEFxUnYUz2tTmtZXiR31q2bRK5YRFrMJaPtbV6fTUCrtkbbJMzFtU+tPeuQedrD7AWP8cVHh+zrO/9RklKu0bpZLxrX6+CwOZWE5E7PybR58ZOjZHdZx66yq5mZmbz88st4enrStGlTrl+/DsBrr73GggVl/4cmcAyS7K4b6M24tcc4HXvH2VMS6JF2Jc7o8eKS3BKmblA0ajUaddH2kqlVKjQaC0vV9bixf5fBzUDDZ0ZQ94khVG/fBYDs27fYO30sGUkJJq93bNG7OsldtXELGg4ZQeiAQfjUCQMgdvd2ji16F7XKfPVoV+LE6Z0PXp/ZZdcYuw78DMCdu9rVH64quSWkm4KG/iE8v24GJ+PP2zyGJLujU66WiyfpAoFAIBAIBAKBrYQ+0V0nu2O2mM5T7gzJrdFoihy3mYprTY2bkZTA3uljdZI7pN8T1B88jNq9B+jOObFsHjf2m47LLMW1zuTO3WSysjNs7peVnaGLJaXY0hGU1EruF9bPJD0n0+bxJ3V5gendR7Jo31qWHvzR5v7lYWexXSu6Z82axX///ceePXvo37+/7nifPn147733mDlTBPDFRcIHX5TYtU7euMqz3y+nYUAQPw8fD16Ov4ZY2e26nFz0QyGRXdyS2xhpMdc4+qHhA7T2b83At26oVf01Gg3Xtu3g4oZNumNewUG0nz0dNw8Pk/3uXo/hzNrlAIQ99jT1Bz6LTG8bpvrl1zn04Qzu3bjGwXcm0/fLXwzaAc79vJrbF7RPx9tNf5+qjfVrGAwnPeEGB956ndsXznJp8zpaPf2hVZ/JmWg0GtZvWYj8frXq9VsW0qZZb5tWdSckx7DviFb+5+ZmubzklnD0ym6BQCAQCAQCgUBQmII1nAqK7JKW3PlZWZz49AvSLl95MMfHBlB30ONWx0GW4tpLm9fRYPAwg3aNWs3+WRMA8AwIosOsuVTwraxrbzJsDNd2/R/nflrNmbXL8akThk9twzjZmrjWmeTmZvH33u94sp9teaP/3vstuXnZAOw7/BtPPzqF6tVCrO5vbAFcSaYrkRY/2ZPWsqgru8s6donuzZs3s379ejp27Gjwn7pp06ZcvnzZYZMTOI+Cktu7QkVsf9ZkHUJ2uybewQEGstspkvtyDEfnLix0/OjchXSYMxOfEMuV2KO++Y6Ef48YHMuIS2D3hKn0/moZcqXSaL8za1cA0HDoS4T2e6JQu1yppP3Mj9g1cTjq/DxSzkRSrUUbXbsqN4fY3dsB6Dp3Gd5BNQuN4R1Uk3bT39et+rZGdGemJpJ6ORLPqsH4hdlX/Df1ymkyb8XhV7cVnn7aPHqrfp5JUEAoQYF1ad20ly634tXYKC5fiyQ+6QoJyVeIjY8mMTlGN1bM9dNMmtOVmjUaEBQQRo3AMOrWaUVIraYAqNVqTkb9Q0LSZRKSY4hPvMy1G1HIZXJUGjUajaZUSG4JR8pugUAgEAgEAoFAYBxTsrukJbc6L4/dE6YWOh7zv63k3Emj6ajhFsewJq698tfv1H3iGYMaTilnTgIgV7rT9cMlRmPXOr0fIT0ultg92zn749d0nD3foN2auNZWVHm5JEcdBCCgaRe76k5Jca1Uu+lQxBaCg+prY9KAMJo17Iqvj7YmVtrdFM5EHyAh+QoJyTHEJVzk6o2zOlktl8mZPf8R6tRsSo3qda2Oa/eO2qubT0nn5C5qDSchu01jl+i+efMmAQEBhY5nZGTYlatV4FoYk9zFjZDdrkeracOI/OQnTi76gRavDeHKxj0lKrkB3c1Ai/FjCGyrlchJx09wasUqjnywgL5rVprtn5eRoZPc3T9bQAVfXzQaDYfnfER6XDzXd+4mZMDDhfrlZ2Vy95r2oV0dvS1hBVF6eulE9ek1n9NryVpdW9IJ7XV96oQZldwSVRs3x69xc1LPnTb7WQDSYqPZO28Y+dnax07Nh06nfv+RFvvpc2HrGs78uhgAt4pe9Jj9I761GvL33u+QyWS8MmyhQQEhNzcl6/5YyJ27ychkMqNPvhOSr5B4MwaNRkNlnwDem/abrk0ul5N6O4HvNrxnck7OlNxLD/5Y5KrV9spugUAgEAgEAoFAYJqCslt6XZLpSq7v2gNApTq16fDOm8jkcnLv3mXv5DeJP/AvDYY+hdLL/PZ3a+PapIgj1OjYTdfv0pZftf3GTDIrpBs/N4rYPdu5cyma/KxM3Dy00tTauNYWVHm57F84ktRLWgnvV6813d9ci9zNetldMK5VqfK4fO0/rlw/hUajIbzFw7Rv9WC+nh6V2H90ExGn/jYak6rU+dzLuM2Z6AOciT5gc1zrjMKTxmo4CdntGOwS3W3btuWvv/7itddeA9DJ7dWrV9OpUyfHzU5Q4jhDcksI2e1auHlUoNW0YUTMW8vJhT8A0HbO6BKT3Pp5y6SbgYKvNWp1oXQh+tyKOgdAyICHqeDrC2h/X4W/OYW9r0/n4oZNRkV3duot3Wu5m/kn3J4B2hXRuXfTDI5nJMYDULVJS7P9Aao1b22V6L64bS2q3Bzd+6jfl1Dv4RfN/gz00ajVnN24VPdelZvNxW1raTtmATKZjHEjPqNn56EGfWoGNeCDNzYxZ/Eg7qWnotLkGx1bLlNQqZIfH7yxiaDAMIO2vt2H46ZQsvIH7UqIgjcmzlzJ7Yiq1fbKboFAIBAIBAKBZf6b90iJX3PV7kss33mRXz/rXeLXFhhSUHaXdOHJS79tBqD15PG6uMvdx4fQx/oT879tpJyOIqhje5P9rY1rATKT4g36pl25CICHf+GFpvroS/Ds1Ft4B3vqXuvOsRDXWkty1EGd5AZIvXSSpDMHCWrV0+oxCsa1+rRvNYDJY75EqSfOlcoKTBu7iiWrXuXYf9tMjiu5SVviWmdIbgkhu4sHu4pRzps3j9mzZzNu3Djy8/NZunQpDz/8MGvXrmXu3LmOnqOghHCm5JYQBSoFjkSj0t4wyBQKw+P5qvvH7foVaEBBwV0Q90qVLI9x726R51FUXhv5eaGbAYmgwDA+mrEFX59qutzc+sjlCnx9qvHRm38WktwSPbs8y2sjPzfa5izJveLgsiIX8ihqgUqBQCAQCAQCgesxpmc9JvSp7+xpCFwASUIXjCnVUkxp5aKjouBeyafYr+FsZDIZncKfYMorXxlIbgmlmztTXvmKjm3M50W3Ja51puSWkGS3lLPbGQUqyxp2/Y/s2rUr//33H/n5+TRv3py///6bgIAADh06RHh4uKPnKCgBXEFySwjZ7RpIObmzUu7QesZwfOvV4uSiH0i7Elci19e/YUg6fsLoa0s3Ff4ttHmiY/63ldy7WpmsUas5uVRbjKP+04OM9qvoV1X3Wp2XZ/YaWbe0lZ596zYwOO5VvQYAKacjzfYHuBX1n8VzAOr3H4nC/cH/zaaDJ9t0YyWTy2k6eLLuvcK9oi71SbcOg832DaxWhwE9Rxlt02g0PNLrZQL9a5sdo1uHwdQObmz1fE3hKMm9dP9ih1StFrJbIBAIBAKBoPhwVjw4pmc9p1xXYIh+Tu6wp3pyZeNuYrbsK7Hr1xs8EICIjz/T7UzNSUvj2rYdAFRtaj6+sSWu9QysYdDX3Ue7KzkzOdHsNdT5D2JW/VjWlrjWWgKadsGvXmvde796rQls1sWmMQrGtaCNKZ8fNBs3hemV524KJcMGzTaaThOgdnATm+JaZ0tuCSG7HYvNqUvy8vIYO3Ys77zzDqtWrSqOOQlKGFeS3BIF05isWdDD2VMqdxQsPOkTGqzL2V1Sebo7zJnJkQ8WcGpF4d817d+aYbG/0suLGl07EX/gEHsnv1movVYv4/+u3Dw88a3bgLTLFzi3bg1Nh481ep46L49Tq5YBUO/xIQZtgeEdAbh17hS3zp2mamPjhSPT429w93qM0baC+NZqSJ+5fxapGGX9/iOp2qBtoWKUarXaIIeZMW4kXECG9um5tLJbrVYhl8mJTbhg8dpqtZr4xAcFi+VyBWq1yqb5O1Ryd3sDKPp2L5HGRCAQCAQCgaD4EGktyy+mCk8WLFBZnNR5uBeXft9Melw8O18eb9AW1LmDxfzcYH1cGxjewaCt+aiJRCyZy7FF79L7ix9Qehq/1rWd/weAb2g9XX5usD6utQWF0p3ub64l6Yy2GGVgsy425ecGw7j26IoHhT5vJFywuHjqRoG4Uz+mjE+6bFNc6wqSW0KkMXEcNotupVLJ77//zjvvvFMc8xGUMK4ouSX0Zbeg5ClYeFLK2V2SstsnpA7t35pRqEK1VJnaGpqMHE6FKlWI+fP/dMcq16tL6ykTzRb0aDp8LP++N43Y3dvxrlGLOr0NcwPmpN3hyPzZqPNyAfBv1sqgXaF0J+zRwVz563eOLXqXrh8tw7uGYVHKW2dPcWzxewDU6tnPqs/j6VcdT7/+Vp1rCr+w5oUk+c1bsQRWM/8zvXojCpVam8usRmBdQHuToFLnczU2yuJ1k29dJy8/R1f8o2WTHpy7eMTqeTtacmtvarR574TsFggEAoFAIDBNwgdfFNvY6TnZPP/DCqKTE1g3YgKta4YYtEs7fYXsLl+YktwFc3YXt+yWK5X0XP4px+YtJj3uQQ7t+kOeok6/PlaNYU1cW6tnPxRKQ2Hs36w1Mjc3NPn5HF3wNp3eWWgQw2rUai5u+pkrf20EoOmL4wpd25q41rOyVR9Dh9zN3aac3MaQ4tpTFd4hJzcLjUZNbNx5wpub/5lev7+DViaTU7GCJw3rtiMyajcymYy8vGyb4lpnSe5T8ZF0Ci780ELIbsdgVzHKgQMHsnnzZqZMmeLo+QhKEFsld0ZOPl4V7PonYzeS7I5DbflkgUMxJrKdIbt964bSd83KB7nRbMyBJpPJqDfoceoNehy1SoVMLjeb00vCp3YozUZO4Mza5Zz7aTXnflpN6CNP4V7Jh5w7qVzdvkV3bvcFK4zOq97AZ7l98Ry3L5zlwNuv4xUUTHDnnsjc3Lh5KkJXgLJKgyY0fn60TZ/L0cTGR5u9IVCpVcQnXaFa1Zo8N3AWXdoOBODg8c38snk+CUmXUalVKIzk8Na/BkCrpj15fuBsQmo1JT3jDiRbnl/xSG5DhOwWCAQCgUAgKFksSW7AYKevkN3lA1OSW6KkZbebhwedPnwHjUaDRq1GrjAd85jCUlxrLB6UyeV0/XAp+2dN4N6Na/w9dijV23fBN6QeaDREb/hed26zkRPwqR1aaAxr4tqn1p61+fM4ipXzj/PH38v5a+cqncQ2R2zceZTKijzWewxPPDweb6/KxMSe4ZfN8zl55h+b4lpbcZTkHrl+GOenbjbaLmR30bHLWtavX58PPviAgwcPEh4ejleBrRqvv277UxFByWLPSu7x3x5jxUvtnCK7IbdErynApMAubtkds2UfdR8bUui4I4p82HpDUrNbb9wr+XJi2Tzt3P5vo0G7b2h9Wo1/A4+q1Uxer9309zn382pid28nIyGOC78b5ssKe3Qw9QY+a9fNkiNxs1CFOz39NiOHvE/PLs8Z5E3r1v4pOoU/zu6Dv5CefhtfH3+TYyjd3Plw+h80qvegKrm3V2XAfL64kpDcEkJ2CwQCgUAgEJQM1khuKJzWUsjuso0lyS1R0rIbtIuoChaltHkME3GtqXjQKzCIHou+4si82WTfvkXi0YMkHj1ocE6b12cT0KqtyWtaimudibdXZYYNeotHeo3m6MmtFs9vWLcdw59+lyq+AbpjobWaMfu1nzh36Qg5FvJb68e1XLd+no6U3PX9G5g9T8juomGXsfzmm2+oXLkyERERREREGLTJZDIhul0ce9OVXE5Kd5rsFrgWxSW7pZsaY6LbWQS0aku/b37nzuVoUs+fQZ2bi7KSL9XbdqRilaoW+8sVCpoOH0ujZ0eSFHGEzCTtdjfPwBoEhncotD3NWbRqan7rma+PP327jzDa5qZQmmyz5RrGKEnJLSFkt0AgEAgEAkHxYq3klhCyu/xgjeSWcIbsdgYeVavx0CertKL7+GHy7qUhd3fHr1FzKtdtYNWOZXNxrStQxTeQfg+9ZPE8c+c0rtfBZJuEYVxrXYFOR0vu1UN+ABLMni9kt/3YZStjYqwrnCZwPYqSk3vlyHaMW3us3MnuiKPmKxwXJ1crmP/CUuVrtz1dPZRk/zXu9736byIKtwdPl706m+/naNmt/+Te1ZDJZFSp14gq9eyXlQqlOzU6dnPgrMo+zpDcEkJ2CwQCgUAgEBQPtkpuifImu105Di1IfOQtbb+D9s1ZP661VnJLlBfZDVCxSlVC+j5qd39HxLXljeKQ3NbGtUJ220eRTaVGowGw6gmSwLkUtfBk81qVy6Xs/uU7y3minE3i6VQST6cWaYxja6IN3ve2ILrBcbK70PY0kZK93ONMyS0hZLdAIBAIBAKBY7FXckuUJ9ldGuLQghz++lyR+ieeTqX3NNtFtSNkd9qVOKqEVLe5n6Ds4kzJLVEcsnshvWzqX9qw21J+//33LFq0iIsXLwLQoEEDpk+fzvDhwx02OYHjKKrkliiPsnvBEuc9EY7wNv8ASZ2v5vdX99NuVENCOtv3pXz130SOrYlm8JfdkOut6LZWmxdVdlubg01QfnAFyS0hZLdAIBAIBAKBYyiq5JYoL7LblePQguxfcgqAbpNb2HU9/bjWXooiu9OuxHFy0Q/0Wh5u9/UFZQtXkNwSjpbdC1lj1zxKC3YZyk8//ZR33nmHiRMn0qVLFwAOHDjAq6++SkpKClOmTHHoJAVFw1GSW6K8yW43ZdGLINqLQmndDYbCTY7CznlK6UrkBcfQWD+GvbLbFSX3tlFP4RtWn3pPDMG/WWurimDmZWZw/Z+tXNz4MwAyNzdqdOxB6ICBeAdZJ/3TE+LwdBEBmp5xh5FTG/P6y8vp1v4pu8bYf3Qjy76ZwNpPz90vOGkdriS5JYTsFggEAoFAICgajpLcEuVBdpeGOFRCWjBlb0yqu65b0frbI7slye0dHGDx3OIkPSGO6PXfcvOUtg5eBd8qhA4YSHDXXig9vSz216jVpJw5yek1X5B7Nw3A5rjW1Xhzbj9kMhkLZm+zq7+9ca0rSW4JR8ruso5d/9I///xzVq5cyccff8wTTzzBE088wcKFC1mxYgXLli1z9BwFRcDRkltCkt1SgcqMnHyHjCsovUiy2zs4gJOLfiDtSpzZ811RckukXblIxJK5bB/9NBlJ5otExB/ex66Jw3WSG0CTn0/cgV0ceOs1IpbMRZ1vusiFOj+PyC8/5cBbrzls/kXl9PkDAPy8aZ7dY/y0cS4AZ6IPWN3HFSW3xKQuLzC9+0gW7VvL0oM/2txfkt0N/UN4ft0MTsaXvq2oAoFAIBAIBPbgaMktIcnuuoHejFt7jNOxdxwyrqB0E/pEd8Ke6smVjbuJ2bLP7Ln6krvVtGElNEPjHHjrNZ3kBshJu835dWvZNXE48Yf2mu2bkZTA368+S8SSuTrJDbbFta5GVnY6V66f4vK1/8jKzrBrDHvi2pKS3PbEg5Lsjk65ygvrZ5Kek2nzGOVBdtsluhMSEujcuXAC386dO5OQULr+85RliktySwjZLSiItbLb0ZJbo1aTcjqK+IOHSI+z73dQXnoGCYeO0mnOIlqNn45c6Q7A/lkTyLp102if+EN7OfX1EgD8GjWj69xldF+wgg6z5tLouVEA3DwVwYll89GoCyce16hVHFv8PolHrZfBxY1Go2HT1qUA3LodT0Ky7cWHE5KukHpH+/ewcesyXS0Hc7iy5JZwtOwWCAQCgUAgKOsUl+SWELJbYAxrZHdBye3mUYGEQ0dJPHqcvAz7xOq9G3HEHzjErahzRuM/SzQcMoIei7+m64dLaTttDkEduwFwatVS4g8b/xxZt26yf9YENPlaH9Nu+vv0WPSVTXGtK/L33u+MvrYWe+LaklzJbe/iJ0fI7rKOXaK7Xr16/Prrr4WOr1+/nvr16xd5UoKiU9ySW0LIbkFBLMluayR36rloo8dNnbtz9AROfvYFUd98z6F3PmDHqHFW35xoNBqi1nzPntff4MyqtfiG1KV62070+eIHPAO0ec8jVywq1C8vM4NTq7RfnA2HjKDd9PfxDqqJZ0B1qtRvTEjfx+jywWcApJyJJPHYv4XGiD+8n9sXzgLQ6Z2Prf7MxUVGZhp7Dv1KTOwZAOQyBev++Jhbt+NRW3GjplaruXU7nl/++Bi5TAFAzPXT7Dn0KxmZaSb7lQbJLeFI2S0QCAQCgUBQliluyS0hZLfAGOZktzHJDXBm1VpOf/kNe157gzOrv7VqwQ5AXkYGO0aN4/Ccj4ha8z0nPlnGztETbIprO73zMaH9B+Lh5493cC38m7ai5StTaPjMCABOfb2E/KzCUvPIvNkAVKpZh95f/EDVxs3xqFrNprjWlcjLyyE2PppN277QHdu8/Qti46PJy8uxagx74tqSTldSlJ2+Qnabx67Eyu+//z5Dhw5l3759uhzdBw8eZNeuXUYFuKBkKSnJLVHecnYLLGMqZ7e1K7kjFi2h2yfzqVilstnrZN++Q8SiJQBUbdYE/+ZNif5lAwB7XnuDvmtWWpzr5U1/En/gEAChjw3QHZcrlXSYNY/dU0aRFnOJ7Nu3qFilqq497sA/AAR17EZo/4FGx65Usw6tJ8zg5PKF/PfVpwR16GrQfnq1NtVTx7c/xje0PnlZGZz/cyWZKXFUb9Gd2l0GIpPZlh8vLTaai9u0uaTr9x+Jby3rCrq8NKWRTkbL5QrUahUqdT7/Hv+Df4//gdKtAgH+tahVoxEvDXmfqlVqAJCSGse3G97lRnw0ySmx5OUb3nzI5QpWfDeZFd+Bl6cvQQFhNG3QmecHzUZ+P0+cMyV3ek5mkatW25uzWyAQCAQCgaCsUlKSW6I85OwW2I6xnN2mJDdo40GNSsXVrX+T8O8RvINrEDLgYYvX2fPaG7rXjYYN5eZ/p7l15qxNca1vqPG4LaT/kySfiuB2dBTX/9lG2KMPck1np6aQffsWAJ3eWYhcqSzU35q41rffgyKcGo2G6wc3k3hqH57+wTR6fBxKD8s5wvWxNa49cvL/+Hvvd9xIuMjttCQ0GrXB+RmZd5j6/kPIZHL8KlcnuHo9Hu7xIh1aPwJoF1z9vGkeURf+JSH5itVx7XcPrwCck5O7qDWcHJGzu6xi14ruwYMHc+TIEfz9/dm8eTObN2/G39+fo0ePMmjQIEfPUWADJS25JcTKbkFBCq7s/m/pOpvSlZxd+4PFc6J/Xg9Ao+HP0Wbqa9Tu24s+36zQtd+7YT5POEDM/7YC0GPJx9R76gmDtgq+lQnppz2WePywYb+tmwEI7tLT7PiB4R11r7PvpD54ffvBa9/QegAcWTGFi9u+Je74DiK+eYvrBzdbnL8+mamJ7J03jNjDfxF7+C/2znuBzNREq/rqr7hWq1WF2vPyc4hPukyrpj11khvA3y+Y1k17Ep90uZDkLjhWRmYaN2/F0rPLszrJDTh1JXdRcpsVdWW3QCAQCAQCQVmkpCW3hFjZLTCG/sru/5auMym5Aeo99QT1nxlE98+0i1Iubthkcfz0uHjd6z7frKBW74doM/U1Gj73DGBbXGsMmUxG0xFjAbjwu2HckRihjVGrt+9qVHJLWBPXSlw/uJmIb94i7vgOLm77lqMrp1icf0FsjWvbNOuNUlmR1DsJaDTaFdf6q+ml1xqNdgexu7sHbZr11rXL5XJ6dhnKzVuxNsW14LzCk46o4SRWdhvH7rKr4eHh/Pjjj0RERBAREcGPP/5I69atHTk3gY04S3JLCNktKIgku1VZOaScjMa/dUOrc3LfOnPW4jnJEZEABHVqrzsmk8loNGwoAPeuXTfbX6168MXn7uNj9ByldyUA8u4Zpt7ISbsNQMXKVQv1MUVe+j2913cN5gyQHHUQNGrtH5mcxFPmi6cUJPVyJPnZmWjUKjRqFfnZGaRejrSqbxXfQBRy4zsxZDIZMpmMCS8tpXfX5wu19+46jAkvLtGdZwyF3I0qvoF8OGMLwdXrGbQ5S3Kn56QXuZBHUWS3QCAQCAQCQVnDWZJbQshugTFCn+iOf+uGpJyMRpWVY1Ry61PB11f3Wj9mNMbdq9cAaPjcMwaxUPBD2vzatsS1ppArjMdpufe0MaVX9RpG2/WxFNdKJJ7aBzK5Li5NOnMQjRFhbAqNWmVzXKtUVmDa2FV0aP2I2ZXfMpmMDm0eZdrYVSiVhn9/wdXr8+GMLTbFtc6S3BJCdhcPdonu//u//2P79u2Fjm/fvp2tW7cWeVIC23G25JYQsltQkNgdR3Svb5+/arJAZUE8AqpZPEfprd1ClZWcYnD8ZuQpACpUrmy2v0xvVbGpYiE591dey93djbYrPDwszlNCKgRiajyPyoHam4r7ePnXtHpsAM+qwVYdM8aHM7bg6Wlc9ms0Gia8tJQeHZ8x2b9HpyFMeGmpyTx2Xl6+fDTjT4ICQgu1OUtyj/51uEOqVgvZLRAIBAKBQOB8yS0hZLegIGlX4rh9/qruvX6Magz9mEY/ZjSGtGAq5XSUwfHMBO3OWlviWlPk3NHKaLmb4aptxf2Y0s0K/2MprpXw9NeLH2VyPKoEIpMrLI7/oIvCrrhW6ebOlDFf0a5lf5PntGvZnymjv8RNYXz1elBAqE1xrTMlt4SQ3Y7HLtE9c+ZMVEaeamk0GmbOnFnkSQlsw1Ukt4SQ3QIJ/ZzcPVbONFmg0hhNXrKc97jlBO0WrsPvzeXe9VhUeXlc3/EPt6LOAeDX2Hx+aplMhm+9MACOfLiAnDuFCyZe/0f78M6vUXOD49VatgUg8egBs9fIvftgTM9qAbrXHv6ButeZKckAdJi4FI8qASCTE9isM42eeNXs2AXxC2tO86HTkbspkbspaT50On5hzS13BAL9a1OxgukbrJCaTS2OUSe4icm2ihW8CPCvZdVcLOEoyX0x5YJDbgqE7BYIBAKBQFDecRXJLSFkt0BCPyd3j5UzTRaoBMi5k0ZuejqH53wEgHdwDYs1k6o2bQxoV27H7t5HfnY2967Hcvi9eYBtca3Jz3D1EgA1OvcwOO7XqBkA0Ru+N7lwS8JSXCvR6PFxBDbrDDIZHlUC6DhxmcX5F8TeuFahcKNmUAMURkS2QqGkVo2GKEysbpewJa51tuSWELLbsdglui9evEiTJoWFRqNGjbh06VKRJyWwHleT3BJCdgsKFp4smLPbnOwObB+OX6MGFq9RpWF9AttrC2ccfm8e/4x9XVeMMnzGFItP3wHaTJkIwL1rseybavig7tqu/9O9rlzXcD4Nh7wIQPSv33PnykWjY+fnZHN08XsA1On7mMGTcLlCQeiAgQAc+mA6qpwc/MKaM+CTfxi0+j+6TP0at4q2Ff0AbQHKJ786yZNfnaR+/5FW98vJzSIlVft3Irv/9F1/y9d1K75sY/XOkfrK73/mm7dukJuXbfV8TOFIyb126E+AY24KhOwWCAQCgUBQXnE1yS0hZLfAWOFJ/ZzdBWX3vqkz2fv6dF3e7Xaz3zA2rAEyuZzw6ZMBOP/DL+weP0UnuW2Ja1U5hesdAdy7cY3z69YCENp/oEFb5boPFnZd3PSLyfGtiWsllB5edJn6NYNWn2LAJ/9QJbSZxfkXpChxbWz8edRqrT+SyeS62FStzrcqJrUlrnUFyS0hZLfjsEt0+/r6cuXKlULHL126hJeX7WJGYB+uKrklhOwuvxSU3BLWyu4Wr462+lotXh1N+Iwpui1hVZs1odsn8626odDOyYPeXy2jwdDByBT3v0Tz8oj6/ivO/bQagDavzy70JN87KJigDtq8a4c/epPUC2cNcpdlpiSzd/pY0m9oc7bVfbxw2g/pRiUv/R6H5j6Q7LZsDTOGTC63SvLrE594SVf4o3pAKONHfEbbVv107bFxlr9o9W8a2rXqz/gRnxFYLQTQFg6JSyzag1BHS279mxohuwUCgUAgEAhsx1Ult4SQ3eUXY5JbwpTsluLBek8PpPdXy3CzMk2lX+OGdPtkPgFt2wCg9Pam7ZtTbYprD82dSdatmwbHkiIOc3COthhktZZtC+XilslktHl9NgBX/vqdmK2byc/O0rVbG9cao6gxqb1jxMSeQaPRUMHdg2cem8Yzj02jgrsHGo2Gq7FnLPa3Ja51luRecdD4Knkhux2D+TX/JnjyySeZPHkymzZtom7duoBWck+bNo0nnnjCoRMUGMfVJbeEJLvHrT3G+G+PseKldnhVsOufnaCUYEpyS0iyO/KTnzi56AdaTx+Ob5h1eaRN4deoAV0XfGB3f7lSSZ1+fajTrw/bRj1l0NZs5AQCWrU12q/5yxPJvn2L2xfOcnTB2ybH7zr3c9zvF//Qx72SD13nfs6Bt17TCXFncT3+PNWq1mLo42/QtcNgFHIFPbs8y+Vr//HL5vlcj4+2OEZsXDQtmzzEcwNnUrdOSwC6d3qGA0d+Z/2fi4mNO09oLdtXBEDxSm4J6abg+XUzeGH9TH4cugDvCp42XWdSF+3WxEX71hq8FwgEAoFAIChruLrklpBk9/hvjzFu7TFWjmxH81qVnT0tQTFiTnJLSLHqlY27de/7rFpu9zUrVqlMy/Fj7O6ffuMae6cbT2Hi36wVrSdMN9oW0KotzUZO4Mza5URv+J7oDd8bPc9cXOsqZOdkci89lcf7jmNQ/4lU8vYDoN9DL7J523L+3vstObmZVHA3HaM5Iq41haMk99L9i5nedYDRdkl2v7B+Js+vm8HPzy6kdY1GNl3DEXFtacauFd0LFy7Ey8uLRo0aERoaSmhoKI0aNaJq1aosXrzY0XMUFKC0SG4JsbK7/GBJckvYksbEWVSu15DO731CzW69TZ4jd1PSfsYHtBgzyaDYhkTogEH0WvadyafkoH2C3mvZd4T0f9Ih87aXBmFtWfrBAXp0GoJC78l73ToteXvSOoYNmm1xjGFPvcXbk37RSW4AhVxBj05DWPr+fhrUte/GqiQkt4RY2S0QCAQCgUBgmdIiuSXEyu7ygzWSW8JcGpOSxlQ82HLsVMInv12oEKU+Nbv1pvN7n1ClQeEUw9bEta6CSp3P0g8OMuLpOTrJDeDjXZURT89h6QcHyVeZ90mOiGuN4UjJPamb+ZQ4YmV30bBraa2vry///vsvO3bs4L///sPDw4OWLVvSrVs3R89PUIDSJrklxMruso+1kluiOFZ2O4KuHy6lol9V3Dyse+Ipk8up0akHNTr1IPtOKnnp95Ar3fHwD0CusG6rlrt3JRoVyI1W0gQFhJptr1XDfGFPS+colRWofj+NiS2UpOSWECu7BQKBQCAQCExT2iS3hFjZXfaxRXJL6K/srvvYkOKeokkaDXmRBoNfICslGXVeLkrvSlSs7Ge54318aofSYeZH5Gdlkp16C8CmuNYV8PLwwcvDx2R71SpBFsdwRFxbEEdLbm1ca7zOl4RY2W0/NpnGQ4cOcevWLR577DFkMhkPP/wwCQkJvPvuu2RmZjJw4EA+//xzKlSw/MtEYB+lUXJLlFbZvebL00679h0LnlSdr809dfjrc1w/mmzXNeIjtV+C+5ecQu72YFVy48mWZbWErZJbwhVlt3dwLbv7VqzsZ9PNiMA8zpDcEkJ2CwQCgUAgEBSmtEpuidIku105Di2IFFPu++yUXdfTj2t7d7ZrCLskt4QtMWxxIlco8Aq0LHPN4ebhiXdw2ZeZJUXxSG7rKC7ZXdaxKXXJBx98QFRUlO796dOnGTNmDH379mXmzJn8+eefzJ8/3+GTFDygtEpuCZHGpOxhr+SWKA1pTAQljzMlt4RIYyIQCAQCgUDwgNIuuSVEGhPXxp54sCiSW8JVZLfAdXCm5JYojjQmZR2bltNGRkby4Ycf6t6vW7eO9u3bs2rVKgBq1arFu+++y3vvvefQSQoe4CzJnZ6TbWeim8KUtpXdo15t7rRrH/eWmW1X5anZMHovHV9pTEiX6nZd4+rBRA5/fY5uk1ugUD549nVTY7lvUSW3RMGV3b2Wh9s9VnGhVqnISklCnZuL0tuHilVsX7ldmreRFSQ3LxsAd6X9v48ysu7iWbFwkU5XkNwSYmW3QCAQCAQCQdmR3BKlYWW3K8ehBZFWcnef0sKu6+nHtcds3OnrCMldmsi+nUpe+l3k7u54+AdanS5TwhFxrSPRaDTIZLb9e9Pvm5l9z2y6E3txBckt4eiV3WUdm+zi7du3CQwM1L3fu3cvAwY8qBTarl07YmNjHTc7QSGcJbmf/2EFq1+17T+SOUqb7BYUxlGSW0JfdrsSuffuErNtMzFbNxdqaz76dWp07I5Mbn5zzN3rMUT98BVply8YHPepU5dmI8fjU9t8HjFXZNhE7Zw3fJVgV//Em1d57e1OdGv/FKt7LdEddyXJLSFkt0AgEAgEgvJOWZLcEqVBdpdHpJ2+1sju8iK5NWo1CUcPcurrzwq11enzKHUffwb3SuZlr7m49qm1Zx01VZv5fM1EXn95uV19l62ZwIGjm/jio8MEVqvjsDm5kuSWcKTsLuvYlLokMDCQmJgYAHJzczlx4gQdO3bUtd+7dw+l0nQlWEHpQ//JvaMRaUxcm/ysHJNtjpbcEpLsdhXSE27wz6SXjN4MAJxevYyjC+egzs8zOcaN/bv4971phSQ3wN1rl/n3vWnc2L/LUVMuEeKTLuteJyRdsWuMwxH/A+DUuQfVzV1RckuINCYCgUAgEAjKM2VNckuINCauh7VpLcuL5Fbn5xGx5COjkhvg2s6/+GfSS6QnmP5ZWYprncmpc/vQaKzYTl4AjUajiyUPn/if4+ZTQpLbnnjQUWlMyjo2ie5HHnmEmTNnsn//fmbNmoWnpyfdunXTtZ86dYq6des6fJIC51Bwe1pxIGS36xL5yU9GZXdxSW6Jgjco6rw8Yv7azo5R49g5egJXt+1AnWdaLBsj+/YdTnyyjB2jxnFg5hxSz0Vb7JN77y4H3tJ+KSm9fei+8Ev6r9lI/zUb6ffN73R8+2MAbl84y+lvvjA6RnLkMc6s1T6drtWzHw9/tV43xsNfr6fhkBcBOLN2OcmRx236TM5k49ZlyOUK5HIFG7cus7l/dk4mm7drfy5p91IA15bcEo6W3QKBQCAQCASlhbIouSWE7HYtrKnh5AzJfTv6Intef4Mdo8bx3/KvyL59x6b+luJaU/HgyS8WknImEoDWE2bo4sn+azbSa8lavGtqVzIfeOs1ctPvFepvbVzrLNLupXDyzD829ztxZhd372nTgm7a9gXZdsRmBSnJldz2Ln5yhOwu69gkuj/88EPc3Nzo0aMHq1atYtWqVbi7u+va16xZw8MPP+zwSQpKnpLMwSZkt2uSHpdcSHYXt+QuSH5WFrvGvs6l3zcD2i1bF3/dyK6xr5OflWXVGKnnotk/bRa3os4BkJV8k4hFSzj15Wqz/S7/uQEA75p1eGjRV3j6B+jaZDIZlcPq624KEo7sJyMx3qC/RqPhxDJtcd7Gw0bTdPhY5Ho7XuRuSkL7P0nYo08BcGLZPKs+D4BGrbL6XGvHiIr+l1u3E4w+Tc/Iusvlq5EcOLqJX/9czL7Dv6FWq1CrVew9vIFf/1zMgaObuHztPzKzCt9caTQabt1OICr6X3bu/5Hl304iMysNALlcUSokt4QjZbdAIBAIBAJBacFZknvJ3m0lch0hu10Lc7LbGZL71MrVHP/4U/LSMwBIjohk/7RZVi2gAuvi2hPL5hWKxdIT4rh5KgKALh98RmB4R4N2dx9fOr41H6W3tu6RsRXb1sS19qBRq9Go1Xb1fTCGCrlcwbcb3uV/O78i4tQO4hIvkZefW+jcvPxc4hIvEXFqB//b+RXfbXgPuVybnzwzK43l305i5/4fbYpr9SnpdCVF2ekrZLd5bEqK7O/vz759+0hLS8Pb2xtFgaT3GzZswNvb9n8MAtfCGYVGRM5u16P19OGcXPQDkZ/8RKtpw4jdcaREJTfA0XmLAPAOrkH4m1PQqFScXLKce9diOfHZF7SfPd1sf41aTcSiJQA0fH4IwT26kpmQxOH35pJ0NILbPXtQpWHhL3a1SsW1nX8B0P6N91BUMH7zVDmsPg2fGUH0hu85/+u3hL8+W9d25/KDm546vR8xOcf6g57nyl8bzX4OibysDI6smEJy1EE8KgfSYeJS/MJsK1KTeuU0h794new7yQQ07ULHCZ/hVtGL9z4dDECn8CeYPHol8vt5xw8e+4Mlq1/V9VfIC/+/3Ph/S1GptQ+oZDIZk0d/See2TwCgVqtZsnochyK2PBhDodTdeKjVKqdK7pPx54tUyKOoObsFAoFAIBAIBMZZsncbC//5i+d7D7B8sgMwlrO7QsPSWzy+tKNfw0nK2Q2UuOROPRdN0jGtbO743lt4BPiTcOgo53/4hYhFS+izernFmk3WxrV3Ll+gSr2Gun6SuG703Cgq1TSeg9qtQkU6zVnEvhmvErN1Ew0GP4/svgC2Nq61lQtb13B241IAmg6eTP3+ti3i0Y9r0WhIvnmN73/7AI1GK86rVqnBRzO24O+nzdGekhrHWx8/TuodbUpdmUyOXCZHfX/xlkaj4dh/2w1SmFgT137UbBLgnJzcRa3h5Iic3WUVm1Z0S/j6+haS3AB+fn4GK7wFpQ9nVtMWK7tdC9+wYFpPH056XDJ7xy0occmt0WjIiNN+kXX84G3cvb2p4OtLh3dmApB26YrFXF7SE/aqzZpQu09PFEollWrXpON7WiH93/KvjfbLSknSvXb38TV7jeodugJws8BWs9TzZwCobUZyA8jkcho+M8LsORLn/1xJctS/oNGQdSeZI8snWdVPn8NfvE5OWor2hiLqX85v+VLX1q5lP14b9bnuZgCgS7snGTfiU2QyGTKZDJU6X3cDAqDRqFGp83Xt44Z/qpPcAHK5nNdGLqNty366YyqVYeoZZ67kLkpus6Ku7BYIBAKBQCAQGEeS3DN6PVqi1y24slvgXPRXdh//YDXHP1hd4jm5z373E6CV3JVq18StYkVq9exO1WZNAHQ7h01hS1ybev60Qd/4w3sB8A0xnyJYf5V25s1k3Wtb4lprSb1yijO/Lkadn4c6P4/T6xeReuW05Y566Me1gC7GlMsV+PpUY86UX3WSG8DfL5g5U37Ft5I/crlCF4Pqox9jWhvXgnMLTxa1hpNY2W0cu0S3oGziTMktIWS3a+EbFkyVRiG697X6diixa+tvg5LJZA9e631ZWdoqlXM/b1q1loarnj0CqgGQl55utJ86t/BWKVPkZxkXnKr7Y1SoXMXyGDnZVl0rM0UvP51GTdbtJJvSmGjUKrLvJBv0yUi5AUCn8MeZOnYVSrfCDyt7dXmOiS9pc3Hr/11ISMcmvrSMnl2eLdSuVFZg2thVdGzzmNH+zpLcp+Iji1zIQ8hugUAgEAgEAseiL7kn9+hf4tfXl90C5+PmUYGwpx7SvQ976qESLTyZlXwTAM+gQIPjkujOvXvXbH9b4tqCcagmX+tDKvhajikl1HkPxrAlrrWWzFvxRo6ZLoRpdIyUwucr5G74+lTjoxlbqBFYWOwHV6/HR2/+ia9PNaO7jEH787UlrnWm5JYQstvxCNEtAFxDcksI2e06xGzZR8rJaPxbN0ThUcFkgcriQK63ayQnLU33Wv9GQm5kZ4k+lUJqA3D+p/UGq78TDh0BIKBtG6P9lN4+Vs8z53YqABUq+xkcd6+kHSPPSEGQgmQkWndjUL1Fd9CoQSYHmZzAZl1029KsQSZXENC0i64/GrV2TGDSyytwUyhN9u3e8WkG9Tf9xf3UgEl07/i0yXY3hZLJo1cSFBBm9XxN4SjJPXL9MIdUrRayWyAQCAQCgcAxGJPczogHJdktcD5pV+I49fmveNcKxLtWIKc+/9VogcriomrTxgDE7T2gO6bRaLiw7jcAfEJCzPa3Ja5VVjK+6lqtMv9/QD/WlfJ1a19bH9dai1/dVrhV9EImVyCTK3Cr6IVf3VY2jWEQ195Hpc7nrdd+onq1ENP9qoXw1ms/FVrNLVEjsK5Nca2zJbeEkN2ORYhugUtJbomCsltQ8ugXnmw56VldGpOSlN31nxkEwL4pM7nw60YubfyDvZPfBCD0cfMpQQAq1Xyw3Wnny+O59vcuTnz6Oed/WAdAw+eeMdqvYpUH0jop4rDZa8Qd3K2dT/8nDY5Xv18o5Or2LeSk3THZX52XR+LRg2avIVG7y0DCX55LcNu+NOg/kg7jP7Oqnz4dJ3xGg/4jCW7bl/CX51K7y0AAFArLOfGzstORG3l6Lpe7kZltWegrFG7cS0+1ec76OFJy1/dv4JCbAiG7BQKBQCAQCIqOqZXczlr8JGpGOR/9wpPhs0cSPnuk0QKVxUmTUdo0k9E//8qJT5YRu2sPO18er2v3Dg6yOIa1cW31tobFJusPHgbA2R++Npu2M+3KRd3rinoLsGyJa63F0686PWb/SK2Oj1Kr46P0mP0jnn7VbRpDP67Vx5qYMjPL9Ar6u+mpNsW1riC5JYTsdhxCdJdzXFFyS+jLbkHJUzAnt37O7pKS3XX69yWoszZdyrVtO4j5n7bqeo2unak78DGrxnjo8wfVlC+s+41bZ84CED59MhWrVDbZr8UrUwA4uXwh925cK9Su0WiI2bqZhCP7AajZrbdBe0U/f3xD6wFwZP5s1Hl5hcbIy8zg0IczdOdbQiaTUafrIDqM/4xmQ6bhVtHLYp+CuFX0otmQaXQY/xl1ug7SbZ+7m37LYt9rcWdRqfJQyN10uc0UcjdUqjyu3zCfmw4g7V4K9zJu3/8scl2VbGtxtORePeQHwDE3BUJ2CwQCgUAgENiPuXQlYqdv+URfcks5ufVzdpeU7K5YpTLh0ycD2nzc539ar2vTjzXNYU1cW9HPn4pVqhr0q91T+38h9fwZrm77w+jYd65c5PBcbb7vFmMK13CyJq61Fd9aDWk7ZgFtxyzAt1ZDyx0KoB/Xat9r1WRsnOU47Hp8tEEfuVyhe30vPdWmuNZZkjs9x7jjErLbMQjRXY5xZcktIcluQcljrPBkSctumUxGs9Ev8dDni2n+6ss0e2UkD32+mKajhhvN9WwMpZcXfdespNOHc2j68ghaT5lIn9XL8Wts/gs5qH0X/Ju1AuDgnClc/t9v3L0eQ3ZqCukJNzi6cA7RG74HoMUrk3HzKFyRvdV4bfXszOREdkwYRvyhvWQmJ5KdmsKtc6fZNXG47majw6y51v5YioXY+zcM5rh+/8ajTfPefDJnN5/M2U2b5r3vt1kW3TfiLwDgrqzIU49MYs0nUUx9xXhB0IIUh+TWv6kRslsgEAgEAoHAOVjKyS3SWpY/jEluCWfIbr/GDemzejltpr1O05dH0OmDt+m7ZiVKL+sWHlkT1xqLB5WeXrR4ZTIA0Ru+5+TyhaRE/UdWagpZqSnEbN3M4Y+0K8OrNGhCUIduhcawJq51JlNf+ZqAqrWABxLbHJIMD/CvzdRXvmbNJ1E8NeB13JUVte02xLXOktyjfx1usl3I7qIj9uKUU+yR3Kt2X2JMz3rFP7kCNK9VmQs4voiCwDwFJbeEJLtPLvqByE9+cnjF67QrcVQJMdz6pPTyonr7tkUa1zs4yKptZRIyuZw2r8/i5PJF3PzvOBc3/szFjT8XOq/FK5Op0dH4z8qjajW6zV/O/lkT0OTnc2rV0kLnyN2UdP1oKR5Vq1n/YYqBxOSrNG3Q2WR72t0UQms147mBs6gf2lp3fMb4b7kYc4KfN80n7W4Kvj6mV6Yn3ozh0d5jGNT/dd15ncIfh5jCq931KW7JLSHdFLywfibPr5vBz88upHWNRjZdR5Ldz6+bwQvrZ/Lj0AV4Vyj8EEQgEAgEAoFAYF3hSWnx07i1xxj/7TFWvNROpBUpw5iT3BKS7I785CdOLvqB1tOH4xsWbGQ0xyGTy3X5uu3FXFxrKh6s0bE7aDScWrWUpIjDRlOQBHXsRvNREw0KXOrP25q41ll0Cn+c9q0GsOfQr5y9eMji+ZnZ9xg34lN6dHxGl6bk2SffZEDPl9m4dalNca2tOEpyX0y5YPa8SV1eAGDRvrUG763FEXFtaUas6C6H2LuSe/nOi6zafal4JycoFRTXym7ppsZVkLspCZ80m65zP6fWQ/0M2hoMfoE+y380KbklvAKD6Lf6N8KnvE3lug9Wkbv7VCZ88tv0/fIXPANsy2lWHDzUeajZdm/vKsyZ8quB5JaoH9qGd6duwNvbfDXwhzo/y0tDPjArwwtSUpJbQqzsFggEAoFAICgZrJHcEgVrOImV3WUXS5Jbwhkru51FjU496P3FDzQeNho3zweryKu1aku3eV/Q8pUpyN1MF2C0FNc6G4XCjd5dn2f8i0ssnjvhpaX06vJcoVzcvj7+jBz6YZHiWnM4UnKvHfqTxfPFym77EY9ByxlFSVcyoU99lu/UFjlwxspuZ5Gfp3batVV55tNzqPO1c1Plq1HZOU/V/TGksXRY+O3g6JXd+k/uXQ3voGCajhhL0xFj7eovk8up1rwN1Zq3cfDMHIfCQr5sS+2OGkOfkpbcEmJlt0AgEAgEAkHxYovklihvK7tdOQ4tiC4utXPO+nGtNZJbwhkru52F0tOLOr0foU7vRyyfbIKixrXFTUnEnLbGpOB4ya2Nay9a6iZWdttJ2f1WEBSiqDm5Jbld3mT3zMn7nD0FixxbE82xNZZzUZnj91f3G7zv/e1DFvs4SnYX3J4mEDhLcksI2S0QCAQCgUBQPNgjuSXKk+wuDXFoQTaM3luk/sfWRNNj5UybYkpHye78rBzcHZeRU1BGKB7JbT3FIbuD6WXTGKUNl/hGWL58OYsWLSIxMZGWLVvy+eef0759e6Pnrlq1iu+//54zZ84AEB4ezrx580yeL9DiqMKT5VF2P/ei8554xVQw/yRdla/m2Jpoqjf3I6RToF3XuHooicTTqbQb1RCFm+3ZjIoqu43mYHPe4gWBC+BsyS0hZLdAIBAIBAKBYymK5JYoL7LblePQghz+WluYvuMr9uWw1o9r7Vk4VVTZnZ+VQ+QnP9F+9mybry0ouzhbcks4Wnbf/Xa8XfMoLTj922D9+vVMnTqVL7/8kg4dOrBkyRL69etHdHQ0AQGFUxjs2bOH5557js6dO1OxYkU+/vhjHn74YaKioggOLptbVIqKoyS3RHmT3eHtnZc/WeNtQXTnaW8IQjoFEtLF/nkmnk4lpHN1FMoHovumxvr+9spuawqNlDQXN6/Dq3oNAsM7olC6W90vOzWFxIjD5N67i8LdHb9Gzahct6GuirYlNBoN2HY/WWxoNBp++WMBj/V5BR/vqnaNcTf9Fv/b+TXP21jkw1Ukt4SQ3QKBQCAQCASOwRGSW6I8yG5XjkMLcv1oMoDdMal+XGsv9spuSXKnxyXbfW1HoNFouHPpPKnRUahyc3Gv5EP18I5U9LO+vpEqL5ekiMNkJMYD2BXXuhLH//sbmUxGeIu+dvUvSlzrKpJbwpGyu6zj9G+CTz/9lDFjxjBy5EgAvvzyS/766y/WrFnDzJmF/wJ++skwafvq1av5/fff2bVrFyNGjCiROZcmHC25Jcqb7BZYxlbZ7YqSG+Dyll91r2v17Efj50cjV5jO45V16yaRKxaRFmO8UGub12cT0Mp4ZW2J5MjjnFg2j6fWnrVv0g7m/KUjbNq6jH+Pb+GLjyxXvjbG7PmPkpRyjdbNetG4Xger+ria5JYoLtktEAgEAoFAUF5wpOSWKA+yW2AbtspufcndevrwEpxpYba/PLjQsfO/rKFilap0mD0Pj6rVTPZVq1Rc2ryOK3/9brTdmrjW1dBoNHy84kUAfv0y3uoFZPrYG9eWhOROz8m0efGTo2R3Wcf2PAUOJDc3l4iICPr06aM7JpfL6dOnD4cOWfePMDMzk7y8PPz8/Iy25+TkcPfuXYM/5YXiktwSY3rW0xWoXLXbuOQTlC8k2Z0el0zkJz+Rn5Vj9LzikNwajQa1SlW0MdRqQgcMwqdOGACxu7dzbNG7JsfNSEpg7/SxOskd0u8J6g8eRu3eA3TnnFg2jxv7d5m85o39uzixbF6R5u1odh34GYA7d5PJys6wuX9WdgZ37iYbjGUJV5XcEo6oWi3J7uiUq+XiSbpAIBAIBAIBFI/klpBk9+WkdMZ/e4yMnHyHji8ofUiy2zs4gJOLfiDtSpzR8wpKbt+wYDRqNRp10fJoqlUq7W5dO2n4zAjqPjGE6u27AJB9+xZ7p48lIynB5PWOLXpXJ7mrNm5BwyEjbIprXZETp3c+eH3GdDxtDnvi2pJayf3C+pmk52TaPP6kLi8wvftIFu1by9KDP9rcvzzsLHaq6E5JSUGlUhEYaLg9JTAwkMTERKvGePPNN6lRo4aBLNdn/vz5+Pr66v7UqlWryPMuDRS35JYQsltQEEuy2xrJnZ+VZfX18rOyODp3ETtfHs+uMRPZMWoclzZusenmIi3mGjtGjWPn6Ak0fGY4nd9dTNe5ywC4feEslzavK9RHo1azf9YEADwDguj52RoaDX2Juo8OpsmwMfRfs5HGw0YDcGbtcu5ejyk0xt3rMZxZuxyAsMeetnq+xUlCcgz7jmhvknJzs/h773c2j/H33m/JzcsGYN/h30i8edXs+a4uuSUcLbsFAoFAIBAIyjrFKbklhOwWFMSS7DYmuQF2jp7AztET2DFqHGmXC8dvptBoNFzd+jc7Ro1j15iJ7Hx5PP++84FNcW3YY0/Tb/VvhA4YSP2Bz9Lq1Wk8/NV6KtWsA8DBdyYblfDnfl7N7QvancHtpr9Pu+nvEdp/oE1xrauh0WhYv2UhcrkCuVzB+i0LbX54YE9cW5LpSqTFT86Q3WWdUr2vZ8GCBaxbt449e/ZQsWJFo+fMmjWLqVOn6t7fvXu3zMvukpLcEiKNiaAgptKYWLuSe/eEqfT+ahlypdLsddR5eeyeMLXQ8Zj/bSXnThpNR1nefpZ2OYajcxcWOu4dVJN209/XPR2v+8QzBrnNUs6cBECudKfrh0uMzrVO70dIj4slds92zv74NR1nzzdoP7N2BQANh75EaL8nAEi9cprMW3H41W2Fp5/tOe5UebkkRx0EIKBpF6vzsa36eSbxiZe5diMKuUyOSqPW5TQ7FLGF4KD6BAWEEhQQRrOGXfH10eaKS7ubwpnoAyQkXyEhOYa4hItcvXFWdyMil8mZPf8R6tRsSo3qdbVjBNalddNeyOXaZ62lQXJLODKNiUAgEAgEAkFZpiQkt4RIYyIoiKk0JqYkd0GOzl1Ihzkz8QmpY/FaUd98R8K/RwyOZcQl2BTXNnjq+ULH5Uol7Wd+xK6Jw1Hn55FyJpJqLdro2lW5OcTu3g5A17nL8A6qWWgMa+Jar5ZNDPpkpiaSejkSz6rB+IU1t/j5jWFLXHs1NorL1yKJT7pCQvIVYuOjSUx+8KAh5vppJs3pSs0aDQgKCKNGYBh167QipFZTANRqNSej/iEh6TIJyTFWx7Ut/B/VXaOkc3IXtYZTUdOYlGWc+pvf398fhUJBUlKSwfGkpCSqVzf/H2Hx4sUsWLCAnTt30qJFC5PnVahQgQoVXCP3b0lQ0pJbQshuQUEKyu6wpx7i1Oe/Wp2u5PrO3YQMeNj8Obv2AFCpTm06vPMmMrmc3Lt32Tv5TeIP/EuDoU+h9PIyO4YkuVuMH0Ng2zYGbVUbN8evcXNSz50mKeIINTp207Vdup/Lu8WYSWZvXBo/N4rYPdu5cyma/KxM3Dy0X2D5WZncvXYZgDr3U51c2LqGM78uBsCtohc9Zv+Ib62GZuevjyovl/0LR5J6SSvh/eq1pvuba5G7WZbdpp5wq1R5XL72H1eun0Kj0RDe4mHat3qQmsXToxL7j24i4pS2UEjBJ+0qdT73Mm5zJvoAZ6IPIJPJeGXYQp3kBpwquZce/LHIVavtld0CgUAgEAgEZZWSlNwSQnYLClJQdrd4bQhXNu4xKbn7rlkJQNLxE5xasYojHyzQHTNFXkaGTnJ3/2wBFXx90Wg0HJ7zEelx8VbHtXX7G49JlJ5eOlF9es3n9FqyVteWdEJ7XZ86YUYlt4SluLZOy2G692mx0eydN4z8bO1K4+ZDp1O//0iz8y+IrXGtm5uSdX8s5M7dZKMxJUBC8hUSb8ag0Wio7BPAe9N+07XJ5XJSbyfw3Yb3TF7DWFw77hGt6HZG4UljNZyE7HYMTk1d4u7uTnh4OLt2Pci3o1ar2bVrF506dTLZb+HChXz44Yds27aNtm3NF3krTzhLckuINCaCgkiyO+1SLCcX/oCHf2Wrc3Jf3LDJ4jmXftsMQOvJ45HdF6fuPj6EPqa9mU45HWW2v/42rYKSW6Ja89YAZCbFGxxPu6J9qOPhH2D2GvoSPDv1ltHXcjftOWc3LtUdU+Vmc3Hbg5sYa0iOOqiT3ACpl06SdOagVX1lMpnFAh/tWw1g2thVKJUP/v6UygpMG7vKQH6bG3/ciM/o083wC9iZK7mLktusqGlMBAKBQCAQCMoizpDcEiKNiaAgkuz28K/MyYU/kHYp1mKRSv3Y0FJqj1tR5wAIGfAwFXx9AW3sE/7mFMC2uNYUngHahaC5d9MMjmckamPUqk1aWryGpbhW4uK2tahyH6Qfjfp9iU3pTTRqtc1xbc2gBnzwxiYq+wQgl5kumCmXKajsE8AHb2wiuHp9g7a+3YczfsRnNsW14BzJLVGwhpNIY+IYnCq6AaZOncqqVav47rvvOHfuHOPGjSMjI4ORI7VPjEaMGMGsWbN053/88ce88847rFmzhpCQEBITE0lMTCQ9Pd1ZH8ElcLbklhCyW+AoZArLv56kL1xZgerR6nxtkQ2ZvOi/4nLvmS9g617Jp8jXcAVeG/m5yTaZTEan8CeY8spXKI2sDle6uTPlla/o2OZxszcVr438nJ6dhxY67izJveLgsiIX8hCyWyAQCAQCWL58OSEhIVSsWJEOHTpw9OhRk+dGRUUxePBgQkJCkMlkLFmypNA5UlvBPxMmTCjGTyFwBM6U3BJCdgtKEo3KeEyqkWJSG+JaUxQU3AVxr1TJ4jUsxbXOJigwjI9mbMHXpxpyeWHZLZcr8PWpxkdv/klQYJjRMXp2edamuNaZkltCyG7H43TRPXToUBYvXsycOXNo1aoVkZGRbNu2TVeg8vr16yQkPKguu3LlSnJzc3n66acJCgrS/Vm8eLGzPoLTcRXJLSFkt0BCysntW68WrWcMJyvljtEClcao//Qgi+fUGzwQgIiPP9Ntb8pJS+Path0AVG3a2Gx/fRGedPyE0XNuRf0HgGdgDYPj7j7ap/WZyeYL56rz83SvK/pVNfpanac9p+ngybpjCveKNm8RC2jaBb96rXXv/eq1JrBZF6v6duswmNrBxn9eGo2G5wfNxk1hOkWLm0LJsEGzTRYJqR3chG4dBls1F0s4SnIv3b/YIVWrhewWCAQCQXlm/fr1TJ06lXfffZcTJ07QsmVL+vXrR3JystHzMzMzCQsLY8GCBSbTVR47doyEhATdnx07tPd2zzzzTLF9DkHRcQXJLSFkt0BCysmdlXKH1jOG41uvltEClfrox4aWFk/5t9DmiY7531Zy72plskat5uTS5YBtca0psm5pf5/61m1gcNyrujZGTTkdafEaluJaifr9R6Jwf1ADr+ngyTYtIJPJ5XbHtYHV6jCg5yijbRqNhkd6vUygf22zY9gS1zpbcksI2e1YXCJh1cSJE5k4caLRtj179hi8v3r1avFPqBThapJbQuTsFhgrPGmsQKUpavXqYfEadR7uxaXfN5MeF8/Ol8cbtAV17mAxPzdAhzkzOfLBAk6t0G5d6r9mo64tPf4Gd69ri2AEhncw6Nd81EQilszl2KJ36f3FDyg9jV/r2s7/A8A3tJ4uPzeAm4cnvnUbkHb5AufWraHp8LHU7z+Sqg3a2l2MUqF0p/uba3XpSgKbdbEqPzdo00bFJ17WvZfLFajVKt37GwkXLN5U3Ei4YPBef4z4pMuo1WqD3Nz24FDJ3e0NoOi5zRyRs1sgEAgEgtLKp59+ypgxY3Q7cr/88kv++usv1qxZw8yZMwud365dO9q1awdgtB2gWrVqBu8XLFhA3bp16dHD8v2hwDm4kuSWEDm7BcYKT/qEBhcqUCmxY9Q4g/7t35ph8RpKLy9qdO1E/IFD7J38ZqF2a+NaU6jz8ji1ahkA9R4fYtAWGN4RgFvnTnHr3GmqNjZeONJSXKuPb62G9Jn7Z5GKURYlrr2RcAEZ2l3C0sputVqFXCYntkC8aQxb4lpXkNwSIme343D6im6B/biq5JYQK7vLL8YkNzzI2Z0el2x2ZXfP5Z9ZrEwN2vzXPZd/inew4VPp+kOeounLL1o1V5+QOkZvYG6dPcWBt7VferV69jOoTA3g36w1MjftjfLRBW/rVmVLaNRqLvz+I9G/aos8Nn3R8KYJoOnwsQDE7t7OtV1aIe4X1pya7frbLLkl5G7uBLXqSVCrnlZLboDkW9fJy8/RpR5p2aQHFSt4IZNpvyZi4yyvVL5+fzWzTCbHo6I3LRp3v/9eRl5eNjdvxdr6cQxwtOTWv6kRK7sFAoFAILCd3NxcIiIi6NOnj+6YXC6nT58+HDp0yGHX+PHHHxk1apTFvKsC5+CKkltCrOwuvxiT3PAgZ7d3cIDZld0d5szEt26oVddqMnI4oY8/YnCscr26NsW1UjyoT07aHQ68Mwl1Xi4A/s1aGbQrlO6EPardNXts0bukx98oNIY1cW1BPP2qU7Ndf7skt4S9ce3VG1Go1Nr/pzUC61IjsC4AKnU+V2PN1+AC2+JaZ0nuU/GRRo+Lld2OQTzOLKW4uuSWECu7yx+mJLeEJLvNrex286iItbh5eNDpw3fQaDRo1GrkCtPFK0zhWzeUvmtWolGridm+hZunIkg9dxqAKg2a0Pj50YX6yORyun64lP2zJnDvxjX+HjuU6u274BtSDzQaojd8rzu32cgJ+NQufJPkUzuUZiMncGbtcs79tJrGfabaPHdHERsfDUCrpj15fuBsQmo1JT3jDn/8vZy/dq7SSWyzY8SdR6msyGO9x/DEw+Px9qpMTOwZftk8n5Nn/iE2PprAanXsml9xSm4JsbJbIBAIBALbSElJQaVS6dJOSgQGBnL+vGMe+m7evJk7d+7w0ksvOWQ8gWNxZcktIVZ2lz9MSW4JSXYXXNndZ7U23YittZ5kMhn1Bj1OvUGPo1apkMnlNj+YO/fTas79tJrQR57CvZIPOXdSubp9i669+4IVRudVb+Cz3L54jtsXznLg7dfxCgomuHNPZG5uVse1roJKrSI+6QrVqtbkuYGz6NJ2IAAHj2/ml83zSUi6jEqtQmEkh7eEI+JaUzhKco9cP4zzUzcbbRcru4uO+O1eCiktkltCyO7ygyXJLWGN7LYVmUxWqACIzWPI5USv/1b3PuzRwdQb+KxJee4VGESPRV9xZN5ssm/fIvHoQRKPHjQ4p83rswlo1dbkNWt26417JV9OLJtXpLkXFaWbOx9O/4NG9drrjnl7VWbYoLd4pNdojp7canGMhnXbMfzpd6niG6A7FlqrGbNf+4lzl46QY8cTaSgZyS0hZLdAIBAIBK7FN998w4ABA6hRo4blkwUlSmmQ3BJCdpcfLEluCWOyu0qIfbtq9bFn4ZU+Mf+30eC9b2h9Wo1/A4+q1YyeL1coaDf9fc79vJrY3dvJSIjjwu+GK3ktxbWuQnr6bUYOeZ+eXZ4zqA/Vrf1TdAp/nN0HfyE9/Ta+Pv4mx3BEXGt0bg6U3PX9G5g9T8juoiF+s5cySpvklhCyu+xjreSWKA7Z7QjqPTkUz8AaBIZ3KLStyxgeVavx0CertKL7+GHy7qUhd3fHr1FzKtdtYNWT/IBWben3ze+OmL7dtGra02RbFd9A+j30ksUxzJ3TuF4Hk23mKEnJLSFkt0AgEAgE1uHv749CoSApKcngeFJSkslCk7Zw7do1du7cycaNGy2fLChRSpPklhCyu+xjreSWKCi7ey0PL6GZFqbfN79z53I0qefPoM7NRVnJl+ptO1KxSlWLfeUKBU2Hj6XRsyNJijhCZlI8gE1xrSvg6+NP3+4jjLa5KZQm2/RxRFxbEEdL7tVDfgASzJ4vZLf9iN/qpYzSKLklSqvsjjia6LRrX61gXpKq8tXa8w4lmT3P7DXu9736byIKtwfblrw6Wz+GrZJbwhVld70nh9rVr2KVqoT0fdTu64qck4VxhuSWELJbIBAIBALLuLu7Ex4ezq5duxg4cCCgLQS2a9cuJk6caL6zFaxdu5aAgAAefdT+eyyB4ymNkluiNMluV45DCxIfeUvb76B9c9aPa5vaEIfqY6vkltCX3c5EJpNRpV4jqtSzP2ZQKN2p0bGbA2clKA7JbW1cK2S3fbjmb3SBSUqr5JYojbL7l+9cv6hc4ulUEk+nFmmMY2uiDd73tvIGw17JLWFMdrs7f2G3wMk4U3JLCNktEAgEAoFlpk6dyosvvkjbtm1p3749S5YsISMjg5EjRwIwYsQIgoODmT9/PqAtLnn27Fnd67i4OCIjI/H29qZevQexgVqtZu3atbz44ou4uYmw1VUozZJborTI7tIQhxbk8NfnitQ/8XQqDbNybI4p7ZXcEpLsFgj0cabkligO2b2QXjb1L2243m9zgVlKs+SWKG2ye8GS7k67doS3+Sfp6nw1v7+6n3ajGhLS2b7toVf/TeTYmmgGf9kNud6Kbmu0eVElt0RB2d1+9my7xhGUDVxBcksI2S0QCAQCgXmGDh3KzZs3mTNnDomJibRq1Ypt27bpClRev34duV6xs/j4eFq3bq17v3jxYhYvXkyPHj3Ys2eP7vjOnTu5fv06o0aNKrHPIjBPWZDcEqVBdrtyHFqQ/UtOAdBtcgu7rqcf19q607eoklvCzaMCqO3qKiiDuILklnC07F7IGrvmUVpwrd/kAos4S3Iv2buN53s7brzSJLvdlM6rQqxQWneDoXCTo7BznlK6EnnBMTTm+zlKckvoy25XIz0hjuj133LzVAQAFXyrEDpgIMFde6H09LLYX6NWk3LmJKfXfEHu3TQAfMPqU++JIfg3a+1Sla6t5c25/ZDJZCyYvc2u/ukZdxg5tTGvv7ycFtUe1x13JcktIWS3QCAQCATmmThxoslUJfryGiAkJASNxsKNJvDwww9bdZ6gZChLklvC1WV3aYhDJaQFU/bGpLrruslJj0u2WnY7SnKXBvIyM7j+z1YubvwZAJmbGzU69iB0wEC8g6z73Kbi2ib9nBeb7D+6kW7tn7K777JvJrD203N4e1V22JxcSXJLOFJ2l3VKn10RlDjSTY2jGdOzHhP61Gf5zous2n3J4eMLig9HS24JSXa7Cur8PCK//JQDb72muxkAyEm7zfl1a9k1cTjxh/aaHSMjKYG/X32WiCVzdZIbIO3KRSKWzGX76KfJSDJfiMLVyMpO58r1U1y+9h9Z2Rl2jXH6/AEAft40T3fMFSW3xKQuLzC9+0gW7VvL0oM/Wu5QAEl2N/QP4fl1MzgZX/q2ogoEAoFAICi/lDXJLSHJ7stJ6Yz/9hgZOfnOnlK5p/X04TrZnZ+VY/K88iS54w/vY9fE4TrJDaDJzyfuwC4OvPUaEUvmos7PM9nfUlzrTPTjQVv5aeNcAM5EH3DUdEpMctsTD0qyOzrlKi+sn0l6TqbNY5QH2S1Et8As+k/uiwMhu12XmC37jB4vLsktYewGJS89g4RDR0k8epy8DPvE6r0bccQfOMStqHNo1Jb3pGnUKo4tfp/Eo9ovzYZDRtBj8dd0/XApbafNIeh+kY9Tq5YSf9j4zyrr1k32z5qAJl97w9xu+vv0WPQVneYsotX46cjvV7/eP2sCWbdu2vW5nMHfe78z+tpaNBoNm7YuBeDWbW1FcFeW3BKOlt0CgUAgEAgEpYWyKLklhOx2LaTFT+ZktzMkt0atJuV0FPEHD5EeZ99CJXNxral4MP7QXk59vQQAv0bN6Dp3Gd0XrKDDrLk0ek6b2unmqQhOLJtvNM61Nq51Frdux5OQHGNzv4SkK6Te0f49bNy6zCE7gEpyJbe9i58cIbvLOq6zL0fgchTcnpbM/xXLdUpTGpPyxJWNuwEIfeJBbrjiltwF0Wg0nF37A/EHDhkcD+rcgaYvv4hMZnlLXV5GBntee6PQ8fDpk/Fr3NBkv/jD+7l9QVsoqdM7H+MbWl/X5h1cC/+mrfCpFUr0hu859fUSAlq2xc3DcOvQkXnaXOOVatah/cyPdGlOPKpWwzekLgEt23LgnUlkJicSuWIRvee4doXsvLwcEm9eZdO2L3THNm//gjbNe1O9WghKpeV/DxmZaRyN3EZM7BkA5DJFqZDcEo5MYyIQCAQCgUBQWnCW5D554yrBIcV/HVdPY1LeKFjDST/2dIbkTj0XTcSiJYWOP/T5YpReVqSytCKujVyxiE7vLDRoz8vM4NQq7QKhhkNGENLvSV0M7BlQnSr1G1O1cXMOzplCyplIEo/9S1CHrgZjWBPXOhO5TMG6Pz5mxNNzqOJb3aCmgzHUajW30xL55Y+PkcsUqDT5xFw/zZ5Dv9K+VX+8PH0tXlOKa1sQpjtW0ulKpMVP9qS1dEQak7KM+M0tMEpJ52ATstv1CHuqp4HsLmnJDXB505+6m4HQxwagUam4uvVvEv49gndwDUIGPGxxDH3J3WjYUG7+d5pbZ84SsWgJ3T6ZT8UqlY32O716GQAd3za8GdAnpP+TJJ+K4HZ0FNf/2UbYow9yi2WnppB9+xYAnd5ZiFypLNRfrlTSYdY8dk8ZRVqM5R0NGo2G6wc3k3hqH57+wTR6fBxKD8s3VvrkZWVw/s+VZKbEUb1Fd2p3GYhMJuOlKY0ICgijaYPOPD9otu4G48jJ/+Pvvd9xI+Eit9OS0GjUBg8YMjLvMPX9h5DJ5PhVrk5w9Xo83ONFOrR+BNDeiPy8aR5RF/4lIfkKGZna9C1yuQK1WoVKne9UyZ2ek1nkqtX2ym6BQCAQCAQCgWlO3rjKs98vZ/+cniVyPWOyG+GOnIYx2Q2UuOTOvn1HJ7mrNmuCf/OmRP+yAdDGmn3XrLQ4hjVxbVrMJbJv36Jilaq6fnEH/gEgqGM3QvsPNDp2pZp1aD1hBieXL+S/rz4tJLqtiWttJS02movbtLFQ/f4j8a1legGZMfTjWpU6n3+P/8G/x/9A6VaBAP9a1KrRiJeGvE/VKjUASEmN49sN73IjPprklFjy8g1X+cvlClZ8N5kV34GXp6/Vce2jM68DzsnJXdQaTkJ2m0akLhEUwlmFRkQaE9ci9InuOtn939J1JS65AWL+txWAHks+pt5TT1D/mUF0/0wrCC9u2GSxf3pcvO51n29WUKv3Q7SZ+hoNn3sGgLNrjRe+zL6dqnvtG2r6oYtMJqPpiLEAXPjdMJVFYsRhAKq372pUcktU8K1MSL8nLHwSLdcPbibim7eIO76Di9u+5ejKKVb10+fIiilc3PYtccd3EPHNW1w/uBnQrrS+eSuWnl2eNXiK3qZZb5TKiqTeSUCj0W6F098WJr3WaNTcuh2Pu7sHbZo9qFwrl8vp2WUoN2/F6iQ3gFqt0r125kruouQ2K2oaE4FAIBAIBAKBcSTJ3TAgqESvWzCNicC56KcxiZi3loh5a0s8J3f0z+sBaDT8OdpMfY3afXvR55sVuvZ7N+IsjmFtXJt4/LBhv62bAQjuYv5hT2B4R93r7DsPYllr41pbyExNZO+8YcQe/ovYw3+xd94LZKYm2jSGflyrT15+DvFJl2nVtKdOcgP4+wXTumlP4pMuF5LcYBhb2hLXgvMKTzqihpNIY2IcIboFBji7mraQ3a5F6BPd8W/dkJST0aiyckpUcqtVD76s3H18dK8r+PoaPccYd69eA6Dhc88YrEIOfkibIuTWmbNG++Wl39W9tpQeRa4wvjEm9552DK/qNYy266P0rmTxHIDEU/tAJgeNGjRqks4cRKM2/zPQR6NWkRx1UNcfmVw7JlDFN5APZ2whuLrhDZBSWYFpY1fRofUjZn8WMpmMDm0eZdrYVYVSmARXr8+HM7ZQxTcQhbzwz8tZkjs9J73IhTyKIrsFAoFAIBAIBIXRl9w/Dx9f4tfXl90C5+MbFkyL14aQHptEemwSLV4bUqKFJ5MjIgEI6tRed0wmk9Fo2FAA7l27bra/LXFt3r0HC4NAWywSoGLlqlhLXvo9vdfWx7XWkno5kvzsTDRqFRq1ivzsDFIvR9o0hkFcqzc/mUzGhJeW0rvr84X69O46jAkvLtGdZwyF3M2muNZZkltCyO7iQYhugQ5nS24JIbtdh7Qrcdw+f1X3PnbHkRK7tkzv6at+UQ391cQyC/m7pBuJlNNRBsczE7RPnD0CqhntJ3d3t3qeOXe0Nx9yN8NV24r7Y7hVqGh5DL0n7ebw9Ne7oZPJ8agSiEyusHKmIJMr8KgcqL2puI+Xf00APprxJ0EBoUb7Kd3cmTLmK9q1NP17oV3L/kwZ/SVuCuOr14MCQvlwxhY8PX0KtTlLco/+dbhDqlYL2S0QCAQCgUDgGApKbm8r7qWLA0l2C5xPflYOVzbu0b2/snGP0QKVxYXSW5sqMis5xeD4zchTAFSoXNlsf1viWlNxqMLDw+r5ypUPxrAlrrUWz6qFHzIYO2Z2DP/C52s0Gia8tJQeHZ8x2a9HpyFMeGmpycKTXl6+NsW1zpTcEkJ2Ox4hugWA60huCSG7nY9+Tu4eK2fq0pjEbNlXIteXyWT41tMWhzjy4QJy7qSRm57O4TkfAeAdXMPiU+mqTRsD2pXbsbv3kZ+dzb3rsRx+bx4ATV4ynlvZwz9Q9zozJdnsNdKuav991ujcw+C4X6NmAERv+N5o9Wt9rv+z1Wy7RKPHxxHYrDPIZHhUCaDjxGVW9dOnw8SleFQJAJmcwGadafTEqwAE+Ncy20+hcKNmUAMURkS2QqGkVo2GKEysbpcI9K9NxQq25RQ3hqMk98WUCw65KRCyWyAQCAQCgaDomJLczooHm9eq7JTrCh6gX3iy7ZzRtJ0zmvS4ZCI/+anEZHfLCdpUlYffm8u967Go8vK4vuMfbkWdA8Cvsfn81LbEtX6Nmhv0rdayLQCJRw+YvUbu3QcrwT2rBehe2xLXWotfWHOaD52O3E2J3E1J86HT8QtrbrmjHvpxrT4hNZta7FsnuInJtooVvGyKa50tuSWE7HYsohilwOUkt4R+gcoej9Z28mzKHwVzcoc+0R3AoEBlcdNmykR2T5jKvWux7Js606Ct3ew3TPR6gEwuJ3z6ZCIWLeH8D79w/odfdG2B7cPxa9TAaD+5QkHogIHEbN3MoQ+m89Cir1FUKJyy5d6Na5xfpy3CUbA4SOW6D254Lm76hQaDhxm91rVd/2fxc0goPbzoMvVrNGqVTSu59fELa86AT/4pNEZuXjbuSvMrZmLjz6NW5wMgu78qXKNRo1bnc92KL+Oc3CxSUuN0/TUatdFUJuZwpOReO/QnwDGFPIpaoFIgEAgEgtJG3Ox/DN4vPfgji/atZXr3kXZ9D6bnZPLC+plEp1y1WJgreF4vm8cXuDbmVnIv33kReBAfCsoH+pJbPyd3wQKVxZ1as0rD+gS2DyfpaIRuwZRE+IwpFncZg/VxbeW6hvFpwyEvcvO/40T/+j1VGjSlcljhYpL5OdkcXfweAHX6PmYQ41kb19pK/f4jqffwi4DlXdbG0I9rt4xpjep+jHk9/jx1apoW2aCNSSUUcjdU6nzkcgVqtYqbt27YFNe6guSWkGS3KFBZdMSK7nKOq0puCWllt6DkMVZ4Ur9AZUms7Hbz8KD3V8toMHQwMoX211W9pwfS+6tluFm5fcuvcUO6fTKfgLZtAFB6e9P2zam0eHW02X6SuM5Lv8ehuTPJunXToD0p4jAH52iLQVZr2bZQLm6ZTEab12cDcOWv34nZupn87Cxduzovj6jvv+LcT6sBdOdag72S29wYcYmWV8rExJ5Bo9FQwd2DZx6bxjOPTaOCuwcajYarsWcs9o9PvKQr/FE9IJTxIz6jbat+Vs/Z0ZJb/6ZGrOwWCAQCgaBoOKJQc1FXtAlKJ5bSlYidvuUPU5IbDAtUltTK7havjiZ8xhRd6suqzZrQ7ZP5JhdOFcSauLbN67ML7Vj2DgomqIO2vtThj94k9cJZgxpNmSnJ7J0+lvQb2tpUdR8vnPbDmrjWHmRyuV2S23AMhUE8GBtn+fe+/gKrdq36M37EZwRWCwG0i7BsiWudJblXHDS+M1us7HYMYkV3OcZWyX069o5Ttm+N6VmPC+SW+HXLO6aejhf3yu78rBzc9S4rVyqp068Pdfr1sXvMilUq03L8GJv6uFfyoevczznw1muk37jG3uljjZ7n36wVrSdMN9oW0KotzUZO4Mza5URv+J7oDd8bPa/ZyAkEtGpr0/wcTWzceUJrNTPZnp2Tyb30VB7vO45B/SdSydsPgH4Pvcjmbcv5e++35ORmUsHd9BPj6/HnqVa1FkMff4OuHQajkCvo2eVZiMmzOL/ilNwSYmW3QCAQCARFo6jfg45Y0SYoXViTk1t/p6/+e0HZxJzklpBkd0mu7PZr1ICuCz6wu7+luNZUPNj85Ylk377F7QtnObrgbZPjd537Oe7elQodtyauDV37knUfohh4Y+xqLl/7j182z+d6fLTF82PjomnZ5CGeGziTunVaAtC90zMcOPI76/9cbFNcayuOktxL9y9metcBRtvFyu6iI1Z0l1PsWck9bu0xTsfeKd6JCVwGczcKxbWyW7qpcRW8g4Lptew7Qvo/abS95diphE9+u1AhSn1qdutN5/c+oUqDwluwKtdrSOf3PqFmt94Om7O9NKhrXrSr1Pks/eAgI56eo5PcAD7eVRnx9ByWfnCQfFW++WuEtWXpBwfo0WkIChtWpZeE5JYQK7sFAoFAICgaYmW3wFpsKTwpajiVHyxJbglnrOx2BnI3Je1nfECLMZNAVljhhQ4YRK9l3+EdZPpnZSmudTZ167Tk7UnrGDbI8i7nYU+9xduTftFJbgCFXEGPTkNY+v5+m+JaW3Ck5J7UzXwaVrGyu2iIFd3lEHvTldQN9Gbc2mOsHNlOFOYQOHxlt/6Te1fC3bsSjYa8SIPBL5CVkow6LxeldyUqVvaz3Pk+PrVD6TDzI/KzMslOvQVARb+quHm4zlPV6ve3e5nCy8MHLw8fk+1VqwRZvIap6tfmKEnJLSFWdgsEAoFAUDTEym6BJWyR3BJiZXf5wBrJLeGMld3OQCaXU6NTD2p06kH2nVTy0u8hV7rj4R+AXGHdAiJHxLXFTa0a5gt7WjpHqaxQ5LjWGI6W3Nq49qLZ88XKbvsRorucUZSc3Cteasf4b4+VO9m95svTTrv2HQvfWep8bb7jw1+f4/pR+wRxfKRWvO5fcgq524MnxI0nWxbXjpLdBbenuSJyhQKvQMsy1xxuHp54B5f9LxZH4QzJLSFkt0AgEAgERUPIboEp7JHcEuVJdrtyHFoQKabc99kpu66nH9e2nTPaKsktUV5kt0TFyn5FktOOiGvLE8Ujua2juGR3WUekLilHFLXwpFcFN1a81E63slukMRFA0dOYWJODTVD+cKbklhBpTAQCgUAgKBoijYmgIEWR3BIijUnZxp540BFpTByZklNQNnCm5JYojjQmZR2xorucUFTJLSHJ7vK0snvUq82ddu3j3jKz7ao8NRtG76XjK40J6VLdrmtcPZjI4a/P0W1yCxTKB8++bmqsH8Peld2uKLnT42LtTiuSfTuVvPS7yN3d8fAPtHobmSuSkXUXz4qVClX+thaNRkNm9j2bt4WBa0huCbGyWyAQCASColEcK7uD6eXweQqKH0dIbonysLLblePQgkgrubtPaWHX9fTjWnspysrumC37uLJxN3UfG2L39R2BWqUiKyUJdW4uSm8fKlaxfeW2K6fLtJXcvGwA3JX2/66wN651Bckt4eiV3WUdsaK7HOAoyS0hVnYLjGHrym5XlNwAB96ZxM4JL/Dv+9O5ez3G4vkatZr4w/vZNuop9kwbzcF3p7J/1kT+HvMM537+htx7dy2OkXvvLtEbvnfE9B1C4s2rvDS5IZ+vmWj3GMvWTOClyQ1JunnNpn6uJLklxMpugUBQFJYvX05ISAgVK1akQ4cOHD161OS5UVFRDB48mJCQEGQyGUuWLCl0zsqVK2nRogU+Pj74+PjQqVMntm7dWoyfQCAoOo5e2S0ofThSckuIld2CgtizsluS3GFP9SyBGZomesP3/D3mGfbPmsjBd6eyZ9poto16irh/96BRqy32v3s9hkNzZ7JzwgsceGeSzXGtKzJsYijDJtpe50nC3rjWlSS3hCNXdpd1hOgu4zhacksI2S0whrWy21Ultz53r13m3/emcWP/LpPnqPPziFjyEae+/sxo+7Wdf/HPpJdIT4gzOUZ6wg3+mfQSMVs3F3XKDuNwxP8AOHVuHxqNDUv776PRaDh1Tvv3f/jE/6zu54qSW0LIboFAYA/r169n6tSpvPvuu5w4cYKWLVvSr18/kpON19XIzMwkLCyMBQsWUL268Z1aNWvWZMGCBURERHD8+HF69erFk08+SVRUVHF+FIGgyDhSdgtKF8UhuSWE7BYUxBbZrS+57a035ShMxYOnVy/j6MI5qPPzTPa9sX8X/743jbTLFwq1WRPXuiLxSZd1rxOSrtg1hj1xbUlJbmel8yoPtS6E6C7DFJfklhCyW2AMS7K7OCT37eiL7Hn9DXaMGsd/y78i+/Ydm/qr8/KI+Ws7O0aNo/+ajTz89XoaDnkRgDNrl5Mcedxov5NfLCTlTCQArSfMoP+ajbo/vZasxbtmHQAOvPUauen3CvXPvXeXA29pv/iU3ran+CgOsnMy2bx9OQBp91I4eeYfm8c4cWYXd+9pt8tt2vYF2VZIYVeW3BKOlt0CgaDs8+mnnzJmzBhGjhxJkyZN+PLLL/H09GTNmjVGz2/Xrh2LFi3i2WefpUIF49utH3/8cR555BHq169PgwYNmDt3Lt7e3hw+fLg4P4pAUAhnPPSVgnxB6aE4JbeEkN2Cglgju41J7h2jxrFz9ASubtuBOs+0WDZG9u07nPhkGTtGjePAzDmknou2qb/S24fuC7/UxZP9vvmdjm9/DMDtC2c5/c0XRvslRx7jzFpt/FarZz8e/mq9bgxr41pXZOPWZcjlCuRyBRu3LrO5vz1xbUmu5Ba1K4oPIbrLKMUtuSWE7BYYw5TstlZyn/pytdXXOrVyNcc//pS89AwAkiMi2T9tltU3FvlZWewa+zqXft+sOyZ3UxLa/0nCHn0KgBPL5hV6ApyeEMfNUxEAdPngMwLDOxq0u/v40vGt+Si9KwHGn9Bf/nMDAN416/DQoq90xzVqlVVzN4VGrbZqe5s+UdH/snP/jyz/dhKZWWkAyOUKvt3wLv/b+RURp3YQl3iJvPzcQn3z8nOJS7xExKkd/G/nV3y34T3kcm1+8sysNJZ/O4md+38kKvpfbt1OKPSzLA2SW8KRslsgEJRtcnNziYiIoE+fPrpjcrmcPn36cOjQIYdcQ6VSsW7dOjIyMujUqZNDxhQIrMVZO5xsrZUhcB4lIbklhOwWFMSc7Da3klujVnPx143sGvs6+VlZVl0r9Vw0+6fN4lbUOQCykm8SsWiJTXHtQ4u+wtM/QPdeJpNROay+TnYnHNlPRmK84Vw1Gk4smw9A42GjaTp8LHKlUtdubVxrjKLGpLaOkZF1l8tXIzlwdBO//rmYfYd/Q61WoVar2Ht4A7/+uZgDRzdx+dp/ZGYVXkSm0Wi4dTvB7ri2pNOViELNxYcoRlkGKSnJLVEeC1QKLFOwQGWtvh2sXsmddDSC2z17UKVhfbPXSD0XTdIxrWzu+N5beAT4k3DoKOd/+IWIRUvos3o5Mrn553lH5y0CwDu4BuFvTjFoqz/oea78tRGAO5cvUKVeQ12bJK4bPTeKSvdXbhfErUJFOs1ZxL4ZrxKzdRMNBj+P7L4AVqtUXNv5FwDt33gPRYUKpF45zeEvXif7TjIBTbvQccJnuFX0Mjv/glzYuoazG5cC0HTwZOr3t06ovvfpYAAUCqXu5ketVpF88xrf//YBGo1WnFetUoOPZmzB30/795eSGsdbHz9O6p0EAGQyOXKZHPX9mxqNRsOx/7YbpDDpFP4Ek0evRH7/78aZkvtk/PkiFfIoaoFKgUBQdklJSUGlUhEYGGhwPDAwkPPnixaQnD59mk6dOpGdnY23tzebNm2iSZMmRRpTILAV6aGvKNQsMEZJSm6J8lCgUmAbxgpUxu44YlJy91i2CI1Kxckly7l3LZYTn31B+9nTzV5Do1YTsWgJAA2fH0Jwj65kJiRx+L25NsW11RobT51SOaw+DZ8ZQfSG7zn/67eEvz5b13bn8oPFXXV6P2JyfEtxbcDLvXXv87IyOLJiCslRB/GoHEiHiUvxC7OtMKqtce3BY3+wZPWruvcKeWFVufH/lqJS5wPahwCTR39J57ZPAKBWq1myehyHIrY8GMOKuPbfsdrdcM7IyV0chZrLQ1oSaxArussYJS25JcTKboEx9Fd27x23wKZ0Jf8t/9riOWe/+wnQSu5KtWviVrEitXp2p2ozbbAvi5VAvwABAABJREFUPVE3hUajISNOK2g7fvA27t6GX2gyuZyGz4wAIPX8aYO2+MN7AfANqWv2GvpP5TNvPsjJmpWSpHvt7uMLwOEvXicnLQU0GpKj/uX8li/Njl2Q1CunOPPrYtT5eajz8zi9fhGpV05b7qiHSmW4RU+lzkejUSOXK/D1qcacKb/qJDeAv18wc6b8im8lf+RyBRqNWncDYmzMdi378dqoz3WSG3DqSu6iFvIoyspugUAgsJeGDRsSGRnJkSNHGDduHC+++CJnz5519rQE5QxRu0JgCmdIbgmxsltQEP2V3XvHLTCbk9vd25sKvr50eGcmAGmXrljM7SztJK7arAm1+/REoVRSqXZNOr6nFdK2xLWmqN6hKwA3C6QeST1/BoDaZiQ3WBfXSpz/cyXJUf+CRkPWnWSOLJ9kcf4FsTWu7dLuScaN+BSZTIZMJtPFoBJSjCm1jxv+qU5yg3bH3Gsjl9G2ZT/dMWviWnBu4UlHF2oWK7u1CNFdhnCW5JYQsltgjFp9O+heV2kUYnVO7rz0dIvnZCXfBMAzyHC1nCS6c+/eNdtfP72HTCYzek5+TjYA6lzDrU2afK3MreBbxeI8JdR5D8YoOB5A9p1kg+1dGSk3rB4bIPNWvJFjpgth6tOxzWMmfwYKuRu+PtX4aMYWagQWFvvB1evx0Zt/4utTzejTd9D+fDuFP87UsatQurkbtDlLcp+Kj3RI1WohuwUCgTH8/f1RKBQkJSUZHE9KSjJZaNJa3N3dqVevHuHh4cyfP5+WLVuydOnSIo0pcAzLly8nJCSEihUr0qFDB44ePWry3KioKAYPHkxISAgymYwlS5YUOmflypW0aNECHx8ffHx86NSpE1u3bi3GT2A9olCzwBjOlNwSQnYLCuIbFkyVRiG69/oxqjH0dwVbSgmZc78+VLWWhquePQKqAbbFtabIzzL++1V1P6asUNlyTGoprpXITNGLHzVqsm4n2ZSCRKNW2RXX9uryHBNf0ubiNhaXSscmvrSMnl2eLdSuVFZg2thVNsW1zpTcEkJ2Ox4hussIzpbcEkJ2C/SRcnIrPCrg37ohKSejjRaoNEZA2zYWz6natDEAcXsP6I5pNBourPsNAJ+QELP95QqF7nVOWprRczIStV/0ykq+RtvVqnyjx/XnIyHl69a+Llx8MqBpF5DJtX80aqq3sK3yt1/dVrhV9EImVyCTK3Cr6IVf3VZW9Z08eiVBAWFG21TqfN567SeqVwsx2b96tRDeeu2nQqu5JWoE1mXSyytwUyiNttuCoyT3yPXDHFK1WshugUBgDHd3d8LDw9m1a5fumFqtZteuXQ7Pp61Wq8nJKVxoy5mUJ+ErsX79eqZOncq7777LiRMnaNmyJf369SM5Odno+ZmZmYSFhbFgwQKTDz9q1qzJggULiIiI4Pjx4/Tq1Ysnn3ySqKioQuc642cuZLdAH1eQ3BJCdgv0idmyj5ST0fi3bojCo4LJApUS+gum9GNGY1QKqQ3A+Z/WG8R+CYeOALbFtabIuZ0KQIXKfgbH3StpY8q89MI5qwtiKa6VqN6iO2jUurg0sFkXXfpNa5DJFXbHtd07Ps2g/qZl8VMDJtG949Mm290USpviWmdLbgkhux2LEN1lAFeR3BJCdgugcOHJlpOeNVqg0hQNn3vG4jlNRmm3X0X//CsnPllG7K497Hx5vK7dOzjI4hj1nxkEwL4pM7nw60aDNnVeHolHDwJQva1hscn6g4cBcPaHr81uZ0u7clH3uqLejUnFKg9eJ0Voc4N1nPAZDfqPJLhtX8JfnkvtLgMtzl8fT7/q9Jj9I7U6Pkqtjo/SY/aPePpZt2pQoXDjXnqqyfbMbMs3T5lZplfQ301PRaEoelkIR0ru+v4NHHJTIGS3QCAwxdSpU1m1ahXfffcd586dY9y4cWRkZDBypLZ+wogRI5g1a5bu/NzcXCIjI4mMjCQ3N5e4uDgiIyO5dOmBpJk1axb79u3j6tWrnD59mlmzZrFnzx6GDRtW4p/PFM4Wvs7i008/ZcyYMYwcOZImTZrw5Zdf4unpyZo1a4ye365dOxYtWsSzzz5LhQoVjJ7z+OOP88gjj1C/fn0aNGjA3Llz8fb25vDhwwbnOfNnLmS3AFxLcksI2S0Aw8KTLSc9a7JAJcCFXzdyaeMf7J38JgChj5tPCQJQqeaDHcs7Xx7Ptb93ceLTzzn/wzrAtrjWFHEHtXWvQvs/aXC8erg2Rr26fQs5aXdM9rcmrpWo3WUg4S/PJbhtXxr0H0mH8Z9ZnH9BihLXZmWnIzeyS1gud7MqJrUlrnUFyS0hZLfjEKK7lONqkltCyO7yTUHJLaUr0c/ZbU52h0+fTMUqlS1ep2KVyoRPnwxo83Gf/2m9ru2hzxdbNdc6/fsS1Fm7de3ath2643mZGRz6cIb2On7+VKxS1aBf7Z7a/2+p589wddsfRse+c+Uih+dq87u1GFM4t1mLV7TFL08uX8i9G9dwq+hFsyHT6DD+M+p0HWRyy5U5fGs1pO2YBbQdswDfWg0td7hP2r0U7mXcBu4XlLz/1F4m035NxMZZ/qK8Hh9t0EcuV+he30tP5W76Les/iBEcLblXD/kBcMxNgZDdAoHAGEOHDmXx4sXMmTOHVq1aERkZybZt23QFKq9fv05CQoLu/Pj4eFq3bk3r1q1JSEhg8eLFtG7dmtGjR+vOSU5OZsSIETRs2JDevXtz7Ngxtm/fTt++fUv885nCmcLXWeTm5hIREUGfPn10x+RyOX369OHQoUMOuYZKpWLdunVkZGQU2hXg7J+5kN3lG1eU3BJCdpdv9CW3lJNbP2d3Qdl9bdsOYv63DYAaXTtTd+BjVl1HP/a8sO43bp3R1s2wJa69d+NaoeMajYaYrZtJOLIfgJrdehv28/PHN1RbdPXI/Nmo8/IKjWFtXCshk8mo03UQHcZ/RrMh08wWkTRFUeLaa3FnUanyUMjddDm5FXI3VKo8rt8wX4MLbItrnSW503OMp7MRstsxFH15ncBpuKrklpBk9/hvjzFu7TFWjmxH81qVnT0tQTFjSnJLSDcYVzbuNnivj19j6wWtX+OG9Fm9nNRz0eTcuYNPndp417QuDzhov8ibjX6Jhs89w62oc2SnppCRlMCxRe/qzukwa26hfkpPL1q8MplTXy8hesP33LlygVoP9cMrSHvtxCMHiN7wPQBVGjQhqEO3QmMEte9C/L+7STkTycE5U3hqrfMKid2IvwCAu7Iijz88jsf7jOXUuX38tHEuSSnXdBLbHNJNQ4B/bYYNmk2Lxt35c8eX/LnjS3LzsomNj6Zpg852za84JLf+TY0jqlZLQf7z62bwwvqZ/Dh0Ad4VPG2ep0AgKFtMnDiRiRMnGm3bs2ePwfuQkBCLRa+++eYbR02tWJCEr/5K9eIQvhs2bDAqfJ1FSkoKKpVK9xBDIjAwkPPnixZonj59mk6dOpGdnY23tzebNm2iSZMmunZX+Zk74ntwUpcXAFi0b63Be4Hr4sqSW2JMT60IXL7zosF7QdnGmOSWkGT3yUU/EPnJT7SaNgw3jwo0f/VlNGo1/s2bovSyXvAqvbzou2Yl6XEJ3L16FXcfH6o2bWyQ69sSB+dMof5Tz1OtRTju3pXIz8km6vuvuB2t3UXT4pXJuHkU/p3aavx09k4fS2ZyIjsmDKP5yAlUrtsQuZub1XGtK3H9fkzZpnlvnhuo/V77ZfN8jv23netxlkW3LXGtsyT36F+H8+eIhUbbi/o96Ii4trQjRHcpxdUlt4SQ3eULS5JbwhrZbQsyudxiXjNLKL28qN6+LTtGvaI7JndT0vWjpXhUrWa0T42O3UGj4dSqpSRFHNalINEnqGM3mo+aaPQmRyaX0+b1WZxcvoib/x0v1F6SJN6M4dHeYxjU/3V8ffwB6BT+OO1bDWDPoV85e9FyoJyZfY9xIz6lR8dndGlKnn3yTQb0fJmNW5eSmHzVLtFd3JJbQshugUAgKDrOFL5llYYNGxIZGUlaWhq//fYbL774Inv37tV9dlf6mQvZXb4oDZJbQsju8oU5yS1hTHZXb9+2SNf1Dg6yKn2mKS5u/JmLG38udLzFK5O1sacRPKpWo9v85eyfNQFNfj6nVhUuTm0prnUV0u6mEFqrGc8NnEX90Na64zPGf8vFmBP8vGk+aXdTdPGqMRwR15rCUZL7YsoFs+cJ2V00hOguhZQWyS0hZHf5wFrJLeFo2e1I3H0q03zURPybtbL4FL5Gpx5Ua9mW+EN7ubjpF/IzMwCo1qotjYa8hFf1Gmb7y92UhE+aTXpCnNnzipuHOj+LwkiREYXCjd5dn+ehzkMtjjHhpaVGx/D18Wfk0A9R2VCtW6KkJLeEkN0CgUDgulgSvs7E398fhUJBUlKSwfGkpCSTObCtxd3dnXr1tFIuPDycY8eOsXTpUr766qsijWsN9vzMhewuH5QmyS0hZHf5wBrJLVFQdrefPbuEZlmYrnM/59qO/xG7Z7vuWIPBL1C7V3+jK7n18QoMot/q30iJiuTylg3cuaxdtWxLXOsKeHtXYc6UX4221Q9tw7tTN1iMKR0R1xrDkZJ77dCfLJ4vZLf9CNFdyihtkluiNMvu/Dy1066tyjOfy0qdr52bKl+Nys55qu6PIY2lw4bfDrZKbglXlN3912y0fFIBlJ5e1On9CHV6Wy5WYgrvIOvTrRQHxm4GbGl31Bj6lLTklhCyWyAQCOynrApfS7i7uxMeHs6uXbsYOHAgAGq1ml27dplMXWMvarWanJwHOWVd8WcuZHfZpjRKbonSJLtdOQ4tiC4utXPO+nFtUbBFckvoy25n4h0UTNMRY2k6Yqxd/WVyOdWat6Fa8zYOnlnJURIxp60xKThecmvj2osW+wnZbR9CdJcySqPkliitsnvmZNNFE12FY2uiObbGcg5lc/z+6n6D972/fciqfvZKbglXlN0C5+MsyS0hZLdAIBDYhzOFr7OZOnUqL774Im3btqV9+/YsWbKEjIwMRo4cCcCIESMIDg5m/vz5gDa39tmzZ3Wv4+LiiIyMxNvbWyeXZ82axYABA6hduzb37t3j559/Zs+ePWzf/mDFn6v+zIXsLpuUZsktUVpkd2mIQwuyYfTeIvU/tiaa3naGg/ZIbglJdgsEBSkeyW09xSG7g+ll0xilDSG6SxmlVXJLlEbZ/dyLznviFVPB/JN0Vb6aY2uiqd7cj5BOgWbPNcXVQ0kknk6l3aiGKNxs285UVMktUVB2131siF3jCMoGzpbcEkJ2CwQCgX04S/g6m6FDh3Lz5k3mzJlDYmIirVq1Ytu2bbrc2devX0eut3U8Pj6e1q0f5CBdvHgxixcvpkePHrpCpcnJyYwYMYKEhAR8fX1p0aIF27dvp2/fvgbXdtWfuZDdZYuyILklSoPsduU4tCCHv9YWCez4in11k/Tj2pgt+2wW1UWR3BK+YcHgvEX0AhfE2ZJbwtGy++634+2aR2lBiO5ShrMk98kbVwkOccxYpU12h7cv2pbPoqDxtiC687Q3BCGdAgnpYv88E0+nEtK5Ogrlg+DrpsZ8H0dJbgl92e1qoluj0XDn0nlSo6NQ5ebiXsmH6uEdqehnughGQVR5uSRFHCYjMR4Ar+o1CAzviELpXlzTLlaO//c3MpmM8BZ9LZ9sBI1Gwy9/LOCxPq8APrrjriK5JYTsFggEAttxpvB1NhMnTjS5ilr6LBIhISFoNOZvuL755hurruvKP/PikN0Ly/hqNFekLEluCVeX3a4chxbk+tFkALtjUv249vDXtu30dYTkLi1kp6aQGHGY3Ht3Ubi749eoGZXrNkQms+7vy1Rc61m1mCduhrvpt/Dxtm8Cd9Nv8b+dX/P8wFkOnpXrSG4JR8ruso4Q3QKLSDc1++f0dNiYpU12CwxxtOSWcMUbk+TI45xYNq/Q8fO/rKFilap0mD3PbPVqtUrFpc3ruPLX70bba/XsR+PnRyNX2J4rzFloNBo+XvEiAL9+GW/1jZU+5y8dYdPWZfx7fAv7XtZuy3Q1yS1RXLJbIBAIyjLOEr7lGVf+mTtadi9kjcPmJrCOsia5JVxddpdHwp7qaXVay/IiubNu3SRyxSLSYi4ZbW/z+mwCWrU1O4a5uPaptWcdMk97mL3gMb746JB9fec/SlLKNVo360Xjeh0cNqeSkNzpOZklvsNJimvLOq5fdlXgVPSf3DsaSXbXDfRm3NpjnI694/BrCOwn7Uqc0ePFJbklTN2gaNRqNOqi7SVTq1QWgzp9buzfZXAz0PCZEdR9YgjV23cBIPv2LfZOH0tGUoLJ6x1b9K5Ocldt3IKGQ0YQOmAQPnXCAIjdvZ1ji95FrTJfPdqVOHF654PXZ3bZNcauAz8DcOeudvWHq0puCemmoKF/CM+vm8HJ+PM2jyEF+dEpV8vFk3SBQCAQCPQp+D2YnpNp8xiTurzA9O4ji2F2AkuURcktMaZnPSb0qc/ynRdZtdu4SBSUHKFPdNfJ7pgtpvOUO0NyazSaIsdtpuJaU+NmJCWwd/pYneQO6fcE9QcPo3bvAbpzTiybx439puMyS3GtM7lzN5ms7Ayb+2VlZ+hiSSm2dAQltZK7qN+Di/atZenBH23uXx52FosV3QKTFNyelsk/Dr+GWNntupxc9EMhkV3cktsYaTHXOPqh4VPH9m/NwLduqFX9NRoN17bt4OKGTbpjXsFBtJ89HTcPD5P97l6P4cza5QCEPfY09Qc+i0xvy6/65dc59OEM7t24xsF3JtP3y18M2gHO/bya2xe0T8fbTX+fqo2b67UOJz3hBgfeep3bF85yafM6Wj39oVWfyZloNBrWb1mI/H616vVbFtKmWW+bVnUnJMew74hW/ufmZrm85JZw9MpugUAgEAhKMyfjzzslnZfI0e0cnCW503OyS8RaiJXdrkXBGk4FRXZJS+78rCxOfPoFaZevPJjjYwOoO+hxq+MgS3Htpc3raDB4mEG7Rq1m/6wJAHgGBNFh1lwq+FbWtTcZNoZru/6Pcz+t5sza5fjUCcOntmGcbE1c60xyc7P4e+93PNnPtrzRf+/9lty8bAD2Hf6Npx+dQvVqIVb3N7YAriTTlUgPfUXtCscjRLfAKMZysNn+rMk6hOx2TbyDAwxkt1Mk9+UYjs5dWOj40bkL6TBnJj4hdSyOEfXNdyT8e8TgWEZcArsnTKX3V8uQK5VG+51ZuwKAhkNfIrTfE4Xa5Uol7Wd+xK6Jw1Hn55FyJpJqLdro2lW5OcTu1hZr6jp3Gd5BNQuN4R1Uk3bT39et+rZGdGemJpJ6ORLPqsH4hTW3eL4xUq+cJvNWHH51W+Hpp82jt+rnmQQFhBIUWJfWTXvp8nhejY3i8rVI4pOukJB8hdj4aBKTY3RjxVw/zaQ5XalZowFBAWHUCAyjbp1WhNRqCoBareZk1D8kJF0mITmG+MTLXLsRhVwmR6VRo9FoSoXklnCk7BYIBCXH8uXLWbRoEYmJibRs2ZLPP/+c9u3bGz03KiqKOXPmEBERwbVr1/jss8+YPHmywTn79u1j0aJFREREkJCQwKZNmxg4cKCuvc9375F2JY6Ti37AOziAVtOG4eZRwegxR7PzxfccPqZAYAxRu6J84SzJ/fwPK1j9askUZRSy27UwJbtLWnKr8/LYPWFqoeMx/9tKzp00mo4abnEMa+LaK3/9Tt0nnjGo4ZRy5iQAcqU7XT9cYjR2rdP7EdLjYonds52zP35Nx9nzDdqtiWttRZWXS3LUQQACmnaxq+6UFNdKtZsORWwhOKi+NiYNCKNZw674+mhrYqXdTeFM9AESkq+QkBxDXMJFrt44q5PVcpmc2fMfoU7NptSoXtfquHbvqL26+ZR0Tm5RqLn4EKlLBIVwRqERkcbE9Wg1bZhOdqeevVLikhvQ3Qy0GD+GvmtW0nfNSlqMHwPAkQ8s55bKy8jQSe7uny2g75qV9PlmBd7BNQC4vnO30X75WZncvXYZgDp6W8IKovT0ot309wE4veZzg7akE9rr+tQJMyq5Jao2bo5fY+uEdVpsNDvfeoyjK6ay58OhXNy21qp++lzYuoY9Hw7l6Iqp7HzrcdJiowH4e+93fP/b+9y+k2hQrMrNTcm6Pxay5e8VHP9vu4HklkhIvsLx/7az5e8VrPtjIUq9Gx25XE7q7QS+2/Aef+/9jjPRB7iXcRuVOl93jjMlt73bvRyRxkQgEJQM69evZ+rUqbz77rucOHGCli1b0q9fP5KTk42en5mZSVhYGAsWLKB6deNFtTIyMmjZsiXLly83eV3fsGBaTx9OelwykZ/8ROrZK8UuuQWCksSR6bzs2b4tKNtIkjs62XiKwOJCpDFxLQqmMXFGupLru/YAUKlObfqsXk7fNSvpseRjAOIP/EtehuW0G9bGtUkRhgu0Lm35VdtvzCSzQrrxc6MAuHMpmvysB79PrY1rbUGVl8v+hSM5tHQCh5ZOYP/Ckajzc20aQz+uBVCp8rh87T/2H/md9VsWsf/oJjw9KunO9/SoxP6jm7RtR37n8rX/UKnyHsxJnc+9jNuciT5gV1zrjMKTjkznZW8ak7KKEN0CA5xZTVvIbtfCzaMCraYNw8O/MicX/kDapdgSldz6ecsC27Yx+tpSzu5bUecACBnwMBV8fQGQyWSEvzkFwCCdiT7Zqbd0r+Vu5p9wewZoJUju3TSD4xmJ8QBUbdLSbH+Aas1bWzwH4OK2tahyc3Tvo35fYlPeco1azdmNS3XvVbnZOlkuk8kYN+Iz+nQzfBJcM6gBH7yxico+AchlpgtmymUKKvsE8MEbmwiuXt+grW/34Ywf8Rkymczo1j5nruQuSm6zospugUBQMnz66aeMGTOGkSNH0qRJE7788ks8PT1Zs8Z4Mbt27dqxaNEinn32WSpUMC6jBwwYwEcffcSgQYPMXluS3WmXYjm58Ac8/CsLyS0oMzi6doWQ3QIJfcm9bsSEEr++vuwWOB992e2MwpOXftsMQOvJ43UpP9x9fAh9rD8AKaejzPa3Ja7NTIo36Jt2Rftv0MM/wOw19CW4fixrS1xrLclRB0m9dFL3PvXSSZLOHLRpjIJxrT7tWw1g2thVKJUP7pWUygpMG7uK9q3My3op3rQlrnWG5JYQsrt4EKJboMOZkltCyG6BI9GotDcMMoWhoNXkq+4fL/qvwIKCuyDulSqZbQfIvXe3yPMoKq+N/JyenYcabQsKDOOjGVvw9ammy82tj1yuwNenGh+9+SdBgWFGx+jZ5VleG/m50TZnSe4VB5cVuZCHkN0CgWuTm5tLREQEffr00R2Ty+X06dOHQ4cOOXFmAkHppzgKNQvZLSgouVvXDHHKPCTZLRBIErpgTKmWYkp58Ws190o+xX4NZyOTyegU/gRTXvkKpVvhVChKN3emvPIVHduYz4tuS1zrTMktIWS34xGiWwC4huSWELLbNZBycmel3KH1jOH41qvFyUU/kHYlrkSur3/DkHT8hNHXlm4q/Fto80TH/G8ruXe1MlmjVnNyqXaref2nja/Eq+hXVfdanZdn9ByJrFvare++dRsYHPeqrk2PknI60mx/gFtR/1k8B6B+/5Eo3B/832w6eLJNN1YyuZymgyfr3ivcK1K//0gAunUYbLZvYLU6DOg5ymibRqPhkV4vE+hf2+wY3ToMpnZwY6vnawpHSe6l+xc7pGq1kN0CgeuSkpKCSqUiMDDQ4HhgYCCJiYnFfn0pJ7dvvVq0njGcrJQ7RH7yE/lZxlcxCQSlDSG7BY7ElOR2VjwocnS7BvrpSvTTmJQU9QYPBCDi4890OaFz0tK4tm0HAFWbmo9vbIlrPQNrGPR199HuSs5MNn/Pos5/ELPqx7K2xLXWEtC0C371HuxI9qvXmsBmXWwao2BcC9qY8vlBs3FTmF557qZQMmzQbKOFJAFqBzexKa51tuSWELLbsQjRLXApyS1RUHYLSh79nNx+TcIMcnaXlOzuMGcmAKdWrGLHqHHsGDWOUytWAdoK1ZZQenlRo2snAPZOfpMdo8axc/QE7l2LBaBWrx5G+7l5eOrE9bl1xre2g/Zm4dSqZQDUe3yIQVtgeEcAbp07xa1zp02OkR5/g7vXC+e9NoZvrYb0mfsn7cd/ykPvrNdJaluo338kD72znvbjP6XP3D/xrdVQ+1msSIFyI+ECMrRPz+VyhW51t1wmJzbhgsX+arWa+MTLuvfGVodbwqGSu9sbQNFvCoTsFggExihYeNKvSZhBzm4huwVlBSG7BY7A3Epusfip/FIwJ3fBnN0lQZ2HewGQHhfPzpfHs2PUOPZN0capQZ07oPTysjiGtXFtYHgHg37NR00E4Niid8nLNJ0L/NrO/wPAN7Qebh4PihpaG9fagkLpTvc319Jp0nI6TVpO9zfXIjeyAtsc+nGtPjesiCkLnqMfU8YnXbYprnUFyS0hZLfjcHP2BATOxRUlt4Qku8d/K0S3MyhYeFLK2R35yU+cXPRDieTr9gmpQ/u3ZhSqUC1VpraGJiOHU6FKFWL+/D/dscr16tJ6ykSzBT2aDh/Lv+9NI3b3drxr1KJO70cM2nPS7nBk/mzUedrCG/7NWhm0K5TuhD06mCt//c6xRe/S9aNleNcwLEp56+wpji1+D4BaPftZ9Xk8/arj6dffqnNN4RfWHL8wwwKYN2/FEljN/M/06o0oXRHJGoF1Ae1Ngkqdz9VY87npAJJvXScvPweZTIZGo6Flkx6cu3jEYj8JR0tu7U2NNu9dUatWS0H+C+tn8vy6Gfz87EJRcFIgcAH8/f1RKBQkJSUZHE9KSjJZaNIRFJTcUk5uKWf3yUU/EPnJT2UuX/eAtYa1Ly78/hNX/vodwKrvwabDxxq0a9Rq/n71WTT5+VSqWYdO7yw0+O7WqNVc3PQzV/7aaHJ1V0kgk8moU7Mpi9/Zafa8xJtXee3tTrrvwdbNenHu4hFycrPQaNQMG/QWA/tPNDvGpm2f8/OmechkcipW8CRy8lmb5urIbdrnp27WHXfE96AU5D+/bgYvrJ/Jj0MX4F3B03JHQanHUroSafHTypHtaF6rslPmKCh5TBWelF5f2bjb4H1xIVcq6bn8U47NW0x63IMc2vWHPEWdfn3M9HyANXFtrZ79UCgNhbF/s9bI3NzQ5OdzdMHbZr8HAZq+OK7Qta2Jaz0rW/UxdMjd3Alq1dO2TgWQ4tpTFd7RfQ/Gxp0nvLn5n+n1+w9Tpe/BhnXbERm1G5lMRl5etk1xrbMk96n4SDoFF35A4ojvwaLGtWUBIbrLMbZK7oycfLwqlOw/GUl2x2F9wT2BYzAmsp0hu33rhtJ3zcoHudFszIEmk8moN+hx6g16HLVKhUwuN5vTS8KndijNRk7gzNrlnPtpNed+Wk3oI0/hXsmHnDupXN2+RXdu9wUrjM6r3sBnuX3xHLcvnOXA26/jFRRMcOeeyNzcuHkqgtT7K72rNGhC4+dH2/S5HE1sfLTZGwKVWkV80hWqVa3JcwNn0aXtQAAOHt/ML5vnk5B0GZVahcLMKu3Y+GgAWjXtyfMDZxNSqynpGXcg2fL8ikdyGyJkt0BQ9nB3dyc8PJxdu3YxcOBAQLu7ZNeuXUycaF4oFgVjkluirMtufYr6PSiTy+n64VL2z5rAvRvX+HvsUKq374JvSD3QaIje8H1JfySjVKtas0jfg3/8vZy/dq7SBe/miI07j1JZkcd6j+GJh8db9R0q4ehcpAURsltgD9bk5JYWPwnZXX4wJbklSlp2u3l40OnDd9BoNGjUauQK23emWopri/o92GzkBHxqhxYaw5q49qm1tj00dSQr5x8v0vegt1dlYmLP8Mvm+Zw8849Nca2tOEpyj1w/zOBhsT5CdhcdIbrLKfas5B7/7TFWvNTOKbIbckv0mgJMCuzilt0xW/ZR97EhhY47osiHrTckNbv1xr2SLyeWzdPO7f82GrT7htan1fg38KhazeT12k1/n3M/ryZ293YyEuK48LvhFqKwRwdTb+Czdt0sORI3C1W409NvM3LI+/Ts8pxB3rRu7Z+iU/jj7D74C+npt/H18Tc5htLNnQ+n/0Gjeu11x7y9KgPm88WVhOSWELJbICh7TJ06lRdffJG2bdvSvn17lixZQkZGBiNHatM/jRgxguDgYObPnw9oC1iePXtW9zouLo7IyEi8vb2pV0+brzU9PZ1Lly7prhETE0NkZCR+fn7Url3bpOSWKC+y2xHfg16BQfRY9BVH5s0m+/YtEo8eJPHoQYNz2rw+u9g+gzUs+/DfIn0PDhv0Fo/0Gs3Rk1stXqth3XYMf/pdqvgG3D9iXc7V4im4dbHQeUJ2C2zB2sKT+jt9hewu+1iS3BIlLbtBu4iqYFFKm8cwEdcW9XswoFVbk9e0FNc6k6J/D0JorWbMfu0nzl06Qo6FlB/6cS3XrZ+nIyV3ff8GZs8TsrtoCNFdDrE3XcnlpHSnyW6Ba1Fcslu6qTEmup1FQKu29Pvmd+5cjib1/BnUubkoK/lSvW1HKlaparG/XKGg6fCxNHp2JEkRR8hM0m538wysQWB4h0Lb05xFq6bmt575+vjTt/sIo21uCqXJNluuYYySlNwSQnYLBP/P3nmHR1G9bfjeTYeQUEISCCX0XgOEJkXwBygqoICIIlWlSBUEVCyIKE1AmoiADUEUsX2iiPROINIDUkMqIRBISN/9/lhm2V4n2U1y7uviunZnzjlzJkCeeZ85532LFwMHDuTmzZvMmjWLxMREmjdvzrZt27QFKq9fv45SJ+iMj4+nRYuHRZYWLFjAggUL6Ny5M7t27QLg2LFjdO368Hfa5MmTAXjppZdYv369TcZ1QZjd7pj3Ww4d9KtQkS4LP9cE+McOkXsvDaW3N+XrN6Fsrbo27dQqSOTQwXKBIfToMtTqGLa0MaRgTG7zCLNbYAu2mtwSwuwuOdhicku4wux2BXLooKW41h2QQwcb1I40e05CP6617WWx3Cb3mgFfAwkW2wuz23GEW1nCcCYn98phrRm97miJM7ujjliucFyQXPWxLFj5eZptT1cPJllsZ/EaD/pePZCIh+fDQL90e8v95Da7dd/cuxsKhYJytetTrrbjZqWHlzeV2z4i46yKP64wuSWE2S0QFC/GjRtnNlWJZF5LhIeHW8333KVLF4ttbDWs5TS78zKziV74Lbw616H+BY0cOuhbrgLhjz0h04xKBoVtcksIs1tgCXtNbomSZna7cxxqSHz0LU2//Y7NWTeutdXkligpZjc4r4NyxLUljYIwuW2Na4XZ7Rglw6kUAM4XnmxStWyJNLu/+9L+yvGFTeKpVBJPpTo1xtG1MXrfu1kxukE+s9toe5pIyV7icaXJLSHMboFAXpYvX878+fNJTEykWbNmfPrpp7Rp08Zs+82bN/P2229z9epV6tSpw8cff8zjjz8soJSUlMQbb7zBX3/9xZ07d+jUqROffvopderUKYzbkQ05zG7J5E6PsyNhs6DY4yqTW0KY3QJTOGpyS5Qks7soxKGGHFp9zqn+iadS6TbFfqNaDrM77XIc5cILrlC1oOjhSpNboiDM7nk8alf/okbxdykFgPMmt0RJNLs/Wuy6N8JR/pbfpKvyVPz46l5aD69HeHvHRPnqgUSOro3hmVWPoNRZ0W2rbe6s2W1rDjZBycEdTG4JYXYLBPKwadMmJk+ezKpVq4iMjGTx4sX06NGDmJgYgoODjdofOHCAQYMGMXfuXHr37s2GDRvo06cPx48fp3HjxqjVavr06YOXlxc///wzAQEBLFq0iO7du3P27FlKlzauZO/OOGN265rcLaa+WMAzFRQVXG1ySxSU2S0omjhrckuUFLPbneNQQ/YuPgnAIxObOnQ93bjWUZwxu9Mux3Fi/tc8ujzC4esLihfuYHJLyG12z2OtQ/MoKjhf3U3g9shlcktIZreUszsjO0+mmbonnl5Kl/3xsPJHMqY9PK23NfvnwRhKgzHs+hk9MLv9w4I5Mf9r0i7H2dTPHU3ubcP7cfCDN7h5MkpbEdsaufczuPTbD2wb3o9tw/vx58sDOLV2OekJtv0cALvaFjTpGXfo/0ol9h5xvEjJ3iNb6P9KJdIz7tjVz51MbokJHV5gaqdhzN+zjiX7v7HewQApyK8XFM7zG6dxwoZq4gJBcWPRokWMGjWKYcOG0bBhQ1atWkWpUqVYu9b0g/aSJUvo2bMnU6dOpUGDBsyePZuWLVuybNkyAC5evMihQ4dYuXIlrVu3pl69eqxcuZLMzEy+++67wrw12ZDM7vS4ZKIXfmtTvm1Dk1vO4tCO4KwORi2eox1j56QRXP3rV3LvZxTgjAsOR3VQlzfm9GD6hz3t7ucuJreEHDooBfkxKVd5YdN0p+YjcA1ymdwSktldK8Sf0euOcir2jizzdCfcOQ41FZcaxpP29gf0Umk6Qo2nOlGzX1cub9nJlV/22NRHMrn9w4xfvBcmzuqgWqXi5sko/pk4TDuGvXGtu+GoDko4Gte6k8ktYaiD6VaKbppCimuLO8LoLubIbXJLlDSzW2Ade81udzS5JdIuXyRq8Rz+HPksGUmWi0TEH9rDjnEvcnHLBu0xdV4ecft2sO/N14haPAdVnvkiF6q8XKJXLWLfm6/JNn9nOXV+HwAbfvrQ4TG+3TIHgNMx+2zu444mt4QwuwUCx8nJySEqKoru3btrjymVSrp3787BgwdN9jl48KBee4AePXpo22dna0xgX9+HzzVKpRIfHx/27bP99467YY/Z7W4mty6O6uDNk1Ha49lptzm/cR07xr1I/MHdFq9nTatdgSM6qEtmVjqXr5/k0rV/ycyy3ex3N5NbQm6zW1C0kNvkligJZrfAfuwxu3VN7uZTBhfSDE3jrA7+9epzRC2eQ87dNO1xe+Jad8NRHdTFkbi2sExuOV76Omp2F3eE0V2MKSiTW0KY3QJDbDW75Ta51SoVKafOEL//IOlxjgl4bnoGCQeP0G7WfJqPmYrSyxuAvTPGknnrpsk+8Qd3c3L1YgDK129MxzlL6fTRCiJnzKH+oOEA3DwZxfGlc02+RVer8jm64D0Sj7iPKaNWq/npjyUA3LodT0LyFbvHSEi6TOodzd/Dlj+WWi3qBu5tckvIbXYLBCWFlJQU8vPzCQkJ0TseEhJCYqLpolWJiYkW29evX59q1aoxY8YMbt++TU5ODh9//DE3btwgIaFoBXKG2GJ2u6vJ7awO1hswhM4LVtNx9hJaTZlFpQfFK09+voT4Q6bNisxbN9k7Y2wB3ZFjOKKDhvy1+0uTny3hria3hJxmt6DoUFAmt4QwuwWmsMXsNjS5Pf18SDh4hMQjx8jNcMxYvXcjjvh9B7l15pxDq6id0UF1nsaPaT31PTrP/8yuuNYdcUQHdXEkri3Mldxy7XByxOwu7giju5hS0Ca3hDC7BYZYM7ttMblTz8WYPG6u7d8jx3Lik2Wc+eIrDr79PtuHj7b54UStVnNm7VfsGv86pz9fR2B4LUJbtaP7sq8pFazJex69Yr5Rv9z7GZz8XCOc9QYMofXU9/CvVIVSwaGUq9OA8Md60+H9TwBIOR1N4tEDRmPEH9rL7QtnAWj39sc233NBkXE/jV0Hv+dK7GkAlAoPNv78Mbdux6Oy4UFNpVJx63Y83/38MUqFBwBXrp9i18HvybifZrZfUTC5JeQ0uwUCgeN4eXmxZcsWLly4QPny5SlVqhQ7d+6kV69eKJVF//HWktntriY34LQO1ujZB7/yQfiHVSWoUXOavTyJev2HAHBy9WLyMo2DucMfzizAO7IPR3VQl9zcbGLjY/hp2zLtsa1/LiM2PobcXPMr/F1hcrtqh5OodVF0KGiTW0KY3QJTWDK7TZncAKc/X8epVV+w67XXOb1mvc0vKnMzMtg+fDSHZn3AmbVfcXzhUv4eOdauuNZZHSxTpTrdln1NhQZN8KtQ0a641p1wVAd1cSSuLex0JXKm8xJmtz7Fu4pgCaWwTG6JkligUmAZcwUqbV3JHTV/MY8snItvubIWr5N1+w5R8xcDUKFxQ4KaNCLmu80A7HrtdR5bu9LqXC/99Cvx+zRb4Wv07qU9rvTyInLGh+ycNJy0K/+RdfsWvuUqaM/H7fsHgEptH6FGzz4mxy5TpTotxk7jxPJ5/PvZIipFdtQ7f2rNUgDavvUxgTXqkJuZwflfV3I/JY7Qpp2o1qEPCoV9hWDSYmO4uE1TOLFOz2EEVrWtoMvQSfW1QbhS6YFKlU++Ko8Dx37mwLGf8fL0ITioKlUr12fogPeoUK4yACmpcazf/A434mNIToklN0//4UOp9GDFlxNZ8SWULhVIpeCaNKrbnuf7ztSaUa40udOz7ztdtdrRApUCQUkhKCgIDw8PkpKS9I4nJSURGmq6kHJoaKjV9hEREURHR5OWlkZOTg4VK1YkMjKSVq1ayX8TLsBUgUrAbU1uU9irg6YI7/k0ySejuB1zhuv/bKPmE/2057JSU8i6favgbsBGFnw20ikdPHzi//hr95fcSLjI7bQk1GqVnv5n3L/D5Pe6oFAoKV82lLDQ2vyv80tEtngccN1KblGoWWCJwjK5JUpKgUqBfZgqUGnO5AZNPKjOz+fqH3+RcOAw/mGVCe/1P6vX2fXa69rP9QcP5Oa/p7h1+qxdcW1gDdNxm6062O7teSi9vIz62xLXBvZ4WIRTrVZzff9WEk/uoVRQGPWfHI2Xn31Fvu2Na53VQZVKxYafPuTMhQMkJF+2Oa798n8rANfk5C6IQs32xrXFlaK/5EWgR2Gb3BJiZbfAEMOV3f8u2WhXupKz67622iZmwyYA6r84iJaTX6PaY4/S/YsV2vP3blgvgnXltz8A6Lz4Y2r3e0rvnE9gWcJ7aI4lHjuk3++PrQCEdehqcfyQiLbaz1l3Uh9+vv3wc2CN2gAcXjGJi9vWE3dsO1FfvMn1/Vutzl+X+6mJ7P5wMLGHfif20O/s/vAF7qeaTgtgiO5KM5Uq3+h8bl428UmXaN6oq9bkBggqH0aLRl2JT7pkFNwbjpVxP42bt2Lp2uE5vRWXrlzJ7WwhD2dWdgsEJQVvb28iIiLYsWOH9phKpWLHjh20a9fOZJ927drptQfYvn27yfaBgYFUrFiRixcvcuzYMZ5++ml5b8CF6K7sjvpwHVEfrrPL5F6+fDnh4eH4+voSGRnJkSNHLLbfvHkz9evXx9fXlyZNmvB///d/eueTkpIYOnQolStXplSpUvTs2ZOLFy9aHNMeHTSFQqGg0ZBXALjwo/7v28QojTaHtulo1K8wcVYHWzbuhpeXL6l3ElCrNSvNdFcRSp/Vas2KcW9vP1o27ga4Nl2JqF0hMEdhm9wSYmW3wBS6K7v/XbLRrMkNULvfU9Tp35dOn2gWpVzc/JPV8dPj4rWfu3+xgqrdutBy8mvUG9QfsC+uNYWtOmjK5JawJa6VuL5/K1FfvEncse1c3LaeIysnWZ2/IfbGtc7oIGjqtHTtMJCbt2LtimvBdYUnC6JQs1jZrUEsuy1GuMrklhAruwWGSGb37tEfkXIihqAW9WzOyX3r9FmrbZKjogGo1K6N9phCoaD+4IGc/3YT965dp0wV80aAKv+h8HkHBJhs4+VfBoDce/pbjrPTbgPgW7aCUR9z5Kbfw7ds+Qef7+rNGSD5zH6QBF2hJPHkHqp37Gvz+KmXosnLeihueVkZpF6KplR565WqywWGcPfeLfJVxi+ppPmNHbqEzm37G53v1nEwnh5eLP9yIoDJLX4eSk8CylTgvdd/olJwDb1zrjK507PTtQ8FjrwBd3Zlt0BQkpg8eTIvvfQSrVq1ok2bNixevJiMjAyGDdNUfh8yZAhhYWHMnTsXgAkTJtC5c2cWLlzIE088wcaNGzl27BirV6/Wjrl582YqVqxItWrVOHXqFBMmTKBPnz7873+alVeDvnrUprld3Z/IodXn6L+mMx5ejq0B2fPJSQA6TWr64Ih8RZYDa4bR9LUBnJinCZRbTLPN5N60aROTJ09m1apVREZGsnjxYnr06EFMTAzBwcFG7Q8cOMCgQYOYO3cuvXv3ZsOGDfTp04fjx4/TuHFj1Go1ffr0wcvLi59//pmAgAAWLVpE9+7dOXv2LKVLW1/tZU0HzaH0MP08mXNPM0bp0MomzxcWzuqgl5cPU175nCVrRnMk+g+zW+UVCgVtWjzOxJEr8fTQGBquzMkt1w4nsbK7eOEqk1tCrOwWmKLGU524eyWelBOaVCKmTG5dfAIDtZ9V+fkoPTzMtr179RoA9Qb119OzsC6PEPPdZrviWnPIoYPW4lqJxJN7QKGEB4Zz0un9qFX5KJTmfwa6qFX5dse1zuigRFhoHWZP+4V3FvS1Oa5NP3/bJSa3hBw6KFZ2GyNWdBcTXG1yS4iV3QJDYrcf1n6+ff6q2QKVhvgFV7TaxstfE1RnJqfoHb8ZrTEcfMqWtdhfobOaylyxkOwHK86U3t4mz3v4+Vmdp4RUCMTceH5lQzQPFQ8oHVTF5rEBSlUwNj5MHTPF7Gm/UKqUabNfrVabNbklOrcbwNihS8w+lJQuHcgH0341Cu4Bl5ncI79/UZaq1c6saBMISgoDBw5kwYIFzJo1i+bNmxMdHc22bdu0BSevX7+uV0Syffv2bNiwgdWrV9OsWTN++OEHtm7dSuPGjbVtEhISePHFF6lfvz7jx4/nxRdf5Lvvviv0eyto8jKzubxll/b75S27TBaoNGTRokWMGjWKYcOG0bBhQ1atWkWpUqVYu3atyfZLliyhZ8+eTJ06lQYNGjB79mxatmzJsmWaHJkXL17k0KFDrFy5ktatW1OvXj1WrlxJZmamzT93azpojuw7miBc6akf1Ho8GMPTRc+9ujijgwBent5MGvUZrZuZfzndullPJo1cpRfcu7rwpNyFmsXK7qKNq01uCbGyW2BI2uU4bp+/qv2uG6OaQvd3ucJK7Q9pwVTKqTN6x+8naHbW2hPXmkMOHbQW10qUCtKJHxVK/MqF2Gxya7p4OBTXOqqDulQKrmFXXOtKk1tCrOyWH2F0FwPcxeSWEGa3QEI3J3fnldPNFqg0RcOh1lcFNRur2cJ16N053LseS35uLte3/8OtM+cAKN/Acn5qhUJBYO2aABye/RHZd4wLRV3/R5PapHz9JnrHKzbT5IBNPLLP4jVy7j4cs1TFhyvo/IJCtJ/vpyQDEDluCX7lgkGhJKRxe+o/9arFsQ0pX7MJTQZORenphdLTiyYDp1K+ZhPrHYGQoGr4+ph/wAqv0sjqGNXDGpo95+tTmuCgqjbNxRpybtOW46FAmN0CgW2MGzeOa9eukZ2dzeHDh4mMjNSe27VrF+vXr9dr379/f2JiYsjOzub06dM8/vjjeufHjx9PbGwsOTk5XLt2jdmzZ+Nth3laFNAtPNlq1khazRppskClITk5OURFRdG9e3ftMaVSSffu3Tl48KDJPgcPHtRrD9CjRw9t++xszfV8fR8+ZyqVSnx8fNi3z7wW2qOD5ki7+h8Aldt31jtevr7mxUfM5q8s9i8snNVBDw9PqlSqi4eJAN7Dw4uqlevhYbCqz1Um98n4aO1nYXYLwH1MbglhdgskdHNyd1453WyBSoDsO2nkpKdzaNYHAPiHVba666hCowaAZkdy7M495GVlce96LIfe/RCwL641ew826KC5hVsS1uJaifpPjiakcXtQKPArF0zbcUutzt8QR+NaR3TQEHviWleb3BLC7JYXlxvd9uQOPHPmDM888wzh4eEoFAoWL15ceBN1U9zN5JYQZrfAsPCkYc5uS2Z3SJsIyteva/Ua5erVIaSNpnDGoXc/5J9XxmuLUUZMm2T17TtAy0njALh3LZY9k6frnbu242Fu0rK19OdTb8BLAMR8/xV3LpvOT5qXncWRBe8CUP2x3npvwpUeHtTo1QeAg+9PJT87m/I1m9Br4T/0XfMvHSavxtPXvqIfoClA+fRnJ3j6sxPU6TnM5n7ZOZmkpGr+ThQP3r57KB8+RFy3QWxjddpIfZUP7vnmrRvk5GbZPB9zyJ2LFOR5KBBmt0AgkBtdk1vKya2bs9uS2Z2SkkJ+fr52xbxESEgIiYmmazckJiZabF+/fn2qVavGjBkzuH37Njk5OXz88cfcuHFDbzW+3j3YqYOmuHfjGuc3alJjGBZ/LlvLtoLLhYUcOhgbfx7Vg+3WCoVSq8kqVZ5NWmwNOXOR6iLM7pKNu5ncEsLsFpgqPKmbs9vQ7N4zeTq7x0/V5t1uPfN1U8PqoVAqiZg6EYDzX3/HzjGTtCa3PXGtszp48Sfzu6tsiWslvPxK02HyavquOUmvhf9QrkZj7MWZuNZZHbQnrnUHk1tCmN3y4VKjW8od+M4773D8+HGaNWtGjx49SE42varj/v371KxZk48++ojQ0NBCnq374a4mt4Qwu0suhia3hK1md9NXR9p8raavjiRi2iTtlrAKjRvyyMK5Nj1QaObkR7fPllJ34DMoPB6IaG4uZ776jHPfrgGg5fiZRm/y/SuFUSnyEQAOffAGqRfOotYpdnE/JZndU18h/YYmZ1utJ43TfkgPKrnp9zg456HJbs/WMFMolEqbTH5d4hP/0xb+CA2uwZghn9CqeQ/t+dg460Kr+9DQunlPxgz5hJCK4YCmcEhc4n92zcmQgiy4JcxugUDgTpgyuSVsNbvlxsvLiy1btnDhwgXKly9PqVKl2LlzJ7169dIWVnRWBzNv3dQ7nxR1iP2zNEWwKjZrZZSDVKFQ0HL8TNnu0Vnk0MErsadRq9X4ePvRv/cU+veego+3H2q1mquxp52an9wFtwwRZnfJxF1NbglhdpdcTJncEubMbikerP1sH7p9thRPG9NUlm9Qj0cWziW4VUsAvPz9afXGZLviWmd18PLvP3Llj63kZWVqz9sa15rC2ZjU0TGc1UF74lpXmdwr9pteJS/MbnlwaaVA3dyBAKtWreL3339n7dq1TJ8+3ah969atad26NYDJ8yUJdze5JUSBypKHOZNbQjK7oxd+y4n5XxsF8I5Qvn5dOn70vsP9lV5eVO/Rneo9urNteD+9c42HjSW4eSuT/ZqMGEfW7VvcvnCWIx+9ZXb8jnM+xftB8Q9dvMsE0HHOp+x78zWtEeAqrsefp2KFqgx88nU6Rj6Dh9KDrh2e49K1f/lu61yux8dYHSM2LoZmDbswqM90alVvBkCndv3Zd/hHNv26gNi489Soav+KAChYk1tCjkIeokClQCBwFksmt4Rkdp+Y/zXRC781CuCDgoLw8PAgKSlJr19SUpLZxSKhoaFW20dERBAdHU1aWho5OTlUrFiRyMhIWrXS6OSfI03XcrBVB3dPNb11O6hxc1qMnWryXHDzVjQeNtbkucLGWR3Myr7PvfRUnnxsNH17jqOMv6ZwZ48uL7F123L+2r2e7Jz7+HjbX2RKbpN7zYCvAeOV/KJAZcnC3U1uCVGgsuRhyeSWkGLVy1t2ar93/3y5w9f0LVeWZmNGOdzfWR08vW45MZu/MpvOy1Jc6y7IoYNyxLXmkMvkXrJ3AVM79jJ5XhSodB6Xreh2JHegI2RnZ3P37l29P0WdomJyS4iV3SUHaya3hD1pTFxF2dr1aP/uQqo80s1sG6WnF22mvU/TURP0im1I1OjVl0eXfmn2LTlo3qA/uvRLwns+Lcu8HaVuzVYseX8fndsNwEPnzXut6s14a8JGBve1vmJucL83eWvCd9rgHsBD6UHndgNY8t5e6tZy7MGqMExuCbGyWyAQuBJbTG4JSyu7vb29iYiIYMeOHdpjKpWKHTt20K5dO5PjtWvXTq89wPbt2022DwwMpGLFily8eJFjx47x9NOmNUwOHWz2ymQiJr5lVIBLF0taXZg4q4P5qjyWvL+fIc/O0gb3AAH+FRjy7CyWvL+fvHz7n6MLwuS2FNyLld0lg6JickuIld0lB1tMbglLaUwKG2d1sP27CylX17hWhC1xrbsghw7KEdeaQk6Te8IjllPiiJXdzuGypbWWcgeePy/fw8zcuXN57733ZBvP1RQ1k1tCrOwu/thqcksUxMpuOeg4ewm+5Svg6WfbG0+FUknldp2p3K4zWXdSyU2/h9LLG7+gYJQetm3V8vYvQ32D3GiFTaXgGhbPV61sPQ+qpTZeXj6EPti+bQ+FaXJLiJXdAoHAFdhjckuYWtktMXnyZF566SVatWpFmzZtWLx4MRkZGdqdlEOGDCEsLIy5c+cCMGHCBDp37szChQt54okn2LhxI8eOHWP16tXaMTdv3kzFihWpVq0ap06dYsKECfTp04f//e9/AHRZtMYpHaz7zAtkpiSjys3By78MvmXLW+/sRjirg6X9AijtF2D2fIVyleyeU2Gb3BJiZXfxpqiZ3BJiZXfxxx6TW0J3ZXet3gMKeopmcVYHA6rVIHL6B+Rl3icr9RaAXXGtOyCHDsoR1xoit8mtiWtN1/mSECu7HafYO40zZsxg8uTJ2u93796lalXLFc/dmaJocksUVbN77apTLrv2HSvxoSpPk3vq0OpzXD9iOre9NeKjNSK4d/FJlJ4PVyU3mGjdrJaw1+SWcEez2z/M8d8PvmXLF7mg3J1xhcktIcxugaDos+eTkza1M6eD9iCNIV3THg0Fx0xuCUOzm1c1xvXAgQO5efMms2bNIjExkebNm7Nt2zbtIpPr169rc2sDtG/fng0bNvDWW28xc+ZM6tSpw9atW2nc+GGqjYSEBCZPnkxSUhKVKlViyJAhvP3229rzzuqg0sOD0iH2m7kC07jK5JYQZnfxpKia3BJFyex25zjUEEMdtBfduLZbe4eGcMjklrAnhi1I5NBBT79S+IcVfzOzsCgYk9s2CsrsLu64LHWJI7kDHcHHx4eAgAC9P0WZompyS4g0JsUPR01uiaKQxkRQ+LjS5JYQaUwExZHly5cTHh6Or68vkZGRHDlyxGL7zZs3U79+fXx9fWnSpAn/93//p3deoVCY/DN//vyCvI1ihTMmt4RuGhNdxo0bx7Vr18jOzubw4cNERkZqz+3atYv169frte/fvz8xMTFkZ2dz+vRpHn/8cb3z48ePJzY2lpycHK5du8bs2bPx9va2e76CgsfVJreESGNSvCjqJreESGPi3jgSDzpjcku4i9ktcB9caXJLFEQak+KOy5bT6uYO7NOnD/Awd+C4ceNcNS23x1Umd3p2lmz/Worayu7hrzZx2bWP+Sssns/PVbF55G7avtyA8A6OvSC6uj+RQ6vP8cjEpnh4PXz3dVNtva+zJreE4cruR5dHODxWQaHKzyczJQlVTg5e/gH4lrN/xVpR3kZmSE5uFgDeXo7/PsrIvEspX+PiZO5gckuIld2C4sSmTZuYPHkyq1atIjIyksWLF9OjRw9iYmIIDg42an/gwAEGDRrE3Llz6d27Nxs2bKBPnz4cP35cu8o3IUG/GN0ff/zBiBEjeOaZZwrlnizRaVJTm9qZ00F7kFawSde0RUNBHpNbQjK73Rk5dDDrdiq56XdRenvjFxRic3oUd0TSQYXC8vOeOdRq0//Q3MXkliiIld1hPOrwfASOUVxMbomisLLbneNQQwx10F5049qjdu70lcPkLko4q4NyxLVyolarndLB+1n3LKY7cRR3MLkl5F7ZXdxxqbtob+7AnJwczp49q/0cFxdHdHQ0/v7+1K5d22X3UZi4yuR+/usVrHlVvq2CRc3sFhgjl8ktoWt2uxM59+5yZdtWrvyx1ehck5Hjqdy2EwqlZWPk7vUrnPn6M9IuXdA7HlC9Fo2HjSGgmuU8Yu7I4HGaOW/+LMFKS9Mk3rzKa2+145E2/Vjz6GLtcXcyuSWE2S0oLixatIhRo0Zpn7NWrVrF77//ztq1a5k+3Xh1x5IlS+jZsydTp04FYPbs2Wzfvp1ly5axatUqAKNdeD///DNdu3alZs2aACxc0bUgb8kydgbpcnLllz1WtVFOk1vC1em/zOGsDqpVKhKO7Ofk6k+MzlXv/gS1nuwv63wLA10dHD9iuUNjLF07li8eXaJ3zN1Mbgm5ze6768c4PSeBfRQnk1uiKJjdJRFpp68t2lhSTG5bdNC7jGWz11Jc22/dWbmmajefrh3nlA7uO/ITyz44REjF6rLNyZ1Mbgk5ze7ijstSl4Amd+CCBQuYNWsWzZs3Jzo62ih3oO5Kofj4eFq0aEGLFi1ISEhgwYIFtGjRgpEjR7rqFoo9um/u5UakMXFv8jKzzZ6T2+SWkMxudyE94Qb/TBhq8mEA4NSapRyZNwtVXq7ZMW7s3cGBd6cYBfcAd69d4sC7U7ixd4dcUy4U4pMuaT8nJF12aIxDUb8BcPLcw+rm7mhyS4g0JoKiTk5ODlFRUXTv3l17TKlU0r17dw4ePGiyz8GDB/XaA/To0cNs+6SkJH7//XdGjBgh38SLKJe37OTKL3vMni8Ik9tdcVYHVXm5RC3+wGRwD3Dt79/5Z8JQuaZbaOjqoLmV2ZZQq9V6Ggrua3JLyJnGRFD4FDeTW0KkMXE/bE1rWVJMblt1MD3B/M/KWlzrSuTQwUPHf5NvPoVkcrsqnVdJqHXh8iW048aNM5uqZNeuXXrfw8PDHfoPIHAMw+1pIP9bPrGy232JXvityQeGgjK5JTz9fED18LsqN5drf/3Dfz9uRaFUUvvZPlTr1gWll5fNY2bdvsPZtV9x68w5/IIr0vClwZRvYLnacs69u+x7UyNKXv4BtJs1j1JBmq39arWatCv/ceiDN7h94SynvlhGs1cmGY2RHH2U0+s0b6erdu1Bg+eGa+etysvl2t//R8z3X3J63XK8ywQS3qJoiM6WP5aiVHpoP48dutiu/lnZ99n6p+bnknYvBXBvk1tC7pXd81gr+xwFAnOkpKSQn5+vXUwgERISwvnzph+SExMTTbZPTEw02f7LL7+kTJky9OvXT55JF2Fq9uvK5S07AeOcnyXJ5LZXB4ObtzIa48SyeaScjgagxdhphES01Z7LuZvGkQXvkn7jWsHfjIwY6uCJ0//Qskk3u8Y4fnoHd+/d0n53hcmdnn2/0Hc4SUG+oPApjia3hFjZ7V4YprU0pZWuMLlvx1zk3+WfkZueQXBEc+o9PxDfcmVt7m8trk2OPuaUDu578zUeXfol3v76qSFtiWtdiRw6+NO2ZfToMgxfOzXJkMJcyS0KNRccLl3RLXBfCjMHm1jZ7Z6kxyUTvfBbvZXdBW1yG5KXmcmOV8bz349bAc2WrYvfb2HHK+PJy8y0aYzUczHsnTKDW2fOAZCZfJOo+Ys5uWqNxX6Xft0MgH+V6nSZ/5n2YQA0xdfK1qxD27c+BiDh8F4yEuP1+qvVao4v1aRdajB4JI1efEXPnFd6elGj59PUfEJjCB1f+qFN9wOgVuXb3NbWMc7EHODW7QSTLxMzMu9y6Wo0+478xPe/LmDPoR9QqfJRqfLZfWgz3/+6gH1HfuLStX+5n3nP+FpqNbduJ3Am5gB/7/2G5esncD8zDQCl0qNImNwScq7sFgiKG2vXrmXw4MH4+hbNgtlyUuOpTlqzW3dld0kyuR3RQUMNSk+I4+bJKAA6vP+JXnAP4B0QSNs35+Llb1zvoTBxVgfXb36H3/7+jKiT24lL/I/cvByjMXLzcohL/I+ok9v57e/P+HLzu9qXzq5aye2qHU72musCeXCVyb1497ZCuY5Y2e1eSGa3qZXdrjC5T65cw7GPF5GbngFAclQ0e6fMIPVcjE39bYlr5dBBUyu2bYlrHUGtUqFWqaw3tDhGviw6eD8zjeXrJ/D33m/simt1Kex0JaJQc8Ehls4KjHBFoRGxstv9aDH1RU7M/1q7sjt2++FCNbkBjnw4HwD/sMpEvDEJdX4+JxYv5961WI5/sow2M6da7K9WqYiavxiAes8PIKxzR+4nJHHo3TkkHYnidtfOlKtnLOyq/Hyu/f07AG1efxcPH9MPT2Vr1qFe/yHEbP6K89+vJ2L8TO25O5cePvRU7/a42TnW6fs8l3/fYvE+JHIzMzi8YhLJZ/bjVzaEyHFLKF/TviI1qZdPcWjZeLLuJBPcqANtx36Cp29p3l2kKRzXLuIpJo5cifJB3vH9R39m8ZpXtf09lMb/L7f83xLyVZoXVAqFgokjV9G+1VOApsjw4jWjORj1y8MxPLy0Dx4qVb5LTe4T8eedKuTh7MpugaCwCAoKwsPDg6SkJL3jSUlJRnm2JUJDQ21uv3fvXmJiYti0aZN8ky7iSFopreyu+lhkiTG5wTEdvHPpAuVqP9xxJQXs9QcNp0wV07k3PX18aTdrvgwzdhxndTD55jW++uF91GqNYVChXGU+mPYLQeU1/0ZSUuN48+MnSb2T8OAaSpQKJSpVvkvTlUgvfUXtCkFBsXj3Nub98zvPd+tVKNcztbLbp554seIqTK3sBgrd5E49F0PSUY3Z3PbdN/ELDiLh4BHOf/0dUfMX033Ncqs1m2yNa53RwT3TXuXKHz9R95nnUTwwgG2Na+3lwh9rObtFUyOi0TMTqdPTvkU8unEtarVTOgiaF8pH//1TL4WJLXHtB40nAK7JyV0QhZrFym4NYkW3QA9XVtMWK7vdi8CaYbSY+iLpccnsHv1RoZvcarWajDiNkLV9/y28/f3xCQwk8m1NwbS0/y5bTWUkvWGv0Lgh1bp3xcPLizLVqtD2XY0h/e/y1Sb7ZaY8NHa8AwItXiM0siMAN6OP6V/7/GkAqlkI7gEUSiX1+g+x2Ebi/K8rST5zANRqMu8kc3j5BJv66XJo2Xiy01I0DxRnDnD+l1Xac62b9eC14Z9qHwYAOrR+mtFDFqFQKFAoFOSr8rQPIABqtYp8VZ72/OgXF2mDe9DkAH5t2FJaNeuhPZafr5/T3JUruZ3Jbebsym5ByWX58uWEh4fj6+tLZGQkR44csdh+8+bN1K9fH19fX5o0acL//d//GbU5d+4cTz31FIGBgZQuXZrWrVtz/fp17Xlvb28iIiLYseNhLmSVSsWOHTto166dyeu2a9dOrz3A9u3bTbb/4osviIiIoFmzZhbvpaShu7J79+iPSozJDY7pYOr5U3rn4g/tBiAwvJbFMXRXp7kCZ3VQ0lal0oPAgIrMmvS9NrgHCCofxqxJ3xNYJgil0kOrvYBLc3KL2hWCgkQyuac9+kShXtdwZbfAteiu7D72/hqOvb+m0HNyn/3yW0BjcpepVgVPX1+qdu1EhcYNAbQ7h81hT1wrhw7ev5ms/WxPXGsrqZdPcvr7BajyclHl5XJq03xSL5+y3lEH3bgWnNNBCV1ttTWuBdcWnpSzdoVY2f0QYXQLtLjS5JYQZrd7EVgzjHL1w7Xfqz4WWWjX1t0GpVAoHn7WEStrW6Wyb98BoGIz/VXPfsEVAchNTzfZT5VjvFXKHHmZpgO7/Adj+JQtZ32M7CybrnU/RafAiFpF5u0ku9KYqFX5ZN1J1uuTkXIDgHYRTzL5lc/x8vQ26vdoh0GMG7oU0P+7kJCOjRu6lK4dnjM67+Xlw5RXPqdty94m+7vK5D4ZH+10IQ9hdgvsZdOmTUyePJl33nmH48eP06xZM3r06EFycrLJ9gcOHGDQoEGMGDGCEydO0KdPH/r06cPp06e1bS5dukTHjh2pX78+u3bt4uTJk7z99ttGKUQmT57M559/zpdffsm5c+cYPXo0GRkZDBumWYUzZMgQZsyYoW0/YcIEtm3bxsKFCzl//jzvvvsux44dM6qtcvfuXTZv3iyKg5tBVzvL1Q8vESY3OKaDhvqrztM8B/oEWh/DlTirg6BZWRYYUJEPpv1C5RBjQyMstDYfvPErgQEV9XZXubLwpCjULCgodE3uiZ17Fvr1dc1ugevx9POhZr8u2u81+3Up1MKTmck3AShVSb9uiWR059y9a7G/PXGtHDqoyn04hj1xra3cvxVv4pj5Qpgmx0gxbu+oDuqiUCjsimtdaXJLCLNbfoTRLQDcw+SWEGa3+3Dllz2knIghqEU9PPx8jHJ2FyRKDw/t5+y0NO1n3QcJ3TamKBNeDYDz327SW/2dcPAwAMGtWprs5+UfYPM8s2+nAuBTtrzece8ymjFy041zdRqSkWjbg0Fo006gVoFCCQolIY07aLel2YJC6UFwow7a/qhVmjGBCSNW4OlhvsBnp7bP0reneeHu12sCndo+a/a8p4cXE0eupFJwTZvnaw45t2nLUbVamN0Ce1i0aBGjRo1i2LBhNGzYkFWrVlGqVCnWrjVdnHTJkiX07NmTqVOn0qBBA2bPnk3Lli1ZtmyZts2bb77J448/zrx582jRogW1atXiqaeeIjhYf5XrwIEDWbBgAbNmzaJ58+ZER0ezbds2bcHJ69evk5CQoG3fvn17NmzYwOrVq2nWrBk//PADW7dupXHjxnrjbty4EbVazaBBg+T6MRUbpJzcHn4+BLWoR8qJGL2c3cUZR3TQq4zp1WaqfMvPgu5QrN5ZHcxX5fHma98SWjHc7BihFcN587Vv9VaxucrklhBmt0BuTJncrogHJbNb4HrSLsdx8tPv8a8agn/VEE5++r1ezu6CpkKjBgDE7d6nPaZWq7mw8QcAAsLDLfa3J66VQwd161bYE9faSvlazfH0LY1C6YFC6YGnb2nK12pu1xh6ce0DHNVBXSqH1LIrrnW1yS0hzG55EUa3wK1MbglDs1tQ+OgWnmw24TltGpPCNLvr9O8LwJ5J07nw/Rb+2/Izuye+AUCNJy1vhQYoU+Xhqrm/R4zh2l87OL7oU85/vRGAeoP6m+znW+6haZ0UdcjiNeL2a3Kv1uj5tN7x0AeFQq7++QvZaXfM9lfl5pJ4ZL/Fa0hU69CHiBFzCGv1GHV7DiNyzCc29dOl7dhPqNtzGGGtHiNixByqdegDgIeH9Zz4mVnpKE28PVcqPbmfZd3I8PDw5F56qt1z1kXuXKRyPBQIs1tgKzk5OURFRdG9e3ftMaVSSffu3Tl48KDJPgcPHtRrD9CjRw9te5VKxe+//07dunXp0aMHwcHBREZGsnXrVpPjjRs3jmvXrpGdnc3hw4eJjHy42njXrl2sX79er33//v2JiYkhOzub06dP8/jjxr97X375Ze7fv09goDxbYosLhoUnm014zmSBSrn537pVen+kLboKhYJ6g/oT1KSR9nunRR8ZtTf802nRR9r2QY0b0uCF5/TG1G2riyM6GNpKv8hWnWcGA3D269UWzey0yxcd+VHJhlw6aMsY9zP1Vw66yuResX+p9rMwuwVyYW4lt6sWP4maUa5Ht/BkxMxhRMwcZrJAZUHScLgmvVbMhu85vnApsTt28feIMdrz/mGVrI5ha1wrhw766izAsieutZVS5UPpPPMbqrZ9gqptn6DzzG8oVd50vRdz6Ma1ujiig7rcTU+1K651B5NbQpjd8iGM7hKOO5rcErpmt6DwMczJrZuzu7DM7uo9H6NSe40Jc23bdq78pqm6Xrlje2r16W3TGF0+fVhN+cLGH7h1+iwAEVMn4luurNl+TV+eBMCJ5fO4d+Oa0Xm1Ws2VP7aScHgvAFUe6aZ33rd8EIE1agNweO5MVLm5RmPk3s/g4Oxp2vbWUCgUVO/Yl8gxn9B4wBQ8fUtb7WOIp29pGg+YQuSYT6jesa92+9zd9FtW+16LO0t+fi4eSk+tweGh9CQ/P5frNyznpgNIu5fCvYzbD+5Fqa2SbSsFUXAL5HkoEGa3wBZSUlLIz8/XrqCWCAkJITEx0WSfxMREi+2Tk5NJT0/no48+omfPnvz111/07duXfv36sXv37oK5EYFVDE1uKV2Jbs7uwlrZ7agOSviWK0vE1ImAJg/p+W8fFhzVHduonwM66Fuugt75al01Rlfq+dNc3fazyevcuXyRQ3OmW72PgsRZHVQ8WNEWG2ddf67Hx+j1sRc5t2nrIsxugbNYSlcidvqWTHRNbiknt27O7sIyux3VQV1siWvl0MGmo4xrONkS19pLYNV6tBr1Ea1GfURg1XrWOxigG9dqvjuug0qlh/bzvfRUu+JaV5nc6dmmPS5hdsuDeEVZgnFnk1tCMrsFhY+pwpOS2X1i/tdEL/y2wIuAKBQKGo8cSr1B/bl15hxqlYqgJo3wKm27wetVujSPrV1JelwCd69exTsggAqNGlitjF2pTQfiD+wk5XQ0+2dNok6/56nYNAJv/zLkZWdx5qvPuB1zBoCmL0/E08+4InvzMVPZPfUV7icnsn3sYJoMG0vZWvVQenqSkZTA0fnvaNtGzphj8z0VBLHxMTSq295im+sPHjxaNunGoD6aPL7fbZ3L0X//5Hqc9QD/RvwFALy9fHnyf6N5svsrnDxnm9FTECa37kONHFWrpSD/+Y3TeGHTdL4Z+BH+Psb/LgQCOVE9yOn49NNPM2mSJpBp3rw5Bw4cYNWqVXTu3NmV0yuxmDK5JSRtvbxlp973gsJRHdSlfIN6dF+znNRzMWTfuUNA9Wr4V7Gea9xZHfQqVZqmL0/k5OrFxGz+ijuXL1C1Sw9KV9JcO/HwPmI2f2XzfRQkzujgt1vmkJRyTRu8W0IyAYKDqjG470y75ih3LlJD5NDBCR1eAGD+nnV63wXFG2s5uVcOa83odUcZs/4oK4a2FiutSwCmTG4JyeyOXvgtJ+Z/XShFnh3VQQlb4lpndbBc3YZUinzEaAxb4tqu0zfb+yORjckvr3ZKB5s26MSv21fx6/ZV5ORm2RXXusrkHvn9i/w6ZJ7J887qoBxxbVFHKEQJxRGT+/Od/zGqa+2Cn5wBTaqW5QLyF1EQWMZc4F3QZnfa5TjKhetvffIqXZrQNq2cGtc/rJJN28okFEolLcfP4MTy+dz89xgXt2zg4pYNRu2avjyRym1N/6z8KlTkkbnL2TtjLOq8PE5+vsSojdLTi44fLMGvQkXbb6YASEy+avGBIO1uCjWqNmZQnxnUqdFCe3zamPVcvHKcDT/NJe1uCoEB5lemJ968whPdRtG353htu3YRT8IV41V+uhS0yS0hzG5BQRMUFISHhwdJSUl6x5OSkggNNb3lMzQ01GL7oKAgPD09adiwoV6bBg0asG/fPgSuwZzJLVHYZjfYr4OGKJRKbZ5SW5FDByu37QRqNSc/X0JS1CGTW68rtTUO7AuTD6f/7pQOtmnei10Hv+fsRdMpjHS5n3WP0UMW0bltf832bCsaKlEwBbeMU8YIs1tgL7YUnpQWPwmzu2RgyeSWcIXZ7YgOGmIprnVWB5sMH2fyJbatca2rcFoHgeeefoNeXUew5Y8ldsW19iKXyX0x5YLFdsLsdg6RuqQE4uhK7uV/X+Tznf8V7OQERYKCSmMiPdS4C0pPLyImzKTjnE+p2qWH3rm6z7xA9+XfmDW5JUqHVKLHmh+ImPQWZWs93NblHVCWiIlv8diq7ygVbF9Os4KgS/uBFs/7+5dj1qTv9UxuiTo1WvLO5M34+1uuBt6l/XMMHfC+RRPAkMIyuSVEGhNBQeLt7U1ERAQ7duzQHlOpVOzYsYN27dqZ7NOuXTu99gDbt2/Xtvf29qZ169bExOivgLlw4QLVq1eX+Q4EtmJLwO2KNCauQA4drNyuM92WfU2DwSPxLPVw9VvF5q145MNlNHuwLdtVOKuDHh6edOv4PGNeWmz1WmOHLuHRDoNsykEqUTAmt3lEGhOBrdhicksY1nASaUyKL9ZMbglXpDFxFbbooNLTfAFGa3Gtq5FDBwMDghg2cLZTca0l5DS51w381mp7kcbEccRr0BKGM+lKxnavw/K/NSs3XLGy21Xk5apcdu38XIXF86o8zdzy81TkOzjP/AdjSGNpsfLbQe6V3bpv7t0N/0phNBryCo2GvOJQf4VSScUmLanYpKXMM5MPDyv5sq2dl2sMXQrb5JYQK7sFBcnkyZN56aWXaNWqFW3atGHx4sVkZGQwbNgwAIYMGUJYWBhz584FYMKECXTu3JmFCxfyxBNPsHHjRo4dO8bq1au1Y06dOpWBAwfSqVMnunbtyrZt2/j111/ZtWuXK25RD3fWUG07czpoB1o9fnC/tq4qk3tl95Vf9lCr9wCnxigI5NBBr1Klqd7tcap3s16Mevny5cyfP5/ExESaNWvGp59+Sps2bcy237x5M2+//TZXr16lTp06fPzxx3qFV4cOHcqXX36p16dHjx5s27ZN75izOiiH1hpS2Ca3hFjZLbCGPSa3RElb2V0UNFTCUAftRTeutcXklnDFym5XYY8OmsPZuLagKYyY014dBflNbk1ca72QtljZ7RjFVxUERjibk1syt0ua2T19ovuvsjq6Noaja63ns7LEj6/u1fvebX0Xq33kMrsNt6cJBK4yuSWE2S0oKAYOHMjNmzeZNWsWiYmJNG/enG3btmkLTl6/fh2lzrbT9u3bs2HDBt566y1mzpxJnTp12Lp1K40bN9a26du3L6tWrWLu3LmMHz+eevXq8eOPP9KxY8dCvz9DioKGShjqoCNsHqkpAGqLhkrIZXZf+WUPl7fsdEujuzDZtGkTkydPZtWqVURGRrJ48WJ69OhBTEwMwcHGL9MPHDjAoEGDmDt3Lr1792bDhg306dOH48eP6/0/69mzJ+vWrdN+9/EpuBolcuEqk1tCmN0CczhickuUJLO7KGmohKSDjnJ0bQydV063K6aUy+zOy8zG2/1/tQsKmYIxuW2nIMzuMB61a4yiRvFUBIERchWeLIlm96CXXPfG64qP5Tfp+Xkqjq6NIbRJecLbhTh0jasHk0g8lUrr4fXw8LQ/m5GzZrfJHGyuW7wgcANcbXJLCLNbUFCMGzeOcePGmTxnahV2//796d+/v8Uxhw8fzvDhw+WYnqy4s4ZKOKuDAIdWawoRtn3Zsdydzprdkslds19Xh65fnFi0aBGjRo3S7pJYtWoVv//+O2vXrmX69OlG7ZcsWULPnj2ZOnUqALNnz2b79u0sW7aMVatWadv5+PiYzaXvjrja5JYQZrfAEGdMbomSYnYXBQ2VcFYHdeNaRxZOOWt252VmE73wW9rMtK/Ir6B442qTW0Jus/vu+jEOzaOoUPzUQGCEXCa3REkzuyPauC6oUftbMbpzNQ8E4e1CCO/g+DwTT6US3j4UD6+HAf5Nte39HTW7bSk0Uthc3LqR0qGVCYloi4eXt839slJTSIw6RM69u3h4e1O+fmPK1qqHQmHbQ6JarQb7nicLDLVazXc/f0Tv7i8T4F/BoTHupt/it79X87ydRT7cxeSWEGa3QOAc7qyhupjSQXu4fiQZQKvF9miohKNmt67JXeOpTi59WXx1+29O6eCd/86TGnOG/JwcvMsEEBrRFt/yttd1yMnJISoqihkzHmqPUqmke/fuHDxousDVwYMHmTx5st6xHj16sHXrVr1ju3btIjg4mHLlyvHoo4/ywQcfUKGCeY10VAd1OfbvXygUCiKaPmZXP3cxuSUKwuyeV8xXoxVX5DC5JUqC2V1UNBSMddBedONaR3HU7JZM7vS4ZIevLQdy6GB+bg5JUYfISIwHcCiudScc1UEJZ+JadzG5JeQ0u4s7xUsJBEbIbXJLlDSzW2Ade81udzS5AS798r32c9WuPWjw/EiUHubzeGXeukn0ivmkXTFdqLXl+JkENzddWVsiOfoYx5d+SL91Zx2btMyc/+8wP/2xlAPHfmHZB9YrX5ti5twnSEq5RovGj9KgdqRNfdzN5JYoKLNb4FpclcNXILCEvWa3kcntYs5/t9bomD06aGo833IViJz5IX4VKprtr8rPByAlJYX8/HxtKiCJkJAQzp83XYQpMTHRZPvExETt9549e9KvXz9q1KjBpUuXmDlzJr169eLgwYN4mHlGcEQHdVGr1Xy84iUAvl8Vb/MLA3czuSXkNrvnYfxvTeDeyGlyS5QEs1tgH/aa3bomd4upLxbiTI35c8QzRsfs0cH/tm7k8u8/mjxvS1zrbjiqg7o4GtcWhsmdnn2/0Hc4SXFtcUeoQDGmoExuCWF2Cwyx1ewuCJNbrVajVqmcEm+1SkWNXn25dfZf7l67TOzOP0mPi6X11PdMjpuRlMDeGWO138N7PIWXfxmy76RyfccfABxf+iGNh42lyiPdTF7zxt4dnF633OE5FwQ79m0A4M7dZDKzMvDzLW2lhz6ZWRncuZusHcuWAN9dTW6JgjC7jy7sLdv8BPZR0nL4bl7c3ehYbp6KwVN3Mm5wQzq1quTQuHuOJbDs27N8O78rXuZSfhTNBUQuxVazuyBMbrVKsxxcoXRsZbscOliv/xDysrPISIwj8ch+sm7fYvfUV3hk7nJKhxj/W1Xl53N0/jsw8lmH5mwLzz33nPZzkyZNaNq0KbVq1WLXrl1062Z8X47ooCHHT/398PPpHUQ0Mf5/bIi7mtwScprdgqJFQZjcEsLsFhhiq9ltaHIH1gxDneecDoJGlxRKpUPGLDiug7cvaBZNVWjQlKAmzcm5d8/muNYdcUQHDXEkri2sldyuSudVEnYWO/6/V+DWFLTJLTGqa23Gdq/D8r8v8vlO0ytaBSULyexOj0smeuG35GVm6523xeTOy8y0+Xp5mZkcmTOfv0eMYceocWwfPpr/tvyiSQViI2lXrrF9+Gj+HjmWev1fpP07C+g4ZykAty+c5b+tG436qFUqrcldKrgSXT9ZS/2BQ6n1xDM0HDyKnmu30GDwSABOr1vO3etXjMa4e/2KNriv2bvgAnR7SEi+wp7DmpUAOTmZ/LX7Sys9jPlr93pycrMA2HPoBxJvXrXY3t1NbgnJ7K4XFM7zG6dxIt706kBLSEF+TMpV2ecnsB3dHL4NGzZk1apVlCpVirVrTa8Q1M3h26BBA2bPnk3Lli1ZtmyZXjsph6/0p1y5coVxO4JiSI2nOlGzX1cub9nJlV+Mi5FZMrmd1cG/R45l+/DRpF0y1i1zqNVqrv7xl9M62GPND9To1Yc6fZ6j+atT+N9nmyhTpToA+9+eqDXidTm3YY02uA8KCsLDw4OkpCS9NklJSWbza4eGhtrVHqBmzZoEBQXx33+mn33t1UFD1Go1m36Zh1LpgVLpwaZf5ln9+3SFye2sDr6waTrp2fftHkOY3UWLgjS5JSSz+1JSOmPWHyUjO69AriMoOkhmt39YMCfmf03a5Ti986ZMbsBpHdw+fDQ7Ro3j7xFjOPD2+3bFtXLoYOup79F66rvU6NnHrrjW3XBEBw1xJK4tzHQlzurg1E7DmL9nHUv2f2N3/+KOeNVZDCksk1tCrOwWGGJuZbetK7l3jp1Mt8+WovTysngdVW4uO8dONjp+5bc/yL6TRqPh1refpV26wpE584yO+1eqQuup73F0/jtc/v1Haj3VXy+3WcrpEwAovbzpOHuxyblW7/Y46XGxxO76k7PfrKbtzLl650+vWwFAvYFDqdHjKQBSL5/i/q04ytdqTqny9ue4y8/NIfnMfgCCG3WwOR/b5xumE594iWs3zqBUKMlXq7Q5zQ5G/UJYpTpUCq5BpeCaNK7XkcAATa64tLspnI7ZR0LyZRKSrxCXcJGrN85qH0SUCiUz5z5O9SqNqBxaSzNGSC1aNHoU5YOVEkXB5JaQc2W3wDW4Uw5fgcAS5lZ227KSWw4dPDJnHpGzphMQXt3qGGe++JKEA4ep1Uv/eo7ooC5KLy/aTP+AHeNeRJWXS8rpaCo2bak9n5+TTezOP7Xfvb29iYiIYMeOHfTp0wcAlUrFjh07zBaAbdeuHTt27GDixInaY9u3b6ddu3Zm7/fGjRvcunWLSpU0K+v2H93qlA5ejT3DpWvRxCddJiH5MrHxMSQmPzRYrlw/xYRZHalSuS6VgmtSOaQmtao3J7xqI8B1K7lF7QqBNQrD5JYQK7sFhphb2W3O5DbEER3UJSMuwa64tm6/542O26ODHecsxb9SFaMxbIlrSzdrqNfnfmoiqZeiKVUhjPI1m1i9f1PYE9c6q4MqlYoTZ/4hIekSCclXbI5rmwY9ob1GYefkFoWaCw6xoruYUdgmt4RY2S0wxHBld+rZy3alK7n+906r17i+YxcAZapXo/ua5Ty2diWdF38MQPy+A+RmZFgdQwrum44ZxWNrV+qdq9CgCeUbaIQ9KUr/weW/B7m8m46aYPHBpcGg4QDc+S+GvMyHb2vzMu9z99olAKp36wXAhT/Wsmv2QI6smMzfbz5JWmyM1fnrkp+bw955wzi4ZCwHl4xl77xhqPJybOr71+4vOR2zj3sZt8lXPVwFk5+fy6Vr/7L38I9s+mU+e4/8RCm/MtrzpfzKsPfIT5pzh3/k0rV/yc/Pfdhflce9jNucjtnHX7u/5Ksf3uP2nURtcA+41OR25A24XCu7Ba7BUg5f3Zy8utiaw/err75ix44dfPzxx+zevZtevXqR/yB3sEDgCIYru20xuZ3VwcfWrqTpmFEAHH7feh7H3IwMo+BeF3t00BRepUrTeup7AJxa+6neuaTjmusGVK+pPTZ58mQ+//xzvvzyS86dO8fo0aPJyMhg2LBhAAwZMkTvRdeECRPYtm0bCxcu5Pz587z77rscO3ZMa4ynp6czdepUDh06xNWrV9mxYwdPP/00tWvXpkePHgBO66Cnpxcbf57HL3+t4Ni/f+oF9xIJyZc59u+f/PLXCjb+PA+vByaFK9OVyLXDydEVbQL3pjBNbgmxsltgiOHK7tSzly2a3M7qYKdPPuKxtSvp/sUK/MMqA/bFtaawVQdNmdwStsS1EmmxMfz9Zm+OrJjMrtkDubhtHfZib1zrjA6CZtFK6u0Evtz8rl1xrYQrCk/KtcNJrOw2RhjdxQhXmdwSwuwWGCKZ3Wn/xXJi3tf4BZW1OSf3xc0/WW3z3w9bAWgxcYw2j5p3QAA1emseplNOnbHYX3ebVkirlibbVGzSAoD7SfF6x9Mua3Yw+AUZ5/PVRdcEz0q9ZfKz0lPT5uyWJdpj+TlZdj9UJJ/ZT+p/J7TfU/87QdLp/Tb1VSgUVvPItWneiymvfI6X18O/Py8vH6a88jltmps3KXTHHz3kE7o/ov+22ZUruR19KJDD7BYUL5577jmeeuopmjRpQp8+ffjtt984evQou3btcvXUBEUcXbPblpzccuig7mdrW5pvnTkHQHiv/5k8b48OmqNUsGYlWM7dNL3jGYkaba7QsJn22MCBA1mwYAGzZs2iefPmREdHs23bNu3LquvXr5OQkKBt3759ezZs2MDq1atp1qwZP/zwA1u3btXm3/fw8ODkyZM89dRT1K1blxEjRhAREcHevXu1efid1cEqlery/us/UTYgGKXCfO5UpcKDsgHBvP/6T4SF1gFwaU5uOdN5CbO7eOEKk1tCmN0CQySz2y+oLCfmfU3af7FWi1Q6qoM+gYGA5nd+xBuTAPviWnPYo4PmsBbXSlzcto78nIfpR8/8uNiu9CZqlcruuNYZHZR4rNOLjBnyiV1xLbjG5JYQZnfBIIzuYoKrTW4JYXYL5ELhYf3Xk7ZolkFBDVWeZgWlM0VEJHLu3bV43rtMgNPXcAdeG/ap2XMKhYJ2EU8x6eXP8PI0ToXi5enNpJc/o23LJy0+VLw27FO6th9odNxVJveK/UudeigQZnfRxF1y+AoEBYmcOmgNdb5pLZYTw8DeEO8yZfS+jxs3jmvXrpGdnc3hw4eJjHxYDHLXrl2sX79er33//v2JiYkhOzub06dP8/jjj2vP+fn58eeff5KcnExOTg5Xr15l9erVers8nNVBgEohNflg2i8EBlREqTT+WSqVHgQGVOSDN36lUsjDFeyuLDwpd+0KYXYXD1xpcksIs1tQmJjTQbWkxXbEteawVwdNjmElrnU1juqgLl07PGdXXOtKk1tCmN3yI4zuYoC7mNwSwuwWSEg5uQNrV6XFtBfJTLljskClKeo829dqm9rP9AEg6uNPtLkws9PSuLZtOwAVGjWw2F/XAEg6dtxkm1tn/gWgVEhlvePeAZq39feTTac6kFDlPdy+7Fu+gsnPqlxNm0bPTNQe8/D2pU7PYRbHNiS4UQfK126h/V6+dgtCGnewqe8jkc9QLcz0z0utVvN835l4ephfcefp4cXgvjPNFgmpFtaQRyKfsWku1pB1m7aTDwXC7C566ObwlZBy+JrLySvl8NXF3hy+/374uF1/DrzzP5pXL0cZXy++Gd3e7v66fwRFG910JZYKVErIoYO6n62Z5UFNNfkxr/z2h8nz9uigOTJvJQMQWKuu3vHSoRptTjkVbbF/YSCHDoZUrE6vrsNNnlOr1Tz+6AhCgqrpHXeVyZ2enQ4Is1ugjzuY3BLC7BZISDm5M1Pu0GLaiwTWrmqyQKUujupgzl2NmaxWqTixRFNs2Z641hxy6KC1uFaiTs9heHj7ar83emaiXS/OFUqlw3GtIzpoiD1xratNbglhdsuLMLqLOO5mcksIs1tgWHiyfMOaejm7rZndVR/tbPUa1f/3KADpcfH8PWIM24ePZs+k6QBUah+JV+nSVseInKVpf3LF52wfPlrvXHr8De5e1+QHC4mI1DvXZLgmd+fR+e+Qe998DtRrf/8fAIE1auPp97DAhKdfKe2DyrmNawHNQ0WXtzfRZswius/5lcCq9azOXxcPL286vbGOdhOW027Ccjq9sQ6liRXYplCpVMQnXtJ+N3yLfiPhgtUxDNvojhGfdAmVDBW95c5FCs4/FAizu+jhihy+9lLax5MVQ1tTK8Sf0euOcir2jtP3LSh6GObkNszZbQpndXD78NGcXKHZztvmzWlW+3uVLk3ljuZf+tijg6ZQ5eZy8vOlANR+coDeuZCItgDcOnfS6jwLAzl08EbCBRQotP2lMZQKJbEmtNhVJvfI7x8WHhVmtwDcy+SWEGa3wLDwZPmGNfVydhua3c7q4O6Jb7B9+Gj+HjmWe9diAfviWlPYqoO3zp0yO4Ytca1EYNV6dJ/zK23GLKLL25vsXnwFzsW19uqgIfbEte5gcksIs1s+hNFdhHFXk1tCmN0lF0OTW8rJbVig0pzZ3XX5J1YrU4Mm72fX5Yu0hT4k6gzoR6MRL9k014Dw6iYfYG6dPcm+tzSiV7VrD73K1ABBjVug8NRUcj/y0VtGq9HUKhUXfvyGmO+/BKDRS/omOkCjF18BIHbnn1zboTECytdsQpXWPa1WpjaH0tObSs27Uql5V5tNboDkW9fJzcvWbrlu1rAzvj6lUSg0MhEbZz1ovf4gsFUolPj5+tO0QacH3xXk5mZx81asvbejR0EW3BJmd8nCFTl8HUGY3SUbc4UnbTG75dDByFnTCaxVw6YxGg57kRpP6u8ecFQHdclOu8O+tyegytUUVg5q3FzvvIeXNzWfkGe3kBzIoYNXb5zRFs+qHFKLyiG1AE1Ry6uxlnOu24Kc27R1EWZ3ycYdTW4JYXaXXAxNbiknt2GBSnMru53VwbK1a9kV1zqrg0fnv0N6/A2jMWyJaw0pVT6UKq17Ur5mE6tzN4ejca2zOmhPXOsqk/tkfLTJ48LslgdPV09A4BjubnJLjOpaG4Dlf1/U+y4ovpgzuSUks/vE/K+JXvityTaefr7YiqefH+1mv41arUatUqF0IEdoYK0aPLZ2JWqViit//sLNk1GkPngjXq5uQxo8P9Koj0KppOPsJeydMZZ7N67x1ysDCW3TgcDw2qBWE7P5K23bxsPGElDN+CEpoFoNGg8by+l1yzn37RoadJ9s99zlIjZeUwm7eaOuPN9nJuFVG5GecYef/1rO739/rg3eLY4Rdx4vL196dxvFU/8bg3/pslyJPc13W+dy4vQ/xMbHEFKxukPzK0iTW2JCB01xsPl71ul9txUpyH9h03Se3ziNDc/No0Xl+nbPU1A4jBs3Trsi2xBTBST79+9P//79TbaXcvgWBJLZPWb9UUavO8rKYa1pUrVsgVxL4D6YM7klpGOXt+zU+w7w2NqVdl9PVwfB/tzeCoWC2n2fJHrVQqd08Ny3a6jxeD+8ywSQfSeVq3/+om3b6aMVJudVu89z3L54zq75FhTO6mC+Kp/4pMtUrFCFQX1m0KFVHwD2H9vKd1vnkpB0iXxVPh4mcpfagty5SA2RQwelIP/5jdN4YdN0vhn4Ef4+pax3FLgMdza5JSSze/S6o4xZf5QVQ1tT2kdYIcUZcya3hGR2Ry/8lhPzv9a26b5Gk27EUR2s3fdJVPn5KJRKqwURDXFWB29fOMu+t8ZTulIYYe27ovD0tDmudRfk0EE54lpzyGVyD9s0mPOTt5o8L4cOOhvXFnXEb/ciSFExuSWE2V1ysGZyS9hidtuLQqFwuhCWQqkkZtN67feaTzxD7T7PmTXPS4dUovP8zzj84Uyybt8i8ch+Eo/s12vTcvxMgpu3MnvNKo90w7tMIMeXfujU3J3Fy9Ob2VN/pn7tNtpj/qXLMrjvmzz+6EiOnDCde1WXerVa8+Kz71AuMFh7rEbVxsx87VvO/XeYbAdXZhWGyS0hzG6BOyLM7pKFNZNbwpLZ7SjOBrty6OCV/9uidz6wRh2aj3kdvwoVTfZXenjQeup7Ts1bLpzVwfT02wwb8B5dOwzSq4vxSJt+tIt4kp37vyM9/TaBAUF2z61gCm5dNGonzO6SRVEwuSWE2V1ysGZyS5gyu8uFO7arVhdHFl7p4qgOntuwhtidf5KREMeFH/VX8lqLa90FOXRQjrjW5NxkNLnrBNW12E6Y3c4hfrMXMYqayS0hzO7ij60mt0RBmN1yUPvpgZQKqUxIRKTRti5T+FWoSJeFn2uM7mOHyL2XhtLbm/L1m1C2Vl2b3uQHN29Fjy9+lGP6DtO8UVez58oFhtCjy1CrY1hq06B2pNlzlihMk1tCmN0Cd0SY3SUDW01uiYIwu52h/qARTungnUsxpJ4/jSonB68ygYS2aotvuQpW+7tL4O6sDgYGBPFYpyEmz3l6eJk9Z42CMbnNI8zukkFRMrklhNld/LHV5JYwNLsfXR5RSDM1xlkdbPTiK9R/bhhJUYe5nxQPYFdc6w7IoYNyxLWGyG1yrxnwNZBgsb0wux1H/FYvYhRFk1uiqJrdUUcSXXbtqz6Wg8P8PM0W46sHkxy/xoO+Vw8k4uH5cCVX6fa2j2GvyS3hjmZ37acHOtTPt1wFwh97wuHr2ru1rSTgCpNbQpjdAnekKJrd7qyh2nZmdNAe4qNvacbYr7lfezRUwl6TW8LQ7K7Ve4Cl5gWKszpYrnZ9ytUWvyvlpLBNbglhdhdviqLJLVGUzO6ioKEShjpoL7pxbSMHNBTsN7kldM1uVyKHDnp4eVO57SMyzkpQECa3rXGtMLsdwz1/owvMUlRNbomiaHZ/96X7F5VLPJVK4qlUp8Y4ujZG73s3Gx8wHDW5JUyZ3d6uX9gtcDGuNLklhNktcEeKmtldFDRUwlAHHeHQak2uaFs1VMJRk1tC1+x2pdEtcC9cZXJLCLO7eFKUTW6JomJ2FyUNlZB00FEST6VSLzPb7pjSUZNbQjK7BQJdXGlySxSE2T2PR+3qX9Rwv9/mAosUZZNboqiZ3R8tdt024Ch/y2/SVXkqfnx1L62H1yO8vWP5xK4eSOTo2hieWfUISp2VbLbY5s6a3BKGZnebmTMdGkdQPHAHk1tCmN0Cd6Qomd3urKES5nTQHvYuPgnAIxObArZpqISzJreEq9OWuJpn1ruuIOUgyrns2uZwtcktUVBmt8A1FAeTW6IomN1FQUMlDHXQXnTjWnt3+jprckt4+vmAyqGugmKIO5jcEnKb3fNY69A8igru9ZtcYBVXmdyLd2/j+W7yjVeUzG5PL9dVIfbwsu0Bw8NTiYeD85S2aSsNx1Bb7ieXyS2ha3a7G+kJccRsWs/Nk1EA+ASWo0avPoR1fBSvUqWt9lerVKScPsGptcvIuZsGQGDNOtR+agBBjVu4VaVrW3ljTg8UCgUfzdzmUP/0jDsMm9yA8SOW07Tik9rj7mRySwizW+COFBWzu6hoKJjQQTuQDHKpf9qlOJsCbblMbokaT3VyyyBdDh3MvZ/B9X/+4OKWDQAoPD2p3LYzNXr1wb+SY6aGK9HVwUfa9HNojL1HtuhpqIS7mNwSBWF2H13Y26k5CeynOJncEu5udhcFDZUw1EGHr+upJD0u2WazWy6Tuygghw6ai2sb9nBdbLL3yBandHDpF2NZt+gc/qXLyjYndzK5JeQ0u4s7Rc9dERQ60kON3IzqWpux3euw/O+LfL7zP9nHFxQccpvcEpLZ7S6o8nKJXrWIfW++pn0YAMhOu835jevYMe5F4g/utjhGRlICf736HFGL52iDe4C0yxeJWjyHP0c+S0aS5UIU7kZmVjqXr5/k0rV/yczKcGiMU+f3AbDhpw+1x9zR5JaY0OEFpnYaxvw961iy/xvrHQyQgvx6QeE8v3EaJ+KL3lZUgfshmd21QvwZve4op2LvuHpKggecmP81aZfjLLaR2+R2V+TQwfhDe9gx7kVtcA+gzssjbt8O9r35GlGL5xTY/AsKUzpoL99uMb5vdzO5JeTQQSnIj0m5KsucBPZR3ExuCcnsvpSUzpj1R8nIznP1lEo8Laa+qDW78zKzzbYrSSa3LTqoyss1299aXOtK5NDB0zH75JpOoZnczurgC5umk5593+4xSoLZLYxugUV039wXBMLsdl+u/LLH5PGCMrklTD2g5KZnkHDwCIlHjpGb4Zixeu9GHPH7DnLrzDnUKuvL3dSqfI4ueI/EIxrRrDdgCJ0XrKbj7CW0mjKLSg+KfJz8fAnxh0z/rDJv3WTvjLGo8zQPzK2nvkfn+Z/RbtZ8mo+ZivJB9eu9M8aSeeumQ/flCv7a/aXJz7aiVqv56Y8lANy6rakI7s4mt4TcZrdAIAfC7HZP/MOCLZrdrjC57dVBQ9QqFSmnzhC//yDpcba9oJVDB+MP7ubk6sUAlK/fmI5zltLpoxVEzphD/UHDAfSC9qKAoQ4mJF+xe4yEpMuk3tH/e3BXk1tCTrNbUPgUR5NbQpjd7oW0+MmS2e0Kk9sRHTTEUlzrrA4eXzrXpL7bGte6Cjl0cMsfS1GrrWxJt4HCXMktx0tfR83u4o777MsRuB2G29OS+b8CuU5RSmNSkri8ZSegn/OzoE1uQ9RqNWfXfU38voN6xyu1j6TRiJdQKKxvqcvNyGDXa68bHY+YOpHyDeqZ7Rd/aC+3L5wFoN3bHxNYo472nH9YVYIaNSegag1iNn/FydWLCW7WCk8//a1Dhz/U5BovU6U6baZ/oE1z4lehIoHhtQhu1op9b0/gfnIi0Svm022We1fIzs3NJvHmVX7atkx7bOufy2jZpBuhFcPx8rL+7yHjfhpHordxJfY0AEqFR5EwuSXkTGMiEMhFUUljUpJoPmUw0Qu/5cT8r40C8MI2uR3VQV1Sz8UQNX+x0fEuny7Aq7T5FF726mC7t/UNzNz7GZz8XGMI1xswhPAeT2u1v1RwKOXqNKBCgybsnzXJpvsoKGLjY5zSwY0/f8yQZ2dRLjAUpZU0LiqVittpiXz388coFR7a464wuZfs/8Yl6bxE+i/X4CqT+8SNq4SFF/x13D2NSUnDsIaTbuzpCpPbUR2UsCWudVYHU05Hk3j0AJUiO+qNYUtc60qc1cF8dR5Xrp9i18HvadO8J6VLBVq9phTXNqWm9lhhpyuRXvqKQs3yI35zC0xS2DnYhNntftTs11XP7C5skxvg0k+/ah8GavTuhTo/n6t//EXCgcP4h1UmvNf/rI6hG9zXHzyQm/+e4tbps0TNX8wjC+fiW66syX6n1iwFoO1b+g8DuoT3fJrkk1HcjjnD9X+2UfOJh7nFslJTyLp9C4B2b89D6eVl1F/p5UXkjA/ZOWk4aVes72hQq9Vc37+VxJN7KBUURv0nR+PlZ/3BSpfczAzO/7qS+ylxhDbtRLUOfVAoFAydVJ9KwTVpVLc9z/edqX3AOHzi//hr95fcSLjI7bQk1GqV3guGjPt3mPxeFxQKJeXLhhIWWpv/dX6JyBaPA5oHkQ0/fciZCwdISL5Mxn3NtnWl0gOVKp98VZ5LTe707PtOV612NMgXFD4J7y+z3kgm0rOzeP7rFcQkJ7BxyNgCD9KF2e1eePr5mDS7XbGS21EdlMi6fUcb3Fdo3JCgJo2I+W6zduzH1q403c8BHcy6fQvfchW05+P2/QNApbaPUKNnH5PXKVOlOi3G2r5LxpwO2kNabAwXt2k0oE7PYUx+r69TOnjg2M8cOPYzXp4+BAdVpWrl+gwd8B4VylUGICU1jvWb3+FGfAzJKbHk5umvbnTVSm5Ru0JQ0Jy4cZXnvlrO3lldC+V6psxuhHfkMkyZ3UChm9yO6qAutsS1zurgieXz+PezRUZGty1xrb0Y6mBgVdtenEvoxrVy6KBS6cGKLyey4ksoXSrQ5rj2ienXAdfk5C6IQs3C7NYgUpcIjHBVoRGRxsS9qPFUJ63Z/e+SjYVucgNc+e0PADov/pja/Z6iTv++dPpEYxBe3PyT1f7pcfHaz92/WEHVbl1oOfk16g3qD8DZdaYLX2bdTtV+Dqxh/qWLQqGg0ZBXALjwo34qi8SoQwCEtuloMriX8AksS3iPp6zciYbr+7cS9cWbxB3bzsVt6zmy0v4VbIdXTOLitvXEHdtO1Bdvcn3/VkCzwuzmrVi6dnhO7y16y8bd8PLyJfVOAmq1Ziuc7rYw6bNareLW7Xi8vf1o2fhh5VqlUknXDgO5eStWG9wDqFT52s+uXMntTG4zZ9OYCIovhiZ3YRWSFmlM3AvJ7JbSmPy7ZGOhm9yO6qAuMRs2AVD/xUG0nPwa1R57lO5frNCev3fDdHoWR3Qw8dghvXNX/tgKQFgHyyZXSERbyzehgzkdtJX7qYns/nAwsYd+J/bQ7+z+UGPyOqODErl52cQnXaJ5o67a4B4gqHwYLRp1JT7pktuY3ICoXSEoUCSTu15wpUK9rmEaE4Fr0U1jEvXhOqI+XFfoObkd1UFdbI1r5dDBrDsPY1lb41p7MKWD91MT7RpDN67VxREdBH1NtSeuBdcVnpS7doVIY/IQYXQL9HB1NW1hdrsXNZ7qRFCLeqSciCE/M7tQTW5V/kOx8g4I0H72CQw02cYUd69eA6DeoP56q7XCumhShNw6fdZkv9z0u9rP1lZ5KT1Mb4zJuacZo3RoZZPndfHyL2O1DUDiyT2gUIJaBWoVSaf3ozYRKJtDrcon+cx+bX8USs2YQLnAEGZP+4WwUP0HIC8vH6a88jmRLR63+LNQKBREtnyCKa98brR1Oyy0DrOn/UK5wBA8lMY/L1eZ3OnZ6U4X8nAmyBcUT1xlcksIs9u9kMzu/MxsUk7EENSiXqEWnnRUB3VJjooGoFK7NtpjCoWC+oMHAnDv2nWT/RzRwdx7aXrHs9NuA+BbtoJRH0ewpIO2knopmrys+6hV+ahV+eQ9KMrsjA5K/RUKBWOHLqFbx+eNznfrOJixLy3WtpNwZU5uUahZUFDomtwbXhxT6NfXNbsFriewZhhNXxtAemwS6bFJNH1tQKEWnnRUByXsiWvl0MHc9Hs6n22Pa23FlA6mXoq2awy9uFZnfo7ooC4eSk+74lpXmdwSwuwuGITRLdDiapNbQpjd7kPa5Thun7+q/R67/XChXVuh8/ZVt6iG7mpihZX8XdKDRMqpM3rH7ydo3jj7BVc02U/p7W3zPLPvaB4+lJ76q9U8Hozh6eNrfQydN+2WKBWk80CnUOJXLgSF0sN8BwMUSg/8yoZoHioeUDqoCgAfTPuVSsE1TPbz8vRm0qjPaN3M/O+F1s16MmnkKjw9TK/aqxRcg9nTfqFUqQCjc64yuUd+/6IsVauF2S2QcLXJLWFodgtci6523j5/1WyByoLAUR3UxctfkyIrMzlF7/jN6JMA+JQta7KfIzpoTn89/PysjmELlnTQVkpVMG2uOKODoHm+GTt0CZ3b9jd77c7tBjB26BK9ZyFXF54UZrdAbgxNbn8bfocUBJLZLXA9eZnZXN6yS/v98pZdJgtUFhSO6qCEPXGtHDooFXq2NJ4zmNJBc9podowg4/aO6qAupUsH2hXXutLklhBmt/wIo1sAuI/JLSHMbtejm5O788rp2jQmV36xb+WToygUCgJra4pDHJ79Edl30shJT+fQrA8A8A+rbPWtdIVGDQDNirXYnXvIy8ri3vVYDr37IQANh5rOKekXFKL9fD8l2eI10q5q/n1Wbt9Z73j5+o0BiNn8lcnq17pc/+cPi+cl6j85mpDG7UGhwK9cMG3HLbWpny6R45bgVy4YFEpCGren/lOvAhAcVNViPw8PT6pUqouHiQDew8OLqpXr4WFmdbtESFA1fH3syyluCjm3acvxUCDMbgG4j8ktoWt2C1yHbk7uziuna9OYFJbZ7agO6tJsrCZF16F353Dveiz5ublc3/4Pt86cAzBb0NIRHSxfv4ne8YrNWgGQeGSfxf45d9MsntfFnA7aSvmaTWgycCpKTy+Unl40GThVNh0Mr9LI6vWrhzXU++4qk/tkfLT2szC7BXJhzuR2VTwoal24Ht3Ck61mjaTVrJGkxyUTvfDbQjO7HdVBCXviWjl0sFTFYO1ne+JaWzGlg+VrNrHeUQfduFYXR3RQF1+f0nbFta42uSWE2S0vwugWuJ3JLaFrdgsKH8Oc3Lo5uwvL7G45aRwA967FsmfydHaPn6rNN9p65uuWugKaN+MRUycCcP7r79g5ZpI2uA9pE0H5+nVN9lN6eFCjVx8ADr4/lfxs0w9R925c4/xGTREOw+IgZWs9fOC5+NN3Zud4bcf/Wb0PCS+/0nSYvJq+a07Sa+E/lKvR2Oa+EuVrNqHXwn/ou+ZfOkxejaevJuDOyc2y2jc2/jwqVR4ACoUSxYMVcSpVHtdtEOPsnExSUuO0/QGzW7jNIXcuUpDnoUCY3SUbW0zujOy8Qp+XZHYLXINh4UnDnN2FYXY7qoO6lKtXh5A2EQAcevdD/nllvLYIV8S0SWZ3Vzmig2Vr6c+n3oCXAIj5/ivuXDb9PJiXncWRBe9avQ8JczpoD3V6DuPpz07w9GcnqNNzmGw6aMsYsTpt7NVQkHebti7C7BY4i6WV3GLxU8lE1+SWcnLr5uwuLLPbUR3Uxda41lkdrP5Yb70dv7bGtfZiqIP2ohvXyqGDygf3fPPWDbviWncwuSWE2S0fwugu4biryS0hmd2CwsdU4cnCNrs9/fzo9tlS6g58BoWH5tdV7Wf70O2zpXjauH2rfIN6PLJwLsGtWgLg5e9Pqzcm0/TVkRb7ScZ1bvo9Ds6ZTuatm3rnk6IOsX+WphhkxWatjHKQKhQKWo6fCcDl33/kyh9bycvK1J5X5eZy5qvPOPftGgBtW1uwJ12JrWPEJVoPHq7EnkatVuPj7Uf/3lPo33sKPt5+qNVqrsaetto/PvE/beGP0OAajBnyCa2a97B5zgVZcEuY3QJHsXUl95j1R11mdgsKH0OTW8IVZrejOqhL01dHEjFtkjbVSYXGDXlk4VyLRrkjOmi4U8u/UhiVIjX5xA998AapF87q1aa4n5LM7qmvkH7jms33op2fk1qqUCq15oZcOhgbZz2o1TUBWje379ld7lykhgizW+Ao1tKViJ2+JQ9TJreEK8xuR3RQF1viWjl0sNaTxmk/bIlrHUFXBx1FofRwWgfHDPmEkIrhgKYwtD1xratM7hX7Te/MFma3PIjopwRjr8l9KvaOS7ZvjepamwvkFPp1SzrmCk9KAfvlLTv1vstFXmY23jqXVXp5Ub1Hd6r36O7wmL7lytJszCi7+niXCaDjnE/Z9+ZrpN+4xu6pr5hsF9S4OS3GTjV5Lrh5KxoPG8vpdcuJ2fwVMZu/Mtmu8bCxBDdvZdf85CY27jw1qppfIZ6VfZ976ak8+dho+vYcRxn/8gD06PISW7ct56/d68nOuY+PdymzY1yPP0/FClUZ+OTrdIx8Bg+lB107PAdXcq3OryBNbgnpoeD5jdN4YdN0vhn4Ef4+5u/HFBM6aNIAzN+zTu+7oHhiT7qSS0npjFl/lBVDWwvzuQRgyuSWkMzu6IXfcmL+10YBfEHgiA4aUr5+XTp+9L5dfeTQwSYjxpF1+xa3L5zlyEdv2T3vwuDJx0Y7pYOXrv3Ld1vncj0+xuq1YuNiaNawC4P6TKdW9WY2aSgUTMEtSDBq46wOSkH+C5um8/zGaWx4bh4tKte3e66CooMtOblHddUUlZN2+krfBcUTSya3hGR2n5j/NdELvzUbu8qJIzqoi7W41lkd7DjnU7wfFHfWxZa4tsa6obbdRAHw+itrnNNBoFO7/uw7/CObfl1gV1xrL3KZ3Ev2LmBqx14mz8uhg3LEtUUZsaK7hOLISu7R645yKvZOwU5M4DZYelAoqJXd0kONu+BfKYxHl35JeM+nTZ5v9spkIia+ZVSIUpcqj3Sj/bsLKVfXOJdY2dr1aP/uQqo80k22OTtK3VqWjfZ8VR5L3t/PkGdnaU1ugAD/Cgx5dhZL3t9PXr7l1ap1a7Ziyfv76NxuAB52rKQrDJNbQqzsFtiKvTm5Vw5rrTW7XbGyW1C4mDO5JVyxsttVOKuDSk8v2kx7n6ajJugVkZSo0asvjy79UtY524uzOlirejPemrCRwX2t7+4a3O9N3prwnTa4t4WCMLktBfdiZbfAVuwpPClqOJUcrJncEq5Y2e0KbNVB/0rmf1bW4lpX46wOeig96NxuAEve22tXXGsPcprcEx6xnIZVrOx2DrGkqATiaLqSWiH+jF53lJXDWovCHALZV3brvrl3J7z9y1B/wEvUfeYFMlOSUeXm4OVfBt+y5a13fkBAtRpETv+AvMz7ZKXeAsC3fAU8/dznrWrog+1e5ijtF0BpvwCz5yuUq2T1GuaqX1uiME1uCbGyW2ANRwpPNqlalpXDWjN63VGxsrsEYIsmFsTK7rTLcZQLD3VqjILAWR1UKJVUbteZyu06k3Unldz0eyi9vPELCkbp4Xw6LzmQQwerVrZc0MzWNroUtsktIVZ2C6xhj8ktIVZ2lwxsMbklXLGy2xXIoYNyxLUFjbM66OXl43Rcawq5TW5NXGu5Fp1Y2e04IsIqYTiTk3vF0NaMWX+0xJnda1edctm171jRLFWeJs/jodXnuH7EMYM4PloTcO5dfBKl58M3xA0mWg/S5TK7DbenuSNKDw9Kh1gPYi3h6VcK/7DiLyxy4QqTW0KY3QJzOGJyS5REs9udNVTCnA7agzTGnk9OArZpKMhrdqddjuPE/K95dHmEQ/0LAzl00LdsebcKyt0ZV5ncEsLsFpjDEZNboiSZ3UVBQyUMddBedOPaVrNG2qWFJcXslnBWB+WIa0sSBWNy20ZBmd3FHZG6pAThbOHJ0j6erBjaWruyW6QxEYDzaUxsycEmKHm40uSWEGlMBIY4Y3JLSGa3SGMikJAjjYlkcvuHBRfADAVFEVeb3BIijYnAEGdMbgmRxqR440g8KEcaEzlTcgqKB640uSUKIo1Jcad4LyMSaHHW5JaQzO6StLJ7+KtNXHbtY/4Ki+fzc1VsHrmbti83ILyDY1uVr+5P5NDqczwysSkeXg/ffd1U2z6Goyu73dHkTo+LdTitSNbtVHLT76L09sYvKMRttlM7QkbmXUr5ljGq/G0rarWa+1n37N4WBu5hckuIld0CCTlMbomStLLbnTVUwpwO2oO0gq3TpKaAfRoKzq3s1jW5m08ZbN+FZcZZHVTl55OZkoQqJwcv/wB8yxXdldvO6KBETm4WAN5e9hmB7mJySxTEyu4wHnV4PgLXIYfJLVESVnYXBQ2VMNRBe9GNax3FmZXdV37Zw+UtO6nVe4DD15cDOXTQndNl2oujOqiLo3GtO5jcEnKv7C7uFM+oSqCHXCa3REk0uwXWsdfsdkeTG2Df2xMACKhei8bDxhBQzXI+TbVKRcKR/Zxc/YnRuerdn6DWk/3xLmM5yM25d5cr27bSYsCHjk9cRhJvXuW1t9rxSJt+jB+x3KExlq4dy74jP7Hsg0OEVKxucz93MrklhNktkNPklihJZrfANhwxuw1Nbk8/H1AV0oRNsGvKSL3v9urglT+2Gp1rMnI8ldt2QqEsWhtRHdVBXQaP0zyDbP4sweY+7mZyS8htdt9dP8bpOQkKFzlNbomSYHYL7MMRs1syuWv261pIszRNzOavnNLBu9evcObrz0i7dEHvuK1xrTviiA7q4mhc604mt4ScZndxp2g9MQrsRm6TW0KkMRGYwtY0Ju5qcuty99olDrw7hRt7d5hto8rLJWrxByZNboBrf//OPxOGkp5gfht6esIN/pkw1ORDjas4FPUbACfP7UGttnNZIppVbCfPaf7+Dx3/zeZ+7mhyS4g0JiWXgjC5JUQaE4Eh9qQxMWlyuxly6OCpNUs5Mm8Wqrxcs2NY0mpX4KgO6hKfdEn7OSHpsk193NXklpAzjYmgaFEQJreESGMiMMSeNCa6Jrej9abkwlkdPPDuFCOTG2yLa90RR3TQEEfi2sIyuV2Vzqsk1LoQRncxpqBMbglhdgtMYc3sLgiT+3bMRXaNf53tw0fz7/LPyLp9x67+qtxcrvz+J9uHj6bn2i38b/Um6g14CYDT65aTHH3MZL8Ty+aRcjoagBZjp9Fz7Rbtn0cXr8O/imYF1743XyMn/Z5R/5x7d9n3pkb4vPwd39osJ1nZ99n6p+Ztd9q9FE6c/sfuMY6f3sHde5rtcj9tW0aWDaawO5vcEnKb3QL3pyBNbglhdgsMscXstmRyO6uDf48cy9Vt21Hlmg+oTZF1+47TOthp3ipt/x5f/Ejbtz4G4PaFs5z6YpnJ6yZHH+X0Osd2HxUUjuigIVv+WIpS6YFS6cGWP5Zabe8Kk9sVL32lIF9QdChIk1tCmN0CQ2wxu02Z3M7q4PGFS9k+fDT7ps8i9VyMXf3l0MGqXXvwv882acewNa51R+zVQUMciWsLcyW3qF1RcAiju5hS0Ca3hDC7BaYwZ3bbanKfXLXG5mudXLmGYx8vIjc9A4DkqGj2Tplh84NFXmYmO14Zz38/btUeU3p6UaPn09R8oh8Ax5d+aPQGOD0hjpsnowDo8P4nhES01TvvHRBI2zfn4uVfBjD9hv7Sr5sB8K9SnS7zP9MeV6vybZq7OdQqFWqVfXvXz8Qc4O+937B8/QTuZ6YBoFR6sH7zO/z292dEndxOXOJ/5OblGPXNzcshLvE/ok5u57e/P+PLze+iVGryst7PTGP5+gn8vfcbzsQc4NbtBKOfZVEwuSXkNLsF7k1hmNwSwuwWGGLJ7La2kttZHVSrVFz8fgs7XhlPXmamTWOknoth75QZescc0cFSQQ+LaSoUCsrWrKMN8hMO7yUjMV6vv1qt5vjSuUbjOqKDxmPYrsXO6iBocoheuhrNviM/8f2vC9hz6AdUqnxUqnx2H9rM978uYN+Rn7h07V/uZ+q/NHDVSm5X7XCyN32YwHUUhsktIcxugSGWzG5LK7md0cFbZ84BkJl8k6j5i+2Ka53VwQaDR9LoxVdQenlpz9sa15rC2ZjU3jGc0UHQ/Cxu3U5wOK4t7HQlolBzwSGSQRZDCsvklhA5uwWmMMzZXfWxSJtXcicdieJ2186Uq1fH4jVSz8WQdFRjNrd99038goNIOHiE819/R9T8xXRfs9xqLrMjH84HwD+sMhFvTNI7V6fv81z+fQsAdy5doFztetpzUsBef9BwylQxnXvT08eXdrPms2faq1z54yfqPvM8igeBryo/n2t//w5Am9ffxcPHh9TLpzi0bDxZd5IJbtSBtmM/wdO3tMX5G3Lhj7Wc3bIEgEbPTKROT9sM1XcXPQOAh4eX9uFHpcon+eY1vvrhfdRqjWFQoVxlPpj2C0HlNX9/KalxvPnxk6Te0eRNUyiUKBVKVA8eatRqNUf//VNv63a7iKeYOHIlygd/N640uU/En3eqkIezObsF7klhmtwSIme3wBBTObsBq+lK6r84yCkdVOfnc2Lxcu5di+X4J8toM3Oqxf5qlYqo+YtN34OdOmiKsjXrUK//EGI2f8X579cTMX6m9tydS8ZmvqM6KJGbmcHhFZNIPrMfv7IhRI5bQvmalgvCjXuzrVM6uP/ozyxe86r2vIfS+P/+lv9bQr4q78E1FEwcuYr2rZ5yaboS6aWvqF0hMEVhmtwSIme3wBBTObtjtx82a3J3XjrfKR2s9/wAwjp35H5CEofenWNXXFuxgenUKbbqYPVuj5sd31pcGzyim/a7IzpodD92xrXO6CCASqVi8ZrRHIz65eEYNsS1B145BLgmJ3dBFGouCWlJbEGs6C5mFLbJLSFWdgtMobuye/foj+xKV/Lv8tVW25z98ltAY3KXqVYFT19fqnbtRIXGDQG0b9TNoVaryYjTBKZt338Lb399QVMoldTrPwSA1POn9M7FH9oNQGB4LYvX0H0rf/9msvZzZkqS9rN3QCAAh5aNJzstBdRqks8c4PwvqyyObUjq5ZOc/n4BqrxcVHm5nNo0n9TLp6x31CE/X3+LXr4qD7VahVLpQWBARWZN+l5rcgMElQ9j1qTvCSwThFLpgVqt0j6AmBqzdbMevDb8U21wD7h0Jbczuc2cXdktcE9cYXJLiJXdAkN0V3Yfe38Nx95fYzUnt7M66BMYSOTb0wFI+++y1ZyW0spx6ZqG2KOD5giN7AjATYMt16nnTwNQ7UFwL4cOnv91JclnDoBaTeadZA4vn2C1j7M62KH104wesgiFQoFCodBqr4Q0pnR+9IuLtMG9K3Nyi9oVAnO4wuSWECu7BYboruzePfojizm5ndXBat274uHlRZlqVWj7rsaQtieuNYetOmgOW+JaCUd00BB741pndBBAqVTy2rCltGrWQ3vMlrgWXFt4Us7aFWJl90OE0V2McJXJLSHMboEpqj4Wqf1crn64zTm5c9PTrbbJTL4JQKlKIXrHpWA75+5di/11tzUrFAqTbfKyswBQ5ehvbVLnaYJYn8ByVucpocp9OIbheABZd5L1tndlpNyweWyA+7fiTRwzXwBMl7Yte5v9GXgoPQkMqMgH036hcoixsR8WWpsP3viVwICKJt++g+bn2y7iSSa/8jlent5651xlcp+Mj3a6kIcwu4sXrjS5JYTZLTDE08+Hmv26aL/X7NfFauFJZ3VQdxW4tRQg2Q/ygVdsZn21lzUdNEdepunfr/kPxvApq9FiZ3RQ2z5Fp71aRebtJKtbr53VQYBHOwxi3NCl2ram+gOMG7qUrh2e0x53ZeFJUahZYApXmtwSwuwWGBJYM4xy9cO133VjVFPIoYN+wRUB++Jac9iqgxbHsBLXSjiig3rjqfIdimsd1UEJLy8fprzyuV1xrStNbglhdsuPMLqLCa42uSWE2S3QRcrJ7eHnQ1CLeqSciDFZoNIUwa1aWm1ToVEDAOJ279MeU6vVXNj4AwAB4eEW+ys9PLSfs9PSTLbJSNQIvVcZ06vNVPmWTSjdFQBSnlLNZ+Pik8GNOoBCqfmjVhHa1L7K3+VrNcfTtzQKpQcKpQeevqUpX6u5TX0njlxJpeCaJs/lq/J487VvCa0YbrZ/aMVw3nztW6NVbBKVQ2oxYcQKPD28TJ63Bzm3actRtVqY3cUDdzC5JYTZLdAl7XIcJz/9Hv+qIfhXDeHkp9+bLFApIYcO6hrkum1MUSa8GgDnv91kdj4S1nTQHNm3UwHwKVte77h3Gc0YuQ8KXTqjgxKhTTuBWqXV45DGHbTpViz2k0EHO7V9lr49zQfJ/XpNoFPbZ/WOucrklhBmt0AXdzC5JYTZLdDlyi97SDkRQ1CLenj4+ZgtUCnhqA7qal7CwcOAfXGtOWzVQUtYi2slHNVBCYXSw+G41hEd1MXTw8uuuNbVJreEMLvlRRjdxQB3MbklhNktAOPCk80mPGeyQKU56g3qb7VNw+Ga7VcxG77n+MKlxO7Yxd8jxmjP+4dVsjpGnf59AdgzaToXvt+id06Vm0vikf0AhLbSLzZZ55nBAJz9erXF7Wxply9qP/vqPJj4lnv4OSlKkxus7dhPqNtzGGGtHiNixByqdehjdf66lCofSueZ31C17RNUbfsEnWd+Q6nyoTb19fDw5F56qtnz97OsPzzdzzS/cvBueioeHs7nG5Y7F6kcDwXC7C76uJPJLSHMbgHoF56MmDmMiJnDTBao1MVZHfxvy8/snvgGADWetLwVGqBMFcs7tezRQXPE7dfU+6jR82m946EPCkFf/VOTk9MZHZSo1qEPESPmENbqMer2HEbkmE9s7iuHDmZmpaM0sSpcqfQ0qcWuMrlX7F+q/SzMbgG4l8ktIcxuAegXnmw24TmzBSoBp3Xw7xFjuPbXDo4v+pTzX28E7ItrzWGLDman3THb35a4VsIZHZRwJq61VwcNsSeudQeTW0KY3fIhjO4ijruZ3BLC7C7ZGJrcUroS3ZzdlszuiKkT8S1X1up1fMuVJWLqRECTh1R3NVmXTxfYNNfqPR+jUnvN1rVr27Zrj+fez+Dg7Gma65QPwrdcBb1+1bpq/r+lnj/N1W0/mxz7zuWLHJqjye/WdJRxbrOmL2uKX55YPo97N67h6VuaxgOmEDnmE6p37Gt2y5UlAqvWo9Woj2g16iMCq9az3uEBafdSuJdxG3hQSOvBW3uFQiMTsXHWhfJ6fIxeH6XSQ/v5Xnoqd9Nv2X4jJiiIglsgz0OBMLuLLu5ocksIs7tko2tySzm5dXN2WzK7wXEdvPLbNgAqd2xPrT69bRrD3LXs1UFD1Go1V/7YSsLhvQBUeaSb3nnf8kEE1tAvNueoDkooFAqqd+xL5JhPaDxgil1FoeXQwWtxZ8nPz8VD6anNReqh9CQ/P5frNyznXLcFObdp6yLM7pKNO5rcEsLsLtnomtxSTm7dnN2GZrccOnhh4w/cOn0WsC+udVYHD8+diSo312gMW+NaCWd0UMKZuNZZHbQnrnWVyZ2ebTqdjTC75UEY3UUYdzW5JYTZXTIxZ3JL2GJ2l29ge2BavkE9uq9ZTssp42k0Ygjt3n+Lx9auxKu0bYKsUChoPHIoXT5dQJNXR5CVmsKtc6fYMe5F7cNG5Iw5Rv28SpWm6csTAYjZ/BUnls8j5cy/ZKamkJmawpU/tnLoA81KgHJ1G1Ip8hGjMSq16UBQ4+YA7J81yeZ7LghuxF8AwNvLl36PT2DtwjNMfnk1wRWqAg+Dd0tIDw3BQdWY/PJq1i48Q79e4/H20gQ7sTaMYY6CMLl1H2qE2V0ycWeTW0KY3SUTUya3hDWz21kdbPzyMLp8uoBGw1+0OTD1Kl2ax9audFoHL/32A3evXyErNYX0hBscmTeLmM1fAdD05Yl4+pUyGqP5mKk2zbEwkEMHrz8Yo2WTbiyctZOFs3bSskm3B+ecM7rlzkVqiDC7SybubHJLCLO7ZGLK5JYwZ3Y7q4PtZs+i0YghtJg0ju5rltsV1zqrg/eTE9k+djDxB3dzPznRrrjWnXBWB+2Ja11lco/8/kWz54XZ7TzO7yMXuAR3N7klJLN7zPqjjF53lJXDWtOkallXT0tQQFgzuSWkB43LW3bqfXcUhVJpNa+ZNbxKlya0TSu2D39Ze0zp6UXHD5bgV6GiyT6V23YCtZqTny8hKeqQya3Xldo+QpPh4/QKmujOu+X4GZxYPp+b/x4zOl+YJN68whPdRtG353gCA4IAaBfxJG2a92LXwe85e/Gg1THuZ91j9JBFdG7bX7s9+7mn36BX1xFs+WMJiclXaVS3vd1zK2iTW0J6KHhh03Se3ziNDc/No0Xl+nZdRwryn984jRc2TeebgR/h72P8QCpwPUXB5JaQzO7R644yZv1RVgxtTWkf8QhXXLFkcktIZnf0wm85Mf9rPc31t5JKxBySDjrDsYXvGR2zRwcvbtnAxS0bjNo1fXmiRnNN4FehIo/MXe7UvOXCWR1Mu5tCjaqNGdRnBnVqtNAenzZmPRevHGfDT3NJu5ui1Wl7KJiCWxeN2sihgxM6vADA/D3r9L4L3I+iYHJLjOqqWfW6/O+Let8FxRNLJreEZHafmP810Qu/pfmUwU7roH9YJZvShpnDGR3cO2Ms6rw8Tn6+xKiNtbjWXZBDB+WIa80hl8l9MeWCxXbO6qAccW1RRkRJRZCiYnJLCLO7ZGCryS0ht9ktJ94BZWkyfBxBjZubDMx1qdyuMxWbtSL+4G4u/vQdefczAKjYvBX1BwyldGhli/2Vnl5ETJhJeoL57eeFQZf2z+FhosiIh4cn3To+T5f2A62OMXboEpNjBAYEMWzgbPLtqNYtUVgmt4Qwu0sGRcnklhBmd8nAFpNbwpLZ7So8S5V2Sgevbf+N2F1/as/VfeYFqj3a0+QKNl1KhzhuKsiJszro71+OWZO+N3muTo2WvDN5s0NaWjAmt3mE2V0yKEomt4Qwu0sGtpjcEoZmd5uZMwtplsZ0nPOpUzrYY80PpJyJ5tIvm7lzSbNq2Z641h2QQwfliGtNIafJvW7gt1bbC7PbcUSEVMQoaia3RFE2u/NyVS67dn6u5a1SqjzN3PLzVOQ7OM/8B2NIY2mx47eDvSa3hDua3T3XbrHeyACvUqWp3u1xqnezXqzEHP6VXGtOmHoYsOe8XGPoUtgmt4Qwu4s3RdHkliiKZrc7a6i2nTkdtAOtHkv368Bfiz0mt4Qps7tcuH3FF+Wk+7KvHe7rXymMRkNeodGQV2ScUeHirA7KobWGFLbJLSHM7uJNUTS5JYqS2V0UNFTCSAftRDeudQZ7TG4JXbPblTirgwqlkopNWlKxSUuZZ1Z4FEbMaa+OgvwmtyauNd4VZYgwux3DvaMjgRFF0eSWKKpm9/SJ5osmugtH18ZwdK3j+Y8Bfnx1r973buu72NTPUZNbwh3NboHrcZXJLSHM7uJJUTa5JYqa2V0UNFTCUAcdYfPI3YDtGirhiMktYWh2P7o8wq5rC4ovrjK5JYTZXTwpyia3RFExu4uShkpIOugoR9fG0M3BcNARk1tCMrsFAkMKxuS2nYIwu8N41K4xihruGxkJTFJUTW6Jomh2D3rJdW+8rvhYfpOen6fi6NoYQpuUJ7xdiEPXuHowicRTqbQeXg8PT/u2MzlrcksYmt21eg9waBxB8cDVJreEMLuLH0Xd5JYoSma3O2uohDM6KHFotaY4UtuX7a8X4YzJLaFrdgsE4HqTW0KY3cWL4mBySxQFs7soaKiEMzoI+nHtlV/22G1UO2NySwTWDAPXLaIXuCGuNrkl5Da7764f49A8igruGRUJzOIqk/vEjauEhcszVlEzuyPauG4LsNrfitGdq3kgCG8XQngHx+eZeCqV8PaheHg9DPBvqi33kcvkltA1u93N6Far1dz57zypMWfIz8nBu0wAoRFt8S1vezGo/NwckqIOkZEYD0Dp0MqERLTFw8u7oKZdoBz79y8UCgURTR9zqL9area7nz+id/eXgQDtcXcxuSWE2V28KA4mt0RRMbvdWUN1MaWD9nD9SDKAVosT7mfbZFjLYXJLSGa3uyKHDmalppAYdYice3fx8PamfP3GlK1VD4XCPkPGHdDVwQD/Cg6NcTf9FroaKuEuJrdEQZjd84r5ajR3pDiZ3BLubnYXFQ0FYx20F9249tBq+3b6ymFyFxWc1UFzcW0px2RIFu6m33JKB3/7ezXP95kh86zcx+SWkNPsLu64X0QkcDukh5q9s7rKNmZRM7sF+shtcku444NJcvQxji/90Oj4+e/W4luuApEzP7RYvVqVn89/Wzdy+fcfTZ6v2rUHDZ4fidLD/lxhrkKtVvPxipcA+H5VvEMGw/n/DvPTH0s5cOwX9ozQbMt0N5NboqDMbkHhU1xMbomiYnaXRKIXfmvVuJbT5Jbw9PNxu9Vocuhg5q2bRK+YT9qV/0yebzl+JuEtilbOSV0dXPbBQYfGmDn3CfaM1E+5424mt4TcZvc81soyL4HtFDeTW8Ldze6SSM1+XW1Oa1lSTG5bdDC4eSuLY1iKa/utOyvLPB1h5ke9ndLBpJRrtGj8KA1qR8o2p8IwudOz7xf6Dicpri3uuH/ZVYFL0X1zLzeS2V0rxJ/R645yKvaO7NcQOE7a5TiTxwvK5JYw94CiVqlQq5yL3lX5+ajVVpaq63Bj7w69h4F6/YdQ66kBhLbpAEDW7VvsnvoKGUkJZq93dP472uC+QoOm1BswhBq9+hJQvSYAsTv/5Oj8d1DlW64e7U4cP/X3w8+ndzg0xo59GwC4c1ez+sNdTW4J6aGgXlA4z2+cxon483aPIQX5MSlXS8SbdHfEVSb34t3bCmxsyey+lJTOmPVHycjOK7BrCWwnPS6Z6IXfkpeZbfJ8QZjctmCvDhqiVqvt0is5dDAjKYHdU1/RBvfhPZ6izjODqdatl7aNqcDd3dHVwcysDLv7Z2ZlaDVUwl1NbglDHUzPvm/3GBM6vMDUTsNknZfANoqjyS0xqmttxnavw/K/L/L5TtNGoqDwqPFUJ63ZfeUX83nKXWFy26uDJscwE9c6q4M39pqPy6zFta5EDh2UNFUOCmslt7M6OH/POpbs/8bu/iVhZ7FbLPtZvnw58+fPJzExkWbNmvHpp5/Spk0bs+03b97M22+/zdWrV6lTpw4ff/wxjz/+eCHOuGRguD3tPv/Ifg2xstt9OTH/ayMju6BNblOkXbnGkdn6bx3bvDmNwFo1bOqvVqu5tm07Fzf/pD1WOqwSbWZOxdPPz2y/u9evcHrdcgBq9n6WOn2eQ6F8+G5QNWI8B2dP496Na+x/eyKPrfpO7zzAuQ1ruH1B83a89dT3qNCgic7ZF0lPuMG+N8dz+8JZ/tu6kebPzrbpnlyJWq1m0y/zUD6oVr3pl3m0bNzNrlXdCclX2HNYY3rk5GS6vcktIffKbkHJYPHubcz753ee1wlG5Eas7HY/Wkx9kRPzvza5sruwTW5HdVCXvMxMji9aRtqly9pjNXr3olbfJy3+/rdXB+s+o596Ra1SsXfGWABKBVcicsYcfALLas83HDyKazv+j3PfrrHpPgoKtVrtlA7+tftLnu5hX77Mv3avJyc3S/vdFSb3ifjzLknnJXJ0uwZXmdzp2VmF4lqIld3uhWENJ0Mju7BNbkd1UBdrca2zOnh63XICqtckoJp+nGxLXOtK5NDBPYd+4NknJhFaMdzm/qZe/BdmuhLppa+oXSE/Lo+CNm3axOTJk1m1ahWRkZEsXryYHj16EBMTQ3BwsFH7AwcOMGjQIObOnUvv3r3ZsGEDffr04fjx4zRu3NgFd1A8MZWDzf53TbYhzG73xD8sWM/sdonJfekKR+bMMzp+ZM48ImdNJyC8utUxznzxJQkHDusdy4hLYOfYyXT7bClKLy+T/U6vWwFAvYFDqdHjKaPzSi8v2kz/gB3jXkSVl0vK6WgqNm2pPZ+fk03szj8B6DhnKf6VqhiN4V+pCq2nvqdd7WaL0X0/NZHUS9GUqhBG+ZpNrLY3RerlU9y/FUf5Ws0pVV6TR+/zDdOpFFyDSiG1aNHoUZQPHn6uxp7h0rVo4pMuk5B8mdj4GBKTr2jHunL9FBNmdaRK5bpUCq5J5ZCa1KrenPCqjQBQqVScOPMPCUmXSEi+QnziJa7dOINSoSRfrUKtVhcJk1tCTrNbUPyRTO5pjz5BQeeTEGa3exFYM8yk2e2KldyO6qCEKjeXnWMnGx2/8tsfZN9Jo9HwF032c0QHaz3VXy9nd8rpEwAovbzpOHuxyblW7/Y46XGxFu/BEFM6aA/5uTkkn9kPQHCjDk7r4Hc/f8TBqF8Iq1RHo8XBNWlcryOBAZpaIGl3Uzgds4+E5MskJF8hLuEiV2+c1QbprlrJLWpXlCxcZXI///UK1rxaOKmJhNntXpgzuwvb5HZUB3WxJa51Vgdjd/3J2W9W03bmXL3ztsS19mKog47UnZLiWjl0UKlQMnPu41Sv0ojKobVsjmt3D9+tnU9h5+QWhZoLDpdHQIsWLWLUqFEMG6bZgrZq1Sp+//131q5dy/Tpxlu7lyxZQs+ePZk6dSoAs2fPZvv27SxbtoxVq1YV6tyLK64oNCLMbvej+ZTBRC/8lhPzv6bpawO4vGVXoZrcgPZhoOmYUYS00pjISceOc3LF5xx+/yMeW7vSYv/cjAxtcN/pk4/wCQxErVZzaNYHpMfFc/3vnYT3+p9Rv7zM+9y9dgmA6hZWYXqVKq0N0E+t/ZRHF6/Tnks6rrluQPWaJoN7iQoNmlC+QRNSz52yeC8AabEx7P5wMHlZmtdOTQZOpU5P+7bvXvhjLae/XwCAp29pOs/8hsCq9fhr95coFApeHjxP+zAA4Onpxcaf53HnbjIKhcLkm++E5Msk3ryCWq2mbEAw7075QXtOqVSSejuBLze/a3ZOrjS5l+z/xumq1Y4G+YLija7JPbFzT5L5vwK/pjC73QtDs7tmvy6c/PT7QjW5HdVBXa7v2AVAmerViHz7DRRKJTl377J74hvE7ztA3YH98Cpd2qifIzqYFHWYym0f0Z7775fvAWg6aoLFQLzBoOEW70EXczpoK/m5OeydN4zU/zTmQ/naLUh1Ugfz83O5dO1fLl8/iVqtJqLp/2jT/OHzRym/Muw98hNRJ/8y0mJXpiuR0nkJs1tQEEgmd0xyAlB4z02GZnfnJ6oV2rUFxhia3dLnwkxX4qgO6mJrXOuMDsbu+pM7/8WQl3kfTz/N71Nb41p7MKWDnd5Yh9LTdrPbMK51RgcB8lV53Mu4zemYfZyO2Wd3XOuKwpMFUahZmN0aXBr95OTkEBUVxYwZDyukKpVKunfvzsGDppPRHzx4kMmT9d+m9ejRg61bt5psn52dTXb2w/yIaWlpANy9e9fJ2buGe1mZBTr+yfjrDPvuc+pUDGX1gOGo1WrtNdM9cwv02gDzB7Vg8rfHeXXtET55oSUNwwIBuK/Uzzman69ZGXc/y7FcpLl5mv45uflWx8hyYe7kXA/L26DyczX3kZeTT26mYz+LvBzN/eVm5qHK00+90XjMs0Qv/IYT874GoMUbQyhdKchszlG5UKoy9fKWVWjUgLzMTO1nidyMDKN0Ibokn/gXgGrdH8XD21s7RrPxo9n/xttc3PwTVbo8otdHzX3S4x/mJ1fl5qLKNf9v3ztA8280524aeZkP9z3ci70GQLk6DfWOm6J8/UaknjtFbma6xXYxv60mL/vhNuXTP3xC9Ueesfgz0EWtUnHmx8Xa73nZmcT8tpoWQ98DYMRzc2nf6mnuZ957OLeylZgx7hs+/PR57qXfRo3p/w8KlASUKceMcd9QLjBUb4wOrfugys9nzUbT1bCX9/2cmhVqkZ59z+R5c5xOOMWYLSOpVaE2i59eCajtHmP+nnVk5+Uwuu1Au/oBrOozi1Fb3mHQd1P54pn3aVqprl39i6oOFWUKWkMlVuz/myV7/mJCp/8xIrIz97IyC0VDAWpULM2iwS2Y9M1xXll7hEWDW3Jfabya3B4dNEdOrub3wf2sPLw8Tf8ecmcNlbCkg7aievBcImlxnlqjk6UrBdF0/EBOfPwVJ+Z9TemwijQe86ymTQFpqVL18N+5IzpoiJTypMmrw8l/8Eyt9PKieo/uXPvzb5KiThDaOgLQaKiEIzqYfuMaeZkR2uNplzVGk1eZAKtj5OZb1lCwroO2kHhyjza4B7SfndFB7fzUaiKadGf0kEXk5uWQm5ejPTd6yCJWfjmRKJ06GQBDNz7vlA5+fmgVqw5+yqvtXmNIq2F29XdWB2tXqMqafu8x4sdZDPpuKp/3e4/SPral0xEaWvgUloYCZORkM2LjGi7eTGTdoFGkZ9lfG8UZBrWrTk5ePsv/vkjrbpXNtrNFB61hKa4tChoqYaiD9qIb1xrqY9XHIlHl5mnN7vAnH6HqY5EFoqO6Giphjw6awta4FnBKByXS4+Pwrxym/SxhKa7NVVjXUAlTOhgX9TehTW1/8WAY10o4ooOmsDWuTc++55QOAqRnZ/DaTy9z6dZ/rOi3xua4VqXIcEoHJYa36kt2Xo5dcW1R1FBpzrbUmlGonalI4yTx8fGEhYVx4MAB2rVrpz0+bdo0du/ezeHDh436eHt78+WXXzJo0CDtsRUrVvDee++RlJRk1P7dd9/lvfdsf3gVCAQCgUAgEAgEAoFAIBAIBAKB+xAbG0uVKuZ3CoIbpC4paGbMmKG3AlylUpGamkqFChXsKhojEAgKh7t371K1alViY2MJCAhw9XQEAoFAICgyCA0VCAQCgcAxhIYKBO6LWq3m3r17VK5sfpeNhEuN7qCgIDw8PIxWYiclJREaarowTGhoqF3tfXx88PHRz4FYtmxZxyctEAgKhYCAAPGAIRAIBAKBAwgNFQgEAoHAMYSGCgTuSWBgoE3tHEskJRPe3t5ERESwY8cO7TGVSsWOHTv0Upno0q5dO732ANu3bzfbXiAQCAQCgUAgEAgEAoFAIBAIBMUbl6cumTx5Mi+99BKtWrWiTZs2LF68mIyMDIYNGwbAkCFDCAsLY+7cuQBMmDCBzp07s3DhQp544gk2btzIsWPHWL16tStvQyAQCAQCgUAgEAgEAoFAIBAIBC7C5Ub3wIEDuXnzJrNmzSIxMZHmzZuzbds2QkJCALh+/TpK5cOF5+3bt2fDhg289dZbzJw5kzp16rB161YaN27sqlsQCAQy4uPjwzvvvGOUckggEAgEAoFlhIYKBAKBQOAYQkMFguKBQq1Wq109CYFAIBAIBAKBQCAQCAQCgUAgEAgcxaU5ugUCgUAgEAgEAoFAIBAIBAKBQCBwFmF0CwQCgUAgEAgEAoFAIBAIBAKBoEgjjG6BQCAQCAQCgUAgEAgEAoFAIBAUaYTRLRAIBAKBQCAQCAQCgUAgEAgEgiKNMLoFAoFVhg4dikKh4NVXXzU6N3bsWBQKBUOHDtVra/inZ8+e7Nq1y+Q53T+7du1i/fr1Js/5+vrqXTs2Npbhw4dTuXJlvL29qV69OhMmTODWrVt67bp06aI3Rt26dZk7dy6iFq9AIBAIChtDnaxQoQI9e/bk5MmT2jYKhYKtW7ea7G9JSxMTE7XX6NOnj9m+d+7cMTu/M2fO8MwzzxAeHo5CoWDx4sVO3K1AIBAIBPajq5VeXl7UqFGDadOmkZWVpdfut99+o3PnzpQpU4ZSpUrRunVr1q9fr9fGkvaFh4cb6dzOnTvp3bs3FStWxNfXl1q1ajFw4ED27NljNKYlLTZ3X6b0WSAQyIcwugUCgU1UrVqVjRs3kpmZqT2WlZXFhg0bqFatml7bnj17kpCQoPfnu+++o3379nrHBgwYYNS2ffv2AAQEBBiNce3aNe01Ll++TKtWrbh48SLfffcd//33H6tWrWLHjh20a9eO1NRUvTmNGjWKhIQEYmJimDFjBrNmzWLVqlUF+BMTCAQCgcA0utq3Y8cOPD096d27t11jxMTEGOlkcHCw03O7f/8+NWvW5KOPPiI0NNTp8QQCgUAgcARJKy9fvswnn3zCZ599xjvvvKM9/+mnn/L000/ToUMHDh8+zMmTJ3nuued49dVXef311x265ooVK+jWrRsVKlRg06ZNxMTE8NNPP9G+fXsmTZpk1L6gtFggEDiOp6snIBAIigYtW7bk0qVLbNmyhcGDBwOwZcsWqlWrRo0aNfTa+vj4mA2OdY/7+fmRnZ1tsq1CobAYYI8dOxZvb2/++usv/Pz8AKhWrRotWrSgVq1avPnmm6xcuVLbvlSpUtrxhg0bxrJly9i+fTujR4+28ScgEAgEAoE86OpkaGgo06dP55FHHuHmzZtUrFjRpjGCg4MpW7as7HNr3bo1rVu3BmD69Omyjy8QCAQCgS3oamXVqlXp3r0727dv5+OPPyY2NpYpU6YwceJEPvzwQ22fKVOm4O3tzfjx4+nfvz+RkZE2X+/69etMnDiRiRMnsmjRIr1zTZs2Zfz48UZ9CkqLBQKB44gV3QKBwGaGDx/OunXrtN/Xrl3LsGHDCn0eqamp/Pnnn4wZM0ZrckuEhoYyePBgNm3aZDI1iVqtZu/evZw/fx5vb+/CmrJAIBAIBCZJT0/nm2++oXbt2lSoUMHV0xEIBAKBwO04ffo0Bw4c0MZvP/zwA7m5uSZXbr/yyiv4+/vz3Xff2XWNH3/8kdzcXKZNm2byvEKhsH/iAoGg0BFGt0AgsJkXXniBffv2ce3aNa5du8b+/ft54YUXjNr99ttv+Pv76/3RfdNuC2lpaUZj9OrVC4CLFy+iVqtp0KCByb4NGjTg9u3b3Lx5U3tsxYoV+Pv74+PjQ6dOnVCpVCbfygsEAoFAUNDo6mSZMmX45Zdf2LRpE0ql7Y/mVapU0dPIRo0aFeCMBQKBQCAoXCSt9PX1pUmTJiQnJzN16lQALly4QGBgIJUqVTLq5+3tTc2aNblw4YJd17tw4QIBAQF6u4p//PFHPa09deqUXh+hxQKB+yFSlwgEApupWLEiTzzxBOvXr0etVvPEE08QFBRk1K5r1656aUMAypcvb9e1ypQpw/Hjx/WOGa7etqeY5ODBg3nzzTe5ffs277zzDu3bt9fmAxcIBAKBoDDR1cnbt2+zYsUKevXqxZEjR6hevbpNY+zdu5cyZcpov3t5edk1h+vXr9OwYUPt95kzZzJz5ky7xhAIBAKBoKCQtDIjI4NPPvkET09PnnnmmQK9puGq7R49ehAdHU1cXBxdunQhPz9f77w5Ld67d692kRbAZ599pk3/KRAIChZhdAsEArsYPnw448aNA2D58uUm25QuXZratWs7dR2lUml2jNq1a6NQKDh37hx9+/Y1On/u3DnKlSunl+c0MDBQO973339P7dq1adu2Ld27d3dqngKBQCAQ2IuhTq5Zs4bAwEA+//xzPvjgA5vGqFGjhtm8oAEBAXoFnCXu3LmDh4cHpUuXxt/fn+joaO05e19ICwQCgUBQkOhq5dq1a2nWrBlffPEFI0aMoG7duqSlpREfH0/lypX1+uXk5HDp0iW6du0KaDQRNDuGDXXzzp07BAYGAlCnTh3S0tJITEzUrur29/endu3aeHqats7MaXGrVq30NDYkJMTu+xcIBI4hUpcIBAK76NmzJzk5OeTm5tKjRw+XzKFChQo89thjrFixgszMTL1ziYmJfPvttwwcONBsHjV/f38mTJjA66+/bteqcIFAIBAICgKFQoFSqTTSNEepV68eZ86cITs7W+/48ePHqVGjBl5eXnh6elK7dm3tH2F0CwQCgcBdUSqVzJw5k7feeovMzEyeeeYZvLy8WLhwoVHbVatWkZGRwaBBgwCNga1UKomKitJrd/nyZdLS0qhbty4Azz77LF5eXnz88cdOz9fPz09PY3VXfQsEgoJFrOgWCAR24eHhwblz57SfTZGdnU1iYqLeMU9PT5NpTsyhVquNxgBNZWulUsmyZcto3749PXr04IMPPqBGjRqcOXOGqVOnEhYWxpw5cyyO/8orrzB79mx+/PFHnn32WZvnJRAIBAKBs+jq5O3bt1m2bBnp6ek8+eST2jZXrlzRWw0GmmBdIjk5maysLL3zFSpUwMvLi8GDB/P+++8zZMgQpk2bRmBgIHv27GHx4sXMmzfP4txycnI4e/as9nNcXBzR0dHaVW0CgUAgELiC/v37M3XqVJYvX87rr7/OvHnzmDJlCr6+vrz44ot4eXnx888/M3PmTKZMmUJkZCSgSYk5cuRIpkyZgqenJ02aNCE2NpY33niDtm3batNZVqtWjYULFzJhwgRSU1MZOnQoNWrUIDU1lW+++QYwjn8tabE50tLSjPS9QoUKVK1a1dkfkUAgQBjdAoHAAaTtX+bYtm2bUWGQevXqcf78eZuvcffuXZPFRRISEggNDaVOnTocO3aMd955hwEDBpCamkpoaCh9+vThnXfesboyrXz58gwZMoR3332Xfv362VUATCAQCAQCZ9DVyTJlylC/fn02b95Mly5dtG0mT55s1G/v3r3az/Xq1TM6f/DgQdq2bUvZsmXZu3cv06dP56mnniItLY3atWuzaNEiRowYYXFu8fHxtGjRQvt9wYIFLFiwgM6dO7Nr1y4771QgEAgEAnnw9PRk3LhxzJs3j9GjRzNx4kRq1qzJggULWLJkCfn5+TRq1IiVK1cybNgwvb5Llizho48+4o033uDatWuEhoby2GOPMWfOHL1dwK+99hoNGjRg0aJFPPvss9y9e5cKFSrQrl07tm3bRpMmTfTGtaTF5ti1a5eezgKMGDGCNWvWOPJjEQgEBijUYt++QCAQCAQCgUAgEAgEAoFAIBAIijBiCaNAIBAIBAKBQCAQCAQCgUAgEAiKNMLoFggEAoFAIBAIBAKBQCAQCAQCQZFGGN0CgUAgEAgEAoFAIBAIBAKBQCAo0gijWyAQCAQCgUAgEAgEAoFAIBAIBEUaYXQLBAKBQCAQCAQCgUAgEAgEAoGgSCOMboHg/9m78+gaz/X/4+8tEZkREUHThEYINYcS1SQ1hKoWLaXUVBxFK8ZDqSSmoqa0itYULUpbSr+iplRiqLliSCI1BXViHk5jiJD9+yO/PMcWklAtqc9rrb1W9nPfzz3tnCzn2levR0RERERERERERPI1BbpFREREREREREREJF9ToFtERERERERERERE8jUFukVEREREngImk4nly5c/7mWIiIiIiPwlFOgWEREREfmbdO7cGZPJRM+ePbO19e7dG5PJROfOnfM0VkxMDCaTicuXL+epf0pKCk2bNn2A1YqIiIiI5B8KdIuIiIiI/I08PDxYvHgx169fN67duHGDRYsW8eyzzz7y+W7evAmAu7s7hQoVeuTji4iIiIg8CRToFhERERH5G9WoUQMPDw+WLVtmXFu2bBnPPvss1atXN65lZGTw8ccfU6ZMGezs7KhatSrff/89AMnJyQQFBQFQtGhRi0zwwMBA+vTpQ0hICK6urgQHBwPZS5f8/vvvtGvXDhcXFxwcHPDz82P79u0A7N27l6CgIJycnHB2dqZmzZrs2rXrrzwWEREREZE/xfpxL0BERERE5GnTtWtX5s2bR/v27QGYO3cuXbp0ISYmxujz8ccfs2DBAmbOnEm5cuXYuHEjHTp0oHjx4rz44ossXbqUN954g6SkJJydnbGzszPunT9/Pu+99x5btmy55/ypqakEBARQunRpfvzxR9zd3fn111/JyMgAoH379lSvXp0ZM2ZgZWVFXFwcBQsW/OsORERERETkT1KgW0RERETkb9ahQweGDh3K8ePHAdiyZQuLFy82At1paWmMHTuW9evXU7duXQDKli3L5s2b+eKLLwgICMDFxQUANzc3ihQpYjF+uXLlmDBhwn3nX7RoEefOnWPnzp3GON7e3kb7iRMnGDRoEBUqVDDGExERERF5kinQLSIiIiLyNytevDjNmjUjMjISs9lMs2bNcHV1NdoPHz7MtWvXaNSokcV9N2/etChvcj81a9bMsT0uLo7q1asbQe679e/fn27duvH111/TsGFDWrduzXPPPZeHnYmIiIiIPB4KdIuIiIiIPAZdu3alT58+AHz++ecWbampqQBERUVRunRpi7a8PFDSwcEhx/Y7y5zcS1hYGG+//TZRUVH89NNPhIaGsnjxYlq2bJnr3CIiIiIij4MeRikiIiIi8hg0adKEmzdvkp6ebjwwMkvFihUpVKgQJ06cwNvb2+Ll4eEBgI2NDQC3b99+4LmrVKlCXFwcFy9evG8fHx8f+vXrx9q1a2nVqhXz5s174HlERERERP4uCnSLiIiIiDwGVlZWJCYmkpCQgJWVlUWbk5MTAwcOpF+/fsyfP58jR47w66+/8tlnnzF//nwAPD09MZlMrFy5knPnzhlZ4HnRrl073N3dadGiBVu2bOHo0aMsXbqUrVu3cv36dfr06UNMTAzHjx9ny5Yt7Ny5E19f30e6fxERERGRR0mBbhERERGRx8TZ2RlnZ+d7to0aNYqPPvqIjz/+GF9fX5o0aUJUVBRlypQBoHTp0oSHhzNkyBBKlChhlEHJCxsbG9auXYubmxuvvPIKlStXZty4cVhZWWFlZcWFCxfo2LEjPj4+tGnThqZNmxIeHv5I9iwiIiIi8lcwmc1m8+NehIiIiIiIiIiIiIjIw1JGt4iIiIiIiIiIiIjkawp0i4iIiIiIiIiIiEi+pkC3iIiIiIiIiIiIiORr1o97ASIiIiJ/FbPZzK1bt7h9+/bjXoqIiDxCVlZWWFtbYzKZHvdSRERE5AmhQLeIiIj8I928eZOUlBSuXbv2uJciIiJ/AXt7e0qWLImNjc3jXoqIiIg8AUxms9n8uBchIiIi8ihlZGRw6NAhrKysKF68ODY2Nsr6ExH5hzCbzdy8eZNz585x+/ZtypUrR4ECqsopIiLytFNGt4iIiPzj3Lx5k4yMDDw8PLC3t3/cyxERkUfMzs6OggULcvz4cW7evImtre3jXpKIiIg8ZvraW0RERP6xlOEnIvLPpb/xIiIicif9y0BERERERERERERE8jUFukVEREREREREREQkX1ONbhEREXmqpIycZvF+z+/JtP3qc8q7lWTRO71wLPRo67yWHNHnkY4nIiIiIiIi2SmjW0RERJ5af3WQ+884efIkXbt2pVSpUtjY2ODp6Unfvn25cOHCY1vT3r17adeuHR4eHtjZ2eHr60tERESe7v3uu++oUKECtra2VK5cmVWrVuXYPyYmBpPJlO11+vRpo0/nzp0t2ooVK0aTJk3Yt29fjmMnJydjMpmIi4vL09pzExkZSZEiRR7JWHkRGBhISEjI3zafiIiIiEh+oEC3iIiIPJWe5CD30aNH8fPz49ChQ3zzzTccPnyYmTNnEh0dTd26dbl48eJjWdfu3btxc3NjwYIFxMfHM2zYMIYOHcq0adNyvO+XX36hXbt2vPvuu+zZs4cWLVrQokULDhw4kOucSUlJpKSkGC83NzeL9iZNmhht0dHRWFtb8+qrr/6pff5Vbt68+biXICIiIiLyj6VAt4iIiDx1nuQgN0Dv3r2xsbFh7dq1BAQE8Oyzz9K0aVPWr1/PqVOnGDZsGNOmTeP555837lm+fDkmk4mZM2ca1xo2bMjw4cON9ytWrKBGjRrY2tpStmxZwsPDuXXrltFuMpmYPXs2LVu2xN7ennLlyvHjjz8a7V27diUiIoKAgADKli1Lhw4d6NKlC8uWLctxPxERETRp0oRBgwbh6+vLqFGjqFGjRq4BcgA3Nzfc3d2NV4EClv98LVSokNFWrVo1hgwZwsmTJzl37lyuY2fJyh6Pjo7Gz88Pe3t7/P39SUpKMvrs3buXoKAgnJyccHZ2pmbNmuzatYuYmBi6dOnClStXjMzysLAwALy8vBg1ahQdO3bE2dmZHj16GHNdvnzZGDsuLg6TyURycrJxbcuWLQQGBmJvb0/RokUJDg7m0qVLdO7cmdjYWCIiIoz57rxPRERERORppUC3iIiIPFWe9CD3xYsXWbNmDb169cLOzs6izd3dnfbt27NkyRICAgJISEgwArqxsbG4uroSExMDQHp6Olu3biUwMBCATZs20bFjR/r27UtCQgJffPEFkZGRjBkzxmKO8PBw2rRpw759+3jllVdo3759jhnkV65cwcXFJcc9bd26lYYNG1pcCw4OZuvWrbmeR7Vq1ShZsiSNGjViy5YtOfZNTU1lwYIFeHt7U6xYsVzHvtuwYcOYNGkSu3btwtramq5duxpt7du355lnnmHnzp3s3r2bIUOGULBgQfz9/Zk6dSrOzs5GZvnAgQON+yZOnEjVqlXZs2cPH330UZ7WERcXR4MGDahYsSJbt25l8+bNNG/enNu3bxMREUHdunXp3r27MZ+Hh8cD71VERERE5J9GD6MUERGRp8qTHOQGOHToEGazGV9f33u2+/r6cunSJdzc3HBxcSE2NpY333yTmJgYBgwYYNTM3rFjB+np6fj7+wOZAewhQ4bQqVMnAMqWLcuoUaMYPHgwoaGhxvidO3emXbt2AIwdO5ZPP/2UHTt20KRJk2xr+eWXX1iyZAlRUVE57un06dOUKFHC4lqJEiUs6m3frWTJksycORM/Pz/S0tKYPXs2gYGBbN++nRo1ahj9Vq5ciaOjIwBXr16lZMmSrFy5Mlvmd16MGTOGgIAAAIYMGUKzZs24ceMGtra2nDhxgkGDBlGhQgUAypUrZ9xXuHBhTCYT7u7u2cZ8+eWXGTBggPH+5MmTua5jwoQJ+Pn5MX36dONapUqVjJ9tbGywt7e/53wiIiIiIk8rZXSLiIjIU+VJDnLfyWw259heqFAhXnrpJWJiYrh8+TIJCQn06tWLtLQ0Dh48SGxsLLVq1cLe3h7ILL0xcuRIHB0djVdWVvC1a9eMcatUqWL87ODggLOzM2fPns02/4EDB3j99dcJDQ2lcePGAJw4ccJi/LFjxz70/suXL8+//vUvatasib+/P3PnzsXf358pU6ZY9AsKCiIuLo64uDh27NhBcHAwTZs25fjx4wA0bdrUWM+dweJ7uXPvJUuWBDD23r9/f7p160bDhg0ZN24cR44cydM+/Pz88rznLFkZ3SIiIiIiknfK6BYREZGnypMe5Pb29sZkMpGYmEjLli2ztScmJlK8eHGKFClCYGAgX375JZs2baJ69eo4Ozsbwe/Y2FgjOxkyy3qEh4fTqlWrbGPa2v7vPAoWLGjRZjKZyMjIsLiWkJBAgwYN6NGjh0UN8FKlShEXF2e8zypp4u7uzpkzZyzGOHPmzANnJNeuXZvNmzdbXHNwcMDb29t4P3v2bAoXLsysWbMYPXo0s2fP5vr16/fc293ubDeZTADG3sPCwnj77beJiorip59+IjQ0lMWLF9/zM7p7fXfKyjS/84uM9PR0iz53l6wREREREZHcKaNbREREnipPcpAboFixYjRq1Ijp06cbAdosp0+fZuHChXTu3BnAqNP93XffGbW4AwMDWb9+vfEwwyw1atQgKSkJb2/vbK8HKfMRHx9PUFAQnTp1ylbf29ra2mLcrEB33bp1iY6Otui7bt066tatm+d5ITPTOSvT+n5MJhMFChQwzq506dLGejw9PR9ovrv5+PjQr18/1q5dS6tWrZg3bx6QWUrk9u3beRqjePHiAKSkpBjX7vxyADIzy+8+rzs9yHwiIiIiIk8LBbpFREREnjDTpk0jLS2N4OBgNm7cyMmTJ1m9ejWNGjXCx8eHESNGAJkB0aJFi7Jo0SKLQPfy5ctJS0ujXr16xpgjRozgq6++Ijw8nPj4eBITE1m8eLFFRnZuDhw4QFBQEI0bN6Z///6cPn2a06dPGw/EvJ++ffuyevVqJk2axMGDBwkLC2PXrl306dPH6DN06FA6duxovJ86dSorVqzg8OHDHDhwgJCQEH7++Wd69+5tMXZaWpqxjsTERN5//31SU1Np3rx5nveVm+vXr9OnTx9iYmI4fvw4W7ZsYefOnUYddS8vL1JTU4mOjub8+fMWpWDu5u3tjYeHB2FhYRw6dIioqCgmTZpk0Wfo0KHs3LmTXr16sW/fPg4ePMiMGTM4f/68Md/27dtJTk7m/Pnz2TLuRURERESeRipdIiIiIk+VkiP65N7pMStXrhw7d+4kLCyMNm3acPbsWcxmM61ateLrr7826m6bTCbq169PVFQUL774IpAZ/HZ2dqZ8+fIWZTOCg4NZuXIlI0eOZPz48RQsWJAKFSrQrVu3PK/r+++/59y5cyxYsIAFCxYY1z09PUlOTr7vff7+/ixatIjhw4fz4YcfUq5cOZYvX87zzz9v9ElJSeHEiRPG+5s3bzJgwABOnTqFvb09VapUYf369QQFBVmMvXr1aiPL28nJiQoVKlhkuD8KVlZWXLhwgY4dO3LmzBlcXV1p1aoV4eHhxv569uzJW2+9xYULFwgNDSUsLOyeYxUsWJBvvvmG9957jypVqlCrVi1Gjx5N69atjT4+Pj6sXbuWDz/8kNq1a2NnZ8cLL7xgPCR04MCBdOrUiYoVK3L9+nWOHTuGl5fXI9uviIiIiEh+ZDLn9qQjERERkXzmxo0bHDt2jDJlyljUn87PQkNDmTx5MuvWraNOnTqPezkiIo/dP/FvvYiIiDw8ZXSLiIiI5APh4eF4eXmxbds2ateu/UB1tUVERERERP7pFOgWERERySe6dOnyuJcgIiIiIiLyRFIqkIiIiIiIiIiIiIjkawp0i4iIiIiIiIiIiEi+pkC3iIiIiIiIiIiIiORrCnSLiIiIiIiIiIiISL6mQLeIiIiIiIiIiIiI5GsKdIuIiIiIiIiIiIhIvqZAt4iIiIiIiIiIiIjka9aPewEiIiIif6e9Y1/5W+er+uGqv3U+ebp99NFHnDlzhi+//BIAs9nMv/71L77//nsuXbrEnj17CAkJoVq1akydOvXxLvYvlJycTJkyZdizZw/VqlUjISGBxo0bk5SUhIODw+NenoiIiIj8BZTRLSIiIk+VWRsOP+4l5MnJkyfp2rUrpUqVwsbGBk9PT/r27cuFCxce25r27t1Lu3bt8PDwwM7ODl9fXyIiIvJ073fffUeFChWwtbWlcuXKrFqV8xcAMTExmEymbK/Tp08bfTp37mzRVqxYMZo0acK+fftyHDs5ORmTyURcXFye1p6byMhIihQp8kjGyovAwEBCQkKyXT99+jQREREMGzbMuLZ69WoiIyNZuXIlKSkpPP/88yxbtoxRo0Y90jV17tyZFi1aPNIxH6WKFStSp04dJk+e/LiXIiIiIiJ/EQW6RURE5Kny+fpDT3yw++jRo/j5+XHo0CG++eYbDh8+zMyZM4mOjqZu3bpcvHjxsaxr9+7duLm5sWDBAuLj4xk2bBhDhw5l2rRpOd73yy+/0K5dO95991327NlDixYtaNGiBQcOHMh1zqSkJFJSUoyXm5ubRXuTJk2MtujoaKytrXn11Vf/1D7/Kjdv3vxLx589ezb+/v54enoa144cOULJkiXx9/fH3d0da2trXFxccHJy+kvX8iTq0qULM2bM4NatW497KSIiIiLyF1CgW0RERJ4qvRuWe+KD3b1798bGxoa1a9cSEBDAs88+S9OmTVm/fj2nTp1i2LBhTJs2jeeff964Z/ny5ZhMJmbOnGlca9iwIcOHDzfer1ixgho1amBra0vZsmUJDw+3CPqZTCZmz55Ny5Ytsbe3p1y5cvz4449Ge9euXYmIiCAgIICyZcvSoUMHunTpwrJly3LcT0REBE2aNGHQoEH4+voyatQoatSokWuAHMDNzQ13d3fjVaCA5T9fCxUqZLRVq1aNIUOGcPLkSc6dO5fr2Fmyssejo6Px8/PD3t4ef39/kpKSjD579+4lKCgIJycnnJ2dqVmzJrt27SImJoYuXbpw5coVI7M8LCwMAC8vL0aNGkXHjh1xdnamR48exlyXL182xo6Li8NkMpGcnGxc27JlC4GBgdjb21O0aFGCg4O5dOkSnTt3JjY2loiICGO+rPsWL15M8+bNjTE6d+7M+++/z4kTJzCZTHh5eQHZM8K9vLwYO3YsXbt2xcnJiWeffdYofZLl5MmTtGnThiJFiuDi4sLrr79uzBsWFsb8+fNZsWKFsaaYmJg87TUrG37NmjX4+vri6OhofHlxp9mzZ+Pr64utrS0VKlRg+vTpFu07duygevXq2Nra4ufnx549e7J9zo0aNeLixYvExsZmaxMRERGR/E+BbhEREXmqdA/yfqKD3RcvXmTNmjX06tULOzs7izZ3d3fat2/PkiVLCAgIICEhwQjoxsbG4urqSkxMDADp6els3bqVwMBAADZt2kTHjh3p27cvCQkJfPHFF0RGRjJmzBiLOcLDw2nTpg379u3jlVdeoX379jlmkF+5cgUXF5cc97R161YaNmxocS04OJitW7fmeh7VqlWjZMmSNGrUiC1btuTYNzU1lQULFuDt7U2xYsVyHftuw4YNY9KkSezatQtra2u6du1qtLVv355nnnmGnTt3snv3boYMGULBggXx9/dn6tSpODs7G5nlAwcONO6bOHEiVatWZc+ePXz00Ud5WkdcXBwNGjSgYsWKbN26lc2bN9O8eXNu375NREQEdevWpXv37sZ8Hh4eXLx4kYSEBPz8/IxxIiIiGDlyJM888wwpKSns3LnzvnNOmjTJCBD36tWL9957zwj0p6enExwcjJOTE5s2bWLLli1GQPrmzZsMHDiQNm3aWGTX+/v75/ncr127xsSJE/n666/ZuHEjJ06csDjDhQsXMmLECMaMGUNiYiJjx47lo48+Yv78+UDm5/7qq69SsWJFdu/eTVhYmMX9WWxsbKhWrRqbNm3K89pEREREJP/QwyhFRETkqdM9yBvILGNy5/snwaFDhzCbzfj6+t6z3dfXl0uXLuHm5oaLiwuxsbG8+eabxMTEMGDAAKNm9o4dO0hPTzcCjuHh4QwZMoROnToBULZsWUaNGsXgwYMJDQ01xu/cuTPt2rUDYOzYsXz66afs2LGDJk2aZFvLL7/8wpIlS4iKispxT6dPn6ZEiRIW10qUKGFRb/tuJUuWZObMmfj5+ZGWlsbs2bMJDAxk+/bt1KhRw+i3cuVKHB0dAbh69SolS5Zk5cqV2TK/82LMmDEEBAQAMGTIEJo1a8aNGzewtbXlxIkTDBo0iAoVKgBQrlw5477ChQtjMplwd3fPNubLL7/MgAEDjPcnT57MdR0TJkzAz8/PImu5UqVKxs82NjbY29tbzHfixAnMZjOlSpWyWJeTkxNWVlb3XNudXnnlFXr16gXAv//9b6ZMmcKGDRsoX748S5YsISMjg9mzZ2MymQCYN28eRYoUISYmhsaNG2NnZ0daWlqu89xLeno6M2fO5LnnngOgT58+jBw50mgPDQ1l0qRJtGrVCoAyZcoYX9Z06tSJRYsWkZGRwZw5c7C1taVSpUr8/vvvvPfee9nmKlWqFMePH3/gNYqIiIjIk08Z3SIiIvJUetIzu81mc47thQoV4qWXXiImJobLly+TkJBAr169SEtL4+DBg8TGxlKrVi3s7e2BzNIbI0eOxNHR0XhlZQVfu3bNGLdKlSrGzw4ODjg7O3P27Nls8x84cIDXX3+d0NBQGjduDGQGW+8cf+zYsQ+9//Lly/Ovf/2LmjVr4u/vz9y5c/H392fKlCkW/YKCgoiLiyMuLo4dO3YQHBxM06ZNjWBm06ZNjfXcGSy+lzv3XrJkSQBj7/3796dbt240bNiQcePGceTIkTzt484M67zKyuh+ENevXwfA1tb2gecDy71nBe2z9r53714OHz6Mk5OTcZYuLi7cuHEjz+eQE3t7eyPIDZlnnzX31atXOXLkCO+++67F79bo0aONuRMTE6lSpYrF3uvWrXvPuezs7Cx+30VERETkn0MZ3SIiIvLUehIzu729vTGZTCQmJtKyZcts7YmJiRQvXpwiRYoQGBjIl19+yaZNm6hevTrOzs5G8Ds2NtbITobM8g7h4eFGVuyd7gwQFixY0KLNZDKRkZFhcS0hIYEGDRrQo0cPixrgpUqVIi4uznifVdLE3d2dM2fOWIxx5syZB87+rV27Nps3b7a45uDggLf3/z632bNnU7hwYWbNmsXo0aOZPXu2EQS+e293u7M9K3M5a+9hYWG8/fbbREVF8dNPPxEaGsrixYvv+Rndvb47ZWWa3/lFRnp6ukWfu0vW5IWrqysAly5donjx4g98f06fe2pqKjVr1mThwoXZ7stprrzs9X5zZ92TmpoKwKxZs3jhhRcs+llZWd137vu5ePGiRVBdRERERP45lNEtIiIiT7UnLbO7WLFiNGrUiOnTpxsB2iynT59m4cKFdO7cGcCo0/3dd98ZtbgDAwNZv3698TDDLDVq1CApKQlvb+9srwcp8xEfH09QUBCdOnXKVt/b2traYtysQHfdunWJjo626Ltu3br7Zt3eT1xcnJFpfT8mk4kCBQoYZ1e6dGljPZ6eng803918fHzo168fa9eupVWrVsybNw/ILCVy+/btPI2RFRi+82GLd345AJnZ1Xef153uNd9zzz2Hs7MzCQkJeVrHg6hRowaHDh3Czc0t2+9O4cKF77umvOw1NyVKlKBUqVIcPXo029xlypQBMsv57Nu3jxs3bhj3bdu27Z7jHThwgOrVqz/QGkREREQkf1CgW0RERJ56T1qwe9q0aaSlpREcHMzGjRs5efIkq1evplGjRvj4+DBixAggMyBatGhRFi1aZBHoXr58OWlpadSrV88Yc8SIEXz11VeEh4cTHx9PYmIiixcvtsjIzs2BAwcICgqicePG9O/fn9OnT3P69GnjgZj307dvX1avXs2kSZM4ePAgYWFh7Nq1iz59+hh9hg4dSseOHY33U6dOZcWKFRw+fJgDBw4QEhLCzz//TO/evS3GTktLM9aRmJjI+++/T2pqKs2bN8/zvnJz/fp1+vTpQ0xMDMePH2fLli3s3LnTqKPu5eVFamoq0dHRnD9/PsfSGN7e3nh4eBAWFsahQ4eIiopi0qRJFn2GDh3Kzp076dWrF/v27ePgwYPMmDGD8+fPG/Nt376d5ORkzp8/T0ZGBgUKFKBhw4bZMt4fhfbt2+Pq6srrr7/Opk2bOHbsGDExMXzwwQf8/vvvxpr27dtHUlIS58+fJz09PU97zYvw8HA+/vhjPv30U3777Tf279/PvHnzmDx5MgBvv/02JpOJ7t27k5CQwKpVq5g4cWK2cZKTkzl16lS2B6OKiIiIyD+EWUREROQf5vr16+aEhATz9evXH/dSHtqxY8fMnTp1MpcoUcJsMpnMgLlVq1bmq1evWvR7/fXXzdbW1uY//vjDbDabzbdv3zYXLVrUXKdOnWxjrl692uzv72+2s7MzOzs7m2vXrm3+8ssvjXbA/MMPP1jcU7hwYfO8efPMZrPZHBoaagayvTw9PXPdz7fffmv28fEx29jYmCtVqmSOioqyaO/UqZM5ICDAeD9+/Hjzc889Z7a1tTW7uLiYAwMDzT///HO2e+5ch5OTk7lWrVrm77//Pse1HDt2zAyY9+zZYzabzeYNGzaYAfOlS5eMPnv27DED5mPHjpnT0tLMbdu2NXt4eJhtbGzMpUqVMvfp08fi96tnz57mYsWKmQFzaGio2Ww2mz09Pc1TpkzJNv/mzZvNlStXNtva2prr169v/u6774y5ssTExJj9/f3NhQoVMhcpUsQcHBxsrC8pKclcp04ds52dncV9q1atMpcuXdp8+/ZtY5wpU6Zk+3wCAgLMffv2Nd7fa51Vq1Y19mE2m80pKSnmjh07ml1dXc2FChUyly1b1ty9e3fzlStXzGaz2Xz27Flzo0aNzI6OjmbAvGHDhjztdd68eebChQtbzP3DDz+Y7/6/KQsXLjRXq1bNbGNjYy5atKj5pZdeMi9btsxo37p1q7lq1apmGxsbc7Vq1cxLly61+IzNZrN57Nix5uDg4Gyfh+Rf/4S/9SIiIvLomMzmXJ50JCIiIpLP3Lhxg2PHjlGmTJmHfjjfkyY0NJTJkyezbt066tSp87iXI08gs9nMCy+8QL9+/WjXrt3jXs4T5ebNm5QrV45FixZZ/JcOkr/9E//Wi4iIyMNT6RIRERGRfCA8PJxPP/2Ubdu2ZXs4pAhk1if/8ssvuXXr1uNeyhPnxIkTfPjhhwpyi4iIiPyDKaNbRERE/nGU5Sci8s+nv/UiIiJyJ2V0i4iIiIiIiIiIiEi+pkC3iIiIiIiIiIiIiORrCnSLiIiIiIiIiIiISL6mQLeIiIiIiIiIiIiI5GsKdIuIiIiIiIiIiIhIvqZAt4iIiIiIiIiIiIjkawp0i4iIiIiIiIiIiEi+Zv24FyAiIiLyd/J7vjj/7lb1oe5Nv5VB+0Eb6NO+Ii/5lczTPa1D1j/UXCIiIiIiIpJ3yugWEREReQKdPHmSrl27UqpUKWxsbPD09KRv375cuHDhsa1p7969tGvXDg8PD+zs7PD19SUiIiLX++Lj43njjTfw8vLCZDIxderUXO9JTk7GZDJle23bts3oExYWZtFWuHBh6tevT2xsbK7jm0wmli9fnmu/vIiJicFkMnH58uVHMl5uOnfuTIsWLf6WuURERERE8gsFukVERESeMEePHsXPz49Dhw7xzTffcPjwYWbOnEl0dDR169bl4sWLj2Vdu3fvxs3NjQULFhAfH8+wYcMYOnQo06ZNy/G+a9euUbZsWcaNG4e7u/sDzbl+/XpSUlKMV82aNS3aK1WqZLRt3bqVcuXK8eqrr3LlypUH3t9f7ebNm497CSIiIiIi/1gKdIuIiIg8YXr37o2NjQ1r164lICCAZ599lqZNm7J+/XpOnTrFsGHDmDZtGs8//7xxz/LlyzGZTMycOdO41rBhQ4YPH268X7FiBTVq1MDW1payZcsSHh7OrVu3jHaTycTs2bNp2bIl9vb2lCtXjh9//NFo79q1KxEREQQEBFC2bFk6dOhAly5dWLZsWY77qVWrFp988glt27alUKFCD3QWxYoVw93d3XgVLFjQot3a2tpoq1ixIiNHjiQ1NZXffvstz3NkZY8vW7aMoKAg7O3tqVq1Klu3bjX6HD9+nObNm1O0aFEcHByoVKkSq1atIjk5maCgIACKFi2KyWSic+fOAAQGBtKnTx9CQkJwdXUlODjYmCsuLs4Y+/Lly5hMJmJiYoxr8fHxvPrqqzg7O+Pk5ET9+vU5cuQIYWFhzJ8/nxUrVhiZ7HfeJyIiIiLytFKgW0REROQJcvHiRdasWUOvXr2ws7OzaHN3d6d9+/YsWbKEgIAAEhISOHfuHACxsbG4uroaQc/09HS2bt1KYGAgAJs2baJjx4707duXhIQEvvjiCyIjIxkzZozFHOHh4bRp04Z9+/bxyiuv0L59+xwzyK9cuYKLi8ujO4C7vPbaa7i5ufHiiy9aBN3vJS0tjXnz5lGkSBHKly//wHMNGzaMgQMHEhcXh4+PD+3atTO+COjduzdpaWls3LiR/fv3M378eBwdHfHw8GDp0qUAJCUlkZKSYlHOZf78+djY2LBlyxaLLyFycurUKV566SUKFSrEzz//zO7du+natSu3bt1i4MCBtGnThiZNmhiZ7P7+/g+8VxERERGRfxo9jFJERETkCXLo0CHMZjO+vr73bPf19eXSpUu4ubnh4uJCbGwsb775JjExMQwYMMAIsu7YsYP09HQjCBoeHs6QIUPo1KkTAGXLlmXUqFEMHjyY0NBQY/zOnTvTrl07AMaOHcunn37Kjh07aNKkSba1/PLLLyxZsoSoqKhHegYAjo6OTJo0iXr16lGgQAGWLl1KixYtWL58Oa+99prRb//+/Tg6OgKZJVKcnJxYsmQJzs7ODzznwIEDadasGZB5XpUqVeLw4cNUqFCBEydO8MYbb1C5cmUg8/yyZAX63dzcKFKkiMWY5cqVY8KECcb75OTkXNfx+eefU7hwYRYvXmxksPv4+BjtdnZ2pKWlPXAZGBERERGRfzJldIuIiIg8gcxmc47thQoV4qWXXiImJobLly+TkJBAr169SEtL4+DBg8TGxlKrVi3s7e2BzAdJjhw5EkdHR+PVvXt3UlJSuHbtmjFulSpVjJ8dHBxwdnbm7Nmz2eY/cOAAr7/+OqGhoTRu3BiAEydOWIw/duzYh96/q6sr/fv354UXXqBWrVqMGzeODh068Mknn1j0K1++PHFxccTFxbF7927ee+89Wrduza5duwDo2bOnxZpycufeS5YsCWDs/YMPPmD06NHUq1eP0NBQ9u3bl6d93F1TPC/i4uKoX79+tjItIiIiIiJyfwp0i4iIiDxBvL29MZlMJCYm3rM9MTGR4sWLU6RIEQIDA4mJiWHTpk1Ur14dZ2dnI/gdGxtLQECAcV9qairh4eFGUDguLo79+/dz6NAhbG1tjX53B1dNJhMZGRkW1xISEmjQoAE9evSwqAFeqlQpi/F79uz5KI7E8MILL3D48GGLazY2Nnh7e+Pt7U316tUZN24cpUuXZurUqQCMHDnSYk05uXPvJpMJwNh7t27dOHr0KO+88w779+/Hz8+Pzz77LNc1Ozg4WLwvUCDzn993fpGRnp5u0efukjUiIiIiIpI7BbpFREREniDFihWjUaNGTJ8+nevXr1u0nT59moULFxoPO8yq0/3dd98ZtbgDAwNZv349W7ZsMa4B1KhRg6SkJCMofOcrK/iaF/Hx8QQFBdGpU6ds9b2tra0txn3Utbvj4uKMTOucWFlZGWfn5uZmsaY/w8PDg549e7Js2TIGDBjArFmzgMxgO8Dt27dzHaN48eIApKSkGNfuDsBXqVKFTZs2ZQuAZ7GxscnTXCIiIiIiTxMFukVERESeMNOmTSMtLY3g4GA2btzIyZMnWb16NY0aNcLHx4cRI0YAmQHRokWLsmjRIotA9/Lly0lLS6NevXrGmCNGjOCrr74iPDyc+Ph4EhMTWbx4sUVGdm4OHDhAUFAQjRs3pn///pw+fZrTp08bD8S8n5s3bxoZ1Tdv3uTUqVPExcVZZGdPmzaNBg0aGO/nz5/PN998w8GDBzl48CBjx45l7ty5vP/++xZj37p1y1jHoUOHGD16NAkJCbz++ut53ldehISEsGbNGo4dO8avv/7Khg0bjDrqnp6emEwmVq5cyblz50hNTb3vOHZ2dtSpU4dx48aRmJhIbGxsts+gT58+/Pe//6Vt27bs2rWLQ4cO8fXXX5OUlASAl5cX+/btIykpifPnz983IC4iIiIi8jTRwyhFRETkqbLrQM5B2dy8PfARLSQH5cqVY+fOnYSFhdGmTRvOnj2L2WymVatWfP3110bdbZPJRP369YmKiuLFF18EMoPfzs7OlC9f3qJsRnBwMCtXrmTkyJGMHz+eggULUqFCBbp165bndX3//fecO3eOBQsWsGDBAuO6p6dnjg9Z/M9//kP16tWN9xMnTmTixIkEBAQQExMDwPnz5zly5IjFfaNGjeL48eNYW1tToUIFlixZwptvvmnRJz4+3sjytre357nnnmPGjBl07Ngxz/vKi9u3b9O7d29+//13nJ2dadKkCVOmTAGgdOnSxsM+u3TpQseOHYmMjLzvWHPnzuXdd9+lZs2alC9fngkTJhh1ziEzq//nn39m0KBBBAQEYGVlRbVq1YwvLrp3705MTAx+fn6kpqayYcMGi+x9EREREZGnkcmc25OORERERPKZGzducOzYMcqUKWNRfzo/Cw0NZfLkyaxbt446deo87uWIiDx2/8S/9SIiIvLwlNEtIiIikg+Eh4fj5eXFtm3bqF279gPV1RYREREREfmnU6BbREREJJ/o0qXL416CiIiIiIjIE0mpQCIiIiIiIiIiIiKSrynQLSIiIiIiIiIiIiL5mgLdIiIiIiIiIiIiIpKvKdAtIiIiIiIiIiIiIvmaAt0iIiIiIiIiIiIikq8p0C0iIiIiIiIiIiIi+ZoC3SIiIiIiIiIiIiKSr1k/7gWIiIiI/J0mTQ/6W+cb0GvD3zqfPN3mzJnDkiVLWLt2rXEtLCyMGTNmcPbsWX744QeWL1/O5cuXWb58+eNb6N/AZDLxww8/0KJFC86fP0/FihX59ddfeeaZZx730kRERETkL6CMbhEREZEn0MmTJ+natSulSpXCxsYGT09P+vbty4ULFx7bmvbu3Uu7du3w8PDAzs4OX19fIiIicr0vPj6eN954Ay8vL0wmE1OnTs31nuTkZEwmU7bXtm3bjD5hYWEWbYULF6Z+/frExsbmOr7JZHpkgd6YmBhMJhOXL19+JOPlpnPnzrRo0SLb9Rs3bvDRRx8RGhpqXEtMTCQ8PJwvvviClJQUmjZtSkREBJGRkY90TWFhYVSrVu2Rjvkoubq60rFjR4uzEREREZF/FgW6RURERJ4wR48exc/Pj0OHDvHNN99w+PBhZs6cSXR0NHXr1uXixYuPZV27d+/Gzc2NBQsWEB8fz7Bhwxg6dCjTpk3L8b5r165RtmxZxo0bh7u7+wPNuX79elJSUoxXzZo1LdorVapktG3dupVy5crx6quvcuXKlQfe31/t5s2bf+n433//Pc7OztSrV8+4duTIEQBef/113N3dKVSoEIULF6ZIkSJ/6VqeRF26dGHhwoWP7X8/IiIiIvLXUqBbRERE5AnTu3dvbGxsWLt2LQEBATz77LM0bdqU9evXc+rUKYYNG8a0adN4/vnnjXuWL1+OyWRi5syZxrWGDRsyfPhw4/2KFSuoUaMGtra2lC1blvDwcG7dumW0m0wmZs+eTcuWLbG3t6dcuXL8+OOPRnvXrl2JiIggICCAsmXL0qFDB7p06cKyZcty3E+tWrX45JNPaNu2LYUKFXqgsyhWrBju7u7Gq2DBghbt1tbWRlvFihUZOXIkqamp/Pbbb3meIyt7fNmyZQQFBWFvb0/VqlXZunWr0ef48eM0b96cokWL4uDgQKVKlVi1ahXJyckEBWWWwylatCgmk4nOnTsDEBgYSJ8+fQgJCcHV1ZXg4GBjrri4OGPsy5cvYzKZiImJMa7Fx8fz6quv4uzsjJOTE/Xr1+fIkSOEhYUxf/58VqxYYWSyZ923ePFimjdvbowRFhZmvC9QoAAmkwnInhEeGBjIBx98wODBg3FxccHd3Z2wsDCLM7p8+TLdunWjePHiODs78/LLL7N3714AIiMjCQ8PZ+/evcaaIiMj87TXrGz46Oho/Pz8sLe3x9/fn6SkJIv5c/vdPXToEC+99BK2trZUrFiRdevWZfucK1WqRKlSpfjhhx+ytYmIiIhI/qdAt4iIiMgT5OLFi6xZs4ZevXphZ2dn0ebu7k779u1ZsmQJAQEBJCQkcO7cOQBiY2NxdXU1Aojp6els3bqVwMBAADZt2kTHjh3p27cvCQkJfPHFF0RGRjJmzBiLOcLDw2nTpg379u3jlVdeoX379jlmwF65cgUXF5dHdwB3ee2113Bzc+PFF1+0CLrfS1paGvPmzaNIkSKUL1/+gecaNmwYAwcOJC4uDh8fH9q1a2cEU3v37k1aWhobN25k//79jB8/HkdHRzw8PFi6dCkASUlJpKSkWJRzmT9/PjY2NmzZssXiS4icnDp1ipdeeolChQrx888/s3v3brp27cqtW7cYOHAgbdq0oUmTJkYmu7+/PwCbN2/Gz8/PGGfgwIHMmzcPwOh7P/Pnz8fBwYHt27czYcIERo4caREsbt26NWfPnuWnn35i9+7d1KhRgwYNGnDx4kXeeustBgwYYJFd/9Zbb+Xx1DMNGzaMSZMmsWvXLqytrenatavRltvvbkZGBq1atcLGxobt27czc+ZM/v3vf99zntq1a7Np06YHWpuIiIiI5A96GKWIiIjIE+TQoUOYzWZ8fX3v2e7r68ulS5dwc3PDxcWF2NhY3nzzTWJiYhgwYIARZN2xYwfp6elGEDQ8PJwhQ4bQqVMnAMqWLcuoUaMYPHiwRd3izp07065dOwDGjh3Lp59+yo4dO2jSpEm2tfzyyy8sWbKEqKioR3oGAI6OjkyaNIl69epRoEABli5dSosWLVi+fDmvvfaa0W///v04OjoCmSVSnJycWLJkCc7Ozg8858CBA2nWrBmQeV6VKlXi8OHDVKhQgRMnTvDGG29QuXJlIPP8smQF+t3c3LKVBClXrhwTJkww3icnJ+e6js8//5zChQuzePFiI4Pdx8fHaLezsyMtLc2iDMzly5e5cuUKpUqVMq45Ojoa68mtZEyVKlWM34Ny5coxbdo0oqOjadSoEZs3b2bHjh2cPXvWyMifOHEiy5cv5/vvv6dHjx44Ojoa2fUPY8yYMQQEBAAwZMgQmjVrxo0bN7C1tc31d3f9+vUcPHiQNWvWGPsfO3YsTZs2zTZPqVKl2LNnz0OtUURERESebMroFhEREXkCmc3mHNsLFSrESy+9RExMDJcvXyYhIYFevXqRlpbGwYMHiY2NpVatWtjb2wOZD5IcOXIkjo6Oxqt79+6kpKRw7do1Y9wqVaoYPzs4OODs7MzZs2ezzX/gwAFef/11QkNDady4MQAnTpywGH/s2LEPvX9XV1f69+/PCy+8QK1atRg3bhwdOnTgk08+sehXvnx54uLiiIuLY/fu3bz33nu0bt2aXbt2AdCzZ0+LNeXkzr2XLFkSwNj7Bx98wOjRo6lXrx6hoaHs27cvT/u4u6Z4XsTFxVG/fv1sZVpycv36dQBsbW0feD6w3Dtk7j9r73v37iU1NZVixYpZnOWxY8eMGuB/Vk5nn9vvbmJiIh4eHhZB/rp1695zHjs7O4vfdxERERH551BGt4iIiMgTxNvbG5PJRGJiIi1btszWnpiYSPHixSlSpAiBgYF8+eWXbNq0ierVq+Ps7GwEv2NjY40MWYDU1FTCw8Np1apVtjHvDI7eHVw1mUxkZGRYXEtISKBBgwb06NHDogZ4qVKlLOoxP+qSJi+88EK22ss2NjZ4e3sb76tXr87y5cuZOnUqCxYsYOTIkQwcODBP49+596x61ll779atG8HBwURFRbF27Vo+/vhjJk2axPvvv5/jmA4ODhbvCxTIzDO584uM9PR0iz53l6zJi2LFimEymbh06dID3ws5f+6pqamULFnSooZ4lpweapmXvd5r/rvPPq+/u3lx8eJFihcv/kD3iIiIiEj+oIxuERERkSdIsWLFaNSoEdOnTzeydLOcPn2ahQsXGg87zKrT/d133xm1uAMDA1m/fj1btmwxrgHUqFGDpKQkvL29s72yApJ5ER8fT1BQEJ06dcpW39va2tpi3Ecd6I6LizOyfXNiZWVlnJ2bm5vFmv4MDw8PevbsybJlyxgwYACzZs0CMoPtALdv3851jKwg6531su/8cgAys5s3bdp036CwjY1NtrlsbGyoWLEiCQkJed5PXtWoUYPTp09n+3y9vb1xdXW975ryste8zp/T766vry8nT560mGfbtm33HOvAgQNUr179gdcgIiIiIk8+ZXSLiIjIU2XuzP1/63wDej34PdOmTcPf35/g4GBGjx5NmTJliI+PZ9CgQfj4+DBixAggMyBatGhRFi1axMqVK4HMQPfAgQMxmUzUq1fPGHPEiBG8+uqrPPvss7z55psUKFCAvXv3cuDAAUaPHp2ndR04cICXX36Z4OBg+vfvz+nTp4HMwHJOWbI3b940ArA3b97k1KlTxMXF4ejoaASfp02bxg8//EB0dDTwv4c4ZgUlly1bxty5c5k9e7bF2Ldu3TLW8ccff7BkyRISEhLu+zDChxUSEkLTpk3x8fHh0qVLbNiwwaij7unpiclkYuXKlbzyyivY2dndt0yKnZ0dderUYdy4cZQpU4azZ89aZMUD9OnTh88++4y2bdsydOhQChcuzLZt26hduzbly5fHy8uLNWvWkJSURLFixShcuDAFCxYkODiYzZs3ExIS8kj33rBhQ+rWrUuLFi2YMGECPj4+/Oc//yEqKoqWLVvi5+eHl5cXx44dIy4ujmeeeQYnJ6c87TUvcvvdbdiwIT4+PnTq1IlPPvmE//73vwwbNizbONeuXWP37t1/qqSOiIiIiDy5FOgWERGRp0r8vvOPewm5KleuHDt37iQsLIw2bdpw9uxZzGYzrVq14uuvvzbqbptMJurXr09UVBQvvvgikBn8dnZ2pnz58hZlM4KDg1m5ciUjR45k/PjxFCxYkAoVKtCtW7c8r+v777/n3LlzLFiwgAULFhjXPT09c3zI4n/+8x+LLNqJEycyceJEAgICjHIY58+fz1bvedSoURw/fhxra2sqVKjAkiVLePPNNy36xMfHG1ne9vb2PPfcc8yYMYOOHTvmeV95cfv2bXr37s3vv/+Os7MzTZo0YcqUKQCULl3aeGBily5d6NixI5GRkfcda+7cubz77rvUrFmT8uXLM2HCBKPOOWRm9f/8888MGjSIgIAArKysqFatmvHFRffu3YmJicHPz4/U1FQ2bNhAYGAg7777Ln5+fly5coXChQs/sr2bTCZWrVrFsGHD6NKlC+fOncPd3Z2XXnqJEiVKAPDGG2+wbNkygoKCuHz5MvPmzaNz58657jUvcvvdLVCgAD/88APvvvsutWvXxsvLi08//TTbA1RXrFjBs88+S/369R/NwYiIiIjIE8Vkzu1JRyIiIiL5zI0bNzh27BhlypR56IfzPWlCQ0OZPHky69ato06dOo97OfKEat26NTVq1GDo0KGPeylPnDp16vDBBx/w9ttvP+6lyCPyT/xbLyIiIg9PNbpFRERE8oHw8HA+/fRTtm3blu3hkCJZPvnkk/uWTXmanT9/nlatWtGuXbvHvRQRERER+Ysoo1tERET+cZTlJyLyz6e/9SIiInInZXSLiIiIiIiIiIiISL6mQLeIiIiIiIiIiIiI5GsKdIuIiIiIiIiIiIhIvqZAt4iIiIiIiIiIiIjkawp0i4iIiIiIiIiIiEi+pkC3iIiIiIiIiIiIiORrCnSLiIiIiIiIiIiISL5m/bgXICIiIvJ3Kl3dlZf6VXmoe2+nZ/Bdt1jq9PDFq557nu75puPPDzWXiIiIiIiI5J0yukVERESeQCdPnqRr166UKlUKGxsbPD096du3LxcuXHhsa9q7dy/t2rXDw8MDOzs7fH19iYiIyPW+wMBATCZTtlezZs3u26dEiRK0bt2a48eP5zh2TEwMJpOJy5cv/9ntARAWFka1atUeyVh54eXlxdSpU/+2+URERERE/qkU6BYRERF5whw9ehQ/Pz8OHTrEN998w+HDh5k5cybR0dHUrVuXixcvPpZ17d69Gzc3NxYsWEB8fDzDhg1j6NChTJs2Lcf7li1bRkpKivE6cOAAVlZWtG7d2qJf9+7dSUlJ4T//+Q8rVqzg5MmTdOjQ4a/c0kNLT09/3EsQEREREZE7KNAtIiIi8oTp3bs3NjY2rF27loCAAJ599lmaNm3K+vXrOXXqFMOGDWPatGk8//zzxj3Lly/HZDIxc+ZM41rDhg0ZPny48X7FihXUqFEDW1tbypYtS3h4OLdu3TLaTSYTs2fPpmXLltjb21OuXDl+/PFHo71r165EREQQEBBA2bJl6dChA126dGHZsmU57sfFxQV3d3fjtW7dOuzt7bMFuu3t7XF3d6dkyZLUqVOHPn368Ouvvz7Q2UVGRlKkSBHWrFmDr68vjo6ONGnShJSUFKNPTEwMtWvXxsHBgSJFilCvXj2OHz9OZGQk4eHh7N2718gsj4yMNM5mxowZvPbaazg4ODBmzBhjrjtlfQ53+r//+z9q1aqFra0trq6utGzZEsjMYj9+/Dj9+vUz5hMRERERkYejQLeIiIg8VTJuZXA7/eFeGbcyALj9AGM8qIsXL7JmzRp69eqFnZ2dRZu7uzvt27dnyZIlBAQEkJCQwLlz5wCIjY3F1dWVmJgYIDPjeOvWrQQGBgKwadMmOnbsSN++fUlISOCLL74gMjKSMWPGWMwRHh5OmzZt2LdvH6+88grt27fPMYP8ypUruLi4PNAe58yZQ9u2bXFwcMjxHL799lteeOGFBxob4Nq1a0ycOJGvv/6ajRs3cuLECQYOHAjArVu3aNGiBQEBAezbt4+tW7fSo0cPTCYTb731FgMGDKBSpUpG9vlbb71ljBsWFkbLli3Zv38/Xbt2zdNaoqKiaNmyJa+88gp79uwhOjqa2rVrA5mZ7s888wwjR4405hMRERERkYejh1GKiIjIU+X0gUt81y32T42xc24SO+cm5a3zuw829qFDhzCbzfGEGwMAAH3rSURBVPj6+t6z3dfXl0uXLuHm5oaLiwuxsbG8+eabxMTEMGDAAKNm9o4dO0hPT8ff3x/IDGAPGTKETp06AVC2bFlGjRrF4MGDCQ0NNcbv3Lkz7dq1A2Ds2LF8+umn7NixgyZNmmRbyy+//MKSJUuIiorK8/527NjBgQMHmDNnTra26dOnM3v2bMxmM9euXcPHx4c1a9bkeews6enpzJw5k+eeew6APn36MHLkSAD++9//cuXKFV599VWj/c6zdnR0xNraGnf37A8bffvtt+nSpcsDrWXMmDG0bduW8PBw41rVqlWBzEx3KysrnJyc7jmfiIiIiIjknQLdIiIi8tSp0+PeQeTc3L6Vwc65SbhXdsGrbolHvCpLZrM5x/ZChQrx0ksvERMTQ8OGDUlISKBXr15MmDCBgwcPEhsbS61atbC3twcyHyS5ZcsWiwzu27dvc+PGDa5du2b0q1KlitHu4OCAs7MzZ8+ezTb/gQMHeP311wkNDaVx48YAnDhxgooVKxp9PvzwQz788EOL++bMmUPlypWNrOY7tW/fnmHDhgFw5swZxo4dS+PGjdm9ezdOTk5UqlTJeDhl/fr1+emnn+55Nvb29kYQG6BkyZLGHlxcXOjcuTPBwcE0atSIhg0b0qZNG0qWLHm/ozb4+fnl2uducXFxdO/e/YHvExERERGRB6NAt4iIiDxVSlUrhle9h8uevZ2eGej2qlviocfIjbe3NyaTicTERKOW850SExMpXrw4RYoUITAwkC+//JJNmzZRvXp1nJ2djeB3bGwsAQEBxn2pqamEh4fTqlWrbGPa2toaPxcsWNCizWQykZFhWYIlISGBBg0a0KNHD4sa4KVKlSIuLs54f3dJk6tXr7J48WIju/puhQsXxtvb2ziHOXPmULJkSZYsWUK3bt1YtWqV8RDIu8u63Olee7jzi4N58+bxwQcfsHr1apYsWcLw4cNZt24dderUue+YQLZSKwUKFMj2hcTdD6nMaZ0iIiIiIvLoqEa3iIiIyBOkWLFiNGrUiOnTp3P9+nWLttOnT7Nw4UI6d+4MYNTp/u6774xa3IGBgaxfv54tW7YY1wBq1KhBUlIS3t7e2V4FCuT9n4Tx8fEEBQXRqVOnbPW9ra2tLca9O9D93XffkZaWRocOHfI0l5WVFYBxDp6ensbYpUuXzvOa76V69eoMHTqUX375heeff55FixYBYGNjw+3bt/M0RvHixfnjjz+4evWqce3OQD9kZshHR0ffd4wHmU9ERERERO5PgW4RERGRJ8y0adNIS0sjODiYjRs3cvLkSVavXk2jRo3w8fFhxIgRQGYQtWjRoixatMgi0L18+XLS0tKoV6+eMeaIESP46quvCA8PJz4+nsTERBYvXmyRkZ2bAwcOEBQUROPGjenfvz+nT5/m9OnTxgMxczNnzhxatGhBsWLF7tl+7do1Y8y9e/fy3nvvYWtra5RGeRSOHTvG0KFD2bp1K8ePH2ft2rUcOnTIqNPt5eXFsWPHiIuL4/z586Slpd13rBdeeAF7e3s+/PBDjhw5wqJFi4iMjLToExoayjfffENoaCiJiYns37+f8ePHG+1eXl5s3LiRU6dOcf78+Ue2TxERERGRp41Kl4iIiMhT5dSePxlMfMCHSz6McuXKsXPnTsLCwmjTpg1nz57FbDbTqlUrvv76a6Oetslkon79+kRFRfHiiy8CmcFvZ2dnypcvb1FqIzg4mJUrVzJy5EjGjx9PwYIFqVChAt26dcvzur7//nvOnTvHggULWLBggXHd09OT5OTkHO9NSkpi8+bNrF279r59Zs2axaxZswAoWrQoVapUYdWqVZQvXz7Pa8yNvb09Bw8eZP78+Vy4cIGSJUvSu3dv/vWvfwHwxhtvsGzZMoKCgrh8+TLz5s0zMujv5uLiwoIFCxg0aBCzZs2iQYMGhIWF0aNHD6NPYGAg3333HaNGjWLcuHFGeZksI0eO5F//+hfPPfccaWlpudZmFxERERGRezOZ9a9pERER+Ye5ceMGx44do0yZMhb1p/Oz0NBQJk+enKda0iIiT4N/4t96EREReXjK6BYRERHJB8LDw/Hy8mLbtm3Url37gepqi4iIiIiI/NMp0C0iIiKST3Tp0uVxL0FEREREROSJpFQgEREREREREREREcnXFOgWERERERERERERkXxNgW4RERERERERERERydcU6BYRERERERERERGRfE2BbhERERERERERERHJ1xToFhEREREREREREZF8TYFuEREREREREREREcnXrB/3AkRERET+Ttb2tjiWdqPagPZY2xXK831Xjp5izydfP/C96zuFPeRK5Wn3zjvv4Ovry4cffgjAtWvXeOedd1i3bh1//PEHly5dolq1aoSEhBASEvJ4F/sXiomJISgoiEuXLlGkSBFWr17NkCFD+PXXXylQQHk7IiIiIpJJ/zIUERGRp8rDBLkBCpctTfVB75B66ixxkxZy63raX7TCTCdPnqRr166UKlUKGxsbPD096du3LxcuXPhL583J3r17adeuHR4eHtjZ2eHr60tERESu9wUGBmIymbK9mjVrdt8+JUqUoHXr1hw/fjzHsWNiYjCZTFy+fPnPbg+AsLAwqlWr9kjGygsvLy+mTp2a7frevXtZtWoVH3zwgXFt/vz5bNq0iV9++YWUlBQKFy7Mzp076dGjxyNdU2Bg4BMdOG/SpAkFCxZk4cKFj3spIiIiIvIEUaBbREREnioPE+TO8ncFu48ePYqfnx+HDh3im2++4fDhw8ycOZPo6Gjq1q3LxYsX/5J5c7N7927c3NxYsGAB8fHxDBs2jKFDhzJt2rQc71u2bBkpKSnG68CBA1hZWdG6dWuLft27dyclJYX//Oc/rFixgpMnT9KhQ4e/cksPLT09/S8d/7PPPqN169Y4Ojoa144cOYKvry/PP/887u7umEwmihcvjr29/V+6lidR586d+fTTTx/3MkRERETkCaJAt4iIiDxVHjbIneXvCHb37t0bGxsb1q5dS0BAAM8++yxNmzZl/fr1nDp1imHDhjFt2jSef/55457ly5djMpmYOXOmca1hw4YMHz7ceL9ixQpq1KiBra0tZcuWJTw8nFu3bhntJpOJ2bNn07JlS+zt7SlXrhw//vij0d61a1ciIiIICAigbNmydOjQgS5durBs2bIc9+Pi4oK7u7vxWrduHfb29tkC3fb29ri7u1OyZEnq1KlDnz59+PXXXx/o7CIjIylSpAhr1qzB19cXR0dHmjRpQkpKitEnJiaG2rVr4+DgQJEiRahXrx7Hjx8nMjKS8PBw9u7da2SWR0ZGGmczY8YMXnvtNRwcHBgzZowx152yPoc7/d///R+1atXC1tYWV1dXWrZsCWRmTh8/fpx+/foZ8wHcvn2b77//nubNmxtjBAYGMmnSJDZu3IjJZCIwMBDInhGe22cIcODAAZo2bYqjoyMlSpTgnXfe4fz580BmADk2NpaIiAhjTcnJyXnaa1Y2/Ndff42XlxeFCxembdu2/PHHH0afjIwMPv74Y8qUKYOdnR1Vq1bl+++/txh31apV+Pj4YGdnR1BQEMnJydk+5+bNm7Nr1y6OHDmSrU1EREREnk4KdIuIiIg8oL8y2H3x4kXWrFlDr169sLOzs2hzd3enffv2LFmyhICAABISEjh37hwAsbGxuLq6EhMTA2RmHG/dutUIiG7atImOHTvSt29fEhIS+OKLL4iMjGTMmDEWc4SHh9OmTRv27dvHK6+8Qvv27XPMIL9y5QouLi4PtMc5c+bQtm1bHBwccjyHb7/9lhdeeOGBxobMWtYTJ07k66+/ZuPGjZw4cYKBAwcCcOvWLVq0aEFAQAD79u1j69at9OjRA5PJxFtvvcWAAQOoVKmSkX3+1ltvGeOGhYXRsmVL9u/fT9euXfO0lqioKFq2bMkrr7zCnj17iI6Opnbt2kBmpvszzzzDyJEjjfkA9u3bx5UrV/Dz8zPGWbZsGd27d6du3bqkpKTk+OVCTp/h5cuXefnll6levTq7du1i9erVnDlzhjZt2gAQERFB3bp1jez6lJQUPDw88nz2R44cYfny5axcuZKVK1cSGxvLuHHjjPaPP/6Yr776ipkzZxIfH0+/fv3o0KEDsbGxQGbJnlatWtG8eXPi4uLo1q0bQ4YMyTbPs88+S4kSJdi0aVOe1yYiIiIi/2x6GKWIiIjIQ8gKdu/55GviJi28Z0mUhwmAHzp0CLPZjK+v7z3bfX19uXTpEm5ubri4uBAbG8ubb75JTEwMAwYMMGpm79ixg/T0dPz9/YHM4OeQIUPo1KkTAGXLlmXUqFEMHjyY0NBQY/zOnTvTrl07AMaOHcunn37Kjh07aNKkSba1/PLLLyxZsoSoqKg872/Hjh0cOHCAOXPmZGubPn06s2fPxmw2c+3aNXx8fFizZk2ex86Snp7OzJkzee655wDo06cPI0eOBOC///0vV65c4dVXXzXa7zxrR0dHrK2tcXd3zzbu22+/TZcuXR5oLWPGjKFt27aEh4cb16pWrQpkZrpbWVnh5ORkMd/x48exsrLCzc3NuObi4oK9vT02Njb3XNudcvoMp02bRvXq1Rk7dqzRf+7cuXh4ePDbb7/h4+ODjY2NkV3/oDIyMoiMjMTJyQnIfKBmdHQ0Y8aMIS0tjbFjx7J+/Xrq1q0LZP4ebt68mS+++IKAgABmzJjBc889x6RJkwAoX748+/fvZ/z48dnmKlWqVK413EVERETk6aGMbhEREZGHlFNm963racRNeviH5ZnN5hzbCxUqxEsvvURMTAyXL18mISGBXr16kZaWxsGDB4mNjaVWrVpG/ea9e/cycuRIHB0djVdW1u61a9eMcatUqWL87ODggLOzM2fPns02/4EDB3j99dcJDQ2lcePGAJw4ccJi/DuDqVnmzJlD5cqVjazmO7Vv3564uDj27t3L5s2b8fb2pnHjxkbpi0qVKhljN23a9L5nY29vbwSxAUqWLGnswcXFhc6dOxMcHEzz5s2JiIiwKGuSkzszrPMqLi6OBg0aPNA9169fp1ChQtlKoORVTp/h3r172bBhg8XnVKFCBYBHUgbEy8vLCHKD5dkfPnyYa9eu0ahRI4v5v/rqK2PuxMTEbFn8WUHxu9nZ2Vn87oqIiIjI000Z3SIiIiJ/wr0yuwHiJi0k9VT2AHFuvL29MZlMJCYmGrWc75SYmEjx4sUpUqQIgYGBfPnll2zatInq1avj7OxsBL9jY2MJCAgw7ktNTSU8PJxWrVplG9PW1tb4uWDBghZtJpOJjIwMi2sJCQk0aNCAHj16WNQAL1WqFHFxccb7u0uaXL16lcWLFxvZ1XcrXLgw3t7exjnMmTOHkiVLsmTJErp168aqVauMh0DeXdblTvfaw51fHMybN48PPviA1atXs2TJEoYPH866deuoU6fOfccEspVaKVCgQLYvJO5+SGVO67wfV1dXrl27xs2bN7GxsXng+3P6DFNTU2nevPk9M6RLlix53zHzste8zA2Z5VxKly5t0a9QoQevnX/x4kWKFy/+wPeJiIiIyD+TAt0iIiIif9Kdwe7dY+cBcP38ZaoPeueBxypWrBiNGjVi+vTp9OvXzyJQevr0aRYuXEjv3r0BCAgIICQkhO+++86oxR0YGMj69evZsmULAwYMMO6tUaMGSUlJRiD5YcXHx/Pyyy/TqVOnbPW9ra2tcxz/u+++Iy0tjQ4dOuRpLisrKyAzwxnA09PzIVedXfXq1alevTpDhw6lbt26LFq0iDp16mBjY8Pt27fzNEbx4sX5448/uHr1qhEEvzPQD5nZ1dHR0fcteXKv+apVqwZkfqGQ9fOjUqNGDZYuXYqXlxfW1vf+vwL3WlNe9pqbihUrUqhQIU6cOGHxJcydfH19sz08c9u2bdn63bhxgyNHjlC9evUHWoOIiIiI/HOpdImIiIjII1C4bGmqvN+G1JNnSD15hirvt6Fw2dK533gP06ZNIy0tjeDgYDZu3MjJkydZvXo1jRo1wsfHhxEjRgCZQdSiRYuyaNEii0D38uXLSUtLo169esaYI0aM4KuvviI8PJz4+HgSExNZvHixRUZ2bg4cOEBQUBCNGzemf//+nD59mtOnTxsPxMzNnDlzaNGiBcWKFbtn+7Vr14wx9+7dy3vvvYetra1RGuVROHbsGEOHDmXr1q0cP36ctWvXcujQIaNOt5eXF8eOHSMuLo7z58+Tlnb/OusvvPAC9vb2fPjhhxw5coRFixYRGRlp0Sc0NJRvvvmG0NBQEhMTs9Wb9vLyYuPGjZw6dYrz588DmUHlGjVqsHnz5ke27yy9e/fm4sWLtGvXjp07d3LkyBHWrFlDly5djOC2l5cX27dvJzk5mfPnz5ORkZGnvebGycmJgQMH0q9fP+bPn8+RI0f49ddf+eyzz5g/fz4APXv25NChQwwaNIikpKT7zrNt2zYKFSp037ImIiIiIvIUMouIiIj8w1y/ft2ckJBgvn79+uNeykM7duyYuVOnTuYSJUqYTSaTGTC3atXKfPXqVYt+r7/+utna2tr8xx9/mM1ms/n27dvmokWLmuvUqZNtzNWrV5v9/f3NdnZ2ZmdnZ3Pt2rXNX375pdEOmH/44QeLewoXLmyeN2+e2Ww2m0NDQ81Atpenp2eu+zl48KAZMK9du/ae7QEBARZjFi1a1BwQEGD++eefcxx3w4YNZsB86dIls9lsNs+bN89cuHBhiz4//PCDOeufvadPnza3aNHCXLJkSbONjY3Z09PTPGLECPPt27fNZrPZfOPGDfMbb7xhLlKkiBkw9n6vs8ka29vb22xnZ2d+9dVXzV9++aX57n9iL1261FytWjWzjY2N2dXV1dyqVSujbevWreYqVaqYCxUqZHHf9OnTs32Gffv2NQcEBFhc8/T0NE+ZMsV4n9tnaDabzb/99pu5ZcuW5iJFipjt7OzMFSpUMIeEhJgzMjLMZrPZnJSUZK5Tp47Zzs7ODJiPHTuWp72Ghoaaq1atajH3lClTLH4/MjIyzFOnTjWXL1/eXLBgQXPx4sXNwcHB5tjYWKPP//3f/5m9vb3NhQoVMtevX988d+5ci8/YbDabe/ToYf7Xv/5llqfbP+FvvYiIiDw6JrM5lycdiYiIiOQzN27c4NixY5QpU8ai/nR+FhoayuTJk/NUS1ryv+vXr1O+fHmWLFmirOW7nD9/nvLly7Nr1y7KlCnzuJcjj9E/8W+9iIiIPDzV6BYRERHJB8LDw/Hy8mLbtm3Url2bAgVUge6fzM7Ojq+++sooZyL/k5yczPTp0xXkFhERERELyugWERGRfxxl+YmI/PPpb72IiIjcSalAIiIiIiIiIiIiIpKvKdAtIiIiIiIiIiIiIvmaAt0iIiIiIiIiIiIikq8p0C0iIiIiIiIiIiIi+ZoC3SIiIiIiIiIiIiKSrynQLSIiIiIiIiIiIiL5mgLdIiIiIvlYTEwMJpOJy5cv/+1zR0ZGUqRIkT89jpeXF1OnTn2ge5KTkzGZTMTFxf3p+UVEREREJP+zftwLEBEREfk7NZ4382+db22Xnn/p+P7+/qSkpFC4cOFc+8bExBAUFMSlS5ceSYA6LwIDA4mNjc12/ZVXXiEqKuqhx/Xw8CAlJQVXV9c/szxDZGQkISEhf9sXBoGBgVSrVu2BA/z3EhMTQ//+/YmPj8fDw4Phw4fTuXPn+/ZPSkqiZ8+eJCQkcOXKFUqVKsXbb79NaGgoBQsW/NPrERERERF5HBToFhERkafK+m698X6zBc82CKTAAwT1bly6TMLcr7gQn4idW3EqdmqPi2/5v3CleWNjY4O7u/sjHfPmzZvY2Ng8krGWLVvGzZs3jfcXLlygatWqtG7d+k+Na2Vl9cj3nReP8mwehWPHjtGsWTN69uzJwoULiY6Oplu3bpQsWZLg4OB73lOwYEE6duxIjRo1KFKkCHv37qV79+5kZGQwduzYv3kHIiIiIiKPhkqXiIiIyFPFnJHBoW+XEf2vD7h1/Xqe7rmYmMSmAUO5EJ8IwPWz59j9yVT2zZz9yNcXGBjI+++/T0hICEWLFqVEiRLMmjWLq1ev0qVLF5ycnPD29uann34CspcuOX78OM2bN6do0aI4ODhQqVIlVq1aRXJyMkFBQQAULVoUk8lkZP0GBgbSp08fQkJCcHV1NQKkkydPpnLlyjg4OODh4UGvXr1ITU19oP24uLjg7u5uvNatW4e9vX22QPcff/xBu3btcHBwoHTp0nz++ec5jnt36ZKsc4iOjsbPzw97e3v8/f1JSkoy7tm7dy9BQUE4OTnh7OxMzZo12bVrFzExMXTp0oUrV65gMpkwmUyEhYUBmWVVRo0aRceOHXF2dqZHjx73LBcTFxeHyWQiOTnZuLZlyxYCAwOxt7enaNGiBAcHc+nSJTp37kxsbCwRERHGfHfel+XLL7+kVKlSZGRkWFx//fXX6dq1KwAzZ86kTJkyTJo0CV9fX/r06cObb77JlClT7nt2ZcuWpUuXLlStWhVPT09ee+012rdvz6ZNm3I8cxERERGRJ5kC3SIiIvJUeWnKOJw8PQD4dcq0XPubMzLY/clUAMq/3YaXv/iUOmHDADizYzeXkg498jXOnz8fV1dXduzYwfvvv897771H69at8ff359dff6Vx48a88847XLt2Ldu9vXv3Ji0tjY0bN7J//37Gjx+Po6MjHh4eLF26FMgsXZGSkkJERITFnDY2NmzZsoWZMzPLuxQoUIBPP/2U+Ph45s+fz88//8zgwYP/1N7mzJlD27ZtcXBwsLj+ySefULVqVfbs2cOQIUPo27cv69ate+Dxhw0bxqRJk9i1axfW1tZGQBigffv2PPPMM+zcuZPdu3czZMgQChYsiL+/P1OnTsXZ2ZmUlBRSUlIYOHCgcd/EiRONtX300Ud5WkdcXBwNGjSgYsWKbN26lc2bN9O8eXNu375NREQEdevWpXv37sZ8Hh4e2cZo3bo1Fy5cYMOGDca1ixcvsnr1atq3bw/A1q1badiwocV9wcHBbN26Nc9ndvjwYVavXk1AQECe7xERERERedKodImIiIg8VQoVLswLHw1hfbfeXDl8FLPZjMlkum//i4mZGcHFnq/Isw0zM6Kdnn2GOmEfsi1sLHs//5LATz95pGusWrUqw4cPB2Do0KGMGzcOV1dXunfvDsCIESOYMWMG+/bty3bviRMneOONN6hcuTKQmb2bxcXFBQA3N7dsNbrLlSvHhAkTLK6FhIQYP3t5eTF69Gh69uzJ9OnTH2pfO3bs4MCBA8yZMydbW7169RgyZAgAPj4+bNmyhSlTptCoUaMHmmPMmDFGwHbIkCE0a9aMGzduYGtry4kTJxg0aBAVKlQAMvecpXDhwphMpnuWQ3n55ZcZMGCA8f7kyZO5rmPChAn4+flZnFWlSpWMn21sbLC3t8+x/ErRokVp2rQpixYtokGDBgB8//33uLq6Gtn5p0+fpkSJEhb3lShRgv/+979cv34dOzu7+46f9cVJWloaPXr0YOTIkbnuS0RERETkSaWMbhEREXnqmAr8759A5rvKQtwt7dJlAIpXrWxx3c6tOADpD1jKIy+qVKli/GxlZUWxYsWMwDVgBDbPnj2b7d4PPviA0aNHU69ePUJDQ+8ZDL+XmjVrZru2fv16GjRoQOnSpXFycuKdd97hwoUL98wkP3HiBI6OjsbrXrWe58yZQ+XKlaldu3a2trp162Z7n5iYWSqmZ8+eFmPn5M6zK1myJPC/c+rfvz/dunWjYcOGjBs3jiNHjuQ4VhY/P7889btTVkb3g6hUqZKxx6ZNmwKZWehLly4lLS0NgIULF9K2bVsKFPjz/4xfsmQJv/76K4sWLSIqKoqJEyf+6TFFRERERB4XBbpFRETkqXPzv/81fi5gZZVjXyevZwE4uHAJZrPZuJ6ydTsAbn41Hvn6Ct71kEyTyWRxLSsD/e7azQDdunXj6NGjvPPOO+zfvx8/Pz8+++yzXOe8u5RIcnIyr776KlWqVGHp0qXs3r3bqJt958Mls5QqVYq4uDjj1bNnT4v2q1evsnjxYt59991c13K3kSNHWoydk5zOKSwsjPj4eJo1a8bPP/9MxYoV+eGHH3Kd/+6zyQoy3/n7kJ6ebtEnp0zq+1m1apWxx9mzM+u/N2/eHLPZTFRUFCdPnmTTpk1G2RIAd3d3zpw5YzHOmTNncHZ2znUNHh4eVKxYkXbt2jFu3DjCwsK4ffv2A69bRERERORJoNIlIiIi8lQ5vGwFx1auBqBM81dy7e/0TGnj5/Xv9sKn7ZtcOJDAhQMJAJRv1/p+tz42Hh4e9OzZk549ezJ06FBmzZrF+++/j42NDUCegpm7d+8mIyODSZMmGYHdb7/99r79ra2t8fb2vm/7d999R1paGh06dLhn+7Zt27K99/X1BTJLrbi5ueW65rzw8fHBx8eHfv360a5dO+bNm0fLli2xsbHJc5C3ePHMbP6UlBSKFi0KkC0AX6VKFaKjowkPD7/nGPeaz9PTM1s/W1tbWrVqxcKFCzl8+DDly5enRo3/fblSt25dVq1aZXHPunXrsmXI5yYjI4P09HQyMjKwyuXLHxERERGRJ5EyukVEROSpkhXkLvWiP8+1eDVP9wR+9r+SDr8t/t4IctccFIJt0SKPfI1/RkhICGvWrOHYsWP8+uuvbNiwwQgYe3p6YjKZWLlyJefOnSM1h7Ir3t7epKen89lnn3H06FG+/vpr4yGVD2POnDm0aNGCYsWK3bN9y5YtTJgwgd9++43PP/+c7777jr59+z70fHe7fv06ffr0ISYmhuPHj7NlyxZ27txpnI2XlxepqalER0dz/vz5e5ZnyeLt7Y2HhwdhYWEcOnSIqKgoJk2aZNFn6NCh7Ny5k169erFv3z4OHjzIjBkzOH/+vDHf9u3bSU5O5vz58/fMzs/Svn17oqKimDt3rkU2N2SWdTl69CiDBw/m4MGDTJ8+nW+//ZZ+/foZfaZNm2ZRRmXhwoV8++23JCYmcvToUb799luGDh3KW2+9le2/JhARERERyS8U6BYREZGnyvM9uhD42UQqdX0nx4dQ3qmggwON5s6g7qgRVHq3I9X79aHh7M9x8S3/F6/2wd2+fZvevXvj6+tLkyZN8PHxMR6IWLp0acLDwxkyZAglSpSgT58+9x2natWqTJ48mfHjx/P888+zcOFCPv7444daU1JSEps3b86xbMmAAQPYtWsX1atXZ/To0UyePJng4OCHmu9erKysuHDhAh07dsTHx4c2bdrQtGlTI+Pa39+fnj178tZbb1G8ePFsD+a8U8GCBfnmm284ePAgVapUYfz48YwePdqij4+PD2vXrmXv3r3Url2bunXrsmLFCqytM/+DyoEDB2JlZUXFihUpXrw4J06cuO98L7/8Mi4uLiQlJfH2229btJUpU4aoqCjWrVtH1apVmTRpErNnz7Y4u/Pnz1vUI7e2tmb8+PHUrl2bKlWqEB4eTp8+fYxyKSIiIiIi+ZHJfGdxQREREZF/gBs3bnDs2DHKlCmDra3t416OiIj8BfS3XkRERO6kjG4RERERERERERERydcU6BYRERERERERERGRfE2BbhERERERERERERHJ1xToFhEREREREREREZF8TYFuEREREREREREREcnXFOgWERERERERERERkXxNgW4RERERERERERERydcU6BYRERERERERERGRfE2BbhERERERERERERHJ1xToFhEREREREREREZF8zfpxL0BERETk79R03g+knjrJ5o/6AtBk7rIc+1+/cI7YQf/K1vfQ8sUc+fFbyjRtSfnW79z3/p+6tHwEq76/mJgYgoKCuHTpEkWKFPlL57pbZGQkISEhXL58+U+N4+XlRUhICCEhIXm+Jzk5mTJlyrBnzx6qVav2p+aXe7t58yYVK1bkq6++wt/fH4CDBw/SuXNn4uLiqFChAsuXL38qPoewsDCWL19OXFwcAEOGDOHq1at89tlnj3dhIiIiImJQRreIiIhIDm7+90qO7TZOTn/TSu7N39+flJQUChcunGvfmJgYTCbTnw5MP4jAwEBMJlO2V7Nmzf7UuB4eHqSkpPD8888/knVGRkb+rV8UBAYGPlBgPycxMTHUqFGDQoUK4e3tTWRkZI79k5KSCAoKokSJEtja2lK2bFmGDx9Oenq6Rb+ZM2dSpkwZI8gNEBoaioODA0lJSURHRz/yzyGLyWRi+fLlj3TMR2ngwIHMnz+fo0ePPu6liIiIiMj/p0C3iIiIPHVsXYoZP2fcFdy72/ULZwEo/JyPxXUH91IAnN8f92gX94BsbGxwd3fHZDI9sjFv3rz5yMZatmwZKSkpxuvAgQNYWVnRunXrPzWulZUV7u7uWFv/vf+B4qM8m0fh2LFjNGvWjKCgIOLi4ggJCaFbt26sWbPmvvcULFiQjh07snbtWpKSkpg6dSqzZs0iNDTU6GM2m5k2bRrvvvuuxb1HjhzhxRdfxNPTk2LFij22z+Fxc3V1JTg4mBkzZjzupYiIiIjI/6dAt4iIiDx1rO3sjcB14uK59+2XkZ7OvlmfAuDdvI1FW4madQC4kLiPC4n7H9naAgMDef/99wkJCaFo0aKUKFGCWbNmcfXqVbp06YKTkxPe3t789NNPQPYs7ePHj9O8eXOKFi2Kg4MDlSpVYtWqVSQnJxMUFARA0aJFMZlMdO7c2ZizT58+hISEGAE8gMmTJ1O5cmUcHBzw8PCgV69epKamPtB+XFxccHd3N17r1q3D3t4+W6D7jz/+oF27djg4OFC6dGk+//zzHMdNTk7GZDIZpSSyziE6Oho/Pz/s7e3x9/cnKSnJuGfv3r0EBQXh5OSEs7MzNWvWZNeuXcTExNClSxeuXLliZJyHhYUBmWVVRo0aRceOHXF2dqZHjx73zIyPi4vDZDKRnJxsXNuyZQuBgYHY29tTtGhRgoODuXTpEp07dyY2NpaIiAhjvjvvy/Lll19SqlQpMjIyLK6//vrrdO3aFfhf1vWkSZPw9fWlT58+vPnmm0yZMuW+Z1e2bFm6dOlC1apV8fT05LXXXqN9+/Zs2rTJ6LN7926OHDlikXlvMpnYvXs3I0eONM7oYT4HgBUrVlCjRg0jozw8PJxbt24ZZw7QsmVLTCaT8b5z5860aNHCYpyQkBACAwON94GBgXzwwQcMHjzY+N3L+iyzXL58mW7dulG8eHGcnZ15+eWX2bt3r0WfcePGUaJECZycnHj33Xe5ceNGtnNs3rw5ixcvvu85i4iIiMjfS4FuEREReSpVeiez7vbJDWs4Hr0qW3valcts/qgvGemZGbyuz1ezaLcqaEPZZm8AsPOTUFL/83u2MS4k7Huotc2fPx9XV1d27NjB+++/z3vvvUfr1q3x9/fn119/pXHjxrzzzjtcu3Yt2729e/cmLS2NjRs3sn//fsaPH4+joyMeHh4sXboUyCxdkZKSQkREhMWcNjY2bNmyhZkzZwJQoEABPv30U+Lj45k/fz4///wzgwcPfqg9ZZkzZw5t27bFwcHB4vonn3xC1apV2bNnD0OGDKFv376sW7fugccfNmwYkyZNYteuXVhbWxsBYYD27dvzzDPPsHPnTnbv3s2QIUMoWLAg/v7+TJ06FWdnZyPzfODAgcZ9EydONNb20Ucf5WkdcXFxNGjQgIoVK7J161Y2b95M8+bNuX37NhEREdStW5fu3bsb83l4eGQbo3Xr1ly4cIENGzYY1y5evMjq1atp3749AFu3bqVhw4YW9wUHB7N169Y8n9nhw4dZvXo1AQEBxrVNmzbh4+OD0x2leVJSUqhUqRIDBgzIdkZ3y+lz2LRpEx07dqRv374kJCTwxRdfEBkZyZgxYwDYuXMnAPPmzSMlJcV4n1fz58/HwcGB7du3M2HCBEaOHGnxu9S6dWvOnj3LTz/9xO7du6lRowYNGjTg4sWLAHz77beEhYUxduxYdu3aRcmSJZk+fXq2eWrXrs3vv/9+zy8pREREROTv93T9N4YiIiIi/5/zs2V4vktvDsz7nMSFs0lcOJsyr7TCxsmZtMsXSV7zo9H3pXHTMRXInh/g3aItlw4lcum3BDYP/wCHkqUp7R+Eydqac/t2czFxP3wSmu2+3FStWpXhw4cDMHToUMaNG4erqyvdu3cHYMSIEcyYMYN9+7IH0k+cOMEbb7xB5cqVgczs3SwuLi4AuLm5ZatHXa5cOSZMmGBx7c4a0l5eXowePZqePXveM+iXFzt27ODAgQPMmTMnW1u9evUYMmQIAD4+PmzZsoUpU6bQqFGjB5pjzJgxRsB2yJAhNGvWjBs3bmBra8uJEycYNGgQFSpUADL3nKVw4cKYTCbc3d2zjfnyyy8zYMAA4/3JkydzXceECRPw8/OzOKtKlSoZP9vY2GBvb3/P+bIULVqUpk2bsmjRIho0aADA999/j6urq5Gdf/r0aUqUKGFxX4kSJfjvf//L9evXsbOzu+/4WV+cpKWl0aNHD0aOHGm0HT9+nFKlSln0zypR4ujoaKz7/Pnz9xw7p88hPDycIUOG0KlTJyDzd3TUqFEMHjyY0NBQihcvDkCRIkVyPJ/7qVKlilGGpVy5ckybNo3o6GgaNWrE5s2b2bFjB2fPnqVQoUJA5hcZy5cv5/vvv6dHjx5MnTqVd9991yjbMnr0aNavX58tqzvrfI4fP25knYuIiIjI46OMbhEREXlqPVO/ATU++NB4f2zVMpKWRBpB7sJlyhHwyRfYu9072FbAyopag8LxCMos9XE15RS/LV1A0pLIzCD3Q6pSpYrxs5WVFcWKFTMC14AR2Dx79my2ez/44ANGjx5NvXr1CA0NvWcw/F5q1qyZ7dr69etp0KABpUuXxsnJiXfeeYcLFy7cM5P8xIkTODo6Gq+xY8dm6zNnzhwqV65M7dq1s7XVrVs32/vExEQAevbsaTF2Tu48u5IlSwL/O6f+/fvTrVs3GjZsyLhx4zhy5EiOY2Xx8/PLU787ZWV0P4hKlSoZe2zatCmQmYW+dOlS0tLSAFi4cCFt27alwD2+eHlQS5Ys4ddff2XRokVERUUxceJEo+369evY2to+9Ng5fQ579+5l5MiRFp9pVnb7vX63/szcWfPfOXdqairFihWzmP/YsWPG70NiYiIvvPCCxRh3/34CxpcIj2LNIiIiIvLnKaNbREREnmpu1fwInrOUy0eSuHjwABk3b1LQqTDufnWwLVos1/sLWFlR6Z1/UaFtF87s3s61M/8BwL5EKUrUfCGXu++tYMGCFu9NJpPFtawHT95duxmgW7duBAcHExUVxdq1a/n444+ZNGkS77//fo5z3l1KJDk5mVdffZX33nuPMWPG4OLiwubNm3n33Xe5efMm9vb2Fv1LlSpl1GmG/2WPZ7l69SqLFy+2yBrOq5EjR+ZYJuNOOZ1TWFgYb7/9NlFRUfz000+EhoayePFiWrZsmeOYd59NVpDZbDYb19LveqhpTpnU97Nq1SpjnKz7mzdvjtlsJioqilq1arFp0yaL+tvu7u6cOXPGYpwzZ87g7Oyc6xqyyqVUrFiR27dv06NHDwYMGICVlRWurq7s3//wX9bk9DmkpqYSHh5Oq1atst2XU3C9QIECFmcO2c/97rmz5r9z7pIlSxITE5Ptvrv/K4fcZJU6ycpAFxEREZHHS4FuEREReeqZTCaKelegqHeFhx7DqqANperUf4SrengeHh707NmTnj17MnToUGbNmsX777+PjY0NALdv3851jN27d5ORkcGkSZOMwO6333573/7W1tZ4e3vft/27774jLS2NDh063LN927Zt2d77+voCmaVW3Nzccl1zXvj4+ODj40O/fv1o164d8+bNo2XLltjY2OTpXOB/gc2UlBSKFi0KYBHkh8ys4ujoaMLDw+85xr3m8/T0zNbP1taWVq1asXDhQg4fPkz58uWpUaOG0V63bl1WrbKsMb9u3bp7ZiDnJCMjg/T0dDIyMrCysqJ69erMmDEDs9lsBKoflRo1apCUlJTj70vBggWznU/x4sU5cOCAxbW4uLhsge3c5j59+jTW1tb3LTfi6+vL9u3b6dixo3Ht7t9PgAMHDlCwYEGLkjQiIiIi8viodImIiIjIP0hISAhr1qzh2LFj/Prrr2zYsMEIGHt6emIymVi5ciXnzp0jNTX1vuN4e3uTnp7OZ599xtGjR/n666+Nh1Q+jDlz5tCiRQuKFbt3lvyWLVuYMGECv/32G59//jnfffcdffv2fej57nb9+nX69OlDTEwMx48fZ8uWLezcudM4Gy8vL1JTU4mOjub8+fM5lqPw9vbGw8ODsLAwDh06RFRUFJMmTbLoM3ToUHbu3EmvXr3Yt28fBw8eZMaMGUZNay8vL7Zv305ycjLnz5+/Z3Z+lvbt2xMVFcXcuXONh1Bm6dmzJ0ePHmXw4MEcPHiQ6dOn8+2339KvXz+jz7Rp0yzKqCxcuJBvv/2WxMREjh49yrfffsvQoUN56623jKBxUFAQqampxMfH5/GE827EiBF89dVXhIeHEx8fT2JiIosXLzbq0kPm+URHR3P69GkuXboEZNZK37VrF1999RWHDh0iNDQ0W+A7Nw0bNqRu3bq0aNGCtWvXkpyczC+//MKwYcPYtWsXAH379mXu3LnMmzeP3377jdDQ0Huew6ZNm6hfv/5DZe+LiIiIyKOnjG4RERF5qvzUJecyFfnd7du36d27N7///jvOzs40adLEKHVRunRp40GAXbp0oWPHjkRGRt5znKpVqzJ58mTGjx/P0KFDeemll/j4448tslzzKikpic2bN7N27dr79hkwYAC7du0iPDwcZ2dnJk+eTHBw8APPdT9WVlZcuHCBjh07cubMGVxdXWnVqpWRce3v70/Pnj156623uHDhAqGhoYSFhd1zrIIFC/LNN9/w3nvvUaVKFWrVqsXo0aNp3bq10cfHx4e1a9fy4YcfUrt2bezs7HjhhRdo164dAAMHDqRTp05UrFiR69evc+zYsftmGL/88su4uLiQlJTE22+/bdFWpkwZoqKi6NevHxERETzzzDPMnj3b4uzOnz9vUY/c2tqa8ePH89tvv2E2m/H09KRPnz4WwfFixYrRsmVLFi5cyMcff/xAZ52b4OBgVq5cyciRIxk/fjwFCxakQoUKdOvWzegzadIk+vfvz6xZsyhdujTJyckEBwfz0UcfMXjwYG7cuEHXrl3p2LHjA5VYMZlMrFq1imHDhtGlSxfOnTuHu7s7L730klH7/q233uLIkSPGPG+88Qbvvfcea9assRhr8eLF9/0dEREREZG/n8l8d6E7ERERkXzuxo0bHDt2jDJlyvypB+qJPM327dtHo0aNOHLkSK4PAX3a/PTTTwwYMIB9+/Zhba3cocdFf+tFRETkTipdIiIiIiIi2VSpUoXx48dz7Nixx72UJ87Vq1eZN2+egtwiIiIiTxBldIuIiMg/jrL8RET++fS3XkRERO6kjG4RERERERERERERydcU6BYRERERERERERGRfE2BbhERERERERERERHJ1xToFhEREREREREREZF8TYFuEREREREREREREcnXFOgWERERERERERERkXxNgW4RERGRfMJkMrF8+fLHvQwRw52/k8nJyZhMJuLi4h7rmp4EkZGRFClS5HEvQ0REROSpYv24FyAiIiLyd3ojMvFvnW9pZ98H6t+5c2cuX76c7wLamzdv5t///jcHDx7k2rVreHp68q9//Yt+/frleF9qaipDhgxh+fLlXLhwgTJlyvDBBx/Qs2dPo4+XlxfHjx8HoECBApQoUYKmTZsyceJEihYtet+xIyMjCQkJ4fLly49kj3/3Z2Mymfjhhx9o0aJFnvpv2LCBSZMmsX37dv744w9Kly6Nn58fvXv35qWXXvprFwt4eHiQkpKCq6vrIx03r+cQHx/PiBEj2L17N8ePH2fKlCmEhIRk6/f555/zySefcPr0aapWrcpnn31G7dq1s/Uzm8288sorrF69Otv8O3fuZMiQIezevRuTyUTt2rWZMGECVatW/ZO7FREREZGHpYxuEREREfnTHBwc6NOnDxs3biQxMZHhw4czfPhwvvzyyxzv69+/P6tXr2bBggUkJiYSEhJCnz59+PHHHy36jRw5kpSUFE6cOMHChQvZuHEjH3zwwV+5pYeWnp7+t885ffp0GjRoQLFixViyZAlJSUn88MMP+Pv75/hlw+3bt8nIyHgka7CyssLd3R1r68eTS3Pt2jXKli3LuHHjcHd3v2efJUuW0L9/f0JDQ/n111+pWrUqwcHBnD17NlvfqVOnYjKZsl1PTU2lSZMmPPvss2zfvp3Nmzfj5OREcHDwY/nsRURERCSTAt0iIiIi+dS///1vfHx8sLe3p2zZsnz00UcWgbawsDCqVavG3LlzefbZZ3F0dKRXr17cvn2bCRMm4O7ujpubG2PGjLEYd/LkyVSuXBkHBwc8PDzo1asXqampOa6levXqtGvXjkqVKuHl5UWHDh0IDg5m06ZNOd73yy+/0KlTJwIDA/Hy8qJHjx5UrVqVHTt2WPRzcnLC3d2d0qVLExQURKdOnfj1118f6LyyzuPrr7/Gy8uLwoUL07ZtW/744w+jz/fff0/lypWxs7OjWLFiNGzYkKtXrxIWFsb8+fNZsWIFJpMJk8lETEyMUa5jyZIlBAQEYGtry8KFC4257jR16lS8vLwsrs2dO5dKlSpRqFAhSpYsSZ8+fQCMfi1btsRkMmW7704nTpwgJCSEkJAQ5s+fz8svv4ynpydVqlShb9++7Nq1y+ibVVLjxx9/pGLFihQqVIgTJ06wc+dOGjVqhKurK4ULFyYgICDb+R46dIiXXnoJW1tbKlasyLp16yza71W65MCBAzRt2hRHR0dKlCjBO++8w/nz5432wMBAPvjgAwYPHoyLiwvu7u6EhYUZ7Q9yDrVq1eKTTz6hbdu2FCpU6J59Jk+eTPfu3enSpQsVK1Zk5syZ2NvbM3fuXIt+cXFxTJo0Kdt1gIMHD3Lx4kVGjhxJ+fLlqVSpEqGhoZw5c8b4Lw/udu7cOfz8/GjZsiVpaWn33YOIiIiIPDwFukVERETyKScnJyIjI0lISCAiIoJZs2YxZcoUiz5Hjhzhp59+YvXq1XzzzTfMmTOHZs2a8fvvvxMbG8v48eMZPnw427dvN+4pUKAAn376KfHx8cyfP5+ff/6ZwYMHP9Da9uzZwy+//EJAQECO/fz9/fnxxx85deoUZrOZDRs28Ntvv9G4ceP73nPq1Cn+7//+jxdeeOGB1gSZ57F8+XJWrlzJypUriY2NZdy4cQCkpKTQrl07unbtSmJiIjExMbRq1Qqz2czAgQNp06YNTZo0ISUlhZSUFPz9/Y1xhwwZQt++fUlMTCQ4ODhPa5kxYwa9e/emR48e7N+/nx9//BFvb28gszQGwLx580hJSTHe38vSpUtJT0+/72d0d1bytWvXGD9+PLNnzyY+Ph43Nzf++OMPOnXqxObNm9m2bRvlypXjlVdeMb4EyMjIoFWrVtjY2LB9+3ZmzpzJv//97xz3d/nyZV5++WWqV6/Orl27WL16NWfOnKFNmzYW/ebPn4+DgwPbt29nwoQJjBw50giiP8g55ObmzZvs3r2bhg0bGtcKFChAw4YN2bp1q8X5vP3223z++ef3zAwvX748xYoVY86cOdy8eZPr168zZ84cfH197xmIP3nyJPXr1+f555/n+++/v28QXkRERET+HNXoFhEREcmnhg8fbvzs5eXFwIEDWbx4sUXAMyMjg7lz5+Lk5ETFihUJCgoiKSmJVatWUaBAAcqXL8/48ePZsGGDETi+s66xl5cXo0ePpmfPnkyfPj3XNT3zzDOcO3eOW7duERYWRrdu3XLs/9lnn9GjRw+eeeYZrK2tKVCgALNmzcpWU/rf//43w4cP5/bt29y4cYMXXniByZMn5+WYLGRkZBAZGYmTkxMA77zzDtHR0YwZM4aUlBRu3bpFq1at8PT0BKBy5crGvXZ2dqSlpd0z+BkSEkKrVq0eaC2jR49mwIAB9O3b17hWq1YtAIoXLw5AkSJF7luGI8tvv/2Gs7OzRb+lS5fSqVMn4/3WrVuNvaSnpzN9+nSLetIvv/yyxZhffvklRYoUITY2lldffZX169dz8OBB1qxZQ6lSpQAYO3YsTZs2ve+6pk2bRvXq1Rk7dqxxbe7cuXh4ePDbb7/h4+MDQJUqVQgNDQWgXLlyTJs2jejoaBo1avRA55Cb8+fPc/v2bUqUKGFxvUSJEhw8eNB4369fP/z9/Xn99dfvOY6TkxMxMTG0aNGCUaNGGetes2ZNtrItSUlJNGrUiJYtW963FIqIiIiIPBrK6BYRERHJp5YsWUK9evVwd3fH0dGR4cOHc+LECYs+Xl5eRlAXMoN6FStWpECBAhbX7qxRvH79eho0aEDp0qVxcnLinXfe4cKFC1y7dg0AR0dH43XnQyMBNm3axK5du5g5cyZTp07lm2++AWDhwoUW92WVNPnss8/Ytm0bP/74I7t372bSpEn07t2b9evXW4w7aNAg4uLi2LdvH9HR0QA0a9aM27dv57qmnM6jZMmSxt6rVq1KgwYNqFy5Mq1bt2bWrFlcunQpp4/A4Ofnl6d+Wc6ePct//vMfGjRokOd7Tpw4YbHPOwPIdwdQg4ODiYuLIyoqiqtXrxrnBGBjY0OVKlUs+p85c4bu3btTrlw5ChcujLOzM6mpqcbvU2JiIh4eHkaQG6Bu3bo5rnfv3r1s2LDBYs0VKlQAMjPrs9y9ljs/kwc9hz/rxx9/5Oeff2bq1Kn37XP9+nXeffdd6tWrx7Zt29iyZQvPP/88zZo14/r16xb96tevT6tWrYiIiFCQW0REROQvpoxuERERkXxo69attG/fnvDwcIKDgylcuDCLFy9m0qRJFv0KFixo8d5kMt3zWtYDCZOTk3n11Vd57733GDNmDC4uLmzevJl3332XmzdvYm9vb1GD2dnZ2WKsMmXKAJmZ0GfOnCEsLIx27drx2muvWZQaKV26NNevX+fDDz/khx9+oFmzZkBm0DMuLo6JEydalJhwdXU1ynqUK1eOqVOnUrduXTZs2EDDhg1zXFNu55G1dysrK9atW8cvv/zC2rVr+eyzzxg2bBjbt2839nU/Dg4OFu8LFCiA2Wy2uHZn/XQ7O7scx7uXUqVKWezTxcUFyDyPK1eucPr0aSPr2dHREW9v73s+GNLOzi5b0LVTp05cuHCBiIgIPD09KVSoEHXr1uXmzZsPvM4sqampNG/enPHjx2drK1mypPFzTp/JvdzvHHLj6uqKlZUVZ86csbh+5swZ49x+/vlnjhw5QpEiRSz6vPHGG9SvX5+YmBgWLVpEcnIyW7duNb4wWrRoEUWLFmXFihW0bdsWgEKFCtGwYUNWrlzJoEGDKF26dJ7WKSIiIiIPR4FuERERkXzol19+wdPTk2HDhhnX7vcgvAexe/duMjIymDRpkhHE+/bbby36ZAWcc5ORkWE8eM/Jyckikxrgv//9L+np6RbZ5ZAZcM4p0JnVBzAyaPO6ptyYTCbq1atHvXr1GDFiBJ6envzwww/0798fGxsbi8zonBQvXpzTp09jNpuNoPKdwVknJye8vLyIjo4mKCjonmMULFjQYj5ra+t77vPNN99kyJAhjB8/PluN9rzasmUL06dP55VXXgEy60rf+dBIX19fTp48SUpKihGk3rZtW45j1qhRg6VLl+Ll5XXPgHte5fUccmNjY0PNmjWJjo6mRYsWQObvaHR0tPEQ0CFDhmQrt1O5cmWmTJlC8+bNgcwa3gUKFLD4siDr/Z2/twUKFODrr7/m7bffJigoiJiYGIuMeBERERF5tBToFhEREXnCXLlyxSIoClCsWDGL9+XKlePEiRMsXryYWrVqERUVxQ8//PCn5/b29iY9PZ3PPvuM5s2bs2XLFmbOnJnrfZ9//jnPPvusUZpi48aNTJw4kQ8++OC+9zg7OxMQEMCgQYOws7PD09OT2NhYvvrqq2z1t//44w8jcHzy5EkGDx5M8eLFLR4I+Wdt376d6OhoGjdujJubG9u3b+fcuXP4+voCmWVP1qxZQ1JSEsWKFaNw4cL3HSswMJBz584xYcIE3nzzTVavXs1PP/1kkW0eFhZGz549cXNzo2nTpvzxxx9s2bKF/9fevcf1eP//A3+8K510kkqFepeSFJXFxKilNMkcZqZhOc8UcmY6j5QJOcQQhcR3FmtmOaRyzCmxlCSSUc5MSqX6/dGv69NbZ7NZ87jfbu/brfd1va7X63W9rku3m+f17HlNmzZNGC8+Ph69e/eGnJwcWrVqVetYenp6CAkJwYwZM/D48WOMHTsWBgYGePz4MXbs2AHgfw8G6mJsbIzt27fD2toaf/75p3BNqjg4OKBjx45wc3PD999/jz///FPiIUtt3N3dsWnTJri6umLevHlQV1fH9evXsWvXLmzevLnBOVVp7DqUlJQgPT1d+PnOnTtITU0VstsBYNasWXBzc4O1tTV69OiBVatW4cWLFxg3bhwAQFtbu9Za4Hp6ekJWv6OjI+bOnQt3d3dMmzYN5eXlCAoKgoyMTI2HFtLS0oiKioKrqyvs7e2RmJj4l2uNExEREVHtWKObiIiI6F8mMTERVlZWEh9/f3+JNp9++ilmzpwJDw8PWFpa4tSpU/D29v7LY1tYWGDFihUIDg6Gubk5oqKisHTp0gaPKy8vx8KFC2FpaQlra2usW7cOwcHBCAgIqPe4qkD9qFGj0LlzZwQFBWHJkiU16mz7+PhAR0cHurq6cHFxQcuWLXHo0KEaDwD+ChUVFRw7dgzOzs7o2LEjvLy8EBISIrxwcdKkSTAxMYG1tTU0NTVx8uTJOvsyNTVFWFgY1q1bBwsLC5w9exZz5syRaOPm5oZVq1YhLCwMZmZmcHFxQVZWlrA/JCQEhw8fRvv27WFlZVXv3KdNm4ZDhw7hwYMHGD58OIyNjeHs7IybN28iLi5O4qWatQkPD8eTJ0/QrVs3jBkzBtOnT4eWlpawX0pKCnv37kVRURF69OiBiRMnYsmSJfX2qauri5MnT6KsrAz9+/dHly5d4OnpCTU1tRpZ/PVp7DrcvXtX+PeSl5eH5cuXw8rKSiJD+4svvsDy5cvh4+MDS0tLpKamIi4ursYLKuvTqVMn/PLLL7h8+TJsbGzQp08f3L17F3FxcRIlWarIyMggOjoaZmZmsLe3r7f+OBERERG9OVHF68UDiYiIiJq5ly9f4ubNmzAwMIC8vPy7ng4REf0N+LueiIiIqmNGNxERERERERERERE1awx0ExEREREREREREVGzxkA3ERERERERERERETVrDHQTERERERERERERUbPGQDcRERERERERERERNWsMdBMRERERERERERFRs8ZANxERERERERERERE1awx0ExEREREREREREVGzxkA3ERERERERERERETVrDHQTERERNRMikQj79u1719MgElS/J3NyciASiZCamvpO5/RvEBERATU1tQbbeXt7Y/LkycL3iooKTJ48Gerq6sJa2tnZwdPT8++b7L/A6/dOeno62rVrhxcvXrzbiREREVGzIvOuJ0BERET0T9qzMf8fHW/4ZO0mtR87diyePn3a7ALaJ06cwPz583H16lUUFhZCX18fX3/9NWbOnFnvcQUFBViwYAH27duHR48ewcDAANOnT8eUKVOENmKxGLdu3QIASElJoU2bNhgwYACWL1+OVq1a1dl3REQEPD098fTp07dyjv/0tRGJRNi7dy+GDBnSqPYJCQkICQnBmTNn8Pz5c7Rt2xbW1tZwd3dH3759/97JAmjfvj3y8vKgoaHxVvtt7DpcuXIFPj4+uHDhAm7duoWVK1fWGiBet24dvv/+e+Tn58PCwgJr1qxBjx49arSrqKiAs7Mz4uLiaox/7tw5LFiwABcuXIBIJEKPHj2wbNkyWFhYNPq88vPzERoait9//13YFhcXh4iICCQmJsLQ0BAaGhqIiYlBixYtGt1vY/zbf8907twZPXv2xIoVK+Dt7f2up0NERETNBDO6iYiIiOgva9myJTw8PHDs2DFkZGTAy8sLXl5e2LhxY73HzZo1C3FxcdixYwcyMjLg6ekJDw8PxMbGSrQLCAhAXl4ecnNzERUVhWPHjmH69Ol/5ym9sdLS0n98zLCwMPTr1w+tW7fG7t27kZmZib1796JXr171PmwoKytDeXn5W5mDtLQ0tLW1ISPzbnJpCgsLYWhoiKCgIGhr1/6Aaffu3Zg1axZ8fX2RkpICCwsLODk54f79+zXarlq1CiKRqMb2goICfPLJJ9DT08OZM2dw4sQJKCsrw8nJqUnXfvPmzejVqxf09fWFbdnZ2dDR0UGvXr2EtVRXV4eysnKj+/2vGDduHNavX49Xr16966kQERFRM8FANxEREb1XLlw+/MbHVlRUYOe+pfiz4NFbnNGbmz9/Pjp27AhFRUUYGhrC29tbItDm5+cHS0tLbNmyBXp6elBSUsLUqVNRVlaGZcuWQVtbG1paWliyZIlEvytWrECXLl3QsmVLtG/fHlOnTkVBQUG9c7GysoKrqyvMzMwgFosxevRoODk54fjx4/Ued+rUKbi5ucHOzg5isRiTJ0+GhYUFzp49K9FOWVkZ2traaNu2LT7++GO4ubkhJSWlSetVtR7bt2+HWCyGqqoqRo4ciefPnwtt9uzZgy5dukBBQQGtW7eGg4MDXrx4AT8/P0RGRuLnn3+GSCSCSCRCYmKiUHJh9+7dsLW1hby8PKKiooSxqlu1ahXEYrHEti1btsDMzAxycnLQ0dGBh4cHAAjthg4dCpFIVOO46nJzc+Hp6QlPT09ERkbC3t4e+vr66Nq1K2bMmIHz588LbatKasTGxqJz586Qk5NDbm4uzp07B0dHR2hoaEBVVRW2trY11jcrKwt9+/aFvLw8OnfujMOHJf8t1Va6JC0tDQMGDICSkhLatGmDMWPG4OHDh8J+Ozs7TJ8+HfPmzYO6ujq0tbXh5+cn7G/KOnTv3h3ff/89Ro4cCTk5uVrbrFixApMmTcK4cePQuXNnbNiwAYqKitiyZYtEu9TUVISEhNTYDgBXr17F48ePERAQABMTE5iZmcHX1xf37t0T/vLgdQ8ePIC1tTWGDh2K4uJiAMCuXbswaNAgoc3YsWMxbdo05ObmSpzr66VLxGIxAgMDMX78eCgrK0NPT6/GA6Xbt29jxIgRUFNTg7q6OgYPHoycnBwAqPNeTkxMhEgkkvgLiNTUVIhEIuHYqvvn4MGDMDU1hZKSEj755BPk5eVJjL9582aYmppCXl4enTp1QlhYmMT+s2fPwsrKCvLy8rC2tsbFixdrrJmjoyMeP36MpKSkWteUiIiI6HUMdBMREdF7JWjdV6ioqHijY69eP4O9v63Gt0Eub3lWb0ZZWRkRERFIT09HaGgoNm3ahJUrV0q0yc7Oxm+//Ya4uDhER0cjPDwcAwcOxB9//IGkpCQEBwfDy8sLZ86cEY6RkpLC6tWrceXKFURGRuLo0aOYN29ek+Z28eJFnDp1Cra2tvW269WrF2JjY3Hnzh1UVFQgISEB165dQ//+/es85s6dO/jll1/w4YcfNmlOQOV67Nu3D/v378f+/fuRlJSEoKAgAEBeXh5cXV0xfvx4ZGRkIDExEcOGDUNFRQXmzJmDESNGCEG9vLw89OrVS+h3wYIFmDFjBjIyMuDk5NSouaxfvx7u7u6YPHkyfv/9d8TGxsLIyAhAZWkMANi6dSvy8vKE77X56aefUFpaWuc1ej0rubCwEMHBwdi8eTOuXLkCLS0tPH/+HG5ubjhx4gSSk5NhbGwMZ2dn4SFAeXk5hg0bBllZWZw5cwYbNmzA/Pnz6z2/p0+fwt7eHlZWVjh//jzi4uJw7949jBgxQqJdZGQkWrZsiTNnzmDZsmUICAgQguhNWYeGlJSU4MKFC3BwcBC2SUlJwcHBAadPn5ZYny+//BLr1q2rNTPcxMQErVu3Rnh4OEpKSlBUVITw8HCYmprWGoi/ffs2+vTpA3Nzc+zZswdycnJ4/Pgx0tPTYW1tLbQLDQ1FQEAA2rVr1+C5hoSECAHiqVOn4ptvvkFmZiaAyr8ocHJygrKyMo4fP46TJ08KAemSkpIG7+WGFBYWYvny5di+fTuOHTuG3NxczJkzR9gfFRUFHx8fLFmyBBkZGQgMDIS3tzciIyMBVGbEu7i4oHPnzrhw4QL8/Pwkjq8iKysLS0vLBh+WEREREVVhjW4iIiJ676SkxeODLg4NN3xN/ImdAICnf95H0csXUJBv+ban1iReXl7Cz2KxGHPmzMGuXbskAp7l5eXYsmULlJWV0blzZ3z88cfIzMzEgQMHICUlBRMTEwQHByMhIUEIHL+ePbp48WJMmTKlRlZmbdq1a4cHDx7g1atX8PPzw8SJE+ttv2bNGkyePBnt2rWDjIwMpKSksGnTpho1pefPnw8vLy+UlZXh5cuX+PDDD7FixYrGLJOE8vJyRERECKUgxowZg/j4eCxZsgR5eXl49eoVhg0bJpST6NKli3CsgoICiouLaw1+enp6YtiwYU2ay+LFizF79mzMmDFD2Na9e3cAgKamJgBATU2tzjIcVa5duwYVFRWJdj/99BPc3NyE76dPnxbOpbS0FGFhYRL1pO3t7SX63LhxI9TU1JCUlAQXFxccOXIEV69excGDB6GrqwsACAwMxIABA+qc19q1a2FlZYXAwEBh25YtW9C+fXtcu3YNHTt2BAB07doVvr6+AABjY2OsXbsW8fHxcHR0bNI6NOThw4coKytDmzZtJLa3adMGV69eFb7PnDkTvXr1wuDBg2vtR1lZGYmJiRgyZAi+++47Yd4HDx6sUbYlMzMTjo6OGDp0qEQplNzcXFRUVAhrCQCqqqpQVlYWSsDUx9nZGVOnTgVQ+W9j5cqVSEhIgImJCXbv3o3y8nJs3rxZGG/r1q1QU1NDYmIi+vfvX++93JDS0lJs2LABHTp0AAB4eHggICBA2O/r64uQkBDh34OBgQHS09Pxww8/wM3NDTt37kR5eTnCw8MhLy8PMzMz/PHHH/jmm29qjKWrq1tnljwRERHR65jRTURERO8VKSlp7I5d1uSs7rz7N3HszE8AgJKSIhxKivw7ptcku3fvRu/evaGtrQ0lJSV4eXkhNzdXoo1YLJao79umTRt07twZUlJSEtuq1yg+cuQI+vXrh7Zt20JZWRljxozBo0ePUFhYCABQUlISPtVfGgkAx48fx/nz57FhwwasWrUK0dHRACqzPKsfV5WluWbNGiQnJyM2NhYXLlxASEgI3N3dceTIEYl+586di9TUVFy+fBnx8fEAgIEDB6KsrKzBOdW3Hjo6OsK5W1hYoF+/fujSpQs+//xzbNq0CU+ePKnvEgiqZ+Y2xv3793H37l3069ev0cfk5uZKnGf1APLrWdtOTk5ITU3Fr7/+ihcvXgjrBFRmynbt2lWi/b179zBp0iQYGxtDVVUVKioqKCgoEO6njIwMtG/fXiIwa2NjU+98L126hISEBIk5d+rUCUBlZn2V1+dS/Zo0dR3+qtjYWBw9ehSrVq2qs01RUREmTJiA3r17Izk5GSdPnoS5uTkGDhyIoqIiiXZ9+vTBsGHDEBoaKnGNqtrJy8u/0Tyrr5lIJIK2trawZpcuXcL169ehrKwsrJG6ujpevnwpse5vSlFRUQhyA5LX68WLF8jOzsaECRMkrtHixYuFsTMyMtC1a1eJc6/rXlJQUBB+7xARERE1hBndRERE9F4pLy/DzdzfMcPnI7TT7QgdLUPotjFEB31LiNub/f825bh45Sjy7mUj7/5N3M3Pxq0/rkBKJIWyinJUVFQg+ucgnL4Qi7Y6xtDRMoCOliHMTT6CqooGAODZnw+RlnkCwyfXHXT9K06fPo1Ro0bB398fTk5OUFVVxa5duxASEiLRrkWLFhLfRSJRrduqXkiYk5MDFxcXfPPNN1iyZAnU1dVx4sQJTJgwASUlJVBUVJSowayioiLRl4GBAYDKTOh79+7Bz88Prq6u+PTTTyVKjbRt2xZFRUX49ttvsXfvXgwcOBBAZQAvNTUVy5cvlygxoaGhIZT1MDY2xqpVq2BjY4OEhAQ4ODjUO6eG1qPq3KWlpXH48GGcOnUKhw4dwpo1a7Bo0SKcOXNGOK+6tGwpmd0vJSVV42FK9frpCgoK9fZXG11dXYnzVFdXB1C5Hs+ePUN+fr6QoaukpAQjI6NaXwypoKBQIzDu5uaGR48eITQ0FPr6+pCTk4ONjQ1KSkqaPM8qBQUFGDRoEIKDg2vs09HREX6u75rUpq51aIiGhgakpaVx7949ie337t0T1u3o0aPIzs6GmpqaRJvPPvsMffr0QWJiInbu3ImcnBycPn1aeGC0c+dOtGrVCj///DNGjhwJAJCTk4ODgwP279+PuXPnom3bthJzAYAnT54IWetNUd+aFRQU4IMPPkBUVFSN4+obq+pcqt+3tb1cs7axq46pquW/adOmGqWFpKWl6xy7Lo8fP5YIqhMRERHVh4FuIiIiei/l3b+B/Ac3UVFRATUVLfjN3iPsk5KSwuMneYj80a/O48vKSpF96xJu5F5GRUUFPujaHz0s/1fGQVFBGcfP7gXw9wS6T506BX19fSxatEjY9jb+xP/ChQsoLy9HSEiIEPj6v//7P4k2VQHnhpSXlwsv3lNWVpbIpAaAP//8E6WlpRLZ5UBlQKy+QGdVG+B/mbGNnVNDRCIRevfujd69e8PHxwf6+vrYu3cvZs2aBVlZWYnM6PpoamoiPz8fFRUVQlC5enBWWVkZYrEY8fHx+Pjjj2vto0WLFhLjycjI1Hqew4cPx4IFCxAcHFyjRntjnTx5EmFhYXB2dgZQWVe6+ksjTU1Ncfv2beTl5QlB6uTk5Hr77NatG3766SeIxeJaA+6N1dh1aIisrCw++OADxMfHY8iQIQAq79H4+HjhJaALFiyoUW6nS5cuWLlypfDiyMLCQkhJSUk8LKj6Xv2+lZKSwvbt2/Hll1/i448/RmJiopAR36FDB6ioqCA9PV0o4fK2dOvWDbt374aWlladD31qu5erguB5eXlo1aoVAMl7tjHatGkDXV1d3LhxA6NGjaq1jampKbZv346XL18KWd113UtpaWkYPnx4k+ZARERE7y+WLiEiIqL3lpRIGmoqWgiYsxdttY0l9jn2HYOpX62ESCSqkf36uh6WAzD7601o0UJO2NaihRxmf73pjeb17NkzpKamSnxu374t0cbY2Bi5ubnYtWsXsrOzsXr1auzdu/eNxqvOyMgIpaWlWLNmDW7cuIHt27djw4YNDR63bt06/PLLL8jKykJWVhbCw8OxfPlyjB49us5jVFRUYGtri7lz5yIxMRE3b95EREQEtm3bhqFDh0q0ff78OfLz85GXl4ezZ89i7ty50NTUbNJL9Bpy5swZBAYG4vz588jNzUVMTAwePHgAU1NTAJVlTy5fvozMzEw8fPiw1mzXKnZ2dnjw4AGWLVuG7OxsrFu3Dr/99ptEGz8/P4SEhGD16tXIyspCSkoK1qxZI+yvCoTn5+fXW0JFT08PISEhCA0NhZubGxISEpCTk4OUlBSsXr0aQMPZtMbGxti+fTsyMjJw5swZjBo1SiLr3MHBAR07doSbmxsuXbqE48ePSzxkqY27uzseP34MV1dXnDt3DtnZ2Th48CDGjRvX6AcGTVmHkpIS4d9LSUkJ7ty5g9TUVFy/fl1oM2vWLGzatAmRkZHIyMjAN998gxcvXmDcuHEAAG1tbZibm0t8gMo1rsrqd3R0xJMnT+Du7o6MjAxcuXIF48aNg4yMTI2HFtLS0oiKioKFhQXs7e2Rn58P4H8vwTxx4kSj16GxRo0aBQ0NDQwePBjHjx/HzZs3kZiYiOnTp+OPP/4AUPu9bGRkhPbt28PPzw9ZWVn49ddfa/yFSGP4+/tj6dKlWL16Na5du4bff/8dW7duFWrqf/nllxCJRJg0aRLS09Nx4MABLF++vEY/OTk5uHPnjsRfdhARERHVh4FuIiIiei9JSUlDVUUTi+f/Ap02hrW2+bj3SEwbt6bWfUBl9q/NB59i5uQf0EJGtsb+2rY1RmJiIqysrCQ+/v7+Em0+/fRTzJw5Ex4eHrC0tMSpU6fg7e39RuNVZ2FhgRUrViA4OBjm5uaIiorC0qVLGzyuvLwcCxcuhKWlJaytrbFu3ToEBwdLvKSuNrt27UL37t0xatQodO7cGUFBQViyZEmNOts+Pj7Q0dGBrq4uXFxc0LJlSxw6dAitW7f+S+dbnYqKCo4dOwZnZ2d07NgRXl5eCAkJEV64OGnSJJiYmMDa2hqampo4efJknX2ZmpoiLCwM69atg4WFBc6ePYs5c+ZItHFzc8OqVasQFhYGMzMzuLi4ICsrS9gfEhKCw4cPo3379rCysqp37tOmTcOhQ4fw4MEDDB8+HMbGxnB2dsbNmzcRFxcn8VLN2oSHh+PJkyfo1q0bxowZg+nTp0NLS0vYLyUlhb1796KoqAg9evTAxIkTsWTJknr71NXVxcmTJ1FWVob+/fujS5cu8PT0hJqaWo0s/vo0dh3u3r0r/HvJy8vD8uXLYWVlJZGh/cUXX2D58uXw8fGBpaUlUlNTERcXV+MFlfXp1KkTfvnlF1y+fBk2Njbo06cP7t69i7i4OImSLFVkZGQQHR0NMzMz2NvbC/WsJ06ciF27djX41wtNpaioiGPHjkFPTw/Dhg2DqakpJkyYgJcvXwoZ3rXdyy1atEB0dDSuXr2Krl27Ijg4GIsXL27y+BMnTsTmzZuxdetWdOnSBba2toiIiBAeFCgpKeGXX37B77//DisrKyxatKjW8jbR0dHo37+/8GJYIiIiooaIKpr6JiYiIiKif7mXL1/i5s2bMDAweOOXvRER/Z0qKirw4YcfYubMmXB1dX3X0/lXKSkpgbGxMXbu3InevXvX2Y6/64mIiKg6ZnQTERERERH9w0QiETZu3IhXr16966n86+Tm5uLbb7+tN8hNRERE9DpmdBMREdF/DrP8iIj++/i7noiIiKpjRjcRERERERERERERNWsMdBMRERERERERERFRs8ZANxERERERERERERE1awx0ExEREREREREREVGzxkA3ERERERERERERETVrDHQTERERERERERERUbPGQDcRERERERERERERNWsMdBMRERER/QfY2dnB09PzL/WRk5MDkUiE1NTUtzKn6saOHYshQ4bU2+ZtnAMRERERvZ9k3vUEiIiIiP5J1xbebnTbsJOrEXp8OWb0mYOpvac3eayC4gJ0W2Ha5OMA4Pbt2/D19UVcXBwePnwIHR0dDBkyBD4+PmjduvUb9flXXbp0CUFBQThx4gQePnwIsViMKVOmYMaMGfUeZ2dnh6SkpBrbnZ2d8euvv6K0tBReXl44cOAAbty4AVVVVTg4OCAoKAi6urpCe5FIJPwsLS0NXV1dDB8+HEuXLoWcnFyd4/v5+WHfvn1vLXhrZ2cHS0tLrFq16q30V5+cnBwYGBjg4sWLsLS0/NvHexuuX78OKysrSEtL4+nTp006NiYmBi1atBC+i8VieHp6MvhNRERERA1iRjcRERFRLd5GkHvi/415o7Fv3LgBa2trZGVlITo6GtevX8eGDRsQHx8PGxsbPH78+I36/asuXLgALS0t7NixA1euXMGiRYuwcOFCrF27tt7jYmJikJeXJ3zS0tIgLS2Nzz//HABQWFiIlJQUeHt7IyUlBTExMcjMzMSnn35ao6+tW7ciLy8PN2/eRFhYGLZv347Fixf/Lef7V1RUVODVq1fvehr/uNLSUri6uqJPnz5vdLy6ujqUlZXf8qyIiIiI6H3AQDcRERHRa95WkDvr4bU3Gt/d3R2ysrI4dOgQbG1toaenhwEDBuDIkSO4c+cOFi1ahLVr18Lc3Fw4Zt++fRCJRNiwYYOwzcHBAV5eXsL3n3/+Gd26dYO8vDwMDQ3h7+8vEYwViUTYvHkzhg4dCkVFRRgbGyM2NlbYP378eISGhsLW1haGhoYYPXo0xo0bh5iYmHrPR11dHdra2sLn8OHDUFRUFALdqqqqOHz4MEaMGAETExP07NkTa9euxYULF5CbmyvRl5qaGrS1tdG+fXu4uLhg8ODBSElJadL6VpXQWL58OXR0dNC6dWu4u7ujtLRUaBMWFgZjY2PIy8ujTZs2GD58uHBsUlISQkNDIRKJIBKJkJOTg8TERIhEIvz222/44IMPICcnhxMnTtRarsPT0xN2dnbC9/LycixbtgxGRkaQk5ODnp4elixZAgAwMDAAAFhZWUEkEkkcV5tXr17Bw8MDqqqq0NDQgLe3NyoqKoT9YrEYgYGBGD9+PJSVlaGnp4eNGzfW6Ofq1avo1asX5OXlYW5uXmtGfm28vLzQqVMnjBgxos42/v7+0NTUhIqKCqZMmYKSkhJhX/XSJXZ2drh16xZmzpwprDURERERUV0Y6CYiIiKq5m0Gubd+EdXk4x8/foyDBw9i6tSpUFBQkNinra2NUaNGYffu3bC1tUV6ejoePHgAAEhKSoKGhgYSExMBVGbWnj59WgiMHj9+HF999RVmzJiB9PR0/PDDD4iIiBACqlX8/f0xYsQIXL58Gc7Ozhg1alS9GeTPnj2Durp6k84xPDwcI0eORMuWLevtVyQSQU1Nrc42165dw9GjR/Hhhx82aXwASEhIQHZ2NhISEhAZGYmIiAhEREQAAM6fP4/p06cjICAAmZmZiIuLQ9++fQEAoaGhsLGxwaRJk4QM9fbt2wv9LliwAEFBQcjIyEDXrl0bNZeFCxciKCgI3t7eSE9Px86dO9GmTRsAwNmzZwEAR44cQV5eXoMPFSIjIyEjI4OzZ88iNDQUK1aswObNmyXahISEwNraGhcvXsTUqVPxzTffIDMzU6LN3LlzMXv2bFy8eBE2NjYYNGgQHj16VO/YR48exY8//oh169bV2SY+Ph4ZGRlITExEdHQ0YmJi4O/vX2vbmJgYtGvXDgEBAcJaExERERHVhYFuIiIiov/vbQe5u+paNrmPrKwsVFRUwNS09trepqamePLkCbS0tKCuri5k2iYmJmL27NnC97Nnz6K0tBS9evUCUBnAXrBgAdzc3GBoaAhHR0d89913+OGHHyT6Hzt2LFxdXWFkZITAwEAUFBQIwdbXnTp1Crt378bkyZMbfX5nz55FWloaJk6cWGebly9fYv78+XB1dYWKiorEPldXVygpKUFeXh4mJiYwMzPDwoULGz1+lVatWmHt2rXo1KkTXFxcMHDgQMTHxwMAcnNz0bJlS7i4uEBfXx9WVlaYPr3yflBVVYWsrCwUFRWFDHVpaWmh34CAADg6OqJDhw6NegDw/PlzhIaGYtmyZXBzc0OHDh3w0UcfCeujqakJAGjdujW0tbUb7LN9+/ZYuXIlTExMMGrUKEybNg0rV66UaOPs7IypU6fCyMgI8+fPh4aGBhISEiTaeHh44LPPPoOpqSnWr18PVVVVhIeH1znuo0ePMHbsWERERNS4ZtXJyspiy5YtMDMzw8CBAxEQEIDVq1ejvLy8Rlt1dXVIS0tDWVlZWGsiIiIiorow0E1ERESEf0eQu7rq5SZqIycnh759+yIxMRFPnz5Feno6pk6diuLiYly9ehVJSUno3r07FBUVAVS+SDIgIABKSkrCpyorubCwUOi3ehZyy5YtoaKigvv379cYPy0tDYMHD4avry/69+8PoDJAXL3/wMDAGseFh4ejS5cu6NGjR63nVVpaihEjRqCiogLr16+vsX/lypVITU3FpUuXsH//fly7dg1jxoxp9PhVzMzMJALUOjo6wnk6OjpCX18fhoaGGDNmDKKioiTWqD7W1taNalclIyMDxcXF6NevX6OPOX78uMR5RkX97y8HevbsKVHiw8bGBllZWSgrKxO2Vb/GIpEI2traNa6xjY2N8LOMjAysra2RkZEBoHLtqsYeMGAAAGDSpEn48ssvhcz3ulhYWAj3ZNU4BQUFuH278S+JJSIiIiKqjcy7ngARERHRu/ZvCnIbGRlBJBIhIyMDQ4cOrbE/IyMDmpqaUFNTg52dHTZu3Ijjx4/DysoKKioqQvA7KSkJtra2/5tjQQH8/f0xbNiwGn3Ky8sLP7do0UJin0gkqpFtm56ejn79+mHy5MkSNcB1dXWRmpoqfH89+/jFixfYtWsXAgICaj33qiD3rVu3cPTo0Vozg7W1tWFkZAQAMDExwfPnz+Hq6orFixdDLBbXO3519Z2nsrIyUlJSkJiYiEOHDsHHxwd+fn44d+5cvaVUANQoxyIlJVXjoUX1WuCvl6dpDGtra4nzrCpz0liNucb1OXDggHAOVfM/evQoYmNjsXz5cgCVD2rKy8shIyODjRs3Yvz48U2aIxERERFRUzHQTURERO+1f1OQG6gsUeHo6IiwsDDMnDlTIhCan5+PqKgouLu7AwBsbW3h6emJH3/8UajFbWdnhyNHjuDkyZOYPXu2cGy3bt2QmZkpBInf1JUrV2Bvbw83N7ca9b1lZGTq7f/HH39EcXExRo8eXWNfVZA7KysLCQkJaN26daPmU5WVXVRU1OD4TSEjIwMHBwc4ODjA19cXampqOHr0KIYNGwZZWVmJDOn6aGpqIi0tTWJbamqqEGw2NjaGgoIC4uPjay3nIisrCwAS4ykoKNR5nmfOnJH4npycDGNjY4ns9cZITk4WsrNfvXqFCxcuwMPDAwCgr69fo/3p06cl5vjzzz8jODgYp06dQtu2bYXtly5dQlFRkXBfJycnQ0lJSaLOeXVNWWsiIiIier8x0E1ERETvrX9bkLvK2rVr0atXLzg5OWHx4sUwMDDAlStXMHfuXHTs2BE+Pj4AKktQtGrVCjt37sT+/fsBVAa658yZA5FIhN69ewt9+vj4wMXFBXp6ehg+fDikpKRw6dIlpKWlYfHixY2aV1paGuzt7eHk5IRZs2YhPz8fQGWwuaqWdH3Cw8MxZMiQGkHs0tJSDB8+HCkpKdi/fz/KysqEvtXV1YVgLwA8ffoU+fn5KC8vR1ZWFgICAtCxY8c6a5q/if379+PGjRvo27cvWrVqhQMHDqC8vBwmJiYAALFYjDNnziAnJwdKSkr1Zo7b29vj+++/x7Zt22BjY4MdO3YgLS0NVlZWACqz6efPn4958+ZBVlYWvXv3xoMHD3DlyhVMmDABWlpaUFBQQFxcHNq1awd5eXmoqqrWOV5ubi5mzZqFr7/+GikpKVizZg1CQkKavAbr1q2DsbExTE1NsXLlSjx58qTerOzX1//8+fOQkpKCubm5xPaSkhJMmDABXl5eyMnJga+vLzw8PCAlVXtFRbFYjGPHjmHkyJGQk5ODhoZGk8+FiIiIiN4PrNFNRERE76V/a5AbqMzyPXfuHAwNDTFixAjo6+tjwIAB6NixI06ePAklJSUAlSUn+vTpA5FIhI8++ghAZfBbRUUF1tbWEmU0nJycsH//fhw6dAjdu3dHz549sXLlylqzc+uyZ88ePHjwADt27ICOjo7w6d69e4PHZmZm4sSJE5gwYUKNfXfu3EFsbCz++OMPWFpaSvR96tQpibbjxo2Djo4O2rVrB1dXV5iZmeG3336DjMzby99QU1NDTEwM7O3tYWpqig0bNiA6OhpmZmYAgDlz5kBaWhqdO3eGpqYmcnNz6+zLyckJ3t7emDdvHrp3747nz5/jq6++kmjj7e2N2bNnw8fHB6ampvjiiy+EmtkyMjJYvXo1fvjhB+jq6mLw4MH1zv2rr75CUVERevToAXd3d8yYMaNJLwutEhQUhKCgIFhYWODEiROIjY19K0Hmfv36wdjYGH379sUXX3yBTz/9FH5+fnW2DwgIQE5ODjp06NCohylERERE9P4SVTT0piMiIiKiZubly5e4efMmDAwMJOpPN2e+vr5YsWIFDh8+jJ49e77r6RARvXP/xd/1RERE9OZYuoSIiIioGfD394dYLEZycjJ69OhRZ6kHIiIiIiKi9xED3URERETNxLhx4971FIiIiIiIiP6VmApERERERERERERERM0aA91ERERERERERERE1Kwx0E1EREREREREREREzRoD3URERERERERERETUrDHQTURERERERERERETNGgPdRERERERERERERNSsMdBNRERERERERERERM0aA91ERERERP8BdnZ28PT0/Et95OTkQCQSITU19a3MqbqxY8diyJAh9bZpyjmMGTMGgYGBwvfCwkJ89tlnUFFRgUgkwtOnTyEWi7Fq1ao3n3QzkJiYKJwvAMTFxcHS0hLl5eXvdmJERERE/zCZdz0BIiIion/SnW+P1rkv9OQOfH9sK+b2HYcZvUc3ue+C4kKM3r0AmQ9zsHPkMljpdkLbQPs3muft27fh6+uLuLg4PHz4EDo6OhgyZAh8fHzQunXrN+rzr7p06RKCgoJw4sQJPHz4EGKxGFOmTMGMGTPqPc7Ozg5JSUk1tjs7O+PXX39FaWkpvLy8cODAAdy4cQOqqqpwcHBAUFAQdHV1hfYikUj4WVpaGrq6uhg+fDiWLl0KOTm5Osf38/PDvn373lrw1s7ODpaWlv9IADUnJwcGBga4ePEiLC0t//bx3obr16/DysoK0tLSQvC1sWJiYtCiRQvhu1gshqenZ43g96VLl3DgwAGsX79e2BYZGYnjx4/j1KlT0NDQgKqqKs6dO4eWLVv+ldOp4Z+8/m/ik08+gbe3N6KiojBmzJh3PR0iIiKifwwzuomIiIj+vxm9R2Nu33H4/thWhJ7c0eTjleQUseOLIJhoiPHlrnm4ePfqG83jxo0bsLa2RlZWFqKjo3H9+nVs2LAB8fHxsLGxwePHj9+o37/qwoUL0NLSwo4dO3DlyhUsWrQICxcuxNq1a+s9LiYmBnl5ecInLS0N0tLS+PzzzwFUZuKmpKTA29sbKSkpiImJQWZmJj799NMafW3duhV5eXm4efMmwsLCsH37dixevPhvOd+/oqKiAq9evXrX0/jHlZaWwtXVFX369Hmj49XV1aGsrNxguzVr1uDzzz+HkpKSsC07OxumpqYwNzeHtrY2RCIRNDU1oaio+EZzac7Gjh2L1atXv+tpEBEREf2jGOgmIiIiquZtB7vfhLu7O2RlZXHo0CHY2tpCT08PAwYMwJEjR3Dnzh0sWrQIa9euhbm5uXDMvn37IBKJsGHDBmGbg4MDvLy8hO8///wzunXrBnl5eRgaGsLf318iGCsSibB582YMHToUioqKMDY2RmxsrLB//PjxCA0Nha2tLQwNDTF69GiMGzcOMTEx9Z6Puro6tLW1hc/hw4ehqKgoBLpVVVVx+PBhjBgxAiYmJujZsyfWrl2LCxcuIDc3V6IvNTU1aGtro3379nBxccHgwYORkpLSpPWtKqGxfPly6OjooHXr1nB3d0dpaanQJiwsDMbGxpCXl0ebNm0wfPhw4dikpCSEhoZCJBJBJBIhJydHKB/x22+/4YMPPoCcnBxOnDhRa7kOT09P2NnZCd/Ly8uxbNkyGBkZQU5ODnp6eliyZAkAwMDAAABgZWUFkUgkcVxtXr16BQ8PD6iqqkJDQwPe3t6oqKgQ9ovFYgQGBmL8+PFQVlaGnp4eNm7cWKOfq1evolevXpCXl4e5uXmtGfm18fLyQqdOnTBixIg62/j7+0NTUxMqKiqYMmUKSkpKhH3VS5fY2dnh1q1bmDlzprDWAFBWVoY9e/Zg0KBBEseFhITg2LFjEuv0eumShu5xAEhLS8OAAQOgpKSENm3aYMyYMXj48CGAuq9/REQE1NTUJPqp+jdZxc/PD5aWlti+fTvEYjFUVVUxcuRIPH/+XGhTXl6OpUuXwsDAAAoKCrCwsMCePXsk+j1w4AA6duwIBQUFfPzxx8jJyamxxoMGDcL58+eRnZ1d53UgIiIi+q9hoJuIiIjoNW8z2N1Ujx8/xsGDBzF16lQoKChI7NPW1saoUaOwe/du2NraIj09HQ8ePAAAJCUlQUNDA4mJiQAqM2tPnz4tBPyOHz+Or776CjNmzEB6ejp++OEHRERECAHVKv7+/hgxYgQuX74MZ2dnjBo1qt4M8mfPnkFdXb1J5xgeHo6RI0fWW1Li2bNnEIlENYKH1V27dg1Hjx7Fhx9+2KTxASAhIQHZ2dlISEhAZGQkIiIiEBERAQA4f/48pk+fjoCAAGRmZiIuLg59+/YFAISGhsLGxgaTJk0SMtTbt28v9LtgwQIEBQUhIyMDXbt2bdRcFi5ciKCgIHh7eyM9PR07d+5EmzZtAABnz54FABw5cgR5eXkNPlSIjIyEjIwMzp49i9DQUKxYsQKbN2+WaBMSEgJra2tcvHgRU6dOxTfffIPMzEyJNnPnzsXs2bNx8eJF2NjYYNCgQXj06FG9Yx89ehQ//vgj1q1bV2eb+Ph4ZGRkIDExEdHR0YiJiYG/v3+tbWNiYtCuXTsEBAQIaw0Aly9fxrNnz2BtbS3RdtKkSbCxsWlwneq7x58+fQp7e3tYWVnh/PnziIuLw71794TAfUPXvyHZ2dnYt28f9u/fj/379yMpKQlBQUHC/qVLl2Lbtm3YsGEDrly5gpkzZ2L06NHCg4bbt29j2LBhGDRoEFJTUzFx4kQsWLCgxjh6enpo06YNjh8/3ui5ERERETV3DHQTERER1eJtBbubKisrCxUVFTA1Na11v6mpKZ48eQItLS2oq6sLAbDExETMnj1b+H727FmUlpaiV69eACqDewsWLICbmxsMDQ3h6OiI7777Dj/88INE/2PHjoWrqyuMjIwQGBiIgoICIdj6ulOnTmH37t2YPHlyo8/v7NmzSEtLw8SJE+ts8/LlS8yfPx+urq5QUVGR2Ofq6golJSXIy8vDxMQEZmZmWLhwYaPHr9KqVSusXbsWnTp1gouLCwYOHIj4+HgAQG5uLlq2bAkXFxfo6+vDysoK06dPB1CZfS4rKwtFRUUhQ11aWlroNyAgAI6OjujQoUOjHgA8f/4coaGhWLZsGdzc3NChQwd89NFHwvpoamoCAFq3bg1tbe0G+2zfvj1WrlwJExMTjBo1CtOmTcPKlSsl2jg7O2Pq1KkwMjLC/PnzoaGhgYSEBIk2Hh4e+Oyzz2Bqaor169dDVVUV4eHhdY776NEjjB07FhERETWuWXWysrLYsmULzMzMMHDgQAQEBGD16tW1vjhRXV0d0tLSUFZWFtYaAG7dugVpaWloaWlJtFVUVISsrGyD61TfPb527VpYWVkhMDAQnTp1gpWVFbZs2YKEhARcu3atwevfkPLyckRERMDc3Bx9+vTBmDFjhPuuuLgYgYGB2LJlC5ycnGBoaIixY8di9OjRwr/T9evXo0OHDggJCRGu8dixY2sdS1dXF7du3Wr03IiIiIiaOwa6iYiIiOrwNoLdb6p6uYnayMnJoW/fvkhMTMTTp0+Rnp6OqVOnori4GFevXkVSUhK6d+8u1Ce+dOkSAgICoKSkJHyqslILCwuFfqtnIbds2RIqKiq4f/9+jfHT0tIwePBg+Pr6on///gAqA8TV+w8MDKxxXHh4OLp06YIePXrUel6lpaUYMWIEKioqJF40WGXlypVITU3FpUuXsH//fly7dk144V5jxq9iZmYmEaDU0dERztPR0RH6+vowNDTEmDFjEBUVJbFG9ameZdwYGRkZKC4uRr9+/Rp9zPHjxyXOMyoqStjXs2dPiXIZNjY2yMrKQllZmbCt+jUWiUTQ1taucY1tbGyEn2VkZGBtbY2MjAwAlWtXNfaAAQMAAJMmTcKXX34pZL7XxcLCQqJmto2NDQoKCnD79u1Gn39RURHk5OQkzrMp6rvHL126hISEBIn17dSpEwC8lTIgYrFYogZ59fvu+vXrKCwshKOjo8T427ZtE8bOyMio8RcM1a9VdQoKCo2+b4mIiIj+C2Te9QSIiIiI/s1m9B4NAPj+2FaJ738XIyMjiEQiZGRkYOjQoTX2Z2RkQFNTE2pqarCzs8PGjRtx/PhxWFlZQUVFRQh+JyUlwdbWVjiuoKAA/v7+GDZsWI0+5eXlhZ9btGghsU8kEtXItk1PT0e/fv0wefJkiRrgurq6SE1NFb6/nlX74sUL7Nq1CwEBAbWee1WQ+9atWzh69GitmcHa2towMjICAJiYmOD58+dwdXXF4sWLIRaL6x2/uvrOU1lZGSkpKUhMTMShQ4fg4+MDPz8/nDt3rt5SKgBqlGORkpKq8dCiei3w18vTNIa1tbXEeVaVOWmsxlzj+hw4cEA4h6r5Hz16FLGxsVi+fDmAygc15eXlkJGRwcaNGzF+/PgmzbE+GhoaKCwsRElJCWRlZZt8fH3nX1BQgEGDBiE4OLjGcTo6OnX22dB1buzYAPDrr7+ibdu2Eu3k5OTqHLsujx8/Fv4igIiIiOh9wEA3ERERvVcKigubnGn9Twa7W7duDUdHR4SFhWHmzJkSgdD8/HxERUXB3d0dAGBrawtPT0/8+OOPQi1uOzs7HDlyBCdPnsTs2bOFY7t164bMzEwhSPymrly5Ant7e7i5udWo7y0jI1Nv/z/++COKi4sxenTN9asKcmdlZSEhIQGtW7du1HyqsrKLiooaHL8pZGRk4ODgAAcHB/j6+kJNTQ1Hjx7FsGHDICsrK5EhXR9NTU2kpaVJbEtNTRUCnsbGxlBQUEB8fHyt5VyqArnVx1NQUKjzPM+cOSPxPTk5GcbGxk0qr1F1XFV29qtXr3DhwgV4eHgAAPT19Wu0P336tMQcf/75ZwQHB+PUqVMSQdtLly6hqKhIuK+Tk5OhpKRUZ53r2tba0tISQOUDl6qf35Zu3brhp59+glgshoxM7f9Vqm1OmpqaeP78OV68eCE87Kj+MKIxOnfuDDk5OeTm5ko8pKrO1NS0xsszk5OTa7R7+fIlsrOzYWVl1aQ5EBERETVnLF1CRERE75XRuxegoLjpf87/V8uYNMXatWtRXFwMJycnHDt2DLdv30ZcXBwcHR3RsWNH+Pj4AKgswdCqVSvs3LlTItC9b98+FBcXo3fv3kKfPj4+2LZtG/z9/XHlyhVkZGRg165dEhnZDUlLS8PHH3+M/v37Y9asWcjPz0d+fr7wQsyGhIeHY8iQITWC2KWlpRg+fDjOnz+PqKgolJWVCX2XlJRItH369Cny8/Nx9+5dJCUlISAgAB07dqyzpvmb2L9/P1avXo3U1FTcunUL27ZtQ3l5OUxMTABUlp84c+YMcnJy8PDhw3qzoe3t7XH+/Hls27YNWVlZ8PX1lQh8y8vLY/78+Zg3b55QoiI5OVmoh62lpQUFBQXhpYjPnj2rd+65ubmYNWsWMjMzER0djTVr1mDGjBlNXoN169Zh7969uHr1Ktzd3fHkyZN6s7JNTU1hbm4ufNq2bQspKSmYm5ujVatWQruSkhJMmDAB6enpOHDgAHx9feHh4QEpqdr/WyIWi3Hs2DHcuXMHDx8+BFAZVO7WrRtOnDjR5PNqiLu7Ox4/fgxXV1ecO3cO2dnZOHjwIMaNGycEt2u7/h9++CEUFRXx7bffIjs7Gzt37hRebtpYysrKmDNnDmbOnInIyEhkZ2cjJSUFa9asQWRkJABgypQpyMrKwty5c5GZmVnnOMnJyZCTk6uzrAkRERHRfxEzuomIiOi9cu6PtIYb1WEZ7LEMW97ibGpnbGyMc+fOwc/PDyNGjMD9+/dRUVGBYcOGYfv27UKNY5FIhD59+uDXX3/FRx99BKAy+K2iogITExOJMhpOTk7Yv38/AgICEBwcjBYtWqBTp071vhTydXv27MGDBw+wY8cO7Njxv2C/vr4+cnJy6j02MzMTJ06cwKFDh2rsu3PnjpCl+nqGbkJCghDEB4Bx48YJ566trY2+ffsiMDCwzuzbN6GmpoaYmBj4+fnh5cuXMDY2RnR0NMzMzAAAc+bMgZubGzp37oyioiLcvHmzzr6cnJzg7e2NefPm4eXLlxg/fjy++uor/P7770Ibb29vyMjIwMfHB3fv3oWOjg6mTJkCoDKzfPXq1QgICICPjw/69OmDxMTEOsf76quvUFRUhB49ekBaWhozZsxo0stCqwQFBSEoKAipqakwMjJCbGwsNDQ0mtzP6/r16wdjY2P07dsXxcXFcHV1hZ+fX53tAwIC8PXXX6NDhw4oLi4WyoNMnDgR27ZtE7LM3xZdXV2cPHkS8+fPR//+/VFcXAx9fX188sknQjC+tusvFouxY8cOzJ07F5s2bUK/fv3g5+fX5LX/7rvvoKmpiaVLl+LGjRtQU1NDt27d8O233wIA9PT08NNPP2HmzJlYs2YNevTogcDAwBoPIaKjozFq1CiJeuhERERE/3WiiobedERERETUzLx8+RI3b96EgYGBRP3p5szX1xcrVqzA4cOH0bNnz3c9HaJ3qqioCCYmJti9ezezll/z8OFDmJiY4Pz58zAwMHjX0/lb/Rd/1xMREdGbY0Y3ERERUTPg7+8PsViM5ORk9OjRo85SD0TvAwUFBWzbtk0oZ0L/k5OTg7CwsP98kJuIiIjodczoJiIiov8cZvkREf338Xc9ERERVcdUICIiIiIiIiIiIiJq1hjoJiIiIiIiIiIiIqJmjYFuIiIi+s9ihTYiov8u/o4nIiKi6hjoJiIiov+cFi1aAAAKCwvf8UyIiOjvUvU7vup3PhEREb3fZN71BIiIiIjeNmlpaaipqeH+/fsAAEVFRYhEonc8KyIiehsqKipQWFiI+/fvQ01NDdLS0u96SkRERPQvIKrg33sRERHRf1BFRQXy8/Px9OnTdz0VIiL6G6ipqUFbW5sPMomIiAgAA91ERET0H1dWVobS0tJ3PQ0iInqLWrRowUxuIiIiksBANxERERERERERERE1a3wZJRERERERERERERE1awx0ExEREREREREREVGzxkA3ERERERERERERETVrDHQTERERERERERERUbPGQDcRERERERERERERNWsMdBMRERERERERERFRs8ZANxERERERERERERE1a/8Px5SuJdJaSOEAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_metrics(metrics_df, figsize=(18, 5), ylim=(0, 0.5))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "180e447c-dc1e-4db2-ae1a-50b5e849ae3d", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Experiment 3" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "bc2fa32c-8aa0-4af6-9431-1284972839c6", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Qwen2-0.5B(flash-attn:false): {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.2596076617917167, 'bleu_scores': {'bleu': 0.04409425333473452, 'precisions': [0.33363639542609363, 0.07631808913486746, 0.023275240156433914, 0.007970813340207003], 'brevity_penalty': 0.9458179749826672, 'length_ratio': 0.9472341835044716, 'translation_length': 28597, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3092428110430848, 'rouge2': 0.0866672692806888, 'rougeL': 0.2522299230834305, 'rougeLsum': 0.2525527712447725}}\n", + "Qwen2-0.5B(flash-attn:true): {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.2604145257072387, 'bleu_scores': {'bleu': 0.046012982792608105, 'precisions': [0.3368365553602812, 0.07779038693853645, 0.02424219286859586, 0.009012601690859786], 'brevity_penalty': 0.9406727862276781, 'length_ratio': 0.9423650215303081, 'translation_length': 28450, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.30974071220443333, 'rouge2': 0.08881206363108668, 'rougeL': 0.2545047298857762, 'rougeLsum': 0.2548786273221506}}\n", + "Qwen2-0.5B(finetuned)(flash-attn:false): {'accuracy': 0.00264783759929391, 'correct_ids': [147, 199, 533], 'meteor': 0.29068031845703574, 'bleu_scores': {'bleu': 0.060746076799314715, 'precisions': [0.33972489966727265, 0.09057815845824412, 0.03312637097074023, 0.015400729304057724], 'brevity_penalty': 0.9650542580679092, 'length_ratio': 0.9656508777740974, 'translation_length': 29153, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3200549791044618, 'rouge2': 0.1064928404938012, 'rougeL': 0.2656289309387257, 'rougeLsum': 0.26549973587614306}}\n", + "Qwen2-0.5B(finetuned)(flash-attn:true): {'accuracy': 0.00264783759929391, 'correct_ids': [147, 199, 533], 'meteor': 0.2875255734599462, 'bleu_scores': {'bleu': 0.0641508398737298, 'precisions': [0.33414174077068176, 0.09091227455787078, 0.03518174195194867, 0.01695559610705596], 'brevity_penalty': 0.983232969050409, 'length_ratio': 0.9833719774759854, 'translation_length': 29688, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3199368891131209, 'rouge2': 0.10646415929992675, 'rougeL': 0.2655909857054649, 'rougeLsum': 0.2656870813605584}}\n", + "Qwen2-1.5B(flash-attn:false): {'accuracy': 0.00088261253309797, 'correct_ids': [77], 'meteor': 0.3140903672178899, 'bleu_scores': {'bleu': 0.07105708852581612, 'precisions': [0.3965299918439979, 0.11779729886614296, 0.045886780237122164, 0.01916228590808886], 'brevity_penalty': 0.8876072487625101, 'length_ratio': 0.8934746604836038, 'translation_length': 26974, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3711972499986497, 'rouge2': 0.13222528838162723, 'rougeL': 0.31603590215154154, 'rougeLsum': 0.3160062592092049}}\n", + "Qwen2-1.5B(flash-attn:true): {'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.31196189387338386, 'bleu_scores': {'bleu': 0.07269636467609625, 'precisions': [0.3994544911074578, 0.11993289376146073, 0.046890303623898136, 0.020746000513303105], 'brevity_penalty': 0.8798465412731852, 'length_ratio': 0.8865187148062272, 'translation_length': 26764, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.36696214278654415, 'rouge2': 0.12870506511885213, 'rougeL': 0.3105268126934275, 'rougeLsum': 0.3104771178128985}}\n", + "Qwen2-1.5B(finetuned)(flash-attn:false): {'accuracy': 0.00264783759929391, 'correct_ids': [77, 170, 533], 'meteor': 0.3490204861480539, 'bleu_scores': {'bleu': 0.08451008426675863, 'precisions': [0.34023887851575235, 0.11221172718351324, 0.050447931888938306, 0.026483085740637045], 'brevity_penalty': 1.0, 'length_ratio': 1.1176217290493542, 'translation_length': 33741, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3766697667924311, 'rouge2': 0.15045265594444793, 'rougeL': 0.32089502499900974, 'rougeLsum': 0.32082614116465435}}\n", + "Qwen2-1.5B(finetuned)(flash-attn:true): {'accuracy': 0.00264783759929391, 'correct_ids': [147, 170, 533], 'meteor': 0.3494130868780632, 'bleu_scores': {'bleu': 0.09464090787225404, 'precisions': [0.3858167437495811, 0.12861870754224003, 0.05755213055303717, 0.02944845575171058], 'brevity_penalty': 0.988272274882654, 'length_ratio': 0.988340510102683, 'translation_length': 29838, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.37576306572007256, 'rouge2': 0.1507218344948884, 'rougeL': 0.3188000390638358, 'rougeLsum': 0.3186826481428405}}\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
modelaccuracymeteorbleu_1rouge_lall_metrics
0Qwen2-0.5B(flash-attn:false)0.0000000.2596080.0440940.252230{'accuracy': 0.0, 'correct_ids': [], 'meteor':...
1Qwen2-0.5B(flash-attn:true)0.0000000.2604150.0460130.254505{'accuracy': 0.0, 'correct_ids': [], 'meteor':...
2Qwen2-0.5B(finetuned)(flash-attn:false)0.0026480.2906800.0607460.265629{'accuracy': 0.00264783759929391, 'correct_ids...
3Qwen2-0.5B(finetuned)(flash-attn:true)0.0026480.2875260.0641510.265591{'accuracy': 0.00264783759929391, 'correct_ids...
4Qwen2-1.5B(flash-attn:false)0.0008830.3140900.0710570.316036{'accuracy': 0.00088261253309797, 'correct_ids...
5Qwen2-1.5B(flash-attn:true)0.0000000.3119620.0726960.310527{'accuracy': 0.0, 'correct_ids': [], 'meteor':...
6Qwen2-1.5B(finetuned)(flash-attn:false)0.0026480.3490200.0845100.320895{'accuracy': 0.00264783759929391, 'correct_ids...
7Qwen2-1.5B(finetuned)(flash-attn:true)0.0026480.3494130.0946410.318800{'accuracy': 0.00264783759929391, 'correct_ids...
\n", + "
" + ], + "text/plain": [ + " model accuracy meteor bleu_1 \\\n", + "0 Qwen2-0.5B(flash-attn:false) 0.000000 0.259608 0.044094 \n", + "1 Qwen2-0.5B(flash-attn:true) 0.000000 0.260415 0.046013 \n", + "2 Qwen2-0.5B(finetuned)(flash-attn:false) 0.002648 0.290680 0.060746 \n", + "3 Qwen2-0.5B(finetuned)(flash-attn:true) 0.002648 0.287526 0.064151 \n", + "4 Qwen2-1.5B(flash-attn:false) 0.000883 0.314090 0.071057 \n", + "5 Qwen2-1.5B(flash-attn:true) 0.000000 0.311962 0.072696 \n", + "6 Qwen2-1.5B(finetuned)(flash-attn:false) 0.002648 0.349020 0.084510 \n", + "7 Qwen2-1.5B(finetuned)(flash-attn:true) 0.002648 0.349413 0.094641 \n", + "\n", + " rouge_l all_metrics \n", + "0 0.252230 {'accuracy': 0.0, 'correct_ids': [], 'meteor':... \n", + "1 0.254505 {'accuracy': 0.0, 'correct_ids': [], 'meteor':... \n", + "2 0.265629 {'accuracy': 0.00264783759929391, 'correct_ids... \n", + "3 0.265591 {'accuracy': 0.00264783759929391, 'correct_ids... \n", + "4 0.316036 {'accuracy': 0.00088261253309797, 'correct_ids... \n", + "5 0.310527 {'accuracy': 0.0, 'correct_ids': [], 'meteor':... \n", + "6 0.320895 {'accuracy': 0.00264783759929391, 'correct_ids... \n", + "7 0.318800 {'accuracy': 0.00264783759929391, 'correct_ids... " + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.read_csv(\"results/experiment-3-results.csv\")\n", + "metrics_df = get_metrics(df)\n", + "metrics_df" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "613aecd3-5cc1-4401-90fa-09342889895d", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_utils.py:302: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n", + " barplot.set_xticklabels([\"METEOR\", \"BLEU-1\", \"ROUGE-L\"])\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABI0AAAJzCAYAAACCgONnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdeVxVdf7H8ReLoqBiSKKiJYrkkoojyghqLig6Nrg1GpOhWFZuZZSl5VJpqaXFZCqlmVK59XOdHFcUx4U0TXMj15RcQFEBBWW9vz/unNu93P0Cci98no+Hj9/lnPP93nP6jZyP7/M936+TSqVSIYQQQgghhBBCCCGEFufyPgEhhBBCCCGEEEIIYX8kNBJCCCGEEEIIIYQQeiQ0EkIIIYQQQgghhBB6JDQSQgghhBBCCCGEEHokNBJCCCGEEEIIIYQQeiQ0EkIIIYQQQgghhBB6JDQSQgghhBBCCCGEEHokNBJCCCGEEEIIIYQQeiQ0EkIIIYQQQgghhBB6JDQSQgghhBBCCCGEEHrsIjRasGABjRs3plq1agQHB3Po0CGL2q1atQonJycGDBigs12lUjFt2jTq169P9erVCQsL49y5c2Vw5kIIIYQQQgghhBAVU7mHRqtXryYmJobp06fzyy+/0LZtW8LDw7lx44bJdpcuXeLNN9+kS5cuevs+/vhjPv/8c+Li4jh48CAeHh6Eh4fz4MGDsroMIYQQQgghhBBCiArFSaVSqcrzBIKDg+nQoQNffPEFAEVFRTRq1Ijx48czadIkg20KCwvp2rUrI0eOZO/evWRkZLBhwwZAPcqoQYMGvPHGG7z55psAZGZm4uPjw7Jly3j22WcfynUJIYQQQgghhBBCODLX8vzyvLw8jhw5wuTJkzXbnJ2dCQsLIykpyWi7Dz74gLp16/LCCy+wd+9enX2///47qamphIWFabZ5enoSHBxMUlKSwdAoNzeX3Nxczc9FRUXcvn2bOnXq4OTkVJJLFEIIIYQQQgghhLAbKpWKu3fv0qBBA5ydTb+AVq6hUXp6OoWFhfj4+Ohs9/Hx4bfffjPYZt++fXz99dccO3bM4P7U1FRNH8X7VPYVN2vWLN5//30rz14IIYQQQgghhBDCMf3xxx80bNjQ5DHlGhpZ6+7duzz//PMsXrwYb2/vUut38uTJxMTEaH7OzMzkscce448//qBWrVql9j1CCCGEEEIIIYQQ5SkrK4tGjRpRs2ZNs8eWa2jk7e2Ni4sLaWlpOtvT0tKoV6+e3vEXLlzg0qVL/P3vf9dsKyoqAsDV1ZUzZ85o2qWlpVG/fn2dPgMDAw2eh5ubG25ubnrba9WqJaGREEIIIYQQQgghKhxLpuMp19XTqlatSvv27UlISNBsKyoqIiEhgU6dOukd37x5c06cOMGxY8c0fyIiIujevTvHjh2jUaNG+Pn5Ua9ePZ0+s7KyOHjwoME+hRBCCCGEEEIIIYS+cn89LSYmhuHDhxMUFETHjh2JjY0lOzub6OhoAKKiovD19WXWrFlUq1aNJ598Uqd97dq1AXS2T5gwgZkzZ9KsWTP8/PyYOnUqDRo0YMCAAQ/rsoQQQgghhBBCCCEcWrmHRkOHDuXmzZtMmzaN1NRUAgMD2bp1q2Yi65SUFLOzeRf31ltvkZ2dzUsvvURGRgadO3dm69atVKtWrSwuQQghhBBCCCGEEKLCcVKpVKryPgl7k5WVhaenJ5mZmTKnkRBCCCGEEEIIISoMazKPcp3TSAghhBBCCCGEEELYJwmNhBBCCCGEEEIIIYQeCY2EEEIIIYQQQgghhB4JjYQQQgghhBBCCCGEHgmNhBBCCCGEEEIIIYQeCY2EEEIIIYQQQgghhB4JjYQQQgghhBBCCCGEHgmNhBBCCCGEEEIIIYQeCY2EEEIIIYQQQgghhB4JjYQQQgghhBBCCCGEHgmNhBBCCCGEEEIIIYQeCY2EEEIIIYQQQgghhB4JjYQQQgghhBBCCCGEHgmNhBBCCCGEEEIIIYQeCY2EEEIIIYQQQgghhB4JjYQQQgghhBBCCCGEHgmNhBBCCCGEEEIIIYQeCY2EEEIIIYQQQgghhB4JjYQQQgghhBBCCCGEHgmNhBBCCCGEEEIIIYQeCY2EEEIIIYQQQgghhB4JjYQQQgghhBBCCCGEHgmNhBDCgSxYsIDGjRtTrVo1goODOXTokNFj161bR1BQELVr18bDw4PAwEC+/fZbnWNGjBiBk5OTzp8+ffqU9WUIIYQQQgghHICERkIIYYI9hTSrV68mJiaG6dOn88svv9C2bVvCw8O5ceOGweO9vLx49913SUpK4vjx40RHRxMdHc22bdt0juvTpw/Xr1/X/Fm5cqVF5yOEEEIIIYSo2JxUKpWqvE/C3mRlZeHp6UlmZia1atUq79MRQpST1atXExUVRVxcHMHBwcTGxvLDDz9w5swZ6tatq3d8YmIid+7coXnz5lStWpUff/yRN954g82bNxMeHg6oQ6O0tDS++eYbTTs3NzceeeQRs+cTHBxMhw4d+OKLLwAoKiqiUaNGjB8/nkmTJll0TX/5y1/o168fM2bM0JxPRkYGGzZssKi9EEIIIYQQwrFZk3nISCMhhDDi008/ZdSoUURHR9OyZUvi4uJwd3dn6dKlBo/v1q0bAwcOpEWLFjRt2pTXXnuNNm3asG/fPp3j3NzcqFevnuaPJYFRXl4eR44cISwsTLPN2dmZsLAwkpKSzLZXqVQkJCRw5swZunbtqrMvMTGRunXr8sQTTzB69Ghu3bpltj8hhBBCCCFExSehkRBCGGBvIU16ejqFhYX4+PjobPfx8SE1NdVou8zMTGrUqEHVqlXp168f8+fPp1evXpr9ffr0IT4+noSEBObMmcOePXvo27cvhYWFZs9JCCGEEEIIUbG5lvcJCCGEPTIV0vz2229G22VmZuLr60tubi4uLi4sXLhQL6QZNGgQfn5+XLhwgXfeeYe+ffuSlJSEi4tLqV9HzZo1OXbsGPfu3SMhIYGYmBiaNGlCt27dAHj22Wc1x7Zu3Zo2bdrQtGlTEhMT6dmzZ6mfjxBCCCGEEMJxSGgkhBClqKxCGm9vb1xcXEhLS9PZnpaWRr169Yy2c3Z2xt/fH4DAwECSk5OZNWuW5nyKa9KkCd7e3pw/f15CIyGEEEIIISo5eT1NCCEMKGlIExgYyBtvvMEzzzzDrFmzjB6vHdKYUrVqVdq3b09CQoJmW1FREQkJCXTq1MnCq1K3yc3NNbr/ypUr3Lp1i/r161vcpxBCCCGEEKJiktBICCEMsMeQJiYmhsWLF7N8+XKSk5MZPXo02dnZREdHAxAVFcXkyZM1x8+aNYsdO3Zw8eJFkpOTmTdvHt9++y3Dhg0D4N69e0ycOJGffvqJS5cukZCQQP/+/fH399es9iaEEEIIIYSovOT1NCGEMCImJobhw4cTFBREx44diY2N1QtpfH19NSOJZs2aRVBQEE2bNiU3N5f//Oc/fPvttyxatAhQhzTvv/8+gwcPpl69ely4cIG33nrL4pBm6NCh3Lx5k2nTppGamkpgYCBbt27VzLuUkpKCs/OfzwKys7MZM2YMV65coXr16jRv3pzvvvuOoUOHAuDi4sLx48dZvnw5GRkZNGjQgN69ezNjxgzc3NxK9b+lEEIIIYQQwvE4qVQqVXmfhL3JysrC09OTzMxMatWqVd6nI4QoR1988QWffPKJJqT5/PPPCQ4OBqBbt240btyYZcuWATBlyhRWr16tE9K89tprmpDm/v37DBgwgKNHj+qFNMUn3BZCCCGEEEKIsmBN5iGhkQESGgkhhBBCCCGEEKIisibzkDmNhBBCCCGEEEIIIYQeCY2EEEIIIYQQQgghhB4JjYQQQgghhBBCCCGEHrsIjRYsWEDjxo2pVq0awcHBHDp0yOix69atIygoiNq1a+Ph4UFgYCDffvutzjEjRozAyclJ50+fPn3K+jKEEEIIIYQQQgghKgzX8j6B1atXExMTQ1xcHMHBwcTGxhIeHs6ZM2eoW7eu3vFeXl68++67NG/enKpVq/Ljjz8SHR1N3bp1dZas7tOnD998843mZ1k+WgghhBBCCCGEEMJy5b56WnBwMB06dOCLL74AoKioiEaNGjF+/HgmTZpkUR9/+ctf6NevHzNmzADUI40yMjLYsGGDTeckq6cJIYQQQgghhBCiInKY1dPy8vI4cuQIYWFhmm3Ozs6EhYWRlJRktr1KpSIhIYEzZ87QtWtXnX2JiYnUrVuXJ554gtGjR3Pr1i2j/eTm5pKVlaXzRwghhBBCCCGEEKIyK9fX09LT0yksLMTHx0dnu4+PD7/99pvRdpmZmfj6+pKbm4uLiwsLFy6kV69emv19+vRh0KBB+Pn5ceHCBd555x369u1LUlISLi4uev3NmjWL999/v/QuTAghhBBCCCGEEMLBlfucRraoWbMmx44d4969eyQkJBATE0OTJk3o1q0bAM8++6zm2NatW9OmTRuaNm1KYmIiPXv21Otv8uTJxMTEaH7OysqiUaNGZX4dQgj7kfzhLoPb07Ju0v1j9e+Uzs060KVZB2b9Z6Fm/+mZCWb7/nznN8QlfgfAK92GkV+Yz9d7VwPwRvhLvNBlqF6b1fn/JSkpie3bt9O6dWsGDRpk/NyTk1mzZg0A06dP19mnBOIvvvgivr6+BturVCqWL1/O5cuX6dmzJ507d9bsy8rK4rPPPtMcJ4QQQgghhKg8yvX1NG9vb1xcXEhLS9PZnpaWRr169Yy2c3Z2xt/fn8DAQN544w2eeeYZZs2aZfT4Jk2a4O3tzfnz5w3ud3Nzo1atWjp/hBAC4KMfFwAwLWICXw2fzfMhgzk1Y6dm/9nUi2b7UAKjvZP+j1fDonkj/CX2vP0DAPO2fWW03YEDBwBo27atyf5btGih+Xz37l2Dnxs0aGC0vZOTE08//TQACQm6Idjp06cBePLJJ02egxBCCCGEEKLiKdfQqGrVqrRv317nHylFRUUkJCTQqVMni/spKioiNzfX6P4rV65w69Yt6tevX6LzFUJUPjtO7wXg723/nHvNycmJKU+PB+D0tXMm2xcUFmo+16nxiObzozW9DB6j7d69ewBWBdk5OTkGPzs5OZls5+xs+Hag9FGnTh2Lz0EIIYQQQghRMZRraAQQExPD4sWLWb58OcnJyYwePZrs7Gyio6MBiIqKYvLkyZrjZ82axY4dO7h48SLJycnMmzePb7/9lmHDhgHqf2RNnDiRn376iUuXLpGQkED//v3x9/cnPDy8XK5RCOG4arurA5s/bl/T2b77N/Vk/XVrmQ5TXLTCmMKiP8Mh7Ve9XIwENoqqVatadrJAlSpVNJ9dXS1/A1kJqIrP+6b0p92vEEIIIYQQonIo99Bo6NChzJ07l2nTphEYGMixY8fYunWrZnLslJQUrl+/rjk+OzubMWPG0KpVK0JDQ1m7di3fffcdL774IqD+B8/x48eJiIggICCAF154gfbt27N3717c3NzK5RqFEI7r80j1nECDFrxE8vXz5Obn8W3SOvafPwxAcJN2Jts7OTnR7rFWAAyNG8vNu7fIyMlkwBejAGjm42d0FFBAQAAAp06dMvkd2dnZms+1a9fWfH7kkT9HNmVkZJjs49o1dSjWpk0bne2NGzcGYOfOncWbCCGEEEIIISo4J5XMbKonKysLT09PMjMzZX4jISoJYxNhA7yxegZbTiTqbV82ch4dmwSa7fveg2w6zowwuO/QlE3UqOaht311/n9JT09nwQL1nErGJrLOy8vj66+/5saNGwQHB9OnTx+d/Tt27ODAgQNUr16d119/3eCIobS0NOLi4gAYN26czqtoKpWKDz74QPNZCCGEEEII4disyTzKfaSREELYu3lDp7Js5Dweq6MObTo360DiW6stCowAalTz4Nh7W3ir7yu4Oqtf/4rp/SLH3ttiMDBSeHt7ayagXrJkCZcvX6aoqEizPyMjg9jYWG7cuAFA165d9foIDQ0F4P79+yxZsoTMzEyd/cnJyZrAKCAgQG/uIicnJyIjIy26TiGEEEIIIUTFIiONDJCRRkJUPqZGGpWH1fn/BaCwsJD4+HhSUlJMHj927Fi8vb0N7tMesWRM06ZNiYyM1JvTSHH06FE2btxowZkLIYQQQggh7JmMNBJCiArCxcWFESNGMHDgQINzH4WGhjJx4kSjgRGoRyxNnDjR6KqUgwcP5rnnnjMaGAG0a2d67iYhhBBCCCGKW7BgAY0bN6ZatWoEBwdz6NAho8euW7eOoKAgateujYeHB4GBgXz77bea/fn5+bz99tu0bt0aDw8PGjRoQFRUlGZuTlE2LF9aRwghRLlwcnKiTZs2tGnThrt375KTk0OVKlWoXbs2zmZWXlO4u7vTu3dvwsLCyMjIID8/H3d3d2rWrFnGZy+EEEIIISqj1atXExMTQ1xcHMHBwcTGxhIeHs6ZM2eoW7eu3vFeXl68++67NG/enKpVq/Ljjz8SHR1N3bp1CQ8PJycnh19++YWpU6fStm1b7ty5w2uvvUZERASHDx8uhyusHGSkkRBCOJCaNWvi4+ODl5eXxYGRNmdnZ7y8vPDx8ZHASAghhBBClJlPP/2UUaNGER0dTcuWLYmLi8Pd3Z2lS5caPL5bt24MHDiQFi1a0LRpU1577TXatGnDvn37APD09GTHjh0MGTKEJ554gr/+9a988cUXHDlyxOxUDgoZ+WQ9CY2EEEIIIYQQQghRavLy8jhy5AhhYWGabc7OzoSFhZGUlGS2vUqlIiEhgTNnzhhc7EWRmZmJk5MTtWvXNtunMvJp+vTp/PLLL7Rt25bw8HDNojLFKSOfkpKSOH78ONHR0URHR7Nt2zYAnZFPv/zyC+vWrePMmTNERBheNdlRSWgkhBB26O7duza1y83N5ebNm9y8eZPc3FybvzstLY1bt27prNYmhBBCCCF0ycgVw9LT0yksLMTHx0dnu4+PD6mpqUbbZWZmUqNGDapWrUq/fv2YP38+vXr1MnjsgwcPePvtt4mMjLRoASt7HPnkCGROIyGEsEOffvopAAMGDKBNmzYGJ8HWlpqayubNm7ly5YrO9vr16xMREUG9evVMtlepVJw8eZJ169bp7QsODjb5hEcIIYQQojKSOXtKX82aNTl27Bj37t0jISGBmJgYmjRpQrdu3XSOy8/PZ8iQIahUKhYtWmS2X2Xk0+TJkzXbrB35tGvXLs6cOcOcOXOMHmfNyCdHISONhKjESvPJiHJM7969qVOnDk5OThw7dqyMr6Di27BhA8uWLaOwsNDoMUePHuXLL7/UC4wArl+/zpdffsnRo0eNti8sLOT77783GBgBHDx4kE8++cT6kxdCCCGEqMBk5Ipx3t7euLi4kJaWprM9LS3N5MNMZ2dn/P39CQwM5I033uCZZ55h1qxZOscogdHly5fZsWOHRaOM7HHkk6OQkUZCVFKl/WQEIDs7m86dOzNkyBBGjRr1sC+pRBbvWclnO5bg4uzM671HMeyvA6jqWtXi9jey0nl33SfsP3+YRl4N+GBADMFNrFum/vDvx3l15XQycrKYNm0a165dY8mSJaSkpLBhwwYGDx6s1+bMmTNs2rQJgKCgIMLDw3F1Vf9qLyws5ODBg+zYsYNNmzbh4eFBQECAXh+rV6/mwoULAAwZMoQWLVpo9mVnZxMfH2/0XW8hhBBCiMpIRq6YVrVqVdq3b09CQgIDBgwAoKioiISEBMaNG2dxP0VFRTpTLiiB0blz59i9ezd16tQp7VPXUVYjnxyJjDQSopIq7ScjAM8//zzTpk3TmfDOWuU1+umzHUsAKCwqYu7WLwl8ry/3HmRbdM4HLx6l28dD2X9ePWz4j9vXiF76Jm+snmFRe4CYVTOI+vp1MnKyAHBycsLX15cXX3wRgJMnT3Lr1i2dNiqVilWrVgHQt29f+vXrpwmMAFxcXAgJCaFz584ArFy5EpVKpdNHeno6586dA2D06NE6gRGAh4cHL7zwAtWrV7f4WoQQQgghKjoZuWJeTEwMixcvZvny5SQnJzN69Giys7OJjo4GICoqSid0mzVrFjt27ODixYskJyczb948vv32W4YNGwaog5lnnnmGw4cP8/3331NYWEhqaiqpqank5eWZPBd7G/nkSCQ0EqISelirGVirtFc0gD9HP5l6ggPQzMePA++sY8/bP9CyQTMAXo6fbLINQGFRIdFL3wTgnX7jODp9C+vGfgXAlhOJHP79uNk+Dl48ytaTiQCsG/sVP0/9UbPP19dX8/+n7du367TTfh2tY8eORvvv0aOHwTYA+/fvByA8PNzgCDNQPyl66aWXzF6HEEIIIYQwTRm58vPPP/Phhx8SExNDYmKi3nEVYeTK0KFDmTt3LtOmTSMwMJBjx46xdetWTdCWkpLC9evXNcdnZ2czZswYWrVqRWhoKGvXruW7777TPES9evUqmzZt4sqVKwQGBlK/fn3NnwMHDpg8F+2RTwpl5FOnTp0sviZTI5927txZ5iOfyoOERkJUQg/jyYgtynP004Zxi6nt7smjNb1Y/coCAI6mnNIbmVPcwYvquYI6N+vAsE4DcatSleb1m7J27JcAvLbyPbPXPW2DetLrdWO/onn9pni46Y7qefLJJwE4e/aszvZLly4BpgMjUI9aUq5faaM4ceIEAA0aNDDZh6MNiRZCCCGEKEv2OnLF3uYsHTduHJcvXyY3N5eDBw8SHBys2ZeYmMiyZcs0P8+cOZNz585x//59bt++zYEDBxg6dKhmf+PGjVGpVAb/FH9dzBB7GvnkSGROIyGExSx9p9cWD+u9cGO0VydzcXbRfC4sKsLVxcVQEwDSstIB6PbEX3W2P+blC8CdnEyz3/3HbfUyqn7ejQzu136aoS0/Px+AGjVqmP0O5caltFEoE2xb0ocQQgghhFCzxzl7ZM5S04YOHcrNmzeZNm0aqampBAYG6o18cnb+c1yNMvLpypUrVK9enebNm/Pdd99pgixl5BNAYGCgznft3r27VP6NZA8kNBKiEirpkxFQ/2JMTk5m1qxZpfIL0dTop99++81ou8zMTHx9fcnNzcXFxYWFCxfaNPrp5t3bPFrTC4Bb9+5otpsKjABaNVBPLD3zx/lEBvfXhE//PrYDgN6tzL++F+ofxP7zh/nh8GaGdRqot//u3buAOrTT5u7uDsD9+/fNfocyH5KHh4fB/UVFRSbbmxtxJYQQQghR2cTExDB8+HCCgoLo2LEjsbGxeiNXfH19NSOJZs2aRVBQEE2bNiU3N5f//Oc/fPvtt5rXz5SRK7/88gs//vijZuQKqAOeqlVNL9KiPWofIC4ujs2bN7N06VImTZqkd3zxGv61115j+fLl7Nu3TxMaPf/884D+aHVHNW7cOKOhXvHXBGfOnMnMmTON9qWMfKro5PU0ISqhsnqntzxY+l64OU/N+Qdzt37Jv3YspcvsZwAY3W2Y2XYB9ZpoPreaGkb8gbW8tHwSH/z7XwBM7jfGbB8fDpoIwEebv2DUsrdZ8dMGnf3KMOCQkBCd7S1btgQgKSmJe/fuGe2/oKCAU6dOAehNdK3Md7R582aTN72rV6+avQ4hhBBCiMrEnubssdc5S4Xjk5FGQlRSpf1kBOD27dukpKRw7Zr6daszZ84AUK9ePZMjmMA+Rj8t3bdG83nQX/owrucIi9olvbuBTh8OAGD2fxZqtn8zci4+tR41275uLW++GTmX6KVvsv/8YfafP8z09tNRqVQcOHCAkydPAtCuXTuddrVq1aJBgwZcu3aNpUuXMmbMGJ3V00C96obyrnitWrX03onv0KEDu3bt4tKlSxw4cIDQ0FC987t69Spff/01S5YsMXstQgghhBCVib2MXCnvUfui4pLQSIhKqrTf6QXYtGmTJnQCePbZZwGYPn067733nsnzKav3wi2V9O4GDpw/TGFREV0COuJZvab5Rv/jWb0mp2cmcP7GJU5eOUOdGo8Q4t9eZ24kc4KbtOPEB9s5ePEoN7JucTU9nR9//JHLly8DMGjQINzc3PTaDRkyhNjYWO7cucPs2bOJiIigUaNGuLi4cOvWLeLj4zXHjhw5Uq99tWrVGDRoEOvWrWPnzp1cvXqVoKAgzfvzJ0+eZOfOnRZfhxBCCCGEcBxlOWepqBicVJXhJTwrZWVl4enpSWZmplUz1QshSmb16tUMHz6cL7/8UjP6ac2aNfz222/4+PhYNPpp0qRJLFq0SDPMV3v0U79+/Vi1ahVPPPGE3uin5A93lcs1G9NySk/N50GDBtG6dWujx96+fZv58+cb3e/i4sKYMWPw8vIyeszx48dZv3690f2tW7fm+PHjZs5aCCGEEEKUh7y8PNzd3fm///s/zQNYgOHDh5ORkcHGjRst6ufFF1/kjz/+YNu2bTrbL126hJ+fH0ePHtWb9Fk4HmsyDxlpJISwG/Y2+qm89ezZkw4dOhgcYaTNy8uLadOmceHCBfbs2cOVK1cA9aTX/fv3x9/fX2d1OEPatGlDQEAAx48fZ/fu3Tx48ACAgIAAzTKrQgghhBDCPpX3qH1RcclIIwNkpJEQlY+9jTRanf/f8j4FPfYesgkhhBBCVGblOWpfOBYZaSSEEEIIIYQQQlQi5T1q394ewrZ4t0d5n0KFICONDJCRRkJUPvZ2k5ORRkIIIYQQwpHYWz0toZFxMtJICCEqoKysLE6fPk1OTg5VqlShcePGNGzY0Ox8RQqVSsUff/zB5cuXyc/Px93dnZYtW0o4LoQQQgghKpyuXbvaVO8WFBSQnJzMrVu3AKhTpw4tWrTA1dXy+MRQ3f71119bfQ32QEIjIYSwc5mZmaxZs4Zr164Z3B8ZGUlAQIDJPs6ePcvKlSv1tm/bto1atWoxcuRIPD09jbYvKiqy7qSFEEIIIYQoR3v37gWsq3d3797Nvn37DO4PCgqib9++Oq/4FWeqbnfU0Mj41QohhNChUqkoKCwsUR9FRUVWBTC3b98mNjZWc+Pp1KkTPXr0oEOHDppjVq5cydGjR432cfToUZ3AKCwsjKeeeopWrVoB6ichsbGx3L592+g5L1++3OJzFkIIIYQQlZe19a4hBYWFlGQmHZVKZVO9qwRGfn5+9OrVi9DQUOrXrw/A4cOHWb58udFrs6Rud0Qy0kgIIcy49yCbl5ZP4tgfpzXbXuk2jPE9R1j8atjJq2cYsmiMzraVL39B20YtjLZRqVTMnz8fAC8vL6Kjo6lRo4Zm/9/+9jcOHTrEli1b2LRpE/Xr19dbxSI1NZVNmzYB0KVLF7p3765zzgMGDGDJkiWkpaWxcOFC3n33Xb1r2rJlCykpKRZdpxBCCCFERdS6aTAALw+YYlP7/IJ8Yj4fTFSf1+nQsrtNffx8ejfxWz/j01fX8vq/BtnUR1mypd7VplKpWLpvDfO2faXZ5l+3MSte+pwa1Tws6kO7bp8+fTpgfb0bFRWFn5+fzv709HQWLFhASkoKu3fvpmfPnnrnbq5ud1Qy0kgIIUzIK8ij48wIncAIIC7xO6aun2tRH7/+kax3AwWI/HIcp66eNdru/PnzALi6ujJ69GidG4+iY8eOtG/fHoD//Oc/evuVwKh379706NFD7wbp6urKiBEjACgsLNR8pyI/P5/Dhw+buDohhBBCCCGwqd7V9s7aOTqBEcD5G5foODOCvII8s+2N1e3W1Ltjx47VC4wAvL29iYqKAmDfvn0UFBTonqcFdbujkpFGQlQSX7zxb4PbtZ9YVHGtYlPfX26YCVj35GXcvL/b9F0P23c/bQCgVYMAVr3yBS7OLty6d4cus59h3S9bmdj3FTyr1zTZR+SX4wCIjZxO71ZdAdh+6r9MWPk+/1g0mtMzEwy227NnDwADBw40OfFenz59OHLkCH/88Qe5ubm4ubkBkJuby/Xr1wF1uGRMtWrViIqKIj4+no0bN/Lmm29q9v32228AmmG5QgghhBBCGGNtvavIvH+Xjcd2ALDn7R94tKYXKpWKAV+M4lza73ybtI4Xujxrso/idfv/Fe7X7LO03vX29jbav5+fH35+fvz+++8kJyfTunVrzT5L63ZHJCONhBDChM+2LwZg4fMf4uLsAkCdGo/w8lPPAbD37CGT7bXfeVZuoMU/G3sv+urVqwDUrl3b5Hdo35iysrIMfnZxcTHZh5eXFwDZ2dk625VVI5o0aWKyvRBCCCGEENbWu4oD59UjfV7oMpRHa6rrUicnJ5a/MA+AedsWm/1uQ3W7ttKod/39/XXaKCyt2x1RxYrAhBAO48Nhz1h87O4TySQcVw8z7dmmJd1bW/ZetDUGtdAfTgtQ+L8bXJVioUt+YT4ALiZWTygtHh6WvcNdEsVvnsW5u7uX+TkIIYQQQojKSVlspoqL7psP+YXq18BcDYRAxRmr2xWlUe+a6+Nh1O0Pm4RGokJbsGABn3zyCampqbRt25b58+cbfU1n8eLFxMfHc/LkSQDat2/PRx99pHN8Wloab7/9Ntu3bycjI4OuXbsyf/58mjVr9lCupzJSAqOebVoCaMKjsgiODHm914t8tmMJw79+gw3jFuPk5MTNu7dZum8NACH+QSbbay/Juf3Uf3WG6xo6RpuHhwfZ2dncvn3b5PKghVorutWqVcvg54KCApNDZTMyMgBo2LChzvY6deoA6L37LYQQQgghyo81D2AtcSX9Nt/s2ouPpyc927ZkxX+T8PH0ZHiPUNyqWD6FhbX1rqLrE+qJxuMSv+O5vw6gTo1HKCwqZPS37wIQEz7K7HcXr9uLs6TeDQkJMfkdFy9e1GmjsLRud0TyepqosFavXk1MTAzTp0/nl19+oW3btoSHh3Pjxg2DxycmJhIZGcnu3btJSkqiUaNG9O7dWzPUUKVSMWDAAC5evMjGjRs5evQojz/+OGFhYWYTZ2Eb7cCoe+sWdG/dgp5tWpJw/DS7TyQ/lHMYHjoYgHNpv9Nqahgtp/TkqTn/AKB/YC+z8xkB/DB6EQATVr5Pyyk9aTmlJxNWvg+oV5Qwpn///gDEx8fz4MEDo8cdPHgQgAYNGmjmMwJwc3PT3BS3bdtmtH1BQQHr168HoGvXrjr7WrRQh3O///670fZCCCGEEMJxaQdGw3uE0rReXaJ7dCEtM5Plu/aTm59vcV/W1rsKz+o1GfiXPgB0mf0MLaf0pPW03py+dg6Afwb3N9tH8bpdm6X1rqma9+bNm6Smpuq0UVhatzsiCY1EhfXpp58yatQooqOjadmyJXFxcbi7u7N06VKDx3///feMGTOGwMBAmjdvzpIlSygqKiIhQT1p27lz5/jpp59YtGgRHTp04IknnmDRokXcv3+flStXPsxLqxSKB0aKhx0cVXWtyqEpm2jmo7uKwpt9XuajwW9b1Ecr3wCDN8sfRi8yuQSpv7+/Zi6iZcuW6a3SoFKpSEhIYMcO9aSBf/+7/uTi/fr1A+Dw4cMcOqQ//9K9e/dYuHChpm/lPW2Fq6srnTt3NnV5QgghhBDCQRUPjJRRRQ29vawOjmypd7XNHPgmo7sN09n2l8ee5Oep/6aqa1Wz7Y3V7dbUu/Hx8dy8eVOv74sXL7Jw4UIAgoKC9EbwW1K3Oyp5PU1USHl5eRw5coTJkydrtjk7OxMWFkZSUpJFfeTk5JCfn6+ZMC03NxdQz7yv3aebmxv79u3jxRdfNNunvC5nGWOBkULZ9rBeVatRzYON45egUqkoLCrC1cyk0oa0bdSC0zMTNJMAmhuiC+rJ/8aMGcP8+fNJS0vjww8/pFWrVjRo0ACVSsXOnTs1x0ZERFCvXj29PurVq0dERASbNm1iy5YtbNmyhdDQUDw8PLh7967O34fx48fj5OSk10f37t1JSUmx+pqFEEIIIYT9MhYYKZTg6Jtde1m+a7/ZV9VsqXe1OTk5MT4smvFh0RQUFuLi7GywNjVFu25/fe/HVte7KSkpLFy4EG9vb9q2bYuLiwvnzp3TjEB6/PHH6du3r8FzN1e3v//++1Zdi72QkUaiQkpPT6ewsBAfHx+d7T4+Ppohhea8/fbbNGjQgLAw9dDG5s2b89hjjzF58mTu3LlDXl4ec+bM4cqVK5plzU2R1+UsYy4wUpTHq2pOTk42BUbanJ2drbqBenl5MWHCBM38RKdOnWLHjh06gVFkZCTt2rUz2ke7du2IjIzU/Lx//362b9+uuYH6+voyYcIETUBq6JyHDx9u8TkLIYQQQgj7Zi4wUtgy4sjaetcQVxcXqwMjbU5OTjbVu0FB6vlK09PTSUhIYPv27ZrAqHPnzkRFRRm9NkvqdkckI42EMGD27NmsWrWKxMREzciiKlWqsG7dOl544QW8vLxwcXEhLCyMvn37WjTcUPt1OYC4uDg2b97M0qVLmTRpkt7x33//vc7PS5YsYe3atSQkJBAVFaV5Xe7kyZO0atUKgEWLFlGvXj1Wrlxp0cgne2NpYKR42COOyounpyevv/46WVlZJCcnk52dTZUqVWjcuDENGza06IYaEBDAtGnTuHLlCpcuXSI/Px8PDw9atGihM2G2MSW98QshhBBCCPtgaWCksHbEkb3o0qWL1fVuv379CA8PJzk5mVu3bgHqSa9btGhhclEZham63VHZRWhkzSs769at46OPPuL8+fPk5+fTrFkz3njjDZ5//nnNMSqViunTp7N48WIyMjIIDQ1l0aJFDv/KjrCct7c3Li4upKWl6WxPS0sz+AqPtrlz5zJ79mx27txJmzZtdPa1b9+eY8eOkZmZSV5eHo8++ijBwcGaRNoYe3hd7ufTuw32+3NyIgCHk3fj6mLbL/+TFw+Z/A5DAnQXHLA6MFJUluAI1KuhBQcH29zeycmJRo0a0ahRo1I8KyGEEEKIis+WeldbQaF6hI5Se9tCu25vWde2PqwNjBSOGBz16NHDpnaurq60bt26RN9d0rrdnpR7aKS8shMXF0dwcDCxsbGEh4dz5swZ6tbV/5vg5eXFu+++S/PmzalatSo//vgj0dHR1K1bl/DwcAA+/vhjPv/8c5YvX46fnx9Tp04lPDyc06dP6/wDW1RcVatWpX379iQkJDBgwAAAzaTW48aNM9ru448/5sMPP2Tbtm0mgyBlGcVz585x+PBhZsyYYfJ8TL0u99tvv1l0TaZel/vyyy/x8PDgs88+M/q6XPzWz0z2v2KH+VUNzDH3HdpmPjdY89nWwEhRmYIjIYQQQghRfqypdw1JvnyU5MtHS9THih1f6NTSlrI1MFI4YnAkSq7cQyNrX9np1q2bzs+vvfYay5cvZ9++fYSHh6NSqYiNjWXKlCk6y975+PiwYcMGnn322TK/JmEfYmJiNO+lduzYkdjYWLKzszX/W4uKisLX15dZs2YBMGfOHKZNm8aKFSto3LixZu6jGjVqUKNGDQB++OEHHn30UR577DFOnDjBa6+9xoABA+jdu3eZXktpvC736atrDfZ9OHk3K3Z8wcdjV9k80uirjTMBeKn/FIvb5NxaAZQ8MFKUJDi6kn7b5u8VQgghhBAVX/PH1fNHWlPvaisozOetBc/yz17jCGrR3aY+tOv2bXs/sLreLUlgpJDgqPIp19CopK/sqFQqdu3axZkzZ5gzZw4Av//+O6mpqZrRGKAeFRIcHExSUpLB0Cg3N1fzqg9AVlZWSS5L2ImhQ4dy8+ZNpk2bRmpqKoGBgWzdulUz2iclJUVnnpZFixaRl5fHM888o9PP9OnTee+99wC4fv06MTExpKWlUb9+faKiopg6darZc7GH1+WquBr+Za4ERa4uVYweY47Sh7XtSyswUtgSHCk30Fe7lPjrS9W8efMICQmhXbt2Fo2QVKlUnD9/no0bN2omQvf19eWpp57C39/fonmPHjx4wM8//8yuXbsAcHFxoXXr1oSGhuLt7V2yCxJCCCGEcGC21ruG+ilpze3qUsWmerekgZHCkYOj0qh309PT2b59O+fOnQPUgwysqdsdTbmGRra+spOZmYmvry+5ubm4uLiwcOFCevXqBaAZHWLNqlmzZs1y2OXvhGnjxo0z+jpaYmKizs+XLl0y29+rr77Kq6++avV52NvrcvagtAMjhTXBkfYN1N7cu3eP7du3s337dgYOHKgXGGq7ffs2CxcupLCwUGf71atXWbFCPaJr/PjxRleLADhx4gTr1q3T2VZYWMixY8c4duwYzZo104SnQgghhBCifCmrCIPl9W5phjuGgiN7Z0m9O3ToUFyMrJZcWFjIhg0bOHnypM52a+p2R1Tur6fZombNmhw7dox79+6RkJBATEwMTZo00Xt1zVKTJ08mJiZG83NWVpZMFCtKXUV6Xa40lEVgpLAkOCp+A83IyWLv2UO4urgQ4h+EZ/WaVn/v2dSLnLp2Fp9a3gQ3aYeLs+EbjjGFRYUcOH+EW/fuMGzYMH799VdOnDjB+vXrcXJyMjghX2ZmJvPnz9f8HBUVhZeXF9nZ2WRkZLB+/XoKCgqYP38+EyZM0ASM2o4fP8769esBaNy4Mf369cPZ2Zl79+5x7do1tm3bpnmSIoQQQgghyp8t9W5pjwYqHhxFtnnNbBvtevfJhk/gX7ex1d9rS91uab27cuVKnnvuOb1R+kVFRcTHx5OSkgJAr169aNWqFXl5eWRlZVlUtzuqcg2NbH1lx9nZGX9/fwACAwNJTk5m1qxZdOvWTdNOeX1Iu8/AwECD/bm5ueHm5lbCqxHCNHt6Xc4elFVgpDB1IzV0Aw35aKBO+/6Bvfho8NsWvdaVef8unT4coLf9m5FzCW7SzqLzPXjxKNFL39T8PH36dJo2bYqPjw87d+5k3bp1BAQE6P2uWrp0KaAeTTlixAjNkFhPT08aNGhAQEAACxcu5M6dO6xZs4ZRo0bptH/w4IHmBtqrVy86deqkuWYvLy8ee+wx/Pz8iIuLs+g6hBBCCCHEw2FtvVsWtIMjc4rXu4qkdzdYFPyoVCqmrJ/L+l+26mw3V7dbU+9euHCBU6dO8eSTT+r0ceLECU1g9OKLL+Lr66vZ9+ijj1pUtzsqZ/OHlB3tV3YUyis7nTp1srifoqIizZxEfn5+1KtXT6fPrKwsDh48aFWfQpSFcePGcfnyZXJzczl48KDOMoyJiYksW7ZM8/OlS5dQqVR6f7RfEXr11Vf5448/yMvL4/Lly8yYMYOqVas+xCuy3cNY4ax76xaaobu7TyQDpm+gr3QbxgtdhgKw8dgOlu5bY9H3aAdGU54eT+dmHQCIXvomN7LSzbZPy7qpuYF2btaByX8bo9kXEhLC448/DsDPP/+s0y4rK0szB9uLL75o8B1qV1dXRo4cCcC1a9f05mw7elS9ekfr1q0JCQkxeLP18fFhyJAhZq9DCCGEEEI8XNbWu2VBCY5MMVXvGnr4asj8hGWawMiaut3aenftWv0FhDZs2ADoB0bazNXtjqpcQyNQv7KzePFili9fTnJyMqNHj9Z7ZUd7ouxZs2axY8cOLl68SHJyMvPmzePbb79l2LBhADg5OTFhwgRmzpzJpk2bOHHiBFFRUTRo0EAzl4wQovLQvpF+t+eA0Rvo3kn/x6th0bwR/hJ73v4BgHnbvjLb/7m0S5rPp2bs5J9/HcBXw2czud9YAKasn2u2j49+XADAtIgJfDV8Ns+H/LmEqpOTE08//TSAThgOcPq0+qnSk08+iaur8YGjNWrU0ITmycnJOvsOHDgAQNu2bU2eY4sWZR/yCSGEEEII61la75alht7G584Ew/XuqRk7NfvPpl40+x1xid8B1tftttS7d+/eNfi5QYMGRttbUrc7onKf08jaV3ays7MZM2YMV65coXr16jRv3pzvvvuOoUOHao556623yM7O5qWXXiIjI4POnTuzdevWCjmTuRDCvO6tW3D19h1+u3IdwOANtE6NRzSfH635502voLAQVyOT4QGcunoGgMl/G6Pz1GJI0NPM2ryAfed+NtZUY8dp9XDev7cNM7hf+3egtpycHPW516lj9juqV68OoFlZTXHv3j0AatWqZbYPIYQQQghhnyypd8uToXrXycmJKU+PZ+aP8zl97RwB9ZoYbV+gteCLtXW7LfVuTk4ONWvW1HzWPmdTzNXtjqjcRxqBda/szJw5k3PnznH//n1u377NgQMHdAIjUP8/8oMPPiA1NZUHDx6wc+dOAgICHtblCCHszJX02/yedlPz84HfzusdU1j0541IpVJpPrsY+cWvUG5ae4uFQxfT1e88P1bH8PBVbbXd1TewP25fM7hfudEVX8mhyv8KgSoWFATKExJjxzrKa41CCCGEEEKfJfVueTJW7+7+LQmAurVMPwTVrsltrdutqXe1a2ZTI/qLM1e3OyK7CI2EEKKsaL/TPXVIhN4734qhcWO5efcWGTmZDPhCPVl0Mx8/s08TQvzbA7Dv3M+sOvRvsnPvk3z9PIMXvAzAB/1jTDUH4PPI9wEYtOAlkq+fJzc/T2f/tWvqm2vx5TsbN24MwM6dO3VumIYo71UrbRRKoH7q1CmT7YuPUBJCCCGEEPbB0nq3LOXm55vcb6je/TZpHfvPHwYwu3iMk5MT7R5rBVhft9tS79auXVvz+ZFH/hzZlJGRYbIPc3W7I5LQSAhRLq6k334o31H8nW5DkwUCnL52jqfmDCHko0GcS/sdgO9H/cvsd7g4u/DNSPW8RR9siqXDjKc1gVHf1t3o2CTQbB9Bfm3o27obAIMXvEy79/tq9qWlpbFt2zYAQkNDddo1bNhQ83nXrl1G+z906JDBNqBeQQJgx44dXL161WD7vLw84uPjzV6HEEIIIYR4uKypd8tKbn4+y3ftN3mMoXp31mb1PEfLRs7Dxdn4dBCKL6NmAdbX7dbWu8HBwTqvmTk7OxMSEgLAV199Rb6RgMzSut3RlPucRkKUltD5oeYP+p+i/CJ+euMnmj3fjEc7PGrT9938+Sbnvj3HX+f9Fecq+vnr/vGmf3FWdt/s2kt0jy5mJ82zlalVIwwtT/pW31f4dNtiCooKien9IlEhg6nqatkQ1uAm7dj91ipmbV7I9lP/5RF3T/4V+R5Bfm3MN/6feUOnMrTD35m28VNSbqlvZsnJyaxZo14JIiAgQG/uIicnJyIjI1m5ciX79u2jWrVqdOjQQTP0tqCggK1bt3LkyBEAIiMj9Z7AeHt78+STT3Ly5EmWLFnCiBEjaNSokeZGmZGRwVdffcX9+/ctvhYhhBBCCFH2rK13y4ISGKVlZpo9tni927lZB2YOfJO6tbwt+q4a1Tw49t4WVhzcaFXdbm2927VrV70+QkNDOXDgAPfv32fJkiX885//xNPTU7PfkrrdUUloJIQoFz6enmUWHFmyzGjxG+n8IWMYEfoPm7/Tp9ajxEZOt7k9QMcmgWx9Xf2Eo+WUnprtTZs2NbrkfUBAABEREWzatImdO3eyc+dOg8dFREQYndttwIABZGVlkZKSojOHnBBCCCGEsE+21LulHRxpB0bRPbpY1Ea73rVFVdeqjAj9h9V1u6X17tixY3F3d9fb7u7uztixY1mwYAE3btwgNjbWYHtTdbujktBICFEuhvcIZfmu/aUeHFlyA1Xo3Ejt9Hf74MGDadWqlcmnE+3ataN+/fps2bKFlJQUnX2NGjXib3/7G/Xq1TPa3sXFhREjRnDixAk2bNigNz9SaGioZkiuEEIIIYQofzbVu5RecFQ8MCqrtwdKi6X1rqHASOHt7c3EiRPZt28fSUlJevstqdsdkYRGQpSRPV2fKu9T0NXhzfI+Ax1uVaqUenBkTWCkKKuhuiUVExOjWebTEvXq1SM6Oprc3FyysrIA9bKibm5uFrV3cnKiTZs2tGnThrt375KTk0OVKlWoXbu20aVDhRBCCCFE+bCl3i2t4MjRAiNFadS77u7u9O7dm7CwMDIyMsjPz8fd3d2qut3RSGgkRBl559TJ8j4FHVlXZxrcfvKiepLkrzbOxNXFtqUglT6+3GD4Owx5tlOjUg2ObAmMFPYYHNl643Fzc+PRR22bp0v7uyvyjU8IIYQQwlq21LvaCgrVkyfHb/2MX87aNvepdt1ua71b0uDIUQOj4kpa7zo7O+Pl5ZjXbi15fCyEKFdKcKTMcWTLqmolCYyEEEIIIYRxCxYsoHHjxlSrVo3g4GCdVVmLW7x4MV26dOGRRx7hkUceISwsTO94Jycng38++eQThzyf8mJLvVvSVdUqSmAkrCMjjYQoIx+1erK8T0HHCSOvp/18ejfxWz/jpf5TqOJqW9iiPHF5ecAUi9tkpi3XfC7JiKPKFhgpQ2ldXV155JFHrH51rKioiDt37lBQUFDhh9IKIYQQomRWr15NTEwMcXFxBAcHExsbS3h4OGfOnKFu3bp6xycmJhIZGUlISAjVqlVjzpw59O7dm1OnTuHr6wvA9evXddps2bKFF154gcGDB9v1+TzZpCNgXb2rLb8gn5jPBxPV53U6tOxuUx/adXvOrRU29WHriCNHDIzS0tJsrndtnfJBW/G63VFJaCSEsAu2BEeVJTBSqVScPHmSdevW6e0LDg6ma9euJiftA8jJyWH//v0cOHBAb9+AAQNo06ZNhZu0TwghhBAl8+mnnzJq1Ciio6MBiIuLY/PmzSxdupRJkybpHf/999/r/LxkyRLWrl1LQkICUVFRAHqLc2zcuJHu3bvTpEkThzsfR2VtcOSIgRGo//ehsLTeTU1NZfPmzVy5ckVne/369YmIiDC5uAyYrts/+OADK87efsjraUIIu2HNq2qVJTAqLCzk+++/N3jjATh48CCffPIJ6enpRvtIT0/nk08+MRgYAWzYsIFly5ZRWFhotI+jR49ad+JCCCGEcGh5eXkcOXKEsLAwzTZnZ2fCwsIMrhxlSE5ODvn5+UbnfklLS2Pz5s288MILDnc+js7SV9UcNTAqztJ698svv9QLjEA9Iu3LL780WRObq9sdlYw0EkLYFUtGHJVHYHT49+O8unI6GTlZ9GrZhXeeHotPLcsnnM4ryGP5/rV8tmMJLs7OvN57FMP+OoCqrlVNtlu9ejUXLlwAYMiQIbRo8eeToOzsbOLj47lx4wYLFixg4sSJeiOOcnJyWLBgAaBe7WHUqFHUrl0bUD8JuXbtGkuWLCElJYUNGzYYHBp+5swZNm3aZPG1CiGEEMLxpaenU1hYiI+Pj852Hx8ffvvtN4v6ePvtt2nQoIFO0KNt+fLl1KxZk0GDBjnc+VQE5kYcWRsY2VrvaruRlc676z5h//nDNPJqwAcDYghu0s6q61Lq9unTp9tU7wYFBREeHo6rqzouKSws5ODBg+zYsYNNmzbh4eFBQECAXh/m6nZHJSONRKkq7YnpAJKTk4mIiMDT0xMPDw86dOhASkpKWV6GKGemRhyVR2AUs2oGUV+/TkaO+r3mHaf30v3jZzl40bLRN/ceZBP4Xl8+27EEgMKiIuZu/ZLA9/py74HxG0h6ejrnzp0DYPTo0To3HgAPDw9eeOEFqlevDmBwJNF///tfAOrWrcuECRM0gRGoJ3709fXlxRdfBODkyZPcunVLp71KpWLVqlUWXacQQgghhGL27NmsWrWK9evXU61aNYPHLF26lOeee87o/op8PvbC2IgjW0YY2VLvajt48SjdPh7K/vOHAfjj9jWil77JG6tnWHw9xet2a+vdvn370q9fP01gBODi4kJISAidO3cGYOXKlahUKp0+LKnbHZWERg7OnkIaZWK66dOn88svv9C2bVvCw8O5ceOGweOViel2795NUlISjRo1onfv3ly9elVzzIULF+jcuTPNmzcnMTGR48ePM3Xq1Er1i7yyMhQclUdgdPDiUbaeTARg3div+Hnqj0yLmABA9NI3KSwyPsRV8c+vXgWgmY8fB95Zx563f6Blg2YAvBw/2Wi7/fvVy7GGh4cbnNwRoGrVqrz00kua44uKijT7ioqKOHjwIABRUVFUMfLfy9fXV/PEbfv27Tr7DA3PFUIIIUTF5+3tjYuLC2lpaTrb09LSzM7rMnfuXGbPns327dtp06aNwWP27t3LmTNnNP+Yd7TzqUiKB0cleSXN2npXUVhUSPRS9cI97/Qbx9HpW1g39isAtpxI5PDvx832Ubxu12ZpvduxY0ej/ffo0cNgG7CsbndUEho5MHsLabQnpmvZsiVxcXG4u7uzdOlSg8d///33jBkzhsDAQJo3b86SJUsoKioiISFBc8y7777L3/72Nz7++GPatWtH06ZNiYiIqHB/EYVh2sFR3LbdxG3b/dDnMJq24VNAfeNpXr8pHm7Vebbj3+ncrAMAB84fMdlepVJx/sYlADaMW0xtd08erenF6lfUr4wdTTml96RCceLECQAaNGhg8ju0Rw9lZGRoPt+5c0fz2dzTjSefVK/2d/bsWZ3tly6pz93UDVQIIYQQFU/VqlVp3769Tm2u1OqdOnUy2u7jjz9mxowZbN26laCgIKPHff3117Rv3562bds65PlUNNrB0Yw1m2yew8jaelehjODv3KwDwzoNxK1KVZrXb8rasV8C8NrK98x+d/G6vbiS1rtOTk6a4Elpo7C0bndEEho5MHsKacpiYrqioiI2b95MQECAJrENDg5mw4YNFvUnKga3KlXo2bal5ueebVs+1Emv/7h9DQA/70Y625XQ6Na9O3pttBVqjfzRXq3BxdnF4DE6bf83UV+NGjUsPt/8/HzN54KCAovb5ebmmuzPmnMQQgghRMUQExPD4sWLWb58OcnJyYwePZrs7GzN6mVRUVFMnvznKJI5c+YwdepUli5dSuPGjUlNTSU1NZV79+7p9JuVlcUPP/xg9ageezufiiakub/ms5/PozZNem1tvatIy1Iv6tLtib/qbH/MyxeAOzmZZr/bWN2uKI16Ny8vT6eNwpa63VFIaOSg7C2kMTUxXWpqqkXnU3xiuhs3bnDv3j1mz55Nnz592L59OwMHDmTQoEHs2bPHoj6F47uSfpsV/02iXm1P6tX2ZMV/k0yuqlbaQv3VT6R+OLxZs02lUjH7PwsBeLLhEybbu7r8ebO8effP89YOm7SPMaTIzE1W+8mN9kTYxSfFNuXu3bsA1KxZU2e70sf9+/ct7ksIIYQQFcPQoUOZO3cu06ZNIzAwkGPHjrF161ZNzZ+SksL169c1xy9atIi8vDyeeeYZ6tevr/kzd+5cnX5XrVqFSqUiMjLSoc+nIlFeSXOr4krzhvX57cp1k6uqGWNrvduqgXpi6Zk/ztepbf99bAcAvVt1Nfvdhup2baVR7yrzIRkbxW+ubndEEho5qIoW0hiamE75C9e/f39ef/11AgMDmTRpEk8//TRxcXFlej7CPmjPYTSq91OM6v2Uwcmxy9KHgyYC8NHmLxi17G1W/LSBVlP/DGv96zY228cb4eo5h56a8w/mbv2Sf+1YSpfZzwAwutswo+2U96Y3b95sckiv9ium2jdB7c/JyaZv+seOHQMgJCREZ3vLlupRXpaG0UIIIYSoWMaNG8fly5fJzc3l4MGDBAcHa/YlJiaybNkyzc+XLl1CpVLp/Xnvvfd0+nzppZfIycnB09PT4c+nIig+h9Gwp0IMTo5tCWvrXUVAvSaaz62mhhF/YC0vLZ/EB//+FwCT+40x20fxur04S+rd4qPQtBUUFHDq1CkAvYmuLa3bHZGERpVUaYc0ZTExnbe3N66urpq/xIoWLVrI6mmVgKFJr02tqlZW6tby5puR6qdR+88fZuaP8zX7kt7dYFEfIzsPoX9gLwCW7lvDl3u+B2DQX/owrucIo+06dFC/Anfp0iWDK6OBOjD6+uuvARg4cKDefmXJ2DVr1uj9/QT1KKX9+/dz8uRJANq1013StFatWhXy3WwhhBDCHpX2IjcjRozAyclJ50+fPn3K+jKEAzE26bWxVdVMsaXe1aZdW8/+z0L2nfsZgG9GzsWn1qNm2xev2xXW1LtLly41OMXDgwcPWLJkieb4WrVq6ey3pG53VK7mDxH2qDRCmp07d1oc0uzbt89kn9oT0w0YMAD4c2K6cePGGW338ccf8+GHH7Jt2za9iemqVq1Khw4dOHPmjM72s2fP8vjjj5s8H+HYTK2SpgRHy3ft55tde22aoM9awU3aceKD7Ry8eJQbWbdo5RtAMx8/i9s7OTkx65lJTOo3lgPnD1NYVESXgI54Vq9psl21atUYNGgQ69atY+fOnVy9epWgoCDq1KkDqJcM3blzJwCPPfYYrVu31uvjySef5Ndff+XChQvExcXRo0cPmjVrhru7O3l5efz4449cvnwZUAdMbm5uen0MGTKE2NhYi69XCCGEENZTFrmJi4sjODiY2NhYwsPDOXPmjMH5RZVFbkJCQqhWrRpz5syhd+/enDp1Cl9fX81xffr04ZtvvtH8bOheLyonc6ukdW+tHk2TcPy0zs/G2FLvavOsXpPTMxM4f+MSJ6+coU6NRwjxb68zN5I52nX7qaxUq+vdO3fuMHv2bCIiImjUqBEuLi7cunWL+Ph4zbEjR47Ua29J3e6oJDRyUPYY0sTExDB8+HCCgoLo2LEjsbGxehPT+fr6MmvWLEA9Md20adNYsWKFZmI6UE8epkwgNnHiRIYOHUrXrl3p3r07W7du5d///jeJiYkW/XcSjsdUYKQoj+DIxdmFEH/jK25YwrN6Tfq27m5Vm9atW6NSqVi/fj3JyckGXzNr3bo1/fv315l4UOHk5ERkZCRr1qzh7Nmz7Nq1i127dukdN2jQIIOhE4Cnpyfjx4+36ryFEEIIR7BgwQI++eQTUlNTadu2LfPnzze6gtLixYuJj4/X/OOvffv2fPTRRzrHjxgxguXLl+u0Cw8PZ+vWrWbPRXuRG4C4uDg2b97M0qVLmTRpkt7x33//vc7PS5YsYe3atSQkJBAVFaXZ7ubmZvahsqh8zAVGCmuDI1vq3eL86za2aAoIY5S6/cUpPXW2W1Lvzp8/n8LCQtavX6/fr4sLY8aMMfoaoyV1uyOS0MiB2VtIM3ToUG7evMm0adNITU0lMDBQb2I6Z+c/34jUnphO2/Tp0zXvGQ8cOJC4uDhmzZrFq6++yhNPPMHatWvp3Llzif7bCftkSWCkKI/gqLy0adOGgIAAjh8/zu7du3nw4AEAAQEB9O7dW/MEwxgXFxciIyNJT0/np59+4siRI5p9PXv2pEOHDmafOioT5gshhBAVhT2N7FEWudFe+auki9xon3fdunV55JFH6NGjBzNnzjRbO4iKzdLASGFtcGRvrKl3p02bxoULF9izZw9XrlwB1JNe9+/fH39/f4MPabWZqtsdlYRGDsweQ5px48YZHelUPHi6dOmSRX2OHDnS4BBAe5dnZzPn5xfkG9xeUJiv839tobQ19h2WsCYwUlSm4KhatWp07NjR6NNPS3h7e/P000/z9NNPl+KZCSGEEI7Jnkb2mFrk5rfffrOoj+KL3IA6wBo0aBB+fn5cuHCBd955h759+5KUlISLmdWshK6S1rvaNXdp9GErawMjhSMGR9OnT7e6jZOTE/7+/vj7+9v8vaVRt9sTCY0cnIQ09qv3ftPzQD10Zs7nrQXPlvgrYj4fbPGxM5/781hbAiNFZQqOhBBCCFE6KtrIHmWRm8TERM0iNwDPPvtnfde6dWvatGlD06ZNSUxMpGfPnoa6Ekb8dvkoYF29a8iKHV+wYscXJerjrQXP6tTSlrI1MFI4YnAkSk5CIyHKyDtPPFHep6Djit/fDG7/OTmR5MtH+Wevcbi6WB7WaIvf+hkAUX1et6KVegW8kgRGipIGR7n5tj+tEUIIIYTjsbeRPWWxyI0hTZo0wdvbm/Pnz0toZCPr6t0/FRTms2LHF7R4vB0dWnSzqQ/tuv1K+imr692SBEYKCY4qHwmNhCgjvev6mD/oITrR0viEdMmXjxLUojtVXG0LjX45ux+ADia+o7jMtOWlEhgpbA2OlBtoZJvXbP7usvDTTz/RsmVLveU8TSkoKCA5OZlbt24BUKdOHVq0aIGrq+W/6rOysjh9+jQ5OTlUqVKFxo0b07BhQ7PvbwshhBCVSWmP7CmLRW4MuXLlCrdu3aJ+/foWXKXQ9mQT9atG1tS72vIL1KFRhxbdbO4D/qzbJy+Ks7reLWlgpHD04Kik9a5KpeKPP/7g8uXL5Ofn4+7ubnXd7kgkNBJClIvSDIwU1gZH2jdQe7Nt2za2bdtGrVq1GDlypNFVGkBdVO7evZt9+wy/ghgUFETfvn115jgrLjMzkzVr1nDt2jWD+yMjI627ACGEEMKO2ePIntJe5ObevXu8//77DB48mHr16nHhwgXeeust/P39CQ8P1/v+0Pmhettu/nyTc9+e46/z/opzFeN1hCnJX6lXkGrxknXBwkc/yD9VTfHx9LS63i3NaRwcMTiypN41N2H12bNnWblypd52S+t2R2Tb33whhCih0g6MFEpwpNxIr6TfNnhc8RsoqMOXohJOYF5QWIhKpbK5vUqloqCwkFatWgHqJyGxsbHcvm34OoqKili+fLkmMPLz86NXr16EhoZqniIePnyY5cuXG72227dvExsbq7mBdurUiR49etChQwfNMYZujkIIIYSj0h7Zo1BG9nTq1Mlou48//pgZM2awdevWUh/ZM3ToUObOncu0adMIDAzk2LFjeovcXL9+XXO89iI39evX1/yZO3cuoF499fjx40RERBAQEMALL7xA+/bt2bt3r0Urugn7Zku9W9rzfnZv3YKebVqScPw0u09Yvry8Uu+WhLV1u6X17tGjR432cfToUZ2aOCwsjKeeesriut1RSXwrhCgXZREYKcyNODJ0A205Rffp38qXv6BtI8uemKhUKpbuW8O8bV9ptvnXbcyKlz6nRjUPi/q49yCbl5ZP4tgf6qc106dPZ8CAASxZsoS0tDQWLlzIu+++qzdsdsuWLaSkqOeHioqKws/PT2d/eno6CxYsICUlhd27d+s95VSpVMyfPx9QLzMaHR1NjRo1NPv/9re/cejQIbZs2WLRdQghhBCOorxH9hhSmovcVK9enW3btln0vcLx2FLvlgWdEUdDTB9bvN4FeKXbMMb3HGHxq2Enr55hyKIxOtvM1e3W1LubNm2ifv36eiMOU1NT2bRpEwBdunShe/fuOudsSd3uqGSkkRCiXJRVYKQwNuLI0hto5JfjOHX1rEXf9c7aOTqBEcD5G5foODOCvII8s+3zCvLoODNC5wYK4OrqyogRIwAoLCzk/PnzOvvz8/M5fPgwAGPHjtULjEA9/F5ZBnjfvn0UFBTonuf/+nR1dWX06NE6N1BFx44dad++vdnrEEIIIRyJjOwRjq6k9W5pUUYcmWKs3o1L/I6p6+da9D2//pGsFxiB+brd2nr3P//5j95+JTDq3bs3PXr00AuELKnbHZWERkKIclGWgZH2d2jfSC+k3jB6A42NnM7pmQmcnplAbOR0AP6xaLTZ78i8f5eNx3YAsOftHzg9M4FTM3bSzEcd4HybtM5sH9/9tAGAVg0COPHBdk7P/HOofLVq1TShz8aNG3XaKau71K9fH29vb6P9+/n5aQKl5GTdocN79uwBYODAgSYnzO7Tp4/Z6xBCCCEczbhx47h8+TK5ubkcPHiQ4OBgzb7ExESWLVum+fnSpUuoVCq9P++99x7w58ieGzdukJeXx6VLl/jqq6/0VmgTojRZU++WJXNzGhmqd/dO+j8A1v2ylcz7d81+R+SX6lF41tbt1ta7f/zxB7m5uZrtubm5mgC5Y8eORttbUrc7Ink9Tdgk5YPWBrd/vucG83bd4I0edXn1qboP96QeqZiz1YuSUW6ki7fv4ZuEvQC8Et5d7wbau1VXg5+LiopMTiB94Lx6pM8LXYbyaE11n05OTix/YR4hHw1i3rbFvNDlWaPtAT7bvhiAhc9/iIuz/nK8Xl7qfrOzs3W2K6ukNWnSxGT/AP7+/vz++++aNoqrV68CULt2bZPtrVmBTQghhBBCPDyW1rvlyVC9W6fGI7z81HN8ued79p49xNNtjU8Wrz1/kbV1uy31blZWFo8++qjms8LFRb9W12aubndE8q8AB9F+YrzJ/YV59zn/f3O5n36VZv+YiEf9pmV6PutrGt6uBEXzdt3Q+VmIikqZxK+Ki+7IqfxC9WtgrgZCoOIK/3cTrGLkJlT8plOcu7u72e8w14eHh2VzLwkhhBBCCGEtY/VufqH6tS0XEw9pS8vDqHfN1dyOSF5PqwAedmBkzqtP1eWNHnWZt+sGn++5Ua7nIoTyTved7Gyie3bhMe86BleZ2H7qvwY/mxplBND1CfUw9rjE77h17w4AhUWFjP72XQBiwkeZPcfXe70IwPCv3zC48lpGRgYADRs21Nlep04dAL13pg25ePGiThuFcvM0t8pDYQlXuBBCCCGEEGXD0nq3PBmqd2/evc3SfWsACPE3vRqhdk1ubd1uS71bq1Ytg5+Lzw9anLm63RFJaOTg7C0wUkhwJMzJfQiTwRWfBLBpvbpGlyedsPJ9Wk7pScspPZmw8n1AvRKDOZ7VazLwL+r3n7vMfoaWU3rSelpvTl87B8A/g/ub7WN46GAAzqX9TqupYToruRUUFLB+/XoAunbtqtOuRQv1u+O///47v//+u9H+b968qVnhRWmj6N9ffX7x8fE8ePDAaB8HDx40ex1CCCGEEOLhsqbeLUu7TySb3G+o3n1qzj8A6B/YC8/qRl5l0fLD6EWA9XW7tfVugwYNdCavd3Nz04RAplYktKRud0QSGjkwew2MFBIcCVOW79pfpsGRsVUjjK0yUdwPoxeZXLpT28yBbzK62zCdbX957El+nvpvqrpWNdu+qmtVDk3ZpJk8W3Hv3j0WLlyoeaLh7++vs9/V1ZXOnTsD6pvgzZs39fq+ePEiCxcuBCAoKEhvbiJ/f3/Nu9nLli3Te3qiUqlISEhgx44dZq9DCCGEEEI8PCWtd0vL7hPJJBw/bfIYY/Xum31e5qPBb1v0Pa18AwyGQ+bqdmvr3b///e96ffTr1w+Aw4cPc+jQIb39ltTtjspxz7ySs/fASCFzHAlj0jIzWb5rP8N7hJb6SmrmlhlVbqTLd+3nm117ie7RhdMzEzQT7Jl7Ja04JycnxodFMz4smoLCQlycnfWW4TSnRjUPNo5fgkqlorCoiJe3TicpKUmzf/z48Qb77N69OykpKaSkpLBw4UK8vb1p27YtLi4unDt3TjMC6fHHH6dv374Gz33MmDHMnz+ftLQ0PvzwQ1q1akWDBg1QqVTs3LnTqusQQgghhBBlz5Z6tywmxlYCo55tWpo9tni962pmUmlD2jZqYXXdbk29GxERQb169fT6qFevHhEREWzatIktW7awZcsWQkND8fDw4O7duxbV7Y7KLkKjBQsW8Mknn5Camkrbtm2ZP3++0aXsFi9eTHx8PCdPngSgffv2fPTRRzrHjxgxguXLl+u0Cw8PZ+vWrWV3EQ+RowRGCgmOhCHRPbrwza69pR4cmbuBKorfSF/tYn1YZIgtNz9tTk5OuLq4aG48vr6+/OMf/8DT09Pg8c7OzgwfPpwtW7Zw+PBh0tPTSUhI0Dmmc+fOdO/e3ej1eXl5MWHCBJYuXUpWVhanTp3i1KlTOsdERkaW6LqEEEIIocvYasQlcexKDsPiLxFQtxrxzz9ODTcr6hJZidih2FLvlnZwpB0YdW9t+etXSr1bEtbW7ZbWuwEBAUb7aNeuHR4eHqxcuRKA/fv36+w3V7c7qnIPjVavXk1MTAxxcXEEBwcTGxtLeHg4Z86coW5d/XAhMTGRyMhIQkJCqFatGnPmzKF3796cOnUKX19fzXF9+vThm2++0fys/U6iI3O0wEghwZEorqG3V6kHR5YGRgrtG6m96dOnDy1atNCZeM8YZ2dn+vXrR3h4OMnJyZolPevUqUOLFi0sGg7r6enJ66+/TlZWFsnJyWRnZ1OlShUaN25Mw4YNK9TTEiGEEKKiCmzozndRjRkWf4moby9bHxwJh2FLvVuawZGtgVF5Ko16NyAggGnTpnHlyhUuXbpEfn4+Hh4eFtftjqjcQ6NPP/2UUaNGER0dDUBcXBybN29m6dKlTJo0Se/477//XufnJUuWsHbtWhISEoiKitJsd3NzMziszJHZS2B0PWkj9La+XWULjrbfSCvvU9Bx5fRug9t/Tk4E4HDyblxdbAttTl5Uv9f7s5HvMCSgTukGR9YGRgrlRmpvgoODrW7j6upK69Yle2pZq1Ytm75bCCGEsHftJ8ZbfOz1pI1c37+O+qGDqN/J/KIWtlhvft5fm1SU4MjeaumTF88A1tW72gr+t7S8UnvbQrtut6XeLa3gyBEDI20lrXednJxo1KgRjRo1KsWzsl/lGhrl5eVx5MgRJk+erNnm7OxMWFiYzjuBpuTk5JCfn4+Xl+7/6BMTE6lbty6PPPIIPXr0YObMmUaXucvNzSU3N1fzc1ZWlg1XU7bsKTC6vn8d9H7SpvaVKTj66MyZ8j4FXWbOZ8UO8yuFmRO/9TOLj535nHoFhdIIjmwNjBSlPaeSEEIIIRzXwwiMylpFCI7srpb+H2vqXUOSLx8l+fLREvWxYscXmlraUqUVHDl6YCSsV66hUXp6OoWFhfj4+Ohs9/Hx4bfffrOoj7fffpsGDRoQFham2danTx8GDRqEn58fFy5c4J133qFv374kJSVpZk3XNmvWLN5///2SXUwZs6fAqH7oIOCszf1UluBoe2jn8j4FHSfbv25w++Hk3azY8QUfj11l80ijrzbOBOCl/lMsbpNza4Xmc0mCo5IGRkIIIYQQiooQGCkcPTiyt1r61SvZgHX1rraCwnzeWvAs/+w1jqAW3W3qQ7tuz8v4wer2JQ2OJDCqnMr99bSSmD17NqtWrSIxMZFq1apptj/77LOaz61bt6ZNmzY0bdqUxMREevbsqdfP5MmTiYmJ0fyclZVld0PN7CkwUt9APylRf5UhOKpaCpMql6YqroZDGCUocnWpYvQYc5Q+bG0PtgVHlSkwevDgAT///DO7du0CwMXFhdatWxMaGoq3t7dFfaSnp7N9+3bOnTsHQI0aNQgJCaFdu3Y6v0OFEEKIyqgiBUYKRw6O7K2WLo16V+mnpDW3q0sV8mz8fluDI0cMjN5//32r612VSsX58+fZuHEj2dnqoNDX15ennnoKf39/i+Y9Mla3v/feeyW6nvJSrqGRt7c3Li4upKXpvq+alpZmdj6iuXPnMnv2bHbu3EmbNm1MHtukSRO8vb05f/68wdDIzc3N7ifKtq/AqHRUhuBIWMea4KgyBUYnTpxg3bp1OtsKCws5duwYx44do1mzZgwdOtTgSErl2A0bNmhWnVTcu3eP7du3s337dgYOHGjyd+nt27dLfiFCCCGEnaqIgZHCkYMjUTasDY4cMTBSWFvvLly4kMLCQp3tV69eZcUK9VsS48eP15saR5uput1RlWtoVLVqVdq3b09CQgIDBgwAoKioiISEBMaNG2e03ccff8yHH37Itm3bCAoKMvs9V65c4datW9SvX7+0Tv2hq2iBkUKCI1GcJcFReQRGhUWFHDh/hFv37vBkwyfwr9vY6j4ycrLYe/YQri4uhPgH4Vnd/AyYx48fZ/369QA0btyYfv364ezszL1797h27Rrbtm3j3LlzrFy5kueee07v6UdRURHx8fGkpKQA0KtXL1q1akVeXh5ZWVn8+uuvnDhxgvXr1+Pk5GRwIu3MzEzmz5/P559/bvU1CyGEEPauIgdGCgmORHGWBkfWBka21LvFnU29yKlrZ/Gp5U1wk3a4OFv3v1Wlbh8zZozV9a4iKioKLy8vsrOzycjIYP369RQUFDB//nwmTJiAp6enXh/m6nZHVe6vp8XExDB8+HCCgoLo2LEjsbGxZGdna1ZTi4qKwtfXl1mzZgEwZ84cpk2bxooVK2jcuDGpqamA+jWLGjVqcO/ePd5//30GDx5MvXr1uHDhAm+99Rb+/v6Eh4eX23U6ood1A5XgSBRnKjgqj8Do4MWjRC99U2970rsbLLoRqlQqpqyfy/pftups7x/Yi48Gv210mOuDBw80N55evXrRqVMnzbFeXl489thj+Pn5ERcXx4ULFzh16hRPPqk7Sf2JEyc0gdGLL76Ir6+vZt+jjz5K06ZN8fHxYefOnaxbt46AgAC9kZdLly41e41CCCGEI6oMgZFCgiNRnLngyNrA6N11n1hd72rLvH+XTh8O0Nv+zci5BDdpZ/6C0K3bp0+fbnW96+Pjw4gRIzSvsnl6etKgQQMCAgJYuHAhd+7cYc2aNYwaNUqnvSV1u6Mq9xdFhw4dyty5c5k2bRqBgYEcO3aMrVu3aibHTklJ4fr165rjFy1aRF5eHs888wz169fX/Jk7dy6gfl/w+PHjREREEBAQwAsvvED79u3Zu3ev3b+CZk8e9g301afq8kaPuszbdYPP99wo8+8T9k8JjtIyM1m+az+5+fnlEhilZd3U3Hg6N+vA5L+N0ewzdFMzZH7CMs0N9JVuw3ihy1AANh7bwdJ9a4y2O3pUvbJG69atCQkJMXiz9fHxYciQIQCsXbtWb/+GDRsA/cBIW0hICI8//jgAP//8s86+rKwsu1xRUgghhCipyhQYKZTg6OyNB0R9e5l7uYXmG4kKTQmOfDw9+WbXXq6kq6cksOWVNFvqXW3atfWUp8fTuVkHAKKXvsmNrHSz7YvX7dosrXdffPFFg3Mfubq6MnLkSACuXbumVx9bUrc7qnIPjQDGjRvH5cuXyc3N5eDBgwQHB2v2JSYmsmzZMs3Ply5dQqVS6f1RJpWqXr0627Zt48aNG+Tl5XHp0iW++uorvRXahHHldQOV4EgUpx0cLd6+h8Xb9zz0OYw++nEBANMiJvDV8Nk8HzKYUzN2avafTb1oto+4xO8A2Dvp/3g1LJo3wl9iz9vqFS/mbfvKaLsDBw4A0LZtW5P9t2jx54387t27Bj83aNDAaHsnJyeefvppABISEnT2nT59GkBvBJMQQgjhyOwhMCqveleCI1Fc8eDouz0HbJ7DyNp6V3Eu7ZLm86kZO/nnXwfw1fDZTO43FoAp6+ea7aN43a7N0nrX1dX4y1g1atSgU6dOACQnJ+vss7Rud0R2ERoJ+1HeN1AJjkRxDb29+GfXTqRmZJKakck/u3Z6qJNe7zi9F4C/tw3TbHNycmLK0+MBOH3tnMn2BVoT6dWp8Yjm86M1vQweo+3evXsA1KpVy+LzzcnJMfjZ3NMOZyMrlCh91KlTx+JzEEIIIexZede7ANnXL5RrvSvBkShOCY5y8wv47cp1mjesb9Ok19bWu4pTV88AMPlvY3Tq1iFB6qBn37mfDbbTZqhu11Ya9W716tUBNCurKWyp2x2FhEZCwx5uoCDBkdCVm59Pwq+nNT8n/Hqa3Pz8h/b9td3Vv/j/uK07ed3u35IAqFvL9M3FRevmVFj0581SpVIZPMaQqlWrWnayQBWtScNNPSkpTrnRFV+BTemvipFV7IQQQghHYg/1bvb1C5z74ZNyr3eLB0dCHPjtvObz72k3Na+qWcPWelcJm/YWC4cupqvn5nysjuFpFrQZq9sVpVHvKiP5jR1rTd3uKCQ0EoB93EC1SXAkQHfS61fCu/NKeHedOY4ehs8j3wdg0IKXSL5+ntz8PL5NWsf+84cBzE7K5+TkRLvHWgEwNG4sN+/eIiMnkwFfqCfPa+bjZ3QUUEBAAACnTp0y+R3aTzpq166t+fzII38+6cnIyDDZh7KiQ/FlSBs3bgzAzp07izcRQgghHIo91LtKYFTd29cu6l3t4EhUbtpzGE0dEqE3x5GlrK13FSH+7QH1iKJVh/5Ndu59kq+fZ/CClwH4oH+M2e8uXrcXZ0m9qx10GaLMh6S0UVhatzsiCY2EXdxADbGHG6koP4YmvTY0OXZZC/JrQ9/W3QAYvOBl2r3fl1mb1e9LLxs5z6IlQL+MUq/+ePraOZ6aM4SQjwZxLu13AL4f9S+j7Xr16gXAjh07uHr1qsFj8vLyiI+PByA4OFhn2K2zszMhISEAfPXVV+Qb+e+VlpbGtm3bAAgNDdXZ17BhQ7PXJ4QQQjiC8q53tQMj/2fUk/XaQ72rBEei8io+6bWxybEtYW29q3BxduGbkep5iz7YFEuHGU9rAqO+rbvRsUmg2T6K1+3aLK13d+3aZbT/Q4cOGWwDltXtjkpCo0rOXgMjhT3cSMXDZ2qVtPIIjuYNncqykfM0w2I7N+tA4lurLbp5AdSo5sGx97bwVt9XcP1fyBTT+0WOvbeFGtU8jLbz9vbWTEC9ZMkSLl++TFFRkWZ/RkYGsbGx3Lih/rvRtWtXvT6Um+L9+/dZsmQJmZmZOvuTk5OJi4sD1E9Iir/L7eTkRGRkpEXXKYQQQtgzewqMXKpW1+yzh3o3sKF7uXyvKH/GVkmzJTiypd7VFtykHbvfWkXvVuqa9hF3T+Jf+Ix5Q6dafD3F63awrt7dt28f+/fvJy8vT7O/oKCAH3/8kS1btgAQGRmpN3LKkrrdUVk+4YWocOw9MFK8+lRdAObtuqHzs6iYTAVGCiU4+mbXXpbv2s/wHqG4lfGcOx2bBLL19Xib21d1rcqI0H8wIvQfVrUbMGAAWVlZpKSk6KwkWdzYsWNxd9cv+Nzd3Rk7diwLFizgxo0bxMbGGmzftGlThgwZYnBfQEAAERERVp23EEIIYW/sMTBSSL0ryoOxwEihBEfLd+3nm117za5gbGu9q82n1qPERk63uT38Wbe3nNJTZ7sl9e6mTZvYuXOn0akZIiIiNK+iFWeubjdWh9s7GWlUSTlKYKSwhycwouxZEhgpymPEUXlwcXFhxIgRDBw40OC74KGhoUycOBFvb2+jfXh7ezNx4kTNEqHFDR48mOeee05vUkBt7dqZnrtJCCGEEPosCYwUUu+Kh8lcYKQoyatq9sTSevfll1/mscce09vXqFEjXn75ZZM1sbm63VHJSKNKyNECI4U8ganYrAmMFOUx4qg8ODk50aZNG9q0acPdu3fJycmhSpUq1K5d2+jSocW5u7vTu3dvwsLCyMjIID8/H3d3d2rWrFnGZy+EEEJUTtYERgqpd8XDYGlgpLB2xJG9eOWVV6yud+vVq0d0dDS5ublkZWUBUKtWLdzc3Cxqb6pud1QSGlUypRUYHbuSUy7vPjvSjfSdUyfL+xR0ZF2daXD7yYvqCd2+2jgTVxfbAheljy83GP4OQ57t1Ejz2ZbASFFZgiNFzZo1SxT0ODs74+Vl/zd5IYQQwpHZEhgpHKneLUv2VkufvK0eYWNNvautoFA9Kj5+62f8cna/beegVbcP69zEpj6sDYwUjhgc+fj42NzWzc2NRx99tETfX9K63V7I62mVSGmOMBoWf4ljV3JK6cysI0N3K5aSBEaKyvKqmhBCCCHsX0kCI4XUu8IcW+pdWwMjRUV5VU1YR0YaVRKl/UpaQN1qDIu/xHdRjWXEkREftXqyvE9Bx4kObxrc/vPp3cRv/YyX+k+hiqttI3SUJy4vD5hicZvMtOWlEhgpSjLiaPeJZAZZf98sU3fv3rXpyYStQ2mLf3dOTg6urq488sgjFr8CJ4QQQlR2pREYKRyh3i1L9lZLj7v6ALCu3tWWX5BPzOeDierzOh1adrepD+26PXbFS1bXuyUJjBSOOOLIkJLWu0VFRdy5c4eCgoIKP+WDhEaVQFnMYRT//ONEfXvZroIjBtR66OcgbFeagZHCluBIuYFieCGFcvPpp58C6lUY2rRpY3YyvdTUVDZv3syVK1d0ttevX5+IiAjq1atnsr1KpeLkyZOsW7dOb19wcDBdu3a18gqEEEKIyqU0AyNFZQ+OhHHKCHtr6t2SBkYKRw2OLKl3Da1GrC0nJ4f9+/dz4MABvX2W1u2ORh4fV3BlNel1DTcX4p9/XDPiyB5eVROOpbQDI4U1r6pp30Dt1YYNG1i2bBmFhYVGjzl69ChffvmlXmAEcP36db788kuOHj1qtH1hYSHff/+9wRsowMGDB/nkk0+sP3khhBCikiiLwEghr6oJQ2ypd0sjMFI42qtqlta76enpRvtIT0/nk08+MRgYgWV1uyOSkUYVWFmvkqYER/Yy4ijpoX+zKImyCIwUlow4Kn4DXbxnJZ/tWIKLszOv9x7FsL8OoKprVYu/80ZWOu+u+4T95w/TyKsBHwyIIbiJdcvUH/79OK+unE5GThbTpk3j2rVrLFmyhJSUFDZs2MDgwYP12pw5c4ZNmzYBEBQURHh4OK6u6l/thYWFHDx4kB07drBp0yY8PDwICAjQ62P16tVcuHABgCFDhtCixZ8FRXZ2NvHx8dy4IUWqEEIIYUhZBkYKGXEkirOl3i1txUccvdrFfBvterdXyy688/RYfGpZPuF0XkEey/evtbput7TeXbBgARMnTtQbcZSTk8OCBQsA9arEo0aN0qyIplKpLKrbHZWMNKqgyjowUtjTiCPhWMp6GKupEUeGbqCf7VgCQGFREXO3fknge3259yDbou86ePEo3T4eyv7zhwH44/Y1ope+yRurZ1h8vjGrZhD19etk5KjnI3JycsLX15cXX3wRgJMnT3Lr1i2dNiqVilWrVgHQt29f+vXrpwmMAFxcXAgJCaFz584ArFy5EpVKpdNHeno6586dA2D06NE6N1AADw8PXnjhBapXL/0CWAghhHB0DyMwUsiII1GctfVuWdAecWRO8Xp3x+m9dP/4WQ5eND4iXtu9B9kEvtfX6rrd2nrX0Eii//73vwDUrVuXCRMmaAIjsLxud1QSGlVADyswUthLcCQcy8N479nQjdTYDbSZjx8H3lnHnrd/oGWDZgC8HD/Z7HcUFhUSvVQ9yfg7/cZxdPoW1o39CoAtJxI5/Ptxs30cvHiUrScTAVg39it+nvqjZp+vry9hYWEAbN++Xaed9utoHTt2NNp/jx49DLYB2L9fveRreHg4desaDl+rVq3KSy+9ZPY6hBBCiMrkYQZGCgmORHHW1LtlRQmOTDFU706LmABA9NI3KSwy/0rXP796FbC+bre23t2/fz9FRUWafUVFRRw8eBCAqKgoqhiZQ8rSut3RSGhUwTzswEghwZGwV9o30hlrNhm9gW4Yt5ja7p48WtOL1a+oh54eTTll9qmA8mSkc7MODOs0ELcqVWlevylrx34JwGsr3zN7jtM2qCe9Xjf2K5rXb4qHm27R+eST6tVDzp49q7P90qVLgOnACNRPP5QbmNJGceLECQAaNGhgsg/tpylCCCFEZVcegZFCgiNRnKX1blkyNxm3oXr32Y5/p3OzDgAcOH/EZHuVSsX5G5cA6+t2W+rdjIwMzec7d+5oPnt4eJjsw1zd7ogkNKpAyiswUkhwJOxVQ28v/Hz+fFc6pLm/3jHaqxy4OLtoPhdqPWUwJC1LPVletyf+qrP9MS9fAO7kZJo9vz9uXwPAz7uRwf25ubkGt+f/bwhyjRo1zH5HXl6eThuFMlGfJX0IIYQQonwDI4UER6I4S+rd8mSs3lVCo1v37ui10aZdk1tbt9tS72rXzAUFBRa3M1e3OyIJjSqI8g6MFBIcCXu0+0Qyv125TvOG9XGr4mpwlYmbd/9c8UH7puXq4oIprRqoJ5ae+eN8nacb/z62A4DercwvVR/qHwTAD4c3G9x/9+5dAGrWrKmzXZmg7/79+2a/Q3mv2tjTkSIz4Zgjv4cthBBClBZ7CIwUEhwJbZbUu+XJUL2rUqmY/Z+FADzZ8AmT7bVrclvrdmvqXe2JsItPim2KubrdEUloVAHYS2CkkOBI2BPtd7qHPRVidLLAp+b8g7lbv+RfO5bSZfYzAIzuNsxs/wH1mmg+t5oaRvyBtby0fBIf/PtfAEzuN8ZsHx8OmgjAR5u/YNSyt1nx0wad/ceOHQMgJCREZ3vLli0BSEpK4t69e0b7Lygo4NSpUwB6E/8p8x1t3rzZZDB09epVs9chhBBCVGT2FBgpJDgSYHm9W5aupN82ud9Qvdtqaphmv3/dxma/441w9ZxD1tbtttS72qGP9ufk5GST52iubndEEho5OHsLjBQSHAlzdp8w/Qu3tL6j+DvdplaZWLpvDV/u+R6AQX/pw7ieIyz6nqR3N2g+z/7PQvad+xmAb0bOtWgJ0bq1vPlm5FwA9p8/zMwf5wPqpx379+/n5MmTALRr106nXa1atTTvZi9dutTg0NkHDx6wZMkSzfG1atXS2d+hg3pI8KVLlwyuFAHqG+jXX39t9jqEEEKIisoeAyOFBEeVm7X1blm4kn6bb3btNXmMsXoXdGtpU0Z2HkL/wF6AdXW7tfXuwIED9fYPGjQIgDVr1pCWlqa335K63VG5mj/EvKysLHbt2sUTTzyh9xRblB17DYwUSnAU9e1lhsVf4ruoxgQ2dNxheaJ0JRw/DVBmE/SZWjVCuZF+s2svy3ftZ3iPUJLe3cCB84cpLCqiS0BHPKvXNNKzPs/qNTk9M4HzNy5x8soZ6tR4hBD/9jrvWJsT3KQdJz7YzsGLR7mRdYur6en8+OOPXL58GVDfqNzc3PTaDRkyhNjYWO7cucPs2bOJiIigUaNGuLi4cOvWLeLj4zXHjhw5Uq99tWrVGDRoEOvWrWPnzp1cvXqVoKAg6tSpA6iXDN25c6fF1yGEEEJUNPYcGClefUq9ItS8XTd0fhYVm7X1rrnJqm2hBEY+np5mjy1e77byDaCZj5/F3+Xk5MSsZyYxqd9Yq+p2a+rdxx57jNatW+v18eSTT/Lrr79y4cIF4uLi6NGjB82aNcPd3Z28vDyL6nZHZVNoNGTIELp27cq4ceO4f/8+QUFBXLp0CZVKxapVqxg8eHBpn6coxt4DI4UER8KYnm1alllwZMkyo8VvpJFtXqNv6+4l+l7/uo0tGlprjIuzCyH/e9+75ZSemu2DBg0yePMC8PT0ZPz48cyfP5/CwkLWr1+v36+LC2PGjMHTyM28devWqFQq1q9fT3JyssFht8a+XwghhKjIHCEwUkhwVLnYUu+WdnCkHRgN7xFqURvtetdWntVrWl23W1rv9u/fX2eibYWTkxORkZGsWbOGs2fPsmvXLnbt2qV3nKm63VHZFBr997//5d133wVg/fr1qFQqMjIyWL58OTNnzpTQqIw5SmCkkOBIGKLc3Eo7OLLkBqrQvpHao549e9KhQwezTyq8vLyYNm0aFy5cYM+ePVy5cgVQT3rdv39//P39Dd78tLVp04aAgACOHz/O7t27efDgAQABAQH07t1b8yRGCCGEqCwcKTBSSHBUedhS75ZmcFQ8MCqLUUylraT1rouLC5GRkaSnp/PTTz9x5MgRzT5L63ZHZFNolJmZiZeXFwBbt25l8ODBuLu7069fPyZOnFiqJyh02UNglH39Alj+5g5QOYOjPDOz8z9s+QWG32UuKMzX+b+2UNoa+w5jSjs4siYwUig3Unszffp0q453cnLC398ff3/bl1etVq0aHTt2pGPHjjb3IYQQQlQEjhgYKSpKcGRvtbSt9W7x9gWF+aXShy31bmkFR44YGClKo9719vbm6aef5umnny7FM7NfNoVGjRo1IikpCS8vL7Zu3cqqVasAuHPnDtWqVSvVExR/spfA6NwPn8C7Ta1uW9mCo97795X3Kegycz5vLXi2xF8R87nlowxnPqc+trSCI1sCI0VDby+bvlMIIYQQFY8jB0aKihAc2V0t/T/W1LuGrNjxBSt2fFGiPt5a8KymlrZUaQVHjhwYCdvYFBpNmDCB5557jho1avDYY4/RrVs3QP3aWkV7f89e2FNgVN3b1+Y+KlNw9M4TT5T3Kei44vc3g9t/Tk4k+fJR/tlrHK4utv3Sj9/6GQBRfV63olWK5lNJg6OSBEZCCCGEEIqKEBgpHD04srda+qMzZwBr690/FRTms2LHF7R4vB0dWnSzqQ/tuh2uW92+pMGRBEaVk02h0ZgxY+jYsSN//PEHvXr1wtnZGYAmTZowc+bMUj1BoWZPgZH/M28CtqfjlSU46l3Xp7xPQceJlsYni0u+fJSgFt2p4mrbL/5fzu4HoIOJ7yguM225zs+2BkeVKTDKysri9OnT5OTkUKVKFRo3bkzDhg3NzlekUKlU/PHHH1y+fJn8/Hzc3d1p2bKlQy8BKoQQQpSWihQYKRw5OLK3WnpTvnpBEWvqXW35BerQqEOLbjb3AX/W7Tm3VtjU3tbgyBEDo127dtlU7xYUFJCcnMytW7cAqFOnDi1atMDV1fL4xFDd7qhsCo0AgoKCaNOmDb///jtNmzbF1dWVfv36lea5CS32FBiVxg20sgRHwjrWBkeVJTDKzMxkzZo1XLt2zeD+yMhIAgICTPZx9uxZVq5cqbd927Zt1KpVi5EjRxpdXQ2gyM7mFRBCCCFKU0UMjBSOHByJsmFtcOSIgRHA3r3qxW6sqXd3797Nvn2GX40MCgqib9++mkEzhpiq27/++msrr8A+GL9aE3JycnjhhRdwd3enVatWpKSoXzMZP348s2fPLtUTFGoVKTBSKMFRQN1qDIu/xLErOaXWt3Bc3Vu3oGebliQcP83uE/pLYSrKIzBSqVQUFBaWqI+ioiKrApjbt28TGxurufF06tSJHj160KFDB80xK1eu5OjRo0b7OHr0qE5gFBYWxlNPPUWrVq0A9ZOQ2NhYbt++bfScly9fbnCfEEII4egqcmCkePWpurzRoy7zdt3g8z03yvt0hB1QgqO0zEyW79pPbr7hybltCYysrXcNKSgsRKVS2dxepVLZVO8qgZGfnx+9evUiNDSU+vXrA3D48GGWL19u9NosqdsdkU0jjSZPnsyvv/5KYmIiffr00WwPCwvjvffeY9KkSaV2gqL8PIwbqIw4EoaYG3H0sAOjew+yeWn5JI79cVqz7ZVuwxjfc4TFr4advHqGIYvG6Gxb+fIXtG1k/PxVKhXz588HwMvLi+joaGrUqKHZ/7e//Y1Dhw6xZcsWNm3aRP369alXr55OH6mpqWzatAmALl260L17d51zHjBgAEuWLCEtLY2FCxfy7rvv6l3Tli1bNA8HhBBCiIqkMgRGChlxJIozN+LI2sDIlnpXm0qlYum+Nczb9pVmm3/dxqx46XNqVPOwqA/tul1ZjdjaejcqKgo/Pz+d/enp6SxYsICUlBR2795Nz5499c7dXN3uqGwaabRhwwa++OILOnfurPMfu1WrVly4cKHUTk6Un4d5A5URR8IQYyOOHnZglFeQR8eZETqBEUBc4ndMXT/Xoj5+/SNZ7wYKEPnlOE5dPWu03fnz5wFwdXVl9OjROjceRceOHWnfvj0A//nPf/T2K4FR79696dGjh94N0tXVlREjRgBQWFio+U5Ffn4+hw8fNnF1QgghhGOqTIGRQkYcieKMjTiyZYSRLfWutnfWztEJjADO37hEx5kR5BXkmW1vrG63pt4dO3asXmAE4O3tTVRUFAD79u2joKBA9zwtqNsdlU2h0c2bN6lbVz+Zzs7Otvipu7Bf5XEDleBIGFI8OCqPV9K++2kDAK0aBHDig+2cnpnA3kn/B8C6X7aSef+u2T4ivxwHQGzkdE7PTOD0zARiI9VPPv6xaLTRdnv27AFg4MCBJifeU0Z8/vHHH+Tm5mq25+bmcv26emWNjh07Gm1frVo1zU1w48aNOvt+++03AM2wXCGEEKIisIfAqLzqXQmORHHFg6MLqTdsnsPI2npXkXn/LhuP7QBgz9s/cHpmAqdm7KSZjzrA+TZpndk+itft2iytd729vY327+fnpwmUkpN1p9GwtG53RDaFRkFBQWzevFnzsxIULVmyhE6dOpXOmYlyUZ43UAmOhCHawVF5THr92fbFACx8/kNcnF0AqFPjEV5+6jkA9p49ZLK99jvPvVt1NfjZ2HvRV69eBaB27domv0P7xpSVlWXws4uLi8k+vLy8AHX4r01ZNaJJkyYm2wshhBCOwh4Co8K8++Va70pwJIpTgqOU9Ft8k7CXRzw8bJr02tp6V3HgvHqkzwtdhvJoTXVd6uTkxPIX5gEwb9tis99tqG7XVhr1rr+/v04bhaV1uyOyKQL76KOP6Nu3L6dPn6agoIB//etfnD59mgMHDmgSNuF47OEGKnMcCXtT+L8bXJVioUt+oXrorouJ1RNKi4eHZe9wl0Txm2dx7u7y91AIIYTjs4d6tzDvPuf/by7N//egtLzq3eJzHDHA8iXJhShtymIzVVx0Q6r8QvVrYK4GQqDijNXtitKod8318TDq9ofNpn/tdO7cmV9//ZWCggJat27N9u3bqVu3LklJSZq5NYRjsYcbqEJGHAlt2q+kWbKqWml7vdeLAAz/+g3NCg43795m6b41AIT4B5lsr70k5/ZT/zX42diyncpNx9gqD4pCrRXdatWqZfBz8feui8vIyACgYcOGOtvr1KkDoPfutxBCCOFo7KHeVQKj++lX7aLe1R5xJCo3ZQ6jx7zrEN2zC3eys02uqmaMtfWuousTwYB63tBb9+4AUFhUyOhv3wUgJnyU2e82VLdrK4169+LFizptFJbW7Y7I6tAoPz+fkSNH4uTkxOLFizl06BCnT5/mu+++o3Xr1mVxjqKM2cMNtDgJjgToT3ptbHLssjQ8dDAA59J+p9XUMFpO6clTc/4BQP/AXnhWr2m2jx9GLwJgwsr3aTmlJy2n9GTCyvcB9YoSxvTv3x+A+Ph4Hjx4YPS4gwcPAtCgQQPc3Nw0293c3DQ3xW3bthltX1BQwPr16wHo2rWrzr4WLdSvAv7+++9G2wshhBCOoLzrXe3AqNk/JtpNvasER6LyKj7pddN6dQ1Ojm0Ja+tdhWf1mgz8i3qezi6zn6HllJ60ntab09fOAfDP4P5m+yhet2uztN41VfPevHmT1NRUnTYKS+t2R2R1aFSlShXWrl1bFuciyoE9BkYKe7mRivJhbNLrhx0cVXWtyqEpmzST8Cne7PMyHw1+26I+WvkGGLxZ/jB6kcklSP39/TVzES1btkxvtJBKpSIhIYEdO9STBv7973/X66Nfv34AHD58mEOH9OdfunfvHgsXLtT0rbynrXB1daVz586mLk8IIYRwCPYUGHnUbwrYT72rvKomKh9jq6QZW1XNFFvqXW0zB77J6G7DdLb95bEn+Xnqv6nqWtVse2N1uzX1bnx8PDdv3tTr++LFiyxcuBBQz/FcfLJrS+p2R2XTnEYDBgxgw4YNvP7666VyEgsWLOCTTz4hNTWVtm3bMn/+fKMr/SxevJj4+HhOnjwJQPv27fnoo490jlepVEyfPp3FixeTkZFBaGgoixYtolmzZqVyvhWFPQdGCpnjqHIyt0qasi3h+Gmdn8tKjWoebBy/BJVKRWFREa5mJpU2pG2jFpyemaCZBNDcEF1QT/43ZswY5s+fT1paGh9++CGtWrWiQYMGqFQqdu7cqTk2IiKCevXq6fVRr149IiIi2LRpE1u2bGHLli2Ehobi4eHB3bt3SUpK0hw7fvx4gytgdu/enZSUFKuvWQghhLAn9hYYKaTeFeXFWGCkUIKjb3btZfmu/WYnxral3tXm5OTE+LBoxodFU1BYiIuzs9Wrs2vX7a/v/djqejclJYWFCxfi7e1N27ZtcXFx4dy5c5oRSI8//jh9+/Y1eO7m6vb333/fqmuxFzaFRs2aNeODDz5g//79tG/fXm+yp1dffdXivlavXk1MTAxxcXEEBwcTGxtLeHg4Z86coW5d/cQ7MTGRyMhIQkJCqFatGnPmzKF3796cOnUKX19fAD7++GM+//xzli9fjp+fH1OnTiU8PJzTp09TrVo1Wy65wnGEwEghN9LKxVxgpHjYwRGobwa2BEbarL15enl5MWHCBJYuXUpWVhanTp3i1KlTOsdERkYSEBBgtI927drh4eHBypUrAdi/f7/Ofl9fX/7xj3/g6elp9JyHDx9u1XkLIYQQ9sYeAyOF1LviYTMXGCmsDY7A+nrXkJLW3E5OTmzfvl3zs6X17pYtWzh8+DDp6ekkJCToHNO5c2e6d+9u9PosqdsdkU2h0ddff03t2rU5cuQIR44c0dnn5ORkVWj06aefMmrUKKKjowGIi4tj8+bNLF26lEmTJukd//333+v8vGTJEtauXUtCQgJRUVGoVCpiY2OZMmWKznuFPj4+bNiwgWeffVavz9zcXHJzczU/ay9TXRE5UmCkkBtp5WBpYKQoj+CoPHh6evL666+TlZVFcnIy2dnZVKlShcaNG9OwYUOLnsAEBAQwbdo0rly5wqVLl8jPz8fDw4MWLVroTJhtTGnc/IUQQojKxNLASCH1rnhYLA2MFLYER/agS5cuVte7/fr1Izw8nOTkZG7dugWoJ71u0aKF3itphpiq2x2VTaFRaU2ImpeXx5EjR5g8ebJmm7OzM2FhYTpDyEzJyckhPz8fLy8vzbmlpqYSFvbnxFeenp4EBweTlJRkMDSaNWuWww4Vs1ZpBUb3cgup4Vay9NdajnYj3X4jrbxPQceV07sNbv85ORGAw8m7cXWx7Zf/yYvquXJ+NvIdhgToLjhgdWCkqCzBEahXQwsODra5vZOTE40aNaJRo0aleFZCCCGEKM7awEjhaPVuWbK3WvrkxTOAdfWutoJC9ZxASu1tC+26vaWN01BZGxgpHDE46tGjh03tXF1dS7zIV0nrdntiU2ikTZnQydp3DQHS09MpLCzEx8dHZ7uPjw+//fabRX28/fbbNGjQQBMSKbOZG+pT2Vfc5MmTiYmJ0fyclZVVIf9RVZojjKK+vUz8849LcGTCR2fOlPcp6DJzPit2mF/VwJz4rZ9ZfOzM5wZrPtsaGCkqU3AkhBBCCPtma2CkcKR6tyzZXS39P9bUu4YkXz5K8uWjJepjxY4vdGppS9kaGCkcMTgSJWdzaBQfH88nn3zCuXPqJfACAgKYOHEizz//fKmdnDmzZ89m1apVJCYmlmiuIjc3N51lqiui0n4l7eyNBxIcmbE91L5WnDrZ3vDE9YeTd7Nixxd8PHaVzSONvto4E4CX+k+xuE3OrRVAyQMjRUmCoyvpt23+XiGEEEIIRUkDI4Wj1Ltlyd5q6VevZAPW1bvaCgrzeWvBs/yz1ziCWnS3qQ/tun3b3g+srndLEhgpJDiqfGwKjT799FOmTp3KuHHjCA0NBWDfvn288sorpKenW7yqmre3Ny4uLqSl6Q49TEtLM7gKkLa5c+cye/Zsdu7cSZs2bTTblXZpaWnUr19fp8/AwECLzquiKYs5jL6Lasyw+Et2FRzxiPn3VB+mqnY2B0wVV8O/zJWgyNWlitFjzFH6sLZ9aQVGCluCI+UG+mqXEn99qZo3bx4hISG0a9fOolBcpVJx/vx5Nm7cSHa2uqjx9fXlqaeewt/f36LRoA8ePODnn39m165dALi4uNC6dWtCQ0Px9vYu2QUJIYQQFVxpBUaKyh4c2VstbWu9a6ifktbcri5VbKp3SxoYKRw5OCqNejc9PZ3t27drBtDUqFHDqrrd0dj0N3H+/PksWrSIOXPmEBERQUREBB9//DELFy7k888/t7ifqlWr0r59e51ZyYuKikhISKBTp05G23388cfMmDGDrVu3EhQUpLPPz8+PevXq6fSZlZXFwYMHTfZZUZXVpNeBDd35LqqxZsTRvdzCUunXGsqNNKBuNYbFX3ro3y9KprQDI0X31i3o2aYlCcdPs/tEssljtW+g9ubevXts376dOXPmcPz4cZPH3r59mw8//JAVK1ZoAiOAq1evsmLFCj744ANu3zY9murEiRPMmTNHcwMFKCws5NixYyxYsIAVK1aU7IKEEEKICqy0AyNF8Xr32JWcUulXOD5b6t3SDHeU4CgtM5Plu/aTm59fKv2WJUvq3cJC4/+uLSwsZO3atSxYsEATGIF1dbsjsmmk0fXr1wkJCdHbHhISwvXr163qKyYmhuHDhxMUFETHjh2JjY0lOztbs5paVFQUvr6+zJo1C4A5c+Ywbdo0VqxYQePGjTXzFNWoUYMaNWrg5OTEhAkTmDlzJs2aNcPPz4+pU6fSoEEDBgwYYMvlOqyyXiVNCY7sZcSRcCxlERgpLBlxVPwGmpGTxd6zh3B1cSHEPwjP6jWt/t6zqRc5de0sPrW8CW7SDhdn6/4+FBYVcuD8EW7du8OwYcP49ddfOXHiBOvXr8fJycnghHyZmZnMnz9f83NUVBReXl5kZ2eTkZHB+vXrKSgoYP78+UyYMMHgMqPHjx9n/fr1ADRu3Jh+/frh7OzMvXv3uHbtGtu2bdO5MQohhBDiT2UVGCkq+4gjYZgt9W5pjwYqPuIoss1rZtto17tPNnwC/7qNrf5eW+p2S+vdlStX8txzz+mN0i8qKiI+Pp6UlBQAevXqRatWrcjLyyMrK8uiut1R2RQa+fv7s2bNGt555x2d7atXr6ZZs2ZW9TV06FBu3rzJtGnTSE1NJTAwkK1bt2omsk5JSdFZ6nnRokXk5eXxzDPP6PQzffp03nvvPQDeeustsrOzeemll8jIyKBz585s3bq1Qg4VM6asAyOFPQVHox7qt4qSKqvASGHqRmroBhry0UCd9v0De/HR4Lcteq0r8/5dOn04QG/7NyPnEtyknUXne/DiUaKXvqn5efr06TRt2hQfHx927tzJunXrCAgI0Jt/benSpYB6sv8RI0Zofs95enrSoEEDAgICWLhwIXfu3GHNmjWMGqX7N+XBgweaG2ivXr3o1KmT5pq9vLx47LHH8PPzIy4uzqLrEEIIISqTsg6MFBIcCUOsrXfLgnZwZE7xeleR9O4Gi4IflUrFlPVzWf/LVp3t5up2a+rdCxcucOrUKZ588kmdPk6cOKEJjF588UV8fX01+x599FGL6nZHZdPrae+//z7Tpk2jT58+zJgxgxkzZtCnTx/ef/99PvjgA6v7GzduHJcvXyY3N5eDBw/qLE2XmJjIsmXLND9funQJlUql90cJjEC9ktsHH3xAamoqDx48YOfOnQQEBNhyqQ7pYQVGCnt5VU04loexwpmhV9VM3UBf6TaMF7oMBWDjsR0s3bfGou/RDoymPD2ezs06ABC99E1uZKWbbZ+WdVNzA+3crAOT/zZGsy8kJITHH38cgJ9//lmnXVZWFllZWYD65mUoGHd1dWXkyJEAXLt2TXO84uhR9eodrVu3JiQkxODN1sfHhyFDhpi9DiGEEKIyeViBkUJeVROGWFvvlgUlODLFVL1r6OGrIfMTlmkCI2vqdmvr3bVr1+rt37BhA6AfGGkzV7c7KptCo8GDB3Pw4EG8vb3ZsGEDGzZswNvbm0OHDjFw4EDzHYgy87ADI4U9BEdCGKJ9I/1uzwGjN9C9k/6PV8OieSP8Jfa8/QMA87Z9Zbb/c2mXNJ9PzdjJP/86gK+Gz2Zyv7EATFk/12wfH/24AIBpERP4avhsng/5cwlVJycnnn76aQCdudoATp9WP1V68skncXU1PnC0Ro0amjndkpN133s/cOAAAG3btjV5ji1alH3IJ4QQQjiKhx0YKSQ4EoZYWu+WpYbeXib3G6p3T83Yqdl/NvWi2e+IS/wOsL5ut6XevXv3rsHPDRo0MNrekrrdEdk8JX379u357rvvOHLkCEeOHOG7776jXTvLXsMQZaO8AiOFBEfCXnVv3YLmDevz25Xr5OYXGLyB1qnxiObzozX/vOkVmJgMD+DU1TMATP7bGJ2nFkOC1DeMfed+NthO247T6uG8f28bZnC/s5HVQ3Jy1IVinTp1zH5H9erq3wfaE2WDeuI+gFq17Gv1QSGEEMJelVdgpJDgSBhiSb1bngzVu05OTkx5ejwAp6+Znj9Tuya3tm63pd5V6uzin81NXWGubndENoVG//nPf9i2bZve9m3btrFly5YSn5SwXnkHRgoJjoQ9upJ+m9/Tbmp+PvDbeb1jCov+/N+qSqXSfHYxs9yrctPaWywcupiufuf5sTqGh69qq+2uvoH9cfuawf3Kjc7FRfc1zCr/KwSqWFAQKE9IjB1btWpVs30IIYQQlV15B0YKCY5EcZbUu+XJWL27+7ckAOrWMv0QVLsmt7Vut6be1a6ZTY3oL85c3e6IbAqNJk2aZHApOpVKxaRJk0p8UsI69hIYKSQ4EvZE+53uqUMijC5POjRuLDfv3iIjJ5MBX6gni27m42f2aUKIf3tAPaJo1aF/k517n+Tr5xm84GUAPugfY/YcP498H4BBC14i+fp5cvPzdPZfu6a+ubZp00Zne+PGjQHYuXOnzg3TEOW9aqWNQpnv7dSpUybbFx+hJIQQQlQ29hIYKSQ4EgpL692ylJufb3K/oXr326R17D9/GMDs4jFOTk60e6wVYH3dbku9W7t2bc3nRx75c2RTRkaGyT7M1e2OyKbQ6Ny5c7Rs2VJve/PmzTl/3r4SzYrO3gIjhQRHwpwr6bcfyncUf6fb0GSBoB4S+9ScIYR8NIhzab8D8P2of5n9DhdnF74ZqZ636INNsXSY8bQmMOrbuhsdmwSa7SPIrw19W3cDYPCCl2n3fl/NvrS0NM3IztDQUJ12DRs21HzetWuX0f4PHTpksA2oV5AA2LFjB1evXjXYPi8vj/j4eLPXIYQQQlRU9hYYKSQ4EtbUu2UlNz+f5bv2mzzGUL07a7N6nqNlI+fh4mx+YaMvo2YB1tft1ta7wcHBOq+ZOTs7ExISAsBXX31FvpGAzNK63dHYFBp5enpy8aL+RFXnz5/Hw8OjxCclLGOvgZFCgiNhyje79pZpcGRq1QhDN9K3+r6C6/9uVjG9X+TYe1uoUc2y32fBTdqx+61V9G7VFYBH3D2Jf+Ez5g2davH5zhs6lWUj5+m8zpacnKxZ6j4gIEBv7iInJyciIyMB2LdvH/v37ycv789RSgUFBfz444+a14YjIyP1nsB4e3trlhRdsmQJly9fpqioSLM/IyOD2NhYbty4YfG1CCGEEBWJvQZGCgmOKi9r692yoARGaZmZZo8tXu92btaBxLdWW/SQFaBGNQ+OvbfF6rrd2nq3a9euen0oIdD9+/dZsmQJmcWu15K63VFZ/nKelv79+zNhwgTWr19P06bqX5rnz5/njTfeICIiolRPUBhm74GRQgmOhsVfIurby8Q//zg13MynyKLi8/H05Jtde4nu0cXsagvWsmSZ0e6t1asjJBxXr2Qwf8gYRoT+w+bv9Kn1KLGR021uD9CxSSBbX1c/4Wg5padme9OmTY0ueR8QEEBERASbNm1i586d7Ny50+BxERERmqG5xQ0YMICsrCxSUlJYtmxZia5BCCGEqEjsPTBSKMFR1LeXGRZ/ie+iGhPY0L28T0uUIVvqXeXn0qIdGEX36GJRG+161xZVXasyIvQfVtftlta7Y8eOxd1d/++Ou7s7Y8eOZcGCBdy4cYPY2FiD7U3V7Y7KppFGH3/8MR4eHjRv3hw/Pz/8/Pxo3rw5derUYe5c88tLi5JxlMBIISOOhCHDe4RqgqPSHHFkyQ1Uof0Exl4NHjyY5557Tm8yPW3t2rXj5Zdf5rHHHtPb16hRI15++WWTq1u6uLgwYsQIBg4caPApSGhoKBMnTrTtAoQQQggH5SiBkUJGHFUuttS7pTniqHhgVNoPgUubpfWut7e30T68vb2ZOHEinTp1MrjfkrrdEdk00sjT05MDBw6wY8cOfv31V6pXr07btm3p0sWydFHYzh4Co8K8+1a3kRFHoji3KlUY3iOU5bv2l9qII2sCI0VpP3EpLTExMdSsWdPi4+vVq0d0dDS5ublkZWUB6mVF3dzcLGrv5OREmzZtaNOmDXfv3iUnJ4cqVapQu3Zto0uHCiGEEBWVowVGChlxVHnYUu+W1ogjRwuMFKVR77q7u9O7d2/CwsLIyMggPz8fd3d3q+p2R2NVaJSUlMStW7d4+umncXJyonfv3ly/fp3p06eTk5PDgAEDmD9/vsX/SBHWsZfA6Pz/zYWXq1ndtrIFR++cOlnep6Aj6+pMg9tPXlRPkvzVxpm4uti2FKTSx5cbDH+HIc92alSqwZEtgZHCHoMjW288bm5uPProoyX+7op84xNCCCFMcdTASFFRgiN7q6VP3laPjLem3tVWUKiePDl+62f8ctb0pNFGz0Grbre13i1pcOSogVFxJa13nZ2d8fJyzGu3llWPjz/44AOdZepOnDjBqFGj6NWrF5MmTeLf//43s2bNKvWTFPYVGN1PNzzjvCXkVTVRnBIcleRVtZIERkIIIYQQCkcPjBTyqlrFZ0u9W9JX1SpKYCSsY9VIo/9n78zDa7rWP/7JQBBEScU8NVJTDNeQK6gaQ7kEPzStpkVpFb2q1VFpS6u9aNMSN1QNaS/KLeFWDZGghCoVJaTGElMSMQUhw8n5/XG6jzPsM+YM+yTr8zx5erL3Wmvvrbzre757rfc9cuQIs2bN0v6+Zs0aOnXqxNdffw1ocmfMnDmTDz74wKE3KUBRhlHT4dOA9XaPVVZWHH3SspW7b0GPYx3fkD1+8MRO4rd+wfjB0ynna5/ZIr1xeSlyutV9bmet1H4uyYqjsmYYSUtpfX19eeSRR2zeOlZcXMzNmzcpKioq9UtpBQKBQCCwhdJiGEl4+oojpWnpSZcfALbpXV0KiwqZ+tUwovu9RscWPewaQ1e3511fZdcY9q448kTDKCsry269a2/KB10MdbunYpNpdPPmTYKCgrS/7969m/79+2t/79ixIxcvXnTc3Qm0KMkwcsQEWlaMI4H12GMclRXDSK1Wk5aWxvr1xmZtWFgYTzzxhGyVB13y8vJISUlh3759RuciIyNp3bq1R5cCFQgEAoGgJJQ2w0jC040jgXOw1TjyRMMIIC4uTvvZWr2bmZnJ5s2buXTpkt7x2rVrM2jQIGrVqmW2vznd/tFHH9lw98rBplfUQUFB/PnnnwAUFBRw+PBh/v73v2vP37lzh3Kl9EubuylNhpGE2KomMMSWrWplxTBSqVT85z//kZ14AA4cOMDcuXPJyckxOUZOTg5z586VNYwAEhISWLFiBSqV6X+Dqamptt24QCAQCAQeQmk1jCTEVjWBHNZuVfNUw8gQa/Xu4sWLjQwjgKtXr7J48WKzmtiSbvdUbFpp9NRTT/H222/z2WefkZCQQKVKlfQqph09epTHHitdQVYplDbDSEKsOBIYYs2KI3cYRof+PMqrq2dyKy+XPi268e7AiQRVtT7hdEFRAStTfuCLxKX4eHvzWt9xjPp7JOV9y5vt9/3333P27FkARowYQfPmD98E3bt3j/j4eLKzs4mNjWXatGlGK47y8vKIjY0FNNUexo0bR7Vq1QDNm5ArV66wdOlSMjIySEhIYNiwYUb3cPLkSTZt2mT1swoEAoFA4CmUdsNIQqw4EshhacWRrYaRvXpXl+zcHN5bP5eUM4eoX70OH0VOJaxJO5ueS9LtM2fOtEvvdujQgYiICHx9NXaJSqXiwIEDJCYmsmnTJvz9/QkJCTEaw5Ju91RsWmk0a9YsfH196d69O19//TVff/015cs//AuwbNky+vbt6/CbFLgeV06gYsWRwBBzK47cYRhNXTOL6G9e41aeZl9z4ok99PjX0xw4Z93qm7sP7tH2g/58kbgUAFVxMfO2LqbtB/25+8D0BJKTk8Pp06cBmDBhgt7EA+Dv78/YsWOpWFFjKsutJPr5558BqFmzJlOmTNEaRqApO1q3bl1efPFFANLS0rh+/bpef7VazZo1a6x6ToFAIBAIPImyYhhJiBVHAjlMrTiyZ4WRPXpXlwPnUnnyXyNJOXMIgIs3rjB62Ru8/v0sCz0fYqjbbdW7/fv3Z8CAAVrDCMDHx4fw8HC6du0KwOrVq1Gr1XpjWKPbPRWbTKPAwEB+/vlnbt68yc2bNxkyZIje+XXr1jFz5kyH3qDA9bhjAhXGkcAQOePIHYbRgXOpbE3bBcD6iUs4+P6PzBg0BYDRy95AVWz57+ozS14FoGlQY/a9u57db62jRZ2mALwU/47JfikpmnKsERER1KxZU7ZN+fLlGT9+vLZ9cXGx9lxxcTEHDhwAIDo62uT24bp169K7d28Atm/frndObnmuQCAQCASejhIMI3foXWEcCeQwNI5KsiXNVr0roSpWMXqZpnDPuwMmkTpzC+snLgFgy7FdHPrzqMUxDHW7Ltbq3U6dOpkcv2fPnrJ9wDrd7qnYVnbnLwICAvDxMd5CVL16db2VRwLPw50TqDCOBIboGkdx23YSt22ny3MYzUj4HNBMPM1qP4a/X0We7vQPujbtCMC+M7+Z7a9WqzmTfR6AhElfU61SAI9Wqc73L2u2jKVmHDd6UyFx7NgxAOrUqWP2Grqrh27duqX9fPPmTe1nS283WrXSVCg5deqU3vHz5zX3bm4CFQgEAoHAk1CCYQS4Te8K40ggh65xNGvtJrtzGNmqdyWkFfxdm3ZkVOch+JUrT7Paj/HDxMUA/HP1BxavbajbDSmp3vXy8tIaT1IfCWt1uydil2kkKJ0oYQIVxpHAEL9y5ejVpoX2915tWrg06fXFG1cAaBxYX++4ZBpdv3vTqI8uKp2VP7rVGny8fWTb6PX9K1Ff5cqVrb7fwsJC7eeioiKr++Xn55sdz5Z7EAgEAoFAqShB7wJc3b/RrXpXGEcCOcKbBWs/Nw561K6k17bqXYmsXE1Rlycf/7ve8QbV6wJwM++2xWub0u0SjtC7BQUFen0k7NHtnoIwjQSAciZQEMaRQJ9LOTdY9fN+alULoFa1AFb9vN9sVTVH0yW4AwDrDm3WHlOr1Xz60yIAWtV73Gx/X51VmdfuPLxvXbPJV2blpi7FFiZZ3Tc3uomwDZNim+POnTsAVKlSRe+4NMb9+/etHksgEAgEAiWiFL17df9Grqasd7veNTSOBGUbaUuaXzlfmtWrzR+XrpqtqmYKe/VuyzqaxNKzf1ygp23/dyQRgL4tn7B4bTndrosj9K6UD8nUKn5Lut0TEaaRQDETqC7COBKAftLrcX27M65vd9nk2M7k46HTAPhk80LGrXiLVb8k0PL93trzwTUbWRzj9QhNzqHunw1n3tbFfJm4jG6f/h8AE54cZbKftG968+bNZpf0Xr58WftZdxLU/Zyebn7SP3LkCADh4eF6x1u00Kzy2r9/v9n+AoFAIBAoHSXoXckwqt1lqCL0rq5xJCi7GOYwGtU9XDY5tjXYqnclQmo10X5u+X5v4vf9wPiVb/PR/74E4J0Br1gcw1C3G2KN3r17967J8YuKijh+/DiAUaJra3W7JyJMozKOEg0jCSVMpAL3IZf02lxVNWdRs2ogy8fMAyDlzCFm/7hAe27/ewlWjTGm6wgGt+0DwLK9a1m8+z8ADP1bPyb1esFkv44dNVvgzp8/L1sZDTSG0TfffANgVJwAYOjQoQCsXbuWrKwso/NqtZqUlBTS0tIAaNdOv6Rp1apVS+XebIFAIBCUPdytd3UNo9qdBwPK0LuScSQom5hKem2qqpo57NG7uuhq609/WsTe0wcBWD5mHkFVH7XY31C3S9iid5ctWyab4uHBgwcsXbpU275q1ap6563R7Z6Kr+UmgtKKkg0jCWkiHRV/nuhvLxD/XEMq+5lf2ijwfMxVSZOMo5XJKSxP3mNXgj5bCWvSjmMfbefAuVSyc6/Tsm4ITYMaW93fy8uLOf/3Nm8PmMi+M4dQFRfTLaQTARWrmO1XoUIFhg4dyvr169mxYweXL1+mQ4cO1KhRA9CUDN2xYwcADRo0IDQ01GiMVq1a8fvvv3P27Fni4uLo2bMnTZs2pVKlShQUFPDjjz9y4cIFQGMw+fn5GY0xYsQIYmJirH5egUAgEAiUiNIMIwkl6F2hr8smlqqk9QjVrKZJOnpC73dT2KN3dQmoWIUTs5M4k32etEsnqVH5EcKD2+vlRrKErm4/nptps969efMmn376KYMGDaJ+/fr4+Phw/fp14uPjtW3HjBlj1N8a3e6pCNOojOIJhpGEEiZSgeswZxhJuMM48vH2IfyvfdL2ElCxCv1De9jUJzQ0FLVazYYNG0hPT5fdZhYaGsrgwYP1Eg9KeHl5ERUVxdq1azl16hTJyckkJycbtRs6dKis6QSaipmTJ0+26b4FAoFAIFAaSjSMJITeFbgaS4aRhK3GkT1615Dgmo2sSgFhCkm3vzi9l95xa/TuggULUKlUbNiwwXhcHx9eeeUVAgICZMewRrd7IsI0KoN4kmEkISbSsoE1hpGEO4wjd9G6dWtCQkI4evQoO3fu5MGDBwCEhITQt29f7RsMU/j4+BAVFUVOTg6//PILv/32m/Zcr1696Nixo+wbF12qVy+df7YCgUAgEDgTawwjCaF3Ba7CWsNIwlbjSGnYondnzJjB2bNn2b17N5cuXQI0Sa8HDx5McHCw7EtaXczpdk9FmEZlDE80jCQ8bSItUFjm/MKiQtnjRapCvf/ag9TX1DWswRbDSKIsGUcVKlSgU6dOdOrUye4xAgMDGThwIAMHDnTgnQkEAoFAIJDDFsNIwtP0rjNRmpYuqd7V1dyOGMNebDWMJDzROJo5c6bNfby8vAgODiY4ONju6zpCtysJYRqVIRxpGH21O5tXu9d04N1ZhydNpH1T9rr7FvSxcD9vxj5d4ktM/WqY1W1nP/uwrT2GkURZMo4EAoFAIBB4BvYYRhKepHedieK09F/YonflWJW4kFWJC0s0xpuxT+tpaWux1zCS8ETjSFByhGlURnD0CqP5ydkAwjgyw7uPP+7uW9DjUuOnZI8fTN9F+oVUnukzCV8f680aXeK3fgFAdL/XbOiVobmvEhhGEiU1jvIL7X9bIxAIBAKBQKBLSQwjCU/Ru85EaVr6k5MnAVv17kOKVIWsSlxI84bt6Nj8SbvG0NXtl3KO26x3S2IYSQjjqOwhTKMygDO2pL3es6bijCOl0bdmkLtvQY9jLUwnpEu/kEqH5j0o52ufaXT4VAoAHc1cw5DbWSsdYhhJ2GscSRNoVOt/2n1tZ/DLL7/QokULo3Ke5igqKiI9PZ3r168DUKNGDZo3b46vr/WhPjc3lxMnTpCXl0e5cuVo1KgR9erVs7h/WyAQCAQCgWMMI4mybhwpTUtvKtQkP7ZF7+pSWKQxjTo2f9LuMeChbn/n33E2692SGkYSnm4clVTvqtVqLl68yIULFygsLKRSpUo263ZPQphGpRxn5TCSjCIlGUd+0x5x+T0I7MeRhpGErcaR7gSqNLZt28a2bduoWrUqY8aMMVmlAaC4uJidO3eyd6/8Mu4OHTrQv39/vL29TY5x+/Zt1q5dy5UrV2TPR0VF2fYAAoFAIBCUMRxpGEmUdeNIYJqggACb9a4j0zh4onFkjd61lLD61KlTrF692ui4tbrdEzH9DULg8Tg76fWr3WtqVxx9tTvboWNbizSRnsp+4JbrC+zH0YaRhGQcSRPppZwbsu0MJ1DQmC/FJUy6WKRSoVar7e6vVqspUqlo2bIloHkTEhMTw40b8s9RXFzMypUrtYZR48aN6dOnD126dKF27doAHDp0iJUrV5p8ths3bhATE6OdQDt37kzPnj3p2LGjto3c5CgQCAQCgUCDMwwjCV29G/3tBe7mqxw6vsAzsUfvOjrvZ4/Q5vRq3YKkoyfYecz68vKS3i0Jtup2a/VuamqqyTFSU1P1NHHv3r3p3r271brdUxErjUoprqqSpqQVR5+5/MqCkuAMw0jC0oojuQm0xfReemOsfmkhbepb98ZErVazbO9a5m9boj0WXLMRq8Z/ReUK/laNcffBPcavfJsjFzVva2bOnElkZCRLly4lKyuLRYsW8d577xktm92yZQsZGZr8UNHR0TRu3FjvfE5ODrGxsWRkZLBz50569dJ/TrVazYIFCwBNmdHRo0dTuXJl7fmnnnqKX3/9lS1btlj1HAKBQCAQlDWcaRhJiBVHAkPs0bvOQG/F0QjzbQ31LsDLT45icq8XrN4alnb5JCP+/YreMUu63Ra9u2nTJmrXrk2tWrX0xsjMzGTTpk0AdOvWjR49eujdszW63VMRK41KIa4yjCSUsuJI4Fk4yzCSMLXiyNoJNGrxJI5fPmXVtd794TM9wwjgTPZ5Os0eREFRgcX+BUUFdJo9SG8CBfD19eWFF14AQKVScebMGb3zhYWFHDp0CICJEycaGUYAgYGBREdHA7B3716Kior07/OvMX19fZkwYYLeBCrRqVMn2rdvb/E5BAKBQCAoa7jCMJIQK44EhpRU7zoKacWROUzp3bhd3/H+hnlWXef3i+lGhhFY1u226t2ffvrJ6LxkGPXt25eePXsaGULW6HZPRZhGpQxXG0YSSjCOBJ6FMw0j3WvoTqRnM7NNTqAxUTM5MTuJE7OTiImaCcDwf0+weI3b9++w8UgiALvfWseJ2Ukcn7WDpkEaA+fb/estjvHdLwkAtKwTwrGPtnNidpL2XIUKFbSmz8aNG/X6/fHHHwDUrl2bwMBAk+M3btxYayilp+svHd69ezcAQ4YMMZswu1+/fhafQyAQCASCsoQrDSMJYRwJDLFF7zoTSzmN5PTunrf/C8D6w1u5ff+OxWtELZ4E2K7bbdW7Fy9eJD8/X3s8Pz+fq1evAhpzyRTW6HZPRJhGpQh3GUYSwjgSKBFpIn3E35/lSXvIyLkuO4H2bfmE7GdLe6X3ndGs9BnbbSSPVtGM6eXlxcqx8wGYv+1ri/f4xXZNm0XPfYyPt/FS8+rVNePeu3dP77hUJa1JkyYWrxEcHKzXR+Ly5csAVKtWzWx/WyqwCQQCgUBQ2nGHYSQhjCOBIdbqXXcip3drVH6El7o/C8CeU7+a7a+ryW3V7fbo3dzcXNnPPj7mt4Va0u2eiDCNSgnuNowkhHEkKGtISfzK+eivnCpUabaB+cqYQIao/prgypmYhAwnHUMqVbK8PdPSGP7+1uVeEggEAoGgrONOw0hCGEcCT8OU3i1UabZt+Zip8usoXKF3LWluT0SYRqUApRhGEsI4EigJaU/3zXv3GN2rGw0Ca8hWmdh+/GfZz+bK1AM88XgYoNmPff3uTQBUxSomfPseAFMjxlm8x9f6vAjA89+8Llt57datWwDUq1dP73iNGjUAjPZMy3Hu3Dm9PhLS5GmpyoOqhBUuBAKBQCAoDSjBMJIQxpFAwlq9607k9O61OzdYtnctAOHBHcz219Xktup2e/Ru1apVZT8b5gc1xJJu90TcbhrFxsbSqFEjKlSoQFhYGL/+anpZ2vHjxxk2bBiNGjXCy8uLmJgYozYffPABXl5eej/NmjVz4hO4F6UZRhLCOBJYIt8FyeAMkwA+VqumyfKkU1Z/SIvpvWgxvRdTVn8IaCoxWCKgYhWG/E2z/7nbp/9Hi+m9CJ3RlxNXTgPwTJhlQfl8l2EAnM76k5bv99ar5FZUVMSGDRsAeOKJJ/T6NW+u2Tv+559/8ueff5oc/9q1a2RmZur1kRg8WHN/8fHxPHjwwOQYBw4csPgcAoFAIBCUZpRkGEkI40hgi951JjuPpZs9L6d3u382HIDBbfsQULGKxWusm/BvwHbdbqverVOnDn5+ftrjfn5+WhNo27ZtJvtbo9s9EbeaRt9//z1Tp05l5syZHD58mDZt2hAREUF2trzJkJeXR5MmTfj000+NSuDp0rJlS65evar92bt3r7Mewa0o1TCSEMaRwBwrk1OcahyZqhphqsqEIesm/Nts6U5dZg95gwlPjtI79rcGrTj4/v8o71veYv/yvuX5dfombfJsibt377Jo0SLtGw0pL5GEr68vXbt2BTST4LVr14zGPnfuHIsWLQKgQ4cORrmJgoODtXuzV6xYYfT2RK1Wk5SURGJiosXnEAgEAoGgtKJEw0hCGEdll5LqXUex81g6SUdPmG1jSu++0e8lPhn2llXXaVk3RNYcsqTbbdW7//jHP4zGGDBgAACHDh2SXehijW73VNx6559//jnjxo1j9OjRAMTFxbF582aWLVvG22+/bdS+Y8eOdOzYEUD2vISvr69ZU8mQ/Px8vezouomulIrSDSOJV7vXBGB+crbe7wJB1u3brExO4fmeXRxeSc1SmVFpIl2ZnMLy5D2M7tmNE7OTtMnzLG1JM8TLy4vJvUczufdoilQqfLy9jcpwWqJyBX82Tl6KWq1GVVzMS1tnsn//fu35yZMny47Zo0cPMjIyyMjIYNGiRQQGBtKmTRt8fHw4ffq0dgVSw4YN6d+/v+y9v/LKKyxYsICsrCw+/vhjWrZsSZ06dVCr1ezYscOm5xAIBAKBoLShZMNIQjKORsWfJ/rbC8Q/15DKfpbzKgo8F3v0rjMSY0uGUa/WLSy2NdS7vhaSSsvRpn5zm3W7LXp30KBBsl5CrVq1GDRoEJs2bWLLli1s2bKFLl264O/vz507d6zS7Z6K21YaFRQU8Ntvv9G7d++HN+PtTe/evfX+wO3h9OnT1KlThyZNmvDss8+SkZFhtv2cOXMICAjQ/tSvX79E13c2nmIYSYgVRwI5RvfspjWOHLniyNIEKmH4BgY0MchWw8gQXx+fEk0SXl5e+Pr4aONg3bp1mTJlirYSgyHe3t48//zzdOig2Qeek5NDUlIS27dv1xpGXbt2JTo62uSzVa9enSlTpmj3ax8/fpzExES9CTQqKsruZxIIBAKBwFPxBMNIQqw4KlvYo3cdveJI1zDqEWr99itJ75YEW3W7tXq3Xbt2Jsdo166dniZOSUlh+/btVut2T8VtK41ycnJQqVQEBQXpHQ8KCuKPP/6we9ywsDBWrFjB448/ztWrV/nwww/p1q0baWlpVKkiv0/ynXfeYerUqdrfc3NzFWsceZphJCFWHAkMqRdYndE9u7E8eY/DVhxZaxhJ6L6BURr9+vWjefPmeon3TOHt7c2AAQOIiIggPT1dW9KzRo0aNG/e3KrlsAEBAbz22mvk5uaSnp7OvXv3KFeuHI0aNaJevXql6m2JQCAQCATW4EmGkYRYcVR2sEfvOnLFkb2GkTtxhN4NCQlhxowZXLp0ifPnz1NYWIi/v7/Vut0T8dyNdSbQ3X7RunVrwsLCaNiwIWvXrmXs2LGyffz8/PQSXSkVpRhGV/dvhL629ytrxtH27Cx334Iel07slD1+MH0XAIfSd+LrY59pk3ZOs6/3oIlryBFSw7HGka2GkYQ0kSqNsLAwm/v4+voSGhpaoutWrVrVrmsLBAKBQFCa8ETDSKK0GEdK09Jp504CtuldXYr+Ki0vaW970NXt9uhdRxlHnmgY6VJSvevl5UX9+vUVu9DE0bjNNAoMDMTHx4esLP1gkJWVZVM+IktUq1aNkJAQq0pSKxklGUZXU9ZD31Z29S9LxtEnJ0+6+xb0sXA/qxItVwqzRPzWL6xuO/tZTQUFRxhH9hpGEo7OqSQQCAQCgcBz8WTDSKI0GEeK09J/YYvelSP9QirpF1JLNMaqxIVaLW0tjjKOPN0wEtiO20yj8uXL0759e5KSkoiMjASguLiYpKQkJk2a5LDr3L17l7Nnz/Lcc885bEx3oCTDqHaXocApu8cpK8bR9i5d3X0LeqS1f032+KH0naxKXMi/Jq6xe6XRko2zARg/eLrVffKur9J+LolxVFLDSCAQCAQCgUCiNBhGEp5uHClNS7966R5gm97VpUhVyJuxT/NMn0l0aN7DrjF0dXvBrXU29y+pcSQMo7KJW7enTZ06VZvAtVOnTsTExHDv3j1tNbXo6Gjq1q3LnDlzAE3y7BMnTmg/X758mSNHjlC5cmVtSbs33niDf/zjHzRs2JArV64wc+ZMfHx8PD6Jq5IMI80EOrdE45UF46h8CRMqO5pyvvImjGQU+fqUM9nGEtIY9vYH+4yjsmQYPXjwgIMHD5KcnAyAj48PoaGhdOnShcDAQKvGyMnJYfv27Zw+fRqAypUrEx4eTrt27ahQoYLT7l0gEAgEAk+gNBlGEp5sHClNSztC70rjlFRz+/qUo8DO69trHHmiYfThhx/arHfVajVnzpxh48aN3LunMQrr1q1L9+7dCQ4OtirvkSnd/sEHH5ToedyFW02jkSNHcu3aNWbMmEFmZiZt27Zl69at2uTYGRkZehnRr1y5opfNfN68ecybN4/u3buza9cuAC5dukRUVBTXr1/n0UcfpWvXrvzyyy88+uijLn02R6Msw8gxlAXjSGAbthhHZckwOnbsGOvXr9c7plKpOHLkCEeOHKFp06aMHDkSHxNVKFQqFQkJCaSlpekdv3v3Ltu3b2f79u0MGTKE1q1bm7yHGzccW21DIBAIBAIlURoNIwlPNo4EzsFW48gTDSMJW/XuokWLUKn0Kw9evnyZVas0uyQmT55stjqaOd3uqbg9EfakSZNMbkeTjCCJRo0aoVarzY63Zs0aR92aoihthpGEMI4EhlhjHLnDMFIVq9h35jeu371Jq3qPE1yzkc1j3MrLZc+pX/H18SE8uAMBFeUrOupy9OhRNmzYAGhi4IABA/D29ubu3btcuXKFbdu2cfr0aVavXs2zzz5r9PajuLiY+Ph4MjIyAOjTpw8tW7akoKCA3Nxcfv/9d44dO8aGDRvw8vKSTaR9+/ZtFixYwFdffWXzMwsEAoFAoHRKs2EkIYwjgSHWGke2Gkb26F1DTmWe4/iVUwRVDSSsSTt8vG37uyrp9ldeecVmvSsRHR1N9erVuXfvHrdu3WLDhg0UFRWxYMECpkyZQkBAgNEYlnS7p+J200igXFw1gQrjSGCIOePIHYbRgXOpjF72htHx/e8lWDURqtVqpm+Yx4bDW/WOD27bh0+GvWVymeuDBw+0E0+fPn3o3Lmztm316tVp0KABjRs3Ji4ujrNnz3L8+HFatdJPUn/s2DGtYfTiiy9St25d7blHH32Uxx57jKCgIHbs2MH69esJCQkxqia5bNkyi88oEAgEAoEnUhYMIwlhHAkMsWQc2WoYvbd+rs16V5fb9+/Q+eNIo+PLx8wjrEk74w4y6Or2mTNn2qx3g4KCeOGFF7Rb2QICAqhTpw4hISEsWrSImzdvsnbtWsaNG6fX3xrd7qkoa6OoQDG4egJ9tXtNXu9Zk/nJ2Xy1O9vp1xMoH8k4yrp9m5XJKeQXFrrFMMrKvaadeLo27cg7T72iPSc3qcmxIGmFdgJ9+clRjO02EoCNRxJZtnetyX6pqZrKGqGhoYSHh8tOtkFBQYwYMQKAH374weh8QkICYGwY6RIeHk7Dhg0BOHjwoN653NxccnNzzT2eQCAQCAQeSVkyjCQk4+hU9gOiv73A3XyV5U6CUo1kHAUFBLA8eQ+XcjQpCezZkmaP3tVFV1tPHziZrk07AjB62Rtk5+ZY7G+o23WxVu+++OKLsrmPfH19GTNmDKBJm2Ooj63R7Z6KMI0ERrhrAhXGkcAQXePo6+27+Xr7bpfnMPrkx1gAZgyawpLnP+W58GEcn7VDe/5U5jmLY8Tt+g6APW//l1d7j+b1iPHsfktT8WL+tiUm++3btw+ANm3amB2/efOHE/mdO3dkP9epU8dkfy8vLwYOHAhAUlKS3jmp+IDhCiaBQCAQCDwZJRhG7tK7wjgSGGJoHH23e5/dOYxs1bsSp7POaz8fn7WDZ/4eyZLnP+WdARMBmL5hnsUxDHW7LtbqXV9f05uxKleuTOfOnQFIT0/XO2etbvdEhGkk0MPdE6gwjgSG1AuszjNPdCbz1m0yb93mmSc6uzTpdeKJPQD8o01v7TEvLy+mD5wMwIkrp832L9JJpFej8iPaz49WqS7bRpe7d+8CULVqVavvNy8vT/azpbcd3iYqlEhj1KhRw+p7EAgEAoFAybhb7wLcu3rWrXpXGEcCQyTjKL+wiD8uXaVZvdp2Jb22Ve9KHL98EoB3nnpFT7eO6KAxevaePijbTxc53a6LI/RuxYoVAbSV1STs0e2egjCNBFqUMIGCMI4E+uQXFpL0+wnt70m/nyC/sNBl169WSRP4L97QT16384/9ANSsan5y8dGZnFTFDydL3aT+PhZKypYvX966mwXK6SQNN/emxBBpojOswCaNV85EFTuBQCAQCDwJJejde1fPcnrdXLfrXUPjSCDY98cZ7ec/s65pt6rZgr16VzKb9hiYQ+dyNLk5G9SQT7OgiyndLuEIvSut5DfV1hbd7ikI00gAKGMC1UUYRwLQT3r9ckQPXo7ooZfjyBV8FfUhAENjx5N+9Qz5hQV8u389KWcOAVhMyufl5UW7Bi0BGBk3kWt3rnMr7zaRCzXJ85oGNTa5CigkJASA48ePm72G7puOatWqaT8/8sjDNz23bt0yO4ZU0cGwDGmjRo0A2LFjh2EXgUAgEAg8CiXoXckwqhhYVxF6V9c4EpRtdHMYvT9ikFGOI2uxVe9KhAe3BzQritb8+j/u5d8n/eoZhsW+BMBHg6davLahbjfEGr1rqVq7lA9J6iNhrW73RIRpJFDEBCqHEiZSgfuQS3otlxzb2XRo3Jr+oU8CMCz2Jdp92J85mzX7pVeMmW9VCdDF0XMAzVa27p+NIPyToZzO+hOA/4z70mS/Pn36AJCYmMjly5dl2xQUFBAfHw9AWFiY3rJbb29vwsPDAViyZAmFJv68srKy2LZtGwBdunTRO1evXj2LzycQCAQCgSfgbr2raxgF/58mWa8S9K5kHAnKLoZJr00lx7YGW/WuhI+3D8vHaPIWfbQpho6zBmoNo/6hT9KpSVuLYxjqdl2s1bvJyckmx//1119l+4B1ut1TEaZRGUephpGEEiZSgesxVyXNHcbR/JHvs2LMfO2y2K5NO7Lrze+tmrwAKlfw58gHW3iz/8v4/mUyTe37Ikc+2ELlCv4m+wUGBmoTUC9dupQLFy5QXFysPX/r1i1iYmLIztb823jiiSeMxpAmxfv377N06VJu376tdz49PZ24uDhA84bEcC+3l5cXUVFRVj2nQCAQCARKRkmGkU/5itpzStC7betVcst1Be7HVJU0e4wje/SuLmFN2rHzzTX0banRtI9UCiB+7BfMH/m+1c9jqNvBNr27d+9eUlJSKCgo0J4vKirixx9/ZMuWLQBERUUZrZyyRrd7KtYnvBCUOpRuGEm82r0mAPOTs/V+F5ROzBlGEpJxtDx5DyuTU3i+Zxf8nJxzp1OTtmx9Ld7u/uV9y/NCl+G80GW4Tf0iIyPJzc0lIyODFStWmGw3ceJEKlUyFnyVKlVi4sSJxMbGkp2dTUxMjGz/xx57jBEjRsieCwkJYdCgQTbdt0AgEAgESkOJhpGE0LsCd2DKMJKQjKOVySksT95jsYKxvXpXl6CqjxITNdPu/vBQt7eY3kvvuDV6d9OmTezYscNkaoZBgwZpt6IZYkm3m9LhSkesNCqjeIphJKGENzAC52ONYSThjhVH7sDHx4cXXniBIUOGyO4F79KlC9OmTSMwMNDkGIGBgUybNk1bItSQYcOG8eyzzxolBdSlXTvzuZsEAoFAIBAYY41hJCH0rsCVWDKMJEqyVU1JWKt3X3rpJRo0aGB0rn79+rz00ktmNbEl3e6piJVGZRBPM4wkxBuY0o0thpGEO1YcuQMvLy9at25N69atuXPnDnl5eZQrV45q1aqZLB1qSKVKlejbty+9e/fm1q1bFBYWUqlSJapUqeLkuxcIBAKBoGxii2EkIfSuwBVYaxhJ2LriSCm8/PLLNuvdWrVqMXr0aPLz88nNzQWgatWq+Pn5WdXfnG73VIRpVMZwlGF05FKeW/Y+e9JE+u7xNHffgh65l2fLHk87p0notmTjbHx97DNcpDEWJ8hfQ46nO9fXfrbHMJIoK8aRRJUqVUpk9Hh7e1O9uvIneYFAIBAIPBl7DCMJT9K7zkRpWjrthmaFjS16V5cilWZVfPzWLzh8KsW+e9DR7aO6NrFrDFsNIwlPNI6CgoLs7uvn58ejjz5aouuXVLcrBbE9rQzhyBVGo+LPc+RSnoPuzDbE0t3SRUkMI4myslVNIBAIBAKB8imJYSQh9K7AEvboXXsNI4nSslVNYBtipVEZwdFb0kJqVmBU/Hm+i24kVhyZ4JOWrdx9C3oc6/iG7PGDJ3YSv/ULxg+eTjlf+1boSG9cXoqcbnWf21krHWIYSZRkxdHOY+kMtX3edCp37tyx682EvUtpDa+dl5eHr68vjzzyiNVb4AQCgUAgKOs4wjCS8AS960yUpqUnXX4A2KZ3dSksKmTqV8OI7vcaHVv0sGsMXd0es2q8zXq3JIaRhCeuOJKjpHq3uLiYmzdvUlRUVOpTPgjTqAzgjBxG8c81JPrbC4oyjois6vJ7ENiPIw0jCXuMI2kCRb6Qgtv4/PPPAU0VhtatW1tMppeZmcnmzZu5dOmS3vHatWszaNAgatWqZba/Wq0mLS2N9evXG50LCwvjiSeesPEJBAKBQCAoWzjSMJIo68aRwDTSCntb9G5JDSMJTzWOrNG7ctWIdcnLyyMlJYV9+/YZnbNWt3sa4vVxKcdZSa8r+/kQ/1xD7YojJWxVE3gWjjaMJGzZqqY7gSqVhIQEVqxYgUqlMtkmNTWVxYsXGxlGAFevXmXx4sWkpqaa7K9SqfjPf/4jO4ECHDhwgLlz59p+8wKBQCAQlBGcYRhJiK1qAjns0buOMIwkPG2rmrV6Nycnx+QYOTk5zJ07V9YwAut0uyciVhqVYpxdJU0yjpSy4mi/y68sKAnOMIwkrFlxZDiBfr17NV8kLsXH25vX+o5j1N8jKe9b3uprZufm8N76uaScOUT96nX4KHIqYU1sK1N/6M+jvLp6JrfycpkxYwZXrlxh6dKlZGRkkJCQwLBhw4z6nDx5kk2bNgHQoUMHIiIi8PXVhHaVSsWBAwdITExk06ZN+Pv7ExISYjTG999/z9mzZwEYMWIEzZs/FBT37t0jPj6e7GwhUgUCgUAgkMOZhpGEWHEkMMQevetoDFccvdrNch9dvdunRTfeHTiRoKrWJ5wuKCpgZcoPNut2a/VubGws06ZNM1pxlJeXR2xsLKCpSjxu3DhtRTS1Wm2VbvdUxEqjUoqzDSMJJa04EngWzl7Gam7FkdwE+kXiUgBUxcXM27qYth/05+6De1Zd68C5VJ7810hSzhwC4OKNK4xe9gavfz/L6vudumYW0d+8xq08TT4iLy8v6taty4svvghAWloa169f1+ujVqtZs2YNAP3792fAgAFawwjAx8eH8PBwunbtCsDq1atRq9V6Y+Tk5HD69GkAJkyYoDeBAvj7+zN27FgqVnS8ABYIBAKBwNNxhWEkIVYcCQyxVe86A90VR5Yw1LuJJ/bQ419Pc+Cc6RXxutx9cI+2H/S3WbfbqnflVhL9/PPPANSsWZMpU6ZoDSOwXrd7KsI0KoW4yjCSUIpxJPAsXLHvWW4iNTWBNg1qzL5317P7rXW0qNMUgJfi37F4DVWxitHLNEnG3x0widSZW1g/cQkAW47t4tCfRy2OceBcKlvTdgGwfuISDr7/o/Zc3bp16d27NwDbt2/X66e7Ha1Tp04mx+/Zs6dsH4CUFE3J14iICGrWlDdfy5cvz/jx4y0+h0AgEAgEZQlXGkYSwjgSGGKL3nUWknFkDjm9O2PQFABGL3sDVbHlLV3PLHkVsF2326p3U1JSKC4u1p4rLi7mwIEDAERHR1PORA4pa3W7pyFMo1KGqw0jCWEcCZSK7kQ6a+0mkxNowqSvqVYpgEerVOf7lzVLT1Mzjlt8KyC9GenatCOjOg/Br1x5mtV+jB8mLgbgn6s/sHiPMxI0Sa/XT1xCs9qP4e+nLzpbtdJUDzl16pTe8fPnzwPmDSPQvP2QJjCpj8SxY8cAqFOnjtkxdN+mCAQCgUBQ1nGHYSQhjCOBIdbqXWdiKRm3nN59utM/6Nq0IwD7zvxmtr9areZM9nnAdt1uj969deuW9vPNmze1n/39/c2OYUm3eyLCNCpFuMswkhDGkUCp1AusTuOgh3ulw5sFG7XRrXLg4+2j/azSecsgR1auJlnek4//Xe94g+p1AbiZd9vi/V28cQWAxoH1Zc/n5+fLHi/8awly5cqVLV6joKBAr4+ElKjPmjEEAoFAIBC41zCSEMaRwBBr9K47MaV3JdPo+t2bRn100dXktup2e/SurmYuKiqyup8l3e6JCNOolOBuw0hCGEcCJbLzWDp/XLpKs3q18SvnK1tl4tqdhxUfdCctXx8fzNGyjiax9OwfF+i93fjfkUQA+ra0XKq+S3AHANYd2ix7/s6dOwBUqVJF77iUoO/+/fsWryHtqzb1dqTYgjnmyfuwBQKBQCBwFEowjCSEcSTQxRq9607k9K5arebTnxYB0Kre42b762pye3W7LXpXNxG2YVJsc1jS7Z6IMI1KAUoxjCSEcSRQErp7ukd1DzeZLLD7Z8OZt3UxXyYuo9un/wfAhCdHWRw/pFYT7eeW7/cmft8PjF/5Nh/970sA3hnwisUxPh46DYBPNi9k3Iq3WPVLgt75I0eOABAeHq53vEWLFgDs37+fu3fvmhy/qKiI48ePAxgl/pPyHW3evNmsMXT58mWLzyEQCAQCQWlGSYaRhDCOBGC93nUml3JumD0vp3dbvt9bez64ZiOL13g9QpNzyFbdbo/e1TV9dD+np6ebvUdLut0TEaaRh6M0w0hCGEcCS+w8Zj7gOuoahnu6zVWZWLZ3LYt3/weAoX/rx6ReL1h1nf3vJWg/f/rTIvaePgjA8jHzrCohWrNqIMvHzAMg5cwhZv+4ANC87UhJSSEtLQ2Adu3a6fWrWrWqdm/2smXLZJfOPnjwgKVLl2rbV61aVe98x46aJcHnz5+XrRQBmgn0m2++sfgcAoFAIBCUVpRoGEkI46hsY6vedQaXcm6wPHmP2Tam9C7oa2lzjOk6gsFt+wC26XZb9e6QIUOMzg8dOhSAtWvXkpWVZXTeGt3uqfhabiJQKko1jCQk4yj62wuMij/Pd9GNaFvPc5flCRxL0tETAE5L0GeuaoQ0kS5P3sPK5BSe79mF/e8lsO/MIVTFxXQL6URAxSomRjYmoGIVTsxO4kz2edIunaRG5UcID26vt8faEmFN2nHso+0cOJdKdu51Lufk8OOPP3LhwgVAM1H5+fkZ9RsxYgQxMTHcvHmTTz/9lEGDBlG/fn18fHy4fv068fHx2rZjxowx6l+hQgWGDh3K+vXr2bFjB5cvX6ZDhw7UqFED0JQM3bFjh9XPIRAIBAJBaUPJhpHEq901FaHmJ2fr/S4o3diqdy0lq7YHyTAKCgiw2NZQ77asG0LToMZWX8vLy4s5//c2bw+YaJNut0XvNmjQgNDQUKMxWrVqxe+//87Zs2eJi4ujZ8+eNG3alEqVKlFQUGCVbvdUhGnkoSjdMJIQxpHAFL1at3CacWRNmVHDiTSq9T/pH9qjRNcNrtnIqqW1pvDx9iH8r/3eLab30h4fOnSo7OQFEBAQwOTJk1mwYAEqlYoNGzYYj+vjwyuvvEKAick8NDQUtVrNhg0bSE9Pl112a+r6AoFAIBCUZjzBMJIQxlHZwh6962jjSNcwer5nF6v66OpdewmoWMVm3W6t3h08eLBeom0JLy8voqKiWLt2LadOnSI5OZnk5GSjduZ0u6ciTCMPxFMMIwlhHAnkkCY3RxtH1kygEroTqRLp1asXHTt2tPimonr16syYMYOzZ8+ye/duLl26BGiSXg8ePJjg4GDZyU+X1q1bExISwtGjR9m5cycPHjwAICQkhL59+2rfxAgEAoFAUFbwJMNIQhhHZQd79K4jjSNDw8gZq5gcTUn1ro+PD1FRUeTk5PDLL7/w22+/ac9Zq9s9EWEaeRhKMIzuXT0L1u/cAcqmcVRgITu/qykskt/LXKQq1PuvPUh9TV3DFI42jmwxjCSkiVRpzJw506b2Xl5eBAcHExxsf3nVChUq0KlTJzp16mT3GAKBQFDWiY2NZe7cuWRmZtKmTRsWLFhgNq6uW7eO999/n/Pnz9O0aVM+++wznnrqKe35rKws3nrrLbZv386tW7d44oknWLBgAU2bNnXF45RZPNEwkigtxpHStLS9etewf5Gq0CFj2KN3HWUceaJhJOEIvRsYGMjAgQMZOHCgA+9MuQjTyINQimF0et1ceO8xm/uWNeOob8ped9+CPhbu583Yp0t8ialfDbO67exnNW0dZRzZYxhJ1Ausbtc1BQKBQCDQ5fvvv2fq1KnExcURFhZGTEwMERERnDx5kpo1jb+479u3j6ioKObMmcPAgQNZtWoVkZGRHD58mFatWqFWq4mMjKRcuXJs3LiRqlWr8vnnn9O7d29OnDiBv7+/G56y9OPJhpFEaTCOFKel/8IWvSvHqsSFrEpcWKIx3ox9WqulrcVRxpEnG0YC+xCmkYegJMOoYmBdu8coS8bRu48/7u5b0ONS46dkjx9M30X6hVSe6TMJXx/7gn781i8AiO73mg29MrSfSmoclcQwEggEAoHAUXz++eeMGzeO0aNHAxAXF8fmzZtZtmwZb7/9tlH7L7/8kn79+jFtmqYU9axZs0hMTGThwoXExcVx+vRpfvnlF9LS0mjZsiUA//73v6lVqxarV6/mxRdfdN3DlRFKg2Ek4enGkdK09CcnTwK26t2HFKkKWZW4kOYN29Gx+ZN2jaGr2+Gqzf1LahwJw6hsIkwjD0FJhlHw/70B2O+OlxXjqG/NIHffgh7HWphOFpd+IZUOzXtQzte+wH/4VAoAHc1cw5DbWSv1frfXOCpLhlFubi4nTpwgLy+PcuXK0ahRI+rVq2cxX5GEWq3m4sWLXLhwgcLCQipVqkSLFi08ugSoQCAQKIWCggJ+++033nnnHe0xb29vevfuzf79+2X77N+/n6lTp+odi4iIICEhAYD8/HxAs51Cd0w/Pz/27t0rTCMHU5oMIwlPNo6UpqU3FWoKitiid3UpLNKYRh2bP2n3GPBQt+ddX2VXf3uNI080jJKTk+3Su0VFRaSnp3P9+nUAatSoQfPmzfH1td4+kdPtnoowjTwEJRlGjphAy4pxJLANW42jsmIY3b59m7Vr13LlyhXZ81FRUYSEhJgd49SpU6xevdro+LZt26hatSpjxowxWV0NoFhheQUEAoFAaeTk5KBSqQgK0v+iGxQUxB9//CHbJzMzU7Z9ZmYmAM2aNaNBgwa88847LF68GH9/f7744gsuXbrE1au2rzIQmKY0GkYSnmwcCZyDrcaRJxpGAHv2aIrd2KJ3d+7cyd698lsjO3ToQP/+/fH29jY5hjnd/s0339j4BMrA9NMKFEVpMowkJOMopGYFRsWf58ilPIeNLfBceoQ2p1frFiQdPcHOY8alMCXcYRip1WqKVKoSjVFcXGyTAXPjxg1iYmK0E0/nzp3p2bMnHTt21LZZvXo1qampJsdITU3VM4x69+5N9+7dtVsdcnNziYmJ4caNGybveeXKlbLnBAKBQOA8ypUrx/r16zl16hTVq1enUqVK7Ny50+KXFoFtlGbDSOLV7jV5vWdN5idn89XubHffjkABSMZR1u3brExOIb9QPjm3PYaRrXpXjiKVCrVabXd/tVptl96VDKPGjRvTp08funTpQu3atQE4dOgQK1euNPls1uh2T0SsNBKYxBUTqFhxJJDD0oojVxtGdx/cY/zKtzly8YT22MtPjmJyrxes3hqWdvkkI/79it6x1S8tpE190/evVqtZsGABANWrV2f06NFUrlxZe/6pp57i119/ZcuWLWzatInatWtTq1YtvTEyMzPZtGkTAN26daNHjx569xwZGcnSpUvJyspi0aJFvPfee0bPtGXLFjIyMhAIBAKBaQIDA/Hx8SErK0vveFZWllFslqhVq5bF9u3bt+fIkSPcvn2bgoICHn30UcLCwujQoYPjH6IMUhYMIwmx4khgiKUVR7YaRvboXV3UajXL9q5l/rYl2mPBNRuxavxXVK5gXeJ/Xd0uVSO2Ve9GR0fTuHFjvfM5OTnExsaSkZHBzp076dWrl9G9W9Ltnop4RSGQxZUTqFhxJJDD1IojVxtGBUUFdJo9SM8wAojb9R3vb5hn1Ri/X0w3mkABohZP4vjlUyb7nTlzBgBfX18mTJigN/FIdOrUifbt2wPw008/GZ2XDKO+ffvSs2dPownS19eXF154AQCVSqW9pkRhYSGHDh0y83QCgUAgAChfvjzt27cnKSlJe6y4uJikpCQ6d+4s26dz58567QESExNl2wcEBPDoo49y+vRpDh06xODB7lmFXpooS4aRhFhxJDDE1Ioje1YY2aN3dXn3h8/0DCOAM9nn6TR7EAVFBRb7m9LttujdiRMnGhlGoHkxEB0dDcDevXspKirSv08rdLunIkwjgRHumECFcSSQw9A4cseWtO9+SQCgZZ0Qjn20nROzk9jz9n8BWH94K7fv37E4RtTiSQDERM3kxOwkTsxOIiZK8+Zj+L8nmOy3e/duAIYMGWI28V6/fv0AuHjxojZpKmgSqEo5Lzp16mSyf4UKFbST4MaNG/XOSXk4pGW5AoFAIDDN1KlT+frrr1m5ciXp6elMmDCBe/fuaaupRUdH6yXK/uc//8nWrVuZP38+f/zxBx988AGHDh1i0qRJ2jbr1q1j165dnDt3jo0bN9KnTx8iIyPp27evy5+vNKEEw8hdelcYRwJDDI2js5nZducwslXvSty+f4eNRxIB2P3WOk7MTuL4rB00DdIYON/uX29xDEPdrou1ejcwMNDk+I0bN9YaSunp+mk0rNXtnogwjQR6uHMCFcaRQA5d48gdSa+/2P41AIue+xgfbx8AalR+hJe6PwvAnlO/mu2vu+e5b8snZD+b2hd9+fJlAKpVq2b2GroTU25uruxnHx8fs2NUr14dgHv37ukdl6pGNGnSxGx/gUAgcAexsbE0atSIChUqEBYWxq+/mo/J69ato1mzZlSoUIHQ0FCjFZp3795l0qRJ1KtXj4oVK9KiRQvi4uKsvp+RI0cyb948ZsyYQdu2bTly5Ahbt27VJrvOyMjQS2AdHh7OqlWrWLJkCW3atOG///0vCQkJtGrVStvm6tWrPPfcczRr1oxXX32V5557TrawgcB6lGAYqQruu1XvCuNIYIhkHGXkXGd50h4e8fe3K+m1rXpXYt8ZzUqfsd1G8mgVjS718vJi5dj5AMzf9rXFa8vpdl0coXeDg4P1+khYq9s9EbebRrZM9sePH2fYsGE0atQILy8vYmJiSjym4CFKmECFcSRQGqq/JrhyBqZLoUqzdNfHBYlI/f2t28NdEgwnT0MqVRK5xgQCgbL4/vvvmTp1KjNnzuTw4cO0adOGiIgIsrPlvwDv27ePqKgoxo4dS2pqKpGRkURGRpKWlqZtM3XqVLZu3cp3331Heno6U6ZMYdKkSdqtvtYwadIkLly4QH5+PgcOHCAsLEx7bteuXaxYsUKv/fDhwzl58iT5+fmkpaUZ5b149dVXuXjxIgUFBVy4cIFZs2ZRvnx5q+9HoI8S9K6q4D5n/jvP7XpXGEcCJSEVmynno29SFao028B8ZUwgQ0zpdglH6F1LY7hCt7sat5pGtk72eXl5NGnShE8//dRkQkFbxxRoUMIEKiGMI4EuulvSrKmq5mhe6/MiAM9/87q2gsO1OzdYtnctAOHB5hOR6la32X78Z9nPpirgSJOOqSoPEiqdim5Vq1aV/Wy479qQW7duAVCvXj294zVq1AAw2vstEAgE7ubzzz9n3LhxjB49WrsiqFKlSixbtky2/Zdffkm/fv2YNm0azZs3Z9asWfztb39j4cKF2jb79u3j+eef58knn6RRo0aMHz+eNm3aiBeQpQQl6F3JMLqfc1kRelfXOBKUbaQcRg0CazC6Vzdu3rtntqqaKWzVuxJPPK4x2ON2fcf1uzcBUBWrmPDtewBMjRhn8dpyul0XR+jdc+fO6fWRsFa3eyJuNY1snew7duzI3Llzefrpp/Hz83PImAJlTKCGCONIAMZJr00lx3Ymz3cZBsDprD9p+X5vWkzvRffPhgMwuG0fAipWsTjGugn/BmDK6g9pMb0XLab3YsrqDwFNRQlTSElO4+PjefDggcl2Bw4cAKBOnTp6sdHPz087KW7bts1k/6KiIjZs2ADAE088oXeueXPNVsA///zTZH+BQCBwNQUFBfz222/07t1be8zb25vevXuzf/9+2T779+/Xaw8QERGh1z48PJxNmzZx+fJl1Go1O3fu5NSpUyJ/UCnB3XpX1zBqOnyaYvSuZBwJyi6GSa8fq1VTNjm2NdiqdyUCKlZhyN80eTq7ffp/tJjei9AZfTlx5TQAz4RZTv5vqNt1sVbvmtO8165dIzMzU6+PhLW63RNxm2lkz2TvrDHz8/PJzc3V+ykrKNEwklDKRCpwD6aSXrvaOCrvW55fp2/SJuGTeKPfS3wy7C2rxmhZN0R2slw34d9mS5AGBwdrcxGtWLHCaLWQWq0mKSmJxERN0sB//OMfRmMMGDAAgEOHDsm+Kb979y6LFi3Sji3t05bw9fWla9eu5h5PIBAIXE5OTg4qlUqbK0giKChIK+gNyczMtNh+wYIFtGjRgnr16lG+fHn69etHbGys0RcMgWeiJMPIv/ZjgHL07qvdhWlUVjFVJc1UVTVz2KN3dZk95A0mPDlK79jfGrTi4Pv/o7yv5W25pnS7LXo3Pj6ea9euGY197tw5Fi1aBECHDh2Mkl1bo9s9Fbel9TY32UvZy1015pw5c/jwww/tuqYno2TDSEKaSKO/vcCo+PN8F92ItvVEbpXSjqUqadKxpKMn9H53FpUr+LNx8lLUajWq4mJ8LSSVlqNN/eacmJ2kTQJoaYkuaJL/vfLKKyxYsICsrCw+/vhjWrZsSZ06dVCr1ezYsUPbdtCgQbLbdmvVqsWgQYPYtGkTW7ZsYcuWLXTp0gV/f3/u3LmjZ6hPnjwZLy8vozF69OhBRkaGzc8sEAgEnsaCBQv45Zdf2LRpEw0bNuTnn39m4sSJ1KlTx2iVksDzUJphJCH0rsBdmDKMJCTjaHnyHlYmp1hMjG2P3tXFy8uLyb1HM7n3aIpUKny8vWW1qTl0dftre/5ls97NyMhg0aJFBAYG0qZNG3x8fDh9+rR2BVLDhg3p37+/7L1b0u2e6jmUrlpwdvLOO+8wdepU7e+5ubnUr1/fjXfkfDzBMJIQE2nZwpJhJOFq4wg0k4E9hpEutk6e1atXZ8qUKSxbtozc3FyOHz/O8ePH9dpERUUREhJicox27drh7++vrbaTkpKid75u3boMHz6cgIAAk/f8/PPP23TfAoFA4EwCAwPx8fEhKytL73hWVpbJvJe1atUy2/7+/fu8++67bNiwQbtKs3Xr1hw5coR58+YJ06gUoETDSELoXYGrsWQYSdhqHIHteleOkmpuLy8vtm/frv3dWr27ZcsWDh06RE5ODklJSXptunbtSo8ePUw+nzW63RNxm2lkz2TvrDH9/PxM5kgqjXiSYSQhJtKygbWGkYQ7jCN3EBAQwGuvvUZubi7p6encu3ePcuXK0ahRI+rVq2fVG5iQkBBmzJjBpUuXOH/+PIWFhfj7+9O8eXO9hNmmcMTkLxAIBI6ifPnytG/fnqSkJCIjIwFNOeekpCQmTZok26dz584kJSUxZcoU7bHExEQ6d+4MQGFhIYWFhUbxzsfHx2KpaIFADmsNIwmhdwWuwlrDSMIe40gJdOvWzWa9O2DAACIiIkhPT+f69euAJul18+bNjbakyWFOt3sqbjON7Jns3TFmacNRhtHdfBWV/Urm/tqKp02k27OzLDdyIZdO7JQ9fjB9FwCH0nfi62Nf8E87p8mVc9DENeQI0S84YLNhJFFWjCPQVEPTLd1sK15eXtSvX7/Ur6QUCARlg6lTp/L888/ToUMHOnXqRExMDPfu3WP06NEAREdHU7duXebMmQPAP//5T7p37878+fMZMGAAa9as4dChQyxZsgTQxNju3bszbdo0KlasSMOGDdm9ezfx8fF8/vnnbntOgWdiq2Ek4Wl615koTUunnTsJ2KZ3dSlSaXICSdrbHnR1ews701DZahhJeKJx1LNnT7v6+fr6EhoaWqJrl1S3Kwm3bk+zdbIvKCjgxIkT2s+XL1/myJEjVK5cWZvMytKYZRlHrjCK/vYC8c81FMaRGT45edLdt6CPhftZlWi5qoEl4rd+YXXb2c8O03621zCSKEvGkUAgEAg0jBw5kmvXrjFjxgwyMzNp27YtW7du1ea2zMjI0Fs1FB4ezqpVq5g+fTrvvvsuTZs2JSEhgVatWmnbrFmzhnfeeYdnn32WGzdu0LBhQz7++GNefvlllz+fwHOx1zCS8CS960wUp6X/wha9K0f6hVTSL6SWaIxViQv1tLS12GsYSXiicSQoOW41jWyd7K9cuUK7du20v8+bN4958+bRvXt3du3aZdWYZRVHb0k7lf1AGEcW2N5FWRWn0tq/Jnv8UPpOViUu5F8T19i90mjJxtkAjB883eo+eddXASU3jCRKYhxdyrlh93UFAoFA4D4mTZpkcjW5pA11GT58OMOHDzc5Xq1atVi+fLlV1/7ggw+saucqlHY/ZZWSGkYSnqJ3nYnStPSrl+4BtuldXYpUhbwZ+zTP9JlEh+Y97BpDV7dv2/ORzXq3JIaRhDCOyh5uT4Rty2TfqFEjq0rVmRuzLOKMHEbfRTdiVPx5RRlHPGJ5n6orKa+wHDDlfOWDuWQU+fqUM9nGEtIYtvZ3lGEkYY9xJE2gr3Yr8eUdyvz58wkPD6ddu3ZUqFDBYnu1Ws2ZM2fYuHEj9+5pRE3dunXp3r07wcHBVuU9evDgAQcPHiQ5ORnQ5PEIDQ2lS5cuBAYGluyBBAKBoJShlLibk5PD9u3bhWmkABxlGEmUdeNIaVraXr0rN05JNbevTzm79G5JDSMJTzaOHBl3T58+DUDlypVt0u2ehrL+JQocjrOSXretV4nvohtpVxzdzVc5ZFxbkCbSkJoVGBV/3uXXF5QMRxtGEj1Cm9OrdQuSjp5g57F0s211J1ClcffuXbZv385nn33G0aNHzba9ceMGH3/8MatWrdJ+cQG4fPkyq1at4qOPPuLGDfOrqY4dO8Znn32mnUABVCoVR44cITY2llWrVpXsgQQCQakgNjaWRo0aUaFCBcLCwvj111/Ntl+3bh3NmjWjQoUKhIaG8tNPP+md9/Lykv2ZO3euMx/DIbgi7qpUpvWVSqXihx9+IDY2VvvFReA+HG0YSRjq3SOX8hwyrsDzsUfvOtLckYyjrNu3WZmcQn5hoUPGdSbOiru26HZPRJhGpRhnV0lTmnEk8CycYRhJWGMcGU6gt/Jy+d+RHWw5tpPb9+/Ydd1TmefYcHgr+84cQlVs+78HVbGKPad+JeHwNkaNGqVNwLdhwwaOHTsm2+f27dssWLBAO8FFR0czZcoUxo0bx/Dhw7VVHhYsWMDt27dlxzh69Cjr168HNCs6J06cyOTJkxk9ejQREREA4guJQOAmlGTSfP/990ydOpWZM2dy+PBh2rRpQ0REBNnZ2bLt9+3bR1RUFGPHjiU1NZXIyEgiIyNJS0vTtrl69arez7Jly/Dy8mLYMNtzdbgaV8Td1atXy66yLy4uJj4+Xvtn2adPH4c/n8B6nGUYSQjjSCCHPXrX0auBDI0ja9DVu2eyz9t1XXt0uzPi7pQpU3jllVes1u2eitu3pwmcg7MNIwnJOFLCVrVxLr2qoKQ4yzCSMLdVTW4CDf9kiF7/wW378Mmwt6zaXnD7/h06fxxpdHz5mHmENWln3EGGA+dSGb3sDe3vM2fO5LHHHiMoKIgdO3awfv16QkJC8PPz0+u3bNkyAIKCgnjhhRe0S2IDAgKoU6cOISEhLFq0iJs3b7J27VrGjdP/l/LgwQM2bNgAaCa/zp07a5+5evXqNGjQgMaNGxMXF2fVcwgEAschmTRxcXGEhYURExNDREQEJ0+epGZN47I5kkkzZ84cBg4cyKpVq4iMjOTw4cPaZM9Xr17V67NlyxbGjh1rlUnz+eefM27cOG1xkbi4ODZv3syyZct4++23jdp/+eWX9OvXj2nTpgEwa9YsEhMTWbhwoTam1KpVS6/Pxo0b6dGjB02aNLHiT8i9uCLunj17luPHj+sl6wbN2/KMjAwAXnzxRerWrevsxxWYwNmGkURZ36omkMdWvesMdLeqWcJQ70rsfy+BgIpVLPZXq9VM3zCPDYe36h23pNudHXcfffRRq3S7pyJWGpVCXGUYSShlxZHAs3BFhTO5NzDmJtCXnxzF2G4jAdh4JJFle9dadR1dw2j6wMl0bdoRgNHL3iA7N8di/6zca9oJtGvTjrzz1Cvac+Hh4TRs2BCAgwcP6vXLzc0lNzcX0ExecnuofX19GTNmDKApJiC1l0hN1VTvCA0NJTw8XHayDQoKYsSIERafQyAQOBZdk6ZFixbExcVRqVIlrVlsiK5J07x5c2bNmsXf/vY3Fi58WB2zVq1aej/WmjQFBQX89ttv9O7dW3vM29ub3r17s3//ftk++/fv12sPEBERYbJ9VlYWmzdvZuzYsWbvRSm4Ku7+8MMPRucTEhK09yAMI/fhKsNIQqw4Eshhq951BpJxZA5zelfu5ascC5JWaA0jW3S7q+KuJd3uqQjTqJThasNIQgnGkUAgh+5E+t3ufSYn0D1v/5dXe4/m9Yjx7H5rHQDzty2xOP7prPPaz8dn7eCZv0ey5PlPeWfARACmb5hncYxPfowFYMagKSx5/lOeC3/4xt/Ly4uBAwcCkJSUpNfvxAnNW6VWrVppt0PIUblyZTp37gxAerr+8uV9+/YB0KZNG7P32Ly5800+gUDwEKWZNDk5OahUKqNqtEFBQWRmZsr2yczMtKn9ypUrqVKlCkOHDrV4P0rAlXH3zp07sp/r1Klj/Q0LHIqrDSMJYRwJ5LBW7zqTeoHVzZ6X07vHZ+3Qnj+Vec7iNeJ2fQfYrttdFXet0e2eiDCNShHuMowkhHEkUCo9QpvTrF5t/rh0lfzCItkJtEblR7SfH63ycNIrMpMMD+D45ZMAvPPUK3pvLUZ00EwYe08flO2nS+IJzXLef7TpLXve20T1kLw8jVCsUaOGxWtUrKiJB7oJW0GTuA+galVlVR8UCMo6ZdGkWbZsGc8++2ypqTzjyLgrxXvDz9ZsoRY4HncZRhLCOBLIYY3edSdyetfLy4vpAycDcOKK+fyZuprcVt3uyrhrSbd7IsI0KiW42zCSEMaRQIlcyrnBn1nXtL/v++OMURvdxNW6ye98LJR7lSatPQbm0LkczZ7nBjUsbxuoVkkzgV28cUX2vDTR+fjob8Ms95cQKGeFIJDekJhqW758eYtjCASC0oUtJk1gYCA+Pj5kZWXpHc/KyjLKSyRRq1Ytq9vv2bOHkydP8uKLL9rwBMrGkXFXdwxzK5wEzsfdhpGEMI4Ehlijd92JKb278w/NatiaVc2/BNXV5PbqdlfEXUu63RMRplEpQCmGkYQwjgRKQndP9/sjBpmsMjEybiLX7lznVt5tIhdqkpY2DWps8W1CeHB7QLOiaM2v/+Ne/n3Sr55hWOxLAHw0eKrFe/wq6kMAhsaOJ/3qGfILC/TOX7mimVxbt26td7xRo0YA7NixQ7bKgy7Svmqpj0RISAgAx48fN9vf8E25QCBwLkozacqXL0/79u31ltsXFxeTlJSk3YZlSOfOnY2W5ycmJsq2/+abb2jfvr3ZrQPpHyeT/nEyabMStVXfZg5+jUMzNrN+0hLtsZVjP9e2NfXz7Ysx2vbrJy3hyAdbeXfgJO2xtFmJFsdwZdytVq2a9vMjjzx8w37r1i2zYwgci1IMIwlhHAkkrNW7ziS/sNDseTm9++3+9aScOQRgsXiMl5cX7Rq0BGzX7a6Mu5Z0uyciTCMPR2mGkYQwjgSWuJRzwyXXMNzTbao86Ykrp+n+2QjCPxnK6aw/AfjPuC8tXsPH24flYzR5iz7aFEPHWQO1hlH/0Cfp1KStxTE6NG5N/9AnARgW+xLtPuyvPZeVlcW2bdsA6NKli16/evXqaT8nJyebHF+3RLduH3hYpjkxMZHLly/L9i8oKCA+Pt7icwgEpQFHl7gHTU6bQYMGERAQgL+/Px07dtRWYDGFEkwaQ6ZOncrXX3/NypUrSU9PZ8KECdy7d09bTS06Opp33nlH2/6f//wnW7duZf78+fzxxx988MEHHDp0iEmTJumNm5uby7p166w2sJwVd+ds1uTbWDFmPj7elgtsuCruhoWF6W138Pb2Jjw8HIAlS5ZQaOGLmsAxKM0wkhDGkcAWvess8gsLWZmcYraNI+Lu4ug5gO263VVx11rd7mkI08iDUaphJCGMI4E5lifvcapxZK5qhNxE+mb/l/H9a7Ka2vdFjnywhcoV/K26VliTdux8cw19Wz4BwCOVAogf+wXzR75v9f3OH/k+K8bM19vOlp6eri1LHRISYpS7yMvLi6ioKAD27t1LSkoKBQUPVykVFRXx448/smXLFgCioqKM3sAEBgZqS4ouXbqUCxcuUFxcrD1/69YtYmJiyM7OtvpZBAJPRSpxP3PmTA4fPkybNm2IiIgw+fdfKnE/duxYUlNTiYyMJDIykrS0NG2bs2fP0rVrV5o1a8auXbs4evQo77//vlVbwpRi0kiMHDmSefPmMWPGDNq2bcuRI0fYunWrNo9SRkYGV69e1bYPDw9n1apVLFmyhDZt2vDf//6XhIQEozLGa9asQa1Wa+OZNTgj7nZt2pFdb35vlekErou7TzzxhNG1pS8j9+/fZ+nSpdy+fdvKpxbYg1INIwlhHJVdbNW7zkAyjLKsiEMljbuVK/hz5IMtNut2V8Rda3S7p+KltrS2tgySm5tLQEAAt2/fVkxy2PbT9N/yu9sw2lBlrtVtj1zKY1T8eUJqViD+uYZU9rPsIttD1CPW/78qLizml9d/oelzTXm046N2Xe/awWuc/vY0f5//d7zLGfuvn6xTVs6BYx3fkD1+8MRO4rd+weev/kA5X/v22i5OmA3AS5HTre4zN340WbdvM7pnN4vVFmzF2jKjO4+lk3T0BL1at2DBiAUOvYeS0mJ6L+3nxx57jKioKKO90RKpqals2rTJ7HiDBg2iXTv5Zb8qlYr4+HiLKx/EdCEo7YSFhdGxY0dtifri4mLq16/P5MmTefvtt43ajxw5knv37vHjjz9qj/3973+nbdu2WuH49NNPU65cOb799lu77mnhwoXMnTuXzMxM2rZty1dffUVYWBgATz75JI0aNWLFihXa9uvWrWP69OmcP3+epk2b8q9//YunnnpKb8wlS5YwZcoUrl69SkBAgF335Q7SPza9sscd6MZpORwRdydOnEhgYKDsuZycHGJjY7W/ixhtGUM9bQ3ONIxs0dPWcDdfRfS3FziV/YDvohvRtl4lm/rLaWlLetca0pdoDIvm422rxKo0LT3p8gPANr2rS2FRIVO/GkZ0v9fo2KKHXWPo6vZTJxfarHd7hDq2Gq6uYTS6Zzde7Wbfn42z+L7wZ+1nZ8RdOczp9g8++MDyTbsIWzwPsdLIA3G3YWQrYsWRQI7ne3YhKCDA4SuOrDWMQP8NjFIZNmwYzz77rEnDCKBdu3a89NJLNGjQwOhc/fr1eemll0x+cQFNor4XXniBIUOGyL4F6dKlC9OmTbPvAQQCCyhlO5gzStwXFxezefNmQkJCiIiIoGbNmoSFhZGQkGD2XnSZNGkSFy5cID8/nwMHDmgNI4Bdu3bpGUYAw4cP5+TJk+Tn55OWlmZkGAGMHz+evLw8jzKMlIgr4q6pLy6geXM+bdo0k9sVBSVH6SuMDBErjsoW9uhdR644MjSMHP0S2NG4Iu5ao9s9EWXZtwKLKMEwUhXct7mPZByNij9P9LcXnLriSOAZ+JUrx/M9u7AyOYXlyXscMtnYYhhJOPqNi6OYOnUqVapUsbp9rVq1GD16NPn5+eTm5gKasqJ+fn5W9ffy8qJ169a0bt2aO3fukJeXR7ly5ahWrZrJ0qECQUmRtoPFxcURFhZGTEwMERERnDx5kpo1axq1l7aDzZkzh4EDB7Jq1SoiIyM5fPiwdtm5tB1s7NixfPjhh1StWpXjx49b3A5mrsT9H3/8IdvHUon77Oxs7t69y6effsrs2bP57LPP2Lp1K0OHDmXnzp10797d6j8rgfJQQtytVKkSffv2NTIvBSXH0wwjCck4iv72AqPiz9u14kjgGdijd6UXpSXVv55mGEk4Ou7eunWLwsJCKlWqZJNu9zSEaeRBKMUwOvPfefCS5VwMhpQ14+jd42mWG7mQ3MuzZY+nndO81V+ycTa+PvZtT5PGkLapWcPTnes71DiyxzCSUKJxZO/E4+fnx6OP2rflUvfapXniEyiHzz//nHHjxmlz9MTFxbF582aWLVsmux3syy+/pF+/ftqVb7NmzSIxMZGFCxdqt4O99957PPXUU/zrX//S9nvsMfd82ZPyJQwePJjXXnsNgLZt27Jv3z7i4uKEaVRKUELcFea+Y/FUw0iitBhHStPSaTc0K+Nt0bu6FKk0yZPjt37B4VPmk0abvAcd3W6v3i2pceSphpEhjoi71at75rPbiphhPAQlGUb3c+QzzluD2KomMEQyjkqyVa0khpFAIHAPStsO5owS94GBgfj6+tKiRQu9Ns2bN7e4XU4gELgHTzeMJMRWtdKPPXq3pFvVSothJLANsdLIQ1CSYdR0+DRgvd1jlZUVR5+0bGW5kQuxlAh7/ODpLk2EfTtrpfZzSVYclTXDSFpK6+vryyOPPGLz2+Xi4mJu3rxJUVFRqV9KK1A2StsOplviPjIyEnhY4t6w+piEVOJ+ypQp2mO6Je7Lly9Px44dOXnypF6/U6dO0bBhQ5P3IihdOCLu2rsFTmAbpcUwkvD0FUdK09JKSoQ9fvB08q6vsmsMe1cceaJhlJWV5da4a6jbPRVhGnkISjKMHDGBlhXjSGA99hhHZcUwUqvVpKWlsX69sVkbFhbGE088QaVK5kVgXl4eKSkp7Nu3z+hcZGQkrVu39uhSoAIBlHw72NSpU3n++efp0KEDnTp1IiYmxqjEfd26dZkzZw6gKXHfvXt35s+fz4ABA1izZg2HDh1iyZIl2jGnTZvGyJEjeeKJJ+jRowdbt27lf//7H7t27XLCn4BASTgi7mZmZrJ582YuXbqkd7x27doMGjTI5Co4ge2UNsNIwtONI4FzsNU48kTDCNBuXQfXxV1zuv2jjz6y4e6Vg9ie5iGUJsNIQmxVExhiy1a1smIYqVQq/vOf/8hOPAAHDhxg7ty55OTkmBwjJyeHuXPnyn5xAUhISGDFihWoVKb/Daamptp24wKBBZS4HWzkyJHMmzePGTNm0LZtW44cOcLWrVu1q5syMjK4evWqtn14eDirVq1iyZIltGnThv/+978kJCRok3IDDBkyhLi4OP71r38RGhrK0qVL+eGHH+jatavF+xEoG1fE3cWLFxt9cQG4evUqixcvFrHZQZRWw0hCbFUTyGHtVjVPNYwMcUXctaTbPRWx0shDKG2GkYRYcSQwxJoVR+4wjA79eZRXV8/kVl4ufVp0492BEwmqan3i04KiAlam/MAXiUvx8fbmtb7jGPX3SMr7ljfb7/vvv+fs2bMAjBgxgubNH74JunfvHvHx8WRnZxMbG8u0adOMVhzl5eURGxsLaKo9jBs3jmrVqgGaNyFXrlxh6dKlZGRkkJCQwLBhw4zu4eTJk2zatMnqZxUIrEGp28EmTZpk8vpyq4OGDx/O8OHDzY45ZswYxowZY9X1BabJyr3mkrirS3ZuDu+tn0vKmUPUr16HjyKnEtakHYDL4m6HDh2IiIjA11cj21UqFQcOHCAxMZFNmzbh7+9PSEiI1c8k0Ke0G0YSYsWRQA5LK45sNYycHXetRdLtM2fOdGnctaTbPRVhGglkceUEKowjgSHmjCN3GEZT18xia9ou7e+JJ/aQeGIPy8fMs2oSu/vgHp1mD9L+riouZt7Wxczbuphfp2+icgV/2X45OTmcPn0agAkTJhiVIPf392fs2LHExMRw//599u3bZ5Qk+OeffwagZs2avPjii5TT+fPy8vKibt26vPjiiyxdupS0tDSefPJJatSooW2jVqtZs2aNxWcUCOxBbAdTNh+P+j+b+5iK0TuPpZN09AS9Wrewu2LP9P/84PS4q8uBc6mMXvYwH+DFG1cYvewN+oc+yfyR7wM4Pe7279+fTp066Y3v4+NDeHg49+/fZ+/evaxevZoZM2ZYfB6BMWXFMJIQxpFADlPGkT0rjNp+0F/72Vlx1xKGut2ZcVd3q5s1ut1TEaaRwAh3TKDCOBIYImccAS43jA6cS9VOPOsnLqF+9Tr87/cdfLQphtHL3uDYR9vx8Tb/d/WZJa8C0DSoMSvHzqdQpWLCt+9y4sppXop/h/+M/0q2X0qKphyrVAVKjvLlyzN+/Hi+/PJLUlJS6NmzpzY5dnFxMQcOHAA0X77Lmfjzqlu3Lr1792bHjh1s376dqKgo7Tm55bkCgaMYOXIk165dY8aMGWRmZtK2bVuj7WC6yd6l7WDTp0/n3XffpWnTpia3g82ZM4dXX32Vxx9/3GO2gy18/X9Wt7WnAIEujkjIKke9wOqM7tmN5cl7WJmcwvM9u7DvjzMlNowknB13JVTFKu0Xl3cHTGJ4hwH8mXORobHj2XJsF1GdBgM4Pe4afnHRpWfPnuzdu9eoj8A6lGAY3c1XuVzvCuNIIIehcRTeLNjuLWnOjLsdGrc2O4ahbv+dh1W/nRF369evrz1njW73VEROI4Ee7pxARY4jgSG6OY7itu0kbttOl+cwmpHwOaCZeJrVfgx/v4o83ekfdG3aEYB9Z34z21+tVnMm+zwACZO+plqlAB6tUp3vX9ZsXUjNOI5arZbte+zYMQDq1Klj9hrStgeAW7duaT/fvHlT+9nS2w3pS/epU6f0jp8/r7l3cxOoQFASJk2axIULF8jPz+fAgQOEhYVpz+3atYsVK1botR8+fDgnT54kPz+ftLQ0nnrqKaMxx4wZw+nTp7l//z5Hjhxh8ODBzn4MgQ6ScZR1+zaz1m5ymGHkirgrceBcqvaaozoPwa9ceZrVfowfJi4G4J+rP9C2dVfc9fLy0q5ykvoIrEMJhhHgNr0rchwJ5NDNcTRr7Sa7cxi5Iu6awlC3G+LMuGutbvdEhGkk0KKECVQYRwJD/MqVo1ebh0lte7Vp4dKk1xdvXAGgcWB9vePSl5frd28a9dFF9Vc1J0BvCavuW3LdNnp9/0rUV7lyZavvt7CwUPu5qKjI6n75+flmx7PlHgQCgaBeYHUaBz3MPxTeLLjEY7oi7kpk5WqSXD/5+N/1jjeoXheAm3m3tcfcGXcLCgqM7kFgHiXoXYCr+ze6Ve8K40ggh26sbhz0qF1Jr10Rd01hSrdLODPu2qPbPQVhGgkA5UygIIwjgT6Xcm6w6uf91KoWQK1qAaz6eb/ZqmqOpktwBwDWHdqsPaZWq/n0p0UAtKr3uNn+vj4PJ8trdx7et+6XHt02chRbmGR139zoJmQ1TM5qjjt37gBQpUoVvePSGPfv37d6LIFAINh5LJ0/Ll2lWb3a+JXzZWVyCvklNDZcGXdb1tEkOJ394wK9GPu/I4kA9G35hPaYO+Pu9evXAc/OleFKlKJ3r+7fyNWU9W7Xu4bGkaBsI+Uw8ivnS7N6tfnj0lWzVdVM4Yq4awo53a6LK+KuJd3uiQjTSKCYCVQXYRwJQD+h6ri+3RnXtztBAQEsT97jMuPo46HTAPhk80LGrXiLVb8k0PL9h0lPg2s2sjjG6xHjAej+2XDmbV3Ml4nL6PapJsHshCdHmezXs2dPADZv3mx2Se/lyw/3a+tOgrqf09PNT/pHjhwBNDljdJFKl+/fv99sf4FAIJDQTXo9qnu4dquaI4wjcG7clQip1UT7ueX7vYnf9wPjV77NR//7EoB3BryiPe/MuHv37l2T/YuKijh+/DiAXoUegWmUoHclw6h2l6GK0Lu6xpGg7GKY9HpU93DtVjVbjSNXxF1TGOp2Q5wZd63V7Z6IMI3KOEo0jCSUMJEK3IdcBR7dHEeuMo5qVg1k+Zh5AKScOcTsHxdoz+1/L8GqMcZ0HcHgtn0AWLZ3LYt3/weAoX/rx6ReL5js17GjZivG+fPn2bdvn2yby5cv88033wCaBMCGDB06FIC1a9eSlZVldF6tVpOSkkJaWhoA7drpVyWqWrVqqdybXZaJjY2lUaNGVKhQgbCwMH799Vez7detW0ezZs2oUKECoaGh/PTTT3rnX3jhBby8vPR++vXr58xHECgYuSppujmOSmocOTvumrrWpz8tYu/pgwAsHzOPoKqarXfOjrvLli2T3fL24MEDli5dqm1ftWpVq56prONuvatrGNXurMm1pgS9KxlHgrKJqSppujmOrDWOXBF3zWGo2yVcEXet0e2eiqieVoZRsmEkIaqqlU1MlWwG+apq9uy3toWwJu049tF2DpxLJTv3Oi3rhtA0qLHV/b28vJjzf2/z9oCJ7DtzCFVxMd1COhFQsYrZfhUqVGDo0KGsX7+eHTt2cPnyZTp06KAtEZqWlsaOHTsAaNCgAaGhoUZjtGrVit9//52zZ88SFxdHz549adq0KZUqVaKgoIAff/yRCxcuAJovOn5+fkZjjBgxgpiYGKufV6Bcvv/+e6ZOnUpcXBxhYWHExMQQERHByZMnZSt97Nu3j6ioKObMmcPAgQNZtWoVkZGRHD58WK9iWb9+/Vi+fLn2d7m/R4LSj5xhJCFXVc3W/HQnZidZ3dbeuKtLQMUqnJidxJns86RdOkmNyo8QHtxem6PDFXH35s2bfPrppwwaNIj69evj4+PD9evXiY+P17YdM2aM1c9U1lGaYSShBL0r9HXZxJRhJGFYVc1SQQNnx11r0NXtx3MzXRZ3rdHtnoowjcoonmAYSShhIhW4DnOGkYQ7jCMfbx/C/9onbS8BFavQP9S20tahoaGo1Wo2bNhAenq67HaH0NBQBg8erJd4UMLLy4uoqCjWrl3LqVOnSE5OJjk52ajd0KFDZb/8AAQEBDB58mSb7lvwkNjYWObOnUtmZiZt2rRhwYIFZqtzrFu3jvfff5/z58/TtGlTPvvsM9kKYQAvv/wyixcv5osvvmDKlCkW7+Xzzz9n3LhxjB49GoC4uDg2b97MsmXLePvtt43af/nll/Tr149p0zTLvWfNmkViYiILFy4kLi5O287Pz49atWpZvL6S2P1Ed3ffgjEd33D3HdiNOcNIwhHGka3YE3cNCa7ZSHZLXHR0tNPj7oIFC1CpVGzYsMGojY+PD6+88goBAQG2P1QZRYmGkYTQuwJXY8kwkrDVOHJm3LUWSbe/OL2X3nFnx11rdLsnIranlUE8yTCSUMLSXYHzscYwknDHVjV30bp1a9566y369+9PhQoPcw6EhIQwadIkhg4dio+Z5II+Pj5ERUUxceJE2rdvr3euV69evP322yYnUInq1Z1rypVWpJU9M2fO5PDhw7Rp04aIiAiys7Nl20sre8aOHUtqaiqRkZFERkbKvp3asGEDv/zyi9XbBwsKCvjtt9+0pWIBvL296d27t8mcVfv379drDxAREWHUfteuXdSsWZPHH3+cCRMmaJNECsoG1hhGEo7cquZuXBF3Z8yYwbPPPku9evW0x/39/XnmmWd47733RGz2AKwxjCSE3hW4CmsNIwl7tqopCVfGXXO63VMRK43KGJ5oGEl42huYAoVlzi8skhfmRapCvf/ag9TX1DWswRbDSMIdK47cRYUKFejUqZPZFSqWCAwMZODAgQwcONCBdyYwh7NW9ly+fJnJkyezbds2BgwYYNW95OTkoFKpCAoK0jseFBTEH3/8IdsnMzNTtn1mZqb29379+jF06FAaN27M2bNneffdd+nfvz/79+83+6Xa3SgtRoNtMbSkcVc39pckdttiGEm4Y8WRuyhp3PXy8iI4OJjg4GDLjQWKwxbDSMLT9K4zUVqcVkLcdYRut9UwkrB1xZESmDlzps19HBF3HaHblYQwjcoQjjSMvtqdzavdjfNfOBtPmkj7pux19y3oY+F+3ox9usSXmPrVMKvbzn72YVt7DCOJsmQcCTwLaWXPO++8oz1mzcqeqVOn6h2LiIggISFB+3txcTHPPfcc06ZNo2XLlk65d1t4+umHsSM0NJTWrVvz2GOPsWvXLnr1ergsvPPnnfnl9V9o+lxTHu1oOZmlHNcOXuP0t6f5+/y/413OvsXS6Us0b0hvpt200NIN2DFv2BJ35ViVuJBViQtlz+nGaTnsMYwkypJxJCib2GMYSXiS3nUmitPSf+HMuGstb8Y+bTFGy2GvYSThicaRoOQI06iM4OgVRvOTNVsrhHFkmncff9zdt6DHpcbyOVEOpu8i/UIqz/SZhK+PfYI9fusXAET3e82GXhma+yqBYSRRUuPIk7dHCJSLs1b2fPbZZ/j6+vLqq6/adD+BgYH4+PgYVXPKysoymY+oVq1aNrUHaNKkCYGBgZw5c0bPNFIaSovRYDpOy2Ff3H1IkaqQVYkLad6wHR2bP2miVYbJ/iUxjCSEcSQorZTEMJLwFL3rTJQWpz85eRJwdtw1j65uv5Rz3Ga9WxLDSEIYR2UPYRqVAZyxJe31njUVZxwpjb41gyw3ciHHWphOSJd+IZUOzXtQztc+sX74VAoAHc1cw5DbWSsdYhhJ2GscSRNoVOt/2n1tZ/DLL7/QokULm8ooFxUVkZ6ers0nU6NGDZo3b46vr/WhPjc3lxMnTpCXl0e5cuVo1KgR9erVk032KnA9v/32G19++SWHDx+2+f9J+fLlad++PUlJSURGRgKaVUtJSUlMmjRJtk/nzp1JSkrSS7KdmJhI586dTV7n0qVLXL9+ndq1a9t0f65GaTEazMdpQ+yJu7oUFmm+vHRs/qTJMW5nrZQ97gjDSMLTjSMlxF21Ws3Fixftun+B43GEYSRR1o0jpcXpTYWa5MfOjLvWIOn2d/4dZ7PeLalhJOHpxpGj4u6FCxcoLCykUqVKNut2T0KYRqUcZ+UwkowiJRlHftMecfk9COzHkYaRhK3Gke4EqjS2bdvGtm3bqFq1KmPGjDFbHae4uJidO3eyd6/8Mu4OHTrQv39/vL1Nb+e5ffs2a9eu5cqVK7Lno6KibHsAgVNW9uzZs4fs7GwaNGigPa9SqXj99deJiYnh/PnzZu9p6tSpPP/883To0IFOnToRExPDvXv3tDmXoqOjqVu3LnPmzAHgn//8J927d2f+/PkMGDCANWvWcOjQIZYsWQLA3bt3+fDDDxk2bBi1atXi7NmzvPnmmwQHBxMREWH9H5bAY3CkYSThqcZRUlKS0+OupcSpp06dYvXq1QAsW7bMyjsXOAtHGkYSZd04EphGKgZji951ZBoHTzSOHB13dbFWt3sionpaKcbZSa9f7V5Tu+Loq93ylYCcjW6VCYFn4WjDSMLaqmqGEyhozJfiEiZdLFKpUKvVdvdXq9UUqVTaXDW5ubnExMRw44b8cxQXF7Ny5UrtF5fGjRvTp08funTpol3pcejQIVauXGny2W7cuEFMTIx2Au3cuTM9e/akY8eO2jZyk6PAPLoreySklT2mVupIK3t00V3Z89xzz3H06FGOHDmi/alTpw7Tpk1j27ZtFu9p5MiRzJs3jxkzZtC2bVuOHDnC1q1btVviMjIyuHr1qrZ9eHg4q1atYsmSJbRp04b//ve/JCQk0KpVK0BTIero0aMMGjSIkJAQxo4dS/v27dmzZw9+fn62/YEJFI8zDCOJklRVc1TctRVXxN3U1FST109NTRWxWUE4wzCSEFXVBHLYo3cdnffT3qpq9sZdXWzV7c6Iu71796Z79+5W63ZPRaw0KqW4qkqaklYcfebyKwtKgjMMIwlLK47kJtAW0/Vzr6x+4tP+HwAA0PBJREFUaSFt6lv3pUitVrNs71rmb1uiPRZcsxGrxn9F5Qr+Vo1x98E9xq98myMXNW9rZs6cSWRkJEuXLiUrK4tFixbx3nvvGS2b3bJlCxkZmrwj0dHRNG7cWO98Tk4OsbGxZGRksHPnTqMcM2q1mgULFgCaMqOjR4+mcuXK2vNPPfUUv/76K1u2bLHqOQT6OHplT40aNahRo4beNcqVK0etWrV43MrcD5MmTTK5HW3Xrl1Gx4YPH87w4cNl21esWNEqs0rg+TjTMJKwZcWRM+IuwMtPjmJyrxes3qLg7Li7adMmateubbQ6MTMzk02bNgHQrVs3evSwf6uLoOQ40zCSECuOBIbYo3edgd6KoxHm2zoi7qZdPsmIf7+id8ySbndW3NW9Z2t0u6ciVhqVQlxlGEkoZcWRwLNw9vYDUyuOrJ1AoxZP4vjlU1Zd690fPtP74gJwJvs8nWYPoqCowGL/gqICOs0epDeBAvj6+vLCCy8Ami1IZ86c0TtfWFjIoUOHAJg4caLRFxfQbJGKjo4GNG/Fi4qK9O/zrzF9fX2ZMGGC3gQq0alTJ9q3b2/xOQTGOHplj0DgDlxhGEmYWnFk+BbdWXE3btd3vL9hnlX36qq4+9NPPxmdl7649O3bl549e5aaLyaeiCsMIwmx4khgSEn1rqOQVhyZwxFx9/eL6UaGEVjW7a6Iu9bodk9FEaZRbGwsjRo1okKFCoSFhfHrr7+abb9u3TqaNWtGhQoVCA0NNfqf+sILGqdS96dfv37OfATF4GrDSEIJxpHAs3BFvgrDifRsZrbJCTQmaiYnZidxYnYSMVEzARj+7wkWr3H7/h02HkkEYPdb6zgxO4njs3bQNEjzReLb/estjvHdLwkAtKwTwrGPtnNi9sPtSRUqVNB++di4caNeP6kCV+3atQkMDDQ5fuPGjbVfbNLT9ZcO7969G4AhQ4aYTdxaVmKoM5g0aRIXLlwgPz+fAwcOEBYWpj23a9cuVqxYodd++PDhnDx5kvz8fNLS0njqKfMVtc6fP6+XqFogcBQ7j6W71DCSMDSOzmZmszx5j/a8s+Lunrf/C8D6w1u5ff+OxTFcFXcvXrxIfn6+9nh+fr7WbO7UqZPF+xQ4D1caRhLCOBIYYovedSaW5ghHxN2oxZqV0rbqdlfFXWt0uyfidtPo+++/Z+rUqcycOZPDhw/Tpk0bIiIiyM6WNx727dtHVFQUY8eOJTU1lcjISCIjI0lLS9Nr169fP65evar9KQt7vt1lGEkI40igRKSJ9BF/f5Yn7SEj57rsBNq35ROyny3tld53RrPSZ2y3kTxaRTOml5cXK8fOB2D+tq8t3uMX2zVtFj33MT7exkvNq1fXjHvv3j2941K1niZNmli8RnBwsF4ficuXLwNQrVo1s/1tqQQkEAhKB0lHT7jcMJKQjKOMnOssT9rDI/4Pt5w5K+7WqPwIL3V/FoA9p8y/wLQmj4Yj425ubq7sZx8fsT3JXbjDMJIQxpHAEGv1rjtxZNy1Vbe7Mu5a0u2eiNtNo88//5xx48YxevRoWrRoQVxcHJUqVTJZAeLLL7+kX79+TJs2jebNmzNr1iz+9re/sXDhQr12fn5+1KpVS/vzyCOlu7KWuw0jCWEcCcoaUhK/cj76K6cKVZrtCL4yJpAhqr8muHImJiHDSceQSpUsb8+0NIa/v3U5QAQCgcDdODPuFqo02wd8zFQ9sxYRd0sv7jSMJIRxJPA0XBF3LeGKuGsp9nsibjWNCgoK+O233+jdu7f2mLe3N71792b//v2yffbv36/XHiAiIsKo/a5du6hZsyaPP/44EyZMMOvs5efnk5ubq/fjSSjFMJIQxpFASUh7um/eu8foXt1oEFhDtsrE9uM/y342Vy4Z4InHNVuN4nZ9x/W7NwFQFauY8O17AEyNGGfxHl/r8yIAz3/zumwFoFu3bgFQr149veNSQmTDPdNynDt3Tq+PhDR5WqryoCphhQuBQOB59Grdwq6qOI7gUs4NlifvoUFgDUb36sZNHRHurLh77c4Nlu1dC0B4cAez/S3NDeDYuFu1alXZz4b5kgTORwmGkYQwjgQS1updd+LIuGurbndl3LWk2z0Rt5pGOTk5qFQqbUJQiaCgIDIzM2X7ZGZmWmzfr18/4uPjSUpK4rPPPmP37t3079/f5JeeOXPmEBAQoP2pX79+CZ/MdSjNMJIQxpHAEraUUy7JNXT3dD9Wq6bJ8qRTVn9Ii+m9aDG9F1NWfwhoKjFYIqBiFYb8TbP/udun/0eL6b0IndGXE1dOA/BMmGVB+XyXYQCczvqTlu/31qvkVlRUxIYNGwB44okn9Po1b67ZLvLnn3/y559/mhz/2rVr2hgp9ZEYPFhzf/Hx8Tx48MDkGAcOHLD4HAKBoHTRI7S53eWUS4JkGElVNh+rVZPRPbtpzzsr7nb/TFMlcHDbPgRUrGJxDFfF3Tp16uDn56c97ufnp/0yIqoXuhYlGUYSwjgS2KJ3nYmlOcIRcXfdhH8Dtut2V8Vda3S7J+L27WnO4Omnn2bQoEGEhoYSGRnJjz/+yMGDB2VLCQO888473L59W/tz8eJF196wnSjVMJIQxpHAHLpVcZyBqaoRpqpMGLJuwr/Nlu7UZfaQN5jw5Ci9Y39r0IqD7/+P8r7lLfYv71ueX6dv0iZxlbh79y6LFi3SvtGQ8mNI+Pr60rVrV0AzCV67ds1o7HPnzrFo0SIAOnToYJSbKDg4WLs3e8WKFUZvT9RqNUlJSSQmJlp8DoFAUDpxpXFkaBhJRRMM83I4K+6+0e8lPhn2llX36qq4+49//MPoGgMGDADg0KFDFgvICByDEg0jCWEclV1KqncdhVQ0wRyOiLst64bImkOWdLsr4q41ut1TceudBwYG4uPjQ1ZWlt7xrKwsatWqJdunVq1aNrUHTZLYwMBAzpw5Q69evYzO+/n56TmJnoDSDSOJV7vXBGB+crbe7wKBVBVH90uBo7BUZlSaSFcmp7A8eQ+je3bjxOwkbfI8a7Yd6OLl5cXk3qOZ3Hs0RSoVPt7eNpc/rlzBn42Tl6JWq1EVF/PS1pl6224nT54sO2aPHj3IyMggIyODRYsWERgYSJs2bfDx8eH06dPaN+ENGzakf//+svf+yiuvsGDBArKysvj4449p2bIlderUQa1Ws2PHDpueQyAQlE6kRNjSlwJnJMY2ZRjJ4Yy462tHUmlnx91BgwbJatxatWoxaNAgNm3axJYtW9iyZQsffPCBzfcvsA4lG0YSknE0Kv480d9eIP65hlT2E4nSSzP26F1nJMbWrbJpCUfE3Tb1m9us250Vd7t06YK/vz937tyxSrd7Km5daVS+fHnat29PUtLD8tLFxcUkJSXRuXNn2T6dO3fWaw+QmJhosj3ApUuXuH79OrVr13bMjbsZTzGMJMSKI4EcuuWUHbniyNIEKmH4BgY0k46thpEhvj4+JZokvLy88PXx0U48devWZcqUKdpKDIZ4e3vz/PPP06GDZh94Tk4OSUlJbN++XfvFpWvXrkRHR5t8turVqzNlyhTtfu3jx4+TmJioN4FGRUXZ/UwCgaB04MwVR7YYRoY4Ku7aiivibrt27Uxev127diI2uwBPMIwkxIqjsoU9etfRK450DSNbXibYG3d1sVW3OyPupqSksH37dqt1u6fi9jVSU6dO1X7h6dSpEzExMdy7d4/Ro0cDEB0dTd26dZkzZw4A//znP+nevTvz589nwIABrFmzhkOHDrFkyRJAsyzsww8/ZNiwYdSqVYuzZ8/y5ptvEhwcTEREhNue01F4mmEkIVYcCQyRyikvT97jsBVH1hpGErpvYJRGv379aN68uV7iPVN4e3szYMAAIiIiSE9P1yb+r1GjBs2bN7dqOWxAQACvvfYaubm5pKenc+/ePcqVK0ejRo2oV69eqXpbUlbI+Ci0xGPczVcR/e0FTmU/4LvoRrStZ7lSn0kqVy7x/QjcjzNWHJXEMHInSoi7ISEhzJgxg0uXLpX4eQTGeJJhJCFWHJUd7NG7jlxxZK9h5E4cHXfPnz9PYWEh/v7+Vut2T8TtptHIkSO5du0aM2bMIDMzk7Zt27J161ZtsuuMjAw9BzE8PJxVq1Yxffp03n33XZo2bUpCQgKtWrUCwMfHh6NHj7Jy5Upu3bpFnTp16Nu3L7NmzfK4LWiGKMUwurp/I/S1vV9ZM462Z2dZbuRCLp3YKXv8YPouAA6l78TXxz6hnnZOs6/3oIlryBFSw7HGka2GkYQ0kSqNsLAwm/v4+voSGloyo6Bq1ap2XVsA7afFyx6/d/Usp9fNpWJgXYL/7w18yld0yf1ssJxP0iKV/XyIf64h0d9eYFT8+ZIbRwpCaTEaTMdpOeyJu7oU/VXiWJoD5AgxUejFkcaRpxpGEkqIu15eXh5VxMVT8ETDSKK0GEdKi9Np504Czo27ltDV7fboXUcZR55oGOniqLhbVmKv200jgEmTJjFp0iTZc3LJq4cPH87w4cNl21esWLFUVpJQkmF0NWU99G1lV/+yZBx9cvKku29BHwv3syrRcqUwS8Rv/cLqtrOf1VRQcIRxZK9hJOFpX1IEnoO7DCNHUlqNI8XFaLAYp+WwJe7KkX4hlfQLqbLnpDgthyOMI083jASlF082jCRKg3GkyDiNc+OutaxKXGg2RsvhKOPI0w0jge0owjQSWEZJhlHtLkOBU3aPU1aMo+1durr7FvRIa/+a7PFD6TtZlbiQf01cY/dKoyUbZwMwfvB0q/vkXV+l/VwS46ikhpFA4CxKg2EkURqNI6XFaDAdp+WwJ+7qUqQq5M3Yp3mmzyQ6NO8h20Y3TstREuNIGEYCpVIaDCMJTzeOlBanX710D3Bu3LWErm4vuLXO5v4lNY6EYVQ2EaaRh6Akw0gzgc4t0XhlwTgqX8KEyo6mnK+8IJeMIl+fcibbWEIaw97+YJ9xVJYMowcPHnDw4EGSk5MBzVbc0NBQunTpQmBgoFVj5OTksH37dk6fPg1A5cqVCQ8Pp127dlSoUMFp914WKU2GkURpM46UFqPBthjqiLgrjVOSMewxjsqSYVTSuKtWqzlz5gwbN27k3j3NF9a6devSvXt3goODRb45B1OaDCMJTzaOlBanlRB3dXV7gZ3Xt9c48kTD6MMPP3RL3DWl2z21yqUwjTwEZRlGjqEsGEcC27DFOCpLhtGxY8dYv3693jGVSsWRI0c4cuQITZs2ZeTIkfiYqEKhUqlISEggLS1N7/jdu3fZvn0727dvZ8iQIbRu3drkPdy44dhqG6UZJRhGqoL7Thm3tBlHAsdgi3FUmgwjlUrl9Li7aNEiVCr9CliXL19m1SrNKrDJkyeXuio97qI0GkYSnmwcCZyDrcaRJxpGEq6Ou+Z0u6ciTCMPobQZRhLCOBIYYo1x5A7DSFWsYt+Z37h+9yat6j1OcM1GNo9xKy+XPad+xdfHh/DgDgRUtJyp+OjRo2zYsAGARo0aMWDAALy9vbl79y5Xrlxh27ZtnD59mtWrV/Pss88avf0oLi4mPj6ejIwMAPr06UPLli0pKCggNzeX33//nWPHjrFhwwa8vLxkE7revn2bBQsW8NVXX9n8zGUNpRhGZ/47D15yzuoxYRwJ5LDGOHKHYWRP3DXkVOY5jl85RVDVQMKatMPHW/Nl2xVxVyI6Oprq1atz7949bt26xYYNGygqKmLBggVMmTKFgIAAm59L8JDSbBhJCONIYIi1xpGthpEz4661SLr9lVdecWnctaTbPRVhGglM4qoJVBhHAkPMGUfuMIwOnEtl9LI3jI7vfy/BqolQrVYzfcM8Nhzeqnd8cNs+fDLsLZPLXB88eKCdePr06UPnzp21batXr06DBg1o3LgxcXFxnD17luPHj2srSUocO3ZM+8XlxRdfpG7dutpzjz76KI899hhBQUHs2LGD9evXExISYlRpctmyZRafUaAsw+h+zmXAeS8bhHEkkMOcceQIw6jF9F5Oj7u63L5/h84fRxodXz5mHmFN2rkk7gYFBfHCCy9ot1QEBARQp04dQkJCWLRoETdv3mTt2rWMGzfO4vMI5CkLhpGEMI4Ehlgyjmw1jN5bP9epcdcadHX7zJkzXRZ3rdHtnoqyNooKFIOrJ9BXu9fk9Z41mZ+czVe7s51+PYHykYyjrNu3WZmcQn5hoVsMo6zca9qJp2vTjrzz1Cvac3KTmhwLklZoJ9CXnxzF2G4jAdh4JJFle9ea7JeaqqmsERoaSnh4uOxkGxQUxIgRIwD44YcfjM4nJCQAxl9cdAkPD6dhw4YAHDx4UO9cbm4uubm55h5P8BdKMoyaDp/m9OtJxlFIzQqMij/PkUt5Tr+mQPn0CG1Or9YtSDp6gp3H0gHHrjBydtw1da3pAyfTtWlHAEYve4Ps3BzA+XH3xRdflM3B4evry5gxYwC4cuWKiNN2UpYMIwnJODqV/YDoby9wN19luZOgVCMZR0EBASxP3sOlHE1KAnu2pLki7prDULfr4uy4a41u91SEaSQwwl0TqDCOBIboGkdfb9/N19t3uzyH0Sc/xgIwY9AUljz/Kc+FD+P4rB3a86cyz1kcI27XdwDsefu/vNp7NK9HjGf3W5qKF/O3LTHZb9++fQC0adPG7PjNmz+cyO/cuSP7uU6dOib7e3l5MXDgQACSkpL0zp04oVkxYPgmXWCMkgwjV21pFsaRQA5d4+i73fscviXNmXFX4nTWee3n47N28MzfI1ny/Ke8M2AiANM3zNOed2bc9fU1vSmgcuXKdO7cGYD09HRLjyQwQAmGkbv0rjCOBIYYGkff7d5ndw4jV8RdUxjqdl2cHXet1e2eiDCNBHq4ewIVxpHAkHqB1Xnmic5k3rpN5q3bPPNEZ5cmvU48sQeAf7TprT3m5eXF9IGTAThx5bTZ/kU6ifRqVH5E+/nRKtVl2+hy9+5dAKpWrWr1/ebl5cl+tvS2w9tEhRJpjBo1alh9D2WVsmYYSQjjSCBHj9DmNKtXmz8uXSW/sMghhpEr4q7E8csnAXjnqVf04ueIDpovHHtPP3xL7c64W7GiJuZIFX4E1uFuvQuaLc3u1LvCOBIYIhlH+YVF/HHpKs3q1bYr6bUr4q4p5HS7Ls6Mu/bodk9BmEYCLUqYQEEYRwJ98gsLSfr9hPb3pN9PkF9Y6LLrV6ukCfwXb+gnr9v5x34AalY1P7n46ExOquKHk6VarZZtI0f58uWtu1mgnM6XMnNvSgyRJjrDSkDSeOU8uMKRqyiLhpGEMI4EhlzKucGfWde0v+/740yJx3Rl3JW+9Owx+JJyLkeTq6hBjYfbztwZd6WVTSJGW48S9K6UA8/detfQOBIIdGP1n1nXtFvVbMEVcdcUpnS7hCviri263VMQppEAUMYEqoswjgSgn/T65YgevBzRQy/HkSv4KupDAIbGjif96hnyCwv4dv96Us4cArCYlM/Ly4t2DVoCMDJuItfuXOdW3m0iF2qS5zUNamzybXRISAgAx48fN3sN3Tcd1apV035+5JGHb3pu3bpldgypooNhGdJGjRoBsGPHDsMuAjejFMNIQhhHAgndHEbvjxhklOPIXlwRdyXCg9sDmjfba379H/fy75N+9QzDYl8C4KPBU7VtnRl3db9wySHl5ZD6CMyjBL2rWzRBCXpX1zgSlG10cxi9P2KQUY4ja3FF3DWFoW43xJlx11rd7okI00igiAlUDiVMpAL3IZf0Wi45trPp0Lg1/UOfBGBY7Eu0+7A/czZr9kuvGDPfqhKgi6PnAJotFd0/G0H4J0M5nfUnAP8Z96XJfn369AEgMTGRy5cvy7YpKCggPj4egLCwML1lt97e3oSHhwOwZMkSCk38eWVlZbFt2zYAunTponeuXr16Fp9P4HqsMYzcsdVAGEcCuaTXcsmx7cXZcVfCx9uH5WM0+TM+2hRDx1kDtV9c+oc+SacmbQHnx93k5GST9/jrr7/K9hGYxt1617DKJihD70rGkaDsYpj02lRybGtwdtw1h6Fu18XZcdca3e6pCNOojKNUw0hCCROpwPWYq5LmDuNo/sj3WTFmvnZZbNemHdn15vdWTV4AlSv4c+SDLbzZ/2V8//qyM7Xvixz5YAuVK/ib7BcYGKhNQL106VIuXLhAcXGx9vytW7eIiYkhO1vzb+OJJ54wGkOaFO/fv8/SpUu5ffu23vn09HTi4uIAzRsSw73cXl5eREVFWfWcAtdg7Qojd+WoEMZR2cVclTRHGEeuiLu6hDVpx84319C3pSa2PlIpgPixXzB/5PvaNs6Ou3v37iUlJYWCggLt+aKiIn788Ue2bNkCQFRUVKmq0uNMlGQY6W5pVoLebVuvkluuK3A/pqqk2WMcuSLuWsJQt4Nr4q41ut1TsX7jtaDUoXTDSOLV7jUBmJ+crfe7oHRizjCSkIyj5cl7WJmc4rCKPObo1KQtW1+Lt7t/ed/yvNBlOC90GW5Tv8jISHJzc8nIyGDFihUm202cOJFKlYwFX6VKlZg4cSKxsbFkZ2cTExMj2/+xxx5jxIgRsudCQkIYNGiQTfctcA62bEmTclTEP9eQyn6WV2Y4Esk4iv72AqPiz2veYDer7NJ7ELgWc4aRhPRlJOnoCb3fraVm1UCb2tsbd3UJqvooMVEzZc+5Iu5u2rSJHTt2mNwiPGjQIO2WCIFllGgYSQi9K3AHpgwjCck4WpmcwvLkPRYrGDs77lqLpNtbTO+ld9zZcdeSbjc1HygdsdKojOIphpGEEt7ACJyPNYaRhDtWHLkDHx8fXnjhBYYMGSL7JrlLly5MmzaNwEDTX6YCAwOZNm2atkSoIcOGDePZZ581SgqoS7t25nOICJyPrTmM3F0Vx3DFkaD0Yo1hJOHIrWruxhVx96WXXqJBgwZG5+rXr89LL70kYrMHYI1hJCH0rsCVWDKMJEqyVU1JuCLuWtLtnopYaVQG8TTDSEK8gSnd2GIYSbhjxZE78PLyonXr1rRu3Zo7d+6Ql5dHuXLlqFatmsnSoYZUqlSJvn370rt3b27dukVhYSGVKlWiSpUqTr57gSOwJ+m1lKNiVPx5Raw4EpRObDGMJEq64shTcETcrVWrFqNHjyY/P5/c3FxAU87Zz8/PWbctcCC2GEYSQu8KXIG1hpGErSuOlMLLL7/s8rhrTrd7KsI0KmM4yjA6cinPLXufPWkiffd4mrtvQY/cy7Nlj6ed0yR0W7JxNr4+9hku0hiLE+SvIcfTnetrP9tjGEmUFeNIokqVKiUyery9valeXfmTvOAhJamSpiTjaKxLr2oZpcVoMB2n5bAn7upSpNKszozf+gWHT6XIttGN03LYYxhJlBXjCBwTd/38/Hj00UcddEcCV2CPYSThSXrXmSgtTqfd0KywcWbctXgPOrp9VNcmdo1hq2Ek4YnGUVBQkN19HRF3S6rblYLYnlaGcOQKI3cmNxVLd0sXJTGMJMrKVjVB2aMkhpGEbjlld25VE5QuSmIYSZSmrWoCgS4lMYwkhN4VWMIevWuvYSRRWraqCWxDrDQqIzh6S5qUo+K76EZixZEJPmnZyt23oMexjm/IHj94YifxW79g/ODplPO1b4WO9MblpcjpVve5nbXSIYaRRElWHO08ls5Qhb3kvnPnjl1vJhyxhUFaSuvr68sjjzxi9RY4gWNxhGEkoYQVR0pDaTEaTMdpOeyJu7oUFhUy9athRPd7jY4tesi2uZ21Uva4IwwjidKw4kgJcVe3Qo/AvTjCMJLwBL3rTJQWpyddfgA4N+5aQle3x6wab7PeLYlhJOGJK47kcETcvXnzJkVFRaU+5YMwjcoAzshhZFgVRwnGEZFVXX4PAvtxpGEkYY9xJE2gyBdScBuff/45oKnC0Lp1a4vJ9DIzM9m8eTOXLl3SO167dm0GDRpErVq1zPZXq9WkpaWxfv16o3NhYWGyZaUFzsORhpGEMI4EjsCRhpGEpxpHroi7ctXZdMnLyyMlJYV9+/bx0Ucf2f4QAofiSMNIoqwbRwLTSCvsbdG7JTWMJDzVOHJ03DXEWt3uaYjXx6UcZyW9NqyKo4StagLPwtGGkYQtW9V0J1ClkpCQwIoVK1CpTG8pSk1NZfHixUZfXACuXr3K4sWLSU1NNdlfpVLxn//8R3YCBThw4ABz5861/eYFduEMw0hCCVvVBJ6LMwwjCU/cquaKuJuTk2NyjJycHObOnSv7xUXgepxhGEmIrWoCOezRu4405T1tq5or4q41ut0TESuNSjHOrpKmWxVHCSuO9rv8yoKS4AzDSMKaFUeGE+jXu1fzReJSfLy9ea3vOEb9PZLyvuWtvmZ2bg7vrZ9LyplD1K9eh48ipxLWxLZSyIf+PMqrq2dyKy+XGTNmcOXKFZYuXUpGRgYJCQkMGzbMqM/JkyfZtGkTAB06dCAiIgJfX01oV6lUHDhwgMTERDZt2oS/vz8hISFGY3z//fecPXsWgBEjRtC8+UNBce/ePeLj48nOFiLVFTjTMJIQK44E9uBMw0jC1hVHjo67fVp0492BEwmqan3iU2fH3djYWKZNm2b05jsvL4/Y2FhAU6Vt3LhxNj23wLE40zCSECuOBIbYo3cdjeGKo1e7We5T0rhbUFTAypQfbNbtzoi7UkU0tVptlW73VMRKo1KKsw0jCSWtOBJ4Fs5exmpuxZHcBPpF4lIAVMXFzNu6mLYf9Ofug3tWXevAuVSe/NdIUs4cAuDijSuMXvYGr38/y+r7nbpmFtHfvMatPE1eDC8vL+rWrcuLL74IQFpaGtevX9fro1arWbNmDQD9+/dnwIAB2i8uAD4+PoSHh9O1a1cAVq9ejVqt1hsjJyeH06dPAzBhwgS9CRTA39+fsWPHUrGi4wWwQB9XGEYSYsWRwBZcYRhJmFpxZPgW3RlxN/HEHnr862kOnDO9QkgXV8VduTfaP//8MwA1a9ZkypQpHl3K2dNxhWEkIVYcCQyxVe86A90VR5Yoady9++AebT/ob7Nud3bctVa3eyrCNCqFuMowklCKcSTwLFyx71luIjU1gTYNasy+d9ez+611tKjTFICX4t+xeA1VsYrRyzTJa98dMInUmVtYP3EJAFuO7eLQn0ctjnHgXCpb03YBsH7iEg6+/6P2XN26denduzcA27dv1+unuy2iU6dOJsfv2bOnbB+AlBRNydeIiAhq1pQ3X8uXL8/48eMtPofAflxpGEkI40hgiUs5N1xqGEkYGkdSDjwJZ8XdGYOmADB62Ruoii3/e3BV3E1JSdFLdF1cXMyBAwcAiI6OppwL/p8I5HGlYSQhjCOBIbboXWchGUfmcETcfWbJq4Dtut1Vcdda3e5pCNOolOFqw0hCGEcCpaI7kc5au8nkBJow6WuqVQrg0SrV+f5lzdLT1IzjFt8KSG9GujbtyKjOQ/ArV55mtR/jh4mLAfjn6g8s3uOMBE3S6/UTl9Cs9mP4++mLzlatNNVDTp06pXf8/PnzgPkvLqB5+yFNYFIfiWPHjgFQp04ds2OIt9jOwx2GkYQwjgTmiNu2k7htO11qGEnoGkez1m4i6/Zt7Tlnxd2nO/2Drk07ArDvzG9m+1uaGxwdd2/duqX9fPPmTe1nf39/s2MInIc7DCMJYRwJDLFW7zoTS3OEI+LumezzgO263ZVx15Ju90SEaVSKcJdhJCGMI4FSqRdYncZBD/dKhzcLNmqjW+XAx/thfheVhTLGWbmaZHlPPv53veMNqtcF4GbebaM+hly8cQWAxoH1Zc/n5+fLHi/8awly5cqVLV6joKBAr4+ElKjPmjEEjsedhpGEMI4ElujVpoVLDSMJ3VitG8OdGXelLy/X79406qOLpbkBHBt3dccoKiqyup/AObjTMJIQxpHAEGv0rjtxZNy1Vbe7Mu5a0u2eiDCNSgnuNowkhHEkUCI7j6Xzx6WrNKtXG79yvrJVJq7deVjxQXfS8vUxnyC4ZR1NgtPZPy7Qe7vxvyOJAPRtablUfZfgDgCsO7RZ9vydO3cAqFKlit5xKUHf/fv3LV5D2ldt6u1IsYUvQJ68D1upKMEwkhDGkUCOWtUCqFUtgFU/73d5VRxpS5pfOV+a1avNH5euas85K+6q1Wo+/WkRAK3qPW62v6W5ARwbd3UTsloqBy1wLkowjCSEcSTQxRq9604cGXft1e2uiLuWdLsnIkyjUoBSDCMJYRwJlITunu5R3cNNJgvs/tlw5m1dzJeJy+j26f8BMOHJURbHD6nVRPu55fu9id/3A+NXvs1H//sSgHcGvGJxjI+HTgPgk80LGbfiLVb9kqB3/siRIwCEh4frHW/RogUA+/fv5+7duybHLyoq4vjx4wBGif+kvBubN282awxdvnzZ4nMIrEdJhpGEMI4Ehozr251xfbu7vJyyZBhJVTZHdQ+nV+sW2vPOirst3++tPR9cs5HFMVwZd3W/fOh+Tk9PR+A6lGQYSQjjSADW611nYmmOcETcfT1Ck3PIVt3uyrhrSbd7IsI08nCUZhhJCONIYAndqjjOvIbhnm5zVSaW7V3L4t3/AWDo3/oxqdcLVl1n/3sJ2s+f/rSIvacPArB8zDyrSojWrBrI8jHzAEg5c4jZPy4ANG87UlJSSEtLA6BdO/1S0lWrVtXuzV62bJns0tkHDx6wdOlSbfuqVavqne/YUbMk+Pz587KVIkAzgX7zzTcWn0NgHUo0jCSEcSTQxa9cOb2qOK4wjgwNI6logmFeDmfFXcOxzeGquDtkyBCj80OHDgVg7dq1ZGVlWXW/gpKhRMNIQhhHZRtb9a4zkIommMMRcXdM1xEMbtsHsE23uyLuWqPbPRVfy00ESkWphpGEZBxFf3uBUfHn+S66EW3ree6yPIFjSTp6AjD+IuAozFWNkCbS5cl7WJmcwvM9u7D/vQT2nTmEqriYbiGdCKhYxcTIxgRUrMKJ2UmcyT5P2qWT1Kj8COHB7fX2WFsirEk7jn20nQPnUsnOvc7lnBx+/PFHLly4AGgmKj8/P6N+I0aMICYmhps3b/Lpp58yaNAg6tevj4+PD9evXyc+Pl7bdsyYMUb9K1SowNChQ1m/fj07duzg8uXLdOjQgRo1agCakqE7duyw+jkE5lGyYSQhGUej4s8T/e0F4p9rSGU/6/8uC0ofknG0MjmF5cl79MwcR2LKMJLDGXG3Zd0QmgY1trq/K+JugwYNCA0NNRqjVatW/P7775w9e5a4uDi9am0Cx6Nkw0ji1e6ailDzk7P1fheUbmzVu87IT6dbZdMSJY27Xl5ezPm/t3l7wESbdLuz4m7Tpk2pVKkSBQUFVul2T0WYRh6K0g0jCWEcCUwhVcUBxxtH1pQZNZxIo1r/k/6hPUp03eCajaxaWmsKH28fwv/a791iei/t8aFDh8pOXgABAQFMnjyZBQsWoFKp2LBhg/G4Pj688sorBJiYzENDQ1Gr1WzYsIH09HTZZbemri+wHk8wjCSEcSQwxNnGkS2GkS6OjLv24Oy4O3jwYL2ErxJeXl5ERUWxdu1aTp06RXJyst3PIDCPJxhGEsI4KlvYo3cdbRzpGkbP9+xiVZ+Sxl3QvDiwVbc7I+7KxV5zut1TEaaRB+IphpGEMI4EckiTm6ONI2smUAndiVSJ9OrVi44dO1p8U1G9enVmzJjB2bNn2b17N5cuXQI0yVcHDx5McHCw7OSnS+vWrQkJCeHo0aPs3LmTBw8eABASEkLfvn21b2IE9uFJhpGEMI4EhjjLOLLXMHI37o67Pj4+REVFkZOTwy+//OKYhxLo4UmGkYQwjsoO9uhdRxpHhoaRO6ps2oqj4+5vv/2mPWetbvdEhGnkYSjBMLp39SxYv3MHKJvGUYEV5XhdSWGR/F7mIlWh3n/tQepr6hqmcLRxZIthJCFNpEpj5syZNrX38vIiODiY4GD7y6tWqFCBTp060alTJ7vHEBjjiYaRRGk2jpQWo8G2GGpv3DXsX6QqtGkMRxtHnmoYgXLibmBgIAMHDrS7v0AeTzSMJEqLcaS0OO2uuGtqDHv0rqOMI080jCQcGXfLSuwVppEHoRTD6PS6ufCe7V96yppx1Ddlr7tvQR8L9/Nm7NMlvsTUr4ZZ3Xb2s5q2jjKO7DGMJDzpS4rAs/Bkw0iitBpHiovRYDFOy2FL3JVjVeJCViUulD0nxWlDHGUcebJhJCjdeLJhJFEajCNFxmmcG3et5c3Yp03GaFM4yjjyZMNIYB/CNPIQlGQYVQysa/cYZck4evfxx919C3pcavyU7PGD6btIv5DKM30m4etjX9CP3/oFANH9XrOhV4b2U0mNo5IYRgKBsygNhpFEaTSOlBajwXSclsO+uPuQIlUhqxIX0rxhOzo2f9JEqwwTx0tuHAnDSKBUSoNhJOHpxpHS4vQnJ08Czo675tHV7XDV5v4lNY6EYVQ2EaaRh6Akwyj4/94A7HfHy4px1LdmkLtvQY9jLUwni0u/kEqH5j0o52tf4D98KgWAjmauYcjtrJV6v9trHJUlwyg3N5cTJ06Ql5dHuXLlaNSoEfXq1bOYN0NCrVZz8eJFLly4QGFhIZUqVaJFixYeXQJUqZQmw0iitBlHSovRYD5OG2JP3NWlsEjz5aVj8ydNjmEYpw2x1zgqS4aRI+JuUVER6enpXL9+HYAaNWrQvHlzfH2FjHc0pckwkvBk40hpcXpToSaxvTPjrjVIuj3v+iq7+ttrHHmiYZScnOy2uCun2z0VMdt4CEoyjBwxgZYV40hgG7YaR2XFMLp9+zZr167lypUrsuejoqIICQkxO8apU6dYvXq10fFt27ZRtWpVxowZY7LKD0CxwvIKKJnSaBhJlDbjSFBybDWOypJh5Ii4u3PnTvbuld+i06FDB/r374+3t7fD7rksUxoNIwlPNo4EzsFW48gTDSOAPXs0xW5cGXfN6fZvvvnGxidQBmKW8RBKk2EkIRlHITUrMCr+PEcu5TlsbIHn0iO0Ob1atyDp6Al2HjMuhSnhDsNIrVZTpFKVaIzi4mKbDJgbN24QExOjnXg6d+5Mz5496dixo7bN6tWrSU1NNTlGamqq3heX3r170717d1q2bAlo3oTExMRw48YNk/e8cqX5FQcCDUoxjK7u3+i0sSXj6FT2A6K/vcDd/JL9mxB4PpJxFBQQwPLkPVzKkY8l7jKMbI27chSpVKjVar1jroi70heXxo0b06dPH7p06ULt2rUBOHToECtXrhSmvgMozYaRxKvda/J6z5rMT87mq93Z7r4dgQKQjKOs27dZmZxCfqF8cm57DCNnxV1bUKvVLo+71uh2T0QRK41iY2OZO3cumZmZtGnThgULFpjNZr5u3Tref/99zp8/T9OmTfnss8946qmHeQDUajUzZ87k66+/5tatW3Tp0oV///vfNG3a1BWPU2pwxQQqVhwJ5LC04sjVhtHdB/cYv/Jtjlw8oT328pOjmNzrBau3hqVdPsmIf7+id2z1SwtpU9/0/avVahYsWABA9erVGT16NJUrV9aef+qpp/j111/ZsmULmzZtonbt2tSqVUtvjMzMTDZt2gRAt27d6NGjh949R0ZGsnTpUrKysli0aBHvvfee0TNt2bKFjAzTuU0EGpRkGF1NWQ99WzntGmLFkcAQSyuOHGEYfbVjudPjri5qtZple9cyf9sS7bHgmo1YNf4rKlfwd1ncjY6OpnHjxnrnc3JyiI2NJSMjg507d9KrVy+rnklgTFkwjCTEiiOBIZZWHNlqGDk77lqDrm6XqhG7Iu5ao9s9FbevNPr++++ZOnUqM2fO5PDhw7Rp04aIiAiys+Ud8H379hEVFcXYsWNJTU0lMjKSyMhI0tLStG3+9a9/8dVXXxEXF8eBAwfw9/cnIiKCBw8euOqxPB5XTqBixZFADlMrjlxtGBUUFdBp9iA9wwggbtd3vL9hnlVj/H4x3WgCBYhaPInjl0+Z7HfmzBkAfH19mTBhgt7EI9GpUyfat28PwE8//WR0Xvri0rdvX3r27Gk0Qfr6+vLCCy8AoFKptNeUKCws5NChQ2aeTiChJMOodpehTr+WWHEkMMTUiiNHrTByRdzV5d0fPtP74gJwJvs8nWYPoqCoAHB+3J04caLRFxfQlHuOjo4GYO/evRQVFVn1TAJ9ypJhJCFWHAkMMbXiyJ4VRq6Iu+YwpdtdEXet0e2eittNo88//5xx48YxevRoWrRoQVxcHJUqVWLZsmWy7b/88kv69evHtGnTaN68ObNmzeJvf/sbCxdqEjOr1WpiYmKYPn06gwcPpnXr1sTHx3PlyhUSEhJc+GSeizsmUGEcCeQwNI7csSXtu18SAGhZJ4RjH23nxOwk9rz9XwDWH97K7ft3LI4RtXgSADFRMzkxO4kTs5OIidK8+Rj+7wkm++3evRuAIUOGmE28169fPwAuXrxIfn6+9nh+fj5Xr2oqa5hbvVmhQgXtJLhxo/62pj/++ANAuyxXYBolGUau2tIsjCOBIYbG0dnMbIduSXN23JW4ff8OG48kArD7rXWcmJ3E8Vk7aBqk+SLx7f71gPPjbmBgoMkxGjdurP1ik55ueju3QB4lGEbu0rvCOBIYYmgcnc3MtjuHkbPjrjkMdbsuzo671up2T8StT1NQUMBvv/3GO++8oz3m7e1N79692b9/v2yf/fv3M3XqVL1jERERWkPozz//JDMzk969e2vPBwQEEBYWxv79+3n66aeNxszPz9eb8G/fvg1o9j0qBVX+fZdc517Wn5zd8CUVa9Sh8T8mglr+2nfKOeeLwaIR9XhpzUWeXfknX0c1JLSO9RN40X3r37IVF2n2oaoKVDb100VVoPkzKHpQhHeRsf96T2Ev/e7nywuTgiLN3/0H+XkUqexLaqcqVpm9hhwPTOybNqRzs2CKVCrtVrXuLR+nc7Ngq/tby90H92SPf/7X2465I97jfoFmtaKfb3lGdxnB8pS1JB7fQ79W3U2Oq7vnOfyx9trrhD/WXns8N++OUUK9/KJ8Ll++DEClSpX0YpQ5cnJytJNdTk6O9nhRUZHZN9H+/polv/fu3dO7VlZWFgANGjRQVExUIk0iX6VC9Toui9e6ZP66mawD/yMo7B/U/FtfVPn3nRanDXks0I8lTzdg3OoLPLvyPIv/n707j8ui6h///7rYdxAUAUWWREXFFNEENXAL3G53jVsT1LJSU1I0TRNwrUxT4zY1SzRzyQ37aLmREC6Jmmgs4obi1xt3sXBBlPn9wY+5GS9EUMus9/PxmEfMOWfOOXNmGrnenDnXq65Ymui/qnbPqPj++yOfuxWh3C9eH+HmX3BmRmWeoY/z3C3t3r3iZ+jdewUPreNJnrOvtm5O3I97WJpQvAjp4HatqWpr/UR1/tHP3dJ+zCz+drrX/Htibmyq1rFgwFQ6zB7A7G1fqGX/yOfuo5797u7uZGdnc/HiRXlGV0DJ87miv+/+oX25e/uxft99Wga95EDBvSJm/3iJgntFvNWyWpnP5qf53K3ss/+v9rv0n/HcfZTSv7c/7d+Fq9pa0/9lf75KSGZpQjKOtja82ro5CpX79+CPfO72a9a13LYf/L294J72Gfq0n7ulv4imIr+3/5We0yV9qdC6UcozdP78eQVQ9u7dq0kfO3as0rx58zKPMTY2VlauXKlJ+89//qM4OjoqiqIoe/bsUQDlv//9r6ZMnz59lL59+5ZZZ1RUlALIJptssskmm2yyySabbLLJJptssv0jtnPnzj0ybvP3mjf1mCZMmKCZvVRUVMS1a9dwcHCo8GKLQvyV/fbbb7i6unLu3DlsbGyedXeEEEI8QJ7TQgjx1yXPaPF3oygKv//+Oy4uLo8s+0yDRlWrVsXQ0FCdDlbi4sWLet9EUcLJyanc8iX/vXjxomYNjosXL9K4ceMy6zQ1NcXU1FSTZmdnV5lTEeK5YGNjI//QCSHEX5g8p4UQ4q9LntHi78TW1rZC5Z7pQtgmJiY0bdqUhIQENa2oqIiEhAT8/f3LPMbf319THmDHjh1qeQ8PD5ycnDRlfvvtN/bv3//QOoUQQgghhBBCCCGE1jN/PW306NGEhYXh5+dH8+bNmTt3Ljdv3mTQoEEADBw4kBo1ajBz5kwARo0aRWBgILNnz6Zz586sXr2agwcPsnhx8aJXOp2OiIgIpk2bhpeXFx4eHnzwwQe4uLjQvXv3Z3WaQgghhBBCCCGEEM+VZx406tevH5cvX2by5MlcuHCBxo0bs3XrVqpXrw5ATk6OZpX1gIAAVq5cyaRJk3j//ffx8vIiPj6ehg0bqmXGjRvHzZs3GTp0KHl5ebRq1YqtW7diZmb2p5+fEH8FpqamREVF6b2GKYQQ4q9BntNCCPHXJc9o8U+mU5SKfMeaEEIIIYQQQgghhPgneaZrGgkhhBBCCCGEEEKIvyYJGgkhhBBCCCGEEEIIPRI0EkIIIYQQQgghhBB6JGgkhBBCCCGEEEIIIfRI0EiIZyQ8PBydTsdbb72llzd8+HB0Oh3h4eGasg9uISEhJCYmlplXektMTCQuLq7MvAe/VfDcuXMMHjwYFxcXTExMcHNzY9SoUVy9elVTLigoSFNHnTp1mDlzJrK2vhDi7+rBZ7GDgwMhISEcPXpULaPT6YiPjy/z+PKe1xcuXFDb6N69+0OPzcvLe2j/0tPT6dWrF+7u7uh0OubOnfsEZyuEEH9NpZ/FxsbGeHh4MG7cOO7cuaMpt3nzZgIDA7G2tsbCwoJmzZoRFxenKVPes9Xd3V3vObpr1y66dOlCtWrVMDMz44UXXqBfv3789NNPenWW96x/2HmV9fwX4lmToJEQz5CrqyurV6/m9u3batqdO3dYuXIltWrV0pQNCQkhNzdXs61atYqAgABNWt++ffXKBgQEAGBjY6NXx9mzZ9U2Tp8+jZ+fHydOnGDVqlWcPHmShQsXkpCQgL+/P9euXdP06Y033iA3N5esrCwmTJjA5MmTWbhw4R84YkII8WyVfr4mJCRgZGREly5dKlVHVlaW3rPY0dHxift269YtPD09+fDDD3Fycnri+oQQ4q+q5Fl8+vRpPv30UxYtWkRUVJSa/9lnn9GtWzdatmzJ/v37OXr0KK+++ipvvfUWkZGRj9XmggULaNeuHQ4ODqxZs4asrCw2btxIQEAA7777rl75P+pZL8SfzehZd0CIfzJfX19OnTrFhg0b6N+/PwAbNmygVq1aeHh4aMqampo+9ENA6XRzc3MKCgrKLKvT6cr9IDF8+HBMTEzYvn075ubmANSqVYsmTZrwwgsvMHHiRD7//HO1vIWFhVrfoEGDiI2NZceOHbz99tsVHAEhhHi+lH4WOzk5MX78eFq3bs3ly5epVq1ahepwdHTEzs7uqfetWbNmNGvWDIDx48c/9fqFEOKvovSz2NXVlfbt27Njxw4++ugjzp07x5gxY4iIiGDGjBnqMWPGjMHExISRI0fSp08fXnrppQq3l5OTQ0REBBEREcyZM0eT16hRI0aOHKl3zB/1rBfizyYzjYR4xgYPHszSpUvV/a+++opBgwb96f24du0a27ZtY9iwYWrAqISTkxP9+/dnzZo1Zb5+pigKycnJHDt2DBMTkz+ry0II8Uzl5+ezYsUKateujYODw7PujhBC/COlpaWxd+9e9XfQdevWUVhYWOaMojfffBMrKytWrVpVqTbWr19PYWEh48aNKzNfp9NVvuNCPCckaCTEMzZgwAB2797N2bNnOXv2LHv27GHAgAF65TZv3oyVlZVmK/3Xk4q4ceOGXh0dO3YE4MSJEyiKgre3d5nHent7c/36dS5fvqymLViwACsrK0xNTXn55ZcpKioq8y8tQgjxd1H6WWxtbc13333HmjVrMDCo+K9UNWvW1DyHGzRo8Af2WAgh/n5KnsVmZmb4+Phw6dIlxo4dC8Dx48extbXF2dlZ7zgTExM8PT05fvx4pdo7fvw4NjY2mhn769ev1zzLf/31V80x8qwXfxfyepoQz1i1atXo3LkzcXFxKIpC586dqVq1ql65Nm3aaF4NA7C3t69UW9bW1vzyyy+atAdnFVVmIev+/fszceJErl+/TlRUFAEBAer6SUII8XdU+ll8/fp1FixYQMeOHUlJScHNza1CdSQnJ2Ntba3uGxsbV6oPOTk51K9fX91///33ef/99ytVhxBCPM9KnsU3b97k008/xcjIiF69ev2hbT44myg4OJjU1FTOnz9PUFAQ9+/f1+Q/7FmfnJys/tEWYNGiReoyFUL8FUnQSIi/gMGDBzNixAgA/vOf/5RZxtLSktq1az9ROwYGBg+to3bt2uh0OjIzM+nRo4defmZmJlWqVNGs2WFra6vW9+2331K7dm1atGhB+/btn6ifQgjxV/Xgs3jJkiXY2tryxRdfMG3atArV4eHh8dB1LmxsbDRfUFAiLy8PQ0NDLC0tsbKyIjU1Vc2r7B8QhBDieVf6WfzVV1/x4osv8uWXXzJkyBDq1KnDjRs3+O9//4uLi4vmuLt373Lq1CnatGkDFD9zoXg2/oPP5by8PGxtbQHw8vLixo0bXLhwQZ1tZGVlRe3atTEyKvsj9cOe9X5+fppnePXq1St9/kL8meT1NCH+AkJCQrh79y6FhYUEBwc/kz44ODjQoUMHFixYoPk2N4ALFy7wzTff0K9fv4e+s21lZcWoUaOIjIys1GwlIYR4nul0OgwMDPSem4+rbt26pKenU1BQoEn/5Zdf8PDwwNjYGCMjI2rXrq1uEjQSQvyTGRgY8P777zNp0iRu375Nr169MDY2Zvbs2XplFy5cyM2bNwkNDQWKg0EGBgYcOnRIU+706dPcuHGDOnXqANC7d2+MjY356KOPnri/5ubmmmd46dlIQvwVyUwjIf4CDA0NyczMVH8uS0FBARcuXNCkGRkZlfkq28MoiqJXBxR/u4OBgQGxsbEEBAQQHBzMtGnT8PDwID09nbFjx1KjRg2mT59ebv1vvvkmU6dOZf369fTu3bvC/RJCiOdF6Wfx9evXiY2NJT8/n65du6plsrOzNX9FhuIPJiUuXbrEnTt3NPkODg4YGxvTv39/pkyZwsCBAxk3bhy2trb89NNPzJ07l48//rjcvt29e5eMjAz15/Pnz5Oamqr+NVwIIf6u+vTpw9ixY/nPf/5DZGQkH3/8MWPGjMHMzIzXXnsNY2NjNm3axPvvv8+YMWPUb06ztrbm9ddfZ8yYMRgZGeHj48O5c+d47733aNGihbrsQq1atZg9ezajRo3i2rVrhIeH4+HhwbVr11ixYgWg/zt8ec/6h7lx44bevx8ODg64uro+6RAJ8fgUIcQzERYWpnTr1u2h+d26dVPCwsLUsoDeVrdu3QrXu3Tp0jLrAJTc3Fy13JkzZ5SwsDClevXqirGxseLq6qq88847ypUrVzT1BQYGKqNGjdJr580331QaNGig3L9/v0LjIIQQz4sHn8XW1tZKs2bNlHXr1qllHvacTU5OVnbt2vXQ/H379ql1ZGVlKT169FBcXFwUS0tL5cUXX1S++OILpaioqNz+ZWdnl1l3YGDgHzUkQgjxp3vY77ozZ85UqlWrpuTn5yuKoiibNm1SWrdurVhaWipmZmZK06ZNla+++krvuNu3bytRUVFKvXr1FHNzc8XDw0MZOnSocvnyZb2yO3bsUDp27KjY29srRkZGSvXq1ZXu3bsrW7duVctU9Flf1nmVdcyQIUMeY5SEeHp0iiLvkQghhBBCCCGEEEIILVnTSAghhBBCCCGEEELokaCREEIIIYQQQgghhNAjQSMhhBBCCCGEEEIIoUeCRkIIIYQQQgghhBBCjwSNhBBCCCGEEEIIIYQeCRoJIYQQQgghhBBCCD0SNBJCCCGEEEIIIYQQeiRoJIQQQgghhBBCCCH0SNBICCGEEOIvTqfTER8f/6y7IYQQQoh/GAkaCSGEEEJUQHh4ODqdjrfeeksvb/jw4eh0OsLDwytUV2JiIjqdjry8vAqVz83NpWPHjpXorRBCCCHEk5OgkRBCCCFEBbm6urJ69Wpu376tpt25c4eVK1dSq1atp97e3bt3AXBycsLU1PSp1y+EEEIIUR4JGgkhhBBCVJCvry+urq5s2LBBTduwYQO1atWiSZMmalpRUREzZ87Ew8MDc3NzXnzxRdatWwfAmTNnaNOmDQBVqlTRzFAKCgpixIgRREREULVqVYKDgwH919P+3//7f4SGhmJvb4+lpSV+fn7s378fgCNHjtCmTRusra2xsbGhadOmHDx48I8cFiGEEEL8TRk96w4IIYQQQjxPBg8ezNKlS+nfvz8AX331FYMGDSIxMVEtM3PmTFasWMHChQvx8vLip59+YsCAAVSrVo1WrVqxfv16evXqRVZWFjY2Npibm6vHLlu2jLfffps9e/aU2X5+fj6BgYHUqFGD7777DicnJ3755ReKiooA6N+/P02aNOHzzz/H0NCQ1NRUjI2N/7gBEUIIIcTflgSNhBBCCCEqYcCAAUyYMIGzZ88CsGfPHlavXq0GjQoKCpgxYwY7d+7E398fAE9PT3bv3s2iRYsIDAzE3t4eAEdHR+zs7DT1e3l58fHHHz+0/ZUrV3L58mUOHDig1lO7dm01Pycnh7Fjx1KvXj21PiGEEEKIxyFBIyGEEEKISqhWrRqdO3cmLi4ORVHo3LkzVatWVfNPnjzJrVu36NChg+a4u3fval5he5imTZuWm5+amkqTJk3UgNGDRo8ezeuvv87XX39N+/bt6dOnDy+88EIFzkwIIYQQQkuCRkIIIYQQlTR48GBGjBgBwH/+8x9NXn5+PgBbtmyhRo0amryKLGZtaWlZbn7pV9nKEh0dzb///W+2bNnCDz/8QFRUFKtXr6ZHjx6PbFsIIYQQojRZCFsIIYQQopJCQkK4e/cuhYWF6mLVJerXr4+pqSk5OTnUrl1bs7m6ugJgYmICwP379yvddqNGjUhNTeXatWsPLVOnTh3effddtm/fTs+ePVm6dGml2xFCCCGEkKCREEIIIUQlGRoakpmZSUZGBoaGhpo8a2trIiMjeffdd1m2bBmnTp3il19+4bPPPmPZsmUAuLm5odPp2Lx5M5cvX1ZnJ1VEaGgoTk5OdO/enT179nD69GnWr1/Pvn37uH37NiNGjCAxMZGzZ8+yZ88eDhw4gLe391M9fyGEEEL8M0jQSAghhBDiMdjY2GBjY1Nm3tSpU/nggw+YOXMm3t7ehISEsGXLFjw8PACoUaMGMTExjB8/nurVq6uvulWEiYkJ27dvx9HRkU6dOuHj48OHH36IoaEhhoaGXL16lYEDB1KnTh369u1Lx44diYmJeSrnLIQQQoh/Fp2iKMqz7oQQQgghhBBCCCGE+GuRmUZCCCGEEEIIIYQQQo8EjYQQQgghhBBCCCGEHqNn3QEhhBDiWbt//z6FhYXPuhtCCPFMGRsb6y3sLoQQ4p9NgkZCCCH+sRRF4cKFC+Tl5T3rrgghxF+CnZ0dTk5O6HS6Z90VIYQQfwESNBJCCPGPVRIwcnR0xMLCQj4kCSH+sRRF4datW1y6dAkAZ2fnZ9wjIYQQfwUSNBJCCPGPdP/+fTVg5ODg8Ky7I4QQz5y5uTkAly5dwtHRUV5VE0IIIQthCyGE+GcqWcPIwsLiGfdECCH+OkqeibLOmxBCCJCgkRBCiH84eSVNCCH+R56JQgghSpOgkRBCCCGEEEIIIYTQI0EjIYQQQgghhBBCCKFHFsIWQgghSmk6dvkf3sbN3FOcWDsL86o1yD9/4g9vTzzal19+yZo1a9i+fbuaFh0dzeeff86lS5fYuHEj8fHx5OXlER8f/1Ta1Ol0bNy4ke7duz+V+v5M8fHxREZGkp2dzTvvvMPcuXMfeUxQUBCNGzeuUNlHycjI4JVXXiErKwtLS8snrk8IIYQQZZOZRkIIIcSfqHTAqHbvyMeu59y5cwwePBgXFxdMTExwc3Nj1KhRXL169Sn2tnKOHDlCaGgorq6umJub4+3tzbx58yp07Nq1a6lXrx5mZmb4+Pjw/fffl1s+MTERnU6nt124cEEtEx4erslzcHAgJCSEo0ePauq6c+cOH3zwAVFRUWpaZmYmMTExLFq0iNzcXDp27FiJkXi2goKCiIiI0KSVjFdeXt5TaePNN9+kd+/enDt3jqlTpz6VOiujfv36tGjRgjlz5vzpbQshhBD/JBI0EkIIIf4kDwaMDE3MH6ue06dP4+fnx4kTJ1i1ahUnT55k4cKFJCQk4O/vz7Vr155yzyvm0KFDODo6smLFCtLT05k4cSITJkwgNja23OP27t1LaGgoQ4YM4fDhw3Tv3p3u3buTlpb2yDazsrLIzc1VN0dHR01+SEiImpeQkICRkRFdunTRlFm3bh02Nja0bNlSTTt16hQA3bp1w8nJCVNT04oOw99efn4+ly5dIjg4GBcXF6ytrZ9JPwYNGsTnn3/OvXv3nkn7QgghxD+BBI2EEEKIP8HTChgBDB8+HBMTE7Zv305gYCC1atWiY8eO7Ny5k/PnzzNx4kRiY2Np2LChekx8fDw6nY6FCxeqae3bt2fSpEnq/qZNm/D19cXMzAxPT09iYmI0H8h1Oh1LliyhR48eWFhY4OXlxXfffafmDx48mHnz5hEYGIinpycDBgxg0KBBbNiwodzzmTdvHiEhIYwdOxZvb2+mTp2Kr6/vI4NNAI6Ojjg5OambgYH2VxtTU1M1r3HjxowfP55z585x+fJltczq1avp2rWruh8dHa3uGxgYPPTbpLZu3UqrVq2ws7PDwcGBLl26qMEmgLt37zJixAicnZ0xMzPDzc2NmTNnauq4cuXKQ8ezLFevXiU0NJQaNWpgYWGBj48Pq1atUvPDw8NJSkpi3rx56gyrM2fO0KZNGwCqVKmCTqcjPDwcKJ6VNHLkSMaNG4e9vT1OTk5ER0c/tP3ExEQ1SNS2bVt0Oh2JiYmP7FdZFixYgJeXF2ZmZlSvXp3evXureUVFRcycORMPDw/Mzc158cUXWbduneb4Dh06cO3aNZKSksptRwghhBCPT4JGQgghxB/saQaMrl27xrZt2xg2bBjm5tp6nJyc6N+/P2vWrCEwMJCMjAw1OJKUlETVqlVJTEwEoLCwkH379hEUFARAcnIyAwcOZNSoUWRkZLBo0SLi4uKYPn26po2YmBj69u3L0aNH6dSpE/379y93ZtONGzewt7cv95z27dtH+/btNWnBwcHs27fvkePRuHFjnJ2d6dChA3v27Cm3bH5+PitWrKB27do4ODio6bt378bPz0/dj4yMZOnSpQDqLKWy3Lx5k9GjR3Pw4EESEhIwMDCgR48eFBUVATB//ny+++47vv32W7Kysvjmm29wd3fX1FHZ8bxz5w5NmzZly5YtpKWlMXToUF577TVSUlKA4gCcv78/b7zxhtp3V1dX1q9fD/xvZlbp1waXLVuGpaUl+/fv5+OPP2bKlCns2LFDzQ8PD1fvk4CAALKysgBYv349ubm5BAQEPLJfDzp48CAjR45kypQpZGVlsXXrVl5++WU1f+bMmSxfvpyFCxeSnp7Ou+++y4ABAzQBIhMTExo3bkxycvJDx0sIIYQQT0YWwhZCCCH+QE8zYARw4sQJFEXB29u7zHxvb2+uX7+Oo6Mj9vb2JCUl0bt3bxITExkzZowaLEhJSaGwsJCAgACgOHgxfvx4wsLCAPD09GTq1KmMGzdOs9ZPeHg4oaGhAMyYMYP58+eTkpJCSEiIXl/27t3LmjVr2LJlS7nndOHCBapXr65Jq169umZ9ogc5OzuzcOFC/Pz8KCgoYMmSJQQFBbF//358fX3Vcps3b8bKygooDvI4OzuzefNmdUZSXl4eN27cwMXFRT3GysoKOzs7oDgQ9zC9evXS7H/11VdUq1aNjIwMGjZsSE5ODl5eXrRq1QqdToebm5teHZUZT4AaNWoQGfm/tbDeeecdtm3bxrfffkvz5s2xtbXFxMQECwsLTd9LAneOjo7quZVo1KiReo29vLyIjY0lISGBDh06AMVjXRIIMzExUV8BLJmZVJF+PSgnJwdLS0u6dOmCtbU1bm5uNGnSBICCggJmzJjBzp078ff3B4rvx927d7No0SICAwPVelxcXDh79myZYyWEEEKIJyczjYQQQog/yNMOGJWmKEq5+aamprz88sskJiaSl5dHRkYGw4YNo6CggGPHjpGUlESzZs2wsLAAihexnjJlClZWVupWMlvl1q1bar2NGjVSf7a0tMTGxoZLly7ptZ+Wlka3bt2IiorilVdeAYoDBaXrnzFjxmOff926dXnzzTdp2rQpAQEBfPXVVwQEBPDpp59qyrVp04bU1FRSU1NJSUkhODiYjh07qoGG27dvA2BmZlbpPpw4cYLQ0FA8PT2xsbFRZxHl5OQAxQGh1NRU6taty8iRIzXfzFaivPFs0KCBOlYlC3Hfv3+fqVOn4uPjg729PVZWVmzbtk1t83GU7gMUB4lKX9OSWT/lqWy/OnTogJubG56enrz22mt888036n128uRJbt26RYcOHTT3y/LlyzWv/wGYm5tr7k8hhBBCPF0y00gIIYT4A/xRAaPatWuj0+nIzMykR48eevmZmZlUq1YNOzs7goKCWLx4McnJyTRp0gQbGxs1kJSUlKSZsZGfn09MTAw9e/bUq7N0QMXY2FiTp9Pp1FkoJTIyMmjXrh1Dhw7VrJnk4uJCamqqul8y+8XJyYmLFy9q6rh48WK5s3zK0rx5c3bv3q1Js7S0pHbt2ur+kiVLsLW15YsvvmDatGk4ODig0+m4fv16pdoC6Nq1K25ubnzxxRe4uLhQVFREw4YNuXv3LgC+vr5kZ2fzww8/sHPnTvr27Uv79u01a/OUN57ff/89hYWFAOqriLNmzWLevHnMnTsXHx8fLC0tiYiIUNt8HBW5po9S2X5ZW1vzyy+/kJiYyPbt25k8eTLR0dEcOHCA/Px8ALZs2UKNGjU0xz24IPm1a9d44YUXKtVXIYQQQlScBI2EEEKIp+yPnGHk4OBAhw4dWLBgAe+++65mXaMLFy7wzTffMHz4cAACAwOJiIhg7dq16po0QUFB7Ny5kz179jBmzBj1WF9fX7KysjQBlseRnp5O27ZtCQsL01sPycjIqMz6/f39SUhI0HxN/I4dO9RXkyoqNTUVZ2fncsvodDoMDAzUGUYmJibUr1+fjIwMdUZURVy9epWsrCy++OILWrduDaAXsAKwsbGhX79+9OvXj969exMSEsK1a9ceuc4TUObrbHv27KFbt24MGDAAKF4w+vjx49SvX18tY2Jiwv379zXHmZiYAOilPy0V6deDjIyMaN++Pe3btycqKgo7Ozt+/PFHOnTogKmpKTk5OZrAZlnS0tI0C2gLIYQQ4umSoJEQQgjxFP2RAaMSsbGxBAQEEBwczLRp0/Dw8CA9PZ2xY8dSp04dJk+eDBS/dlSlShVWrlzJ5s2bgeKgUWRkJDqdTvMV85MnT6ZLly7UqlWL3r17Y2BgwJEjR0hLS2PatGkV6ldaWhpt27YlODiY0aNHq2sSGRoaUq1atYceN2rUKAIDA5k9ezadO3dm9erVHDx4kMWLF6tlJkyYwPnz59XXpObOnYuHhwcNGjTgzp07LFmyhB9//FHvFbCCggK1H9evXyc2Npb8/HzNt6UFBweze/duTdDqUapUqYKDgwOLFy/G2dmZnJwcxo8frykzZ84cnJ2dadKkCQYGBqxduxYnJye9NYUqw8vLi3Xr1rF3716qVKnCnDlzuHjxoiY44+7uzv79+zlz5gxWVlbY29vj5uaGTqdj8+bNdOrUCXNzc3Wtp0d5cOwft1+lbd68mdOnT/Pyyy9TpUoVvv/+e4qKiqhbty7W1tZERkby7rvvUlRURKtWrbhx4wZ79uzBxsZGXXfrzJkznD9/Xm8RdSGEEEI8PRI0EkIIIUo5NGvgk1eyIubJ6yiHl5cXBw4cIDo6mr59+3Lp0iUURaFnz558/fXX6jpFOp2O1q1bs2XLFlq1agUUB5JsbGyoW7culpaWap3BwcFs3ryZKVOm8NFHH2FsbEy9evV4/fXXK9yvdevWcfnyZVasWMGKFSvUdDc3N86cOfPQ4wICAli5ciWTJk3i/fffx8vLi/j4eBo2bKiWyc3N1ayPc/fuXcaMGcP58+exsLCgUaNG7Ny5U/1q+RJbt25VZx9ZW1tTr149zcwrgCFDhuDn58eNGzewtbWt0LkaGBiwevVqRo4cScOGDalbty7z58/X1Gttbc3HH3/MiRMnMDQ0pFmzZnz//ffqItyPY9KkSZw+fZrg4GAsLCwYOnQo3bt358aNG2qZyMhIwsLCqF+/Prdv3yY7Oxt3d3d1sfNBgwYxcOBA4uLiKtTmg2P/uP0qzc7Ojg0bNhAdHc2dO3fw8vJi1apVNGjQAICpU6dSrVo1Zs6cyenTp7Gzs8PX15f3339frWPVqlW88sorZc7IEkIIIcTToVMetZKmEEII8Td0584dsrOz8fDweKxFkP9qoqKimDNnDjt27KBFixbPujvPnT59+uDr68uECROedVdEBdy9excvLy9WrlypmTEnntzf7dkohBDiyci3pwkhhBB/AzExMcyfP5+ff/650osYi+KFnCv6upZ49nJycnj//fclYCSEEEL8wWSmkRBCiH8k+Wu6EELok2ejEEKI0mSmkRBCCCGEEEIIIYTQI0EjIYQQQgghhBBCCKFHgkZCCCGEEEIIIYQQQo8EjYQQQgghhBBCCCGEHgkaCSGEEEIIIYQQQgg9EjQSQgghhBBCCCGEEHokaCSEEEIIIYQQQggh9Bg96w4IIYQQfyU5U3yeSj3zky4x+8dLjGnryMhAx4eWqzX516fSnngyX375JWvWrGH79u1qWnR0NJ9//jmXLl1i48aNxMfHk5eXR3x8/FNpU6fTsXHjRrp37/5U6vs7a9GiBWPHjqVXr17PuitCCCHEP4rMNBJCCCH+ACMDHRnT1pHZP15iftKlp17/uXPnGDx4MC4uLpiYmODm5saoUaO4evXqU2+roo4cOUJoaCiurq6Ym5vj7e3NvHnzKnTs2rVrqVevHmZmZvj4+PD999+XWz4xMRGdTqe3XbhwQS0THh6uyXNwcCAkJISjR49q6rpz5w4ffPABUVFRalpmZiYxMTEsWrSI3NxcOnbsWImReLaCgoKIiIjQpJWMV15e3hPXHxcXh52d3RPXUxmTJk1i/PjxFBUV/antCiGEEP90EjQSQggh/iB/VODo9OnT+Pn5ceLECVatWsXJkydZuHAhCQkJ+Pv7c+3atafWVmUcOnQIR0dHVqxYQXp6OhMnTmTChAnExsaWe9zevXsJDQ1lyJAhHD58mO7du9O9e3fS0tIe2WZWVha5ubnq5uiondUVEhKi5iUkJGBkZESXLl00ZdatW4eNjQ0tW7ZU006dOgVAt27dcHJywtTUtKLDIP5/d+/efWp1dezYkd9//50ffvjhqdUphBBCiEeToJEQQgjxB/ojAkfDhw/HxMSE7du3ExgYSK1atejYsSM7d+7k/PnzTJw4kdjYWBo2bKgeEx8fj06nY+HChWpa+/btmTRpkrq/adMmfH19MTMzw9PTk5iYGO7du6fm63Q6lixZQo8ePbCwsMDLy4vvvvtOzR88eDDz5s0jMDAQT09PBgwYwKBBg9iwYUO55zNv3jxCQkIYO3Ys3t7eTJ06FV9f30cGmwAcHR1xcnJSNwMD7a82pqamal7jxo0ZP348586d4/Lly2qZ1atX07VrV3U/Ojpa3TcwMECn05XZ9tatW2nVqhV2dnY4ODjQpUsXNdgExUGTESNG4OzsjJmZGW5ubsycOVNTx5UrVx46nmW5evUqoaGh1KhRAwsLC3x8fFi1apWaHx4eTlJSEvPmzVNnWJ05c4Y2bdoAUKVKFXQ6HeHh4UDxrKSRI0cybtw47O3tcXJyIjo6+qHtJyYmMmjQIG7cuKHWX1Le3d2dqVOnMnDgQGxsbBg6dGiZM5xSU1PVfpXYvXs3rVu3xtzcHFdXV0aOHMnNmzfVfENDQzp16sTq1avLHR8hhBBCPF0SNBJCCCH+YE8zcHTt2jW2bdvGsGHDMDc31+Q5OTnRv39/1qxZQ2BgIBkZGWpwJCkpiapVq5KYmAhAYWEh+/btIygoCIDk5GQGDhzIqFGjyMjIYNGiRcTFxTF9+nRNGzExMfTt25ejR4/SqVMn+vfvX+7Mphs3bmBvb1/uOe3bt4/27dtr0oKDg9m3b98jx6Nx48Y4OzvToUMH9uzZU27Z/Px8VqxYQe3atXFwcFDTd+/ejZ+fn7ofGRnJ0qVLAdRZSmW5efMmo0eP5uDBgyQkJGBgYECPHj3UV6jmz5/Pd999x7fffktWVhbffPMN7u7umjoqO5537tyhadOmbNmyhbS0NIYOHcprr71GSkoKUByA8/f354033lD77urqyvr164H/zcwq/drgsmXLsLS0ZP/+/Xz88cdMmTKFHTt2qPnh4eHqfRIQEMDcuXOxsbFR64+MjFTLfvLJJ7z44oscPnyYDz744KHnUdqpU6cICQmhV69eHD16lDVr1rB7925GjBihKde8eXOSk5MrVKcQQgghng5ZCFsIIYT4E5Qshj37x0ua/co6ceIEiqLg7e1dZr63tzfXr1/H0dERe3t7kpKS6N27N4mJiYwZM0YNFqSkpFBYWEhAQABQHLwYP348YWFhAHh6ejJ16lTGjRunWesnPDyc0NBQAGbMmMH8+fNJSUkhJCREry979+5lzZo1bNmypdxzunDhAtWrV9ekVa9eXbM+0YOcnZ1ZuHAhfn5+FBQUsGTJEoKCgti/fz++vr5quc2bN2NlZQUUB3mcnZ3ZvHmzOiMpLy+PGzdu4OLioh5jZWWlrtnj5OT00D48uCjzV199RbVq1cjIyKBhw4bk5OTg5eVFq1at0Ol0uLm56dVRmfEEqFGjhiZI884777Bt2za+/fZbmjdvjq2tLSYmJlhYWGj6XhK4c3R01FuPqFGjRuo19vLyIjY2loSEBDp06AAUj3VJIMzExARbW1t0Ol2ZY9O2bVvGjBmj7p87d67swStl5syZ9O/fX12HycvLi/nz5xMYGMjnn3+OmZkZAC4uLpw7d46ioiK9GWVCCCGE+GPIv7hCCCHEn+RpzjhSFKXcfFNTU15++WUSExPJy8sjIyODYcOGUVBQwLFjx0hKSqJZs2ZYWFgAxYtYT5kyBSsrK3Urma1y69Yttd5GjRqpP1taWmJjY8OlS/rnkpaWRrdu3YiKiuKVV14BICcnR1P/jBkzHvv869aty5tvvknTpk0JCAjgq6++IiAggE8//VRTrk2bNqSmppKamkpKSgrBwcF07NiRs2fPAnD79m0ANTBRGSdOnCA0NBRPT09sbGzUWUQ5OTlAcUAoNTWVunXrMnLkSM03s5UobzwbNGigjlXJQtz3799n6tSp+Pj4YG9vj5WVFdu2bVPbfByl+wDFQaLS13TmzJksX768QnWVnrFVUUeOHCEuLk5zbwQHB1NUVER2drZaztzcnKKiIgoKCirdhhBCCCEej8w0EkIIIf5ED844+qSSx9euXRudTkdmZiY9evTQy8/MzKRatWrY2dkRFBTE4sWLSU5OpkmTJtjY2KiBpKSkJAIDA9Xj8vPziYmJoWfPnnp1lg6oGBsba/J0Op3eN1plZGTQrl07hg4dqlkzycXFhdTUVHW/ZPaLk5MTFy9e1NRx8eLFcmf5lKV58+bs3r1bk2ZpaUnt2rXV/SVLlmBra8sXX3zBtGnTcHBwQKfTcf369Uq1BdC1a1fc3Nz44osvcHFxoaioiIYNG6oLQPv6+pKdnc0PP/zAzp076du3L+3bt2fdunVqHeWN5/fff09hYSGA+irirFmzmDdvHnPnzsXHxwdLS0siIiKeaNHpilzTirK0tNTsl8wIKh3kLDmnEvn5+bz55puMHDlSr75atWqpP1+7dg1LS0u91zKFEEII8ceRoJEQQgjxJysdOKps0MjBwYEOHTqwYMEC3n33Xc0H6AsXLvDNN98wfPhwAAIDA4mIiGDt2rXqmjRBQUHs3LmTPXv2aF4j8vX1JSsrSxNgeRzp6em0bduWsLAwvfWQjIyMyqzf39+fhIQEzdfE79ixA39//0q1nZqairOzc7lldDodBgYG6gwjExMT6tevT0ZGhjojqiKuXr1KVlYWX3zxBa1btwbQC1gB2NjY0K9fP/r160fv3r0JCQnh2rVrj1znCSjzdbY9e/bQrVs3BgwYAEBRURHHjx+nfv36ahkTExPu37+vOc7ExARAL/1xlFX/w1SrVg0oXhuqSpUqAJrAIRTfexkZGY+899LS0mjSpEnlOyyEEEKIxyavpwkhhBDPQMmrao8jNjaWgoICgoOD+emnnzh37hxbt26lQ4cO1KlTh8mTJwPFrx1VqVKFlStXaoJG8fHxFBQUaL5ifvLkySxfvpyYmBjS09PJzMxk9erVmplCj5KWlkabNm145ZVXGD16NBcuXODChQuabyory6hRo9i6dSuzZ8/m2LFjREdHc/DgQc1CyBMmTGDgwIHq/ty5c9m0aRMnT54kLS2NiIgIfvzxRzVgVqKgoEDtR2ZmJu+88w75+fmab0sLDg4uM+BTnipVquDg4MDixYs5efIkP/74I6NHj9aUmTNnDqtWreLYsWMcP36ctWvX4uTkpLemUGV4eXmxY8cO9u7dS2ZmJm+++abeLC13d3f279/PmTNnuHLlCkVFRbi5uaHT6di8eTOXL18mPz+/wm0+OPbu7u7k5+eTkJDAlStXNK8vPqh27dq4uroSHR3NiRMn2LJlC7Nnz9aUee+999i7dy8jRowgNTWVEydOsGnTJr2FsJOTkysV2BNCCCHEk5OZRkIIIUQptSb/+qe1VdlZRiW8vLw4cOAA0dHR9O3bl0uXLqEoCj179uTrr79W1ynS6XS0bt2aLVu20KpVK6A4kGRjY0PdunU1rxIFBwezefNmpkyZwkcffYSxsTH16tXj9ddfr3C/1q1bx+XLl1mxYgUrVqxQ093c3DRfr/6ggIAAVq5cyaRJk3j//ffx8vIiPj6ehg0bqmVyc3M16/bcvXuXMWPGcP78eSwsLGjUqBE7d+5Uv1q+xNatW9XZR9bW1tSrV08z8wpgyJAh+Pn5cePGDWxtbSt0rgYGBqxevZqRI0fSsGFD6taty/z58zX1Wltb8/HHH3PixAkMDQ1p1qwZ33///RMt4jxp0iROnz5NcHAwFhYWDB06lO7du3Pjxg21TGRkJGFhYdSvX5/bt2+TnZ2Nu7u7utj5oEGDGDhwIHFxcRVq88GxDwgI4K233qJfv35cvXqVqKgooqOjyzzW2NiYVatW8fbbb9OoUSOaNWvGtGnT6NOnj1qmUaNGJCUlMXHiRFq3bo2iKLzwwgv069dPLXP+/Hn27t2rua+EEEII8cfTKY9aSVMIIYT4G7pz5w7Z2dl4eHg81iLIfzVRUVHMmTOHHTt20KJFi2fdnedOnz598PX1ZcKECc+6K6IM7733HtevX2fx4sXPuit/e3+3Z6MQQognI6+nCSGEEH8DMTExzJ8/n59//vmxFzH+J5s1axZWVlbPuhviIRwdHZk6deqz7oYQQgjxjyMzjYQQQvwjyV/ThRBCnzwbhRBClCYzjYQQQgghhBBCCCGEHgkaCSGEEEIIIYQQQgg9EjQSQgghhBBCCCGEEHokaCSEEEIIIYQQQggh9EjQSAghhBBCCCGEEELokaCREEIIIYQQQgghhNAjQSMhhBBCCCGEEEIIocfoWXdACCGE+Ctp+VlL9efLBy5z4usTtJjdAgPjx/s7S+biTAC8h3qXmb/nnT2PVa94ur788kvWrFnD9u3b1bTo6Gg+//xzLl26xMaNG4mPjycvL4/4+Phn19E/gU6nY+PGjXTv3p0rV65Qv359fvnlF2rWrKkpd/XqVby9vUlJScHd3R2APXv28NZbb3Hs2DE6d+5MREQEbdq04fr169jZ2T1x38LDw5/ba3DhwgVee+019u7di7GxMXl5eY88Ji4ujoiIiAqVrYgWLVowduxYevXq9VTqE0II8fcnM42EEEKI59C5c+cYPHgwLi4umJiY4ObmxqhRo7h69eoz69ORI0cIDQ3F1dUVc3NzvL29mTdvXoWOXbt2LfXq1cPMzAwfHx++//77cssnJiai0+n0tgsXLqhlwsPDNXkODg6EhIRw9OhRTV137tzhgw8+ICoqSk3LzMwkJiaGRYsWkZubS8eOHZk3bx5xcXEVH5AKiI6OpnHjxk+1zqepatWqDBw4UDM2JaZPn063bt3UgBHA6NGjady4MdnZ2U99rP5IcXFxZQa13N3dmTt37lNp49NPPyU3N5fU1FSOHz/+VOqsrEmTJjF+/HiKioqeSftCCCGePxI0EkIIIZ4zp0+fxs/PjxMnTrBq1SpOnjzJwoULSUhIwN/fn2vXrj2Tfh06dAhHR0dWrFhBeno6EydOZMKECcTGxpZ73N69ewkNDWXIkCEcPnyY7t270717d9LS0h7ZZlZWFrm5uerm6OioyQ8JCVHzEhISMDIyokuXLpoy69atw8bGhpYt/zfL7NSpUwB069YNJycnTE1NsbW1fSqzZZ43gwYN4ptvvtHcV7du3eLLL79kyJAhmrKnTp2ibdu21KxZ8x85VuU5deoUTZs2xcvLS+8+/bN07NiR33//nR9++OGZtC+EEOL5I0EjIYQQ4jkzfPhwTExM2L59O4GBgdSqVYuOHTuyc+dOzp8/z8SJE4mNjaVhw4bqMfHx8eh0OhYuXKimtW/fnkmTJqn7mzZtwtfXFzMzMzw9PYmJieHevXtqvk6nY8mSJfTo0QMLCwu8vLz47rvv1PzBgwczb948AgMD8fT0ZMCAAQwaNIgNGzaUez7z5s0jJCSEsWPH4u3tzdSpU/H19X1ksAnA0dERJycndTMw0P5qY2pqquY1btyY8ePHc+7cOS5fvqyWWb16NV27dlX3o6Oj1X0DAwN0Oh1QPHOpe/fuarmgoCBGjhzJuHHjsLe3x8nJiejoaE37eXl5vP7661SrVg0bGxvatm3LkSNHgOLZLTExMRw5ckSdDRUXF8eZM2fQ6XSkpqZq6tHpdCQmJgL/m2mVkJCAn58fFhYWBAQEkJWVpWn/Udf0xIkTvPzyy5iZmVG/fn127NihN8YNGjTAxcWFjRs3qmnff/89pqamtGjRAkDt89WrVxk8eLB6Lg+6evUqoaGh1KhRAwsLC3x8fFi1apWmzLp16/Dx8cHc3BwHBwfat2/PzZs3NWU++eQTnJ2dcXBwYPjw4RQWFuq1VdqcOXPw8fHB0tISV1dXhg0bRn5+vjqWgwYN4saNG+p1iI6OJigoiLNnz/Luu++q6SXXzc7Ojm3btuHt7Y2VlZUanHwYd3d31q9fz/Lly9HpdISHhz+yX2U5cuQIbdq0wdraGhsbG5o2bcrBgwfV/N27d9O6dWvMzc1xdXVl5MiRmrEzNDSkU6dOrF69utzxEkIIIUpI0EgIIYR4jly7do1t27YxbNgwzM3NNXlOTk7079+fNWvWEBgYSEZGhhocSUpKomrVqmrQobCwkH379hEUFARAcnIyAwcOZNSoUWRkZLBo0SLi4uKYPn26po2YmBj69u3L0aNH6dSpE/379y93ZtONGzewt7cv95z27dtH+/btNWnBwcHs27fvkePRuHFjnJ2d6dChA3v2lL8+VH5+PitWrKB27do4ODio6bt378bPz0/dj4yMZOnSpQDqLKWHWbZsGZaWluzfv5+PP/6YKVOmaAIvffr04dKlS/zwww8cOnQIX19f2rVrx7Vr1+jXrx9jxoyhQYMGajv9+vV75DmXNnHiRGbPns3BgwcxMjJi8ODBat6jrmlRURE9e/bExMSE/fv3s3DhQt57770y22nevDnJycmaups2baruu7q6kpubi42NDXPnzn3oudy5c4emTZuyZcsW0tLSGDp0KK+99hopKSlA8XiHhoYyePBgMjMzSUxMpGfPniiKotaxa9cuTp06xa5du1i2bBlxcXGPfBXOwMCA+fPnk56ezrJly/jxxx8ZN24cAAEBAcydOxcbGxv1OkRGRrJhwwZq1qzJlClT9O6DW7du8cknn/D111/z008/kZOTQ2RkpJpfEtQ7c+YMAAcOHCAkJIS+ffuSm5urvrZZXr/K0r9/f2rWrMmBAwc4dOgQ48ePx9jYGCieyRQSEkKvXr04evQoa9asYffu3YwYMaLcaymEEEKURxbCFkIIIZ4jJ06cQFEUvL3LXljb29ub69ev4+joiL29PUlJSfTu3ZvExETGjBmjflhNSUmhsLCQgIAAoDgYNH78eMLCwgDw9PRk6tSpjBs3TrOeTXh4OKGhoQDMmDGD+fPnk5KSQkhIiF5f9u7dy5o1a9iyZUu553ThwgWqV6+uSatevbpmfaIHOTs7s3DhQvz8/CgoKGDJkiUEBQWxf/9+fH191XKbN2/GysoKgJs3b+Ls7MzmzZvVGUl5eXncuHEDFxcX9RgrKyv11SonJ6dy+96oUSN1fLy8vIiNjSUhIYEOHTqwe/duUlJSuHTpEqampkDxDJn4+HjWrVvH0KFDsbKywsjI6JHtPMz06dMJDAwEYPz48XTu3Jk7d+5gZmb2yGu6c+dOjh07xrZt29TznzFjBh07dtRrx8XFhcOHD6v7Z8+e1YyZoaEhTk5O6HQ6bG1tH3o+NWrU0ARX3nnnHbZt28a3335L8+bNyc3N5d69e/Ts2RM3NzcAfHx8NHVUqVKF2NhYDA0NqVevHp07dyYhIYE33njjoeMUERGh/uzu7s60adN46623WLBgASYmJtja2qLT6fT6bWhoiLW1tV56YWEhCxcu5IUXXgBgxIgRTJkyRc23sLCgbt26akCnWrVqmJqaYm5urqmrvH6VJScnh7Fjx1KvXj2g+J4rMXPmTPr376/W6eXlxfz58wkMDOTzzz/HzMwMKL6W586do6ioSG9mnhBCCPEg+ZdCCCGEeA6VnnlRFlNTU15++WUSExPJy8sjIyODYcOGUVBQwLFjx0hKSqJZs2ZYWFgAxa+9TJkyBSsrK3V74403yM3N5datW2q9jRo1Un+2tLTExsaGS5cu6bWflpZGt27diIqK4pVXXgGKP/CWrn/GjBmPff5169blzTffpGnTpgQEBPDVV18REBDAp59+qinXpk0bUlNTSU1NJSUlheDgYDp27MjZs2cBuH37NoD6gbqySo8HFAezSsbjyJEj5Ofn4+DgoDnv7Oxsdc2kJ1W6fWdnZwBN++Vd08zMTFxdXTXBH39//zLbMTc319wHt2/ffqwxu3//PlOnTsXHxwd7e3usrKzYtm0bOTk5ALz44ou0a9cOHx8f+vTpwxdffMH169c1dTRo0ABDQ0PNeZec84wZMzTnW1Lvzp07adeuHTVq1MDa2prXXnuNq1evas6pMiwsLNSA0YN9gOLZPMeOHaNGjRrl1lPZfo0ePZrXX3+d9u3b8+GHH2ruoyNHjhAXF6c5/+DgYIqKisjOzlbLmZubU1RUREFBwWOduxBCiH8WmWkkhBBCPEdq166NTqcjMzOTHj166OVnZmZSrVo17OzsCAoKYvHixSQnJ9OkSRNsbGzUQFJSUpI6QwWKX92KiYmhZ8+eenWWDg6UzJwoodPp9L6JKSMjg3bt2jF06FDNmkkuLi6adXpKXltzcnLi4sWLmjouXrxY6dk3zZs3Z/fu3Zo0S0tLateure4vWbIEW1tbvvjiC6ZNm4aDgwM6nU4vMFFR5Y1Hfn4+zs7O6iuBpZW3SHTJ7I/SgcGHrdlTuv2SNXdKt1+Ra1oR165do1q1aup+1apVH2vMZs2axbx585g7d666lk9ERAR3794Fimf27Nixg71797J9+3Y+++wzJk6cyP79+/Hw8ADKH/O33nqLvn37qnkuLi6cOXOGLl268PbbbzN9+nTs7e3ZvXs3Q4YM4e7du2rgtDLK6sOjArkPepx+RUdH8+9//5stW7bwww8/EBUVxerVq+nRowf5+fm8+eabjBw5Uu+4WrVqqT9fu3YNS0tLvddbhRBCiLJI0EgIIYR4jjg4ONChQwcWLFjAu+++q/ngd+HCBb755huGDx8OQGBgIBEREaxdu1ZduygoKIidO3eyZ88exowZox7r6+tLVlaWJsDyONLT02nbti1hYWF66yEZGRmVWb+/vz8JCQmaV3V27Njx0FkvD5OamqrOtnkYnU6HgYGBOsPIxMSE+vXrk5GRoc6Ielp8fX25cOECRkZGmq+lL83ExIT79+9r0kqCM7m5uTRp0gRAE2yrTPvlXVNvb2/OnTtHbm6uOm4///xzmWXT0tLUewigSZMmrFixotJ92rNnD926dWPAgAFAcYDr+PHj1K9fXy2j0+lo2bIlLVu2ZPLkybi5ubFx40ZGjx79yPrt7e311tA6dOgQRUVFzJ49Ww3Iffvtt5oyZV2H8tKfhor0qyx16tShTp06vPvuu4SGhrJ06VJ69OiBr68vGRkZj/x/OC0tTb2vhBBCiEeR19OEEEKI50xsbCwFBQUEBwfz008/ce7cObZu3UqHDh2oU6cOkydPBopfXapSpQorV67UBI3i4+MpKCjQfMX85MmTWb58OTExMaSnp5OZmcnq1as1M4UeJS0tjTZt2vDKK68wevRoLly4wIULFzTfVFaWUaNGsXXrVmbPns2xY8eIjo7m4MGDmgV8J0yYwMCBA9X9uXPnsmnTJk6ePElaWhoRERH8+OOPasCsREFBgdqPzMxM3nnnHfLz8zXflhYcHKw3Q+lpaN++Pf7+/nTv3p3t27dz5swZ9u7dy8SJE9VvvHJ3dyc7O5vU1FSuXLlCQUEB5ubmtGjRgg8//JDMzEySkpIqdR1KPOqatm/fnjp16hAWFsaRI0dITk5m4sSJevXcunWLQ4cOaYJqwcHBpKenV3q2kZeXlzqTKDMzkzfffFMzy2z//v3MmDGDgwcPkpOTw4YNG7h8+fJD1/CqiNq1a1NYWMhnn33G6dOn+frrrzXfIgjF1yE/P5+EhASuXLmivh7m7u7OTz/9xPnz57ly5UqF20xJSaFevXqcP3/+ifpV2u3btxkxYgSJiYmcPXuWPXv2cODAAXVs3nvvPfbu3cuIESNITU3lxIkTbNq0SW8h7OTk5KceIBVCCPE3pgghhBD/QLdv31YyMjKU27dvP+uuPJbs7GwlLCxMqV69uqLT6RRA6dmzp3Lz5k1NuW7duilGRkbK77//riiKoty/f1+pUqWK0qJFC706t27dqgQEBCjm5uaKjY2N0rx5c2Xx4sVqPqBs3LhRc4ytra2ydOlSRVEUJSoqSgH0Njc3t0eez7fffqvUqVNHMTExURo0aKBs2bJFkx8WFqYEBgaq+x999JHywgsvKGZmZoq9vb0SFBSk/Pjjj3rHlO6HtbW10qxZM2XdunWacunp6Yq5ubmSl5enpm3cuFF58NeksLAwpVu3bup+YGCgMmrUKE2Zbt26KWFhYer+b7/9przzzjuKi4uLYmxsrLi6uir9+/dXcnJyFEVRlDt37ii9evVS7OzsFEAdy4yMDMXf318xNzdXGjdurGzfvl0BlF27dimKoii7du1SAOX69etqW4cPH1YAJTs7W0171DXNyspSWrVqpZiYmCh16tRRtm7dqnedV65cqdStW1d5UPPmzZWFCxdq0krfD2X18+rVq0q3bt0UKysrxdHRUZk0aZIycOBAdVwzMjKU4OBgpVq1aoqpqalSp04d5bPPPnvoNVAURRk1apTm3ijLnDlzFGdnZ8Xc3FwJDg5Wli9frjd+b731luLg4KAASlRUlKIoirJv3z6lUaNGiqmpqXo/LF26VLG1tdXU/+D9UnLepa/Fg/dGRfpVuq2CggLl1VdfVVxdXRUTExPFxcVFGTFihOYZlpKSonTo0EGxsrJSLC0tlUaNGinTp09X8//f//t/irGxsXLu3LmHjtXz/mwUQgjxdOkUpZIvYAshhBB/A3fu3CE7OxsPD4/HXgT5ryQqKoo5c+awY8cOWrRo8ay789zp06cPvr6+TJgw4Vl35S+nRYsWjBw5kn//+9+a9C1btjB27FjS0tLkW7ieE++99x7Xr19n8eLFDy3zd3s2CiGEeDKyppEQQgjxNxATE4O7uzs///wzzZs3lw/xlTRr1iz+7//+71l34y/nypUr9OzZk9DQUL28zp07c+LECc6fP4+rq+sz6J2oLEdHxwqtDSWEEEKUkJlGQggh/pHkr+lCCKFPno1CCCFKkz9DCiGEEEIIIYQQQgg9EjQSQgghhBBCCCGEEHokaCSEEEIIIYQQQggh9EjQSAghhBBCCCGEEELokaCREEIIIYQQQgghhNAjQSMhhBBCCCGEEEIIoUeCRkIIIYQQQgghhBBCj9Gz7oAQQgjxV5L0cuCf2l7gT0l/anuibF9++SVr1qxh+/btalp0dDSff/45ly5dYuPGjcTHx5OXl0d8fPyz6+ifQKfTsXHjRrp3786VK1eoX78+v/zyCzVr1tSUu3r1Kt7e3qSkpODu7g7Anj17eOuttzh27BidO3cmIiKCNm3acP36dezs7J64b+Hh4f+Ia/A0LFy4kC1btvB///d/z7orQgghnmMy00gIIYR4Dp07d47Bgwfj4uKCiYkJbm5ujBo1iqtXrz6zPh05coTQ0FBcXV0xNzfH29ubefPmVejYtWvXUq9ePczMzPDx8eH7778vt3xiYiI6nU5vu3DhglomPDxck+fg4EBISAhHjx7V1HXnzh0++OADoqKi1LTMzExiYmJYtGgRubm5dOzYkXnz5hEXF1fxAamA6OhoGjdu/FTrfJqqVq3KwIEDNWNTYvr06XTr1k0NGAGMHj2axo0bk52d/dTH6o8UFxdXZlDL3d2duXPnPpU2dDrdnxrsGjx4ML/88gvJycl/WptCCCH+fiRoJIQQQjxnTp8+jZ+fHydOnGDVqlWcPHmShQsXkpCQgL+/P9euXXsm/Tp06BCOjo6sWLGC9PR0Jk6cyIQJE4iNjS33uL179xIaGsqQIUM4fPgw3bt3p3v37qSlpT2yzaysLHJzc9XN0dFRkx8SEqLmJSQkYGRkRJcuXTRl1q1bh42NDS1btlTTTp06BUC3bt1wcnLC1NQUW1vbpzJb5nkzaNAgvvnmG819devWLb788kuGDBmiKXvq1Cnatm1LzZo1/5Fj9aTu3r371OoyMTHh3//+N/Pnz39qdQohhPjnkaCREEII8ZwZPnw4JiYmbN++ncDAQGrVqkXHjh3ZuXMn58+fZ+LEicTGxtKwYUP1mPj4eHQ6HQsXLlTT2rdvz6RJk9T9TZs24evri5mZGZ6ensTExHDv3j01X6fTsWTJEnr06IGFhQVeXl589913av7gwYOZN28egYGBeHp6MmDAAAYNGsSGDRvKPZ958+YREhLC2LFj8fb2ZurUqfj6+j4y2ATg6OiIk5OTuhkYaH+1MTU1VfMaN27M+PHjOXfuHJcvX1bLrF69mq5du6r70dHR6r6BgQE6nQ4onrnUvXt3tVxQUBAjR45k3Lhx2Nvb4+TkRHR0tKb9vLw8Xn/9dapVq4aNjQ1t27blyJEjQPHslpiYGI4cOaLOhoqLi+PMmTPodDpSU1M19eh0OhITE4H/zbRKSEjAz88PCwsLAgICyMrK0rT/qGt64sQJXn75ZczMzKhfvz47duzQG+MGDRrg4uLCxo0b1bTvv/8eU1NTWrRoAaD2+erVqwwePFg9lwddvXqV0NBQatSogYWFBT4+PqxatUpTZt26dfj4+GBubo6DgwPt27fn5s2bmjKffPIJzs7OODg4MHz4cAoLC/XaKm3OnDn4+PhgaWmJq6srw4YNIz8/Xx3LQYMGcePGDfU6REdHExQUxNmzZ3n33XfV9JLrZmdnx7Zt2/D29sbKykoNTj5MyWysHj16oNPp1P2SmWZLlizBw8MDMzMztfyDM5waN26sub/Ku7dKdO3ale+++47bt2+XOz5CCCHEw0jQSAghhHiOXLt2jW3btjFs2DDMzc01eU5OTvTv3581a9YQGBhIRkaGGhxJSkqiatWqatChsLCQffv2ERQUBEBycjIDBw5k1KhRZGRksGjRIuLi4pg+fbqmjZiYGPr27cvRo0fp1KkT/fv3L3dm040bN7C3ty/3nPbt20f79u01acHBwezbt++R49G4cWOcnZ3p0KEDe/bsKbdsfn4+K1asoHbt2jg4OKjpu3fvxs/PT92PjIxk6dKlAOospYdZtmwZlpaW7N+/n48//pgpU6ZoAi99+vTh0qVL/PDDDxw6dAhfX1/atWvHtWvX6NevH2PGjKFBgwZqO/369XvkOZc2ceJEZs+ezcGDBzEyMmLw4MFq3qOuaVFRET179sTExIT9+/ezcOFC3nvvvTLbad68ueY1p+TkZJo2baruu7q6kpubi42NDXPnzn3oudy5c4emTZuyZcsW0tLSGDp0KK+99hopKSlA8XiHhoYyePBgMjMzSUxMpGfPniiKotaxa9cuTp06xa5du1i2bBlxcXGPfBXOwMCA+fPnk56ezrJly/jxxx8ZN24cAAEBAcydOxcbGxv1OkRGRrJhwwZq1qzJlClT9O6DW7du8cknn/D111/z008/kZOTQ2RkpJpfEtQ7c+YMAAcOHABg6dKl5ObmqvsAJ0+eZP369WzYsEETKHyU8u6tEn5+fty7d4/9+/dXuF4hhBCiNFkIWwghhHiOnDhxAkVR8Pb2LjPf29ub69ev4+joiL29PUlJSfTu3ZvExETGjBmjrjGUkpJCYWEhAQEBQHEwaPz48YSFhQHg6enJ1KlTGTdunGY9m/DwcEJDQwGYMWMG8+fPJyUlhZCQEL2+7N27lzVr1rBly5Zyz+nChQtUr15dk1a9enXN+kQPcnZ2ZuHChfj5+VFQUMCSJUsICgpi//79+Pr6quU2b96MlZUVADdv3sTZ2ZnNmzerM5Ly8vK4ceMGLi4u6jFWVlbqq1VOTk7l9r1Ro0bq+Hh5eREbG0tCQgIdOnRg9+7dpKSkcOnSJUxNTYHiGTLx8fGsW7eOoUOHYmVlhZGR0SPbeZjp06cTGFi8ePv48ePp3Lkzd+7cwczM7JHXdOfOnRw7doxt27ap5z9jxgw6duyo146LiwuHDx9W98+ePasZM0NDQ5ycnNDpdNja2j70fGrUqKEJrrzzzjts27aNb7/9lubNm5Obm8u9e/fo2bMnbm5uAPj4+GjqqFKlCrGxsRgaGlKvXj06d+5MQkICb7zxxkPHKSIiQv3Z3d2dadOm8dZbb7FgwQJMTEywtbVFp9Pp9dvQ0BBra2u99MLCQhYuXMgLL7wAwIgRI5gyZYqab2FhQd26dTE2NgagWrVqANjZ2enVdffuXZYvX66WqYiK3Fsl/bC1teXs2bMVrlsIIYQoTWYaCSGEEM+h0jMvymJqasrLL79MYmIieXl5ZGRkMGzYMAoKCjh27BhJSUk0a9YMCwsLoHgR6ylTpmBlZaVub7zxBrm5udy6dUutt1GjRurPlpaW2NjYcOnSJb3209LS6NatG1FRUbzyyisA5OTkaOqfMWPGY59/3bp1efPNN2natCkBAQF89dVXBAQE8Omnn2rKtWnThtTUVFJTU0lJSSE4OJiOHTuqH6JLXtspeS2oskqPBxQHs0rG48iRI+Tn5+Pg4KA57+zsbHXNpCdVun1nZ2cATfvlXdPMzExcXV01wR9/f/8y2zE3N9fcB7dv336sMbt//z5Tp07Fx8cHe3t7rKys2LZtGzk5OQC8+OKLtGvXDh8fH/r06cMXX3zB9evXNXU0aNAAQ0NDzXmXnPOMGTM051tS786dO2nXrh01atTA2tqa1157jatXr2rOqTIsLCzUgNGDfYDimVnHjh2jRo0aj6zLzc2tUgEjqNy99eC1E0IIISpDZhoJIYQQz5HatWuj0+nIzMykR48eevmZmZlUq1YNOzs7goKCWLx4McnJyTRp0gQbGxs1kJSUlKTOUIHiV7diYmLo2bOnXp2lgwMlMydK6HQ6ioqKNGkZGRm0a9eOoUOHatZMcnFx0bx+U/LampOTExcvXtTUcfHixUrPvmnevDm7d+/WpFlaWlK7dm11f8mSJdja2vLFF18wbdo0HBwc0Ol0eoGJiipvPPLz83F2dlZfCSytvEWiS2ZBlQ4MPmzNntLtl6y5U7r9ilzTirh27ZomsFG1atXHGrNZs2Yxb9485s6dq64xFBERoS4AbWhoyI4dO9i7dy/bt2/ns88+Y+LEiezfvx8PDw+g/DF/66236Nu3r5rn4uLCmTNn6NKlC2+//TbTp0/H3t6e3bt3M2TIEO7evasGTiujrD48KpD7MJaWlnppBgYGevWVvgcqc289eO2EEEKIypCgkRBCCPEccXBwoEOHDixYsIB3331Xs67RhQsX+Oabbxg+fDgAgYGBREREsHbtWnXtoqCgIHbu3MmePXsYM2aMeqyvry9ZWVmaAMvjSE9Pp23btoSFhemth2RkZFRm/f7+/iQkJGheIdqxY8dDZ708TGpqqjrb5mF0Oh0GBgbqDCMTExPq169PRkaGOiPqafH19eXChQsYGRlpvpa+NBMTE+7fv69JK/mAn5ubS5MmTQAqtdZN6fbLu6be3t6cO3eO3Nxcddx+/vnnMsumpaWp9xBAkyZNWLFiRaX7tGfPHrp168aAAQOA4gDX8ePHqV+/vlpGp9PRsmVLWrZsyeTJk3Fzc2Pjxo2MHj36kfXb29vrraF16NAhioqKmD17thqQ+/bbbzVlyroO5aU/DmNj4wrXVa1aNc0aSr/99hvZ2dnqfkXuLSj+Nrs7d+6o95EQQghRWRI0EkIIIUp5P/3RX/P+NJW/dHPZYmNjCQgIIDg4mGnTpuHh4UF6ejpjx46lTp06TJ48GSh+dalKlSqsXLmSzZs3A8VBo8jISPWDeYnJkyfTpUsXatWqRe/evTEwMODIkSOkpaUxbdq0CvUrLS2Ntm3bEhwczOjRo9U1iQwNDcud6TBq1CgCAwOZPXs2nTt3ZvXq1Rw8eJDFixerZSZMmMD58+dZvnw5AHPnzsXDw4MGDRpw584dlixZwo8//sj27ds1dRcUFKj9uH79OrGxseTn52u+LS04OJjdu3drglZPQ/v27fH396d79+58/PHH1KlTh//+979s2bKFHj164Ofnh7u7O9nZ2aSmplKzZk2sra0xNzenRYsWfPjhh3h4eHDp0iXNjK2KetQ1bd++PXXq1CEsLIxZs2bx22+/MXHiRL16bt26xaFDhzSvEwYHBzNhwgSuX79OlSpVKtwnLy8v1q1bx969e6lSpQpz5szh4sWLatBo//79JCQk8Morr+Do6Mj+/fu5fPnyQ9fwqojatWtTWFjIZ599RteuXdmzZ4/mWwSheJ2j/Px8EhISePHFF7GwsMDCwgJ3d3d++uknXn31VUxNTalatWqF2kxJSWHgwIEkJCSor6i5u7uTkJBAy5YtMTU1LXfc2rZtS1xcHF27dsXOzo7JkydrXsmryL0FxQuWe3p6al6lE0IIISpFEUIIIf6Bbt++rWRkZCi3b99+1l15LNnZ2UpYWJhSvXp1RafTKYDSs2dP5ebNm5py3bp1U4yMjJTff/9dURRFuX//vlKlShWlRYsWenVu3bpVCQgIUMzNzRUbGxulefPmyuLFi9V8QNm4caPmGFtbW2Xp0qWKoihKVFSUAuhtbm5ujzyfb7/9VqlTp45iYmKiNGjQQNmyZYsmPywsTAkMDFT3P/roI+WFF15QzMzMFHt7eyUoKEj58ccf9Y4p3Q9ra2ulWbNmyrp16zTl0tPTFXNzcyUvL09N27hxo/Lgr0lhYWFKt27d1P3AwEBl1KhRmjLdunVTwsLC1P3ffvtNeeeddxQXFxfF2NhYcXV1Vfr376/k5OQoiqIod+7cUXr16qXY2dkpgDqWGRkZir+/v2Jubq40btxY2b59uwIou3btUhRFUXbt2qUAyvXr19W2Dh8+rABKdna2mvaoa5qVlaW0atVKMTExUerUqaNs3bpV7zqvXLlSqVu3rvKg5s2bKwsXLtSklb4fyurn1atXlW7duilWVlaKo6OjMmnSJGXgwIHquGZkZCjBwcFKtWrVFFNTU6VOnTrKZ5999tBroCiKMmrUKM29UZY5c+Yozs7Oirm5uRIcHKwsX75cb/zeeustxcHBQQGUqKgoRVEUZd++fUqjRo0UU1NT9X5YunSpYmtrq6n/wful5LxLX4vvvvtOqV27tmJkZKT+PxEVFaW8+OKLev29ceOG0q9fP8XGxkZxdXVV4uLilBdffFHtl6I8+t5SFEV55ZVXlJkzZ5Y7Ng963p+NQgghni6dojzmC9hCCCHEc+zOnTtkZ2fj4eHx2Isg/5VERUUxZ84cduzYQYsWLZ51d547ffr0wdfXlwkTJjzrrvzltGjRgpEjR/Lvf/9bk75lyxbGjh1LWlqa+tqX+OsoeVX0+PHj2NraVvi4v9uzUQghxJORf+GFEEKIv4GYmBjmz5/Pzz//rLcwtXi0WbNmYWVl9ay78Zdz5coVevbsSWhoqF5e586dGTp0KOfPn38GPROPkpuby/LlyysVMBJCCCEeJDONhBBC/CPJX9OFEEKfPBuFEEKUJjONhBBCCCGEEEIIIYQeCRoJIYQQQgghhBBCCD0SNBJCCCGEEEIIIYQQeiRoJIQQQgghhBBCCCH0SNBICCGEEEIIIYQQQuiRoJEQQgghhBBCCCGE0CNBIyGEEEIIIYQQQgihx+hZd0AIIYT4K4kd83/qzwcydrF866fMGbkeYyPjx6pvUfw0AN7sPqnM/BGzuz5WveLpSkhIYMSIEaSlpWFoaAjA4sWLmTp1KufPn2fOnDnk5eURHx9PamrqU2nT3d2diIgIIiIinkp9f6Y9e/bw1ltvcezYMTp37kx8fPwjjwkPD1fH8ElduXKF+vXr88svv1CzZs0nrk8IIYQQZZOZRkIIIcRz6Ny5cwwePBgXFxdMTExwc3Nj1KhRXL169Zn16ciRI4SGhuLq6oq5uTne3t7Mmzfvkcelp6fTq1cv3N3d0el0zJ0795HHnDlzBp1Op7f9/PPPapno6GhNnq2tLa1btyYpKUmvvnHjxjFp0iQ1YPTbb78xYsQI3nvvPc6fP8/QoUMrPhDPWHh4ON27d9eklYzX0wp4jR49msaNG5OdnU1cXNxTqbMyqlatysCBA4mKivrT2xZCCCH+SSRoJIQQQjxnTp8+jZ+fHydOnGDVqlWcPHmShQsXkpCQgL+/P9euXXsm/Tp06BCOjo6sWLGC9PR0Jk6cyIQJE4iNjS33uFu3buHp6cmHH36Ik5NTpdrcuXMnubm56ta0aVNNfoMGDdS8ffv24eXlRZcuXbhx44ZaZvfu3Zw6dYpevXqpaTk5ORQWFtK5c2ecnZ2xsLCoVL/+7k6dOkXbtm2pWbMmdnZ2z6QPgwYN4ptvvnlm97sQQgjxTyBBIyGEEOI5M3z4cExMTNi+fTuBgYHUqlWLjh07snPnTs6fP8/EiROJjY2lYcOG6jHx8fHodDoWLlyoprVv355Jk/732tymTZvw9fXFzMwMT09PYmJiuHfvnpqv0+lYsmQJPXr0wMLCAi8vL7777js1f/DgwcybN4/AwEA8PT0ZMGAAgwYNYsOGDeWeT7NmzZg1axavvvoqpqamlRoLBwcHnJyc1M3YWPsaoZGRkZpXv359pkyZQn5+PsePH1fLrF69mg4dOmBmZgZAXFwcPj4+AHh6eqLT6Thz5oxe2wcOHKBDhw5UrVoVW1tbAgMD+eWXX9R8RVGIjo6mVq1amJqa4uLiwsiRIzV13Lp1i8GDB2NtbU2tWrVYvHhxued7//59hgwZgoeHB+bm5tStW1czmys6Opply5axadMmdYZVYmIiHh4eADRp0gSdTkdQUBDwv1lJn3zyCc7Ozjg4ODB8+HAKCwvLbL9kxtLVq1cZPHgwOp2OuLi4R/arLOvWrcPHxwdzc3McHBxo3749N2/eVPOXLFmCt7c3ZmZm1KtXjwULFmiOb9CgAS4uLmzcuLHcdoQQQgjx+CRoJIQQQpRSeK9Q3e7dL/7gfO9+oSa9Mtu9+4XlHl9Z165dY9u2bQwbNgxzc3NNnpOTE/3792fNmjUEBgaSkZHB5cuXAUhKSqJq1aokJiYWn2dhIfv27VODB8nJyQwcOJBRo0aRkZHBokWLiIuLY/r06Zo2YmJi6Nu3L0ePHqVTp07079+/3JkeN27cwN7evtLnWVH/+te/cHR0pFWrVpoAVlkKCgpYunQpdnZ21K1bV01PTk7Gz89P3e/Xrx87d+4EICUlhdzcXFxdXfXq+/333wkLC2P37t38/PPPeHl50alTJ37//XcA1q9fz6effsqiRYs4ceIE8fHxajCqxOzZs/Hz8+Pw4cMMGzaMt99+m6ysrIeeQ1FRETVr1mTt2rVkZGQwefJk3n//fb799lsAIiMj6du3LyEhIeoMq4CAAFJSUoD/zcwqHcjbtWsXp06dYteuXSxbtoy4uDjNK2fR0dG4u7sD4OrqSm5uLjY2NsydO5fc3Fz69ev3yH49KDc3l9DQUAYPHkxmZiaJiYn07NkTRVEA+Oabb5g8eTLTp08nMzOTGTNm8MEHH7Bs2TJNPc2bNyc5Ofmh4yWEEEKIJ6QIIYQQ/0C3b99WMjIylNu3b2vSgT91q6yff/5ZAZSNGzeWmT9nzhwFUC5cuKA4ODgoa9euVRRFURo3bqzMnDlTcXJyUhRFUXbv3q0YGxsrN2/eVBRFUdq1a6fMmDFDU9fXX3+tODs7a8Zm0qRJ6n5+fr4CKD/88EOZfdmzZ49iZGSkbNu2rcLn5+bmpnz66aePLHf58mVl9uzZys8//6ykpKQo7733nqLT6ZRNmzapZaKiohQDAwPF0tJSsbS0VHQ6nWJjY6PXX1tbW2X58uWatMOHDyuAkp2dranvxRdffGif7t+/r1hbWyv/93//pyiKosyePVupU6eOcvfu3Yee64ABA9T9oqIixdHRUfn8888fef6lDR8+XOnVq5e6HxYWpnTr1k1TJjs7WwGUw4cPa9LDwsIUNzc35d69e2panz59lH79+qn7n332mdK2bVvNcba2tsrSpUsfu1+HDh1SAOXMmTNlHvvCCy8oK1eu1KRNnTpV8ff316S9++67SlBQULn9EJXzsGejEEKIfyb59jQhhBCilIEh76o/H8hMJPPsYf7dYQRGho/37WnLt36qV+/ToPz/MzIextTUlJdffpnExETat29PRkYGw4YN4+OPP+bYsWMkJSXRrFkzda2eI0eOsGfPHs3Movv373Pnzh1u3bqllmvUqJGab2lpiY2NDZcuXdJrPy0tjW7duhEVFcUrr7wCFK8TVL9+fbXM+++/z/vvv/9Y51+1alVGjx6t7jdr1oz//ve/zJo1i3/9619qet26ddUZSL///jtr1qyhT58+7Nq1S51ddPv2bfXVtMq4ePEikyZNIjExkUuXLnH//n1u3bpFTk4OAH369GHu3Ll4enoSEhJCp06d6Nq1K0ZG//v1q/R46nQ6nJyc1PHs2LGjOovGzc2N9PR0AP7zn//w1VdfkZOTw+3bt7l79y6NGzeudP9LNGjQQF0AHMDZ2Zlff/1V3R8xYgQjRox4ZD2V6deLL75Iu3bt8PHxITg4mFdeeYXevXtTpUoVbt68yalTpxgyZAhvvPGGesy9e/ewtbXV1GNubs6tW7cqecZCCCGEqCgJGgkhhBClNKvfRrOfefYwft5tMDZ6vKDRL8f3lFnv46pduzY6nY7MzEx69Oihl5+ZmUm1atWws7MjKCiIxYsXk5ycTJMmTbCxsVEDSUlJSQQGBqrH5efnExMTQ8+ePfXqLB1QeXDNIJ1OR1FRkSYtIyODdu3aMXToUM2aSS4uLppv73rar6299NJL7NixQ5NmYmJC7dq11f0mTZoQHx/P3LlzWbFiBVAcgLp+/Xql2wsLC+Pq1avMmzcPNzc3TE1N8ff35+7du0Dxq1xZWVns3LmTHTt2MGzYMGbNmkVSUpI6juWN55IlS7h9+7am3OrVq4mMjGT27Nn4+/tjbW3NrFmz2L9/f6X7X6Ii1/RRKtsvQ0NDduzYwd69e9m+fTufffYZEydOZP/+/WqA8osvvuCll17SO660a9euUa1atUr1VQghhBAVJ0EjIYQQ4jni4OBAhw4dWLBgAe+++65mXaMLFy7wzTffMHz4cAACAwOJiIhg7dq16tpFQUFB7Ny5kz179jBmzBj1WF9fX7KysjQBlseRnp5O27ZtCQsL01sPycjI6InrL09qairOzs6PLGdoaKgGY6A4kJSRkVHp9vbs2cOCBQvo1KkTAOfOnePKlSuaMubm5nTt2pWuXbsyfPhw6tWrx6+//oqvr+8j669Ro0aZbQYEBDBs2DA17dSpU5oyJiYm3L9/Xy8N0Et/WirSrwfpdDpatmxJy5YtmTx5Mm5ubmzcuJHRo0fj4uLC6dOn6d+/f7l1pKWlqfe2EEIIIZ4+CRoJIYQQz5nY2FgCAgIIDg5m2rRpeHh4kJ6eztixY6lTpw6TJ08Gil99qlKlCitXrmTz5s1AcdAoMjJS/cBeYvLkyXTp0oVatWrRu3dvDAwMOHLkCGlpaUybNq1C/UpLS6Nt27YEBwczevRoLly4ABQHacqbDXL37l01aHP37l3Onz9PamoqVlZWapApNjaWjRs3kpCQAMCyZcswMTGhSZMmAGzYsIGvvvqKJUuWaOq+d++e2o+S19MyMjJ477331DLBwcF6CyxXhJeXF19//TV+fn789ttvjB07VhPEK/lWsZdeegkLCwtWrFiBubk5bm5ulW6rdJvLly9n27ZteHh48PXXX3PgwAH129EA3N3d2bZtG1lZWTg4OGBra4ujoyPm5uZs3bqVmjVrYmZmpveq18M8OPaP26/S9u/fT0JCAq+88gqOjo7s37+fy5cv4+3tDRQvuD5y5EhsbW0JCQmhoKCAgwcPcv36dfW1xFu3bnHo0CFmzJhR0eETQgghRCVJ0EgIIYQoZcTsrqX2urKMOU+xvqfDy8uLAwcOEB0dTd++fbl06RKKotCzZ0++/vpr9fUenU5H69at2bJlC61atQKKA0k2NjbUrVsXS0tLtc7g4GA2b97MlClT+OijjzA2NqZevXq8/vrrFe7XunXruHz5MitWrFBf/YLi9XjK+sr6Ev/973/V4A/AJ598wieffEJgYKD6bW9XrlzRm7kydepUzp49i5GREfXq1WPNmjX07t1bUyY9PV2dfWRhYcELL7zA559/zsCBA9Uy/fv3Z9y4cWRlZWm+Ve1RvvzyS4YOHYqvry+urq7MmDGDyMhINd/Ozo4PP/yQ0aNHc//+fXx8fPi///s/HBwcKtzGg958800OHz5Mv3790Ol0hIaGMmzYMH744Qe1zBtvvEFiYiJ+fn7k5+eza9cugoKCmD9/PlOmTGHy5Mm0bt1aHdtHKWvsH6dfpdnY2PDTTz8xd+5cfvvtN9zc3Jg9ezYdO3YE4PXXX8fCwoJZs2YxduxYLC0t8fHxISIiQq1j06ZN1KpVi9atW1foPIQQQghReTrlUStpCiGEEH9Dd+7cITs7Gw8Pj8daBPmvJioqijlz5rBjxw5atGjxrLvz3Bk7diy//fYbixYtetZdERXUokULRo4cyb///e9n3ZW/lb/bs1EIIcSTMXjWHRBCCCHEk4uJiWH+/Pn8/PPPlV7EWMDEiRNxc3OTsXtOXLlyhZ49exIaGvqsuyKEEEL8rclMIyGEEP9I8td0IYTQJ89GIYQQpclMIyGEEEIIIYQQQgihR4JGQgghhBBCCCGEEEKPBI2EEEIIIYQQQgghhB4JGgkhhBBCCCGEEEIIPRI0EkIIIYQQQgghhBB6JGgkhBBCCCGEEEIIIfRI0EgIIYQQQgghhBBC6DF61h0QQggh/kqmD+gNwK5fM0k4mkG7RvVp4+P92PX9vyvXWPpjMtVtbQlr2xJTY2NN/sQV656ov+LpSEhIYMSIEaSlpWFoaAjA4sWLmTp1KufPn2fOnDnk5eURHx9PamrqU2nT3d2diIgIIiIinkp9f2evvvoqzZo1Y8yYMc+6K0IIIcQ/isw0EkIIIcrQxsebdo3qk3A0g12/Zj5WHY8KGD2Jc+fOMXjwYFxcXDAxMcHNzY1Ro0Zx9erVp9ZGZR05coTQ0FBcXV0xNzfH29ubefPmPfK49PR0evXqhbu7Ozqdjrlz5z7ymDNnzqDT6fS2n3/+WS0THR2tybO1taV169YkJSXp1Tdu3DgmTZqkBox+++03RowYwXvvvcf58+cZOnRoxQfiGQsPD6d79+6atJLxehoBr8TERHQ6HXl5eU9cV0VNmjSJ6dOnc+PGjT+tTSGEEEJI0EgIIYR4qCcJHP2RAaPTp0/j5+fHiRMnWLVqFSdPnmThwoUkJCTg7+/PtWvXnlpblXHo0CEcHR1ZsWIF6enpTJw4kQkTJhAbG1vucbdu3cLT05MPP/wQJyenSrW5c+dOcnNz1a1p06aa/AYNGqh5+/btw8vLiy5dumiCD7t37+bUqVP06tVLTcvJyaGwsJDOnTvj7OyMhYVFpfol4O7du0+troYNG/LCCy+wYsWKp1anEEIIIR5NgkZCCCFEOR4ncPRHBowAhg8fjomJCdu3bycwMJBatWrRsWNHdu7cyfnz55k4cSKxsbE0bNhQPSY+Ph6dTsfChQvVtPbt2zNp0iR1f9OmTfj6+mJmZoanpycxMTHcu3dPzdfpdCxZsoQePXpgYWGBl5cX3333nZo/ePBg5s2bR2BgIJ6engwYMIBBgwaxYcOGcs+nWbNmzJo1i1dffRVTU9NKjYWDgwNOTk7qZvzAWBsZGal59evXZ8qUKeTn53P8+HG1zOrVq+nQoQNmZmYAxMXF4ePjA4Cnpyc6nY4zZ87otX3gwAE6dOhA1apVsbW1JTAwkF9++UXNVxSF6OhoatWqhampKS4uLowcOVJTx61btxg8eDDW1tbUqlWLxYsXl3u+9+/fZ8iQIXh4eGBubk7dunU1s7mio6NZtmwZmzZtUmdYJSYm4uHhAUCTJk3Q6XQEBQUB/5uV9Mknn+Ds7IyDgwPDhw+nsLCwzPbPnDlDmzZtAKhSpQo6nY7w8HAAgoKCGDFiBBEREVStWpXg4OAyZzjl5eWp/SqRlpZGx44dsbKyonr16rz22mtcuXJF03bXrl1ZvXp1ueMjhBBCiKdLgkZCCCHEI1QmcPRHB4yuXbvGtm3bGDZsGObm5po8Jycn+vfvz5o1awgMDCQjI4PLly8DkJSURNWqVdUP6oWFhezbt08NHiQnJzNw4EBGjRpFRkYGixYtIi4ujunTp2vaiImJoW/fvhw9epROnTrRv3//cmc23bhxA3t7+6c3AA/417/+haOjI61atdIEsMpSUFDA0qVLsbOzo27dump6cnIyfn5+6n6/fv3YuXMnACkpKeTm5uLq6qpX3++//05YWBi7d+/m559/xsvLi06dOvH7778DsH79ej799FMWLVrEiRMniI+PV4NRJWbPno2fnx+HDx9m2LBhvP3222RlZT30HIqKiqhZsyZr164lIyODyZMn8/777/Ptt98CEBkZSd++fQkJCVFnWAUEBJCSkgL8b2ZW6UDerl27OHXqFLt27WLZsmXExcURFxen5kdHR+Pu7g6Aq6sr69evByArK4vc3FxN0GrZsmWYmJiwZ88eTYCyPHl5ebRt25YmTZpw8OBBtm7dysWLF+nbt6+mXPPmzUlJSaGgoKBC9QohhBDiyclC2EIIIUQFlCyGnXA0Q7Nf2h8dMAI4ceIEiqLg7V324tze3t5cv34dR0dH7O3tSUpKonfv3iQmJjJmzBj1A35KSgqFhYUEBAQAxcGg8ePHExYWBhTPsJk6dSrjxo0jKipKrT88PJzQ0FAAZsyYwfz580lJSSEkJESvL3v37mXNmjVs2bLlqY4BgJWVFbNnz6Zly5YYGBiwfv16unfvTnx8PP/617/Ucr/++itWVlZA8awea2tr1qxZg42NjVrm7NmzuLi4qPvm5uY4ODgAUK1atYe+Mte2bVvN/uLFi7GzsyMpKYkuXbqQk5ODk5MT7du3x9jYmFq1atG8eXPNMZ06dWLYsGEAvPfee3z66afs2rVLE9QqzdjYmJiYGHXfw8ODffv28e2339K3b1+srKwwNzenoKBA0+9q1aoB/5uZVVqVKlWIjY3F0NCQevXq0blzZxISEnjjjTcAqFq1Ki+88AIAhoaGahDQ0dEROzs7TV1eXl58/PHH6n5ZM7QeFBsbS5MmTZgxY4aa9tVXX+Hq6srx48epU6cOAC4uLty9e5cLFy7g5ub2yHqFEEII8eRkppEQQghRQeXNOPozAkalKYpSbr6pqSkvv/wyiYmJ5OXlkZGRwbBhwygoKODYsWMkJSXRrFkzda2eI0eOMGXKFKysrNTtjTfeIDc3l1u3bqn1NmrUSP3Z0tISGxsbLl26pNd+Wloa3bp1IyoqildeeQUoXieodP2lgwSVVbVqVUaPHs1LL71Es2bN+PDDDxkwYACzZs3SlKtbty6pqamkpqZy6NAh3n77bfr06cPBgwfVMrdv31ZfTauMixcv8sYbb+Dl5YWtrS02Njbk5+eTk5MDQJ8+fbh9+zaenp688cYbbNy4UfO6H2jHU6fT4eTkpI5nyetaVlZWNGjQQC33n//8h6ZNm1KtWjWsrKxYvHix2ubjaNCggboAOICzs7Pmmo4YMYKEhIQK1fXgmlIVceTIEXbt2qW5N+rVqwfAqVOn1HIlM+tK349CCCGE+GPJTCMhhBCiEsqacfRnBoxq166NTqcjMzOTHj166OVnZmZSrVo17OzsCAoKYvHixSQnJ9OkSRNsbGzUQFJSUhKBgYHqcfn5+cTExNCzZ0+9OksHVB5cM0in01FUVKRJy8jIoF27dgwdOlSzZpKLi4tmbZun/draSy+9xI4dOzRpJiYm1K5dW91v0qQJ8fHxzJ07V11UuWrVqly/fr3S7YWFhXH16lXmzZuHm5sbpqam+Pv7qwtAu7q6kpWVxc6dO9mxYwfDhg1j1qxZJCUlqeNY3nguWbKE27dva8qtXr2ayMhIZs+ejb+/P9bW1syaNYv9+/dXuv8lKnJNK8rS0lKzb2BQ/PfJ0kHOB9dLys/Pp2vXrnz00Ud69Tk7O6s/l7wGWTJrSgghhBB/PAkaCSGEEJVUOnB0/tp1si9e/tNmGDk4ONChQwcWLFjAu+++q1nX6MKFC3zzzTcMHz4cgMDAQCIiIli7dq26dlFQUBA7d+5kz549jBkzRj3W19eXrKwsTYDlcaSnp9O2bVvCwsL01kMyMjJ64vrLk5qaqgkyPIyhoaEajIHiQFJGRkal29uzZw8LFiygU6dOAJw7d05v8WZzc3O6du1K165dGT58OPXq1ePXX3/F19f3kfXXqFGjzDYDAgLUV9pAOxsHigNl9+/f10sD9NIfR2XqKgnw5Obm0qRJEwBN4BCK773169fj7u6OkdHDfzVNS0ujZs2aVK1a9TF7LoQQQojKktfThBBCiMfQxsebejWdOfb/cikovPenBIxKxMbGUlBQQHBwMD/99BPnzp1j69atdOjQgTp16jB58mSg+NWnKlWqsHLlSk3QKD4+noKCAlq2bKnWOXnyZJYvX05MTAzp6elkZmayevVqzUyhR0lLS6NNmza88sorjB49mgsXLnDhwgV1Me6HuXv3rvoK2d27dzl//jypqamcPHlSc87t2rVT95ctW8aqVas4duwYx44dY8aMGXz11Ve88847mrrv3bun9uPEiRNMmzaNjIwMunXrppYJDg5m9+7dFT7PEl5eXnz99ddkZmayf/9++vfvrwnixcXF8eWXX5KWlsbp06dZsWIF5ubmT7Qej5eXFwcPHmTbtm0cP36cDz74gAMHDmjKuLu7c/ToUbKysrhy5QqFhYU4Ojpibm6uLjJ948aNCrf54Ni7ubmh0+nYvHkzly9fJj8//6HHmpub06JFCz788EMyMzNJSkrSu6eGDx/OtWvXCA0N5cCBA5w6dYpt27YxaNAgTWAqOTlZfdVRCCGEEH8SRQghhPgHun37tpKRkaHcvn37WXflsWRnZythYWFK9erVFZ1OpwBKz549lZs3b2rKdevWTTEyMlJ+//13RVEU5f79+0qVKlWUFi1a6NW5detWJSAgQDE3N1dsbGyU5s2bK4sXL1bzAWXjxo2aY2xtbZWlS5cqiqIoUVFRCqC3ubm5PfJcyjouMDBQLRMVFaWpJy4uTvH29lYsLCzUvq5du1ZT74P9sbCwUHx8fJTPP/9cU+7q1auKmZmZcuzYMTXt8OHDCqBkZ2dr6nvxxRfV/V9++UXx8/NTzMzMFC8vL2Xt2rWKm5ub8umnnyqKoigbN25UXnrpJcXGxkaxtLRUWrRooezcuVM9vnTZEi+++KISFRX10LG6c+eOEh4ertja2ip2dnbK22+/rYwfP17Tr0uXLikdOnRQrKysFEDZtWuXoiiK8sUXXyiurq6KgYGBOrZhYWFKt27dNG2MGjWq3LFXFEWZMmWK4uTkpOh0OiUsLExRFEUJDAxURo0apdfnjIwMxd/fXzE3N1caN26sbN++XdMvRVGU48ePKz169FDs7OwUc3NzpV69ekpERIRSVFSkKErx/6+2trbKvn37Hjo24ul43p+NQgghni6dojxiJU0hhBDib+jOnTtkZ2fj4eHxWIsg/9VERUUxZ84cduzYQYsWLZ51d547Y8eO5bfffmPRokXPuiuiDJ9//jkbN25k+/btz7orf3t/t2ejEEKIJyOvpwkhhBB/AzExMcyfP5+ff/75sRcx/iebOHEibm5uMnZ/UcbGxnz22WfPuhtCCCHEP47MNBJCCPGPJH9NF0IIffJsFEIIUZrMNBJCCCGEEEIIIYQQeiRoJIQQQgghhBBCCCH0SNBICCGEEEIIIYQQQuiRoJEQQgghhBBCCCGE0CNBIyGEEEIIIYQQQgihR4JGQgghhBBCCCGEEEKPBI2EEEIIIYQQQgghhB6jZ90BIYQQ4q8kc/qPf2p73hPb/qntibIlJCQwYsQI0tLSMDQ0BGDx4sVMnTqV8+fPM2fOHPLy8oiPjyc1NfXZdvYP5u7uTkREBBEREdy9e5c6deqwbt06/Pz8NOXu3r1L/fr1Wb58OQEBAQAcO3aM8PBwUlNTqVevHvHx8Xh4eHD48GEaN278xH2Ljo5+bq/BrVu3eO2119ixYwe///47169fx87OrtxjEhMTadOmTYXKVsSrr75Ks2bNGDNmzBPXJYQQ4p9BZhoJIYQQZTiYfZSAGT2oP6kdo1ZGc/G3y5U6/u69u3yRtIr6k9rhM7kDX+3+lrv37j61/p07d47Bgwfj4uKCiYkJbm5ujBo1iqtXrz61NirryJEjhIaG4urqirm5Od7e3sybN++Rx6Wnp9OrVy/c3d3R6XTMnTv3kcecOXMGnU6nt/38889qmejoaE2era0trVu3JikpSa++cePGMWnSJDVg9NtvvzFixAjee+89zp8/z9ChQ4mMjCQhIaHiA1IBcXFxTyUY8EcxMTEhMjKS9957Ty9v4cKFeHh4qAEjgKioKCwtLcnKynrqY/VHSkxMRKfTkZeXp0kPCgoiIiLiqbSxbNkykpOT2bt3L7m5udja2j6Veitj0qRJTJ8+nRs3bvzpbQshhHg+SdBICCGEeMDo1VMZ+OW75N36DYAdGcm0+fhV9p8+XKHj8+/cpHF0Rz7dsQSA+0VFfLJ1EY2jO5J/5+YT9+/06dP4+flx4sQJVq1axcmTJ1m4cCEJCQn4+/tz7dq1J27jcRw6dAhHR0dWrFhBeno6EydOZMKECcTGxpZ73K1bt/D09OTDDz/EycmpUm3u3LmT3NxcdWvatKkmv0GDBmrevn378PLyokuXLpoPzbt37+bUqVP06tVLTcvJyaGwsJDOnTvj7OyMhYUFVlZWODg4VKp/fwf9+/dn9+7dpKenq2mKohAbG8uQIUM0ZU+dOkWrVq1wc3P7R45VeU6dOoW3tzcNGzbEyckJnU73p/ehYcOGvPDCC6xYseJPb1sIIcTzSYJGQgghRCn7Tx9ma1oiABuGL+bAB5uZ/K8IAAZ9Fcn9ovuPrOPfi0cC4FXdg73vbyDpvbXUd/EC4M3lE564j8OHD8fExITt27cTGBhIrVq16NixIzt37uT8+fNMnDiR2NhYGjZsqB4THx+PTqdj4cKFalr79u2ZNGmSur9p0yZ8fX0xMzPD09OTmJgY7t27p+brdDqWLFlCjx49sLCwwMvLi++++07NHzx4MPPmzSMwMBBPT08GDBjAoEGD2LBhQ7nn06xZM2bNmsWrr76KqalppcbCwcEBJycndTM2NtbkGxkZqXn169dnypQp5Ofnc/z4cbXM6tWr6dChA2ZmZkDx7B8fHx8APD090el0nDlzhujoaM0rVuHh4XTv3p1PPvkEZ2dnHBwcGD58OIWFhWqZgoICIiMjqVGjBpaWlrz00kskJiYCxbNbBg0axI0bN9TZUNHR0epYx8fHa87Fzs6OuLg44H8zrTZs2ECbNm2wsLDgxRdfZN++fZpjdu/eTevWrTE3N8fV1ZWRI0dy8+b/ApeXLl2ia9eumJub4+HhwTfffKM3xlWqVKFly5asXr1aTTt06BCnTp2ic+fOappOp+PQoUNMmTJFcy6l3b9/nyFDhuDh4YG5uTl169bVm42WmJhI8+bNsbS0xM7OjpYtW3L27FlNma+//hp3d3dsbW159dVX+f333/XaerC8n58f1tbWODk58e9//5tLly6pY9mmTRv1XHU6HeHh4YSHh5OUlMS8efPU63PmzBl1VlJCQgJ+fn5YWFgQEBBAVlbWQ9sPCgpi9uzZ/PTTT+h0OoKCgh7Zr7KcPXuWrl27UqVKFSwtLWnQoAHff/+9mp+WlkbHjh2xsrKievXqvPbaa1y5ckVTR9euXTXXUgghhCiPBI2EEEKIUibHzwGKA0b1nF/A0tScV5t3pZVXMwD2njxU7vGKonDy0hkA4kd8gZ2FLdWs7Vnz1n8AOJyTjqIoj92/a9eusW3bNoYNG4a5ubkmz8nJif79+7NmzRoCAwPJyMjg8uXi1+qSkpKoWrWqGrAoLCxk37596ofX5ORkBg4cyKhRo8jIyGDRokXExcUxffp0TRsxMTH07duXo0eP0qlTJ/r371/uzKYbN25gb2//2Of7KP/6179wdHSkVatWmgBWWQoKCli6dCl2dnbUrVtXTU9OTtas19OvXz927twJQEpKCrm5ubi6upZZ565duzh16hS7du1i2bJlxMXFqYEdgBEjRrBv3z5Wr17N0aNH6dOnDyEhIZw4cYKAgADmzp2LjY2NOhsqMjKyUuc/ceJEIiMjSU1NpU6dOoSGhqqBvlOnThESEkKvXr04evQoa9asYffu3YwYMUI9Pjw8nHPnzrFr1y7WrVvHggULygxaNG/enOTkZM2Y1alTB2trazUtNzeXBg0aMGbMmIeeS1FRETVr1mTt2rVkZGQwefJk3n//fb799lsA7t27R/fu3QkMDOTo0aPs27ePoUOHamblnDp1ivj4eDZv3szmzZtJSkriww8/LHecCgsLmTp1KkeOHCE+Pp4zZ84QHh4OgKurK+vXrwcgKyuL3Nxc5s2bx7x58/D39+eNN95Qr0/p+2DixInMnj2bgwcPYmRkxODBg9W8kqBeyf9vGzZs4I033sDf35/c3Fw1kFpev8oyfPhwCgoK+Omnn/j111/56KOPsLKyAiAvL4+2bdvSpEkTDh48yNatW7l48SJ9+/bV1NG8eXNSUlIoKCgod8yEEEIIkIWwhRBCCI1z1/4LgEdVbZCglVczdp84wNX86+Uef7+oSP259AddQwNDTRkjQ0Mex4kTJ1AUBW9v7zLzvb29uX79Oo6Ojtjb25OUlETv3r1JTExkzJgx6qyOlJQUCgsL1fVoYmJiGD9+PGFhYUDxDJupU6cybtw4oqKi1PrDw8MJDQ0FYMaMGcyfP5+UlBRCQkL0+rJ3717WrFnDli1bHutcy2NlZcXs2bNp2bIlBgYGrF+/nu7duxMfH8+//vUvtdyvv/6qfqi+desW1tbWrFmzBhsbG7XM2bNncXFxUffNzc3VV6uqVatW7itzVapUITY2FkNDQ+rVq0fnzp1JSEjgjTfeICcnh6VLl5KTk6PWHxkZydatW1m6dCkzZszA1tYWnU5X6dfySkRGRqqzfWJiYmjQoAEnT56kXr16zJw5k/79+6tr8nh5eTF//nwCAwP5/PPPycnJ4YcffiAlJYVmzYqDol9++WWZ95aLi4tmts+DYwbFQUsjIyOsrKzU83lwlouxsTExMTHqvoeHB/v27ePbb7+lb9++/Pbbb9y4cYMuXbrwwgsvAOj1p6ioiLi4ODVg9dprr5GQkKAX4CytdEDH09OT+fPn06xZM/Lz87GyslIDm46Ojpo1pkxMTLCwsCjz+kyfPp3AwEAAxo8fT+fOnblz5w5mZmYYGxtTt25dLCwsALC3t8fCwgITExNNXY/q14NycnLo1auXZiZcidjYWJo0acKMGTPUtK+++gpXV1eOHz9OnTp1gOJreffuXS5cuICbm9tDx0wIIYQAmWkkhBBCaLSsXTzjZO3B/wU6FEXhw+8XANCwZt0yjytROhh0+ff/zcApHWx63IBRaY+arWRqasrLL79MYmIieXl5ZGRkMGzYMAoKCjh27BhJSUk0a9ZM/VB75MgRpkyZgpWVlbqVzLC4deuWWm+jRo3Uny0tLbGxsSlzZkpaWhrdunUjKiqKV155BSj+wFu6/tIfbiuratWqjB49mpdeeolmzZrx4YcfMmDAAGbNmqUpV7duXVJTU0lNTeXQoUO8/fbb9OnTh4MHD6plbt++rb6aVlkNGjRQF88GcHZ2Vsfj119/5f79+9SpU0dz3klJSZw6deqx2ntQ6evh7OwMoLZ/5MgR4uLiNG0HBwdTVFREdnY2mZmZGBkZadaBqlevXpkLc5ubm2vugycZs//85z80bdqUatWqYWVlxeLFi8nJyQGKgyvh4eEEBwfTtWtX5s2bR25uruZ4d3d3zQyn0mP+zTffaM63ZHbUoUOH6Nq1K7Vq1cLa2loN9pS0+zjKG/saNWpw7NgxmjdvXm4dle3XyJEjmTZtGi1btiQqKoqjR4+qeUeOHGHXrl2a869Xrx6A5n4rmaFY+noKIYQQDyMzjYQQQohSpvccS9DH/ZixJZakrJ9pU8+faZs/U/NrO7o/so4xwUOZvW0xgR/1YXCrvhgbGrMoqXitmLeDBjxR/2rXro1OpyMzM5MePXro5WdmZlKtWjXs7OwICgpi8eLFJCcn06RJE2xsbNRAUlJSkvoBFSA/P5+YmBh69uypV2fp4MCDawbpdDqKSs2uAsjIyKBdu3YMHTpUs2aSi4uL5qvSn/Zray+99BI7duzQpJmYmFC7dm11v8n/196dR0VxpW0Af1qgkR0FDKCyGRZ1lLRxA2VwCbaJiXs0hgmgMeooGlSMCw6Iu1FcGUkcR9DEBU0Mesy4cpAoQREiIt2CSEAyRMY1GlQEsb4/+LhD0azqjJn4/M7hHLvq1u1bt8rWfnnvWyoVEhISsH79elEM2NraGnfuNJxBVp+G5qO0tBR6enrIyMiQBZYA1JlFUruf2oHBmrWS6nr/6sy2mu8/efJkzJgxQ+c4BwcHWV2nxty+fRs2NjbitbW1NS5evNjk46vt2bMHoaGhiIqKgpeXF8zMzLB69WqcPXtWtImNjcWMGTNw5MgRxMfHY+HChTh+/Dh69+4NoOE5Hzp0KHr16iX2tW3bFvfv34darYZarcbOnTthY2ODoqIiqNVqlJc//RMNG5r7pniacU2cOBFqtRrffvstjh07hhUrViAqKgrTp09HaWkp3nnnHaxatUrnuOqgFgCxnLTm9SQiIqoPg0ZEREQ1tDG3RuyENRi/LRQpV9KRcuXfGSmpYQlN6mNC3zG48q8CHMg8jm2n94rtI7sNRvDAoGcan5WVFfz8/LB582bMnDlTVteopKQEO3fuxLRp0wAAvr6+CAkJwb59+0Tton79+uHEiRNISUnB7NmzxbHdunVDbm6uLMDyNDQaDQYMGIDAwECd5UL6+vrP3H9DMjMzZV+O66Onp4eHDx+K1yqVClqt9rmPR6VSobKyEtevX4ePj0+dbZRKJSordYur29jYyDJs8vLymp0Z0q1bN2i12nrn3MPDA48fP0ZGRoZYnpabm6vz2HmgKnNMpVKJ1yqVCjExMZAkqVlPAUtJSYG3tzemTp0qttWVdaVSqaBSqTB//nx4eXlh165dImjUEDMzM1kWElCVzXPr1i2sXLlS1CSqmWkGVF0HADrXor7r8zzk5OQ0Oq66tG/fHlOmTMGUKVMwf/58/O1vf8P06dPRrVs3fP3113BycoK+fv3/xc/Ozka7du1gbW393M6FiIh+v7g8jYiIqJZeLipcXHwMW4NWYfnIT3Bg+lZolybCwsis8YNRlXWwYvQ8pIYlIGrsQnz67gKkhiVg6cg5z+Ux29HR0Xj06BHUajW+++47/PTTTzhy5Aj8/Pzg5uaG8PBwAFXLZ1q1aoVdu3bJgkYJCQl49OgR+vTpI/oMDw/Hjh07EBkZCY1Gg0uXLmHPnj2yTKHGZGdno3///hg0aBBmzZqFkpISlJSUiGLc9SkvLxdLyMrLy1FcXIzMzExcuXJFds4DBw4Ur7dv347du3cjJycHOTk5WL58ObZt24bp06fL+n78+LEYR15eHpYuXQqtVothw4aJNmq1GqdPn27yeTaVm5sb/P39ERAQgP3796OgoABpaWlYsWKFqPPk5OSE0tJSJCYm4ubNmyIwNGDAAERHR+P8+fNIT0/HlClTdDJsGjN37lx8//33CA4ORmZmJvLy8nDgwAFRCNvd3R2DBw/G5MmTcfbsWWRkZGDixIk6BdaBqsLX1csMAaB///4oLS2FRqNp1phcXV2Rnp6Oo0eP4vLly/jLX/6Cc+fOif0FBQWYP38+UlNTcfXqVRw7dgx5eXn11vBqCgcHByiVSmzatAk//vgjDh48iCVLlsjaODo6QqFQ4NChQ7hx4wZKS0sBVF2fs2fPorCwEDdv3mxyJlFxcTE8PDyQlpb2TOOqLSQkBEePHkVBQQF++OEHJCUlibmZNm0abt++jXHjxuHcuXPIz8/H0aNHMX78eFngq/a1JCIiapBERET0Enr48KGk1Wqlhw8fvuihPJWCggIpMDBQeuWVVySFQiEBkEaOHCndv39f1m7YsGGSvr6+9Ouvv0qSJEmVlZVSq1atpN69e+v0eeTIEcnb21syMjKSzM3NpZ49e0pbtmwR+wFI33zzjewYCwsLKTY2VpIkSYqIiJAA6Pw4Ojo2ei51Hefr6yvaREREyPqJi4uTOnbsKBkbG4ux7tu3T9Zv7fEYGxtLXbp0kWJiYmTtbt26JbVs2VLKyckR286fPy8BkAoKCmT9eXp6iteBgYHSsGHDZH19/PHHsnGXl5dL4eHhkpOTk2RgYCDZ2dlJI0aMkLKyskSbKVOmSFZWVhIAKSIiQpIkSSouLpYGDRokmZiYSK6urtI//vEP2VxXz9n58+dFP3fu3JEASElJSWJbWlqa5OfnJ5mamkomJiZS165dpWXLlon9165dk4YMGSIZGhpKDg4O0o4dOyRHR0dp3bp1os33338vWVpaSg8ePJCd65gxY6R58+bJtnl6eopzqGucZWVlUlBQkGRhYSFZWlpKf/7zn6V58+aJeS0pKZGGDx8u2dnZSUqlUnJ0dJTCw8OlysrKOq+BJEnSunXrGr3Hdu3aJTk5OUmGhoaSl5eXdPDgQZ35W7x4sWRrayspFAopMDBQkiRJys3NlXr37i0ZGRmJ+yEpKUkCIN25c0ccW/t+qT7vmtei9r3RlHHVfq/g4GCpQ4cOkqGhoWRjYyN98MEH0s2bN0V/ly9flkaMGCFZWlpKRkZGkoeHhxQSEiI9efJEkqSqzz0LCwspNTW13rn6X/9sJCKi50shSc/w3F8iIqL/UWVlZSgoKICzs/NTF/T9LYmIiMDatWtltV+o6ebMmYN79+7h888/f9FD+c0ZO3YsPD09sWDBAtn2rKws+Pn5IT8/v9EaTfTbEBMTg2+++QbHjh2rt83v7bORiIieDZenERER/Q5ERkZi48aNOHPmTLOK8VKVsLAwODo6cu5qKS8vR5cuXTBz5kydfV27dsWqVatQUFDwAkZGT8PAwACbNm1qvCEREdH/Y6YRERG9lPjbdCIiXfxsJCKimphpREREREREREREOhg0IiIiIiIiIiIiHQwaERERERERERGRDgaNiIiIiIiIiIhIB4NGRERERERERESkg0EjIiIiIiIiIiLSwaARERERERERERHp0H/RAyAiIvotGTRoEI4fPw5fX1/069dPbI+MjAQAfPTRR7C3t2+wj+q2U6dOhY2NDQDgxo0b2Lx5MwAgIiJCtF20aNFzHD09rcTERAQHByM7Oxt6enoAgC1btmDJkiUoLi7G2rVr8csvvyAhIQGZmZkvdrD/YU5OTggJCUFISAjKy8vh5uaGr776Ct27d5e1Ky8vR6dOnbBjxw54e3sDAHJychAUFITMzEx4eHggISEBzs7OOH/+PF577bVnHtuiRYteimvwPBw5cgTz5s3DDz/8gBYt+HtiIiJ6OvwXhIiIqIb79+83uN/ExOS/NJKG/fTTT5gwYQLs7e2hVCrh6OiIjz/+GLdu3XphY7pw4QLGjRuH9u3bw8jICB07dsSGDRsaPU6j0WDUqFFwcnKCQqHA+vXrGz2msLAQCoVC5+fMmTOizaJFi2T7LCws4OPjg+TkZJ3+PvnkEyxcuFAEjO7du4fg4GDMnTsXxcXFmDRpEkJDQ5GYmNj0CWmCuLg4WFpaPtc+nyelUonQ0FDMnTtXZ99nn30GZ2dnETACqgKiJiYmyM3Nfe5z9Z908uRJKBQK/PLLL7Lt/fr1Q0hIyHN5Dycnpybd28/L4MGDYWBggJ07d/7X3pOIiH5/GDQiIiKq4ccffwQAWFlZybZXB4tu377d4PGVlZXiz+bm5nX++fHjx888xu7duyMvLw+7d+/GlStX8NlnnyExMRFeXl6NjvE/JSMjA23atMGXX34JjUaDsLAwzJ8/H9HR0Q0e9+DBA7i4uGDlypWwtbVt1nueOHEC165dEz+vv/66bH/nzp3FvtTUVLi6uuLtt9/G3bt3RZvTp08jPz8fo0aNEtuKiopQUVGBIUOGwM7ODsbGxjA1NdW5L14G/v7+OH36NDQajdgmSRKio6Px4Ycfytrm5+ejb9++cHR0fCnn6llVVlbiyZMnz62/oKAgbNy48bn1R0RELx8GjYiIiGooKSkBAHTs2FG2fdiwYQCAHTt2oKysrN7jz549CwCwt7eHoaGh2G5oaIh27doBAI4ePfpMY5w2bRqUSiWOHTsGX19fODg44M0338SJEydQXFyMsLAwREdH4w9/+IM4JiEhAQqFAp999pnY9sYbb2DhwoXi9YEDB9CtWze0bNkSLi4uiIyMlAW4FAoFtm7dihEjRsDY2Biurq44ePCg2D9hwgRs2LABvr6+cHFxwZ/+9CeMHz8e+/fvb/B8evTogdWrV+O9996TzVlTWFlZwdbWVvwYGBjI9uvr64t9nTp1wuLFi1FaWorLly+LNnv27IGfnx9atmwJoCr7p0uXLgAAFxcXKBQKFBYWYtGiRbIlVkFBQRg+fDjWrFkDOzs7WFlZYdq0aaioqBBtHj16hNDQULRt2xYmJibo1asXTp48CaAqu2X8+PG4e/euyIaqXq6oUCiQkJAgOxdLS0vExcUB+Hem1f79+9G/f38YGxvD09MTqampsmNOnz4NHx8fGBkZoX379pgxY4Ysm+769et45513YGRkBGdn5zqzUlq1aoU+ffpgz549YltGRgby8/MxZMgQsU2hUCAjIwOLFy+WnUtNlZWV+PDDD+Hs7AwjIyO4u7vrZKOdPHkSPXv2hImJCSwtLdGnTx9cvXpV1uaLL76Ak5MTLCws8N577+HXX3/Vea/a7bt37w4zMzPY2tri/fffx/Xr18Vc9u/fX5yrQqFAUFAQgoKCkJycjA0bNojrU1hYKLKSEhMT0b17dxgbG8Pb2xu5ubn1vn+/fv1w9epVzJw5U/QF/DvT7ODBg+jUqRMMDQ1RVFRUZ4bT8OHDERQUJF43dG9Ve+edd5Ceno78/PwG54eIiKg+DBoRERHV0r17d+jry8v+vfrqq2LpUlxcnE62kCRJSExMxPHjxwFUfVmrrfoLdnp6OtLS0p5qbLdv38bRo0cxdepUGBkZyfbZ2trC398f8fHx8PX1hVarxY0bNwAAycnJsLa2Fl8qKyoqkJqaKuo2nTp1CgEBAfj444+h1Wrx+eefIy4uDsuWLZO9R2RkJMaMGYOsrCy89dZb8Pf3bzCz6e7du2jduvVTnWtTDB06FG3atEHfvn1lAay6PHr0CLGxsbC0tIS7u7vYfurUKVm9nrFjx+LEiRMAgLS0NFy7dg3t27evs8+kpCTk5+cjKSkJ27dvR1xcnAjsAEBwcDBSU1OxZ88eZGVl4d1338XgwYORl5cHb29vrF+/Hubm5iIbKjQ0tFnnHxYWhtDQUGRmZsLNzQ3jxo0T92Z+fj4GDx6MUaNGISsrC/Hx8Th9+jSCg4PF8UFBQfjpp5+QlJSEr776Cps3bxbBlJp69uyJU6dOyebMzc0NZmZmYtu1a9fQuXNnzJ49u95zefLkCdq1a4d9+/ZBq9UiPDwcCxYswN69ewFUZeENHz4cvr6+yMrKQmpqKiZNmiSCLNXnlZCQgEOHDuHQoUNITk7GypUrG5yniooKLFmyBBcuXEBCQgIKCwtFAKZ9+/b4+uuvAQC5ubm4du0aNmzYgA0bNsDLywsfffSRuD4174OwsDBERUUhPT0d+vr6mDBhgthXHdSr/vu2f/9+tGvXDosXLxZ9VXvw4AFWrVqFrVu3QqPRoE2bNg2eS7WG7q1qDg4OeOWVV2TXjoiIqDlYCJuIiKgGR0dHvPnmmzrbFQoFpk6dik2bNuFf//oXli1bhs6dO8Pe3h6SJIkgA1AVyKhrmZWtrS2GDh2KgwcP4vDhwzh8+HCzC2Hn5eVBkiSdTKhqHTt2xJ07d9CmTRu0bt0aycnJGD16NE6ePInZs2eLrI60tDRUVFSIejSRkZGYN28eAgMDAVRl2CxZsgSffPKJrHB3UFAQxo0bBwBYvnw5Nm7ciLS0NAwePFhnLN9//z3i4+Px7bffNuscm8LU1BRRUVHo06cPWrRoga+//hrDhw9HQkIChg4dKtpdvHgRpqamAKq+nJuZmSE+Pl62XPDq1auy4uZGRkZiaZWNjU2DS+ZatWqF6Oho6OnpwcPDA0OGDEFiYiI++ugjFBUVITY2FkVFRaL/0NBQHDlyBLGxsVi+fDksLCygUCiavSyvWmhoqAhGRkZGonPnzrhy5Qo8PDywYsUK+Pv7i4wVV1dXbNy4Eb6+voiJiUFRUREOHz6MtLQ09OjRAwDw97//vc57y97eXpbtU3vOgKr7W19fH6ampuJ8bt68KWtjYGAgCsUDgLOzM1JTU7F3716MGTMG9+7dw927d/H222+jQ4cOAHSz/p48eYK4uDgRsPrggw+QmJioE+CsqWZAx8XFBRs3bkSPHj1QWloKU1NTEdhs06aNrMaUUqmEsbFxnddn2bJl8PX1BQDMmzcPQ4YMQVlZGVq2bAkDAwO4u7vD2NgYANC6dWvo6emJTKeaKioqsHnzZnh6etY7/tqacm9Vq33tiIiImoNBIyIiohoCAgLqfdJQ69atERISgm3btuHevXvQaDSyOi8AMG7cOLi5udXbv0qlgomJCXbv3v1M45QkqcH9hoaG+OMf/4iTJ0/ijTfegFarxdSpU/Hpp58iJycHycnJ6NGjh/hSe+HCBaSkpMi+eFdWVqKsrAwPHjwQ7bp27Sr2m5iYwNzcvM7MlOzsbAwbNgwREREYNGgQgKovup06dRJtFixYgAULFjzV+VtbW2PWrFnidY8ePfDzzz9j9erVsqCRu7u7yED69ddfER8fj3fffRdJSUkiu+jhw4diaVpzde7cWWSgAYCdnR0uXrwIoCpgVVlZqXM/PHr06LnV+6l5Pezs7ABULTnz8PDAhQsXkJWVJVtyJkkSnjx5goKCAly+fBn6+vqyOlAeHh51FuY2MjLCgwcPxOtnmbO//vWv2LZtG4qKivDw4UOUl5eLZX+tW7dGUFAQ1Go1/Pz88MYbb2DMmDHi3ICqgtI1M5zs7OzEPbhz505MnjxZ7Dt8+DB8fHyQkZGBRYsW4cKFC7hz546oG1T7nmyO+ubewcEBbdu2RU5OTpP6USqVsr6aojn3Vu1rR0RE1BwMGhEREdXQ2KOpLSwsMHPmTNy7dw+XLl3C/fv3YWBgACcnJ7Rr1062jKY+bm5uCA8Pxz//+c9mj+/VV1+FQqHApUuXMGLECJ39ly5dgo2NDSwtLdGvXz9s2bIFp06dgkqlgrm5uQgkJScniywJACgtLUVkZCRGjhyp02fN4EDtmkEKhUKncK9Wq8XAgQMxadIkWc0ke3t72aPSn/eytV69eonlgdWUSiVeffVV8VqlUiEhIQHr16/Hl19+CaAqAHXnzp2nes+G5qO0tBR6enrIyMiQBZYAiOyn+igUCp3AYM1aSXW9f/W9V/P9J0+ejBkzZugc5+DgIKvr1Jjbt2/DxsZGvLa2thbBsebYs2cPQkNDERUVBS8vL5iZmWH16tWiFhgAxMbGYsaMGThy5Aji4+OxcOFCHD9+HL179wbQ8JwPHToUvXr1Evvatm2L+/fvQ61WQ61WY+fOnbCxsUFRURHUajXKy8ubfQ7VGpr75jAyMtL53GjRokWD178591bta0dERNQcDBoRERE9BXNzc9mX0+ZSKBT11slpiJWVFfz8/LB582bMnDlTVteopKQEO3fuxLRp0wAAvr6+CAkJwb59+0Tton79+uHEiRNISUnB7NmzxbHdunVDbm6uLMDyNDQaDQYMGIDAwECd5UL6+vrP3H9DMjMzZRkp9dHT08PDhw/Fa5VKBa1W+9zHo1KpUFlZievXr8PHx6fONkqlUvbEvWo2Njayujd5eXnNzhbp1q0btFptvXPu4eGBx48fIyMjQyxPy83N1XnsPFCVOaZSqcRrlUqFmJgYSJLUpEBptZSUFHh7e2Pq1KliW11FmlUqFVQqFebPnw8vLy/s2rVLBI0aYmZmJstCAqqKdt+6dQsrV64Uf+fS09NlbZRKJQDoXIv6rs/TaE5fta9/ZWUlsrOzRcHuptxbAFBWVob8/HzZtSMiImoOBo2IiIhqaG6NoRchOjoa3t7eUKvVWLp0KZydnaHRaDBnzhyRxQRULZ9p1aoVdu3ahUOHDgGoChqFhoZCoVCgT58+os/w8HC8/fbbcHBwwOjRo9GiRQtcuHAB2dnZWLp0aZPGlZ2djQEDBkCtVmPWrFniSXR6enoNZjqUl5eLoE15eTmKi4uRmZkJU1NTEfCIjo7GN998g8TERADA9u3boVQqxZfh/fv3Y9u2bdi6daus78ePH4txVC9P02q1mDt3rmijVquxffv2Jp1jc7i5ucHf3x8BAQGIioqCSqXCjRs3kJiYiK5du2LIkCFwcnJCaWkpEhMT4enpCWNjYxgbG2PAgAGIjo6Gl5cXKisrMXfuXJ0Mm8bMnTsXvXv3RnBwMCZOnAgTExNotVocP34c0dHRcHd3x+DBgzF58mTExMRAX18fISEhOgXWgarC10uWLBGv+/fvj9LSUmg0GtlT+hrj6uqKHTt24OjRo3B2dsYXX3yBc+fOwdnZGQBQUFCALVu2YOjQobC3t0dubi7y8vIQEBDQrHOvycHBAUqlEps2bcKUKVOQnZ0tOxegqpaZQqHAoUOH8NZbb8HIyAimpqZwcnLC2bNnUVhYKKt91Jji4mIMHDgQO3bsQM+ePQFULav77rvvxFMCra2t6z1+wIABmDVrFr799lt06NABa9eulQXzmnJvAcCZM2dgaGgILy+vZs4aERFRFT49jYiI6H+Mq6srzp07BxcXF4wZM0YU73Zzc0NKSopYnqJQKODj4wOFQoG+ffsCqAokmZubo3v37jAxMRF9qtVqHDp0CMeOHUOPHj3Qu3dvrFu3Do6Ojk0e11dffYUbN27gyy+/hJ2dnfipzmKpz88//ywyS65du4Y1a9ZApVJh4sSJos3Nmzd1MlKWLFmC119/Hb169cKBAwcQHx+P8ePHy9poNBoxjtdeew179+5FTEyMLAjh7+8PjUbT4CPTn1ZsbCwCAgIwe/ZsuLu7Y/jw4Th37hwcHBwAAN7e3pgyZQrGjh0LGxsbfPrppwCAqKgotG/fHj4+Pnj//fcRGhoq6ko1VdeuXZGcnIzLly/Dx8cHKpUK4eHhsgLWsbGxsLe3h6+vL0aOHIlJkybpPL0rNTUVd+/exejRo8U2KysrjBgxQlYvqSkmT56MkSNHYuzYsejVqxdu3bolyzoyNjZGTk4ORo0aBTc3N0yaNAnTpk2T1SlqLhsbG8TFxWHfvn3o1KkTVq5ciTVr1sjatG3bVhSDf+WVV8QT5kJDQ6Gnp4dOnTqJZW1NUVFRgdzcXFl22OLFi1FYWIgOHTo0ulxswoQJCAwMREBAAHx9feHi4iKyjKo1dm8BwO7du+Hv79/se4eIiKiaQmqskiYREdHvUFlZGQoKCuDs7PzUBX1/SyIiIrB27VpZ7Rdqujlz5uDevXv4/PPPX/RQfnPGjh0LT09PnaLlWVlZ8PPzQ35+fqM1mui/7+bNm3B3d0d6errI5GqK39tnIxERPRtmGhEREf0OREZGYuPGjThz5sxTFeN92YWFhcHR0ZFzV0t5eTm6dOmCmTNn6uzr2rUrVq1ahYKCghcwMmpMYWEhNm/e3KyAERERUW3MNCIiopcSf5tORKSLn41ERFQTM42IiIiIiIiIiEgHg0ZERPRSY8ItEdG/8TORiIhqYtCIiIheStWPL6/5dCMiopdd9Wdi9WckERG93PRf9ACIiIheBD09PVhaWuL69esAqh71rVAoXvCoiIheDEmS8ODBA1y/fh2WlpbQ09N70UMiIqLfABbCJiKil5YkSSgpKcEvv/zyoodCRPSbYGlpCVtbWwbRiYgIAINGREREqKysREVFxYseBhHRC2VgYMAMIyIikmHQiIiIiIiIiIiIdLAQNhERERERERER6WDQiIiIiIiIiIiIdDBoREREREREREREOhg0IiIiIiIiIiIiHQwaERERERERERGRDgaNiIiIiIiIiIhIB4NGRERERERERESk4/8AAcWMO44Sl+4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_metrics(metrics_df, ylim=(0, 0.4))" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "a3359e60-7820-4e35-bdc7-0081e7dac1b4", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
modelmeteorbleu_1rouge_l
0Qwen2-0.5B(flash-attn:false)0.2596080.0440940.252230
1Qwen2-0.5B(flash-attn:true)0.2604150.0460130.254505
2Qwen2-0.5B(finetuned)(flash-attn:false)0.2906800.0607460.265629
3Qwen2-0.5B(finetuned)(flash-attn:true)0.2875260.0641510.265591
4Qwen2-1.5B(flash-attn:false)0.3140900.0710570.316036
5Qwen2-1.5B(flash-attn:true)0.3119620.0726960.310527
6Qwen2-1.5B(finetuned)(flash-attn:false)0.3490200.0845100.320895
7Qwen2-1.5B(finetuned)(flash-attn:true)0.3494130.0946410.318800
\n", + "
" + ], + "text/plain": [ + " model meteor bleu_1 rouge_l\n", + "0 Qwen2-0.5B(flash-attn:false) 0.259608 0.044094 0.252230\n", + "1 Qwen2-0.5B(flash-attn:true) 0.260415 0.046013 0.254505\n", + "2 Qwen2-0.5B(finetuned)(flash-attn:false) 0.290680 0.060746 0.265629\n", + "3 Qwen2-0.5B(finetuned)(flash-attn:true) 0.287526 0.064151 0.265591\n", + "4 Qwen2-1.5B(flash-attn:false) 0.314090 0.071057 0.316036\n", + "5 Qwen2-1.5B(flash-attn:true) 0.311962 0.072696 0.310527\n", + "6 Qwen2-1.5B(finetuned)(flash-attn:false) 0.349020 0.084510 0.320895\n", + "7 Qwen2-1.5B(finetuned)(flash-attn:true) 0.349413 0.094641 0.318800" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "metrics_df.drop(columns=[\"all_metrics\", \"accuracy\"], inplace=True, errors=\"ignore\")\n", + "metrics_df" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f3158e09-7eb4-4ec6-b072-7283e3484f25", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "def get_minutes(time_str):\n", + " parts = time_str.split(\":\")\n", + " if len(parts) == 3:\n", + " h, m, s = parts\n", + " else:\n", + " h, m = parts\n", + " s = 0\n", + " return int(h) * 60 + int(m) + int(s) / 60" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9681e0ca-8c1c-45ad-b98b-094e8a5c262f", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "def get_times(metrics_df, df_time):\n", + " train_time = []\n", + " eval_time = []\n", + " for idx, row in metrics_df.iterrows():\n", + " model_name = row[\"model\"]\n", + " with_flash_attn = \"true\" in model_name\n", + " finetuned = \"finetuned\" in model_name\n", + " model_name = model_name.split(\"(\")[0]\n", + " # print(model_name, with_flash_attn)\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + " df_time[\"with_flash_attn\"] == with_flash_attn\n", + " ].iloc[0]\n", + "\n", + " if finetuned:\n", + " train_time.append(model_time[\"train_time\"])\n", + " eval_time.append(get_minutes(model_time[\"fine_tuned_model_eval_time\"]))\n", + " else:\n", + " train_time.append(model_time[\"train_time\"])\n", + " eval_time.append(get_minutes(model_time[\"base_model_eval_time\"]))\n", + "\n", + " return train_time, eval_time" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "95115d7f-5602-45de-b25e-d4a1c1b054de", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "def get_perf_df(metrics_df, bnb_4bit=False):\n", + " df_time = pd.read_csv(\"results/model_training_evaluation_times.csv\")\n", + " df_time = df_time[df_time[\"4bit\"] == bnb_4bit]\n", + " df_time.drop(columns=[\"4bit\"], inplace=True)\n", + " # print(df_time.to_markdown())\n", + " perf_df = metrics_df.copy()\n", + " perf_df.drop(columns=[\"bleu_1\", \"rouge_l\"], inplace=True)\n", + "\n", + " perf_df[\"train-time(mins)\"], perf_df[\"eval-time(mins)\"] = get_times(metrics_df, df_time)\n", + " if bnb_4bit:\n", + " perf_df.drop(columns=[\"meteor\"], inplace=True)\n", + " \n", + " return perf_df" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7a95bcf7-2aa6-40fc-83b9-91ddd0fb4056", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAPdCAYAAABba9tpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3RU1fr/8ffMZCa995AQuvSmIgFURBQs2LCCgooNsYEF8YsNEdSrggVR7+Uq/gD1Xuu1gSCCCgiCIFJEek2H9Drl98dAwkACISQ5KZ/XWrMy55x99nlOxuPKPDx7b5PL5XIhIiIiIiIiIiJSh8xGByAiIiIiIiIiIk2PklIiIiIiIiIiIlLnlJQSEREREREREZE6p6SUiIiIiIiIiIjUOSWlRERERERERESkzikpJSIiIiIiIiIidU5JKRERERERERERqXNeRgdQH9jtdtauXUt0dDRms/J0IiIiIiIiIlJznE4nqamp9OjRAy8vpWKO0G8CWLt2Lb169TI6DBERERERERFpxFatWsXZZ59tdBj1hpJSQHR0NOD+jyM2NtbgaERERERERESkMUlOTqZXr15l+QdxU1IKyobsxcbGEh8fb3A0IiIiIiIiItIYacogT/ptiIiIiIiIiIhInVNSSkRERERERERE6pySUiIiIiIiIiIiUuc0p1QVOZ1OSkpKjA5DaojNZtNYXhERERERaTQcDgelpaVGh9FkWa1WLBaL0WE0OEpKVUFJSQk7d+7E6XQaHYrUELPZTMuWLbHZbEaHIiIiIiIiUm0ul4uUlBSysrKMDqXJCwkJISYmBpPJZHQoDYaSUifhcrlITk7GYrGQkJCg6ppGwOl0cuDAAZKTk2nevLn+hyEiIiIiIg3WkYRUVFQUfn5++n5jAJfLRUFBAWlpaQDExsYaHFHDoaTUSdjtdgoKCoiLi8PPz8/ocKSGREZGcuDAAex2O1ar1ehwRERERERETpnD4ShLSIWHhxsdTpPm6+sLQFpaGlFRURrKV0Uq+zkJh8MBoGFejcyRz/PI5ysiIiIiItLQHJlDSgUU9cORz0Fze1WdklJVpBLIxkWfp4iIiIiINBb6flM/6HM4dUpKiYiIiIiIiIhInVNSSk6qRYsWTJ8+vcH1LSIiIiIiIiL1lyY6r6bN7TvU6fU6/LX5lNr379+f7t2710jC57fffsPf3/+0+nj//fd56KGHjlumtCb6FhERERERkepzOF2s2nmQtNwiogJ96NUyDItZQ9Gk9ikp1US5XC4cDgdeXif/TyAyMrLW4qjNvkVEREREROTE5m9I5tmvNpGcXVS2LzbYh6eHdGRw51gDIzs1NVmYIXVHw/caoVtvvZWlS5fy2muvYTKZMJlMvP/++5hMJr777jvOPPNMvL29+eWXX9i+fTtXXnkl0dHRBAQEcPbZZ7No0SKP/o4dYmcymfjXv/7F1VdfjZ+fH23btuV///tfpfEsWbKE2267jezs7LJ4nnnmmUr7fuedd7j88svx8/OjQ4cOrFixgm3bttG/f3/8/f3p06cP27dv97jGl19+Sc+ePfHx8aFVq1Y8++yz2O320/5dioiIiIiINFbzNyQzes7vHgkpgJTsIkbP+Z35G5INiqz+crlc+q5Zg5SUaoRee+01kpKSuPPOO0lOTiY5OZmEhAQAHn/8cV544QU2b95M165dycvL49JLL+WHH35g7dq1DB48mCFDhrBnz54TXuPZZ5/l+uuvZ/369Vx66aUMHz6cgwcPVti2T58+TJ8+naCgoLJ4HnnkkUr7fu655xgxYgTr1q2jffv2DBs2jLvvvpsJEyawevVqXC4X9913X1n7n3/+mREjRvDggw+yadMm3nnnHd5//32ef/75avz2REREREREGiaXy0VBib1Kr9yiUp7+30ZcFfVz+Ocz/9tEblFplfpzuSrqqWL9+/fn/vvv56GHHiI0NJTo6Gj++c9/kp+fz2233UZgYCBt2rThu+++Kztnw4YNXHLJJQQEBBAdHc0tt9xCRkYGUHFhxq5du056HkBxcTEPPPAAUVFR+Pj40K9fP3777bey40uWLKmwwENqhobvNULBwcHYbDb8/PyIiYkB4K+//gJg0qRJXHTRRWVtw8LC6NatW9n2c889x+eff87//vc/j8TPsW699VZuuukmAKZMmcLrr7/OqlWrGDx48HFtbTYbwcHBmEymsnhO5LbbbuP6668HYPz48SQlJfHkk08yaNAgAB588EFuu+22svbPPvssjz/+OCNHjgSgVatWPPfcczz22GM8/fTTJ72eiIiIiIhIY1BY6qDjUwtqpC8XkJJTRJdnvq9S+02TBuFnq3qKYfbs2Tz22GOsWrWKjz/+mNGjR/P5559z9dVX88QTTzBt2jRuueUW9uzZQ0lJCQMGDOCOO+5g2rRpFBYWMn78eK6//noWL17Ma6+9xt9//03nzp2ZNGkS4J4qJisr64TnATz22GN8+umnzJ49m8TERF566SUGDRrEtm3bCAsLK4v38ccf5+WXX6ZVq1aEhoZW/RcpJ6SkVBNz1llneWzn5eXxzDPP8M0335CcnIzdbqewsPCklVJdu3Yte+/v709QUBBpaWkAdOrUid27dwNw7rnnemS3q+LovqOjowHo0qWLx76ioiJycnIICgrijz/+YNmyZR6VUQ6Hg6KiIgoKCvDz8zul64uIiIiIiEjt6tatGxMnTgRgwoQJvPDCC0RERHDnnXcC8NRTTzFz5kzWr1/PokWL6NGjB1OmTCk7/9///jcJCQn8/ffftGvX7rjCDIA333zzhOc1a9aMmTNn8v7773PJJZcA8M9//pOFCxcya9YsHn300bLzji3wkJqhpFQTc+xKd4888ggLFy7k5Zdfpk2bNvj6+nLttddSUlJywn6sVqvHtslkwul0AvDtt99SWloKgK+v7ynHeHTfJpOp0n1HrpeXl8ezzz7LNddcc1xfPj4+p3x9ERERERGRhsjXamHTpEFVartq50Fufe+3k7Z7/7az6dUy7KTtfK2WKl33iKOLESwWC+Hh4ccVIwCkpaXxxx9/8OOPPxIQEHBcP9u3b6ddu3YVXuNk5xUVFVFaWkrfvn3L9lutVnr16sXmzZs92h9b4CE1Q0mpRspms+FwOE7abtmyZdx6661cffXVgDvBc2TsbXUlJiZWO57q6NmzJ1u2bKFNmza10r+IiIiIiEhDYDKZqjyE7ty2kcQG+5CSXVThvFImICbYh3PbRmIxm2o0Tqi40KGyYoS8vDyGDBnCiy++eFw/sbGVrxB4svOOXUDrRI4t8JCaoaRUI9WiRQtWrlzJrl27CAgIKKsqOlbbtm357LPPGDJkCCaTiSeffLLStqcbT15eHj/88APdunXDz8+vxobVPfXUU1x++eU0b96ca6+9FrPZzB9//MGGDRuYPHlyjVxDRERERESkMbGYTTw9pCOj5/yOCTwSU0dSUE8P6VgrCalT1bNnTz799FNatGiBl1fFaYyKCiFOdl7r1q2x2WwsW7asrLiitLSU3377jYceeqjG70OOp9X3GqlHHnkEi8VCx44diYyMrHSOqFdffZXQ0FD69OnDkCFDGDRoED179qzxePr06cM999zDDTfcQGRkJC+99FKN9T1o0CC+/vprvv/+e84++2x69+7NtGnTKqzYEhEREREREbfBnWOZeXNPYoI9pz2JCfZh5s09Gdy58iqkujRmzBgOHjzITTfdxG+//cb27dtZsGABt912W1ki6ujCjIyMDJxO50nP8/f3Z/To0Tz66KPMnz+fTZs2ceedd1JQUMCoUaMMvuumQZVS1dThr80nb2Sgdu3asWLFCo99t95663HtWrRoUbbqwBFjxozx2D52OF9FS31mZWWdNKaZM2cyc+bMU+q7RYsWx+3r37//cfsGDRpUtjqfiIiIiIiIVM3gzrFc1DGGVTsPkpZbRFSgD71ahtWLCqkj4uLiWLZsGePHj+fiiy+muLiYxMREBg8ejNnsrrV55JFHGDlyJB07dqSwsJCdO3fSokWLk573wgsv4HQ6ueWWW8jNzeWss85iwYIFWmGvjphcFWUYmph9+/aRkJDA3r17iY+P9zhWVFTEzp07admypSbNbkT0uYqIiIiISEOn7zX1y4k+jxPlHZoyDd8TEREREREREZE6p6SUiIiIiIiIiIjUOSWlRERERERERESkzikpJSIiIiIiItKAaaro+kGfw6lTUkpERERERESkAbJarQAUFBQYHIlA+edw5HORk/MyOgAREREREREROXUWi4WQkBDS0tIA8PPzw2QyGRxV0+NyuSgoKCAtLY2QkBAsFovRITUYSkqJiIiIiIhUxzx9+a8XhjXtIVMxMTEAZYkpMU5ISEjZ5yFVo6SUiIiIiIiISANlMpmIjY0lKiqK0tJSo8NpsqxWqyqkqkFJKam2999/n4ceeoisrKxTOm/Xrl20bNmStWvX0r1791qJTUREREREpCmxWCxKikiDo6RUddV1qW4DLUm99dZbycrK4osvvijbl5CQQHJyMhEREcYFJiIiIiIiIiKGUlJK6pzFYtE4WxEREREREZEmzmx0AFJ7nE4nU6dOpWXLlvj6+tKtWzc++eQTnE4n8fHxzJw506P92rVrMZvN7N69G4BXX32VLl264O/vT0JCAvfeey95eXlVvv4zzzzD7Nmz+fLLLzGZTJhMJpYsWcKuXbswmUysW7cOgCVLlmAymViwYAE9evTA19eXAQMGkJaWxnfffUeHDh0ICgpi2LBhHkudVnZ/IiIiIiIiIlL/qVKqEZs6dSpz5szh7bffpm3btvz000/cfPPNLFiwgJtuuol58+YxevTosvZz586lb9++JCYmAmA2m3n99ddp2bIlO3bs4N577+Wxxx7jrbfeqtL1H3nkETZv3kxOTg7vvfceAGFhYRw4cKDC9s888wxvvvkmfn5+XH/99Vx//fV4e3szb9488vLyuPrqq3njjTcYP378Ce8vMjKS888//3R+dSIiIiIiIiJSy5SUaqSKi4uZMmUKixYtIikpCYBWrVrxyy+/8M477/DYY4/xyiuvsGfPHpo3b47T6eSjjz5i4sSJZX089NBDZe9btGjB5MmTueeee6qclAoICMDX15fi4uIqDdebPHkyffv2BWDUqFFMmDCB7du306pVKwCuvfZafvzxR8aPH3/S+1NSSkRERERERKR+0/C9Rmrbtm0UFBRw0UUXERAQUPb64IMP2L59O927d6dDhw7MmzcPgKVLl5KWlsZ1111X1seiRYu48MILadasGYGBgdxyyy1kZmZ6DKE7Ys+ePR7XmTJlyinH3LVr17L30dHR+Pn5lSWkjuxLS0ur0v2JiIiIiDQWP22GIS9D3BgwDYcvVnsef+ZTaP8I+N8OoXfCwCmwclvFfRWXQvcJ7n7W7ara9V0uuOTF46+dmQuDX3TH5T0SEu6H+96HnOO/LoiIVEiVUo3UkbmfvvnmG5o1a+ZxzNvbG4Dhw4czb948Hn/8cebNm8fgwYMJDw8HYNeuXVx++eWMHj2a559/nrCwMH755RdGjRpFSUkJfn5+Hn3GxcWVzREF7mF6p8pqtZa9N5lMHttH9jmdzirfn4iIiIhIY5BfDN2aw+3nwzXTjz/eLgbevBVaRUFhCUz7Di5+Aba9CpFBnm0f+xDiQuGPPVW//vT5YKpg8XGzGa48EyZfB5GBsC0VxrwPB/Ng3n2ncIMi0mQpKdVIdezYEW9vb/bs2VPpULZhw4YxceJE1qxZwyeffMLbb79ddmzNmjU4nU5eeeUVzGZ3Qd1//vOfSq/n5eVFmzZtjttvs9lwOByneTfHq8r9iYiIiIg0Bpd0d78qM6yv5/arw2HWEli/By7sXL7/u3Xw/Z/w6YPw3R9Vu/a6XfDKN7B6MsSO8TwW6g+jB5ZvJ0bCvQPhH99UrW8RESWlGqnAwEAeeeQRxo4di9PppF+/fmRnZ7Ns2TKCgoIYOXIkLVq0oE+fPowaNQqHw8EVV1xRdn6bNm0oLS3ljTfeYMiQISxbtswjaVVVLVq0YMGCBWzZsoXw8HCCg4Pr7P5ERERERJqaEju8+yME+0G3xPL9qdlw57/gi3HgV8WBBQXFMGwGzLgVYkJO3v7AIfhsNZzfoTqRi0hTpKRUdQ1zGR3BST333HNERkYydepUduzYQUhICD179uSJJ54oazN8+HDuvfdeRowYga+vb9n+bt268eqrr/Liiy8yYcIEzjvvPKZOncqIESNOKYY777yTJUuWcNZZZ5GXl8ePP/5IixYt6uz+RERERESagq9/hxvfhIISiA2BhY9DRKD7mMsFt74N91wIZ7WCXelV63PsHOjTDq4868TtbnoTvlzjHjo4pCf8647TuhURaUJMLper/mdXatm+fftISEhg7969xMfHexwrKipi586dtGzZEh8fH4MilJqmz1VERERETtu8CiZaqmWm4fD5WLjqmERRfhEkZ0FGLvzzR1i8CVY+C1HB8Pp8+M9KWPokWMzupFTLh2Dt89C9RcXX+d8aeHgurJ0CAT4nvnZKFmQVwN/JMOFjd6XUW7fV7H2fUAMoGBA5Ud6hKVOllIiIiIiISAPn7wNtYtyv3m2h7Tj3vFITrnQnqFZsda+Qd7SznoThfWH2Pcf3t3gTbE+DkDs99w+dDue2hyUTy/fFhLhf7eMgLADOnQRPXgWxoTV6iyLSCCkpJSIiIiIi0sg4XVBsd79/fYR7hbwjDhyCQS/Cx/fDOa0rPv/xIXBHf899XR6HaTe7h+hVel33Ytll1xYRORGz0QEc8cILL2AymXjooYfK9hUVFTFmzBjCw8MJCAhg6NChpKamepy3Z88eLrvsMvz8/IiKiuLRRx/Fbtf/AUVEREREpHHIK3Kvgrdul3t7Z7r7/Z4M97C9Jz6GX7fC7nRYsxNufxf2H4LrznG3bx4BnRPKX+1i3ftbR0F8uPv9/oPQ/hFYtd29HRPieU7nhPK+Wka533+7Dt5bChv2uocEfrMW7vk39G0HLSJr/dciIo1AvaiU+u2333jnnXfo2rWrx/6xY8fyzTff8N///pfg4GDuu+8+rrnmGpYtWwaAw+HgsssuIyYmhuXLl5OcnMyIESOwWq1MmTLFiFsRERERERGpUat3wAXPl2+Pm+P+OfJcePt2+OsAzP7ZPZ9UeACc3Qp+fhI6ncK0NaUO2JLsXnGvqnyt7vmrxs6B4lJICIdrznZXWYmIVIXhE53n5eXRs2dP3nrrLSZPnkz37t2ZPn062dnZREZGMm/ePK699loA/vrrLzp06MCKFSvo3bs33333HZdffjkHDhwgOjoagLfffpvx48eTnp6OzWarUgxVmei8RYsWHqvTScNWWFjIrl27NNG5iIiIiFSfAROdSwU00fkJOZwuVu08SFpuEVGBPvRqGYbFrP9265omOq+Y4cP3xowZw2WXXcbAgQM99q9Zs4bS0lKP/e3bt6d58+asWLECgBUrVtClS5eyhBTAoEGDyMnJYePGjZVes7i4mJycnLJXbm5upW0tFgsAJSUl1bo/qZ+OfJ5HPl8REREREZHGZv6GZPq9uJib/vkrD360jpv++Sv9XlzM/A3JRocmAhg8fO+jjz7i999/57fffjvuWEpKCjabjZCQEI/90dHRpKSklLU5OiF15PiRY5WZOnUqzz77bJVi9PLyws/Pj/T0dKxWK2az4Xk8OU1Op5P09HT8/Pzw8qoXI1hFRERERERq1PwNyYye8zvH1pGlZBcxes7vzLy5J4M7xxoSm8gRhn0j37t3Lw8++CALFy6s8+FTEyZMYNy4cWXb+/fvp2PHjhW2NZlMxMbGsnPnTnbv3l1XIUotM5vNNG/eHJNJZasiIiIi0vQ4XGZW5XcirTSUKOshevlvxGJyGh2W1BCH08WzX206LiEF4AJMwLNfbeKijjEayieGMiwptWbNGtLS0ujZs3w9UYfDwU8//cSbb77JggULKCkpISsry6NaKjU1lZiYGABiYmJYtWqVR79HVuc70qYi3t7eeHt7l23n5OScMFabzUbbtm01hK8RsdlsqnoTERERkSZpfnYSzx64i+TS8iXyYq3pPB33LoODVxgYWePkcrlwOF3YnS5KHU7sDvfPUqcLu8NJqeOo/U4npXbnCdq629sd7jYlh9vYD7c5cu6+QwUkZxdVHhOQnF3Eqp0HSWodXne/DJFjGJaUuvDCC/nzzz899t122220b9+e8ePHk5CQgNVq5YcffmDo0KEAbNmyhT179pCUlARAUlISzz//PGlpaURFudclXbhwIUFBQZVWPlWX2WzWhNgiIiIiItKgzc9OYvTuJ44f0lUazujdTzAzcUq9S0y5XFDq8sKOhVKnF6V4YXd5UeryotRlwZ6a65GsKUvyOI/sO5zYOZzIsTuclByd2LG727rbeZ57dGLoSKKoPIF0JAnk3i5LEB17rtOJscuLVS4tt/LElUhdMCwpFRgYSOfOnT32+fv7Ex4eXrZ/1KhRjBs3jrCwMIKCgrj//vtJSkqid+/eAFx88cV07NiRW265hZdeeomUlBQmTpzImDFjPCqhREREREREmjqHy8yzB+46nJDyHLLlwgy4+L99Y/A3FeIwmY9K/Hhhd1k83ttdXpS43Mkhu8vi8b683eGkEV6Hk0mWKvZpKTu31GXFwUkWJ9ryUy39xmqPyQRWsxkviwmrxYzVYsLr8LbN4v7pZTZj9TJjNZuOamfGy+x+f+y5Ze8tJlKyi/j09/0njSMqUIUXYqx6PcvztGnTMJvNDB06lOLiYgYNGsRbb71VdtxisfD1118zevRokpKS8Pf3Z+TIkUyaNMnAqEVEREREROqfVfmdPIbsHc9EpiOEW3ZNrrOYqssLO14mO1aTA6tv6HGJmiPb7iRNRYkbd7LnuOSO5UgS6Ki2h7fLkkUVnnv4/bGJpkra1vY8Tg6ni+XbM0nJLqpwXikTEBPsQ6+WYbUah8jJ1Kuk1JIlSzy2fXx8mDFjBjNmzKj0nMTERL799ttajkxERERERKRhSysNrVK7WGs6YZZsvEwOrCY7VpP9mPfuZJDX4e3y9w6sptLjz+PI+aVlbb1MDmyH+3K/d59X3k9F17HjhfuYx3pFw+rp2DgDWcwmnh7SkdFzfscEHompI7+6p4d01CTnYrh6lZQSERERERGR2hFlPVSldq8mvEpSwJ8nbyj12uDOscy8uSfPfrXJY9LzmGAfnh7SkcGdYw2MTsRNSSkREREREZEmoJf/RmKt6SSXRnDsnFIAJpzEWDPp5b+x7oOTWjG4cywXdYxh1c6DpOUWERXoHrKnCimpL5SUEhERERERaQIsJiePRH/Aw/vGHXfMhBMw8XTcu1hMzroPrr6b13CTOBYg6egdKw0KpCZoqGajYzY6ABEREREREakb24sTABNe2D32x1gzmZk4hcHBK4wJTESaJFVKiYiIiIiINAEZ9mDey7gCgLcSpxJoKSCtNJQo6yF6+W9UhZSI1DklpURERERERJqAmWnXUejyoZvvFi4KWum5gp2IiAE0fE9ERERERKSRSykNZ07mJQA8HDNHCSkRqReUlBIREREREWnkZqRdR7HLm17+Gzg3YK3R4YiIAEpKiYiIiIiINGr7SiL56OAgAMZFq0pKROoPJaVEREREREQasTdSb6TUZaVfwFp6B2wwOhwRkTJKSomIiIiIiDRSu4pj+eTQQMBdJSUiUp8oKSUiIiIiItJIvZZ6Ew4sDAhcRU//LUaHIyfx02YY8jLEjQHTcPhitefxZz6F9o+A/+0QeicMnAIrt3m2OZgHw2dA0CgIuRNGvQt5RVW7vssFl7x4/LXfX+reV9ErLfv07lmaNi+jAxAREREREZGat7UogS+y+gMwLnquscFIleQXQ7fmcPv5cM3044+3i4E3b4VWUVBYAtO+g4tfgG2vQmSQu83wGZCcBQsnQKkDbnsH7voXzLvv5NefPp8K5xy7IQkGd/Pcd+s7UFQKUcGneJMiR1FSSkREREREpBGaljocF2YGBy2js992o8ORKriku/tVmWF9PbdfHQ6zlsD6PXBhZ9i8H+avh9+eg7Naudu8MRIu/Qe8PBziQivve90ueOUbWD0ZYsd4HvO1uV9HpOfA4o0w686q35tIRTR8T0REREREpJHZWNiSb7P7YcLJ2Jh5RocjtaDEDu/+CMF+0C3RvW/FVgjxK09IAQzsDGbT8cP8jlZQDMNmwIxbISbk5Nf+4Gfw84ZrzzmdOxBRpZSIiIiIiEijMy3lZgCuCPmJM3x2GxyN1KSvf4cb34SCEogNgYWPQ0Sg+1hK9vHD6bwsEBbgPlaZsXOgTzu48qyqxTBrCQzr41k9JVIdqpQSERERERFpRNYWtGNR7jmYcfBgtKqkGpsLOsK6KbD8aRjcFa5/4/QmG//fGvdQvOm3VK39iq2w+QCM6l/9a4ocoaSUiIiIiIhII/Lq4SqpoaGLaeV9wOBopKb5+0CbGOjdFmbdBV5md+USQEzw8Qkqu8O9Il9MJROSL94E29PcK/V53eJ+AQydDv0nH9/+Xz9C90Q4s2VN3ZE0ZRq+JyIiIiIi0kiszOvEz3k9sZpKeSD6Q6PDkTrgdEGx3f0+qS1kFcCaneVJo8Ub3W3OaVPx+Y8PgTv6e+7r8jhMuxmG9PTcn1cE/1kJU2+o0VuQJkxJKRERERERkUbA5YJXUt1VUteHLiTBlmZwRHKq8opgW0r59s5096p4YQEQHgDPfwlX9HTPJZWRBzMWwv5DcN3hCcc7NHMP6bvzX/D27VBqh/tmw429y1fe238QLpwCH4yGXq3dE5tXNLl58whoGeW57+Nf3ZVXN/c9vr1IdSgpJSIiIiIi0ggsy+vGqvwu2Ewl3Bf1sdHhSDWs3gEXPF++PW6O++fIc91Jpr8OwOyfISPXnaQ6uxX8/CR0ii8/Z+4YuO99d+LJbIKhveD1EeXHSx2wJdm94t6pmrUErjkbQvyrc3cix1NSSkREREREpIFzueDlVPdkQDeHf0usLdPgiKQ6+ncE19zKj3829uR9hAXAvPsqP94i8sTXgMqPL3/m5NcXORWa6FxERERERKSBW5x7NusK2uNrKmJ05CdGhyMiUiVKSomIiIiIiDRgTpeJV1OGAzAy4isirVnGBiQiUkVKSomIiIiIiDRgC3KS2FjUhgBzAXdHfmZ0OCIiVaaklIiIiIiISAPlcJnLqqRuj/iCUK9cgyMSEak6JaVEREREREQaqK+zzmVrcSLBllxGRX5pdDgiIqdESSkREREREZEGyO4yMy3VXSV1V+RnBFvyDY5IROTUKCklIiIiIiLSAH126EJ2lcQRbsni1vCvjA5HROSUKSklIiIiIiLSwJQ4vXgt9UYARkd9gr+lyOCIREROnZJSIiIiIiIiDczHhy5if2k0UV6Z3Bz+rdHhiIhUi5JSIiIiIiIiDUiR08abh6uk7ov6Dz7mEoMjEhGpHiWlREREREREGpC5mYNJtYfTzJrGDWELjA5HRKTalJQSERERERFpIPIdPsxMvw6AB6I/xNtsNzgiEZHqU1JKRERERESkgZideTkZ9lASbQe4JnSx0eGIiJwWJaVEREREREQagByHH++kDwXgoeh5WE0OgyMSETk9SkqJiIiIiIg0AP/OuJJsRyBtvPdwRchPRocjInLalJQSERERERGp57LsAcxKvwqAsdFzsZicxgYkIlIDlJQSERERERGp595Nv4Zcpz8dfHZwSfByo8MREakRSkqJiIiIiIjUYxn2YN7LuAKAh2PmYDa5DI5IRKRmKCklIiIiIiJSj81Mu45Clw/dfLdwYeAqo8MREakxSkqJiIiIiIjUUyml4czJvARwV0mZTAYHJCJSg5SUEhERERERqadmpF1Hscubs/02cm7AWqPDERGpUUpKiYiIiIiI1EP7SiL56OAgAB6O+X+qkhKRRkdJKRERERERkXrojdQbKXVZ6Rewlt4BG4wOR0SkxikpJSIiIiIiUs/sKo7lk0MDARgXPcfgaEREaoeSUiIiIiIiIvXMa6k34cDCgMBV9PTfYnQ4IiK1QkkpERERERGRemRrUQJfZPUHYFz0XGODERGpRUpKiYiIiIiI1CPTUofjwszgoGV09ttudDgiIrVGSSkREREREZF6YmNhS77N7ocJJ2Nj5hkdjohIrVJSSkREREREpJ6YlnIzAFeE/MQZPrsNjkZEpHYpKSUiIiIiIlIPrC1ox6LcczDj4MFoVUmJSOOnpJSIiIiIiEg98OrhKqmhoYtp5X3A4GhERGqfklIiIiIiIiIGW5nXiZ/zemI1lfJA9IdGhyMiUie8jA5ARERERESkKXO54JVUd5XU9aELSbClGRyRiNQXH6zYxTtLd5CeV0yH2CCevaIT3RNCKmw7f0MyM37czq7MfOwOFy0i/Lnz3JZc0zO+wvZPfP4n81bu4cnLOzKqX8tavIvKKSklIiIiIiJioGV53ViV3wWbqYT7oj42OhwRqSe++uMAk7/ezOSrO9MjIYR/L9vJiFkrWfxIfyICvI9rH+xrY8wFbWgT5Y/VYuaHzWk8+sl6wgO8Ob9dpEfb+RtSWLsni+ig4/upSxq+JyIiIiIiYhCXC15OvQWAm8O/JdaWaXBEIlKbcnNzycnJKXsVFxdX2vZfv+zkxl4JXH9WAm2jA3n+qi742iz8Z/XeCtsntQ5ncOcY2kQFkhjuz+39WtI+JpDVuw56tEvJLuKZ/23ktRu742U2Ni2kpJSIiIiIiIhBFueezbqC9viaihgd+YnR4YhILevYsSPBwcFlr6lTp1bYrsTuZMP+bPq2iSjbZzab6Nsmgt93Z530Oi6Xi2XbMtiRnk+vlmFl+51OF2M/Xsdd57WiXXTgad/P6dLwPREREREREQM4XSZeTRkOwMiIr4i0ZhkbkIjUuk2bNtGsWbOybW/viofPHSooweF0HTdMLzLAm+3p+ZX2n1NUSu8pP1Bid2I2m5h8ZWfObVs+dG/m0u14WUzc1rfF6d1IDVFSSkRERERExAALcpLYWNSGAHMBd0d+ZnQ4IlIHAgMDCQoKqrX+A2xefPvAueSX2Fm+LZPnvtlEQpgfSa3D+XNfNu8t28U3D/TDZDLVWgynQkkpERERERGROuZwmcuqpG6P+IJQr1yDIxKR+iTUz4bFbCIjz3POqfS8YiIrmOT8CLPZRIsIfwA6xQWzLS2Pt5ZsI6l1OKt2HSQzv5g+Lywua+9wunj+m038+5edLHt8QO3czAkoKSUiIiIiIlLHvs46l63FiQRbchkV+aXR4YhIPWPzMtO5WTDLt2UwqFMM4J4Pavm2TEb0SaxyP06XixK7E4BrejSj31FzVAGM+PdKru4Rz3Vnxddc8KdASSkREREREZE6ZHeZmZbqrpK6K/Izgi2Vzw8jIk3XHf1a8vB//6BLfAjdE4KZ9csuCkrsXHdmAgDjPl5HdLAP4we3B2DGj9voGh9MYpg/JQ4HP/6Vzudr9zP5qs4AhPrbCPW3eVzDy2wmMtCb1pEBdXtzR65vyFVFRERERESaqM8OXciukjjCLVncGv6V0eGISD01pFscB/NLmLbwb9Jzi+kQF8Ts23sRGegevrc/q9BjbqjCEgdPfrGB5OwifKwWWkf6M+2G7gzpFmfULZyUyeVyuYwOwmj79u0jISGBvXv3Eh9vTMmaiIiIiIg0MPNOfaLgEqcXF2x5h/2l0UyM/Rd3RH5R83E1NcPq4CttNT5rqQV18VnXEuUdKmY2OgAREREREZGm4uNDF7G/NJoor0xuDv/W6HBERAxl6PC9mTNnMnPmTHbt2gVAp06deOqpp7jkkksA6N+/P0uXLvU45+677+btt98u296zZw+jR4/mxx9/JCAggJEjRzJ16lS8vDQyUUREROoR/St7/dCA/5VdGr4ip403U28E4L6o/+BjLjE4IhERYxmauYmPj+eFF16gbdu2uFwuZs+ezZVXXsnatWvp1KkTAHfeeSeTJk0qO8fPz6/svcPh4LLLLiMmJobly5eTnJzMiBEjsFqtTJkypc7vR0REREREpDJzMweTag+nmTWNG8IWGB2OiIjhDE1KDRkyxGP7+eefZ+bMmfz6669lSSk/Pz9iYmIqPP/7779n06ZNLFq0iOjoaLp3785zzz3H+PHjeeaZZ7DZbBWeV1xcTHFxcdl2bm5uDd2RiIiIiIjI8fIdPsxMvw6AB6I/xNtsNzgiERHj1Zs5pRwOBx999BH5+fkkJSWV7Z87dy4RERF07tyZCRMmUFBQUHZsxYoVdOnShejo6LJ9gwYNIicnh40bN1Z6ralTpxIcHFz26tixY+3clIiIiIiICDA783Iy7KEk2g5wTehio8MREakXDJ946c8//yQpKYmioiICAgL4/PPPy5JEw4YNIzExkbi4ONavX8/48ePZsmULn332GQApKSkeCSmgbDslJaXSa06YMIFx48aVbe/fv1+JKRERERERqRU5Dj/eSR8KwEPR87CaHAZHJCJSPxielDrjjDNYt24d2dnZfPLJJ4wcOZKlS5fSsWNH7rrrrrJ2Xbp0ITY2lgsvvJDt27fTunXral/T29sbb2/vsu2cnJzTugcREREREZHK/DvjSrIdgbTx3sMVIT8ZHY6ISL1h+PA9m81GmzZtOPPMM5k6dSrdunXjtddeq7DtOeecA8C2bdsAiImJITU11aPNke3K5qESERERERGpK1n2AGalXwXA2Oi5WExOYwMSEalHDE9KHcvpdHpMQn60devWARAbGwtAUlISf/75J2lpaWVtFi5cSFBQkIbjiYiIiIiI4d5Nv4Zcpz8dfHZwSfByo8MREalXDB2+N2HCBC655BKaN29Obm4u8+bNY8mSJSxYsIDt27czb948Lr30UsLDw1m/fj1jx47lvPPOo2vXrgBcfPHFdOzYkVtuuYWXXnqJlJQUJk6cyJgxYzyG54mIiIiIiNS1DHsw72VcAcC46DmYTS6DIxIRqV8MTUqlpaUxYsQIkpOTCQ4OpmvXrixYsICLLrqIvXv3smjRIqZPn05+fj4JCQkMHTqUiRMnlp1vsVj4+uuvGT16NElJSfj7+zNy5EgmTZpk4F2JiIiI1C8tHoTdGcfvv3cgzLgN7p4FizbAgUMQ4AN92sKLN0H7uIr7K7XDxP/Ct+tgRzoE+8LAzvDCjRAXWt7uildg3W5Iy4FQfxjYyd3v0W1EGrOZaddR6PKhm+/fDAxaZXQ4IiL1jqFJqVmzZlV6LCEhgaVLl560j8TERL799tuaDEtERESkUfntOXAcNY3Nhn1w0VS4zj1dJ2e2hOF9oHkEHMyDZz6Di1+AndPBUsFkDwUl8PsuePJq6NYcDuXDg//PnYRaPbm83QUd4YkrIDYE9h+CR+bBta/B8mdq715F6ouU0nDmZF4CwMMx/w+TyeCARETqIcNX3xMRERGR2hUZ5Ln9wlfQOhrO7+DevmtA+bEWkTD5Oug2AXalu9sdK9gPFk7w3PfmSOj1FOzJcCe3AMZeUn48MRIeHwJXTXNXWln1V6g0cjPSrqPY5c3Zfhs5N2Ct0eGIiNRL9W6icxERERGpPSV2mPML3H4+FVZu5BfBe0uhZSQkhFe93+xCd38hfhUfP5gHc5e5hwYqISWN3b6SSD46OAhQlZSIyInoTwIRERGRJuSL1ZBVALee57n/rYXw2IeQXwxnxLoroWxV/EuxqATGfwg3JUHQMUmp8R/CmwuhoBh6t4GvH6mZ+xCpz95IvZFSl5V+AWvpHbDB6HBEROotVUqJiIiINCGzlsAl3Y6fbHx4X1g7BZZOhHaxcP3r7mTTyZTa4fo3wAXMvO34449eDmufh+8fd89PNeJtcGkBMmnEdhXH8smhgYB7xT0REamcKqVEROqjearzrzeG6duzNB67092r7H320PHHgv3cr7Yx0LsthN4Fn6+Gm/pU3t+RhNTuDFj8xPFVUgARge5Xu1joEAcJD8Cv2yCpbY3dlki98lrqTTiwMCBwFT39txgdjohIvaaklIiIiEgT8d5PEBUMl/U4cTuXy/0qLq28zZGE1NYU+PH/IDzw5Nd3Hs7xnqhfkYZsa1ECX2T1B2Bc9FxjgxERaQCUlBIRERFpApxO9wTmI88FL0v5/h1p8PEKuLgrRAbCvoPu1fl8bXBp9/J27R+BqTfA1We7E1LXvga/73LPEeVwQkqWu11YgHsuqpXb4Lcd0K8dhPrD9jR48r/u1fxUJSWN1bTU4bgwMzhoGZ39thsdjohIvaeklIiIiEgTsGgD7Ml0r7p3NB8r/LwFps+HQ/kQHQzntYflT7urqo7YkgzZBe73+w/B/353v+/+hGd/P/4f9O8Ifjb47Dd4+lP35OmxITC4K0y8CryttXWXIsbZWNiSb7P7YcLJ2Jh5RocjItIgKCklIiIi0gRc3BVcFYwmiguFbx87+flHn9sisuK+jtalOSz+v1OLUaQhm5ZyMwBXhPzEGT67DY5GRKRh0Op7IiIiIiIip2FtQTsW5Z6DGQcPRqtKSkSkqpSUEhEREREROQ2vHq6SGhq6mFbeBwyORkSk4VBSSkREREREpJpW5nXi57yeWE2lPBD9odHhiIg0KEpKiYiIiIiIVIPLBa+kuqukrg9dSIItzeCIREQaFiWlRESauJ82w5CXIW4MmIbDF6s9j7tc8NQnEDsGfG+FgVNga4pnm4N5MHwGBI2CkDth1LuQV3Ti625PhaunQeQ97vOufx1Ss8uPL9nkjqei129aZVtEROqBZXndWJXfBZuphPuiPjY6HBGRBkdJKRGRJi6/GLo1hxm3Vnz8pa/h9QXw9m2wchL4e8OgF6CopLzN8BmwcR8snABfPwI//QV3/esE1yyCi18AE7D4CVj2NJTY3ckxp9Pdpk87SJ7h+bqjP7SMhLNa1dDNi4iIVJPL5eLl1FsAuDn8W2JtmQZHJCLS8HgZHYCIiBjrku7uV0VcLpg+HyZeBVee5d73wWiIvhe+WAM3JsHm/TB/Pfz2XHmy6I2RcOk/4OXh7uXmj7Xsb9iVDmufhyA/977Z90DoXbB4EwzsDDYviAkpP6fUDl/+DvdfDCZTzdy7iIhIdS3+K411Be3xNRUxOvITo8MREWmQVCklIiKV2pkOKVkwsFP5vmA/OKc1rNjq3l6xFUL8PKuXBnYGswlWbqu432K7O7HkbS3f52N1n/PLlorP+d/vkJkLt513WrckIiJy2pxOF68u/BuAkRFfEWnNMjYgEZEGSkkpERGpVEqW+2d0sOf+6ODyYynZEHXMcS8LhAW4j1Wkdxv3MMDxH0FBsXs43yPzwOGE5KyKz5m1BAZ1hfjw6t2LiIhITVmwMYWNB3IIMBdwd+RnRocjItJgKSklIiJ1LjII/vsAfPU7BIyC4DshKx96tnBXSx1rXyYsWA+j+td1pCIiIp4cR1VJ3R7xBaFeuQZHJCLScGlOKRERqdSROZ1SsyH2qLmhUrOhe+LhNsGQdkxFlN3hXpEv5pgKqqNd3BW2T4OMXPAyQ4g/xNwLraKOb/veTxAeCFf0PK3bEREROW1frz/A1rQ8gn2tjIr80uhwREQaNFVKiYhIpVpGuhNTP2ws35dTACu3Q1Jb93ZSW8gqgDU7y9ss3ghOF5zT5uTXiAh0J6QWb4S0nOMTTy4XvLcURvQDq/4pRUREDGR3OJl2uErqrvNaEWzJNzgiEZGGTX/ei4g0cXlFsC2lfHtnOqzb5Z4TqnkEPDQYJn8BbWPcSaonP4G4ELjqTHf7Ds1gcFe481/w9u3uVfLumw039i5feW//Qbhwinvlvl6t3fveWwod4txD+VZshQf/H4wdDGfEeca3eKM7pjsuqOVfhIhITZmnJULrhWGuGu/ys9/3syuzgHB/G7f2aQGf1vglRESaFCWlRESauNU74ILny7fHzXH/HHkuvH8PPHY55BfDXbPcFVH92sH88eBjKz9n7hi473134slsgqG94PUR5cdLHbAl2T2p+RFbkmHCx+5hfi0i4f+uhLGXHB/frCXQpy20jzv+mIiISF0psTt57Qf30rOj+7fG31tfpURETpf+Tyoi0sT17wiuuZUfN5lg0rXuV2XCAmDefZUfbxF5/DVeuNH9OpkT9SsiIlJXPl69l/1ZhUQFenNz70SjwxERaRSUlBIRETGShvnUH7Uw1EdEGoeiUgdvLnZXSd03oA0+VovBEYmINA6a6FxEREREROQE5q7cQ2pOMc1CfLnh7ASjwxERaTSUlBIREREREalEfrGdmUu2AXD/gDZ4e6lKSkSkpigpJSIiIiIiUonZK3aRkVdCYrgfQ8+MNzocEZFGRUkpERERERGRCuQUlfLO0h0APDSwLVaLvj6JiNQk/V9VRERERESkAv/+ZSfZhaW0iQrgim7NjA5HRKTR0ep7IiIiIiKNxP6DMP4j+O4PKCiGNtHw3t1wViv38bwiePwj+GI1ZOZBy0h4YBDcM/DE/f53JTz5X9iVAW2j4cWb4NLu5cdNwys+76Wb4NHLa+TW6lxWQQmzft4JwNiB7bCYtVqqiEhNU1JKRERERKQROJQPfZ+FCzrCd49BZCBsTYFQ//I24+bA4k0w515oEQnf/wn3vgdxoXDFmRX3u/xvuOlNmHoDXN4D5i2Hq16F35+HzocXokue4XnOd3/AqH/C0F61c6914d2fdpBbbKdDbBCXdI4xOhwRkUZJSSkRERERkUbgxa8gIdxdGXVEyyjPNsu3wshzoX9H9/ZdA+CdH2DV9sqTUq/Nh8FdyyuenrsOFv4Jb34Pb49y74sJ8TznyzXu5FirY67fUGTkFfPesl0AjLuoHWZVSYmI1ArNKSUiItJE/LQZhrwMcWPcQ22+WO153OWCpz6B2DHgeysMnOKusjhiVzqMehdaPuQ+3nosPP0JlNhPfN2iEhjzHoTfDQG3w9DpkJrt2WZPBlz2D/C7DaJGw6PzwO44/XsWaUr+twbOagnXveZ+jno8Af9c7NmmT1v43+/uYX4uF/y4Ef5OgYu7VN7vim0wsLPnvkFd3fsrkpoN36yDUeef1u0YauaS7RSWOugWH8zADg00syYi0gAoKSUiItJE5BdDt+Yw49aKj7/0Nby+AN6+DVZOAn9vGPSCO6kE8NcBcLrgndth40sw7WZ4+wd44uMTX3fsHPhqLfz3AVj6JBw4BNdMKz/ucLoTUiV2WP40zL4b3v/JnSATkarbkQ4zf4C2MbBgPIweCA98ALN/Km/zxkjo2Azi7wfbSBj8kvv/Ced1qLzflCyIDvbcFx3s3l+R2T9BoA9cc/Zp3pBBUrKLmPPrbgAevvgMTCZVSYmI1BYN3xMREWkiLunuflXE5YLp82HiVXDlWe59H4yG6HvhizVwYxIM7uZ+HdEqCrYkw8xF8HIlkxxnF8CsJTBvDAzo5N733t3Q4VH4dSv0bgvfr4dN+2HRE+4vut2B5651T9b8zFCw6a8VkSpxOt0Tmk+5wb3dowVs2OtOHo88z73vje/h123wv4chMQJ++gvGvO+eU+rYaqjq+vdSGN4XfGw1019dm/HjNortTs5uEcq5bSOMDkdEpFFTpZSIiIiwM91d9TCwU/m+YD84pzWs2Fr5edkFEBZQ+fE1O6HU4fllt30cNA8vH/qzYht0SfCsxBjUFXIKYeO+at2OSJMUG+Kugjpah2awJ9P9vrDEXdn46nAY0hO6Nof7LoYbesPL31Teb0zI8UNuU7OPn0cK4Oe/3MnqO/pX/z6MtO9QAR/9tgdQlZSISF1QUkpERETKhuGcyhCdbSnuqou7B5y4X5sXhPh77j+638qGBh0dl4icXN927oTQ0f5OdldEAZTa3UniY+fstpjdVVaVSWoDP2z03Ldwg3v/sWYtgTNbQrfEUw6/Xnjjh22UOlz0axNB71bhRocjItLoqSC+sZinf8WpF4a5jI5ARKRO7D/onovmunPgzhMkpUSk7oy9BPo8C1O+hOvPca+o9+6P8O7hFfKC/OD8DvDoh+Brcyerlm6GD36GV28u72fETGgWClNvdG8/OBjOnwyvfAOX9YCPVsDqHeX9HpFTAP9dBa8Mq5v7rWm7MvL55Hd3eea4i9sZHI2ISNOgpJSIiIiUDcNJzYbY0PL9qdnQ/ZiKhwOH4ILn3at4HfultKJ+S+yQle9ZLXX00J+YEPeX56MdGSpU0fAgEanY2a3h84dgwscw6XNoGQnTb3bP73TER/e5jw9/Cw7muRNTz18P91xY3mZPpmc1VZ927nnhJv4XnviPeyL1L8ZB5wTP63/0q3t+upv61Opt1prXftiKw+liQPsoejYPPfkJIiJy2pSUEhEREVpGuhNAP2yE7i3c+3IKYOV29wpeR+w/6E5IndnSPWG5+SQTAZzZEqwWd79De7n3bTng/tJ7ZOhPUht4/gtIy4aow8P2Fm6AIN/j58cRkRO7vKf7VZmYEPezeyJLJh6/77pz3K8TuWuA+9UQbU3N5Yt1+wEYd5GqpERE6oqSUiIiIk1EXpF7HqgjdqbDul3uicqbR8BDg2HyF+4qiJaR8OQnEBcCV53pbr//IPSf7K6seHkYpOeU93Wkomn/Qbhwinvlvl6t3ZOlj+oP4+ZAmL97+ND9syGprXvlPYCLu7qTT7fMhJdugpRsd0XGmIvA21rrvxYREaYt+huXCwZ3iqFzs+CTnyAiIjVCSSkREZEmYvUOd5XTEePmuH+OPBfevwceuxzyi+GuWZBVAP3awfzx5cu6L/wTtqW6X/H3e/btmuv+WepwT7RcUFx+bNrN7qFAQ1+DYjsM6gJv3VZ+3GKGrx+B0e9B0jPg7+2OadK1Nf4rEBE5zsYD2Xz7ZwomE4xVlZSISJ1SUkpERKSJ6N+xPHlUEZPJnQiqLBl06/nu14m0iDz+Gj42mHGb+1WZxEj49rET9y0iUhumLfwbgCu6xXFGTKDB0YiINC0nmQlCRERERESkcVq75xCLNqdhNsGDF7Y1OhwRkSZHSSkREREREWmSXj1cJTW0ZzytIgMMjkZEpOlRUkpERERERJqclTsy+XlrBlaLiQdUJSUiYgjNKSWnpcWDsDvj+P33DnTPHZKSBY/Ocy/tnVsEZ8TC/11Zvix4RaZ+CZ+thr8OgK8N+rSFF2+EM+Lcx3elQ8uHKj73Pw+cfLliEREREWnaXC4Xrxyukrr+rAQSwvwMjkhEpGlSUkpOy2/PgcNZvr1hH1w0tTwxNGKmewWn/z0MEYEwbxlc/zqsngw9WlTc59K/YMxAOLs12B3wxH/g4hdg00vg7wMJ4ZA8w/OcdxfDP76BS7rVym2KiIiISCOybFsmq3YexOZl5r4BbYwOR0SkyVJSSk5LZJDn9gtfQetoOL+De3v5Vph5G/Rq7d6eeDVMmw9rdlaelJo/3nP7/bsharT7nPM6uJcOjwnxbPP5arj+HAjwOd07EhEREZHGzOVy8fL3WwC4+ZxEYoN9DY5IRKTp0pxSUmNK7DDnF7j9fPey4uAeevfxr3AwD5xO+GgFFJVC/w5V7ze7wP0zrJK5J9fshHW7YVT/0wpfRERERJqAxX+lsW5vFr5WC6P7tzY6HBGRJk2VUlJjvljtHqp363nl+/7zANzwBoTfDV4W8LPB5w9Bm5iq9el0wkP/D/q2g84JFbeZtQQ6xEGfdqd7ByIiIiLSmDmdrrIV90b2aUFkoLfBEYmING2qlJIaM2uJe06nuNDyfU9+4k5ULZoAq5+DcZfA9W/An3uq1ueY993zVH10X8XHC0tg3nJVSYmIiIjIyS3YmMLGAzkEeHtx93mtjA5HRKTJU6WU1Ijd6bBoA3z2UPm+7anw5vew4UXoFO/e1y0Rft4CMxbC26NO3Od978PXa+GnJyE+vOI2n6yEgmIYcW5N3IWIiIiINFaOo6qkbu/XklB/m8ERiYiIklJSI977CaKC4bIe5fsKit0/zSbPthYzOF2V9+Vywf2z3ZOXL5kILaMqbztrKVzR8/gJ10VEREREjvb1+gNsTcsj2NfKqH4tjQ5HRETQ8D2pAU4nvLcURp7rnjfqiPZx0CYa7p4Fq7a7K6de+QYWboCrzipvd+EUd0XVEWPehznLYN4YCPSBlCz3q7DE87rbUuCnv+COC2rx5kRERESkwbM7nEw7XCV113mtCPa1GhyRiIiAKqWkBizaAHsy3avuHc3qBd8+Bo9/BENehrxid5Jq9t1waffydttTISO3fHvmIvfP/pM9+3vvLrj1qGv8eynEh8HFXWr0dkRERESkkfns9/3syiwgzN/GrX1aGB2OiIgcpqSUnLaLu4JrbsXH2sbApw+d+Pxdr3luV9bXsabc4H6JiIiIiFSmxO7ktR+2AnBv/9b4e+srkIhIfaHheyIiIiIi0mh9vHov+7MKiQr05ubeiUaHIyIiR1FSSkREREREGqWiUgdvLnZXSd03oA0+VstJzhARkbqkpJSIiIiIiDRKc1fuITWnmGYhvtxwdoLR4YiIyDGUlBIRERERkUYnv9jOzCXbALh/QBu8vVQlJSJS3ygpJSIiIiIijc7sFbvIyCshMdyPoWfGGx2OiIhUQEkpERERERFpVHIcfryzdAcADw1si9Wirz0iIvWR/u8sIiIiIiKNyr8zriS7sJQ2UQFc0a2Z0eGIiEglDE1KzZw5k65duxIUFERQUBBJSUl89913ZceLiooYM2YM4eHhBAQEMHToUFJTUz362LNnD5dddhl+fn5ERUXx6KOPYrfb6/pWRERERESkHsiyBzAr/SoAxg5sh8VsMjYgERGplKFJqfj4eF544QXWrFnD6tWrGTBgAFdeeSUbN24EYOzYsXz11Vf897//ZenSpRw4cIBrrrmm7HyHw8Fll11GSUkJy5cvZ/bs2bz//vs89dRTRt2SiIiIiIgY6N30a8h1+tMhNohLOscYHY6IiJyAl5EXHzJkiMf2888/z8yZM/n111+Jj49n1qxZzJs3jwEDBgDw3nvv0aFDB3799Vd69+7N999/z6ZNm1i0aBHR0dF0796d5557jvHjx/PMM89gs9mMuC0RERERETFAhj2Y9zKuAGDcRe0wq0pKRKReqzdzSjkcDj766CPy8/NJSkpizZo1lJaWMnDgwLI27du3p3nz5qxYsQKAFStW0KVLF6Kjo8vaDBo0iJycnLJqq4oUFxeTk5NT9srNza29GxMRERERkToxM+06Cl0+dPP9m4EdoowOR0RETsLwpNSff/5JQEAA3t7e3HPPPXz++ed07NiRlJQUbDYbISEhHu2jo6NJSUkBICUlxSMhdeT4kWOVmTp1KsHBwWWvjh071uxNiYiIiIhInUopDWdO5iUAPBzz/zCZVCUlIlLfGZ6UOuOMM1i3bh0rV65k9OjRjBw5kk2bNtXqNSdMmEB2dnbZq7avJyIiIiIitWtG2nUUu7w5228j5wasNTocERGpAkPnlAKw2Wy0adMGgDPPPJPffvuN1157jRtuuIGSkhKysrI8qqVSU1OJiXFPWBgTE8OqVas8+juyOt+RNhXx9vbG29u7bDsnJ6embkdEREREROrYvpJIPjo4CDhSJWVwQCIiUiWGV0ody+l0UlxczJlnnonVauWHH34oO7Zlyxb27NlDUlISAElJSfz555+kpaWVtVm4cCFBQUEakiciIiIi0kS8kXojpS4r/QLW0jtgg9HhiIhIFRlaKTVhwgQuueQSmjdvTm5uLvPmzWPJkiUsWLCA4OBgRo0axbhx4wgLCyMoKIj777+fpKQkevfuDcDFF19Mx44dueWWW3jppZdISUlh4sSJjBkzxqMSSkREREREGqddxbF8csi9ONK46DkGRyMiIqfC0KRUWloaI0aMIDk5meDgYLp27cqCBQu46KKLAJg2bRpms5mhQ4dSXFzMoEGDeOutt8rOt1gsfP3114wePZqkpCT8/f0ZOXIkkyZNMuqWRERERESkDr2WehMOLAwIXEVP/y1GhyMiIqfA0KTUrFmzTnjcx8eHGTNmMGPGjErbJCYm8u2339Z0aCIiIiIiUs9tLUrgi6z+AIyLnmtsMCIicsrq3ZxSIiIiIiIiVTEtdTguzAwOWkZnv+1GhyMiIqfI8NX3ROQUzNNSMvXGMJfREYiIiDRpGwtb8m12P0w4GRszz+hwRESkGlQpJSIiIiIiDc60lJsBuCLkJ87w2W1wNCIiUh1KSomIiIiISIOytqAdi3LPwYyDB6NVJSUi0lApKSUiIiIiIg3Kq4erpIaGLqaV9wGDoxERkepSUkpERERERBqMlXmd+DmvJ1ZTKQ9Ef2h0OCIichqUlBIRERERkQbB5YJXUt1VUteHLiTBlmZwRCIicjqUlBIRERERkQZhWV43VuV3wWYq4b6oj40OR0RETpOSUiIiIiIiUu+5XPBy6i0A3Bz+LbG2TIMjEhGR06WklIiIiIiI1HuLc89mXUF7fE1FjI78xOhwRESkBngZHYCINBz7D8L4j+C7P6CgGNpEw3t3w1mt3MdNwys+76Wb4NHLKz42c5H7tSvdvd0pHp66Gi7pXt7m7lmwaAMcOAQBPtCnLbx4E7SPq7FbExERkXrM6TLxaor7D42REV8Rac0yNiAREakRSkqJSJUcyoe+z8IFHeG7xyAyELamQKh/eZvkGZ7nfPcHjPonDO1Veb/xYfDCjdA2xl2WP/tnuPJVWDvFnaACOLMlDO8DzSPgYB488xlc/ALsnA4W1XuKiIg0egtykthY1IYAcwF3R35mdDgiIlJDlJQSkSp58StICHdXRh3RMsqzTUyI5/aXa9xJrFbHtDvakJ6e289f766c+nVbeVLqrgHlx1tEwuTroNsEd3VV6+hTvhURERFpQBwuc1mV1O0RXxDqlWtwRCIiUlNUYyAiVfK/NXBWS7juNYgaDT2egH8urrx9ajZ8sw5GnV/1azic8NEKyC+GpDYVt8kvgveWQstId5JMREREGrevs85la3EiwZZcRkV+aXQ4IiJSg1QpJSJVsiMdZv4A4y6BJ66E33bAAx+AzQtGnnd8+9k/QaAPXHP2yfv+cw8kPQNFpe45oz4fCx3jPdu8tRAe+9CdsDojFhZOcF9bREREGi+7y8y0VHeV1F2RnxFsyTc4IhERqUmqlBKRKnE6oWcLmHID9GjhHlJ35wXw9g8Vt//3UhjeF3xsJ+/7jDhYNwVWToLRF8LIt2HTPs82w/u655laOhHaxcL1r0NRyenelYiIiNRnnx26kF0lcYRZsrk1/CujwxERkRqmpJSIVElsCHRs5rmvQzPYk3l825//gi3JcEf/qvVt84I2Me4JzafeCN2aw2sLPNsE+7knQz+vA3zyIPyVDJ+vrs6diIiISENQ4vTitdQbAbg36r/4W4oMjkhERGqaklIiUiV927kTTUf7OxkSI45vO2uJO8HULbF613K6oLi08uMul/t1ojYiIiLSsH186CL2l0YT5ZXJzeHfGh2OiIjUAiWlRKRKxl7iXhFvypewLQXmLYN3f4QxF3m2yymA/66qvErqwinw5vfl2xM+gp82u1fS+3OPe3vJZvdwPYAdaTD1S1izE/ZkwPK/4brXwdcGl3avjTsVERERoxU5bbx5uErqvqj/4GPWmH0RkcZI0wSLSJWc3Ro+fwgmfAyTPnevfjf95vLk0REf/equYrqpT8X9bE+FjKNWck7LgRFvQ3KWe4he1wRYMB4u6uI+7mOFn7fA9PlwKB+ig+G89rD8aYgKro07FREREaPNzRxMqj2cZtY0bghbcPITRESkQVJSSkSq7PKe7teJ3DXA/arMrtc8t2fddeL+4kLh28eqFp+IiIg0fPkOH2amXwfA/VEf4W22GxyRiIjUFg3fExERERGRemN25uVk2ENJtB1gaFgly/yKiEijoKSUiIiIiIjUCzkOP95JHwrAQ9HzsJocBkckIiK1SUkpERERERGpF/6dcSXZjkDaeO/hipCfjA5HRERqmZJSIiIiIiJiuCx7ALPSrwJgbPRcLCansQGJiEit00TnIiIiInVg86T2RocgQIdhRkcglXk3/Rpynf508NnBJcHLjQ6nSvRc1w96rkUaLlVKiYiIiIiIoTLswbyXcQUA46LnYDa5DI5IRETqgpJSIiIiIiJiqJlp11Ho8qGb798MDFpldDgiIlJHlJQSERERERHDpJSGMyfzEgAejvl/mEwGByQiInVGSSkRERERETHMjLTrKHZ5c7bfRs4NWGt0OCIiUoc00bmIiIiIiBhiX0kkHx0cBKhKSkSkIh+s2MU7S3eQnldMh9ggnr2iE90TQipsO39DMjN+3M6uzHzsDhctIvy589yWXNMzHoBSh5OXv9/Ckr/S2XOwgEAfL/q1iWD8Je2JDvKpw7sqp6SUiIiIiIgY4o3UGyl1WekXsJbeARuMDkdEpF756o8DTP56M5Ov7kyPhBD+vWwnI2atZPEj/YkI8D6ufbCvjTEXtKFNlD9Wi5kfNqfx6CfrCQ/w5vx2kRSWOti4P4f7L2xDh9ggsgtLefarTdwxezVf3d/PgDvU8D0RERERETHAruJYPjk0EHCvuCci0hTk5uaSk5NT9iouLq607b9+2cmNvRK4/qwE2kYH8vxVXfC1WfjP6r0Vtk9qHc7gzjG0iQokMdyf2/u1pH1MIKt3HQQgyMfKnDvO4fKucbSODKBn81AmXdGJP/dnsz+rsFbu92SUlBIRERERkTr3WupNOLAwIHAVPf23GB2OiEid6NixI8HBwWWvqVOnVtiuxO5kw/5s+raJKNtnNpvo2yaC33dnnfQ6LpeLZdsy2JGeT6+WYZW2yy2yYzJBkI8xA+k0fE9ERESkEXk3M5NFebnsKC7Bx2yiu68vD0dG0tLmLvPfX1rCRTt2VHjuq3FxDA4MAuDPwkJezUhnU1ERJqCLj7uf9j4nnnNiXWEhr2Wks76wELPJRHtvb/4Zn4CP2f1voWP27WNzcREHHQ6CzGaS/P15ODKSKC9rzf0SpN7bWpTAF1n9ARgXPdfYYBoAPdcijcemTZto1qxZ2ba39/HD8AAOFZTgcLqOG6YXGeDN9vT8SvvPKSql95QfKLE7MZtNTL6yM+e2jaywbVGpgxfmb+aKbnEE+hjzvCopJSIiItKIrC4o4KaQEDr7+OJwuZiekc4de/fyVctW+JnNxHhZWdq6jcc5/83K4t8HD3KufwAA+U4nd+3bywUBgTyVGI3dBTMyMrhz314Wt26DtZLZqNcVFnLXvr3cGRbOE1HReJngr6Jij9L8Xn5+3BUeToSXF2n2Uv6Rls5D+w8wLzGxtn4lUg9NSx2OCzODg5bR2W+70eHUe3quRRqPwMBAgoKCaq3/AJsX3z5wLvkldpZvy+S5bzaREOZHUutwj3alDif3zfsdlwsmX9W51uI5GSWlRERERBqRdxMSPLanxMTSb/s2NhUVcZafHxaTiUgvzz8BF+XlMjgoEP/DVQ87S4rJdjq5PyKCWKv7X07vjQjnql25HCgtJdFmq/DaL6SlcnNoKHeGl//he6SS44iRYeVDCJpZrdwRHsb9+/dT6nJV+qVYGpeNhS35NrsfJpyMjZlndDgNgp5rkaYn1M+GxWwiI89zzqn0vGIiK5jk/Aiz2USLCH8AOsUFsy0tj7eWbPNISpU6nIyZ+zv7DhXy4Z29DauSAs0pJSIiItKo5TqdAARbLBUe31hUxF/FxQwNDi7b19JmI8Ri4dPsLEpcLoqcTj7NzqaVzUYza8V/uGba7awvKiLMYmHY7t2cu20rI/bsZk1BQaWxZTkcfJ2TQw9fX31xbUKmpdwMwBUhP3GGz26Do2mY9FyLNH42LzOdmwWzfFtG2T6n08XybZn0TAypcj9Ol4sSu7Ns+0hCaldmPnPvOIdQ/4oT0nVFlVIiIiIijZTT5eKFtFR6+vrStpI5Kz7NzqKVzUYPX7+yff5mC7MTmnPf/n28nZkJQKLNxrvxCXhV8iVzX2kp4B4O9GhUFO29ffhfTja379vLly1a0uKoKoxX0tOYd+gQhS4X3Xx8mBmfUGGf0visLWjHotxzMOPgwWhVSVWHnmuRpuOOfi15+L9/0CU+hO4Jwcz6ZRcFJXauO9P9fI37eB3RwT6MH9wegBk/bqNrfDCJYf6UOBz8+Fc6n6/dXzY8r9ThZPSc39l4IJtZI8/G4XKRllsEQIivDZtX3dctKSklIiIi0kg9l5rK1uJi5jSveF6XIqeTb3JyuCc8/Lj9E1OS6enry8uxcTiB9w4eZPS+vfwnsUXZ5MZHc+IC4PqQUK4JDgGgo48Pv+YX8Fl2FuMio8ra3h4axtDgEA6UlvJWZgaPJx9gZrN4TKqqaPRePVwlNTR0Ma28DxgcTcOk51qk6RjSLY6D+SVMW/g36bnFdIgLYvbtvYgMPLzIQVahxzNWWOLgyS82kJxdhI/VQutIf6bd0J0h3eIASMkuYtHmVAAuff1nj2t9eGfv4+adqgtKSomIiIg0QpNTU1ian8cHCc2JqWRozve5uRQ6nVwZFOyx/5ucHA6UlvJh80TMh//YfSkujqStf7M4L49LK5igNdLi/rOy9THz0rTytpFcavfYF+rlRSjQwmajlc3GgB3b+aOoiO6+vtW9XWkAVuZ14ue8nlhNpTwQ/aHR4TRIeq5Fmp6RfVowsk+LCo99fHeSx/Yjg87gkUFnVNpXQpgfu164rCbDO22aU0pERESkEXG5XExOTWFRXh7/TmhOfCWTF4N7iM+AgEDCjpkgudDlxAQcXd9w5I/GI5UTx2pmtRLl5cWu0hKP/btKSoir5Muzuz+3Epez0jbS8Llc8Eqqu0rq+tCFJNjSDI6oYdFzLSKNlZJSIiIiIo3Ic2mpfJWTwz9i4/A3m0m320m32ylyen453F1SwurCQoaGBB/XRx8/f3KcTp5LS2V7cTFbi4v5v5RkvEwmzvFzr+iTWlrKZTt3sL6wEACTycTtoWHMOXSIBbk57C4p4fWMdHaWlJRNtvxHYSFzDx1ic1ER+0tL+TU/n0cPHCDBaqW7j6opGrNled1Yld8Fm6mE+6I+NjqcBkfPtYg0Vhq+JyIiItKIfJSVBcDIvXs89j8fE8PVh+eEAfgsO5toLy/6Hv4yerRW3t681SyetzIzGLZnNyagg48P78YnlC07b8fFzpISio6qhBgRFkaxy8WLaWlkOxyc4e3Dv+ITaH64qsPXbGJRXi5vZqRT6HIR6eVFPz9/Xg2Pw1bBfDbSOLhc8HLqLQAMD/+OWFumwRE1PHquRaSxMrlcroprNZuQffv2kZCQwN69e4mPjzc6nOqZpwkE64Vhtfw46XOuP/RZNx36rJuOWv6sN7fvUKv9S9V0+Gtz7V9Ez7WHH3LOZtSup/E1FbG0/R1EWbPq5sK1/f9v9FzXF3qum5A6eK5rS6PIO9QCpa5FRERERKRWOF0mXk0ZDsDIiK/qLiElIiINgpJSIiIiIiJSKxbkJLGxqA0B5gLujvzM6HBERKSeUVJKRERERERqnMNlLquSuj3iC0K9cg2OSERE6hslpUREREREpMZ9nXUuW4sTCbbkMiryS6PDERGRekir74mIiIiI1KDNk9obHYLhHCYzL154KwTAVX/+zIFPEzhQxzF0GFbHF5RGTc91/aDnuvFRpZSIiIiIiNSoRQlnciAgkqDiPK7c/ovR4YiISD2lpJSIiIiIiNSYUpOFee0vAuCGvxfj6ygxOCIREamvlJQSEREREZEasyCxF2l+YYQVZnPZzuVGhyMiIvWYklIiIiIiIlIjis1efHTGQABu/PsHvJ12gyMSEZH6TEkpERERERGpEd+2TCLTN5iogkMM2r3S6HBERKSeU1JKREREREROW6HFxn/aDgDgpi0LsTkdBkckIiL1nZJSIiIiIiJy2r5q1Zcsn0Bi8zIYuGe10eGIiEgDoKSUiIiIiIiclnwvH/7b9gIAbv7re7xcToMjEhGRhsDL6ABEREREROTUvZuZyaK8XHYUl+BjNtHd15eHIyNpafP2aLeusJDXMtJZX1iI2WSivbc3/4xPwMfs/vfpMfv2sbm4iIMOB0FmM0n+/jwcGUmUl7XC62Y5HLyZkc7y/AKS7aWEWiy0PX8EeTY/EnJSOX/fWgAOlJYyKTWFVQUF+JnNXBkUzNjISLxMptr9xYg0YPXpub4wIJAHIiIItFjK2um5lpqmpJSIiIiISAO0uqCAm0JC6Ozji8PlYnpGOnfs3ctXLVvhd/iL6brCQu7at5c7w8J5IioaLxP8VVTsMVyil58fd4WHE+HlRZq9lH+kpfPQ/gPMS0ys8LrpdjvpdjuPRkXS2ubNdpOVl7oMAuCWvxZgwYXD5WL0vn1EeFmY2zyRdLudCSnJeJlMjI2MrO1fjUiDVV+e6wOlpTybmkK63c70Zs0A9FxLrVBSSkRERESkAXo3IcFje0pMLP22b2NTURFn+fkB8EJaKjeHhnJneHhZu2MrLkaGhZW9b2a1ckd4GPfv30+py4W1guqHtt7evNYsvmx7cYdLwNuf0tQdnLN/PZhgWX4+20uKmZXQhggvLzoA90dE8Gp6OmMiIrCpqkKkQvXluW5us/FgZCTjk5Oxu1x4mUx6rqVWaE4pEREREZFGINfpnscp+PBQm0y7nfVFRYRZLAzbvZtzt21lxJ7drCkoqLSPLIeDr3Ny6OHrW+EX1+Pa2wL4ovW5AJQu/xDb4VP+KCqkrbc3EV7l/wbez8+fPKeTbcXF1b1FkSbHiOf6iDyHkwCzuWxonp5rqQ1KSomIiIiINHBOl4sX0lLp6etLW293xcS+0lIAZmRkcG1IMO/EJ9DRx4fb9+1lV0mJx/mvpKdx5t9b6LNtK8mlpbx5VMXEifyn3QUUe9lwJv/NVRl/l+3PsNuJsHgOygg//EU2w26v9n2KNCVGPdcAh+x2ZmZmcF1wSNk+PddSG5SUEhERERFp4J5LTWVrcTEvx8aV7XPiAuD6kFCuCQ6ho48Pj0dF09Jq47PsLI/zbw8N49MWLflXfAIWk4nHkw/gcrlOeM0MnyC+bpkEQPSq/3JfRETN3pRIE2fEcw2Q53Bwz/59tPb2Zoyea6llmlNKRERERKQBm5yawtL8PD5IaE6MtXxlrcjDFQ2tbTaP9q28bSSXelY1hHp5EQq0sNloZbMxYMd2/igqoruvb6XXndN2AKUWG9YDm3nXkYnVXP7v3RFeXqwvKvJon3m4kuLooT8iUjGjnut8p4O79u3D32zmjbhmHsP99FxLbTC0Umrq1KmcffbZBAYGEhUVxVVXXcWWLVs82vTv3x+TyeTxuueeezza7Nmzh8suuww/Pz+ioqJ49NFHsat8UEREREQaMZfLxeTUFBbl5fHvhObEH/MltZnVSpSXF7tKPYf07CopIc5a8bLwAM7DP0tczkrb7PQOYkGLcwB4atvismXoj+jm48vW4uKyL6wAywsKCDCbaXNMnCJSzsjnOs/h4I69e7GaYEazeLz1XEsdMDSduXTpUsaMGcPZZ5+N3W7niSee4OKLL2bTpk34+/uXtbvzzjuZNGlS2bbf4VUHABwOB5dddhkxMTEsX76c5ORkRowYgdVqZcqUKXV6PyIiIiKNkQMTGyNacdA7kLDiXDpl7MDCyYeASO16Li2Vb3JyeLNZPP5mM+mHvygGms34mM2YTCZuDw3jzcwMzvD2pr23D1/mZLOzpITpccEA/FFYyIaiInr6+hJksbC3pIQ3MjJIsFrp7uOupkgtLeX2fXuZGhNLV19f8hwOHkzoAxYrHVL+IjFtK+mHYwqzWLCYTPT196e1zZvHk5N5ODKSDIed1zPSuSkkFJtZM4iIVMbI5/qOfXspcrp4sVkceU4neYcnWddzLbXJ0KTU/PnzPbbff/99oqKiWLNmDeedd17Zfj8/P2JiYirs4/vvv2fTpk0sWrSI6OhounfvznPPPcf48eN55plnsCljKyIiIlJty2I783bXq8jwDSnbF1GYxT3rv6Bv8gbjAhM+ysoCYOTePR77n4+J4erDkxOPCAuj2OXixbQ0sh0OzvD24V/xCTQ//Deyr9nEorxc3sxIp9DlItLLi35+/rwaHlf2JdOOi50lJRQdrrD42RpISccLMAGLF/6T+Qe2lV17YatWNLPasJhMvBUfz6TUFIbt2Y2v2cyVQcHcr/lpRE7IqOd6U3FR2dC8wTt3eFxbz7XUpno18DM7OxuAsLAwj/1z585lzpw5xMTEMGTIEJ588smyaqkVK1bQpUsXoqOjy9oPGjSI0aNHs3HjRnr06HHcdYqLiyk+asnK3Nzc2rgdERERkQZtWWxnJvcaedz+DJ9gJvcaycRVs5WYMtCmM9pXqd2d4eHcGR5e4bF23j68l9D8hOc3s9o8rrWhx5WYzBZ6pWzi2UATVBJHM6uVd+ITqhSjiLgZ9Vz38vOv0rX1XEtNqzdJKafTyUMPPUTfvn3p3Llz2f5hw4aRmJhIXFwc69evZ/z48WzZsoXPPvsMgJSUFI+EFFC2nZKSUuG1pk6dyrPPPltLdyIiIiLS8Dkw8XbXq9wbR010W7btcvFOlyvpnbxRQ/makN2B0fwY7/5H31s2LzA4GhERaejqTVJqzJgxbNiwgV9++cVj/1133VX2vkuXLsTGxnLhhReyfft2WrduXa1rTZgwgXHjxpVt79+/n44dO1YvcBEREZFGaGNEK48he8cxmUj3C2VOh4s5M+1vIgqzCS/Mxupy1FmMUvfmtL8Yl8lM3wPraZO93+hwRESkgasXSan77ruPr7/+mp9++on4+PgTtj3nHPcqH9u2baN169bExMSwatUqjzapqakAlc5D5e3tjbe3d9l2Tk7O6YQvIiIi0ugc9A6sUruPzriIj864qGw7tCiHiMJsIguziCjMIrIwm4gj74vciSuvE6z+JPXX9uA4fmnWDZPLyc2bvzc6HBERaQQMTUq5XC7uv/9+Pv/8c5YsWULLli1Pes66desAiI2NBSApKYnnn3+etLQ0oqKiAFi4cCFBQUGqfhIRERGpprDiqs252SprP0VeNtJ9Qyi1WDnkE8QhnyC2hlY854jJ5SS0KJeIoiOJq+zDyaussmRWWFGOElf10P9rPwiA8/eto0VuxdNkiIiInApDk1Jjxoxh3rx5fPnllwQGBpbNARUcHIyvry/bt29n3rx5XHrppYSHh7N+/XrGjh3LeeedR9euXQG4+OKL6dixI7fccgsvvfQSKSkpTJw4kTFjxnhUQ4mIiIhI1XXK2EFEYRYZPsHHzykF4HIRWZjF60umY8GFC8i2+ZPhG0yGbwgZviGk+waTftT7DJ8Q7BYvDvoGc9A3mL9DK56I13wkcXVUoiqiKJvIgsPJq6IswopysShxVWf+Cm3OythOmF1Ohv+lKikREakZhialZs6cCUD//v099r/33nvceuut2Gw2Fi1axPTp08nPzychIYGhQ4cyceLEsrYWi4Wvv/6a0aNHk5SUhL+/PyNHjmTSpEl1eSsiIiIijYoFF13St/Nj8zPB5fJMTLncE5vf/eeXZZOcm4CQknxCSvJpk32gwj7diasAd4LKN+Rwwiq47GfG4Z92sxeZvsFk+gazpZL43ImrHCKPGiJ4bOVVaFGuJmGvIUeqpC7cs5r4/AyDoxERkcbC8OF7J5KQkMDSpUtP2k9iYiLffvttTYUlIiIi0uT9HRLPT/HdAQgoLSTP5ld2LLIwi7v//JK+yRtOqU934iqPkJI82lYySbYTE9ne/sckrELI8Cl/n+kbdDhxFUKmbwh/VXI9s9NB+NFzXBVl0+mXncQF+xAT7ENciC8RAd5YzBVUgkmZP8Nb8Xv0GVicDoZtWWR0OCIi0ojUi4nORURERKT+KLTYePGs4TjMFvrt/4Pxv81hU0QrDnoHElacS6eMHbVWgWTGRWhxHqHFebTL2ldhGycmsrwDPCqs0o8MESxLYgXhNFtI9wsl3S+UzYfP/ezrTR59eZlNRAf5EHtUoiomyIe4EB9ig32JDfYhIsAbcxNNXLmADzq4q6QG7V5JTMFBYwMSEZFGRUkpEREREfEws+tVHAiIJLLgEA+s+wQvXHTN2G50WGXMuAgrziWsOJczsvZW2MaBiSyfQM+KK59giofeREp2EclZhaTmFmN3utifVcj+rMJKr3ckcXV0osqdxPIt2xfub2uUiat1kW3ZENEaq6OUG7f8YHQ4IiLSyCgpJSIiIiJlforrxsLEXphcTh5d8yGBpZUna+ozCy7Ci3IIL8qBQ+X7O3z6j7L3DqeL9NxiDmQXkpJdxIGsQpKzi9zvD+9LzSk6JnF16PiLATaLmehgb2KDfPE7c1jZkEH3PFfuea+CS/JoSGkrFzC7w2AALt21gsiibGMDEhGRRkdJKREREREBINU3lNd7XAvADX8vpkvmDoMjql0Ws4mYw8P2KmN3OEnLLSY5u4jksuRVESk5hRzIcu9Lyy2mxOFk78FC9h4shISeFfbl5bATUXT8xOxHElcRhdkEl+TXm8TVqugObAlLxNtewvV//2h0OCIi0ggpKSUiIiIiODDx8pk3kW/15YyDuxn+1/dGh1QveFnMxIX4EhfiC4RW2Kb0cOIqJdudqFr3f5PKVxP0cc91dcgnELvFixT/CFL8Iyq9ntVR6jExe2TB4YRVUXnyKqikoNYSVw5MbIxoRYZ3EHM7XAzAFTt+Iaw4t5auKCIiTVm1k1J79uxh9+7dFBQUEBkZSadOnfD29q7J2ERERESkjvyn3QA2RLTCt7SI8avn4uVyGh1Sg2G1mGkW4kuzEF/OTIQ2245fPbrUZOGgT5DHZOzpR03SnuEbzCGfIEotVpIDIkgOqDxx5W0vIaIom4jDFVdHhgeWDxXMIqC08JQTV8tiO/N216vI8A0p3+lykZCbeoo9iYiIVM0pJaV27drFzJkz+eijj9i3bx8uV/mqKzabjXPPPZe77rqLoUOHYjabazxYqdzmSe2NDkGADsOMjkAaCz3T9Yeea2kKNocmMqe9uypmzPrPiNUKazXO6nIQXXiI6MKK56QCd+Iq0zfYY2L2dD/3zyPJqyyfQIq9bOwPiGR/QGSlfXnbS8qHCJZVWXkOGzw6cbUstjOTe42ssK9Xe96In72YvskbTudXICIicpwqJ6UeeOABZs+ezaBBg5g8eTK9evUiLi4OX19fDh48yIYNG/j555956qmnePbZZ3nvvfc4++yzazN2ERERETlN+V7evHTWMJxmC/33/s6Avb8bHVKTZXU5iCk4SMwJkoIlZguZZUmqiquusr0D3ImrwCj2B0ZV2pePvbgsUbU5LNG903RMfZXJBC4X73S5kt7JG7HgOr4jERGRaqpyUsrf358dO3YQHh5+3LGoqCgGDBjAgAEDePrpp5k/fz579+5VUkpERESknnur2zWk+IcTnX+Q+/74rN5Msi0VszkdxBYcPGE1W4nZy11ddbjK6sjwwPSjhgrmeAdQ5OXNvsAo9p0gcQWAyUS6XygbI1rRNWN7Dd+RiIg0ZVVOSk2dOrXKnQ4ePLhawYiIiIhI3Vkc35PFCWdidjp4bM1c/O1FRockNcDmtBNXkElcQWalbYrNXmQeTlT9FNeNb1v1OWm/B70DazJMERERqjXxU2FhIQUFBWXbu3fvZvr06SxYsKDGAhMRERGR2pPsF8ab3a4BYNiWRXQ8uNvgiKQueTvtxOVn0i1jO+cf+KNK52gFPhERqWnVSkpdeeWVfPDBBwBkZWVxzjnn8Morr3DVVVcxc+bMGg1QRERERGqWw2TmpbOGUWj1oVPGDm78+wejQxIDdcrYQURhFrgqmS/K5SKy4BCdMnbUaVwiItL4VSsp9fvvv3PuuecC8MknnxAdHc3u3bv54IMPeP3112s0QBERERGpWXPPuIi/wlrgX1rIo2s+xOJyGh2SGMiCi3vWf+HeODYxdXj77j+/1CTnIiJS46qVlCooKCAw0D2m/Pvvv+eaa67BbDbTu3dvdu9W6beIiIhIfbUhvCUfn3EhAPet+5TowkMGRyT1Qd/kDUxcNZuIomyP/ZGFWUxcNZu+yRsMikxERBqzKk90frQ2bdrwxRdfcPXVV7NgwQLGjh0LQFpaGkFBQTUaoIiIiIjUjFyrLy+dOQynycxFu3+j//51Rock9Ujf5A30Tt7IxohWHPQOJKw4l04ZO1QhJSIitaZaSamnnnqKYcOGMXbsWC688EKSkpIAd9VUjx49ajRAqV/ezcxkUV4uO4pL8DGb6O7ry8ORkbS0eR/X1uVycff+ffySn8/rcc0YGFi+YsvzqamsLSxga0kJrWw2Pm/RssoxnKjfA6WlTEpNYVVBAX5mM1cGBTM2MhIvkxa4FqmInmmRpsMFvNltKOl+ocTmZXDPn18YHZLUQxZcdM3YbnQYIiLSRFQrKXXttdfSr18/kpOT6datW9n+Cy+8kKuvvrrGgpP6Z3VBATeFhNDZxxeHy8X0jHTu2LuXr1q2ws/sORr0g0OHONHXxmuCQ1hfVMiW4uJTiqGyfh0uF6P37SPCy8Lc5omk2+1MSEnGy2RibGTkKV1DpKnQMy3SdCxsfjY/xXfH4nQwfvVc/Oyn9qyKiIiI1LRqzSkFEBMTQ48ePTAf9aWlV69etG/fvkYCk/rp3YQErg4Ooa23N+19fJgSE0uy3c6moiKPdpuLinj/0EEmx8RW2M//RUczLDSUeKv1lK5/on6X5eezvaSYF2Pj6ODjw3kBAdwfEcGHWYcoqWw1GZEmTs+0SNOw3z+CmV2vAmDE5vmckbXX2IBEREREqGZSKj8/nyeffJI+ffrQpk0bWrVq5fGSpiPX6V6tJ9hiKdtX6HTyaPIBJkZFE+lVrWK8Cp2s3z+KCmnr7U3EUcf6+fmT53Sy7RQrN0SaKj3TIo1PqcnCi2cNp8jLm67p2xi6dYnRIYmIiIgA1Ry+d8cdd7B06VJuueUWYmNjMWlujybJ6XLxQloqPX19aetdPv/MC2lp9PD15cKj5oWpCSfrN8NuJ8Li+Z90+OEvsxl2e43GItIY6ZkWaZz+X4dBbA1NIKCkgEfWfKhJq0VERKTGzd+QzPRFW5n/0HmndF61klLfffcd33zzDX379q3O6dJIPJeaytbiYuY0Tyzbtzgvl5UF+Xx6CpMcV0Vt9Ssi5fRMizQ+6yLa8Enb/gA8tPY/RBZlGxuQiIiINFhzV+7ml60ZWC1mbuvbgh7NQ1m+LYPJ32xmZ0Y+1/Rsdsp9VispFRoaSlhYWHVOlUZicmoKS/Pz+CChOTFHzSGzsqCAvaWl9N76t0f7hw7s50xfX2Yf9WX3VFSl3wgvL9YfMw9O5uFqiogaHHIk0hjpmRZpfHKsfrx85k24TGYG7/qVvskbjA5JREREGqi3lmxj2sK/aR8TxPb0PBZuSuW+AW14f/kubuvbguG9Egn2O7X5ZaGaSannnnuOp556itmzZ+Pn51edLqSBcrlcPJ+WyqK8PN5PaE68zeZx/I6wcK4NDvHYd+WunYyPiuIC/4BqX7cq/Xbz8eWdzEwy7fayIT7LCwoIMJtpc0ycIuKmZ1qkcXIBr/W4jkzfYOJz07j7z/8ZHZKIiIg0YJ+s3sfUa7py7ZnxrNp5kBveXcGa3YdY+mh//GzV/wfjap35yiuvsH37dqKjo2nRogXWY1Zb+v3336sdkNRvz6Wl8k1ODm82i8ffbCb9cNVCoNmMj9lMpJdXhRMWx3pZPb7s7i4pocDpJMPuoNjpYvPhaojW3t7YTCZSS0u5fd9epsbE0tXXt0r99vX3p7XNm8eTk3k4MpIMh53XM9K5KSQUm7naC02KNGp6pkUap+8Sz2F5XBe8nHbGr56Dj6PE6JBERESkAdufVUif1uEA9GoZhtVsZuzAdqeVkIJqJqWuuuqq07qoNFwfZWUBMHLvHo/9z8fEcPUxVQ8n8lRKMr8VFpZtD929C4CFrVrRzGrDjoudJSUUuZxV7tNiMvFWfDyTUlMYtmc3vmYzVwYFc39ERJX7EGlq9EyLND57AqJ4t8uVANy66TvaZB8wOCIRERFp6EocTry9yv9h2GoxEVKN4XrHqlZS6umnnz7tC0vDtOmM9jVyzsnmoWlmtZ30WhUdb2a18k58wqkFKNKE6ZkWaVxKzBZePGs4xV42eqRt4eptPxkdkoiIiDQSryz8G1+rBYBSh4s3Fm8l0MczMfXk5R1PqU/NFCsiIiLSSMzueCk7QpoRVJzHI2s+wozL6JBERESkEejVIowd6Xll2z0TQ9hzsMCjjQnTKfdb5aRUWFgYf//9NxEREYSGhmIyVX6xgwcPnnIgIiIiIlJ9ayLb8Vmb8wEYu/Y/hBXnGhyRiIiINBYf351UK/1WOSk1bdo0AgMDAZg+fXqtBCMiJ7Z50qkPtZLa0WGY0RFIY6Hnuv5oyM91li2AV868EYAhO5bRO2WTwRGJiIhIY3Yw372ISpj/6a2KXeWk1MiRIyt8LyIiIiLGcQHTel7PIZ8gEnNSGLXhK6NDEhERkUYou7CUlxds4ev1B8guLAUg2NfKkG5xPHzxGQT7nvrE56c1p1RaWhppaWk4nZ6rKXXt2vV0uhURERGRKvqqZV9WxXTE6ihl/Oq5eDvtRockIiIijUxWQQnXvLWclJwiruzejDZRAQBsS8vlkzX7WLYtg89G9yX4FFfkq1ZSas2aNYwcOZLNmzfjcnlOoGkymXA4HNXpVkREREROwa7AGP7V+XIARm38mpY5yQZHJCIiIo3Raz9sxWoxs/TRC4gM9PY4NvaidoyYtYrXftjKU0NObfU9c3WCuf3222nXrh3Lly9nx44d7Ny5s+y1Y8eO6nQpIiIiIqeg2OzFi2cNp9Ri5eyUzVyxY5nRIYmIiEgj9f3GVJ64rMNxCSmAqEAfHr+kPQs2ppxyv9WqlNqxYweffvopbdq0qc7pIiIiInKaZnW+nF3BsYQW5TDu94+rsQiziIiISNWk5xbTLjqg0uNnxASSnld8yv1Wq1Lqwgsv5I8//qjOqSIiIiJymlZFd+CrVv0AGPf7x4SU5BkckYiIiDRmof5W9h0qrPT43oOFhNTVROf/+te/GDlyJBs2bKBz585YrZ4XvuKKK6rTrYiIiIicxEHvQF7peQMAV237ibPSthgckYiIiDR257WN5B8LtjBn1DnYvDzrm4rtDl75fgvnt4s85X6rlZRasWIFy5Yt47vvvjvumCY6FxEREakdTky80vNGcrwDaJW9n9s2fWN0SCIiItIEjLu4HUPeWMYFLy/hlqREWkcG4HK52Jaex5wVuylxOJl2Q/dT7rdaSan777+fm2++mSeffJLo6OjqdCEi9dy7mZksystlR3EJPmYT3X19eTgykpY298R2WQ4Hb2akszy/gGR7KaEWCxcGBPJARASBFgsAn2dn8X8pFU9293PrNoR7Hf+/oP2lJczMzGRlQQEZdjtRXl5cHhTE3eER2EzuGVNWFeQz+9Ah/iwsJN/ppLnNxu1hYQwJCq6l34ZIw6dnunH4snU/fo8+A297CY+tnofNqX8IFBERkdoXG+zL5/f2YeIXG3hp/l+4Du83Af3aRjLpik7Ehfiecr/VSkplZmYyduxYJaREGrHVBQXcFBJCZx9fHC4X0zPSuWPvXr5q2Qo/s5l0u510u51HoyJpbfPmQGkpz6amkG63M71ZMwAuCQyin7/nZHj/l5xMsctZ4ZdXgB0lJThd8Ex0DM2tVraWFPN0SgqFThePRUUBsLawkDO8vbkjLIxwixdL8/OYkJxMoNlC/4DKJ98Tacr0TDd824Lj+HenywC4c8NXJOamGhyRiIiINCUJYX7Mvr0X2QWl7MzMB6BFuB8hfrZq91mtpNQ111zDjz/+SOvWrat9YRGp395NSPDYnhITS7/t29hUVMRZfn609fbmtWbxZceb22w8GBnJ+ORk7C4XXiYTPmYzPuby8cYH7XZ+Lchnckxspdc91z+Ac4/60ptgs7EzrISPs7LKvsDeHR7hcc4ttjCW5eezMDdXX2BFKqFnumErslh56azh2M1eJCVv4NJdK4wOSURERJqoYD8r3f1CPPa5XC4y80uICPA+pb6qlZRq164dEyZM4JdffqFLly7HTXT+wAMPVKdbEanHcp1OAIIPD+OpSJ7DSYDZjJep4oXJv8zJxtds5uLAwFO6dp7DSbC58usC5DmdtLKduI2IlNMz3bD8s/MV7A2MJrwwmwfX/oeKPxERERGR2tH+ye9YNn4A4YeTTre9t4oXh3YlKsgHgIy8Es6ZsogdUy87pX6rvfpeQEAAS5cuZenSpR7HTCaTklIijYzT5eKFtFR6+vrS1rvizPchu52ZmRlcFxxSaT+fZmdzWVCQR6XFyewuKWFu1iEejYyqtM13OTn8WVTE09ExVe5XpCnTM92wLIvtzLctkzC5nDy85kOCSwqMDklERESamGK7s2weKYBVOw9SVOr0aOPi1FUrKbVz587qnCYiDdRzqalsLS5mTvPECo/nORzcs38frb29GRMRUWGbdYWF7Cgp4cXYuCpfN7W0lLv27WVQYCDXhYRU2GZlQT4TU5KZFB1T6ZdrEfGkZ7rhyPAJ4rUe1wEwdOtSemRsMzgiERERkYpVp5K7WkkpEWk6JqemsDQ/jw8SmhNzzFBdgHyng7v27cPfbOaNuGZYKxnm80l2Fu29venk41Ol66bZS7l17x56+PrybCXVEr8VFHDvvn2Mj4rmymCt0iVSFXqmGw4HJl4+cxi5Nn/aHtrLiM3zjQ5JREREpEZVud7+hRdeoLCwsEptV65cyTfffFPtoETEeC6Xi8mpKSzKy+PfCc2Jtx2/okKew8Ede/diNcGMZvF4VzKEJ9/pZH5OLkNPMAzoaKmlpYzcs4dOPj48HxOLuYIvxasK8rln314ejozi+koqLkSknJ7phufTtv35I7IN3vYSHls9F6vLYXRIIiIi0kSZ8KyEMplMVPJvl6ekypVSmzZtonnz5lx33XUMGTKEs846i8jISADsdjubNm3il19+Yc6cORw4cIAPPvjg9KMTEcM8l5bKNzk5vNksHv/Dy8UDBB5efSvP4eCOfXspcrp4sVkceU4neYcnTg6zWLAc9X+o+Tk5OHAxJCjouOusLyxkQkoy/45PINpqdX953buHOKuVRyOjOOgo/xIWeXjJ+ZUF+dy7bx83h4ZxUWBgWWxWk4mQE0zaLNKU6ZluWLaEJPBBh8EAjF7/OfH5GQZHJCIiIk2ZC7jg5SWYDv9NmF9i59LXfy77x0aXqzozSp1CUuqDDz7gjz/+4M0332TYsGHk5ORgsVjw9vamoMA94WaPHj244447uPXWW/GpYjm/iNRPH2VlATBy7x6P/c/HxHB1cAibiotYX1QEwOCdOzzaLGzVimbW8iqMT7OzGRgQSFAFXy6LXE52lpRgPzwt3vKCfPaUlrKntJQLdmz3aLvpjPYAfJmdTaHLxT8PZvLPg5llx8/29WV2JXPkiDR1eqYbjkKLjZfOGobDbOHc/X9w8Z7fjA5JREREmrh/XNutVvo1uaqRznI6naxfv57du3dTWFhIREQE3bt3J6KSyVDru3379pGQkMDevXuJj483Opxq2dy+g9EhCNDhr8212r8+5/pDn3XToc+66agvn/WrPa5nYWIvIgsOMePHVwksrdr0CVI1tf05g57r+kKfddOhz7rpqIvPurY09LzDyh2ZnJkYipel6qsuV0W1Jjo3m810796d7t2712gwIiIiIk3ZT3HdWJjYC5PLyaNrPlRCSkREROqFm/75K6v+byARATW7OnLNprhEREREpFpSfUN5vce1ANz49w90ydxxkjNERERE6kb1Zow6OSWlRERERAzmwMQ/zryJfKsvZxzczbC/FhodkoiIiIiHGlhs7zjVGr4nIiIiIjXn4zMuZGNEK3xLixi/ei5eLqfRIYmIiIh4eOS/f2DzOnFt0zu3nHVKfSopJSIiImKgzaGJzD3jIgDu++MzYgsOGhyRiIiIyPH8vb3wsR6/+vLpOK2k1LZt29i+fTvnnXcevr6+uFwuTKbaKOgSERERaXzyvXx48azhOM0W+u/9nQH7fjc6JBEREZEKPXNFp/ox0XlmZiYDBw6kXbt2XHrppSQnJwMwatQoHn744RoNUERERKSxmtHtalL9w4jOP8h9f3xmdDgiIiIiFaqt8qNqJaXGjh2Ll5cXe/bswc/Pr2z/DTfcwPz582ssOBEREZHGanF8T35MOBOz08H41XPxtxcZHZKIiIhIhWpr9b1qDd/7/vvvWbBgAfHx8R7727Zty+7du2skMBEREZHGKtkvjDe7XQPAsC2L6HBIfz+JiIhI/fXhnb0J8bXWeL/VqpTKz8/3qJA64uDBg3h71+z4QhEREZHGxG4y89JZwyi0+vD/2bvv8KjqtI3j96T3BiQhEEJCLyEgSFcBERBFKWJ9pemKLiqCFQsIdlwboqCrUlwFESkWRJEqiPTeCTGhhEAS0ntm3j8ioyEBKZOcZOb7ua5cMmdOhnv2bJjMPb/znBbJR3TnweVGRwIAALig6atjlVNYbL394arDSs8ttN4+k12gnm+vvuTHvaxS6pprrtHs2bOtt00mk8xmsyZPnqzu3btfzkMCAAA4hC+b3KD9QfXlXZirJ7fMkbPFbHQkAACAC1pz8LQKiv76neXDlbFKz/mrlCoyW3TkdNYlP+5lnb43efJkXX/99dq8ebMKCgr01FNPac+ePUpNTdW6desu5yEBAADs3q4aUfqqyfWSpIe3f6OQ3DMGJwIAAPhn586UslhsM2XqslZKtWzZUgcPHlTXrl116623Kjs7WwMHDtS2bdvUoEEDmwQDAACwJ5munnqz7V0ym5x0Q/wmdTu+3ehIAAAAhrqslVKS5O/vr+eee86WWQAAAOySxWLR1JhBOu0VqNpZyXpw1yKjIwEAAFw0059fpbadu+EyXHYplZeXp507d+rUqVMym0vPQrjllluuOBgAAIC9+HrLMa2p21rO5mI9vfkLeRXlGx0JAADgolkkPfH1Drm5lJxwl19k1rMLd8nLzVmSSs2buhSXVUotXbpUQ4YMUXJycpn7TCaTiouLy/kuAAAAxxOXnK0Xv90jSRqyb6mapB01OBEAAMClGdimbqmVUf3b1Cm7z1V1L/lxL6uUeuSRRzR48GCNHz9eISEhl/MQAAAAdq+gyKzRc7cpp6BYrU4f1qBDq4yOBAAAcMneuj2mQh73skqppKQkjR07lkIKAADgAt5edlA7j6XL39NVT2yZI+cy164BAACo+p78esc/7mMySZNvu7Ty6rKuvnfbbbdp1apVl/Otpbz22mu6+uqr5evrq+DgYPXv318HDhwotU9eXp5GjRqlGjVqyMfHR4MGDVJSUlKpfRISEnTTTTfJy8tLwcHBevLJJ1VUVHTF+QAAAC7Xb4eT9dGaWEnSG4OiVSsv3eBEAAAAl2f+1mNafyRFGXmFSs8t/ystp/CSH/eyVkpNnTpVgwcP1q+//qro6Gi5urqWuv/RRx+9qMdZvXq1Ro0apauvvlpFRUV69tln1atXL+3du1fe3t6SpDFjxuiHH37Q119/LX9/fz388MMaOHCg1q1bJ0kqLi7WTTfdpNDQUP32229KTEzUkCFD5OrqqldfffVynh4AAMAVOZNdoDHztstike5qH64+LWtrn9GhAAAALtP/dYjQtztO6Ghqrga3q6sBbeoowMvtih/3skqpOXPm6Oeff5aHh4dWrVol09+mXZlMposupZYuXVrq9syZMxUcHKwtW7bo2muvVXp6uj799FN9+eWX6tGjhyRpxowZatasmX7//Xd17NhRP//8s/bu3atffvlFISEhat26tV566SU9/fTTevHFF+XmVvZ/pPz8fOXn/3XVm8zMzMv5nwEAAKAMi8Wip7/ZqaSMfEXV8tYLNzc3OhIAAMAVeal/Sz1/czMt3X1SX28+pslLD6hH02DdfnW4rm1Us1QvdCku6/S95557ThMnTlR6err++OMPxcXFWb+OHDlyWUEkKT29ZFl7UFCQJGnLli0qLCxUz549rfs0bdpU9erV0/r16yVJ69evV3R0dKn5Vr1791ZGRob27NlT7t/z2muvyd/f3/rVvDm/LAIAANv4cmOCft6bJFdnk6bc2UZebpf1GSAAAECV4u7irFtb19H/7u+gZWOvVaMQH72waLe6vrFS2fmXN0LpskqpgoIC3XHHHXJyuqxvL5fZbNZjjz2mLl26qGXLlpKkkydPys3NTQEBAaX2DQkJ0cmTJ637nDtw/ezts/uca9y4cUpPT7d+7d2712bPAwAAOK7DpzL10vclv1c81bupWtbxNzgRAACA7TmZTDLJJIssKjZf/oVcLuuju6FDh+qrr77Ss88+e9l/8blGjRql3bt3a+3atTZ7zPNxd3eXu7u79XZGRkaF/50AAMC+5RcV65E525VXaNY1jWrqvq6RRkcCAACwmfyiYuvpe5v+SNX1zYI16ZaWuq5xLTk5Xd7pe5dVShUXF2vy5Mn66aef1KpVqzKDzt9+++1LeryHH35Y33//vdasWaO6detat4eGhqqgoEBpaWmlVkslJSUpNDTUus/GjRtLPd7Zq/Od3QcAAKCiTV56QPsSMxTk7aa3Bsdc9i9nAAAAVc3zi3bpux2Jqu3vodvbhWvKXW0U5G3QoPNdu3apTZs2kqTdu3eXuu9ShltZLBY98sgjWrhwoVatWqXIyNKfKLZt21aurq5avny5Bg0aJEk6cOCAEhIS1KlTJ0lSp06d9Morr+jUqVMKDg6WJC1btkx+fn7MigIAAJVi9cHT+nRtnCTpzdtaKdjPw+BEAAAAtvPFhgSF+XuqXpCXNsSlaENcSrn7fXRvu0t63MsqpVauXHk531bGqFGj9OWXX2rx4sXy9fW1zoDy9/eXp6en/P39dd9992ns2LEKCgqSn5+fHnnkEXXq1EkdO3aUJPXq1UvNmzfXvffeq8mTJ+vkyZN6/vnnNWrUqFKn6AEAAFSE5Kx8PT5vhyRpSKcIXd8s5B++AwAAoHoZ2KauLvMCexdk6OVgpk2bJknq1q1bqe0zZszQsGHDJEnvvPOOnJycNGjQIOXn56t379768MMPrfs6Ozvr+++/10MPPaROnTrJ29tbQ4cO1aRJkyrraQAAAAdlsVj05Nc7lJyVr8YhPnq2bzOjIwEAANjcW7fHVMjjXnQpNXDgQM2cOVN+fn4aOHDgBfddsGDBRT2mxfLPE9o9PDz0wQcf6IMPPjjvPhEREVqyZMlF/Z0AAAC2Mnt9vFYeOC03FydNuauNPFydjY4EAABQbVx0KeXv72+dF+Xvz+WNAQCAY9t/MkOvLNknSXr2xqZqGupncCIAAIDq5aJLqRkzZmjSpEl64oknNGPGjIrMBAAAUKXlFRbr0TnbVFBkVvcmtTS0c32jIwEAAFQ7Tpey88SJE5WVlVVRWQAAAKqF15bs08GkLNX0cdebg2Mu6erDAAAAKHFJpdTFzIACAACwZ8v3JWnW+nhJ0n8Gt1JNH672CwAAcDkuqZSSxCeBAADAYZ3KyNOT83dKku7rGqluTYINTgQAAFB9XfRMqbMaN278j8VUamrqZQcCAACoisxmix7/eodSswvUrLafnurTxOhIAADAzs1e/4c+Wn1Ep7Py1ay2nybe0kKtwwPK3Xfp7kR9sDJWf6Rkq6jYovo1vfWvayI18Kq61n0sFoveWXZQczYdVUZuodrVD9TL/aMVWdO7kp5RaZdcSk2cOJGr7wEAAIfz2bo4/XooWR6uTppyZ2u5uzgbHQkAANix73ac0Mvf79PLA1qqTXiAPlsXpyGfbtCKJ7qVOz7A39NNo7o3VMNgb7k6O2n5vlN6cv5O1fBx13WNa0mSpq8+ohm//aG3BscoPMhLb/18UEM+26BlY66Th2vl/25zyaXUnXfeqeBglqoDAADHsft4ut5Yul+S9MLNzdUoxNfgRAAAwN59sjZOd7YP1+3twiVJr/SP1or9pzRv81H9u1vDMvt3alCj1O0RXSP1zdZj2vxHqq5rXEsWi0WfrYvTIz0aqleLUEnS23fEqN3Lv+jnvUm6JSas4p/UOS5pphTzpAAAgKPJKSjS6LnbVFhs0Q3NQ3R3+3pGRwIAANVUZmamMjIyrF/5+fnl7ldQZNbu4+nq0rCmdZuTk0ldGtbU1vi0f/x7LBaL1h1O1pHT2WofGSRJOpqaq9OZ+aUe08/DVa3DA7Q1/syVPbHLdEkrpbj6HgAAcDQvfb9PsaezFeLnrjcGteJDOgAAcNmaN29e6vaECRP04osvltnvTE6Bis2WMqfp1fJxV+zp7PM+fkZeoTq+ulwFRWY5OZn08q0tdU2jklP3TmflWR/j3Mc8nVV+OVbRLqmUMpvNFZUDAACgylm6+6TmbEyQySS9fXtrBXm7GR0JAABUY3v37lWdOnWst93dy86GuhI+bi5a8ug1yi4o0m+HU/TSD3sVHuRV5tS+quKSZ0oBAAA4gsT0XD2zYKck6YFro0otdQcAALgcvr6+8vPz+8f9Ar3c5OxkUvI5K5hOZ+WXWen0d05OJtX/80p6LcL8dfhUlj5cdVidGtRQLR8P62ME+3mUeszmtf85U0W4pJlSAAAAjqDYbNHYr3YoLadQ0XX89fgNTYyOBAAAHIibi5Na1vHXb4eTrdvMZot+O5yiqyICLvpxzBaLCopKznoLD/JULV93/XY4xXp/Zl6hth9N01URgTbLfilYKQUAAHCOj9bEav2RFHm5Oeu9O1vLzYXP8QAAQOW6v2ukHv96h6LrBqh1uL8+XfuHcgqKNLhtydX4xn61XSH+Hnq6T1NJ0gcrD6tVXX9FBHmroLhYK/ef1sJtx/Vy/5aSSi5eN6JLpN5fcUj1a3orPMhTb/18UCF+7urVPMSQ50gpBQAA8Dc7jqbp7Z8PSpJe7NdCUbV8DE4EAAAcUb+YMKVmF+idZQd1OjNfzcL8NGtEe9XyLTl973habqkLsOQWFOuFRbuVmJ4nD1dnNajlrXfuaK1+MWHWfR68Lkq5BUUat2CXMvIKdXX9QM0a3l4ers6V/vwkSikAAACr7PwijZ67TUVmi26Krq3B7eoaHQkAADiwoZ3ra2jn+uXe99XITqVuP9G7iZ7ofeGRAyaTSWN7NdHYXlVjNAFr0QEAAP704rd79EdKjsL8PfTqgOhSnz4CAADAtiilAAAAJH2344S+3nJMJpP0zh2t5e/lanQkAAAAu0YpBQAAHF6SZ6CeXbhLkvRw94bqEFXD4EQAAAD2j1IKAAA4tGKZ9Gbbu5SZV6TW4QF69PpGRkcCAABwCJRSAADAoX3V5HrtqRklH3cXTbmzjVyd+fUIAACgMvBbFwAAcFj7AiP0RZMbJEkv9W+hejW8DE4EAADgOCilAACAQ8p28dAb7e6R2clZ3Y5u1YA2dY2OBAAA4FAopQAAgEP6IGaAkryDFJKdqod3LDA6DgAAgMOhlAIAAA5nRd2rtDK8rZzMxXp68xfyLsozOhIAAIDDoZQCAAAOJdErSFNjBkqS7j7wi5qdiTc4EQAAgGOilAIAAA6jyOSkN9rdo1xXD7VIPqI7Dy43OhIAAIDDopQCAAAO48umN+hAUIS8C3P15JY5craYjY4EAADgsCilAACAQ9hVI0pfNb5ekvTI9vkKyT1jcCIAAADHRikFAADsXqarp95se5fMJifdEL9J1x3fYXQkAAAAh0cpBQAA7JpF0vutb9Npr0CFZZ3Wg7sWGR0JAAAAopQCAAB2blm9q/VrnRg5m4v11OYv5VWUb3QkAAAAiFIKAADYsWPeNTWtVX9J0pB9S9Uk7aixgQAAAGBFKQUAAOxSoclZk9vdozwXd8WcPqxBh1YZHQkAAAB/QykFAADs0ufNeutQYLh8CnL0+JY5cpbF6EgAAAD4G0opAABgd7bVbKj5jbpJkh7bNk+18tKNDQQAAIAyKKUAAIBdyXD10ltt75LF5KQb49arS+JuoyMBAACgHJRSAADAblgkvdtmsFI8/VU385Qe2P2d0ZEAAABwHpRSAADAbvxYv6PWh0XLxVykpzf/Tx7FBUZHAgAAwHlQSgEAALuQ4BOsj1veIkkavmeJGqafMDgRAAAALoRSCgAAVHsFTs564+p7lO/ipquSDqh/7K9GRwIAAMA/oJQCAADV3szmfXXEv4788rP0+Na5cpLF6EgAAAD4B5RSAACgWtsc3EQLG14nSRq79SsF5WcanAgAAAAXg1IKAABUW2luPnr7qjskSf2OrFOHpH0GJwIAAMDFopQCAADVkkXS21fdoTMeforIOKn7dn9ndCQAAABcAkopAABQLX0b1UWbQpvJtbhQT2/+n9zNRUZHAgAAwCWglAIAANVOnF+oPm1xsyTpvj3fKzLjpMGJAAAAcKkopQAAQLWS7+SiN9r9nwqdXXX1yX265cg6oyMBAADgMlBKAQCAauXTlv0U7xeqwLwMjd36lUxGBwIAAMBloZQCAADVxoaQZvouqosk6fGtcxVQkGVwIgAAAFwuSikAAFAtpLr76u2r7pAkDTi8Wm1PHTQ4EQAAAK4EpRQAAKjyzDLpP23vVIa7j6LSj2vY3iVGRwIAAMAVopQCAABV3qIG12hbcBO5FxXo6U1fyM1cbHQkAAAAXCFKKQAAUKUd9g/TjBZ9JUkP7P5W9bJOGZwIAAAAtkApBQAAqqw8Zze90e7/VOTkok6Ju3XjH78bHQkAAAA2QikFAACqrI9b9tMx32DVyE3X6G3zZDI6EAAAAGyGUgoAAFRJ62q31I+RnWSymPX4ljnyL8gxOhIAAABsiFIKAABUOac9/PVum9slSYMOrVab5MMGJwIAAICtUUoBAIAqpVgmvdX2LmW5eanRmaMasm+p0ZEAAABQASilAABAlfJNo27aUauhPIry9dTmL+RqKTY6EgAAACoApRQAAKgyDgSEa3azPpKkB3cuUt3sZIMTAQAAoKJQSgEAgCohx8Vdk9vdrWInZ11zfId6JWwyOhIAAAAqEKUUAACoEqZH99cJn1qqlXNGj2yfL5PRgQAAAFChKKUAAIDhVteJ0bKIq+VkMevJLXPkW5hrdCQAAABUMBejAwAA7EuxTNpTM0qp7r4Kys9Ui+QjcpbF6FiowpI8A/V+69skSXccXK7olCMGJwIAAEBlMHSl1Jo1a9SvXz+FhYXJZDJp0aJFpe4fNmyYTCZTqa8+ffqU2ic1NVX33HOP/Pz8FBAQoPvuu09ZWVmV+CwAAGetq91Sw3o/p6e7PqQ3rv4/Pd31IQ3r/ZzW1W5pdDRUUcUmJ01ud7eyXT3VNPUP3b1/mdGRAAAAUEkMLaWys7MVExOjDz744Lz79OnTR4mJidavOXPmlLr/nnvu0Z49e7Rs2TJ9//33WrNmjR544IGKjg4AOMe62i31cvuhSvbwL7U92cNfL7cfSjGFcs1tfL321oiUZ2Gentr8pVwsZqMjAQAAoJIYevrejTfeqBtvvPGC+7i7uys0NLTc+/bt26elS5dq06ZNateunSTp/fffV9++ffWf//xHYWFhNs8MACirWCZNb9W/5IbpnPHUJpNkseij6FvVMXEPp/LBam9QhL5s0lOS9PCOBaqdk2pwIgAAAFSmKj/ofNWqVQoODlaTJk300EMPKSUlxXrf+vXrFRAQYC2kJKlnz55ycnLShg0bzvuY+fn5ysjIsH5lZmZW6HMAAHu3p2aUkj0DyhZSZ5lMOu0VqF/qtVOBk3OlZkPVlO3ioclt75HZyVndj25Rj2NbjY4EAACASlalB5336dNHAwcOVGRkpGJjY/Xss8/qxhtv1Pr16+Xs7KyTJ08qODi41Pe4uLgoKChIJ0+ePO/jvvbaa5o4cWJFxwcAh5Hq7ntR+7171R2a0mawamclKyIzSfX+/IrIOKm6WaflZi6q4KSoKqbGDFSSd5BCslM1asdCo+MAAADAAFW6lLrzzjutf46OjlarVq3UoEEDrVq1Stdff/1lP+64ceM0duxY6+3jx4+refPmV5QVABxZUP7FrTj1KMpXnou7jvsG67hvsH5TtPU+J4tZtbNTSoqqjCTVyzypeplJCs88JXfKKruyou5VWhV+lZzMxXp68xfyLsozOhIAAAAMUKVLqXNFRUWpZs2aOnz4sK6//nqFhobq1KlTpfYpKipSamrqeedQSSVzqtzd3a23MzIyKiwzADiCFslHVDM3rWTIeXmn8FksqpWbps9+flXpHr5K8A1Rgm+I4s/+1y9UWW5eOu5TS8d9amn934aimyxmhWan/rWqKvOk6mUkKTzrlDyKCyvxWcIWTnjV0NSYgZKkew4sU7Mz8QYnAgAAgFGqVSl17NgxpaSkqHbt2pKkTp06KS0tTVu2bFHbtm0lSStWrJDZbFaHDh2MjAoADsVZFo3Y/YMmt7u77J2WksHmI3ctlossqpGXoRp5GWpz+tBfu0g64+77Z0EVYi2tEnxDleHurUSfmkr0qakNtVtYv8dkMSsk58zfVlb99eVZXFDRTxmXocjkpMnt7lauq4daJB/RHQeWGx0JAAAABjK0lMrKytLhw4ett+Pi4rR9+3YFBQUpKChIEydO1KBBgxQaGqrY2Fg99dRTatiwoXr37i1Jatasmfr06aN//etfmj59ugoLC/Xwww/rzjvv5Mp7AFDJDgfUkUwmOZnNMjv9dR2NWrlpGrlrsbok7j7v95pUcgpgUH6mWif/9bpgkZTu5lOqqIr3C1WCb4jS3X100ruGTnrX0MbQ0qdgB+ekKiKj9Myq8KxT8irKt/XTxiX4sukNOhAUIe/CXD25ZQ5XYgQAAHBwhpZSmzdvVvfu3a23z855Gjp0qKZNm6adO3dq1qxZSktLU1hYmHr16qWXXnqp1Kl3X3zxhR5++GFdf/31cnJy0qBBgzRlypRKfy4A4MgO+4dpUYNrJEkvbJghr+ICpbr7Kig/Uy2Sj1x2+WCSFFCQpYDkLMUkx5a6L83N+68VVX4lq6oSfIN1xsNPp7yCdMorSJtCm5X6nlo5Z/4asJ5x8s/S6hQzjSrBzhpRmtu4ZB7ko9vmKyT3jMGJAAAAYDRDS6lu3brJYjn/G5WffvrpHx8jKChIX375pS1jAQAuQbFMmtJ6sMxOzrr22HZ1TNpXKX9vQEG2AlKOqFXKkVLbM1y9/iypSs+sOuPhp9NegTrtFajNIU1LfU/NnLS/5lX97TRAn0LKKlvIdPXUm23vlsXkpBviN+naEzuMjgQAAIAqoFrNlAIAVD3fRXXRocBweRfmauSuxUbHkV9hjlqmxKllSlyp7Zmunn8VVX5/zaxK8fRXsleAkr0CtDWkSanvqZGbrmafbFCjEB81CvZV4z//6+/lWplPqVqzSJrS+jYlewUoLOu0Hty1yOhIAAAAqCIopQAAl+2UZ4BmNb9RkjRizw8Kys80ONH5+RbmqkXqH2qR+kep7VmuHjrq89eA9XjfkplVyV4BSvH019rDyVp7OLnU99TydbcWVA2DfdQ4xFeNgn0U6O1Wic+oevi53tVaWydGzuZiPbX5S+Z6AQAAwIpSCgBwWSySPmw1QHku7mqeEqc+f2wwOtJl8SnMU7Mz8Wp2Jr7U9mwXdyX4hqhoyic6dCpTB5OydPhUlo6n5ep0Zr5OZ+Zr3eGUUt9T08dNjYJ9S1ZW/VlUNQr2UQ0fdzmiY941Na3VAEnSkH1L1STtqMGJAAAAUJVQSgEALsu6sGhtqN1CLuYiPbp9vpzs7Epq3kX5anYmQc2uDi+1PSu/SIdPZelQUqYO/e2/x87kKjmrQMlZKVp/pHRZVcPbTQ2DfdQopGRV1dnVVTXseGVVoclZk9vdo3wXN8WcPqxBh1YZHQkAAABVDKUUAOCSZbt4aFp0f0nS4EMrFZGZZGygSuTj7qLW4QFqHR5Qant2fpFiT2fpYFKWDp3K1OGkLB08lamjqblKyS5QSlyqNsSllvqeQC9X1en6b9U7O2A9I0kRmUkKzM+UqRKfU0WY3ayPDgWGy7cgW49vmXPZV2AEAACA/aKUAgBcspnNb1Sqp7/qZJ3WnQeWGx2nSvB2d1GrugFqVTeg1PacgiIdOZ2tg+esrEpIzdGZnEKdqRml3TWjSn2PT0GOIjL+uhJgxJ//DcrLqBZl1baaDTW/cXdJ0uhtX6tWXrrBiQAAAFAVUUoBAC7JvsAI/RDZSZL08PZv5GYuMjhR1ebl5qKWdfzVso5/qe15hcU6fCpLq4c+VHIlQL+SAeuJ3jWU5ealPTWjtKecsupsUXV2ZVW9zCTVzEuvMmVVupuX3mp7lyTpxrj16pK42+BEAAAAqKoopQAAF63I5KQprW+TxeSkngmb1Dr5sNGRqi0PV2e1rOMv52PbSm0vcHLRMZ9aJVcC/LOoSvAN0Yk/y6q9NSK1t0Zkqe/xKsz9a1VVxtnVVSdVM7dyyyqLpPfa3K4UT3+FZybpgd3fVeLfDgAAgOqGUgoAcNG+adhNf/jXll9+lu6ncKgQbuYiRWUkKiojUTr+1/YCJ2cd96mleN+/iqoEvxAd966pHFdP7Q+qr/1B9Us9lmdhnuplnrLOrDpbWNXKTbPJYPpimbSnZpRS3X0VlJ+peJ9gra/dUi7mIj21+Qt5FBdc8d8BAAAA+0UpBQC4KCe8aujLpjdIkh7Y/a38C3IMTuRY3MzFisw4qciMk6W2F5qcdcKnZklZ5VdSVsX7hui4Ty3lunroQFA9HQiqV+p7PIryS50CeHZmVXDOmYsuq9bVbqnprfor2TPgr42Wku8dvmeJGqafuKLnCwAAAPtHKQUA+EcWSVNbD1KBs6tanzqoHke3Gh0Jf3K1FCviz2Ho+lsPVGRy0vE/TwP8+9cx31rKc3HXwcB6OhhYuqxyLyooPbPqz9IqNCe1VFm1rnZLvdx+aNkwJpNksahWzpmKeroAAACwI5RSAIB/tLLuVdoW3FhuxYV6ZMc3VWaoNs7PxWL+q6z6myKTkxK9a5SZWXXUJ1j5Lm46FBiuQ4Hhpb7HvahAdbNOqV5mksIzkrSo4XUld5jK/3/Cf6NvUefE3XK2wSmCAAAAsF+UUgCAC0p389JH0bdIku46sExh2SkGJ8KVcLGYFZ51WuFZp0tdGa/Y5KREr6BSRVWC319lVWxAXcUG1P3nv8Bk0mmvQO2pGaVWybEV+EwAAABQ3VFKAQAu6NMW/ZTh7qP66YkadGi10XFQQZwtZtXNTlbd7OTSZZVMSvIOsg5Y3xjaVHtrRP3j46W6+1ZkXAAAANgBJ6MDAACqrh01G2hZxNUyWcx6dPt8uVqKjY6ESuYsi8KyU9Tp5B7dcWiFhu776aK+Lyg/s4KTAQAAoLqjlAIAlKvAyUXvt75NktQ37nc1OxNvcCJUBS2Sj6hmbpr1Sntl/DnovEXykUrNBQAAgOqHUgoAUK65ja/XcZ9aCspN1/C9S4yOgyrCWRY9uHNRyY1zi6k/b4/ctZgh5wAAAPhHlFIAgDLifUP0dePukqQHdy2Wd1GewYlQlXRJ3K3nN85Szbz0Uttr5abp+Y2zSs2kAgAAAM6HQecAgFLMMmlK69tU5OSiDol71PXETqMjoQrqkrhbHRP3aE/NKKW6+yooP1Mtko+wQgoAAAAXjVIKAFDK0oj22lsjUh5F+fr3zoUyGR0IVZazLGqVHGt0DAAAAFRTnL4HALBKdffVZy1vliQN2bdUwblpxgYCAAAAYLcopQAAVh9F36JsV081OnNUt8SuNToOAAAAADtGKQUAkCRtDGmqNXXbyMli1qPb5zMbCAAAAECFopQCACjP2U0fxAyUJPU/vEYN048bnAgAAACAvaOUAgDo86a9dMorSME5qbp3/89GxwEAAADgACilAMDBHfYP06IG10iSRu1YII/iAoMTAQAAAHAElFIA4MCKZdKU1oNldnLWtce2q33SfqMjAQAAAHAQlFIA4MC+bdBVhwLD5V2Yq5G7FhsdBwAAAIADoZQCAAd1yjNAs5v1kSSN2P29gvIzDU4EAAAAwJFQSgGAA7JI+rDVAOW5uKt5Spz6xG80OhIAAAAAB0MpBQAOaF1YtDbUbiEXc5Ee3T5fTrIYHQkAAACAg6GUAgAHk+3ioWnR/SVJgw+tVERmkrGBAAAAADgkSikAcDAzmvdVqqe/6mSd1p0HlhsdBwAAAICDopQCAAeyLzBCSyI7SpIe2T5fbuYigxMBAAAAcFSUUgDgIIpMTprS+jZZTE66IX6TYpJjjY4EAAAAwIFRSgGAg/imYTf94V9bfvlZun/3d0bHAQAAAODgKKUAwAGc8K6hL5veIEl6YPe38ivMMTgRAAAAAEdHKQUAds4i6f2YQSpwdlXrUwfV4+hWoyMBAAAAAKUUANi7FeFXaXtwY7kVF+qRHd/IZHQgAAAAABClFADYtXQ3L33c8hZJ0t37lyksO8XgRAAAAABQglIKAOzYpy36KcPdR/XTEzXo8Cqj4wAAAACAFaUUANip7TUbalnE1TJZzHp0+3y5WMxGRwIAAAAAK0opALBDBU4umtp6kCSpb9zvanYm3uBEAAAAAFAapRQA2KG5Ta7XcZ9aCspN1/C9S4yOAwAAAABlUEoBgJ2J9w3R1426S5Ie2rVI3kV5BicCAAAAgLIopQDAjphl0pTWt6nIyUUdEveoy4ldRkcCAAAAgHJRSgGAHVlav4P21oiUR1G+/r1zoUxGBwIAAACA86CUAgA7keruq89a3CRJGrJvqYJz04wNBAAAAAAXQCkFAHbio+hble3qqUZnjuqW2LVGxwEAAACAC6KUAgA7sDGkqdbUbS0nc7Ee3f61nGUxOhIAAAAAXBClFABUc7nObvogZqAkqX/sr2qYfsLgRAAAAADwzyilAKCa+1+z3jrlFaTgnFTdu/9no+MAAAAAwEWhlAKAauywfx0tanCNJGnUjgXyKC4wOBEAAAAAXBxKKQCopopNTprS+jaZTU669tg2tU/ab3QkAAAAALholFIAUE19G9VFhwLD5VOQo5G7vjU6DgAAAABcEkopAKiGTnkGaHazPpKk4Xt+UFB+psGJAAAAAODSUEoBQDVjkfRBzEDlubireUqc+sRvNDoSAAAAAFwySikAqGbWhrXSxtDmcjEX6dHt8+Uki9GRAAAAAOCSuRgdAABw8bJdPDQ9+lZJ0uCDKxWRmWRwIgAAAAAVZfb6P/TR6iM6nZWvZrX9NPGWFmodHlDuvnM2JmjB1mM6cLJktEd0XX892btpqf2z84v0xtL9+nlPks7kFCg8yEvDOtfX/3WMqIRnUxYrpQCgGpnRvK9SPf1VJ/OU7jy43Og4AAAAACrIdztO6OXv92l0z0b64ZGual7bV0M+3aDkrPxy9//9SIpuiQnTnAc6asG/u6i2v6fu/XSDTqbnWfd5+Ye9Wn3wtN65o7V+GXudRnSJ1IRv92jZXmM+7KaUAoBqYm9QhJZEdpQkPbLjG7mZiwxOBAAAAKCifLI2Tne2D9ft7cLVKMRXr/SPlqebs+ZtPlru/u/d2Ub3dqqvFmH+ahjsozcGtZLFIq07nGzdZ0v8GQ26qq46Naih8CAv3d2hnprV9tWOo2mV9KxKo5QCgGqg0OSsKa0Hy2Jy0g3xmxSTHGt0JAAAAACXKDMzUxkZGdav/PzyVz0VFJm1+3i6ujSsad3m5GRSl4Y1tTU+7aL+rtzCYhUWmxXg5Wrd1jYiUL/sS9LJ9DxZLBb9FpusuNPZuqZRzQs8UsWhlAKAauCbRtcp3i9UfvlZun/3d0bHAQAAAHAZmjdvLn9/f+vXa6+9Vu5+Z3IKVGy2qKaPe6nttXzcdfo8p++d6/Uf9ynEz6NUsfXiLS3UMNhHHV9brkbP/ahhn23SpFtbqkNUjct/UleAQecAUMWd8K6hL5vcIEkauetb+RXmGJwIAAAAwOXYu3ev6tSpY73t7u5+gb0v34erDuu7HYma+0BHebg6W7fP+u0PbU9I0ydD2qlOoKc2xqVq/OLdCvHzUFcDVktRSgFAFWaR9H7MIBU6u6rNqQPqfmyr0ZEAAAAAXCZfX1/5+fn9436BXm5ydjKVGWp+OitftXwuXGR9vCZW01bF6ov7O6hZ7b/+rrzCYr350wF9dG9b9WgaIklqVttPe09k6ONfjxhSSnH6HgBUYSvCr9L24MZyKy7Uw9sXyGR0IAAAAAAVzs3FSS3r+Ou3vw0pN5st+u1wiq6KCDjv901fHav3lx/WrBHt1apu6f0Ki80qLLbIZCr9rsLJySSLxWLL+BfN0FJqzZo16tevn8LCwmQymbRo0aJS91ssFo0fP161a9eWp6enevbsqUOHDpXaJzU1Vffcc4/8/PwUEBCg++67T1lZWZX4LACgYqS7eenjlrdKku7ev0xhOSkGJwIAAABQWe7vGqk5m45q/pZjOnwqU88t2q2cgiINbhsuSRr71Xa9sXS/df9pq2L19s8HNfm2Vqob6KlTmXk6lZmn7PySq3b7eriqQ2SQXluyT+tjU3Q0NUdfbz6qBVuPqVeLUEOeo6Gn72VnZysmJkYjRozQwIEDy9w/efJkTZkyRbNmzVJkZKReeOEF9e7dW3v37pWHh4ck6Z577lFiYqKWLVumwsJCDR8+XA888IC+/PLLyn46AGBTn7Tspwx3b9VPT9Sgw6uMjgMAAACgEvWLCVNqdoHeWXZQpzPz1SzMT7NGtFct35LT946n5ZZa9fS/3+NVUGzWQ1+UHvkx+vpGGnNDY0nS+3e30eSlB/TYV9uUllOoOoGeerJ3E/1fh3qV98T+xtBS6sYbb9SNN95Y7n0Wi0Xvvvuunn/+ed16a8lKgdmzZyskJESLFi3SnXfeqX379mnp0qXatGmT2rVrJ0l6//331bdvX/3nP/9RWFhYuY+dn59f6rKLmZmZNn5mAHBlttdsqF/qXS2TxaxHt38tF4vZ6EgAAAAAKtnQzvU1tHP9cu/7amSnUrfXPdPjHx8v2NdD/xkcY4toNlFlZ0rFxcXp5MmT6tmzp3Wbv7+/OnTooPXr10uS1q9fr4CAAGshJUk9e/aUk5OTNmzYcN7Hfu2110pdgrF58+YV90QA4BLlFRZrautBkqSb4tar2ZkEgxMBAAAAgO1V2VLq5MmTkqSQkJBS20NCQqz3nTx5UsHBwaXud3FxUVBQkHWf8owbN07p6enWr71799o4PQBcvg9WHtZxn1oKyk3XsL0/Gh0HAAAAACqEoafvGcXd3V3u7n9dQjEjI8PANADwl4NJmZq2KlaS9NCuRfIuyjM4EQAAAABUjCq7Uio0tGTye1JSUqntSUlJ1vtCQ0N16tSpUvcXFRUpNTXVug8AVBdms0XjFuxSkdmijom71eXELqMjAQAAAECFqbKlVGRkpEJDQ7V8+XLrtoyMDG3YsEGdOpUM8+rUqZPS0tK0ZcsW6z4rVqyQ2WxWhw4dKj0zAFyJOZsStCX+jLzdnPXvnQtl+udvAQAAAIBqy9DT97KysnT48GHr7bi4OG3fvl1BQUGqV6+eHnvsMb388stq1KiRIiMj9cILLygsLEz9+/eXJDVr1kx9+vTRv/71L02fPl2FhYV6+OGHdeedd573ynsAUBWdysjT6z/ulyQ93quJas1LNzgRAAAAAFQsQ0upzZs3q3v37tbbY8eOlSQNHTpUM2fO1FNPPaXs7Gw98MADSktLU9euXbV06VJ5eHhYv+eLL77Qww8/rOuvv15OTk4aNGiQpkyZUunPBQCuxMTv9yozr0it6vpraOf6Omh0IAAAAACoYIaWUt26dZPFYjnv/SaTSZMmTdKkSZPOu09QUJC+/PLLiogHAJVixf4k/bAzUc5OJr06IFrOTpy4BwAAAMD+VdmZUgDgCLLzi/TCoj2SpPu6RqplHX+DEwEAAABA5aCUAgADvbPsoI6n5apOgKce69nI6DgAAAAAUGkopQDAILuPp+uzdXGSpJcHtJSXm6FnVAMAAABApaKUAgADFBWbNW7BLpkt0s2taqt7k2CjIwEAAABApaKUAgADzPztD+06ni4/DxeN79fc6DgAAAAAUOkopQCgkh1Py9Xbyw5Kksb1baZgXw+DEwEAAABA5aOUAoBKZLFYNH7RbuUUFOvq+oG6o1240ZEAAAAAwBCUUgBQiX7cfVLL95+Sq7NJrw2MlpOTyehIAAAAAGAISikAqCTpuYV68ds9kqSHrmughsG+BicCAAAAAONQSgFAJZm8dL9OZeYrqqa3/t29odFxAAAAAMBQlFIAUAm2xKfqiw0JkqRXBkTLw9XZ4EQAAAAAYCxKKQCoYAVFZo1bsEuSNLhtXXVqUMPgRAAAAABgPEopAKhg//31iA4mZSnI203P9m1mdBwAAAAAqBIopQCgAsUlZ+u95YckSS/c3EyB3m4GJwIAAACAqoFSCgAqiMVi0XMLd6mgyKxrGtVU/9Z1jI4EAAAAAFUGpRQAVJAFW4/rt9gUubs46eX+LWUymYyOBAAAAABVBqUUAFSA1OwCvfzDXknS6J6NFFHD2+BEAAAAAFC1UEoBQAV45Yd9OpNTqKahvvrXNVFGxwEAAACAKodSCgBs7LfDyfpm6zGZTNKrA6Pl6sw/tQAAAABwLt4pAYAN5RUW69mFuyRJ/9chQlfVCzQ4EQAAAABUTZRSAGBDU1cc1h8pOQrxc9eTfZoYHQcAAAAAqixKKQCwkYNJmZq+OlaSNPGWFvLzcDU4EQAAAABUXZRSAGADZrNF4xbsUpHZop7NQtS7RajRkQAAAACgSqOUAgAbmLMpQVviz8jbzVmTbm0hk8lkdCQAAAAAqNIopQDgCp3KyNPrP+6XJD3eq4nCAjwNTgQAAAAAVR+lFABcoYnf7VVmXpFa1fXX0M71jY4DAAAAANUCpRQAXIEV+5P0w65EOTuZ9NrAaDk7cdoeAAAAAFwMSikAuEzZ+UV6YdEeSdJ9XSPVIszf4EQAAAAAUH1QSgHAZXpn2UEdT8tVnQBPPdazkdFxAAAAAKBaoZQCgMuw61i6PlsXJ0l6eUBLebm5GJwIAAAAAKoXSikAuETFJieNW7hTZovULyZM3ZsEGx0JAAAAAKodSikAuETfRnXR7uMZ8vNw0fibmxsdBwAAAACqJUopALgESZ6Bmt2sjyRpXN9mquXrbnAiAAAAAKieKKUA4CJZJH0YM0B5Lu66un6g7mgXbnQkAAAAAKi2KKUA4CKtDWuljaHN5WIu0msDo+XkZDI6EgAAAABUW5RSAHARslw9NK1Vf0nS7QdXqGGwr7GBAAAAAKCao5QCgIswo3lfnfHwU53MU7rj4Aqj4wAAAABAtUcpBQD/YE9QfS2J7CxJenTHN3IzFxmcCAAAAACqP0opALiAQpOzprS+TZJ0Q/xGtUqONTgRAAAAANgHSikAuID5jbopwS9UfvlZun/390bHAQAAAAC7QSkFAOdx3Lum5jTpKUkauWux/ApzDE4EAAAAAPaDUgoAymGR9H7rQSp0dtVVSQfU/dg2oyMBAAAAgF2hlAKAciwPb6sdtRrJrbhQo3YskMnoQAAAAABgZyilAOAc6W5e+m/LWyRJ9+z/WWE5KQYnAgAAAAD7QykFAOf4b8tblOHurfrpJzTw8Gqj4wAAAACAXaKUAoC/2VazoZbXayeTxaxHt8+Xi8VsdCQAAAAAsEuUUgDwp3wnF01tPUiSdHPcejU7k2BwIgAAAACwX5RSAPCnuU166oRPLdXITdfQvUuMjgMAAAAAdo1SCgAk/eEbqq8bdZckPbRzobyL8g1OBAAAAAD2jVIKgMMzy6T3Ww9SsZOzOibuVufE3UZHAgAAAAC7RykFwOH9WL+j9taIlGdhnv69c6FMRgcCAAAAAAdAKQXAoaW6+2pGi76SpKH7lqpWbrrBiQAAAADAMVBKAXBo01v1V7arpxqfSdDNR9YZHQcAAAAAHAalFACHtSGkmX6tEyMnc7Ee3T5fzrIYHQkAAAAAHAalFACHlOvspg9jBkqSBsT+qgbpJwxOBAAAAACOhVIKgEP6vFlvnfIKVEh2qv5v/89GxwEAAAAAh0MpBcDhHPKvo8UNrpEkPbzjG3kUFxicCAAAAAAcD6UUAIdSbHLSlDaDZTY56bpj29Tu1AGjIwEAAACAQ6KUAuBQFkd11eGAuvIpyNHIXYuNjgMAAAAADotSCoDDSPIM1OxmfSRJI/b8oMD8LIMTAQAAAIDjopQC4BAskj6MGaB8Fze1TD6i3vEbjY4EAAAAAA6NUgqAQ1gb1kobQ5vLxVykR7bPl5MsRkcCAAAAAIdGKQXA7mW5emhaq/6SpNsPrlC9rFPGBgIAAAAAUEoBsH8zmt+kMx5+qpN5SnccXGF0HAAAAACAKKUA2Lk9QfW1JLKTJOnRHd/IzVxkcCIAAAAAgFTFS6kXX3xRJpOp1FfTpk2t9+fl5WnUqFGqUaOGfHx8NGjQICUlJRmYGEBVUmhy1pTWt0mSesVvUKvkWIMTAQAAAADOqtKllCS1aNFCiYmJ1q+1a9da7xszZoy+++47ff3111q9erVOnDihgQMHGpgWQFUyv1E3JfiFyj8/S/fv/t7oOAAAAACAv3ExOsA/cXFxUWhoaJnt6enp+vTTT/Xll1+qR48ekqQZM2aoWbNm+v3339WxY8fKjgqgCjnuXVNzmvSUJD2wa7F8C3MNTgQAAAAA+Lsqv1Lq0KFDCgsLU1RUlO655x4lJCRIkrZs2aLCwkL17NnTum/Tpk1Vr149rV+//oKPmZ+fr4yMDOtXZmZmhT4HAJXLIun91rep0NlVVyUdUPdj24yOBAAAAAA4R5UupTp06KCZM2dq6dKlmjZtmuLi4nTNNdcoMzNTJ0+elJubmwICAkp9T0hIiE6ePHnBx33ttdfk7+9v/WrevHkFPgsAle2X8HbaUauh3IoLNWrHApmMDgQAAAAAKKNKn7534403Wv/cqlUrdejQQREREZo3b548PT0v+3HHjRunsWPHWm8fP36cYgqwE+luXvqkZT9J0j37f1ZYTorBiQAAAAAA5anSK6XOFRAQoMaNG+vw4cMKDQ1VQUGB0tLSSu2TlJRU7gyqv3N3d5efn5/1y9fXtwJTA6hM/215izLcvRWZfkIDD682Og4AAAAA4DyqVSmVlZWl2NhY1a5dW23btpWrq6uWL19uvf/AgQNKSEhQp06dDEwJwCjbajbU8nrtZLKY9ej2+XKxmI2OBAAAAAA4jyp9+t4TTzyhfv36KSIiQidOnNCECRPk7Oysu+66S/7+/rrvvvs0duxYBQUFyc/PT4888og6derElfcAB5Tv5KL3W98mSbo5br2ankkwOBEAAAAA4EKqdCl17Ngx3XXXXUpJSVGtWrXUtWtX/f7776pVq5Yk6Z133pGTk5MGDRqk/Px89e7dWx9++KHBqQEYYU6Tnkr0qakauekauneJ0XEAAAAAAP+gSpdSc+fOveD9Hh4e+uCDD/TBBx9UUiIAVdEfvqGa36i7JOmhnQvlXZRvcCIAAAAAwD+pVjOlAOBcZpn0futBKnZyVqfE3eqSuNvoSAAAAACAi0ApBaBa+7F+R+2tESnPwjw9tHOh0XEAAAAAABeJUgpAtZXi4afPWvSVJA3dt1S1ctMNTgQAAAAAuFiUUgCqrenRtyrH1VONzyTo5iPrjI4DAAAAALgElFIAqqUNIc20tk6MnMzFenT7fDnLYnQkAAAAAMAloJQCUO3kOrvpw5iBkqSBsWvUIP2EwYkAAAAAAJeKUgpAtfN5s9465RWokOxU3bN/mdFxAAAAAACXgVIKQLVyyL+OFje4RpL08I5v5FFcYHAiAAAAAMDloJQCUG0Um5z0XpvBMpucdN2xbWp36oDRkQAAAAAAl4lSCkC1sTiqq2ID6sqnIEcjdy02Og4AAAAA4ApQSgGoFpI8AzW7WR9J0n17vldgfpbBiQAAAAAAV4JSCkCVZ5H0YcwA5bu4qWXyEfWK32R0JAAAAADAFaKUAlDl/RoWo42hzeViLtIj2+fLSRajIwEAAAAArhClFIAqLcvVQ9Nb3SpJuv3gCtXLOmVwIgAAAACALVBKAajSZjS/SWc8/FQ385TuOLjC6DgAAAAAABuhlAJQZe0Oqq8lkZ0kSY9sny83c5HBiQAAAAAAtkIpBaBKKjQ56/3Wt0mSesVvUKuUIwYnAgAAAADYEqUUgCppfqNuSvALlX9+lu7f/b3RcQAAAAAANkYpBaDKOeZdU3Oa9JQkPbBrsXwLcw1OBAAAAACwNUopAFWKRdLU1rep0NlVVyUdUPdj24yOBAAAAACoAJRSAKqUX8LbaUethnIvKtDDO76RyehAAAAAAIAKQSkFoMpIc/PWf6P7SZLuPrBMtXNSDU4EAAAAAKgolFIAqoxPWvZTppu3ItNPaODh1UbHAQAAAABUIEopAFXCtlqNtLxeO5ksZj26fb5cLGajIwEAAAAAKhClFADD5Tu56P2YQZKkm+N+U9MzCQYnAgAAAABUNEopAIab06SnEn1qqkZumobu/dHoOAAAAACASkApBcBQcX6hmt+ouyTpoZ2L5F2Ub3AiAAAAAEBloJQCYBizTHo/5jYVOzmrU+JudUncbXQkAAAAAEAloZQCYJgl9TtqX4368izM00M7FxodBwAAAABQiSilABgixcNPM1r0lSQN3fejauWmG5wIAAAAAFCZXIwOAMBxFMukPTWjlOruqx/qd1SOq6eapCbo5iO/GR0NAAAAAFDJKKUAVIp1tVtqeqv+SvYM+GujxaJrj2+TsyyG5QIAAAAAGIPT9wBUuHW1W+rl9kOV7OFf5r7/trxF62q3NCAVAAAAAMBIlFIAKlSxTJreqn/JDZOp9J1/3v4o+lYV65z7AAAAAAB2jdP3AFyQWSYVOLsqz9lNeS5u1v/ml7ut5M/5f9uW5BlY+pS9c5lMOu0VqD01o9QqObbSnhcAAAAAVHWz1/+hj1Yf0emsfDWr7aeJt7RQ6/CAcvedszFBC7Ye04GTmZKk6Lr+erJ30zL7Hz6Vqdd/3K8NR1JVZLaoUYiPpv1fW9UJ8KzgZ1MWpRQM9/fh10H5mWqRfIQZQ5fAIqnQybmkEPqzDMo/pyzKc75wcVTyPa5lvif/z/0qQ6q7b6X8PQAAAABQHXy344Re/n6fXh7QUm3CA/TZujgN+XSDVjzRTTV93Mvs//uRFN0SE6arbgmUu4uzpq+O1b2fbtCyMdcp1N9DkhSfkq3bpq/XHe3C9VjPxvL1cNHBpCy5uxhzIh2lFAxV3vDrmrlpenDnInVJ3G1cMBsrNluUU1Ck3IJi5RQUK7fwz/8WFJds/9vtkj8X/e3+km25BcVKuebhclcrmZ2cK+V5uBcVyL24QB7FBfIoKvmve3Gh9c/Wbdb7CpTi4adFDa/7x8cOys+shGcAAAAAANXDJ2vjdGf7cN3eLlyS9Er/aK3Yf0rzNh/Vv7s1LLP/e3e2KXX7jUGttHT3Sa07nKxBbetKkt786YC6NwnWuL7NrPtF1PCuwGdxYZRSMMzZ4dfnSvbw18vth+r5jbMqrZiySNaCp9SKozKrjC58yprT9PXKKSw6p2AqVkGR2TZBa9S/4N0u5iJ5nC2Oiv4sjP5WFnlYt5/z56K/3z7ne/78r1txoZwuYwVbsUxaWyemZMj5uTOlJMliUa3cNLVIPnLJjw0AAAAA1UlmZqYyMjKst93d3eXuXnbVU0GRWbuPp+vf3RpYtzk5mdSlYU1tjU+7qL8rt7BYhcVmBXi5SpLMZotW7j+lkdc10L2fbtDeExmqG+Slf3droN4tQq/siV0mSikY4h+HX1ss+ij6VnVM3GM9la/Q5FxuMZR/oVPWSu3vbi2Qzn2cfGdXWUw2WK74R+oF7zaZJC9XZ3m6ucjLzVmers7ydHOW159fnm4uf97/53br/SX7J499rEzBdHZFkkdRgVwsNiq/bMhZFj24c1FJAWmxlD7elpJjO3LXYk7ZBAAAAGD3mjdvXur2hAkT9OKLL5bZ70xOgYrNljKn6dXycVfs6eyL+rte/3GfQvw81KVhTUlScna+sguKNW1VrB7v1VjP3NhUqw+e1oP/26I5/+qojlE1Lu9JXQFKKRhiT82oixp+ffeNE1Ts5Kw8ZzcVV9Ipam5/no52vtVEpU5XO6cYavCfyX+WS6WLJk/Xkj+7uzjJVN5qoYu0r5qe0tglcbee3zirzKmatXLTNHLXYrs6VRMAAAAAzmfv3r2qU6eO9XZ5q6Rs4cNVh/XdjkTNfaCjPFxL3kv/uSZANzQP0f3XREmSWoT5a2v8GX2xIYFSCo7jYodaZ7j7lNnmZC4+/ylpFzj9rFSx9LfvdS8q/NufC65oxU6zVrUv+3vtXZfE3eqYuIeh9gAAAAAclq+vr/z8/P5xv0AvNzk7mZSclV9q++msfNUqZ8j53328JlbTVsXqi/s7qFntv/6uQC83uTiZ1Ci49PvsBsE+2vzHmUt4FrZDKQVDXOxQ60e3fa3olCN/lUhFBXK1FFdwOlQUZ1nUKjnW6BgAAAAAUKW5uTipZR1//XY42TrvyWy26LfDKRrSOeK83zd9daw+WHFYs+5rr1Z1A8o8Zqu6/jqSXPr0v7jT2aoT4Gnz53AxKKVgiBbJR1QzN+0fh1/3it/IShoAAAAAgMO5v2ukHv96h6LrBqh1uL8+XfuHcgqKNLhtydX4xn61XSH+Hnq6T1NJ0rRVsXpn2UG9d2dr1Q301KnMPEmSt5uLvN1L6p8Hrm2gR+ZsVfvIIHWKqqHVB09r+f5TmvtAR0OeI6UUDMHwawAAAAAAzq9fTJhSswv0zrKDOp2Zr2Zhfpo1or1q+Zacvnc8LbfUzOL//R6vgmKzHvpia6nHGX19I425obEkqU/LUL3SP1ofrjqsF7/do6haPpp2z1W6un5Q5T2xv6GUgmEYfg0AAAAAwPkN7VxfQzvXL/e+r0Z2KnV73TM9Luoxb786XLdfHX6l0WyCUgqGYvg1AAAAAACOiVIKhmP4NQAAAAAAjsfJ6AAAAAAAAABwPJRSAAAAAAAAqHSUUgAAAAAAAKh0lFIAAAAAAACodJRSAAAAAAAAqHSUUgAAAAAAAKh0lFIAAAAAAACodJRSAAAAAAAAqHSUUgAAAAAAAKh0lFIAAAAAAACodJRSAAAAAAAAqHSUUgAAAAAAAKh0lFIAAAAAAACodJRSAAAAAAAAqHSUUgAAAAAAAKh0lFIAAAAAAACodJRSAAAAAAAAqHSUUgAAAAAAAKh0lFIAAAAAAACodHZTSn3wwQeqX7++PDw81KFDB23cuNHoSAAAAAAAADgPuyilvvrqK40dO1YTJkzQ1q1bFRMTo969e+vUqVNGRwMAAAAAAEA5XIwOYAtvv/22/vWvf2n48OGSpOnTp+uHH37QZ599pmeeeabM/vn5+crPz7feTk9PlyQlJiZWTuAKcLKwwOgIkOR77FiFPj7HuergWDsOjrXj4Fg7hoo+zhLHuqrgWDsOjrXjqIxjXVHO9g1ms9ngJFWLyWKxWIwOcSUKCgrk5eWl+fPnq3///tbtQ4cOVVpamhYvXlzme1588UVNnDixElMCAAAAAABHt3HjRl199dVGx6gyqv1KqeTkZBUXFyskJKTU9pCQEO3fv7/c7xk3bpzGjh1rvV1UVKR9+/YpPDxcTk52cUZjtZOZmanmzZtr79698vX1NToOKhDH2jFwnB0Hx9pxcKwdB8facXCsHQfH2nhms1lJSUlq06aN0VGqlGpfSl0Od3d3ubu7l9rWpUsXg9JAkjIyMiRJderUkZ+fn8FpUJE41o6B4+w4ONaOg2PtODjWjoNj7Tg41lVDvXr1jI5Q5VT7ZUE1a9aUs7OzkpKSSm1PSkpSaGioQakAAAAAAABwIdW+lHJzc1Pbtm21fPly6zaz2azly5erU6dOBiYDAAAAAADA+djF6Xtjx47V0KFD1a5dO7Vv317vvvuusrOzrVfjQ9Xn7u6uCRMmlDmtEvaHY+0YOM6Og2PtODjWjoNj7Tg41o6DY42qqtpffe+sqVOn6s0339TJkyfVunVrTZkyRR06dDA6FgAAAAAAAMphN6UUAAAAAAAAqo9qP1MKAAAAAAAA1Q+lFAAAAAAAACodpRQAAAAAAAAqHaUUAAAAAAAAKh2lFAAAAAAAACqdi9EBADiW/Px8ubu7Gx0DlYBjDQDVS2FhoU6ePKmcnBzVqlVLQUFBRkdCBeFYA6gqKKVgCLPZrNWrV+vXX39VfHy89QWxTZs26tmzp8LDw42OCBv58ccfNXfuXP366686evSozGazvL291aZNG/Xq1UvDhw9XWFiY0TFhAxxrx5Kfn68NGzaU+Tc8MjLS6GiwobS0NC1cuLDc1+vevXurc+fORkfEFcrMzNT//vc/zZ07Vxs3blRBQYEsFotMJpPq1q2rXr166YEHHtDVV19tdFRcIY6144mLiyv33+9OnTrJw8PD6HiAJMlksVgsRoeA48jNzdVbb72ladOmKTU1Va1bt1ZYWJg8PT2Vmpqq3bt368SJE+rVq5fGjx+vjh07Gh0Zl2nhwoV6+umnlZmZqb59+6p9+/ZljvWvv/6q9evXa9iwYXrppZdUq1Yto2PjMnCsHcu6dev03nvv6bvvvlNhYaH8/f2txzo/P19RUVF64IEH9OCDD8rX19fouLhMJ06c0Pjx4/XFF18oLCys3J/rLVu2KCIiQhMmTNAdd9xhdGRchrfffluvvPKKGjRooH79+p333+9FixapQ4cOev/999WoUSOjY+MycKwdyxdffKH33ntPmzdvVkhISKljHRsbKw8PD91zzz16+umnFRERYXRcODhKKVSq8PBwderUScOGDdMNN9wgV1fXMvvEx8fryy+/1EcffaTnnntO//rXvwxIiivVqVMnPf/887rxxhvl5HT+8XXHjx/X+++/r5CQEI0ZM6YSE8JWONaO45ZbbtHWrVt19913q1+/fmrXrp08PT2t9x85ckS//vqr5syZox07dmj27Nm64YYbDEyMyxUSEqKhQ4dq2LBhat68ebn75ObmatGiRZoyZYoGDRqkJ554opJT4krdddddev7559WiRYsL7pefn68ZM2bIzc1NI0aMqKR0sCWOteNo06aN3NzcNHToUPXr16/MGSj5+flav3695s6dq2+++UYffvihBg8ebFBagFIKlWzfvn1q1qzZRe1bWFiohIQENWjQoIJTAQAuxkcffaQRI0aU+4HCufbu3avExERdf/31lZAMtpaSkqIaNWpU2P4AgIrx008/qXfv3he1b0pKiv744w+1bdu2glMB50cpBaDSFBQUKC4uTg0aNJCLCyPt7BnHGgCqp8OHDys2NlbXXnutPD09rTOHYH841gCqgvOfZwFUgl9//VX/93//p06dOun48eOSpM8//1xr1641OBlsKScnR/fdd5+8vLzUokULJSQkSJIeeeQRvf766wangy1xrB1LWlqaPvnkE40bN06pqamSpK1bt1r/PYf9+Pzzz9WlSxeFhYUpPj5ekvTuu+9q8eLFBieDraSkpKhnz55q3Lix+vbtq8TEREnSfffdp8cff9zgdLAljrVjiY2N1fPPP6+77rpLp06dklRycZo9e/YYnAwoQSkFw3zzzTfq3bu3PD09tW3bNuXn50uS0tPT9eqrrxqcDrY0btw47dixQ6tWrSp1pY+ePXvqq6++MjAZbI1j7Th27typxo0b64033tB//vMfpaWlSZIWLFigcePGGRsONjVt2jSNHTtWffv2VVpamoqLiyVJAQEBevfdd40NB5sZM2aMXFxclJCQIC8vL+v2O+64Q0uXLjUwGWyNY+04Vq9erejoaG3YsEELFixQVlaWJGnHjh2aMGGCwemAEpRSMMzLL7+s6dOn67///W+p+SRdunTR1q1bDUwGW1u0aJGmTp2qrl27lloW3qJFC8XGxhqYDLbGsXYcY8eO1bBhw3To0KFSBWTfvn21Zs0aA5PB1t5//33997//1XPPPSdnZ2fr9nbt2mnXrl0GJoMt/fzzz3rjjTdUt27dUtsbNWpkXR0H+8CxdhzPPPOMXn75ZS1btkxubm7W7T169NDvv/9uYDLgL5RSMMyBAwd07bXXltnu7+9v/cQd9uH06dMKDg4usz07O5vZBXaGY+04Nm3apJEjR5bZXqdOHZ08edKARKgocXFxatOmTZnt7u7uys7ONiARKkJ2dnapVTNnpaamyt3d3YBEqCgca8exa9cuDRgwoMz24OBgJScnG5AIKItSCoYJDQ3V4cOHy2xfu3atoqKiDEiEitKuXTv98MMP1ttny4lPPvlEnTp1MioWKgDH2nG4u7srIyOjzPaDBw+qVq1aBiRCRYmMjNT27dvLbF+6dOlFX1EXVd8111yj2bNnW2+bTCaZzWZNnjxZ3bt3NzAZbI1j7TgCAgKsM8P+btu2bapTp44BiYCyuCQSDPOvf/1Lo0eP1meffSaTyaQTJ05o/fr1euKJJ/TCCy8YHQ829Oqrr+rGG2/U3r17VVRUpPfee0979+7Vb7/9ptWrVxsdDzbEsXYct9xyiyZNmqR58+ZJKnlTk5CQoKefflqDBg0yOB1saezYsRo1apTy8vJksVi0ceNGzZkzR6+99po++eQTo+PBRiZPnqzrr79emzdvVkFBgZ566int2bNHqampWrdundHxYEMca8dx55136umnn9bXX39tLR/XrVunJ554QkOGDDE6HiBJMlksFovRIeCYLBaLXn31Vb322mvKycmRVPLJ+xNPPKGXXnrJ4HSwtdjYWL3++uvasWOHsrKydNVVV+npp59WdHS00dFgYxxrx5Cenq7bbrtNmzdvVmZmpsLCwnTy5El16tRJS5Yskbe3t9ERYUNffPGFXnzxRetsuLCwME2cOFH33XefwclgS+np6Zo6dWqpf79HjRql2rVrGx0NNsaxdgwFBQUaNWqUZs6cqeLiYrm4uKi4uFh33323Zs6cWWpOIGAUSikYrqCgQIcPH1ZWVpaaN28uHx8foyMBAC7S2rVrtXPnTuubmp49exodCRUoJydHWVlZ5c6OAwBUTQkJCdq9e7eysrLUpk0bNWrUyOhIgBWlFKqMjIwMrVixQk2aNGFGhZ1JSEi44P316tWrpCSoaFu3bpWrq6t1VdTixYs1Y8YMNW/eXC+++GKpK78AqB5yc3NlsVisg5Hj4+O1cOFCNW/eXL169TI4HWzln66aWd7FaVA9LV26VD4+Puratask6YMPPtB///tfNW/eXB988IECAwMNTgjAkVBKwTC33367rr32Wj388MPKzc1V69atFRcXJ4vForlz5zKTxI44OTld8MprxcXFlZgGFenqq6/WM888o0GDBunIkSNq3ry5Bg4cqE2bNummm27Su+++a3RE2MikSZMueP/48eMrKQkqWq9evTRw4EA9+OCDSktLU5MmTeTm5qbk5GS9/fbbeuihh4yOCBtwcip7/aO/v3bzWm0/oqOj9cYbb6hv377atWuX2rVrp8cff1wrV65U06ZNNWPGDKMjwkZGjBhxwfs/++yzSkoCnB+DzmGYNWvW6LnnnpMkLVy4UGazWWlpaZo1a5ZefvllSik7sm3btlK3CwsLtW3bNr399tt65ZVXDEqFinDw4EG1bt1akvT111/ruuuu05dffql169bpzjvvpJSyIwsXLix1u7CwUHFxcXJxcVGDBg0opezI1q1b9c4770iS5s+fr9DQUG3btk3ffPONxo8fTyllJ86cOVPq9tnX6hdeeIHXajsTFxen5s2bS5K++eYb9evXT6+++qq2bt2qvn37GpwOtlTez/Xu3buVlpamHj16GJQKKI1SCoZJT09XUFCQpJJlxIMGDZKXl5duuukmPfnkkwangy3FxMSU2dauXTuFhYXpzTff1MCBAw1IhYpgsVhkNpslSb/88otuvvlmSVJ4eLiSk5ONjAYbO7dslkpOwx42bJgGDBhgQCJUlJycHPn6+kqSfv75Zw0cOFBOTk7q2LGj4uPjDU4HW/H39y+z7YYbbpCbm5vGjh2rLVu2GJAKFcHNzc16kaFffvnFehW2oKAgZWRkGBkNNnbuB0iSZDab9dBDD6lBgwYGJALKKrtOF6gk4eHhWr9+vbKzs7V06VLrXIozZ87Iw8PD4HSoDE2aNNGmTZuMjgEbateunV5++WV9/vnnWr16tW666SZJJZ/KhoSEGJwOFc3Pz08TJ07UCy+8YHQU2FDDhg21aNEiHT16VD/99JP19frUqVPy8/MzOB0qWkhIiA4cOGB0DNhQ165dNXbsWL300kvauHGj9bX64MGDqlu3rsHpUNGcnJw0duxY6wpYwGislIJhHnvsMd1zzz3y8fFRRESEunXrJqnktD4uHW9fzv3UzWKxKDExUS+++CJX/7Az7777ru655x4tWrRIzz33nBo2bCip5JSfzp07G5wOlSE9PV3p6elGx4ANjR8/XnfffbfGjBmj66+/Xp06dZJUsmqqTZs2BqeDrezcubPU7bOv1a+//rr1tGzYh6lTp+rf//635s+fr2nTpqlOnTqSpB9//FF9+vQxOB0qQ2xsrIqKioyOAUhi0DkMtnnzZh09elQ33HCDfHx8JEk//PCDAgIC1KVLF4PTwVbKG3RusVgUHh6uuXPnWt/gwH7l5eXJ2dlZrq6uRkeBjUyZMqXU7bNvYD///HPrLDHYj5MnTyoxMVExMTHWgdgbN26Un5+fmjZtanA62MLZ1+pz3xp07NhRn332GccZqIbGjh1b6vbZ1+offvhBQ4cO1dSpUw1KBvyFUgpAhVu9enWp205OTqpVq5YaNmwoFxcWbALVUWRkZKnbZ3+ue/TooXHjxllnEAGoHs6dD3b2Z5qRCvbhUmZFcVqu/ejevXup239/rR4xYgS/h6NKoJRCpTq3rb+Qt99+uwKToLIUFhZq5MiReuGFF8q8iYV9CAwMLLMS7nxSU1MrOA0AW7iUC1AsWLCgApOgMhQWFqpPnz6aPn06p9XbqfJWrZ/LYrHIZDKpuLi4klKhIlksFh09elS1atWSp6en0XGA86IaRaUq72pN5bnYN7io+lxdXfXNN98w+NiOvfvuu0ZHQCUrLCyUp6entm/frpYtWxodBxWgvCuxwX65urqWmSkF+7Jy5UqjI6CSWSwWNWzYUHv27KFsRpXGSikAFW7o0KFq3bq1xowZY3QUADYSFRWlhQsXKiYmxugoAGxgzJgxcnd31+uvv250FAA20qJFC3366afq2LGj0VGA82KlFIAK16hRI02aNEnr1q1T27Zt5e3tXer+Rx991KBkqEh5eXkqKCgotY05Ffbjueee07PPPqvPP/9cQUFBRscBcIWKior02Wef6Zdffin3tZqxCvYnJydHCQkJZV6rW7VqZVAi2Nrrr7+uJ598UtOmTWNlM6osVkrBUJs3b9a8efPKfUFkRoX9uNAsKZPJpCNHjlRiGlSk7OxsPf3005o3b55SUlLK3M+cCvvRpk0bHT58WIWFhYqIiCjzBnbr1q0GJUNFmD9//nlfrznW9uHcgcjn4vQv+3H69GkNHz5cP/74Y7n381ptPwIDA5WTk6OioiK5ubmVmS3FrE9UBayUgmHmzp2rIUOGqHfv3vr555/Vq1cvHTx4UElJSRowYIDR8WBDcXFxRkdAJXnqqae0cuVKTZs2Tffee68++OADHT9+XB999BGnhNiZW2+9lfl/DmLKlCl67rnnNGzYMC1evFjDhw9XbGysNm3apFGjRhkdDzZC6eQ4HnvsMaWlpWnDhg3q1q2bFi5cqKSkJL388st66623jI4HG3rnnXd4rUaVx0opGKZVq1YaOXKkRo0aJV9fX+3YsUORkZEaOXKkateurYkTJxodETYyadIkPfHEE/Ly8iq1PTc3V2+++abGjx9vUDLYWr169TR79mx169ZNfn5+2rp1qxo2bKjPP/9cc+bM0ZIlS4yOCOASNW3aVBMmTNBdd91lfb2OiorS+PHjlZqaqqlTpxodETYwYsQIvffee/L19S21PTs7W4888og+++wzg5LB1mrXrq3Fixerffv28vPz0+bNm9W4cWN9++23mjx5stauXWt0RAAOxMnoAHBcsbGxuummmyRJbm5uys7Olslk0pgxY/Txxx8bnA62NHHiRGVlZZXZnpOTQ/loZ1JTUxUVFSWpZH7U2WXhXbt21Zo1a4yMBhuLiooq9xTNtLQ06/8HYB8SEhLUuXNnSZKnp6cyMzMlSffee6/mzJljZDTY0KxZs5Sbm1tme25urmbPnm1AIlSU7OxsBQcHSyo5vev06dOSpOjoaE7HtTPOzs46depUme0pKSlydnY2IBFQFqUUDBMYGGj9xbZOnTravXu3pJI3NDk5OUZGg41ZLJZylw7v2LGDAcl2Jioqynq6ZtOmTTVv3jxJ0nfffaeAgAADk8HW/vjjj3LnjuTn5+vYsWMGJEJFCQ0NtRbM9erV0++//y6p5NRsFtxXfxkZGUpPT5fFYlFmZqYyMjKsX2fOnNGSJUusBQbsQ5MmTXTgwAFJUkxMjD766CMdP35c06dPV+3atQ1OB1s637/R+fn5cnNzq+Q0QPmYKQXDXHvttVq2bJmio6M1ePBgjR49WitWrNCyZct0/fXXGx0PNhAYGCiTySSTyaTGjRuXKqaKi4uVlZWlBx980MCEsLXhw4drx44duu666/TMM8+oX79+mjp1qgoLC7lyk5349ttvrX/+6aef5O/vb71dXFys5cuXX/DiBqh+evTooW+//VZt2rTR8OHDNWbMGM2fP1+bN2/WwIEDjY6HKxQQEFDqtfpcJpOJVc12ZvTo0UpMTJQkTZgwQX369NEXX3whNzc3zZw509hwsIkpU6ZIKvn5/eSTT+Tj42O9r7i4WGvWrFHTpk2NigeUwkwpGCY1NVV5eXkKCwuT2WzW5MmT9dtvv6lRo0Z6/vnnFRgYaHREXKFZs2bJYrFoxIgRevfdd0u9eXVzc1P9+vXVqVMnAxOiosXHx2vLli1q2LAhl5i2E05OJYusTSZTmU9gXV1dVb9+fb311lu6+eabjYiHCmA2m2U2m+XiUvJZ5ty5c62v1yNHjuTT9mpu9erVslgs6tGjh7755ptSK5jd3NwUERGhsLAwAxPCFjIyMuTn51fufTk5Odq/f7/q1aunmjVrVnIyVISzHw7Fx8erbt26pU7VO/s7+KRJk9ShQwejIgJWlFKoVGPHjtVLL70kb29vrVmzRp07d7b+kgv7tXr1anXu3Fmurq5GR0EFCAoK0sGDB1WzZs3zDsqF/YmMjNSmTZt4A2OnBg4cqJkzZ8rPz0+zZ8/WHXfcIXd3d6NjoQLFx8crPDzcWjzDvjg7OysxMVHBwcHq0aOHFixYwGn1DqB79+5asGABH/ajSqOUQqVydXXVsWPHFBISUurFEUD15ePjo507dyoqKkrOzs46efKkatWqZXQsAFfAzc1N8fHxql27Nq/XgB3w9/fX77//rmbNmsnJyUlJSUm8VgOoEliigkpVv359TZkyRb169ZLFYtH69evP29xfe+21lZwOFal79+6KiIgoNatg6NChOnr0qFasWGFcMFyxTp06qX///mrbtq0sFoseffRReXp6lrsvlxS3LwkJCXJ1dS01GDcxMVGFhYWqV6+egclwpZo2bapx48ape/fuslgsmjdv3nlP/RkyZEglp0NFiYyMVMOGDbVs2TLrtp49e+rIkSM6cuSIgclwpXr27Knu3burWbNmkqQBAwac99Rbfi+zL2vWrJGXl5fatWtn3bZ582bl5OTwfgtVAqUUKtWbb76pBx98UK+99ppMJpMGDBhQ7n4mk6ncqzqh+qpfv36ZK7rUqVOH0wTswP/+9z+98847io2NlclkUnp6uvLy8oyOhUpQv359NW3aVHv37rVu69Gjhw4ePMi/4dXc9OnTNXbsWP3www8ymUx6/vnny72KqslkopSyI0OHDi2zembAgAFKTk42KBFs5X//+59mzZql2NhYrV69Wi1atJCXl5fRsVAJunXrVua1+t577+W1GlUGp+/BEFlZWfLz89OBAwfOezrA34diA6geIiMjtXnzZtWoUcPoKKgEq1evlpeXl66++mrrtk2bNiknJ0fXXXedgclgS05OTjp58iSn7wF2onv37lq4cCEzpRxEfHy8XF1dS12w4MSJEyosLFRERISByYASlFIwzOrVq9WlSxcGnQMAUIXFx8erXr165a6UAgAAuBKUUgAqxfLly7V8+XKdOnVKZrO51H3MGbIvzBkCgOqnuLhYM2fOPO9rNXOG7AtzhhxLQUFBuT/X/F6GqoAlKjAUw68dw8SJEzVp0iS1a9dOtWvX5tN2O8ecIceQlJSkJ554wvoG9tzPuDjW9oUB2PZv9OjRmjlzpm666Sa1bNmS12o7x5whx3Do0CGNGDFCv/32W6ntFouFGb6oMiilYCiGXzuG6dOna+bMmbr33nuNjoJKsHLlyjLDU2fPnq2cnByDEqEiDBs2TAkJCXrhhRcomx0AA7Dt39y5czVv3jz17dvX6CioBHFxcXJ1dS21bfny5SosLDQoESrCsGHD5OLiou+//57XalRZnL4HoMLVqFFDGzduVIMGDYyOAsBGfH199euvv6p169ZGRwFgA2FhYVq1apUaN25sdBQANuLt7a0tW7aoadOmRkcBzovlKAAq3P33368vv/zS6BgAbCg8PLzMKXsAqq/HH39c7733Hj/XDqSgoEDHjh1TQkJCqS/Yj+bNm7OiFVUeK6VgKIZfO4bRo0dr9uzZatWqlVq1alVmufjbb79tUDLYGnOGHMfPP/+st956Sx999JHq169vdBxUIAZgO4YBAwZo5cqVCgoKUosWLcq8Vi9YsMCgZLA15gw5jhUrVuj555/Xq6++qujo6DI/135+fgYlA/7CTCkYhuHXjmPnzp3WU3x2795tbBhUKOYMOY477rhDOTk5atCggby8vMr8opuammpQMtgaA7AdQ0BAgAYMGGB0DFQC5gw5jp49e0qSrr/++lLbKSBRlbBSCoapXbu2Jk+ezPBrwI4wZ8hxzJw584JvZIYOHVqJaVCRatasqdmzZzMAG7ATzBlyHKtXr77g/dddd10lJQHOj5VSMExBQYE6d+5sdAxUghEjRui9996Tr69vqe3Z2dl65JFHOFXTjjBnyHEMGzbM6AioJG5ubmrYsKHRMVDBevTooQULFiggIKDU9oyMDPXv35/TNO0Ic4YcR2RkpMLDw8t8iGSxWHT06FGDUgGlMegchmH4teOYNWuWcnNzy2zPzc3V7NmzDUiEivLuu+/qmWee0R9//GF0FFQwZ2dnnTp1qsz2lJQUOTs7G5AIFYUB2I5h1apVKigoKLM9Ly9Pv/76qwGJUFHeeOMNPfXUU1q1apVSUlKUkZFR6gv2IzIyUqdPny6zPTU1VZGRkQYkAspipRQMk5eXp48//li//PILw6/tVEZGhiwWiywWizIzM+Xh4WG9r7i4WEuWLFFwcLCBCWFrzBlyHOcrKPLz8+Xm5lbJaVCR1q5dq5UrV+rHH39kALYd2rlzp/XPe/fu1cmTJ623i4uLtXTpUtWpU8eIaKggzBlyHGeP6bmysrJK/V4OGIlSCoZh+LX9CwgIkMlkkslkUuPGjcvcbzKZNHHiRAOSoaK88847DEy1c1OmTJFU8vP7ySefyMfHx3pfcXGx1qxZw5wSO8MAbPvWunVr62t1jx49ytzv6emp999/34BkqCgrV640OgIq2NixYyWVvFa/8MIL8vLyst5XXFysDRs2MP8TVQaDzgFUmNWrV8tisahHjx765ptvFBQUZL3Pzc1NERERCgsLMzAhgEt1drl/fHy86tatW+pUPTc3N9WvX1+TJk1Shw4djIoI4BLEx8fLYrEoKipKGzduVK1ataz3ubm5KTg4mFNy7UxCQsIF5wzVq1fPoGSwle7du0sq+V28U6dOpVYwn32tfuKJJ9SoUSOjIgJWlFIwDMOvHUd8fLzq1atX7gqahIQEfvmxI87OzkpMTCxzWmZKSoqCg4M5JcCOdO/eXQsWLFBgYKDRUVDBGIDtGFavXq0uXbrIxaX0iRTFxcVat26drr32WoOSwdZ4rXYcw4cP13vvvSc/Pz+jowDnxaBzGIbh144jKiqq3CGLKSkpDFm0M8wZchzdu3eXu7t7me25ubmaNGmSAYlQURiA7Rh69OhR7ty/tLQ066oL2AfmDDmOs6fmnis7O1sjRowwIBFQFjOlUOkYfu14zldU8MuP/WDOkOOZOHGiHnzwwVJzKiQpJydHEydO1Pjx4w1KBlthALZjOV9RkZKSIm9vbwMSwdaYM+R4Zs2apddff73MmSlnFwFwZgqqAkopVDqGXzuOv//yM378eH75sWPvvPOOpJI3NdOnTy93ztD06dONiocKcL43sDt27Cg1Pw7VFwOwHcPAgQMllbxWDxs2rNQKyOLiYu3cuVOdO3c2Kh5saNu2bZJK/v3etWtXmTlDMTExeuKJJ4yKBxtiEQCqE0opVLqVK1cy/NpB8MuP44iLi5PEnCFHEBgYWOqDhb8XU8XFxcrKytKDDz5oYELYSlxcHAOwHYC/v7+kktdqX19feXp6Wu9zc3NTx44d9a9//cuoeLChs1fdY86Q/WMRAKoTSilUuuuuu05SyS+7DL+2b/zy43guNGfozTff5JQuO/Duu+/KYrFoxIgRmjhxovUNrfTXqrhOnToZmBC2EhERIank3/LWrVuXOwB7zZo1DMCu5mbMmCFJ1qtxcaqe/bvQnCEuNmQfWASA6oSr78EwXPkDsD/8XDuO1atXq3PnznJ1dTU6CioYP9eAfTnfz3RycrJCQ0NVVFRkUDLYWnx8vMLDw+XkxPXNUHWxUgqGYfi1Y9m8ebPmzZunhISEMldxWrBggUGpYGvMGXIcZ1e9SiVXYTv355qVkfaDAdiOY/78+ed9rd66datBqWArzBlyPGdXvObk5JT7c92qVSsjYgGlUEqh0jH82vHMnTtXQ4YMUe/evfXzzz+rV69eOnjwoJKSkjRgwACj48EGmDPkeHJycvTUU09p3rx5SklJKXM/q2eqPwZgO5YpU6boueee07Bhw7R48WINHz5csbGx2rRpk0aNGmV0PNgAc4Ycz+nTpzV8+HD9+OOP5d7PazWqAkopVDqGXzueV199Ve+8845GjRolX19fvffee4qMjNTIkSNVu3Zto+PBBpgz5HiefPJJrVy5UtOmTdO9996rDz74QMePH9dHH32k119/3eh4sAEGYDuWDz/8UB9//LHuuusuzZw5U0899ZSioqI0fvx4paamGh0PNsCcIcfz2GOPKS0tTRs2bFC3bt20cOFCJSUl6eWXX9Zbb71ldDxAEjOlYCCGXzsOb29v7dmzR/Xr11eNGjW0atUqRUdHa9++ferRo4cSExONjggbYc6Q46hXr55mz56tbt26yc/PT1u3blXDhg31+eefa86cOVqyZInREWEjEydOZAC2A/Dy8tK+ffsUERGh4OBgLVu2TDExMTp06JA6duxY7opIVE/MGXIctWvX1uLFi9W+fXv5+flp8+bNaty4sb799ltNnjxZa9euNToiwEopGOfs1V5g/wIDA5WZmSlJqlOnjnbv3q3o6GilpaUpJyfH4HSwJeYMOY7U1FRFRUVJKjmuZ1dSdO3aVQ899JCR0WBjEyZMMDoCKkFoaKhSU1MVERGhevXq6ffff1dMTIzi4uLOOwcU1RNzhhxHdna2dU5YYGCgTp8+rcaNGys6Opo5cagyKKVgKIZfO4Zrr71Wy5YtU3R0tAYPHqzRo0drxYoVWrZsma6//nqj48GGmDPkOKKiohQXF6d69eqpadOmmjdvntq3b6/vvvtOAQEBRseDjTEA2/716NFD3377rdq0aaPhw4drzJgxmj9/vjZv3mydLwb7wJwhx9GkSRMdOHBA9evXV0xMjD766CPVr19f06dPZ4QGqgzWbMIwc+fOVefOnbVv3z4tXLhQhYWF2rNnj1asWFFqHg2qv6lTp+rOO++UJD333HMaO3askpKSNGjQIH366acGp4MtPfnkk1qxYoWmTZsmd3d3ffLJJ5o4caLCwsI0e/Zso+PBhoYPH64dO3ZIkp555hl98MEH8vDw0JgxY/Tkk08anA62NGXKFA0fPlwhISHatm2b2rdvrxo1aujIkSO68cYbjY4HG/n444/13HPPSZJGjRqlzz77TM2aNdOkSZM0bdo0g9PBlv4+Z8jT01NLly7VrFmz1KhRI3377bdGx4MNjR492jomY8KECfrxxx9Vr149TZkyRa+++qrB6YASzJSCYVq1aqWRI0dah1/v2LGj1PBrrv4BVD/MGXJc8fHx2rJlixo2bMipH3amadOmmjBhgu666y7r6/XfB2BPnTrV6IgALgFzhhxXTk6O9u/fr3r16qlmzZpGxwEksVIKBoqNjdVNN90kqeSKH9nZ2TKZTBozZow+/vhjg9Ohotx0000MNrdjF5oztGbNGiOjoQIdO3ZM4eHhGjhwIIWUHUpISFDnzp0lSZ6entYZgffee6/mzJljZDRUkOjoaB09etToGKgg5c0ZksScITu3bt06OTs766qrrqKQQpVCKQXDlDf8WhLDr+3cmjVrlJuba3QMVJCzc4YkWecMSWLOkJ1r3ry5/vjjD6NjoIKcHYAtyToAWxIDsO3YH3/8ocLCQqNjoIKcnTMkyTpn6Pjx48wZsnM33nijjh8/bnQMoAwGncMwDL8G7M/ZOUPXXXednnnmGfXr109Tp05VYWGh3n77baPjoYJQTNg3BmAD9uXcOUN9+vTRF198ITc3N82cOdPYcKgwvFajqmKmFAyTmpqqvLw8hYWFyWw2a/Lkyfrtt9/UqFEjPf/88woMDDQ6IipAy5Yt9eOPPyo8PNzoKKgEzBlyDH+fMwT7YzabZTab5eJS8lnm3Llzra/XI0eOlJubm8EJYWt9+/bVp59+yqoZB8GcIcfAazWqKkopAECFOHbsmMLCwuTkxJni9u61117TQw89xCmaAFDNrFu3Tu3atZO7u7vRUVDBvvzyS916663y9vY2OgpQCqUUqoSbbrpJn3zyCZ/I2bG0tDRt3LhRp06dktlsLnXfkCFDDEqFiuTn56ft27fziRxgR6Kjo7VkyRJWu9qpQ4cOaeXKleW+Vo8fP96gVKhIvFYDMBozpVAlMPzavn333Xe65557lJWVJT8/P5lMJut9JpOJUspO8ZmHfSsuLtbMmTO1fPnyct/ArlixwqBkqEgMwLZf//3vf/XQQw+pZs2aCg0NLfNaTSlln3ittm/Z2dl6/fXXz/tafeTIEYOSAX+hlAJQ4R5//HGNGDFCr776qry8vIyOA8AGRo8erZkzZ+qmm25Sy5YtS72BBVD9vPzyy3rllVf09NNPGx0FgI3cf//9Wr16te69917Vrl2b12pUSZRSqBIiIiLk6upqdAxUkOPHj+vRRx+lkHIwzz77rIKCgoyOgQoyd+5czZs3T3379jU6CirRNddcI09PT6NjoAKcOXNGgwcPNjoGKtlHH32kkJAQo2Oggvz444/64Ycf1KVLF6OjAOfF9FlUCbt372Y+hR3r3bu3Nm/ebHQMVLJx48Yx+NqOubm5qWHDhkbHQCVbsmQJ8x/t1ODBg/Xzzz8bHQOV7O6772bwtR0LDAzkA0JUeQw6h6EYfm2/vv32W+ufT58+rUmTJmn48OGKjo4usyrulltuqex4qCDMGXIcb731lo4cOaKpU6dyOoADYAC2fZoyZYr1z9nZ2Xr77bd10003lfta/eijj1Z2PFQQ5gw5jv/9739avHixZs2axRkLqLIopWCYfxp+nZqaamA6XCknp4tbiGkymVRcXFzBaVBZHn74YeucofJmF7zzzjsGJYMtDBw4sNTtFStWKCgoSC1atCjzBnbBggWVGQ0V6J8GYG/dutXAdLgSkZGRF7WfyWSiqLAjd9111wXnDI0ePdqgZLCFNm3alDqmhw8flsViUf369cu8VvPvN6oCSikYpnHjxurbty/DrwE7UrNmTc2ePZs5Q3Zq+PDhF73vjBkzKjAJKlNERIT+/e9/MwAbsBMBAQHMGbJjEydOvOh9J0yYUIFJgItDKQXDeHt7a9euXYqKijI6CgyQlpbGvCE7FBYWplWrVqlx48ZGRwFgI35+ftq+fTuv1w6muLhYu3btUkREhAIDA42OAxuKjIzUkiVL1KxZM6OjAACDzmEchl87jjfeeENfffWV9fbgwYMVFBSkOnXqaMeOHQYmg609/vjjeu+998TnHfYvNzdXOTk51tvx8fF69913GZRshxiA7Rgee+wxffrpp5JKCqlrr71WV111lcLDw7Vq1Spjw8GmXnrpJY0fP77Uv+GwT0ePHtWxY8estzdu3KjHHntMH3/8sYGpgNJYKYVKxfBrxxQZGakvvvhCnTt31rJly3T77bfrq6++0rx585SQkMCbnWqOOUOOqVevXho4cKAefPBBpaWlqUmTJnJzc1NycrLefvttPfTQQ0ZHxBVgALbjqVu3rhYtWqR27dpp0aJFGjVqlFauXKnPP/9cK1as0Lp164yOiCvAnCHHdM011+iBBx7Qvffeq5MnT6px48Zq2bKlDh06pEceeYQLVaBKoJRCpWL4tWPy9PTUwYMHFR4ertGjRysvL08fffSRDh48qA4dOujMmTNGR8QVYM6QY6pZs6ZWr16tFi1a6JNPPtH777+vbdu26ZtvvtH48eO1b98+oyPiCjAA2/F4eHjo8OHDqlu3rh544AF5eXnp3XffVVxcnGJiYpSRkWF0RFwB5gw5psDAQP3+++9q0qSJpkyZoq+++krr1q3Tzz//rAcffJB/v1EluBgdAI7l3EvOwjEEBgbq6NGjCg8P19KlS/Xyyy9LkiwWC+WjHaBockw5OTny9fWVJP38888aOHCgnJyc1LFjR8XHxxucDlcqLi7O6AioZCEhIdq7d69q166tpUuXatq0aZJKftadnZ0NTocrRdHkmAoLC+Xu7i5J+uWXX6xnojRt2lSJiYlGRgOsmCmFKiUtLc3oCKgAAwcO1N13360bbrhBKSkpuvHGGyVJ27ZtU8OGDQ1OB1tizpDjaNiwoRYtWqSjR4/qp59+Uq9evSRJp06dkp+fn8HpUJGKi4u1fft2VrnameHDh+v2229Xy5YtZTKZ1LNnT0nShg0b1LRpU4PTwZaYM+Q4WrRooenTp+vXX3/VsmXL1KdPH0nSiRMnVKNGDYPTASUopWAYhl87jnfeeUcPP/ywmjdvrmXLlsnHx0eSlJiYqH//+98Gp4Mt3XrrrZo9e7akkpK5ffv2euutt3TrrbdaP3WHfRg/fryeeOIJ1a9fXx06dFCnTp0klayaatOmjcHpYEsMwHYML774oj755BM98MADWrdunXV1hbOzs5555hmD08GW7r77bq1cuVKSdPLkSfXs2VMbN27Uc889p0mTJhmcDrb0xhtv6KOPPlK3bt101113KSYmRlLJnN/27dsbnA4owUwpGIbh14D9Yc6QYzl58qQSExMVExNjnRm4ceNG+fn5sbLCjjAAG7AvzBlyLMXFxcrIyFBgYKB12x9//CEvLy8FBwcbmAwowUwpGObkyZMKDw+XJH3//fe6/fbb1atXL+un7rA/e/fuVUJCggoKCkpt50qL9oM5Q44lNDRUoaGhpbbxyav9SU5Oth7nJUuWaPDgwWrcuLFGjBih9957z+B0sKXs7GytXr263NdqrrJoP5gz5FicnZ1LFVKSVL9+fWPCAOWglIJhGH7tOI4cOaIBAwZo165dMplMOrtA8+yliTne9uPsnKEBAwbop59+0pgxYyQxZ8hebd682bq69dw3sAsWLDAoFWyNAdiOYdu2berbt69ycnKUnZ2toKAgJScnW1dTUErZj7Nzhm666SYtW7ZML730kiTmDNmr+fPnn/e1euvWrQalAv7CTCkYhuHXjmP06NGKjIzUqVOn5OXlpT179mjNmjVq164d80jsDHOGHMfcuXPVuXNn7du3TwsXLlRhYaH27NmjFStWyN/f3+h4sCEGYDuGMWPGqF+/fjpz5ow8PT31+++/Kz4+Xm3bttV//vMfo+PBhpgz5DimTJmi4cOHKyQkRNu2bVP79u1Vo0YNHTlyxPreCzAaM6VgmMLCQr333ns6evSohg0bZn3D+s4778jX11f333+/wQlhKzVr1tSKFSvUqlUr+fv7a+PGjWrSpIlWrFihxx9/XNu2bTM6ImyIOUOOoVWrVho5cqRGjRolX19f7dixQ5GRkRo5cqRq166tiRMnGh0RNjR//nwdPXpUgwcPVt26dSVJs2bNUkBAgG699VaD08EWAgICtGHDBjVp0kQBAQFav369mjVrpg0bNmjo0KHav3+/0RFhQ8wZcgxNmzbVhAkTdNddd1lfq6OiojR+/HilpqZq6tSpRkcEKKUAVLzAwEBt3bpVkZGRatCggT755BN1795dsbGxio6OVk5OjtERAVwib29v7dmzR/Xr11eNGjW0atUqRUdHa9++ferRowdzSYBqplatWvrtt9/UqFEjNW7cWO+//7569+6t/fv3q23btsrOzjY6IoBL5OXlpX379ikiIkLBwcFatmyZYmJidOjQIXXs2FEpKSlGRwSYKQXjMfza/rVs2dK6iqJDhw6aPHmy3Nzc9PHHHysqKsroeLAx5gw5hsDAQGVmZkqS6tSpo927dys6OlppaWkUzXaIAdj2r02bNtq0aZMaNWqk6667TuPHj1dycrI+//xztWzZ0uh4sDHmDDmG0NBQpaamKiIiQvXq1dPvv/+umJgYxcXFibUpqCoopWAYhl87jueff976CeukSZN0880365prrlGNGjX01VdfGZwOtjR37lwNGTJEvXv31s8//6xevXrp4MGDSkpK0oABA4yOBxu69tprtWzZMkVHR2vw4MEaPXq0VqxYoWXLlun66683Oh5siAHYjuHVV1+1Fs2vvPKKhgwZooceekiNGjXSZ599ZnA62NKUKVP03HPPadiwYVq8eLGGDx+u2NhYbdq0SaNGjTI6HmyoR48e+vbbb9WmTRsNHz5cY8aM0fz587V582YNHDjQ6HiAJE7fg4H69esnZ2dnffLJJ4qMjNTGjRuVkpKixx9/XP/5z390zTXXGB0RFSg1NVWBgYHWEhL2gTlDjiM1NVV5eXkKCwuT2WzW5MmTraf+PP/882UuP43qq1u3bmrcuLGmT58uf39/7dixQ66urvq///s/jR49mjc2QDXDnCHHYTabZTab5eJSshZl7ty51tfqkSNHys3NzeCEAKUUDMTwa8D+MGcIsD8MwAbsC3OGAFQlnL4HwxQXF8vX11dSSUF14sQJNWnSRBERETpw4IDB6XClLuWTc+YM2Q/mDNm3jIyMi97Xz8+vApOgMrm6ulqvpBkcHKyEhAQ1a9ZM/v7+Onr0qMHpcCXatGlz0SuWmTNkP5gzZN927tx50fu2atWqApMAF4dSCoZh+LV98/f3NzoCDMCcIfsWEBDwj29gLRaLTCYTcwHtCAOw7Vf//v2NjgADMGfIvrVu3brUvN7z4bUaVQWn78EwP/30k7KzszVw4EAdPnxYN998sw4ePGgdft2jRw+jI+IKfPvtt+rTpw/nqjsY5gzZt9WrV1/0vtddd10FJkFl2rx5szIzM9W9e3edOnVKQ4YMsf5cf/bZZ4qJiTE6Ii7TlClT9MADD8jDw0MJCQmqW7eudVUc7BdzhuxbfHz8Re8bERFRgUmAi0MphSqF4df2w9nZWSdPnlStWrXk7OysxMREBQcHGx0LwBUYOHCgZs6cKT8/P82ePVt33HGH3N3djY4F4DK5uLjoxIkTCg4O5rUasBNXXXWVli9frsDAQE2aNElPPPGEvLy8jI4FnBelFIAKERoaqv/+97/q16+fnJyclJSUpFq1ahkdCxWAOUOOw83NTfHx8apduzZvYAE7UK9ePY0bN059+/ZVZGSkNm/erJo1a553X1RfzBlyHJ6enjp06JDq1q3LazWqBUopVCqGXzuOF198UZMmTbqoVW+cz169OTk5MWfIQbRq1UpXXXWVunfvruHDh2vKlCnnLRqHDBlSyelgSwzAdgwff/yxHnnkERUVFZ13H/79tg9nX6uZM2T/OnXqJB8fH3Xt2lUTJ07UE088IR8fn3L3HT9+fCWnA8qilEKlGj58+EXvO2PGjApMgsqwf/9+HT58WLfccotmzJihgICAcve79dZbKzcYbIo5Q47jt99+09ixYxUbG6vU1FT5+vqWW1yYTCalpqYakBC2MnHixIved8KECRWYBBUtMzNT8fHxatWqlX755RfVqFGj3P2YHVa9MWfIcRw4cEATJkxQbGystm7dqubNm1vnh/2dyWTiQwVUCZRSqFQMv3ZMEydO1JNPPsn57HaKOUOOycnJSSdPnuSUADvFAGzHM2vWLN155538+22nmDPkmHitRnVAKYVKxfBrwP4wZ8gxxcfHq169elyYwk4xANsxnD01D/aPOUMAqqqy6/iAClSrVi39/vvv6tevH78I2bk+ffroxRdfVMeOHS+4X2Zmpj788EP5+Pho1KhRlZQOttS0aVONGzdO3bt3l8Vi0bx585gzZKcSEhKsw44v5vSO48ePq06dOhUdCxUgLCxM33zzjfr27SuLxaJjx44pLy+v3H0ZgF19tWjRQuPHj9fAgQMvuIr90KFDevvttxUREaFnnnmmEhPCVlq3bq3hw4era9euslgs+s9//sOcITv1+++//+Pv32fl5OQoLi5OLVq0qOBUwPmxUgqViuHXjuPTTz/V+PHj5e/vr379+qldu3YKCwuTh4eHzpw5o71792rt2rVasmSJbrrpJr355pu8sammmDPkOEJCQtS/f3/df//9uvrqq8vdJz09XfPmzdN7772nBx54QI8++mglp4QtMADbMSxfvlxPP/20jhw5ohtuuOG8r9V79uzRww8/rGeffVb+/v5Gx8ZlYM6Q42jUqJGioqJ0//33q2/fvvL29i6zz969e/W///1PM2bM0BtvvMGHhjAUpRQqHcOvHUd+fr6+/vprffXVV1q7dq3S09MllfzC07x5c/Xu3Vv33XefmjVrZnBS2AqzC+xbSkqKXnnlFX322Wfy8PBQ27Zty7yB3bNnj6666iq98MIL6tu3r9GRcQUYgO041q5dq6+++kq//vqr4uPjlZubq5o1a6pNmzbq3bu37rnnHgUGBhodEzbCa7V9Kyws1LRp0/TBBx/oyJEjaty4canX6v379ysrK0sDBgzQs88+q+joaKMjw8FRSsEwDL92POnp6crNzVWNGjXk6upqdBxUAOYMOYbc3Fz98MMPWrt2bblvYFu2bGl0RNgQA7ABoHravHlzua/V3bt3V1BQkNHxAEmUUgCAK/T3OUMXgzlDQNXH3EfAvjBnCEBVxbV9Uan69Omj33///R/3y8zM1BtvvKEPPvigElKhoqWkpFj/fPToUY0fP15PPvmk1qxZY2Aq2MrVV1+tkSNHatOmTefdJz09Xf/973/VsmVLffPNN5WYDsDlaNGihebOnauCgoIL7nfo0CE99NBDev311yspGSpCZmamtmzZoqysLEnS1q1bNWTIEA0ePFhffPGFwelgC/fee6969+6tr7/+WtnZ2eXus3fvXj377LNq0KCBtmzZUskJATgqVkqhUjH82rHs2rVL/fr109GjR9WoUSPNnTtXffr0UXZ2tpycnJSdna358+erf//+RkfFFWDOkONJS0vTwoULrfNncnJyVKtWLbVp00a9evVSly5djI6IK8QAbMexZs2a/2/v3sNruvP2j987URHnqGMyIiJF45g4VAhSQtBRUdrpQ0tVtagaz7QOnVaQdlJlWprRDjUIM9NSmpqh5SkhIcUomgTboYhD1aF1ToVI4veHn9RuklYnO3vtrPV+XZfrufb67tG74xmfne/+rnvpt7/9rbKysuTj46MPP/xQAwcOlJ+fnzw9PbVv3z7NnTtXI0aMMDoqSoCeIevZt2+fli5dWuysHjhwILdlwy2wKQWXo/zaOnr37q1y5cpp0qRJ+vvf/67Vq1crKipK8+fPlyS98MIL2rlz512dnoP7o2fI/L799lvFxMTon//8p3x9fdW+fXv5+vrK29tb58+f1549e7Rz5041aNBAU6ZM0e9+9zujI6OEKMA2vy5duui+++5TbGysFi5cqLffflujRo1SXFycJOn111/XihUrlJaWZmxQOA09Q+a2a9cuTZgwQampqerUqVORs3rz5s26fPmyJkyYoHHjxrE5BUOxKQXDUX5tXjVr1tSGDRvUsmVLZWVlqWrVqvryyy/Vpk0bSbeexNihQwddvHjR2KAA7kqdOnU0dOhQPfXUUwoODi7yPdnZ2Vq5cqXi4+M1YMAAvfTSSy5OCeDXqF69urZt26amTZsqJydH3t7e2rVrV8FTFQ8dOqSQkBBduXLF4KQA7kbDhg01fvx4DRo0qNinnEvS1q1b9c4776hly5b64x//6LqAwE+wKQWg1Pz0kcNVqlRRenq6AgMDJUlnzpyRr6+v8vLyjIwJ4C6dO3dO9957b6m9H4DrMasBc7lx48av+qL/174fcLZyRgeAdd35w8qJEyc0f/58ZWdnq2/fvurSpYvB6eAsP316E09zMi96hszv124wsSFV9l25ckUHDx5UkyZNVLlyZe3atUuzZ89Wdna2oqOjNXjwYKMjooRsNpvDbP7pa5gLPUPm92s3mNiQgtE4KQWXo/zaOjw8PNS7d++CDzerVq1St27dVKlSJUm3+sXWrl3Lt69lHD1D1vHvf//7rt/78MMPl2ISuAIF2Nbg4eGh5s2bq1y5W99VZ2RkqGnTpipfvrwkKTc3V3v37mVWl3H0DFlHfHz8Xb937NixpZgEuDtsSsHlKL+2jmHDht3V+xYtWlTKSVCa6BmyDg8PD4fXNptNd36MuPN0BT/Aln0UYFvDtGnT7up9U6ZMKeUkKE30DFlHw4YNHV5/9913unr1asGf+8WLF1WxYkXVrl1bR44cMSAh4IhNKbgc5deAudAzZE3r16/XxIkTFRcXp7CwMEm3fph59dVXFRcXpx49ehicECVFATZgHvQMWdMHH3yg9957TwsWLFCTJk0kSQcOHNCIESP03HPPcQs23AKbUnA5CjUBoOxr3ry55s6dq/DwcIfrmzdv1rPPPqt9+/YZlAzOwrwGgLKtUaNGWrFihUJCQhyu79y5UwMHDlRmZqZByYAfUXQOQ1B+bQ1nz54t+GFGktLS0jRr1iwdOnRI9erV05gxYxQREWFcQDgFPUPWdPjw4SJvAalWrZqOHj3q8jxwPgqwrWH79u1q06aNPD09JUmrV6/WzJkzC2b12LFjNWTIEINToqToGbKmU6dOKTc3t9D1vLw8nTlzxoBEQGGclILLUX5tHZ6enjp16pRq166tLVu2KCIiQh07dlT79u2VlpamjRs3KikpiactlnH0DFlTly5dVKFCBf39739XnTp1JN06OTNkyBBdu3ZNKSkpBidESVGAbQ13zupVq1YpOjpaTzzxhB544AF99dVXSkhI0EcffaT+/fsbHRUlQM+QNfXt21cnT57U3/72N4WGhkq6dUrq2WeflZ+f36/6YhEoLWxKweUov7aOO2/96Nmzp+rXr68FCxYUrI8bN067d+9WUlKSgSnhTPQMWcehQ4fUv39/HTx4UPXr15ekgqeqrly5UkFBQQYnRElRgG0Nd87qzp07Kzw8XG+88UbBub3R5wAAM1xJREFUelxcnFatWqWtW7camBLORM+QdXz33XcaOnSo1q5dW9ARlpubq6ioKCUkJDjc0QAYhU0pAKXmzg+6vr6+SkxMVIcOHQrW9+7dq4iICH333XcGpoQz0TNkLTdv3tS6deu0f/9+SdL999+vyMhIbvECypA7Z3WdOnX02WefFTx8Rrq1WdGhQwdduHDBwJRwJnqGrOfgwYMFs7pp06Zq3LixwYmAH9EpBaBUXblyRRUqVFCFChUKbtm8rUKFCrp69apByVAa6BmyFpvNpp49e6pLly7y8vJiMwooo+x2u06fPi1vb2/l5+cXWi+qkwZlFz1D1hMQEKCbN2+qUaNGBbdkA+7C45ffAjjX2bNnHV6npaVp6NCh6tSpkwYOHKjk5GRjgqFUNG7cWD4+Pjp69Kh27NjhsLZ37175+voalAyloV27dvrDH/7g8KH2zJkzGj9+vNq3b29gMjhbfn6+XnvtNfn5+aly5coF36xPnjzZ4TZdlF3bt2936ItavXq1unbtKj8/P7Vt21ZLliwxMB2cqXv37mrdurWOHz+uL774wmHtq6++kr+/v0HJUBq6d++u5557Trt27Sq4tnPnTo0aNUqRkZEGJoOzXb16VcOHD1fFihXVrFkzHT9+XJL0wgsvaPr06QanA25hUwouV69evYKNqS1btqh9+/Y6duyYOnXqpMuXL6tHjx7atGmTwSnhDBs3btSGDRu0YcMGbdy4UV27dnVYz8zM1LPPPmtQOpSGhQsX6tSpU/L391dQUJCCgoLk7++vkydPslFhMq+//roSEhI0Y8aMguJr6dYtnH/7298MTAZnCQsL07lz5yTdeihJv379FBAQoFdeeUUhISEaPny4PvnkE4NToqQyMzN15MgRZWZmKjMzU08++aTDek5OjiZOnGhQOpSGhQsXqm7dumrbtq28vLzk5eWl9u3bq06dOvz9bTIvv/yy0tPTlZycrAoVKhRcj4yM1LJlywxMBvyITim4HOXXgLnRM2QNQUFBmjdvnrp3764qVaooPT1dgYGB2r9/v8LCwuifMQEKsAFzo2fI/Bo0aKBly5apQ4cODrP60KFDCg0N1eXLl42OCNApBWPt2bNHsbGxDtdGjBihiIgIYwIBKDF6hqzh5MmTRT5hLz8/Xzdu3DAgEUrTwYMHNXv2bIdrAwYM0MyZM40JBKDE6Bkyv++++67IJ+z98MMPfD6D2+D2PRjiypUrunz5MuXXFjd06FB169bN6BhwInqGrCM4OFibN28udL2oJzqh7LLb7crIyKAA28IiIyMVGBhodAw4ET1D1tG2bVt9+umnBa9vb0T97W9/U1hYmFGxAAdsicMQt48H37x5Uzt27HD4AYbya+vw8/OThwd742by+uuva/HixZoxY4ZGjBhRcL158+aaPXu2hg8fbmA6OFNMTIyGDh2qkydPKj8/X4mJiTpw4ICWLFmi1atXGx0PTtK9e3fdbnr44osv1K5du4I1CrCtoX///vr++++NjgEnurNnqFevXgXXIyMjNXXqVE2aNMnAdHCmuLg49e7dW3a7Xbm5uXrnnXdkt9u1ZcsWpaSkGB0PkESnFAzw078A69Wr53AP+zvvvKOcnByNHz/e1dEAlBA9Q9ayefNmxcbGKj09XVlZWQoNDVVMTIx69uxpdDQ4wbFjxxxeV65cWffee2/B69tP3xsyZIhLcwEoGXqGrOXw4cOaPn26w6yeOHGiWrRoYXQ0QBKbUgAAJ/L29tb+/fvVoEEDhw+6drtd7du3V1ZWltERAQCwtIoVK2rPnj0KDAx0mNXp6enq0qWLLl26ZHREABbC7XsASpXdbtecOXO0detWnT59WpJUt25dhYWFacyYMQoODjY4IZzpds9QgwYNHK7TM2ReOTk5Onv2bKG+IW7rAszh8OHDGjFihDZs2GB0FDjJ7Z6hF154QRI9Q2aXn5+vQ4cOFTmru3TpYlAq4EdsSsHtDB06VCdOnODDjwmsWbNG0dHRCg0NVb9+/VSnTh1J0pkzZ7Ru3TqFhobqX//6l6KiogxOCmehZ8g6vv76az399NPasmWLw/WbN2/KZrMpLy/PoGRwlcjISB05ckRHjhwxOgpKUVZWFt0zJkPPkHVs27ZNgwYN0rFjx/TTG6SY1XAXbErB7VB+bR6TJk3SxIkTFRsbW2ht6tSpmjp1qsaPH8+mlIn069dPq1atUmxsrCpVqqSYmBiFhoZq1apV6tGjh9Hx4ERPPfWUypUrp9WrV6tevXo8WtqCKMA2h/j4+J9dP3nypIuSwFXCw8OVlpam6dOnq0WLFvr8888VGhqqrVu30jNkMiNHjiw4GceshruiUwpAqfH29lZaWpqaNGlS5PqBAwfUunVrZWdnuzgZgJKqVKmSdu7cqaZNmxodBUAJeHh4qF69eipfvnyR6zk5OTp9+jQnKoAyqFKlSkpPT1dQUJDRUYBicVIKQKkJCAjQp59+Wuym1KefflqoewjmQM+Q+QUHB3NKBjCBBg0a6M0339Rjjz1W5HpaWpratGnj4lQobfQMWcMDDzygQ4cOsSkFt8amFAxB+bU1xMbGatCgQUpOTlZkZKRDp1RSUpLWrl2rDz74wOCUcCZ6hqzjzTff1IQJExQXF6cWLVronnvucVivWrWqQcngKhRgm0ObNm20c+fOYjelbDZboS4alG30DFnHCy+8oBdffFGnT58ucla3bNnSoGTAj7h9Dy53Z/l1VFRUofLrnTt3Un5tIlu2bFF8fHyRG5C///3vecqLyXTq1EnlypXTpEmTiuwuaNWqlUHJ4Gy3u/9++mfMBqR1pKenKzQ0lD/rMs5ut+vq1atq27Ztkes3btzQt99+y8lmE2ndurUaN26sadOmFTmrq1WrZlAyOFtRPb23N5qZ1XAXbErB5Vq1aqV+/foVWX4t3SrATkxMVEZGhouTASgpeoas45ee0NS1a1cXJUFpuZsC7D//+c/8UAOUMfQMWcexY8d+dp3NZrgDNqXgcpRfW9v06dM1cuRIVa9e3egoKAXt2rXTrFmzFB4ebnQUACVEAbZ1MavNrVu3bpowYYJ69epldBQAoFMKrkf5tbXFxcXpscce44OuSdEzZB2bNm362XWKcss+CrCti1ltbvQMWceSJUt+dn3IkCEuSgIUj5NScLnly5dr0KBB6t2798+WXw8YMMDgpCgNVapUUXp6ugIDA42OglJAz5B1FNdTcRt/1mXfwIED1ahRI7355ptFrqenpyskJKTQk7tQ9jGrzY2eIevw8fFxeH3jxg1dvXpV5cuXV8WKFXX+/HmDkgE/4qQUXO7RRx+Vn5+f4uPj9dZbbxUqv05OTqb8GiijNm7caHQEuMiFCxccXt+4cUNfffWVJk+erD/96U8GpYIzxcbG6urVq8WuBwcHKzMz04WJADgD/7u1jp/OaunWk5JHjRql8ePHG5AIKIyTUgBc6sSJE/L19ZWnp6fRUQCUgpSUFP3hD3/Qzp07jY4C4L/ErAbMbceOHXriiSe0f/9+o6MAnJSCe6BQ0zrq169vdASUInqGUKdOHR04cMDoGCglzGtrYFabGz1DKFeunL799lujYwCSOCkFN1G1alWlpaXRXWBC7733nhITE1WjRg0999xz6t69e8Ha999/r/bt2+vIkSMGJoQz0TNkHRkZGQ6vb968qVOnTmn69OnKzc1VamqqQclQmpjX5sSsthZ6hqzj3//+t8Pr27N6zpw5ql+/vtasWWNQMuBHnJSCW2Bv1Jzi4+P18ssva9iwYbp06ZL69OmjqVOn6uWXX5Z0a4Pi2LFjBqeEM9EzZB2tW7cuKMa9U4cOHbRw4UKDUqG0Ma/Nh1ltPfQMWUd0dLTDa5vNplq1aqlbt2566623jAkF/ASbUgBKzbx58zR//nwNGjRIkjRq1ChFR0crOztbsbGxBqdDaahWrVqhaz169FD58uXpGTKZnxblenh4qFatWqpQoYJBiQD8N5jVkKT77rtP06dPp2fIZHg6KsqCwvdZAAaw2+1q0KCB0THgZJmZmerYsWPB644dO2rDhg16//33C76BhTXQM2Q+KSkpqlu3rho0aKAGDRqofv36qlChgnJycn6xrwRlF/PafJjVuI2eIfMp7imqbDrDndApBaDU+Pv765///Kc6d+7scN1ut6tbt26KiorSP/7xD3qGTISeIevw9PTUqVOnVLt2bYfr586dU+3atfnfNVBGMKuth54h62BWoyzg9j0YgkJNawgPD1diYmKhD7rBwcFKSkrSgw8+aFAylBZ6hqzj5s2bDiX2t33zzTdF3saJsol5bX7MauuhZ8g6ipvV6enpqlGjhgGJgMLYlILLUahpHZMmTSq2Q6hZs2basGGDPv74YxenQmmiZ8j8QkJCZLPZZLPZ1L17d5Ur9+NHiby8PGVmZqpXr14GJoSzMK+tgVltPfQMmZ+Pj0/BrG7cuHGhJyFnZWVp5MiRBiYEfsTte3C5Zs2a6ZVXXiko1NyyZYuio6M1cuRIxcbG6syZM/L19eU4KVAGLVmyRL/73e/k5eXlcD0nJ0dLly7VkCFDDEoGZ5k2bVrB/33xxRdVuXLlgrXy5csrICBAAwYMUPny5Y2KCCdhXgPmFBsbq5deekkVK1Z0uJ6dna2ZM2cqJibGoGRwlsWLF+vmzZt6+umnNXv2bIcTzLdndVhYmIEJgR+xKQWXq1ixoux2uwICAgqu7dmzR5GRkRo2bJjGjRvHh1wTunjxorZv366zZ886fENns9n05JNPGpgMzkR3gXUsXrxYv/vd7zgFZ2LMa+thVlsDs9o6UlJS1KlTJ4dTzYC74f874XI1a9bUiRMnHD7kNm/eXBs2bFC3bt146ocJrVq1SoMHD1ZWVpaqVq3qcISYD7rmQs+QdQwdOlQXL17UP/7xDx0+fFjjx49XjRo1tGvXLtWpU0d+fn5GR0QJMa+thVltHfQMWUfXrl11+PBhLVq0SIcPH9Y777yj2rVra82aNfL391ezZs2MjghwUgquN2jQINWpU0ezZs0qtLZ37149+OCDOnfuHN/SmEjjxo3Vp08fxcXFFToqDnO43TOUnp6uZs2aFdsz9NFHHxmYEs6UkZGhyMhIVatWTUePHtWBAwcUGBioV199VcePH9eSJUuMjogSYl5bC7Pa/G73DF26dKnQxuOdPUPvvvuugSnhTCkpKerdu7c6deqkTZs2ad++fQoMDNT06dO1Y8cOrVixwuiIACel4HoUalrPyZMnNXbsWD7kmtjtJ/mkpaUpKiqq2J4hmMf//u//6qmnntKMGTNUpUqVgut9+vQp6CBC2ca8thZmtfnNnj27oGdo2rRp9AxZwKRJk/T666/rD3/4g8Os7tatm+bMmWNgMuBHnJQCUOoeeeQRPf7443rssceMjoJSRs+QdVSrVk27du1So0aNVKVKFaWnpyswMFDHjh1TkyZNdO3aNaMjAvgVmNXWQc+QdVSuXFm7d+9Ww4YNHWb10aNH1bRpU2Y13AJ/E8FQFGpaw0MPPaTx48fLbrerRYsWuueeexzWH374YYOSwdnoGbIOLy8vXb58udD1gwcPqlatWgYkQmliXpsfs9o66BmyjurVq+vUqVNq2LChw/WvvvqKz2RwG5yUgmF+qVDz/PnzBqaDM3l4eBS7ZrPZ6CMxEXqGrOOZZ57RuXPn9NFHH6lGjRrKyMiQp6enoqOj1aVLF82ePdvoiHAS5rU1MKutg54h63jppZf0n//8R8uXL1fjxo21a9cunTlzRkOGDNGQIUM0ZcoUoyMCbErBOBRqAubTvXt3tWnTpqBn6PYx8S1btmjQoEE6evSo0RHhJJcuXdLAgQO1Y8cOXblyRb6+vjp9+rTCwsL02WefqVKlSkZHhJMwrwFzCQsL06OPPlrQM3R7Vm/fvl2PPPKIvvnmG6MjwklycnL0/PPPKyEhQXl5eSpXrpzy8vI0aNAgJSQkyNPT0+iIAJtSME6lSpW0e/duBQYGGh0FLnTt2jX6hkyMniHrSU1NVUZGhrKyshQaGqrIyEijI8HJmNfWw6w2N3qGrOf48ePas2ePsrKyFBISovvuu8/oSECB4s/pAqUsKipKO3bsMDoGXCAvL0+vvfaa/Pz8VLlyZR05ckSSNHnyZC1YsMDgdHAmeoasJzw8XKNHj9aECRPYkDIp5rU1MKut43bP0E/RM2Re/v7+6tOnjx577DE2pOB2KDqHYSjUtI4//elPWrx4sWbMmKERI0YUXG/evLlmz56t4cOHG5gOzvTwww8rNjZWH330kaRbPSTHjx/XxIkTNWDAAIPTwdmSkpKUlJRUqPxakhYuXGhQKjgb89oamNXW8fjjj2vixIlavny5bDab8vPz9cUXX+ill17SkCFDjI4HJ8rLy1NCQkKxs3rDhg0GJQN+xO17MAyFmtYRFBSkefPmqXv37g7HxPfv36+wsDBduHDB6IhwEnqGrGPatGmKjY1V27ZtVa9ePYfya0n65JNPDEoGZ2NeWwOz2jroGbKOMWPGKCEhQQ899FCRs3rWrFkGJQN+xEkpGOanO/Uwr5MnTyooKKjQ9fz8fN24ccOARCgt1apV07p16+gZsoC5c+cqISFBTz75pNFRUMqY19bArLaO8uXLa/78+Zo8eTI9Qya3dOlSffTRR+rTp4/RUYBisSkFt0ChprkFBwdr8+bNatCggcP1FStWKCQkxKBUKE3h4eEKDw83OgZKUU5Ojjp27Gh0DLgY89q8mNXW4+/vL39/f6NjoBSVL1++yM1mwJ2wKQXD5OXlKS4uTnPnztWZM2d08OBBBQYGavLkyQoICKC7wERiYmI0dOhQnTx5Uvn5+UpMTNSBAwe0ZMkSrV692uh4cDJ6hqzhmWee0QcffKDJkycbHQWljHltDcxq66BnyDpefPFFvfPOO5ozZ06hW/cAd8GmFAxDoaZ19OvXT6tWrVJsbKwqVaqkmJgYhYaGatWqVerRo4fR8eBEv9QzBPO4du2a3n//fa1fv14tW7YsVH799ttvG5QMzsa8tgZmtXX8/ve/L+gZat68ObPaxFJTU7Vx40atWbNGzZo1KzSrExMTDUoG/IiicxiGQk3AfOrVq6cZM2bQM2QBDz74YLFrNpuNb9pNhHkNmEvNmjW1ZMkSeoYsYNiwYT+7vmjRIhclAYrHSSkYhkJN64iJidGDDz6osLAwukhMjp4h69i4caPREeAizGtrYFZbBz1D1sGmE8qC4p/xC5Sy24WaP0Whpvls3bpVffv2VfXq1dW5c2e9+uqrWr9+vbKzs42OBie73TMEwDyY19bArLaO2z1D3DADwB1wUgqGoVDTOtatW6fc3Fz95z//0aZNm5SSkqL4+Hhdv35d7dq1U2pqqtER4ST0DJnbI488ooSEBFWtWlWPPPLIz76XngrzYF5bA7PaOugZMrfQ0FAlJSXJx8dHISEhP9sZtmvXLhcmA4rGphQMQ6GmtZQrV06dOnVSrVq1VKNGDVWpUkUrV67U/v37jY4GJ8rIyFDr1q0lSXv27HFYo0i17KtWrVrBn2PVqlX5M7UI5rV1MKutoXr16urfv7/RMVBK+vXrJy8vL0lSdHS0sWGAu0DROYBS9/777ys5OVkpKSm6fv26OnfurIiICEVERKhly5b8YAuUEf/+97/Vu3fvQt+qAyj7mNWAOcTHx+vZZ59VhQoVdPz4cf3mN7+RhwetPXBfbErBMBRqWoeHh4dq1aqlF198UaNHj1blypWNjgTgv+Dp6anTp0+rVq1a8vT01KlTp1S7dm2jY6GUMa+tgVkNmEO5cuX07bffqnbt2sxqlAlsSsEwPXr00NatW5Wbm6t27dqpa9euioiIUKdOneTt7W10PDjRypUrtWnTJiUnJ2vfvn0KCQkp+PY1PDxcFStWNDoiSoCeIeuoW7eu5s+fr759+8rDw0NnzpxRrVq1jI6FUsa8tgZmtbnRM2Qd/v7+evnll9WnTx81bNhQO3bsUM2aNYt9L2A0OqVgGAo1rSM6OrrgnvZLly5p8+bNWr58uX7729/Kw8ND165dMzYgSoSeIesYOXKk+vXrJ5vNJpvNprp16xb73ry8PBcmQ2liXlsDs9rc6BmyjldffVUvvPCCxowZI5vNpnbt2hV6z82bN2Wz2ZjVcAtsSsFQFGpax7lz55SSkqLk5GQlJydr79698vHxUefOnY2OhhLq379/wS09CQkJxoZBqZo6daoef/xxHTp0SA8//LAWLVqk6tWrGx0LLsC8tgZmtXn5+PgU9AoNGzaMniETe/bZZ/U///M/OnbsmFq2bKn169fr3nvvNToWUCxu34NhKNS0jhYtWshut6tGjRrq0qWLIiIi1LVrV7Vs2dLoaHACeoasadq0aRo/fjy39FgA89oamNXmRs+QNS1evFiPP/54wSk5wB2xKQXDUKhpHe+++64iIiJUu3ZteXl5qWrVqkZHghPRMwSYG/PaGpjV5kbPkHXcvjUPKCs4swnDJCYmavDgwVq6dKlq1aqljh076o9//KM+//xzXb161eh4cJKLFy/Kbrera9euqlu3rnx8fFS3bl29/PLL/DmbxO2eIU9Pz4KeIU9PzyJ/oWzr1auXtm3b9ovvu3Llit588029++67LkiF0sa8Nj9mtfm9+uqrGjdunAIDAwt6hho2bOjwKyAgQA0bNjQ6KkqoWbNmWrp0qXJycn72fV9//bVGjRql6dOnuygZUDROSsEt3Fmo+eGHH1KoaRLnz59XWFiYTp48qcGDB+v++++XJNntdn3wwQdq2rSpUlNTlZGRoW3btmns2LEGJ8Z/a//+/XfVM9SvXz/XBoNTLViwQDExMapWrZr69u2rtm3bytfXVxUqVNCFCxdkt9uVmpqqzz77TA899JBmzpzJN+4mw7w2H2a1dVy5cuWueoZatWrl4mRwpqSkJE2cOFFHjhxRjx49ip3Ve/fu1ZgxY/THP/5R1apVMzo2LIxNKRjq5wo1P/nkE6PjoYTGjRunpKQkrV+/XnXq1HFYO336tHr27KkmTZro888/V3x8vIYOHWpQUjgLPUPmd/36dS1fvlzLli1TamqqLl26JEmy2WwKDg5WVFSUhg8fXvCDLcyBeW1ezGrroWfIGlJTU7Vs2TJt3rxZx44dU3Z2tmrWrKmQkBBFRUVp8ODB8vHxMTomwKYUjEOhpvkFBARo3rx5ioqKKnJ97dq16tOnj6ZMmaIpU6a4OB0AZ7h06ZKys7N177336p577jE6DkoB89rcmNXWQM8QAHfFphQMQ6Gm+Xl5eenw4cP6zW9+U+T6N998o4CAAOXm5ro4GZypV69emjp1qjp06PCz77ty5Yree+89Va5cWc8//7yL0gEoKea1uTGrrSE4OFgxMTF65JFHVL58+WLf9/XXX+vtt99WgwYNNGnSJBcmBGBV5YwOAGu6Xag5ZcoUXbhwQZJUq1YtDRs2TJMnT+bWH5OoWbOmjh49WuwH3czMTB5HbAKPPvqoBgwY8Kt6hlD2nTt3rqCP5MSJE5o/f76ys7PVt29fdenSxeB0cBbmtfkxq63hL3/5iyZOnKjRo0ffVc/QqFGjjI6MErpy5YoOHjyoJk2aqHLlytq1a5dmz56t7OxsRUdHa/DgwUZHBCRxUgoGoFDTOp5++mkdPnxY69atK/St3PXr1xUVFaXAwEAtXLjQoIRwFnqGrGP37t3q27evTpw4ofvuu09Lly5Vr1699MMPP8jDw0M//PCDVqxYoejoaKOjooSY19bArLYWeoasYdOmTfrtb3+rrKws+fj46MMPP9TAgQPl5+cnT09P7du3T3PnztWIESOMjgqwKQXXo1DTOr755hu1bdtWXl5eev7559W0aVPdvHlT+/bt03vvvafr16/ryy+/5OlcJkTPkHn17t1b5cqV06RJk/T3v/9dq1evVlRUlObPny9JeuGFF7Rz505t27bN4KQoKea1NTCrAfPp0qWL7rvvPsXGxmrhwoV6++23NWrUKMXFxUmSXn/9da1YsUJpaWnGBgXEphQMQKGmtWRmZmr06NH6/PPPdfuvG5vNph49emjOnDkKCgoyOCGAX6NmzZrasGGDWrZsqaysLFWtWlVffvml2rRpI0nav3+/OnTooIsXLxobFCXGvLYOZjVgLtWrV9e2bdvUtGlT5eTkyNvbW7t27VKrVq0kSYcOHVJISIiuXLlicFKATSkYgEJNa7pw4YK+/vprSVJQUJBq1KhhcCKUBnqGzM/Dw0OnT58u6JipUqWK0tPTFRgYKEk6c+aMfH19lZeXZ2RMOAHz2nqY1eZHz5A1MKtRllB0DpejUNOafHx81L59e6NjoJT8Us/QrFmz6BkykZ8+VpzHjJsT89p6mNXm9ks9Q4mJibp69So9QyZgs9kcZvNPXwPuhJNScDkKNQHzoWfIOjw8PNS7d295eXlJklatWqVu3bqpUqVKkm79Pb527Vq+fTUB5jVgLvQMWYeHh4eaN2+ucuVunUHJyMhQ06ZNC/4uz83N1d69e5nVcAtsSsHlKNQEzIeeIesYNmzYXb1v0aJFpZwEpY15DZgLPUPWMW3atLt6H32AcAdsSsEQFGoC5kJ3AWBOzGvAPJjVANwRnVIwRMOGDbVmzRoKNQEToWcIMB/mNWAe9AwBcEeclAIAlBg9Q9Zx9uxZh3LrtLQ0zZo1S4cOHVK9evU0ZswYRUREGBcQAFAkeoasY/v27WrTpo08PT0lSatXr9bMmTMLZvXYsWM1ZMgQg1MCt7ApBQAoMXqGrMPT01OnTp1S7dq1tWXLFkVERKhjx45q37690tLStHHjRiUlJalLly5GRwUA3IGeIeu4c1avWrVK0dHReuKJJ/TAAw/oq6++UkJCgj766CP179/f6KgAm1IAAODu3dlJ0rNnT9WvX18LFiwoWB83bpx2796tpKQkA1MCAGBdd87qzp07Kzw8XG+88UbBelxcnFatWqWtW7camBK4xcPoAAAAoGzas2ePRowY4XBtxIgRysjIMCgRAAC408GDBzVw4ECHawMGDND+/fsNSgQ4YlMKAFBiZ8+edXidlpamoUOHqlOnTho4cKCSk5ONCYZSceXKFV2+fFkVKlQo6BG7rUKFCrp69apByQAAxdm+fbtDX9Tq1avVtWtX+fn5qW3btlqyZImB6eBsdrtdGRkZ8vb2Vn5+fqH13NxcA1IBhbEpBQAosXr16hVsTG3ZskXt27fXsWPH1KlTJ12+fFk9evTQpk2bDE4JZ2ncuLF8fHx09OhR7dixw2Ft79698vX1NSgZAKA4YWFhOnfunKRbDyTp16+fAgIC9MorrygkJETDhw/XJ598YnBKOEv37t3VunVrHT9+XF988YXD2ldffSV/f3+DkgGOyhkdAABQ9t1ZTzh16lQ9+eSThXqGpk2bRs+QCWzcuNHhdb169RxeZ2Zm6tlnn3VlJADAXbhzVs+YMUMTJkxw6Blq2LChZsyYQfm1CWRmZjq8rly5ssPrnJwcTZw40ZWRgGJRdA4AKLE7CzV9fX2VmJioDh06FKzv3btXERER+u677wxMCQCAdd05q+vUqaPPPvtMbdq0KVg/cOCAOnTooAsXLhiYEoDVcFIKAOAUV65cUYUKFegZAgDATdntdp0+fZqeIQBug04pAIBT0DMESRo6dKi6detmdAwAQBHoGYIkRUZGKjAw0OgYgCROSgEAnICeIdzm5+cnDw++8wIAd0PPEG7r37+/vv/+e6NjAJLolAIAAAAAAIAB+CoTAAAAAAAALsemFACg1NEzZC52u12jR49WSEiI6tWrp3r16ikkJESjR4+W3W43Oh4A4L9Az5B1HD58mM9lcBt0SgEASh09Q+axZs0aRUdHKzQ0VP369VOdOnUkSWfOnNG6desUGhqqf/3rX4qKijI4KQDg16BnyDqysrKUkpJidAxAEp1SAADgV2jVqpX69eun2NjYItenTp2qxMREZWRkuDgZAACQpPj4+J9dP3nypP785z8rLy/PRYmA4rEpBQAA7pq3t7fS0tLUpEmTItcPHDig1q1bKzs728XJAACAJHl4eKhevXoqX758kes5OTk6ffo0m1JwC9xLAQBwCnqGrCEgIECffvppseuffvqpGjRo4MJEAABnoGfIPBo0aKBZs2YpMzOzyF8/N8cBV6NTCgBQYvQMWUdsbKwGDRqk5ORkRUZGOvxZJyUlae3atfrggw8MTgkA+LXoGTKPNm3aaOfOnXrssceKXLfZbOKGKbgLbt8DAJQYPUPWsmXLFsXHx2vr1q06ffq0JKlu3boKCwvT73//e4WFhRmcEADwU/QMWYfdbtfVq1fVtm3bItdv3Lihb7/9lpPNcAtsSgEASoyeIQAA3Bs9QwDcEZ1SAIASo2fI2qZPn66LFy8aHQMA8DPoGbI2ZjXcFSelAAAltnz5cg0aNEi9e/f+2Z6hAQMGGJwUpaFq1apKS0tTYGCg0VEAAMUYOHCgGjVqpDfffLPI9fT0dIWEhCg/P9/FyeAKzGq4K4rOAQAl9uijj8rPz0/x8fF66623CvUMJScn0zNkYny/BQDuLzY2VlevXi12PTg4WJmZmS5MBFdiVsNdcVIKAACUSJUqVZSens63rwAAuClmNdwVnVIAgFJBd4F12O12OsMAoAxiVlsHsxruipNSAIBSQXcBAADujVkNwGiclAIAlAq+8zCv9957T5GRkXrssceUlJTksPb999/zww0AlBHMavNiVqOsYFMKAADctfj4eI0fP15NmzaVl5eX+vTpozfeeKNgPS8vT8eOHTMwIQAA1sasRlnC0/cAAKXCbrfL19fX6Bhwsnnz5mn+/PkaNGiQJGnUqFGKjo5Wdna2YmNjDU4HAPg1mNXmxKxGWcKmFACgVNSvX9/oCCgFmZmZ6tixY8Hrjh07asOGDYqMjNSNGzc0btw448IBAH4VZrU5MatRlnD7HgDAKegusIaaNWvqxIkTDteaN2+uDRs2aNGiRZowYYJByQAAv4RZbQ3MapQlbEoBAEqM7gLrCA8PV2JiYqHrwcHBSkpK0po1awxIBQD4Jcxq62BWoyzh9j0AQInRXWAdkyZN0s6dO4tca9asmTZs2KCPP/7YxakAAL+EWW0dzGqUJbabPAcUAFBCFStWlN1uV0BAQMG1PXv2KDIyUsOGDdO4cePk6+urvLw840ICAGBhzGoA7oiTUgCAErvdXXDnB93b3QXdunXTt99+a1w4lJqLFy9q+/btOnv2rPLz8wuu22w2PfnkkwYmAwD8FLPampjVcHeclAIAlNigQYNUp04dzZo1q9Da3r179eCDD+rcuXN8+2oiq1at0uDBg5WVlaWqVavKZrMVrNlsNp0/f97AdACAn2JWWw+zGmUBm1IAgBLLyMjQzp07NWzYsCLX9+zZo48//lhTpkxxcTKUlsaNG6tPnz6Ki4tTxYoVjY4DAPgFzGrrYVajLGBTCgAA/GqVKlXS7t27eXw4AABuilmNsoBOKQCAU9FdYA1RUVHasWMHH3QBoAxiVlsDsxplAZtSAACn+aXuAj7omsdDDz2k8ePHy263q0WLFrrnnnsc1h9++GGDkgEAfg6z2jqY1SgLuH0PAOA0dBdYh4eHR7FrNpuNolwAcFPMautgVqMsYFMKAOA0dBcAAODemNUA3EnxW6cAAPxKt7sLYC3Xrl0zOgIA4C4xq62JWQ13RacUAMBp6C6wjry8PMXFxWnu3Lk6c+aMDh48qMDAQE2ePFkBAQEaPny40REBAEVgVlsHsxplAbfvAQCchu4C64iNjdXixYsVGxurESNGaM+ePQoMDNSyZcs0e/Zsbd261eiIAIAiMKutg1mNsoDb9wAATpOfn1/sLz7kmsuSJUv0/vvva/DgwfL09Cy43qpVK+3fv9/AZACAn8Ostg5mNcoCNqUAAKWC7gJzO3nypIKCggpdz8/P140bNwxIBAD4tZjV5sasRlnAphQAwGny8vL02muvyc/PT5UrV9aRI0ckSZMnT9aCBQsMTgdnCg4O1ubNmwtdX7FihUJCQgxIBAC4G8xq62BWoyyg6BwA4DR/+tOftHjxYs2YMUMjRowouN68eXPNnj2bQk0TiYmJ0dChQ3Xy5Enl5+crMTFRBw4c0JIlS7R69Wqj4wEAisGstg5mNcoCis4BAE4TFBSkefPmqXv37qpSpYrS09MVGBio/fv3KywsTBcuXDA6Ipxo8+bNio2NVXp6urKyshQaGqqYmBj17NnT6GgAgGIwq62FWQ13x0kpAIDT0F1gLZ07d9a6deuMjgEA+BWY1dbCrIa7o1MKAOA0dBdYR0xMjDZu3EhJLgCUMcxq62BWoyzgpBQAwGnoLrCOrVu36u2331Zubq7atWunrl27KiIiQp06dZK3t7fR8QAAxWBWWwezGmUBnVIAAKeiu8A6cnNz9Z///EebNm1SSkqKtmzZouvXr6tdu3ZKTU01Oh4AoBjMautgVsPdsSkFAABK5ODBg9q4caPWr1+vlStXqlq1avr++++NjgUAAP4/ZjXcFZ1SAACnobvAOt5//30NGjRIfn5+6tixo9auXavw8HDt2LFD3333ndHxAADFYFZbB7MaZQEnpQAATtOjRw9t3bqV7gIL8PDwUK1atfTiiy9q9OjRqly5stGRAAB3gVltHcxqlAVsSgEAnIruAmtYuXKlNm3apOTkZO3bt08hISGKiIhQRESEwsPDVbFiRaMjAgCKway2BmY1ygI2pQAApYLuAuu4dOmSNm/erOXLl+vDDz+Uh4cHt4UAQBnArLYOZjXcVTmjAwAAzOP9999XcnKyUlJSdP36dXXu3FkRERF69dVX1bJlS6PjwcnOnTunlJQUJScnKzk5WXv37pWPj486d+5sdDQAQDGY1dbCrIa746QUAMBp6C6wjhYtWshut6tGjRrq0qWLIiIi1LVrV36gAQA3x6y2DmY1ygI2pQAATkN3gXW8++67ioiIUO3ateXl5aWqVasaHQkAcBeY1dbBrEZZwKYUAKBU0F1gXhcvXtQrr7yiZcuW6cKFC5KkWrVqadiwYZo8eTI/0ABAGcGsNi9mNcoKOqUAAE5Fd4G5nT9/XmFhYTp58qQGDx6s+++/X5Jkt9v1l7/8RevWrVNqaqoyMjK0bds2jR071uDEAICfYlabG7MaZQknpQAATkN3gfmNGzdOSUlJWr9+verUqeOwdvr0afXs2VNNmjTR559/rvj4eA0dOtSgpACAojCrzY9ZjbKETSkAgNPQXWB+AQEBmjdvnqKioopcX7t2rfr06aMpU6ZoypQpLk4HAPglzGrzY1ajLPEwOgAAwBwuXrwou92url27qm7duvLx8VHdunX18ssv6+rVq0bHg5OcOnVKzZo1K3a9efPm8vDw4EMuALghZrU1MKtRltApBQAoMboLrKNmzZo6evSofvOb3xS5npmZqdq1a7s4FQDglzCrrYNZjbKE2/cAACVGd4F1PP300zp8+LDWrVun8uXLO6xdv35dUVFRCgwM1MKFCw1KCAAoCrPaOpjVKEvYlAIAlBjdBdbxzTffqG3btvLy8tLzzz+vpk2b6ubNm9q3b5/ee+89Xb9+XV9++aX8/f2NjgoAuAOz2jqY1ShL2JQCAJSYl5eXDh8+XOwx8W+++UYBAQHKzc11cTKUhszMTI0ePVqff/65bn+MsNls6tGjh+bMmaOgoCCDEwIAfopZbS3MapQVdEoBAEqM7gJradiwodasWaMLFy7o66+/liQFBQWpRo0aBicDABSHWW0tzGqUFZyUAgCUGN0FAAC4N2Y1AHfEphQAoMToLgAAwL0xqwG4IzalAABOQXcBAADujVkNwN2wKQUAcCq6CwAAcG/MagDugk0pAAAAAAAAuJyH0QEAAAAAAABgPWxKAQAAAAAAwOXYlAIAAAAAAIDLsSkFAAAAAAAAl2NTCgAA4L+UnJwsm82mixcv3vV/JiAgQLNnzy61TAAAAGUFm1IAAMC0nnrqKdlsNo0cObLQ2vPPPy+bzaannnrK9cEAAADAphQAADC3+vXra+nSpcrOzi64du3aNX3wwQfy9/c3MBkAAIC1sSkFAABMLTQ0VPXr11diYmLBtcTERPn7+yskJKTg2vXr1zV27FjVrl1bFSpUUHh4uL788kuH3+uzzz5T48aN5e3trQcffFBHjx4t9M9LTU1V586d5e3trfr162vs2LH64Ycfisx28+ZNTZ06Vf7+/vLy8pKvr6/Gjh3rnH9xAAAAN8emFAAAML2nn35aixYtKni9cOFCDRs2zOE9EyZM0Mcff6zFixdr165dCgoKUlRUlM6fPy9JOnHihB555BH17dtXaWlpeuaZZzRp0iSH3+Pw4cPq1auXBgwYoIyMDC1btkypqakaM2ZMkbk+/vhjzZo1S/PmzdPXX3+tlStXqkWLFk7+twcAAHBPbEoBAADTe+KJJ5Samqpjx47p2LFj+uKLL/TEE08UrP/www/661//qpkzZ6p3794KDg7W/Pnz5e3trQULFkiS/vrXv6pRo0Z666231KRJEw0ePLhQH9Ubb7yhwYMHa9y4cbrvvvvUsWNHxcfHa8mSJbp27VqhXMePH1fdunUVGRkpf39/tW/fXiNGjCjV/y4AAADcBZtSAADA9GrVqqWHHnpICQkJWrRokR566CHVrFmzYP3w4cO6ceOGOnXqVHDtnnvuUfv27bVv3z5J0r59+/TAAw84/L5hYWEOr9PT05WQkKDKlSsX/IqKilJ+fr4yMzML5Xr00UeVnZ2twMBAjRgxQp988olyc3Od+a8OAADgtsoZHQAAAMAVnn766YLb6N59991S+WdkZWXpueeeK7IXqqhS9fr16+vAgQNav3691q1bp9GjR2vmzJlKSUnRPffcUyoZAQAA3AUnpQAAgCX06tVLOTk5unHjhqKiohzWGjVqpPLly+uLL74ouHbjxg19+eWXCg4OliTdf//92r59u8N/btu2bQ6vQ0NDZbfbFRQUVOhX+fLli8zl7e2tvn37Kj4+XsnJydq6dat2797tjH9lAAAAt8ZJKQAAYAmenp4Ft+J5eno6rFWqVEmjRo3S+PHjVaNGDfn7+2vGjBm6evWqhg8fLkkaOXKk3nrrLY0fP17PPPOMdu7cqYSEBIffZ+LEierQoYPGjBmjZ555RpUqVZLdbte6des0Z86cQpkSEhKUl5enBx54QBUrVtQ//vEPeXt7q0GDBqXzXwIAAIAb4aQUAACwjKpVq6pq1apFrk2fPl0DBgzQk08+qdDQUB06dEj/93//Jx8fH0m3br/7+OOPtXLlSrVq1Upz585VXFycw+/RsmVLpaSk6ODBg+rcubNCQkIUExMjX1/fIv+Z1atX1/z589WpUye1bNlS69ev16pVq3Tvvfc6918cAADADdlu3rx50+gQAAAAAAAAsBZOSgEAAAAAAMDl2JQCAAAAAACAy7EpBQAAAAAAAJdjUwoAAAAAAAAux6YUAAAAAAAAXI5NKQAAAAAAALgcm1IAAAAAAABwOTalAAAAAAAA4HJsSgEAAAAAAMDl2JQCAAAAAACAy7EpBQAAAAAAAJf7f2nnIZlspemUAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from llm_toolkit.translation_utils import plot_times\n", + "perf_df = get_perf_df(metrics_df)\n", + "plot_times(perf_df, ylim=0.358)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "432827f5-93be-44f3-b290-60f1d17f5a74", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(252.61249999999998, 94.34583333333333, 441.3041666666667)" + ] + }, + "execution_count": 47, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "perf_df[\"train-time(mins)\"].mean(), perf_df[\"eval-time(mins)\"].mean(), perf_df[\n", + " \"train-time(mins)\"\n", + "].mean() + 2 * perf_df[\"eval-time(mins)\"].mean()" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c7b2a963-ea0f-4d32-966b-9df1f2d45f4d", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n", + "/tmp/ipykernel_22202/2495696642.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n", + " model_time = df_time[df_time[\"model\"] == model_name][\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAPdCAYAAABba9tpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAADaN0lEQVR4nOzdeZxe0+E/8M9MkpnJNomsk8hmJ2orSlBVVCzVha6UaJVStKVU06rao74t2lLV1o8qqa60VcRSUhS1VCmqRCRBJiGRjGwzmcz8/nhkYsgmZu4E7/frdV/z3HvPPffcyXXj+eScc8uam5ubAwAAAAAFKu/oBgAAAADw7iOUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACte5oxuwNmhqasoLL7yQnj17pqysrKObAwAAAPC21dzcnFdeeSWDBw9OefmK+0MJpZK88MILGTp0aEc3AwAAAOAdY9q0aRkyZMgK9wulkvTs2TNJ6ZdVXV3dwa0BAAAAePuqq6vL0KFDW/KWFRFKJS1D9qqrq4VSAAAAAG1gVVMkmegcAAAAgMIJpQAAAAAonFAKAAAAgMKZU2o1NTU1paGhoaObQRupqKhY6WspAQAAgPYllFoNDQ0NmTx5cpqamjq6KbSR8vLyrLfeeqmoqOjopgAAAMC7klBqFZqbmzN9+vR06tQpQ4cO1bvmHaCpqSkvvPBCpk+fnmHDhq3ybQAAAABA2xNKrUJjY2MWLFiQwYMHp1u3bh3dHNpI//7988ILL6SxsTFdunTp6OYAAADAu45uP6uwZMmSJDHM6x1m6Z/n0j9fAAAAoFhCqdVkiNc7iz9PAAAA6FhCKQAAAAAKJ5RilUaMGJELL7zwbVc3AAAAsPYy0fkaemLTzQo932b/feJNld9tt92y9dZbt0ngc//996d79+5vqY4rrrgiX/va1zJnzpw2rxsAAAB4+xFKvUs1NzdnyZIl6dx51bdA//79260d7Vk3AAAAsPYyfO8d6LDDDsvEiRPzwx/+MGVlZSkrK8sVV1yRsrKy3Hjjjdl2221TWVmZu+66K5MmTcpHP/rRDBw4MD169Mj222+fW2+9tVV9rx9iV1ZWll/84hf5+Mc/nm7dumWjjTbKn//85xW254477sjnP//5zJ07t6U9p5122grrvvTSS/PhD3843bp1y2abbZZ77rknTz/9dHbbbbd07949O+20UyZNmtTqHH/605/y3ve+N1VVVVl//fVz+umnp7Gx8S3/LgEAAID2IZR6B/rhD3+YUaNG5Ygjjsj06dMzffr0DB06NEnyzW9+M+eee26eeOKJbLnllpk3b1723Xff3HbbbfnXv/6VvffeO/vvv3+mTp260nOcfvrp+dSnPpVHHnkk++67bw4++ODMnj17uWV32mmnXHjhhamurm5pz4knnrjCus8888wceuihefjhh7PpppvmoIMOype+9KWMHTs2DzzwQJqbm3Pssce2lL/zzjtz6KGH5qtf/Woef/zxXHrppbniiity9tlnr8FvDwAAACiCUOodqFevXqmoqEi3bt1SU1OTmpqadOrUKUlyxhln5EMf+lA22GCD9OnTJ1tttVW+9KUv5T3veU822mijnHnmmdlggw1W2vMpKfXG+uxnP5sNN9ww55xzTubNm5d//vOfyy1bUVGRXr16paysrKU9PXr0WGHdn//85/OpT30qG2+8cU4++eQ8++yzOfjggzN69Ohsttlm+epXv5o77rijpfzpp5+eb37zmxkzZkzWX3/9fOhDH8qZZ56ZSy+99M3/8gAAAIBCCKXeZbbbbrtW6/PmzcuJJ56YzTbbLL17906PHj3yxBNPrLKn1JZbbtnyuXv37qmurs7MmTOTJJtvvnl69OiRHj16ZJ999nnTbXxt3QMHDkySbLHFFq22LVq0KHV1dUmSf//73znjjDNaztmjR4+WXmILFix40+cHAAAA2p+Jzt9lXv+muxNPPDG33HJLvv/972fDDTdM165d84lPfCINDQ0rradLly6t1svKytLU1JQkueGGG7J48eIkSdeuXd90G19bd1lZ2Qq3LT3fvHnzcvrpp+eAAw54Q11VVVVv+vwAAABA+xNKvUNVVFRkyZIlqyx3991357DDDsvHP/7xJKWA59lnn31L5x4+fPgat2dNvPe9782TTz6ZDTfcsF3qBwAAANqeUOodasSIEbnvvvvy7LPPpkePHi29il5vo402yh//+Mfsv//+KSsry3e+850Vln2r7Zk3b15uu+22bLXVVunWrVu6devWJnWfeuqp+fCHP5xhw4blE5/4RMrLy/Pvf/87//nPf3LWWWe1yTkAAACAtmVOqXeoE088MZ06dcrIkSPTv3//Fc4Rdf7552edddbJTjvtlP333z+jR4/Oe9/73jZvz0477ZSjjjoqn/70p9O/f/+cd955bVb36NGjc/311+fmm2/O9ttvnx133DEXXHDBcntsAQAAAGuHsubm5uaObkRHq6urS69evTJ37txUV1e32rdo0aJMnjw56623nvmJ3kH8uQIAAED7WFnO8lp6SgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQuM4d3QAAAAAKMr6so1vAihzU3NEtgMKtNT2lzj333JSVleVrX/tay7ZFixblmGOOSd++fdOjR48ceOCBmTFjRqvjpk6dmv322y/dunXLgAEDctJJJ6WxsbHg1gMAAADwZqwVodT999+fSy+9NFtuuWWr7ccff3z+8pe/5He/+10mTpyYF154IQcccEDL/iVLlmS//fZLQ0ND/vGPf+SXv/xlrrjiipx66qlFX8K70hVXXJHevXu/6eOeffbZlJWV5eGHH27zNgEAAABvDx0+fG/evHk5+OCD8/Of/zxnnXVWy/a5c+fmsssuy/jx47P77rsnSS6//PJsttlmuffee7Pjjjvm5ptvzuOPP55bb701AwcOzNZbb50zzzwzJ598ck477bRUVFQs95z19fWpr69vWa+rq3vzDS+62+vbtCvnYYcdljlz5uS6665r2TZ06NBMnz49/fr167iGAQAAAB2qw3tKHXPMMdlvv/2y5557ttr+4IMPZvHixa22b7rpphk2bFjuueeeJMk999yTLbbYIgMHDmwpM3r06NTV1eWxxx5b4TnHjRuXXr16tSxDhw5t46tiZTp16pSampp07tzhmSgAAADQQTo0lLrmmmvy0EMPZdy4cW/YV1tbm4qKijcMDxs4cGBqa2tbyrw2kFq6f+m+FRk7dmzmzp3bskybNu0tXsnaqampKePGjct6662Xrl27Zquttsrvf//7NDU1ZciQIbnkkktalf/Xv/6V8vLyTJkyJUly/vnnZ4sttkj37t0zdOjQfPnLX868efNW+/ynnXZafvnLX+ZPf/pTysrKUlZWljvuuOMNw/fuuOOOlJWVZcKECdlmm23StWvX7L777pk5c2ZuvPHGbLbZZqmurs5BBx2UBQsWrPL6AAAAgLVfh3VVmTZtWr761a/mlltuSVVVVaHnrqysTGVlZaHn7Ajjxo3LVVddlZ/+9KfZaKON8ve//z2f+9znMmHChHz2s5/N+PHjc/TRR7eUv/rqq7Pzzjtn+PDhSZLy8vL86Ec/ynrrrZdnnnkmX/7yl/ONb3wjP/nJT1br/CeeeGKeeOKJ1NXV5fLLL0+S9OnTJy+88MJyy5922mm56KKL0q1bt3zqU5/Kpz71qVRWVmb8+PGZN29ePv7xj+fHP/5xTj755JVeX//+/fOBD3zgrfzqAAAAgHbWYaHUgw8+mJkzZ+a9731vy7YlS5bk73//ey666KJMmDAhDQ0NmTNnTqveUjNmzEhNTU2SpKamJv/85z9b1bv07XxLy7xb1dfX55xzzsmtt96aUaNGJUnWX3/93HXXXbn00kvzjW98Iz/4wQ8yderUDBs2LE1NTbnmmmtyyimntNTx2jchjhgxImeddVaOOuqo1Q6levToka5du6a+vn61/jzOOuus7LzzzkmSww8/PGPHjs2kSZOy/vrrJ0k+8YlP5Pbbb8/JJ5+8yusTSgEAAMDarcOG7+2xxx559NFH8/DDD7cs2223XQ4++OCWz126dMltt93WcsyTTz6ZqVOntoQQo0aNyqOPPpqZM2e2lLnllltSXV2dkSNHFn5Na5Onn346CxYsyIc+9KH06NGjZbnyyiszadKkbL311tlss80yfvz4JMnEiRMzc+bMfPKTn2yp49Zbb80ee+yRddddNz179swhhxySWbNmtRpCt9TUqVNbneecc855021+7dsXBw4cmG7durUEUku3Lf2zXtX1AQAAa4e/P5Hs//1k8DFJ2cHJdQ+03n/aH5JNT0y6fyFZ54hkz3OS+55etv+Ox0vHLW+5fyX/6187JznkJ0nNl0t1v/fbyR9a92nIiK++sc5z/9xmlw6sQof1lOrZs2fe8573tNrWvXv39O3bt2X74YcfnhNOOCF9+vRJdXV1jjvuuIwaNSo77rhjkmSvvfbKyJEjc8ghh+S8885LbW1tTjnllBxzzDHviuF5K7N07qe//vWvWXfddVvtW/q7OfjggzN+/Ph885vfzPjx47P33nunb9++SZJnn302H/7wh3P00Ufn7LPPTp8+fXLXXXfl8MMPT0NDQ7p169aqzsGDB7fMEZWUhum9WV26dGn5XFZW1mp96bampqbVvj4AAKDjza9PthqWfOEDyQEXvnH/xjXJRYcl6w9IFjYkF9yY7HVu8vT5Sf/qZKeNk+kXtz7mO79Lbnss2W79N9a31KGXJHMWJH/+etKvZzL+7uRTP0oeOCvZZsSycmd8Ijnig8vWexY7uwy8q63Vrz+74IILUl5engMPPDD19fUZPXp0q6FjnTp1yvXXX5+jjz46o0aNSvfu3TNmzJicccYZHdjqtcPIkSNTWVmZqVOnrnAo20EHHZRTTjklDz74YH7/+9/npz/9acu+Bx98ME1NTfnBD36Q8vJSh7rf/va3Kzxf586ds+GGG75he0VFRZYsWfIWr+aNVuf6AACAjrfP1qVlRQ7aufX6+Qcnl92RPDI12eM9SUXnpKb3sv2LG5M/PZQct1dSVrbiev/xVHLJ55P3bVBaP+XjyQU3JQ9Obh1K9axqXT9QnLUqlLrjjjtarVdVVeXiiy/OxRdfvPwDkgwfPjw33HBDO7fs7adnz5458cQTc/zxx6epqSm77LJL5s6dm7vvvjvV1dUZM2ZMRowYkZ122imHH354lixZko985CMtx2+44YZZvHhxfvzjH2f//ffP3Xff3Sq0Wl0jRozIhAkT8uSTT6Zv377p1atXYdcHAAC8vTQ0Jj+7PenVLdlq+PLL/PmhZNYryed3XXldO22U/ObeZL9tkt7dkt/elyxanOy2Wety5/4lOfO6ZFjf5KCdkuP3STp3apPLAVZhrQql3lYOau7oFqzSmWeemf79+2fcuHF55pln0rt377z3ve/Nt771rZYyBx98cL785S/n0EMPTdeuXVu2b7XVVjn//PPzve99L2PHjs2uu+6acePG5dBDD31TbTjiiCNyxx13ZLvttsu8efNy++23Z8SIEYVdHwAAsPa7/qHkMxclCxqSQb2TW75ZGnK3PJfdkYzeMhnSd+V1/vYryad/nPT9Uilk6laRXPu1ZMPXvIPpK6OT945I+vRI/vG/ZOxvkulzkvM/1yaXBaxCWXNz89qfrrSzurq69OrVK3Pnzk11dXWrfYsWLcrkyZOz3nrrparK4OJ3Cn+uAAC8K41fyXi3ApQdnFx7fPKx7Vpvn7+oFAa99Ery89uTvz2e3Hd6MuB1Ay2em5UM/2opcDrwfSs/13G/TP45KTnnU6WA67oHSsP37vxOssWw5R/z/+5IvvT/knmXJZVdll+m3bwNOj7A6lpZzvJaHfb2PQAAAEiS7lWlHkw7bpRcdmTSubzUI+r1Lv970rdn8pH3rry+STOSi25O/t+RpXmpthqefPfAZLv1kotvWfFxO2yYNC5Jnn3xLV0OsJoM3wMAAGCt0tSc1De23tbcnFw+MTl0l6TLKr7JLqgv/Sx/XcewTuWlulfk4SmlY17fQ+sdpYN7y7EK77Iec3pKAQAA0G7mLUoefra0JMnkF0ufp75UGrb3rd8k9z6VTHmx9Ga8L/wsef7l5JM7tK7nb4+Vjv3iB994judnJ5ueWBqulySbDk42HJh86bLStkkzkh/8NbnlP8uGDt7zVHLhjcm/pyTPzEyuvjs5/qrkc7sk63Rvp18G0IqeUgAAALSbB55JPnj2svUTrir9HPP+5KdfSP77QvLLO0vzSfXtkWy/fmnep82HtK7nsjtKb9TbdPAbz7F4SfLk9GU9pLp0Tm74RvLNa5L9v5/Mqy+FVL/8UrLv1qUylZ2Ta+5JTvtjUr84Wa9/cvzeyQn7tvVvAFgRE51n9SY6HzFiRKu30/H2tnDhwjz77LMmOgcA4N3F0K21V1HDttwDa7d3yPA9E523kU6dOiVJGhoaOrgltKWlf55L/3wBAACAYhm+twqdO3dOt27d8uKLL6ZLly4pL5fjvd01NTXlxRdfTLdu3dK5s/8EAAAAoCP4Rr4KZWVlGTRoUCZPnpwpU6Z0dHNoI+Xl5Rk2bFjKynRdBQAAgI4glFoNFRUV2WijjQzhewepqKjQ6w0AAAA6kFBqNZWXl5sQGwAAAKCNCKUAAACAdvX3J5L/+2vy4ORk+pzk2uOTj223bP9pf0iuuSeZNjup6JRsu15y9qeSHTZcVubs65K/Ppw8PCWp6JzM+fmqz7s69f5venLS+OTu/yUNjcmWw5IzP5F8cPM2unhWyPglAAAAoF3Nr0+2GpZcfNjy929ck1x0WPLoucld301G9E/2Ojd5sW5ZmYbG5JM7JEfvsfrnXZ16P/z9pLEp+du3kwfPLrXzwz9Iaue8+evkzdFTCgAAAGhX+2xdWlbkoJ1br59/cHLZHckjU5M93lPadvonSj+vmLj6511VvS+9kjxVm1x2RKmHVJKc+5nkJ7cm/3kuqem9+ufizdNTCgAAAFhrNDQmP7s96dUt2Wp4+9bbt0eyyaDkyjuT+YuSxiXJpX9LBlSXhvrRvvSUAgAAADrc9Q8ln7koWdCQDOqd3PLNpF/P9q23rCy5dWzysQuSnl9MystKgdRNJyfrdH/r52bl9JQCAAAAOtwHRyYPn5P847vJ3lsmn/pxMnNu+9bb3Jwcc0UpiLrzO8k/zyhNwL7/95PpL7/1c7NyQikAAACgw3WvSjasSXbcKLnsyKRzeWn+p/as92+PJdf/K7nm2GTnTZL3rpf85PNJ14rkl3e+9XOzckIpAAAAYK3T1JzUN7ZvvQsaSj/LX5eOlJeXytG+zCkFAAAAtKt5i5Kna5etT34xefjZpE+P0mTjZ/8p+ch7S3M+vTQvufiW5PmXk0/usOyYqS8ls+clU2clS5pKxyelXlA9qkqfNz0xGffp5OPblyYuX1W9ozYqzR015qfJqR8v9ZD6+e3J5JnJflu3+6/lXU8oBQAAALSrB55JPnj2svUTrir9HPP+5KdfSP77Qmm43EuvlEKq7dcvzfG0+ZBlx5z6+9ZD6rb5dunn7d9OdhtZ+vzk9GTugtLnTuWrrrdfz9Kk5t/+bbL7OcnixtK+P53Qtm/+Y/nKmpub3/Ud0urq6tKrV6/MnTs31dXVHd0cAACA9jG+rKNbwIocVNBXc/fA2q2o+6CdrW7OYk4pAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAonlAIAAACgcJ07ugEAALSz8WUd3QJW5qDmjm4BAHQIPaUAAAAAKJxQCgAAAIDCGb4HAADvBoZxrt0M4wTehfSUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACtehodQll1ySLbfcMtXV1amurs6oUaNy4403tuzfbbfdUlZW1mo56qijWtUxderU7LfffunWrVsGDBiQk046KY2NjUVfCgAAAABvQoe+fW/IkCE599xzs9FGG6W5uTm//OUv89GPfjT/+te/svnmmydJjjjiiJxxxhktx3Tr1q3l85IlS7LffvulpqYm//jHPzJ9+vQceuih6dKlS84555zCrwcAgBV7ZWHynd8n196fzKxLthmR/PCQZPsNksWNySm/S254OHnmxaRX12TP9yTnfiYZvM6K6zztD8npf2y9bZNByX+/X/o8e17y3T8kNz+aTH0p6V+dfGzb5MxPJr26vbE+AKA4HRpK7b///q3Wzz777FxyySW59957W0Kpbt26paamZrnH33zzzXn88cdz6623ZuDAgdl6661z5pln5uSTT85pp52WioqKdr8GAABWzxd/nvznueRXR5eCpqvuTvYclzx+XtKjKnno2eQ7H0+2Gpa8PD/56q+Sj/wgeeCslde7+ZDk1rHL1jt3Wvb5hZdLy/cPSkaum0x5KTnq/5W2/f5r7XGVAMDq6tBQ6rWWLFmS3/3ud5k/f35GjRrVsv3qq6/OVVddlZqamuy///75zne+09Jb6p577skWW2yRgQMHtpQfPXp0jj766Dz22GPZZpttlnuu+vr61NfXt6zX1dW101UBdKDxZR3dAlbmoOaObgEUamFD8of7kz+dkOy6WWnbaQcmf3koueTW5KxPJbeMbX3MRWOS951a6uE0rN+K6+5cntT0Xv6+9wxN/vC1ZesbDEzO/lTyuZ8kjUtaB1gAQLE6PJR69NFHM2rUqCxatCg9evTItddem5EjRyZJDjrooAwfPjyDBw/OI488kpNPPjlPPvlk/vjHUh/t2traVoFUkpb12traFZ5z3LhxOf3009vpigAAeL3GJcmSpqSqS+vtXSuSu/63/GPmLkzKypLeqxhm99SMZPAxpbpHbZSM+/TKQ6y5C5LqrgKpjrKyYZxJaUjmNfck02YnFZ2SbdcrBYk7bLjiOlc1jDNJdjsrmfhE6zJf2j356eFtclkArIEOD6U22WSTPPzww5k7d25+//vfZ8yYMZk4cWJGjhyZI488sqXcFltskUGDBmWPPfbIpEmTssEGG6zxOceOHZsTTjihZb2uri5Dhw59S9cBAMCK9exaCozOvC7ZbN1kYK/k1/9I7nkq2XA5MzUsakhO/nXy2VFJ9UpCqR02SK74UimAmD6nFEy8/4zkP98rnfP1XnolOfPa5Mjd2+rKeLNWNoxz3T7JxjXJRYcl6w8o9bC74MZkr3OTp88vzQm2IisbxrnUER9MzvjEsvVuZvsA6FAd+va9JKmoqMiGG26YbbfdNuPGjctWW22VH/7wh8stu8MOOyRJnn766SRJTU1NZsyY0arM0vUVzUOVJJWVlS1v/Fu6AADQvn51dNLcnKx7bFI5JvnRhOSzOyXlrxttvLgx+dSPk+Ykl3x+5XXus3XyyR2SLYclo7dMbjgpmbMg+e19byxbtyDZ7/9Kc0uddkBbXRVvxtJhnOd9tjSMc8Oa0jDODQeWhnEmyUE7lya5X39AKWg6/+CkbmHyyNSV1710GOfSpV/PN5bpVtm6zMoCTwDaX4eHUq/X1NTUar6n13r44YeTJIMGDUqSjBo1Ko8++mhmzpzZUuaWW25JdXV1yxBAAADWDhsMTCZ+J5l3WTLtR8k/zywFUOsPWFZmaSA15aXklm+++dCgd/dk40HJ06+byeGVhcne5yU9q5Jrj0+6dPh4gXenNzuMs6Ex+dntpTclbjV85XUvHca5/teSgy8uzUX2elffnfT7UvKek5Ox1yQLlv+1A4CCdOhfx2PHjs0+++yTYcOG5ZVXXsn48eNzxx13ZMKECZk0aVLGjx+ffffdN3379s0jjzyS448/Prvuumu23HLLJMlee+2VkSNH5pBDDsl5552X2tranHLKKTnmmGNSWVnZkZcGAMAKdK8qLS/PTyY8Wuo1kywLpJ6qTW7/dtJ3OT1dVmXeomTSjOSQnZdtq1uQjP5eUtkl+fPXkypDtjrM6g7jvP6h5DMXJQsakkG9SwHl8no+LbU6wzgP2ikZ3i8Z3Dt5ZFppeOiT05M/Ht9+1wvAynVoKDVz5swceuihmT59enr16pUtt9wyEyZMyIc+9KFMmzYtt956ay688MLMnz8/Q4cOzYEHHphTTjml5fhOnTrl+uuvz9FHH51Ro0ale/fuGTNmTM4444wOvCoAAJZnwiOl4XubDEqenpGcND7ZdFDy+V1LgdQnfpg89Gxy/Yml3jS1c0rH9emRVLz6f617nJN8fLvk2L1K6ydenez/3lLY8MLLyXf/kHQqLw0LTEqB1F7nlsKNq75cGgZWt7C0r391qSzF+tXRyRd+VhrG2ak8ee+I0p/Xg5OXlfngyOThc0pzgP389lJYed/pyYBey69zn62Xfd5yWCmkGv7V0jDOw3crbX/tPGJbDCuFXXucUwoxN2j97iQACtKhodRll122wn1Dhw7NxIkTV1nH8OHDc8MNN7RlswAAaAdzFyRjf5M8N7sUNB24femtal06J8++mPz5oVK5rb/V+rjbv53s9urMDJNmlIKKpZ6bnXz2omTWvKR/z2SXTZJ7T182IfZDzyb3TSp93vCEVtVm8oXJiP5tfZWsytJhnPMXlQLCQeskn/5R62Gc3atKPac2rEl23CjZ6ITksjuSsR9dvXOsaBjna+3w6nuTnhZKAXQYo+kBACjEp3YsLcszon/SfPWq63j2de/Duea4lZffbeTq1UvxVjSMc3mampP6xtWve3nDOF/v4Smln4N6r369ALQtHZYBaDN/fyLZ//uliWbLDk6ue6D1/j/en+w1Lun7pdL+h59tvX/2vOS4XyabnJh0PSwZ9pXkK78s9a5YXUddVqr7whuXbbvj8dK25S33T1rTqwVgTUx4JLnp38nkmcktjyYfPGvZMM75i5Jv/Sa596lkyoulIX1f+Fny/Multywutcc5yUU3L1s/8epk4hOlHnf/+F/y8QtaD+OcNCM589pSfc++mPz5weTQnya7bloa7gdAx9BTCoA2M78+2WpY8oUPJAdcuJz9i0pDaz61Y3LEL964/4WXS8v3Dyq9sn3KS8lR/6+07fdfW/X5r70/uffpZPA6rbfvtHEy/eLW277zu+S2x5Lt1l/dqwOgLaxsGOeSpuS/LyS/vLM0TLNvj2T79ZM7v5NsPmRZHW92GGdF5+TW/yQX3lT6u2pon9J5T/lYoZcOwOsIpQBoM/ts3Xqy2dc75P2ln8++uPz97xma/OFry9Y3GFj6ovK5n5ReI96504rrfn52qZfVhG8m+/1f630VnZOa3svWFzcmf3ooOW6vpKxsxXUC0PZWNoyzqmL13ob3ZodxDu1bmscKgLWL4XsArNXmLkiqu648kGpqSg65JDnpw63/JX1F/vxQMuuV0lARAACgYwilAFhrvfRKaQ6Q177Ge3m+95ekc3nyldGrV+9ldySjt0yG9H3LTQQAANaQUAqAtVLdgtIwvJHrJqcdsOJyD05OfjghueKo1RuK99ys0iS7h+/WZk0FAADWgDmlAFjrvLIw2fu8pGdVcu3xpclvV+TO/yYz60pv6ltqSVPy9atLE9q+ft6Ry/+e9O2ZfOS97dN2AABg9QilAFir1C1IRn8vqeyS/PnrpUlvV+aQXZI939N62+jvlba/fs6o5ubk8onJobusPOgCAADan/8lB6DNzFuUPF27bH3yi8nDz5Ze+T2sXzJ7XjL1peSFOaX9T04v/azpXVrqFiR7nZssaEiu+nJSt7C0JKXXend6ddD5picm4z6dfHz7Uq+nvj1bt6NLp6SmV7LJ4Nbb//ZYqU1f/GDbXjcAAPDmCaUAaDMPPJN88Oxl6ydcVfo55v2lOZ/+/GDy+Z8t2/+Zi0o/v3tActqByUPPJvdNKm3b8ITWdU++MBnRv/T5yemlt/K9WZfdkey0UbLp4FUWBQAA2plQCoA2s9vIpPnqFe8/7AOlZU2PX2pVZV4/j9RS449ddd0AAEAxvH0PAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAoXOeObgAA0I7Gl3V0C1iZg5o7ugUAAB1GTykAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwnTu6AQDAO98rC5Pv/D659v5kZl2yzYjkh4ck229Q2j9jbnLyr5ObH03mLEh23TT58Zhko5oV17nbWcnEJ964fd+tk7+eVPo8b1HyzWuS6x5IZs1L1uuffGV0ctSebX2FAAC8WUIpAKDdffHnyX+eS351dDJ4neSqu5M9xyWPn1da/9j5SZdOyZ9OSKq7JuffmOx5Tml/96rl1/nHryUNjcvWZ81LthqbfPJ9y7adcFXyt8eTq76cjOhfCr2+fHnpnB/Ztl0vGQCAVTB8DwBoVwsbkj/cn5z32WTXzZINa5LTDkw2HJhccmvyVG1y79PJJV8o9ZzaZHByyeeThYuTX9+z4nr79Ehqei9bbnk06VaRfHKHZWX+8VQy5v3JbiNLodSRuydbDUv+OamdLxoAgFUSSgEA7apxSbKkKanq0np714rkrv8l9YtL66/dX16eVHZO7npy9c9z2R3JZ0a17lm100bJnx9Knp+dNDcntz+W/K822WuLNb4cAADaiFAKAGhXPbsmozZKzrwueeHlUkB11V3JPU8l0+ckmw5OhvVNxv4meXl+aUje9/6SPDe7tH91/HNSaXjgFz/YevuPxyQj102GHJdUjEn2Pi+5+LBSjy0AADqWOaUAgHb3q6OTL/wsWffYpFN58t4RyWd3Sh6cnHTpnPzx+OTwnyV9jizt3/M9yT5blXo3rY7L7ki2GJq8b4PW2398c2lo4J+/ngzvl/z9v8kxV5TmlNrzPW18kQAAvClCKQCg3W0wMJn4nWT+oqRuYTJoneTTP0rWH1Dav+16ycPjkrkLSj2l+lcnO5yabLfequuevyi55p7kjE+03r6wIfnWb5Jrj0/226a0bcthycNTku//VSgFANDRhFLvNOPLOroFrMxBq/lP/gDvUN2rSsvL85MJj5YmP3+tXt1KP5+qTR54JjnzE2+s4/V+d19S35h8bufW2xc3JouXJOWv+6uxU3nS1LTm1wAAQNsQSgEA7W7CI6WheJsMSp6ekZw0Ptl0UPL5XUv7f3df0r9nMqxf8ujU5Ku/Sj62XbLXlsvqOPSSZN11knGfaV33ZROTj22b9O3Zent1t+QDmyUn/bo0qfrwfsnEJ5Ir70zO/1z7Xi8AAKsmlAIA2t3cBaWJzJ+bnfTpkRy4fXL2p0rzSSXJ9JeTE65KZsxNBvVODn1/8p2Pt65j6qw39np68oXSG/pu/ubyz3vNsaXzHvyTZPa8UjB19qeSo/Zo80sEAOBNEkoBAO3uUzuWlhX5yt6lZWXuOOWN2zYZnDRfveJjanonl39ptZoIAEDByju6AQAAAAC8+wilAAAAACicUAoAAACAwgmlAAAAACicUAoAAACAwnn7Hm1uSVNy2h+Sq+5Oauckg9dJDts1OeVjSdmrr/L+4/3JT29NHny29Iruf52dbD1i5fUubkzG/Tn55Z3J8y8nmwxKvveZZO+tlpUZ8dVkyktvPPbLeyYXf75trg8AAAB464RStLnv/SW55Nbkl0clmw9JHngm+fzPkl5dl73ue/6iZJdNSq8HP+IXq1fvKb8rBV0//2Ky6eBkwiPJxy9I/nFass2IUpn7zyyFYkv957nkQ+OST+7QllcIAAAAvFVCKdrcP/6XfHTbZL9tSusj+ie/vif55zPLyhzy/tLPZ19c/Xp/dVfy7Y8m+25dWj96z+TW/yQ/uCG56sulbf2rWx9z7l+SDQYmH9hsjS4FAAAAaCfmlKLN7bRxcttjyf+ml9b/PSW568lkn61Wftyq1DcmVRWtt3WtKNW9PA2NyVV3JV/4wLJhgwAAAMDaQU8p2tw390/qFiabnpR0Ki8Npzv7k8nBO7+1ekdvkZx/Q7LrpskGA0rB1x/vbz1c77WueyCZs6A0nxUAAACwdhFK0eZ+e19y9d3J+GOSzddNHp6SfO2q0oTnY95CQPTDQ0vzT216Yqnn0wYDk8/vmvy/icsvf9kdpd5Zg9dZ83MCAAAA7UMoRZs7aXypt9RnRpXWtxhWeiPeuD+/tVCqf3Vy3QnJooZk1rxS2PTNa5L1B7yx7JQXS/NN/fFra34+AAAAoP2YU4o2t6AhKX/dndWpPGlqbpv6qyqSdfskjUuSP9xfmlT99S7/ezKg17LJ1gEAAIC1i55StLn9t0nOvi4Z1jfZfEjyr2eT828sTTi+1Ox5ydSXkhfmlNaffHVS9JrepSVJDr0kWXedZNxnSuv3PZ08/3Ky9fDk+dnJaX9MmpqSb3y49fmbmpLLJyZj3p907tRulwkAAAC8BUIp2tyPxyTf+X3y5cuTmXWlYXZf2j059YBlZf78YPL5ny1b/8xFpZ/fPSA57cDS56mzkvLXvDVv0eLklN8mz7yY9KhM9t06+dXRSe/urc9/639Kx742BAMAAADWLkIp2lzPrsmFh5SWFTnsA6VlZe44pfX6BzZLHv+/VZ9/ry2T5qtXXQ4AAADoOOaUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACtehodQll1ySLbfcMtXV1amurs6oUaNy4403tuxftGhRjjnmmPTt2zc9evTIgQcemBkzZrSqY+rUqdlvv/3SrVu3DBgwICeddFIaGxuLvhQAAAAA3oQODaWGDBmSc889Nw8++GAeeOCB7L777vnoRz+axx57LEly/PHH5y9/+Ut+97vfZeLEiXnhhRdywAEHtBy/ZMmS7LfffmloaMg//vGP/PKXv8wVV1yRU089taMuCQAAAIDV0LkjT77//vu3Wj/77LNzySWX5N57782QIUNy2WWXZfz48dl9992TJJdffnk222yz3Hvvvdlxxx1z88035/HHH8+tt96agQMHZuutt86ZZ56Zk08+OaeddloqKiqWe976+vrU19e3rNfV1bXfRQIAAADwBmvNnFJLlizJNddck/nz52fUqFF58MEHs3jx4uy5554tZTbddNMMGzYs99xzT5LknnvuyRZbbJGBAwe2lBk9enTq6upaelstz7hx49KrV6+WZejQoe13YQAAAAC8QYeHUo8++mh69OiRysrKHHXUUbn22mszcuTI1NbWpqKiIr17925VfuDAgamtrU2S1NbWtgqklu5fum9Fxo4dm7lz57Ys06ZNa9uLAgAAAGClOnT4XpJssskmefjhhzN37tz8/ve/z5gxYzJx4sR2PWdlZWUqKyvb9RwAAAAArFiHh1IVFRXZcMMNkyTbbrtt7r///vzwhz/Mpz/96TQ0NGTOnDmtekvNmDEjNTU1SZKampr885//bFXf0rfzLS0DAAAAwNqnw4fvvV5TU1Pq6+uz7bbbpkuXLrntttta9j355JOZOnVqRo0alSQZNWpUHn300cycObOlzC233JLq6uqMHDmy8LYDAAAAsHo6tKfU2LFjs88++2TYsGF55ZVXMn78+Nxxxx2ZMGFCevXqlcMPPzwnnHBC+vTpk+rq6hx33HEZNWpUdtxxxyTJXnvtlZEjR+aQQw7Jeeedl9ra2pxyyik55phjDM/j3W18WUe3gJU5qLmjWwAAANDhOjSUmjlzZg499NBMnz49vXr1ypZbbpkJEybkQx/6UJLkggsuSHl5eQ488MDU19dn9OjR+clPftJyfKdOnXL99dfn6KOPzqhRo9K9e/eMGTMmZ5xxRkddEgAAAACroUNDqcsuu2yl+6uqqnLxxRfn4osvXmGZ4cOH54YbbmjrpgEAAADQjta6OaUAAAAAeOcTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQuM4d3QDgnev52cnJ1yQ3/jtZUJ9sODC5/EvJdusvK/PE86UyE59IGpuSkesmf/hqMqzf8uvc7axS2dfbd+vkryeVPjc3J9/9Q/Lz25M585OdN04u+UKyUU2bXyIAAABrSCgFtIuX5yc7n558cGRy4zeS/j2Tp2qTdbovKzNpRrLLGcnhH0hOPzCp7po89lxS1WXF9f7xa0lD47L1WfOSrcYmn3zfsm3nXZ/8aELyyy8l6w1IvvO7ZPS5yePnJVUVbX6pAAAArAGhFNAuvveXZGjfUs+opdYb0LrMt3+b7LtVct5By7ZtMHDl9fbp0Xr9mnuSbhXJJ3corTc3JxfelJzyseSj25W2XXl0MvDLyXUPJp8ZtUaXAwAAQBszpxTQLv78YLLdesknf5gMODrZ5lvJz/+2bH9TU/LXh5ONB5V6MQ04Otnh1OS6B97ceS67oxQ0da8qrU9+Mamdk+y5+bIyvbolO2yQ3PPUW7woAAAA2oxQCmgXz7yYXHJbaR6nCScnR++ZfOXK5Jd/L+2fWZfMW5Sc+5dk762Sm09OPr5dcsCFy58zann+OSn5z3PJFz+4bFvtnNLPgb1alx3Ya9k+AAAAOp7he0C7aGoqTWh+zqdL69uMSP4zLfnpbcmYXZOm5tL2j743OX6f0uetRyT/eKpU5gObrfocl92RbDE0ed8G7XABAAAAtCs9pYB2Mah36U16r7XZusnUWaXP/XomnTstp8zgZOpLq65//qLSfFKH79Z6e03v0s8Zc1tvnzF32T4AAAA6nlAKaBc7b5w8Ob31tv9NT4b3K32u6Jxsv/5yytQuK7Myv7svqW9MPrdz6+3r9S+FT7c9tmxb3YLkvknJqI3e9GUAAADQToRSQLs4fp/k3qeTc/6UPF2bjL87+dntyTEfWlbmpP2S39xbmgD96drkopuTvzyUfPk1ZQ69JBl7zRvrv2xi8rFtk749W28vK0u+tndy1nWlydYfnZoc+tNkcO9SeQAAANYO5pQC2sX2GyTXfi0Z+5vkjGtLPZgu/Fxy8Gt6Nn18++SnX0jG/bk0Cfomg5I/fDXZZZNlZabOSsrLWtf95AvJXU8mN39z+ef+xoeT+fXJkZclcxYku2yc3HRyUlXR5pcJAADAGhJKAe3mw+8tLSvzhd1Ky4rcccobt20yOGm+esXHlJUlZ3yitAAAALB2MnwPAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAonFAKAAAAgMIJpQAAAAAoXOeObgAAAO3riTM27egmsBKbHdTRLeDdxPNg7eVZwLuRnlIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhTHQOAADvAia4XruZ5JqieBas3d5tzwI9pQAAAAAonFAKAAAAgMIJpQAAAAAonDmlAABocz+bNSu3znslz9Q3pKq8LFt37Zqv9++f9SoqW5V7eOHC/PClF/PIwoUpLyvLppWV+fmQoakqL/3b6Z6Tns4LjY2tjjm+X/8c0bfvCs/92zlz8te6uXm8vj7zm5py74YbpbpTpzeUmzhvXn4y66X8r74+lWVl2a5bt1y07pA2uHpgKc8CYGWEUgAAtLkHFizIZ3v3znuqumZJc3MufOnFfHHatPxlvfXT7dUvmQ8vXJgjn5uWI/r0zbcGDEznsuS/i+rf0JX/uL798onevVvWu5evvLP/oqam7NK9R3bp3iMXvPTicsvc/EpdTq2tzdf698+O3bqnsbk5T9XXv5VLBpbDswBYGaEUAABt7mdDh7ZaP6dmUHaZ9HQeX7Qo23XrliQ5d+aMfG6ddVr1dHh974mk9MWzf+fV/9/WQ/v0SZL8c8H85e5vbG7OuJkzc1L/ATnwNV9wN6x847mBt8azAFgZoRQAAO3ulaamJEmvV4fOzGpszCOLFuXD1dU5aMqUTFvckPUqKvLVfv2z7atfVJf6+exZuWTWSxnUpUv2q67OmHX6pHNZ2Rq35fFFizKjsTFlZckBz07OS42N2bSqKif1H5CNfBmFduVZALyWic4BAGhXTc3NOXfmjLy3a9eWL3rPLV6cJLn4pZfyid69cumQoRlZVZUvPDctzzY0tBz7uXX65AeDB+eKocPyqd698/NZs/KDF2e+pfa89txH9e2bS4YMTa/yThkzbWrmLFnyluoGVsyzAHg9PaUAAGhXZ86Ykafq63PVsOEt25rSnCT5VO91ckCv3kmSkVVVuXf+gvxx7pyc0H9AkuSwV4ffJMkmVVXpUlaW02trc3y//qlYxXwyK7L03F/q2y979axOkpxdU5MPPjMpE16py6d7r7NG9dJaW01w/dNZL+Xv8+bnv/WL0qWsLPdttPEqz/1SY2POf3Fm7p6/IK80Lcl2XbvlWwMHZkRFRUuZ+qamnPfizNxQV5eG5ubs0r17vjOwJv3exPAw3hzPgncvzwNWRE8pAADazVkzajNx/rxcMXRYarp0adnev1Ppf/Q3eM2XgiRZv7Ii0xe3fsPWa21Z1TWNSZ5vXLzGbVo6J81rz11RXp4hXbqs9Ny8OUsnuP718OH5xZChaWxuzhenTcuCV4dvJcsmuN6pW/dcM3xEfjt8eA7qvU6rLymLm5szumfPfPo1c/6sTHNzc457/rlMW7w4F627bv4wYkQGdemSw6dNbXXuc2fOzO3z5uWCwevmymHDM7OxMV99/vk2unpez7Pg3c3zgBURSgEA0Oaam5tz1oza3DpvXv7f0GEZ8rovnOt26ZIBnTvn2cUNrbY/29CQwa/5wvp6/61flPIkfTqt+b9eb15ZlYqyslZDgxY3N+eFxYtXem7enJ8NHZqP9+qdjSors2lVVc6pGZTpjY15fNGiljKvneB6o8rKrFdRmX2qq1v1fDmuX/+M6dMnG6/mHD9TFi/OvxctyqkDa7JF165Zr6Iy3x04MPXNzbmhri5J8sqSJfnD3Dk5ecCA7Ni9ezavqsrZNYPyr0UL8++FC9v2F/Eu51lA4nnAigmlAABoc2fOnJG/1NXl/wYNTvfy8rzY2JgXGxuz6NV/mS4rK8sX1umTq15+ORNeqcuUhob86KUXM7mhIQf26pWk9K/mV86enf8uWpRpDQ35S93cfG/mzOxfXd0ySfKMxYuz3+Rn8shrvji82NiYJxYtytSGUg+K/9XX54lFi1rmiOnRqVM+3bt3Lpr1Uu6ePz+TG+pzxozaJMnonj0L+x2926xogus+nTrloClT8v6nn8qhU6fkwQUL3tJ5GppL56l8zQTY5WVlqSgry0MLS3U/tmhRGpOM6ta9pcz6lZUZ1LlzHvYltE15FrA8ngcsZYAkAABt7po5c5IkY6ZNbbX97JqafPzVeWMO7dMn9c3N+d7MmZm7ZEk2qazKL4YMzbBXe1JUlJXlhlfqcvGsl9LQ3Jx1u3TJoev0yWHrLJvnpTHNmdzQkEXNy4Zh/GbOy/nJrFkt64e+2obXnvvE/gPSKWX55vQXsqi5OVtWVeX/DR3W8gWJtrWqCa5PGjAgm1ZW5c91c/OF56blTyPWazXfy5uxXkXpy+QFL72Y0wbWpGt5ea6cPTu1jY15sbEURry0pDFdyspS/bo/736dO+elJYZttSXPAl7P84DXEkoBANDmHt9k09Uqd0Tfvjmib9/l7htZVZVrho9Y6fHrdql4w7mO7dc/x/brv9LjupSV5RsDBuQbAwasVjt5a97KBNdvVpeysvxo3SE5pXZ6Rj39VDql1APi/d27v3pGiuRZwOt5HvBaHTp8b9y4cdl+++3Ts2fPDBgwIB/72Mfy5JNPtiqz2267paysrNVy1FFHtSozderU7LfffunWrVsGDBiQk046KY2NEk0AAOhobT3B9erYvKoq145YL/dtuFEmbrBhfjZ0aOYsWZKhr56/X6fOWdzcnLpXh3Et9VJjY/q9hTmKgJXzPOD1OjSUmjhxYo455pjce++9ueWWW7J48eLstddemT9/fqtyRxxxRKZPn96ynHfeeS37lixZkv322y8NDQ35xz/+kV/+8pe54oorcuqppxZ9OQAAwKvaa4LrN6Nnp07p07lznm1oyGOLFmX3HqV5gjavqkrnJPcuWPa9Y3JDfaY3Nmbrrl3b5NzAMp4HrEiHxn433XRTq/UrrrgiAwYMyIMPPphdd921ZXu3bt1SU1Oz3DpuvvnmPP7447n11lszcODAbL311jnzzDNz8skn57TTTkvFcsae1tfXp76+vmW97tVZ9wEAgLZx5swZ+WtdXS5ad0jLBNdJ0rO8PFXl5S0TXF8066VsUlmZTSur8qe6uZnc0JALB/dqqeeFxYszd8mSTF/cmCXNyROvvq1rWEVFur/6Vq79Jj+T4/v1z56vTk590yt16dOpUwZ17pL/1ddn3MwZ2aNHj+zcvTSRcc9OnXJgr9753syZ6dWpU3qUd8rZM2Zk66qu2cqXUGhzngesyFrVF23u3LlJkj59+rTafvXVV+eqq65KTU1N9t9//3znO99Jt27dkiT33HNPtthiiwwcOLCl/OjRo3P00UfnscceyzbbbPOG84wbNy6nn356O14JAAC8u7XFBNdJctFLL+a61/wj8oFTnk2SXDF0aN736tuyJjc05JWmZUNvXmxszHkzZ+alxsb079w5H+3VK0f17deqHd8cMCDlLyZfff75LG5uzs7du+c7A5f/D+HAW+N5wIqUNTc3rxXzezU1NeUjH/lI5syZk7vuuqtl+89+9rMMHz48gwcPziOPPJKTTz4573vf+/LHP/4xSXLkkUdmypQpmTBhQssxCxYsSPfu3XPDDTdkn332ecO5ltdTaujQoZk7d26qq6vb8SoLML5s1WXoOAcV9J+b+2Dt5j4gcR9QUtB98MSmmxVyHtbMZv99opDzuA/Wbu4D3AMkxd0H7a2uri69evVaZc6y1vSUOuaYY/Kf//ynVSCVlEKnpbbYYosMGjQoe+yxRyZNmpQNNthgjc5VWVmZyldfPQkAAABA8Tp0ovOljj322Fx//fW5/fbbM2TIkJWW3WGHHZIkTz/9dJKkpqYmM2bMaFVm6fqK5qECAAAAoGN1aCjV3NycY489Ntdee23+9re/Zb311lvlMQ8//HCSZNCgQUmSUaNG5dFHH83MmTNbytxyyy2prq7OyJEj26XdAAAAALw1HTp875hjjsn48ePzpz/9KT179kxtbW2SpFevXunatWsmTZqU8ePHZ999903fvn3zyCOP5Pjjj8+uu+6aLbfcMkmy1157ZeTIkTnkkENy3nnnpba2NqecckqOOeYYQ/QAAAAA1lId2lPqkksuydy5c7Pbbrtl0KBBLctvfvObJElFRUVuvfXW7LXXXtl0003z9a9/PQceeGD+8pe/tNTRqVOnXH/99enUqVNGjRqVz33uczn00ENzxhlndNRlAQAAALAKHdpTalUv/hs6dGgmTpy4ynqGDx+eG264oa2aBQAAAEA7WysmOgcAAADg3UUoBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhhFIAAAAAFE4oBQAAAEDhOnd0A2hbT5yxaUc3gZXY7KCObgHvJp4HazfPAwAA3u30lAIAAACgcGvcU2rq1KmZMmVKFixYkP79+2fzzTdPZWVlW7YNAAAAgHeoNxVKPfvss7nkkktyzTXX5Lnnnktzc3PLvoqKirz//e/PkUcemQMPPDDl5TphAQAAALB8q50cfeUrX8lWW22VyZMn56yzzsrjjz+euXPnpqGhIbW1tbnhhhuyyy675NRTT82WW26Z+++/vz3bDQAAAMDb2Gr3lOrevXueeeaZ9O3b9w37BgwYkN133z277757vvvd7+amm27KtGnTsv3227dpYwEAAAB4Z1jtUGrcuHGrXenee++9Ro0BAAAA4N1hjSZ+WrhwYRYsWNCyPmXKlFx44YWZMGFCmzUMAAAAgHeuNXr73kc/+tEccMABOeqoozJnzpzssMMO6dKlS1566aWcf/75Ofroo9u6nazFfjZrVm6d90qeqW9IVXlZtu7aNV/v3z/rVSx7G+N3a2tz74L5mdnYmG7l5aUy/fpn/Vff2Hjt3Dn5dm3tcuu/c4MN07fz8m/VZxsa8n8vzsy/Fi7M4ubmbFJZmeP69csO3bq/oeycJUvy8WcnZ0ZjY+7dcKNUd+rUBlcPvJbnAQAAsLrWqKfUQw89lPe///1Jkt///vcZOHBgpkyZkiuvvDI/+tGP2rSBrP0eWLAgn+3dO78ePjy/GDI0jc3N+eK0aVnQ1NRSZvOqqpxdMyjXr7defj5kaNKcfPG5aVny6hsc9+lZnYkbbNhq2aVb92zftesKv4AmydGv1nH5kKH53fAR2aSyMl9+7rm82Nj4hrKn1E7PxpWVy6kFaCueBwAAwOpao1BqwYIF6dmzZ5Lk5ptvzgEHHJDy8vLsuOOOmTJlSps2kLXfz4YOzcd79c5GlZXZtKoq59QMyvTGxjy+aFFLmU/17p3tunXLul0qMrKqKl/p1y+1jY15fvHiJElVeXn6d+7csnRKcu+C+TmwV+8VnvflxsZMWbw4X+zTN5tUVWVERUVO6N8/C5ub81R9fauy17z8cl5ZsiSfX6dPe/wKgFd5HgAAAKtrjUKpDTfcMNddd12mTZuWCRMmZK+99kqSzJw5M9XV1W3aQN5+Xnm1R0SvFQyHWdDUlGvr5mZIly6p6dJluWX+VDc3XcvLs9er4efy9O7UKetVVOTPdXOzoKkpjc3N+c2cOenbqVM2r6pqKfd0fX1+MuuljBs0OOVlb+HCgDfN8wAAAFiRNZpT6tRTT81BBx2U448/PnvssUdGjRqVpNRraptttmnTBvL20tTcnHNnzsh7u3bNRq8bGvPrl1/O91+cmYXNzVmvoiK/GDI0FWXL/1b4h7lzs191darKV5yblpWV5bIhQ3Pc889n+6f+l/IkfTp1zqVDhrZ8AW5oaspJ01/Iif0HZHCXLnlucUObXeva7IkzNu3oJrASmx3U0S0ohufB2sHzYO32bnkeAAAszxr1lPrEJz6RqVOn5oEHHshNN93Usn2PPfbIBRdc0GaN4+3nzBkz8lR9fb4/aPAb9n24ujp/GLFerhw6LCO6VOSEF55P/WvmmVnq4YUL80xDw0qH6iRJc3Nzzpw5I306d8qvhg7Lb4aPyB49e+SY55fNIXPBSy9m/YqKfKRXrza5PmD1eR4AAAArs0Y9pZKkpqYmNTU1rba9733ve8sN4u3rrBm1mTh/Xq4cOmy5w3B6duqUnp06ZURFRbbs2jWjnvpfbp03L/u9bsjn7+fOyaaVla2G3CzPvQsWZOK8ebl3w43S49WeEKdW1eQf8yflurlzc0Tfvrl3wYI8VV+fm5/8b5Kk+dVjd376qRzZt2+O69f/rV848AaeBwAAwKqsUSg1f/78nHvuubntttsyc+bMNL3uX7efeeaZNmkcbw/Nzc05e+aM3DpvXq4YOixDKipW56A0J2lobn3vzG9qyk11r+T4/qv+crjo1WPLXjfkpzxlaXr16+YPB6+b+ubmln2PLlqYU2pr86thwzN0BfPXAGvO8wAAAFhdaxRKffGLX8zEiRNzyCGHZNCgQW/4EsC7y5kzZ+SvdXW5aN0h6V5e3jJUpmd5earKyzOtoSE3vvJKdu7ePet06pQZjYvzi1mzU1lWll2792hV1011dVmS5uy/nAnzH1m4MGNrp+f/DRmagV26ZOuqrqnu1Cnfmv5Cju7bL1Xl5fndnDl5bnFDPvBqvcNe94X45SWltq1fUZHqFUy8DKw5zwMAAGB1rVEodeONN+avf/1rdt5557ZuD29D18yZkyQZM21qq+1n19Tk4716p7K8LA8uXJBfvTw7c5csSb/OnbNt124ZP3x4+nZufQv+Ye7c7Nmj53K/IC5qbsrkhoY0vtrrYZ3OnfOzIUPywxdfyuenTU1jkg0rKnLRukOy6SqG+gDtw/MAAABYXWsUSq2zzjrp06dPW7eFt6nHN1n5m50GdO6SS4cMXa26xg8fvsJ97+vW/Q3nek9V1/x86OrVvaI6gLbjeQAAAKyuNXr73plnnplTTz01CxYsaOv2AAAAAPAusEY9pX7wgx9k0qRJGThwYEaMGJEur5sg9qGHHmqTxgEAAADwzrRGodTHPvaxNm4G8Hb3s1mzcuu8V/JMfUOqysuyddeu+Xr//lmvorKlzHdra3PvgvmZ2diYbuXlpTL9+mf9ymVlXli8OGfMqM0/FyxIt/LyfLS6V47v3z+dV/JChTlLluTsGTNyx/x5KU/yoZ49M3bAwHQvL3UGveilF/OTWbPecFzXsrI8uPEmbfdLAJJ4HgAAsHrWKJT67ne/29btAN7mHliwIJ/t3TvvqeqaJc3NufClF/PFadPyl/XWT7dXvwxuXlWV/aurM6hL58xd0pSLX3opX3xuWm5Zf4N0KivLkubmHP3cc+nXuVOuHjY8LzY2Zmzt9HQuK8vx/fuv8NzfmP5CXmxszC+GDE1jc3O+XTs9p9XW5v8GD06SfL5P33y69zqtjvnCtKnZwgTY0C48DwAAWB1rNKcUwOv9bOjQfLxX72xUWZlNq6pyTs2gTG9szOOLFrWU+VTv3tmuW7es26UiI6uq8pV+/VLb2JjnFy9Oktw9f34mNdTne4MGZ7Oqquzao0eO69cvv57zchqam5d73kn19blr/vycWVOTrbp2zbbduuXbAwfmhlfqMrOxVG/38vL079y5ZZnV2JhJDQ05oFfvdv+9wLuR5wEAAKtjtUOpPn365KWXXkqy7O17K1oAXmlqSpL06tRpufsXNDXl2rq5GdKlS2penZfu34sWZqPKyvTrvKwT5y7dumdeU1Oerq9fbj0PL1qY6vLyvKeqa8u2Ud26pzzJIwsXLfeY38+dkxFdKrJdt25rcmnAm+R5AADA8qz28L0LLrggPXv2TJJceOGF7dUe4B2gqbk5586ckfd27ZqNXjM/TJL8+uWX8/0XZ2Zhc3PWq6jIL4YMTcWr88O81NiYfp1aP5b6vvqF9KXGxuWe66XGxvR53TGdy8rSq1On5R5T39SU6+vqckSfvmt8fcDq8zwAAGBFVjuUGjNmzHI/A7zemTNm5Kn6+lw1bPgb9n24ujqjunfPS42NuXz27JzwwvO5etjwVJYXM5r41nnzsqCpKR/t1auQ88G7necBAAArskYTnS81c+bMzJw5M02vdstfasstt3xLjQLevs6aUZuJ8+flyqHDWobhvFbPTp3Ss1OnjKioyJZdu2bUU//LrfPmZb/q6vTr3DmPLGo9xGbWq70bXjuE57X6de6c2Uta94BobG7O3CVLlnvM7+fOyQd69FhhfUDb8TwAAGBl1uj/wh588MGMGTMmTzzxRJpfN9loWVlZlixZ0iaNA94+mpubc/bMGbl13rxcMXRYhlRUrM5BaU7S0FwKtreq6ppLZ83KrMbGlmE6/1iwID3Ky7PhCurbuqpr6pqa8tiiRdn81bdn3bdgQZqSbNm19du0nmtoyD8XLMjF6w5Z4+sEVs3zAACA1bFG/eO/8IUvZOONN84//vGPPPPMM5k8eXLL8swzz7R1G4G3gTNnzshf6uryf4MGp3t5eV5sbMyLjY1Z9GpPymkNDfnZrFl5bNGivLB4cf61cEGOf+GFVJaVZdfuPZIkO3fvng0qKvPN6dPz30WLctf8efnRSy/ms73XScWrw3keWbgw+01+JjNefUPXBpWV2aV795xaOz2PLFyYhxYsyFkzarNvz+oM6Ny6Z8Yf6+amf+fOeX/37gX+ZuDdx/MAAIDVsUY9pZ555pn84Q9/yIYbbtjW7QHepq6ZMydJMmba1Fbbz66pycd79U5leVkeXLggv3p5dstQmm27dsv44cNbekF0KivLT4YMyRkzanPQ1CnpWl6ej1b3ynH9+rXUt6i5KZMbGtKYZb00zxs0OGfPmJEvTJuW8rLkQz165lsDB7ZqR1Nzc66bOzcfq+6VTq9OpAy0D88DAABWxxqFUnvssUf+/e9/C6WAFo9vsulK9w/o3CWXDhm6ynrW7bLycu/r1v0N5+rdqVP+b/DgldZbXlaWv23gmQVF8DwAAGB1rFEo9Ytf/CJjxozJf/7zn7znPe9Jl9dNXvqRj3ykTRoHAAAAwDvTGoVS99xzT+6+++7ceOONb9hnonMAAAAAVmWNJjo/7rjj8rnPfS7Tp09PU1NTq0UgBQAAAMCqrFEoNWvWrBx//PEZ+LqJQwEAAABgdaxRKHXAAQfk9ttvb+u2AAAAAPAusUZzSm288cYZO3Zs7rrrrmyxxRZvmOj8K1/5Sps0DgAAAIB3pjV++16PHj0yceLETJw4sdW+srIyoRQAAAAAK7VGodTkyZPbuh0AAAAAvIus0ZxSAAAAAPBWrHYode6552bhwoWrVfa+++7LX//61zVuFAAAAADvbKsdSj3++OMZNmxYvvzlL+fGG2/Miy++2LKvsbExjzzySH7yk59kp512yqc//en07NmzXRoMAAAAwNvfas8pdeWVV+bf//53Lrroohx00EGpq6tLp06dUllZmQULFiRJttlmm3zxi1/MYYcdlqqqqnZrNAAAAABvb29qovOtttoqP//5z3PppZfmkUceyZQpU7Jw4cL069cvW2+9dfr169de7QQAAADgHWSN3r5XXl6erbfeOltvvXUbNwcAAACAdwNv3wMAAACgcEIpAAAAAAonlAIAAACgcEIpAAAAAAr3lkKpp59+OhMmTMjChQuTJM3NzW3SKAAAAADe2dYolJo1a1b23HPPbLzxxtl3330zffr0JMnhhx+er3/9623aQAAAAADeedYolDr++OPTuXPnTJ06Nd26dWvZ/ulPfzo33XRTmzUOAAAAgHemzmty0M0335wJEyZkyJAhrbZvtNFGmTJlSps0DAAAAIB3rjXqKTV//vxWPaSWmj17diorK99yowAAAAB4Z1ujUOr9739/rrzyypb1srKyNDU15bzzzssHP/jB1a5n3Lhx2X777dOzZ88MGDAgH/vYx/Lkk0+2KrNo0aIcc8wx6du3b3r06JEDDzwwM2bMaFVm6tSp2W+//dKtW7cMGDAgJ510UhobG9fk0gAAAAAowBoN3zvvvPOyxx575IEHHkhDQ0O+8Y1v5LHHHsvs2bNz9913r3Y9EydOzDHHHJPtt98+jY2N+da3vpW99torjz/+eLp3756kNH/VX//61/zud79Lr169cuyxx+aAAw5oOc+SJUuy3377paamJv/4xz8yffr0HHrooenSpUvOOeecNbk8AAAAANrZGoVS73nPe/K///0vF110UXr27Jl58+blgAMOyDHHHJNBgwatdj2vnxT9iiuuyIABA/Lggw9m1113zdy5c3PZZZdl/Pjx2X333ZMkl19+eTbbbLPce++92XHHHXPzzTfn8ccfz6233pqBAwdm6623zplnnpmTTz45p512WioqKtbkEgEAAABoR2sUSiVJr1698u1vf7st25K5c+cmSfr06ZMkefDBB7N48eLsueeeLWU23XTTDBs2LPfcc0923HHH3HPPPdliiy0ycODAljKjR4/O0UcfncceeyzbbLPNG85TX1+f+vr6lvW6uro2vQ4AAAAAVm6NQ6lFixblkUceycyZM9PU1NRq30c+8pE3XV9TU1O+9rWvZeedd8573vOeJEltbW0qKirSu3fvVmUHDhyY2traljKvDaSW7l+6b3nGjRuX008//U23EQAAAIC2sUah1E033ZRDDz00L7300hv2lZWVZcmSJW+6zmOOOSb/+c9/ctddd61Jk96UsWPH5oQTTmhZr6ury9ChQ9v9vAAAAACUrNHb94477rh88pOfzPTp09PU1NRqWZNA6thjj83111+f22+/PUOGDGnZXlNTk4aGhsyZM6dV+RkzZqSmpqalzOvfxrd0fWmZ16usrEx1dXWrBQAAAIDirFEoNWPGjJxwwglvGDb3ZjU3N+fYY4/Ntddem7/97W9Zb731Wu3fdttt06VLl9x2220t25588slMnTo1o0aNSpKMGjUqjz76aGbOnNlS5pZbbkl1dXVGjhz5ltoHAAAAQPtYo+F7n/jEJ3LHHXdkgw02eEsnP+aYYzJ+/Pj86U9/Ss+ePVvmgOrVq1e6du2aXr165fDDD88JJ5yQPn36pLq6Oscdd1xGjRqVHXfcMUmy1157ZeTIkTnkkENy3nnnpba2NqecckqOOeaYVFZWvqX2AQAAANA+1iiUuuiii/LJT34yd955Z7bYYot06dKl1f6vfOUrq1XPJZdckiTZbbfdWm2//PLLc9hhhyVJLrjggpSXl+fAAw9MfX19Ro8enZ/85CctZTt16pTrr78+Rx99dEaNGpXu3btnzJgxOeOMM9bk0gAAAAAowBqFUr/+9a9z8803p6qqKnfccUfKyspa9pWVla12KNXc3LzKMlVVVbn44otz8cUXr7DM8OHDc8MNN6zWOQEAAADoeGsUSn3729/O6aefnm9+85spL1+jaakAAAAAeBdbo0SpoaEhn/70pwVSAAAAAKyRNUqVxowZk9/85jdt3RYAAAAA3iXWaPjekiVLct5552XChAnZcsst3zDR+fnnn98mjQMAAADgnWmNQqlHH30022yzTZLkP//5T6t9r530HAAAAACWZ41Cqdtvv72t2wEAAADAu4iZygEAAAAo3Gr3lDrggANyxRVXpLq6OgcccMBKy/7xj398yw0DAAAA4J1rtUOpXr16tcwX1atXr3ZrEAAAAADvfKsdSl1++eU544wzcuKJJ+byyy9vzzYBAAAA8A73puaUOv300zNv3rz2agsAAAAA7xJvKpRqbm5ur3YAAAAA8C7ypt++t3ReKQAAAABYU6s9p9RSG2+88SqDqdmzZ69xgwAAAAB453vTodTpp5/u7XsAAAAAvCVvOpT6zGc+kwEDBrRHWwAAAAB4l3hTc0qZTwoAAACAtuDtewAAAAAU7k0N32tqamqvdgAAAADwLvKmekoBAAAAQFsQSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIXr0FDq73//e/bff/8MHjw4ZWVlue6661rtP+yww1JWVtZq2XvvvVuVmT17dg4++OBUV1end+/eOfzwwzNv3rwCrwIAAACAN6tDQ6n58+dnq622ysUXX7zCMnvvvXemT5/esvz6179utf/ggw/OY489lltuuSXXX399/v73v+fII49s76YDAAAA8BZ07siT77PPPtlnn31WWqaysjI1NTXL3ffEE0/kpptuyv3335/tttsuSfLjH/84++67b77//e9n8ODByz2uvr4+9fX1Let1dXVreAUAAAAArIm1fk6pO+64IwMGDMgmm2ySo48+OrNmzWrZd88996R3794tgVSS7LnnnikvL8999923wjrHjRuXXr16tSxDhw5t12sAAAAAoLW1OpTae++9c+WVV+a2227L9773vUycODH77LNPlixZkiSpra3NgAEDWh3TuXPn9OnTJ7W1tSusd+zYsZk7d27LMm3atHa9DgAAAABa69Dhe6vymc98puXzFltskS233DIbbLBB7rjjjuyxxx5rXG9lZWUqKyvbookAAAAArIG1uqfU662//vrp169fnn766SRJTU1NZs6c2apMY2NjZs+evcJ5qAAAAADoeG+rUOq5557LrFmzMmjQoCTJqFGjMmfOnDz44IMtZf72t7+lqakpO+ywQ0c1EwAAAIBV6NDhe/PmzWvp9ZQkkydPzsMPP5w+ffqkT58+Of3003PggQempqYmkyZNyje+8Y1suOGGGT16dJJks802y957750jjjgiP/3pT7N48eIce+yx+cxnPrPCN+8BAAAA0PE6tKfUAw88kG222SbbbLNNkuSEE07INttsk1NPPTWdOnXKI488ko985CPZeOONc/jhh2fbbbfNnXfe2Wo+qKuvvjqbbrpp9thjj+y7777ZZZdd8rOf/ayjLgkAAACA1dChPaV22223NDc3r3D/hAkTVllHnz59Mn78+LZsFgAAAADt7G01pxQAAAAA7wxCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHAdGkr9/e9/z/7775/BgwenrKws1113Xav9zc3NOfXUUzNo0KB07do1e+65Z5566qlWZWbPnp2DDz441dXV6d27dw4//PDMmzevwKsAAAAA4M3q0FBq/vz52WqrrXLxxRcvd/95552XH/3oR/npT3+a++67L927d8/o0aOzaNGiljIHH3xwHnvssdxyyy25/vrr8/e//z1HHnlkUZcAAAAAwBro3JEn32effbLPPvssd19zc3MuvPDCnHLKKfnoRz+aJLnyyiszcODAXHfddfnMZz6TJ554IjfddFPuv//+bLfddkmSH//4x9l3333z/e9/P4MHDy7sWgAAAABYfWvtnFKTJ09ObW1t9txzz5ZtvXr1yg477JB77rknSXLPPfekd+/eLYFUkuy5554pLy/Pfffdt8K66+vrU1dX12oBAAAAoDhrbShVW1ubJBk4cGCr7QMHDmzZV1tbmwEDBrTa37lz5/Tp06elzPKMGzcuvXr1almGDh3axq0HAAAAYGXW2lCqPY0dOzZz585tWaZNm9bRTQIAAAB4V1lrQ6mampokyYwZM1ptnzFjRsu+mpqazJw5s9X+xsbGzJ49u6XM8lRWVqa6urrVAgAAAEBx1tpQar311ktNTU1uu+22lm11dXW57777MmrUqCTJqFGjMmfOnDz44IMtZf72t7+lqakpO+ywQ+FtBgAAAGD1dOjb9+bNm5enn366ZX3y5Ml5+OGH06dPnwwbNixf+9rXctZZZ2WjjTbKeuutl+985zsZPHhwPvaxjyVJNttss+y999454ogj8tOf/jSLFy/Osccem8985jPevAcAAACwFuvQUOqBBx7IBz/4wZb1E044IUkyZsyYXHHFFfnGN76R+fPn58gjj8ycOXOyyy675KabbkpVVVXLMVdffXWOPfbY7LHHHikvL8+BBx6YH/3oR4VfCwAAAACrr0NDqd122y3Nzc0r3F9WVpYzzjgjZ5xxxgrL9OnTJ+PHj2+P5gEAAADQTtbaOaUAAAAAeOcSSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIUTSgEAAABQOKEUAAAAAIVbq0Op0047LWVlZa2WTTfdtGX/okWLcswxx6Rv377p0aNHDjzwwMyYMaMDWwwAAADA6lirQ6kk2XzzzTN9+vSW5a677mrZd/zxx+cvf/lLfve732XixIl54YUXcsABB3RgawEAAABYHZ07ugGr0rlz59TU1Lxh+9y5c3PZZZdl/Pjx2X333ZMkl19+eTbbbLPce++92XHHHYtuKgAAAACraa3vKfXUU09l8ODBWX/99XPwwQdn6tSpSZIHH3wwixcvzp577tlSdtNNN82wYcNyzz33rLTO+vr61NXVtVoAAAAAKM5aHUrtsMMOueKKK3LTTTflkksuyeTJk/P+978/r7zySmpra1NRUZHevXu3OmbgwIGpra1dab3jxo1Lr169WpahQ4e241UAAAAA8Hpr9fC9ffbZp+XzlltumR122CHDhw/Pb3/723Tt2nWN6x07dmxOOOGElvW6ujrBFAAAAECB1uqeUq/Xu3fvbLzxxnn66adTU1OThoaGzJkzp1WZGTNmLHcOqteqrKxMdXV1qwUAAACA4rytQql58+Zl0qRJGTRoULbddtt06dIlt912W8v+J598MlOnTs2oUaM6sJUAAAAArMpaPXzvxBNPzP7775/hw4fnhRdeyHe/+9106tQpn/3sZ9OrV68cfvjhOeGEE9KnT59UV1fnuOOOy6hRo7x5DwAAAGAtt1aHUs8991w++9nPZtasWenfv3922WWX3Hvvvenfv3+S5IILLkh5eXkOPPDA1NfXZ/To0fnJT37Swa0GAAAAYFXW6lDqmmuuWen+qqqqXHzxxbn44osLahEAAAAAbeFtNacUAAAAAO8MQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACieUAgAAAKBwQikAAAAACveOCaUuvvjijBgxIlVVVdlhhx3yz3/+s6ObBAAAAMAKvCNCqd/85jc54YQT8t3vfjcPPfRQttpqq4wePTozZ87s6KYBAAAAsBydO7oBbeH888/PEUcckc9//vNJkp/+9Kf561//mv/3//5fvvnNb76hfH19ferr61vW586dmySpq6srpsHtaN6SJR3dBFaiqHvMfbB2cx+QuA8ocR+QuA8ocR/gHiB5Z+QSybLraG5uXmm5suZVlVjLNTQ0pFu3bvn973+fj33sYy3bx4wZkzlz5uRPf/rTG4457bTTcvrppxfYSgAAAIB3l2nTpmXIkCEr3P+27yn10ksvZcmSJRk4cGCr7QMHDsx///vf5R4zduzYnHDCCS3rTU1NmT17dvr27ZuysrJ2bS+rr66uLkOHDs20adNSXV3d0c2hg7gPSNwHlLgPSNwHlLgPSNwHuAfWZs3NzXnllVcyePDglZZ724dSa6KysjKVlZWttvXu3btjGsMqVVdXe8DgPiCJ+4AS9wGJ+4AS9wGJ+wD3wNqqV69eqyzztp/ovF+/funUqVNmzJjRavuMGTNSU1PTQa0CAAAAYGXe9qFURUVFtt1229x2220t25qamnLbbbdl1KhRHdgyAAAAAFbkHTF874QTTsiYMWOy3Xbb5X3ve18uvPDCzJ8/v+VtfLw9VVZW5rvf/e4bhlry7uI+IHEfUOI+IHEfUOI+IHEf4B54J3jbv31vqYsuuij/93//l9ra2my99db50Y9+lB122KGjmwUAAADAcrxjQikAAAAA3j7e9nNKAQAAAPD2I5QCAAAAoHBCKQAAAAAKJ5QCAAAAoHBCKQAAAAAK17mjGwCwMvX19amsrOzoZtDB3AcAJMnixYtTW1ubBQsWpH///unTp09HN4kO4D6Adw6hFGuVpqamTJw4MXfeeWemTJnS8hfNNttskz333DNDhw7t6CbSzm688cZcc801ufPOOzNt2rQ0NTWle/fu2WabbbLXXnvl85//fAYPHtzRzaSduQ9Yqr6+Pvfdd98b/k5Yb731OrppFGTOnDm59tprl/v/BqNHj85OO+3U0U2knb3yyiu56qqrcs011+Sf//xnGhoa0tzcnLKysgwZMiR77bVXjjzyyGy//fYd3VTakfuA15o8efJy/14YNWpUqqqqOrp5vAllzc3NzR3dCFi4cGF+8IMf5JJLLsns2bOz9dZbZ/DgwenatWtmz56d//znP3nhhRey11575dRTT82OO+7Y0U2mjV177bU5+eST88orr2TffffN+973vjfcA3feeWfuueeeHHbYYTnzzDPTv3//jm42bcx9wFJ33313fvjDH+Yvf/lLFi9enF69erXcB/X19Vl//fVz5JFH5qijjkrPnj07urm0gxdeeCGnnnpqrr766gwePHi5z4MHH3www4cPz3e/+918+tOf7ugm0w7OP//8nH322dlggw2y//77r/Dvheuuuy477LBDfvzjH2ejjTbq6GbTxtwHLHX11Vfnhz/8YR544IEMHDiw1X0wadKkVFVV5eCDD87JJ5+c4cOHd3RzWQ1CKdYKQ4cOzahRo3LYYYflQx/6ULp06fKGMlOmTMn48eNz6aWX5tvf/naOOOKIDmgp7WXUqFE55ZRTss8++6S8fMXT3T3//PP58Y9/nIEDB+b4448vsIUUwX1AknzkIx/JQw89lIMOOij7779/tttuu3Tt2rVl/zPPPJM777wzv/71r/Pvf/87V155ZT70oQ91YItpDwMHDsyYMWNy2GGHZeTIkcsts3Dhwlx33XX50Y9+lAMPPDAnnnhiwa2kvX32s5/NKaecks0333yl5err63P55ZenoqIiX/jCFwpqHUVxH5Ak22yzTSoqKjJmzJjsv//+bxhFU19fn3vuuSfXXHNN/vCHP+QnP/lJPvnJT3ZQa1ldQinWCk888UQ222yz1Sq7ePHiTJ06NRtssEE7twqAjnDppZfmC1/4wnL/geL1Hn/88UyfPj177LFHAS2jSLNmzUrfvn3brTwAby8TJkzI6NGjV6vsrFmz8uyzz2bbbbdt51bxVgmlgLVWQ0NDJk+enA022CCdO5sC793KfQDAaz399NOZNGlSdt1113Tt2rVlXiHeXdwH8M6w4rER0IHuvPPOfO5zn8uoUaPy/PPPJ0l+9atf5a677urgllGEBQsW5PDDD0+3bt2y+eabZ+rUqUmS4447Lueee24Ht46iuA9Yas6cOfnFL36RsWPHZvbs2UmShx56qOXvB94dfvWrX2XnnXfO4MGDM2XKlCTJhRdemD/96U8d3DKKMmvWrOy5557ZeOONs++++2b69OlJksMPPzxf//rXO7h1FMV9wFKTJk3KKaecks9+9rOZOXNmktLLch577LEObhlvhlCKtc4f/vCHjB49Ol27ds2//vWv1NfXJ0nmzp2bc845p4NbRxHGjh2bf//737njjjtavT1jzz33zG9+85sObBlFch+Q5P+3d+dxNeX/H8BftyglJUtUWiVNhLKMrMkSGWOfhbFkxljHvg6ixhjL2Bpm7NJs1hgMRqTIMiQKWZOyZSmhhVL9/ujR/XUrhplv53O65/V8PDy+c8+5M/Oar9T7fO7nvA5iYmLg6OiIBQsW4Pvvv0dqaioAIDg4GNOnTxcbjiTz008/YcKECfD29kZqaipycnIAAJUrV8ayZcvEhiPJjB8/HuXKlUNiYiIMDQ3Vxz/++GMcOHBAYDKSEr8OCADCw8Ph4uKCv//+G8HBwUhLSwMAREdHY/bs2YLT0bvgohTJzty5c7Fq1SqsXbtWo0+kZcuWiIqKEpiMpLJr1y6sWLECrVq10tiGXa9ePcTFxQlMRlLi1wEBwIQJEzB48GBcv35dY3HS29sbR48eFZiMpPTDDz9g7dq1mDFjBnR1ddXHmzRpggsXLghMRlI6ePAgFixYgFq1amkcr1Onjnr3HGk/fh0QAEybNg1z585FSEgI9PT01Mc9PT1x6tQpgcnoXXFRimTn6tWraNOmTbHjJiYm6k/ISbs9evQIZmZmxY6np6ezK0BB+HVAAHDmzBkMGzas2HFLS0skJSUJSEQixMfHw9XVtdhxfX19pKenC0hEIqSnp2vsjCmQkpICfX19AYlIBH4dEABcuHABPXv2LHbczMwMjx8/FpCI/i0uSpHs1KxZEzdu3Ch2PCIiAvb29gISkdSaNGmCP//8U/26YAFi3bp1cHd3FxWLJMavAwLyFx2ePXtW7Pi1a9dQvXp1AYlIBDs7O5w/f77Y8QMHDrz103up7GvdujWCgoLUr1UqFXJzc7Fw4UK0a9dOYDKSEr8OCMi/fbugT6ywc+fOwdLSUkAi+rf4GCOSnaFDh2Ls2LHYsGEDVCoV7t27h5MnT2LSpEmYNWuW6HgkgXnz5qFLly6IjY3Fq1evsHz5csTGxuLEiRMIDw8XHY8kwq8DAoAPP/wQ/v7+2Lp1K4D8i4/ExERMnToVvXv3FpyOpDJhwgSMGjUKL168QF5eHk6fPo3ff/8d3333HdatWyc6Hklk4cKFaN++PSIjI5GVlYUpU6bg0qVLSElJwfHjx0XHI4nw64AA4JNPPsHUqVOxbds29cLk8ePHMWnSJAwcOFB0PHoHqry8vDzRIYgKy8vLw7x58/Ddd98hIyMDQP4n5ZMmTcI333wjOB1JJS4uDvPnz0d0dDTS0tLg5uaGqVOnwsXFRXQ0khC/Dujp06fo06cPIiMj8fz5c1hYWCApKQnu7u7Yt28fKlasKDoiSeTXX3/FnDlz1J1yFhYW8PPzw+effy44GUnp6dOnWLFihcbPhVGjRsHc3Fx0NJIQvw4oKysLo0aNQmBgIHJyclCuXDnk5OSgX79+CAwM1OgfJHnjohTJVlZWFm7cuIG0tDQ4OzvDyMhIdCQiIhIkIiICMTEx6ouPDh06iI5EgmRkZCAtLa3EzjkiIlKWxMREXLx4EWlpaXB1dUWdOnVER6J3xEUpkr1nz54hNDQUdevWZW+EQiQmJr7xvLW1tURJSKSoqCiUL19evSvqjz/+wMaNG+Hs7Iw5c+ZoPGmFiLRbZmYm8vLy1OXGCQkJ2LlzJ5ydndGpUyfB6Ugq//TEzZIelEPa58CBAzAyMkKrVq0AACtXrsTatWvh7OyMlStXwtTUVHBCInoXXJQi2fnoo4/Qpk0bjB49GpmZmWjUqBHi4+ORl5eHzZs3s0NEAXR0dN74dLWcnBwJ05AoTZs2xbRp09C7d2/cvHkTzs7O6NWrF86cOYOuXbti2bJloiOSBPz9/d943tfXV6IkJFKnTp3Qq1cvDB8+HKmpqahbty709PTw+PFjLFmyBCNGjBAdkSSgo1P8GU2F5wXOB8rg4uKCBQsWwNvbGxcuXECTJk0wceJEHDlyBE5OTti4caPoiCSBIUOGvPH8hg0bJEpC/xWLzkl2jh49ihkzZgAAdu7cidzcXKSmpmLTpk2YO3cuF6UU4Ny5cxqvs7Ozce7cOSxZsgTffvutoFQktWvXrqFRo0YAgG3btqFt27b47bffcPz4cXzyySdclFKInTt3arzOzs5GfHw8ypUrh9q1a3NRSiGioqKwdOlSAMD27dtRs2ZNnDt3Djt27ICvry8XpRTiyZMnGq8L5oNZs2ZxPlCQ+Ph4ODs7AwB27NiBbt26Yd68eYiKioK3t7fgdCSVkr4fXLx4EampqfD09BSUiv4NLkqR7Dx9+hRVqlQBkL89t3fv3jA0NETXrl0xefJkwelICg0bNix2rEmTJrCwsMCiRYvQq1cvAalIanl5ecjNzQUAHDp0CB988AEAwMrKCo8fPxYZjSRUdJEayL+te/DgwejZs6eARCRCRkYGKlWqBAA4ePAgevXqBR0dHTRv3hwJCQmC05FUTExMih3r2LEj9PT0MGHCBJw9e1ZAKpKanp6e+mFIhw4dUj9prUqVKnj27JnIaCShoh9aAUBubi5GjBiB2rVrC0hE/1bxPbBEgllZWeHkyZNIT0/HgQMH1F0RT548QYUKFQSnI5Hq1q2LM2fOiI5BEmnSpAnmzp2Ln3/+GeHh4ejatSuA/E9Ia9SoITgdiWRsbAw/Pz/MmjVLdBSSiIODA3bt2oXbt2/jr7/+Us8GDx8+hLGxseB0JFqNGjVw9epV0TFIIq1atcKECRPwzTff4PTp0+r54Nq1a6hVq5bgdCSSjo4OJkyYoN5ZS2UDd0qR7IwbNw79+/eHkZERbGxs4OHhASD/tj4+Bl4Zin7KlZeXh/v372POnDl8ooaCLFu2DP3798euXbswY8YMODg4AMi/dadFixaC05FoT58+xdOnT0XHIIn4+vqiX79+GD9+PNq3bw93d3cA+bumXF1dBacjqcTExGi8LpgP5s+fr77dm7TfihUrMHLkSGzfvh0//fQTLC0tAQD79+9H586dBacj0eLi4vDq1SvRMegdsOicZCkyMhK3b99Gx44dYWRkBAD4888/UblyZbRs2VJwOiptJRWd5+XlwcrKCps3b1ZfjJAyvXjxArq6uihfvrzoKCSBgIAAjdcFF6E///yzumeMlCEpKQn3799Hw4YN1YXXp0+fhrGxMZycnASnIykUzAdFL1+aN2+ODRs28OuASEEmTJig8bpgPvjzzz8xaNAgrFixQlAyeldclCIi2QkPD9d4raOjg+rVq8PBwQHlynGDJ5GS2NnZabwu+H7g6emJ6dOnq3uGiEj7Fe0PK/h+wHoH7fcuXVG8pVcZ2rVrp/G68HwwZMgQXjOUIVyUIlkoutL9JkuWLCnFJCRadnY2hg0bhlmzZhW7GCXtZ2pqWmyX3OukpKSUchoiEuldHmoRHBxciklIDrKzs9G5c2esWrWKt/IrUEm76IvKy8uDSqVCTk6ORKlIlLy8PNy+fRvVq1eHgYGB6Dj0H3H5kGShpKcrleRtL1ap7Cpfvjx27NjBAmOFWrZsmegIJCPZ2dkwMDDA+fPnUb9+fdFxSGIlPWmNlKt8+fLFOqVIOY4cOSI6AslIXl4eHBwccOnSJS5SawHulCIi2Rk0aBAaNWqE8ePHi45CRILZ29tj586daNiwoegoRCTY+PHjoa+vj/nz54uOQkSC1atXD+vXr0fz5s1FR6H/iDuliEh26tSpA39/fxw/fhyNGzdGxYoVNc6PGTNGUDIS5cWLF8jKytI4xs4IZZgxYwa+/vpr/Pzzz6hSpYroOEQk0KtXr7BhwwYcOnSoxPmAFQ/KkpGRgcTExGLzQYMGDQQlIinNnz8fkydPxk8//cTd1GUcd0qRLEVGRmLr1q0l/qBhb4T2e1OXlEqlws2bNyVMQ6Kkp6dj6tSp2Lp1K5KTk4udZ2eEMri6uuLGjRvIzs6GjY1NsYvQqKgoQclIatu3b3/tbMCvA2UoWmxcFG/xUoZHjx7Bx8cH+/fvL/E85wNlMDU1RUZGBl69egU9Pb1i3VLsHi07uFOKZGfz5s0YOHAgvLy8cPDgQXTq1AnXrl3DgwcP0LNnT9HxSALx8fGiI5AMTJkyBUeOHMFPP/2EAQMGYOXKlbh79y5Wr17NWzcUpHv37uwTJAQEBGDGjBkYPHgw/vjjD/j4+CAuLg5nzpzBqFGjRMcjiXDRiQBg3LhxSE1Nxd9//w0PDw/s3LkTDx48wNy5c7F48WLR8UgiS5cu5XygJbhTimSnQYMGGDZsGEaNGoVKlSohOjoadnZ2GDZsGMzNzeHn5yc6IpUyf39/TJo0CYaGhhrHMzMzsWjRIvj6+gpKRlKytrZGUFAQPDw8YGxsjKioKDg4OODnn3/G77//jn379omOSEQScXJywuzZs/Hpp5+qZwN7e3v4+voiJSUFK1asEB2RJDBkyBAsX74clSpV0jienp6Or776Chs2bBCUjKRkbm6OP/74A82aNYOxsTEiIyPh6OiI3bt3Y+HChYiIiBAdkYjegY7oAERFxcXFoWvXrgAAPT09pKenQ6VSYfz48VizZo3gdCQFPz8/pKWlFTuekZHBRUkFSUlJgb29PYD8/qiCbditWrXC0aNHRUYjCdnb25d4+2Zqaqr664O0X2JiIlq0aAEAMDAwwPPnzwEAAwYMwO+//y4yGklo06ZNyMzMLHY8MzMTQUFBAhKRCOnp6TAzMwOQfwvXo0ePAAAuLi68lVdBdHV18fDhw2LHk5OToaurKyAR/VtclCLZMTU1VQ+blpaWuHjxIoD8C5CMjAyR0UgieXl5JW7HjY6OZtGxgtjb26tv5XRycsLWrVsBAHv27EHlypUFJiMp3bp1q8R+kJcvX+LOnTsCEpEINWvWVC9MW1tb49SpUwDyb/fmpn/t9+zZMzx9+hR5eXl4/vw5nj17pv715MkT7Nu3T71IQdqvbt26uHr1KgCgYcOGWL16Ne7evYtVq1bB3NxccDqSyuu+9798+RJ6enoSp6H/gp1SJDtt2rRBSEgIXFxc0LdvX4wdOxahoaEICQlB+/btRcejUmRqagqVSgWVSgVHR0eNhamcnBykpaVh+PDhAhOSlHx8fBAdHY22bdti2rRp6NatG1asWIHs7Gw+YUkBdu/erf7rv/76CyYmJurXOTk5OHz48BsfikDaxdPTE7t374arqyt8fHwwfvx4bN++HZGRkejVq5foeFTKKleurDEfFKVSqbiTWkHGjh2L+/fvAwBmz56Nzp0749dff4Wenh4CAwPFhqNSFxAQACD/z/26detgZGSkPpeTk4OjR4/CyclJVDz6F9gpRbKTkpKCFy9ewMLCArm5uVi4cCFOnDiBOnXqYObMmTA1NRUdkUrJpk2bkJeXhyFDhmDZsmUaF6F6enqwtbWFu7u7wIQkUkJCAs6ePQsHBwc+7lkBdHTyN3OrVKpin4aWL18etra2WLx4MT744AMR8Uhiubm5yM3NRbly+Z+nbt68WT0bDBs2jJ+Ka7nw8HDk5eXB09MTO3bs0Ng1raenBxsbG1hYWAhMSKXt2bNnMDY2LvFcRkYGrly5Amtra1SrVk3iZCS1gg+kEhISUKtWLY1b9QquF/z9/fH++++LikjviItSJAsTJkzAN998g4oVK+Lo0aNo0aKFevAk5QkPD0eLFi1Qvnx50VFIYlWqVMG1a9dQrVq11xbakrLY2dnhzJkzvNBQoF69eiEwMBDGxsYICgrCxx9/DH19fdGxSKCEhARYWVmpF61JOXR1dXH//n2YmZnB09MTwcHBvJVf4dq1a4fg4GBuWNACXJQiWShfvjzu3LmDGjVqaPzQISJlMTIyQkxMDOzt7aGrq4ukpCRUr15ddCwiEkBPTw8JCQkwNzfnbECkcCYmJjh16hTee+896Ojo4MGDB5wPiLQEt6KQLNja2iIgIACdOnVCXl4eTp48+dpV7zZt2kicjkRo164dbGxsNLoBBg0ahNu3byM0NFRcMCpV7u7u6NGjBxo3boy8vDyMGTMGBgYGJb6Xj/5WjsTERJQvX16jwPb+/fvIzs6GtbW1wGRUmpycnDB9+nS0a9cOeXl52Lp162tv3xk4cKDE6UgUOzs7ODg4ICQkRH2sQ4cOuHnzJm7evCkwGZWmDh06oF27dnjvvfcAAD179nztbbucE5Xj6NGjMDQ0RJMmTdTHIiMjkZGRwWvGMoSLUiQLixYtwvDhw/Hdd99BpVKhZ8+eJb5PpVKV+BQm0j62trbFnqBiaWnJLfta7pdffsHSpUsRFxcHlUqFp0+f4sWLF6JjkWC2trZwcnJCbGys+pinpyeuXbvGnwlabNWqVZgwYQL+/PNPqFQqzJw5s8Qns6pUKi5KKcigQYOK7ZDp2bMnHj9+LCgRSeGXX37Bpk2bEBcXh/DwcNSrVw+GhoaiY5FgHh4exeaDAQMGcD4oY3j7HslKWloajI2NcfXq1ddu0S9cfk1E2svOzg6RkZGoWrWq6CgkWHh4OAwNDdG0aVP1sTNnziAjIwNt27YVmIykoqOjg6SkJN6+R0Ro164ddu7cyU4pQkJCAsqXL6/xoIN79+4hOzsbNjY2ApPRu+CiFMlOeHg4WrZsyaJzIiIiApB/4WFtbV3iTikiIiIqu7goRUSydPjwYRw+fBgPHz5Ebm6uxjl2CSkHu4SIiKhATk4OAgMDXzsfsEtIOdglRAWysrJK/H7AObHs4FYUkiWWXCubn58f/P390aRJE5ibm/OTcQVjlxA9ePAAkyZNUl+EFv0sjV8HysGCaxo7diwCAwPRtWtX1K9fn/OBgrFLiK5fv44hQ4bgxIkTGsfz8vLYQ1zGcFGKZIkl18q2atUqBAYGYsCAAaKjkGBHjhwpVmQaFBSEjIwMQYlIaoMHD0ZiYiJmzZrFRWqFY8E1bd68GVu3boW3t7foKCRYfHw8ypcvr3Hs8OHDyM7OFpSIpDZ48GCUK1cOe/fu5XxQxvH2PSKSnapVq+L06dOoXbu26ChEJFilSpVw7NgxNGrUSHQUIhLMwsICYWFhcHR0FB2FiASrWLEizp49CycnJ9FR6D/ithMikp0vvvgCv/32m+gYRCQDVlZWxW7ZIyJlmjhxIpYvX87vCQQgv0vozp07SExM1PhFyuDs7MydslqCO6VIllhyrWxjx45FUFAQGjRogAYNGhTbnr1kyRJByUhK7BIiADh48CAWL16M1atXw9bWVnQcEoQF1wTk36555MgRVKlSBfXq1Ss2HwQHBwtKRlJilxAB+d/3Z86ciXnz5sHFxaXY9wNjY2NByehdsVOKZIcl1xQTE6O+VefixYtiw5Aw7BIiAPj444+RkZGB2rVrw9DQsNjQmZKSIigZSYkF1wQAlStXRs+ePUXHIMHYJURA/oMuAKB9+/Yax7k4WfZwpxTJjrm5ORYuXMiSayKFY5cQAUBgYOAbLzgGDRokYRoSpVq1aggKCmLBNRGxS4gAAOHh4W8837ZtW4mS0H/FnVIkO1lZWWjRooXoGCTQkCFDsHz5clSqVEnjeHp6Or766ivewqkQ7BIiIP8TcSI9PT04ODiIjkGCeXp6Ijg4GJUrV9Y4/uzZM/To0YO3cSoEu4QIAOzs7GBlZVXsg6u8vDzcvn1bUCr6N1h0TrLDkmvatGkTMjMzix3PzMxEUFCQgEQkwrJlyzBt2jTcunVLdBQSSFdXFw8fPix2PDk5Gbq6ugISkQgsuCYACAsLQ1ZWVrHjL168wLFjxwQkIhEWLFiAKVOmICwsDMnJyXj27JnGL1IGOzs7PHr0qNjxlJQU2NnZCUhE/xZ3SpHsvHjxAmvWrMGhQ4dYcq0wz549Q15eHvLy8vD8+XNUqFBBfS4nJwf79u2DmZmZwIQkJXYJEYDXLkK8fPkSenp6EqchUSIiInDkyBHs37+fBdcKFBMTo/7r2NhYJCUlqV/n5OTgwIEDsLS0FBGNBGCXEAH///tdVFpamsY1BMkfF6VIdlhyrVyVK1eGSqWCSqWCo6NjsfMqlQp+fn4CkpEIS5cuZXmpggUEBADI/3O/bt06GBkZqc/l5OTg6NGj7BNREBZcK1ujRo3U84Gnp2ex8wYGBvjhhx8EJCMRjhw5IjoCCTRhwgQA+fPBrFmzYGhoqD6Xk5ODv//+m32kZQyLzolINsLDw5GXlwdPT0/s2LEDVapUUZ/T09ODjY0NLCwsBCYkIqkUbL1PSEhArVq1NG7V09PTg62tLfz9/fH++++LikhEEklISEBeXh7s7e1x+vRpVK9eXX1OT08PZmZmvJ1XQRITE9/YJWRtbS0oGUmhXbt2APKvG9zd3TV2TRfMB5MmTUKdOnVERaR3xEUpkh2WXFNCQgKsra1L3CWTmJjIYUMhdHV1cf/+/WK3bCYnJ8PMzIzb8xWiXbt2CA4OhqmpqegoJBALrgnIvwht2bIlypXTvNkjJycHx48fR5s2bQQlIylxPiAA8PHxwfLly2FsbCw6Cv1HLDon2WHJNdnb25dYXJicnMziQgVhlxAB+YtS+vr6xY5nZmbC399fQCISgQXXBOQvTpbUJ5iamqrePUHaj11CBEB9S29R6enpGDJkiIBE9G+xU4pkgyXXVOB1ixEcNpSBXUJUmJ+fH4YPH67RGQEAGRkZ8PPzg6+vr6BkJAUWXFNhr1uMSE5ORsWKFQUkIimxS4gK27RpE+bPn1/s7pqCjQy8u6bs4KIUyQZLrqnwsOHr68thQ6GWLl0KIP/iY9WqVSV2Ca1atUpUPJLY6y5Co6OjNXrnSDux4JoAoFevXgDy54PBgwdr7J7MyclBTEwMWrRoISoeSeTcuXMA8n8uXLhwoViXUMOGDTFp0iRR8Ugi3MigfbgoRbJx5MgRllwrHIcNAoD4+HgA7BJSOlNTU40PKgovTOXk5CAtLQ3Dhw8XmJCkEB8fz4JrgomJCYD8+aBSpUowMDBQn9PT00Pz5s0xdOhQUfFIIgVP3WOXkLJxI4P24aIUyUbbtm0B5A+gLLlWJg4bVNibuoQWLVrE27a03LJly5CXl4chQ4bAz89PfVEK/P+OOXd3d4EJSQo2NjYA8n8+NGrUqMSC66NHj7LgWstt3LgRANRP1eKtesr2pi4hPhRJ+3Ejg/bh0/dIdvhEDSIC+L2A8oWHh6NFixYoX7686CgkEL8fEFGB130/ePz4MWrWrIlXr14JSkZSSkhIgJWVFXR0+Oy2so47pUh2WHJNABAZGYmtW7ciMTGx2BOXgoODBaUiKbFLiID/30UL5D9prej3A+6oVAYWXFOB7du3v3Y+iIqKEpSKpMAuISqsYCdtRkZGid8PGjRoICIW/QtclCLZYMk1Fdi8eTMGDhwILy8vHDx4EJ06dcK1a9fw4MED9OzZU3Q8KmXsEqLCMjIyMGXKFGzduhXJycnFznOHjHZjwTUVFhAQgBkzZmDw4MH4448/4OPjg7i4OJw5cwajRo0SHY9KGbuEqLBHjx7Bx8cH+/fvL/E854Oyg4tSJBssuaYC8+bNw9KlSzFq1ChUqlQJy5cvh52dHYYNGwZzc3PR8aiUsUuICps8eTKOHDmCn376CQMGDMDKlStx9+5drF69GvPnzxcdj0oZC66psB9//BFr1qzBp59+isDAQEyZMgX29vbw9fVFSkqK6HhUytglRIWNGzcOqamp+Pvvv+Hh4YGdO3fiwYMHmDt3LhYvXiw6Hr0DdkqR7LDkmipWrIhLly7B1tYWVatWRVhYGFxcXHD58mV4enri/v37oiOSBNglRABgbW2NoKAgeHh4wNjYGFFRUXBwcMDPP/+M33//Hfv27RMdkSTg5+fHgmuCoaEhLl++DBsbG5iZmSEkJAQNGzbE9evX0bx58xJ3U5L2YZcQAYC5uTn++OMPNGvWDMbGxoiMjISjoyN2796NhQsXIiIiQnREekvcKUWyU/CEFVIuU1NTPH/+HABgaWmJixcvwsXFBampqcjIyBCcjqTCLiECgJSUFNjb2wPI/z0v2A3RqlUrjBgxQmQ0ktDs2bNFRyAZqFmzJlJSUmBjYwNra2ucOnUKDRs2RHx8/Gs7SUn7sEuIgPynLRZ0iJmamuLRo0dwdHSEi4sL++XKGC5KkSyx5FrZ2rRpg5CQELi4uKBv374YO3YsQkNDERISgvbt24uORxJhlxABgL29PeLj42FtbQ0nJyds3boVzZo1w549e1C5cmXR8UhCLLgmT09P7N69G66urvDx8cH48eOxfft2REZGqvvHSPuxS4gAoG7durh69SpsbW3RsGFDrF69Gra2tli1ahXrPsoY7nkk2dm8eTNatGiBy5cvY+fOncjOzsalS5cQGhqq0S1D2mvFihX45JNPAAAzZszAhAkT8ODBA/Tu3Rvr168XnI6kMnnyZISGhuKnn36Cvr4+1q1bBz8/P1hYWCAoKEh0PJKIj48PoqOjAQDTpk3DypUrUaFCBYwfPx6TJ08WnI6kEhAQAB8fH9SoUQPnzp1Ds2bNULVqVdy8eRNdunQRHY8ksmbNGsyYMQMAMGrUKGzYsAHvvfce/P398dNPPwlOR1Ip3CVkYGCAAwcOYNOmTahTpw52794tOh5JZOzYsepKj9mzZ2P//v2wtrZGQEAA5s2bJzgdvQt2SpHsNGjQAMOGDVOXXEdHR2uUXPOpGkTKwC4hKklCQgLOnj0LBwcH3qKhIE5OTpg9ezY+/fRT9WxQuOB6xYoVoiMSkUTYJUQlycjIwJUrV2BtbY1q1aqJjkPvgDulSHbi4uLQtWtXAPlP0khPT4dKpcL48eOxZs0awelIal27dmWxuUK9qUvo6NGjIqORIHfu3IGVlRV69erFBSmFSUxMRIsWLQAABgYG6t7BAQMG4PfffxcZjQRxcXHB7du3RccgAUrqEgLALiEFO378OHR1deHm5sYFqTKIi1IkOyWVXANgybVCHT16FJmZmaJjkAAFXUIA1F1CANglpGDOzs64deuW6BgkQEHBNQB1wTUAFlwr2K1bt5CdnS06BglQ0CUEQN0ldPfuXXYJKViXLl1w9+5d0THoX2LROckOS66JCPj/LqG2bdti2rRp6NatG1asWIHs7GwsWbJEdDwSgIsPysWCayIqULRLqHPnzvj111+hp6eHwMBAseFICM4HZRs7pUh2UlJS8OLFC1hYWCA3NxcLFy7EiRMnUKdOHcycOROmpqaiI5KE6tevj/3798PKykp0FBKMXUJUuEuIlCU3Nxe5ubkoVy7/89TNmzerZ4Nhw4ZBT09PcEKSmre3N9avX8+dMcQuIeJ8UMZxUYqIiGTvzp07sLCwgI4O7zpXsu+++w4jRozg7ZtERAQgv0uoSZMm0NfXFx2FBPrtt9/QvXt3VKxYUXQU+he4KEWy1rVrV6xbt46fgilQamoqTp8+jYcPHyI3N1fj3MCBAwWlIlGMjY1x/vx5fgJGRHBxccG+ffu4g1ahrl+/jiNHjpQ4H/j6+gpKRaJwPiAq+9gpRbLGkmtl2rNnD/r374+0tDQYGxtDpVKpz6lUKi5KKRA/P1GunJwcBAYG4vDhwyVehIaGhgpKRqKw4Fq51q5dixEjRqBatWqoWbNmsfmAi1LKw/lAudLT0zF//vzXzgc3b94UlIzeFReliEh2Jk6ciCFDhmDevHkwNDQUHYeIBBo7diwCAwPRtWtX1K9fX+MilIiUZe7cufj2228xdepU0VGISLAvvvgC4eHhGDBgAMzNzTkflGFclCJZs7GxQfny5UXHIIndvXsXY8aM4YIUqX399deoUqWK6BgkwObNm7F161Z4e3uLjkIy0bp1axgYGIiOQQI8efIEffv2FR2DZGT16tWoUaOG6BgkwP79+/Hnn3+iZcuWoqPQf8TGWJK1ixcvsjNCgby8vBAZGSk6BsnI9OnTWW6tUHp6enBwcBAdg2Rk37597JpUqL59++LgwYOiY5CM9OvXj+XWCmVqasoPLLUEi85JllhyrTy7d+9W//WjR4/g7+8PHx8fuLi4FNst9+GHH0odjwRglxABwOLFi3Hz5k2sWLGCW/MVjgXXyhQQEKD+6/T0dCxZsgRdu3YtcT4YM2aM1PFIAHYJEQD88ssv+OOPP7Bp0ybeXVHGcVGKZOefSq5TUlIEpqPSoqPzdhs3VSoVcnJySjkNycHo0aPVXUIldQUsXbpUUDIqbb169dJ4HRoaiipVqqBevXrFLkKDg4OljEaC/FPBdVRUlMB0VJrs7Oze6n0qlYqLEQrx6aefvrFLaOzYsYKSUWlzdXXV+P2+ceMG8vLyYGtrW2w+4M+FsoOLUiQ7jo6O8Pb2Zsk1kcJVq1YNQUFB7BJSIB8fn7d+78aNG0sxCcmFjY0NRo4cyYJrIkLlypXZJaRQfn5+b/3e2bNnl2IS+l/iohTJTsWKFXHhwgXY29uLjkIykpqayk4hhbGwsEBYWBgcHR1FRyEiwYyNjXH+/HnOBqQhJycHFy5cgI2NDUxNTUXHIYnY2dlh3759eO+990RHIaL/ARadk+yw5JoWLFiALVu2qF/37dsXVapUgaWlJaKjowUmIylNnDgRy5cvBz87UbbMzExkZGSoXyckJGDZsmUsO1YYFlwTAIwbNw7r168HkL8g1aZNG7i5ucHKygphYWFiw5FkvvnmG/j6+mr8bCDluX37Nu7cuaN+ffr0aYwbNw5r1qwRmIr+De6UIllgyTUVZmdnh19//RUtWrRASEgIPvroI2zZsgVbt25FYmIiL0y0GLuEqKhOnTqhV69eGD58OFJTU1G3bl3o6enh8ePHWLJkCUaMGCE6IpUSFlxTUbVq1cKuXbvQpEkT7Nq1C6NGjcKRI0fw888/IzQ0FMePHxcdkUoJu4SoqNatW+PLL7/EgAEDkJSUBEdHR9SvXx/Xr1/HV199xQdglCFclCJZYMk1FWZgYIBr167BysoKY8eOxYsXL7B69Wpcu3YN77//Pp48eSI6IpUSdglRUdWqVUN4eDjq1auHdevW4YcffsC5c+ewY8cO+Pr64vLly6IjUilhwTUVVaFCBdy4cQO1atXCl19+CUNDQyxbtgzx8fFo2LAhnj17JjoilRJ2CVFRpqamOHXqFOrWrYuAgABs2bIFx48fx8GDBzF8+HD+XChDyokOQASg2KNcSdlMTU1x+/ZtWFlZ4cCBA5g7dy4AIC8vj4uSWo4LTVRURkYGKlWqBAA4ePAgevXqBR0dHTRv3hwJCQmC01Fpio+PFx2BZKZGjRqIjY2Fubk5Dhw4gJ9++glA/vcJXV1dwemoNHGhiYrKzs6Gvr4+AODQoUPqu2mcnJxw//59kdHoHbFTisqE1NRU0RFIQr169UK/fv3QsWNHJCcno0uXLgCAc+fOwcHBQXA6kgq7hAgAHBwcsGvXLty+fRt//fUXOnXqBAB4+PAhjI2NBacjUXJycnD+/HnunFUYHx8ffPTRR6hfvz5UKhU6dOgAAPj777/h5OQkOB1JhV1CBAD16tXDqlWrcOzYMYSEhKBz584AgHv37qFq1aqC09G74KIUyQ5Lrmnp0qUYPXo0nJ2dERISAiMjIwDA/fv3MXLkSMHpSCrdu3dHUFAQgPyF6WbNmmHx4sXo3r27+tNx0n6+vr6YNGkSbG1t8f7778Pd3R1A/q4pV1dXwelIKiy4JgCYM2cO1q1bhy+//BLHjx9X75LQ1dXFtGnTBKcjqfTr1w9HjhwBACQlJaFDhw44ffo0ZsyYAX9/f8HpSCoLFizA6tWr4eHhgU8//RQNGzYEkN9V3KxZM8Hp6F2wU4pkhyXXRASwS4j+X1JSEu7fv4+GDRuqOwhPnz4NY2Nj7o5QCBZcE1EBdglRgZycHDx79gympqbqY7du3YKhoSHMzMwEJqN3wU4pkp2kpCRYWVkBAPbu3YuPPvoInTp1Un9KTsoRGxuLxMREZGVlaRznExiVgV1CVKBmzZqoWbOmxjF+Cqosjx8/Vn8N7Nu3D3379oWjoyOGDBmC5cuXC05HUkpPT0d4eHiJ8wGfwqgM7BKiArq6uhoLUgBga2srJgz9a1yUItlhyTXdvHkTPXv2xIULF6BSqVCwobPgUcD8OlCGgi6hnj174q+//sL48eMBsEtIiSIjI9W7ZYtehAYHBwtKRVJiwTUB+d2S3t7eyMjIQHp6OqpUqYLHjx+rd0VwUUoZCrqEunbtipCQEHzzzTcA2CWkRNu3b3/tfBAVFSUoFb0rdkqR7LDkmsaOHQs7Ozs8fPgQhoaGuHTpEo4ePYomTZqwO0RB2CVEALB582a0aNECly9fxs6dO5GdnY1Lly4hNDQUJiYmouORRFhwTQAwfvx4dOvWDU+ePIGBgQFOnTqFhIQENG7cGN9//73oeCQRdgkRAAQEBMDHxwc1atTAuXPn0KxZM1StWhU3b95UXz9S2cBOKZKd7OxsLF++HLdv38bgwYPVF59Lly5FpUqV8MUXXwhOSKWtWrVqCA0NRYMGDWBiYoLTp0+jbt26CA0NxcSJE3Hu3DnREUki7BKiBg0aYNiwYRg1ahQqVaqE6Oho2NnZYdiwYTA3N4efn5/oiCSR7du34/bt2+jbty9q1aoFANi0aRMqV66M7t27C05HUqhcuTL+/vtv1K1bF5UrV8bJkyfx3nvv4e+//8agQYNw5coV0RFJIuwSIicnJ8yePRuffvqpej6wt7eHr68vUlJSsGLFCtER6S1xUYqIZMfU1BRRUVGws7ND7dq1sW7dOrRr1w5xcXFwcXFBRkaG6IhEJJGKFSvi0qVLsLW1RdWqVREWFgYXFxdcvnwZnp6e7A8hUpDq1avjxIkTqFOnDhwdHfHDDz/Ay8sLV65cQePGjZGeni46IhFJxNDQEJcvX4aNjQ3MzMwQEhKChg0b4vr162jevDmSk5NFR6S3xE4pki2WXCtX/fr11bsh3n//fSxcuBB6enpYs2YN7O3tRccjCbFLiExNTfH8+XMAgKWlJS5evAgXFxekpqZygVphWHBNrq6uOHPmDOrUqYO2bdvC19cXjx8/xs8//4z69euLjkcSYpcQ1axZEykpKbCxsYG1tTVOnTqFhg0bIj4+Htx3U7ZwUYpkhyXXNHPmTPWnnf7+/vjggw/QunVrVK1aFVu2bBGcjqSyefNmDBw4EF5eXjh48CA6deqEa9eu4cGDB+jZs6foeCSRNm3aICQkBC4uLujbty/Gjh2L0NBQhISEoH379qLjkURYcE0AMG/ePPUi9bfffouBAwdixIgRqFOnDjZs2CA4HUklICAAM2bMwODBg/HHH3/Ax8cHcXFxOHPmDEaNGiU6HknE09MTu3fvhqurK3x8fDB+/Hhs374dkZGR6NWrl+h49A54+x7JTrdu3aCrq4t169bBzs4Op0+fRnJyMiZOnIjvv/8erVu3Fh2RBEhJSYGpqal6cZK0H7uECMj/s//ixQtYWFggNzcXCxcuVN++M3PmzGKPgibt5OHhAUdHR6xatQomJiaIjo5G+fLl8dlnn2Hs2LG8ACFSEHYJEQDk5uYiNzcX5crl77PZvHmzej4YNmwY9PT0BCekt8VFKZIdllwTEcAuISL6fyy4JqIC7BIi0i68fY9kJycnB5UqVQKQv0B179491K1bFzY2Nrh69argdFRa3uVTbnYJKQO7hJTr2bNnb/1eY2PjUkxCclG+fHn1EzjNzMyQmJiI9957DyYmJrh9+7bgdFSaXF1d33qXNLuElIFdQsoVExPz1u9t0KBBKSah/yUuSpHssORamUxMTERHIJlhl5ByVa5c+R8vQvPy8qBSqdgzqBAsuFauHj16iI5AMsMuIeVq1KiRRufw63A+KFt4+x7Jzl9//YX09HT06tULN27cwAcffIBr166pS649PT1FR6RSsHv3bnTu3Jn3f5Mau4SUKzw8/K3f27Zt21JMQnIRGRmJ58+fo127dnj48CEGDhyo/n6wYcMGNGzYUHREKiUBAQH48ssvUaFCBSQmJqJWrVrqXXOkTOwSUq6EhIS3fq+NjU0pJqH/JS5KUZnAkmvtp6uri6SkJFSvXh26urq4f/8+zMzMRMciIgF69eqFwMBAGBsbIygoCB9//DH09fVFxyIiAcqVK4d79+7BzMyM8wGRwrm5ueHw4cMwNTWFv78/Jk2aBENDQ9Gx6D/iohQRyULNmjWxdu1adOvWDTo6Onjw4AGqV68uOhZJjF1CBAB6enpISEiAubk5L0KJFM7a2hrTp0+Ht7c37OzsEBkZiWrVqr32vaSd2CVEAGBgYIDr16+jVq1anA+0CBelSBZYck1z5syBv7//W+2G4z3i2ktHR4ddQoQGDRrAzc0N7dq1g4+PDwICAl67CDlw4ECJ05FUWHBNALBmzRp89dVXePXq1Wvfw58L2q9gPmCXkLK5u7vDyMgIrVq1gp+fHyZNmgQjI6MS3+vr6ytxOvq3uChFsuDj4/PW7924cWMpJiGRrly5ghs3buDDDz/Exo0bUbly5RLf1717d2mDkWTYJUQAcOLECUyYMAFxcXFISUlBpUqVSlycUKlUSElJEZCQpODn5/fW7509e3YpJiHRnj9/joSEBDRo0ACHDh1C1apVS3wfu8W0F7uECACuXr2K2bNnIy4uDlFRUXB2dlZ3ixWmUqn4YUUZwkUpkgWWXFNhfn5+mDx5Mu8RVyB2CVFROjo6SEpK4vZ8BWLBNRW1adMmfPLJJ/y5oEDsEqKiOB9oDy5KkSyw5JqIAHYJUXEJCQmwtrbmgy4UiAXXBPz/rXmkbOwSItJexfe6EQlQvXp1nDp1Ct26dePwoVCdO3fGnDlz0Lx58ze+7/nz5/jxxx9hZGSEUaNGSZSOpOLk5ITp06ejXbt2yMvLw9atW9klpECJiYnqwuK3uQ3j7t27sLS0LO1YJDELCwvs2LED3t7eyMvLw507d/DixYsS38uCa+1Vr149+Pr6olevXm/cUX/9+nUsWbIENjY2mDZtmoQJSQqNGjWCj48PWrVqhby8PHz//ffsElKgU6dO/eO1QoGMjAzEx8ejXr16pZyK/ivulCJZYMk1rV+/Hr6+vjAxMUG3bt3QpEkTWFhYoEKFCnjy5AliY2MRERGBffv2oWvXrli0aBEvQrQQu4QIAGrUqIEePXrgiy++QNOmTUt8z9OnT7F161YsX74cX375JcaMGSNxSiptLLgmADh8+DCmTp2KmzdvomPHjq+dDy5duoTRo0fj66+/homJiejY9D/GLiECgDp16sDe3h5ffPEFvL29UbFixWLviY2NxS+//IKNGzdiwYIF/BCzDOCiFMkGS67p5cuX2LZtG7Zs2YKIiAg8ffoUQP6A4ezsDC8vL3z++ed47733BCclKbArQLmSk5Px7bffYsOGDahQoQIaN25c7CL00qVLcHNzw6xZs+Dt7S06MpUSFlxTgYiICGzZsgXHjh1DQkICMjMzUa1aNbi6usLLywv9+/eHqamp6JgkAc4HypWdnY2ffvoJK1euxM2bN+Ho6KgxH1y5cgVpaWno2bMnvv76a7i4uIiOTG+Bi1IkOyy5pgJPnz5FZmYmqlativLly4uOQxJjlxBlZmbizz//RERERIkXofXr1xcdkSTCgmsiIiosMjKyxPmgXbt2qFKliuh49A64KEVERLJRuEvobbBLiEh7sWOSiAqwS4hIe/G5uiQLnTt3xqlTp/7xfc+fP8eCBQuwcuVKCVKRKMnJyeq/vn37Nnx9fTF58mQcPXpUYCqSQtOmTTFs2DCcOXPmte95+vQp1q5di/r162PHjh0SpiMiKdWrVw+bN29GVlbWG993/fp1jBgxAvPnz5coGYnw/PlznD17FmlpaQCAqKgoDBw4EH379sWvv/4qOB2VtgEDBsDLywvbtm1Denp6ie+JjY3F119/jdq1a+Ps2bMSJySif4s7pUgWWHJNAHDhwgV069YNt2/fRp06dbB582Z07twZ6enp0NHRQXp6OrZv344ePXqIjkqlhF1CVFhqaip27typ7pDJyMhA9erV4erqik6dOqFly5aiI1IpYsE1FTh69Cg++OADpKWlwdTUFL///jv69OkDS0tL6Orq4vLly1i1ahWGDh0qOiqVEnYJUWGXL1/G5s2bXzsf9OnTh7d7lyFclCLZYMk1denSBeXKlcO0adPw888/Y+/evfDy8sLatWsBAF999RXOnj37VrvqqGxjl5Cy3bt3D76+vvj1119hYWGBZs2awcLCAgYGBkhJScHFixdx9uxZ2NjYYPbs2fj4449FR6ZSxIJratOmDerUqQN/f39s2LABS5YswYgRIzBv3jwAwNy5c7F9+3acP39ebFCSBLuElCsqKgpTpkxBREQEWrZsWeJ8cOzYMTx79gxTpkzBuHHjuDhVBnBRimSLJdfKU61aNYSGhqJBgwZIS0uDsbExzpw5g8aNGwPIf0Jj8+bNkZqaKjYoEZWqGjVqYNCgQRg8eDCcnZ1LfE9mZiZ27dqFgIAA9O7dG5MmTZI4JRFJpXLlyjh16hScnJyQlZUFAwMDREVFqZ+6eOPGDbi6uuL58+eCkxJRabKzs8PkyZPRr1+/1z6pHQBOnjyJ5cuXo0GDBvj666+lC0j/CheliEg2ij7it1KlSoiOjoa9vT0A4MGDB7CwsEBOTo7ImERUypKTk1G1atVSez8RlS2cD4gIyL+N8102K7zr+0mMcqIDEBVV+OLi9u3bWLt2LTIzM9GtWze0adNGcDoqbUWftMQnLykTu4SU7V0XmLggpd2eP3+Oa9euoW7dujAyMkJUVBSWLVuGzMxM9OjRA/379xcdkUqZSqXSmAeKviblYJeQsr3rAhMXpMoG7pQi2WDJNeno6KBLly7qYWLPnj3w9PRExYoVAeT3jh04cICfhGoxdgkRAOzevfut3/vhhx+WYhISjQXXBOTPB/Xr10e5cvmfp8fExMDJyQl6enoAgFevXuHSpUucD7QYu4QIAAICAt76vWPGjCnFJPS/xEUpkg2WXJOPj89bvW/jxo2lnIREYZcQAfkXoIWpVCoUHlcK75DgRah2Y8E1AYCfn99bvW/27NmlnIREYZcQAflfB4U9evQIGRkZ6q+J1NRUGBoawszMDDdv3hSQkP4NLkqRbLDkmojYJURFHTp0CFOnTsW8efPg7u4OIP+iY+bMmZg3bx46duwoOCGVJhZcExHALiEq7rfffsOPP/6I9evXo27dugCAq1evYujQoRg2bBhv7S5DuChFssESSyIiKqp+/fpYtWoVWrVqpXH82LFj+PLLL3H58mVByUgKnA2IiKgktWvXxvbt2+Hq6qpx/OzZs+jTpw/i4+MFJaN3xaJzkhWWXCvbw4cP1RceAHD+/HksXboUN27cgLm5OUaPHg0PDw9xAanUsUuIioqLiyvxVg0TExPcunVL8jwkLRZcEwCcPn0ajRs3hq6uLgBg7969WLRokXo+GDNmDAYOHCg4JZUmdglRUffv38erV6+KHc/JycGDBw8EJKJ/izulSDZYck26urq4f/8+zMzMcOLECXh4eKBFixZo1qwZzp8/jyNHjuDw4cN8CqMWY5cQFdWmTRtUqFABP//8M2rUqAEgf3fMwIED8eLFC4SHhwtOSKWJBdcEaM4He/bsQY8ePfDZZ5/h/fffx7lz5xAYGIitW7eiZ8+eoqNSKWGXEBXVrVs33L17F+vWrYObmxuA/F1SX375JSwtLd/pg04Si4tSJBssuabCt2l06tQJVlZWWL9+vfr8uHHjcOHCBRw+fFhgSpIKu4QIyO8M6tmzJ65duwYrKysAUD+lddeuXXBwcBCckEoTC64J0JwPWrdujVatWuG7775Tn583bx727NmDkydPCkxJUmGXEAH5C5ODBg3CgQMH1P1hr169gpeXFwIDAzXuviB546IUEclG4aHTwsICwcHBaN68ufr8pUuX4OHhgUePHglMSVJhlxAVyMvLQ0hICK5cuQIAeO+999ChQwfexkWkEIXngxo1amDfvn3qB+EA+QsSzZs3x5MnTwSmJKmwS4gKu3btmno+cHJygqOjo+BE9K7YKUVEsvL8+XNUqFABFSpUUN/KWaBChQrIyMgQlIykxi4hKqBSqdCpUye0adMG+vr6XIwiUqDY2FgkJSXBwMAAubm5xc6X1C1D2oldQlSYra0t8vLyULt2bfWt3lS26PzzW4ik8fDhQ43X58+fx6BBg9CyZUv06dMHYWFhYoKRpBwdHWFqaopbt24hMjJS49ylS5dgYWEhKBlJrWnTppgwYYLGgPngwQNMnjwZzZo1E5iMpJSbm4tvvvkGlpaWMDIyUn8CPmvWLI3be0k7nT59WqMvau/evWjbti0sLS3RpEkTBAUFCUxHUmrfvj0aNWqExMREHD9+XOPcuXPnYG1tLSgZSa19+/YYNmwYoqKi1MfOnj2LESNGoEOHDgKTkZQyMjLw+eefw9DQEPXq1UNiYiIA4KuvvsL8+fMFp6N3wUUpkg1zc3P1wtSJEyfQrFkzJCQkoGXLlnj27Bk6duyIo0ePCk5JpenIkSMIDQ1FaGgojhw5grZt22qcj4+Px5dffikoHUltw4YNuH//PqytreHg4AAHBwdYW1vj7t27XIxQkLlz5yIwMBALFy5Ul1sD+bd3rlu3TmAykoK7uzuSk5MB5D8ApXv37rC1tcWMGTPg6uqKzz//HDt37hSckkpbfHw8bt68ifj4eMTHx2PAgAEa57OysjB16lRB6UhqGzZsQM2aNdGkSRPo6+tDX18fzZo1Q40aNfhzQUGmT5+O6OhohIWFoUKFCurjHTp0wJYtWwQmo3fFTimSDZZcE1FR7BIiBwcHrF69Gu3bt0elSpUQHR0Ne3t7XLlyBe7u7uyQ0XIsuCai12GXkLLZ2Nhgy5YtaN68ucZ8cOPGDbi5ueHZs2eiI9Jb4k2XJEsXL16Ev7+/xrGhQ4fCw8NDTCAiEoJdQnT37t0Sn7CXm5uL7OxsAYlIlGvXrmHZsmUax3r37o1FixaJCUREQrFLSNkePXpU4hP20tPTOS+WMbx9j2Tl+fPnePbsGUuuqUSDBg2Cp6en6BgkEXYJEQA4Ozvj2LFjxY6X9OQl0k6xsbGIiYlhwTW9VocOHWBvby86BkmEXUIEAE2aNMGff/6pfl2wELVu3Tq4u7uLikX/ApeUSVYKtt3m5eUhMjJS44KDJddkaWkJHR2upSvF3LlzsWnTJixcuBBDhw5VH69fvz6WLVuGzz//XGA6koqvry8GDRqEu3fvIjc3F8HBwbh69SqCgoKwd+9e0fFIAu3bt0dB28Tx48fRtGlT9TkWXBMA9OzZE48fPxYdgyRSuEuoc+fO6uMdOnTAnDlzMG3aNIHpSCrz5s1Dly5dEBsbi1evXmH58uWIjY3FiRMnEB4eLjoevQN2SpFsFP3mYW5urnFv+PLly5GVlYXJkydLHY2IBGCXEBU4duwY/P39ER0djbS0NLi5ucHX1xedOnUSHY1KWUJCgsZrIyMjVK1aVf264Ol7AwcOlDQXEYnDLiEqEBcXh/nz52vMB1OnToWLi4voaPQOuChFRESyZGBggCtXrsDGxkZj6IyNjUWzZs2QlpYmOiIRERFJzNDQEBcvXoS9vb3GfBAdHY02bdrg6dOnoiMS0Tvg7XtEJCuxsbFYsWIFTp48iaSkJABAzZo14e7ujtGjR8PZ2VlwQpJKQZeQjY2NxnF2CSlTVlYWHj58WKxTiLduEVFcXByGDh2K0NBQ0VFIAgVdQl999RUAdgkpWW5uLm7cuFHifNCmTRtBqehdcVGKyoxBgwbh9u3bHDi02P79+9GjRw+4ubmhe/fuqFGjBgDgwYMHCAkJgZubG/744w94eXkJTkpSYJcQAcD169cxZMgQnDhxQuN4Xl4eVCoVcnJyBCUjOejQoQNu3ryJmzdvio5CAqWlpbFDRkHYJUQAcOrUKfTr1w8JCQkoevMX54OyhYtSVGaw5Fr7TZs2DVOnToW/v3+xc3PmzMGcOXMwefJkLkopRPfu3bFnzx74+/ujYsWK8PX1hZubG/bs2YOOHTuKjkcSGTx4MMqVK4e9e/fC3Nycj3kmDSy4VoaAgIA3nr97965ESUgOWrVqhfPnz2P+/PlwcXHBwYMH4ebmhpMnT7JLSEGGDx+u3jXH+aBsY6cUEcmGgYEBzp8/j7p165Z4/urVq2jUqBEyMzMlTkZEolSsWBFnz56Fk5OT6ChEJIiOjg7Mzc2hp6dX4vmsrCwkJSVxZwSRglSsWBHR0dFwcHAQHYX+I+6UIiLZsLW1xZ9//vnaRak///yzWL8QaT92CSmbs7Mzd8IQKZyNjQ0WLFiAjz76qMTz58+fR+PGjSVORSKxS4jef/993Lhxg4tSWoCLUiQrLLlWNn9/f/Tr1w9hYWHo0KGDRqfU4cOHceDAAfz222+CU5JU2CVEALBgwQJMmTIF8+bNg4uLC8qXL69x3tjYWFAykgMWXCtD48aNcfbs2dcuSqlUqmKdMqS92CVEAPDVV19h4sSJSEpKKnE+aNCggaBk9K54+x7JRuGSay8vr2Il12fPnmXJtQKcOHECAQEBJS5Mjh07lk9VUZCWLVuiXLlymDZtWoldAQ0bNhSUjKRU0CVY9Pefi5MEANHR0XBzc+PXgZaLjY1FRkYGmjRpUuL57Oxs3Lt3j7upFaJRo0ZwdHSEn59fifOBiYmJoGQkpZK6hgsWqDkflC1clCLZaNiwIbp3715iyTWQX3QdHByMmJgYiZMRkQjsEiIA//gkpbZt20qUhER4m4Lr77//nhcfRArCLiECgISEhDee5yJ12cFFKZINllxTSebPn4/hw4ejcuXKoqOQxJo2bYqlS5eiVatWoqMQkSAsuKbX4XygXJ6enpgyZQo6d+4sOgoR/Q+wU4pkgyXXVJJ58+bho48+4tCpQOwSIgA4evToG8+z0Fa7seCaXofzgXKxS4gAICgo6I3nBw4cKFES+q+4U4pkY9u2bejXrx+6dOnyxpLr3r17C05KUqpUqRKio6Nhb28vOgpJjF1CBLy+M6IAvw60W58+fVC7dm0sWLCgxPPR0dFwdXUt9vQt0n6cD5SLXUIEAKamphqvs7OzkZGRAT09PRgaGiIlJUVQMnpX3ClFstG3b19YWloiICAAixcvLlZyHRYWxpJrIgU5cuSI6AgkA0+ePNF4nZ2djXPnzmHWrFn49ttvBaUiqfj7+yMjI+O1552dnREfHy9hIiISjX/mCSg+HwD5T24eMWIEJk+eLCAR/VvcKUVEsnb79m1YWFhAV1dXdBQikpHw8HBMmDABZ8+eFR2FiATgfEBEJYmMjMRnn32GK1euiI5Cb4k7pUjWWGJJVlZWoiOQIOwSojepUaMGrl69KjoGCcDZgADOB0rGLiF6k3LlyuHevXuiY9A74E4pkjVjY2OcP3+efQEK8uOPPyI4OBhVqlTBsGHD0L59e/W5x48fo1mzZrh586bAhCQVdgkRAMTExGi8zsvLw/379zF//ny8evUKERERgpKRKJwNlInzARVglxABwO7duzVeF8wHK1asgJWVFfbv3y8oGb0r7pQiWeOaqbIEBARg+vTp8PHxwdOnT+Ht7Y05c+Zg+vTpAPIXIRISEgSnJKmwS4gAoFGjRuoC28KaN2+ODRs2CEpFInE2UB7OB1QYu4QIAHr06KHxWqVSoXr16vD09MTixYvFhKJ/hYtSRCQbq1evxtq1a9GvXz8AwIgRI9CjRw9kZmbC399fcDqSmomJSbFjHTt2hJ6eHruEFKRooa2Ojg6qV6+OChUqCEpERFLjfED/pE6dOpg/fz67hBSET13VHsXvjSCSkdjYWNjY2IiOQRKJj49HixYt1K9btGiB0NBQrFmzRv1pKBG7hJQlPDwcNWvWhI2NDWxsbGBlZYUKFSogKyvrH3tFSDtxNlAezgf0NtglpCyvezorF6vLHnZKEZFsWFtb49dff0Xr1q01jsfGxsLT0xNeXl745Zdf2CWkEOwSIgDQ1dXF/fv3YWZmpnE8OTkZZmZm/H5ApACcD6gwdgkRwPlAm/D2PZIVllgqW6tWrRAcHFxs6HR2dsbhw4fRrl07QclIBHYJEZB/sVG44L7AnTt3SrzFk7QPZwPifECFsUuIgNfPB9HR0ahSpYqARPRvcVGKZIMlljRt2rTX9gTVq1cPoaGh2LFjh8SpSBR2CSmbq6srVCoVVCoV2rdvj3Ll/n9kycnJQXx8PDp37iwwIUmBswEBnA9IE7uElM3U1FQ9Hzg6OhZ7MnNaWhqGDx8uMCG9K96+R7JRr149zJgxQ11ieeLECfTo0QPDhw+Hv78/Hjx4AAsLC27FJFKIoKAgfPzxx9DX19c4npWVhc2bN2PgwIGCkpEU/Pz81P87ceJEGBkZqc/p6enB1tYWvXv3hp6enqiIJAHOBkRUlL+/PyZNmgRDQ0ON45mZmVi0aBF8fX0FJSMpbNq0CXl5eRgyZAiWLVumsWu6YD5wd3cXmJDeFRelSDYMDQ0RGxsLW1tb9bGLFy+iQ4cO8PHxwbhx4zh4KkhqaipOnz6Nhw8fanwiplKpMGDAAIHJSCrsCiAgf/j8+OOPuUNOoTgbUFGcD4jzAQH5D0Jp2bKlxk5qKpv4O0iyUa1aNdy+fVtj8Kxfvz5CQ0Ph6enJp2koyJ49e9C/f3+kpaXB2NhYY1suh07lYJcQAcCgQYOQmpqKX375BXFxcZg8eTKqVKmCqKgo1KhRA5aWlqIjUinibECFcT4ggF1ClK9t27aIi4vDxo0bERcXh+XLl8PMzAz79++HtbU16tWrJzoivSXulCLZ6NevH2rUqIGlS5cWO3fp0iW0a9cOycnJ/PRDARwdHeHt7Y158+YV25pN2q+gSyg6Ohr16tV7bZfQ1q1bBaYkqcTExKBDhw4wMTHBrVu3cPXqVdjb22PmzJlITExEUFCQ6IhUijgbUGGcD5StoEvo6dOnxRYlC3cJrVy5UmBKkkp4eDi6dOmCli1b4ujRo7h8+TLs7e0xf/58REZGYvv27aIj0lviTimSDZZYUoG7d+9izJgxHDgVquCpOufPn4eXl9dru4RIGcaPH4/Bgwdj4cKFqFSpkvq4t7e3umeItBdnAyqM84GyLVu2TN0l5Ofnxy4hhZs2bRrmzp2LCRMmaMwHnp6eWLFihcBk9K64U4qIZKdXr1745JNP8NFHH4mOQgKxS4gAwMTEBFFRUahduzYqVaqE6Oho2NvbIyEhAXXr1sWLFy9ERyQiiXA+IIBdQpTPyMgIFy5cgJ2dncZ8cOvWLTg5OXE+KEP4J5lkiSWWyta1a1dMnjwZsbGxcHFxQfny5TXOf/jhh4KSkZTYJUQAoK+vj2fPnhU7fu3aNVSvXl1AIhKFswFxPiCAXUKUr3Llyrh//z7s7Ow0jp87d44zYhnDnVIkO/9UYpmSkiIwHUlBR0fntedUKhW7QxSCXUIEAF988QWSk5OxdetWVKlSBTExMdDV1UWPHj3Qpk0bLFu2THREkgBnAwI4H1A+dgkRAEyaNAl///03tm3bBkdHR0RFReHBgwcYOHAgBg4ciNmzZ4uOSG+Ji1IkOyyxJCIAaN++PRo3bqzuEirYln3ixAn069cPt27dEh2RJPD06VP06dMHkZGReP78OSwsLJCUlAR3d3fs27cPFStWFB2RJMDZgIgKuLu7o2/fvuouoYL54PTp0+jVqxfu3LkjOiJJICsrC6NGjUJgYCBycnJQrlw55OTkoF+/fggMDISurq7oiPSWuChFslOxYkVcuHAB9vb2oqOQDLx48YKdQgrFLiEqLCIiAjExMUhLS4Obmxs6dOggOhJJiLMBFcX5QLnYJUSFJSYm4uLFi0hLS4Orqyvq1KkjOhK9o9fvgSUSxMvLC5GRkaJjkEA5OTn45ptvYGlpCSMjI9y8eRMAMGvWLKxfv15wOpIKu4SosFatWmHkyJGYMmUKF6QUiLMBAZwPKF9Bl1BR7BJSJmtra3h7e+Ojjz7iglQZxaJzkh2WWNK3336LTZs2YeHChRg6dKj6eP369bFs2TJ8/vnnAtORVD788EP4+/tj69atAPL7QhITEzF16lT07t1bcDqS0uHDh3H48OFiBdcAsGHDBkGpSEqcDQjgfED5PvnkE0ydOhXbtm2DSqVCbm4ujh8/jkmTJmHgwIGi45FEcnJyEBgY+Nr5IDQ0VFAyele8fY9khyWW5ODggNWrV6N9+/Ya27KvXLkCd3d3PHnyRHREkgC7hAgA/Pz84O/vjyZNmsDc3Fyj4BoAdu7cKSgZSYmzAQGcDygfu4QIAEaPHo3AwEB07dq1xPlg6dKlgpLRu+JOKZKdoqvcpDx3796Fg4NDseO5ubnIzs4WkIhEMDExQUhICLuEFG7VqlUIDAzEgAEDREchgTgbEMD5gPLp6elh7dq1mDVrFruEFGzz5s3YunUrvL29RUeh/4iLUiRrLLFUJmdnZxw7dgw2NjYax7dv3w5XV1dBqUiUVq1aoVWrVqJjkCBZWVlo0aKF6BgkI5wNlIvzARVmbW0Na2tr0TFIED09vRIXqans4aIUyU5OTg7mzZuHVatW4cGDB7h27Rrs7e0xa9Ys2Nrasi9AAXx9fTFo0CDcvXsXubm5CA4OxtWrVxEUFIS9e/eKjkcSYpcQffHFF/jtt98wa9Ys0VFIIM4GBHA+oHzsEiIAmDhxIpYvX44VK1YUu3WPyhYuSpHssMSSunfvjj179sDf3x8VK1aEr68v3NzcsGfPHnTs2FF0PJLIP3UJkTK8ePECa9aswaFDh9CgQYNiBddLliwRlIykxNmAAM4HlG/s2LHqLqH69etzPlCoiIgIHDlyBPv370e9evWKzQfBwcGCktG7YtE5yQ5LLIkIAMzNzbFw4UJ2CSlcu3btXntOpVLxE3GF4GxARAWqVauGoKAgdgkpnI+PzxvPb9y4UaIk9F9xpxTJDkssydfXF+3atYO7uzt7QxSMXUIEAEeOHBEdgWSAswEBnA8oH7uECOCikzZ5/fN1iQQpKLEsiiWWynHy5El069YNlStXRuvWrTFz5kwcOnQImZmZoqORhAq6hIiIOBsQwPmA8hV0CfGGHyLtwJ1SJDsssaSQkBC8evUKf//9N44ePYrw8HAEBATg5cuXaNq0KSIiIkRHJAmwS0i5evXqhcDAQBgbG6NXr15vfC87I5SBswEBnA8oH7uElMvNzQ2HDx+GqakpXF1d39gnFhUVJWEy+i+4KEWywxJLAoBy5cqhZcuWqF69OqpUqYJKlSph165duHLliuhoJJGYmBg0atQIAHDx4kWNcyw11W4mJibq32NjY2P+fhNnA1LjfECVK1dGz549RccgAbp37w59fX0AQI8ePcSGof8ZFp0TkeysWbMGYWFhCA8Px8uXL9G6dWt4eHjAw8MDDRo04AUqkZbbvXs3unTpUuzTbyJSNs4HRMoWEBCAL7/8EhUqVEBiYiJq1aoFHR02EpV1XJQi2WGJJeno6KB69eqYOHEiRo4cCSMjI9GRiEhCurq6SEpKQvXq1aGrq4v79+/DzMxMdCwSiLMBAZwPiJSuXLlyuHfvHszMzDgfaBEuSpHsdOzYESdPnsSrV6/QtGlTtG3bFh4eHmjZsiUMDAxExyMJ7Nq1C0ePHkVYWBguX74MV1dX9SehrVq1gqGhoeiIVErYJUQAULNmTaxduxbdunWDjo4OHjx4gOrVq4uORQJxNiCA84GSsUuIAMDa2hrTp0+Ht7c37OzsEBkZiWrVqr32vVQ2sFOKZIclltSjRw/1feJPnz7FsWPHsG3bNnzwwQfQ0dHBixcvxAakUsMuIQKA4cOHo3v37lCpVFCpVKhZs+Zr35uTkyNhMhKFswEBnA+UjF1CBAAzZ87EV199hdGjR0OlUqFp06bF3pOXlweVSsX5oAzhohTJEkssKTk5GeHh4QgLC0NYWBguXboEU1NTtG7dWnQ0KkU9e/ZU35oTGBgoNgwJM2fOHHzyySe4ceMGPvzwQ2zcuBGVK1cWHYsE42xAAOcDpTI1NVV3B/n4+LBLSKG+/PJLfPrpp0hISECDBg1w6NAhVK1aVXQs+o94+x7JDkssycXFBbGxsahSpQratGkDDw8PtG3bFg0aNBAdjUoZu4SoKD8/P0yePJm35SgcZwMCOB8oGbuEqKhNmzbhk08+Ue+go7KLi1IkOyyxpJUrV8LDwwNmZmbQ19eHsbGx6EgkEXYJEVFJOBsQwPlAydglRMD/35pH2oV7Hkl2goOD0b9/f2zevBnVq1dHixYt8PXXX+PgwYPIyMgQHY9KWWpqKmJjY9G2bVvUrFkTpqamqFmzJqZPn87ffwUo6BLS1dVVdwnp6uqW+Iu0V+fOnXHq1Kl/fN/z58+xYMECrFy5UoJUJBJnA+J8oGwzZ87EuHHjYG9vr+4SsrOz0/hla2sLOzs70VGpFNWrVw+bN29GVlbWG993/fp1jBgxAvPnz5coGf0X3ClFsla4xPL3339niaWWS0lJgbu7O+7evYv+/fvjvffeAwDExsbit99+g5OTEyIiIhATE4NTp05hzJgxghNTabhy5cpbdQl1795d2mAkmfXr18PX1xcmJibo1q0bmjRpAgsLC1SoUAFPnjxBbGwsIiIisG/fPnTt2hWLFi3iJ+MKwtlAeTgfEJD/QcTbdAk1bNhQ4mQklcOHD2Pq1Km4efMmOnbs+Nr54NKlSxg9ejS+/vprmJiYiI5N/4CLUiRLbyqx3Llzp+h4VErGjRuHw4cP49ChQ6hRo4bGuaSkJHTq1Al169bFwYMHERAQgEGDBglKSlJgl5CyvXz5Etu2bcOWLVsQERGBp0+fAgBUKhWcnZ3h5eWFzz//XH1xStqPs4FycT6gwtglRBEREdiyZQuOHTuGhIQEZGZmolq1anB1dYWXlxf69+8PU1NT0THpLXFRimSHJZbKZWtri9WrV8PLy6vE8wcOHIC3tzdmz56N2bNnS5yOiER6+vQpMjMzUbVqVZQvX150HJIYZwNl43xA7BIi0l5clCLZYYmlcunr6yMuLg61atUq8fydO3dga2uLV69eSZyMpNK5c2fMmTMHzZs3f+P7nj9/jh9//BFGRkYYNWqUROmISBTOBsrG+YCcnZ3h6+uLXr16QU9P77Xvu379OpYsWQIbGxtMmzZNwoRE9G+VEx2AqLCCEsvZs2fjyZMnAIDq1avDx8cHs2bN4m08Wq5atWq4devWa4fO+Ph4Pv5Xy/Xt2xe9e/d+py4h0m7Jycnq3pDbt29j7dq1yMzMRLdu3dCmTRvB6UgKnA2I8wH98MMPmDp1KkaOHPlWXUIjRowQHZlK0fPnz3Ht2jXUrVsXRkZGiIqKwrJly5CZmYkePXqgf//+oiPSO+BOKZINlljSkCFDEBcXh5CQkGKfgr18+RJeXl6wt7fHhg0bBCUkKbBLiADgwoUL6NatG27fvo06depg8+bN6Ny5M9LT06Gjo4P09HRs374dPXr0EB2VShFnAwI4H9D/Y5cQHT16FB988AHS0tJgamqK33//HX369IGlpSV0dXVx+fJlrFq1CkOHDhUdld4SF6VINlhiSXfu3EGTJk2gr6+PUaNGwcnJCXl5ebh8+TJ+/PFHvHz5EmfOnOFTthSGXULK1KVLF5QrVw7Tpk3Dzz//jL1798LLywtr164FAHz11Vc4e/YsTp06JTgplSbOBgRwPiCi/9emTRvUqVMH/v7+2LBhA5YsWYIRI0Zg3rx5AIC5c+di+/btOH/+vNig9Na4KEWywRJLAvK34I8cORIHDx5EwbcnlUqFjh07YsWKFXBwcBCckIikUK1aNYSGhqJBgwZIS0uDsbExzpw5g8aNGwMArly5gubNmyM1NVVsUCpVnA2oAOcDIgKAypUr49SpU3ByckJWVhYMDAwQFRWFhg0bAgBu3LgBV1dXPH/+XHBSeltclCLZYIklFfbkyRNcv34dAODg4IAqVaoITkRSY5eQsuno6CApKUndE1OpUiVER0fD3t4eAPDgwQNYWFggJydHZEwqZZwNqCjOB8rGLiHifKB9WHROssESSyrM1NQUzZo1Ex2DBPinLqGlS5eyS0ghij7+m48DVx7OBlQU5wPl+qcuoeDgYGRkZLBLSMupVCqNeaDoayp7uFOKZIMllkQEsEuI8uno6KBLly7Q19cHAOzZsweenp6oWLEigPyfCwcOHOAnoVqOswERFWCXEAH580H9+vVRrlz+/pqYmBg4OTmpf0a8evUKly5d4nxQhnBRimSDJZZEBLBLiPL5+Pi81fs2btxYyklIJM4GRFSAXUIEAH5+fm/1PvYMlh1clCJZYYklEbErgIgK42xARADnAyJtxU4pkhU7Ozvs37+fJZZECscuISIqwNmAiAB2CRFpK+6UIiIiWWGXEAHAw4cPNQqsz58/j6VLl+LGjRswNzfH6NGj4eHhIS4gERFJil1CBACnT59G48aNoaurCwDYu3cvFi1apJ4PxowZg4EDBwpOSe+Ci1JERCQr7BIiANDV1cX9+/dhZmaGEydOwMPDAy1atECzZs1w/vx5HDlyBIcPH0abNm1ERyUiIgmwS4gAzflgz5496NGjBz777DO8//77OHfuHAIDA7F161b07NlTdFR6S1yUIiIiItkp3B3SqVMnWFlZYf369erz48aNw4ULF3D48GGBKYmIiEhKheeD1q1bo1WrVvjuu+/U5+fNm4c9e/bg5MmTAlPSu9ARHYCIiIjoTS5evIihQ4dqHBs6dChiYmIEJSIiIiLRrl27hj59+mgc6927N65cuSIoEf0bXJQiIiJZefjwocbr8+fPY9CgQWjZsiX69OmDsLAwMcFIcs+fP8ezZ89QoUIFdcdYgQoVKiAjI0NQMiIiktrp06c1+qL27t2Ltm3bwtLSEk2aNEFQUJDAdCSl2NhYxMTEwMDAALm5ucXOv3r1SkAq+re4KEVERLJibm6uXpg6ceIEmjVrhoSEBLRs2RLPnj1Dx44dcfToUcEpSQqOjo4wNTXFrVu3EBkZqXHu0qVLsLCwEJSMiIik5u7ujuTkZAD5D0Hp3r07bG1tMWPGDLi6uuLzzz/Hzp07BackKbRv3x6NGjVCYmIijh8/rnHu3LlzsLa2FpSM/o1yogMQEREVVrjqcM6cORgwYECxLiE/Pz92CWm5I0eOaLw2NzfXeB0fH48vv/xSykhERCRQ4flg4cKFmDJlikaXkJ2dHRYuXMiCay0XHx+v8drIyEjjdVZWFqZOnSplJPqPWHRORESyUrjA0sLCAsHBwWjevLn6/KVLl+Dh4YFHjx4JTElERERSKjwf1KhRA/v27UPjxo3V569evYrmzZvjyZMnAlMS0bviTikiIpKd58+fo0KFCuwSIiIiIrXY2FgkJSWxS4hIi7BTioiIZIddQvRPBg0aBE9PT9ExiIhIQuwSon/SoUMH2Nvbi45B74A7pYiISFbYJURvw9LSEjo6/GyNiEgp2CVEb6Nnz554/Pix6Bj0DtgpRUREREREREREkuNHjEREREREREREJDkuShERUZnCLiHliI2NxciRI+Hq6gpzc3OYm5vD1dUVI0eORGxsrOh4REQkI+wSIgCIi4vjnFjGsFOKiIjKFHYJKcP+/fvRo0cPuLm5oXv37qhRowYA4MGDBwgJCYGbmxv++OMPeHl5CU5KRERywC4hAoC0tDSEh4eLjkHvgJ1SREREJDsNGzZE9+7d4e/vX+L5OXPmIDg4GDExMRInIyIiIlECAgLeeP7u3bv4/vvvkZOTI1Ei+q+4KEVERESyY2BggPPnz6Nu3bolnr969SoaNWqEzMxMiZMRERGRKDo6OjA3N4eenl6J57OyspCUlMRFqTKE9z8QEZHssEuIbG1t8eeff772/J9//gkbGxsJExERkZyxS0gZbGxssHTpUsTHx5f4602zA8kTO6WIiEhW2CVEAODv749+/fohLCwMHTp00Pg6OHz4MA4cOIDffvtNcEoiIpILdgkpQ+PGjXH27Fl89NFHJZ5XqVTgzWBlC2/fIyIiWWGXEBU4ceIEAgICcPLkSSQlJQEAatasCXd3d4wdOxbu7u6CExIRkVTYJURA/m76jIwMNGnSpMTz2dnZuHfvHndTlyFclCIiIllhlxAREREVxS4hIu3ETikiIpIVdgnR68yfPx+pqamiYxARkQDsEqLX4XxQtnGnFBERycq2bdvQr18/dOnS5Y1dQr179xaclKRmbGyM8+fPw97eXnQUIiKSWJ8+fVC7dm0sWLCgxPPR0dFwdXVFbm6uxMlINM4HZRuLzomISFb69u0LS0tLBAQEYPHixcW6hMLCwtglpFD8HI2ISLn8/f2RkZHx2vPOzs6Ij4+XMBHJBeeDso07pYiIiKhMqFSpEqKjo/lJKBEREalxPijb2ClFRESyx64AAvKfuMM+MSIiKsD5gADOB2Udd0oREZHssSuAiIiIiuJ8QFT2cacUERHJHj8/UaYff/wRHTp0wEcffYTDhw9rnHv8+DEvQoiIFI7zgTJxPtAuXJQiIiIi2QkICMDkyZPh5OQEfX19eHt747vvvlOfz8nJQUJCgsCEREREJDXOB9qHT98jIiLZi42NhYWFhegYJKHVq1dj7dq16NevHwBgxIgR6NGjBzIzM+Hv7y84HRERyQHnA+XhfKB9uChFRESyZ2VlJToCSSw+Ph4tWrRQv27RogVCQ0PRoUMHZGdnY9y4ceLCERGRLHA+UB7OB9qHt+8REZHssCuAqlWrhtu3b2scq1+/PkJDQ7Fx40ZMmTJFUDIiIhKF8wFxPtA+XJQiIiJZYVcAAUCrVq0QHBxc7LizszMOHz6M/fv3C0hFRESicD4ggPOBNuLte0REJCvsCiAAmDZtGs6ePVviuXr16iE0NBQ7duyQOBUREYnC+YAAzgfaSJXH52gSEZGMGBoaIjY2Fra2tupjFy9eRIcOHeDj44Nx48bBwsICOTk54kISERGRpDgfEGkn7pQiIiJZKegKKDx0FnQFeHp64t69e+LCkRCpqak4ffo0Hj58iNzcXPVxlUqFAQMGCExGRERS4XxARXE+0A7cKUVERLLSr18/1KhRA0uXLi127tKlS2jXrh2Sk5P5SahC7NmzB/3790daWhqMjY2hUqnU51QqFVJSUgSmIyIiqXA+oMI4H2gPLkoREZGsxMTE4OzZs/Dx8Snx/MWLF7Fjxw7Mnj1b4mQkgqOjI7y9vTFv3jwYGhqKjkNERIJwPqDCOB9oDy5KERERkWxVrFgRFy5c4GO+iYiISI3zgfZgpxQREckWuwLIy8sLkZGRHDqJiEiN8wFxPtAeXJQiIiJZ+qeuAA6dytC1a1dMnjwZsbGxcHFxQfny5TXOf/jhh4KSERGRCJwPCOB8oE14+x4REckSuwIIAHR0dF57TqVSsdCWiEhhOB8QwPlAm3BRioiIZIldAURERFQU5wMi7fL65UUiIiKBCroCiAq8ePFCdAQiIhKM8wEVxfmgbGOnFBERyRK7AggAcnJyMG/ePKxatQoPHjzAtWvXYG9vj1mzZsHW1haff/656IhERCQhzgcEcD7QJrx9j4iIZIldAQQA/v7+2LRpE/z9/TF06FBcvHgR9vb22LJlC5YtW4aTJ0+KjkhERBLifEAA5wNtwtv3iIhIlnJzc1/7iwOncgQFBWHNmjXo378/dHV11ccbNmyIK1euCExGREQicD4ggPOBNuGiFBERyR67ApTr7t27cHBwKHY8NzcX2dnZAhIREZFccD5QLs4H2oOLUkREJEs5OTn45ptvYGlpCSMjI9y8eRMAMGvWLKxfv15wOpKKs7Mzjh07Vuz49u3b4erqKiARERGJxPmAAM4H2oRF50REJEvffvstNm3ahIULF2Lo0KHq4/Xr18eyZctYYKkQvr6+GDRoEO7evYvc3FwEBwfj6tWrCAoKwt69e0XHIyIiiXE+IIDzgTZh0TkREcmSg4MDVq9ejfbt26NSpUqIjo6Gvb09rly5And3dzx58kR0RJLIsWPH4O/vj+joaKSlpcHNzQ2+vr7o1KmT6GhERCQxzgdUgPOBduBOKSIikiV2BVCB1q1bIyQkRHQMIiKSAc4HVIDzgXZgpxQREckSuwIIyN+ef+TIEZbZEhERAM4HlI/zgfbgTikiIpIldgUQAJw8eRJLlizBq1ev0LRpU7Rt2xYeHh5o2bIlDAwMRMcjIiKJcT4ggPOBNmGnFBERyRa7AggAXr16hb///htHjx5FeHg4Tpw4gZcvX6Jp06aIiIgQHY+IiCTG+YAAzgfagotSREREVCZcu3YNR44cwaFDh7Br1y6YmJjg8ePHomMRERGRQJwPyjZ2ShERkSyxK4AAYM2aNejXrx8sLS3RokULHDhwAK1atUJkZCQePXokOh4REUmM8wEBnA+0CXdKERGRLHXs2BEnT55kV4DC6ejooHr16pg4cSJGjhwJIyMj0ZGIiEggzgcEcD7QJlyUIiIi2WJXAO3atQtHjx5FWFgYLl++DFdXV3h4eMDDwwOtWrWCoaGh6IhERCQxzgfE+UB7cFGKiIhkj10BBABPnz7FsWPHsG3bNvz+++/Q0dHh7RtERArG+YAAzgdlXTnRAYiIiEqyZs0ahIWFITw8HC9fvkTr1q3h4eGBmTNnokGDBqLjkYSSk5MRHh6OsLAwhIWF4dKlSzA1NUXr1q1FRyMiIolxPqACnA+0A3dKERGRLLErgADAxcUFsbGxqFKlCtq0aQMPDw+0bduWFx5ERArF+YAAzgfahItSREQkS+wKIABYuXIlPDw8YGZmBn19fRgbG4uOREREAnE+IIDzgTbhohQREckeuwKUKTU1FTNmzMCWLVvw5MkTAED16tXh4+ODWbNm8cKDiEjhOB8oE+cD7cJOKSIiki12BShXSkoK3N3dcffuXfTv3x/vvfceACA2NhY//PADQkJCEBERgZiYGJw6dQpjxowRnJiIiKTC+UC5OB9oH+6UIiIiWWJXgLKNGzcOhw8fxqFDh1CjRg2Nc0lJSejUqRPq1q2LgwcPIiAgAIMGDRKUlIiIpMT5QNk4H2gfLkoREZEssStA2WxtbbF69Wp4eXmVeP7AgQPw9vbG7NmzMXv2bInTERGRKJwPlI3zgfbRER2AiIioqNTUVMTGxqJt27aoWbMmTE1NUbNmTUyfPh0ZGRmi45EE7t+/j3r16r32fP369aGjo8OBk4hIQTgfEOcD7cNOKSIikhV2BRAAVKtWDbdu3UKtWrVKPB8fHw8zMzOJUxERkSicDwjgfKCNePseERHJCrsCCACGDBmCuLg4hISEQE9PT+Pcy5cv4eXlBXt7e2zYsEFQQiIikhLnAwI4H2gjLkoREZGssCuAAODOnTto0qQJ9PX1MWrUKDg5OSEvLw+XL1/Gjz/+iJcvX+LMmTOwtrYWHZWIiCTA+YAAzgfaiItSREQkK/r6+oiLi3vttuw7d+7A1tYWr169kjgZSS0+Ph4jR47EwYMHUTCuqFQqdOzYEStWrICDg4PghEREJBXOB1SA84F2YacUERHJCrsCqICdnR3279+PJ0+e4Pr16wAABwcHVKlSRXAyIiKSGucDKsD5QLtwpxQREckKuwKIiIioKM4HRNqJi1JERCQr7AogIiKiojgfEGknLkoREZHssCuAiIiIiuJ8QKR9uChFRESyxa4AIiIiKorzAZH24KIUERERERERERFJTkd0ACIiIiIiIiIiUh4uShERERERERERkeS4KEVERERERERERJLjohQREREREREREUmOi1JEREREZUxYWBhUKhVSU1Pf+u+xtbXFsmXLSi0TERER0bviohQRERHR/9jgwYOhUqkwfPjwYudGjRoFlUqFwYMHSx+MiIiISEa4KEVERERUCqysrLB582ZkZmaqj7148QK//fYbrK2tBSYjIiIikgcuShERERGVAjc3N1hZWSE4OFh9LDg4GNbW1nB1dVUfe/nyJcaMGQMzMzNUqFABrVq1wpkzZzT+Wfv27YOjoyMMDAzQrl073Lp1q9i/LyIiAq1bt4aBgQGsrKwwZswYpKenl5gtLy8Pc+bMgbW1NfT19WFhYYExY8b8b/7DiYiIiN4SF6WIiIiISsmQIUOwceNG9esNGzbAx8dH4z1TpkzBjh07sGnTJkRFRcHBwQFeXl5ISUkBANy+fRu9evVCt27dcP78eXzxxReYNm2axj8jLi4OnTt3Ru/evRETE4MtW7YgIiICo0ePLjHXjh07sHTpUqxevRrXr1/Hrl274OLi8j/+ryciIiJ6My5KEREREZWSzz77DBEREUhISEBCQgKOHz+Ozz77TH0+PT0dP/30ExYtWoQuXbrA2dkZa9euhYGBAdavXw8A+Omnn1C7dm0sXrwYdevWRf/+/Yv1UX333Xfo378/xo0bhzp16qBFixYICAhAUFAQXrx4USxXYmIiatasiQ4dOsDa2hrNmjXD0KFDS/X/CyIiIqKiuChFREREVEqqV6+Orl27IjAwEBs3bkTXrl1RrVo19fm4uDhkZ2ejZcuW6mPly5dHs2bNcPnyZQDA5cuX8f7772v8c93d3TVeR0dHIzAwEEZGRupfXl5eyM3NRXx8fLFcffv2RWZmJuzt7TF06FDs3LkTr169+l/+pxMRERH9o3KiAxARERFpsyFDhqhvo1u5cmWp/DvS0tIwbNiwEnuhSipVt7KywtWrV3Ho0CGEhIRg5MiRWLRoEcLDw1G+fPlSyUhERERUFHdKEREREZWizp07IysrC9nZ2fDy8tI4V7t2bejp6eH48ePqY9nZ2Thz5gycnZ0BAO+99x5Onz6t8fedOnVK47WbmxtiY2Ph4OBQ7Jeenl6JuQwMDNCtWzcEBAQgLCwMJ0+exIULF/4X/8lEREREb4U7pYiIiIhKka6urvpWPF1dXY1zFStWxIgRIzB58mRUqVIF1tbWWLhwITIyMvD5558DAIYPH47Fixdj8uTJ+OKLL3D27FkEBgZq/HOmTp2K5s2bY/To0fjiiy9QsWJFxMbGIiQkBCtWrCiWKTAwEDk5OXj//fdhaGiIX375BQYGBrCxsSmd/xOIiIiISsCdUkRERESlzNjYGMbGxiWemz9/Pnr37o0BAwbAzc0NN27cwF9//QVTU1MA+bff7dixA7t27ULDhg2xatUqzJs3T+Of0aBBA4SHh+PatWto3bo1XF1d4evrCwsLixL/nZUrV8batWvRsmVLNGjQAIcOHcKePXtQtWrV/+1/OBEREdEbqPLy8vJEhyAiIiIiIiIiImXhTikiIiIiIiIiIpIcF6WIiIiIiIiIiEhyXJQiIiIiIiIiIiLJcVGKiIiIiIiIiIgkx0UpIiIiIiIiIiKSHBeliIiIiIiIiIhIclyUIiIiIiIiIiIiyXFRioiIiIiIiIiIJMdFKSIiIiIiIiIikhwXpYiIiIiIiIiISHJclCIiIiIiIiIiIsn9H47UdeRJ50W+AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from llm_toolkit.translation_utils import plot_times\n", + "\n", + "perf_df = get_perf_df(metrics_df, bnb_4bit=True)\n", + "plot_times(perf_df, ylim=0.358)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "e1954c60-5af6-4a05-b54a-0f56588ee6e9", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(248.74249999999998, 102.02708333333334, 452.7966666666666)" + ] + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "perf_df[\"train-time(mins)\"].mean(), perf_df[\"eval-time(mins)\"].mean(), perf_df[\n", + " \"train-time(mins)\"\n", + "].mean() + 2 * perf_df[\"eval-time(mins)\"].mean()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "e50a05ae-b79b-494d-8ca3-bd9e30c71172", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Experiment 4 - Performance vs Epochs" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7aeefdd4-3a36-4e65-87a8-3dadf3a98322", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chineseenglishQwen/Qwen2-0.5B-Instruct_checkpoint-560Qwen/Qwen2-0.5B-Instruct_checkpoint-1120Qwen/Qwen2-0.5B-Instruct_checkpoint-1680Qwen/Qwen2-0.5B-Instruct_checkpoint-2240Qwen/Qwen2-0.5B-Instruct_checkpoint-2800Qwen/Qwen2-0.5B-Instruct_checkpoint-3360Qwen/Qwen2-0.5B-Instruct_checkpoint-3920Qwen/Qwen2-0.5B-Instruct_checkpoint-4480...Qwen/Qwen2-1.5B-Instruct_checkpoint-560Qwen/Qwen2-1.5B-Instruct_checkpoint-1120Qwen/Qwen2-1.5B-Instruct_checkpoint-1680Qwen/Qwen2-1.5B-Instruct_checkpoint-2240Qwen/Qwen2-1.5B-Instruct_checkpoint-2800Qwen/Qwen2-1.5B-Instruct_checkpoint-3360Qwen/Qwen2-1.5B-Instruct_checkpoint-3920Qwen/Qwen2-1.5B-Instruct_checkpoint-4480Qwen/Qwen2-1.5B-Instruct_checkpoint-5040Qwen/Qwen2-1.5B-Instruct_checkpoint-5600
0老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞...Old Geng picked up his shotgun, squinted, and ...Old Trinket raised his gun and squinted his tr...Old Geng raised his gun, his eyes narrowed. Th...Old Geng held his gun up, half-closed, and coc...Old Geng raised his gun, his triangular eye ha...Old Geng took out his pistol, squinted over a ...Old Geng held his rifle up and cocked it over ...Old Geng held his gun to his chest, eyes on a ...Old Geng took up his gun and raised a triangul......Grannie Geng held up his gun with one eye, nar...Old Geng raised his rifle and squinted at it t...Old Geng took his gun off the table and raised...Old Geng raised his rifle and squeezed the tri...Old Geng took aim and squeezed the trigger; do...Old Geng took a step forward, raised his pisto...Old Geng raised his pistol, opened it up, and ...Old Geng took a shot with his rifle. A spray o...Old Geng took a step forward, raised his rifle...Old Geng reached for his rifle, wedged it to h...
1次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，...Next day Grannie Liu was up before dawn. As so...In the morning she was up early for breakfast ...In the morning, however, when the sun was just...In the morning when the sun was just rising, G...It was still dark before she got up for breakf...In the early hours of the next day, when it wa...By day's dawn her old lady had risen from bed ...By the time the next morning was over, Grannie...It was just now six o'clock that the old woman......By morning of the next day, Old Liu got up ver...At dawn the next day, Grannie Liu got up and w...By midnight, Grannie Liu had risen from her be...When she arose from her bed at daybreak the ne...As soon as it was light outside, Grannie Liu r...By daybreak she was up and dressed, having ins...At daybreak the old woman got up and dressed h...When she woke from her nap, Aunt Liu dressed h...Then at daybreak the old woman was up and abou...Grannie Liu got up very early the morning of t...
\n", + "

2 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "1 次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "1 Next day Grannie Liu was up before dawn. As so... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-560 \\\n", + "0 Old Trinket raised his gun and squinted his tr... \n", + "1 In the morning she was up early for breakfast ... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-1120 \\\n", + "0 Old Geng raised his gun, his eyes narrowed. Th... \n", + "1 In the morning, however, when the sun was just... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-1680 \\\n", + "0 Old Geng held his gun up, half-closed, and coc... \n", + "1 In the morning when the sun was just rising, G... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-2240 \\\n", + "0 Old Geng raised his gun, his triangular eye ha... \n", + "1 It was still dark before she got up for breakf... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-2800 \\\n", + "0 Old Geng took out his pistol, squinted over a ... \n", + "1 In the early hours of the next day, when it wa... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-3360 \\\n", + "0 Old Geng held his rifle up and cocked it over ... \n", + "1 By day's dawn her old lady had risen from bed ... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-3920 \\\n", + "0 Old Geng held his gun to his chest, eyes on a ... \n", + "1 By the time the next morning was over, Grannie... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-4480 ... \\\n", + "0 Old Geng took up his gun and raised a triangul... ... \n", + "1 It was just now six o'clock that the old woman... ... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-560 \\\n", + "0 Grannie Geng held up his gun with one eye, nar... \n", + "1 By morning of the next day, Old Liu got up ver... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-1120 \\\n", + "0 Old Geng raised his rifle and squinted at it t... \n", + "1 At dawn the next day, Grannie Liu got up and w... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-1680 \\\n", + "0 Old Geng took his gun off the table and raised... \n", + "1 By midnight, Grannie Liu had risen from her be... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-2240 \\\n", + "0 Old Geng raised his rifle and squeezed the tri... \n", + "1 When she arose from her bed at daybreak the ne... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-2800 \\\n", + "0 Old Geng took aim and squeezed the trigger; do... \n", + "1 As soon as it was light outside, Grannie Liu r... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-3360 \\\n", + "0 Old Geng took a step forward, raised his pisto... \n", + "1 By daybreak she was up and dressed, having ins... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-3920 \\\n", + "0 Old Geng raised his pistol, opened it up, and ... \n", + "1 At daybreak the old woman got up and dressed h... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-4480 \\\n", + "0 Old Geng took a shot with his rifle. A spray o... \n", + "1 When she woke from her nap, Aunt Liu dressed h... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-5040 \\\n", + "0 Old Geng took a step forward, raised his rifle... \n", + "1 Then at daybreak the old woman was up and abou... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-5600 \n", + "0 Old Geng reached for his rifle, wedged it to h... \n", + "1 Grannie Liu got up very early the morning of t... \n", + "\n", + "[2 rows x 22 columns]" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "df = pd.read_csv(\"results/mac-results_lf.csv\")\n", + "df.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7014bd20-5420-4933-9424-6069e3a9b276", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['chinese',\n", + " 'english',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-560',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-1120',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-1680',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-2240',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-2800',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-3360',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-3920',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-4480',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-5040',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-5600',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-560',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-1120',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-1680',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-2240',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-2800',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-3360',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-3920',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-4480',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-5040',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-5600']" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.columns.to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "5c147aa2-acb2-423d-89f4-c0c98af08505", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "df2 = pd.read_csv(\"results/experiment-2-results.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "ff46e45c-ad5d-49d1-b7e5-e95152793336", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chineseenglishunsloth/Qwen2-0.5B-Instructunsloth/Qwen2-0.5B-Instruct(finetuned)unsloth/Qwen2-1.5B-Instructunsloth/Qwen2-1.5B-Instruct(finetuned)unsloth/Qwen2-7B-Instructunsloth/Qwen2-7B-Instruct(finetuned)unsloth/mistral-7b-instruct-v0.3unsloth/mistral-7b-instruct-v0.3(finetuned)gradientai/Llama-3-8B-Instruct-Gradient-1048kgradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned)unsloth/Qwen2-72B-Instruct-bnb-4bitunsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned)
0老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞...Old Geng picked up his shotgun, squinted, and ...Old Teng holds his gun up, his eyes narrowed a...Old Geng raised his rifle and tilted his head ...Old Jin raises his gun, squints one eye as he ...Old Geng raised his pistol, squinted through t...Old Geng raised his gun, squinted one of his t...Old Geng raised his rifle and squinted into th...Geng Da initiates firing, squinting to form a ...Old Geng aimed and fired. A triangular slit op...The old man pulled out his gun, squinting one ...Old Geng raised his rifle, squinting through t...Lao Geng raised his gun, narrowed one of his t...Old Geng raised his gun, narrowed one of his t...
1次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，...Next day Grannie Liu was up before dawn. As so...The next morning, Liu Geo woke up at five o'cl...But not before noon did Grannie Liu rise up an...At dawn the next day, Liu Langlang got up earl...She got up about dawn with a purpose already e...The next morning, before the dawn had fully br...First thing in the morning Grannie Liu rose to...The next day, when it was still dark, Liu Lao ...Before dawn next day Grannie Liu got up and bu...The next day, when the sun had not yet risen, ...Grannie Liu got up before daylight was even vi...Before dawn next morning, Granny Liu got up to...As soon as it was light, Grannie Liu got up an...
\n", + "
" + ], + "text/plain": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "1 次日天未明时，刘老老便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "1 Next day Grannie Liu was up before dawn. As so... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct \\\n", + "0 Old Teng holds his gun up, his eyes narrowed a... \n", + "1 The next morning, Liu Geo woke up at five o'cl... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct(finetuned) \\\n", + "0 Old Geng raised his rifle and tilted his head ... \n", + "1 But not before noon did Grannie Liu rise up an... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct \\\n", + "0 Old Jin raises his gun, squints one eye as he ... \n", + "1 At dawn the next day, Liu Langlang got up earl... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct(finetuned) \\\n", + "0 Old Geng raised his pistol, squinted through t... \n", + "1 She got up about dawn with a purpose already e... \n", + "\n", + " unsloth/Qwen2-7B-Instruct \\\n", + "0 Old Geng raised his gun, squinted one of his t... \n", + "1 The next morning, before the dawn had fully br... \n", + "\n", + " unsloth/Qwen2-7B-Instruct(finetuned) \\\n", + "0 Old Geng raised his rifle and squinted into th... \n", + "1 First thing in the morning Grannie Liu rose to... \n", + "\n", + " unsloth/mistral-7b-instruct-v0.3 \\\n", + "0 Geng Da initiates firing, squinting to form a ... \n", + "1 The next day, when it was still dark, Liu Lao ... \n", + "\n", + " unsloth/mistral-7b-instruct-v0.3(finetuned) \\\n", + "0 Old Geng aimed and fired. A triangular slit op... \n", + "1 Before dawn next day Grannie Liu got up and bu... \n", + "\n", + " gradientai/Llama-3-8B-Instruct-Gradient-1048k \\\n", + "0 The old man pulled out his gun, squinting one ... \n", + "1 The next day, when the sun had not yet risen, ... \n", + "\n", + " gradientai/Llama-3-8B-Instruct-Gradient-1048k(finetuned) \\\n", + "0 Old Geng raised his rifle, squinting through t... \n", + "1 Grannie Liu got up before daylight was even vi... \n", + "\n", + " unsloth/Qwen2-72B-Instruct-bnb-4bit \\\n", + "0 Lao Geng raised his gun, narrowed one of his t... \n", + "1 Before dawn next morning, Granny Liu got up to... \n", + "\n", + " unsloth/Qwen2-72B-Instruct-bnb-4bit(finetuned) \n", + "0 Old Geng raised his gun, narrowed one of his t... \n", + "1 As soon as it was light, Grannie Liu got up an... " + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df2.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "454895be-fb1e-40bd-a6cc-144e952058d9", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Qwen2-0.5B-Instruct checkpoint-560 1\n", + "Qwen2-0.5B-Instruct checkpoint-1120 2\n", + "Qwen2-0.5B-Instruct checkpoint-1680 3\n", + "Qwen2-0.5B-Instruct checkpoint-2240 4\n", + "Qwen2-0.5B-Instruct checkpoint-2800 5\n", + "Qwen2-0.5B-Instruct checkpoint-3360 6\n", + "Qwen2-0.5B-Instruct checkpoint-3920 7\n", + "Qwen2-0.5B-Instruct checkpoint-4480 8\n", + "Qwen2-0.5B-Instruct checkpoint-5040 9\n", + "Qwen2-0.5B-Instruct checkpoint-5600 10\n", + "Qwen2-1.5B-Instruct checkpoint-560 1\n", + "Qwen2-1.5B-Instruct checkpoint-1120 2\n", + "Qwen2-1.5B-Instruct checkpoint-1680 3\n", + "Qwen2-1.5B-Instruct checkpoint-2240 4\n", + "Qwen2-1.5B-Instruct checkpoint-2800 5\n", + "Qwen2-1.5B-Instruct checkpoint-3360 6\n", + "Qwen2-1.5B-Instruct checkpoint-3920 7\n", + "Qwen2-1.5B-Instruct checkpoint-4480 8\n", + "Qwen2-1.5B-Instruct checkpoint-5040 9\n", + "Qwen2-1.5B-Instruct checkpoint-5600 10\n" + ] + }, + { + "data": { + "text/plain": [ + "{'epoch': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],\n", + " 'Qwen2-0.5B-Instruct': [0.26453254295068257,\n", + " 0.28906766286950575,\n", + " 0.3075388134142166,\n", + " 0.3232125016634757,\n", + " 0.3141676906431015,\n", + " 0.31468732087511564,\n", + " 0.3060953047058868,\n", + " 0.29569751947150547,\n", + " 0.29297589531864165,\n", + " 0.2833319356953958,\n", + " 0.28432663251720675],\n", + " 'Qwen2-1.5B-Instruct': [0.3108076173265163,\n", + " 0.3555548051770412,\n", + " 0.364551066769633,\n", + " 0.3723931629938662,\n", + " 0.35847259317675817,\n", + " 0.35988930837184085,\n", + " 0.3460642024871934,\n", + " 0.3479480952549209,\n", + " 0.33844145976530193,\n", + " 0.3380289789419591,\n", + " 0.3339867178782917]}" + ] + }, + "execution_count": 54, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import re # Import the re module for regex operations\n", + "\n", + "dict = {\n", + " \"epoch\": [],\n", + "}\n", + "\n", + "for col in df.columns[2:]:\n", + " # Split using regex\n", + " parts = re.split(r\"[/|_]\", col) # Use re.split() to split by regex\n", + " # print(parts)\n", + " model_name = parts[1]\n", + " checkpoint = parts[2]\n", + " epoch = int(checkpoint.split(\"-\")[1]) // 560\n", + " print(model_name, checkpoint, epoch)\n", + "\n", + " if model_name not in dict:\n", + " dict[model_name] = []\n", + " metrics = calc_metrics(df[\"english\"], df2[f\"unsloth/{model_name}\"])\n", + " dict[model_name].append(metrics[\"meteor\"])\n", + " dict[\"epoch\"].append(0)\n", + "\n", + " metrics = calc_metrics(df[\"english\"], df[col])\n", + " dict[model_name].append(metrics[\"meteor\"])\n", + " dict[\"epoch\"].append(epoch)\n", + "\n", + "dict[\"epoch\"] = dict[\"epoch\"][: len(dict[model_name])]\n", + "dict" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "4007d7c1-11a1-4008-a3e3-8f954fe27fa3", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# create df from dict\n", + "perf_df = pd.DataFrame(dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "e43244f3-4655-4e88-a492-6f2c9599fed5", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA94AAAHqCAYAAADyGZa5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAADCUklEQVR4nOzdd3xN9x/H8de92YkMEUFiiwiKIGLUXkHttnRSbXX6VavVUq1RVFsdWh26q3ToUG1RRYyiVhFb7BUSYoQkZN3z++NwSROE4ma8n4/HeTj3fL/n3M+5jsjnfpfFMAwDEREREREREbkhrI4OQERERERERKQwU+ItIiIiIiIicgMp8RYRERERERG5gZR4i4iIiIiIiNxASrxFREREREREbiAl3iIiIiIiIiI3kBJvERERERERkRtIibeIiIiIiIjIDaTEW0REREREROQGUuItIiIichN89dVXWCwW/vnnH0eHIiIiN5kSbxERcYjzSYjFYmHp0qU5yg3DoFy5clgsFjp37pyt7Px5uW2PPfYYixYtumydi7d/x5LbtmLFimzvn5KSwujRo6lduzaenp74+vrSrFkzvv76awzDyHEv/76ej48PLVq0YNasWdfxExUREZH8ytnRAYiISNHm7u7Ot99+S9OmTbMdX7x4MQcPHsTNzS3X89q1a0efPn1yHA8NDaVChQpMmTIl2/GhQ4dSrFgxhg0bdslYXnnlFSpVqpTjeEhIiH0/ISGBNm3asHXrVu666y4GDBjA2bNn+fnnn+nbty+zZ8/mm2++wcnJKdd4DcNg3759fPTRR3Tp0oU//viDqKioS8YkIiIiBZ8SbxERcahOnTrx448/8t577+HsfOG/pW+//Zb69euTmJiY63mhoaHcd999l7zuv8tee+01AgICLntOx44diYiIuGy8ffv2ZevWrfzyyy907drVfvypp55i8ODBvPnmm9StW5cXXnjhsvHefvvt1KhRg3fffbdAJN5nz57F1dUVq1Wd5URERK6W/vcUERGHuvvuuzl27Bjz5s2zH0tPT+enn37innvucWBkOa1YsYI///yTBx54IFvSfd64ceOoWrUqr7/+OmfOnLnstapXr05AQAC7du3K03tPnDiRmjVr4unpSfHixYmIiODbb7/NVicuLo6HHnqIoKAg3NzcqFSpEo8//jjp6en2Ort37+bOO+/E398fT09PGjVqlKPL+/mu+t9//z0vvfQSwcHBeHp6curUKQBWrlxJhw4d8PX1xdPTkxYtWrBs2bLLxp+QkICzszOjRo3KURYbG4vFYuH9998HICMjg1GjRlG1alXc3d0pUaIETZs2zfaMXMrJkyd5+umnKVeuHG5uboSEhPD6669js9nsdfbu3YvFYuHNN9/knXfeoUKFCnh4eNCiRQs2bdqU45oLFiygWbNmeHl54efnR7du3di6dWuOenn5/AHS0tIYNGgQJUuWxMvLix49enD06NEr3puIiBRcavEWERGHqlixIo0bN+a7776jY8eOAPzxxx8kJSVx11138d577+V63tmzZ3NtDffx8cHV1fWaYklKSspxTYvFQokSJQD4/fffAXLt4g7g7OzMPffcw6hRo1i2bBlt27a97HudOHGCKlWqXDGuTz/9lKeeeoo77riDgQMHcvbsWTZs2MDKlSvtX04cOnSIyMhITp48ySOPPEJYWBhxcXH89NNPpKam4urqSkJCAk2aNCE1NZWnnnqKEiVKMHnyZLp27cpPP/1Ejx49sr3v6NGjcXV15bnnniMtLQ1XV1cWLFhAx44dqV+/PiNGjMBqtfLll1/SunVrlixZQmRkZK73UKpUKVq0aMEPP/zAiBEjspVNmzYNJycn7rzzTgBGjhzJuHHjePjhh4mMjOTUqVP8888/rF27lnbt2l3yc0pNTaVFixbExcXx6KOPUr58ef7++2+GDh3K4cOHmTBhQrb6X3/9NadPn+bJJ5/k7NmzvPvuu7Ru3ZqNGzdSqlQpAObPn0/Hjh2pXLkyI0eO5MyZM0ycOJFbb72VtWvXUrFixTx//uf973//o3jx4owYMYK9e/cyYcIEBgwYwLRp0674LIiISAFliIiIOMCXX35pAMbq1auN999/3/D29jZSU1MNwzCMO++802jVqpVhGIZRoUIF47bbbst2LnDJ7bvvvsv1/WrWrGm0aNHisrHktrm5udnrde/e3QCMEydOXPK+pk+fbgDGe++9ly3ehx56yDh69Khx5MgR459//jE6dOhgAMb48eOv+Fl169bNqFmz5mXr9OnTx7Barcbq1atzlNlsNsMwDOPpp582AGPJkiX2stOnTxuVKlUyKlasaGRlZRmGYRgLFy40AKNy5cr2v5Pz16lataoRFRVlv6ZhGEZqaqpRqVIlo127dpeN8eOPPzYAY+PGjdmO16hRw2jdurX9dZ06dXL8nefF6NGjDS8vL2P79u3Zjg8ZMsRwcnIy9u/fbxiGYezZs8cADA8PD+PgwYP2eitXrjQA45lnnrEfCw8PNwIDA41jx47Zj61fv96wWq1Gnz597Mfy8vmff87atm2b7fN75plnDCcnJ+PkyZNXfc8iIlIwqKu5iIg4XK9evThz5gwzZ87k9OnTzJw584rdzLt168a8efNybK1atbrmOD744IMc1/vjjz/s5adPnwbA29v7ktc4X3a+W/Z5n3/+OSVLliQwMJCIiAiio6N5/vnnGTRo0BXj8vPz4+DBg6xevTrXcpvNxowZM+jSpUuuY9TPz94+e/ZsIiMjs01kV6xYMR555BH27t3Lli1bsp3Xt29fPDw87K9jYmLYsWMH99xzD8eOHSMxMZHExERSUlJo06YNf/31V7Yu3f/Ws2dPnJ2ds7Xsbtq0iS1bttC7d+9s97t582Z27NhxhU8mux9//JFmzZpRvHhxe2yJiYm0bduWrKws/vrrr2z1u3fvTnBwsP11ZGQkDRs2ZPbs2QAcPnyYmJgYHnjgAfz9/e31ateuTbt27ez18vr5n/fII49kO9asWTOysrLYt2/fVd2viIgUHOpqLiIiDleyZEnatm3Lt99+S2pqKllZWdxxxx2XPads2bKX7cp9LSIjIy87udr5pPr06dP4+fnlWudSyXm3bt0YMGAA6enprF69mldffZXU1NQ8TVb2wgsvMH/+fCIjIwkJCaF9+/bcc8893HrrrQAcPXqUU6dOccstt1z2Ovv27aNhw4Y5jlevXt1efvE1/j3D+/lEuG/fvpd8j6SkJIoXL55rWUBAAG3atOGHH35g9OjRgNnN3NnZmZ49e9rrvfLKK3Tr1o3Q0FBuueUWOnTowP3330/t2rUve387duxgw4YNlCxZMtfyI0eOZHtdtWrVHHVCQ0P54YcfAOyJcLVq1XLUq169On/++ScpKSkkJyfn6fM/r3z58tlen/+8Tpw4kafzRUSk4FHiLSIi+cI999xD//79iY+Pp2PHjpdMbB2pevXqzJgxgw0bNtC8efNc62zYsAGAGjVqZDt+8RcFnTp1IiAggAEDBtCqVatsSeel3jc2NpaZM2cyZ84cfv75Zz788EOGDx+e62Rl18vFrd2AvTV7/PjxhIeH53pOsWLFLnvNu+66i379+hETE0N4eDg//PADbdq0ISAgwF6nefPm7Nq1i19//ZW5c+fy2Wef8c477zBp0iQefvjhS17bZrPRrl07nn/++VzLQ0NDLxvbzfLvpebOM3JZA15ERAoHJd4iIpIv9OjRg0cffZQVK1bk20mmOnfuzLhx4/j6669zTbyzsrL49ttvKV68uL01+lIeffRR3nnnHV566SV69OiRozvyv3l5edG7d2969+5Neno6PXv2ZOzYsQwdOpSSJUvi4+OT64zcF6tQoQKxsbE5jm/bts1efjnnJ4Lz8fG55t4G3bt359FHH7X/HW/fvp2hQ4fmqOfv70+/fv3o168fycnJNG/enJEjR1428a5SpQrJycl5ji23ruzbt2+3T5h2/vO41GcWEBCAl5cXHh4eefr8RUSk6NIYbxERyReKFSvGRx99xMiRI+nSpYujw8lVkyZNaNu2LV9++SUzZ87MUT5s2DC2b9/O888/n6O1+N+cnZ159tln2bp1K7/++utl6x47dizba1dXV2rUqIFhGGRkZGC1WunevTu///47//zzT47zz7ekdurUiVWrVrF8+XJ7WUpKCp988gkVK1bM0Ur/b/Xr16dKlSq8+eabJCcn5yjPy5JYfn5+REVF8cMPP/D999/j6upK9+7dL3u/xYoVIyQkhLS0tMteu1evXixfvpw///wzR9nJkyfJzMzMdmzGjBnExcXZX69atYqVK1faZ9cvU6YM4eHhTJ48mZMnT9rrbdq0iblz59KpUyeAPH/+IiJSdKnFW0RE8o3LjR3+t+3btzN16tQcx0uVKnXZJacu548//rC3/l6sSZMmVK5cGTCXoGrTpg3dunXjnnvuoVmzZqSlpTF9+nQWLVpE7969GTx4cJ7e74EHHmD48OG8/vrrOZLPi7Vv357SpUtz6623UqpUKbZu3cr777/PbbfdZh9L/uqrrzJ37lxatGjBI488QvXq1Tl8+DA//vgjS5cuxc/PjyFDhtiXbXvqqafw9/dn8uTJ7Nmzh59//vmK482tViufffYZHTt2pGbNmvTr14/g4GDi4uJYuHAhPj4+9iXXLqd3797cd999fPjhh0RFReUYVlCjRg1atmxJ/fr18ff3559//uGnn35iwIABl73u4MGD+e233+jcuTMPPPAA9evXJyUlhY0bN/LTTz+xd+/ebF3aQ0JCaNq0KY8//jhpaWlMmDCBEiVKZOuqPn78eDp27Ejjxo156KGH7MuJ+fr6MnLkSHu9vHz+IiJShDl2UnURESmqLl5O7HKudjmxSy0Zdq3LiQHGl19+ma3+6dOnjZEjRxo1a9Y0PDw8DG9vb+PWW281vvrqq2zLRF0c75NPPpnre48cOdIAjIULF17yM/j444+N5s2bGyVKlDDc3NyMKlWqGIMHDzaSkpKy1du3b5/Rp08fo2TJkoabm5tRuXJl48knnzTS0tLsdXbt2mXccccdhp+fn+Hu7m5ERkYaM2fOzHad88uJ/fjjj7nGs27dOqNnz572eCpUqGD06tXLiI6OvuQ9XOzUqVOGh4eHARhTp07NUT5mzBgjMjLS8PPzMzw8PIywsDBj7NixRnp6+hWvffr0aWPo0KFGSEiI4erqagQEBBhNmjQx3nzzTfv555cTGz9+vPHWW28Z5cqVM9zc3IxmzZoZ69evz3HN+fPnG7feeqvh4eFh+Pj4GF26dDG2bNmSo96VPv9LPfPnP+/LPQMiIlKwWQxD/Z9ERESk6Ni7dy+VKlVi/PjxPPfcc44OR0REigCN8RYRERERERG5gZR4i4iIiIiIiNxASrxFREREREREbiCN8RYRERERERG5gdTiLSIiIiIiInIDKfEWERERERERuYGcHR3A9WCz2Th06BDe3t5YLBZHhyMiIiIiIiKFnGEYnD59mqCgIKzWy7dpF4rE+9ChQ5QrV87RYYiIiIiIiEgRc+DAAcqWLXvZOoUi8fb29gbMG/bx8XFwNJeXkZHB3Llzad++PS4uLo4ORyQHPaOS3+kZlfxOz6jkd3pGJb8rKM/oqVOnKFeunD0fvZxCkXif717u4+NTIBJvT09PfHx88vVDJEWXnlHJ7/SMSn6nZ1TyOz2jkt8VtGc0L8OdNbmaiIiIiIiIyA2kxFtERERERETkBlLiLSIiIiIiInIDFYox3iIiIiIicmNlZWWRkZHh6DCkCMjIyMDZ2ZmzZ8+SlZXlsDhcXFxwcnK6LtdS4i0ichkffPAB48ePJz4+njp16jBx4kQiIyNzrTt9+nReffVVdu7cSUZGBlWrVuXZZ5/l/vvvt9e51OQbb7zxBoMHD2bv3r2MHj2aBQsWEB8fT1BQEPfddx/Dhg3D1dX1htyjiIjI5RiGQXx8PCdPnnR0KFJEGIZB6dKlOXDgQJ4mLruR/Pz8KF269H+OQ4m3iMglTJs2jUGDBjFp0iQaNmzIhAkTiIqKIjY2lsDAwBz1/f39GTZsGGFhYbi6ujJz5kz69etHYGAgUVFRABw+fDjbOX/88QcPPfQQt99+OwDbtm3DZrPx8ccfExISwqZNm+jfvz8pKSm8+eabN/6mRURE/uV80h0YGIinp6fDEyEp/Gw2G8nJyRQrVgyr1TGjow3DIDU1lSNHjgBQpkyZ/3Q9Jd4iIpfw9ttv079/f/r16wfApEmTmDVrFl988QVDhgzJUb9ly5bZXg8cOJDJkyezdOlSe+JdunTpbHV+/fVXWrVqReXKlQHo0KEDHTp0sJdXrlyZ2NhYPvroIyXeIiJy02VlZdmT7hIlSjg6HCkibDYb6enpuLu7OyzxBvDw8ADgyJEjBAYG/qdu55pcTUQkF+np6axZs4a2bdvaj1mtVtq2bcvy5cuveL5hGERHRxMbG0vz5s1zrZOQkMCsWbN46KGHLnutpKQk/P39r+4GREREroPzY7o9PT0dHImIY5x/9v/r/AZq8RYRyUViYiJZWVmUKlUq2/FSpUqxbdu2S56XlJREcHAwaWlpODk58eGHH9KuXbtc606ePBlvb2969ux5yevt3LmTiRMnqrVbREQcSt3Lpai6Xs++Em8RkevI29ubmJgYkpOTiY6OZtCgQVSuXDlHN3SAL774gnvvvRd3d/dcrxUXF0eHDh2488476d+//w2OXERERERuFCXeIiK5CAgIwMnJiYSEhGzHExIScozTvpjVaiUkJASA8PBwtm7dyrhx43Ik3kuWLCE2NpZp06blep1Dhw7RqlUrmjRpwieffPLfbkZEREREHEpjvEVEcuHq6kr9+vWJjo62H7PZbERHR9O4ceM8X8dms5GWlpbj+Oeff079+vWpU6dOjrK4uDhatmxJ/fr1+fLLLx06qYiIiEhBd+DAAR588EGCgoJwdXWlQoUKDBw4kGPHjjkspvXr13P33XdTrlw5PDw8qF69Ou+++26ezv3xxx8JCwvD3d2dWrVqMXv27MvWX7RoERaLJccWHx9vr/PAAw9kKytRogQdOnRgw4YNl7323r17sVgsxMTE5Cn2K/nqq6/w8/O7LtfKi5YtW/L000/flPfSb3MiIpcwaNAgPv30UyZPnszWrVt5/PHHSUlJsc9y3qdPH4YOHWqvP27cOObNm8fu3bvZunUrb731FlOmTOG+++7Ldt1Tp07x448/8vDDD+d4z/NJd/ny5XnzzTc5evQo8fHx2f5zFBERkbzZvXs3ERER7Nixg++++46dO3cyadIk+xfpx48fd0hca9asITAwkKlTp7J582aGDRvG0KFDef/99y973t9//83dd9/NQw89xLp16+jevTvdu3dn06ZNV3zP2NhYDh8+bN/+vTRqhw4d7GXR0dE4OzvTuXPn/3SfN0p6erqjQ7h6RiGQlJRkAEZSUpKjQ7mi9PR0Y8aMGUZ6erqjQxHJlZ7R7CZOnGiUL1/ecHV1NSIjI40VK1bYy1q0aGH07dvX/nrYsGFGSEiI4e7ubhQvXtxo3Lix8f333+e45scff2x4eHgYJ0+ezFH25ZdfGkCum5j0jEp+p2dU8rureUbPnDljbNmyxThz5sxNiOz669Chg1G2bFkjNTU12/HDhw8bnp6exmOPPWZMnDjRqFmzpr3sl19+MQDjo48+sh9r06aNMWzYMPvrGTNmGHXr1jXc3NyMSpUqGSNHjjQyMjLs5YDx6aefGt27dzc8PDyMkJAQ49dff71srE888YTRqlWry9bp1auXcdttt2U71rBhQ+PRRx+95DkLFy40AOPEiROXrNO3b1+jW7du2Y4tWbLEAIwjR45c8rw9e/YYgLFu3bps7zV//nyjfv36hoeHh9G4cWNj27Zt9nNiYmKMli1bGsWKFTO8vb2NevXqGatXr7afe/E2fPhwwzAMo0KFCsYrr7xi3H///Ya3t7fRt2/fXO9r3bp1BmDs2bPHfmzp0qVGixYtDA8PD8PPz89o3769cfz4caNv37453u/i88673L+Bq8lD1eItInIZAwYMYN++faSlpbFy5UoaNmxoL1u0aBFfffWV/fWYMWPYsWMHZ86c4fjx4/z999/0bhsBcWvAMOz1HnnkEVJTU/H19c3xfg888ACGYeS6iYiI5AeGYZCanumQ7Wr+Pzx+/Dh//vknTzzxhH095vNKly7Nvffey7Rp02jRogVbtmzh6NGjACxevJiAgAAWLVoEmMtILV++3D5fy5IlS+jTpw8DBw5ky5YtfPzxx3z11VeMHTs223uMGjWKXr16sWHDBjp16sS999572Rb2vCwfunz58mxLnQJERUXlaanT8PBwypQpQ7t27Vi2bNll6yYnJzN16lRCQkKuaf32YcOG8dZbb/HPP//g7OzMgw8+aC+79957KVu2LKtXr2bNmjUMGTIEFxcXmjRpwoQJE/Dx8SEuLo5t27bx7LPP2s978803qVOnDuvWrePll1/OUxwxMTG0adOGGjVqsHz5cpYuXUqXLl3Iysri3XffpXHjxvTv39/e0l+uXLmrvte80uRqIiLXk2HA4RjYOhO2zYSj55YeK9cQol6FshEODU9EROS/OpORRY3hfzrkvbe8EoWna95SmB07dmAYBtWrV8+1vHr16pw4cYLAwED8/f1ZvHgxd9xxB4sWLeLZZ5+1j7letWoVGRkZNGnSBDAT6iFDhtC3b18AKleuzOjRo3n++ecZMWKE/foPPPAAd999NwCvvvoq7733HqtWraJDhw45Yvn777+ZNm0as2bNuuw9xcfH57rU6eWGpJUpU4ZJkyYRERFBWloan332GS1btmTlypXUq1fPXm/mzJkUK1YMgJSUFMqUKcPMmTOvaa6ZsWPH0qJFCwCGDBnCbbfdxtmzZ3F3d2f//v0MHjyYsLAwAKpWrWo/z9fXF4vFQunSpfH09LTHA9C6detsifiBAweuGMcbb7xBREQEH374of1YzZo17fuurq54enpeduLc60Ut3iIi/1VWJuxZAn+8ABNqwSctYcmbZtJtdQZndziwEj5rAz89BCf3OzpiERGRIuNKreRubm40b96cRYsWcfLkSbZs2cITTzxBWloa27ZtY/HixTRo0ABPT0/AnBjtlVdeoVixYvbtfKtpamqq/bq1a9e273t5eeHj48ORI0dyvP+mTZvo1q0bI0aMoH379gDs378/2/VfffXVa77/atWq8eijj1K/fn2aNGnCF198QZMmTXjnnXey1WvVqhUxMTHExMSwatUqoqKi6NixI/v27QOgY8eO9nguTl5zc/G9lylTBsB+74MGDeLhhx+mbdu2vPbaa+zatStP9xERcfWNF+dbvPMDtXiLiFyLjDOwa6HZqh37B5y5qOuYiyeEtIXqXaFqO8hIhQVjIOZb2PQTbP0dGj8JTZ8Bdx/H3YOIiMg18HBxYssrUQ5777wKCQnBYrGwdetWevTokaN869atlCxZEj8/P1q2bMknn3zCkiVLqFu3Lj4+PvZkfPHixfbWWzC7YY8aNYqePXvmuKa7u7t938XFJVuZxWLBZrNlO7ZlyxbatGnDI488wksvvWQ/HhQUlG2m8PNd0EuXLn3VS53mJjIykqVLl2Y75uXlZV8SFeCzzz7D19eXTz/9lDFjxvDZZ59x5syZXO/t3y4ut1gsAPZ7HzlyJPfccw+zZs3ijz/+YMSIEXz//fe5/h39O76LnW+Jv/iLlYyMjGx1/j3EwJGUeIuI5NWZk7Bjrpk474yGjJQLZR7+UK0TVO8MlVuCy0U/6D38oPuHEPkIzH0J9i6BpW/DuinQ6kWo2wec9ONYREQKBovFkufu3o5UokQJ2rVrx4cffsgzzzyTLQmLj4/nm2++4cknnwSgRYsWPP300/z444/2sdwtW7Zk/vz5LFu2LFsX53r16hEbG5stSb0WmzdvpnXr1vTt2zfH+HBnZ+dcr9+4cWOio6OzLYE1b968q1rqFMyW4PMt0ZdisViwWq32ZDs4OPiq3uNyQkNDCQ0N5ZlnnuHuu+/myy+/pEePHri6upKVlZWna5QsWRKAw4cPU7x4cYAcy5rVrl2b6OhoRo0ales1rub9/qv8/y9GRMSRTsfDtllmy/aeJWC76JtUn7Jmoh3WGco3vnLyHBQOfX83W8jnvgTHd8HMZ2DlJxA1xmwlFxERkevm/fffp0mTJkRFRTFmzBgqVarE5s2bGTx4MKGhoQwfPhwwE7TixYvz7bffMnPmTMBMvJ977jksFgu33nqr/ZrDhw+nc+fOlC9fnjvuuAOr1cr69evZtGkTY8aMyVNcmzZtonXr1kRFRTFo0CD7GG0nJyd7QpmbgQMH0qJFC9566y1uu+02vv/+e/755x8++eQTe52hQ4cSFxfH119/DcCECROoVKkSNWvW5OzZs3z22WcsWLCAuXPnZrt2WlqaPY4TJ07w/vvvk5ycTJcuXfJ0T3lx5swZBg8ezB133EGlSpU4ePAgq1ev5vbbbwegYsWKJCcnEx0dTeXKlXF2ds42zvtiISEhlCtXjpEjRzJ27Fi2b9/OW2+9la3O0KFDqVWrFk888QSPPfYYrq6uLFy4kDvvvJOAgAAqVqzIypUr2bt3L8WKFcPf3/+axrTnhcZ4i4j827FdsOxd+KwdvBUGswbBrgVm0l0yDJo9B48sgmc2QcfXoVKzvLdYWywQ1gmeWAEdXgd3Pzi6FabeDlN6QsKWG3lnIiIiRUrVqlVZvXo1lStXplevXlSoUIGOHTsSGhrKsmXL7EmdxWKhWbNmWCwWmjZtCpjJuI+PDxEREdm6OUdFRTFz5kzmzp1LgwYNaNSoEe+88w4VKlTIc1w//fQTR48eZerUqZQpU8a+NWjQ4LLnNWnShG+//ZZPPvmEOnXq8NNPPzFjxgxuueUWe53Dhw+zf/+F+WTS09N59tlnqVWrFi1atGD9+vXMnz8/x9jnOXPm2ONo2LAhq1evztYD4HpwcnLi2LFj9OnTh9DQUHr16kXHjh3tLdJNmjThscce4+677yYkJITx48df8louLi589913bNu2jdq1a/P666/n+OIjNDSUuXPnsn79eiIjI2ncuDG//vorzs7m723PPfccTk5O1KhRg5IlS2b73K43i1EI1qg5deoUvr6+JCUl4eOTv8dLZmRkMHv2bDp16nTFsREijlAkn1HDgPgNF2YiP/Kv5Dc44lzLdhcI+G/dynJIPQ5/vQmrPjETe4sV6vU1u6AXC7y+71VIFMlnVAoUPaOS313NM3r27Fn27NlDpUqVso1fLshGjBjB22+/zbx582jUqJGjw5Fc2Gw2Tp06hY+Pzw1rgc6ry/0buJo8VF3NRaRosmXB/hVmor1tZvaZxq3OULGp2YU87DbwCbpxcXj6Q4dXocFDMH+EOX58zZew8Sdo9gw0eiL7eHERERH5T0aNGkXFihVZsWIFkZGRDk/spGhQ4i0iRUfGWdiz2ExuY/+A1MQLZc4eENIGqneB0CjwKH5zYytRBXpPhb3L4M8XzbXAo1+Bf76EtiPhltvNbuoiIiLyn/Xr18/RIUgRo8RbRAq3s6fMmci3zYQd8yA9+UKZux9U62i2bFdpDa6eDgvTruKt0H8hbPwRokdB0gH4+SFY8SFEvQrl1SVOREREpKBR4i0ihU/yEYidbY7Z3rMYstIvlHkHmd3Hq3eGCreCUz4cf2m1Qp3eZuv78g9g6TsQtwa+iIIa3c0WcP9Kjo5SRERERPJIibeIFA4n9l6YHG3/CuCieSNLVL0wOVpQXTOxLQhcPaHFYKh3PywcC2unwJYZ5pcKDR81Z1f38HN0lCIiIiJyBUq8RaRgMgxI2Gwm2ltnQsLG7OVBdc0u5NW7QMlqjonxevEuDV0nQuQj5vrfuxfB3xNh3Tfm7Of1H8ifLfciIiIiAijxFpGCxJYFB1ZdmIn8xN4LZRYnqNDETLTDbgPfsg4L84YpXQvun2GOVZ87DBK3w+znYOXH0H6MOSmcJmATERERyXeUeItI/paZBnv+ujATecqRC2XO7uakaGGdzUnSPP0dF+fNYrFAaHuo0grWfAWLxsGxHfBdb6jUAqLGmgm6iIiIiOQbSrxFJP9JO2226p6fiTzt1IUyd18I7WAm2yFtwNXLcXE6kpMLRPaH2r1gyVuw4iNzIrlJzaDufdD6JbOLuoiIiIg4nBJvEckfUhIvzES+exFkpV0oK1b6wkzkFZtpPPPF3H2h3SsQ8SDMHwmbf4F1U2DTdGj6NDQekD+WSRMREREpwgrI1L4iUiid3A/LP4QvO8GbVeG3/8GOP82k278K3DoQHpoPg7ZC57fNbuVKunNXvCLc+RU8OBeCIyAjxZwJfWJ9WP892GyOjlBERMQhDhw4wIMPPkhQUBCurq5UqFCBgQMHcuzYMYfFtH79eu6++27KlSuHh4cH1atX5913373ieZs3b+b222+nYsWKWCwWJkyYcMVz9u7di8ViybGtWLHCXmfkyJHZynx9fWnWrBmLFy++4vUtFgszZsy4Yr28WLRoERaLhZMnT16X613JAw88QPfu3W/Ke6nFW0RuHsOAI1vPzUT+O8RvyF5epo655Ff1zlAyTBOFXYvyDeHh+bDpZ5g/CpL2wy+Pml3Ro8ZCxaaOjlBEROSm2b17N40bNyY0NJTvvvuOSpUqsXnzZgYPHswff/zBihUr8Pe/+XPErFmzhsDAQKZOnUq5cuX4+++/eeSRR3BycmLAgAGXPC81NZXKlStz55138swzz1zVe86fP5+aNWvaX5coUSJbec2aNZk/fz4Ax48f580336Rz584cPHgQX1/fq3qvGy09PR1XV1dHh3FV1OItIjeWzWbORD73ZZhYDz5qbLbExm8AixUqNIUOr8HTG+HRv8x1qwOrK+n+LywWqHUHDFgNbUeCqzccjoGvboPv74VjuxwdoYiIyE3x5JNP4urqyty5c2nRogXly5enY8eOzJ8/n7i4OIYNG8b777/PLbfcYj9nxowZWCwWJk2aZD/Wtm1bXnrpJfvrX3/9lXr16uHu7k7lypUZNWoUmZmZ9nKLxcJnn31Gjx498PT0pGrVqvz222/28gcffJB3332XFi1aULlyZe677z769evH9OnTL3s/DRo0YPz48dx11124ubld1WdRokQJSpcubd9cXLL3InR2draX1ahRg1deeYXk5GS2b9+e5/c437o+ffp0WrVqhaenJ3Xq1GH58uX2Ovv27aNLly4UL14cLy8vatasyezZs9m7dy+tWrWyx1q8eHH69esHQMuWLRkwYABPP/00AQEBREVF2d8rJibGfu2TJ09isVhYtGiR/djmzZvp3LkzPj4+eHt706xZM3bt2sXIkSOZPHkyv/76q72l/+Lzrje1eIvI9ZeZDnuXnFv2azYkx18oc3IzZ+Q+PxO5V4Dj4izsXNyh6TMQfh8setWcBX3bTNj+pzkxW/PBRWMmeBERub4MAzJSHfPeLp55/nL++PHj/Pnnn4wdOxYPD49sZaVLl+bee+9l2rRpLF68mKeeeoqjR49SsmRJFi9eTEBAAIsWLeKxxx4jIyOD5cuXM2TIEACWLFlCnz59eO+99+xJ3COPPALAiBEj7O8xatQo3njjDcaPH8/EiRO599572bdv3yVb2JOSkm5o63vXrl05e/YsoaGhPP/883Tt2vWSddPS0vjyyy/x8/OjWrVqV/1ew4YN480336Rq1aoMGzaMu+++m507d+Ls7MyTTz5Jeno6f/31F15eXmzZsoVixYpRrlw5fv75Z26//Xa2bt2KxWIhMDDQfs3Jkyfz+OOPs2zZsjzHERcXR/PmzWnZsiULFizAx8eHZcuWkZmZyXPPPcfWrVs5deoUX375JcAN/fyVeIvI9ZGeAjvnm5Ojbf8T0pIulLn5QNX2ZhfykLbg5u24OIuiYiWh8zsQ+QjMfcn8e1rxIcR8Cy2HQMRD4FywumuJiIgDZaTCq0GOee8XD+V5RZMdO3ZgGAbVq1fPtbx69eqcOHGCwMBA/P39Wbx4MXfccQeLFi3i2WeftY+5XrVqFRkZGTRp0gQwE+ohQ4bQt29fACpXrszo0aN5/vnnsyXeDzzwAHfffTcAr776Ku+99x6rVq2iQ4cOOWL5+++/mTZtGrNmzcr7Z5FHxYoV46233uLWW2/FarXy888/0717d2bMmJEt+d64cSPFihUDzC7t3t7eTJs2DR8fn6t+z+eee47bbrsNMD+vmjVrsnPnTsLCwti/fz+33347tWqZy59WrlzZft75xDcwMBCr1ZrtvatWrcobb7xhf713794rxvHBBx/g6+vL999/b2/hDw0NtZd7eHiQlpZG6dI3fiUYJd4icu1Sj5tra2+bCbsWQObZC2VegRDWyRyzXam5Erv8ILA63PezmXj/+RIc3QpzhsCqT6DdaHPmeHXxFxGRQsYwjMuWu7m50bx5cxYtWkTbtm3ZsmULTzzxBG+88Qbbtm1j8eLFNGjQAE9Pc5WQ9evXs2zZMsaOHWu/RlZWFmfPniU1NdVer3bt2vZyLy8vfHx8OHLkSI7337RpE926dWPEiBG0b98egP3791OjRg17nRdffJEXX3zxmu4/ICCAQYMG2V83aNCAQ4cOMX78+GyJd7Vq1ezd4U+fPs20adO48847WbhwIRERETz22GNMnTrVXj85OfmS73nxvZcpUwaAI0eOEBYWxlNPPcXjjz/O3Llzadu2Lbfffnu2+pdSv379vN/0OTExMTRr1ixHt3pHUOItIlcn6SBsm2VOjrbvbzCyLpQVr2h2Ia/eBco2AKuTw8KUywhpC5VamsuOLRwLx3fDtHvN8fZRYyEo3MEBiohIvubiabY8O+q98ygkJASLxcLWrVvp0aNHjvKtW7dSsmRJ/Pz8aNmyJZ988glLliyhbt26+Pj42JPxxYsX06JFC/t5ycnJjBo1ip49e+a4pru7+4VQ/5XsWSwWbP9aZWTLli20adOGRx55JNsY8qCgoGxjl693F+iGDRsyb968bMdcXV0JCQmxv65bty4zZsxgwoQJTJ06lVdeeYXnnnsuT9e/+N4t577UP3/vDz/8MFFRUcyaNYu5c+cybtw43nrrLf73v/9d9ppeXtl7Olit5nRlF3+xkpGRka3Ov4cYOJISbxG5sqOxsPU3sxv54ZjsZaVqmV3IwzpDqZpqMS0onJwhoh/ccjssmwB/vw/7lsInLaHOXdD6ZfANdnSUIiKSH1ksee7u7UglSpSgXbt2fPjhhzzzzDPZkrD4+Hi++eYbnnzySQBatGjB008/zY8//kjLli0Bc0Kv+fPns2zZMp599ln7ufXq1SM2NjZbknotNm/eTOvWrenbt2+21nMwJzr7r9e/nJiYGHtL9OU4OTlx5swZwOz+ffGY6/+iXLlyPPbYYzz22GMMHTqUTz/9lP/973/2mcqzsrLsifWllCxZEoDDhw9Tt25dgGxfVoDZ8j558mQyMjJybfV2dXUlKysrx/EbQYm3iORk2LDErYEdf5it28d2XFRogfKNzrVsdzZbuaXgcveBNsOh/gMQ/Qps/BHWfwebZ8CtT0GTp8CtmKOjFBERuSbvv/8+TZo0ISoqijFjxmRbTiw0NJThw4cDZoJWvHhxvv32W2bOnAmYifdzzz2HxWLh1ltvtV9z+PDhdO7cmfLly3PHHXdgtVpZv349mzZtYsyYMXmKa9OmTbRu3ZqoqCgGDRpEfLw5Ea2Tk5M9ocxNeno6W7Zsse/HxcURExNDsWLF7In6+++/zy+//EJ0dDRgTkrm6upqT06nT5/OF198wWeffZbt2pmZmfY4znc137JlCy+88EKe7imvnn76aTp27EhoaCgnTpxg4cKF9nH4FSpUwGKxMHPmTJo1a5ZjnPfFPDw8aNSoEa+99hqVKlXiyJEj2XoNAAwYMICJEydy1113MXToUHx9fVmxYgWRkZFUq1aNihUr8ueffxIbG0uJEiXw9fW9Yd3StZyYiJgMAw6swjrnBdpvfgbnr6LMltBjO8DJFULaQZd34bnt8OAcaDJASXdh4lcebv8MHl4A5RpB5hlY/DpMrA9rp4Dt5nwbLCIicj1VrVqV1atXU7lyZXr16kWFChXsSd+yZcvsk4lZLBaaNWuGxWKhadOmgJmM+/j4EBERka2bc1RUFDNnzmTu3Lk0aNCARo0a8c4771ChQoU8x/XTTz9x9OhRpk6dSpkyZexbgwYNLnveoUOHqFu3LnXr1uXw4cO8+eab1K1bl4cfftheJzExkV27si8dOnr0aOrXr0/Dhg359ddfmTZtmn2prvM2b95sjyM8PJwffviBjz76iD59+uT5vvIiKyuLJ598kurVq9OhQwdCQ0P58MMPAQgODmbUqFG8+OKLhIaGXrH7+RdffEFmZib169fn6aefzvHFR4kSJViwYAHJycm0aNGC+vXr8+mnn9qT6/79+1OtWjUiIiIoWbLkVc2YfrUsxpVmGygATp06ha+vL0lJSdc0697NlJGRwezZs+nUqVO+GOQvwvHdsOEH2DDN3D/HcPXCUjXKnHCranuzZVSKBsOALb/CvOFwcp95rFQtiBoDlVs6NDTQz1HJ//SMSn53Nc/o2bNn2bNnD5UqVco2frkgGzFiBG+//Tbz5s2jUaNGjg5HcmGz2Th16hQ+Pj5X7HJ+o13u38DV5KHqai5SFKUeh83TzYT7wMoLx128sIXdxqqUstS/81lcPNTFuEiyWKBmd3Od9VWfwOLxkLARvu4GoR2h/WgIqOroKEVERK7JqFGjqFixor3LsaMTOykalHiLFBWZaeb62hummX/azs36aLGarZi174LqncmyuJIwezY4uzk0XMkHnN2gyf+gzj2w+DVY/Tls/wN2zoOIB6HFEPAq4egoRURErtq/u1mL3GhKvEUKM8MwW7TXfw+bf4GzJy+Ula5lJtu17gDv0heO/2sZBhG8SkCn8dCgv9n9fPsfZkv4+mnQYjBEPqIvakREREQuQ4m3SGF0bJeZbG+YdmGMLoB3ENS+00y4S9VwXHxSMJUMhXu+h92L4M+XzO7nc1+C1Z9B21FQo5uWkxMRERHJhRJvkcIi5Zg5bnv99xD3z4XjrsWgeleo0xsqNgOrk+NilMKhckt4dLG57Fj0aDixF37sa86GHvUqlK3v6AhFRERE8hUl3iIFWcZZ2D7HbNneMRdsmeZxixWqtDZbtsNuA1dPx8YphY/VCereBzW6w9/vwbL34MAK+Kw11LoT2owAv3KOjlJERK4Tm83m6BBEHOJ6PftKvEUKGpvNTHDWfw+bZ0Ba0oWyMnXMZPuW28G7lMNClCLErRi0ehHq9YUFo81W8I0/wtbfofGT0PQZcPN2dJQiInKNXF1dsVqtHDp0iJIlS+Lq6opFw4rkBrPZbKSnp3P27FmHzTpvGAbp6ekcPXoUq9WKq6vrf7qeEm+RgiJxh5lsb/wBTu6/cNyn7IVx24FhjotPijbfYOgxCRo+ao7/3rcUlrwFa7+GVsOg7v3gpP9yREQKGqvVSqVKlTh8+DCHDh1ydDhSRBiGwZkzZ/Dw8HD4Fz2enp6UL1/+P38BoN+CRPKzlETY9LOZcB9ae+G4q7c5kVWd3lChKWj9SckvgurCAzNh2yyY9zIc3w0znzZnQW8/BkLaODpCERG5Sq6urpQvX57MzEyysrIcHY4UARkZGfz11180b94cFxcXh8Xh5OSEs7PzdUn+lXiL5DcZZyD2D3Pc9s75F43bdoKQtlC7F1TrpHHbkn9ZLFC9M1RtD/98DotegyNbYGpP8xluPwYCqzs6ShERuQoWiwUXFxeHJkFSdDg5OZGZmYm7u3uheeauqZnsgw8+oGLFiri7u9OwYUNWrVp1ybrTp08nIiICPz8/vLy8CA8PZ8qUKTnqbd26la5du+Lr64uXlxcNGjRg//79uVxRpBCy2WDPEvj1SXgzFH7qZ06aZss0WxA7vA7PxsK9P5jrbivploLA2RUaPQ5PrYNGT4DV2fwy6aMmMPMZSD7q6AhFREREboqrbvGeNm0agwYNYtKkSTRs2JAJEyYQFRVFbGwsgYGBOer7+/szbNgwwsLCcHV1ZebMmfTr14/AwECioqIA2LVrF02bNuWhhx5i1KhR+Pj4sHnzZtzd3f/7HYrkZ0djz43b/hGSDlw47lvObNmu3RtKVnNcfCLXg6c/dBgHDR6GecNh20z45wvY8CM0G2Qm5S76eS8iIiKF11Un3m+//Tb9+/enX79+AEyaNIlZs2bxxRdfMGTIkBz1W7Zsme31wIEDmTx5MkuXLrUn3sOGDaNTp0688cYb9npVqlS52tBECobko7DpJzPhPhxz4bibz7lx23dB+SYaty2FT4kqcNc3sHcp/PkiHF4P0aPgny+h7QhzNn7NlCsiIiKF0FX9Zp+ens6aNWto27bthQtYrbRt25bly5df8XzDMIiOjiY2NpbmzZsD5lTxs2bNIjQ0lKioKAIDA2nYsCEzZsy4ujsRyc/SU2HjT/DNnfBWNZgzxEy6rc4Q2hHu/Aqe2w7d3oeKmixNCrmKTaH/IujxMXgHQdJ++Pkh+LwdHLj00CURERGRguqqWrwTExPJysqiVKns6wOXKlWKbdu2XfK8pKQkgoODSUtLw8nJiQ8//JB27doBcOTIEZKTk3nttdcYM2YMr7/+OnPmzKFnz54sXLiQFi1a5LheWloaaWlp9tenTp0CzNnvMjIyruaWbrrz8eX3OOU6MGxY9i3FuvFHLNt+x5KebC+yBdXDuKUXthrdwSvgwjn54LnQMyo3TY3boWpHrCs+xLp8IpaDq+HzdtiqdyOr9XDwq5DraXpGr8AwICMV0k5BWjKWtFPn9k9D2iks5/4095PNfddiZDV8QpPeXSd6RiW/0zMq+V1BeUavJj6LYRhGXisfOnSI4OBg/v77bxo3bmw//vzzz7N48WJWrlyZ63k2m43du3eTnJxMdHQ0o0ePZsaMGbRs2dJ+zbvvvptvv/3Wfk7Xrl3x8vLiu+++y3G9kSNHMmrUqBzHv/32Wzw9NemUOJb3mYOUO76MsieW45Fx3H48xTWAg8WbcND/VpLdyzgwQpH8xy3jJNUP/0z5Y39hwSDL4szuklFsL92FTKei83PdYmThnJWKS9ZZ80/bGZyzzuCSlXruz3P79uMXldsuvLZiu+r3NrCwr0QLtpXpSZqL3/W/ORERkUImNTWVe+65h6SkJHx8fC5b96pavAMCAnByciIhISHb8YSEBEqXLn3J86xWKyEhIQCEh4ezdetWxo0bR8uWLQkICMDZ2ZkaNWpkO6d69eosXbo01+sNHTqUQYMG2V+fOnWKcuXK0b59+yvesKNlZGQwb9482rVrV2imxhcgOQHr5p/N1u2EjfbDhpsPRvVu2Gr1wrVcQypbrFR2YJh5oWdUHOceMhM24TR/OE57/6LqkVmEJK/A1uwFbPX6mEMzyKfPaLZW5tPZWpXN1xe1OJ89Demns5edPQXpyVgyUq9fSBYruHmb80e4+WC4edtfG/bj5p+WvUuwxs6k4rFFVDi9Glvjp7A1fBxcis6XHtdTvnxGRS6iZ1Tyu4LyjJ7veZ0XV5V4u7q6Ur9+faKjo+nevTtgtmZHR0czYMCAPF/HZrPZu4q7urrSoEEDYmNjs9XZvn07FSrk3s3Qzc0NNze3HMcL0tqCBSlWuYT0FNg2y5wkbfdCMM61MFldzPWL6/TGUjUKi4v7ta3b52B6RsUhytaFvr/Bjrkw9yUsidtx+vN5nNZ8bq7/XbWdvep1e0azMsyk+GySPTk2X5+6kDyfvZA459w/99rI+u+xnOfsAe4+F5Jj9/NJsu9F+xeXndsuOsfi6pVtsrrLTlvX+DHY9zf8OQzLobU4LR6H09rJ0OZlqH2X5p24Rvo5KvmdnlHJ7/L7M3o1sV31rOaDBg2ib9++REREEBkZyYQJE0hJSbHPct6nTx+Cg4MZN24cAOPGjSMiIoIqVaqQlpbG7NmzmTJlCh999JH9moMHD6Z37940b96cVq1aMWfOHH7//XcWLVp0teGJ3Fi2LNjzF2yYBlt/h4vGbVO2gbn81y23m8snici1sVggNAqqtIY1X8HCVyExFr69Eyq3gtYjzXqGAWnJuSTB/06cz+8nXbR/0TmZZ65j7NZck+BsyXKORPniJNrX/NPJAb9kVGgCD0fD5ukwf5Q56d2Mx2HFRxA1Fio1v/kxiYiIFBJXnXj37t2bo0ePMnz4cOLj4wkPD2fOnDn2Cdf279+P9aJvxlNSUnjiiSc4ePAgHh4ehIWFMXXqVHr37m2v06NHDyZNmsS4ceN46qmnqFatGj///DNNmza9Drco+dkHH3zA+PHjiY+Pp06dOkycOJHIyMhc606fPp1XX32VnTt3kpGRQdWqVXn22We5//777XUeeOABJk+enO28qKgo5syZk+3YrFmzeOWVV9iwYQPu7u60aNHi8jPpx2+CDd+bM5OfPnzhePGKZrJdu7e5VJKIXD9OLhDZH2rdCUvegpWTYPdCnHcvoqOTJ84xZ69vK7OLZ55akrMn0b7Zz/lXK3OBY7VCrTsgrLP5eS95C+I3wOQu5goM7V6BkqGOjlJERKTAuerEG2DAgAGX7Fr+71bqMWPGMGbMmCte88EHH+TBBx+8lnCkgJo2bRqDBg1i0qRJNGzYkAkTJhAVFUVsbCyBgYE56vv7+zNs2DDCwsJwdXVl5syZ9OvXj8DAQPua8AAdOnTgyy+/tL/+97CEn3/+mf79+/Pqq6/SunVrMjMz2bRpU84ATx2GjT+ardsJF5W7+8EtPc3ul+UiC/Yv2SIFgYcftB8NEQ/C/JFYtszANSvlQvn5VuaLE+VsiXNuSfS/W58d1MqcX7m4Q9Onoe59sOg1+OcL2P6HOQQgoh+0HJp9RQYRERG5rGtKvEWuh7fffpv+/fvbhylMmjSJWbNm8cUXXzBkyJAc9Vu2bJnt9cCBA5k8eTJLly7Nlni7ubldcrK/zMxMBg4cyPjx43nooYfsx+2T+6Ulw7aZ5rjtPYuzj9sOjYI6d5njt51zzjEgIjeYfyXoNZmMxN38Ff0nzdvdhkuxEmZLtb4AuzG8AuC2N6HhozBvOMTOhtWfwYYfoNkgaPi4maSLiIjIZWm2FHGI9PR01qxZQ9u2be3HrFYrbdu2Zfny5Vc83zAMoqOjiY2NpXnz7OMOFy1aRGBgINWqVePxxx/n2LFj9rK1a9cSFxeH1Wqlbt26lClTho4dOrBp9qcw/RF4syr88uiFydLKNYTb3obntsNd30D1Lkq6RRzNtxzJ7kHgXabgd+0uKAKqwt3fQd/foUwdc3z8/JHwfgRs+BFsV798mYiISFGiFm9xiMTERLKysuxzA5xXqlQptm3bdsnzkpKSCA4OJi0tDScnJz788EPatbswy3GHDh3o2bMnlSpVYteuXbz44ot07NiR5cuX4+TkxO7duwFzLfi3Xx5IxbObeOvzH2l5x1y2/68Y/h4WKF7JbNmu3Qv88/viXyIiN1Gl5tB/kTkEJ/oVSDoA0x+GFR+aE7BVaOLoCEVERPIlJd5SoHh7exMTE0NycjLR0dEMGjSIypUr27uh33XXXfa6tWrVonbt2lSpUoVFixbRpk0bbMmJAAy71YnbD44C4MvbLJTdYeHHlIY8OuBVKBuhFjQRkUuxWiH8bqjRDVZ8AEsnwKG18GVHc1K2dq9oskkREZF/UVdzcYiAgACcnJxISEjIdjwhIeGS47PB7I4eEhJCeHg4zz77LHfccYd96brcVK5cmYCAEuxc8A1M7kqZJebY8RruCeDkCtW74nbfd1SuUZ/9vpFQroGSbhGRvHD1hOaD4X9rof4D5iR322bCB5HwxxBIPe7oCEVERPINJd7iEK6urtSvX5/o6Gj7MZvNRnR0NI0bN87zdWw2G2lpaTkLsjJhxzwOfnwXx44do8xuc7K0+kFW3JytxAb3Msdt955CRpX27N23jwoVKlyPWxMRKVq8S0GXd+HxvyGkHdgyYeVH8F44/D0RMnP5GS0iIlLEqKu5OMygQYPo27cvERERREZGMmHCBFJSUuyznPfp04fg4GB7i/a4ceOIiIigSpUqpKWlMXv2bKZMmcJHH30EQPLp04x6fgC3hxqUPvIXu+KO8vz8s4QUtxLVIBTq34NP7V48dmYCI778iXLNVlOhQgXGjx8PwJ133umYD0JEpDAIrA73/QS7FsCfL8GRzTD3JXMW9LYjoUZ39SgSEZEiS4m3OEzv3r05evQow4cPJz4+nvDwcObMmWOfcG3//v1YrRc6ZaSkpPDEE09w8OBBPDw8CAsLY+rUqfTucCsseRunf75jw7x1TP7axsmzBkG+zrSPvIXR48bjFt7O/gvf+PHjcXZ25v777+fMmTM0bNiQBQsWULx4cYd8DiIihUqV1vDYEoj5BhaMgRN74ccHoGykOQFbuUhHRygiInLTKfEWhxowYAADBgzItWzRokXZXo8ZM4YxY8ZkrxQ9Gt55FDDwAP7s6w/VOpqzkoe0BSeXHNd1cXHhzTff5M0337w+NyEiItlZnaBeH6jZ0+xu/vd7cHAVfN4OavYwW8CLV3R0lCIiIjeNEm8puA7FwJK3AAMq3Aq1e5uz7Hr4OTgwEREBwK0YtBpqTr62cAys+wY2/wLbZkHDR6HZc/qZLSIiRYImV5OCyTDMsYMYUOtO6Dcb6vfVL3AiIvmRTxno9oHZBb1yS8hKN1vC3wuHFZMgK8PREYqIiNxQSrylYNoxF/YuASc3aP2yo6MREZG8KF0L7p8B9/wIJcPgzAmY8wJ80BC2zjS/VBURESmElHhLwZOVCXPPJduNHoPiWgZMRKTAsFggtD08tgw6vwNeJeH4Lph2L3x1G8StdXSEIiIi150Sbyl41k2BxFjw8IemgxwdjYiIXAsnZ4h4EP63Fpo9C87usG8ZfNoKfu4PJw84OkIREZHrRom3FCxpp2Hhq+Z+ixc0pltEpKBz94E2w+F/a6D2XeaxjT/AxPowfyScPeXQ8ERERK4HJd5SsCx7D1KOgH9ls6VEREQKB9+y0PNjeGQRVGgKWWmw9B14ry6s/swcZiQiIlJAKfGWguPUIXMWXIC2o8DZ1bHxiIjI9RdUFx6YCXd9CyVCIDURZj0LHzWG2DmagE1ERAokJd5ScCwYC5lnoFwjqN7F0dGIiMiNYrFA2G3wxAroON6c0yNxO3zXG77uCoc3ODpCERGRq6LEWwqG+I0Q8425HzXW/KVMREQKNycXaPgIPLUOmjwFTq6w5y/4uDnMeMLsCSUiIlIAKPGWgmHecMCAmj2hbISjoxERkZvJww/aj4YB/8AttwOG+WXse/XM3lBpyY6OUERE5LKUeEv+t3M+7FoAVhdz5lsRESmaileAO76Ah6PNYUeZZ+CvN2BiPVgzGWxZjo5QREQkV0q8JX+zZcHcl839ho+CfyXHxiMiIo5XNgIenAO9vobilSA5AX5/CiY1Nb+sFRERyWeUeEv+FvMNHNkC7n7Q7FlHRyMiIvmFxQI1usGTqyDqVfP/iSNbYOrtMKUnJGx2dIQiIiJ2Srwl/0pLNsfuAbR4Hjz9HRuPiIjkP86u0PhJcwK2Rk+aw5J2RZut37/9D04nODpCERERJd6Sjy1/H5LjoXhFaPCwo6MREZH8zNMfOrwKT66E6l3BsMHar+G9urD4DUhPdXSEIiJShCnxlvzpdDwse8/cbzsSnN0cGo6IiBQQJapA7ynw4J8QHAEZKbBwrDkB27pvwGZzdIQiIlIEKfGW/Gnhq+YvS2UbQI3ujo5GREQKmvKN4OH5cPvn4FseTh+GX5+AT5rD7sWOjk5ERIoYJd6S/yRsgXVTzP32Y80JdERERK6WxQK17oABq6HtKHDzgfiN8HVX+LY3HI11dIQiIlJEKPGW/GfecHNsXvWuUL6ho6MREZGCzsUdmj4NT8VA5CNgcYLtc+DDxjBzECQfdXSEIiJSyCnxlvxl1wLYOQ+szubYbhERkevFqwR0Gm9OwFbtNjCy4J/PzQnYlrwNGWccHaGIiBRSSrwl/7BlwdyXzf0G/c0JckRERK63gKpw97fQdyaUqQPppyF6FLzfADb8oAnYRETkulPiLfnH+u8hYRO4+ZrrdouIiNxIlZpB/0XQ4xPwCYakAzC9P3zWBvb97ejoRESkEFHiLflDeiosGGPuN3/OXI9VRETkRrNaoU5v+N8aaP0yuBaDQ2vhy47w/b2QuNPREYqISCGgxFvyhxUfwOlD5pIvkY84OhoRESlqXDzML36fWgf1+4HFCttmwocN4Y8XIPW4oyMs2gzD0RGIiPwnzo4OQITkI7B0grnfdoQ5+6yIiIgjFAuELhOg4WMw72XYMRdWToKY78zEvOGj4Ozm6CgLLsOA9BRIPQZnjpt/pp741+vjF70+V+bkAjW7Q90+UDZCS42KSIGjxFscb9E4SE+GoHpQs6ejoxEREYHAMLj3R9i1EOa+ZM5BMu9lWP2ZuepGzR5K/gwDziZlT5BzTZ7PbefLstKv/r0yz8Dar80toBrUvQ/q3GV+USIiUgAo8RbHOrIN1kw299uPMcfaiYiI5BdVWsGjf8H67yB6NJzcBz/1gxUfQtSrUC7S0RFeHzYbnD2ZPUHOljwfy5lAnzkBtsxrez8nN3M+F88S4FHc/NP+2j/nftJBWDcVtvwKibHmlyDRo6BqFNS7H0LagZN+rRWR/Es/ocSx5o8w11EN6wwVb3V0NCIiIjlZncwW1po94O+JsOxdOLgaPm8HNbqbLeD+lRwd5QVZmWZSfHHCfKnk+XxyffYkGNe4jJqL50UJ9MUJc4nck2sPf3D1uroeA/6VzFnoO70Bm6abSXjcPxA7y9yKlTJbwMPvg5Kh13YfIiI3kBJvcZw9f8H2OWB1hrajHB2NiIjI5bl6QcshUK8vLBwD676BLTMgdrY5MWjz58wE83rKTL+ou/a/Wp/PnMi9Zfps0rW/n6v3uWQ5t9bnS7RMu3hcv/u9EndfiOhnbke2mgn4+u8hOcH8QmTZu1Cu4YUvSty8b15sIiKXocRbHMNmM8fMAUQ8CAEhjo1HREQkr3zKQLcPoOHj5v9luxfC8vch5hto8QKE98n9vIwzl+jKfeLSLdPpp689TnffvLU+X7zv7Hrt73ezBVaHqLHQZgTs+NNMwnfMhQMrze2PIWbyXfc+KN9IY/JFxKGUeItjbPwRDq8HNx/zlxQREZGCpvQtcP8vsDPaTMCPboU5Q3Be+TH1jVI4ffuFmUifT6wzUq/tfSxWM1nObexzbl27PfzN+kVlzLOzK1TvYm6nDsOG780k/NhOiJlqbv5Vzk3Idrf5xYmIyE1WRH4iS76ScQaiXzH3mz4DXgGOjUdERORaWSxQtS1UbgnrpsDCsVhO7KEse+BkbvWd8tb6fHFy7e6nyUfzyqeM+bvFrU+brd7rpsCmX+D4LnMytgWjzYnY6t1vTsxWkFr4RaRAU+ItN9+Kj+DUQfApC40ed3Q0IiIi/52TsznuuNYdZK37li0bY6hevynOxUqeS6bPJdhuPuryfDNYLGb38vKNoMNrsHmG2Qp+YIXZLX3Hn+AZYE7IVvc+s9u6iMgNpMRbbq6URFjytrnfZvjNnZBFRETkRnPzxlb/QXYnzCasVidwcXF0ROLmbbZw17sfEnecm5DtO3NCtuXvm1twfah7P9zS0xwbLyJynanfktxci183J4opUwdq3enoaERERKQoCagK7UbBM1vg7u/N5UytzhC3BmY+DW9Wg+mPwp4l5kSwIiLXiVq85eZJ3AH/fGHutx+j8WoiIiLiGE7OUK2juSUfgQ3TYO0USIw1J2fb8D0Ur2iuCx5+D/gGOzpiESnglPnIzTN/JNgyIbQDVGru6GhEREREoFggNPkfPLkSHppvrtPu6g0n9prrtb9TE6beDpt/gcw0R0crIgWUWrzl5ti7DLbNNGdzbfeKo6MRERERyc5igXINzK3DONjymzkefN9S2Dnf3DyKQ+3e5oRspWs5OmIRKUCUeMuNZ7OZ65sC1O8LJas5Nh4RERGRy3H1gvC7ze3YLoj51txOH4KVk8ytTLiZgNe6w0zIRUQuQ13N5cbbPB0OrQXXYtByqKOjEREREcm7ElWgzcvwzCa49yeo0Q2sLnA4BmY/Z07I9tNDsGuhJmQTkUtSi7fcWBlnYf4oc7/p0+Y4KhEREZGCxuoEVduZW8ox2PiDOSHbkc2w6Sdz8y1vTsZW917wK+/oiEUkH1GLt9xYqz6BpP3gHQSNnnR0NCIiIiL/nVcJaPQ4PL4MHlkEEQ+Bm6/5O8/i12BCbfi6G2z8CTLOODpaEckH1OItN07qcfjrTXO/zcvg6unYeERERESuJ4sFguqaW9RY2DoT1k2BPYth9yJzc/eFWnea48HLhJvniEiRo8RbbpzFb0BaEpSqZc4AKiIiIlJYuXhA7TvN7cReiPkOYr6BpAOw+jNzK3UL1L0favcCT39HRywiN5G6msuNcWwXrP7U3G8/2hwXJSIiIlIUFK8IrYbCwA1w/y9wy+3g5AYJm2DOC/BWNfihL+yYD7YsR0crIjeBWrzlxpg/EmyZENIOqrRydDQiIiIiN5/VClVam1vqcdj0s9kV/fB62DLD3HyCoc7d5oRs/pUdHbGI3CBq8Zbrb/8K2PobWKzQ7hVHRyMiIiLieJ7+ENkfHv0LHl0CkY+Cux+cioMlb8J7deGrzrD+e0hPdXS0InKdKfGW68swYO5L5n7d+6FUDcfGIyIiIpLflKkNnd6AZ2Phji+hShvAAnuXwC+Pml3Rfx8IB/8xf7cSkQLvmhLvDz74gIoVK+Lu7k7Dhg1ZtWrVJetOnz6diIgI/Pz88PLyIjw8nClTplyy/mOPPYbFYmHChAnXEpo42pYZcHA1uHhBqxcdHY2IiIhI/uXiDrf0hPunw9MbodVL4FcB0k7Bmq/gszbwYWP4+31IPuroaEXkP7jqxHvatGkMGjSIESNGsHbtWurUqUNUVBRHjhzJtb6/vz/Dhg1j+fLlbNiwgX79+tGvXz/+/PPPHHV/+eUXVqxYQVBQ0NXfiTheZpo5thvg1qfAu7RDwxEREREpMPzKQYvB8FQM9P3dXBHG2R2OboW5w+DtMPj+XoidA1mZjo5WRK7SVSfeb7/9Nv3796dfv37UqFGDSZMm4enpyRdffJFr/ZYtW9KjRw+qV69OlSpVGDhwILVr12bp0qXZ6sXFxfG///2Pb775BhcXl2u7G3Gs1Z+Zy2cUKw1N/ufoaEREREQKHqsVKjWHnp/Ac9uh8zsQVM+ctHbbTPiuN7xT02zsSNzp6GhFJI+ualbz9PR01qxZw9ChQ+3HrFYrbdu2Zfny5Vc83zAMFixYQGxsLK+//rr9uM1m4/7772fw4MHUrFnzitdJS0sjLS3N/vrUqVMAZGRkkJGRcTW3dNOdjy+/x3nVzpzAefEbWIDMFkMwLK5Q2O6xiCi0z6gUGnpGJb/TMyrXjZMn1Lnf3I5swbr+W6ybfsSSHA9L34Gl72Ar1whbnXswqncF12J5uqyeUcnvCsozejXxXVXinZiYSFZWFqVKlcp2vFSpUmzbtu2S5yUlJREcHExaWhpOTk58+OGHtGvXzl7++uuv4+zszFNPPZWnOMaNG8eoUaNyHJ87dy6enp55vBvHmjdvnqNDuK5qHvyWkLMnOeVeloVxfnBotqNDkv+osD2jUvjoGZX8Ts+oXH9NsFSNpPSpdZQ/9helTm3AemAF1gMryJw9mDi/huwv0ZzjXlXBYrni1fSMSn6X35/R1NS8r0BwU9bx9vb2JiYmhuTkZKKjoxk0aBCVK1emZcuWrFmzhnfffZe1a9diycMPCIChQ4cyaNAg++tTp05Rrlw52rdvj4+Pz426jesiIyODefPm0a5du8LTpf7EXpw/XgCAZ/e36VSltYMDkv+iUD6jUqjoGZX8Ts+o3HhdAcg8fRjrxh+wrv8G5+O7qXD8Lyoc/wujRAi2Ovdgu6VXrnPu6BmV/K6gPKPne17nxVUl3gEBATg5OZGQkJDteEJCAqVLX3oiLavVSkhICADh4eFs3bqVcePG0bJlS5YsWcKRI0coX768vX5WVhbPPvssEyZMYO/evTmu5+bmhpubW47jLi4u+fov5mIFKdYrWvwqZKVDldY4h0U5Ohq5TgrVMyqFkp5Rye/0jMoN518eWjwHzZ+F/cth3VTY/AuWYztxWvAKTgvHQtX2UPc+CI0Cp+zPo55Rye/y+zN6NbFd1eRqrq6u1K9fn+joaPsxm81GdHQ0jRs3zvN1bDabfYz2/fffz4YNG4iJibFvQUFBDB48ONeZzyWfObAaNk8HLNButKOjERERESl6LBao0AS6f2hOyNZ1IpRrCEYWbP8Dpt0Lb1eHuS/BkUsPDxWRG+equ5oPGjSIvn37EhERQWRkJBMmTCAlJYV+/foB0KdPH4KDgxk3bhxgjseOiIigSpUqpKWlMXv2bKZMmcJHH30EQIkSJShRokS293BxcaF06dJUq1btv96f3EiGYf4ABwi/F0rf4th4RERERIo6N2+o18fcjm6HdVNg/feQcgT+ngh/T8QpOII6Z72xzlkEzq5gdc6+OTnnPJat3AWsThcd+9frqyp3OfenU57GpYsUVFedePfu3ZujR48yfPhw4uPjCQ8PZ86cOfYJ1/bv34/VeqEhPSUlhSeeeIKDBw/i4eFBWFgYU6dOpXfv3tfvLsQxtv4OB1aAswe0HuboaERERETkYiVDof1oaDMcdswzu6Jvn4M17h8qAhxb6OAA/8XidC4pd849Mbcn7Teo3F52vvxfry9X7lYM/CuDc87hsCJwjZOrDRgwgAEDBuRatmjRomyvx4wZw5gxY67q+rmN65Z8JjMd5o8w95v8D3yCHBuPiIiIiOTOyQXCOpnb6QSyNs1g+4YVhFapjJPFgKwMsGWZa4XbMsH2r9f/uTzzorJMswt8bowsyLxEWUFgsZrJd0A1KFkNSoaZX34EhIKrl6OjEwe7KbOaSyG05ks4vhu8AuHWvC0DJyIiIiIO5l0KW8SDbD9SmpAWnXByxMRVNpuZZOeauGec+zProrLM/F9+5iSkJcGxneYWOyv7PfuVNxPxgNBzCfm5pNzd9+Z//uIQSrzl6p05CYteM/dbvWiOJRIRERERyQurFbDmmGW9QDMMSE6Ao9vMsfVHt8HRWPPP1EQ4ud/cdszNfp53mYtax6uday0PA68Sub+PFFhKvOXqLX0bzhw3fyjUvd/R0YiIiIiIOJbFYq6Z7l0aKrfMXpZyDBJjL0rGz22nD8Hpw+a2e1H2czwDLrSKn0/KS4ZBsVIFYhK6Dz74gPHjxxMfH0+dOnWYOHEikZGRudadPn06r776Kjt37iQjI4OqVasycOBA/P397XVGjhzJ999/z4EDB+wrbY0dO5aGDRvmuF5aWhoNGzZk/fr1rFu3jvDw8Bt1m1dFibdcnRP7YMUkc7/dK+aslyIiIiIikjuvEuDVxFzy7WJnk8zW8WxJ+TazZTw1EfYtNbeLufmeS8KrZW8p9yl7rieB402bNo1BgwYxadIkGjZsyIQJE4iKiiI2NpbAwMAc9f39/Rk2bBhhYWG4uroyc+ZM+vfvz0svvUSnTp0ACA0N5f3336dy5cqcOXOGd955h/bt27Nz505KliyZ7XrPP/88QUFBrF+//qbcb14pa5Krs2A0ZKVBpeZQtb2joxERERERKZjcfaFcA3O7WHoKJO4wE/HE2AsJ+fHd5jjyg6vM7WIuXhBQNXvreMlqULyiOQP7TfT222/Tv39/+3LTkyZNYtasWXzxxRcMGTIkR/2WLVtmez1w4EC++uortmzZYj92zz335HiPzz//nA0bNtCmTRv78T/++IO5c+fy888/88cff1zHu/rvlHhL3sWthY0/AhZoP6ZAdHMRERERESlQXL0gKNzcLpaZBsd2ZW8dT9xuJukZKXA4xtwu5uR2LiGvln22df/K5hru11l6ejpr1qxh6NCh9mNWq5W2bduyfPnyK55vGAYLFixg+/btdO/e/ZLv8cknn+Dr60udOnXsxxMSEujfvz8zZszA09PzP9/L9abEW/LGMGDuy+Z+nbugTJ3L1xcRERERkevH2Q1K1TC3i2Vlwok9/xpDvs1MyDPPQMImc7uY1dlMvu2t4+dmXA+oCi4e1xxiYmIiWVlZlCpVKtvxUqVKsW3btkuel5SURHBwMGlpaTg5OTFx4sQc3dJnzpzJXXfdRWpqKmXKlGHevHkEBAQAZsL+wAMP8NhjjxEREZEvl6dW4i15E/uHOcbE2R1av+ToaEREREREBMw5lwKqmlv1LheO22yQtP9CIn7xxG7pp8+1lm+Hrb9fdDGL2T3932PIA0Jv6EpG3t7exMTEkJycTHR0NIMHD+a5556zj/EGaNWqFTExMSQmJvLpp5/Sq1cvVq5cSWBgIBMnTuT06dPZWtrzGyXecmVZGTBvuLnf+EnwLevYeERERERE5PKsVjOJLl4RQqMuHDcMOHXoQlf1i7uunzlhtp6f2APb52S/nk/Zi5Lxi2Zb9yhurxIQEICTkxMJCQnZTk1ISKB06dKXCdVKSEgIAOHh4WzevJmff/6ZF154wV7Hy8uLkJAQQkJCaNSoEVWrVuXzzz9n6NChLFiwgOXLl+Pm5pbtuhEREdx7771Mnjz56j67G0CJt1zZmq/g2A5zWYNbn3Z0NCIiIiIicq0sFvANNreQCxOTYRiQknguEd+WfXK35AQ4ddDcdkVnv16xUmaLeMkwXEtWo/4toUTP+Z3u3bqBxYLNZiM6OpoBAwbkOUSbzUZGRsYV66SlpQHw3nvvMWbMGHvZoUOHiIqKYtq0abkuOeYISrzl8s4mwaJx5n7LIeDu49h4RERERETk+rNYoFhJc6vULHvZmRPm0mf/ntgt6YCZlCcnwN4lAAwKyaDvZ1uJOP4rkbXDmLA0iZRTx+nXMgSS4ujzv6EEBwczbpyZY4wbN46IiAiqVKlCWloas2fP5ptvvuHRRx8FICUlhbFjx9K1a1fKlClDYmIiH3zwAXFxcdx5550AlC9fPlu4xYoVA6BKlSqULZs/eusq8ZbLWzoBUo9BiapQ/wFHRyMiIiIiIjebR3Eo39DcLpZ2bqz4RePHexffxtHUWIbPPU789GWEl7Yyp5c7peY8BHNg/9KzWEv7w4wkKFmNlP0beOKzjzl4KAEPDw/CwsL46quv7Mmzk5MT27ZtY/LkySQmJlKiRAkaNGjAkiVLqFmzpgM+jGujxFsuLekgrPjQ3G/3Cji5ODYeERERERHJP9y8Ibi+uV1kwBNnGHBsZ86J3Y7vYlEfdyAVYr4BYEwpGHM/4OxrX/osy/8gqw+cATrh7u7O9OnTryqsihUrYhjG9bnH60SJt1zagjGQeRYqNIVqHR0djYiIiIiIFAQuHlC6lrldLDMdju++MHbcPpb83NJn8RsgfgNOQA33ssAwR0R/QyjxltwdioH135v77UebYz5ERERERESulbMrBIaZ28VsWXBirz0Ztx3ZRvyRVCo5JMgbQ4m35GQYMPclwIBavSC4nqMjEhERERGRwsrqBCWqmFtYJ7IyMtg6e3ahSrytjg5A8qEdc81ZCZ3coM3Ljo5GRERERESkQFPiLdllZcLcc8l2o8fAr/zl64uIiIiIiMhlKfGW7NZ9bU524OEPTQc5OhoREREREZECT4m3XJB2GhaaC9nTcgh4+Dk0HBERERERkcJAibdcsOw9SDkC/lWgfj9HRyMiIiIiIlIoKPEW06lD8PdEc7/dKHOqfxEREREREfnPlHiLacFYc9H6co0grLOjoxERERERESk0lHgLxG+EmG/M/aixYLE4Nh4REREREZFCRIm3nFs+zICaPaFshKOjERERERERKVSUeBd1O+fD7oXg5AptRzg6GhERERERkUJHiXdRZss619oNRD4CxSs6NBwREREREZHCSIl3URbzDRzZAu5+0Pw5R0cjIiIiIiJSKCnxLqrSks2ZzAFavAAexR0bj4iIiIiISCGlxLuoWv4+JMeb3csbPOzoaERERERERAotJd5F0el4WPauud92JDi7OjQcERERERGRwkyJd1G0cCxkpELZBlCju6OjERERERERKdSUeBc1CVtg3VRzv/1YsFgcG4+IiIiIiEghp8S7qJk3HAwb1OgG5Rs6OhoREREREZFCT4l3UbJrAeycB1YXaDPC0dGIiIiIiIgUCUq8iwpbFsx92dxv8DCUqOLYeERERERERIoIJd5FxfrvIWETuPlCi+cdHY2IiIiIiEiRocS7KEhPhQWjzf3mz4Gnv2PjERERERERKUKUeBcFKz6A04fBrzxEPuLoaERERERERIoUJd6FXfIRWDrB3G8zAlzcHRqOSEHzwQcfULFiRdzd3WnYsCGrVq26ZN3p06cTERGBn58fXl5ehIeHM2XKFHt5RkYGL7zwArVq1cLLy4ugoCD69OnDoUOHsl2na9eulC9fHnd3d8qUKcP999+fo46IiIiIFBxKvAu7ReMgPRmC68Mttzs6GpECZdq0aQwaNIgRI0awdu1a6tSpQ1RUFEeOHMm1vr+/P8OGDWP58uVs2LCBfv360a9fP/78808AUlNTWbt2LS+//DJr165l+vTpxMbG0rVr12zXadWqFT/88AOxsbH8/PPP7Nq1izvuuOOG36+IiIiI3BjOjg5AbqAj22DNZHO//RiwWBwbj0gB8/bbb9O/f3/69esHwKRJk5g1axZffPEFQ4YMyVG/ZcuW2V4PHDiQyZMns3TpUqKiovD19WXevHnZ6rz//vtERkayf/9+ypcvD8AzzzxjL69QoQJDhgyhe/fuZGRk4OLicp3vUkRERERuNLV4F2bzR4CRBWGdoUITR0cjUqCkp6ezZs0a2rZtaz9mtVpp27Yty5cvv+L5hmEQHR1NbGwszZs3v2S9pKQkLBYLfn5+uZYfP36cb775hiZNmijpFhERESmglHgXVnv+gu1zwOoMbUc5OhqRAicxMZGsrCxKlSqV7XipUqWIj4+/5HlJSUkUK1YMV1dXbrvtNiZOnEi7du1yrXv27FleeOEF7r77bnx8fLKVvfDCC3h5eVGiRAn279/Pr7/++t9vSkREREQcQol3YWSzwdyXzP2IByEgxLHxiBQh3t7exMTEsHr1asaOHcugQYNYtGhRjnoZGRn06tULwzD46KOPcpQPHjyYdevWMXfuXJycnOjTpw+GYdyEOxARERGR601jvAujjT/C4fXg5gMtXnB0NCIFUkBAAE5OTiQkJGQ7npCQQOnSpS95ntVqJSTE/LIrPDycrVu3Mm7cuGzjv88n3fv27WPBggU5WrvPv39AQAChoaFUr16dcuXKsWLFCho3bnx9blBEREREbhq1eBc2GWcg+hVzv9kg8ApwbDwiBZSrqyv169cnOjrafsxmsxEdHX1Vya/NZiMtLc3++nzSvWPHDubPn0+JEiXydA0g23VEREREpOBQi3dhs+IjOHUQfMtBw8ccHY1IgTZo0CD69u1LREQEkZGRTJgwgZSUFPss53369CE4OJhx48YBMG7cOCIiIqhSpQppaWnMnj2bKVOm2LuSZ2RkcMcdd7B27VpmzpxJVlaWfby4v78/rq6urFy5ktWrV9O0aVOKFy/Orl27ePnll6lSpYpau0VEREQKKCXehUlKIix529xv/TK4eDg2HpECrnfv3hw9epThw4cTHx9PeHg4c+bMsU+4tn//fqzWCx2HUlJSeOKJJzh48CAeHh6EhYUxdepUevfuDUBcXBy//fYbYHZDv9jChQtp2bIlnp6eTJ8+nREjRpCSkkKZMmXo0KEDL730Em5ubjfnxkVERETkulLiXZgseg3ST0OZOlDrTkdHI1IoDBgwgAEDBuRa9u9J08aMGcOYMWPsr7NsBt+t2s+Lv2ykRWhJWlUrf8UJ0mrVqsWCBQv+c9wiIiIikn8o8S4sEnfAmi/N/fZjwKrh+yKOtPlQEkOnb2TDwSQAvl25n+KeLnSuHUSPesHULeeHxWJxcJQiIiIicjMo8S4s5o8EWyaEdoRKzR0djUiRdSY9iwnzt/PZ0j1k2Qy83Z3pdEsZFsYe4cjpNKas2MeUFfuoFOBFj7rB9KgbTDl/T0eHLSIiIiI3kBLvwmDvMtg2EyxO0G6Uo6MRKbIWbz/KSzM2cuD4GQBuq1WGEV1qEOjjTpbNYNnORKavPcifmxPYk5jC2/O28/a87URW9KdnvWA61iqDr4eLg+9CRERERK43Jd4Fnc0Gc18y9+s/ACWrOTQckaIoMTmNMTO3MCPmEABBvu6M7n4LbaqXstdxslpoHlqS5qElSU7L5M9N8Uxfd5C/dx1j1d7jrNp7nOG/baZd9VL0rBdM89CSuDhpyIiIiIhIYaDEu6DbPB0OrQVXb2g51NHRiBQphmHw45qDvDp7KydTM7Ba4IEmlXi2fShebpf+8VrMzZnb65fl9vplOZx0hhnrDvHLuoNsT0hm1sbDzNp4mBJernSpE0TPesHUCvbVeHARERGRAkyJd0GWcRbmn+ta3nQgFCvp2HhEipDdR5MZ9ssmlu8+BkD1Mj681rMWdcr5XdV1yvh68HjLKjzWojKbD51i+to4flsfR2JyOl/9vZev/t5LlZJe9KxXlu51gwn20zKBIiIiIgWNEu+CbNXHkLQfvIOg0ZOOjkakSEjPtPHJX7t4b8FO0jNtuLtYeaZtKA82rfSfuoZbLBZuCfbllmBfXuwUxpIdiUxfF8fczfHsOprC+D9jeXNuLI0qlaBHvWA63lIab3eNBxcREREpCK7pt8QPPviAihUr4u7uTsOGDVm1atUl606fPp2IiAj8/Pzw8vIiPDycKVOm2MszMjJ44YUXqFWrFl5eXgQFBdGnTx8OHTp0LaEVHanH4a+3zP02L4OrZkUWudHW7DtO54lLeHPudtIzbTQPLcm8Z1rwaIsq13U8trOTlVZhgUy8uy6rX2rLG7fXplFlfwwDlu8+xvM/baDB2Pk89d06FsYeITPLdt3eW0RERESuv6tu8Z42bRqDBg1i0qRJNGzYkAkTJhAVFUVsbCyBgYE56vv7+zNs2DDCwsJwdXVl5syZ9OvXj8DAQKKiokhNTWXt2rW8/PLL1KlThxMnTjBw4EC6du3KP//8c11uslBa/AakJUHpWlC7t6OjESnUTp3N4I052/hm5X4MA0p4uTK8Sw261gm64WOvfdxd6NWgHL0alOPgiVR+jTnE9LUH2XU0hd/WH+K39YcIKOZGt/AgetQNpmaQj8aDi4iIiOQzV514v/322/Tv359+/foBMGnSJGbNmsUXX3zBkCFDctRv2bJlttcDBw5k8uTJLF26lKioKHx9fZk3b162Ou+//z6RkZHs37+f8uXLX22Ihd+xXbD6U3O//RiwOjk2HpFCyjAM/twcz/BfN3PkdBoAd9Yvy4udqlPcy/Wmx1O2uCdPtgrhiZZV2HAwiV/WxfHb+kMkJqfx+dI9fL50D9VKedOjXjDdw4Mp7et+02MUERERkZyuKvFOT09nzZo1DB16YfZsq9VK27ZtWb58+RXPNwyDBQsWEBsby+uvv37JeklJSVgsFvz8/HItT0tLIy0tzf761KlTgNltPSMjI4934xjn4/svcTrNG4HVlomtSluyyt0K+fyepWC5Hs9oYXA46SyvzNzK/G1HAajg78nobtVpXLkE4PjPp0ZpL2p0DOX59iH8tSORX2MOEx17lNiE07z2xzZen7ONxpX96V4niPY1Ai87y3pBo2dU8js9o5Lf6RmV/K6gPKNXE5/FMAwjr5UPHTpEcHAwf//9N40bN7Yff/7551m8eDErV67M9bykpCSCg4NJS0vDycmJDz/8kAcffDDXumfPnuXWW28lLCyMb775Jtc6I0eOZNSoUTmOf/vtt3h6Fu6xzv7J22m2YwwGFhaGjeW0R1lHhyRSqNgMWBpvYeZ+K2k2C1aLQdsgg3bBNlzzeeeS1EyIOWZh9VEru09f6G7uajWo7W/QoKRBqK+BVT3RRURERP6z1NRU7rnnHpKSkvDx8bls3ZvSBOLt7U1MTAzJyclER0czaNAgKleunKMbekZGBr169cIwDD766KNLXm/o0KEMGjTI/vrUqVOUK1eO9u3bX/GGHS0jI4N58+bRrl07XFyuckZiw8Bp8nvmbvh9NLvtkRsQoRR1/+kZLeC2xZ9m2K+b2XDQ7EVTt5wvY7rVILSUt4Mjy7s7zv154EQqv8YcZkbMYfYdT+WfRAv/JEIpbzc61y5Nj/AgqpUuOPd1saL8jErBoGdU8js9o5LfFZRn9HzP67y4qsQ7ICAAJycnEhISsh1PSEigdOnSlzzParUSEhICQHh4OFu3bmXcuHHZEu/zSfe+fftYsGDBZRNoNzc33Nzcchx3cXHJ138xF7umWDf/AnH/gIsX1jYvYS0g9yoFU0H69/Rfnc3I4t3oHXz6124ybQbebs483zGMeyPLYy2gzcOVA315pr0vT7erxroDJ/llbRy/bzhEwuk0Pl+2j8+X7aN6GR961g2mW3gQgT4Fbzx4UXpGpWDSMyr5nZ5Rye/y+zN6NbFdVeLt6upK/fr1iY6Opnv37gDYbDaio6MZMGBAnq9js9myjdE+n3Tv2LGDhQsXUqJEiasJq2jITIP5I839WweC96W/6BCRvFu6I5FhMzay71gqAB1qlmZk15qFZmIyi8VCvfLFqVe+OC93rsHC2CNMX3uQBduOsPXwKcYePsW4P7bStGpJbq8XTPsapfHI733qRURERAqYq+5qPmjQIPr27UtERASRkZFMmDCBlJQU+yznffr0ITg4mHHjxgEwbtw4IiIiqFKlCmlpacyePZspU6bYu5JnZGRwxx13sHbtWmbOnElWVhbx8fGAuRSZq+vNnzk4X1r9GZzYC8VKQ5O8f8khIrk7npLOmFlbmL42DoDSPu680q0m7WsW3i+1XJ2tRNUsTVTN0pxMTWfmhsNMX3uQtftP8tf2o/y1/Sherk50rFWGnnWDaVS5RIFt8RcRERHJT6468e7duzdHjx5l+PDhxMfHEx4ezpw5cyhVqhQA+/fvx2q12uunpKTwxBNPcPDgQTw8PAgLC2Pq1Kn07m2uPR0XF8dvv/0GmN3QL7Zw4cIc48CLpNTj5rrdAK1fAlcvx8YjUoAZhsH0tXGMmbWFE6kZWCzQt3FFnm0fird7/u3KdL35ebpyX6MK3NeoAnsSU/hlXRy/rDvIgeNn+GnNQX5ac5Ayvu50rxtMz7rBVC1A49xFRERE8ptrmlxtwIABl+xavmjRomyvx4wZw5gxYy55rYoVK3IVE6sXTUvegrMnIbAmhN/j6GhECqy9iSkMm7GRZTuPARBW2ptxPWtRt3xxB0fmWJUCvBjULpRn2lbln30nmL42jlkbDnE46SwfLdrFR4t2cUuwDz3rlqVreBABxXLOsSEiIiIil1Z4FnYtrI7vgZUfm/vtXwGrxl6KXK2MLBufLtnNu/N3kJZpw83ZysC2VenfrDIuTtYrX6CIsFgsNKjoT4OK/ozoUoMF244wfW0ci2KPsCnuFJvitjB29lZahJakR91g2tUohbuLfiaJiIiIXIkS7/wu+hWwZUCV1hDS1tHRiBQ46/afYOj0jWyLPw1A05AAxva4hQolNGTjctxdnOhUqwydapXhWHKaOR58XRzrD5xkwbYjLNh2BG83ZzrVKkOPesFEVvTXeHARERGRS1DinZ8dWA2bpwMWaDfa0dGIFCinz2bw5p+xfL1iH4YBxT1deLlzDXrUDcZiUYJ4NUoUc6Nvk4r0bVKRXUeT+WVtHL+siyPu5Bmm/XOAaf8cINjPgx51g+lRL5gqJYs5OmQRERGRfEWJd35lGDD3JXO/7r1Q+hbHxiNSgMzdHM/wXzcTf+osAD3rBfPSbTXw99IqCf9VlZLFeC6qGoPahbJq73F+WRvH7I2HiTt5hvcX7uT9hTupU86PnnWD6VInSJ+5iIiICEq886+tv8OBFeDiCa2GOToakQIhPuksI3/bzJzN5pKE5f09ebVHLZpWDXBwZIWP1WqhUeUSNKpcglHdajJvSwK/rItj8fajrD9wkvUHTjJ65hZaVgukZ71gWocFajy4iIiIFFlKvPOjzHSYP8Lcb/I/8AlybDwi+ZzNZvDNyn28MSeW02mZOFktPNK8Mk+1roqHq5K9G83dxYkudYLoUieIo6fT+H39IaavO8imuFPM35rA/K0J+Lg7c1vtIG6vF0z9CsXV3V9ERESKFCXe+dE/X8Dx3eAVCE2ecnQ0IvlabPxphk7fwNr9JwGoU86P13rWonoZH8cGVkSV9HbjwaaVeLBpJXYknGb6ujhmrIvjcNJZvlu1n+9W7ae8v6c5HrxuMBUDNMmdiIiIFH5KvPObMydh8evmfqsXwU2TFInk5mxGFu8v2MmkxbvItBl4uTrxfIcw7mtUASfNrp0vVC3lzQsdwniufTVW7j7Gz2vjmLPpMPuPp/Ju9A7ejd5BvfJ+9KxXls61y+DnqfHgIiIiUjgp8c5vlr4NZ45DyTCoe7+joxHJl/7elciwXzaxJzEFgLbVS/FKt5oE+Xk4ODLJjZPVQpOQAJqEBDC6uzke/Oe1cSzdcZS1+0+ydv9JXvl9C63DAulRL5hW1QJxddb66iIiIlJ4KPHOT07sgxWTzP12o8FJfz0iFzuRks7Y2Vv5ac1BAAK93XilW02iapbWmOECwtPVmW7hwXQLD+bIqbP8tv4QP6+NY+vhU8zZHM+czfH4ebrQpXYQPeoFU7ecn/5uRUREpMBTZpefLBgNWWlQqQVUbefoaETyDcMw+DXmEK/M3MLxlHQsFrivYQUGd6iGj7uLo8OTaxTo487DzSrzcLPKbD18il/OjQc/cjqNKSv2MWXFPioFeNnHg5fz93R0yDeUzWaQYbORmWWQmWWQnmUj89zr9Czzz4wsGxlZNjJt5/cNMs8dy8gyyLTZyMi8cJ0cdWwGGZkXn2/W83Jzpl2NUjSpUgJnJ/U2EBERud6UeOcXcWtg44+ABdqPAbXwiACw/1gqw2ZsZMmORABCSxVjXM9a1K/g7+DI5HqqXsaH6mV8eKFDGH/vSmT62jjmbIpnT2IKb8/bztvzthNZ0Z8e9YJpH5b78nCGYeSSkBr/SlSzJ6QZ55Jb+/7FyWq249mvlZ6ZMynOtNlIzzT+dTxnDNkS6CwzSc7IMsiyGTf5U89uyop9BBRzpVOtMnSpE0T98sWxar4EERGR60KJd35gGDD3ZXO/zt1QprZj4xHJBzKzbHy+dA/vzN/O2Qwbrs5WnmodwiPNq2j8byHmZLXQrGpJmlUtyZjumczZFM8v6+JYtiuRVXuPs2rvcUY4W/GwODFqw8JzCfaFJLmwcXGy4Gy14uJkwcXJiouTFWf7fvYy54vrWC24OFtxsVpwPnfMXt/Zgos1+3X2Hkvlj42HSUxO5+vl+/h6+T6CfN3ty8TVDPJRl38REZH/QIn3Vfrggw8YP3488fHx1KlTh4kTJxIZGZlr3enTp/Pqq6+yc+dOMjIyqFq1KgMHDsTf3z9bnUlvjWHNuhiOnzFY91cPwv91nU8++YRvv/2WtWvXcvr0aU6cOIGfn98Nu0cRR1t/4CRDp29ky+FTADSuXIKxPW6hcknN8l+UeLk5c3v9stxevyyHk87wa8whpq89yPaEZNKxQEbGFa/hbLXkTEjPJ6HZEtLcE1vznNyS238nvVZcz13T2WrB1dk8z9nJguu585ytVlydz593cQyWfyXL5+tbbmqyO6prTZbuTOT39YeYuzmBQ0ln+fiv3Xz8124ql/SiS+0guoYHUUX/DkVERK6aEu+rMG3aNAYNGsSkSZNo2LAhEyZMICoqitjYWAIDA3PU9/f3Z9iwYYSFheHq6srMmTPp378/L730Ep06dQIg5XQSTX3j6dXWjf6/nwXvnNdJTU2lQ4cOdOjQgaFDh97w+xRxlJS0TN6au52v/t6DzQA/TxeGdarOHfXLqrWtiCvj68FjLarwaPPK7IhPYt7CxbRq3hx3N5fsCfO5Ft3zSbGem7xzcbLSqlograoFcjYji4XbjvD7hkNEbz3C7qMp9iXgapTxoWu42RIerJUERERE8kSJ91V4++236d+/P/369QNg0qRJzJo1iy+++IIhQ4bkqN+yZctsrwcOHMhXX33Fli1b7Mfur54JkSnsPVsSfj+Q6/s+/fTTACxatOi63IdIfhS9NYGXZ2ziUNJZALqHB/FS5xoEFHNzcGSSn1gsFioFeFHWC6qWKoaLiybXuxHcXZzoWKsMHWuV4fTZDOZtSeC39YdYuiORLYdPseXwKV77Yxv1KxSna50gOtUqQ0lv/VsVERG5FCXeeZSens6aNWuytThbrVbatm3L8uXLr3i+YRgsWLCA7du30717d/Pg2SRYNM7cb/wEoNZsKXqOnDrLqN+3MGvjYQDK+XswpnstWoSWdHBkIgLg7e5Cz3pl6VmvLMdT0vlj02F+iznEqr3HWbPvBGv2nWDU75u5NSSALnWCiKpZGl8PfSEiIiJyMSXeeZSYmEhWVhalSpXKdrxUqVJs27btkuclJSURHBxMWloaTk5OTJw48UK39KUTIPUYBITCLbejxFuKEpvN4PvVBxj3x1ZOn83EyWrh4aaVGNi2Kp6u+tEkkh/5e7lyb8MK3NuwAvFJZ5m54RC/rz/E+oNJLNmRyJIdibz0yyZaVCtJ1zpBtKkeqH/PIiIiKPG+4by9vYmJiSE5OZno6GgGDx7Mc889R6dba8OKD81K7V4BJ7UOSNGxI+E0Q6dv5J99JwCoFezLuJ61uCXY18GRiUhelfa9sA77vmMp/L7+EL+tP8T2hGTmbUlg3pYEPF2daFu9FF3rBNE8tKRWJBARkSJLiXceBQQE4OTkREJCQrbjCQkJlC5d+pLnWa1WQkJCAAgPD2fz5s38/PPPvFh9H2SehQpNIbQD7Nt3Q+MXyQ/OZmTx4aJdfLRoJxlZBp6uTjzXvhp9m1TESesFixRYFUp4MaB1VQa0rsq2+FP2JPzA8TP8dm7fx92ZjreUoWt4EI0ql9C/eRERKVKUeOeRq6sr9evXJzo62j5G22azER0dzYABA/J8HZvNhpF2GsvGH80D7UeDZt2VImDF7mO8+MtGdh9NAaB1WCCju9+iWZFFCpmw0j6ElfbhufbViDlwkt/WH2LWhsMcOZ3GtH8OMO2fAwQUc6Nz7TJ0qRNEvfJ+mn1eREQKPSXeV2HQoEH07duXiIgIIiMjmTBhAikpKfZZzvv06UNwcDDjxpkTpo0bN46IiAiqVKlCWloas2fP5ptvvuGN7uWwYECtXhz3qMj+mBgOHToEQGxsLAClS5e2t6THx8cTHx/Pzp07Adi4cSPe3t6UL18+25rgIvlRUmoG4/7YyverzVn7A4q5MaprTTrVKq1ftkUKMYvFQt3yxalbvjgv3VaDlXuO8fv6Q8zeGE9ichpf/b2Xr/7eS9niHnSpE0SX2kFUL+OtnwsiIlIoKfG+Cr179+bo0aMMHz6c+Ph4wsPDmTNnjn3Ctf3792O1Xhi/lpKSwhNPPMHBgwfx8PAgLCyMya8N4p7TkzCc3LC0eZnfZvxmT9wB7rrrLgBGjBjByJEjAXPZslGjRtnrNG/eHIAvv/ySBx544Abftci1MQyD3zcc5pXfN5OYnA7A3ZHlGdIhDF9PzWkgUpQ4WS00qRJAkyoBjOp6C0t3HuW3mEPM3ZLAwRNn+GjRLj5atIuQwGJ0rWOuEV4pwMvRYYuIiFw3FsMwDEcH8V+dOnUKX19fkpKS8PHxcXQ4l5aVifFRYyyJ28lq/BROUaMdHZFIDhkZGcyePZtOnTpd8xrJB46n8vKvm1gUexSAkMBijOtZiwYV1UND/rvr8YxK/nAmPYsF247w2/o4FsYeJT3TZi+rFexL1zpBdK5ThjK+BWtIip5Rye/0jEp+V1Ce0avJQ9XifTOt+xpL4nbSnIphbfI0To6OR+Q6y8yy8dXfe3lr7nbOZGTh6mTlyVYhPNayMm7OeuJFJDsPVyduq12G22qX4dTZDOZuTuC39YdYtjORjXFJbIxLYuzsrURW9KdLeBCdbilNiWJujg5bRETkqinxvpkqNsMW2pHY1ACqu+fjlnmRa7DxYBJDf9nAprhTAERW8ufVHrUICSzm4MhEpCDwcXfhjvpluaN+WRKT0/hjUzy/xxxi1d7j9m3kb5tpGhJAlzpBRNUshbd7/m0FERERuZgS75spoCpZd05hz6xZVHd0LCLXSUpaJu/M284Xy/ZgM8DH3Zlht1XnzvrlsGq5IBG5BgHF3Li/UQXub1SBQyfPMHPDIX5ff5iNcUks3n6UxduP8uIvVlpXC6RLnSDaVA/E3UW9akREJP9S4u0ImrFVComFsUd46ZdNxJ08A0CXOkG83Lk6gd7uDo5MRAqLID8PHmlehUeaV2H30WR+X3+Y39bHsetoCnM2xzNnczxerk60r1marnWCaFo1ABcn65UvLCIichMp8RaRq3b0dBqvzNzC7+vNZfCC/TwY0/0WWoUFOjgyESnMKpcsxsC2VXmqTQhbD5/mt/WH+H39IeJOnuGXdXH8si4OP08XOt5Shq51gois5I+Tet6IiEg+oMRbRPLMMAx++OcAY2dt5dTZTKwWePDWSjzTLhQvN/04EZGbw2KxUCPIhxpBPrzQoRpr95/k9/WHmLnhMInJaXy3aj/frdpPKR83bqsVRNfwIOqU9dUa4SIi4jD6TVlE8mTX0WRenL6RlXuOA1AzyIfXetamVllfB0cmIkWZxWKhfoXi1K9QnJduq87KPcf5LeYQf2w6TMKpNL5Ytocvlu2hQglPutQ21wivVtrb0WGLiEgRo8RbRC4rLTOLSYt288HCnaRn2fBwcWJQu1D63VoRZ42jFJF8xNnJyq0hAdwaEsAr3WuyZHsiv60/xLwtCew7lsr7C3fy/sKdVCvlTdfwILrUDqJ8CU9Hhy0iIkWAEm8RuaTVe48zdPpGdh5JBqBFaEnGdL+Fcv76RVVE8jc3Zyfa1ihF2xqlSE3PZP7WI/y+/hCLY48Sm3Ca8X/GMv7PWOqU86NrnSA61y5DKR9NDCkiIjeGEm8RySE1E17+bQvfrz4IQEAxV4Z3qUmX2mU0RlJEChxPV2e61gmia50gklIz+HNzPL+tP8TfuxJZf+Ak6w+cZMysLTSs5E/XOsF0vKU0xb1cHR22iIgUIkq8RSSbPzbFMy7GiVMZZtLdO6IcQzuF4eepX0JFpODz9XShV4Ny9GpQjqOn05i98TC/rT/Emn0nWLH7OCt2H2f4r5toVjWAruFBtKtRmmKaPFJERP4j/U8iIgCkpGXy8q+bmL42DrBQqYQn426vTaPKJRwdmojIDVHS242+TSrSt0lFDp5IZeaGw/wWc4gth0+xMPYoC2OP4ua8kTbVA+laJ4iW1QJxd3FydNgiIlIAKfEWEbbFn+LJb9ay62gKVgu0DbLx9kONKeap8Y4iUjSULe7JYy2q8FiLKuw8kszv59YI352Y8v/27jwu6mr/4/hrZthFQUTZRHDfBRUll9KuqGWbt8UlS7Kb/rrFLaW6LpW2aKg3vV6XtDQrU1NvZlEapSiWifuauW/gAoimyD4y8/vDnCK1KybMAO/n4/F9yHznzOHzzaPy7pzv97BidxordqdR1dWJHs39uS8sgE4NfHHWAyZFROQGKXiLVGJWq5XFm1MZE7+HgksW/Kq5MunhlpzduwFXzeqISCXVoJYnw7o3YmhUQ/acyrKF8FMX8lm67QRLt53Ap4oLvVr6c39YEBEh1TEa9fwLERG5PgVvkUoqu+ASoz7bTfzOU8DlJ5ZP7hNGNVcjK/bauTgREQdgMBhoEeRFiyAvht/VhK0pP/PlzlMs33WaszmFzN+QwvwNKQR4uXFvqwDuDwuiRVA1e5ctIiIOSMFbpBLac+oCMQu3czQzB5PRwIs9GvN/d9TDaDRgNpvtXZ6IiMMxGg20C/WhXagPo+9txvrDZ/ly5ykS9qRx+kI+s78/yuzvj1LXtwq9Wvjhk2/vikVExJEoeItUIlarlfkbjvPm8r0UXrIQ4OXGtP6tiQj1sXdpIiLlhpPJyB2NanJHo5q82bsFaw+cIX7nKRL3pnM0M4cZSUcAJ77L3kZ0p1C6NKqFSUvRRUQqNQVvkUoiK9/MiKW7WLE7DYBuTWrx9iNh2qtWRORPcHM20bO5Pz2b+5NTcIlVe9NZujWV7w9msvaXI9jHncdvC6FPRLC2ZhQRqaQUvEUqgV0nzhOzcDsp53JxMhoYcXcT/ta5LgaDZmBERG6VKq5OPBAeRK/mtZj32QpOedTn022nSD2Xx1sr9jHp2wPcHxZIdMdQWgR52btcEREpQwreIhWY1Wrlgx+OEff1XsxFVoK83Zn+aGta16lu79JERCo0XzcYeFdjXuzZlPidJ/lo/XF+Op3Ff7ee4L9bT9C6jjfRHUK5u6U/rk7aRUJEpKJT8BapoC7kmnnp0518+1M6AD2b+zHxoTC8PJztXJmISOXh7mKib7s69IkIZlvKz8xLPs6K3afZnnKe7Sk7ePMrF/q1D2ZAZAiB3u72LldEREqJgrdIBbQt5Wf+sXA7J8/n4WIyMqpXE6I7hmppuYiInRgMBtqG+NA2xIdX7mnGok0pLNiYQlpWPjPWHGZm0mG6N/NjYIdQOtavob+vRUQqGAVvkQrEYrEyZ90RJibs55LFSh0fD2Y82oaWtXUvoYiIo6hZ1ZV/dGvI37vWZ9XedD5af5zkI2f5Zk863+xJp0EtTx6/LYQH2wRR1U2rlEREKgIFb5EK4uecQl74705W78sA4J5WAcQ92JJq+qFNRMQhOZmM3NUigLtaBHAw/SIfbzjO0q0nOJSRzZj4PUxM2MeDbWozsEMIDf2q2rtcERH5ExS8RSqAzcfO8dwn2zl9IR8XJyOj723GgMg6WqooIlJONPSryhsPtOClno1Ztv0kH60/xuEzOXy84TgfbzhOh3o1GNghhO7N/HAyGe1droiIlJCCt0g5ZrFYmbn2MJNXHqDIYqWebxWmP9qGZoHV7F2aiIjchKpuzgzsEMrjt4WQfPgs85KP8+1PaSQfOUvykbP4V3NjQGQd+rWvQ82qrvYuV0REbpCCt0g5lZldwLDFO/j+YCYAvcMDGfvXlni66o+1iEh5ZzAY6NjAl44NfDl1Po+FG1NYtPnyw9gmrTzA1NUHubtFANEdQ2hTp7pWOImIODj9hC5SDiUfPsvzi7aTcbEAN2cjb9zfgkciausHLxGRCijQ250XezbmH90a8PXuNOYlH2Nbynnid54ifucpmgVUI7pjCPeHBeHuoj3BRUQckYK3SDlSZLEyffUh/pN4AIsVGtTyZMajbWjsr4fuiIhUdK5OJnq3DqJ36yB+PHmBecnH+GLHKX46ncXwpbt5a8U+Hmlbm8duCyHUt4q9yxURkd9Q8BYpJzIu5jN00Q7WHz4LwMNta/PGA83xcNEfYxGRyqZFkBcTHw5jVK+mLNmSyvwNKaScy2XOuqPMWXeUro1rMrBDCF0b1cJo1GooERF700/sIuXAuoOZDF28g8zsAtydTYzt3YKH2ta2d1kiImJn3h4uDLmjPk91rsfaA2f4KPkYSfvP2I46Ph48dlsd+kQE4+3hYu9yRUQqLQVvEQd2qcjCfxIPMn3NIaxWaOJflemPtqFBLU97lyYiIg7EaDRwZ5Na3NmkFscyc5i/4ThLtqSSci6Xt1bsY9K3B3ggPJCBHUJpEeRl73JFRCodBW8RB5V2IZ/nFm1n09FzAPRvH8yY+5rj5qwH54iIyPWF+lbhlXub8UKPxsTvPMlH64/z0+kslmw5wZItJ2hTx5uBHUK5u6U/rk76N0VEpCwYb+ZDM2bMIDQ0FDc3NyIjI9m0adN123722WdERETg7e1NlSpVCA8P5+OPPy7Wxmq1Mnr0aAICAnB3dycqKoqDBw/eTGkiFULS/gx6Tf2eTUfPUcXFxH/6hRP3YCuFbhERuWHuLib6tqvD8uc6s/TvHXggPBBnk4FtKecZungHncav5u1v9nPqfJ69SxURqfBKHLwXL15MbGwsY8aMYdu2bYSFhdGzZ08yMjKu2d7Hx4eXX36Z5ORkdu3axaBBgxg0aBDffPONrc3EiROZOnUqs2bNYuPGjVSpUoWePXuSn59/81cmUg6ZiyyM/3ofT3ywmXM5hTQLqMZXz93OA+FB9i5NRETKKYPBQNsQH/7TrzXrR3Tjhe6N8K/mRmZ2IdPXHKLzhNX838dbWH8oE6vVau9yRUQqpBIH78mTJzN48GAGDRpEs2bNmDVrFh4eHsydO/ea7bt27cpf//pXmjZtSv369Xn++edp1aoV69atAy7Pdk+ZMoVXXnmFBx54gFatWjFv3jxOnTrF559//qcuTqQ8OXU+j37vbWDW2sMADOwQwmfPdKSutoQREZFbpGZVV/7RrSHrht/JzAFt6FCvBhYrfLMnnUfnbKT7v79jXvIxLuab7V2qiEiFUqJ7vAsLC9m6dSsjR460nTMajURFRZGcnPw/P2+1Wlm9ejX79+9nwoQJABw9epS0tDSioqJs7by8vIiMjCQ5OZl+/fpd1U9BQQEFBQW211lZWQCYzWbMZsf+h+JKfY5ep5StxH0ZjPhsD+fzzHi6OvFW72bc3cIfsGA2W8q0Fo1RcXQao+LoyssYjWriS1QTXw6mZ7NgUyqf7zjFoYxsRn+xhwlf76N3eCADIoNpqAd6VjjlZYxK5VVexmhJ6itR8M7MzKSoqAg/P79i5/38/Ni3b991P3fhwgWCgoIoKCjAZDLxzjvv0L17dwDS0tJsffy+zyvv/V5cXByvv/76Vee//fZbPDw8SnJJdrNy5Up7lyAO4JIFvkwxknT68uKT4CpWnmiUjzVlGytS7Fubxqg4Oo1RcXTlaYy2N0GrMNh0xsC6dCPpeUUs2JTKgk2pNKxmobO/lZY+VkzaErxCKU9jVConRx+jubm5N9y2TJ5qXrVqVXbs2EF2djaJiYnExsZSr149unbtelP9jRw5ktjYWNvrrKwsgoOD6dGjB9WqVbtFVZcOs9nMypUr6d69O87OzvYuR+wo9edchi7Zxa7Tl1dsPNGhDi/1aISL00098/CW0RgVR6cxKo6uPI/RB7m8QjH5yDnmb0wlcV8GB7OMHMwC/2qu9GsXTN+IIHw9Xe1dqvwJ5XmMSuVQXsbolZXXN6JEwdvX1xeTyUR6enqx8+np6fj7+1/3c0ajkQYNGgAQHh7O3r17iYuLo2vXrrbPpaenExAQUKzP8PDwa/bn6uqKq+vVf+E7Ozs79G/Mb5WnWuXWS/gxjZc+3cnF/EtUc3Pi7UfC6NH8+n+G7EFjVBydxqg4uvI8Rrs08adLE39Ons9j4cbjLNqUSlpWAVMSDzEj6TC9WgYwsEMIbepUx2DQNHh5VZ7HqFQOjj5GS1JbiabWXFxcaNu2LYmJibZzFouFxMREOnTocMP9WCwW2z3adevWxd/fv1ifWVlZbNy4sUR9ipQHBZeKeC1+D0/P38rF/Eu0ruPNiudvd7jQLSIiAhDk7c5LPZuwfuRfmNI3nNZ1vDEXWflixykempnMvdPWsXhzCnmFRfYuVUTEoZV4qXlsbCzR0dFERETQvn17pkyZQk5ODoMGDQJg4MCBBAUFERcXB1y+HzsiIoL69etTUFDAihUr+Pjjj5k5cyZweYuLoUOHMnbsWBo2bEjdunV59dVXCQwMpHfv3rfuSkXs7FhmDjGfbOPHk5eXpPzfHfV4sWdjnE32XVouIiLyv7g6mejdOojerYPYfeIC85KPEb/zFHtOZTF86W7eWrGPPhG1eey2EEJqaDcOEZHfK3Hw7tu3L2fOnGH06NGkpaURHh5OQkKC7eFoKSkpGI2/BomcnByeeeYZTpw4gbu7O02aNGH+/Pn07dvX1uaf//wnOTk5DBkyhPPnz9O5c2cSEhJwc3O7BZcoYn9f7TrFiKW7yS64RHUPZyb1CeMvTfz+9wdFREQcTMvaXvzrkTBG9WrKf7em8vGG46Sey2P290eZs+4oXRrVJLpDKF0a1cRo1DJ0ERG4yYerxcTEEBMTc833kpKSir0eO3YsY8eO/cP+DAYDb7zxBm+88cbNlCPisPLNRbz51U8s2Hj5EeXtQqsztX9rArzc7VyZiIjIn1O9igtD7qjP3zrXY+2BDOYlHydp/xnbUcfHg8duq0OfiGC8PVzsXa6IiF2VyVPNRSqjw2eyeXbBNvalXcRggGe61mdYVCOctLRcREQqEJPRwF+a+PGXJn4cy8xh/objLNmSSsq5XN5asY9J3x7ggfBABnYIpUWQl73LFRGxCwVvkVLw+faTjFq2m9zCImpUceHffcO5o1FNe5clIiJSqkJ9q/DKvc14oUdjvthxknnJx/npdBZLtpxgyZYTtKnjzcAOodzd0h9XJ5O9yxURKTMK3iK3UF7h5aeWL96SCsBt9XyY2q81tarpeQUiIlJ5uLuY6Ne+Dn3bBbP1+M/MSz7O1z+eZlvKebal7GDschf6tavDo5F1CPTW7VciUvEpeIvcIgfTL/Lswm0cSM/GYIDn/tKQ57o1xKQHy4iISCVlMBiICPUhItSHjItNWbQplYUbU0jLymf6mkPMXHuY7k39GNghhA71a2hPcBGpsBS8RW6B/25JZfQXe8gzF1Gzqiv/6RtOxwa+9i5LRETEYdSq6sZz3Rry9671WfVTOh8lH2PDkXMk7EkjYU8aDWp5MrBDCH9tHURVN2d7lysickspeIv8CTkFl3j1ix/5bNtJAG5v6MvkPuHUrOpq58pEREQck7PJyN0tA7i7ZQAH0i8yL/kYn207yaGMbEZ/sYcJX+/jwTa1GdghhIZ+Ve1drojILaHgLXKT9qVl8eyCbRw+k4PRALHdG/FM1wbas1REROQGNfKrytjeLRl+VxM+23aSecnHOHwmh483HOfjDcfpUK8GAzuE0L2Zn3YFEZFyTcFbpISsViuLNqfyWvweCi5Z8KvmytR+rYmsV8PepYmIiJRLVd2cie4YysAOIaw/fJZ5ycdY+VM6yUfOknzkLN4eztTx8SDAy40AL3f8vdwI8HLDv9rl135ernpKuog4NAVvkRK4mG9m1LIf+XLnKQC6Nq7JpEfCqOGppeUiIiJ/lsFgoFMDXzo18OXk+TwWbjzOok2pnM0p5HzuBXaduHDdz9ao4mIL5MXC+ZXX1dxwd1E4FxH7UPAWuUE/nrxAzMJtHDubi8lo4J89GzP49npaWi4iIlIKgrzdealnE57r1pCD6dmcvpBP2oW8X37N59SFPNIu5HP6Qj4FlyyczSnkbE4he05lXbdPbw/nX2bJ3fD3cifwt8H8l6BexVU/HovIrae/WUT+B6vVyvwNx3nzq70UFlkI9HJj2qNtaBtS3d6liYiIVHiuTiZaBHnRIsjrmu9brVbO55ovB/KsX4P5bwP66fP55JmLOJ9r5nyumX1pF6/7/aq6OdmCeUA1t19n0b3dbTPoVV2dtPWZiJSIgrfIH8jKNzNi6S5W7E4DIKqpH28/0gpvDxc7VyYiIiJweXl69SouVK/iQrPAatdsY7Vaycq/9EsgzysWzE9n5XP6/OVzFwsucTH/EhfzszmQnn3d71nFxXTVTPlvl7kHeLnh5e6scC4iNgreItexM/U8MZ9sI/VcHs4mAyPubsqTnUL1j6iIiEg5YzAY8HJ3xsvdmcb+19+i7GK+mfSsy6G8+Mx5nu3chTwzOYVFHD6Tw+EzOdfty83ZaLu3/LfB3N/r15nzGlVc9HOFSCWh4C3yO1arlQ9+OEbc13sxF1mpXd2dGY+2ISzY296liYiISCmq6uZMVTdnGtS6fjjPLbw8c24L5VnFZ9FPX8jnXE4h+WYLRzNzOJp5/XDuYjLiXyyUu/2yvN39l9lzN3w9XfU8GZEKQMFb5DfO5xby0qe7WPlTOgB3NfdnwsOt8HJ3tnNlIiIi4gg8XJyoV9OTejU9r9sm31xkmzn//ax5WlY+p87nk5ldQGGRhZRzuaScy71uX05GA35/MGse4OVGTU9X7XMu4uAUvEV+sS3lZ/6xcDsnz+fhYjLyyr1Nefy2EC0BExERkRJxczYRUqMKITWqXLdN4SUL6VlXZsyLP7H9yq8ZF/O5ZLFy8nweJ8/nXbcvowFqVXUjwPvK/ubuvwvqbvhVc8NZ4VzEbhS8pdKzWKzMWXeEiQn7uWSxElLDgxmPtrnu01NFRERE/iwXJyPBPh4E+3hct425yMKZiwW/CeR5tgfCXVnunpaVT5HFStovIX77dfoyGMDX05UALzf8qrpSNc9Ax1wzNb20qk+kLCh4S6V2LqeQF/+7k9X7MgC4t1UAcQ+2pKqb/hESERER+3I2GQn0difQ2/26bYosVjKzC647a34663JYNxdZOXOxgDMXC375pImv/rWWns396dcumA71auhecpFSpOAtldbmY+f4x8LtpGXl4+Jk5LX7mtO/fbCWlouIiEi5YfrlHnC/am5wnQfBWixWzuUWcvr85VnzQxlZLPj+ACdzLXy58xRf7jxFsI87fdoG83BEbQK8rh/0ReTm6EYPsasZM2YQGhqKm5sbkZGRbNq06bptZ8+eze2330716tWpXr06UVFRV7XPzs4mJiaG2rVr4+7uTrNmzZg1a1axNgcPHiKscw9ua1aXTa/fR+7X/+L9Pg15NLKOQreIiIhUOEajAV9PV1rW9qJHc38Gd67LS62KWPb0bTx2Wx2qujqRei6PSSsP0Gn8agZ9sImEH9MwF1nsXbpIhaHgLXazePFiYmNjGTNmDNu2bSMsLIyePXuSkZFxzfZJSUn079+fNWvWkJycTHBwMD169ODkyZO2NrGxsSQkJDB//nz27t3L0KFDiYmJIT4+HoDj6edo06krB89k49f/LR4b+xGRIV688NSjWCz6x0VEREQqB4MBWgRVY2zvlmx6OYrJfcJoX9cHixXW7D/D0/O30iEukbgVezl8Jtve5YqUewreYjeTJ09m8ODBDBo0yDYz7eHhwdy5c6/ZfsGCBTzzzDOEh4fTpEkT5syZg8ViITEx0dZm/fr1REdH07VrV0JDQxkyZAhhYWFs2rSJ5MNn6TH8PbIzTxP0wAtM+fv9fPTiw8z/eB5btmxh9erVZXXpIiIiIg7D3cXEg21qs+T/OrD6hS483aU+vp6uZGYX8u53R+g2aS2PzFrPp1tPkFt4yd7lipRLCt5iF4WFhWzdupWoqCjbOaPRSFRUFMnJyTfUR25uLmazGR8fH9u5jh07Eh8fz8mTJ7FaraxZs4YDBw5wwacpA+Zs4OfsXAwGWBbThT4Rl+/ndnNzw2g0sm7dult+nSIiIiLlSb2anoy4uwnJI//Ce4+3pVuTWhgNsPnYz7z43520H5fIqGW72XXiPFar1d7lipQbCt5iF5mZmRQVFeHn51fsvJ+fH2lpaTfUx/DhwwkMDCwW3qdNm0azZs2oXbs2Li4u3HXXXbR4ZBhfZnhjsUKfu++kqqcncyaPJTc3l5ycHF588UWKioo4ffr0Lb1GERERkfLK2WSkR3N/3n+iHetHdOOlno2p4+NBdsElFm5M4f7pP3D3f77ngx+Ocj630N7lijg8BW8pl8aPH8+iRYtYtmwZbm5utvPTpk1jw4YNxMfHM/uzVfh1H8zGBW9jPbGLf/cNY/rf7uS///0vX375JZ6ennh5eXH+/HnatGmD0ag/DiIiIiK/5+/lxrN3NiDpxa4sHBzJA+GBuDgZ2Zd2kde//In2byXy3Cfb+eFQJhaLZsFFrkXbiYld+Pr6YjKZSE9PL3Y+PT0df3//P/zs22+/zfjx41m1ahWtWrWync/Ly2PUqFF8+ulS9rs0YkbSIYwt7iYw8xghp1bz19YjAejRoweHDx8mMzMTJycnvL298ff3p169erf+QkVEREQqCKPRQMf6vnSs78vruYV8seMUizansvd0FvE7TxGvbclErktTfGIXLi4utG3bttiD0a48KK1Dhw7X/dzEiRN58803SUhIICIioth7ZrMZs9nMv1YeZPqaQ1it8GhkHe5qGYib09XbhPn6+uLt7c3q1avJyMjg/vvvv3UXKCIiIlKBeXu4EN0xlBXPdebLmM4MiLx6W7InP9ysbclEfqEZb7Gb2NhYoqOjiYiIoH379kyZMoWcnBwGDRoEwMCBAwkKCiIuLg6ACRMmMHr0aBYuXEhoaKjtXnBPT088PT3Zejofz9BWbFz0H4J6PcuovnfgnLGVf8//mMmTJ9u+7wcffEDTpk2pWbMmycnJPP/88wwbNozGjRuX/X8EERERkXLMYDDQsrYXLWu35JV7mrFi92kWb0ll09FzrN6Xwep9Gfh6uvBQm9r0aRdM/Zqe9i5ZxC4UvMVu+vbty5kzZxg9ejRpaWmEh4eTkJBge+BaSkpKsfuuZ86cSWFhIQ8//HCxfl55dTQeHfrx7tojeN/zEmxZSN63U3huyRhCQkIYN24cTz/9tK39/v37GTlyJOfOnSM0NJSXX36ZYcOGlc1Fi4iIiFRQ7i4mHmpbm4fa1ubwmWyWbEll6daTZGYX8O53R3j3uyO0C61O33Z16NXSHw8XRRGpPDTaxa5iYmKIiYm55ntJSUnFXh87duyqNqfO5/GPT7azde0RAJ7sHs6oKf1xdTJd93uOHz+e8ePH33TNIiIiIvLH6tf0ZOTdTXmxR2NW78tgyeZU1uzPYPOxn9l87Gdei9/D/eGB9GsXTMsgLwyGq28LFKlIFLyl3DpyJpv+szeQnlVAVTcnJj7UirtbBti7LBERERH5hbPJSM/m/vRs7k/ahXw+3ZrKki0nSDmXy8KNKSzcmEIT/6r0axdM79ZBeHu42LtkkVKh4C3l0qGMbB6dvYGMiwU08vPk/eh2BPt42LssEREREbkOfy83Yv7SkGe6NmDDkbMs3pLK1z+msS/tIq99+RNvfb2Pu5r7069dMLfVq4HRqFlwqTgUvKXcOZRxkX7vbSQzu4Am/lVZ8FQkNTxd7V2WiIiIiNwAo9FAxwa+dGxweVuyz7efZNHmVPalXSy2LVnfiGAebhuMv5ebvUsW+dMUvKVcOZh+kf6zN5CZXUgT/6osHHwbPlW0JElERESkPPL2cOGJTnWJ7hjK7pMXWLw5lfgdp0g9l8fb3x5g8soDdG1ci77tgvlLk1o4m7QbspRPCt5SbuxPu8ijszdwNqeQZgHVWPBUJNUVukVERETKPYPBQKva3rSq7c3L9zRlxe40lmxOZdOx325L5spDbYK0LZmUS/pfRlIu7D2dRf9fQneLoGosHKzQLSIiIlIRebg48XDb2ix5ugOJL3Th/7rUw9fTxbYtWbdJa+kzK5lPt54gt/CSvcuVa5gxYwahoaG4ubkRGRnJpk2brtt29uzZ3H777VSvXp3q1asTFRXF5s2bi7XJzs4mJiaG2rVr4+7uTrNmzZg1a9Y1+7Nardx9990YDAY+//zzW3lZf4qCtzi8n05l8ejsDZzLKaRlkBcL/nabnngpIiIiUglc2ZYseWQ33n28LX9pUgujATYdO8eL/91J5LhERi3bza4T57FarfYuV4DFixcTGxvLmDFj2LZtG2FhYfTs2ZOMjIxrtk9KSqJ///6sWbOG5ORkgoOD6dWrF2fPnrW1iY2NJSEhgfnz57N3716GDh1KTEwM8fHxV/U3ZcoUh9yeTsFbHNqPJy/w6JwN/JxrJqy2F/OfisTLw9neZYmIiIhIGbqyLdncJ9qxfkQ3XuzRiGAfdy4WXGLhxhTun/4Dvaau48MfjnI+t9De5VZqkydPZvDgwQwaNMg2M+3h4cHcuXOv2X7BggU888wzhIeH06RJE+bMmYPFYmHXrl22NuvXryc6OpquXbsSGhrKkCFDCAsLu2omfceOHUyaNOm638ueFLzFYe0+cYEBczZyPtdMeLA3Hz8ViZe7QreIiIhIZXZlW7K1L97JwqciuT8sEBcnI3tPZ/Halz/R/q1EnvtkO+sPZWKxaBa8LBUWFrJ161aioqJs54xGI1FRUSQnJ99QH7m5uZjNZjw9f72Pv2PHjsTHx3Py5EmsVitr1qzhwIED9OjRo9jnHn30UWbMmIG/v/+tu6hbRA9XE4e068R5Hpuzkaz8S7Sp481HT7anqptCt4iIiIhc9tttyc5fZ1uyOj4e9ImorW3JykhmZiZFRUX4+fkVO+/n58e+fftuqI/hw4cTGBhIWFiY7dy0adMYMmQItWvXxsnJCaPRyOzZs7njjjtsbYYNG0bHjh154IEHbs3F3GIK3uJwdqSe5/H3N3Ix/xJtQ6rz4aB2Ct0iIiIicl2/35Zs0S/bkqWcy9W2ZOXI+PHjWbRoEStXruTEiRO289OmTWPDhg3Ex8cTEhLCd999x7PPPktgYCBRUVHEx8ezevVqtm/fbsfq/5iCtziUbSk/E/3+Ji4WXKJdaHU+GNQeT1cNUxERERH53367Ldkrf7QtWdsg+kYEU0/bkt1Svr6+mEwm0tPTi51PT0//n8u/3377bcaPH8+qVato1aqVLXjn5eUxatQoli1bxj333ANAq1at2LFjB2+//TZRUVGsXr2aw4cP4+3tXazPhx56iNtvv52kpKRbdo03S4lGHMbW4+eInruZ7IJLtK/rwwdPtKOKQreIiIiI3IQr25I93LY2h89ks2RLKku3nri8LdnaI7y79gjtQ33o2y6YXi0DcHcx2bvkcs/FxYW2bduSmJhI7969AbBYLCQmJhITE3Pdz02cOJFx48bxzTffEBERgdlstr1nNpsxm80YjcVXKZhMJiwWCwAjRozgqaeeKvZ+y5Yt+fe//8199913i67uz1GqEYew5dg5ouduIqewiNvq+TD3iXZ4uGh4ioiIiMifd2Vbshd7NCZxbwZLtqSStD+DTcfOsenYOV6L38P94YH0bRdMyyAvh9yOqryIjY0lOjqaiIgI2rdvz5QpU8jJyWHQoEEADBw4kKCgIOLi4gCYMGECo0ePZuHChYSGhpKWlobZbCYvLw+AatWq0aVLF1566SXc3d0JCQlh7dq1zJs3j8mTJwPg7+9/zRn1OnXqULdu3TK68j+mZCN2t+noOZ74YBO5hUV0rF+D96Pb6f84ioiIiMgt52wyclcLf+5q4c/pC3ks3XqCxVtSST2Xx4KNKSzYmELTgGr0axdM7/AgbWN7E/r27cuZM2cYPXo0aWlphIeHk5CQYHvgWkpKSrHZ65kzZ1JYWMjDDz98VT8PPfQQAIsWLWLkyJEMGDCAc+fOERISwrhx43j66afL7sL+JAVvsasNR87y5IebyS0sonMDX2YPjFDoFhEREZFSF+DlTsxfGvJM1wYkHznL4s2pJPyYxt7TWYyJ38O4FXu5u4U/fSOCua1eDYxGzYLfqJiYmOsuLf/9/dbHjh3DYrFScMlCnrmI3MJLXMwtYN3339na+Pv788EHH5SoBqvVsbaSU/AWu1l/OJO/fbiFPHMRtze8HLrdnBW6RURERKTsGI0GOjXwpdM1tiX7YscpvthRubcls1ovh+LcwiLyzEXkFRaRby4q9jrPfIm8wsvBOf+X8HzVa7OF/F8+k1t4iXyzpdjXvxfqaeKJsr/cUqPgLXbxw6FM/vbRZvLNFro0qsm7j7dV6BYRERERu/rttmS7Tlxg8RbH3pbsSijOswXay0H3SiD+/etiv175+nrnf/NrWXJxMuLhbMLVVFCm37e0KXhLmfv+4Bme+mgLBZcs3Nm4JjMfU+gWEREREcdhMBgIC/YmLPjXbckWb05h87Gfb3hbsiuh+Pezw799nf+bgPzr7PAvX/8mTP++rS1Mm4soyxXVLk5G3J1Nlw+X4r+6OZvw+O25K187m3BzMeFxjc9cqw+T0YDZbGbFihVld2FlQMFbytTaA2cYPG8LhZcsdGtSi3cea4Ork0K3iIiIiDim325Ldigjm/9uSWXptuLbkjXy88Rq5arwXKah2GTEzdn4mzDrhLvttdMvvxp/854Jdxfjr19feW1rWzxEuzkZcbLzDH95puAtZSZpfwZDPt5K4SULUU39mDGgtUK3iIiIiJQbDWp5MrJXU17seXlbssWbU1h74AwH0rP/8HPOJgNuvwRZD5fLM7tXwq3t9TVmin/f9re/ejg74eZitM0U23vZu/wxBW8pE6v3pfP0x9soLLLQs7kf0/q3wcVJfzmIiIiISPnz+23J9p7Ows3p8pJq998sub7yWqFYFLyl1K36KZ2/L9iKucjK3S38mdq/tf7yEREREZEKIcDLnQAvd3uXIQ5O6UdK1bd70myh+56WAQrdIiIiIiJS6WjGW0pNwo9pxCzcxiWLlXtbBTClb7geyCAiIiIiIpWOgreUiq93n+Yfn2znksXK/WGBTO4TptAtIiIiIiKVkpKQ3HLLd50m5pfQ3TtcoVtERERERCq3m0pDM2bMIDQ0FDc3NyIjI9m0adN1286ePZvbb7+d6tWrU716daKioq5qn52dTUxMDLVr18bd3Z1mzZoxa9asmylN7OzLnad4btF2iixWHmwTxKQ+Wl4uIiIiIiKVW4kT0eLFi4mNjWXMmDFs27aNsLAwevbsSUZGxjXbJyUl0b9/f9asWUNycjLBwcH06NGDkydP2trExsaSkJDA/Pnz2bt3L0OHDiUmJob4+PibvzIpc1/sOMnzv4Tuh9vW5l8Ph2EyGuxdloiIiIiIiF2VOHhPnjyZwYMHM2jQINvMtIeHB3Pnzr1m+wULFvDMM88QHh5OkyZNmDNnDhaLhcTERFub9evXEx0dTdeuXQkNDWXIkCGEhYX94Uy6OJZl208wbPEOLFboGxHMxIdaKXSLiIiIiIhQwuBdWFjI1q1biYqK+rUDo5GoqCiSk5NvqI/c3FzMZjM+Pj62cx07diQ+Pp6TJ09itVpZs2YNBw4coEePHiUpT+xk6dYTxC7ZicUK/dsHE/dgS4wK3SIiIiIiIkAJn2qemZlJUVERfn5+xc77+fmxb9++G+pj+PDhBAYGFgvv06ZNY8iQIdSuXRsnJyeMRiOzZ8/mjjvuuGYfBQUFFBQU2F5nZWUBYDabMZvNJbmkMnelPkev80Z9uu0koz7fg9UK/dvV5rV7mlBUdImiIntXJjeroo1RqXg0RsXRaYyKo9MYFUdXXsZoSeor0+3Exo8fz6JFi0hKSsLNzc12ftq0aWzYsIH4+HhCQkL47rvvePbZZ68K6FfExcXx+uuvX3X+22+/xcPDo1Sv4VZZuXKlvUv405LTDSw+YsSKgc5+FiJNx0hIOGbvsuQWqQhjVCo2jVFxdBqj4ug0RsXROfoYzc3NveG2BqvVar3RxoWFhXh4ePDpp5/Su3dv2/no6GjOnz/PF198cd3Pvv3224wdO5ZVq1YRERFhO5+Xl4eXlxfLli3jnnvusZ1/6qmnOHHiBAkJCVf1da0Z7+DgYDIzM6lWrdqNXo5dmM1mVq5cSffu3XF2drZ3OTdt8ZYTvPLFTwA8HhnMq/c0wWDQ8vKKoKKMUam4NEbF0WmMiqPTGBVHV17GaFZWFr6+vly4cOF/5tASzXi7uLjQtm1bEhMTbcH7yoPSYmJirvu5iRMnMm7cOL755ptioRt+XR5uNBa/3dxkMmGxWK7Zn6urK66urledd3Z2dujfmN8qT7X+3vwNx22he1CnUEbf20yhuwIqz2NUKgeNUXF0GqPi6DRGxdE5+hgtSW0lXmoeGxtLdHQ0ERERtG/fnilTppCTk8OgQYMAGDhwIEFBQcTFxQEwYcIERo8ezcKFCwkNDSUtLQ0AT09PPD09qVatGl26dOGll17C3d2dkJAQ1q5dy7x585g8eXJJy5NS9nHyMV79Yg8Af+tcl1fuaarQLSIiIiIi8gdKHLz79u3LmTNnGD16NGlpaYSHh5OQkGB74FpKSkqx2euZM2dSWFjIww8/XKyfMWPG8NprrwGwaNEiRo4cyYABAzh37hwhISGMGzeOp59++k9cmtxqH60/xpj4y6F7yB31GHm3lpeLiIiIiIj8Lzf1cLWYmJjrLi1PSkoq9vrYsWP/sz9/f38++OCDmylFysjcdUd546vLy8uf7lKf4Xc1VugWERERERG5AWX6VHMpn+Z8f4Sxy/cC8EzX+rzUU6FbRERERETkRil4yx9677vDvLXi8h7t//hLA2K7N1LoFhERERERKQEFb7mumUmHmZBwOXQ/360hQ6MaKnSLiIiIiIiUkIK3XNOMNYf41zf7ARgW1YjnoxrauSIREREREZHyScFbrjIt8SCTVh4A4IXujfhHN4VuERERERGRm6XgLcVMWXWAKasOAvBSz8Y8e2cDO1ckIiIiIiJSvil4CwBWq5V/rzrI1MTLoXvE3U14ukt9O1clIiIiIiJS/il4C1arlckrDzBt9SEARvVqwpA7FLpFRERERERuBQXvSs5qtfKvb/bzTtJhAF65pylP3V7PzlWJiIiIiIhUHArelZjVamV8wj7eXXsEgNH3NuPJznXtXJWIiIiIiEjFouBdSVmtVuK+3sd7310O3a/f35zojqH2LUpERERERKQCUvCuhKxWK2OX7+X9dUcBePOB5jzeIdS+RYmIiIiIiFRQCt6VjNVq5Y2vfuKDH44BMO6vLRgQGWLfokRERERERCowBe9KxGq18lr8Hj5KPg5A3IMt6d++jp2rEhERERERqdgUvCsJi8XKmPg9fLzhOAYDjH+wJX3bKXSLiIiIiIiUNgXvSsBisfLKFz+ycGMKBgNMfKgVj0QE27ssERERERGRSkHBu4KzWKyMWrabRZtTMRjg7YfDeKhtbXuXJSIiIiIiUmkoeFdgFouVEZ/tYsmWExgNMKlPGH9trdAtIiIiIiJSlhS8K6gii5XhS3fx6dbLofvffcN5IDzI3mWJiIiIiIhUOgreFVCRxcpL/93JZ9tPYjIamNI3nPvCAu1dloiIiIiISKWk4F3BFFmsvLBkB5/vOIXJaGBqv9bc0yrA3mWJiIiIiIhUWgreFcilIguxS3YSv/MUTkYD0/q35u6WCt0iIiIiIiL2pOBdQVwqsjB08Q6+2nUaJ6OB6Y+24a4W/vYuS0REREREpNJT8K4AzEUWhi7awfLdp3E2GZjxaBt6NFfoFhERERERcQQK3uWcucjCc59s5+sf03AxGXlnQBuimvnZuywRERERERH5hYJ3OVZ4yULMwm18+1M6LiYj7z7eljub1LJ3WSIiIiIiIvIbCt7lVOElC88s2Maqvem4OBl57/G2dG2s0C0iIiIiIuJoFLzLoYJLRTwzfxuJ+zJwdTLy3sAIujSqae+yRERERERE5BoUvMuZfHMRf5+/lTX7z+DqZOT96HZ0buhr77JERERERETkOhS8y5F8cxH/9/FW1h44g5uzkbnR7ejYQKFbRERERETEkSl4lxP55iIGz9vC9wczcXc2MfeJdnSoX8PeZYmIiIiIiMj/oOBdDuQVXg7d6w5l4uFyOXTfVk+hW0REREREpDxQ8HZwuYWXeOqjLaw/fJYqLiY+GNSe9nV97F2WiIiIiIiI3CAFbweWW3iJJz/czIYj56jiYuKjJ9sTEarQLSIiIiIiUp4oeDuonIJLDPpwM5uOnsPT1YmPnmxH2xCFbhERERERkfLGaO8CypsZM2YQGhqKm5sbkZGRbNq06bptZ8+eze2330716tWpXr06UVFRbN68+ap2e/fu5f7778fLy4sqVarQNiKCPpPi2XT0HFVdnXD79k0iQmtgMBhsx9NPP12alykiIiIiIiK3iIJ3CSxevJjY2FjGjBnDtm3bCAsLo2fPnmRkZFyzfVJSEv3792fNmjUkJycTHBxMr169OHv2rK3N4cOH6dy5M02aNCEpKYn1m7fi0u4Rdp3Oo6qbEx8/FYmnqxODBw/m9OnTtmPixIllddkiIiIiIiLyJ2ipeQlMnjyZwYMHM2jQIABmzZrF8uXLmTt3LiNGjLiq/YIFC4q9njNnDkuXLmXXrl22cy+//DK9evVi4sSJZOWbeWLuJk57taC6mxPzn4qkVW1vADw8PPD39y+9ixMREREREZFSoRnvG1RYWMjWrVuJioqynTMajURFRZGcnHxDfeTm5mI2m/H09ATAYrGwfPlyGjVqRLfuPahZsxbLxz4JxzazcPBtttANl0O8r68vLVq0YOTIkeTm5t7S6xMREREREZHSoRnvG5SZmUlRURF+fn7Fzvv5+bFv374b6mP48OEEBgYSFhYGQEZGBtnZ2YwfP566PZ+kxsP3YT2xg5Qlb3D273dCUBcAHn30UUJCQggMDGTXrl0MHz6c/fv389lnn93aixQREREREZFbTsG7jIwfP55FixaxcuVKTpw4AVye8Qao3rQj2Y3uws/DmQXjohnx9BlmzZpFly6Xg/eQIUNs/bRs2ZKAgAC6devG4cOHqV+/ftlfjIiIiIiIiNwwLTW/Qb6+vphMJtLT04udT09P/5/3Xr/99tuMHz+eb7/9llatWtnOO3lUw2A0ke3hj08VFxYOvo1mgdVo2rQpKSkp1+0vMjISgEOHDv2JKxIREREREZGyoOB9g1xcXGjbti2JiYm2cxaLhcTERDp06HDdz02cOJE333yThIQEIiIibOd/zi3kiY+24+LfEFPWaT4ZfBtNA6oBcODAAUJCQq7b544dOwAICAj4k1clIiIiIiIipU1LzUsgNjaW6OhoIiIiaN++PVOmTCEnJ8f2lPOBAwcSFBREXFwcABMmTGD06NEsXLiQ0NBQ0tLSMJvNZGblMXDuFvalZxPUpS+pn77Fd18uwnTnnSQkJPDll1+SlJQEXN5ubOHChfTq1YsaNWqwa9cuhg0bxh133FFs9lxEREREREQck4J3CfTt25czZ84wevRo0tLSCA8PJyEhwfbAtZSUFIzGXxcRzJw5k8LCQh5++OFi/QR17Y9T5ABqVnXlkykv8F1UHeLi4njuuedo3LgxS5cupXPnzsDlmfZVq1bZQn5wcDAPPfQQr7zyStlduIiIiIiIiNw0Be8SiomJISYm5prvXZmlvuLYsWPFXp+5WMCjs5M5mJFDraqufDLkNurX9KTBk0/y5JNPXrPP4OBg1q5deytKFxERERERETvQPd5laMmWVA5m5ODlbGX+kxHUr+lp75JERERERESklGnGuwz9vUt9svML8blwkLq+VexdjoiIiIiIiJQBzXiXIaPRQGxUQ2q527sSERERERERKSsK3iIiIiIiIiKlSMFbREREREREpBQpeIuIiIiIiIiUIgVvERERERERkVKk4C0iIiIiIiJSihS8RUREREREREqRgreIiIiIiIhIKVLwFhERERERESlFNxW8Z8yYQWhoKG5ubkRGRrJp06brtp09eza333471atXp3r16kRFRV2z/d69e7n//vvx8vKiSpUqtGvXjpSUlJspT0RERERERMRhlDh4L168mNjYWMaMGcO2bdsICwujZ8+eZGRkXLN9UlIS/fv3Z82aNSQnJxMcHEyPHj04efKkrc3hw4fp3LkzTZo0ISkpiV27dvHqq6/i5uZ281cmIiIiIiIi4gCcSvqByZMnM3jwYAYNGgTArFmzWL58OXPnzmXEiBFXtV+wYEGx13PmzGHp0qUkJiYycOBAAF5++WV69erFxIkTbe3q169f0tJEREREREREHE6JZrwLCwvZunUrUVFRv3ZgNBIVFUVycvIN9ZGbm4vZbMbHxwcAi8XC8uXLadSoET179qRWrVpERkby+eefl6Q0EREREREREYdUohnvzMxMioqK8PPzK3bez8+Pffv23VAfw4cPJzAw0BbeMzIyyM7OZvz48YwdO5YJEyaQkJDAgw8+yJo1a+jSpctVfRQUFFBQUGB7nZWVBYDZbMZsNpfkksrclfocvU6pvDRGxdFpjIqj0xgVR6cxKo6uvIzRktRX4qXmf8b48eNZtGgRSUlJtvu3LRYLAA888ADDhg0DIDw8nPXr1zNr1qxrBu+4uDhef/31q85/++23eHh4lOIV3DorV660dwkif0hjVBydxqg4Oo1RcXQao+LoHH2M5ubm3nDbEgVvX19fTCYT6enpxc6np6fj7+//h599++23GT9+PKtWraJVq1bF+nRycqJZs2bF2jdt2pR169Zds6+RI0cSGxtre52VlWV7aFu1atVKckllzmw2s3LlSrp3746zs7O9yxG5isaoODqNUXF0GqPi6DRGxdGVlzF6ZeX1jShR8HZxcaFt27YkJibSu3dv4PKMdWJiIjExMdf93MSJExk3bhzffPMNERERV/XZrl079u/fX+z8gQMHCAkJuWZ/rq6uuLq62l5brVYA8vLyHPo3Bi4PotzcXPLy8rh06ZK9yxG5isaoODqNUXF0GqPi6DRGxdGVlzGal5cH/JpH/5C1hBYtWmR1dXW1fvjhh9affvrJOmTIEKu3t7c1LS3NarVarY8//rh1xIgRtvbjx4+3uri4WD/99FPr6dOnbcfFixdtbT777DOrs7Oz9b333rMePHjQOm3aNKvJZLJ+//33N1RTamqqFdChQ4cOHTp06NChQ4cOHTrK9EhNTf2fmdVgtd5IPC9u+vTp/Otf/yItLY3w8HCmTp1KZGQkAF27diU0NJQPP/wQgNDQUI4fP35VH2PGjOG1116zvZ47dy5xcXGcOHGCxo0b8/rrr/PAAw/cUD0Wi4VTp05RtWpVDAZDSS+nTF1ZFp+amurwy+KlctIYFUenMSqOTmNUHJ3GqDi68jJGrVYrFy9eJDAwEKPxjzcMu6ngLTcvKysLLy8vLly44NCDSCovjVFxdBqj4ug0RsXRaYyKo6uIY7RE+3iLiIiIiIiISMkoeIuIiIiIiIiUIgXvMubq6sqYMWOKPZVdxJFojIqj0xgVR6cxKo5OY1QcXUUco7rHW0RERERERKQUacZbREREREREpBQpeIuIiIiIiIiUIgVvERERERERkVKk4F2GZsyYQWhoKG5ubkRGRrJp0yZ7lyQCQFxcHO3ataNq1arUqlWL3r17s3//fnuXJXJd48ePx2AwMHToUHuXImJz8uRJHnvsMWrUqIG7uzstW7Zky5Yt9i5LBICioiJeffVV6tati7u7O/Xr1+fNN99Ej3sSe/nuu++47777CAwMxGAw8Pnnnxd732q1Mnr0aAICAnB3dycqKoqDBw/ap9hbQMG7jCxevJjY2FjGjBnDtm3bCAsLo2fPnmRkZNi7NBHWrl3Ls88+y4YNG1i5ciVms5kePXqQk5Nj79JErrJ582beffddWrVqZe9SRGx+/vlnOnXqhLOzM19//TU//fQTkyZNonr16vYuTQSACRMmMHPmTKZPn87evXuZMGECEydOZNq0afYuTSqpnJwcwsLCmDFjxjXfnzhxIlOnTmXWrFls3LiRKlWq0LNnT/Lz88u40ltDTzUvI5GRkbRr147p06cDYLFYCA4O5h//+AcjRoywc3UixZ05c4ZatWqxdu1a7rjjDnuXI2KTnZ1NmzZteOeddxg7dizh4eFMmTLF3mWJMGLECH744Qe+//57e5cick333nsvfn5+vP/++7ZzDz30EO7u7syfP9+OlYmAwWBg2bJl9O7dG7g82x0YGMgLL7zAiy++CMCFCxfw8/Pjww8/pF+/fnas9uZoxrsMFBYWsnXrVqKiomznjEYjUVFRJCcn27EykWu7cOECAD4+PnauRKS4Z599lnvuuafY36cijiA+Pp6IiAgeeeQRatWqRevWrZk9e7a9yxKx6dixI4mJiRw4cACAnTt3sm7dOu6++247VyZytaNHj5KWllbs33svLy8iIyPLbX5ysncBlUFmZiZFRUX4+fkVO+/n58e+ffvsVJXItVksFoYOHUqnTp1o0aKFvcsRsVm0aBHbtm1j8+bN9i5F5CpHjhxh5syZxMbGMmrUKDZv3sxzzz2Hi4sL0dHR9i5PhBEjRpCVlUWTJk0wmUwUFRUxbtw4BgwYYO/SRK6SlpYGcM38dOW98kbBW0SKefbZZ/nxxx9Zt26dvUsRsUlNTeX5559n5cqVuLm52bsckatYLBYiIiJ46623AGjdujU//vgjs2bNUvAWh7BkyRIWLFjAwoULad68OTt27GDo0KEEBgZqjIqUAS01LwO+vr6YTCbS09OLnU9PT8ff399OVYlcLSYmhq+++oo1a9ZQu3Zte5cjYrN161YyMjJo06YNTk5OODk5sXbtWqZOnYqTkxNFRUX2LlEquYCAAJo1a1bsXNOmTUlJSbFTRSLFvfTSS4wYMYJ+/frRsmVLHn/8cYYNG0ZcXJy9SxO5ypWMVJHyk4J3GXBxcaFt27YkJibazlksFhITE+nQoYMdKxO5zGq1EhMTw7Jly1i9ejV169a1d0kixXTr1o3du3ezY8cO2xEREcGAAQPYsWMHJpPJ3iVKJdepU6ertmE8cOAAISEhdqpIpLjc3FyMxuI/+ptMJiwWi50qErm+unXr4u/vXyw/ZWVlsXHjxnKbn7TUvIzExsYSHR1NREQE7du3Z8qUKeTk5DBo0CB7lybCs88+y8KFC/niiy+oWrWq7d4ZLy8v3N3d7VydCFStWvWqZw5UqVKFGjVq6FkE4hCGDRtGx44deeutt+jTpw+bNm3ivffe47333rN3aSIA3HfffYwbN446derQvHlztm/fzuTJk3nyySftXZpUUtnZ2Rw6dMj2+ujRo+zYsQMfHx/q1KnD0KFDGTt2LA0bNqRu3bq8+uqrBAYG2p58Xt5oO7EyNH36dP71r3+RlpZGeHg4U6dOJTIy0t5liWAwGK55/oMPPuCJJ54o22JEblDXrl21nZg4lK+++oqRI0dy8OBB6tatS2xsLIMHD7Z3WSIAXLx4kVdffZVly5aRkZFBYGAg/fv3Z/To0bi4uNi7PKmEkpKSuPPOO686Hx0dzYcffojVamXMmDG89957nD9/ns6dO/POO+/QqFEjO1T75yl4i4iIiIiIiJQi3eMtIiIiIiIiUooUvEVERERERERKkYK3iIiIiIiISClS8BYREREREREpRQreIiIiIiIiIqVIwVtERERERESkFCl4i4iIiIiIiJQiBW8RERERERGRUqTgLSIiIiWWlJSEwWDg/Pnz9i5FRETE4Sl4i4iIiIiIiJQiBW8RERERERGRUqTgLSIiUg5ZLBbi4uKoW7cu7u7uhIWF8emnnwK/LgNfvnw5rVq1ws3Njdtuu40ff/yxWB9Lly6lefPmuLq6EhoayqRJk4q9X1BQwPDhwwkODsbV1ZUGDRrw/vvvF2uzdetWIiIi8PDwoGPHjuzfv790L1xERKQcUvAWEREph+Li4pg3bx6zZs1iz549DBs2jMcee4y1a9fa2rz00ktMmjSJzZs3U7NmTe677z7MZjNwOTD36dOHfv36sXv3bl577TVeffVVPvzwQ9vnBw4cyCeffMLUqVPZu3cv7777Lp6ensXqePnll5k0aRJbtmzBycmJJ598skyuX0REpDwxWK1Wq72LEBERkRtXUFCAj48Pq1atokOHDrbzTz31FLm5uQwZMoQ777yTRYsW0bdvXwDOnTtH7dq1+fDDD+nTpw8DBgzgzJkzfPvtt7bP//Of/2T58uXs2bOHAwcO0LhxY1auXElUVNRVNSQlJXHnnXeyatUqunXrBsCKFSu45557yMvLw83NrZT/K4iIiJQfmvEWEREpZw4dOkRubi7du3fH09PTdsybN4/Dhw/b2v02lPv4+NC4cWP27t0LwN69e+nUqVOxfjt16sTBgwcpKipix44dmEwmunTp8oe1tGrVyvZ1QEAAABkZGX/6GkVERCoSJ3sXICIiIiWTnZ0NwPLlywkKCir2nqura7HwfbPc3d1vqJ2zs7Pta4PBAFy+/1xERER+pRlvERGRcqZZs2a4urqSkpJCgwYNih3BwcG2dhs2bLB9/fPPP3PgwAGaNm0KQNOmTfnhhx+K9fvDDz/QqFEjTCYTLVu2xGKxFLtnXERERG6OZrxFRETKmapVq/Liiy8ybNgwLBYLnTt35sKFC/zwww9Uq1aNkJAQAN544w1q1KiBn58fL7/8Mr6+vvTu3RuAF154gXbt2vHmm2/St29fkpOTmT59Ou+88w4AoaGhREdH8+STTzJ16lTCwsI4fvw4GRkZ9OnTx16XLiIiUi4peIuIiJRDb775JjVr1iQuLo4jR47g7e1NmzZtGDVqlG2p9/jx43n++ec5ePAg4eHhfPnll7i4uADQpk0blixZwujRo3nzzTcJCAjgjTfe4IknnrB9j5kzZzJq1CieeeYZzp49S506dRg1apQ9LldERKRc01PNRUREKpgrTxz/+eef8fb2tnc5IiIilZ7u8RYREREREREpRQreIiIiIiIiIqVIS81FRERERERESpFmvEVERERERERKkYK3iIiIiIiISClS8BYREREREREpRQreIiIiIiIiIqVIwVtERERERESkFCl4i4iIiIiIiJQiBW8RERERERGRUqTgLSIiIiIiIlKKFLxFREREREREStH/AyrI2QDjcgZBAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "from numpy import argmax\n", + "\n", + "# Assuming perf_df.set_index(\"epoch\").plot() generates a line plot\n", + "ax = perf_df.set_index(\"epoch\").plot(\n", + " figsize=(10, 5), title=\"METEOR score vs epoch\", grid=True\n", + ")\n", + "\n", + "# Loop through each line to annotate the last point\n", + "for line in ax.lines:\n", + " # Get the data\n", + " xdata, ydata = line.get_data()\n", + " for index in [0, 1, argmax(ydata), -1]:\n", + " ax.annotate(\n", + " f\"{ydata[index]:.3f}\",\n", + " xy=(xdata[index], ydata[index]),\n", + " textcoords=\"offset points\",\n", + " xytext=(0, 1),\n", + " ha=\"center\",\n", + " )\n", + "\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Experiment 5 - Llama Factory: Performance/Repetition vs Epochs" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chineseenglishQwen/Qwen2-0.5B-InstructQwen/Qwen2-0.5B-Instruct_checkpoint-560Qwen/Qwen2-0.5B-Instruct_checkpoint-1120Qwen/Qwen2-0.5B-Instruct_checkpoint-1680Qwen/Qwen2-0.5B-Instruct_checkpoint-2240Qwen/Qwen2-0.5B-Instruct_checkpoint-2800Qwen/Qwen2-0.5B-Instruct_checkpoint-3360Qwen/Qwen2-1.5B-InstructQwen/Qwen2-1.5B-Instruct_checkpoint-560Qwen/Qwen2-1.5B-Instruct_checkpoint-1120Qwen/Qwen2-1.5B-Instruct_checkpoint-1680Qwen/Qwen2-1.5B-Instruct_checkpoint-2240Qwen/Qwen2-1.5B-Instruct_checkpoint-2800Qwen/Qwen2-1.5B-Instruct_checkpoint-3360unsloth/qwen2-7b-instruct-bnb-4bit
0老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞...Old Geng picked up his shotgun, squinted, and ...Old Ge lifted his gun and lowered his eyes as ...Old Goong cocked his gun and lowered his eyes ...Old Geng held his gun up, his eyes narrowed. T...Old Geng took his pistol from his holster and ...Old Geng raised his rifle, narrowed his eyes, ...Old Geng raised his rifle,眯着眼睛，the trigger cli...Old Geng held his gun at someone's head, his e...Old耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝...Old Geng took up his rifle and squinted throug...Old Geng raised his gun, squinted at it throug...Old Geng took a step forward with his gun rais...Old Geng took up his weapon with a squinted lo...Old Geng took a step forward, raised his pisto...Old Geng took a step forward, lifted his pisto...Old Geng raised his gun, squinted one of his t...
1次日天未明时，刘姥姥便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，...Next day Grannie Liu was up before dawn. As so...The next day, when the sun was still very earl...It was still night when she rose to dress up; ...It was still dark before her eyes when she got...When she awoke again from her nap, she was alr...By dawn's hour again the next day, Grannie Liu...By day's dawn she was out of bed, having alrea...At about three o'clock in the morning the next...The next morning, when it was still dark, Liu ...Misty rose at dawn the next morning, having dr...When it was just dawn outside her room, Granni...By daybreak she got up, dressed herself, and w...When she got up the next morning before daybre...When she arose from bed at midnight, she had a...At daybreak the old woman woke up from her slu...The next morning, before daybreak, Mrs. Liu ro...
\n", + "
" + ], + "text/plain": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "1 次日天未明时，刘姥姥便起来梳洗了， 又将板儿教了几句话； 五六岁的孩子，听见带了他进城逛去，... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "1 Next day Grannie Liu was up before dawn. As so... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct \\\n", + "0 Old Ge lifted his gun and lowered his eyes as ... \n", + "1 The next day, when the sun was still very earl... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-560 \\\n", + "0 Old Goong cocked his gun and lowered his eyes ... \n", + "1 It was still night when she rose to dress up; ... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-1120 \\\n", + "0 Old Geng held his gun up, his eyes narrowed. T... \n", + "1 It was still dark before her eyes when she got... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-1680 \\\n", + "0 Old Geng took his pistol from his holster and ... \n", + "1 When she awoke again from her nap, she was alr... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-2240 \\\n", + "0 Old Geng raised his rifle, narrowed his eyes, ... \n", + "1 By dawn's hour again the next day, Grannie Liu... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-2800 \\\n", + "0 Old Geng raised his rifle,眯着眼睛，the trigger cli... \n", + "1 By day's dawn she was out of bed, having alrea... \n", + "\n", + " Qwen/Qwen2-0.5B-Instruct_checkpoint-3360 \\\n", + "0 Old Geng held his gun at someone's head, his e... \n", + "1 At about three o'clock in the morning the next... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct \\\n", + "0 Old耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝... \n", + "1 The next morning, when it was still dark, Liu ... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-560 \\\n", + "0 Old Geng took up his rifle and squinted throug... \n", + "1 Misty rose at dawn the next morning, having dr... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-1120 \\\n", + "0 Old Geng raised his gun, squinted at it throug... \n", + "1 When it was just dawn outside her room, Granni... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-1680 \\\n", + "0 Old Geng took a step forward with his gun rais... \n", + "1 By daybreak she got up, dressed herself, and w... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-2240 \\\n", + "0 Old Geng took up his weapon with a squinted lo... \n", + "1 When she got up the next morning before daybre... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-2800 \\\n", + "0 Old Geng took a step forward, raised his pisto... \n", + "1 When she arose from bed at midnight, she had a... \n", + "\n", + " Qwen/Qwen2-1.5B-Instruct_checkpoint-3360 \\\n", + "0 Old Geng took a step forward, lifted his pisto... \n", + "1 At daybreak the old woman woke up from her slu... \n", + "\n", + " unsloth/qwen2-7b-instruct-bnb-4bit \n", + "0 Old Geng raised his gun, squinted one of his t... \n", + "1 The next morning, before daybreak, Mrs. Liu ro... " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "df = pd.read_csv(\"results/mac-results_lf-r2.csv\")\n", + "df.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['chinese',\n", + " 'english',\n", + " 'Qwen/Qwen2-0.5B-Instruct',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-560',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-1120',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-1680',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-2240',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-2800',\n", + " 'Qwen/Qwen2-0.5B-Instruct_checkpoint-3360',\n", + " 'Qwen/Qwen2-1.5B-Instruct',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-560',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-1120',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-1680',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-2240',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-2800',\n", + " 'Qwen/Qwen2-1.5B-Instruct_checkpoint-3360',\n", + " 'unsloth/qwen2-7b-instruct-bnb-4bit']" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.columns.to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading: /Users/inflaton/code/engd/papers/rapget-finetuning/eval_modules/calc_repetitions.py\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package wordnet to\n", + "[nltk_data] /Users/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /Users/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to\n", + "[nltk_data] /Users/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n" + ] + } + ], + "source": [ + "from eval_modules.calc_repetitions import *\n", + "\n", + "import re # Import the re module for regex operations\n", + "\n", + "def calc_metrics_for_epochs(df, start_col=2, end_col=-3, alpha=1):\n", + " dict = {\n", + " \"epoch\": [],\n", + " }\n", + "\n", + " columns = df.columns[start_col:end_col]\n", + " # print(columns)\n", + "\n", + " for col in columns:\n", + " # Split using regex\n", + " parts = re.split(r\"[/|_]\", col) # Use re.split() to split by regex\n", + " model_name = parts[1]\n", + " if len(parts) == 3:\n", + " checkpoint = parts[2]\n", + " epoch = int(checkpoint.split(\"-\")[1]) // 560\n", + " else:\n", + " epoch = 0\n", + " checkpoint = \"base\"\n", + "\n", + " dict[\"epoch\"].append(epoch)\n", + "\n", + " if model_name not in dict:\n", + " dict[model_name] = []\n", + " dict[model_name + \"(RAP)\"] = []\n", + "\n", + " metrics = calc_metrics(df[\"english\"], df[col])\n", + " dict[model_name].append(metrics[\"meteor\"])\n", + " print(\"*****\", model_name, checkpoint, epoch)\n", + "\n", + " df[[\"ews_score\", \"repetition_score\", \"total_repetitions\"]] = df[col].apply(\n", + " detect_scores\n", + " )\n", + " print(\"ews_score:\", df[\"ews_score\"].mean())\n", + " print(\"repetition_score:\", df[\"repetition_score\"].mean())\n", + " print(\"total_repetitions:\", df[\"total_repetitions\"].mean())\n", + "\n", + " # find the record with the highest total_repetitions\n", + " print(\"highest total_repetitions:\", df[\"total_repetitions\"].max())\n", + "\n", + " index = df[\"total_repetitions\"].idxmax()\n", + " print(\"\\t@\", index, \":\", df[col][index])\n", + "\n", + " rap = dict[model_name][-1] / math.log10(10 + alpha * df[\"total_repetitions\"].mean())\n", + " dict[model_name + \"(RAP)\"].append(rap)\n", + "\n", + " print(\"meteor:\", dict[model_name][-1])\n", + " print(\"rap:\", rap)\n", + "\n", + " dict[\"epoch\"] = dict[\"epoch\"][: len(dict[model_name])]\n", + " # print(dict)\n", + " return pd.DataFrame(dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "***** Qwen2-0.5B-Instruct base 0\n", + "ews_score: 0.0\n", + "repetition_score: 8.962047661076788\n", + "total_repetitions: 8.962047661076788\n", + "highest total_repetitions: 10070\n", + "\t@ 1079 : Peter said that he would not have dared to disturb him if it was not for his birthday on the 2nd of January; because they had been looking for him in various places; so he is thin and long, with a thick and long stalk, and fresh, sweet, juicy, yellow, and white flowers; large, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big, big\n", + "meteor: 0.26081317536040693\n", + "rap: 0.20409749518480566\n", + "***** Qwen2-0.5B-Instruct checkpoint-560 1\n", + "ews_score: 0.0\n", + "repetition_score: 15.847308031774052\n", + "total_repetitions: 15.847308031774052\n", + "highest total_repetitions: 17851\n", + "\t@ 327 : A little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter, a little longer, a little shorter,\n", + "meteor: 0.29049497377443884\n", + "rap: 0.2056724887994191\n", + "***** Qwen2-0.5B-Instruct checkpoint-1120 2\n", + "ews_score: 0.0\n", + "repetition_score: 0.07413945278022947\n", + "total_repetitions: 0.07413945278022947\n", + "highest total_repetitions: 27\n", + "\t@ 193 : Have... No... Have... No...'\n", + "meteor: 0.31049607994218004\n", + "rap: 0.309503206546529\n", + "***** Qwen2-0.5B-Instruct checkpoint-1680 3\n", + "ews_score: 0.0\n", + "repetition_score: 0.21359223300970873\n", + "total_repetitions: 0.21359223300970873\n", + "highest total_repetitions: 67\n", + "\t@ 1129 : Old man asked, kneeling on the ground: 'Give me mercy, eight-grade Devil! Give me mercy, eight-grade Devil!'\n", + "meteor: 0.3229970135854747\n", + "rap: 0.3200593440591159\n", + "***** Qwen2-0.5B-Instruct checkpoint-2240 4\n", + "ews_score: 0.0\n", + "repetition_score: 0.13062665489849956\n", + "total_repetitions: 0.13062665489849956\n", + "highest total_repetitions: 51\n", + "\t@ 798 : Oh yes! What's that? What's that?' said Pocky Cheng to them when they came up to him. 'What's that? What's that?'\n", + "meteor: 0.3162958975982396\n", + "rap: 0.3145231474647379\n", + "***** Qwen2-0.5B-Instruct checkpoint-2800 5\n", + "ews_score: 0.0\n", + "repetition_score: 12.927625772285966\n", + "total_repetitions: 12.927625772285966\n", + "highest total_repetitions: 14322\n", + "\t@ 160 : Boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom - boom\n", + "meteor: 0.31283536857877553\n", + "rap: 0.229965287947366\n", + "***** Qwen2-0.5B-Instruct checkpoint-3360 6\n", + "ews_score: 0.0\n", + "repetition_score: 7.321270962047661\n", + "total_repetitions: 7.321270962047661\n", + "highest total_repetitions: 8180\n", + "\t@ 809 : I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I'm . . . I\n", + "meteor: 0.31152790941615477\n", + "rap: 0.2515202658684176\n", + "***** Qwen2-1.5B-Instruct base 0\n", + "ews_score: 0.0\n", + "repetition_score: 0.12533097969991175\n", + "total_repetitions: 0.12533097969991175\n", + "highest total_repetitions: 66\n", + "\t@ 327 : Short, short, short, long, long, short, short, long, long, short, short. This is 1108:21:37.\n", + "meteor: 0.31463018790549185\n", + "rap: 0.3129374378105848\n", + "***** Qwen2-1.5B-Instruct checkpoint-560 1\n", + "ews_score: 0.0\n", + "repetition_score: 0.22241835834068843\n", + "total_repetitions: 0.22241835834068843\n", + "highest total_repetitions: 164\n", + "\t@ 327 : Shorts and longs, short and long, short and long, short and long, short and long, shorts and longs, short and long, short and long, short and long, short and long, short and long. This is 1108:21:37.\n", + "meteor: 0.3601984358327376\n", + "rap: 0.356789790786125\n", + "***** Qwen2-1.5B-Instruct checkpoint-1120 2\n", + "ews_score: 0.0\n", + "repetition_score: 0.17917034421888792\n", + "total_repetitions: 0.17917034421888792\n", + "highest total_repetitions: 48\n", + "\t@ 327 : Short and long, short and long, short and long, short and long. This is 1108:21:37.\n", + "meteor: 0.3715145486123948\n", + "rap: 0.3686712153068281\n", + "***** Qwen2-1.5B-Instruct checkpoint-1680 3\n", + "ews_score: 0.0\n", + "repetition_score: 18.94792586054722\n", + "total_repetitions: 18.94792586054722\n", + "highest total_repetitions: 21397\n", + "\t@ 327 : Short long short long long short long short long long short long long long short long short long long long short long short long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long long short long short long long long long long short long short long long long long long long short long short long long long long long short long short long long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long short long short long long long long long short long short long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long short long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long short long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long long\n", + "meteor: 0.3720444789500285\n", + "rap: 0.254542991755422\n", + "***** Qwen2-1.5B-Instruct checkpoint-2240 4\n", + "ews_score: 0.0\n", + "repetition_score: 0.205648720211827\n", + "total_repetitions: 0.205648720211827\n", + "highest total_repetitions: 106\n", + "\t@ 327 : Short: Long, Short: Long, Short: Short, Long: Long, Long: Short, Short: Long, Short: Short, Long: Short, Short: Long, Short: Short, Long: Short, This is 1108:21:37.\n", + "meteor: 0.36344098311977724\n", + "rap: 0.36025609738059056\n", + "***** Qwen2-1.5B-Instruct checkpoint-2800 5\n", + "ews_score: 0.0\n", + "repetition_score: 0.08737864077669903\n", + "total_repetitions: 0.08737864077669903\n", + "highest total_repetitions: 29\n", + "\t@ 62 : He said that I used to talk like Yangzhou people before, perhaps . . . perhaps . . . he might know what happened to my eyes?\n", + "meteor: 0.3590942297170212\n", + "rap: 0.3577425628167956\n", + "***** Qwen2-1.5B-Instruct checkpoint-3360 6\n", + "ews_score: 0.0\n", + "repetition_score: 0.30714916151809357\n", + "total_repetitions: 0.30714916151809357\n", + "highest total_repetitions: 101\n", + "\t@ 327 : SHANG, SHANG, SHANG, ZONGZONG, ZONGZONG, ZONGZONG, ZONGZONG, ZONGZONG, ZONGZONG, ZONGZONG, ZONGZONG, ZONGZONG, ZONGZONG is the sequence of 1108:21:37.\n", + "meteor: 0.3486547719340687\n", + "rap: 0.3441333548414984\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
epochQwen2-0.5B-InstructQwen2-0.5B-Instruct(RAP)Qwen2-1.5B-InstructQwen2-1.5B-Instruct(RAP)
000.2608130.2040970.3146300.312937
110.2904950.2056720.3601980.356790
220.3104960.3095030.3715150.368671
330.3229970.3200590.3720440.254543
440.3162960.3145230.3634410.360256
550.3128350.2299650.3590940.357743
660.3115280.2515200.3486550.344133
\n", + "
" + ], + "text/plain": [ + " epoch Qwen2-0.5B-Instruct Qwen2-0.5B-Instruct(RAP) Qwen2-1.5B-Instruct \\\n", + "0 0 0.260813 0.204097 0.314630 \n", + "1 1 0.290495 0.205672 0.360198 \n", + "2 2 0.310496 0.309503 0.371515 \n", + "3 3 0.322997 0.320059 0.372044 \n", + "4 4 0.316296 0.314523 0.363441 \n", + "5 5 0.312835 0.229965 0.359094 \n", + "6 6 0.311528 0.251520 0.348655 \n", + "\n", + " Qwen2-1.5B-Instruct(RAP) \n", + "0 0.312937 \n", + "1 0.356790 \n", + "2 0.368671 \n", + "3 0.254543 \n", + "4 0.360256 \n", + "5 0.357743 \n", + "6 0.344133 " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "perf_df = calc_metrics_for_epochs(df, end_col=16)\n", + "perf_df" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA90AAAHqCAYAAAAZLi26AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdeVxU1fvA8c/MwDDDNuwguIIL7pamZYuZW9m3Mq0sS81vmS2mqbl9tczS0DSzXLLNtMVflraqWaYtZmqlGa6guLKD7AMMw8z9/TEwMg4oKDioz7sXL+Dec+88F28Mzz3nPEelKIqCEEIIIYQQQgghap3a1QEIIYQQQgghhBBXKkm6hRBCCCGEEEKIOiJJtxBCCCGEEEIIUUck6RZCCCGEEEIIIeqIJN1CCCGEEEIIIUQdkaRbCCGEEEIIIYSoI5J0CyGEEEIIIYQQdUSSbiGEEEIIIYQQoo5I0i2EEEIIIYQQQtQRSbqFEEIIIVxkxYoVqFQq/v77b1eHIoQQoo5I0i2EEKLeKE9AVCoVv//+u9N+RVFo1KgRKpWK//znPw77yo+r7OPJJ5/kl19+OWebih9nx1LZx44dOxxe32g08sorr9ChQwc8PT0xGAzcfPPNfPTRRyiK4nQtZ5/P19eXHj16sH79+lr8iQohhBDC1dxcHYAQQghxNp1Ox6pVq7jpppsctv/6668kJibi4eFR6XF9+vRh2LBhTttbtmxJkyZN+Pjjjx22T506FW9vb6ZNm1ZlLC+//DLNmjVz2t68eXP712lpafTq1YuDBw/y4IMPMnr0aIqLi1m7di3Dhw9nw4YNfPrpp2g0mkrjVRSFEydO8Pbbb3PXXXfx/fff069fvypjEkIIIcTlQ5JuIYQQ9U7//v354osveOutt3BzO/NWtWrVKjp37kxmZmalx7Vs2ZJHHnmkyvOevW/OnDkEBQWd85g77riDLl26nDPe4cOHc/DgQb766ivuvvtu+/YxY8YwceJE5s+fzzXXXMPkyZPPGe+gQYNo06YNb7755mWRdBcXF6PValGrZeCcEEIIURV5lxRCCFHvPPTQQ5w+fZpNmzbZt5WUlLBmzRqGDBniwsic7dixgx9++IFHH33UIeEuFxMTQ4sWLZg7dy5FRUXnPFfr1q0JCgoiISGhWq+9aNEi2rZti6enJ/7+/nTp0oVVq1Y5tElKSuKxxx4jPDwcDw8PmjVrxlNPPUVJSYm9zdGjR7n//vsJCAjA09OT66+/3mmYe/nw/M8++4zp06cTERGBp6cneXl5AOzcuZPbb78dg8GAp6cnPXr0YNu2beeMPy0tDTc3N2bOnOm0Ly4uDpVKxeLFiwEwm83MnDmTFi1aoNPpCAwM5KabbnK4R6qSk5PDc889R6NGjfDw8KB58+bMnTsXq9Vqb3P8+HFUKhXz58/njTfeoEmTJuj1enr06MG+ffuczrllyxZuvvlmvLy88PPz45577uHgwYNO7arz8wcwmUyMHz+e4OBgvLy8uPfee8nIyDjvtQkhhKj/pKdbCCFEvdO0aVNuuOEG/u///o877rgDgO+//57c3FwefPBB3nrrrUqPKy4urrQX3NfXF61We0Gx5ObmOp1TpVIRGBgIwHfffQdQ6bB2ADc3N4YMGcLMmTPZtm0bvXv3PudrZWdnExUVdd643nvvPcaMGcN9993H2LFjKS4uJjY2lp07d9ofTCQnJ9O1a1dycnJ44okniI6OJikpiTVr1lBYWIhWqyUtLY3u3btTWFjImDFjCAwMZOXKldx9992sWbOGe++91+F1X3nlFbRaLc8//zwmkwmtVsuWLVu444476Ny5MzNmzECtVvPhhx9y2223sXXrVrp27VrpNYSGhtKjRw8+//xzZsyY4bBv9erVaDQa7r//fgBeeuklYmJiePzxx+natSt5eXn8/fff7N69mz59+lT5cyosLKRHjx4kJSUxatQoGjduzB9//MHUqVNJSUlh4cKFDu0/+ugj8vPzeeaZZyguLubNN9/ktttuY+/evYSGhgLw008/cccddxAZGclLL71EUVERixYt4sYbb2T37t00bdq02j//cs8++yz+/v7MmDGD48ePs3DhQkaPHs3q1avPey8IIYSo5xQhhBCinvjwww8VQPnrr7+UxYsXKz4+PkphYaGiKIpy//33Kz179lQURVGaNGmi3HnnnQ7HAlV+/N///V+lr9e2bVulR48e54ylsg8PDw97uwEDBiiAkp2dXeV1ffnllwqgvPXWWw7xPvbYY0pGRoaSnp6u/P3338rtt9+uAMq8efPO+7O65557lLZt256zzbBhwxS1Wq389ddfTvusVquiKIry3HPPKYCydetW+778/HylWbNmStOmTRWLxaIoiqL8/PPPCqBERkba/03Kz9OiRQulX79+9nMqiqIUFhYqzZo1U/r06XPOGN955x0FUPbu3euwvU2bNsptt91m/75jx45O/+bV8corryheXl5KfHy8w/YpU6YoGo1GOXnypKIoinLs2DEFUPR6vZKYmGhvt3PnTgVQxo0bZ9/WqVMnJSQkRDl9+rR927///quo1Wpl2LBh9m3V+fmX32e9e/d2+PmNGzdO0Wg0Sk5OTo2vWQghRP0iw8uFEELUSw888ABFRUWsW7eO/Px81q1bd96h5ffccw+bNm1y+ujZs+cFx7FkyRKn833//ff2/fn5+QD4+PhUeY7yfeVDsct98MEHBAcHExISQpcuXdi8eTOTJk1i/Pjx543Lz8+PxMRE/vrrr0r3W61Wvv76a+66665K56SXV2nfsGEDXbt2dSha5+3tzRNPPMHx48c5cOCAw3HDhw9Hr9fbv9+zZw+HDx9myJAhnD59mszMTDIzMzEajfTq1YvffvvNYRj32QYOHIibm5tDj+6+ffs4cOAAgwcPdrje/fv3c/jw4fP8ZBx98cUX3Hzzzfj7+9tjy8zMpHfv3lgsFn777TeH9gMGDCAiIsL+fdeuXenWrRsbNmwAICUlhT179vDoo48SEBBgb9ehQwf69Oljb1fdn3+5J554wmHbzTffjMVi4cSJEzW6XiGEEPWPDC8XQghRLwUHB9O7d29WrVpFYWEhFouF++6775zHNGzY8JzDty9E165dz1lIrTyhzs/Px8/Pr9I2VSXm99xzD6NHj6akpIS//vqLV199lcLCwmoVJps8eTI//fQTXbt2pXnz5vTt25chQ4Zw4403ApCRkUFeXh7t2rU753lOnDhBt27dnLa3bt3avr/iOc6u5F6eBA8fPrzK18jNzcXf37/SfUFBQfTq1YvPP/+cV155BbANLXdzc2PgwIH2di+//DL33HMPLVu2pF27dtx+++0MHTqUDh06nPP6Dh8+TGxsLMHBwZXuT09Pd/i+RYsWTm1atmzJ559/DmBPglu1auXUrnXr1vzwww8YjUYKCgqq9fMv17hxY4fvy39e2dnZ1TpeCCFE/SVJtxBCiHpryJAhjBw5ktTUVO64444qk1pXat26NV9//TWxsbHccsstlbaJjY0FoE2bNg7bKz4k6N+/P0FBQYwePZqePXs6JJxVvW5cXBzr1q1j48aNrF27lqVLl/Liiy9WWpistlTs5Qbsvdjz5s2jU6dOlR7j7e19znM++OCDjBgxgj179tCpUyc+//xzevXqRVBQkL3NLbfcQkJCAt988w0//vgj77//Pm+88QbLli3j8ccfr/LcVquVPn36MGnSpEr3t2zZ8pyxXSpnLydXTqlkjXchhBCXF0m6hRBC1Fv33nsvo0aNYseOHfW2oNR//vMfYmJi+OijjypNui0WC6tWrcLf39/eC12VUaNG8cYbbzB9+nTuvfdepyHIZ/Py8mLw4MEMHjyYkpISBg4cyOzZs5k6dSrBwcH4+vpWWnm7oiZNmhAXF+e0/dChQ/b951Je9M3X1/eCRxkMGDCAUaNG2f+N4+PjmTp1qlO7gIAARowYwYgRIygoKOCWW27hpZdeOmfSHRUVRUFBQbVjq2z4enx8vL04WvnPo6qfWVBQEF5eXuj1+mr9/IUQQlz5ZE63EEKIesvb25u3336bl156ibvuusvV4VSqe/fu9O7dmw8//JB169Y57Z82bRrx8fFMmjTJqZf4bG5ubkyYMIGDBw/yzTffnLPt6dOnHb7XarW0adMGRVEwm82o1WoGDBjAd999x99//+10fHkPav/+/fnzzz/Zvn27fZ/RaOTdd9+ladOmTr3zZ+vcuTNRUVHMnz+fgoICp/3VWfbKz8+Pfv368fnnn/PZZ5+h1WoZMGDAOa/X29ub5s2bYzKZznnuBx54gO3bt/PDDz847cvJyaG0tNRh29dff01SUpL9+z///JOdO3faq+g3aNCATp06sXLlSnJycuzt9u3bx48//kj//v0Bqv3zF0IIceWTnm4hhBD12rnmCp8tPj6eTz75xGl7aGjoOZeVOpfvv//e3utbUffu3YmMjARsy0z16tWLe+65hyFDhnDzzTdjMpn48ssv+eWXXxg8eDATJ06s1us9+uijvPjii8ydO9cp8ayob9++hIWFceONNxIaGsrBgwdZvHgxd955p33u+KuvvsqPP/5Ijx49eOKJJ2jdujUpKSl88cUX/P777/j5+TFlyhT70mxjxowhICCAlStXcuzYMdauXXve+eVqtZr333+fO+64g7Zt2zJixAgiIiJISkri559/xtfX176s2rkMHjyYRx55hKVLl9KvXz+nqQRt2rTh1ltvpXPnzgQEBPD333+zZs0aRo8efc7zTpw4kW+//Zb//Oc/PProo3Tu3Bmj0cjevXtZs2YNx48fdxjG3rx5c2666SaeeuopTCYTCxcuJDAw0GF4+rx587jjjju44YYbeOyxx+xLhhkMBl566SV7u+r8/IUQQlwFXFs8XQghhDij4pJh51LTJcOqWhbsQpcMA5QPP/zQoX1+fr7y0ksvKW3btlX0er3i4+Oj3HjjjcqKFSscloKqGO8zzzxT6Wu/9NJLCqD8/PPPVf4M3nnnHeWWW25RAgMDFQ8PDyUqKkqZOHGikpub69DuxIkTyrBhw5Tg4GDFw8NDiYyMVJ555hnFZDLZ2yQkJCj33Xef4ufnp+h0OqVr167KunXrHM5TvmTYF198UWk8//zzjzJw4EB7PE2aNFEeeOABZfPmzVVeQ0V5eXmKXq9XAOWTTz5x2j9r1iyla9euip+fn6LX65Xo6Ghl9uzZSklJyXnPnZ+fr0ydOlVp3ry5otVqlaCgIKV79+7K/Pnz7ceXLxk2b9485fXXX1caNWqkeHh4KDfffLPy77//Op3zp59+Um688UZFr9crvr6+yl133aUcOHDAqd35fv5V3fPlP+9z3QNCCCEuDypFkfFNQgghhLi6HT9+nGbNmjFv3jyef/55V4cjhBDiCiJzuoUQQgghhBBCiDoiSbcQQgghhBBCCFFHJOkWQgghhBBCCCHqiMzpFkIIIYQQQggh6oj0dAshhBBCCCGEEHVEkm4hhBBCCCGEEKKOuLk6gEvFarWSnJyMj48PKpXK1eEIIYQQQgghhLiMKYpCfn4+4eHhqNVV92dfNUl3cnIyjRo1cnUYQgghhBBCCCGuIKdOnaJhw4ZV7r9qkm4fHx/A9gPx9fV1cTRVM5vN/Pjjj/Tt2xd3d3dXhyPqMblXRE3I/SJqQu4XUV1yr4iakPtFVNflcq/k5eXRqFEje65Zlasm6S4fUu7r61vvk25PT098fX3r9Q0mXE/uFVETcr+ImpD7RVSX3CuiJuR+EdV1ud0r55u+LIXUhBBCCCGEEEKIOiJJtxBCCCGEEEIIUUck6RZCCCGEEEIIIeqIJN1CCCGEEEIIIUQdkaRbCCGEuMIsWbKEpk2botPp6NatG3/++WeVbb/88ku6dOmCn58fXl5edOrUiU8++cShjUqlqvRj3rx5ABw/fpzHHnuMZs2aodfriYqKYsaMGZSUlNTpdQohhBCXgwtKui/2zfzjjz92aHO+N3OApk2bOu2fM2fOhYQvhBCXHUmiRHWtXr2a8ePHM2PGDHbv3k3Hjh3p168f6enplbYPCAhg2rRpbN++ndjYWEaMGMHIkSP5559/7G1SUlIcPpYvX45KpWLQoEEAHDp0CKvVyjvvvMP+/ft54403WLZsGf/73/8uyTULIYQQ9VmNlwwrfzNftmwZ3bp1Y+HChfTr14+4uDhCQkKc2pe/mUdHR6PValm3bh0jRowgJCSEfv36AbY384q+//57HnvsMfubebmXX36ZkSNH2r8/33poQghxJaiN37sjR45k+vTp9O/fHzj/792KSVTz5s3Zt28fI0eOxGg0Mn/+/Lq/aHHBFixYwMiRIxkxYgQAy5YtY/369SxfvpwpU6Y4tb/11lsdvh87diwrVqzgwIED9m1hYWEObb755ht69uxJZGQkALfffju33367fX9kZCRxcXG8/fbbcr8IIYS46tU46a6NN/OVK1fy+++/25Pu872Zl/Px8XFqK4QQVzpJokR1lZSUsGvXLqZOnWrfplar6d27N9u3bz/v8YqisGXLFuLj4xkwYEClbdLS0li/fj0rV64857lyc3MJCAioUfxCCCHElahGSXdtvZnHxcUxd+7cStuc6818zpw5vPLKKzRu3JghQ4Ywbtw43NwqvwSTyYTJZLJ/n5eXB9gWWjebzeeN1VXKY6vPMYr6Qe6Vq0P5792JEyc6/FvfdtttbNu27bz//oqi8PPPP9uTqMral//e/eCDD855vqysLPz9/eWeq8dSUlKwWCwEBgY6/DsFBQVx8ODBKv/tcnNzadq0KSaTCY1GwxtvvEF4eHil7ZcvX46Pjw933XVXlec7cuQIixYtYu7cuXK/XOHkvUjUhNwvoroul3uluvHVKOnOzMzEYrEQGhrqsD00NJRDhw5VeVxubi4RERH2N/OlS5fSp0+fStuuXLkSHx8fBg4c6LB9zJgxXHvttQQEBPDHH38wdepUUlJSWLBgQaXniYmJYebMmU7bf/zxRzw9Pc93qS63adMmV4cgLhNyr1zZsrKysFgsJCQksGHDBvv2goIC4uLiHLZVZDQaeeyxxzCbzajVakaNGkWnTp0qvV++/PJLdDoder2+yvOlpKTw1ltv8eijj1bZRrheVlYWAH/88Yf9a4CjR4+Sk5NT5b+d1Wpl/vz5FBUVERsby/PPP+/wgL2iJUuWcMMNN7Bly5ZK958+fZpp06bRtWtXGjRoIPfLVULei0RNyP0iqqu+3yuFhYXValfj4eUXwsfHhz179lBQUMDmzZsZP348kZGRTkMgwfYE/eGHH0an0zlsHz9+vP3rDh06oNVqGTVqFDExMXh4eDidZ+rUqQ7H5OXl0ahRI/r27Yuvr2/tXVwtM5vNbNq0iT59+uDu7u7qcEQ9JvfK1SE5ORmA7t27c/3119u3//bbbyQlJdnnaJ/NarVyzTXXUFBQwM8//8yrr75KaGgo48ePd7pfJk2axLBhw6ocTpyUlMSECRMYPHgwCxcurJXrEnWjpKSEkSNHEhUV5XBvrFmzhlatWlV5v5xNURTWrl3rdL/8/vvvJCUl8fXXX9OxY0en45KTk+nduze9evXigw8+QK2WRVKudPJeJGpC7hdRXZfLvVI+mvp8apR0BwUFodFoSEtLc9ielpZ2zrnWarWa5s2bA9CpUycOHjxITEyMU9K9detW4uLiWL169Xlj6datG6WlpRw/fpxWrVo57ffw8Kg0GXd3d6/X/3DlLpc4hevJvXJla9CgARqNhtOnTzv8O2dmZtKgQYNz/tu3bt0agOuuu45Dhw6xdu1aJk+e7HDM1q1biY+P5/PPP6/0XMnJyfTt25fu3bvz/vvvSxJVz7m7u9O5c2d+/fVX7rvvPsD2AObnn39m9OjRNfpdYTabnX6/rFy5ks6dO9OlSxen9klJSfTp04cuXbqwcuVKNBrNxV+QuGzIe5GoCblfRHXV93ulurHV6K8nrVZL586d2bx5s32b1Wpl8+bN3HDDDdU+j9VqdZhvXe6DDz6gc+fOlT49P9uePXtQq9WVVu4VQogrRW3+3q1s3tG5fu8mJSVx66230rlzZz788ENJuC8T48eP57333mPlypUcPHiQp556CqPRaC/EN2zYMIeh4zExMWzatImjR49y8OBBXn/9dT799FOnB+N5eXl88cUXPP74406vWX6vNG7cmPnz55ORkUFqaiqpqal1eq1CCCHE5aDGw8vHjx/P8OHD6dKlC127dmXhwoVOb+YRERHExMQAtjfzLl26EBUVhclkYsOGDXz88ce8/fbbDuctfzN//fXXnV5z+/bt7Ny5k549e+Lj48P27dsZN24cjzzyCP7+/hdy3UIIcdmojd+7n376KaNGjXI477l+75YnUU2aNLEnUeVkFYn6bfDgwWRkZPDiiy+SmppKp06d2Lhxo70ey8mTJx0eoBiNRp5++mkSExPR6/VER0ezYsUKvL29Hc772WefoSgKDz30kNNrbtq0iSNHjnDkyBEaNmzosE9RlDq4SiGEEOLyUeOkuzbezD/55BMGDx7scN5zvZl7eHjw2Wef8dJLL2EymWjWrBnjxo1zmLMthBBXKkmirkyKomBRLCiKghUrVsVq+1qxYsVq319xu0LZ/oofVDiurM1tD93GrQ/e6rBtV9ourIqV1/7vNaxY+SPpD6xY6ftkX/qM6uNwrlJLKQn/JJBjyiHILQiVSsUTTzzBE088Uem1PProozz66KOX9gcohBBCXCZUylXy11NeXh4Gg4Hc3Nx6X0htw4YN9O/fv17PXxCuJ/eKqC5FUUjNT+Xbn77lpptuQq1ROyVqFZMz+9cVksHKtlVMEB32V5IIVnr+yhLICtvLj7colkrjcthfSVzlx9j3nxVX+TEWq8XpNStNeitJbs+VFNuT6ip+ppcLb3dvGvk0oqFPQxr6NLR97W37HOYVhpv6ktRkFfWUvBeJmpD7RVTX5XKvVDfHlHdKIYS4ghSVFpGQk0B8djyHsw/bP2ebsgFY/P1iF0coLoRapUaNGpVKZfu6/OOsbSpsX6tUKjQqjcO28u3lx9j3l22zf61SU2op5XjWcfKVfArMBRzMOsjBrINOcbmp3Gjg3cCehJcn5+Wfvdy9XPDTEkIIIeoXSbqFEOIyZLFaSCxIdEisD+cc5mTeSRScBzCpVWr06NF76FGrz52wVfyoLGFz2F9JwlaxTcVtTgnj2eev7PUv5vzVub6zz1/Z9Z3j/BXbVHb+C4qhkvNfauU9DLf1vY10Uzqn8k+RmJ9o+1xg+5yUn0SJtYRT+ac4lX+K7Snbnc4ToAugoXfDSnvJgz2DUaukOJ8QQogrnyTdQghRz2UXZ59JrnMOE58VT0JuAkWlRZW2D9AF0NK/JS38W9DCrwUtA1rS2LMxW37cUu+HaYn6ReemI0ofRZRflNM+q2IlvdA5IU/Mt31km7LJKs4iqziL2MxYp+M9NB5EeEc49o6XJeQRPhF4aJyX/RRCCCEuR5J0CyFEPWGymDiac9SeWB/OsSXamUWZlbb30HgQ5RdlS6zLk2z/FgTpg5zaVrZcmBAXQ61SE+YVRphXGNeFXee0P78kn6SCJIekvPzrFGOK7X7PPcrR3KOVnj/EM8QhEa84bN3fw98lIwCEEEKICyFJtxBCXGKKopBsTHYYGh6fHc+JvBNYFEulxzT0bmhPrMs/N/ZpjEatucTRC1E9PlofogOiiQ6IdtpntppJNabak/DE/ET7sPVT+acwmo2kF6aTXpjOrrRdTsd7uXs5zSNv6NOQRt6NCPMOw10tozmEEELUH5J0CyFEHcorybPNty77iM+O50jOEQrMBZW299X6OiXXzf2aS0EqcUVxV7vbE+azKYpCjimn0nnkifmJpBWmYTQbicuOIy47zul4jUpDmFeYQ+94xR5zb6230zFCCCFEXZKkWwghaoHZauZ47nHHudfZ8aQaUytt76Z2I9IQ6Zhg+7UgxDPkoofNKopCaXIyxYcOUXzwEKa4QxQdPESEXo/Spw/InG5Rj6lUKvx1/vjr/OkQ3MFpv8liIik/ySERt/eYFyTa9hckkVSQBCnO5/fz8LMn4RWHrDfyaUSIZ4gUdxNCCFHrJOkWQogaUBSF9MJ0h8T6cPZhjuYepdRaWukxDbwanClqVpZkNzU0rZUhsNaSEkqOHKH4UBzFhw5iOniI4rg4rHl5Tm29gNzVqwkeMeKiX1cIV/HQeBDpF0mkX6TTPqtiJbMo02H+eMUCb1nFWeSYcsgx5bA3c6/T8Vq1lgifCOd55GUJus5Ndyku8ZJasmQJ8+bNIzU1lY4dO7Jo0SK6du1aadsvv/ySV199lSNHjmA2m2nRogVjx44lICDAod3BgweZPHkyv/76K6WlpbRp04a1a9fSuHFjAEaNGsVPP/1EcnIy3t7edO/enblz5xId7TwVQQghrgSSdAshRBUKzYUczjnsNPc6r8Q5oQXbPNOKiXVL/5Y092+Or9a3VuIpzc7GFBdn670+dIjiQ4cwJSRAaSXJvrs7Hs2bo2vVCl3raEoyM8l+732ylr1DwMCBaAyGWolJiPpErVIT4hlCiGcInUM7O+03mo32OeT2xLysxzylIIUSawnHco9xLPdYpecP1gc7ziGvMGw9QBdw2RV3W716NePHj2fZsmV069aNhQsX0q9fP+Li4ggJCXFqHxAQwLRp04iOjkar1bJu3TpGjhzJ9OnT6d+/PwAJCQncdNNNPPbYY8ycORNfX1/279+PTnfmgUXnzp15+OGHady4MVlZWbz00kv07duXY8eOodFInQohxJVHkm4hxFXPYrVwMv+kQ2J9OPswiQWJlbbXqDQ09W3qMO+6hX8Lwr3Ca+WPbsVqxXzqlGPv9aFDlKZWPlRdbTCgi45GFx2NR3Q0utbReERGotJq7W1KiotJ/e47PFLTyHx7GaFTJl90nEJcbrzcvWgV0IpWAa2c9pVaS0k1plY6bP1U/ikKzAVkFGWQUZTB7vTdTsd7unk6JeLl3zfwaoC7pv5N61iwYAEjR45kRNnol2XLlrF+/XqWL1/OlClTnNrfeuutDt+PHTuWFStWcODAAfu2adOm0b9/f1577TX7tqgoxyXnnnjiCfvXTZs2ZdasWXTs2JHjx487tRVCiCuBJN1CiKtKZlGmQ2J9OOcwCTkJmCymStsH64MdkuuW/i1pZmhWa2sIW4uLMR0+TPHBg2W913GYDh3CWlhYaXv3xo3RtWqFR+todNGt0bWOxi0s7LzJvkqjIaP/nTRcvpysTz/Ff8hDaMuGegohbHUWynuwr29wvcM+RVHINeU6J+Rl36cZ0ygsLSQ+O5747Hinc6tVahp4NbAPU3co8ObTsNZGw9RESUkJu3btYurUqWfiVKvp3bs327dvP+/xiqKwZcsW4uPjGTBgAABWq5X169czadIk+vXrxz///EOzZs2YOnWqvc3ZjEYjH374Ic2aNaNRI+fCekIIcSWQpFsIcUUqLi0mITfBYb3rw9mHySrOqrS93k1Pc7/mDkXNWvi3wF/nX2sxlWZm2gubFZf1XpccOwZWq1NblVaLR8uWtl7rVtGs3BvLm599Rupvv9IxN4dFQx+p+bzLVi3xvLE7hdv+IP31BeQ/Oeqc8y4TEhJ4/vnn+f333zGZTNx+++0sWrSI0NDQWvuZCHE5UKlU+On88NP50S6ondN+k8VEckFypfPIE/MTKbYU24u77Uzd6XS8wcNQ6Tzy8uJudbE0YGZmJhaLxen/59DQUA4dOlTlcbm5uURERGAymdBoNCxatMg+FD09PZ2CggLmzJnDrFmzmDt3Lhs3bmTgwIH8/PPP9OjRw36epUuXMmnSJIxGI61atWLTpk1oK4zOEUKIK4kk3UKIy5pVsZKUn0R8juPQ8JP5J7EqlSSzqGjs29hp7nVDn4a1VrVYsVgoOXHCofe6+NBBLBmZlbbXBATYhoZX6L3WNm2Kys32K3r16tVMWbiwVuZd9powgcLtO9j/3XcMWbmCx554otJ5l0ajkb59+9KxY0e2bNkCwAsvvMBdd93Fjh07UKulwrMQ5Tw0HjQzNKOZoZnTPkVR7MXdKhu6frr4NLmmXHJNuew/vd/peHe1OxHeEbYe8rMS8wjvCDzdPS/FJdr5+PiwZ88eCgoK2Lx5MxMnTuT555+nf//+WMseIN5zzz2MGzcOgE6dOvHHH3+wbNkyh6T74Ycfpk+fPqSkpDB//nweeOABtm3b5jD3WwghrhSSdAshLhu5plz78E372tc5hykqLaq0vb+Hv9Oa15GGyFr9I9VqNFIcF+/Qe22Kj0cpLnZurFKhbdrU1nsd3RpddCs8oqNxCw4+5/Dw2px36dGiBX6DBjHhzYXcEhDA3Llz7a9dcS7ltm3bOH78OP/88w++vrahrytXrsTf358tW7bQu3fvmv6ohLgqqVQqgj2DCfYM5trQa532F5oLq1z+LKkgybYcYd5xjucdr/T8QfqgSueRN/RpSKAusMrfLUFBQWg0GtLS0hy2p6WlERYWVuX1qNVqmjdvDtgS6v3797N27VomT55MUFAQbm5utGnTxuGY1q1b8/vvvztsMxgMGAwGWrRowfXXX4+/vz9fffUVDz30UJWvLYQQlytJuoVwoYtZqqV58+b07NnTXjG23PmWaikuLmbChAl89tlnmEwm+vXrx9KlS+vVkOESi62C8NnLcqUXplfaXqvWEuUX5TA0vGVAy3P+wVlTiqJQmp7uOPf64EFKTp4ERXFqr9LrbXOvo1vZe689WrRA7VmzhL8u5l0Gjn6GX1+dzWNaD3p37szexESneZcmkwmVSoWHx5m56zqdDrVaze+//y5JtxC1xNPdk5b+LWnp39Jpn8VqIa0wzSEhr9hjnl+ST2ZRJplFmfyT/o/T8Xo3PRHeEQ7JeHmCHuEdQefOndm8ebPDnOzNmzczevToasdvtVoxm80AaLVarrvuOuLi4hzaxMfH06RJkyrPoSgKiqJgMlVeW0MIIS53knQL4SIXu1TLN998w8SJE+nTpw933nknUL2lWsaNG8f69ev54osvMBgMjB49moEDB7Jt27ZLdu3lFEUh1ZhqT6zLk+vjuccpVSpf8zrCO+LMmtcBLWnp15LGvo1xU9ferzPFbMZ09BimQwcdKohbcnIqbe8WEnJmaHhZ77W2cWNUtbD0TV3Mu8xSFAqtVt7POs04fz/mrVvHj1u2OMy7vP766/Hy8mLy5Mm8+uqrKIrClClTsFgspKSkXPR1CSHOT6PWEO4dTrh3ON0adHPan2vKtSXjBaeclkJLNaZSVFrEkZwjHMk54nSsChV0h7eXvM0pwyk6X9eZ7au3k1+Qz6AhgwAYNmwYERERxMTEABATE0OXLl2IiorCZDKxYcMGPv30U0aNGmU/78SJExk8eDC33HILPXv2ZOPGjXz33Xf88ssvABw9epTVq1fTt29fgoODSUxMZM6cOej1eqeHyEIIcaWQpFsIF7nYIcPPPvssS5YsYdu2bfak+3xLteTm5vLBBx+watUqbrvtNgA+/PBDWrduzY4dO7j+eseKvbWpoKTAac3rw9mHyTfnV9reR+vjvOa1X3O8td61GpclP9/Wc10+NPzQIUyHD6OU9dw40GjwiIy0LctVvjRXdDRuAQG1GlNtqM68y15BwTyicSMkNpYpU6Y4zLsMDg7miy++4KmnnuKtt95CrVbz0EMPce2118p8biHqCYOHAYOHgbZBbZ32mS1mkgqSKq24npifaJuW0wlCBoewYdkGvp37LbrGOhqMacDAnwfio/XhyD9HCE4PJnRXKA19GhKfFs+7T71LalIqer2e6OhoVqxYgbf3md/L9957L8uWLSMmJoYxY8bQqlUr1q5dy0033QTYRsxs3bqVhQsXkp2dTWhoKLfccgt//PFHpQ+chRDiSiBJtxAuUFtDhpOSkrj55puB6i3VsmvXLsxms8PQ4OjoaBo3bsz27dtrJekutZZyIu+EQ3Idnx1PsjG50vZuKjeaGpo6JNct/VsS6hlaa0PDwfYzMyclO/Vem5OSKm2v9vY+MzQ8uhUe0a3xaNEctUftLBVWXXU577Jj796wezeZy97BMHCg07zLvn37kpCQQGZmJm5ubvj5+REWFkZkZGTdXKwQota4a9xpamhKU0NTp32KonC6+LQtEb/pFIkTHJdAyyzKJL8kn9DnbSNsPtj3ge3Aa8D7Gm+i1dG24m7eDUnwSiDnVA6FhwsJ8AzAV+vL9fdcz8/3/4yvhy/e7t4ORSrDw8PZsGHDpfgRiDpwMdPi7CtpVHhQ/eijj7Jy5UqH4/r168fGjRvt3+/evZvJkyfz119/odFoGDRoEAsWLHB42CNEfSdJtxAuUFtDhkeOHGlPoKuzVEtqaiparRY/Pz+n101NTa3RNZRX5K2YWJeveW22VtJLDIR6hjqvee3bDHeNe41e+3ysJSWUHDni0HtdfOgQ1vzKe9Xdw8PtvdceraPRtW6Ne0RErSb9F0qr1dbZvMsTahUebVpjOnCQzMVLqpx3GRQUBMCWLVtIT0/n7rvvvvgLE0K4jEqlIkgfRJA+iE4hnZz2F5oLSSpIqnwJtIJE+8PVE3kn7Mds+mtTpa+lVqnx0fpg0Brw1fpi8LB99vXwdfi+4naD1oCvhy86ja5e/B4WNhc7La7iShoVpxLcfvvtfPjhh/bvK9YSSU5Opnfv3gwePJjFixeTl5fHc889x6OPPsqaNWvq9oKFqEWSdAtxGak4ZPjHH39k5syZ3HPPPfTu3btGS7XUVKG5kIScBIeiZvHZ8eSYcipt7+nmaZt3XT73uizJNngYLjiGqpRmZ58pbHbooG0d7KNHobSSOeHu7ng0b24bGl7We62LboXGUPtx1abx48czfPhwunTpQteuXVm4cCFGo9E+NeFi5l12mzCB5nv+5dO3l/JdRoZ93iWcmXoQHBzM9u3bGTt2LOPGjaNVq1aX9PqFEJeWp/uZ3+Fns1gtpBem23vFj+ccZ8+RPRhCDOSV5Nk+TLbPxZZirIrVviRaTWnVWockvPzzuZJ2g4cBH60P7urafZgrancljYo8PDyqHLm1bt063N3dWbJkiX1q07Jly+jQoQNHjhyxj+gSor6TpFsIF6iNIcNt27Zl06ZNvPbaa/Tu3btaS7WEhYVRUlJCTk6OQ293+etarBYSCxIde6+zD3Mq/xQKzhW61So1TXybOM29DvcOr7U1r8spVivmU6fKeq9tQ8OL4+IoraKHXmMwOPVeezRrhkqrrdW4LoXBgweTkZHBiy++SGpqKp06dWLjxo32kRInT550mGdtNBp5+umnSUxMrNa8y1MnT9DUzY13evexz7sEiIuLY+rUqWRlZdG0aVOmTZtmf6AjhLg6adQaGng3oIF3A64Luw6z2cyG5A30v6U/7u6Oia7JYiLPlEeuKZe8kjOfK35t32Zy3G5RLJRYS+zV2WvKy93LsQe9Gr3sBg8DXu5etf7+dSWoi5U0yv3yyy+EhITg7+/PbbfdxqxZswgMDARsK2lotVqH9zi9Xg/A77//Lkm3uGxI0i2EC9TWkGFFUSgpKbGf83xLtXTu3Bl3d3c2b95Mzzt7cjj7ML/t/o2TJ0+ywbKBVatWUWypZH1pIFAXWOma1zo3XaXtL4a1qAjT4cO2Xuuy9a9NcXFYCwsrbe/euPGZwmatbJ/dwsKuqGGJo0ePrvLeqNg7DTBr1ixmzZrlsM1sNjvNo/zvf//Lf//7X0xHj3L0rrvh+HGMO3bidb2tSvKcOXOYM2dO7V2EEOKq4qHxsK9RXhOKomA0G6tM0KtK2PNMefbinEazEaPZSIqxZqst1HQ4fMX9V/Jw+LpYSQNsQ8sHDhxIs2bNSEhI4H//+x933HEH27dvR6PRcNtttzF+/HjmzZvH2LFjMRqN9l51WUlDXE4k6RbCRS52yHD5EixLliyxn7OqpVpWfLOCb458w+HswzTv25yHn3yYBo81QKPXkPxJMvrmelKDU8ECOo3Occ3rsiHigfrAOvk5lGZkOBQ2K46Lo+TYMSgbLl+RysMDj5Yt7cty6Vq3xqNlSzRSTOWieERG4j94MNmrVpH22lyarVmDSiqUCyFcRKVS4a31xlvrTTjhNTq21FpKQUkBuSW5tl72Kj5XlrTX9XD4it9fLcPhz7WSBsCDDz5ob9u+fXs6dOhAVFQUv/zyC7169aJt27asXLmS8ePHM3XqVDQaDWPGjCE0NFRW0hCXFUm6hXCRix0y3KpVK8aNG8d///tfwNYzcF3v63hu9nNMnzWdrNFZeIZ70mh0I+ZmzoWy0Xmqe1R4FXlxavEpKIWm1zXlsRmP0aVFF1r4taCRTyM06otfX/psisVCyfHjDr3XxYcOYcmsfNigJjDQqfda27QpKjf5tVUXgkY/Q+6332I6cJDcb77F794Brg5JCCFqzE3thp/ODz+dX42PPXs4fHV61+tqOHzFYfH1YTh8XaykUZnIyEiCgoI4cuQIvXr1AmDIkCEMGTKEtLQ0vLy8UKlULFiwQFbSEJcV+etVCBe6mCHDWcYsXl//OrP/nM2R3CMcyTmC0WyEEAh5OYQQzgzfMngYbL3W5XOvB7SguV9zPN096+S6LAVGTPHxZ3qvy9e+Lq5k6LpKhbZZszOFzVrb5mG7BddsOKK4OG4BAQQ9OYr0+a+TsXAhvrf3Q102b04IIapS+Md2An/4kazkZNy9vFDp9ah1etR6HSr7Zx1qvR61Tle237atvg3Fvpjh8IWlhecdBl9Xw+HtyXkdDoevi5U0KpOYmMjp06dp0KCB077yTonly5ej0+no06dPtV9XCFeTpFuIy9A/6f8w+bfJpBSlwJEz293V7kQaIp3mXgfrg+vkjxtFUShNS6P44EF7BfHiQwcxnzhZaXuVpye6li1thc3Keq89WrRA7Vk3yb+oGf+hQ8le9X+Yk5M5/eGHBD/9tKtDEkLUY6ajR0l+5hkCS0vJ2rKlxsfbE3C9zpaoV0zKy7fp9RW+PiuRr7jN8+ykvmyfpvZHbjldh0qFl7sXXu5etT4c/lxJe/lw+BxTTpWriZxLTYfDP/LkI0x8aiKdru1E9+u7X/RKGgUFBcycOZNBgwYRFhZGQkICkyZNonnz5vTr188e5+LFi+nevTve3t5s2rSJiRMnMmfOHKflT4WozyTpFuIyUmot5Z3Yd3g39l2sihV/tT8DogfQOqg1Lfxa0MTQpM7mhSlmM6ajR23JdYX1ry05OZW2dwsNxSO6FboKvdfujRvLXOF6TO3hQfCE8SRPeJ7T73+A33334V7J2qtCCKEoCmmzZkFpKcXh4QR364bKZMJaXIxSXIS1sMj2dZHtc/nXSlnxTwClqAhLURFk112cKnd3x0Re73lWUl+WoFe2TV+NHnu93vYaF/hg+7IaDq+GgPsDeGriU5TmluLT1IfO0zrzvz3/w+Bh4Ld9vxGcFUzU3ih8tb7EJsWy9N2lZKRmoNPpaBXdig9XfIiPtw8AGo2G2NhYVq5cSU5ODuHh4fTt25dXXnnFYa3uP//8kxkzZlBQUEB0dDTvvPMOQ4cOrfHPSwhXkqRbiMvEqfxTTN06lX8z/gXgP83+Q6esTgzsNNBpmZaLZcnLsyfV5b3XJYePoFQ2JEyjwSMy0tZ7XbbutUd0NG4BAbUak7g0fPv3J+ujjyj+N5bMRYto8Morrg5JCFEP5W/ciPGP7ai0WlIeeZh2Q4dW671IsVhQypJwa1ExSlFh2ddFtu1FZUl7UTHW4jPbrEWFKEXFZ5L68v1FxY7JfVERSlHRmdczm1HMZqx5eXX3w9BoHBLxKhP5sqH3jsm9LZF36LnX652Tew8Pp4fWdT0c/uwe9zxTHvSGwN5nCqtmkEFGagYAPmN9KKaYN3e/adt5Hfhd54cffgAUUsh8y3z88/zZ/Ntmmvs359m3nyXSEEkzQzP0bpVPafroo49qdH1C1EeSdAtxGfgu4Ttm75yN0WzEx92HF254gd4NezstAVVTiqJgTkpy6r02JyVV2l7t7e3Qe+0RHY1H8+aoKzyRFpc3lUpF6OQpnBgyhJy1X+L/yFB0rVq6OiwhRD1iKTCSFmNbTtD/8ccwB1Z/dQuVRoPKywu1l1ddhYeiKCgmk3Mif1ZyX9NE3tZjX7a/qAgsFtsLWixYjUYwGrHU2VVRlphXTO71lQzHryTh99Q7D8fX6wjQ6QjSe6HWBaLyKTvmPMVKqxoOX51e9mJLMRbFQqaSyc+JP/Nz4s9nrg0V4d7hRPlFEWWIItIv0v7Zy73u7hUhLhVJuoWox/JL8pm1YxYbjtmS62tDriXm5hjCvcPPWYikMtaSEkyHD9sTbNMh2/Jc1vz8Stu7h4fj0bo1uuhoW6LdujXuERH1rvCNqH2e116DT79+5P/wA+mvvUbjD953dUhCiDqyZMkS5s2bR2pqKh07dmTRokV07dq10rZffvklr776Kof378dsMtHEy4uJej1np9wHDx5k8uTJ/Prrr5SWltKmTRvWrl1L48aNASguLmbChAl89tlnmEwm+vXrx9KlS53WgL5QKpXKnqDWJcVsPiuRL3JM1MuTe3siX7atsMgxubcn/GcSfaWoyHaOisPxi4uxFBdDFdO6aoPTcPwq5tp76PWE6HWEOSTyQah0Dc/00vs5zrU3u6vJtOSz5tevCY4O5nj+cRJyE0jISSDHlENSQRJJBUn8lvibQ0xhXmEOiXiUXxTNDM0weBjq7OcgRG2TpFuIeuqf9H+YunUqSQVJaFQanuz4JI+3fxw39fn/ty3Nzq7Qe22rIG46dgxKS50bu7vj0aL5mcJm0dHoWrVCY5A3s6tZyITx5G/ZgnHbNgq2bsX75ptdHZIQopatXr2a8ePHs2zZMrp168bChQvp168fcXFxhFRSzyEgIIBJI0bguXAh7hYre/rfwRNPPcX06dPt6y4nJCRw00038dhjjzFz5kx8fX3Zv38/ugoJ8Lhx41i/fj1ffPEFBoOB0aNHM3DgQLZt23bJrr02qNzd0bi7o/HxqbPXcBqOX5agO/TcFzkPx6+s596hZ/+sufYoiu31LsFw/F6BgQRcfz1eHTui73g3Hre1JsdaQEJOAkdzjnI09ygJubavM4oySDWmkmpMZVuy4/0RrA8m0hDp0Cse5RdFgE6mt4n6R5JuIeqZUmsp78a+yzux72BVrER4RzDn5jl0Cunk3NhqpeTECYqOHDnTe33oEKVnraNZTmMwOPVeezRrhkqrrduLEpcdbePGBDz8MFkrVpD+2mt43XCDrJEuxBVmwYIFjBw50l59etmyZaxfv57ly5czZcoUp/Y9evTg5PIPKdS44d23F/0WL+azbds4cOCAvc20adPo378/r732mn1bVFSU/evc3Fw++OADVq1axW233QbAhx9+SOvWrdmxYwfXX399XV3uZemSDccvKal8OH1Vc+3PmchXPkS//MG/9vRpCtavp2D9elsA7u7ooqNp1KEDLTt2QNf+QbRdm6JSqcg15XIs9xgJOQn2RDwhN4FUYyoZRRlkFGWwM3Wnw/X4e/g7JeJRhiiC9EEyWk+4jPwFJUQ9kpifyNStU9mTsQeAuyLv4n/d/oe31tuhnVJaSta77xL17nucNJkqPZd7k8aOvdfR0biFhckbjqi2oKeeJPerrzAdPmKb3z34AVeHJISoJSUlJezatYupU6fat6nVanr37s327dsrPSZv/QYK//wTlU5HyJQpbN68mfj4eId1m9evX8+kSZPo168f//zzD82aNWPq1Kn2Nrt27cJsNtO7d2/7eaOjo2ncuDHbt2+XpNsFVCoVKg8P8PCgLhdYU8xmik+f5vePPqaDXk/Jvn0UxcZiyc6meO9eivfuJfvTTwFQGwzo27dH36E9UR060K5DT9xaDrKfq6CkwJaMV0jEj+YcJakgiWxTNrvSdrErbZfD6/tofRyGp5cn42Fe8reRqHuSdAtRT6w7uo7ZO2ZTYC7A292bF65/gf6R/Z3amY4dI3nKFIr/jUUDqDw88GjZ0tZ7XbY0l0fLVmi8pfCIuDgag4GgZ54m7dUYMt56C98775T7SogrRGZmJhaLxWkedWhoKIcOHXJqbykoIH3uXPItFnoeOkhJZDM0Gg2LFi2yD0VPT0+noKCAOXPmMGvWLObOncvGjRsZOHAgP//8Mz169CA1NRWtVuu0xnJoaCipqal1dr3C9VTu7rgFBlLYqiUB/fvj7u5uL+ha9O+/FMfGUhS7l+IDB7Dm5mL8/XeMv/9uP969YUP0HTqg69AefYeOtG3TmvbB7R1eo6i0iOO5x88k4zkJHM09ysn8k+SX5LMnY4+9Y6Ocp5vnmWHqFQq5RXhHoFbJMqeidkjSLYSL5ZfkM3vnbNYftQ2zuibkGmJujiHCO8KhnaIoZK9aRfq8+SjFxah9fEjqfwc3TZmCVl/5MhtCXCz/Bx8k69NPMZ84yen33yPkuedcHZIQwgUyFy2mNCMD/2bN+OebryksKWHz5s1MnDiR559/nv79+2O1WgG45557GDduHACdOnXijz/+YNmyZfTo0cOVlyDqIZVKhbZhQ7QNG2K4806grEc8Lp7ivbEU/RtLUWwsJUePYk5MxJyYSF75yi1ubuhatkTXsQP6Dh3Rd2iPrlkzWge2pnVga4fXKbGUcDzvuL1XPCEngWO5xzied5zC0kL2nd7HvtP7HI7RaXQ0MzQj0i+SSMOZ4eqNfBpVq76OEBXJHSOEC+1J38OUrVNIKkhCrVLzZIcnGdlhpNMvc3NqKin/m4bxjz8A8LzheoJffplDu3fLPFtRp1RaLSHPP0/Ss2PI+nAF/oMH496ggavDEkJcpKCgIDQaDWln1QBJS0sjLCzMYVtxXDxZn3wCQIMXXsC7TRvAllDv37+ftWvXMnnyZIKCgnBzc6NN2f5yrVu35veyHsuwsDBKSkrIyclx6O2u7HVF/XMh1e6PHDmC2WymRYsWjB07loCAM4XOHn30UVauXOlwXL9+/di4cSP6dm3xf+gh4uPjmTxuHNu2baPEZCLaz49nA4O4rrSU4gMHKD5wgJz/+wwAtY8P+vbt0LXvgL5jB/QdOuAWFIRWo6Wlf0ta+jsugWm2mjmVf8reK17eQ34s9xjFlmIOZh3kYNZBh2Pc1e408W3itLxZE98muGvOv1a9uDrJX+tCuECptZT3Yt/jndh3sCiWKoulKYpC3rr1pL7yCta8PNs8uuefx3/IQ5Ra6nI1UCHO8OndG32XzhT9vYuMhQsJnzvX1SEJIS6SVqulc+fObN682WFO9ubNmxk9erS9naIopL78Mlgs+PTti/fNNzmcx2q12pew1Gq1XHfddcTFxTm0iY+Pp0mTJgB07twZd3d3Nm/ezKBBtjm6cXFxnDx5khtuuKGuLlfUggupdj9t2jSio6PRarWsW7eOkSNHOlS7B7j99tv58MMP7d97eHg4nOc///kPLVq04OfffkOv17Nw4UKeWrGCQ9u2YUhNtfWG742leN9+rPn5GP/YjvGPM3UJ3MIblPWEd7D1hrdti7pshKC72t02tNwQSe8mZ+oMWKwWkgqSnAq4Hcs9RlFpEUdyjnAk54hDnBqVhsa+jZ3WGW/q2xSdW90uXyfqP0m6hbjEkgqSmPLbFPZk7OH0T6cx/mTkcPZhRnUc5fDEuDQ7m9SZL5O/cSMAvwQE8H5+Hgmjn8E86gmaN29Oz549Hd64zvXEGOCXX36hZ8+elcb1559/ct1119XFJYvLnEqlInTyFI7ffz+533yL/9Bh6Nu1dXVYQoiLNH78eIYPH06XLl3o2rUrCxcuxGg02quZDxs2jKDCQkbu249Kr2eFtxfdNm0iKioKk8nEhg0b+PTTTxk1apT9nBMnTmTw4MHccsst9OzZk40bN/Ldd9/xyy+/AGAwGHjssccYP348AQEB+Pr68uyzz3LDDTdIEbV6rqbV7m+99VaH78eOHcuKFSscqt2DLcmuapRDZmYmhw8f5oMPPqBDhw4AzJkzh6VLlxKXmUnv22/H9/bbAVuRWdPhwxTF7qUo1jZH3HQkgdLkFPKTU+x/T6HR4NGihS0J79gBXfv2eERFodKcKSOnUdsS6Ma+jenJmb+brIqVVGOqfa54xaS8wGwr7nYs9xicPHMNKlQ09GnoVE29maEZnu6e1fvhi8ueJN1CXELrj65n1o5ZFJgLKP67mIzPM3h32btOT4z1Bw6Q8sILWDIywc2N4GeeJrplS6bn5dmfGH/zzTdMnDiRPn36cGfZPCg49xPj7t27k5KS4hDTCy+8wObNm+nSpUvd/wDEZUvfvh2+d91F3nffkT53Lo0/WinVXoW4zA0ePJiMjAxefPFFUlNT6dSpExs3brQXVztx7BgFcfEQEEDQ009RfOIETz/9NImJiej1eqKjo1mxYgXe3mdW2Lj33ntZtmwZMTExjBkzhlatWrF27VpuuulMD/kbb7yBWq1m0KBBmEwm+vXrx9KlSy/59Yvqu5Bq9xUpisKWLVscqt2X++WXXwgJCcHf35/bbruNWbNmERgYCEBgYCCtWrXio48+4tprr8XDw4N33nmHkJAQOnfu7HAelZsbutat0bVubV9tw1JgpLisSnr5HPHS9HRMh2zLrOZ8/rntWry80LVrZ+sJ72CbI+4e6tx7r1apCfcOJ9w7nJsb3uxwfemF6U7V1I/kHCGvJI9T+ac4lX+KXxJ/cThfuFe4vVc8yi/KPn/cR1t3a78L11ApiqK4OohLIS8vD4PBQG5uLr6+vq4Op0pms5kNGzbQv6yqo7gyFJQUMHvnbNYdXQdAp+BO7HtxHzdefyOLFy8GbEP0GjVsyPDo1gxNTgZA2zyK8Llz0bd17lU0m81ER0czePBgXn31VcDW052Tk8PXX39drbjMZjMRERE8++yzvPDCC7VwpaK+qo3fLebkZBLu6I9iMtFwyWJ8evWq5ShFfSHvRQIgddZssj/5BG1kJJFff4VKq3VqI/fK1SE5OZmIiAj++OMPh2kAkyZN4tdff2Xnzp2VHpebm0tERAQmk8mh2n35/fLZZ5/h6elJs2bNSEhI4H//+x/e3t5s374dTVnPc2JiIgMGDGD37t2o1WpCQkJYv34911xzzQVdizk11ZaEx5YVatu/H6Ww0KmdW2hohd7wDujbta3xeumKonC6+DRHc47ae8bLP58uPl3lcSH6EHuveKThTO+4n86vppd72bpcfrdUN8eUnm4h6lhlxdKGRw/H925fZkyfYW9XvHs33RTY8defDG3YiIDhwwke9xzqs+Y2wZknxklJSdx8880O+871xPhs3377LadPn7YPFRPiXNzDwwkYPpzT775L+rz5eN9yC6p6/EYohLhwxQcPkr1qFQBhL0yvNOEW4nx8fHzYs2cPBQUFTtXuAR588EF72/bt29OhQweioqL45Zdf6NWrF4qi8MwzzxASEsLWrVvR6/W8//773HXXXfz11180uIDCnu5hYbiHheHbty8AisWC6UhC2ZD0vRTFxmI6fJjStDTyN20if9Mm24FqNR7Nm5ctWWYr0ubRvPk5C9qqVCqC9EEE6YPo2sCx4FxOcY4tAa+wvFlCbgLphemkF9k+dqTscDgmQBfglIhH+kUSqAusd6PParvoXkVPPvkk77zzDm+88QbPVVhVZffu3UyePJm//voLjUbDoEGDWLBggcNoHFeRpFuIOlJqLeW9ve/xzr/OxdKSk5Pt66NaS0rIePNNspZ/iL/JRIJKTeMVK/Dq5vyL6ewnxiNHjqR37zOFP26//XYGDhzo8MT4jjvucHhiXNEHH3xAv379aNiwYZ3+LMSVI/CJkeSsXUvJ8eNkf7aagKGPuDokIUQtU6xWUme+DFYrvv3vwEsKnF31alLtviK1Wk3z5s0B52r3lYmMjCQoKIgjR47Qq1cvtmzZwrp168jOzrb3Ii5dupRNmzaxcuXKSueS15RKo0HXqiW6Vi3h/vsBsBqNFB84QFHZ2uFFsbGUpqRgio/HFB9P7pq1tmP1evRt25YNSbf1iruFhVUrAfbT+XGt7lquDb3WYXt+ST5Hc4869Y4nFSSRVZxFVmoWf6X+5XCMr9a30mQ81DPUJcl4XRXdA/jqq6/YsWMH4eHhDtuTk5Pp3bs3gwcPZvHixeTl5fHcc8/x6KOPsmbNmjq93uqQpFuIOpBUkMTUrVP5J/0fAO6MvJNp3aY5zdExHT/O8VdjMB0+DIBHq5Z4lJZWmnCD4xPjH3/8kZkzZ3LPPffYE+/zPTGuKDExkR9++IHPy+YzCVEdGm9vgp8dTepLM8lcsgTDPXejqcdTdoQQNZf71dcU7dmD2tOTkCqSI3F1qW61+/OpWO2+MomJiZw+fdreg11YNuxbrVY7tFOr1fZ14euC2ssLz+uuw7NCgVlzejrFe/fa1w4v3rsXq9FI4d9/U/j33/Z2muAgx2rp7dujqUFPq4/Wh47BHekY3NFhe6G5kGN5x5yWN0ssSCSvJI9/0v+x/91Zzsvdy6maepRfFA28GqBWOf5Ma1NdFd1LSkri2Wef5YcffnCoZwSwbt063N3dWbJkif1+WbZsGR06dODIkSP2hz+uckFJ98UOF5gwYQJDhw61tzlfxWWArKwsnn32Wb777jt78Y0333yzXgwXEKKiDUc38MqOVygwF+Dl7sW0btO4K+ouhzaBfn5o1Gr2TplCqN4TTWAgDV55GePKlTTIyany3BWfGLdt25ZNmzbx2muvOfR2V3T2E+OKPvzwQwIDA7n77rsv7oLFVcfvvvvI+uQTSo4kkLnsHUInTXR1SEKIWmLJzSV9/nwAgkaPxr2sqJoQ1al2HxERQUxMDAAxMTF06dKlymr3BQUFzJw5k0GDBhEWFkZCQgKTJk2iefPm9OvXD4AbbrgBf39/hg8fzosvvoher+e9997j2LFjTklXXXMPCcG9Vy97PRPFaqXk6FH7kmVFsbGY4uKxZGRSsHkzBZs32w5UqdBGRaJvf6Zauq5lyxpPz/J096RtYFvaBjrW+SkuLeZE3gl7In4s9xgJOQmczDuJ0WwkNjOW2MxYh2P0bnqaGZrZe8bLPzf0bohG7Twysibqquie1Wpl6NChTJw4kbaV1DoymUxotVqHBzT6sqXhfv/998sv6a6N4QIjRowgJCTE/j8UnH+NvocffpiUlBQ2bdqE2WxmxIgRPPHEE6wqm28khKsVlBTw6s5X+e7odwB0DO5IzM0xNPJp5NCu5MQJUiZPoY1Wy468PO69+27CZs5E7efH5mHDavTEWFEUSkpKqtx/9hPjisd9+OGHDBs2rF4XpxD1k8rNjdCJEzk16kmyP/4Y/yEPoZUpCkJcEdIXLsSSnY1Hi+YyfUQ4OF+1+5MnTzokPEaj8ZzV7jUaDbGxsaxcuZKcnBzCw8Pp27cvr7zyij0PCAoKYuPGjUybNo3bbrsNs9lM27Zt+eabb+jYsaNzkJeQqmyOt0fz5vgNGgiAtaiI4oMHy3rDbXPEzUlJlBxJoORIArlffWU7VqdD16YN+vbtbYl4h464R4Rf0FBwnZuOVgGtaBXQymG72WLmZP5Jp7XGj+cep6i0iAOnD3DgtGNPslatpamhqdPyZo18G+Gurt7fi5mZmfYplBWFhoZy6NChKo+rquheublz5+Lm5saYMWMqPf62225j/PjxzJs3j7Fjx2I0Gu296mev3OMKNU66a2O4wMqVK/n9998dku5zrdF38OBBNm7cyF9//WVf1mjRokX079+f+fPnO43pF+JS+zfjX6b8NoXEgkTUKjVPdHiCUR1G4aY+87+YoijkrF5N2tzXUIqKGNEgnKmJp+jTuTPd0tJYOHVqjZ4Yl697umTJEqB6T4zLbdmyhWPHjvH4449fop+QuNJ43XILXt1vwPjHdjIWLCBiwQJXhySEuEhF+/aT89lqAEJfeEEKJQono0ePrrJzoHwt9nKzZs1i1qxZDtvKK1KDrRfyhx9+OO9rdunSpVrt6gO1Xo/ntdfiee2ZedqlmZm2eeF7Yyn+N5aivXux5udTtHs3Rbt329tpAgPPJOHtbUPTL2b6lrvG3ZY0+0U5bC+1lpKYn+i0vNmx3GMUW4qJz44nPjve4Rg3lRtNfJs4VFSPNETS1NAUD41zwd8Lca6ie7t27eLNN99k9+7dVT6YaNu2LStXrmT8+PFMnToVjUbDmDFjCA0NdZqe4Ao1Srpra7hAXFwcc+fOddh3rorL27dvx8/Pz2Ed4d69e6NWq9m5cyf33ntvTS5DiFpjsVp4b+97LPt3GRbFQrhXOHNumcM1IY7LWJjT0kiZNh3j778D4Hn99Yx+dTbuX37JjBkzLuiJcatWrRg3bhz//e9/geo9MS73wQcf0L17d6Kjo+vyxyOuYCqVipBJkzh270DyNnxPwLBh6Dt1cnVYQogLpFitpL78MigKvnfdhVcV0waFEDXjFhSEz2098bmtJ1A2LP34ibKecFuhtuJDh7CcPk3BL79QUOHhhbZZM4e1w3WtWl70SgJuajeaGprS1NCUXo3PTD20KlaSC5LtxdsqLm9WWFpIQq6tx3zTiU32Y9QqNY18GjkMU2/k2ajWi+5t3bqV9PR0GjdubG9vsViYMGECCxcu5Pjx4wAMGTKEIUOGkJaWhpeXFyqVigULFhAZGXlRP7PaUKOku7aGCyxdupQ+ffrY95+v4nJqaqrT0HU3NzcCAgJITU2t9DVNJhMmk8n+fV5eHmB7wnauAg6uVh5bfY5R2CQbk5n+x3T2ZOwB4I4mdzDluin4aH0c/v3yv/+ejFmzseblofLwIHDccxgeegjUakaNGmWf21Su/NhNZUtUlH8/Y8YMZsyY4dCufLoF2P6fWLduXaWxnn0/lddQkPvs6lEXv1s0UVH4DBhA/ldfkRozh4iPP6p3S5aICyPvRVef3DVrKI6NRe3tTcC456r9by/3iqgJuV9s1I0a4tWoIV5l89KtJhOmg4cw7dtLcexeivfupTQxkZJjxyg5dozcb74FQKXVoo2ORtehPbp27dB16IBbw4a19t4bqgslVBfKDaFnVixQFIW0wjRbRfU8W4/40VxbZfV8cz4n8k5wIu8EP5/62X6MtomWse+M5QevH4g0RNLEuwkbN23kySefrPa/fWlpqT1ve/DBB+nRo4fD/v/85z8MGTKE4cOHO52zfKmxFStWoNPpuPXWW+vsnqvueS9J9fKzhwuMHz+eyMhI+9DzmlRcrq6YmBhmzpzptP3HH3/E09Pzgs55KZUnXKJ+ii2J5dvCbymmGA88uMvzLjrldmLrT1vtbdRGIyFff4NvrK14RXHDhqQMfoA4f3+oUCTwYsm9Imqitu8XTZvWNFu3juJ//2Xr3LkUdOhQq+cXriW/X64OaqORZvNfRwOk9ryVQ3/9db5DnMi9ImpC7pcqBATArT3g1h6ojUb0p06hO3kK3alT6BIT0RQWYoqNxRQbS27ZIRZPT4obNaKoUSOKGzWiuFFDrF5edRNe2X+d6YziqVCgFJBuSSfDmkG6Jd3+dVC/IE69d4rvGn6HPlLP6R9Pk5uby+d+n/PT//3EqfdOERgQyJ0P30mIJoQ/vvqD1i1aExYWhtlsZteuXaxatYonn3yyynvFbDaTlZVFQkICCQkJAKxfv57o6Gh0Oh3//vsvK1asYNiwYfzxxx918vOAMxX2z6dGSXdtrdF38OBBYmJinOZ7lzu74nJYWBjp6ekObUpLS8nKyqrydadOncr48ePt3+fl5dGoUSP69u1rX+evPirvvezTp48UuKqHjGYjc/+ey7pjth7l9oHtmX3jbBp6OxaRMm7dSvr817FkZIBGQ8CoUfg//hjtavHfVO4VURN1eb9kZWeTtfRtGv/yK03Gj7/ooW/C9eT3y9Ul/aWZ5BUWom3RghtefhmVW/X/PJR7RdSE3C8XTlEUzCdPYtq7j+K9eyneG4vpUByawkK84uLwiouzt3Vv3BiPdu1sPeLt2+MRHX1J35uz781mbvhcPlr6EamZqfhH+tPpf50oMZSQp+RxOuM0+Uo+3xR9A0BaXhrfLvuWkqwStB5aGkY15IWFL9AiokWV94qnpydt2rRxWMf7iy++YPbs2RQUFNCqVSuWLVvGI4/UbUHI8tHU51OjpLs21+irOPT7bGdXXL7hhhvIyclh165ddO7cGbAVgrJarXTr1q3Sc3h4eDjNYwVwd3e/LP4nv1zivJrEZsQy+bfJ5yyWZjUaSXttHjmrbYVotJGRhM+di759uzqLS+4VURN1cb8EP/44eWvWUpqURP7qzwn874haPb9wHfn9cuUr+vdf8r78EoAGL81AW7bETk3JvSJqQu6XC6Nt3hyv5s3h3gEAWEtKMB06ZCvUFvsvxf/GUnLiBOaTJzGfPElBWdE63N3RRUefWTu8Qwe0TZvW2ZSwEPcQXp/+Oq9Pf91he64p1zY8vbvjWuMMgtBBjtOX17AG8qBpVlO6N+zu9Brl87gr+uSTT2r1OqqjuvdxjYeX18YafR9//DFvv/02UL2Ky61bt+b2229n5MiRLFu2DLPZzOjRo3nwwQelcrmocxarhff3vs/b/76NRbHQwKsBc26ew7Wh1zq0K9y9m+TJUzCfOgVAwPBhBI8bh1qnc0XYQlwyak9PgseOJWXaNDKXLcNw7wDc/P1dHZYQ4jwUi4XUl18BRcEwYACeZR0bQojLg1qrLUukOwAPA2DJyaFo7z77kmVFsbFYsrPLesf3kv1p2bEGA/p27cqWLLOdw61sLnRdMXgY6BTSiU4hnRy2G81G+/ri9qrqOQkkFSTR1LdpncZ0qdQ46a6NNfo++eQTBg8eDFS/4vKnn37K6NGj6dWrF2q1mkGDBvHWW29d7PULcU4pBSlM2TqF3em2JR3uaHoH02+Yjq/2zBQFa0kJmYsWc/qDD8BqxS28AeGvxuB1feWjMIS4EhkG3EPWxx9jOnSIzKVvEzbtf64OSQhxHjmff07x/v2ofXwIeX6Cq8MRQtQCjZ8f3jffhPfNNwFlw9ITEymKjbVVS/83luIDB7Dm5mLctg3jtm32Y90bNkTfoQO6Du1t1dLbtL4knUde7l60C2pHu6AzI0PNZjNfr/+aIF1Qnb/+pXBBhdQudo2+iqq7Rl9AQACrVq2qUZxCXIyNxzby8vaXyTfn4+nmybTrp3FX5F0OQ3GK4+JInjQZU9k8GsO99xL6v6lofHxcFbYQLqHSaAidPImTI/5L9v/9H/5DHsKjWTNXhyWEqEJpVhbpbywEIHjsWNyCrow/bIUQjlQqFdpGjdA2aoShrFq6YjZTHBfv0BtecvQo5sREzImJ5JUPS3dzQ9eyJbqOHdC374C+Ywe0zZqhukTrXmtV2itmVZRLUr1ciMuJ0Wzk1Z2v8m2CbXmGDkEdmHPzHBr5NrK3USwWTi9fTsZbi8BsRhMQQIOXZ+LTu7erwhbC5bxuuAHvHj0o+PVX0l9/nUaLF7s6JCFEFdJffx1rXh4ebVrj/9CD5z9ACHHFULm7o2/XFn27tjDEts2Sn0/xXlsCXlQ+LD0zk+IDByg+cICc//sMALW3N7r27dB36Ii+Ywf07dvjFhzswqu5PEjSLUQFsRmxTNk6hVP5p1Cr1IxsP5JRHUfhrj5TJKHk5EmSJ0+h6J9/APDu1YsGL8/ELTDQVWELUW+ETJpIwe+/U/DTZox//olX166uDkkIcZbC3f+Qu9ZWPC3shRdQaTQujkgI4WoaHx+8unfHq7utaJmiKJQmJ1O0dy9F/8bahqfv34+1oIDC7Tso3L7DfqxbeANbEt6+vW2OeNu2qC+wKOOVSpJuIbAVS/tg3wcs3bPUXiwt5uYYOoeeKSqjKAo5qz8n7bXXUAoLUXt5ETp9OoYB91wxQ1+EuFgeUVH4PXA/Of/3GelzX6PpF59fsmFoQojzU0pLSX3lFQAMgwbiec01Lo5ICFEfqVQq3CMicI+IwPf22wHb7w/T4cO2JHyvbY646UgCpckp5CenkL9xo+1gjQaPFi1sRd46drAtWxYVdVU/4JOkW1z1UgpSmPr7VHal7QIqL5ZmTksnZfp0jFu3AuDZrRvhr87GPSLCJTELUZ8Fjx5N3rffUbx/P3nr1mG4+25XhySEKJP9f59hOngQtcFAyAQpniaEqD6Vmxu61q3RtW6N/4O2otiWggKK9+0vG5ZuW7asNCMD06FDmA4dIufzzwHbSie69u3tS5bpO3TAPTT0XC93RZGkW1zVNh4vK5ZWUnWxtLwNG0iZ+TLW3FxUWi0hE8bjP3So9N4JUQW3wEACR40iY8EC0t9YiE/fvrJ0nhD1QGlmJhlvvglAyLjn6nx5ICHElU/j7Y3X9d0cVu0xp6baqqTvtVVLL9q/H2thIYU7d1K4c6e9nVtoqEO1dH27tqi9vFxxGXVOkm5xVTKajcTsjOGbhG+AyoulWXJySH1lFnnr1wOga9uW8Llz8Gje3CUxC3E5CRg+jOzP/o/S5BSyVqwk6MlRrg5JiKte+rz5WAsK0LVrh9/997s6HCHEFco9LAz3sDB8+/UFbAWITUcSyqql2wq1mQ4fpjQtjfxNm8jftMl2oFqNR1QUuo4d0LZtizYnB6W0FNzdz/FqlwdJusVVZ2/GXiZvncyp/FOoUPF4+8d5qtNTDsXSCrb+Tsq0aZSmp4NGQ9CTTxL05ChUV8D/9EJcCmoPD0LGjSd54kROv/sufvcNkiWJhHChwr//Jvebb0ClImzGi1f13EohxKWl0mjQtWqJrlVLKHvgZzUaKdq/31YxvaxQW2lqKqbDhzEdPgxr1tIUKGrTBm2PHi6NvzZI0i2uGharheX7lrN0z1JKlVLCvMKIuSmGLmFd7G2shYWkzZtnXxZB26wZ4a/NRd++vavCFuKy5Xtnf7I++ojivXvJWLSYBjNfcnVIQlyVFLOZ1JkvA+B3//3yniaEcDm1lxdeXbs6rHJiTku3D0kvjP2Xgj3/4tGunQujrD2SdIurQqoxlalbp/J32t8A9GvajxeufwGDh8HepnD3PyRPnYL5xEkA/IcOJWT8OFnyQIgLpFKrCZ0ymRMPP0LOF18Q8MjDeLRo4eqwhLjqZK9ahenwYTR+fgSPe87V4QghRKXcQ0NwD+2NT+/emM1mNqxbRytf3/MfeBmQSlDiivfD8R8Y+O1A/k77G72bnldufIV5t8yzJ9xKSQnpC97gxCOPYD5xErewMBp/uJywaf+ThFuIi+TZuTM+ffqA1UravHmuDkeIq445LZ2MtxYBEDxhPG7+/i6OSAghqukKKlosPd3iilVoLiTmzxi+PvI1AO2D2jPn5jk09m1sb1McF0/y5MmYDh0CwHDPPYRO+x+aK+SpmhD1QcjzE8j/5ReMv22lYNs2vG+80dUhCXHVSJ83D6vRiK5jB/wGDXJ1OEIIcVW6ch4fCFHBvsx93P/d/Xx95GtUqBjZfiQr71hpT7gVi4XT77/P8fvuw3ToEBp/fyLeepPwuXMk4RailmmbNCFgyEMApM99DcVicXFEQlwdjDt2krduna142osvylKXQohLZsmSJTRt2hSdTke3bt34888/q2z75Zdf0qVLF/z8/PDy8qJTp0588sknVbZ/8sknUalULFy4sNL9JpOJTp06oVKp2LNnz0VeSe2Q377iimKxWnh/7/sM3TCUk/knCfMKY3m/5Yy5doy9OnnJqVOcGDac9Pmvo5jNePfsSeR33+Lbt6+LoxfiyhX01FOoDQZM8fHkfvWVq8MR4oqnmM2kvvIKAP4PPYi+bVsXRySEuFqsXr2a8ePHM2PGDHbv3k3Hjh3p168f6enplbYPCAhg2rRpbN++ndjYWEaMGMHIkSP5559/nNp+9dVX7Nixg/Dw8Cpff9KkSefc7wqSdIsrRqoxlcd/fJw3d79JqVJKv6b9WHPXGnt1ckVRyP78c47eM4CiXbtQe3nRYPZsGi5dIksZCVHHNH5+BD31JADpb76J1Wh0cURCXNmyPvqYkoQENAEBBI8d6+pwhBBXkQULFjBy5EhGjBhBmzZtWLZsGZ6enixfvrzS9rfeeiv33nsvrVu3JioqirFjx9K+fXsOHDjg0C4pKYlnn32WTz/9FPcqlvH9/vvv+fHHH5k/f36tX9fFkKRbXBF+PP4jg74dZC+W9nL3lx2KpZnT0zn15JOkvjgDpbAQz+uuo9k33+A3aCAqlcrF0QtxdQgYMgT3xo2xZGRy+oPK33iFEBfPnJpKxpIlAIQ8/zwag+E8RwghRO0oKSlh165d9O7d275NrVbTu3dvtm/fft7jFUVh8+bNxMfH07bCCB2r1crQoUOZOHGiw/aK0tLSGDlyJB9//DGenp4XfzG1SJJucVkrNBfy4rYXmfDrBPJK8mgX2I4v7vqCe1vca0+m8zZu5Nhdd2P89TdUWi0hkyfTeOUKtA0jXBy9EFcXlVZLyIQJAJxevhxzWpqLIxLiypQ2dy5KYSH6a67BMOAeV4cjhLiKZGZmYrFYCA0NddgeGhpKampqlcfl5ubi7e2NVqvlzjvvZOHChXTq1Mm+f+7cubi5uTFmzJhKj1cUhUcffZQnn3ySLl261Mq11CZJusVla3/mfh5Y9wBfHfkKFSoeb/84H/X/iCa+TQCw5OaS9PxEkp4bhyU3F482rWm2dg2BIx6VYjJCuIhP3z7oO3dGKS4mY+Gbrg5HiCuO8Y8/yP9+I6jVhM2Q4mlCiMuDj48Pe/bs4a+//mL27NlMnDiRvXv3ArBr1y7efPNNVqxYUeUI1UWLFpGfn8/UqVMvZdjVJr+JxWWnvFjaIxse4UTeCUI9Q/mg3weMvXasvVhawe/bOHr3PbaqrRoNQU8/RbPPPsOjRQsXRy/E1U2lUhE6eRIAuV9/TfHBgy6OSIgrh7WkhNRXZgHg//DD6KKjXRyREOJqExQUhEajIe2s0WxpaWmEhYVVeZxaraZ58+Z06tSJCRMmMHDgQNauXQvA1q1bSU9Pp3Hjxri5ueHm5saJEyeYMGECTZs2BWDLli1s374dDw8P3NzcaN68OQBdunRh+PDhdXOxNSBJt7ispBpTGblppL1YWp8mfVh791quC7sOAGthIakvv8ypxx+nNC0NbdOmNF31KcFjxqDSal0cvRACQN+hA7533gmKQtrc11AUxdUhCXFFyFqxkpJjx9AEBRE85llXhyOEuApptVo6d+7M5s2b7dusViubN2/mhhtuqPZ5rFYrZrMZgKFDhxIbG8uePXvsH+Hh4UycOJEffvgBgLfeeot///3Xvn/Dhg2ArZL67Nmza/EKL4ybqwMQoro2ndjES3+8RF5JHno3PVO7TmVA8wH2YSZFe/aQPHkKJSdOAOD/yCOETBiPWq93ZdhCiEqEjB9H/qZNFO7YQcEvv+DTs6erQxLismZOTibz7bcBCJ00EY2Pj4sjEkJcrcaPH8/w4cPp0qULXbt2ZeHChRiNRkaMGAHAsGHDiIiIICYmBoCYmBi6dOlCVFQUJpOJDRs28OmnnzJq1CgAAgMDCQwMdHgNd3d3wsLCaNWqFQCNGzd22O/t7Q1AVFQUDRs2rNPrrQ5JukW9V2guZO5fc/ny8JcAtA1sy9xb5trnbislJWQsXcrpd98DqxW3sDDCX52NV/furgxbCHEO7hERBAwfxun33id93ny8b7oJVRXLfwghzi8tZg5KURH6Lp3xvesuV4cjhLiKDR48mIyMDF588UVSU1Pp1KkTGzdutBdXO3nyJOoK9SaMRiNPP/00iYmJ6PV6oqOjWbFihT1xvhJI0i3qtf2Z+5m8dTIn8k6gQsVj7R/j6U5P2+duF8fHkzx5CqayeaG+d99F2PTpaHx9XRm2EKIaAp94gpw1ayk5epTsL74gYMgQV4ckxGWpYOtW8jdtAo2GsBdelKUwhRAuN3r0aEaPHl3pvl9++cXh+1mzZjFr1iyHbWaz2T5EvDLHjx8/5+s3bdq0Xk1fkzndol6yKlY+2PuBvVhaiGeIQ7E0xWLh9AfLOT7oPkwHD6Lx8yNi4UIiXntNEm4hLhMaHx+CnrW9IWcuWowlP9/FEQlx+bGaTKSW/bEaMHQoulYtXRyREEKIs0nSLeqdVGMqI38cycLdC+3F0r68+0t7sbSSxERODB9O+rx5KGYz3rfeSuR33+J7ez8XRy6EqCn/Bx5AGxmJJTub0+++6+pwhLjsZC1fjvnESdyCgwka/YyrwxFCCFEJSbpFvfLTiZ8Y9O0g/kz9E72bnpndZ/J6j9cxeBhQFIXsL77g2N33UPT3LtSenjSY9QoN316KW3Cwq0MXQlwAlZsbIROfByBr5UeUJCa5OCIhLh8liUlkLnsHgJDJk9FcQfMfhRDiSiJzukW9UGgu5LW/XmPtYdt6fG0C2zD35rk0NTQFoDQjg5QXXqSgbA6IvktnwmNi0DZq5KKIhRC1xfvWW/Hs1o3CnTvJeOMNIl6f7+qQhLgspL36KorJhGe3bvje2d/V4QghhKiC9HQLl9t/ej+D1w1m7eG1qFDx33b/5ZM7PrEn3Hk//MjRu+6m4JdfULm7EzJxIk1WrpSEW4grhEqlInTyJFCpyFu/nqLYWFeHJES9l//zzxRs2QJuboS9MF2KpwkhRD0mSbdwGatiZfm+5Tyy4RGO5x0nxDOE9/u+z7jO43DXuGPJyyNp0iSSxo7FkpODR+vWNF27hsDH/otKo3F1+EKIWqRr0wbDPfcAkDZnbr2qOCpEfWMtLiZt9qsABAwfhkfz5i6OSAghxLlI0i1cIs2YxhM/PsEbu96g1FpK78a9WXvXWro26AqA8Y8/OHr3PeR9+x2o1QQ+OYpmqz9D11KqsgpxpQoe9xwqnY6i3bvJ/3GTq8MRot46/d77mBMTcQsNJfjpp10djhBCiPOQOd3iktt8YjMzts8g15SL3k3PlK5TuLf5vahUKqxFRaS/voDsTz4BQNukCeFz56Dv1Mm1QQsh6px7aCiB/x1B5tK3SX/9dXx63opKq3V1WELUKyUnT3L6vfcACJ06BbWXl4sjEkIIcT7S0y0umUJzIS/98RLP/fIcuaZc2gS24fP/fM7AFgNRqVQU/fsvx+4daE+4/YcModlXX0rCLcRVJPCxx9AEB2E+eZKsVatcHY4Q9YqiKKTOno1SUoJX9+749JOlMoUQ4nIgSbe4JA6cPuBQLG1EuxH2YmlKSQnpb77J8YeGUHL8OG6hoTR6/33CXnwBtaenq0MXQlxCai8vgseMASDz7WVYcnJcG5AQ9UjBli0Yf/0N3N0JnS7F04QQ4nIhSbeoU1bFyof7PuThDQ/biqXpQ3iv73uM7zwed407psOHOf7gQ5x+exlYrfj+5z9EfvsN3jfd6OrQhRAu4jdwIB4tW2LNzSXz7bddHY4Q9YK1qMhePC1wxAg8Ipu5OCIhhBDVJUm3qDNpxjSe2PQEC3YtoNRaSq/GvVh791q6NeiGYrVy+sMVHBt0H8UHDqAxGIh4YwER8+ehMRhcHboQwoVUGg0hkyYBkLXq/yg5ccLFEQnhepnvvIM5ORm38AYEPTnK1eEIIYSoAUm6RZ3YfHIzg74bxM6Unejd9My4YQZv3PoGfjo/ShKTODn8UdLnzrXNS+txC82++xbfO+5wddhCiHrC+6Yb8br5ZjCbSZ//uqvDEcKlTMeOkfXBcgBCp06VqVdCCHGZkerlolYVmguZ9/c81sSvAaB1QGvm3jKXZoZmKIpCztq1pL0ag9VoROXpSeiUyfjdf7/MSxNCOAmdNJGj27aRv2kThX//jWeXLq4OSYhLTlEU0mbNRjGb8br5Znx693Z1SEIIIWpIerpFrTl4+iCD1w22J9wj2o7g0/6f0szQjNLMTBKfGU3KtOlYjUb0nTsT+fVX+D/wgCTcQohKebRogd999wGQNvc1FKvVxREJcenl/7gJ47ZtqNzdCZs+Td4zhRDiMiRJt7hoVsXKin0rGLJhiGOxtC62Yml5P/7I0bvupmDLFlTu7oRMfJ4mH61E27ixq0MXQtRzwWOeRe3pSfHeveSt3+DqcIS4pKyFhaTFxAAQOPJxtE2auDgiIYQQF0KSbnFR0gvTGbVpFK/vet2hWNr1Da7HkpdH8uTJJI0ZiyU7G4/oaJquWUPgY4+h0mhcHboQ4jLgFhRE4BMjAUh/YwHW4mIXRyTEpZP59jJKU1Nxj4ggcORIV4cjhBDiAknSLS7YlpNbGPTtIHak7ECn0fHiDS/ai6UZt2/n6D0DyP3mW1CrCXziCZp+vhpdq5auDlsIcZkJGD4ct7AwSpNTyProY1eHI8QlYTp6lNMrVgAQOm0aar3etQEJIYS4YJJ0ixorKi3i5e0vM/bnseSYcmgd0JrVd63m/pb3oxQXkzr7VU6O+C+lKSm4N25Mk08+IWT8ONRaratDF0JchtR6PSHjngPg9DvvUHr6tGsDEqKOKYpC6iuvgNmM96234nNbT1eHJIQQ4iJI0i1qpLxY2hfxXwDwaNtH+aT/J0QaIinau5djAweR/bGtJ8rvoQeJ/OpLPK+9xpUhCyGuAL533YWubVusRiMZixe7Ohwh6lT+xo0Ubt+BysOD0OnTXB2OEEKIiyRJt6gWq2Jl5f6VDNkwhGO5xwjWB/Nun3eZ0GUC7lYVGW8t4viDD1Fy7BhuwcE0eu9dGsyYgdrLy9WhCyGuACq1mpDJkwDI+fwLTAkJLo6ofluyZAlNmzZFp9PRrVs3/vzzzyrbfvnll3Tp0gU/Pz+8vLzo1KkTn3zyiX2/2Wxm8uTJtG/fHi8vL8LDwxk2bBjJyckO57n77rtp3LgxOp2OBg0aMHToUKc24vwsBUbSYuYAEPjESLQNG7o4IiGEEBdLkm5xXhmFGTy56Unm/z2fUmspPRv1ZO3da7kh/AZMCQkcf/AhMpcuBYsF3/79ifzuW7xvvtnVYQshrjBeXbvi3asXWCykvzbP1eHUW6tXr2b8+PHMmDGD3bt307FjR/r160d6enql7QMCApg2bRrbt28nNjaWESNGMHLkSP755x8ACgsL2b17Ny+88AK7d+/myy+/JC4ujrvvvtvhPD179uTzzz8nLi6OtWvXkpCQwH1lS76J6stcupTS9HTcGzcm8PHHXR2OEEKIWnBBSffFPkH/+OMzhXCq+wS9adOmqFQqh485c+ZcSPiiBn4++TMDvx3I9pTt6DQ6Xrj+Bd7s+SZ+WgNZK1dy7N6BFO/fj9pgIPz1+UQseB2Nn5+rwxZCXKFCnp8Abm4U/Porxu3bXR1OvbRgwQJGjhzJiBEjaNOmDcuWLcPT05Ply5dX2v7WW2/l3nvvpXXr1kRFRTF27Fjat2/PgQMHADAYDGzatIkHHniAVq1acf3117N48WJ27drFyZMn7ecZN24c119/PU2aNKF79+5MmTKFHTt2YDabL8l1XwlMhw+T9dFHAIRNn4baw8PFEQkhhKgNNU66a+MJ+ogRI/jhhx+A6j9BB3j55ZdJSUmxfzz77LM1DV9UU1FpEa9sf4UxP48hx5RDdEA0q/+zmgdaPUBpcjInHx1BWswclJISvG6+mchvv8Vw552uDlsIcYXzaNYM/wcfBCBt7msoFouLI6pfSkpK2LVrF71797ZvU6vV9O7dm+3VeEihKAqbN28mPj6etm3bVtkuNzcXlUqFXxUPWbOysvj000/p3r077u7uNb6Oq5GiKKS+/AqUluLduxfet9zi6pCEEELUEreaHlDxCTrAsmXLWL9+PcuXL2fKlClO7W+99VaH78eOHcvKlSv5/fff6devn/0JekWLFy+ma9eunDx5ksaNG9u3+/j4EBYWVtOQRQ0dyjrE5N8mczT3KADD2wxnzLVjcFe7k/PlV6TNno3VaESl1xM6eTJ+gx9ApVK5OGohxNUi6Jmnyf3mG0yHDpH79Tf4DRro6pDqjczMTCwWC6GhoQ7bQ0NDOXToUJXH5ebmEhERgclkQqPRsGjRIkJCQiptW1xczOTJk3nooYfw9fV12Dd58mQWL15MYWEh119/PevWrbv4i7pK5K1bT+Fff6HS6QibOtXV4QghhKhFNerprq0n6HFxcdxyjie4VT1BnzNnDoGBgVxzzTXMmzeP0tLSmoQvzsNeLG39EI7mHiVYH8w7fd7h+eueR52TT+LoZ0n53/+wGo3or7mGyK+/wv/BwZJwCyEuKTd/f4KefBKAjDffxFpY6OKILn8+Pj7s2bOHv/76i9mzZzNx4kT27t3r1M5sNvPAAw+gKApvv/220/6JEyfyzz//8OOPP6LRaBg2bBiKolyKS7isWfLzSXttLgBBTz6Je0SEiyMSQghRm2rU011bT9CXLl1Knz59Km1b1RP0MWPGcO211xIQEMAff/zB1KlTSUlJYcGCBZWex2QyYTKZ7N/n5eUBtj8Y6vP8svLYLnWMGUUZzNg+gx2pOwDoEdGDF7u9iL/On+yNP5Dx8kwsWdng5kbg6NH4PToclUZTr3+WVzpX3Svi8nSl3S8+Dw4ma9UqSpOSyHj/AwKeetLVIdULBoMBjUZDUlKSw791SkoKISEh5/z3b9KkCQBt27Zl7969rF27lvHjx9v3m81mHnroIY4fP86PP/6IXq93Op/BYMBgMNCsWTOaN29OZGQkv//+O9dff30tX+mVJePNt7BkZOLepAm+Qx+5rP4/vdJ+t4i6JfeLqK7L5V6pbnw1Hl5+IcqfoBcUFLB582bGjx9PZGSk09Dzcz1Br/jG36FDB7RaLaNGjSImJgaPSgqNxMTEMHPmTKftP/74I56enrVzYXXo7CH3demQ+RBfFn5JoVKIO+7cob+D6wquY+f6nwn+7lsMu3YDYAoLI2XwA8Q3CIOyOfnC9S7lvSIuf1fS/eLdowfhq1aR+f77/OXvh+Wsoc5Xq8jISD788EP7XGqr1cr3339P//792bBhQ7XOkZiYiNlstt8vpaWlzJs3j5SUFF555RV27tx53nNkZGQA8Ouvv5KVlXWBV3Pl06ak0GTVKlTAsd692P/TT64O6YJcSb9bRN2T+0VUV32/VwqrOdquRkl3UFAQGo2GtLQ0h+1paWnnnGutVqtp3rw5AJ06deLgwYPExMQ4JN3lCfeJEyfYsmWL0zyxs3Xr1o3S0lKOHz9Oq1atnPZPnTrVIVHPy8ujUaNG9O3b97zndqXyP3L69OlT58VnikqLWPjPQr44/AUALf1a8uqNrxJpiKTwzz9Jn/4CpSkpoFLhN2IEgc88TVuttk5jEtV3Ke8Vcfm7Eu8X5Y47SNq/n+J//+WagwcJqeRB69WooKCAxx57jIEDB3LdddexaNEiLBYLr776KqGhoYwYMYLw8HBmz54NwNy5c+ncuTORkZGYTCY2btzIr7/+yqhRo+yj0gYPHkxSUhJfffWVw2i3gIAAtFotf/75J3///Tfdu3fH39+fo0ePsnz5cqKionjuuecqfTgubNPukh59lGKrFa8+fbj1uedcHVKNXYm/W0TdkftFVNflcq+Uj6Y+nxol3Vqtls6dO7N582YGDBgA2J6gb968mdGjR1f7PFar1WHod3nCffjwYX7++WcCAwPPe449e/agVqurLPTi4eFR6Zu8u7t7vf6HK1fXccZlxTHpt0n2YmnD2gxj7LVjcTNbyZg/n6yVtiVL3Bs1InxODJ6dO9dZLOLiXC73tKgfrrT7JXTKZE48NIS8r74mcPhwdJU8hL3aPPzww2RnZ/Pyyy+TmppKp06d2LhxIw0bNgRsvdhubm72+6C4uJgxY8aQmJiIXq8nOjqaFStW4O3tjbu7O0lJSfaCaNddd53Da/3888/ceuut+Pr68s033/Dyyy9jNBpp0KABt99+O9OnT8fb2/vS/gAuIzlff03x7n9QeXrS4H9TL+v/N6+03y2ibsn9Iqqrvt8r1Y2txsPLx48fz/Dhw+nSpQtdu3Zl4cKFGI1GezXzYcOGERERQUxMDGAb5t2lSxeioqIwmUxs2LCBjz/+2D583Gw2c99997F7927WrVuHxWIhNTUVOPMEffv27ezcuZOePXvi4+PD9u3bGTduHI888gj+/v41vYSrmlWx8smBT1i4eyFmq5kgfRCzb5xN94juFO3dx6nJkyk5akvE/QYPJnTSRNReXi6OWgghKud5zTX43HE7+d9vJH3uazT64H0p7giMHj26yofhv/zyi8P3L770Mg8+NZF9SbnsS85lX1Iecw4X4K6YWZn0Jw0MemZ+u59QXw/CDDpCfXWE+eoIM+jQuWsAaN++PVu2bKnry7qiWPLySJ83H4Dgp5/CvUEDF0ckhBCirtQ46R48eDAZGRm8+OKLDk/Qy4ebnTx5ErX6TFF0o9HI008/7fAE/ZNPPmHw4MEAJCUl8e233wK2oecVlT9B9/Dw4LPPPuOll17CZDLRrFkzxo0b5zB8XJxfZlEm03+fzrbkbQDc2vBWZt44E3+NDxmLl5D59ttgsaAJDiJ81iy8e/RwccRCCHF+IRMmUPDTZox//IFx61ZZ3/gcis0WDqbksS85j/1lSXZcaj5mS2UVxlVkncwBcqo8n0HvTpivjlCDjjBfjwpf25LzUF8dgV5a1Gp5EHK2jDffwnL6NNrISAKGDXN1OEIIIerQBRVSq8kT9FmzZjFr1qwqz9W0adPzLidy7bXXsmPHjhrHKc749dSvvLDtBbJN2XhoPJjYZSIPtHqAkmPHOD5pFMX79gHgc8fthL34Im4ygkAIcZnQNmyI/9ChZC1fTtprr+HVvTsqt0tSJ7ReM5pKOZiSx94kW+/1/uRcDqcXYLE6v+ca9O60i/ClXbiBdhEGIgN1bPplK5FtryXTWEpaXjGpecWk5hbbvy42W8ktMpNbZCYuLb/KONw1KkJ8dJX2lFfWa341KD5wgOz/+z8Awl58AZXUSxFCiCua/FVyhSsuLWb+3/NZHbcagFb+rZh7y1wifZuR/fHHpL++AMVkQu3rS9iMFzHceaeLIxZCiJoLenIUuV9+ScmRBHLWrMX/wcGuDumSyi0ycyA5r8IQ8VyOZhqp7Jl2oJeWdhEG2kX40j7CQNtwAw399Q7D8s1mM0d84I52YZXOV1MUhbyiUlsinmdLxNNyz3xtS9BNnDaaMFsUknKKSMopOuc1+OrczpmUXym95orVSurMl8Fqxbd/f7xkOTUhhLjiSdJ9BYvLimPyb5NJyE0AYGiboTx37XOo0jI5+dxjFJaNHvC68UYavDob97PWXxdCiMuFxteXoGeeIW32bDIWLcL3P3eiuUILeGUZS9ifnMvepFz2J+WxLzmXE6crX7IkzFdHuwhf2oYbaB9h68UO9fW46HnvKpUKg6c7Bk93WoX5VNnObLGSnm8600Ne9jnNnqzb9hWZLeQVl5JXXEB8WkGV5zu71zzEx5aQlw9nL/9ar62/vea5X31F0b//ovb0JGTyJFeHw5IlS5g3bx6pqal07NiRRYsW0bVr10rbfvnll7z66qscOXIEs9lMixYtGDt2LAEBAQ5tli1bxq5du8jKyuKff/5xmj4IsH37dqZNm8bOnTvRaDR06tSJH374Ab1eX1eXKoQQLiNJ9xXIqlj59OCnvLHrDYdiaTeE30DuN9+QNms21oICVHo9oZMm4vfgg1J4SAhx2fN/cDDZn35KyfHjnH7vfULGPefqkC5aen4x+5PKh4jnsj85r8oe44b++rLh4b60jTDQLtxAsI9rl+py16iJ8NMT4Vd1IqUoCnnFpfakPLWSXvO0PBOZBRfXax5S/rWvjlCDB0FeHpe819ySk0P6/NcBCBo92uUPu1evXs348eNZtmwZ3bp1Y+HChfTr14+4uLhKV4cJCAhg2rRpREdHo9VqWbduHSNHjmT69On0798fsNXyuemmm3jggQcYOXJkpa+7fft2br/9dqZOncqiRYtwc3Pj33//dagJJIQQVxJJuq8wZxdL69GwBy/f+DK+hZA0Zgz5m34CQN+xI+Fz56Bt2tSF0QohRO1RubsTMvF5Ep8ZTdaKFfg/OPiyqQitKAopucW24eFJuewrGyqenm+qtH2zIC/ahvvahomXJdp+nhc5L9iUDxnxkHEIdWYCzdMSUcXmgW8YeAWBV7Dtw612E3mVSoVB745B707L0HP3mmfkmxyScscEvWa95m5qFSE+Hg6F38IcisDZetM9tbX3p1L6woVYsrPxaNGcgKGP1Np5L9SCBQsYOXKkfQWaZcuWsX79epYvX86UKVOc2t96660O348dO5YVK1Zw4MAB+7ahQ4cCcPz48Spfd9y4cYwZM8bhNVrJcn9CiCuYJN1XkN8Sf+OFbS+QVZyFh8aD57s8z+BWgyn4+WeOvvAiltOnwd2d4NGjCXzsv1JoSAhxxfG+7TY8r7uOwr/+Iv2NN4h47TVXh+REURROZRWxr2yIeHkPdpaxxKmtWgVRwd60izDYk+y24b746C5izdLiPMiIg4xDFT7iIPeUvYkGaAuQvNr5eA/DmSTcO/hMMu70EQR6f6ilkVTuGjXhfnrCq9Frnl5J4bfUXJP968wCE6VWheTcYpJzi8/5uj46N6c55uWJek16zYv27iNn9ecAhL7wAioXrztbUlLCrl27mDp1qn2bWq2md+/ebN++/bzHK4rCli1biI+PZ8CAAdV+3fT0dHbu3MnDDz9M9+7dSUhIIDo6mtmzZ3PTTTddyKUIIUS9J1lXLXPF3KhbetzC1t+2Omx78NEHuX/Af0iZNp3cL78EwKNFC8Jfm4uudevavWghhKgnVCoVIZMnc/y++8j79jsChg5D376dy+KxWhWOnTae6cEuqyKeV1zq1NZNraJFqA/tynuwI3xp3cD3wntai3IgMx7SD5Yl2WWf85KqPsY7DIJbYfFvRvKxw0T4e6AuzARjhu3DWgqmXNtHVsL5Y1C7nUnAq0rOyxN3zyBw113YtZap2Gveopq95un2Ye0mh3nnqXnFFJZYyC8uJb+4gMPp5+81Dzm7CJzBw9Zr7q2FmTNBUfC96y68qvi74FLKzMzEYrHYl3wtFxoayqFDh6o8Ljc3l4iICEwmExqNhkWLFlU6FL0qR48eBeCll15i/vz5dOrUiY8++ohevXqxb98+WrRocWEXJIQQ9Zgk3bXIFXOj4rPjOXD6AP49/Am5N4T7Wt7HyA4j0cYlcOzuezAnJ4NKRcB/RxA8ZgxqD9fO7xNCiLqmb9cWwz13k/vNt6TPnUvjjz+6JHUrSi1WEjJsCfbepFz2J+dyIDkPY4nFqa1Woya6gQ9ty4aGtws30CrM58KWzSrKtiXT9uS6rPc6P6XqY3waQHC07SOk7HNQS/C0PfS1ms3s3rCBsP79UZf3yCoKFOeAMRMK0s8k4sZMMKZX+DoDCjJsibm11BbHuWKpyMPXsafcnpiHOCfuOj+4wDnA1e01zzeVnhnKbi8CZzoz3zy3mIxq9JrfcXwHY/bto9BdxzTDTXh9sPPMEPazhrYHenugqccV2n18fNizZw8FBQVs3ryZiRMn8vzzz9v/bjkfq9UKwKhRo+zD2q+55ho2b97M8uXLiYmJqbPYhRDCVSTprkWXcm6UoiisOrSKBX8voKi0CH9Pf96/7326B11HxoI3yFq5EgD3hg0JnxODZ5cutXilQghRvwU/9xx5G3+g8O+/Kdi8GZ/evWv1/CWlVuLT8u1VxPcl5XEwJQ9TqdWprc5dTZsGZ+Zft43wpUWID1q3GiaMhVlnEur0CsPCC1KrPsY34kxyHdwKQlrbkmu9X81eG2zDxPX+to+gavRGlprOJOFOHxUT97I2VjOY8mwf1e1F96xsmHt5cn5Wol7DXnSVSoWvzh1f3bl7zUstVjIKHCu0V+w1L8jIZMSBDQB8FN2P3QUaOJxZ5fk05XPNz+o1tyfoZcm5l8fF/QkXFBSERqMhLS3NYXtaWhphYWFVHqdWq2nevDkAnTp1Yv/+/axdu5bJkydX63UblNVZaNOmjcP21q1bc/LkyZpcghBCXDYk6a4ll3JuVGZRJtO3TWdbkq1Ymp+HH8V/FTOg3V0ElJbSQ6vlqcAgGgweTMjkyWi8vS7q2oQQ4nLj3qABASMe5fSyd0ifNx/vW25Bpb2wQmPFZguHUvPL5l7bkuy41HzMFudFsL093GgT7msvbtYuwkBkkBdumhok2MbTZQl1hZ7r9EO23uSqGBrZkuqyBHvJhr3Me281qWlH6djRh0WLptC184VNdTKbzUyfPp0NGzZw9OhRDAYDvXv3Zs6cOYSHh9vPk5WVxbPPPst3332HWq1m0KBBvPnmm3iHR5z/miv2opcn5gXpVSftxWW96AWpto+0875CWS96VcPcgyr0qNesF91No6aBQU8DQ+W95ikvvEBOSSHuLVsy+s3J3G+0OPSUV/w6s8CExWorqpdyvrnmHm5nFYFz7DUP9dURdI5ec61WS+fOndm8ebP97w6r1crmzZsZPXp0ta69/Biz2Vzt9k2bNiU8PJy4uDiH7fHx8dxxxx3VPo8QQlxOJOmuJZdqbtSutF2MjRtrL5Y2ocsEcp/MwX//fnQbfyCu0MiC06dJa9uWb195udauTwghLjeBj48kZ81aSk6cIPuz1QQMG3reY4ymUg6m5JUNEbfNvz6cXoDF6pxg++rcaBdhW//atkSXL00Dvaq/DFVBhnMxs/SDUFh1LyiGxmXDwcsT7NYQ3BI8zvTErl69mvGvvFlrU50KCwvZvXs3L7zwAh07diQ7O5uxY8dy99138/fff9vP8/DDD5OSksKmTZswm82MGDGCJ554glWrVp3/Z3FRvegVE/OzEvWCjEp60Y9WIx5NWSJ+Vm+5U4962X73yhPuoj17yPliDQDhM17EM9yfc11dqcVKZkGJUxG4s6u1G0ss5JtKyU8v4Mg55ppr1CqCvcsrtHs4FYF7+PGnmfTsKLp06ULXrl1ZuHAhRqPRPmJv2LBhRERE2Id8x8TE0KVLF6KiojCZTGzYsIFPP/2UUaNG2V8zKyuLkydPkpycDGBPrsPCwggLC0OlUjFx4kRmzJhBx44d6dSpEytXruTQoUOsWbPm/P82wqVcUbto1KhR/PTTTyQnJ+Pt7U337t2ZO3cu0dHRdXmpQtQqSbpdrLpzo0yltmVjYnbGoG+ip4V/C167+TUaZWtI/nkKxbGx4OZG5/sf4JrbetJ3wAASEhKIiopyxWUJIYTLaby9CH72WVJnzCBzyRIM99yNxmCw788rNrO/LLEun4d9NNOI4pxfE+iltRc3s/ViG2jorz//XHFFsSV9Z8+3zjgEhaerPs6vieN86+BWENQKPLzPe921PdXJYDCwadMmhzaLFy+ma9eunDx5ksaNG3Pw4EE2btzIX3/9RZey6UyLFi2if//+zJ8/36FHvFa4eYAhwvZxPopi6xl3mHtexRB3Y7qtrWKBgjTbR3Vofc7qLQ9C0QeSOm8zAIbeN+DZyMs2ikHvX2UvuptGbZvbbdBBo6pfLr/YfGZ++Vm95eWJeka+rde8PFn/t9IzBeF586OMHDuJ0oJsAhq34Pbn32LlP9mEGYrYc/AIOUW2NdSDvD0wGo08/fTTJCYmotfriY6OZsWKFXh7n7kvv/32W/u9B/Dggw8CMGPGDF566SUAnnvuOYqLixk3bhxZWVl07NiRTZs2yd8s9Zyr1nXv3LkzDz/8MI0bNyYrK4uXXnqJvn37cuzYMTSaC6iDIYQLqBSlsj8vrjx5eXkYDAZyc3Px9fWt9fOXlJTg6enJmjVrHIaHDx8+nJycHL755ptqnee///0v//zzD3/++SfuZcVr4rPjGf3FaDaN2kTUzChG3j6SsdeMofCztaS//jpKcTFqX1/CXngB3//cSWFhId7e3mzcuJF+/frV+rWK+sFsNrNhwwb69+9vv1eEqMrVer8opaUcu/deTIePUDhgMH/ePrRsHexcTpwurPSYUF8Pe2JdnmiH+erOnWArii1BO3u+dcZBW6GzSqnAv6njfOvgVrY519oLmxZ0se9F5VOd7r77biZNmsT//ve/Su+Xn376ib59+5KTk4Ovry/Lly9nwoQJZGefudbS0lJ0Oh1ffPEF99577wVdj0uUlthGG1TsKa+qaFxBuq0XvRJZhz1J2+WH2t1K1J3puOnK5vvbe9HPMf+8Yo96Fb3o572MSnrN7T3n9m0mCkzOlfQrc3aveWjZUPZgL3eOHvyX2266AT9vHd4ebnjr3PDSutXrgnCi5rp168Z1113H4sWLAdvUgkaNGvHss89W+kCvMtdccw0tWrTg008/dfjdcvz4cZo1a1ZpT/fZYmNj6dixI0eOHJEHNVewy+XvlurmmNLTXUvqYm5UxWJpBXm24WPTrp/GI41uJ/mJpyncvgOA/2fvvsOaut4Ajn+TQNh7uhHFgXsrdYtg7bKuam1drda2ttZZV93WPetqXXVWa2vVn1rFBVqlarVoBcS9ZShLZkJyf39EIhEUVCCA5/M8PCa5673xEvLec857rHyaUer77zF9XPgkJCQEeFKsRBAE4XUS/SiN0LuJ+uRaquDH0MtXMN31G+uSPYm0ctKvW8begpplbPVdxGuUtsXV5jkFtyRJV4k7a3fwzBbstPhnbCQDx4qPu4JXfdKC7eQFSst8PffCGOqUlpbGt99+S8+ePfVfMCIjI7Otb2JigqOjI5GRzyn0VhSZKMG2tO4nNwat6E+6uGfcv03Mzu1ABi6tnDEpY/N4LHr8S7SiWz9n/HnWMequBq3oeW01T0rPeKoIXNYEPZ2ohDSiH6Xl0mquYEX4qWyvWikVWD1Owm0e/2ttZoK1mSk2mY/1rxk+z7rczES0ZhqbseZ1f1pycjJr166lYsWKlCv3nAtbMLrCHooQGxvLxIkTCQgI4NatW7i4uNCpUyemTp2KXZZebsYiku58NGzYMPr06ZMvY6Mepj5k8qnJBEYEon6opqqiKpe4BAfC2DN0Ho7p6bhaW5Pc+2O2JSXx1t27OKWlcf78eYYOHUrLli2pXbu2Md8OQRCEAiVJuoJTuuQ6kdDHXcSjH6UbrmjtSUvXKjSIvsTYu4e5N+Q7XRXx0rY4WD2juJokQeK97MXMYiJ0U2HlRCYHR88s1cIzu4V7vXRrZWHJ61AntVpN9+7dkSSJ5cuXGynaIkQm01WCt7AH58r6l6PHjkObmoGZd3UcFm2DzC6wWVvRn9nFPcuPRgWqJN1P3PU8xCPPQ0X3LM+VllibmVDZ1ZrKrs8euqDRSjxIyt6VPTIxjcj4VG5GPkRuZqkba56m1hcZTFZpSFZpsv9OviClQq5rPTdT6BL2rMl6ZkKf2cJuZpjg65J3U6zNTbA0VeS95oJgwFjzumdatmwZo0aNIjk5mapVq3LgwAGUL1kcUyh4xhiKcO/ePe7du8fcuXPx9vbm5s2bDBo0iHv37hWJehEi6c5HH3zwATExMUyYMIHIyEjq1q3Lvn379B9Qt27dQp5lHNezxkbdNbvLB39+QGxaLKnnUrm58iZX0U2f0n/uXAC+9vZm9h/biTYx4dBHH7Fo0SKSk5MpV64cXbp0Yfz48YX/BgiCUGCMUbwmLS2N4cOHs2XLFtLT0/H392fZsmXZvnQVBkmSuB2byoXH468zk+yHyaps68pkUMnFmpqlbfVdxL16eRD9QXe8wk7ia5OIpVelzB1Dwp0nXcH13cIjdEW3ciJTgFMlg2rhuFQDp8ovPC1VfivIaaAyE+6bN29y+PBhg2507u7uREcbVlfPyMggNjb2ucctiVLOniVh+3YASk2YgCzrmNMXbUVPT8zjvOjRj1vRtY+XRcNzit3rKa2fX9H9cTV3hZULbtYOuNmaU+epXTzpAtpC3wU0PUNDUloGSekZPHr8b+Zz/Y/BcrX+tUfpGSQ/fpw5x71KoyU2WUVsMkBqHk4sZzIZWCufJOcGLerZWuRNsTJTPEnan2p9N32RGQleY686r3umXr160b59e+7fv8/cuXPp3r07x48fx9zcuJ+5Qs4KcxrlTDVr1uT333/XP69UqRLTp0/no48+IiMjAxMT46a9IunOZ4MHD35md/LAwECD59OmTWPatGn65+madOadmscvl34BoLJ9ZWbPmo17jzvcnzABzYMHYGKCy+Avcfr0U2QmJpQDgoKCCup0BEEoAoxVvGbo0KHs2bOHbdu2YWdnx+DBg+ncuTPHjx8v0PPVaiWuP0x+PEXX427idxNITMs+9lQhl+Hlaq2vIl6zjC3V3G2zz2GsdUD1th/xu/YRPX4oFQZUR/bgcXKtekb1Z7kJOFYyLGbmUl2XcJuYFcCZv7qCmgYqM+G+fPkyR44cwcnJyWD9Zs2aER8fz5kzZ2jQoAEAhw8fRqvV0qRJk1c/sWJCysggcspUAOy6dsEil7GpzyWTgbmd7scpD+NWM1S64nzPq+KeNWk3aEW/kYd45GDplG38udzCCYckw9ZjMxMFZtYKnKxf7fdEo5VIVj2doD9OyNN1CXpSlqT9WQn+o7QMNFpJNzrk8XavysxErk/CsyXw5k91oX+8js3TrfPmJliYKnIvyFgEGGte90x2dnbY2dnh5eVF06ZNcXBw4I8//qBnz54vfjJCgSoqQxEA/ThrYyfcIJLuIiX0QShbLm0BoGeVngz1/oy4OQu485vuro2ZV2VKzZyJRY0axgxTEIRCZow7xgkJCaxevZrNmzfTtm1bANauXUv16tX5+++/adq0aT6cma7Y09WYZP3468xq4pktXFkpFXKquttQs4wtNUrrkuyq7jaYm2ZpSdRqIeEW3HiqmFnMJVxkqSSYuJJ6LYpH/wvHtvzjeZDlprpW6qzFzFyq6RJuk+LXfTE/hzqBLuHu2rUrZ8+eZffu3Wg0Gv04bUdHR5RKJdWrV6dDhw4MGDCAFStWoFarGTx4MD169Mj/yuVFWNwvW0i/eBG5nR2uw4YV7sFNlGBbSveTm6yt6HmZFz017nEr+uPnWSiAlkDG5erg/Va+npJCLsPW3BRb81croiRJEukZWoOk/FG6Wp+cGybwT1rcc2qdT1XrPpvSM7SkJ6l4kJS9t82LkMt4nLCbPu4+b4K1ualBl/mnE/psXegft8qbFGDru7Hmdc+JJEm6/9P0Vxu2IBQMYw9FyBrH1KlTGThw4EvvIz+JpLsIqe9Wn8F1BpNwJYHPaM7dzh+gvnsXZDIc+/XDZcjXyM2KZuuKIAgFw1h3jM+cOYNarcbX11f/WrVq1ShfvjzBwcEvlXSrMrRcinr0eIquRC7cSyD8fiJpam22dc1N5VQvlTk9l66buJerDUqTx18qtVqIvwHXgg2LmT24BOqcq5KbWJni1NCSB3+nER1RHuuvv0VeppZuHLai6FZGfVH5NdQpcxqou3fvsmvXLoBsQxCOHDmiv8mzadMmBg8eTLt27ZDL5XTp0oXFixcX/AkXERkxMcQsWgSA69ChmGQZzlHkvGgrukb9VEL+pIu79u5Z5DeOofjfl1C6NtgXveJWMpkMc1MF5qYKXGxe7XtUhkZLcrpGl7Q/laAnp2dvkU/SJ/Tqxwm+btx7UnoGWgm0EiSmZeTYk+dFWZgq9K3oVjl2mX9GF/rHY94zk3szE3mOre/5fUMPcp/X/dq1a2zduhU/Pz9cXFy4c+cOM2fOxMLC4oW7qAtFW34NRQBdRfG33noLb29v/VSFxiaS7iKmT+UP+WfpMO7+tRYkCdMyZSg9cwaWjRoZOzRBEIzAWHeMIyMjUSqV2NvbZztuXqpRp6k1XIx89LiLuC7Jjoh8hEqTPcG2UiqoUdqOGo/nwK5V1g5PZytdq41Wo+v6GvMXXM0yHdeDy5DxjLGdCqVu2q3M7uCZLdgOFXFSqYnv8CbqqCjiziTgVLdqnt+T4uRVhjrBk3G6AB4eHuRldlFHR0c2b978cgGXANFz56JNSsK8Zk3su3U1djj5S2H6zFZ0TWoSCT+0wCHlGvzWH/rtLVE3sZ5mopBjZynHzvLVW99T1ZpsreqPsrS+G4x5f7qLfWYLfVoG6Rm6z9VUtYZUtYaYVyxcZyKXPaOqvBctPxrKkJFjSIp/QPnK3oyYv44z0VqsEmIIvXSNpHQN9+JTsTY34VFS0ivP625ubs6xY8dYuHAhcXFxuLm50bJlS06cOPFKraBCwTH2UIRHjx7RoUMHbGxs+OOPP4rMdGMi6S5C0q9e5c6QIThe0RVNs+vaBbfRo1FYP7uiqCAIQk7y845xblJUGYRljr1+/O/l6CQ02uyJmq25ib64WY3Suqm6PJyskEuZyfW/EHER/orQJdgPLoHmGV8gFWbgUsVwvLVLNd3c14qc/7zJLUxw+eYb7o8Zw4MVP2LXuTMmDg7592YIr6WU06dJ2LkLZDLcJz5VPK2kMzHjH48v8b02BdmdU3BwEvhPN3ZURZ5MJsNSaYKl0oRXTR1VGdrsrezpah6l6VrWM5P2p7vMPz1GPkmVgSRBhlYiPkVNfEoO3cCdm2Pfvzn2gBZYGQFEnNUtaz2Gh4DPzMOPV/bBqmcLamdJ4HcnK0i4/YD/xf2L0lSBiVkdhm0NwVQhw0Qhw0QuR2kix0QuY/6BS5jKZbz77Q90lsswVcgxUej+DUuRcencPUwVma/LMZXLdP8qnqxrIpejzHyskGEql2P6eP+mCrmYS74AGHMoQmJiIv7+/piZmbFr164iVWhPJN1FiNzamoyoaDKsrSk343vs27c3dkiCIBiZse4Yu7u7o1KpiI+PN2jtjoyMIs3EhlXHrumT7KsxSeTUEOpopdQl2JlVxEvbUc7eFFns9cfjrS/Clcfjrh9cfnZybWKua7nOOt46M7mWv3hyY/feu8RuWE96WDgPlizF/Tsx24Pw8iS1Wl88zb57dyxq1TJyRIUvxcwFzds/YPJbbwheAhXegGqi629hUZrIUZoonz0FYh5ptRIpao2+ON2jp8a4J2frMv/UGPh0tb77/NPTxkWR9fNdTlh8TM5BFDKZDEyfkbCbyp8k+U+Set1zU8WTxF13w+Dxv3J5ltcMbyIY7P+pmwiZ25ooZI9vEhju/3k3EYriNHjGGIqQmJiIn58fKSkpbNy4kcTERBITdbOQuLi4oDDyzVCRdBchpm5ulFq8iKPXrlHtqUJIgiC8nox1x7hBgwaYmpryw8btONdsxYV7CZz+9z9u377Fr7cs2Lkn3GB9N1sz3dzXj5PsWqUscc+4hyzmIsQcgYvhcOxxcq19Rhymlo+7hVfLUjG8GtiXf6nk+llkcjluo77lVt++xG3dikOvXph5Vsy3/Quvl9hNm0i/fBmFvT0u3wwxdjgvLL+mI5SqdoSmX7J97UJWvNOZMw8tiI2Lz3E6wtatW2ebeeWzzz5jxYoVBXWaQh7I5TJ9izS8Wgvhs6aNi09J58y/IXjXqIUkk6HK0JKhlcjQaFFrJNQa3XO1RkvG4+dqjUSGVvdcpdGSkWUdtebJthnaLPt4/PzJ/qUchzdJkq6ngK4cXvYCnsWBQi7LnsQ/TvKfJPGZNwSefxPBRCFHqXiyrenjmwi69R7vwySHmxRPHb9y0/YMHTeFMePG8yAmGu+atVm3dQeSuR0Pk9K5duMmWnTvvalClmttEch9KMLZs2c5efIkgL7RIdP169fx8PAo8P+L5xFJdxFj0bAhmui8TK4pCMLrojDuGJ8OucDp67FEZ5hzK03JhbuJmNXwZfK40Th3HIrMzJK4AyswK10Nzxr1dMXNSttRy92CWpYPcEq5DtGnda3Xly7CwyugfUZhIFOrLC3WWSqG25UHeeHMfWvVtAnWbdqQdOQI0XPnUm7Z0kI5rlCyqKOiefDDEgBcRwwvdkMV8n06Qt9JJP+6g+Zlr9G9mRsDNsQ/89gDBgxgypQp+ueWlpYFcYqCkTxr2ji1Wo3p3X/p2KhsoY+1lSQJjVbKntTrk/7MJF5CrdWizsjhBsDjdbPfAMhcL+v+smyrzekGwZPXMrTSc29APLmRkL1bmUarO6/Msf1FRw2UH62gNBAPfH04CQ4f0i3yGcUdoMr4PwEwkftg0uMNKj9O6jPkMpbekqFKS8WrYRLeZRzo27cvffv2febRWrdunaf6I8Yikm5BEIQiLj+rUUuSxP2EVOYt38C88d/otxnYTzeFmN0bPbFv3gsAx3YDUFsoif3fTCSNmjeaNmLl6A+pZBqsS67DI+D41Wcn10prw2JmmS3YtmULLbl+HteRI0g6epSkw4dJPnkKqyY5t+4JwrNEz56NNjkZ8zq1sevc2djhvLB8n47QRMnH8/fBihbciLz73GNbWlo+d4iMIOQ3mexxl28FhlNNFiOSJGVr5X92wp7lJkJm4v50D4EcbjhkaLWPbyg8Wed5NyBy7o2QZdss22Q8vkHwtIzHN0PS0GIwEgEZ2hzWL45E0p3P8qubVtZ1VqxYwZkzZ4iNjc2xm9Znn33GwYMHuXfvHtbW1vj4+DBr1iyqVatWkKcqCEIhetlq1JIkcTHyEfv+u8eGM5eZfD6Q2GQ1UJkK3+7WbyOTQSUXa+q6m+FjH09N5X3Ka25j3tAKYrwg9hpIIRAckj0Apc3j7uBZipm5VAW7srodF1Fmnp44fPABcZs3Ez1rFh6/bUNWBG4GCMVD8t8nSdyzB+Ry3CdMKHbXToFNR2hfHt7/EZZ30z2/GghPfW8B3RRzGzduxN3dnXfeeYfvvvtOtHYLQi5kMpm+S3dxpdXqWv4zNE+6/Wcm7OosNw3SVGqO/XWcco4Wxg45X4ikOx/lezctdC1WzZs3p3v37gwYMCDH4zZo0IBevXpRvnx5YmNjmTRpEn5+fly/ft3oRQMEQSh8Gq3EmZtxBIRGEhAWxa3YzHmr5YAahVxGDRdTWjsl0MAyiiryO7ik3cDkQQRcvg7SM7qomdk97g6eZby1SzWwLV2kk+vncR78JQm7dpEWFkbCrl3Yv8Bc5sLrS1KpiJyqK57m0KMHFjVqGDmiF1eg0xFW7QD1+wJLIHAmtHkPHJ/UTfjwww+pUKECpUuX5vz583z77bdERESwffv2fDxDQRCKIrlchplcgVkuWahareauLVgqS0a6WjLOoojI925awMcf67p83rhx45nHHThwoP6xh4cH06ZNo06dOty4cYNKlSq9whkJglBcpKk1HL/ygIDQKA6GR/EwWaVfpjSR82HZWFo8+pP6jqnYJV1DHn8DEp6RXJvbGc5vndmCbeNebJPrZzFxdMR50GdEz51HzIKF2Pr7I7coGXfVhYITu2EDqqtXUTg64jLka2OHU6jyPB3hG18DS0CdBNv6wicBYKIb35v1e0utWrUoVaoU7dq14+rVq+J7iyAIJZJIuvNJgXXTekHJycmsXbuWihUrUq5cuZfejyAIRV9CqpojF6MJCIskMCKGFNWTyqu25ib4VnfDr4YbLR0eYrH6Y2RaNSRn2YGFQw7JdTWwditxyfXzOHz8MXGbf0F97x6xP/+M8+efGzskoQhTR0YSs3QZAK4jRqCwszNyRC+nwKcjVDwukmVmC/dDYP84eGtujvts0qQJAFeuXBFJtyAIJZJIuvNJgXbTyoNly5YxatQokpOTqVq1KgcOHECpfLX5GgVBKHoiE9I4EKbrNh589aFBJVN3W3P8arjhX8OdxhUddWO+JAl+/hS0amItK2HXfACKUjV0ybWVy2uVXD+L3MwMl+HDuDd8BA9WrsK+a1dMXFyMHZZQREXNnIWUkoJF/frYdXrP2OG8tEKbjrDtBPj3Wzi9Eir4QM3sBedCQkIAKFWq1IucgiAIQrEhkm4jy3M3rVz06tWL9u3bc//+febOnUv37t05fvw45uavNs+iIAjGdyX6EftDowgIi+Lc7XiDZV6u1vpEu1YZO2RPJ9HntsDNv5BMLPjH4wvaNOqDopCnaSkObDt2JHb9etLOnSdm8Q+Umjol942E107S8eM82rfvcfG074pd8bSnFcZ0hBFpDuDeE/crm3Df9TVX0+zZ/OdxOnbsiJOTE+fPn2fo0KG0bNmS2rVrF/6bIAiCUAhE0p1PCrybVi7s7Oyws7PDy8uLpk2b4uDgwB9//EHPnj1f/GQEQTAqrVYi5E48AaFRBIRFci0m2WB5/fL2+NVwx8/bDU8X62fvKCUWAsbr9tliJKnxovX2WWQyGW7fjubmhx8S//vvOHz8EeZVqhg7LKEI0apURE3VzQrg0KsX5iVghpD8nI4w065du/RJO0CPHj0AmPhuZSbVi0YZ8C0HD5jrE/xy5crRpUsXxo8fX0hnXTxoNJrn9yAoptRqNSYmJqSlpaHRaHLfQHhtFZVrRaFQYGJikr1R4wWJpDufFFo3rTyQJAlJkkhPT899ZUEQigRVhpbgaw8JCI3kQFgU0Y+e/P6aKmT4VHLGr4Yb7au74Wqbxx4sh6ZAygNwqYa2ySDYf7CAoi8ZLOvXw8bfn0f79xM9ew7lV600dkhCERK79mdUN26gcHbG5euvjB1OvnnZ6QgzqdVq9u7dq3/et29f+vbtm31nifdhRXPKpVwiaGQ/eOfYq4ZeYiUlJXHnzh0kqWTMT5yVJEm4u7tz+/btV05ihJKtKF0rlpaWlCpV6pWG7oqkOx8VSjetiAgA3N3dcXd359q1a2zduhU/Pz9cXFy4c+cOM2fOxMLC4oW7qAuCULiS0jMIjIgmIDSKIxejeZSeoV9mbWZC66ou+NVwp01VF9av/olxH8zhk8hI6tSpww8//EDjxo1z3O/27dv5ftJ4rlwKR60Fr8qxDLH8FUdHR4N1VqxYwZkzZ4iNjeXff/+l7lNz6aalpTF8+HC2bNlCeno6/v7+LFu2LFvtipLEdfgwHh0+TPJff5F07C+sWzQ3dkhCEaC+d48Hy5cD4DZqJAobGyNHVAzZloIuK2FDZzizFjyaQ62uxo6qyNFoNNy5cwdLS0tcXFyMnmzkN61WS1JSEtbW1ga9KAThaUXhWpEkCZVKRUxMDNevX8fLy+ulYxFJdz4q1G5aEycyadIkzM3NOXbsGAsXLiQuLg43NzdatmzJiRMnXqogmyAIBSvmUToHw6MICI3k+JWHqDRPpu1ytjajvbcb/jXcaFbJCTMTBQBbt25l2LBhrFixgiZNmrBw4UL8/f2JiIjI8ffc0c6OcT4S1VpYoaz5LrtVTRgwYADjx4/X34xLTk6mefPmdO/enQEDBuQY69ChQ9mzZw/btm3Dzs6OwYMH07lzZ44fP14A70zRoCxfHscPPyR23TqiZ8/GyqcZMoXC2GEJRhY1YwZSWhqWDRti+847xg6n+KrUFlqOhKOz4X9DoFQdcPYydlRFilqtRpIkXFxcsCiB0xdqtVpUKhXm5uYi6Raeq6hcKxYWFpiamnLz5k19PC9DJpXEvis5SExMxM7OjoSEBGxtbY0dzjNldtPq2LEjpqLYkfAc4lopPm48SCYgLJKA0CjO3Ioj66euh5Ml/jXc8avhTr1y9sjl2Vs1mjRpQqNGjViyZAmg+0NUrlw5vvrqK0aPHp39gMHLYP8YMLeHr86AlTP16tXDy8uLTZs2GVwvN27coGLFitlauhMSEnBxcWHz5s107aprjbp48SLVq1cnODiYpk2b5st7UxRp4uO54t8BbUIC7lMm49C9u7FDKnTi8+WJpKNHuT3wM1AoqPjHdjHW/ykvfK1oNbD+PbhxDFxrwKcHQWlZ8IEWE2lpaVy/fp2KFSuWyGK4Wq2WxMREbG1tRdItPFdRulae93uZ1xxTtHQLgiDkM0mSuHA3UZ9oR0Q9Mlheu6wdft66iuOVXa2f231QpVJx5swZxowZo39NLpfj6+tLcHBw9g0S78GR6brHvpOQLJ04fOgQly5d0tebyIszZ86gVqvx9fXVv1atWjXKly9f4pNuhb09Ll98TtSMmcQs/gHbjm+hsLYydliCEWjT04mcpvt9cvz4Y5Fw5we5ArqshhXNIToU/hwF7y0xdlSCIAgFSiTdgiAI+SBDo+XU9VgCwnRdx+8lpOmXKeQymno64uftTntvN0rb573L4IMHD9BoNNnGUbu5uXHx4sXsG+wbA6okEhzrUab1V6SnD0ShUPDDDz+80JCTyMhIlEol9vb22Y4bGRmZ5/0UVw49exK7eTPqm7d4uHoVrkOGGDskwQgerl6N+tYtTFxdcX6BoqhCLmzcoMsq2NAJ/t2gG99dp4exoxIEQSgwIukWBEF4SakqDUcvx7A/NJLDF6OJT3ky84CFqYJWVVzwq+FG22qu2Fu+fMXLPLtyEMJ2gEyBTddFhHS0IikpiUOHDjFy5EhGjBghCizmkUypxHX4cO5+PYTYtT/j8MEHmD5n+keh5FHducPDH38CwPXbUaK3Q37zbAWtRkPg97B7KJSqC67Ffxo2QRCEnIjBFIIgCC8gLlnFb2fuMGD9P9SbGsBnG86w/exd4lPUOFia0q1BWVb2bsi/E9qz4uMGdK5f9pUSbmdnZxQKBVFRUQavR0VF4Z41CVSnwp4RusdNBiEvXYfKlStTt25dhg8fTufOnfn999/zfFx3d3dUKhXx8fHPP24JZtO+PRYNGyClpRGzYKGxwxEKWdT3M5DS07Fs0gRbcbOqYLQcAZ6tQZ0C2/qAKtnYEQmv6Pbt2/Tv35/SpUujVCqpUKECQ4YM4eHDh0aL6dy5c/Ts2ZNy5cphYWFB9erVWbRoUZ623bZtG9WqVcPc3JxatWoZTI+Xk8DAQGQyWbafrD3E+vbta7DMycmJDh06cP78+efu+8aNG8hkMkJCQvIUe25+/vnnbL3ZClLr1q355ptvCu14RY1IugVBEHJxJy6Ftcev0+OnYBpOP8iIbec4EBZFmlpLGXsL+r9RkS0Dm3J6nC9zutWhvbcb5qb5U/FaqVTSoEEDDh06pH9Nq9Vy6NAhmjVr9mTFvxZA3HWwKQ1txmTbj1arRa1WZ3v9WRo0aICpqanBcSMiIrh165bhcUswmUyG27ffApCwcyepoaFGjkgoLI+OHCHp8GEwMcF9wnclbtqmIkOugM6rwNodYi4+uXEoFEvXrl2jYcOGXL58mV9++YUrV66wYsUK/d+r2NhYo8R15swZXF1d2bhxI6GhoYwbN44xY8boi5M+y4kTJ+jZsyeffPIJ//77L506daJTp05cuHAh12NGRERw//59/c/Tw7s6dOigX3bo0CFMTEx4++23X+k8C4pKpTJ2CCWCSLoFQRCeIkkSFyMTWXTwMm8tPkbzWUeY/L8w/r4Wi0YrUb2ULUPaebHn6+b89W0bJrzjTVNPJ0wUBfOROmzYMFauXMm6desIDw/n888/Jzk5WT+dYO8P3mfMxGm6lTvMYMb8JRw4cIBr164RHh7OvHnz2LRpE61bt9bvMzY2lpCQEMLCwgDdF4SQkBD93Xg7Ozs++eQThg0bxpEjRzhz5gz9+vWjWbNmJbqI2tMsatXC9vEXoehZs3lNJvx4rWnT0oia/j0ATn37YFapkpEjKuGsXaDrapDJ4dxm+HejsSMqUiRJIkWVYZSfF/28+/LLL1EqlQQEBNCqVSvKly/Pm2++ycGDB7l79y7jxo1j6dKlBjdud+zYgUwmY8WKFfrXfH19GT9+vP75zp07qV+/Pubm5nh6ejJ58mQyMjL0y2UyGatWreL999/H0tISLy8vdu3apV/ev39/Fi1aRKtWrfD09OSjjz6iX79+bN++/bnns2jRIjp06MDIkSOpXr06U6dOpX79+rkm6wCurq64u7vrf56uvm1mZqZfVrduXUaPHs3t27eJiYnJdd+ZMlvVDx06RMOGDbG0tMTHx4eIiAj9OufOnaNNmzbY2Nhga2tLgwYN+OeffwgMDKRfv34kJCToW9wnTZoEgIeHB1OnTqV3797Y2toycOBA/bGy9n4LCQlBJpNx48YN/WvHjx+ndevWWFpa4uDggL+/P3FxcfTt25egoCAWLVqkP17W7V4HYky3IAgCoNFKnL0Vx/4LkQSERXErNkW/TC6Dhh6O+Hm74eftTnmnwp3e5oMPPiAmJoYJEyYQGRlJ3bp12bdvn664miRx69wx5OYZUPlN8H6P5F/+4YsvvuDOnTtYWFhQrVo1fv75Z6ytrfX73LVrlz5pB+jRQ1fEaOLEifo/vAsWLEAul9OlSxfS09Px9/dn2bJlhXruRYHr0G94FBBAyqlTJB05gk3btsYOSShAD39aifrOHUzc3XH+/HNjh/N68GgObcbB4am61u7S9cHN29hRFQmpag3eE/Yb5dhhU/yxVOYtVYiNjWX//v1Mnz492/zi7u7u9OrVi61bt3LkyBGGDBlCTEwMbm5uBAUF4ezsTGBgIIMGDUKtVhMcHKyfDvPYsWP07t2bxYsX06JFC65evcrAgQMB3d+rTJMnT2b27NnMmTOHH374gV69enHz5k0cHR1zjDchIeGZyzIFBwczbNgwg9f8/f3ZsWNHru9H3bp1SU9Pp2bNmkyaNIk33njjmesmJSWxceNGKleujJOTU677ftq4ceOYN28eLi4uDBo0iP79+3P8+HEAevXqRb169Vi+fDkKhYKQkBBMTU3x8fFh4cKFTJgwQZ+kZ/2OMHfuXCZMmKB/j2/fvp1rHCEhIbRr105/k8PExIQjR46g0WhYtGgRly5dombNmkyZMgUAFxeXFz7X4kwk3YIgvLbS1BpOXH3A/gtRHAyP4mHyky5UShM5Lb2c8fN2p111V5yszYwYKQwePJjBOVVPvvA7gT3UYOIAHeeATMa0adOYNm2awWqZc+lm6tu3L3379n3uMc3NzVm6dClLly7Nj1MotkzLlMGxTx8erlxJ9Ow5WLdogew1n7u6pFLdvMnDVasAcBs9GrmVKJ5WaJoPg1vBuoKQ2/rAgCNgZp37dkKRcPnyZSRJonr16jkur169OnFxcbi6uuLg4EBQUBDdu3cnMDCQ4cOH68dYnzp1CrVajY+PD6BLpkePHk2fPn0A8PT0ZOrUqYwaNcog6e7bty89e/YE4Pvvv2fx4sWcOnWKDh06ZIvlxIkTbN26lT179jz3nCIjI3OcOeR5M3iUKlWKFStW0LBhQ9LT01m1ahWtW7fm5MmT1K9fX7/e7t279UlucnIypUqVYvfu3S81H/X06dNp1aoVAKNHj+att94iLS0Nc3Nzbt26xciRI6lWTVek0MvLS7+dnZ0dMpksxzotbdu2Zfjw4frneUm6Z8+eTcOGDQ1uzteoUUP/WKlUYmlp+drUhXmaSLoFQXitJKSqCYyIZn9oJIERMaSoNPpltuYmtKvuhp+3Gy2ruGBlVsQ/ItMSYP9Y3eMWI8DR07jxlGBOnw0k/vffUd24QdzWX3H8qJexQxLymSRJRE6fjqRSYeXjg42/n7FDer3I5fD+j7CiBTy4pKto3vkneM3H01uYKgib4m+0Y7+o3Lqkm5mZ4ePjQ1BQEH5+foSFhfHFF18we/ZsLl68SFBQEI0aNcLSUtej7Ny5cxw/fpzp06fr96HRaEhLSyMlJUW/Xu3atfXLrayssLW1JTo6OtvxL1y4wHvvvcfEiRPx89P9jt+6dQtv7yc9K8aOHcvYsWNf+NwBqlatStWqVfXPfXx8uHr1KgsWLGDDhg3619u0acPy5csBiIuLY9myZbz55pucOnWKChUq8Oabb3Ls2DEAKlSoQOhzaopkPfdSpUoBEB0dTfny5Rk2bBiffvopGzZswNfXl27dulEpD0NmGjZs+GInjq6lu1u3bi+83evipQYgLl26FA8PD8zNzWnSpAmnTp165rrbt2+nYcOG2NvbY2VlRd26dQ0uOtD9gk6YMIFSpUphYWGBr68vly9fNlgnNjaWXr16YWtri729PZ988glJSUkvE74gCK+ZyIQ0Nvx9k49Xn6TB1AMM2RLC3v8iSVFpcLc1p3ezCmz8pAlnvmvPgg/q8matUkU/4QY4PA2SosDJC9742tjRlGgKa2tcvtL1NHiwZAmaxEQjRyTkt6RDh0g+egxMTXEbP14UTzMGK2fougZkCvjvVzi7ztgRGZ1MJsNSaWKUnxf5HahcuTIymYzw8PAcl4eHh+Pi4oK9vT3NmzcnKCiIY8eOUa9ePWxtbWnZsiWBgYEEBQXpW21B1/V68uTJhISE6H/+++8/Ll++jLm5uX4906d6H8lkMrRarcFrYWFhtGvXjoEDBxqMGS9durTB/gcNGgTousXnOnNIHjRu3JgrV64YvGZlZUXlypWpXLkyjRo1YtWqVSQnJ7Ny5UoAVq1apY8nt4rpWc898/8s89wnTZpEaGgob731FocPH8bb25s//vgj15itnurlk9kCn/WmytPFWZ8eViAYeuGke+vWrQwbNoyJEydy9uxZ6tSpg7+/f453kwAcHR0ZN24cwcHBnD9/nn79+tGvXz/2738yPmX27NksXryYFStWcPLkSaysrPD39yctLU2/Tq9evQgNDeXAgQPs3r2bo0eP6sd0CIIgPO1KdBLLAq/w3tLjNJ1xiO92XODY5QdkaCW8XK35sk0ldn75BsFj2jLlvZo093LGtIAKoRWIu2fhtK4bLG/NAxPjdn9/Hdh364ayUiU08fE8+PFHY4cj5CNtaiqR3z8unta/P2aeFY0c0WusQjNo953u8d5REPmfceMR8sTJyYn27duzbNkyUlNTDZZFRkayadMm/ZAmHx8fwsLC2LZtm77AZ+vWrTl48KC+EFem+vXrExERoU9Qs/68SFfs0NBQ2rRpQ58+fQxazQFMTEwM9ps51rtZs2YGM3gAHDhw4IVn8AgJCdG3QD+LTCZDLpfr37syZcro46lQocILHe9pVapUYejQoQQEBNC5c2fWrl0L6Lp7azSaXLbWyRx/ff/+ff1rT09dVrt27WzvV1YvcryS6IWbcubPn8+AAQP0BXhWrFjBnj17WLNmjb7oQVZZf3EAhgwZwrp16/jrr7/w9/dHkiQWLlzI+PHjee+99wBYv349bm5u7Nixgx49ehAeHs6+ffs4ffq0vrvDDz/8QMeOHZk7dy6lS5d+0dMQBKGE0Wolzt2JJyAsiv2hkVyLMZzvtX55e/xquOPn7YanSzEfJ6jV6LpeSlqo1Q08W+W+jfDKZCYmuI4cwZ1BnxO3fgMOPXuiLFvW2GEJ+eDBjz+Sce8+JqVL4fyZuKFvdD5D4OYJuBwAv/aBgYFgbmvsqIRcLFmyBB8fH/z9/Zk2bRoVK1YkNDSUkSNHUqVKFSZMmABAzZo1cXBwYPPmzezevRvQ5QsjRoxAJpMZFB2bMGECb7/9NuXLl6dr167I5XLOnTvHhQsXstUueZYLFy7Qtm1b/P39GTZsmH5MtkKheG4xryFDhtCqVSvmzZvHW2+9xZYtW/jnn3/46aef9OuMGTOGu3fvsn79egAWLlxIxYoVqVGjBmlpaaxatYrDhw8TEBBgsO/09HR9HHFxcSxZsoSkpCTeeeedPJ1TXqSmpjJy5Ei6du1KxYoVuXPnDqdPn6ZLly6Arkp5UlIShw4dok6dOlhaWuq76z+tcuXKlCtXjkmTJjF9+nQuXbrEvHnzDNYZM2YMtWrV4osvvmDQoEEolUqOHDlCt27dcHZ2xsPDg5MnT3Ljxg2sra1xdHR8qTHsxdULJd0qlYozZ84wZsyTOWDlcjm+vr4EBwfnur0kSRw+fJiIiAhmzZoFwPXr14mMjMTX11e/np2dHU2aNCE4OJgePXoQHByMvb29wfgCX19f5HI5J0+e5P333892rPT0dNLT0/XPEx93BVSr1S80V21hy4ytKMcoFA3iWgFVhpaTN2I5GB7NofAYoh49+Z03Vcho5umIb3VX2lVzxdXmSUtwcX/P5P+sRnE/BMnMloy2kyAP5yOul/xh5uODRZMmpJ48SdTcebjPmW3skArE63S9qG7c4OHqNQA4j/oWjakpmtfgvPNLgV0rby/BZHUbZLFX0e76Ck2nla/F+G61Wo0kSWi12mzdo4u6SpUqcfLkSSZPnkz37t2Jjo5GkiTef/991q9fj6WlJZIkIZPJaN68OXv37sXHxwetVkvNmjWxtbWlatWqWFhY6M+9ffv27Nq1i2nTpjFr1ixMTU2pVq0a/fv3N3h/cnq/Ml/btm0bMTExbNy4kY0bn0xJV6FCBa5du/bM82natCkbN25kwoQJjB07Fi8vL7Zv3463t7f+WPfu3ePWrVv65+np6QwfPpy7d+9iaWlJ7dq1CQgIoE2bNvp1JEli3759+tZvGxsbqlWrxtatW2nZsuUz/98zX888r6efP72OTCbjwYMH9O7dm6ioKJydnXn//feZOHEiWq2Wpk2b8tlnn/HBBx/w8OFDg2rlmddgJoVCwaZNm/jyyy+pXbs2jRo1YsqUKXzwwQf641euXJl9+/Yxfvx4GjdujIWFBY0bN9avM2zYMPr164e3tzepqalcvXoVDw+PZ77/mV3Zn47FGLRaLZIkoVarUSgMax3k9bNPJr3AJHz37t2jTJkynDhxwqBrxahRowgKCuLkyZM5bpeQkECZMmVIT09HoVCwbNky+vfvD+gqCL7xxhvcu3fPoOtF9+7dkclkbN26le+//55169YZzDsHujnwJk+ezOc5TOkxadIkJk+enO31zZs3P/MujiAIRV+aBsLjZfwXKyMsTkaq5smXMDOFhLe9RG1Hier2EhbFYFj2izJTx9Mu7FtMtamcL9ub6y6+uW8k5CvlvXtUWPwDMkni1hdfkFahvLFDEl6WJFFm9RqsLl8muWpV7vbr+1okdsWFQ/Jlml/6HjkazpXtww2XdsYOqcCZmJjg7u5OuXLlUCqVxg7nlc2YMYNly5axfft2GjVqZOxwBOGlqFQqbt++TWRkpMEc8QApKSl8+OGHJCQkYGv77B45hfKV1MbGhpCQEH0XhmHDhuHp6Zmt63l+GjNmjMHceomJiZQrVw4/P7/nviHGplarOXDgAO3bt89WFEIQsnqdrpUHSekcvhjDgfBoTlyLRZXx5I6ns7WSdtVcaV/dhaaeTpiZlOyuSoodnyHXpqItVZfqH8+hujxv1WVfp+ulMETdvMWjHTuocvw4ZQZ9VuKKbr0u10tSQACRly8jUyqpPn8edcqLGygvqqCvFelvEzg0kdr3f8HbrzeUqpPvxyhK0tLSuH37NtbW1gaFwoqrGTNmUKVKFS5cuECbNm2QyWQ8evQIGxubEve5KeQvSZKKzLWSlpaGhYUFLVu2zPZ7mZjHwqovlHQ7OzujUCheuJKfXC6ncuXKgG6y+PDwcGbMmEHr1q3120VFRRm0dEdFRVG3bl1AVz3w6UJtGRkZxMbGPvO4ZmZmmJllLyxkampaLL5AFJc4BeMrqdfKzYfJBIRGERAWyT8348jaJ8fDyRL/Gu741XCjbjkHFPLX5A/3tUAI/R1kcuTvLERu9uJfyErq9VLY3IYOJWn/ftLOnSPt8BFsOxhnSp+CVpKvF21yMg/mzAXA6dNPscrDNDrCsxXYtdJ8CNw5iSxiL6Z/fAKfHQVzu/w/ThGh0Wj0RbVKynjXTz75RP84s5tw5jkKwrMUpWtFLpcjk8ly/JzL6+feC52BUqmkQYMGBpXptFothw4deqFKflqtVj/eumLFiri7uxvsMzExkZMnT+r32axZM+Lj4zlz5ox+ncOHD6PVamnSpMmLnIIgCEWUJElcuJvA/IAI/BccpdWcQKbvDef0DV3CXauMHSP8qhAwtCVHRrRmTMfqNKjg+Pok3BnpsGe47nGjT6F0PePG85ozdXPF6fEwqeh589CqVEaOSHhRD1asICMyEtMyZXAaOMDY4QjPIpNBp2VgVx7ibsDOwZD3kZGCIAhFwgt3Lx82bBh9+vShYcOGNG7cmIULF5KcnKyvZt67d2/KlCnDjBkzAF23koYNG1KpUiXS09PZu3cvGzZs0E8IL5PJ+Oabb5g2bRpeXl5UrFiR7777jtKlS9OpUycAqlevTocOHRgwYAArVqxArVYzePBgevToISqXC0IxlqHRcupGLAGhURwIi+Ju/JNpRhRyGU0qOuJfw5323m6Utn/N5388vggeXgFrN2g7Pvf1hQLn9El/4n/9FfXt28Rt2oxTv77GDknIo/SrV3m49mcA3MaNQ14CuvGWaBYO0O1nWOMP4bvg1E/Q5DNjRyUIgpBnL5x0f/DBB8TExDBhwgQiIyOpW7cu+/btw83NDYBbt24ZdAFITk7miy++4M6dO1hYWFCtWjU2btzIBx98oF9n1KhRJCcnM3DgQOLj42nevDn79u0z6DO/adMmBg8eTLt27ZDL5XTp0oXFixe/yrkLgmAEqSoNRy/HEBAaxaGLUcSnPKn6aGGqoFUVF/xquNG2miv2lsW/iEy+iL0GR3XdYPH/vkR3rSxO5JaWuHwzhPvjxvNg+XLsOr2HiYODscMSciFJEpFTp0FGBtZt2mDTto2xQxLyomwD8JsK+0bD/nFQtiGUaWDsqARBEPLkpQqpDR48mMGDB+e4LDAw0OD5tGnTcp1HTyaTMWXKFKZMmfLMdRwdHdm8efMLxyoIgvHFJas4dDGagNBIjl6OIU39pBCag6UpvtXd8KvhTvPKzlgo81YY7LUhSbB3JGjSoWIrqNnF2BEJWdh16kTs+g2kR0TwYPly3MeONXZIQi4e/fknKX//jczMDLdx4v+rWGkyCG4eh/D/wba+uvHdFuJGlyAIRV8JnFBHEISi4E5cCgfCoggIjeLUjVg02idj8MrYW+gLoTWs4ICJQhRTeaawnXDlICiU8NZ8MZ1RESNTKHD7dhS3+n9C3OZfcPzwQ5TPmXdUMC5NUjJRM2cB4DRwAMqyZY0ckfBCZDJ4dwncPw/xN2HHl9Bjk/hcFAShyBNJtyAI+UKSJCKiHukrjl+4aziFQjV3G32i7V3K1ujTPxQL6Y9g3xjd4+ZDwbmyceMRcmTl44NVq5YkBx0let48yv7wg7FDEp7hwdKlZERHY1q+PE6ffmrscISXYWEP3dfBaj+I2AN/L4NmXxo7KkEQhOcSSbcgCC9No5U4eyuOgNBIAsKiuPkwRb9MJoNGFRzxq+GGn7c75Z0sjRhpMXVkBjy6Bw4VofkwY0cjPIfbyJFc++s4jw4cJOX0aSwbNTJ2SMJT0i5dInb9egDcx49DnsO0okIxUbqerr7F3hFwYAKUbQzlxO+cIAhFl+jTKQhGtHTpUjw8PDA3N6dJkyacOnXqmeuuXLmSFi1a4ODggIODAx06dODSpUsG60RFRdG3b19Kly6NpaUlHTp04PLlywbrpKWl8eWXX+Lk5IS1tTVdunQhKioqzzGnqTUcvhjF6N/P0+T7g3RbEczKY9e5+TAFpYmcdtVcmd2lNqfH+fLroGZ82sJTJNwv4/55OLlC9/ituWAqqisXZWaVK2PfrSsAUbNmI2m1uWwhFCZJkoiaMhU0Gqx922HdsqWxQxJeVaNPwbsTaDPgt36QEmvsiARBeEnfffcdAwcONMqxe/Towbx58wr8OCLpFgQj2bp1K8OGDWPixImcPXuWOnXq4O/vT3R0dI7rBwYG0rNnT44cOUJwcDBly5Zl0qRJ3L17F9B9qezUqRPXrl1j586d/Pvvv1SoUAFfX1+Sk5P1+xk6dCj/+9//2LZtG0FBQdy7d4/OnTs/N9bENDU7Q+7y5aazNJh6gP4//8OW07d5kKTCxtyETnVLs7xXff79rj2r+zaie6NyOFuLVqSXptXCnmEgaXRfKiv7GjsiIQ9cvvoKuZUVaRcukLhnj7HDEbJI3L2blH/+QWZujvuYMcYOR8gPMhm8+wM4ekLCbdjxue6zUzCq27dv079/f0qXLo1SqaRChQoMGTKEhw8fGi2mc+fO0bNnT8qVK4eFhQXVq1dn0aJFedp227ZtVKtWDXNzc2rVqsXevXufu35gYCAymSzbT2RkpH6dvn37GixzcnKiQ4cOnD9//rn7vnHjBjKZjJCQkDzFnpuff/4Ze3v7fNlXXrRu3Zpvvvkm2+uRkZEsWrSIcePG6V/r27cvCoUCBwcHzMzMqFixIqNGjSItLS3b9nfu3EGpVFKzZs0cj5v1vbazs+ONN97g8OHD+uXjx49n+vTpJCQkvPpJPodIugXBSObPn8+AAQPo168f3t7erFixAktLS9asWZPj+ps2beKLL76gbt26VKtWjR9//BFJkvQfHJcvX+bvv/9m+fLlNGrUiKpVq7J8+XJSU1P55ZdfAEhISGD16tXMnz+ftm3b0qBBA9auXcuJEyf4+++/DY4XlZjGhr9v8vHqkzSYeoAhW0LY8999klUa3GzN+LhpBTZ+0oSz37VnYY96vFmrFFZmYsRKvji7Du6cBqU1dJhh7GiEPDJxcsLp8Z366PkL0Obw5UAofJpHj4iaNRsA50GDMC1TxsgRCfnG3Ba6rQOFGVzaB8GinoIxXbt2jYYNG3L58mV++eUXrly5wooVKzh06BDNmjUjNtY4vRHOnDmDq6srGzduJDQ0lHHjxjFmzBiWLFny3O1OnDhBz549+eSTT/j333/p1KkTnTp14sKFC7keMyIigvv37+t/XF1dDZZ36NBBv+zQoUOYmJjw9ttvv9J5FhSVSlWg+1+1ahU+Pj5UqFDB4HV/f38uXrzIlStXWLBgAT/++CMTJ07Mtv3PP/9M9+7dSUxM5OTJkzkeY+3atdy/f5/jx4/j7OzM22+/zbVr1wCoWbMmlSpVYuPGjfl/clmIpFsQjEClUnHmzBl8fZ+0YMrlcnx9fQkODs7TPlJSUtBoNDg6OgKQnp4OYDC/vVwux8zMjL/++gvQ/eFRq9UGx61WrRrly5cnODiYqzFJLAu8Qqelx2ny/SG+23GBY5cfoNZIVHa15ovWldj55RsEj27H1E41ae7ljKmoPJ6/kmLg4CTd47bjwba0UcMRXoxjn96YlC5Fxv37xK5bb+xwBCDmhx/QPHiA0sMDx/79jB2OkN9K1YY3Z+oeH5wMt/5+/vrFjSSBKtk4P5KUe3xZfPnllyiVSgICAmjVqhXly5fnzTff5ODBg9y9e5dx48axdOlSmjVrpt9mx44dyGQyVqxYoX/N19eX8ePH65/v3LmT+vXrY25ujqenJ5MnTyYjI0O/XCaTsWrVKt5//30sLS3x8vJi165d+uX9+/dn0aJFtGrVCk9PTz766CP69evH9u3bn3s+ixYtokOHDowcOZLq1aszdepU6tevn2uyDuDq6oq7u7v+Ry43/K5kZmamX1a3bl1Gjx7N7du3iYmJyXXfmTJb1Q8dOkTDhg2xtLTEx8eHiIgI/Trnzp2jTZs22NjYYGtrS4MGDfjnn38IDAykX79+JCQk6FuBJ02aBICHhwdTp06ld+/e2NraMnDgQP2x4uPj9fsOCQlBJpNx48YN/WvHjx+ndevWWFpa4uDggL+/P3FxcfTt25egoCAWLVqkP17mdlu2bOGdd97Jdn5mZma4ublRrlw5OnXqhK+vLwcOHDBYR5Ik1q5dy8cff8yHH37I6tWrc3yv7O3tcXd3p2bNmvoGqaz7euedd9iyZUue3/uXIZqlBMEIHjx4gEajwc3NzeB1Nzc3Ll68mKd9jB07FgcHB9q1awc8SZ7HjBnDjz/+iJWVFQsWLODOnTvcv38f0HXhUSqV+u5EWq3E+bsJaM3tWLjrFIuiqhgco155e/y8dRXHK7lYv+JZC3lyYAKkxYN7LWg0wNjRCC9Ibm6O69Ch3Bs5ioc//YR91y6YODkZO6zXVtrFi8Rt3ASA2/jxyJVKI0ckFIgG/eDGcbjwG2zrB4P+AqsS8nunToHvjXTzdew9UFrladXY2Fj279/P9OnTsbCwMFjm7u5Or1692Lp1K0eOHGHIkCHExMTg5uZGUFAQzs7OBAYGMmjQINRqNcHBwYwePRqAY8eO0bt3bxYvXkyLFi24evWqfuxv1lbPyZMnM3v2bObMmcMPP/xAr169uHnzpr5h4mkJCQnPXJYpODiYYcMMi5j6+/uzY8eOXN+PunXrkp6eTs2aNZk0aRJvvPHGM9dNSkpi48aNVK5cGaeX+Hsxbtw45s2bh4uLC4MGDaJ///4cP34cgF69elGvXj2WL1+OQqEgJCQEU1NTfHx8WLhwIRMmTNAn6dbWT77nzZ07lwkTJujf49u3b+caR0hICO3atdPf5DAxMeHIkSNoNBoWLVrEpUuXqFmzJlOmTAHAxcWF2NhYwsLCaNiw4XP3feHCBU6cOJGtNfzIkSOkpKTg6+tLmTJl8PHxYcGCBVhZPfu6zbw+s7bgN27cmOnTp5Oeno5ZARXZFEm3IBRDM2fO5Ndff2XixIn6lm1TU1O2b9/OJ598gqOjIwqFAl9fX958802kp+5WH7scw/7QSA6ERRGVmE70ozTMU9W4KmQ0q+SMn7cbft5uuNqK4l2F6sZxOLcZkMHbC0EhPqKLI9u33iJ23XrSLlwg5ocfKPW49UAoXJJWS+SUqaDVYuPvj3XzZ3/pFYo5mQzeWQj3Q+DhFfhjIHy4DeSiJ1ZhuXz5MpIkUb169RyXV69enbi4OFxdXXFwcCAoKIju3bsTGBjI8OHD9WOsT506hVqtxsfHB9Al06NHj6ZPnz4AeHp6MnXqVEaNGmWQdPft25eePXsC8P3337N48WJOnTpFhw4dssVy4sQJtm7dyp5cam9ERkbm2DiSdXz200qVKsWKFSto2LAh6enprFq1itatW3Py5Enq16+vX2/37t36JDc5OZlSpUqxe/fubC3ieTF9+nRatWoFwOjRo3nrrbdIS0vD3NycW7duMXLkSKpVqwaAl5eXfjs7OztkMhnu7u7Z9tm2bVuGDx+uf56XpHv27Nk0bNiQZcuW6V+rUaOG/rFSqcTS0tLgeLdu3UKSJEqXzn5jac+ePZQtW5aMjAzS09ORy+XZehmsXr2aHj16oFAoqFmzJp6enmzbto2+ffvmGGNKSgrjx49HoVDo3zOA0qVLo1KpiIyMzJbY5xfxjU4QjMDZ2RmFQpGtanhUVFSOH35ZzZ07l5kzZ7Jv375s2zdo0ICQkBASEhJQqVS4uLjQpEkT6tSrz97/7vNb2CNUKhW9lh5Gbq77sLdSKlCqHtG1eU0WfNceW3PT/D1ZIW8yVLriaQAN+kLZ59/1FYoumVyO2+hvufnRx8T/ug3Hjz7CrLKYY72wJezcRerZs8gsLXEbM9rY4QgFzcxGN757VTu4chCOL4AWw3PfrqgztdS1OBvr2C/o6Zv8TzMzM8PHx4egoCD8/PwICwvjiy++YPbs2Vy8eJGgoCAaNWqEpaXu2OfOneP48eNMnz5dvw+NRkNaWhopKSn69WrXrq1fbmVlha2tbY6FaS9cuMB7773HxIkT8fPzA3SJn7e3t36dsWPHMnbs2Bc+d4CqVatStWpV/XMfHx+uXr3KggUL2LBhg/71Nm3asHz5cgDi4uJYtmwZb775JqdOnaJChQq8+eabHDt2DIAKFSoQGhr6zGNmPfdSpUoBEB0dTfny5Rk2bBiffvopGzZswNfXl27dulGpUqVczyO3luechISE0K1btxfaJjU1FTAcGpmpdevWzJ49G5lMpm8579Kli355fHw827dv1w+hBPjoo49YvXp1tqS7Z8+eKBQKUlNTcXFxYfXq1QbvW2brd0pKCgVFJN2CYARKpZIGDRpw6NAhOnXqBIBWq+XQoUMMHjz4mdvNnj2b6dOns3//fho0aPDMKpp2dnY8SEpn4e9BnD79D3cqvUvAprNo0x1AboJpVCjdu3XFr4Y7zhkPqD01kgFdO4iE25iCl0DMRbB0Bt/shUKE4sWyYUNs2vvy6MBBoubMofyPPxo7pNeKJjGR6DlzAHD54nNMc7mZKZQQ7jWh4xzY9RUcngblmoJHMe/hIJPluYu3MVWuXBmZTEZ4eDjvv/9+tuXh4eG4uLhgb29P8+bN2bBhA8eOHaNevXrY2trSsmVLAgMDCQoKMmiBTEpKYvLkyTnOspI1UTM1Nfz+IpPJ0D5VzT4sLIx27doxcOBAgzHjpUuXNqgIntnt3N3d/aUaR57WuHFjg8QQdDcGKme5Gbtq1Srs7OxYuXIl06ZNY9WqVfqE9Olze1rW5TKZDEB/7pMmTeLDDz9kz549/Pnnn0ycOJEtW7bk+H/0dHxZZbbAZ72polarDdZ5elhBXjg7OwO6Gw8uLi7ZYvD09MTW1pY1a9ZQp04dVq9ezSeffALA5s2bSUtLo0mTJvptJElCq9Vy6dIlqlR5MmRywYIF+Pr6Ymdnl+04gL7IX07L8ovodyMIRjJs2DBWrlzJunXrCA8P5/PPPyc5OZl+/XSFfnr37s2YLFPbzJo1i++++441a9bg4eFBZGQkcXFxJCUl6ddZvmYjIxZt5s2p26jRZxoj+nfHwqsppuXrUsHJkkHta9Gl50dknFiHn8NDbJNu89mAT2nWrBlNmzYt9PdAeCzuJgTpqivjNw0sHIwbj5AvXIcPBxMTkoOOknzihLHDea3ELFyEJjYWZaVKOPbubexwhMJU72Oo3QMkLfzWX1ecUihwTk5OtG/fnmXLlumTxUyRkZFs2rRJ3/ro4+NDWFgY27Zto3Xr1oCuVfPgwYP6QlyZ6tevT0REBJUrV8728yJdsUNDQ2nTpg19+vQxaDUHMDExMdhvZtLdrFkzDh06ZLDugQMHDArB5UVISIi+BfpZZDIZcrlc/96VKVNGH8+rdneuUqUKQ4cOJSAggM6dO7N27VpA1wCk0WjytI/MZDSzRhCQbeqy2rVrZ3u/ssrpeJUqVcLW1pawsLDnHl8ulzN27FjGjx+vf49Wr17N8OHDCQkJ0f+cO3eOFi1aZJsJyN3dncqVKz8zqb5w4QJly5bV3wQoCKKlWxCM5IMPPiAmJoYJEyYQGRlJ3bp12bdvn3780K1btwz+oCxfvhyVSkXXrl0N9vPZkJFU6fgJAWFRnNp9jMRT29Ekx6OwdsCzWUc+/2YkHeuWp4qbNTKZjLS2yxk+fDhdunQhPT0df39/g/E3ghH8+S1kpEKF5lCnh7GjEfKJ0sMDhw97Erd+A1GzZlNx++/IFApjh1XipYaGEve4Cq37d+ORieJprxeZDN6eD/f+hQcRsH0AfPQ7yMXvXkFbsmQJPj4++Pv7M23aNCpWrEhoaCgjR46kSpUqTJgwAdBN0eTg4MDmzZvZvXs3oEu6R4wYgUwmMyg6NmHCBN5++23Kly9P165dkcvlnDt3jgsXLjBt2rQ8xXXhwgXatm2Lv78/w4YN04/JVigUz23ZHDJkCK1atWLevHm89dZbbNmyhX/++YeffvpJv86YMWO4e/cu69frZqtYuHAhFStWpEaNGqSlpbFq1SoOHz5MQECAwb7T09P1ccTFxbFkyRKSkpJyrOL9slJTUxk5ciRdu3alYsWK3Llzh9OnT+u7aHt4eJCUlMShQ4eoU6cOlpaW+u76T6tcuTLlypVj0qRJTJ8+nUuXLjFv3jyDdcaMGUOtWrX44osvGDRoEEqlkiNHjtCtWzecnZ3x8PDg5MmT3LhxA2traxwdHfUz9/z111/6np/P0q1bN0aOHMnSpUvx9fXl7NmzbNq0ST9ePVPPnj2ZMmUK06ZNw8Qkb6nusWPH9MMNCoz0mkhISJAAKSEhwdihPJdKpZJ27NghqVQqY4ciFGHxKSpp8YGLUr0J/5MqfLtb/+M5Zo/U86dgae1f16Q7cSnGDlPIi/DdkjTRVpImO0lS9MUCO4z4bDGOjLg46WKjxlJY1WpS3G+/GTucPCuu14tWo5Gud/9ACqtaTbozdJixw3ktFNlrJSpMkqa56z5fA2cZO5o8S01NlcLCwqTU1FRjh/JSrl+/LvXp00dyc3OTZDKZBEidO3eWkpOTJUmSJI1GI8XFxUnvvvuuZGJiIj169Ej/uoODg9S0adNs+9y3b5/k4+MjWVhYSLa2tlLjxo2ln376Sb8ckP744w+Dbezs7KS1a9dKkiRJEydOlIBsPxUqVMj1fH799VepSpUqklKplGrUqCHt2bPHYHmfPn2kVq1a6Z/PmjVLqlSpkmRubi45OjpKrVu3lg4fPpxtm6xx2NjYSI0aNZJ+y+VvxPXr1yVA+vfffyVJkqQjR45IgBQXF6df599//5UA6fr161J6errUo0cPqVy5cpJSqZRKly4tDR482ODaGjRokOTk5CQB0sSJEyVJkqQKFSpICxYsyHb8v/76S6pVq5Zkbm4utWjRQtq2bZv+WJkCAwMlHx8fyczMTLK3t5f8/f318UVEREhNmzaVLCwsDLbbu3evVKZMGUmj0Ri8R++++64UFxdn8PqMGTMkFxcX6dNPP5W8vb1zfJ/u378vyeVyaefOnZIk5Xx9ZJWamirZ2dlJwcHBz13nWb+Xec0xZY+DKfESExOxs7MjISEBW1tbY4fzTGq1mr1799KxY8dcx3AIr5+oxDRW/3WdzSdvkZSum6PS3FROqyou+Hm70666K/aWolWn2FAlw9ImkHAbmg8r0LHc4rPFeB6u/ZnoWbMwcXGh0r4/kT9nKpOiorheL/G//cb98d8ht7TE88+9mD5VeVjIf0X6WgnZDDs+B5kceu+Eii2NHVGu0tLSuH79OhUrVsyxuFRxM3HiRObPn8+BAwdo2rQpWq2WxMREbG1tX6pSt1DySJJEkyZNGDp0qL4CPVBo18ry5cv5448/svVGyOp5v5d5zTFF93JBKAauxiTxU9A1/vj3LiqNrjhGFVdrGtkm8G3PdthaFf8/zK+loFm6hNu+PLQcaexohALi0OtD4jZvRn37Ng/XrMXlq2cXSxReniY+nui5uu6Ozl99JRJuAep+qJuKMWQj/PaJbv5uG3FdFKbJkyfj4eHB33//TePGjY0djlAEyWQyfvrpJ/777z+jHN/U1JQffvihwI8jkm5BKML+vRXHiqCrBIRFkdknpbGHI4Nae9Lc04E///wTC6UYp1YsRYVB8FLd4zfngPLFp2YRige5Uonr8OHc/eYbHq5Zg3337pi6uRo7rBInesFCNPHxmHlVxvGjXsYORygqOs6Be2chOgx+/0TX4i3GdxeqzAKxQLaK4oIAULduXerWrWuUY3/66aeFchzRr0MQihhJkgiMiKbHT8G8v+wE+0N1CbdvdTd+/7wZvw5qRttqbvppIYRiSKvVzcmtzYBqb0PVDsaOSChgNv5+WNSvj5SaSsyiRcYOp8RJ/e8/4n/9FQD3CROQFbVuzoLxKC1183ebWsGNY7oeRoIgCIVMJN2CUERkaLTsDLlLx8V/0Xftaf6+FouJXEaX+mU5MLQlq/o0pEEFR2OHKeSHc5vhVrDuS2CHmcaORigEMpkMt29HAZDwxx+khYcbOaKSQ9JoiJw8BSQJ23ffwbJRI2OHJBQ1LlXgnYW6x0Gz4epho4YjCMLrR3QvFwQjS1Vp2HbmNj8dvcadON3cg5ZKBT0bl+eT5hUpbW9h5AiFfJUSCwHf6R63Hg325Ywbj1BoLOrUwbZjRxL37iVq9mzKr1kjeqzkg/htv5F24QJya2vcRoraCMIz1O4ON/6Cs+vg9wG68d22z587WRAEIb+IpFsQjCQ+RcX64Jv8fOIGsckqAJyslPT18eDjZhVEFfKS6uBESI0FV29o+rmxoxEKmcuwYTw6eJCU4L9JCgrCpnVrY4dUrGXExRG9YAEALl9/jclz5twVBN6cBXfPQtR/j8d37wKF+CosCELBE580glDI7sWnsvqv6/xy6hYpKg0AZR0sGNjSk24NyonCaCXZrZNwdr3u8dsLQCHGnb5ulGXL4Nj7Yx6uWk30nLlYN2+OzET8KX5ZMfPno01IwKxqVRw+7Jn7BsLrzdQCuv0MP7WCm8ch8HtoN8HYUQmC8BoQf+kFoZBcjnrEiqBr7Ay5S4ZWV4q8eilbBrXy5K1apTBRiBILJZpGDbuH6h7X+wjKNzVuPILROH32GfG/b0d19Srx27bh0FMkiy8jNSSE+G2/AeA+cYK4eSHkjXNleHcx/NYfjs2D8j7g5WvsqARBKOHEt3xBKGD/3Ijl03Wnab/gKL+fvUOGVqKppyM/92vE3q+b817dMiLhfh2cXAHRoWDhAL5TjB2NYEQKGxucB38JQMwPS9AkJRk5ouJH0mi4P0X3e2T3/vtY1q9v5IiEYqVmF2j4ie7x9gGQcNe48QiCUOKJ28KCUAC0WokjEdGsCLrK6RtxAMhk4O/tzqDWlahbzt64AQqFK+EOHJmhe9x+Clg5GTcewegcuncnbuMmVNev8/DHn3AdPszYIRUrcVu3kh4WjtzGBtcRw40djlAc+X8Pd05D5Hldq3ff3WLIjyAIBUY0rwlCPlJrtPx+5g4dFh3lk3X/cPpGHEqFnB6NynFwWCtWfNxAJNyvo32jQZ0M5ZpC3Y+MHY1QBMhMTXF9XGk7dt061HdFS1teZTx8SMxC3VznLt8MwcRJ3MQSXoKpOXRfB2a2cPtvODzV2BGVCLdv36Z///6ULl0apVJJhQoVGDJkCA8fPjRaTOfOnaNnz56UK1cOCwsLqlevzqJFi3LdLjQ0lC5duuDh4YFMJmPhwoW5bnPjxg1kMlm2n7///lu/zqRJkwyW2dnZ0aJFC4KCgnLdv0wmY8eOHbmulxeBgYHIZDLi4+PzZX+56du3L506dSqUYxVFIukWhHyQospgzV/XaT0nkOHbznEpKglrMxM+a+nJsW/bMLNLbSq5WBs7TMEYLgVA+P9ApoC354NcfOwKOtZtWmPZpAmSSkX0goXGDqfYiJ47D21iImbe1XHo0cPY4QjFmaMnvLdE9/j4IojYZ9x4irlr167RsGFDLl++zC+//MKVK1dYsWIFhw4dolmzZsTGxholrjNnzuDq6srGjRsJDQ1l3LhxjBkzhiVLljx3u5SUFDw9PZk5cybu7u4vdMyDBw9y//59/U+DBg0MlteoUUO/LDg4GC8vL95++20SEhJe+PwKmkqlMnYIJYL49icIryA2WcX8A5fwmXmYKbvDuBufirO1GaM6VOX46LaM6VgdN1tzY4cpGIsqBfaO0D1u9gW41TBuPEKRIpPJcPt2FMhkJO7eTer588YOqchLOXuWhD/+AKDUhAnIFGK2B+EVeb8HjT/TPd4xCOJvGzeep0iSRIo6xSg/kiS9UKxffvklSqWSgIAAWrVqRfny5XnzzTc5ePAgd+/eZdy4cSxdupRmzZrpt9mxYwcymYwVK1boX/P19WX8+PH65zt37qR+/fqYm5vj6enJ5MmTycjI0C+XyWSsWrWK999/H0tLS7y8vNi1a5d+ef/+/Vm0aBGtWrXC09OTjz76iH79+rF9+/bnnk+jRo2YM2cOPXr0wMzM7IXeCycnJ9zd3fU/pqaGQxdMTEz0y7y9vZkyZQpJSUlcunQpz8fIbFXfvn07bdq0wdLSkjp16hAcHKxf5+bNm7zzzjs4ODhgZWVFjRo12Lt3Lzdu3KBNmzYAODg4IJPJ6Nu3LwCtW7dm8ODBfPPNNzg7O+Pv768/VkhIiH7f8fHxyGQyAgMD9a+Fhoby9ttvY2tri42NDS1atODq1atMmjSJdevWsXPnTn0Lf9btXgdiTLcgvIQ7cSmsOnadLadvkabWAlDByZKBLT3pUr8s5qbii6AAHJsL8TfBtiy0Gm3saIQiyNzbG7v33iNhxw6iZs2mwsYNyGQyY4dVJEkZGURO1hVPs+/WFYu6dY0bkFBy+E3Vje++dxZ+6wd994KJ0thRAZCakUqTzU2McuyTH57E0tQyT+vGxsayf/9+pk+fjoWFhcEyd3d3evXqxdatWzly5AhDhgwhJiYGNzc3goKCcHZ2JjAwkEGDBqFWqwkODmb0aN3fzGPHjtG7d28WL16sT+AGDhwIwMSJE/XHmDx5MrNnz2bOnDn88MMP9OrVi5s3b+Lo6JhjvAkJCc9clh/effdd0tLSqFKlCqNGjeLdd9995rrp6emsXbsWe3t7qlat+sLHGjduHHPnzsXLy4tx48bRs2dPrly5gomJCV9++SUqlYqjR49iZWVFWFgY1tbWlCtXjt9//50uXboQERGBra2twf/bunXr+Pzzzzl+/Hie47h79y4tW7akdevWHD58GFtbW44fP05GRgYjRowgPDycxMRE1q5dC1Cg739RJJJuQXgB4fcT+THoKv87fx/N42m/apWxY1CrSnSo6Y5CLr4sC4/FRMDxxbrHb84EMzG8QMiZy9BvSNy3j9QzZ3h04AC2fn7GDqlIitv8C+kREcjt7HAZJgrPCfnIxAy6rYUfW+qS70OTwX+6saMqVi5fvowkSVSvXj3H5dWrVycuLg5XV1ccHBwICgqie/fuBAYGMnz4cP0Y61OnTqFWq/Hx8QF0yfTo0aPp06cPAJ6enkydOpVRo0YZJN19+/al5+PpF7///nsWL17MqVOn6NChQ7ZYTpw4wdatW9mzZ0++vgcA1tbWzJs3jzfeeAO5XM7vv/9Op06d2LFjh0Hi/d9//2FtrftekJKSgo2NDVu3bsXW1vaFjzlixAjeeustQPd+1ahRgytXrlCtWjVu3bpFly5dqFWrFqB7/zJlJr2urq7Y29sb7NPLy4vZs2frn9+4cSPXOJYuXYqdnR1btmzRt+xXqVJFv9zCwoL09PQX7qpfUoikWxByIUkSp67HsjzoKoERMfrXm1d25vPWlfCp5CRapgRDkgR7hoNWDV7+UO1tY0ckFGGmbm449e/Hg2XLiZ43D5vWrZEpi0YrW1Ghjo4mZrHuJpbr0KGYODgYOSKhxHHwgPeWwdZeELwEKvhAtbeMHRUWJhac/PCk0Y79onLrkm5mZoaPjw9BQUH4+fkRFhbGF198wezZs7l48SJBQUE0atQIS0tdC/u5c+c4fvw406c/uQmi0WhIS0sjJSVFv17t2rX1y62srLC1tSU6Ojrb8S9cuMB7773HxIkT8Xt8g/PWrVt4e3vr1xk7dixjx4594XMHcHZ2ZliWm4KNGjXi3r17zJkzxyDprlq1qr4L/KNHj9i6dSvdunXjyJEjNGzYkEGDBrFx40b9+knPmVoy67mXKlUKgOjoaKpVq8bXX3/N559/TkBAAL6+vnTp0sVg/Wd5egx6XoSEhNCiRYtsXekFHZF0C8IzaLUSB8KjWBF0lX9vxQMgl8GbtUoxqGUlapW1M26AQtF1/le4cQxMLKDjbN18cYLwHE6ffELctm2ob94i7pdfcHzcqiPoRM+dizYpCfNatbDv1tXY4QglVfW3oemX8PdS2PE5fHZUl4wbkUwmy3MXb2OqXLkyMpmM8PBw3n///WzLw8PDcXFxwd7enubNm7NhwwaOHTtGvXr1sLW1pWXLlgQGBhIUFESrVq302yUlJTF58mQ6d+6cbZ/m5k9q5jyd6MlkMrRarcFrYWFhtGvXjoEDBxqMGS9durTBWOX87vbcpEkTDhw4YPCaUqmkcuXK+uf16tVjx44dLFy4kI0bNzJlyhRGjBiRp/1nPffMRqDMc//000/x9/dnz549BAQEMGPGDObNm8dXX3313H1aWVkZPJc/LgKb9aaKWq02WOfpYQWCIVFITRCeosrQ8uvp27RfEMRnG87w7614lCZyPmxSnsPDW7P0w/oi4RaeLTUOAsbpHrcaafQvbELxILeywuXrrwGIWbYcTSFN4VIcpJw+TeKu/4FMhvuE70TxNKFg+U6CMg0hLQG29YMMUbk5L5ycnGjfvj3Lli0jNTXVYFlkZCSbNm3SF+ry8fEhLCyMbdu20bp1a0BXvOvgwYMcP35c/xpA/fr1iYiIoHLlytl+5C8wG0hoaCht2rShT58+Bq3moCtqlnW/+Z10h4SE6Fugn0ehUOjfO1dXV4OYXkW5cuUYNGgQ27dvZ/jw4axcuRLQJf6g6zmQGxcXFwDu37+vfy3rjQrQtbgfO3YsWzKeSalU5ulYJZVIugXhsaT0DFYevUbL2UcY9ft5rsYkY2NuwhetK/HXt234/v1aeDhb5b4j4fV2aCokx4BzVWj2/DvJgpCVfefOmFWpgjYhgQfLV+S+wWtAUquJnPK4eFr37lg8HpcoCAXGRKkb321uryusduA7Y0dUbCxZsoT09HT8/f05evQot2/fZt++fbRv354qVaowYcIEAGrWrImDgwObN282SLp37NhBeno6b7zxhn6fEyZMYP369UyePJnQ0FDCw8PZsmWLQUt1bi5cuECbNm3w8/Nj2LBhREZGEhkZSUxMzHO3U6lUhISEEBISgkql4u7du4SEhHDlyhWDc27Xrp3++bp16/jll1+4ePEiFy9e5Pvvv2fNmjXZWpYzMjL0cVy+fJlp06YRFhbGe++9l+fzyotvvvmG/fv3c/36dc6ePcuRI0f04+4rVKiATCZj9+7dxMTEPLcLu4WFBU2bNmXmzJmEh4cTFBSU7f9g8ODBJCYm0qNHD/755x8uX77Mhg0biIiIAMDDw4Pz588TERHBgwcPnpmcl1Qi6RZeezGP0pmz/yI+Mw4xfW84kYlpuNqYMbZjNU6MbsuoDtVwtRHTfgl5cOcM/LNG9/iteUWm+q1QPMgUClxHjQIgdvNmVDdvGjki44vduIn0y1dQ2NvjOvQbY4cjvC7sy8P7P+oen1wBYTuNG08x4eXlxenTp/H09KR79+5UqFCBN998kypVqnD8+HF94TCZTEbz5s31/4KuldTW1paGDRsadG329/dn9+7dBAQE0KhRI5o2bcqCBQuoUKFCnuP67bffiImJYePGjZQqVUr/06hRo+dud+/ePerVq0e9evW4f/8+c+fOpV69enz66af6dR48eMDVq1cNtps6dSoNGjSgSZMm7Ny5k61bt9KvXz+DdUJDQ/Vx1K1bl19//ZXly5fTu3fvPJ9XXmg0Gr788kuqV69Ohw4dqFKlCsuWLQOgTJky+kJ1bm5uDB48+Ln7WrNmDRkZGTRo0IBvvvmGadOmGSx3cnLi8OHDJCUl0apVKxo0aMDKlSv13d8HDBhA1apVadiwIS4uLi9UGb0kkEkvOglfMZWYmIidnR0JCQkvVRmwsKjVavbu3UvHjh1FIYICduthCj8du8q2f+6QnqEb++LpYsVnLT3pVK8MZiZFuwujuFaKGE0GrGwDkeehdg/o/KOxIzIgrpfi49aAgSQfO4aNnx9lFy8ySgxF4XpRR0Vx7c2OaFNSKDVtKvZdxVjuoqgoXCsFJuA7OLEYzGzhsyBw9Mx9m1eUlpbG9evXqVixosGY5eJq4sSJzJ8/nwMHDtC0aVO0Wi2JiYnY2tq+UPdw4fVTlK6V5/1e5jXHFIXUhNfOhbsJrAi6yt7/7vN41i/qlLPn81aV8PN2Qy6m/RJexulVuoTb3A78puW+viA8g9uokVw7fpxHAQGknDmD5UtUkS0JomfNRpuSgkWdOtjlUERJEApcuwlw+6TuZ1tf6B8ApsU/ES5MkydPxsPDg7///pvGjRsbOxxBMBqRdAuvBUmSCL76kOVBVzl2+YH+9VZVXBjUqhJNPR3FtF/Cy0u8D4cfJ9q+k8DaxajhCMWbmZcX9l27Ev/rr0TNmo3Hll+QvWYtQsl//03i3r0gl+M24bvX7vyFIkJhCl3XwormcP+crkjmW/OMHVWxk7Vr9dMVxQXhdSGSbqFE02gl9odGsiLoKufvJAC6ab/eqVOaz1pWwrt00R1qIBQj+8eC6pGu4m39vsaORigBXL4aTOLu3aSdP0/i3j+xe9v48wUXFkmlInLKVAAcevTAokYNI0ckvNbsykDnn2BTV12Ppgo+ULOLsaMSBKGYEUm3UCKlqTX88e9dfjp6jesPkgEwM5HzQaNyDGjhSTnHoj/npVBMXDkEodtBJoe354NokRPygYmLC04DPiVm0WJi5s/Hpr0vcjMzY4dVKGLXr0d17RoKR0dchnxt7HAEAbzaQ/Nh8Nd82DUEStUFp0rGjkoQhGJEJN1CiZKYpmbT37dYc/w6MY/SAbCzMKVPswr08fHAyfr1+NIqFBJ1GuwdoXvc+DMoVce48QglimPfvsRt2Yr63j1i16/HecAAY4dU4NT37xOzbDkAriNHorCzM3JEgvBYm3G6sd03j8OvfeDTA2BqYeyoBEEoJkSTjFAiRCemMePPcN6YcZhZ+y4S8yidUnbmjH+rOidGt2WYX1WRcAv5768FEHsNbEpBm7HGjkYoYeQWFrg8nibr4Y8/kREba9yACkHUzFlIKSlY1K+P3XvvGjscQXhCYQJdVoOlM0T9B/tGGzsiQRCKEZF0C8Xa9QfJjNl+nuazjvBj0DUepWfg5WrN3G51CBrZhk9beGJlJjp0CAXg4VVdV0MA/+/BXNQHEPKf3bvvYu7tjTYpiQdLlhg7nAKV9NdxHu3fDwoF7hMniOJpQtFjWwq6rARkcOZnOL/N2BEJglBMiL9oQrF07nY8n288Q9t5gfxy6jYqjZYGFRxY2bsh+79pSdcGZVGaiMtbKCCSBHuGg0YFldpBjfeNHZFQQsnkcly//RaAuK2/kn71qpEjKhhalYqoaboZABx6fYh51apGjkgQnqFSW2g5Uvf4f0PgwWXjxiMIQrHwUlnJ0qVL8fDwwNzcnCZNmnDq1Klnrrty5UpatGiBg4MDDg4O+Pr6ZltfJpPl+DNnzhz9Oh4eHtmWz5w582XCF4opSZI4eimGD1f+zXtLj/PnhUgkCdpVc2XboGb8/rkP7cU820JhCN0O146Awgw6zgEx3ZxQgKyaNMa6bVvQaIieM9fY4RSI2DVrUd24gcLFGZevvjJ2OILwfK1Hg0cLUCfrxnerUowdkSAUa6tXr8bPz88oxx49ejRfFcLfnRdOurdu3cqwYcOYOHEiZ8+epU6dOvj7+xMdHZ3j+oGBgfTs2ZMjR44QHBxMuXLl8PPz4+7du/p17t+/b/CzZs0aZDIZXboYTskwZcoUg/UK4w0SjC9Do2XXuXu8/cNf9F5zihNXH2Iil9G5fhn2f9OS1X0b0cjD0dhhCq+LtETY93j8dovhooKtUChcR4wAExOSAgNJ/vtvY4eTr9R37/JgxQoA3EaNQmFjY+SIBCEXcoVufLeVK0SHwp+jjB1RkXH79m369+9P6dKlUSqVVKhQgSFDhvDw4UOjxXTu3Dl69uxJuXLlsLCwoHr16ixatCjX7UJDQ+nSpYu+4W/hwoW5bnPjxo0cGxL/zvK5PWnSJINldnZ2tGjRgqCgoFz3L5PJ2LFjR67r5UVgYCAymYz4+Ph82V9u+vbtS6dOnbK9npaWxnfffcfEiRP1r02aNAmFQoGDgwOmpqaUK1eOgQMHEptDbZPU1FQcHR1xdnYmPT092/KsDbdWVlbUr1+fbdueDA0ZMWIE69at49q1a/lzos/wwkn3/PnzGTBgAP369cPb25sVK1ZgaWnJmjVrclx/06ZNfPHFF9StW5dq1aqxatUqtFothw4d0q/j7u5u8LNz507atGmDp6enwb5sbGwM1rOysnrR8IViJE2tYcPfN2k7L4ivf/mX0HuJWJgq6PeGB0Gj2jC/e12quosvZ0IhOzIdkiLBsRI0/8bY0QivCTPPijh88AEAUbNmI2k0Ro4o/0TOmIGUloZlw4bYvv22scMRhLyxcYMuq3TTRf67Ac5tMXZERnft2jUaNmzI5cuX+eWXX7hy5QorVqzg0KFDNGvWLMeEqTCcOXMGV1dXNm7cSGhoKOPGjWPMmDEsyaVORkpKCp6ensycORN3d/cXOubBgwcNGgobNGhgsLxGjRr6ZcHBwXh5efH222+TkJDwwudX0FQqVYHu/7fffsPW1pY33njD4PUaNWpw8eJFbty4wdq1a9m3bx+ff/55tu1///13atSoQbVq1Z55QyKz4fbff/+lUaNGfPDBB5w4cQIAZ2dn/P39Wb58eb6fW1YvVGFKpVJx5swZxowZo39NLpfj6+tLcHBwnvaRkpKCWq3G0THnlsmoqCj27NnDunXrsi2bOXMmU6dOpXz58nz44YcMHToUE5OcTyE9Pd3gbkdiYiIAarUatVqdp1iNITO2ohxjQUtIVbPp5G3W/32Lh8m6X3QHS1M+blqej5qUw8FSCbze7xGIa8Uo7p/D5NRPyICMDrORJDkUk/dfXC/Fn/1nA0nYtYv08HBit/+Bbaf3CuxYhXW9JB89RtLBQ6BQ4DR2DBkZGQV6PCH/vdafLeV8kLcYieLoLKTdQ8lwqQkur1aPQK1WI0kSWq0WrVabT4EWji+++AKlUsm+ffuwsNBNp1a2bFnq1KmDl5cXY8eOxdvbm+XLl3PhwgW0Wi07duygS5cuLF26lEGDBgHg5+dHkyZNmDp1KgA7d+5k6tSphIWFUbp0aXr37s3YsWP1OYBCoeDHH39k7969BAQEUKZMGebMmcO77+pmQOjbt69BnB4eHpw4cYLt27fzxRdfPPN8GjRooE+WR48erf9/eZ7M5Q4ODri6uua4TJIkTExM9MtdXV2ZNGkSa9eu5eLFizRq1CjXY2i1Wm7cuEGlSpXYtm0bS5cu5eTJk3h5ebFs2TKaNWsGwM2bN/nqq684fvw4KpUKDw8PZs2ahbe3N23atNHHCtC7d2/Wrl1L27ZtqVGjBiYmJmzatIlatWqxevVqKlWqxJkzZ6hbty4A8fHxODk5cejQIVq3bg3oegeMHj2aY8eOIUkSdevWZc2aNWzcuFGf28keD8nL3O6XX37h7bffNnhvM98jNzc3bGxsKFOmDF27duXnn3/O9n+wevVqPvzwQyRJYtWqVXTr1i3be2ZtbY2rqyuurq788MMPbNy4kV27dtG0aVMA3nrrLb777jtmzZr1zPdckiTUajUKhcJgWV4/+14o6X7w4AEajQY3NzeD193c3Lh48WKe9vHtt99SunRpfH19c1y+bt06bGxs6Ny5s8HrX3/9NfXr18fR0ZETJ04wZswY7t+/z/z583Pcz4wZM5g8eXK21wMCArC0tMxTrMZ04MABY4dQ6OLT4ch9OcFRMtK1ul9IRzOJNqW0NHHNwCw1guDACCNHWfS8jteKUUhaWl6agoOk5Y59U86EJ0P4XmNH9cLE9VK8ObRojsveP7k3Zw7HkZCUygI9XkFeLzK1mgoLFqIEYn18uHT5MlwWRamKq9f2s0WqTjObGrg+CiV1fXeOVpmERvHyU5SamJjg7u5OUlISKpUKSZKQ0tLyMeC8k5mb6xOk3MTFxREQEMD48eOzNXBZWlrSrVs3tm7dyu7du4mIiOD69es4Oztz8OBBnJycOHjwIB9++CFqtZrg4GAGDx5MYmIiJ06coE+fPsyaNYtmzZpx/fp1vvnmG9LT0/n2cZFJgMmTJzN58mQmTJjATz/9xMcff8z58+f1CeXTHjx4gI2Njb5RLjdarZa0tLRc109KSgLg3XffJT09nUqVKvH111/TsWNH/Trp6eloNBr9vtLT0/nxxx+xs7OjVKlSuR4jNTWVxMRE/bHGjRvHlClTmDNnDtOmTaNnz56cPXsWExMTBg0ahFqtZvfu3VhZWXHx4kV9l/b169fTu3dvTp8+jY2NDebm5iQmJpKRkcH69evp168ff/75p8F5JScn6+N79OgRoGtQTUxM5N69e7Rq1YrmzZuzc+dObGxsOHnyJPHx8QwYMID//vuPxMREli5dCuiS/cTERP766y+6dOlicN6Z71HmcW7dusWff/6JqampwXrXr18nODiYtWvXIkkSw4cP58KFC5QvX/65/3cmJiYkJSXpX/P29ubOnTvZts2kUqlITU3l6NGj2W4Op6TkraZDoc6lNHPmTLZs2UJgYCDm5uY5rrNmzRp69eqVbfmwYcP0j2vXro1SqeSzzz5jxowZmJll/3AbM2aMwTaJiYn68eS2tkV3ah+1Ws2BAwdo3749pqamxg6nUFyJTmLV8RvsOncftUYCoKqbNQNaVKRjTTdMFaIKeU5ex2vFmORn1qAIuYZkZoNb71V0tHmxrmbGJq6XkkHbrh23zp2Du/docj8Sx88HFchxCuN6iV2+gtiHD1G4utJwzmzkYshYsSQ+W4DkJkirWmObdJeOHELT8eWn90tLS+P27dtYW1tjbm6ONiWFy8188jHYvPP65zTyPDZUhYeH61s2c/qeXbt2bdatW0fFihVxcHDg7NmzdO3aleDgYIYPH87ixYuxtbXl+PHjqNVqfH19sbS0ZP78+YwePZrPPvtMv59Hjx4xevRopk+frt9/v3796N+/PwBz5szhxx9/JDw8nA4dOmSL5cSJE/zxxx/873//y3NOIJfLMTc3z3V9d3d35s6di4+PD3K5nO3bt/PRRx+xfft2fcu7mZkZYWFhlC1bFtAlbTY2Nvzyyy/6157HwsICW1tbrK2tAd2Y5MzW3WnTplGrVi2io6OpVq0a9+/fp3PnzvqW79q1a+v3U6ZMGQA8PT2xt7fXv25iYoKXl5fBGPYbN24AYGVlpX8PMlucLS0tsbW1ZdasWdjb27Nt2zb9Z0H9+vX1+7CxsUGj0eDl5aV/LT4+nsTERCpVqmTw3mZ9jzQaDWmPbzzNmzfPYL1t27bRoUMHfaLs5+fH77//bjA+POv/nUqlYv78+SQmJuLv76/fV5UqVQB4+PAhNWvWzPaep6WlYWFhQcuWLbPlqHm9cfNCSbezszMKhYKoqCiD16OionId6zB37lxmzpzJwYMHDf7Dszp27BgRERFs3bo111iaNGlCRkYGN27coGoOU4uYmZnlmIybmpoWiz8KxSXOV3HmZhwrgq5yIOzJ9dS4oiOft65E6youeb67+rp7Ha4Vo0uK1o3lBmRtJ2DqWM7IAb08cb0Uc6amuA0fzt1hw4n7+Wcce3yA6VNdGPP3cAVzvahu3yZu9WoA3L4dhVmWL3xC8fRaf7bYl4aua2DdO8jPb0FesQXU++ildqXRaJDJZMjlcuRyORhxvnp9DHlcF9DH/rTM73Tm5ub4+PgQFBSEn58fYWFhfPnll8yZM4dLly5x7NgxGjVqpE8oz507x/Hjx/n+++/1+8pMwtLS0vS9V+vUqaM/ro2NDba2tjx48CBbLBcuXOD9999n4sSJ+oT81q1beHt769cZO3YsY8eOzfEccns/XF1dGT58uP55kyZNuH//PvPmzdMXEZPJZFStWpVdu3YBupbcrVu38sEHH3DkyBEaNmzIoEGD2Lhxo34/mS3Nme911v+bunXr6h9nJtKZ5/7111/z+eefc+DAAXx9fenSpYs+D8vcJqf/5wYNGhi8ltO6T7927tw5WrRokWP+lXneT7+HmUOBLS0tDV7PfI82btyIiYkJmzdvJiQkhK+//lq/nkajYf369SxatEj/2scff8yIESOYOHGiwf5Gjx7Nd999R1paGtbW1sycOZN33nlHvzyzTlhaWlqO/8dyuRyZTJbj51xeP/deKOlWKpU0aNCAQ4cO6S+czKJogwcPfuZ2s2fPZvr06ezfv5+GDRs+c73Vq1fToEED6tSpk2ssISEhyOXybOMlhKJNkiQCI2JYHnSVU9efFNTw83ZjUOtK1C+fczcgQTCqgPGQngCl6kKjT4wdjfCas3nzTSzWrSf13DliFi+m9OP5rYuTqO9nIKWnY9m0KbZZul0KQrHl0RzajIPDU2HPCChdH9y8c98uFzILC6qePZMPAb7csfOqcuXKyGQywsPDef/997MtDw8Px8XFBXt7e5o3b86GDRs4duwY9erVw9bWlpYtWxIYGEhQUBCtWrXSb5eUlMTkyZOzDTsFDFocn058ZDJZtrG/YWFhtGvXjoEDBzJ+/Hj966VLlyYkJET//Fl1p15WkyZNsg2/UCqVVK5cWf+8Xr167Nixg4ULF7Jx40amTJnCiBEj8rT/rOeeeXMj89w//fRT/P392bNnDwEBAcyYMYN58+blOgPU08WqMxNRSZL0rz09ltniBa6XTE5OTshkMuLi4rItUyqVeHp6Ymtry8yZM3nrrbeYPHmyfqz//v37uXv3Lh88LjKaSaPRcOjQIdq3b69/beTIkfTt2xdra2vc3NyyNexlFvlzcXF54XPIqxfuXj5s2DD69OlDw4YNady4MQsXLiQ5OZl+/foBukH4ZcqUYcaMGQDMmjWLCRMmsHnzZjw8PIiMjAR0A9oz72KBrml+27ZtzJs3L9sxg4ODOXnyJG3atMHGxobg4GCGDh3KRx999MyxGkLRotZo2X3+Hj8GXeNipG4MiKlCRqe6ZfislSeVXUUVcqGIun4Uzm8FZPD2At1UMYJgRDKZDNdvv+Xmhx+S8Pt2HD/+GPMcenwVVY8OHyHpyBEwNcX9u/GiV5NQcjQfBreC4cpB2NYHBhwBM+vct3sOmUyGrBjUInJycqJ9+/YsW7aMoUOHGiRgkZGRbNq0iS+//BIAHx8fxowZw7Zt2/QFuFq3bs3Bgwc5fvy4QUtx/fr1iYiIMEhQX0ZoaCht27alT58+Bt3SQded+lX3/zwhISGUKlUq1/UUCgWpqakA+qJf+aFcuXIMGjSIQYMGMWbMGFauXMlXX32F8nFNEE0eZsPITEbv379PvXr1AAxuVMCTIQRqtTrH1l+lUpntWEqlEm9vb8LCwnKdp3v8+PG0bduWzz//nNKlS7N69Wp69OjBuHHjDNabPn06q1evNki6nZ2dn/t/fOHCBUxNTalRo8ZzY3gVL9xn5YMPPmDu3LlMmDCBunXrEhISwr59+/TF1W7dusX9+/f16y9fvhyVSkXXrl0pVaqU/mfu3LkG+92yZQuSJNGzZ89sxzQzM2PLli20atWKGjVqMH36dIYOHcpPP/30ouELhSxFlcHPx6/Tek4gQ7ee42LkI6yUCga0qMjRUW2Y062OSLiFoisjHXY/rg3R6BMoU//56wtCIbGsXw+bDh1AkoiePcfY4eSZNi2NqMdfeJ369sGskpjnXihB5HJ4/yewKQ0PLsHuoZClZbCkW7JkCenp6fj7+3P06FFu377Nvn37aN++PVWqVGHChAkA1KxZEwcHBzZv3myQdO/YsYP09HSDqaMmTJjA+vXrmTx5MqGhoYSHh7NlyxaDlurcXLhwgTZt2uDn58ewYcOIjIwkMjKSmJiY526nUqkICQkhJCQElUrF3bt3CQkJ4cqVKwbn3K5dO/3zdevW8csvv3Dx4kUuXrzI999/z5o1a7K1LGdkZOjjuHz5MtOmTSMsLIz33svfWSm++eYb9u/fz/Xr1zl79ixHjhyhevXqAFSoUAGZTMbu3buJiYkx6ML+NAsLC5o2bcrMmTMJDw8nKCgo2/9BZvG7Hj168M8//3D58mU2bNhARISuCLKHhwfnz58nIiKCBw8e6FvK/f39+euvv3I9l2bNmlG7dm2+//57YmJi+N///kefPn2oWbOmwU/v3r3ZsWPHC01Rd+zYMVq0aPFSrfV5Jr0mEhISJEBKSEgwdijPpVKppB07dkgqlcrYobyS2KR0acGBCKnu5P1ShW93SxW+3S01mBogLTl8WYpPLt7nVlSUlGulSAuaLUkTbSVpdmVJSokzdjSvRFwvJU/6rVtSeM1aUljVatKjo0fzdd8Fdb1EL1oshVWtJl1q1VrSJCXl674F4xCfLTm4GSxJkxx0fz/+WftCm6ampkphYWFSampqwcRWwK5fvy716dNHcnNzk2QymQRInTt3lpKTkyVJkiSNRiPFxcVJ7777rmRiYiI9evRI/7qDg4PUtGnTbPvct2+f5OPjI1lYWEi2trZS48aNpZ9++km/HJD++OMPg23s7OyktWvXSpIkSRMnTpSAbD8VKlTI9Vxy2q5Vq1b6dSZOnGiwn59//lmqXr26ZGlpqY9127ZtBvt9Oh5LS0upVq1a0vLly3N5dw3PNTO+f//9V788Li5OAqQjR45IkiRJgwcPlipVqiSZmZlJLi4u0scffyw9ePBAv/6UKVMkd3d3SSaTSX36SuArrQAATnZJREFU9JEkSZJatWolDRkyJNuxw8LCpGbNmkkWFhZS3bp1pYCAAINjSZIknTt3TvLz85MsLS0lGxsbqUWLFtLVq1clSZKk6OhoqX379pK1tbXBdqGhoZKFhYUUHx9v8B7VqVNHiouLkzQajf71X375RTIzM5MmTZok2dvb5/i5k56eLtnb20uLFi2SJEmSKlSoIC1YsOC572vVqlWlX3755ZnLn/d7mdccUyZJr8ctuMTEROzs7EhISCjy1cv37t1Lx44di2VBkrvxqaw6do0tp26TqtZ1ISnvaMmAlp50a1AWc1PRNTe/FPdrpciLvQ7LmkJGGnReCbW7GzuiVyKul5IpatZsYteuxcyrMhX/+AOZSf5MSlIQ14vq5k2uvfMukkpFmYULse3gny/7FYxLfLY8w18L4eBEUJjBgEPgXitPm6WlpXH9+nUqVqz4zJl+ipOJEycyf/58Dhw4QNOmTdFqtSQmJmJra5vnIm1CydetWzfq16/PmDFj9K8V1rXy559/Mnz4cM6fP6+f+/1pz/u9zGuOKa52IV9ERD5i2NYQWs0+wtrjN0hVa/AuZcsPPetxeHgrPm5aQSTcQvEhSfDnKF3CXbEl1Opm7IgEIUfOgz5DYWdH+uUrxP++3djhPJMkSUROn46kUmH1xhvY+D9/7J4gFHs+X4OXP2jS4dc+kJa3aYVKmsmTJ7N48WL+/vvvbIXNBCHTnDlzDGp9Fabk5GTWrl37zIQ7vxTqPN1CyXP6RizLA69y+GK0/jWfSk4MalWJFl7OokCOUDyF/w8uB4BCCW/NB3EdC0WUws4O5y+/JOr774lZvBjbt95CYV305rt+dPAgyUeP6aY8Gz9O/G0QSj65HN5fAStaQOxV+N8Q3bRir+G1n1lsGRCJt5AjDw+PXCuqF5SuXbsWynFES7fwwrRaiQNhUXRZfoJuK4I5fDEamQw61nJn55dvsHlAU1qKebaF4ir9EewbrXv8xhBw9jJuPIKQC4ceH6CsUAHNw4c8XLXS2OFko01JIerxjCZO/ftjVrGikSMShEJi6Qjd1oLcBEK3wz+rjR2RIAhGIpJuIc9UGVq2/XMb/4VHGbD+H87cjEOpkNOzcTkODWvFsl4NqFPO3thhCsKrCZwJiXfBwQNaDM91dUEwNplSietI3XyusWt/Rp1lBpGi4MGPP5Fx7z4mpUvhPOgzY4cjCIWrXGPwnaR7vG8M3AsxZjSCIBiJ6F4u5Co5PYNfTt1i9V/XuZ+QBoCNmQm9mlag/xseuNoW/0IfggBA5AX4e7nucce5YFqAU0cIQj6ybtcOy4YNSfnnH2IWLqT0rFnGDgmA9GvXebhmDQDuY8ciL8jpWAShqGo2GG6egIi9uvm7PzsK5nbGjkoQhEIkkm7hmR4mpfPziRusD75JQqpuLj0XGzP6v1GRXk3LY2suqpQKJYhW+3hOVQ1Ufxe82hs7IkHIM5lMhuu333KjWzcSdu7C4ePeWNSsYdSYJEkiato0UKuxatkC6yxz2QrCa0Umg07LYEVLiLsBOwdD9/XPHd/9mkwuJAjFQn78Poru5UI2t2NTmLDzAj4zD/PD4SskpKqp6GzFjM61ODaqDZ+3riQSbqHk+XcD3DkFSmvoMNPY0QjCC7OoVRPbd98BIHrWLKN/aX+0P4DkEyeQKZW4jx8v6nwIrzcLB+j2M8hNIXwXnPopx9UUCt1MLyqVqhCDEwTheVJSUgBeaVpE0dIt6IXdS2RF0FX2/HcfjVb3Za12WTs+b1UJvxruKOTiC5NQQiU/1M2nCtBmLNiVMW48gvCSXIcO5dH+AFJOnybp8GFsjNS6rE1OflI87dNPUZYvb5Q4BKFIKdsA/KbqinXuHwdlG0KZBgarmJiYYGlpSUxMDKampiVuLmutVotKpSItLa3EnZuQv4rCtSJJEikpKURHR2Nvb6+/KfYyRNL9mpMkib+vxbI86CpHL8XoX2/h5cznrSrRrJKTaJ0QSr4DEyA1DtxqQWNR6EkovkxLlcKxb18e/vgj0XPmYt2yJbJXuDP/sh4sX05GVBSmZcviNHBAoR9fEIqsJoPg5nHd1JTb+urGd1s46BfLZDJKlSrF9evXuXnzpvHiLCCSJJGamoqFhYX4fik8V1G6Vuzt7XF3d3+lfYik+zWl1UoEhEWyPOga527HAyCXwVu1S/NZS09qlhEFPoTXxM0TELJR9/jt+aAQH4tC8eY0YADxv/2G6sYN4rZsxfHjjwr1+OlXr/Lw53UAuI0bi9xcFNsUBD2ZDN5dAvfPQ/xN2PEl9NhkML5bqVTi5eVVIruYq9Vqjh49SsuWLV+pq65Q8hWVa8XU1PSVWrgziW+Xr5n0DA1/nL3LT0evce1BMgBmJnK6NSzLgBaeVHCyMnKEglCINGrYPUz3uH4f3dQuglDMKaytcPnqKyInTeLB0qXYvfcuClvbQjm2JElETpkKGRlYt2mDTZs2hXJcQShWLOyh+zpY7QcRe+DvZdDsS4NV5HI55iXwhpVCoSAjIwNzc3ORdAvPVdKuFZF0vyYepanZfFI37Vf0o3QAbM1N6N3Mg75veOBsbWbkCAXBCIKXQkw4WDo9mUdVEEoA+65diN24AdWVqzxY8SNuo0YWynET9+4l5eRJZGZmuI0bWyjHFIRiqXQ98P8e9o7QDXEq2xjKNTJ2VIIgFBBRwSCfLV26FA8PD8zNzWnSpAmnTp165rorV66kRYsWODg44ODggK+vL6dPn862Xnh4OO+++y52dnZYWVnRqFEjbt26pV/+008/0bp1a2xtbZHJZMTHx+uXRT9KY9a+i/jMPMyMPy8S/Sgdd1tzxnWszokx7RjhX1Uk3MLrKf4WBD2ey9hvGlg6GjceQchHMhMT3EaNAiBuwwZUd+4U+DE1SUlEz9T9Tjl9NhBl2bIFfkxBKNYafQrenUCbAb/1g5RYY0ckCEIBEUl3Ptq6dSvDhg1j4sSJnD17ljp16uDv7090dHSO6wcGBtKzZ0+OHDlCcHAw5cqVo2PHjjx8+FC/ztWrV2nevDnVqlUjMDCQ8+fP89133xl0OUpJSaFDhw6MHfukVeHGg2TG/vEfzWcdYXngVR6lZVDJxYrZXWtzdFQbBrT0xNpMdHQQXmN/jgZ1ClR4A+r0NHY0gpDvrFq0wMrHB0mtJnrevAI/3oMlS8mIicG0fHmcPvmkwI8nCMWeTAbv/gCOnpBwG3Z8DlqtsaMSBKEAiKwrH82fP58BAwbQr18/AFasWMGePXtYs2YNo0ePzrb+pk2bDJ6vWrWK33//nfPnz+tfGzduHB07dmT27Nn61ypVqmSw3TfffAPokniAEb+e4/D1JB7P+kW98vYMalWJ9tXdkItpvwQBLu7VjaOTm8Bb8wwK2AhCSSGTyXD9dhTXO73Poz/3kdK7N5b16hXIsdIuXSJ2wwYA3L8bj9xM9KAShDwxt4Vu62CVL1zaB8E/wBtDjB2VIAj5TLR05xOVSsWZM2fw9fXVvyaXy/H19SU4ODhP+0hJSUGtVmNtbQ3o5qfbs2cPVapUwd/fH1dXV5o0acKOHTsMtpMkib8uP2DanjAAAsIi0UrQpqoLWwc2ZfvnPvjXcBcJtyAAqJLhT123W5oNBtfqxo1HEAqQedWq2HXpDED0zFlIkpTvx5AkiagpU0Gjwaa9L9YtWuT7MQShRCtVG96cqXt8cDLc+tu48QiCkO9E0p1PHjx4gEajwc3NzeB1Nzc3IiMj87SPb7/9ltKlS1OnTh0AoqOjSUpKYubMmXTo0IGAgADef/99OnfuTFBQEBqtxO7z93hnyV98tPokF+4mALppv/Z904K1/RrTxFPMsy0IBoJm67rx2ZWHVqOMHY0gFDiXr79GZmlJ6rlzPNq3L9/3n/i//5Hyzz/IzM1xy6FXlyAIedCgH9TsCpIGtvWD5Ie5byMIQrEhupcXETNnzmTLli0cOHCAO48L3mgfj+t57733GDp0KAB169blr7+OM3r6PJTttdx8mAKAuamc1jXcWQfM6FwLe/vCmR5GEIqV6HAIXqJ7/OYsUIop8oSSz9TVFadP+vPghyVEz5uPdbt2yJXKfNm35tEjombPAcD5888xLVMmX/YrCK8dmQze+X97dx4XVb3/cfw1wyqiAi4gouK+FbijaaWJe6WluaS53LKbRXWlxaVc0lJyu2Z59aZZ9rt5rdwqFxQXrIzUNK+7uWTmAq64CwMzvz+OjJIbKHgGeD8fj3l0OOc7Z94Hvs34mXO+3zMJjm6Gk3thwfPw9Ndg1fkxkfxA/yfnkBIlSuDm5kZSUlKm9UlJSQQFBd3yuePHjycmJobly5cTFhaWaZ/u7u7UrFkTgDOXbExZvZdfzvjw6469/HHyIn4+Hrzaogo/DWpBnyYVcv7ARPILhwMWv2bMElutPVRvZ3YikXumeN++uJcqhe3QIU7/339ybL/HJ39I+okTeIaGEtC3T47tV6RA8ipijO9294a9K2DtP81OJCI5REV3DvH09KRevXqsXLnSuc5ut7Ny5UoaN2580+eNHTuWUaNGERsbS/369a/bZ4MGDfjfth2MXrKTJjGrGLdsN2eTDuJbPIhhj9bkp0GPMKBlVQIK58xZC5F863//hT/WgoePcZZbpACx+vhQ8sqkmyemTSPt9Om73uflXbs4fWVC0MChb+fY2XORAi3oPmhnXD3CqnfhwFpz84hIjlDRnYOio6OZPn06s2bNYufOnfTv358LFy44ZzPv1asXgwcPdrZ///33GTp0KDNnziQ0NJTExEQSExO5dOkSAPuOnyewyVPM/fprJkz+F6cTD+L923JS9m9g7uQR/K1pBXw83UlMTGTz5s3s3bsXgK1bt7J582ZOndL9HkUA496ny982lh8eCH5lzc0jYoJiHTvgVaMG9nPnODHlX3e1L4fdTuI7I8Fup0ibNvg2aZJDKUWEOs9AWDdw2GHu3+D8cbMTichd0pjuHNS1a1eOHz/OsGHDSExMpHbt2sTGxjonVzt48CDWa8bmTJ06ldTUVDp37pxpP62f6MrKlEqs2HUMh1tVird+kZRf5nM+fgYlq1dj/rx5NHv4IWf7adOm8c477zh/fughY9unn35Knz59cvGIRfKIFSPg4kkoWQMav2R2GhFTWKxWAge+ycE+fTk9Zw7+PZ7Gq8KdDUs6s/AbLv36KxYfHwIHDczhpCIFnMUCj06EI7/Cid0wvx/0nAdWN7OTicgd0pnuHBYVFcUff/xBSkoK69atIyIiwrktPj6ezz77zPnzgQMHcDgczseepHN0mbaWXVWfIW7nMRwOiKwRyLKpwzl37E9SLl9i8+bNdOjQIdNrjhgxItN+Mh4quEWAP9fDplnG8qMTwc3D3DwiJircqBG+zZpBWhrHxk+4o32knznDsfHjASj50ot43GbeEhG5A56FocssY0jU/tXww539/yoirkFFtwvx8/Fg859nsFocPFEnmLgBDzGjd33qlQ8wO5pI3pSeBouijeXaPaD8A+bmEXEBpd58A9zcOL9yJRfWr8/2849/8AHpp07hWakSAc88kwsJRQSAUjWg/ZViO34M/P69uXlE5I6p6HYhJXy9GN/5fobVSWfsk/dRJbCI2ZFE8rb1/4akrVDIH1qONDuNiEvwqlgR/65dADgW8z6OK7enzIpL27Zz+r9zAAgaOhSLJk8TyV21n4baPa+M734WziXd/jki4nJUdLuYNrUC8fcyO4VIPnDmMKwebSxHvgOFS5ibR8SFlIiKwurry+UdOzj73XdZeo7Dbidx1EhwOCjavj2FG0Xc/kkicvfajYNSNeHCMZj3LNjTzU4kItmkoltE8qdlgyH1PIQ0NGaCFREn94AAiv/9eQCO/XMS9it3zbiV5HnzuPy/LVh9fCj15pu5HVFEMnj6GPfv9igMB36ANbrtpUheo6JbRPKfPXGw4xuwuMGj/wSr3upE/iqgVy88goNJS0zk1KxZt2ybdvo0xydMBKDEyy/jEVjqXkQUkQwlq8Jjk4zlNWNh3ypT44hI9uhfoiKSv9guwZLXjeVG/SHoPnPziLgoq5cXJaONiQZPfjydtOM3vxfw8UkfkJ6cjFeVKgT07HGvIorItcK6QN3egAPm9YOzR81OJCJZpKJbRPKXHybA6QNQJBiaDTI7jYhLK9q+Hd5hYdgvXuT4hx/dsM2lrVtJ/uorAIKGDcXiodvuiZim7fsQeD9cPGGM705PMzuRiGSBim4RyT9O7IEfJxnLbWPAS3cAELkVi8VC4KCBACTPncvl337LtN2Rnk7iO1cmT3v8MXwaNDAjpohk8CgET30Gnr7wx1qIH212IhHJAhXdIpI/OBywOBrsNqjSCmo8bnYikTzBp25dirRqBXY7x8aNz7Tt7Lx5XN62DauvL4FvvGFSQhHJpERleHyysfzDBNizwtw8InJbKrpFJH/YOhd+/x7cvaHtWLBYzE4kkmeUev018PDgwg8/cP7HtQC4nT/PyQ+Mf9iXfOUV3EuWNDOiiFzrvk5Q/1ljeX4/4zaZIuKyVHSLSN53KRmWDTGWH3odAiqYGkckr/EsV46Ap58G4NjYsTjS0ykRG4v97Fm8qlfH/+nuJicUkeu0Hg1BYXDpFMz9G6TbzE4kIjeholtE8r5Vo+DCMSheBR54xew0InlSif4vYC1WjJTffuP4e6MptuEX4Mrkae7uJqcTket4eEOXWeBVFP782fgsFBGXpKJbRPK2wxthwyfGcvsJ4O5lbh6RPMrNz4+SL/YH4OzXXwNQpEMHfOrWNTOWiNxKQEXocOXOA2s/gN2x5uYRkRtS0S0ieZc9HRZFAw64vwtUfNjsRCJ5mn/37niULwdAeiFvig/4h7mBROT2anaAhn83lhe+AMl/mptHRK6joltE8q4Nn8DRzeBVDFq/Z3YakTzP4ulJ6REjcC9dmmMdn8C9eHGzI4lIVrQaBcF14dJpmNsX0lLNTiQi11DRLSJ507nEq+PXIoeBbylz84jkE4UbNyZ0+TLO1Q43O4qIZJW7Fzz1KXgXg0MbYOU7ZicSkWuo6BaRvGnZW5By1vhmv15fs9OIiIiYyz8UOvzLWE74CHYtNjWOiFyloltE8p59q2DbXLBY4dF/gtXN7EQiIiLmq/EoNHrJWF7YH04fMDWOiBhUdItI3mK7DItfN5Yb9IPg2qbGEXFFU6ZMITQ0FG9vbyIiIli/fv1N206fPp0HH3wQf39//P39iYyMZMOGDZna9OnTB4vFkunRpk2bTG1CQ0OvaxMTE5MrxycitxA5AsrUh8tn4GuN7xZxBXdUdN/th/lf22flw/zUqVP06NGDokWL4ufnx7PPPsv58+fvJL6I5GVrP4BT+8A3EB55y+w0Ii7nyy+/JDo6muHDh7Np0ybCw8Np3bo1x44du2H7+Ph4unfvzurVq0lISKBs2bK0a9eOkydPZmrXpk0bjh496nz897//vW5fI0eOzNTm5ZdfzpVjFJFbcPe8Mr7bD45sgrihZicSyZ6Uc1i2zaX2H9PBYTc7TY7IdtGdEx/mrVq14vDhw5na3e7DvEePHmzfvp24uDgWLVrE999/z/PPP5/d+CKSl53cBz9MMJbbjDEmjBGRTCZOnEi/fv3o27cvNWvWZNq0afj4+DBz5swbtv/iiy948cUXqV27NtWrV2fGjBnY7Xa2bNmSqZ2XlxdBQUHOh7+//3X7KlKkSKY2hQsXzpVjFJHb8CsHT/zbWF43DXZ8Y24ekdtJvQDb5sOXPWFcZdy/eYHyp37AcvgXs5PliGwX3Tn1Yb5y5cpM7W71Yb5z505iY2OZMWMGERERNG3alA8//JA5c+Zw5MiR7B6CiORFDgcseQPSU6Bic6j1pNmJRFxOamoqGzduJDIy0rnOarUSGRlJQkJClvZx8eJFbDYbvr6+mdbHx8dTqlQpqlWrRv/+/a87Ew4QExND8eLFqVOnDuPGjSMtLe3uDkhE7ly1NvDAK8byN1Fwar+5eUT+ynbJ+ELo6z4wrrJxu7ud30HaZRwBFdkd+DiOIsFmp8wR7tlpnPFhPnjwYOe6O/0wDwgIyLQ+48Pc39+fRx55hHfffZfiV+4PmpCQgJ+fH/Xr13e2j4yMxGq1sm7dOp544onsHIaI5EU7FsK+leDmBe0ngMVidiIRl3PixAnS09MJDAzMtD4wMJBdu3ZlaR8DBw4kODiY8PCrtwxr06YNTz75JBUqVGDfvn0MGTKEtm3bkpCQgJubMZHhK6+8Qt26dQkICOCnn35i8ODBHD16lIkTJ+bcAYpI9rQYBn+uMx5f94G/LQcPb7NTSUFmu2z8e277Ati9FFKvGS7sH2qcVKn1BGnFq7Nr6VIqFgsxLWpOylbRnZMf5td+C3+7D/PExERKlcp8D153d3cCAgJITEy84eukpKSQkpLi/Pns2bMA2Gw2bDZblrKaISObK2cU11Cg+krKOdyXDsQCpD/wCvai5aAgHHcOKlD9pQDL+PumpaVl+lunp6fjcDhu+/cfO3Ysc+bMYenSpSQmJjrbd+rUydmmevXq1KhRg+rVq7NixQoeeeQRgEzjt2vUqIGbmxsvvvgiI0eOxMvLK8eOUVyL3lvygI7TcZ/RDMvR/5EeOxh7m7GmRVF/KaDSU7Hsj8e6cyGW35ZiSTnn3OQoVhZ7jQ44anTAUbq286RKXukrWc2XraL7bsXExDBnzhzi4+Px9r76LVu3bt2cy/fffz9hYWFUqlSJ+Ph4WrRocUevNWbMGN55553r1i9fvhwfH5872ue9FBcXZ3YEySMKQl+579B/qHQ+ifNegaw+Ww37kiVmR8qzCkJ/KchsNhtWq5UlS5Zw6tQp5/pff/0Vi8XCklv8v7Nw4UK++uorRo4c6fxC+1b9pWjRonzzzTdcvnz5htsvX75MWloan3/+OWXKlLnDI5K8Qu8trq1U6b403j8Bt40z2XTSmyP+jUzNo/6S/y1Z/B3fLpjHqeSzhAe68VFbTxqWMa6MuuThz2G/hhzxb8Rpn4os/y6O1dF9OHjwIACVKlWiZ8+eVK1a1dlXPvjgA1avXp3pNerUqcPw4cMBSEpK4quvvmLr1q0kJyfj7+9Ps2bN6Ny5Mx4eHrl2nBcvXsxSu2wV3SVKlMDNzY2kpKRM65OSkggKCrrlc8ePH09MTAwrVqwgLCzslm0rVqxIiRIl2Lt3Ly1atCAoKOi6idrS0tI4derUTV938ODBREdHO38+e/ascxK3okWL3vL1zWSz2YiLi6Nly5a52kEk7yswfSVxC+6bVwDg/eRHtKnY3ORAeVOB6S9CvXr1OHv2LO3atQPAbrfz0ksv0b9/f+e6vxo/fjzz589n2bJlRERE3La/HDp0iHPnzhEZGXnTfc6ePRur1Urnzp1vOOma5A96b8kr2pG+Oh23nyZR/8jnpLXtDQGV7nkK9Zd8zp6G5Y+1fP3xOD6buZxp7b2JCCnEpJ9Taf3FJXZ8/HdKPvA07iENKG+xUv7K0+Z8+SX9+/enUaNGeHt7M378eN577z0mTpxIt27d8PDwYN68ebRu3Zrp06c7X87Ly8v5+bJs2TLKlClDdHQ0lSpVYvv27fTv35/g4GDef//9XDvkjKupbydbRbenpyf16tVj5cqVdOzYEcA5KVpUVNRNnzd27Fjee+89li1blmlc9s0cOnSIkydPUrp0aQAaN25McnIyGzdupF69egCsWrUKu91ORETEDffh5eV1w8vZPDw88sT/5Hklp5gvX/cVux1i3zRuF1HrSdyrtTI7UZ6Xr/uLAPDaa6/Ru3dvGjZsSMOGDZk0aRIXLlzgueeew8PDg169elGmTBnGjBkDwPvvv8+IESOYPXs2lStX5uTJk9hsNi5duoSHhwcpKSm88847dOrUiaCgIPbt28ebb75J5cqVad++PR4eHiQkJLBu3TqaN29OkSJFSEhI4I033qBnz57XDQ+T/EnvLXlAi6FweAOWP9biMf85eC4OPAqZEkX9JR+xp8MfP8H2+bDjW7h4gklzztOvrgd9m5aBmh2Y9loHFj/YnVmHyjGoYtPrdvHXu1bNnDkTf39/tmzZwjPPPIOHhwdWqxVvb2/Kli17wxiPPvoojz76qPPnatWqsW/fPqZOnZqrc4tktR9n+/Ly6OhoevfuTf369TN9mPft2xfghh/mw4YNY/bs2YSGhjovWfP19cXX15fz58/f9MO8devWgDE2rE2bNvTr149p06Zhs9mIioqiW7duBAfnjxntROQGNn0Gh38BzyLQerTZaUTyhK5du3L8+HGGDRtGYmIitWvXJjY21jkfy8GDB7Far968ZOrUqaSmptK5c+fr9tOpUyfc3NzYsmULs2bNIjk5meDgYFq1asWoUaOcX257eXkxZ84cRowYQUpKChUqVGDAgAGZrjgTEZO5uUOnT2BaU0jaCrGD4LEPzE4leZHdDn/+bEyGtuMbOH/1KuhUD382Hj3H4LeHQb9B4OaOFXL0Lho3mnj7Rs6cOXPd5N1myXbRnVMf5sOHD2fEiBFZ+jAH49ZjUVFRtGjRAqvVSqdOnZg8efKdHreIuLrzx2HFCGP5kbehaGlT44jkJVFRUTe9Ai0+Pj7TzwcOHLiujc1mc47/LlSoEMuWLbvl69WtW5eff/75jrKKyD1UtDR0mg7/9yRs/AzKN4Wwp8xOJXmBwwGHNhiF9vaFcO6a2zZ7+0GNR6HWk5zwqkz626EEhrcwvui54l7cReNae/fu5cMPP2T8+PF3esQ56o4mUrvbD/NrZeXDHCAgIIDZs2dnNaKI5HXL34bLZyAoDBo8Z3YaERGR/KHSI/DQG/D9WPjuVSgdDiWrmp1KXJHDAUc2wbb5xhntM39e3eZVFKo/CrWegIrNwN3TWH/kyA13lVUZE2/HxcVx6NAh5/rsTLx9+PBh2rRpw1NPPUW/fv3uKk9Ouaezl4uIZMnvP8CWOYAFHp2U6ZtSERERuUvNBsHBBDjwA3zdG55bCZ6uf3cfuQccDkjcYhTa2xdA8h9Xt3n6QrV2RqFduQW4Xz9/Vk5OvH1t0f1Xf514O8ORI0do3rw5DzzwAB9//HEWDzr36V+yIuJa0lJh8WvGcv2+EFLP3DwiIiL5jdXt6vjuYztg6RvQYYrZqcQsDgckbb9y6fh8OLX/6jYPH6jaBu57EipH3nbyvZyaePt297/+68TbYJzhbt68OfXq1ePTTz/NNOTZbCq6RcS1JHwIJ3ZD4ZLQYpjZaURERPKnIoHQaQb8X0f49T/G+O7a3c1OJffSsV1Gkb19AZz47ep6d2+o0sootKu0zvZVEDkx8XbGXTSALE28ffjwYZo1a0b58uUZP348x48fd+a53Rn2e0FFt4i4jtMHYM04Y7nVe1BI9/YVERHJNRUfhocHQfxoWBwNwXWgVHWzU0luOrH3aqF9bMfV9W5eUKWlcel41Tbg5XvzfdyGGXfRiIuLY+/evezdu5eQkJBM+3E4HHd8LDlFRbeIuAaHA5a8CWmXIPRBCOtidiIREZH876HX4eBPsD/eGN/dbxV4FjY7leSkU/uNInvbAuN2cRmsHsbY7FpPQrW24F00x17yXt9Fo0+fPvTp0+dOot4TKrpFxDXsWgR7lhkfAO0ngsVidiIREZH8z+oGT84wxncf32XMq9Jxqj6H87rTf8COhcaEaEc3X11vdTdmG6/1JFRvp6sK7xEV3SJivpTzsHSgsdzkFd26RERE5F7yLQmdP4FZj8H//gvlm0DdZ8xOJdl15pBxD+3tC+DwL1fXW9ygwkPGpeM1HgOfANMiFlQqukXEfGti4Oxh8CsHD75udhoREZGCJ7QpNH8LVo2CJa9DmboQWMvsVHI7Z48a99DePh/+XHd1vcVqfHly35NQ43EoXMK8jKKiW0RMlrQdEv5lLLeboPuEioiImKVptHH/7r0r4Kve8Pxq8Cpidir5q/PHrhTaC+CPn4CMicIsUK7x1UK7SKCZKeUaKrpFxDx2OyyKBke6cblT1VZmJxIRESm4rFZ44mNjfPfJPbBoADw5XeO7XcGFE7DzW6PQPvAjOOxXt5WNMC4dr9kBigabl1FuSkW3iJhn8xfw58/gURjaxJidRkRERAoXh6c+hU/bwdavjUuU6/c1O1XBdPGUMdHstvnw+/fGSYoMZeoZk6HV7AB+Zc3LKFmioltEzHHhJMQNNZabD4ZiIbduLyIiIvdGuUbQYhisGG5MdFqmHpQOMztVwXApGXYvMQrt/avBnnZ1W+lwo9Cu1RH8Q00KKHdCRbeImGPFMLh0GkrVgogXzE4jIiIi13rgFWO88J5lxv27n1+To/dxlmtcPgu7lxqXju9bCempV7cF3m8U2bWegOKVTIsod0dFt4jcewd/hl//Yyw/OhHcPMzNIyIiIplZrfDENJj2IJzaD9+9Ap0/1fjunJJyHn6LNQrtPXGQnnJ1W8kaRpFd6wndRjWfUNEtIvdWus2YmAWgzjPGJWwiIiLienwCrozvbmsUh+WbQMN+ZqfKu1Ivwp7lxu29flsOaZeubitexZh1vNYTUKqGeRklV6joFpF76+epcGwHFAqAliPNTiMiIiK3UrYhRI6A5W/DsiEQUh+C65idKu+wXYa9ccaXFrtjwXbh6jb/ClcK7SeNe6LrKoJ8S0W3iNw7Zw5B/JVZyluNMr5BFxEREdfWOMoY3717CXzdB/7+PXgXMzuV60pLgX2rjEJ71xJIPXd1m1+5K5eOP2lMjKZCu0BQ0S0i987SgcY3vOUaQ/jTZqcRERGRrLBYoOO/YNpDcPoAfBMFXT5XwXittFT4fY0x6/iuxZBy5uq2oiFXJkN7EsrU1e+tAFLRLSL3xu5Y416TVndoP9GYoEVERETyhkL+8NRnMLM17PwW1n8MEX83O5W50tPgwPdGob3zO7icfHVbkdJQs6NxVjukgf7dU8Cp6BaR3Jd6EZa+YSw3ehECa5qbR0RERLIvpJ4xPCx2ECx7yxjfXaae2anuLXs6HPjRuHR857dw8eTVbYVLQc0Oxjjtso1UaIuTim4RyX3fj4Pkg8blVQ8PNDuNiIiI3KmIF+CPtcaZ3Yzx3YX8zU6Vu+zpxu1Ot8+HHd/ChWNXt/kUhxqPG4V2+SZgdTMvp7gsFd0ikruO74afPjSW240FL19z84iIiMids1jg8Y/g6BZI/gMWvgTdvsh/45Ttdji03jijvX0hnE+8uq2QP9R4zLh0PPQhcFNJJbemHiIiucfhgMWvgd0GVdtC9fZmJxIREZG7VcgPusyCT1rB7sXw87+g8Utmp7p7Dgcc3nil0F4AZw9f3eZVDGo8akyGVvFhcPMwL6fkOSq6RST3bPkSDvwA7oWg7ftmpxEREZGcElwHWo+GJa9D3DAIaQhlG5idKvscDji62ZgMbftCOHPw6jbPIlC9nVFoV2oO7l5mpZQ8TkW3iOSOi6eMSVYAHn4T/Mubm0dERERyVoPnjEnFdiyEuX2N8d0+AWanuj2HA5K2XSm0F8Dp369u8ygM1doYhXblSPDwNi+n5BsqukUkd6wcCRdPQMnq0DjK7DQiIiKS0ywWePxDSNwCp/bDwv7Q7b+uO2t30o4rl47Ph5N7r653LwRVWxuToVVuCZ4+5mWUfElFt4jkvEO/wMbPjOX2E8Dd09Q4IiIikku8i8JTs2BGJPwWCwkfQpNXzU511fHfjCJ7+wI4vuvqejcvqNLSKLSrtgHPwuZllHxPRbeI5Kz0NFj0D8AB4U9DaFOzE4mIiEhuKh0GbWNg0QBY8Q6UjYByjczLc3LflUJ7oXEZeQY3T+OS8VpPQLW24FXEtIhSsKjoFpGctWE6JG4Fbz9oNcrsNCIiInIv1OsLB9bCtrnwdV944UcoXPzevf7pA8bZ7G3zjcvdM1jdodIjxhjtam2NmddF7jEV3SKSc84egVXvGcuRI6BwCVPjiIiIyD1iscBjk4yZwE/uhQXPw9Nf5+747uQ/jUncts2HI5uuyeJm3Nar1pPG7UrzwuRukq+p6BaRnBM7GFLPQUgDqNvb7DQiIiJyL3kVuTK+uwXsXQFr/wkPvpazr3H2iHHZ+PYFcGj91fUWK4Q+aFw6XuPxe3uWXeQ2VHSLSM7Yu8L4ttlihfYTXXfmUhEREck9QfdBu3Hw7cuw6l0o2whCm9zdPs8lwY5vjHHaBxOu2WCB8k2gVkeo2QF8S93d64jkEhXdInL3bJdg8evGcsQLxoQqIiIiUjDVecYY371lDsz9mzG+27dk9vZx/jjs/NY4o33gR8BxdVvZRsas4zUeh6KlczS6SG5Q0S0id+/Hf8Lp36FIMDQfYnYaERERMZPFAo9OhCO/wondML8f9Jx3++ddPHW10P79e3DYr24rU98otGt2gGIhuZddJBeo6BaRu3Nir1F0A7QZo9tviIiIiHHf6y6zYPojsH81/DABHhhwfbtLp2HXYmMytP3x4Ei/ui24jjFGu2ZH8C9/r5KL5DgV3SJy5xwOWPIapKca972s2cHsRCIiIuIqStWA9hNgYX+IH4MluL6x/vJZ2BFnFNr7VoHddvU5QfcbhXatJyCgojm5RXKYim4RuXPb5hnfSrt7G5OmWCxmJxIRERFXUvtpY3z35v/gtvDvNHQPwX3Lc8YX9hlK1TRu71XrCShR2bysIrlERbeI3JnLZ2DZlfHbD76mb6NFRETkxtqNgyObsBzbQWmOGetKVL1aaJeqbm4+kVymoltE7syqd+F8EhSvDE1eNTuNiIiIuCpPH+j6H+zLh7In2Y2Kj7+OR3CYrpCTAkM30hWR7DvyK2yYYSy3nwDuXubmEREREddWvBLpnWexK7izcTm5Cm4pQFR0i0j22NNh0QDjNh73PwUVm5mdSERERETEZanoFpHs+WWmcabbqxi0es/sNCIiIiIiLk1Ft4hk3bkkWDnSWG4xFIoEmptHRERE7pkpU6YQGhqKt7c3ERERrF+//qZtp0+fzoMPPoi/vz/+/v5ERkayYcOGTG1GjBhB9erVKVy4sLPNunXrMrU5deoUPXr0oGjRovj5+fHss89y/vz5XDk+kdyioltEsm75W5ByFoLrQP2/mZ1GRERE7pEvv/yS6Ohohg8fzqZNmwgPD6d169YcO3bshu3j4+Pp3r07q1evJiEhgbJly9KuXTtOnjzpbFO1alU++ugjtm7dyo8//khoaCitWrXi+PHjzjY9evRg+/btxMXFsWjRIr7//nuef/75XD9ekZx0R0X33X7LdW17m83GwIEDuf/++ylcuDDBwcH06tWLI0eOZNpPaGgoFosl0yMmJuZO4ovIndgfD1u/BizQfiJY3cxOJCIiIvfIxIkT6devH3379qVmzZpMmzYNHx8fZs6cecP2X3zxBS+++CK1a9emevXqzJgxA7vdzpYtW5xtnn76aSIjI6lYsSK1atVi4sSJnD171tlm586dxMbGMmPGDCIiImjatCkffvghc+bMua5WEHFl2S66c+JbrlatWnH48GEALl68yKZNmxg6dCibNm1i/vz57N69m8cff/y6fY0cOZKjR486Hy+//HJ244vInUhLgcWvGcsNnoMydc3NIyIiIvdMamoqGzduJDIy0rnOarUSGRlJQkJClvZx8eJFbDYbvr6+N32Njz/+mGLFihEeHg5AQkICfn5+1K9f39kuMjISq9V63WXoIq4s2/fpvvZbLoBp06axePFiZs6cyaBBg65r/8UXX2T6ecaMGcybN4+VK1fSq1cvihUrRlxcXKY2H330EQ0bNuTgwYOUK1fOub5IkSIEBQVlN7KI3K21k+HkXvANNMZyi4iISIFx4sQJ0tPTCQzMPJdLYGAgu3btytI+Bg4cSHBwsLOgzrBo0SK6devGxYsXKV26NHFxcZQoUQKAxMRESpUqlam9u7s7AQEBJCYm3sURidxb2TrTnZPfcgUEBNy0zZkzZ7BYLPj5+WVaHxMTQ/HixalTpw7jxo0jLS0tO/FF5E6c2g8/jDeWW48G72Lm5hEREZE8JSYmhjlz5vDVV1/h6emZaVvz5s3ZvHkzP/30E23atKFLly43vYJWJK/K1pnunPyW69rC/VqXL19m4MCBdO/enaJFizrXv/LKK9StW5eAgAB++uknBg8ezNGjR5k4ceIN95OSkkJKSorz57NnzwLGGHKbzZalrGbIyObKGcU13JO+4nDgtuh1rGmXsYc+RHq1x0F9M0/Se4tkh/qLZJX6SsFQrFgx3NzcOHz4cKa/9dGjRylVqtQt//4TJ04kJiaG2NhYatSowaFDhzK19/T0pHz58pQvX5569epRs2ZNPv74YwYOHEjJkiU5duxYpvZpaWmcOnWKEiVKqN/lY3nlvSWr+bJ9efndyPiWKz4+Hm9v7+u222w2unTpgsPhYOrUqZm2RUdHO5fDwsLw9PTk73//O2PGjMHLy+u6fY0ZM4Z33nnnuvXLly/Hx8cnB44md/31knuRm8nNvlL69HoaHlhJusWd1YXac2Hp0lx7Lbk39N4i2aH+IlmlvpL/VaxYkU8//RQPDw8A7HY7S5cupV27dixZsuSGz5k/fz5z585l+PDhJCUlOfvJrfrLuXPn2LZtG0uWLCElJYXk5GQmT55M5cqVAfj111+x2+2cP3/+pq8r+Yerv7dcvHgxS+2yVXSXKFECNzc3kpKSMq1PSkq67Vjr8ePHExMTw4oVKwgLC7tue0bB/ccff7Bq1apMZ7lvJCIigrS0NA4cOEC1atWu2z548OBMhfrZs2edk7jdbt9mstlsxMXF0bJlS+ebmsiN5HpfSTmH+78HGstNXuXhh5/N+deQe0bvLZId6i+SVeorBcf58+d59tlnefLJJ2nQoAEffvgh6enpjB49msDAQPr27UtwcDDvvfceAOPGjWPOnDl8/vnnPPDAA4DRX9avX8/jjz9OamoqY8aM4bHHHiMoKIiTJ08ydepUkpOTGTRoELVq1QKMMd//+c9/mDJlCjabjS+++IIuXbrQs2dP034XkvvyyntLxtXUt5OtotvT05N69eqxcuVKOnbsCBjfcq1cuZKoqKibPm/s2LG89957LFu2LNPsgxkyCu49e/awevVqihcvftssmzdvxmq1Xje5QgYvL68bngH38PBw6T9chrySU8yXa31l5Xg4dxT8Q3F7+A3c1B/zBb23SHaov0hWqa/kfz169OD06dOMHDmSxMREateuTWxsLCEhIQAcOnQId3d3Zz/4+OOPSU1NpVu3bpn207VrVzp16oTVamXPnj107dqVEydOULx4cRo0aMAPP/xA7dq1ne1nz55NVFQUrVu3xmq10qlTJyZPnqz+VkC4+ntLVrNl+/Ly6OhoevfuTf369WnYsCGTJk3iwoULztnMe/XqRZkyZRgzZgwA77//PsOGDWP27NmEhoY6Zxr09fXF19cXm81G586d2bRpE4sWLSI9Pd3ZJiAgAE9PTxISEli3bh3NmzenSJEiJCQkMGDAAHr27Im/v392D0FEbidxK6ybZiy3mwAehczNIyIiIqaLioq66Ym2+Pj4TD8fOHDgujY2m815Sbi3tzfz58+/7WsGBAQwe/bsbGcVcSXZLrq7du3K8ePHGTZsWKZvuTImVzt48CBW69VJ0adOnUpqaiqdO3fOtJ/hw4czYsQIDh8+zLfffguQ6VstgNWrV9OsWTO8vLyYM2cOI0aMICUlhQoVKjBgwIBMl4+LSA6x22HRAHCkQ82OUOXGkx6KiIiIiMjt3dFEanf7Lde1QkNDcTgct2xTt25dfv755+xEFJE7tWkWHNoAnr7QZozZaURERERE8rRs3adbRPK588dhxQhjuflbUDTY1DgiIiIiInmdim4RuSpuGFxOhqD7oeHzZqcREREREcnzVHSLiOHAWvjfbMAC7f8Jbnc0+kRERERERK6holtEIC0VFl+ZmLBeHyjbwNQ4IiIiIiL5hYpuEYGfp8DxXeBTAiKHm51GRERERCTfUNEtUtCd/gPi3zeWW70LhfzNzSMiIiIiko+o6BYp6JYOhLRLUL4phHczO42IiIiISL6iolukINu1GH5bClZ3aD8BLBazE4mIiIiI5CsqukUKqtQLxllugAdehlLVzc0jIiIiIpIPqegWKajWvA9n/gS/cvDQm2anERERERHJl1R0ixRESTsgYYqx3HYcePqYm0dEREREJJ9S0S1S0Njtxj257WlQ/VGo1sbsRCIiIiIi+ZaKbpGC5n+z4WACePhAmxiz04iIiIiI5GsqukUKkounYPlQY7nZIPAra24eEREREZF8TkW3SEGyYjhcOgWlakKjF81OIyIiIiKS76noFikoDq6DTZ8by+0ngpuHuXlERERERAoAFd0iBUF6mjF5GkCdnlC+sbl5REREREQKCBXdIgXBummQtA0K+UPkSLPTiIiIiIgUGCq6RfK7M4dg9WhjueVIKFzc3DwiIiIiIgWIim6R/C52ENguQNkIqN3T7DQiIiIiIgWKim6R/Oy35bDzO7C4GZOnWfW/vIiIiIjIvaR/gYvkV6kXYcnrxnKj/hB0n7l5REREREQKIBXdIvnVDxMg+Q8oWgaaDTY7jYiIiIhIgaSiWyQ/Ov4brP3AWG77Pnj5mptHRERERKSAUtEtkt84HMY9ue02qNIaqj9qdiIRERERkQJLRbdIfrPlKzjwA7gXgnZjwWIxO5GIiIiISIGlolskP7l0Gpa/ZSw/9Dr4h5oaR0RERESkoFPRLZKfrBwFF45DiarwwCtmpxERERERKfBUdIvkF4c2wi8zjeX2E8Hd09w8IiIiIiKiolskX0hPg0X/ABwQ1g0qPGh2IhERERERQUW3SP6wYQYkbgHvYtDqXbPTiIiIiIjIFSq6RfK6c0dh1ZVCu8Vw8C1pbh4REREREXFS0S2Sx7mtGAqp56BMPajX1+w4IiIiIiJyDXezA4jInSt5dhvWfQvBYoVH/wlWfY8mIiIiIuJKVHSL5AUOB6SnGo+0K/+9fI6wQ7OM7Q3/DqXDzc0oIiIiIiLXUdEtAsbs3xlF7bWPjAI33QbpKVeX01JuvD7Tc262r2sfN9pXxn6uWbbbrovsceXh8A3C0nzIPf+ViYiIiIjI7anoltx3o7O0WSpkb1B83nR9ForbWz3HYTf7t5Q9Vnccbp6kODxwbzcRd++iZicSEREREZEbUNGdH+TBs7Quz80L3DzBzQPcvYz/unleWX9lOdP6ax7u1/7scYvnXLve8/r93HS9B1jdSLPZWLZkCe2qtDL7tyUiIiIiIjehotuVXDyF24IXaJR4BLf/fAz2tKyd2c1rZ2ktbndRfGYUsdcWxJ7cvvjNQiGbsS+rO1gsZv+WREREREQkH1DR7Uocdqx7lhEIcO4u9pOjhWwWztJmd19Wtxz6hYmIiIiIiLg2Fd2uxKsIae3+yZZtOwmr1wB3z0LZL4p1ljZPmTJlCuPGjSMxMZHw8HA+/PBDGjZseMO206dP5/PPP2fbtm0A1K1blzZt2tCuXTtnG4fDwfDhw5k+fTrJyck0adKEqVOnUqVKlUz7Wrx4MSNHjmTLli14e3vz8MMPs3Dhwlw7ThERERGRgko39XUl7l446jzDn8Wb4qj5BNR4DKq2hkqPQGgTKNsAgmtDqRpQvBL4lYMigeATAF6+RvGtgjvP+PLLL4mOjmb48OFs2rSJ8PBwWrduzbFjx27YPj4+nu7du7N69WoSEhIICQlhxIgRHD582Nlm7NixTJ48mWnTprFu3ToKFy5M69atuXz5srPNvHnzeOaZZ+jbty//+9//WLt2LU8//XSuH6+IiIiISEGkolvEJBMnTqRfv3707duXmjVrMm3aNHx8fJg5c+YN23/xxRe8+OKL1K5dm+rVq/Pvf/8bh8PBqlWrAOMs96RJk3j77bfp0KEDYWFhfP755xw5csR5FjstLY1XX32VcePG8cILL1C1alVq1qxJly5d7tVhi4iIiIgUKCq6c9iUKVMIDQ3F29ubiIgI1q9ff9O206dP58EHH8Tf3x9/f38iIyPZsGFDpjYOh4Nhw4ZRunRpChUqRGRkJHv27Lnh/lJSUqhduzYWi4XNmzfn5GFJDktNTWXjxo1ERkY611mtViIjI0lISMjSPi5evEh6ejoBAQEA/P777yQmJmbaZ7FixYiIiHDuc9OmTRw+fBir1UqdOnUoXbo0bdu2dV6yLiIiIiIiOeuOiu67LSz/2j4rheWpU6fo0aMHRYsWxc/Pj2effZbz58/fSfxcc7eXC5ctW5Z27dpx8uRJZ5usXC6c4c033yQ4ODjXjk9yzokTJ0hPTycwMDDT+sDAQBITE7O0jyFDhuDv70+LFi0AnM+71T73798PwIgRI3j77bdZtGgR/v7+NGvWjFOnTt3VMYmIiIiIyPWyXXTnRGHZqlWrbI9D7dGjB9u3bycuLo5Fixbx/fff8/zzz9/BIeeeu71ceMaMGdjtdrZs2QJk7XLhDEuXLmX58uWMHz8+tw9TXEBMTAxfffUVgwcPxtvbO8vPs9uN28u99dZbdOrUiXr16vHpp59isVj4+uuvcyuuiIiIiEiBle2iO6cKy5UrVwJZKyx37txJbGwsM2bMICIigqZNm/Lhhx8yZ84cjhw5cudHn4Ny6nJhm82Gr68vkLXLhQGSkpLo168f//d//4ePj08OHZHkphIlSuDm5kZSUlKm9UlJSQQFBd3yuePHjycmJoYlS5YQGhrqXJ/xvFvts3Tp0gDUrFnTud3Ly4uKFSty8ODBOz4eERERERG5sWzdMiyjsBw8eLBz3Z0Wllkdh9qtWzcSEhLw8/Ojfv36zjaRkZFYrVbWrVvHE088cd3rpKSkkJKS4vz57NmzANhsNmw2W3YOO0uOHj1Keno6xYsXz7T/EiVKsHPnziy95htvvEHp0qUJDw/HZrNx6NAhAAICAjI9v2TJkhw5cgSbzYbD4aB3797069eP8PBwDhw4AOTecUrOsFgs1K1bl7i4ONq3bw/g/DKqf//+N/3bZRTcixcvJiwsjLi4OGfbkJAQgoKCWL58ObVq1QKMfr9u3Tr69euHzWYjLCwMLy8vduzYQUREBGD0lQMHDhASEqI+k49l/G31N5asUH+RrFJfkexQf5Gsyit9Jav5slV032oc6q5du7K0j4EDBxIcHOwssrMyDjUxMZFSpUplDu7uTkBAwE3Hv44ZM4Z33nnnuvXLly/PlbPBGeNhf/rpp0xjY/fv309ycjJLliy55fPnzZvHggULePfdd/H09CQuLs75O125cqXzSwowCnyLxcKSJUtYtGgRBw8e5Pnnn2fJkiXOs5w//vijy1wFIDf28MMP88EHH+Du7k6VKlX47rvvSE5OJiQkhCVLljBp0iSKFy/OM888A8D8+fOZPXs20dHR7N27l7179wLw7bffUqhQIQBatWrFyJEjOXPmDKVKlWL27Nn4+fnh5eXl7IMtW7ZkyJAhHDlyhFKlSrFgwQJSUlIoWrTobfup5H1xcXFmR5A8RP1Fskp9RbJD/UWyytX7ysWLF7PULltF992KiYlhzpw5xMfHZ2sc6p0YPHgw0dHRzp/Pnj3rHE9etGjRHH+91NRU+vXrR6VKlWjXrp1z/dy5c6lWrVqmdX81ceJEvv32W+Li4pxnL1u2bEn16tUZNGgQ9913H7Vr13a2nzBhAuHh4bRr145PPvmE3bt3X3fLpzfeeIPu3bvf9LJ/MV+7du0ICQlh4sSJJCYmEh4ezrJly2jYsCFg9AsfHx9n33n11VdJS0tj7NixmfYzZMgQRowYAUDbtm0JCQnhk08+ITk5mSZNmrBy5UqqVq3qbN+yZUvefvttpk6dyqVLl2jYsCHx8fHOs+OSP9lsNud7i4eHh9lxxMWpv0hWqa9Idqi/SFbllb6ScTX17WSr6M6JcagrVqwgLCzMuf7acagZ400zfs4oNIOCgq6bqC0tLY1Tp07d9HW9vLzw8vK6br2Hh0eu/OE8PDyoV68ea9asoXPnzoBxufDq1auJioq66WuOHTuW0aNHs2zZMho1auS8RMHDw4OqVasSFBTE999/T4MGDQDjD7t+/XpefPFFPDw8+Oijjxg9erRzf0eOHKF169Z8+eWXREREuHQnFaOQfvXVV2+4bc2aNZl+zhg6kMFms7FkyRLatWuX6e/83nvv8d577930NT08PJg4cSITJ0688+CSZ+XWe6DkT+ovklXqK5Id6i+SVa7eV7KaLVsTqXl6elKvXj3nJGhwdRxq48aNb/q8sWPHMmrUKGJjYzONywaoUKECQUFBmfaZMQ41Y5+NGzcmOTmZjRs3OtusWrUKu93uHJfqCqKjo5k+fTqzZs1i586d9O/fnwsXLtC3b18AevXqlWk8/Pvvv8/QoUOZOXMmoaGhJCYmkpiYyKVLlwBj3O8//vEP3n33Xb799lu2bt1Kr169CA4OpmPHjgCUK1eO++67z/nIOKNZqVIlQkJC7u0vQERERERERDLJ9uXl0dHR9O7dm/r169OwYUMmTZp0XWFZpkwZxowZAxiF5bBhw5g9e7azsATw9fXF19c3U2FZpUoVKlSowNChQzMVljVq1KBNmzb069ePadOmYbPZiIqKolu3bi51X+quXbty/Phxhg0bRmJiIrVr1yY2NtY5Xv3gwYNYrVe/55g6dSqpqanOM+PX7qdTp06Ace/tCxcu8Pzzz5OcnEzTpk2JjY3N9cvzRURERERE5O5lu+jOqcJy+PDhznGoWSksv/jiC6KiomjRogVWq5VOnToxefLkOznmXBUVFUVUVNQNt8XHx2f6+a+XC8PVS4YzWCwWRo4cyciRI7P0+qGhoTgcjiznFRERERERkdxzRxOp3W1h+VdZKSwDAgKYPXt2dmKKiIiIiIiImCpbY7pFREREREREJOtUdIuIiIiIiIjkknt6n24zZYxzzuq91Mxis9m4ePEiZ8+edenp8cV86iuSHeovkh3qL5JV6iuSHeovklV5pa9k1Ja3m1OrwBTd586dA6Bs2bImJxEREREREZH84ty5cxQrVuym2y2OAjLVtd1u58iRIxQpUgSLxWJ2nJs6e/YsZcuW5c8//6Ro0aJmxxEXpr4i2aH+Itmh/iJZpb4i2aH+IlmVV/qKw+Hg3LlzBAcHZ7qD118VmDPdVquVkJAQs2NkWdGiRV26g4nrUF+R7FB/kexQf5GsUl+R7FB/kazKC33lVme4M2giNREREREREZFcoqJbREREREREJJeo6HYxXl5eDB8+HC8vL7OjiItTX5HsUH+R7FB/kaxSX5HsUH+RrMpvfaXATKQmIiIiIiIicq/pTLeIiIiIiIhILlHRLSIiIiIiIpJLVHSLiIiIiIiI5BIV3S5kypQphIaG4u3tTUREBOvXrzc7krio77//nscee4zg4GAsFgsLFy40O5K4qDFjxtCgQQOKFClCqVKl6NixI7t37zY7lrigqVOnEhYW5rwnauPGjVm6dKnZsSQPiImJwWKx8I9//MPsKOKCRowYgcViyfSoXr262bHEhR0+fJiePXtSvHhxChUqxP33388vv/xidqy7oqLbRXz55ZdER0czfPhwNm3aRHh4OK1bt+bYsWNmRxMXdOHCBcLDw5kyZYrZUcTFrVmzhpdeeomff/6ZuLg4bDYbrVq14sKFC2ZHExcTEhJCTEwMGzdu5JdffuGRRx6hQ4cObN++3exo4sI2bNjAv//9b8LCwsyOIi6sVq1aHD161Pn48ccfzY4kLur06dM0adIEDw8Pli5dyo4dO5gwYQL+/v5mR7srmr3cRURERNCgQQM++ugjAOx2O2XLluXll19m0KBBJqcTV2axWFiwYAEdO3Y0O4rkAcePH6dUqVKsWbOGhx56yOw44uICAgIYN24czz77rNlRxAWdP3+eunXr8q9//Yt3332X2rVrM2nSJLNjiYsZMWIECxcuZPPmzWZHkTxg0KBBrF27lh9++MHsKDlKZ7pdQGpqKhs3biQyMtK5zmq1EhkZSUJCgonJRCS/OXPmDGAUUyI3k56ezpw5c7hw4QKNGzc2O464qJdeeon27dtn+veLyI3s2bOH4OBgKlasSI8ePTh48KDZkcRFffvtt9SvX5+nnnqKUqVKUadOHaZPn252rLumotsFnDhxgvT0dAIDAzOtDwwMJDEx0aRUIpLf2O12/vGPf9CkSRPuu+8+s+OIC9q6dSu+vr54eXnxwgsvsGDBAmrWrGl2LHFBc+bMYdOmTYwZM8bsKOLiIiIi+Oyzz4iNjWXq1Kn8/vvvPPjgg5w7d87saOKC9u/fz9SpU6lSpQrLli2jf//+vPLKK8yaNcvsaHfF3ewAIiJyb7z00kts27ZNY+nkpqpVq8bmzZs5c+YMc+fOpXfv3qxZs0aFt2Ty559/8uqrrxIXF4e3t7fZccTFtW3b1rkcFhZGREQE5cuX56uvvtLQFbmO3W6nfv36jB49GoA6deqwbds2pk2bRu/evU1Od+d0ptsFlChRAjc3N5KSkjKtT0pKIigoyKRUIpKfREVFsWjRIlavXk1ISIjZccRFeXp6UrlyZerVq8eYMWMIDw/ngw8+MDuWuJiNGzdy7Ngx6tati7u7O+7u7qxZs4bJkyfj7u5Oenq62RHFhfn5+VG1alX27t1rdhRxQaVLl77ui94aNWrk+SEJKrpdgKenJ/Xq1WPlypXOdXa7nZUrV2osnYjcFYfDQVRUFAsWLGDVqlVUqFDB7EiSh9jtdlJSUsyOIS6mRYsWbN26lc2bNzsf9evXp0ePHmzevBk3NzezI4oLO3/+PPv27aN06dJmRxEX1KRJk+tubfrbb79Rvnx5kxLlDF1e7iKio6Pp3bs39evXp2HDhkyaNIkLFy7Qt29fs6OJCzp//nymb4h///13Nm/eTEBAAOXKlTMxmbial156idmzZ/PNN99QpEgR5zwRxYoVo1ChQianE1cyePBg2rZtS7ly5Th37hyzZ88mPj6eZcuWmR1NXEyRIkWumxeicOHCFC9eXPNFyHVef/11HnvsMcqXL8+RI0cYPnw4bm5udO/e3exo4oIGDBjAAw88wOjRo+nSpQvr16/n448/5uOPPzY72l1R0e0iunbtyvHjxxk2bBiJiYnUrl2b2NjY6yZXEwH45ZdfaN68ufPn6OhoAHr37s1nn31mUipxRVOnTgWgWbNmmdZ/+umn9OnT594HEpd17NgxevXqxdGjRylWrBhhYWEsW7aMli1bmh1NRPKwQ4cO0b17d06ePEnJkiVp2rQpP//8MyVLljQ7mrigBg0asGDBAgYPHszIkSOpUKECkyZNokePHmZHuyu6T7eIiIiIiIhILtGYbhEREREREZFcoqJbREREREREJJeo6BYRERERERHJJSq6RURERERERHKJim4RERERERGRXKKiW0RERERERCSXqOgWERERERERySUqukVERERERERyiYpuERERuWvx8fFYLBaSk5PNjiIiIuJSVHSLiIiIiIiI5BIV3SIiIiIiIiK5REW3iIhIPmC32xkzZgwVKlSgUKFChIeHM3fuXODqpd+LFy8mLCwMb29vGjVqxLZt2zLtY968edSqVQsvLy9CQ0OZMGFCpu0pKSkMHDiQsmXL4uXlReXKlfnkk08ytdm4cSP169fHx8eHBx54gN27d+fugYuIiLg4Fd0iIiL5wJgxY/j888+ZNm0a27dvZ8CAAfTs2ZM1a9Y427zxxhtMmDCBDRs2ULJkSR577DFsNhtgFMtdunShW7dubN26lREjRjB06FA+++wz5/N79erFf//7XyZPnszOnTv597//ja+vb6Ycb731FhMmTOCXX37B3d2dv/3tb/fk+EVERFyVxeFwOMwOISIiIncuJSWFgIAAVqxYQePGjZ3rn3vuOS5evMjzzz9P8+bNmTNnDl27dgXg1KlThISE8Nlnn9GlSxd69OjB8ePHWb58ufP5b775JosXL2b79u389ttvVKtWjbi4OCIjI6/LEB8fT/PmzVmxYgUtWrQAYMmSJbRv355Lly7h7e2dy78FERER16Qz3SIiInnc3r17uXjxIi1btsTX19f5+Pzzz9m3b5+z3bUFeUBAANWqVWPnzp0A7Ny5kyZNmmTab5MmTdizZw/p6els3rwZNzc3Hn744VtmCQsLcy6XLl0agGPHjt31MYqIiORV7mYHEBERkbtz/vx5ABYvXkyZMmUybfPy8spUeN+pQoUKZamdh4eHc9lisQDGeHMREZGCSme6RURE8riaNWvi5eXFwYMHqVy5cqZH2bJlne1+/vln5/Lp06f57bffqFGjBgA1atRg7dq1mfa7du1aqlatipubG/fffz92uz3TGHERERG5PZ3pFhERyeOKFCnC66+/zoABA7Db7TRt2pQzZ86wdu1aihYtSvny5QEYOXIkxYsXJzAwkLfeeosSJUrQsWNHAF577TUaNGjAqFGj6Nq1KwkJCXz00Uf861//AiA0NJTevXvzt7/9jcmTJxMeHs4ff/zBsWPH6NKli1mHLiIi4vJUdIuIiOQDo0aNomTJkowZM4b9+/fj5+dH3bp1GTJkiPPy7piYGF599VX27NlD7dq1+e677/D09ASgbt26fPXVVwwbNoxRo0ZRunRpRo4cSZ8+fZyvMXXqVIYMGcKLL77IyZMnKVeuHEOGDDHjcEVERPIMzV4uIiKSz2XMLH769Gn8/PzMjiMiIlKgaEy3iIiIiIiISC5R0S0iIiIiIiKSS3R5uYiIiIiIiEgu0ZluERERERERkVyioltEREREREQkl6joFhEREREREcklKrpFREREREREcomKbhEREREREZFcoqJbREREREREJJeo6BYRERERERHJJSq6RURERERERHKJim4RERERERGRXPL/nZ7itBGyAcsAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "from numpy import argmax\n", + "\n", + "# Assuming perf_df.set_index(\"epoch\").plot() generates a line plot\n", + "ax = perf_df.set_index(\"epoch\").plot(\n", + " figsize=(10, 5), title=\"METEOR score vs epoch\", grid=True\n", + ")\n", + "\n", + "# Loop through each line to annotate the last point\n", + "for line_index, line in enumerate(ax.lines):\n", + " # Get the data\n", + " xdata, ydata = line.get_data()\n", + " for index in range(xdata.size):\n", + " ax.annotate(\n", + " f\"{ydata[index]:.3f}\",\n", + " xy=(xdata[index], ydata[index]),\n", + " textcoords=\"offset points\",\n", + " xytext=(0, 1 if line_index % 2 == 0 else -10),\n", + " ha=\"center\",\n", + " )\n", + "\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": 2652223487066960, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 4 + }, + "notebookName": "00_Data_Analysis", + "widgets": {} + }, + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "036fc5746f43416db18c19ad8fd36677": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "06e806c82c7b4cbea31c5358dd9c3434": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "087b76a8b7514269b1f0ab29b062e444": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a069d2ab23824f29aa320ac256e2cfe9", + "placeholder": "​", + "style": "IPY_MODEL_06e806c82c7b4cbea31c5358dd9c3434", + "value": "Map (num_proc=2): 100%" + } + }, + "09b76013aa9e45efb6deb23a7a0d0925": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dea41c5260884aa6879b5e1d1697b14f", + "placeholder": "​", + "style": "IPY_MODEL_89965917796a4f81b899fdc7685f33df", + "value": "config.json: 100%" + } + }, + "0a92c56bfa134ef583220d7ef0b13e17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c34be936c8145d3ab41282f30a70713": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f8b6bfe16894500838793f2491d403f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "177c78fce95d4b4ab33057c5a048d693": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1f44c9ce1adf470cbb19784493ed209f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c34be936c8145d3ab41282f30a70713", + "placeholder": "​", + "style": "IPY_MODEL_0a92c56bfa134ef583220d7ef0b13e17", + "value": "model.safetensors: 100%" + } + }, + "201b59ccd9f845e197029b57e424aefc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2157f01726d748f8a9ae4a00664430da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21db8a77b00d4a4e82fdfa608657531f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26e4202cca81496a90d15a0dd4ca9cf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba90fdb8822d47dab7ba203bee297f37", + "IPY_MODEL_61560ff6a36b44f4a9dfdae5c52791d4", + "IPY_MODEL_95fbe66647904c06a20f640630d6dc0e" + ], + "layout": "IPY_MODEL_57182a263d324a3dbf1471c74290a0d5" + } + }, + "27155728b6b84cb199c91c940095d0a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6b91feeed5464877991ac2c207aebe7c", + "IPY_MODEL_cca8113c54c0495daedce1327bf9c68b", + "IPY_MODEL_2e63a29e2f7247bba5beede9a568c99f" + ], + "layout": "IPY_MODEL_5c9d781c28944f3eb86e2a6d44efdf18" + } + }, + "271ddaa553a042d09b6db7b450643d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a58d04b428c46f4b3dbadd3bc6cd529": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d18ddf6482c4d97829ac0e5a7b9868f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "IPY_MODEL_f2df530d22c74977b249dd9fb5f4829b", + "IPY_MODEL_89b2ef0dbfea47ab8e6f8d659e3351d1" + ], + "layout": "IPY_MODEL_3056b148aa9f4e6e8aa3b61d26886255" + } + }, + "2e5087c76f98437cb5dc729230358cba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e63a29e2f7247bba5beede9a568c99f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b993eaec6b224440bf80c0958c6fb536", + "placeholder": "​", + "style": "IPY_MODEL_de868e26e7154f62aa86223a539ad421", + "value": " 464/464 [00:00<00:00, 27.1kB/s]" + } + }, + "2f6c70dd266c4816bfad3fd3d192929a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "30307300bc4e4baf96560e30969a82b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e36a3f9eff0e4cf68834d66b0213ae96", + "placeholder": "​", + "style": "IPY_MODEL_a0037bdccf254159becde630bee3d1db", + "value": "generation_config.json: 100%" + } + }, + "3056b148aa9f4e6e8aa3b61d26886255": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30cdc32298134cb0be4d41615b9e5774": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3572201bd4d74a58b7a665f9bdfdcdba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35b0e8c26d6640e9bd0ed7b242a423d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e5087c76f98437cb5dc729230358cba", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_036fc5746f43416db18c19ad8fd36677", + "value": 51760 + } + }, + "36166c7bcb854b34aca1f41a5d6ea50b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "370692d819df41828b48c4ad446f977b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "39b29a75374b45c0a22506010be2b84e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_30cdc32298134cb0be4d41615b9e5774", + "max": 1179, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_47928317548c454bba6358ab132e8dee", + "value": 1179 + } + }, + "3cf2dd993b5e4d3daecf61e4bab5a404": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_087b76a8b7514269b1f0ab29b062e444", + "IPY_MODEL_35b0e8c26d6640e9bd0ed7b242a423d8", + "IPY_MODEL_54ad89e05fd74576b9b8b5b5a10eaf8d" + ], + "layout": "IPY_MODEL_a41dc44766444a998bec2d777f249d23" + } + }, + "43dec2ede91341f5af60eb522e18e984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4463edd481c1467f914c7dcd6c6e6ffc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47928317548c454bba6358ab132e8dee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "49277aeeac16434a865a4d12308b1abc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4ae7e449e4ea4c729b5f34607c18ebae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b2061b8a73c43ffb0c2f83daf0d0183": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c4c88d4c701450692fa0f6b0c5764b0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c666f4ace3943f8b80ecd20e7503236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ccedf0d93094e63b57a0f8a434fba06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4463edd481c1467f914c7dcd6c6e6ffc", + "max": 44307561, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6d3b9a05db0b4dadb638c686faa0c40a", + "value": 44307561 + } + }, + "4dcf6ff672d24983a1877a8431709aa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5807d5fb827d490fb3bc698f801ffff5", + "placeholder": "​", + "style": "IPY_MODEL_c4f2b06a82fd4987b8b659524a7b503b", + "value": "Generating train split: 100%" + } + }, + "4ea63adfce694725bdba878aef709dd3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5234566b1bfc4655b8d582ea5b46ed9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "54ad89e05fd74576b9b8b5b5a10eaf8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdb1941405ed4e4aa06019933892deb3", + "placeholder": "​", + "style": "IPY_MODEL_668d5377ca56426a99753867e6e24862", + "value": " 51760/51760 [01:02<00:00, 1131.51 examples/s]" + } + }, + "56aee4853b7740e6a977254f5d1fa66d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "57182a263d324a3dbf1471c74290a0d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5807d5fb827d490fb3bc698f801ffff5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c9d781c28944f3eb86e2a6d44efdf18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f40db8173dd4d76b6ef5ed6d9ec8b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61560ff6a36b44f4a9dfdae5c52791d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db19fc8d37db4e45a5790a876836d8c4", + "max": 11610, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36166c7bcb854b34aca1f41a5d6ea50b", + "value": 11610 + } + }, + "6578fd7acdb54c4c93528ea431fd0144": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_370692d819df41828b48c4ad446f977b", + "placeholder": "​", + "style": "IPY_MODEL_a0bf9160eb2647409b3200270914b90f", + "value": " 50.6k/50.6k [00:00<00:00, 2.71MB/s]" + } + }, + "668d5377ca56426a99753867e6e24862": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697f027529b54ee9956bae78a11e0611": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69ac12aec0714318bf2c83d4f4e745f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b2012c3f88547af8884a9ea90e3164b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_938f45f1b3e24118b815d96ae34ba86a", + "placeholder": "​", + "style": "IPY_MODEL_9367047a800747f79c6b225d92397846", + "value": " 44.3M/44.3M [00:01<00:00, 31.0MB/s]" + } + }, + "6b91feeed5464877991ac2c207aebe7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b2061b8a73c43ffb0c2f83daf0d0183", + "placeholder": "​", + "style": "IPY_MODEL_69ac12aec0714318bf2c83d4f4e745f5", + "value": "special_tokens_map.json: 100%" + } + }, + "6d3b9a05db0b4dadb638c686faa0c40a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6dbbedeca9314e66ae50e44ffa31a414": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6e34619b45934040b6092e6fb01ea7fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71ce208e20d6483abb9ed923510c86d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d69dc491b3ab44d7852b21873ed7bb7f", + "placeholder": "​", + "style": "IPY_MODEL_f401d53bf28e44eb906bce6c05412662", + "value": " 51760/51760 [00:01<00:00, 45512.81 examples/s]" + } + }, + "7358cdad832342c983e31efb8754ab78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e352a3404f4c7dad0737f57d29e92f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_988a0e8c1f89446086858da0a891a79c", + "IPY_MODEL_4ccedf0d93094e63b57a0f8a434fba06", + "IPY_MODEL_6b2012c3f88547af8884a9ea90e3164b" + ], + "layout": "IPY_MODEL_7e29cb8dd4df4d5b94407cd8fd3f2011" + } + }, + "74501720ac7e4dbb911a4a99b3633bc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "78e5400bff924a92a4cc61c4ff18b182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9b313fd861948f5aba25b24b1518d30", + "placeholder": "​", + "style": "IPY_MODEL_4c666f4ace3943f8b80ecd20e7503236", + "value": " 1.18k/1.18k [00:00<00:00, 31.3kB/s]" + } + }, + "7975adbc2ec5489ea7fa0167e620d85c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e34619b45934040b6092e6fb01ea7fe", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_271ddaa553a042d09b6db7b450643d8f", + "value": 51760 + } + }, + "7e29cb8dd4df4d5b94407cd8fd3f2011": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "810ff6c0e17d4fa09a30fef27eacff90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89965917796a4f81b899fdc7685f33df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "89b2ef0dbfea47ab8e6f8d659e3351d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8908fa0df3743ecb9d12983a739104f", + "placeholder": "​", + "style": "IPY_MODEL_177c78fce95d4b4ab33057c5a048d693", + "value": " 9.09M/9.09M [00:00<00:00, 32.6MB/s]" + } + }, + "8b3505352a5a42bf910428c40ce40465": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_49277aeeac16434a865a4d12308b1abc", + "placeholder": "​", + "style": "IPY_MODEL_2157f01726d748f8a9ae4a00664430da", + "value": " 5.70G/5.70G [01:02<00:00, 30.1MB/s]" + } + }, + "8fc142b628fb40568730234de1cafde2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ae7e449e4ea4c729b5f34607c18ebae", + "max": 172, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3572201bd4d74a58b7a665f9bdfdcdba", + "value": 172 + } + }, + "9367047a800747f79c6b225d92397846": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "938f45f1b3e24118b815d96ae34ba86a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "95fbe66647904c06a20f640630d6dc0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b0a370dc20654b279b9680692e34418e", + "placeholder": "​", + "style": "IPY_MODEL_cfeb365ddf7548d58b2557f22737fcf5", + "value": " 11.6k/11.6k [00:00<00:00, 716kB/s]" + } + }, + "988a0e8c1f89446086858da0a891a79c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ad2be500fc164c0f86f33e914ef8e6a0", + "placeholder": "​", + "style": "IPY_MODEL_5234566b1bfc4655b8d582ea5b46ed9f", + "value": "Downloading data: 100%" + } + }, + "98c58f23f4d549518832cb2d18f796e8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_09b76013aa9e45efb6deb23a7a0d0925", + "IPY_MODEL_39b29a75374b45c0a22506010be2b84e", + "IPY_MODEL_78e5400bff924a92a4cc61c4ff18b182" + ], + "layout": "IPY_MODEL_2a58d04b428c46f4b3dbadd3bc6cd529" + } + }, + "99fdbb0300c14c139d1937c646f0cfe7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7358cdad832342c983e31efb8754ab78", + "placeholder": "​", + "style": "IPY_MODEL_e9adf418296e436fb48bb9f78885598b", + "value": " 51760/51760 [00:01<00:00, 38665.95 examples/s]" + } + }, + "9f679ad3ec7f4fe8ad0510ffb57bc2ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ea63adfce694725bdba878aef709dd3", + "placeholder": "​", + "style": "IPY_MODEL_74501720ac7e4dbb911a4a99b3633bc6", + "value": "tokenizer.json: 100%" + } + }, + "a0037bdccf254159becde630bee3d1db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a069d2ab23824f29aa320ac256e2cfe9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0bf9160eb2647409b3200270914b90f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a41dc44766444a998bec2d777f249d23": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8464a4c711e4e00aafdfc919b60d07e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fb995c740590427b882572c81d4e848c", + "placeholder": "​", + "style": "IPY_MODEL_201b59ccd9f845e197029b57e424aefc", + "value": " 172/172 [00:00<00:00, 12.0kB/s]" + } + }, + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad2be500fc164c0f86f33e914ef8e6a0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b0240cd9a4554b29ae11f8051984a1c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_edaf890370314a218f138015faa0b05d", + "placeholder": "​", + "style": "IPY_MODEL_697f027529b54ee9956bae78a11e0611", + "value": "Map: 100%" + } + }, + "b0a370dc20654b279b9680692e34418e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b518dcee69074b87be73957cd810e7ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d891f8d0b1fc462f8008d02bb2a15692", + "placeholder": "​", + "style": "IPY_MODEL_cced8fd7e998472794f3f3e3018956a5", + "value": "tokenizer_config.json: 100%" + } + }, + "b8908fa0df3743ecb9d12983a739104f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b993eaec6b224440bf80c0958c6fb536": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9b313fd861948f5aba25b24b1518d30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba90fdb8822d47dab7ba203bee297f37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f8b6bfe16894500838793f2491d403f", + "placeholder": "​", + "style": "IPY_MODEL_bb19f6c747754682a514373a3a0535ba", + "value": "Downloading readme: 100%" + } + }, + "bb19f6c747754682a514373a3a0535ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc883d4cf13e4f8b8a4fe5f410cb6efd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9159e03e61f4f56978ece9c3bca49b2", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_810ff6c0e17d4fa09a30fef27eacff90", + "value": 51760 + } + }, + "c161d94df0f04feba9542237e0856c22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c22f71b1f85843209d7e5321506b9cb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f44c9ce1adf470cbb19784493ed209f", + "IPY_MODEL_f1addc4479d849879e743cf9089e6540", + "IPY_MODEL_8b3505352a5a42bf910428c40ce40465" + ], + "layout": "IPY_MODEL_4c4c88d4c701450692fa0f6b0c5764b0" + } + }, + "c4f2b06a82fd4987b8b659524a7b503b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cca8113c54c0495daedce1327bf9c68b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e02f9b7849c64531835eb77b860d1c93", + "max": 464, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_56aee4853b7740e6a977254f5d1fa66d", + "value": 464 + } + }, + "cced8fd7e998472794f3f3e3018956a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cf245afeb1c04f29a24d291608c3d157": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b518dcee69074b87be73957cd810e7ed", + "IPY_MODEL_e29104486d594b2992d7285e0ef77371", + "IPY_MODEL_6578fd7acdb54c4c93528ea431fd0144" + ], + "layout": "IPY_MODEL_d35db8148a354c56aaac56dbae22536f" + } + }, + "cfe8cae0e22b495bafa221a63d13b283": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfeb365ddf7548d58b2557f22737fcf5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1b47d39450d4019ae85c9b2f943eeaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4dcf6ff672d24983a1877a8431709aa9", + "IPY_MODEL_7975adbc2ec5489ea7fa0167e620d85c", + "IPY_MODEL_71ce208e20d6483abb9ed923510c86d7" + ], + "layout": "IPY_MODEL_cfe8cae0e22b495bafa221a63d13b283" + } + }, + "d35db8148a354c56aaac56dbae22536f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69dc491b3ab44d7852b21873ed7bb7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d891f8d0b1fc462f8008d02bb2a15692": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8e5318cead340c4adbeaccc05d39225": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "daf4cd890b35422683d22fd30bc71e83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b0240cd9a4554b29ae11f8051984a1c6", + "IPY_MODEL_bc883d4cf13e4f8b8a4fe5f410cb6efd", + "IPY_MODEL_99fdbb0300c14c139d1937c646f0cfe7" + ], + "layout": "IPY_MODEL_c161d94df0f04feba9542237e0856c22" + } + }, + "db19fc8d37db4e45a5790a876836d8c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de868e26e7154f62aa86223a539ad421": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dea41c5260884aa6879b5e1d1697b14f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e02f9b7849c64531835eb77b860d1c93": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e29104486d594b2992d7285e0ef77371": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "max": 50641, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2f6c70dd266c4816bfad3fd3d192929a", + "value": 50641 + } + }, + "e36a3f9eff0e4cf68834d66b0213ae96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9159e03e61f4f56978ece9c3bca49b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9adf418296e436fb48bb9f78885598b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "edaf890370314a218f138015faa0b05d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f1addc4479d849879e743cf9089e6540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43dec2ede91341f5af60eb522e18e984", + "max": 5702746405, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d8e5318cead340c4adbeaccc05d39225", + "value": 5702746405 + } + }, + "f2df530d22c74977b249dd9fb5f4829b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_21db8a77b00d4a4e82fdfa608657531f", + "max": 9085698, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6dbbedeca9314e66ae50e44ffa31a414", + "value": 9085698 + } + }, + "f401d53bf28e44eb906bce6c05412662": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fb995c740590427b882572c81d4e848c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fce7a61c25ec4390af43d92b7c473a45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_30307300bc4e4baf96560e30969a82b6", + "IPY_MODEL_8fc142b628fb40568730234de1cafde2", + "IPY_MODEL_a8464a4c711e4e00aafdfc919b60d07e" + ], + "layout": "IPY_MODEL_5f40db8173dd4d76b6ef5ed6d9ec8b6e" + } + }, + "fdb1941405ed4e4aa06019933892deb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/01_Qwen2-0.5B_Unsloth.ipynb b/notebooks/01_Qwen2-0.5B_Unsloth.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..7e9bcb68563636d461948106258c4814520fb330 --- /dev/null +++ b/notebooks/01_Qwen2-0.5B_Unsloth.ipynb @@ -0,0 +1,5072 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "8fe0634c-c88a-4c89-b956-1f4247c2d503", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "e2fc659d-8a38-4a94-bf3c-81b2778c780a", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /home/inflaton/code/projects/courses/novel-translation\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6601e3ff-e856-4353-98d8-42fcb158f230", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f30d283f-4759-403b-8cc4-94e360a76c04", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('unsloth/Qwen2-0.5B-Instruct',\n", + " True,\n", + " 'models/Qwen2-0.5B-Instruct-MAC-',\n", + " 'Qwen2-0.5B-Instruct-MAC-',\n", + " 2048,\n", + " 10,\n", + " None,\n", + " 'datasets/mac/mac.tsv')" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "model_name = os.getenv(\"MODEL_NAME\")\n", + "token = os.getenv(\"HF_TOKEN\") or None\n", + "load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n", + "local_model = os.getenv(\"LOCAL_MODEL\")\n", + "hub_model = os.getenv(\"HUB_MODEL\")\n", + "num_train_epochs = int(os.getenv(\"NUM_TRAIN_EPOCHS\") or 0)\n", + "data_path = os.getenv(\"DATA_PATH\")\n", + "\n", + "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n", + "dtype = (\n", + " None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n", + ")\n", + "\n", + "model_name, load_in_4bit, local_model, hub_model, max_seq_length, num_train_epochs, dtype, data_path" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6f80b3a1-a6e6-43c7-b54f-da10ef37df32", + "showTitle": false, + "title": "" + }, + "id": "r2v_X2fA0Df5" + }, + "source": [ + "* We support Llama, Mistral, Phi-3, Gemma, Yi, DeepSeek, Qwen, TinyLlama, Vicuna, Open Hermes etc\n", + "* We support 16bit LoRA or 4bit QLoRA. Both 2x faster.\n", + "* `max_seq_length` can be set to anything, since we do automatic RoPE Scaling via [kaiokendev's](https://kaiokendev.github.io/til) method.\n", + "* With [PR 26037](https://github.com/huggingface/transformers/pull/26037), we support downloading 4bit models **4x faster**! [Our repo](https://huggingface.co/unsloth) has Llama, Mistral 4bit models.\n", + "* [**NEW**] We make Phi-3 Medium / Mini **2x faster**! See our [Phi-3 Medium notebook](https://colab.research.google.com/drive/1hhdhBa1j_hsymiW9m-WzxQtgqTH_NHqi?usp=sharing)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9df0e65d-07a4-4d5e-8848-c41872280e6f", + "showTitle": false, + "title": "" + }, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 353, + "referenced_widgets": [ + "98c58f23f4d549518832cb2d18f796e8", + "09b76013aa9e45efb6deb23a7a0d0925", + "39b29a75374b45c0a22506010be2b84e", + "78e5400bff924a92a4cc61c4ff18b182", + "2a58d04b428c46f4b3dbadd3bc6cd529", + "dea41c5260884aa6879b5e1d1697b14f", + "89965917796a4f81b899fdc7685f33df", + "30cdc32298134cb0be4d41615b9e5774", + "47928317548c454bba6358ab132e8dee", + "b9b313fd861948f5aba25b24b1518d30", + "4c666f4ace3943f8b80ecd20e7503236", + "c22f71b1f85843209d7e5321506b9cb9", + "1f44c9ce1adf470cbb19784493ed209f", + "f1addc4479d849879e743cf9089e6540", + "8b3505352a5a42bf910428c40ce40465", + "4c4c88d4c701450692fa0f6b0c5764b0", + "0c34be936c8145d3ab41282f30a70713", + "0a92c56bfa134ef583220d7ef0b13e17", + "43dec2ede91341f5af60eb522e18e984", + "d8e5318cead340c4adbeaccc05d39225", + "49277aeeac16434a865a4d12308b1abc", + "2157f01726d748f8a9ae4a00664430da", + "fce7a61c25ec4390af43d92b7c473a45", + "30307300bc4e4baf96560e30969a82b6", + "8fc142b628fb40568730234de1cafde2", + "a8464a4c711e4e00aafdfc919b60d07e", + "5f40db8173dd4d76b6ef5ed6d9ec8b6e", + "e36a3f9eff0e4cf68834d66b0213ae96", + "a0037bdccf254159becde630bee3d1db", + "4ae7e449e4ea4c729b5f34607c18ebae", + "3572201bd4d74a58b7a665f9bdfdcdba", + "fb995c740590427b882572c81d4e848c", + "201b59ccd9f845e197029b57e424aefc", + "cf245afeb1c04f29a24d291608c3d157", + "b518dcee69074b87be73957cd810e7ed", + "e29104486d594b2992d7285e0ef77371", + "6578fd7acdb54c4c93528ea431fd0144", + "d35db8148a354c56aaac56dbae22536f", + "d891f8d0b1fc462f8008d02bb2a15692", + "cced8fd7e998472794f3f3e3018956a5", + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "2f6c70dd266c4816bfad3fd3d192929a", + "370692d819df41828b48c4ad446f977b", + "a0bf9160eb2647409b3200270914b90f", + "2d18ddf6482c4d97829ac0e5a7b9868f", + "9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "f2df530d22c74977b249dd9fb5f4829b", + "89b2ef0dbfea47ab8e6f8d659e3351d1", + "3056b148aa9f4e6e8aa3b61d26886255", + "4ea63adfce694725bdba878aef709dd3", + "74501720ac7e4dbb911a4a99b3633bc6", + "21db8a77b00d4a4e82fdfa608657531f", + "6dbbedeca9314e66ae50e44ffa31a414", + "b8908fa0df3743ecb9d12983a739104f", + "177c78fce95d4b4ab33057c5a048d693", + "27155728b6b84cb199c91c940095d0a8", + "6b91feeed5464877991ac2c207aebe7c", + "cca8113c54c0495daedce1327bf9c68b", + "2e63a29e2f7247bba5beede9a568c99f", + "5c9d781c28944f3eb86e2a6d44efdf18", + "4b2061b8a73c43ffb0c2f83daf0d0183", + "69ac12aec0714318bf2c83d4f4e745f5", + "e02f9b7849c64531835eb77b860d1c93", + "56aee4853b7740e6a977254f5d1fa66d", + "b993eaec6b224440bf80c0958c6fb536", + "de868e26e7154f62aa86223a539ad421" + ] + }, + "id": "QmUBVEnvCDJv", + "outputId": "a0e2d781-4934-415a-90b4-35165b9e44c5" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.5\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.2.2+cu121. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.25.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 10.6 s, sys: 2.07 s, total: 12.6 s\n", + "Wall time: 51.9 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "from llm_toolkit.translation_engine import *\n", + "\n", + "model, tokenizer = load_model(model_name, load_in_4bit)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "adfadd0f-8c01-4f67-b643-cff930c1ce00", + "showTitle": false, + "title": "" + }, + "id": "SXd9bTZd1aaL" + }, + "source": [ + "We now add LoRA adapters so we only need to update 1 to 10% of all parameters!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "2cd85242-237f-4cca-a706-b7664ec9d3e5", + "showTitle": false, + "title": "" + }, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6bZsfBuZDeCL", + "outputId": "bc6d9ce7-f82a-4191-d0c5-ec8247d9b9eb" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Unsloth 2024.5 patched 24 layers with 0 QKV layers, 24 O layers and 24 MLP layers.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 9.31 s, sys: 0 ns, total: 9.31 s\n", + "Wall time: 2.12 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "model = FastLanguageModel.get_peft_model(\n", + " model,\n", + " r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128\n", + " target_modules=[\n", + " \"q_proj\",\n", + " \"k_proj\",\n", + " \"v_proj\",\n", + " \"o_proj\",\n", + " \"gate_proj\",\n", + " \"up_proj\",\n", + " \"down_proj\",\n", + " ],\n", + " lora_alpha=16,\n", + " lora_dropout=0, # Supports any, but = 0 is optimized\n", + " bias=\"none\", # Supports any, but = \"none\" is optimized\n", + " # [NEW] \"unsloth\" uses 30% less VRAM, fits 2x larger batch sizes!\n", + " use_gradient_checkpointing=\"unsloth\", # True or \"unsloth\" for very long context\n", + " random_state=3407,\n", + " use_rslora=False, # We support rank stabilized LoRA\n", + " loftq_config=None, # And LoftQ\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "2c3fdf26-130f-4ce7-9c51-d62e1ce17629", + "showTitle": false, + "title": "" + }, + "id": "vITh0KVJ10qX" + }, + "source": [ + "\n", + "### Data Prep\n", + "We now use the Alpaca dataset from [yahma](https://huggingface.co/datasets/yahma/alpaca-cleaned), which is a filtered version of 52K of the original [Alpaca dataset](https://crfm.stanford.edu/2023/03/13/alpaca.html). You can replace this code section with your own data prep.\n", + "\n", + "**[NOTE]** To train only on completions (ignoring the user's input) read TRL's docs [here](https://huggingface.co/docs/trl/sft_trainer#train-on-completions-only).\n", + "\n", + "**[NOTE]** Remember to add the **EOS_TOKEN** to the tokenized output!! Otherwise you'll get infinite generations!\n", + "\n", + "If you want to use the `llama-3` template for ShareGPT datasets, try our conversational [notebook](https://colab.research.google.com/drive/1XamvWYinY6FOSX9GLvnqSjjsNflxdhNc?usp=sharing).\n", + "\n", + "For text completions like novel writing, try this [notebook](https://colab.research.google.com/drive/1ef-tab5bhkvWmBOObepl1WgJvfvSzn5Q?usp=sharing)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "a378fd31-1620-42bc-b97a-82f4ffbdcb11", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n" + ] + } + ], + "source": [ + "import os\n", + "from llm_toolkit.translation_engine import *\n", + "\n", + "datasets = load_translation_dataset(data_path, tokenizer)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6ac9ac82-aaf3-482b-8cdb-8eab19fde5ae", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "({'chinese': '全仗着狐仙搭救。',\n", + " 'english': 'Because I was protected by a fox fairy.',\n", + " 'text': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n全仗着狐仙搭救。<|im_end|>\\n<|im_start|>assistant\\nBecause I was protected by a fox fairy.<|im_end|>',\n", + " 'prompt': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n全仗着狐仙搭救。<|im_end|>\\n<|im_start|>assistant\\n'},\n", + " {'chinese': '老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。',\n", + " 'english': 'Old Geng picked up his shotgun, squinted, and pulled the trigger. Two sparrows crashed to the ground like hailstones as shotgun pellets tore noisily through the branches.',\n", + " 'text': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。<|im_end|>\\n<|im_start|>assistant\\nOld Geng picked up his shotgun, squinted, and pulled the trigger. Two sparrows crashed to the ground like hailstones as shotgun pellets tore noisily through the branches.<|im_end|>',\n", + " 'prompt': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。<|im_end|>\\n<|im_start|>assistant\\n'})" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "datasets[\"train\"][0], datasets[\"test\"][0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "87f73aab-12df-4e4e-b758-ee055e17ed58", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "({'chinese': '周瑞家的道：“太太说：‘他们原不是一家子； 当年他们的祖和太老爷在一处做官，因连了宗的。',\n", + " 'english': \"'She said they don't really belong to the family but were adopted into the clan years ago when your grandfather and theirs were working in the same office.\",\n", + " 'text': \"<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n周瑞家的道：“太太说：‘他们原不是一家子； 当年他们的祖和太老爷在一处做官，因连了宗的。<|im_end|>\\n<|im_start|>assistant\\n'She said they don't really belong to the family but were adopted into the clan years ago when your grandfather and theirs were working in the same office.<|im_end|>\",\n", + " 'prompt': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n周瑞家的道：“太太说：‘他们原不是一家子； 当年他们的祖和太老爷在一处做官，因连了宗的。<|im_end|>\\n<|im_start|>assistant\\n'},\n", + " {'chinese': '“听到了吗？',\n", + " 'english': \"'Did you hear that?'\",\n", + " 'text': \"<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n“听到了吗？<|im_end|>\\n<|im_start|>assistant\\n'Did you hear that?'<|im_end|>\",\n", + " 'prompt': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n“听到了吗？<|im_end|>\\n<|im_start|>assistant\\n'})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "datasets[\"train\"][1000], datasets[\"test\"][1000]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "dd425707-88cc-43db-8bc0-e858c8084e16", + "showTitle": false, + "title": "" + }, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 145, + "referenced_widgets": [ + "26e4202cca81496a90d15a0dd4ca9cf1", + "ba90fdb8822d47dab7ba203bee297f37", + "61560ff6a36b44f4a9dfdae5c52791d4", + "95fbe66647904c06a20f640630d6dc0e", + "57182a263d324a3dbf1471c74290a0d5", + "0f8b6bfe16894500838793f2491d403f", + "bb19f6c747754682a514373a3a0535ba", + "db19fc8d37db4e45a5790a876836d8c4", + "36166c7bcb854b34aca1f41a5d6ea50b", + "b0a370dc20654b279b9680692e34418e", + "cfeb365ddf7548d58b2557f22737fcf5", + "73e352a3404f4c7dad0737f57d29e92f", + "988a0e8c1f89446086858da0a891a79c", + "4ccedf0d93094e63b57a0f8a434fba06", + "6b2012c3f88547af8884a9ea90e3164b", + "7e29cb8dd4df4d5b94407cd8fd3f2011", + "ad2be500fc164c0f86f33e914ef8e6a0", + "5234566b1bfc4655b8d582ea5b46ed9f", + "4463edd481c1467f914c7dcd6c6e6ffc", + "6d3b9a05db0b4dadb638c686faa0c40a", + "938f45f1b3e24118b815d96ae34ba86a", + "9367047a800747f79c6b225d92397846", + "d1b47d39450d4019ae85c9b2f943eeaf", + "4dcf6ff672d24983a1877a8431709aa9", + "7975adbc2ec5489ea7fa0167e620d85c", + "71ce208e20d6483abb9ed923510c86d7", + "cfe8cae0e22b495bafa221a63d13b283", + "5807d5fb827d490fb3bc698f801ffff5", + "c4f2b06a82fd4987b8b659524a7b503b", + "6e34619b45934040b6092e6fb01ea7fe", + "271ddaa553a042d09b6db7b450643d8f", + "d69dc491b3ab44d7852b21873ed7bb7f", + "f401d53bf28e44eb906bce6c05412662", + "daf4cd890b35422683d22fd30bc71e83", + "b0240cd9a4554b29ae11f8051984a1c6", + "bc883d4cf13e4f8b8a4fe5f410cb6efd", + "99fdbb0300c14c139d1937c646f0cfe7", + "c161d94df0f04feba9542237e0856c22", + "edaf890370314a218f138015faa0b05d", + "697f027529b54ee9956bae78a11e0611", + "e9159e03e61f4f56978ece9c3bca49b2", + "810ff6c0e17d4fa09a30fef27eacff90", + "7358cdad832342c983e31efb8754ab78", + "e9adf418296e436fb48bb9f78885598b" + ] + }, + "id": "LjY75GoYUCB8", + "outputId": "7e2045fb-9ce9-49b1-b6e7-d5c9bc92455c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "<|im_start|>system\n", + "You are an expert in translating Chinese into English.<|im_end|>\n", + "<|im_start|>user\n", + "Translate from Chinese to English.\n", + "“听到了吗？<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "----------------------------------------\n", + "<|im_start|>system\n", + "You are an expert in translating Chinese into English.<|im_end|>\n", + "<|im_start|>user\n", + "Translate from Chinese to English.\n", + "“听到了吗？<|im_end|>\n", + "<|im_start|>assistant\n", + "Did you hear it?<|im_end|>\n", + "CPU times: user 1.62 s, sys: 160 ms, total: 1.78 s\n", + "Wall time: 1.8 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "prompt1 = datasets[\"test\"][\"prompt\"][1000]\n", + "print(prompt1)\n", + "print(\"--\" * 20)\n", + "test_model(model, tokenizer, prompt1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "53cc91a1-1623-4197-bf82-78cdadad933e", + "showTitle": false, + "title": "" + }, + "id": "idAEIeSQ3xdS" + }, + "source": [ + "\n", + "### Train the model\n", + "Now let's use Huggingface TRL's `SFTTrainer`! More docs here: [TRL SFT docs](https://huggingface.co/docs/trl/sft_trainer). We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. We also support TRL's `DPOTrainer`!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f1053974-9253-4d4d-a172-1f5fea046745", + "showTitle": false, + "title": "" + }, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 122, + "referenced_widgets": [ + "3cf2dd993b5e4d3daecf61e4bab5a404", + "087b76a8b7514269b1f0ab29b062e444", + "35b0e8c26d6640e9bd0ed7b242a423d8", + "54ad89e05fd74576b9b8b5b5a10eaf8d", + "a41dc44766444a998bec2d777f249d23", + "a069d2ab23824f29aa320ac256e2cfe9", + "06e806c82c7b4cbea31c5358dd9c3434", + "2e5087c76f98437cb5dc729230358cba", + "036fc5746f43416db18c19ad8fd36677", + "fdb1941405ed4e4aa06019933892deb3", + "668d5377ca56426a99753867e6e24862" + ] + }, + "id": "95_Nn-89DhsL", + "outputId": "bce9db22-b022-4e43-de3f-c7ea4c9c3c4e" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c9df9c466cc24f5e8715c02eb6764c3c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Map (num_proc=2): 0%| | 0/4528 [00:00\n", + " \n", + " \n", + " [5660/5660 1:02:58, Epoch 10/10]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
1002.137700
2002.001500
3001.938200
4001.935400
5001.899800
6001.819500
7001.689600
8001.737300
9001.665900
10001.664600
11001.723000
12001.520200
13001.381000
14001.423000
15001.419400
16001.436500
17001.401500
18001.119500
19001.130700
20001.139100
21001.120000
22001.166200
23001.062000
24000.858400
25000.846800
26000.892000
27000.887700
28000.907300
29000.728300
30000.644400
31000.652400
32000.683000
33000.673200
34000.670000
35000.460900
36000.487600
37000.501000
38000.491400
39000.501700
40000.452200
41000.352000
42000.368500
43000.368500
44000.360600
45000.374900
46000.294500
47000.270000
48000.270800
49000.285300
50000.282200
51000.285100
52000.216900
53000.228700
54000.223900
55000.226100
56000.229100

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "trainer_stats = trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "ee4dfe12-ac7d-4af0-b8ee-9a3361eb9a48", + "showTitle": false, + "title": "" + }, + "cellView": "form", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pCqnaKmlO1U9", + "outputId": "98f78253-86cf-4673-ff2b-923460c2b3fd" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3779.1598 seconds used for training.\n", + "62.99 minutes used for training.\n", + "Peak reserved memory = 1.855 GB.\n", + "Peak reserved memory for training = 0.656 GB.\n", + "Peak reserved memory % of max memory = 15.466 %.\n", + "Peak reserved memory for training % of max memory = 5.469 %.\n" + ] + } + ], + "source": [ + "# @title Show final memory and time stats\n", + "used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "used_memory_for_lora = round(used_memory - start_gpu_memory, 3)\n", + "used_percentage = round(used_memory / max_memory * 100, 3)\n", + "lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)\n", + "print(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\n", + "print(\n", + " f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\"\n", + ")\n", + "print(f\"Peak reserved memory = {used_memory} GB.\")\n", + "print(f\"Peak reserved memory for training = {used_memory_for_lora} GB.\")\n", + "print(f\"Peak reserved memory % of max memory = {used_percentage} %.\")\n", + "print(f\"Peak reserved memory for training % of max memory = {lora_percentage} %.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "4de3b67e-eeda-4c45-9472-35f105b4c30e", + "showTitle": false, + "title": "" + } + }, + "source": [ + "\n", + "### Inference\n", + "Let's run the model! You can change the instruction and input - leave the output blank!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "12c0141c-6a60-4fcd-a285-27eb59aa002b", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "<|im_start|>system\n", + "You are an expert in translating Chinese into English.<|im_end|>\n", + "<|im_start|>user\n", + "Translate from Chinese to English.\n", + "老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "----------------------------------------\n", + "<|im_start|>system\n", + "You are an expert in translating Chinese into English.<|im_end|>\n", + "<|im_start|>user\n", + "Translate from Chinese to English.\n", + "老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。<|im_end|>\n", + "<|im_start|>assistant\n", + "Old Geng lifted his gun and squinted over his shoulder, because the pistol started firing.<|im_end|>\n", + "CPU times: user 1.71 s, sys: 161 ms, total: 1.87 s\n", + "Wall time: 1.86 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "prompt1 = datasets[\"test\"][\"prompt\"][0]\n", + "print(prompt1)\n", + "print(\"--\" * 20)\n", + "test_model(model, tokenizer, prompt1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1ea3d5bc-314e-4c80-9419-bb5dfdd0172a", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Old Geng picked up his shotgun, squinted, and pulled the trigger. Two sparrows crashed to the ground like hailstones as shotgun pellets tore noisily through the branches.\n" + ] + } + ], + "source": [ + "print(datasets[\"test\"][\"english\"][0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f620a94b-4106-4cc9-a2cd-10ddc5f53b7a", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 1133/1133 [26:28<00:00, 1.40s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 23min 55s, sys: 2min 33s, total: 26min 28s\n", + "Wall time: 26min 28s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "predictions = eval_model(model, tokenizer, datasets[\"test\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "a4da8fd3-c055-429b-812f-0988fb7cd228", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'accuracy': 0.00088261253309797,\n", + " 'correct_ids': [147],\n", + " 'bleu_scores': {'bleu': 0.06508609399238363,\n", + " 'precisions': [0.3407579117113485,\n", + " 0.09377291935878182,\n", + " 0.03598822203642444,\n", + " 0.01652015762352228],\n", + " 'brevity_penalty': 0.9858565320713017,\n", + " 'length_ratio': 0.9859556144418682,\n", + " 'translation_length': 29766,\n", + " 'reference_length': 30190},\n", + " 'rouge_scores': {'rouge1': 0.32340459562777546,\n", + " 'rouge2': 0.11259712507132531,\n", + " 'rougeL': 0.2671219091010598,\n", + " 'rougeLsum': 0.2670685844265569}}" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "calc_metrics(datasets[\"test\"][\"english\"], predictions, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c65f8c79-a8fe-4256-9a9d-2451ee7164ba", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " chinese \\\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct(finetuned) \n", + "0 Old Geng lifted his rifle and narrowed his eye... \n" + ] + } + ], + "source": [ + "from llm_toolkit.translation_engine import save_results\n", + "\n", + "save_results(model_name + \"(finetuned)\", \"results/mac-results.tsv\", datasets[\"test\"], predictions, debug=True)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": {}, + "notebookName": "05_MAC_+_Qwen2-0.5B_Unsloth_train", + "widgets": {} + }, + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "036fc5746f43416db18c19ad8fd36677": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "06e806c82c7b4cbea31c5358dd9c3434": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "087b76a8b7514269b1f0ab29b062e444": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a069d2ab23824f29aa320ac256e2cfe9", + "placeholder": "​", + "style": "IPY_MODEL_06e806c82c7b4cbea31c5358dd9c3434", + "value": "Map (num_proc=2): 100%" + } + }, + "09b76013aa9e45efb6deb23a7a0d0925": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dea41c5260884aa6879b5e1d1697b14f", + "placeholder": "​", + "style": "IPY_MODEL_89965917796a4f81b899fdc7685f33df", + "value": "config.json: 100%" + } + }, + "0a92c56bfa134ef583220d7ef0b13e17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c34be936c8145d3ab41282f30a70713": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f8b6bfe16894500838793f2491d403f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "177c78fce95d4b4ab33057c5a048d693": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1f44c9ce1adf470cbb19784493ed209f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c34be936c8145d3ab41282f30a70713", + "placeholder": "​", + "style": "IPY_MODEL_0a92c56bfa134ef583220d7ef0b13e17", + "value": "model.safetensors: 100%" + } + }, + "201b59ccd9f845e197029b57e424aefc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2157f01726d748f8a9ae4a00664430da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21db8a77b00d4a4e82fdfa608657531f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26e4202cca81496a90d15a0dd4ca9cf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba90fdb8822d47dab7ba203bee297f37", + "IPY_MODEL_61560ff6a36b44f4a9dfdae5c52791d4", + "IPY_MODEL_95fbe66647904c06a20f640630d6dc0e" + ], + "layout": "IPY_MODEL_57182a263d324a3dbf1471c74290a0d5" + } + }, + "27155728b6b84cb199c91c940095d0a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6b91feeed5464877991ac2c207aebe7c", + "IPY_MODEL_cca8113c54c0495daedce1327bf9c68b", + "IPY_MODEL_2e63a29e2f7247bba5beede9a568c99f" + ], + "layout": "IPY_MODEL_5c9d781c28944f3eb86e2a6d44efdf18" + } + }, + "271ddaa553a042d09b6db7b450643d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a58d04b428c46f4b3dbadd3bc6cd529": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d18ddf6482c4d97829ac0e5a7b9868f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "IPY_MODEL_f2df530d22c74977b249dd9fb5f4829b", + "IPY_MODEL_89b2ef0dbfea47ab8e6f8d659e3351d1" + ], + "layout": "IPY_MODEL_3056b148aa9f4e6e8aa3b61d26886255" + } + }, + "2e5087c76f98437cb5dc729230358cba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e63a29e2f7247bba5beede9a568c99f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b993eaec6b224440bf80c0958c6fb536", + "placeholder": "​", + "style": "IPY_MODEL_de868e26e7154f62aa86223a539ad421", + "value": " 464/464 [00:00<00:00, 27.1kB/s]" + } + }, + "2f6c70dd266c4816bfad3fd3d192929a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "30307300bc4e4baf96560e30969a82b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e36a3f9eff0e4cf68834d66b0213ae96", + "placeholder": "​", + "style": "IPY_MODEL_a0037bdccf254159becde630bee3d1db", + "value": "generation_config.json: 100%" + } + }, + "3056b148aa9f4e6e8aa3b61d26886255": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30cdc32298134cb0be4d41615b9e5774": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3572201bd4d74a58b7a665f9bdfdcdba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35b0e8c26d6640e9bd0ed7b242a423d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e5087c76f98437cb5dc729230358cba", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_036fc5746f43416db18c19ad8fd36677", + "value": 51760 + } + }, + "36166c7bcb854b34aca1f41a5d6ea50b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "370692d819df41828b48c4ad446f977b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "39b29a75374b45c0a22506010be2b84e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_30cdc32298134cb0be4d41615b9e5774", + "max": 1179, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_47928317548c454bba6358ab132e8dee", + "value": 1179 + } + }, + "3cf2dd993b5e4d3daecf61e4bab5a404": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_087b76a8b7514269b1f0ab29b062e444", + "IPY_MODEL_35b0e8c26d6640e9bd0ed7b242a423d8", + "IPY_MODEL_54ad89e05fd74576b9b8b5b5a10eaf8d" + ], + "layout": "IPY_MODEL_a41dc44766444a998bec2d777f249d23" + } + }, + "43dec2ede91341f5af60eb522e18e984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4463edd481c1467f914c7dcd6c6e6ffc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47928317548c454bba6358ab132e8dee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "49277aeeac16434a865a4d12308b1abc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4ae7e449e4ea4c729b5f34607c18ebae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b2061b8a73c43ffb0c2f83daf0d0183": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c4c88d4c701450692fa0f6b0c5764b0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c666f4ace3943f8b80ecd20e7503236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ccedf0d93094e63b57a0f8a434fba06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4463edd481c1467f914c7dcd6c6e6ffc", + "max": 44307561, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6d3b9a05db0b4dadb638c686faa0c40a", + "value": 44307561 + } + }, + "4dcf6ff672d24983a1877a8431709aa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5807d5fb827d490fb3bc698f801ffff5", + "placeholder": "​", + "style": "IPY_MODEL_c4f2b06a82fd4987b8b659524a7b503b", + "value": "Generating train split: 100%" + } + }, + "4ea63adfce694725bdba878aef709dd3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5234566b1bfc4655b8d582ea5b46ed9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "54ad89e05fd74576b9b8b5b5a10eaf8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdb1941405ed4e4aa06019933892deb3", + "placeholder": "​", + "style": "IPY_MODEL_668d5377ca56426a99753867e6e24862", + "value": " 51760/51760 [01:02<00:00, 1131.51 examples/s]" + } + }, + "56aee4853b7740e6a977254f5d1fa66d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "57182a263d324a3dbf1471c74290a0d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5807d5fb827d490fb3bc698f801ffff5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c9d781c28944f3eb86e2a6d44efdf18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f40db8173dd4d76b6ef5ed6d9ec8b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61560ff6a36b44f4a9dfdae5c52791d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db19fc8d37db4e45a5790a876836d8c4", + "max": 11610, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36166c7bcb854b34aca1f41a5d6ea50b", + "value": 11610 + } + }, + "6578fd7acdb54c4c93528ea431fd0144": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_370692d819df41828b48c4ad446f977b", + "placeholder": "​", + "style": "IPY_MODEL_a0bf9160eb2647409b3200270914b90f", + "value": " 50.6k/50.6k [00:00<00:00, 2.71MB/s]" + } + }, + "668d5377ca56426a99753867e6e24862": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697f027529b54ee9956bae78a11e0611": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69ac12aec0714318bf2c83d4f4e745f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b2012c3f88547af8884a9ea90e3164b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_938f45f1b3e24118b815d96ae34ba86a", + "placeholder": "​", + "style": "IPY_MODEL_9367047a800747f79c6b225d92397846", + "value": " 44.3M/44.3M [00:01<00:00, 31.0MB/s]" + } + }, + "6b91feeed5464877991ac2c207aebe7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b2061b8a73c43ffb0c2f83daf0d0183", + "placeholder": "​", + "style": "IPY_MODEL_69ac12aec0714318bf2c83d4f4e745f5", + "value": "special_tokens_map.json: 100%" + } + }, + "6d3b9a05db0b4dadb638c686faa0c40a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6dbbedeca9314e66ae50e44ffa31a414": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6e34619b45934040b6092e6fb01ea7fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71ce208e20d6483abb9ed923510c86d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d69dc491b3ab44d7852b21873ed7bb7f", + "placeholder": "​", + "style": "IPY_MODEL_f401d53bf28e44eb906bce6c05412662", + "value": " 51760/51760 [00:01<00:00, 45512.81 examples/s]" + } + }, + "7358cdad832342c983e31efb8754ab78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e352a3404f4c7dad0737f57d29e92f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_988a0e8c1f89446086858da0a891a79c", + "IPY_MODEL_4ccedf0d93094e63b57a0f8a434fba06", + "IPY_MODEL_6b2012c3f88547af8884a9ea90e3164b" + ], + "layout": "IPY_MODEL_7e29cb8dd4df4d5b94407cd8fd3f2011" + } + }, + "74501720ac7e4dbb911a4a99b3633bc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "78e5400bff924a92a4cc61c4ff18b182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9b313fd861948f5aba25b24b1518d30", + "placeholder": "​", + "style": "IPY_MODEL_4c666f4ace3943f8b80ecd20e7503236", + "value": " 1.18k/1.18k [00:00<00:00, 31.3kB/s]" + } + }, + "7975adbc2ec5489ea7fa0167e620d85c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e34619b45934040b6092e6fb01ea7fe", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_271ddaa553a042d09b6db7b450643d8f", + "value": 51760 + } + }, + "7e29cb8dd4df4d5b94407cd8fd3f2011": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "810ff6c0e17d4fa09a30fef27eacff90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89965917796a4f81b899fdc7685f33df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "89b2ef0dbfea47ab8e6f8d659e3351d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8908fa0df3743ecb9d12983a739104f", + "placeholder": "​", + "style": "IPY_MODEL_177c78fce95d4b4ab33057c5a048d693", + "value": " 9.09M/9.09M [00:00<00:00, 32.6MB/s]" + } + }, + "8b3505352a5a42bf910428c40ce40465": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_49277aeeac16434a865a4d12308b1abc", + "placeholder": "​", + "style": "IPY_MODEL_2157f01726d748f8a9ae4a00664430da", + "value": " 5.70G/5.70G [01:02<00:00, 30.1MB/s]" + } + }, + "8fc142b628fb40568730234de1cafde2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ae7e449e4ea4c729b5f34607c18ebae", + "max": 172, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3572201bd4d74a58b7a665f9bdfdcdba", + "value": 172 + } + }, + "9367047a800747f79c6b225d92397846": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "938f45f1b3e24118b815d96ae34ba86a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "95fbe66647904c06a20f640630d6dc0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b0a370dc20654b279b9680692e34418e", + "placeholder": "​", + "style": "IPY_MODEL_cfeb365ddf7548d58b2557f22737fcf5", + "value": " 11.6k/11.6k [00:00<00:00, 716kB/s]" + } + }, + "988a0e8c1f89446086858da0a891a79c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ad2be500fc164c0f86f33e914ef8e6a0", + "placeholder": "​", + "style": "IPY_MODEL_5234566b1bfc4655b8d582ea5b46ed9f", + "value": "Downloading data: 100%" + } + }, + "98c58f23f4d549518832cb2d18f796e8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_09b76013aa9e45efb6deb23a7a0d0925", + "IPY_MODEL_39b29a75374b45c0a22506010be2b84e", + "IPY_MODEL_78e5400bff924a92a4cc61c4ff18b182" + ], + "layout": "IPY_MODEL_2a58d04b428c46f4b3dbadd3bc6cd529" + } + }, + "99fdbb0300c14c139d1937c646f0cfe7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7358cdad832342c983e31efb8754ab78", + "placeholder": "​", + "style": "IPY_MODEL_e9adf418296e436fb48bb9f78885598b", + "value": " 51760/51760 [00:01<00:00, 38665.95 examples/s]" + } + }, + "9f679ad3ec7f4fe8ad0510ffb57bc2ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ea63adfce694725bdba878aef709dd3", + "placeholder": "​", + "style": "IPY_MODEL_74501720ac7e4dbb911a4a99b3633bc6", + "value": "tokenizer.json: 100%" + } + }, + "a0037bdccf254159becde630bee3d1db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a069d2ab23824f29aa320ac256e2cfe9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0bf9160eb2647409b3200270914b90f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a41dc44766444a998bec2d777f249d23": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8464a4c711e4e00aafdfc919b60d07e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fb995c740590427b882572c81d4e848c", + "placeholder": "​", + "style": "IPY_MODEL_201b59ccd9f845e197029b57e424aefc", + "value": " 172/172 [00:00<00:00, 12.0kB/s]" + } + }, + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad2be500fc164c0f86f33e914ef8e6a0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b0240cd9a4554b29ae11f8051984a1c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_edaf890370314a218f138015faa0b05d", + "placeholder": "​", + "style": "IPY_MODEL_697f027529b54ee9956bae78a11e0611", + "value": "Map: 100%" + } + }, + "b0a370dc20654b279b9680692e34418e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b518dcee69074b87be73957cd810e7ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d891f8d0b1fc462f8008d02bb2a15692", + "placeholder": "​", + "style": "IPY_MODEL_cced8fd7e998472794f3f3e3018956a5", + "value": "tokenizer_config.json: 100%" + } + }, + "b8908fa0df3743ecb9d12983a739104f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b993eaec6b224440bf80c0958c6fb536": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9b313fd861948f5aba25b24b1518d30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba90fdb8822d47dab7ba203bee297f37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f8b6bfe16894500838793f2491d403f", + "placeholder": "​", + "style": "IPY_MODEL_bb19f6c747754682a514373a3a0535ba", + "value": "Downloading readme: 100%" + } + }, + "bb19f6c747754682a514373a3a0535ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc883d4cf13e4f8b8a4fe5f410cb6efd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9159e03e61f4f56978ece9c3bca49b2", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_810ff6c0e17d4fa09a30fef27eacff90", + "value": 51760 + } + }, + "c161d94df0f04feba9542237e0856c22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c22f71b1f85843209d7e5321506b9cb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f44c9ce1adf470cbb19784493ed209f", + "IPY_MODEL_f1addc4479d849879e743cf9089e6540", + "IPY_MODEL_8b3505352a5a42bf910428c40ce40465" + ], + "layout": "IPY_MODEL_4c4c88d4c701450692fa0f6b0c5764b0" + } + }, + "c4f2b06a82fd4987b8b659524a7b503b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cca8113c54c0495daedce1327bf9c68b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e02f9b7849c64531835eb77b860d1c93", + "max": 464, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_56aee4853b7740e6a977254f5d1fa66d", + "value": 464 + } + }, + "cced8fd7e998472794f3f3e3018956a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cf245afeb1c04f29a24d291608c3d157": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b518dcee69074b87be73957cd810e7ed", + "IPY_MODEL_e29104486d594b2992d7285e0ef77371", + "IPY_MODEL_6578fd7acdb54c4c93528ea431fd0144" + ], + "layout": "IPY_MODEL_d35db8148a354c56aaac56dbae22536f" + } + }, + "cfe8cae0e22b495bafa221a63d13b283": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfeb365ddf7548d58b2557f22737fcf5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1b47d39450d4019ae85c9b2f943eeaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4dcf6ff672d24983a1877a8431709aa9", + "IPY_MODEL_7975adbc2ec5489ea7fa0167e620d85c", + "IPY_MODEL_71ce208e20d6483abb9ed923510c86d7" + ], + "layout": "IPY_MODEL_cfe8cae0e22b495bafa221a63d13b283" + } + }, + "d35db8148a354c56aaac56dbae22536f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69dc491b3ab44d7852b21873ed7bb7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d891f8d0b1fc462f8008d02bb2a15692": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8e5318cead340c4adbeaccc05d39225": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "daf4cd890b35422683d22fd30bc71e83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b0240cd9a4554b29ae11f8051984a1c6", + "IPY_MODEL_bc883d4cf13e4f8b8a4fe5f410cb6efd", + "IPY_MODEL_99fdbb0300c14c139d1937c646f0cfe7" + ], + "layout": "IPY_MODEL_c161d94df0f04feba9542237e0856c22" + } + }, + "db19fc8d37db4e45a5790a876836d8c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de868e26e7154f62aa86223a539ad421": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dea41c5260884aa6879b5e1d1697b14f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e02f9b7849c64531835eb77b860d1c93": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e29104486d594b2992d7285e0ef77371": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "max": 50641, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2f6c70dd266c4816bfad3fd3d192929a", + "value": 50641 + } + }, + "e36a3f9eff0e4cf68834d66b0213ae96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9159e03e61f4f56978ece9c3bca49b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9adf418296e436fb48bb9f78885598b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "edaf890370314a218f138015faa0b05d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f1addc4479d849879e743cf9089e6540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43dec2ede91341f5af60eb522e18e984", + "max": 5702746405, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d8e5318cead340c4adbeaccc05d39225", + "value": 5702746405 + } + }, + "f2df530d22c74977b249dd9fb5f4829b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_21db8a77b00d4a4e82fdfa608657531f", + "max": 9085698, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6dbbedeca9314e66ae50e44ffa31a414", + "value": 9085698 + } + }, + "f401d53bf28e44eb906bce6c05412662": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fb995c740590427b882572c81d4e848c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fce7a61c25ec4390af43d92b7c473a45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_30307300bc4e4baf96560e30969a82b6", + "IPY_MODEL_8fc142b628fb40568730234de1cafde2", + "IPY_MODEL_a8464a4c711e4e00aafdfc919b60d07e" + ], + "layout": "IPY_MODEL_5f40db8173dd4d76b6ef5ed6d9ec8b6e" + } + }, + "fdb1941405ed4e4aa06019933892deb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/02_Qwen2-1.5B_Unsloth.ipynb b/notebooks/02_Qwen2-1.5B_Unsloth.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..dec98d4f0151f2bdd177702b0d9c6d7beb7d0025 --- /dev/null +++ b/notebooks/02_Qwen2-1.5B_Unsloth.ipynb @@ -0,0 +1,5632 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "5e06060e-f3d7-4e1e-b97e-dc57d8d17ce5", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6831b89a-0776-4014-a3db-9e1860a4c80c", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /home/inflaton/code/projects/courses/novel-translation\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1bdd4cdb-cb26-4527-862d-66ea2a7a1f05", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "14807e21-2648-48a3-9916-6c576fc61d2e", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('unsloth/Qwen2-1.5B-Instruct',\n", + " True,\n", + " 'models/Qwen2-1.5B-Instruct-MAC-',\n", + " 'Qwen2-1.5B-Instruct-MAC-',\n", + " 2048,\n", + " 10,\n", + " None,\n", + " 'datasets/mac/mac.tsv')" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "model_name = os.getenv(\"MODEL_NAME\")\n", + "token = os.getenv(\"HF_TOKEN\") or None\n", + "load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n", + "local_model = os.getenv(\"LOCAL_MODEL\")\n", + "hub_model = os.getenv(\"HUB_MODEL\")\n", + "num_train_epochs = int(os.getenv(\"NUM_TRAIN_EPOCHS\") or 0)\n", + "data_path = os.getenv(\"DATA_PATH\")\n", + "\n", + "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n", + "dtype = (\n", + " None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n", + ")\n", + "\n", + "model_name, load_in_4bit, local_model, hub_model, max_seq_length, num_train_epochs, dtype, data_path" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "bc44b98b-6394-4b2c-af2f-8caa40b28453", + "showTitle": false, + "title": "" + }, + "id": "r2v_X2fA0Df5" + }, + "source": [ + "* We support Llama, Mistral, Phi-3, Gemma, Yi, DeepSeek, Qwen, TinyLlama, Vicuna, Open Hermes etc\n", + "* We support 16bit LoRA or 4bit QLoRA. Both 2x faster.\n", + "* `max_seq_length` can be set to anything, since we do automatic RoPE Scaling via [kaiokendev's](https://kaiokendev.github.io/til) method.\n", + "* With [PR 26037](https://github.com/huggingface/transformers/pull/26037), we support downloading 4bit models **4x faster**! [Our repo](https://huggingface.co/unsloth) has Llama, Mistral 4bit models.\n", + "* [**NEW**] We make Phi-3 Medium / Mini **2x faster**! See our [Phi-3 Medium notebook](https://colab.research.google.com/drive/1hhdhBa1j_hsymiW9m-WzxQtgqTH_NHqi?usp=sharing)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "b952e9b9-edf1-4bb8-b52b-bb714852c721", + "showTitle": false, + "title": "" + }, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 353, + "referenced_widgets": [ + "98c58f23f4d549518832cb2d18f796e8", + "09b76013aa9e45efb6deb23a7a0d0925", + "39b29a75374b45c0a22506010be2b84e", + "78e5400bff924a92a4cc61c4ff18b182", + "2a58d04b428c46f4b3dbadd3bc6cd529", + "dea41c5260884aa6879b5e1d1697b14f", + "89965917796a4f81b899fdc7685f33df", + "30cdc32298134cb0be4d41615b9e5774", + "47928317548c454bba6358ab132e8dee", + "b9b313fd861948f5aba25b24b1518d30", + "4c666f4ace3943f8b80ecd20e7503236", + "c22f71b1f85843209d7e5321506b9cb9", + "1f44c9ce1adf470cbb19784493ed209f", + "f1addc4479d849879e743cf9089e6540", + "8b3505352a5a42bf910428c40ce40465", + "4c4c88d4c701450692fa0f6b0c5764b0", + "0c34be936c8145d3ab41282f30a70713", + "0a92c56bfa134ef583220d7ef0b13e17", + "43dec2ede91341f5af60eb522e18e984", + "d8e5318cead340c4adbeaccc05d39225", + "49277aeeac16434a865a4d12308b1abc", + "2157f01726d748f8a9ae4a00664430da", + "fce7a61c25ec4390af43d92b7c473a45", + "30307300bc4e4baf96560e30969a82b6", + "8fc142b628fb40568730234de1cafde2", + "a8464a4c711e4e00aafdfc919b60d07e", + "5f40db8173dd4d76b6ef5ed6d9ec8b6e", + "e36a3f9eff0e4cf68834d66b0213ae96", + "a0037bdccf254159becde630bee3d1db", + "4ae7e449e4ea4c729b5f34607c18ebae", + "3572201bd4d74a58b7a665f9bdfdcdba", + "fb995c740590427b882572c81d4e848c", + "201b59ccd9f845e197029b57e424aefc", + "cf245afeb1c04f29a24d291608c3d157", + "b518dcee69074b87be73957cd810e7ed", + "e29104486d594b2992d7285e0ef77371", + "6578fd7acdb54c4c93528ea431fd0144", + "d35db8148a354c56aaac56dbae22536f", + "d891f8d0b1fc462f8008d02bb2a15692", + "cced8fd7e998472794f3f3e3018956a5", + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "2f6c70dd266c4816bfad3fd3d192929a", + "370692d819df41828b48c4ad446f977b", + "a0bf9160eb2647409b3200270914b90f", + "2d18ddf6482c4d97829ac0e5a7b9868f", + "9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "f2df530d22c74977b249dd9fb5f4829b", + "89b2ef0dbfea47ab8e6f8d659e3351d1", + "3056b148aa9f4e6e8aa3b61d26886255", + "4ea63adfce694725bdba878aef709dd3", + "74501720ac7e4dbb911a4a99b3633bc6", + "21db8a77b00d4a4e82fdfa608657531f", + "6dbbedeca9314e66ae50e44ffa31a414", + "b8908fa0df3743ecb9d12983a739104f", + "177c78fce95d4b4ab33057c5a048d693", + "27155728b6b84cb199c91c940095d0a8", + "6b91feeed5464877991ac2c207aebe7c", + "cca8113c54c0495daedce1327bf9c68b", + "2e63a29e2f7247bba5beede9a568c99f", + "5c9d781c28944f3eb86e2a6d44efdf18", + "4b2061b8a73c43ffb0c2f83daf0d0183", + "69ac12aec0714318bf2c83d4f4e745f5", + "e02f9b7849c64531835eb77b860d1c93", + "56aee4853b7740e6a977254f5d1fa66d", + "b993eaec6b224440bf80c0958c6fb536", + "de868e26e7154f62aa86223a539ad421" + ] + }, + "id": "QmUBVEnvCDJv", + "outputId": "a0e2d781-4934-415a-90b4-35165b9e44c5" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a44529371839466cae7797d068873634", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "config.json: 0%| | 0.00/707 [00:00 0 ! Suggested 8, 16, 32, 64, 128\n", + " target_modules=[\n", + " \"q_proj\",\n", + " \"k_proj\",\n", + " \"v_proj\",\n", + " \"o_proj\",\n", + " \"gate_proj\",\n", + " \"up_proj\",\n", + " \"down_proj\",\n", + " ],\n", + " lora_alpha=16,\n", + " lora_dropout=0, # Supports any, but = 0 is optimized\n", + " bias=\"none\", # Supports any, but = \"none\" is optimized\n", + " # [NEW] \"unsloth\" uses 30% less VRAM, fits 2x larger batch sizes!\n", + " use_gradient_checkpointing=\"unsloth\", # True or \"unsloth\" for very long context\n", + " random_state=3407,\n", + " use_rslora=False, # We support rank stabilized LoRA\n", + " loftq_config=None, # And LoftQ\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "16e3c2ff-36ff-4895-bfd0-59ab1b2130cc", + "showTitle": false, + "title": "" + }, + "id": "vITh0KVJ10qX" + }, + "source": [ + "\n", + "### Data Prep\n", + "We now use the Alpaca dataset from [yahma](https://huggingface.co/datasets/yahma/alpaca-cleaned), which is a filtered version of 52K of the original [Alpaca dataset](https://crfm.stanford.edu/2023/03/13/alpaca.html). You can replace this code section with your own data prep.\n", + "\n", + "**[NOTE]** To train only on completions (ignoring the user's input) read TRL's docs [here](https://huggingface.co/docs/trl/sft_trainer#train-on-completions-only).\n", + "\n", + "**[NOTE]** Remember to add the **EOS_TOKEN** to the tokenized output!! Otherwise you'll get infinite generations!\n", + "\n", + "If you want to use the `llama-3` template for ShareGPT datasets, try our conversational [notebook](https://colab.research.google.com/drive/1XamvWYinY6FOSX9GLvnqSjjsNflxdhNc?usp=sharing).\n", + "\n", + "For text completions like novel writing, try this [notebook](https://colab.research.google.com/drive/1ef-tab5bhkvWmBOObepl1WgJvfvSzn5Q?usp=sharing)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "4426fdab-78f7-4a28-abf7-dc55b19db864", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading train/test data files\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "95a1b6aa815f461a8281e33633a28a9b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Map: 0%| | 0/4528 [00:00system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n全仗着狐仙搭救。<|im_end|>\\n<|im_start|>assistant\\nBecause I was protected by a fox fairy.<|im_end|>',\n", + " 'prompt': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n全仗着狐仙搭救。<|im_end|>\\n<|im_start|>assistant\\n'},\n", + " {'chinese': '老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。',\n", + " 'english': 'Old Geng picked up his shotgun, squinted, and pulled the trigger. Two sparrows crashed to the ground like hailstones as shotgun pellets tore noisily through the branches.',\n", + " 'text': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。<|im_end|>\\n<|im_start|>assistant\\nOld Geng picked up his shotgun, squinted, and pulled the trigger. Two sparrows crashed to the ground like hailstones as shotgun pellets tore noisily through the branches.<|im_end|>',\n", + " 'prompt': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。<|im_end|>\\n<|im_start|>assistant\\n'})" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "datasets[\"train\"][0], datasets[\"test\"][0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3e839830-d2da-48e3-b6f4-63da7a7b9dab", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "({'chinese': '周瑞家的道：“太太说：‘他们原不是一家子； 当年他们的祖和太老爷在一处做官，因连了宗的。',\n", + " 'english': \"'She said they don't really belong to the family but were adopted into the clan years ago when your grandfather and theirs were working in the same office.\",\n", + " 'text': \"<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n周瑞家的道：“太太说：‘他们原不是一家子； 当年他们的祖和太老爷在一处做官，因连了宗的。<|im_end|>\\n<|im_start|>assistant\\n'She said they don't really belong to the family but were adopted into the clan years ago when your grandfather and theirs were working in the same office.<|im_end|>\",\n", + " 'prompt': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n周瑞家的道：“太太说：‘他们原不是一家子； 当年他们的祖和太老爷在一处做官，因连了宗的。<|im_end|>\\n<|im_start|>assistant\\n'},\n", + " {'chinese': '“听到了吗？',\n", + " 'english': \"'Did you hear that?'\",\n", + " 'text': \"<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n“听到了吗？<|im_end|>\\n<|im_start|>assistant\\n'Did you hear that?'<|im_end|>\",\n", + " 'prompt': '<|im_start|>system\\nYou are an expert in translating Chinese into English.<|im_end|>\\n<|im_start|>user\\nTranslate from Chinese to English.\\n“听到了吗？<|im_end|>\\n<|im_start|>assistant\\n'})" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "datasets[\"train\"][1000], datasets[\"test\"][1000]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "03a3c02c-d3d9-49f4-87b5-2e568c174175", + "showTitle": false, + "title": "" + }, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 145, + "referenced_widgets": [ + "26e4202cca81496a90d15a0dd4ca9cf1", + "ba90fdb8822d47dab7ba203bee297f37", + "61560ff6a36b44f4a9dfdae5c52791d4", + "95fbe66647904c06a20f640630d6dc0e", + "57182a263d324a3dbf1471c74290a0d5", + "0f8b6bfe16894500838793f2491d403f", + "bb19f6c747754682a514373a3a0535ba", + "db19fc8d37db4e45a5790a876836d8c4", + "36166c7bcb854b34aca1f41a5d6ea50b", + "b0a370dc20654b279b9680692e34418e", + "cfeb365ddf7548d58b2557f22737fcf5", + "73e352a3404f4c7dad0737f57d29e92f", + "988a0e8c1f89446086858da0a891a79c", + "4ccedf0d93094e63b57a0f8a434fba06", + "6b2012c3f88547af8884a9ea90e3164b", + "7e29cb8dd4df4d5b94407cd8fd3f2011", + "ad2be500fc164c0f86f33e914ef8e6a0", + "5234566b1bfc4655b8d582ea5b46ed9f", + "4463edd481c1467f914c7dcd6c6e6ffc", + "6d3b9a05db0b4dadb638c686faa0c40a", + "938f45f1b3e24118b815d96ae34ba86a", + "9367047a800747f79c6b225d92397846", + "d1b47d39450d4019ae85c9b2f943eeaf", + "4dcf6ff672d24983a1877a8431709aa9", + "7975adbc2ec5489ea7fa0167e620d85c", + "71ce208e20d6483abb9ed923510c86d7", + "cfe8cae0e22b495bafa221a63d13b283", + "5807d5fb827d490fb3bc698f801ffff5", + "c4f2b06a82fd4987b8b659524a7b503b", + "6e34619b45934040b6092e6fb01ea7fe", + "271ddaa553a042d09b6db7b450643d8f", + "d69dc491b3ab44d7852b21873ed7bb7f", + "f401d53bf28e44eb906bce6c05412662", + "daf4cd890b35422683d22fd30bc71e83", + "b0240cd9a4554b29ae11f8051984a1c6", + "bc883d4cf13e4f8b8a4fe5f410cb6efd", + "99fdbb0300c14c139d1937c646f0cfe7", + "c161d94df0f04feba9542237e0856c22", + "edaf890370314a218f138015faa0b05d", + "697f027529b54ee9956bae78a11e0611", + "e9159e03e61f4f56978ece9c3bca49b2", + "810ff6c0e17d4fa09a30fef27eacff90", + "7358cdad832342c983e31efb8754ab78", + "e9adf418296e436fb48bb9f78885598b" + ] + }, + "id": "LjY75GoYUCB8", + "outputId": "7e2045fb-9ce9-49b1-b6e7-d5c9bc92455c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "<|im_start|>system\n", + "You are an expert in translating Chinese into English.<|im_end|>\n", + "<|im_start|>user\n", + "Translate from Chinese to English.\n", + "“听到了吗？<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "----------------------------------------\n", + "<|im_start|>system\n", + "You are an expert in translating Chinese into English.<|im_end|>\n", + "<|im_start|>user\n", + "Translate from Chinese to English.\n", + "“听到了吗？<|im_end|>\n", + "<|im_start|>assistant\n", + "Did you hear that?<|im_end|>\n", + "CPU times: user 1.8 s, sys: 873 ms, total: 2.68 s\n", + "Wall time: 2.72 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "prompt1 = datasets[\"test\"][\"prompt\"][1000]\n", + "print(prompt1)\n", + "print(\"--\" * 20)\n", + "test_model(model, tokenizer, prompt1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "22ad05ed-04e7-420f-82bf-8f990efce37c", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 1133/1133 [30:01<00:00, 1.59s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 27min 10s, sys: 2min 52s, total: 30min 2s\n", + "Wall time: 30min 1s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "predictions = eval_model(model, tokenizer, datasets[\"test\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "eeba4278-d952-4391-8f63-c123e6098ffd", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'accuracy': 0.00176522506619594,\n", + " 'correct_ids': [658, 659],\n", + " 'bleu_scores': {'bleu': 0.08285381577653864,\n", + " 'precisions': [0.40636974021865224,\n", + " 0.12583290620194773,\n", + " 0.051405438435685916,\n", + " 0.02290685609386224],\n", + " 'brevity_penalty': 0.9405675222192741,\n", + " 'length_ratio': 0.9422656508777741,\n", + " 'translation_length': 28447,\n", + " 'reference_length': 30190},\n", + " 'rouge_scores': {'rouge1': 0.38844471682897896,\n", + " 'rouge2': 0.14120062297432684,\n", + " 'rougeL': 0.3280668137668106,\n", + " 'rougeLsum': 0.3280344032501499}}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "calc_metrics(datasets[\"test\"][\"english\"], predictions, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "2485caac-9b06-42f5-a4da-213d3e522a06", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Unnamed: 0 chinese \\\n", + "0 0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct(finetuned) \\\n", + "0 Old Geng lifted his rifle and narrowed his eye... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct \n", + "0 Old Geng took up his gun, squinted one of its ... \n" + ] + } + ], + "source": [ + "save_results(\n", + " model_name,\n", + " \"results/mac-results.csv\",\n", + " datasets[\"test\"],\n", + " predictions,\n", + " debug=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "5c3f9939-9068-4edf-b057-e4898efeb94e", + "showTitle": false, + "title": "" + }, + "id": "idAEIeSQ3xdS" + }, + "source": [ + "\n", + "### Train the model\n", + "Now let's use Huggingface TRL's `SFTTrainer`! More docs here: [TRL SFT docs](https://huggingface.co/docs/trl/sft_trainer). We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. We also support TRL's `DPOTrainer`!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "053bd880-409c-4ae0-a5a5-06084ada19d5", + "showTitle": false, + "title": "" + }, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 122, + "referenced_widgets": [ + "3cf2dd993b5e4d3daecf61e4bab5a404", + "087b76a8b7514269b1f0ab29b062e444", + "35b0e8c26d6640e9bd0ed7b242a423d8", + "54ad89e05fd74576b9b8b5b5a10eaf8d", + "a41dc44766444a998bec2d777f249d23", + "a069d2ab23824f29aa320ac256e2cfe9", + "06e806c82c7b4cbea31c5358dd9c3434", + "2e5087c76f98437cb5dc729230358cba", + "036fc5746f43416db18c19ad8fd36677", + "fdb1941405ed4e4aa06019933892deb3", + "668d5377ca56426a99753867e6e24862" + ] + }, + "id": "95_Nn-89DhsL", + "outputId": "bce9db22-b022-4e43-de3f-c7ea4c9c3c4e" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6b952d520d494e58811bae80cf5ae883", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Map (num_proc=2): 0%| | 0/4528 [00:00\n", + " \n", + " \n", + " [5660/5660 1:32:43, Epoch 10/10]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
1001.919100
2001.774900
3001.722600
4001.721900
5001.695700
6001.612500
7001.473700
8001.518000
9001.452100
10001.454900
11001.509600
12001.272200
13001.128400
14001.161200
15001.165600
16001.169700
17001.140900
18000.796500
19000.812800
20000.815000
21000.806600
22000.850100
23000.737200
24000.533900
25000.521600
26000.562600
27000.557700
28000.563000
29000.418500
30000.343000
31000.353900
32000.368300
33000.367600
34000.361000
35000.230000
36000.244000
37000.246400
38000.245400
39000.256800
40000.232000
41000.178700
42000.186600
43000.189200
44000.189600
45000.190100
46000.160900
47000.155000
48000.155300
49000.157400
50000.159500
51000.157000
52000.138300
53000.138600
54000.139500
55000.141400
56000.144900

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 1h 23min 59s, sys: 8min 44s, total: 1h 32min 43s\n", + "Wall time: 1h 32min 45s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "trainer_stats = trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "e843842b-295f-4020-accf-393934732322", + "showTitle": false, + "title": "" + }, + "cellView": "form", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pCqnaKmlO1U9", + "outputId": "98f78253-86cf-4673-ff2b-923460c2b3fd" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5564.5261 seconds used for training.\n", + "92.74 minutes used for training.\n", + "Peak reserved memory = 4.152 GB.\n", + "Peak reserved memory for training = 0.519 GB.\n", + "Peak reserved memory % of max memory = 34.617 %.\n", + "Peak reserved memory for training % of max memory = 4.327 %.\n" + ] + } + ], + "source": [ + "# @title Show final memory and time stats\n", + "used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "used_memory_for_lora = round(used_memory - start_gpu_memory, 3)\n", + "used_percentage = round(used_memory / max_memory * 100, 3)\n", + "lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)\n", + "print(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\n", + "print(\n", + " f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\"\n", + ")\n", + "print(f\"Peak reserved memory = {used_memory} GB.\")\n", + "print(f\"Peak reserved memory for training = {used_memory_for_lora} GB.\")\n", + "print(f\"Peak reserved memory % of max memory = {used_percentage} %.\")\n", + "print(f\"Peak reserved memory for training % of max memory = {lora_percentage} %.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "72e8aeca-cd4c-44ee-82cd-04fa6728b40a", + "showTitle": false, + "title": "" + } + }, + "source": [ + "\n", + "### Inference\n", + "Let's run the model! You can change the instruction and input - leave the output blank!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3a76b619-1a84-4852-9be7-0f9b2bfa4c05", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "<|im_start|>system\n", + "You are an expert in translating Chinese into English.<|im_end|>\n", + "<|im_start|>user\n", + "Translate from Chinese to English.\n", + "老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "----------------------------------------\n", + "<|im_start|>system\n", + "You are an expert in translating Chinese into English.<|im_end|>\n", + "<|im_start|>user\n", + "Translate from Chinese to English.\n", + "老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。<|im_end|>\n", + "<|im_start|>assistant\n", + "Old Geng raised the pistol, squinted one eye, squeezed the trigger, and let a shower of jumbo pigeons drop down from skyward, coursing through the willows as though carried on silkworm tails, tossing tin cans in the air as they fell.<|im_end|>\n", + "CPU times: user 3.71 s, sys: 352 ms, total: 4.07 s\n", + "Wall time: 4.04 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "prompt1 = datasets[\"test\"][\"prompt\"][0]\n", + "print(prompt1)\n", + "print(\"--\" * 20)\n", + "test_model(model, tokenizer, prompt1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "55d0f54b-a9fc-4eb5-970e-0e9c118619bf", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Old Geng picked up his shotgun, squinted, and pulled the trigger. Two sparrows crashed to the ground like hailstones as shotgun pellets tore noisily through the branches.\n" + ] + } + ], + "source": [ + "print(datasets[\"test\"][\"english\"][0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1aa24900-40c4-45de-b8af-3d1da7070ff7", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 1133/1133 [34:09<00:00, 1.81s/it] " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 30min 48s, sys: 3min 21s, total: 34min 10s\n", + "Wall time: 34min 9s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "predictions = eval_model(model, tokenizer, datasets[\"test\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "68c4520d-e356-491c-9aa9-ecc57316d177", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n" + ] + }, + { + "data": { + "text/plain": [ + "{'accuracy': 0.00264783759929391,\n", + " 'correct_ids': [147, 170, 194],\n", + " 'meteor': {'meteor': 0.35503843183028994},\n", + " 'bleu_scores': {'bleu': 0.09734851870184895,\n", + " 'precisions': [0.38486636126948554,\n", + " 0.12903115371448134,\n", + " 0.05879839025606325,\n", + " 0.030757244091566802],\n", + " 'brevity_penalty': 1.0,\n", + " 'length_ratio': 1.0050679032792316,\n", + " 'translation_length': 30343,\n", + " 'reference_length': 30190},\n", + " 'rouge_scores': {'rouge1': 0.3809259470501297,\n", + " 'rouge2': 0.1543849804952549,\n", + " 'rougeL': 0.32312000381943484,\n", + " 'rougeLsum': 0.32320284655253784}}" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "calc_metrics(datasets[\"test\"][\"english\"], predictions, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "580351cd-ed04-47eb-82d7-d9fdda0dbeea", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " index chinese \\\n", + "0 0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... \n", + "\n", + " english \\\n", + "0 Old Geng picked up his shotgun, squinted, and ... \n", + "\n", + " unsloth/Qwen2-0.5B-Instruct(finetuned) \\\n", + "0 Old Geng lifted his rifle and narrowed his eye... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct \\\n", + "0 Old Geng took up his gun, squinted one of its ... \n", + "\n", + " unsloth/Qwen2-1.5B-Instruct(finetuned) \n", + "0 Old Geng raised the rifle, squeezed one tiny t... \n" + ] + } + ], + "source": [ + "from llm_toolkit.translation_engine import save_results\n", + "\n", + "save_results(model_name + \"(finetuned)\", \"results/mac-results.csv\", datasets[\"test\"], predictions, debug=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c510db20-d3c0-4fcc-a5db-ccda6b022f68", + "showTitle": false, + "title": "" + }, + "id": "uMuVrWbjAzhc" + }, + "source": [ + "\n", + "### Saving, uploading finetuned models\n", + "To save the final model as LoRA adapters, either use Huggingface's `push_to_hub` for an online save or `save_pretrained` for a local save." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "5c8ebb0f-88a4-4fd6-ba0d-d3fe2ebcca51", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "def save_model(model, tokenizer, save_method, publish=True):\n", + " model.save_pretrained_merged(\n", + " local_model + save_method,\n", + " tokenizer,\n", + " save_method=save_method,\n", + " )\n", + "\n", + " if publish:\n", + " model.push_to_hub_merged(\n", + " hub_model + save_method,\n", + " tokenizer,\n", + " save_method=save_method,\n", + " token=token,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "d8315f14-d351-42e6-8215-be7e39033e02", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... This might take 10 minutes for Llama-7b... Done.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving 4bit Bitsandbytes model. Please wait...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4888e1237402445b809b3b4bbab4ac25", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "README.md: 0%| | 0.00/575 [00:00\n", + " \n", + " \n", + " Support our work if you can! Thanks!\n", + "" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": {}, + "notebookName": "06_MAC_+_Qwen2-1.5B_Unsloth_train", + "widgets": {} + }, + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "036fc5746f43416db18c19ad8fd36677": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "06e806c82c7b4cbea31c5358dd9c3434": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "087b76a8b7514269b1f0ab29b062e444": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a069d2ab23824f29aa320ac256e2cfe9", + "placeholder": "​", + "style": "IPY_MODEL_06e806c82c7b4cbea31c5358dd9c3434", + "value": "Map (num_proc=2): 100%" + } + }, + "09b76013aa9e45efb6deb23a7a0d0925": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dea41c5260884aa6879b5e1d1697b14f", + "placeholder": "​", + "style": "IPY_MODEL_89965917796a4f81b899fdc7685f33df", + "value": "config.json: 100%" + } + }, + "0a92c56bfa134ef583220d7ef0b13e17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c34be936c8145d3ab41282f30a70713": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f8b6bfe16894500838793f2491d403f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "177c78fce95d4b4ab33057c5a048d693": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1f44c9ce1adf470cbb19784493ed209f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c34be936c8145d3ab41282f30a70713", + "placeholder": "​", + "style": "IPY_MODEL_0a92c56bfa134ef583220d7ef0b13e17", + "value": "model.safetensors: 100%" + } + }, + "201b59ccd9f845e197029b57e424aefc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2157f01726d748f8a9ae4a00664430da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21db8a77b00d4a4e82fdfa608657531f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26e4202cca81496a90d15a0dd4ca9cf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba90fdb8822d47dab7ba203bee297f37", + "IPY_MODEL_61560ff6a36b44f4a9dfdae5c52791d4", + "IPY_MODEL_95fbe66647904c06a20f640630d6dc0e" + ], + "layout": "IPY_MODEL_57182a263d324a3dbf1471c74290a0d5" + } + }, + "27155728b6b84cb199c91c940095d0a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6b91feeed5464877991ac2c207aebe7c", + "IPY_MODEL_cca8113c54c0495daedce1327bf9c68b", + "IPY_MODEL_2e63a29e2f7247bba5beede9a568c99f" + ], + "layout": "IPY_MODEL_5c9d781c28944f3eb86e2a6d44efdf18" + } + }, + "271ddaa553a042d09b6db7b450643d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a58d04b428c46f4b3dbadd3bc6cd529": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d18ddf6482c4d97829ac0e5a7b9868f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "IPY_MODEL_f2df530d22c74977b249dd9fb5f4829b", + "IPY_MODEL_89b2ef0dbfea47ab8e6f8d659e3351d1" + ], + "layout": "IPY_MODEL_3056b148aa9f4e6e8aa3b61d26886255" + } + }, + "2e5087c76f98437cb5dc729230358cba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e63a29e2f7247bba5beede9a568c99f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b993eaec6b224440bf80c0958c6fb536", + "placeholder": "​", + "style": "IPY_MODEL_de868e26e7154f62aa86223a539ad421", + "value": " 464/464 [00:00<00:00, 27.1kB/s]" + } + }, + "2f6c70dd266c4816bfad3fd3d192929a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "30307300bc4e4baf96560e30969a82b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e36a3f9eff0e4cf68834d66b0213ae96", + "placeholder": "​", + "style": "IPY_MODEL_a0037bdccf254159becde630bee3d1db", + "value": "generation_config.json: 100%" + } + }, + "3056b148aa9f4e6e8aa3b61d26886255": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30cdc32298134cb0be4d41615b9e5774": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3572201bd4d74a58b7a665f9bdfdcdba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35b0e8c26d6640e9bd0ed7b242a423d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e5087c76f98437cb5dc729230358cba", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_036fc5746f43416db18c19ad8fd36677", + "value": 51760 + } + }, + "36166c7bcb854b34aca1f41a5d6ea50b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "370692d819df41828b48c4ad446f977b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "39b29a75374b45c0a22506010be2b84e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_30cdc32298134cb0be4d41615b9e5774", + "max": 1179, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_47928317548c454bba6358ab132e8dee", + "value": 1179 + } + }, + "3cf2dd993b5e4d3daecf61e4bab5a404": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_087b76a8b7514269b1f0ab29b062e444", + "IPY_MODEL_35b0e8c26d6640e9bd0ed7b242a423d8", + "IPY_MODEL_54ad89e05fd74576b9b8b5b5a10eaf8d" + ], + "layout": "IPY_MODEL_a41dc44766444a998bec2d777f249d23" + } + }, + "43dec2ede91341f5af60eb522e18e984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4463edd481c1467f914c7dcd6c6e6ffc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47928317548c454bba6358ab132e8dee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "49277aeeac16434a865a4d12308b1abc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4ae7e449e4ea4c729b5f34607c18ebae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b2061b8a73c43ffb0c2f83daf0d0183": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c4c88d4c701450692fa0f6b0c5764b0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c666f4ace3943f8b80ecd20e7503236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ccedf0d93094e63b57a0f8a434fba06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4463edd481c1467f914c7dcd6c6e6ffc", + "max": 44307561, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6d3b9a05db0b4dadb638c686faa0c40a", + "value": 44307561 + } + }, + "4dcf6ff672d24983a1877a8431709aa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5807d5fb827d490fb3bc698f801ffff5", + "placeholder": "​", + "style": "IPY_MODEL_c4f2b06a82fd4987b8b659524a7b503b", + "value": "Generating train split: 100%" + } + }, + "4ea63adfce694725bdba878aef709dd3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5234566b1bfc4655b8d582ea5b46ed9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "54ad89e05fd74576b9b8b5b5a10eaf8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdb1941405ed4e4aa06019933892deb3", + "placeholder": "​", + "style": "IPY_MODEL_668d5377ca56426a99753867e6e24862", + "value": " 51760/51760 [01:02<00:00, 1131.51 examples/s]" + } + }, + "56aee4853b7740e6a977254f5d1fa66d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "57182a263d324a3dbf1471c74290a0d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5807d5fb827d490fb3bc698f801ffff5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c9d781c28944f3eb86e2a6d44efdf18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f40db8173dd4d76b6ef5ed6d9ec8b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61560ff6a36b44f4a9dfdae5c52791d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db19fc8d37db4e45a5790a876836d8c4", + "max": 11610, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36166c7bcb854b34aca1f41a5d6ea50b", + "value": 11610 + } + }, + "6578fd7acdb54c4c93528ea431fd0144": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_370692d819df41828b48c4ad446f977b", + "placeholder": "​", + "style": "IPY_MODEL_a0bf9160eb2647409b3200270914b90f", + "value": " 50.6k/50.6k [00:00<00:00, 2.71MB/s]" + } + }, + "668d5377ca56426a99753867e6e24862": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697f027529b54ee9956bae78a11e0611": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69ac12aec0714318bf2c83d4f4e745f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b2012c3f88547af8884a9ea90e3164b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_938f45f1b3e24118b815d96ae34ba86a", + "placeholder": "​", + "style": "IPY_MODEL_9367047a800747f79c6b225d92397846", + "value": " 44.3M/44.3M [00:01<00:00, 31.0MB/s]" + } + }, + "6b91feeed5464877991ac2c207aebe7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b2061b8a73c43ffb0c2f83daf0d0183", + "placeholder": "​", + "style": "IPY_MODEL_69ac12aec0714318bf2c83d4f4e745f5", + "value": "special_tokens_map.json: 100%" + } + }, + "6d3b9a05db0b4dadb638c686faa0c40a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6dbbedeca9314e66ae50e44ffa31a414": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6e34619b45934040b6092e6fb01ea7fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71ce208e20d6483abb9ed923510c86d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d69dc491b3ab44d7852b21873ed7bb7f", + "placeholder": "​", + "style": "IPY_MODEL_f401d53bf28e44eb906bce6c05412662", + "value": " 51760/51760 [00:01<00:00, 45512.81 examples/s]" + } + }, + "7358cdad832342c983e31efb8754ab78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e352a3404f4c7dad0737f57d29e92f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_988a0e8c1f89446086858da0a891a79c", + "IPY_MODEL_4ccedf0d93094e63b57a0f8a434fba06", + "IPY_MODEL_6b2012c3f88547af8884a9ea90e3164b" + ], + "layout": "IPY_MODEL_7e29cb8dd4df4d5b94407cd8fd3f2011" + } + }, + "74501720ac7e4dbb911a4a99b3633bc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "78e5400bff924a92a4cc61c4ff18b182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9b313fd861948f5aba25b24b1518d30", + "placeholder": "​", + "style": "IPY_MODEL_4c666f4ace3943f8b80ecd20e7503236", + "value": " 1.18k/1.18k [00:00<00:00, 31.3kB/s]" + } + }, + "7975adbc2ec5489ea7fa0167e620d85c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e34619b45934040b6092e6fb01ea7fe", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_271ddaa553a042d09b6db7b450643d8f", + "value": 51760 + } + }, + "7e29cb8dd4df4d5b94407cd8fd3f2011": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "810ff6c0e17d4fa09a30fef27eacff90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89965917796a4f81b899fdc7685f33df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "89b2ef0dbfea47ab8e6f8d659e3351d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8908fa0df3743ecb9d12983a739104f", + "placeholder": "​", + "style": "IPY_MODEL_177c78fce95d4b4ab33057c5a048d693", + "value": " 9.09M/9.09M [00:00<00:00, 32.6MB/s]" + } + }, + "8b3505352a5a42bf910428c40ce40465": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_49277aeeac16434a865a4d12308b1abc", + "placeholder": "​", + "style": "IPY_MODEL_2157f01726d748f8a9ae4a00664430da", + "value": " 5.70G/5.70G [01:02<00:00, 30.1MB/s]" + } + }, + "8fc142b628fb40568730234de1cafde2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ae7e449e4ea4c729b5f34607c18ebae", + "max": 172, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3572201bd4d74a58b7a665f9bdfdcdba", + "value": 172 + } + }, + "9367047a800747f79c6b225d92397846": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "938f45f1b3e24118b815d96ae34ba86a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "95fbe66647904c06a20f640630d6dc0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b0a370dc20654b279b9680692e34418e", + "placeholder": "​", + "style": "IPY_MODEL_cfeb365ddf7548d58b2557f22737fcf5", + "value": " 11.6k/11.6k [00:00<00:00, 716kB/s]" + } + }, + "988a0e8c1f89446086858da0a891a79c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ad2be500fc164c0f86f33e914ef8e6a0", + "placeholder": "​", + "style": "IPY_MODEL_5234566b1bfc4655b8d582ea5b46ed9f", + "value": "Downloading data: 100%" + } + }, + "98c58f23f4d549518832cb2d18f796e8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_09b76013aa9e45efb6deb23a7a0d0925", + "IPY_MODEL_39b29a75374b45c0a22506010be2b84e", + "IPY_MODEL_78e5400bff924a92a4cc61c4ff18b182" + ], + "layout": "IPY_MODEL_2a58d04b428c46f4b3dbadd3bc6cd529" + } + }, + "99fdbb0300c14c139d1937c646f0cfe7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7358cdad832342c983e31efb8754ab78", + "placeholder": "​", + "style": "IPY_MODEL_e9adf418296e436fb48bb9f78885598b", + "value": " 51760/51760 [00:01<00:00, 38665.95 examples/s]" + } + }, + "9f679ad3ec7f4fe8ad0510ffb57bc2ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ea63adfce694725bdba878aef709dd3", + "placeholder": "​", + "style": "IPY_MODEL_74501720ac7e4dbb911a4a99b3633bc6", + "value": "tokenizer.json: 100%" + } + }, + "a0037bdccf254159becde630bee3d1db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a069d2ab23824f29aa320ac256e2cfe9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0bf9160eb2647409b3200270914b90f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a41dc44766444a998bec2d777f249d23": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8464a4c711e4e00aafdfc919b60d07e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fb995c740590427b882572c81d4e848c", + "placeholder": "​", + "style": "IPY_MODEL_201b59ccd9f845e197029b57e424aefc", + "value": " 172/172 [00:00<00:00, 12.0kB/s]" + } + }, + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad2be500fc164c0f86f33e914ef8e6a0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b0240cd9a4554b29ae11f8051984a1c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_edaf890370314a218f138015faa0b05d", + "placeholder": "​", + "style": "IPY_MODEL_697f027529b54ee9956bae78a11e0611", + "value": "Map: 100%" + } + }, + "b0a370dc20654b279b9680692e34418e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b518dcee69074b87be73957cd810e7ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d891f8d0b1fc462f8008d02bb2a15692", + "placeholder": "​", + "style": "IPY_MODEL_cced8fd7e998472794f3f3e3018956a5", + "value": "tokenizer_config.json: 100%" + } + }, + "b8908fa0df3743ecb9d12983a739104f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b993eaec6b224440bf80c0958c6fb536": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9b313fd861948f5aba25b24b1518d30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba90fdb8822d47dab7ba203bee297f37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f8b6bfe16894500838793f2491d403f", + "placeholder": "​", + "style": "IPY_MODEL_bb19f6c747754682a514373a3a0535ba", + "value": "Downloading readme: 100%" + } + }, + "bb19f6c747754682a514373a3a0535ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc883d4cf13e4f8b8a4fe5f410cb6efd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9159e03e61f4f56978ece9c3bca49b2", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_810ff6c0e17d4fa09a30fef27eacff90", + "value": 51760 + } + }, + "c161d94df0f04feba9542237e0856c22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c22f71b1f85843209d7e5321506b9cb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f44c9ce1adf470cbb19784493ed209f", + "IPY_MODEL_f1addc4479d849879e743cf9089e6540", + "IPY_MODEL_8b3505352a5a42bf910428c40ce40465" + ], + "layout": "IPY_MODEL_4c4c88d4c701450692fa0f6b0c5764b0" + } + }, + "c4f2b06a82fd4987b8b659524a7b503b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cca8113c54c0495daedce1327bf9c68b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e02f9b7849c64531835eb77b860d1c93", + "max": 464, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_56aee4853b7740e6a977254f5d1fa66d", + "value": 464 + } + }, + "cced8fd7e998472794f3f3e3018956a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cf245afeb1c04f29a24d291608c3d157": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b518dcee69074b87be73957cd810e7ed", + "IPY_MODEL_e29104486d594b2992d7285e0ef77371", + "IPY_MODEL_6578fd7acdb54c4c93528ea431fd0144" + ], + "layout": "IPY_MODEL_d35db8148a354c56aaac56dbae22536f" + } + }, + "cfe8cae0e22b495bafa221a63d13b283": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfeb365ddf7548d58b2557f22737fcf5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1b47d39450d4019ae85c9b2f943eeaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4dcf6ff672d24983a1877a8431709aa9", + "IPY_MODEL_7975adbc2ec5489ea7fa0167e620d85c", + "IPY_MODEL_71ce208e20d6483abb9ed923510c86d7" + ], + "layout": "IPY_MODEL_cfe8cae0e22b495bafa221a63d13b283" + } + }, + "d35db8148a354c56aaac56dbae22536f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69dc491b3ab44d7852b21873ed7bb7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d891f8d0b1fc462f8008d02bb2a15692": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8e5318cead340c4adbeaccc05d39225": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "daf4cd890b35422683d22fd30bc71e83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b0240cd9a4554b29ae11f8051984a1c6", + "IPY_MODEL_bc883d4cf13e4f8b8a4fe5f410cb6efd", + "IPY_MODEL_99fdbb0300c14c139d1937c646f0cfe7" + ], + "layout": "IPY_MODEL_c161d94df0f04feba9542237e0856c22" + } + }, + "db19fc8d37db4e45a5790a876836d8c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de868e26e7154f62aa86223a539ad421": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dea41c5260884aa6879b5e1d1697b14f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e02f9b7849c64531835eb77b860d1c93": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e29104486d594b2992d7285e0ef77371": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "max": 50641, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2f6c70dd266c4816bfad3fd3d192929a", + "value": 50641 + } + }, + "e36a3f9eff0e4cf68834d66b0213ae96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9159e03e61f4f56978ece9c3bca49b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9adf418296e436fb48bb9f78885598b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "edaf890370314a218f138015faa0b05d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f1addc4479d849879e743cf9089e6540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43dec2ede91341f5af60eb522e18e984", + "max": 5702746405, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d8e5318cead340c4adbeaccc05d39225", + "value": 5702746405 + } + }, + "f2df530d22c74977b249dd9fb5f4829b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_21db8a77b00d4a4e82fdfa608657531f", + "max": 9085698, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6dbbedeca9314e66ae50e44ffa31a414", + "value": 9085698 + } + }, + "f401d53bf28e44eb906bce6c05412662": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fb995c740590427b882572c81d4e848c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fce7a61c25ec4390af43d92b7c473a45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_30307300bc4e4baf96560e30969a82b6", + "IPY_MODEL_8fc142b628fb40568730234de1cafde2", + "IPY_MODEL_a8464a4c711e4e00aafdfc919b60d07e" + ], + "layout": "IPY_MODEL_5f40db8173dd4d76b6ef5ed6d9ec8b6e" + } + }, + "fdb1941405ed4e4aa06019933892deb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/03_Qwen2-0.5B_1.5B-4bit.ipynb b/notebooks/03_Qwen2-0.5B_1.5B-4bit.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..ab51e29ab2d8740a229405cd00b32af7f42dd7aa --- /dev/null +++ b/notebooks/03_Qwen2-0.5B_1.5B-4bit.ipynb @@ -0,0 +1,5098 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "0ea8b46b-839b-445b-8043-ccdf4e920ace", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6d394937-6c99-4a7c-9d32-7600a280032f", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /home/inflaton/code/projects/courses/novel-translation\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9f67ec60-2f24-411c-84eb-0dd664b44775", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f1597656-8042-4878-9d3b-9ebfb8dd86dc", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('unsloth/Qwen2-0.5B-Instruct-bnb-4bit',\n", + " True,\n", + " None,\n", + " None,\n", + " 2048,\n", + " 10,\n", + " None,\n", + " 'datasets/mac/mac.tsv',\n", + " 'results/mac-results.csv')" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "model_name = os.getenv(\"MODEL_NAME\")\n", + "token = os.getenv(\"HF_TOKEN\") or None\n", + "load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n", + "local_model = os.getenv(\"LOCAL_MODEL\")\n", + "hub_model = os.getenv(\"HUB_MODEL\")\n", + "num_train_epochs = int(os.getenv(\"NUM_TRAIN_EPOCHS\") or 0)\n", + "data_path = os.getenv(\"DATA_PATH\")\n", + "results_path = os.getenv(\"RESULTS_PATH\")\n", + "\n", + "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n", + "dtype = (\n", + " None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n", + ")\n", + "\n", + "model_name, load_in_4bit, local_model, hub_model, max_seq_length, num_train_epochs, dtype, data_path, results_path" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fri Jun 21 08:19:33 2024 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 545.23.07 Driver Version: 546.12 CUDA Version: 12.3 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 NVIDIA GeForce RTX 4080 ... On | 00000000:01:00.0 On | N/A |\n", + "| N/A 52C P8 5W / 150W | 1156MiB / 12282MiB | 20% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "| No running processes found |\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "!nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/novel-translation\n", + "Tuning unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/novel-translation/translation_engine.py\n", + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n", + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit True 2048 10 None datasets/mac/mac.tsv results/mac-results.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.633 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "100%|███████████████████████████████████████| 1133/1133 [41:53<00:00, 2.22s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Teng raised his gun and looked up at a pai...\n", + "\n", + "[1 rows x 3 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.023 GB of memory reserved.\n", + "Unsloth 2024.6 patched 24 layers with 0 QKV layers, 24 O layers and 24 MLP layers.\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.023 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 8,798,208\n", + "{'loss': 1.9401, 'grad_norm': 0.9639493823051453, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.7762, 'grad_norm': 0.807873547077179, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.7147, 'grad_norm': 0.9143684506416321, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.7154, 'grad_norm': 0.753039538860321, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.6864, 'grad_norm': 0.8280349373817444, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.6079, 'grad_norm': 0.8070634603500366, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.492, 'grad_norm': 0.906767725944519, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.5407, 'grad_norm': 0.9825711250305176, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.4724, 'grad_norm': 1.0809648036956787, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.4687, 'grad_norm': 0.9906221032142639, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.5277, 'grad_norm': 1.0263370275497437, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.3495, 'grad_norm': 1.1578023433685303, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 1.2153, 'grad_norm': 1.1819669008255005, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.2577, 'grad_norm': 1.3301641941070557, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.2556, 'grad_norm': 1.3167439699172974, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.2661, 'grad_norm': 1.1702375411987305, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.235, 'grad_norm': 1.1984246969223022, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.9804, 'grad_norm': 1.4932502508163452, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.9925, 'grad_norm': 1.5164968967437744, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.9981, 'grad_norm': 1.529332160949707, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.9808, 'grad_norm': 1.5573309659957886, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 1.0228, 'grad_norm': 1.630372166633606, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.9318, 'grad_norm': 1.5458662509918213, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.7395, 'grad_norm': 1.7877501249313354, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.7408, 'grad_norm': 1.831641435623169, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.7743, 'grad_norm': 2.053892135620117, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.7734, 'grad_norm': 2.142253875732422, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.7944, 'grad_norm': 2.088791847229004, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.6303, 'grad_norm': 1.8649368286132812, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.5557, 'grad_norm': 1.5683393478393555, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.56, 'grad_norm': 1.6868833303451538, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.5884, 'grad_norm': 1.896024465560913, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.5797, 'grad_norm': 2.199939489364624, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.5782, 'grad_norm': 1.5285065174102783, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.3976, 'grad_norm': 2.0019567012786865, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.411, 'grad_norm': 1.8241479396820068, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.4288, 'grad_norm': 2.22087025642395, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.4213, 'grad_norm': 1.6254255771636963, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.4276, 'grad_norm': 1.4894341230392456, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.3844, 'grad_norm': 1.7346247434616089, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.2934, 'grad_norm': 1.558210015296936, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.3127, 'grad_norm': 1.2885392904281616, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.3137, 'grad_norm': 1.7340631484985352, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.3042, 'grad_norm': 1.8956586122512817, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.3177, 'grad_norm': 1.6758400201797485, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.2466, 'grad_norm': 1.4491084814071655, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.226, 'grad_norm': 1.810203194618225, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.2294, 'grad_norm': 1.1841636896133423, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.2356, 'grad_norm': 1.1479746103286743, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.2364, 'grad_norm': 1.9742052555084229, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.2392, 'grad_norm': 1.2130563259124756, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.181, 'grad_norm': 1.0886257886886597, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1906, 'grad_norm': 1.0989885330200195, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1877, 'grad_norm': 1.1791963577270508, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.1881, 'grad_norm': 1.712857961654663, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1891, 'grad_norm': 0.9620760083198547, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 5102.7155, 'train_samples_per_second': 8.874, 'train_steps_per_second': 1.109, 'train_loss': 0.7989002002001652, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [1:25:02<00:00, 1.11it/s]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5102.7155 seconds used for training.\n", + "85.05 minutes used for training.\n", + "Peak reserved memory = 3.023 GB.\n", + "Peak reserved memory for training = 0.0 GB.\n", + "Peak reserved memory % of max memory = 25.204 %.\n", + "Peak reserved memory for training % of max memory = 0.0 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "100%|███████████████████████████████████████| 1133/1133 [36:52<00:00, 1.95s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his rifle, squinted his eyes, ...\n", + "\n", + "[1 rows x 4 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.023 GB of memory reserved.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... This might take 10 minutes for Llama-7b... Done.\n", + "make: Entering directory '/home/inflaton/code/projects/courses/novel-translation/llama.cpp'\n", + "I ccache not found. Consider installing it for faster compilation.\n", + "I llama.cpp build info: \n", + "I UNAME_S: Linux\n", + "I UNAME_P: x86_64\n", + "I UNAME_M: x86_64\n", + "I CFLAGS: -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_OPENMP -DGGML_USE_LLAMAFILE -std=c11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -fopenmp -Wdouble-promotion \n", + "I CXXFLAGS: -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread -fopenmp -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_OPENMP -DGGML_USE_LLAMAFILE \n", + "I NVCCFLAGS: -std=c++11 -O3 \n", + "I LDFLAGS: \n", + "I CC: cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n", + "I CXX: c++ (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n", + "\n", + "rm -vrf *.o tests/*.o *.so *.a *.dll common/build-info.cpp *.dot *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report libllava.a llama-baby-llama llama-batched llama-batched-bench llama-bench llama-benchmark-matmult llama-cli llama-convert-llama2c-to-ggml llama-embedding llama-eval-callback llama-export-lora llama-finetune llama-gbnf-validator llama-gguf llama-gguf-split llama-gritlm llama-imatrix llama-infill llama-llava-cli llama-lookahead llama-lookup llama-lookup-create llama-lookup-merge llama-lookup-stats llama-parallel llama-passkey llama-perplexity llama-q8dot llama-quantize llama-quantize-stats llama-retrieval llama-save-load-state llama-server llama-simple llama-speculative llama-tokenize llama-train-text-from-scratch llama-vdot llama-cvector-generator tests/test-c.o tests/test-autorelease tests/test-backend-ops tests/test-double-float tests/test-grad0 tests/test-grammar-integration tests/test-grammar-parser tests/test-json-schema-to-grammar tests/test-llama-grammar tests/test-model-load-cancel tests/test-opt tests/test-quantize-fns tests/test-quantize-perf tests/test-rope tests/test-sampling tests/test-tokenizer-0 tests/test-tokenizer-1-bpe tests/test-tokenizer-1-spm\n", + "rm -vrf ggml-cuda/*.o\n", + "rm -vrf ggml-cuda/template-instances/*.o\n", + "find examples pocs -type f -name \"*.o\" -delete\n", + "make: Leaving directory '/home/inflaton/code/projects/courses/novel-translation/llama.cpp'\n", + "Unsloth: Merging 4bit and LoRA weights to 16bit...\n", + "Unsloth: Will use up to 30.26 out of 47.05 RAM for saving.\n", + "100%|███████████████████████████████████████████| 24/24 [00:00<00:00, 43.09it/s]\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... This might take 5 minutes for Llama-7b...\n", + "Done.\n", + "Unsloth: Converting qwen2 model. Can use fast conversion = False.\n", + "==((====))== Unsloth: Conversion from QLoRA to GGUF information\n", + " \\\\ /| [0] Installing llama.cpp will take 3 minutes.\n", + "O^O/ \\_/ \\ [1] Converting HF to GUUF 16bits will take 3 minutes.\n", + "\\ / [2] Converting GGUF 16bits to ['q5_k_m'] will take 10 minutes each.\n", + " \"-____-\" In total, you will have to wait at least 16 minutes.\n", + "\n", + "Unsloth: [0] Installing llama.cpp. This will take 3 minutes...\n", + "Unsloth: [1] Converting model at models/Qwen2-0.5B-Instruct-bnb-4bit-MAC-q5_k_m into bf16 GGUF format.\n", + "The output location will be ./models/Qwen2-0.5B-Instruct-bnb-4bit-MAC-q5_k_m/unsloth.BF16.gguf\n", + "This will take 3 minutes...\n", + "INFO:hf-to-gguf:Loading model: Qwen2-0.5B-Instruct-bnb-4bit-MAC-q5_k_m\n", + "INFO:gguf.gguf_writer:gguf: This GGUF file is for Little Endian only\n", + "INFO:hf-to-gguf:Set model parameters\n", + "INFO:hf-to-gguf:gguf: context length = 32768\n", + "INFO:hf-to-gguf:gguf: embedding length = 896\n", + "INFO:hf-to-gguf:gguf: feed forward length = 4864\n", + "INFO:hf-to-gguf:gguf: head count = 14\n", + "INFO:hf-to-gguf:gguf: key-value head count = 2\n", + "INFO:hf-to-gguf:gguf: rope theta = 1000000.0\n", + "INFO:hf-to-gguf:gguf: rms norm epsilon = 1e-06\n", + "INFO:hf-to-gguf:gguf: file type = 32\n", + "INFO:hf-to-gguf:Set model tokenizer\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "INFO:gguf.vocab:Adding 151387 merge(s).\n", + "INFO:gguf.vocab:Setting special token type eos to 151645\n", + "INFO:gguf.vocab:Setting special token type pad to 151643\n", + "INFO:gguf.vocab:Setting special token type bos to 151643\n", + "INFO:gguf.vocab:Setting chat_template to {% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\n", + "You are a helpful assistant.<|im_end|>\n", + "' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n", + "' + message['content'] + '<|im_end|>' + '\n", + "'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n", + "' }}{% endif %}\n", + "INFO:hf-to-gguf:Exporting model to 'models/Qwen2-0.5B-Instruct-bnb-4bit-MAC-q5_k_m/unsloth.BF16.gguf'\n", + "INFO:hf-to-gguf:gguf: loading model part 'model.safetensors'\n", + "INFO:hf-to-gguf:token_embd.weight, torch.bfloat16 --> BF16, shape = {896, 151936}\n", + "INFO:hf-to-gguf:blk.0.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.0.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.0.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.0.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.0.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.0.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.0.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.0.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.0.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.0.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.0.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.0.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.1.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.1.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.1.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.1.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.1.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.1.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.1.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.1.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.1.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.1.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.1.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.1.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.10.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.10.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.10.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.10.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.10.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.10.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.10.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.10.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.10.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.10.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.10.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.10.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.11.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.11.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.11.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.11.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.11.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.11.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.11.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.11.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.11.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.11.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.11.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.11.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.12.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.12.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.12.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.12.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.12.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.12.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.12.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.12.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.12.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.12.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.12.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.12.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.13.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.13.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.13.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.13.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.13.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.13.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.13.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.13.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.13.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.13.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.13.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.13.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.14.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.14.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.14.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.14.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.14.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.14.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.14.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.14.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.14.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.14.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.14.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.14.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.15.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.15.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.15.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.15.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.15.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.15.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.15.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.15.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.15.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.15.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.15.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.15.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.16.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.16.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.16.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.16.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.16.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.16.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.16.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.16.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.16.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.16.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.16.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.16.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.17.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.17.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.17.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.17.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.17.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.17.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.17.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.17.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.17.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.17.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.17.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.17.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.18.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.18.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.18.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.18.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.18.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.18.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.18.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.18.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.18.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.18.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.18.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.18.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.19.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.19.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.19.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.19.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.19.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.19.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.19.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.19.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.19.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.19.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.19.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.19.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.2.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.2.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.2.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.2.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.2.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.2.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.2.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.2.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.2.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.2.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.2.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.2.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.20.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.20.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.20.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.20.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.20.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.20.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.20.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.20.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.20.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.20.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.20.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.20.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.21.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.21.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.21.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.21.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.21.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.21.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.21.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.21.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.21.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.21.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.21.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.21.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.22.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.22.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.22.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.22.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.22.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.22.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.22.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.22.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.22.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.22.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.22.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.22.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.23.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.23.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.23.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.23.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.23.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.23.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.23.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.23.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.23.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.23.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.23.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.23.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.3.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.3.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.3.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.3.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.3.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.3.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.3.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.3.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.3.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.3.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.3.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.3.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.4.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.4.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.4.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.4.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.4.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.4.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.4.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.4.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.4.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.4.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.4.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.4.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.5.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.5.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.5.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.5.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.5.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.5.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.5.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.5.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.5.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.5.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.5.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.5.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.6.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.6.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.6.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.6.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.6.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.6.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.6.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.6.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.6.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.6.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.6.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.6.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.7.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.7.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.7.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.7.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.7.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.7.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.7.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.7.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.7.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.7.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.7.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.7.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.8.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.8.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.8.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.8.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.8.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.8.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.8.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.8.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.8.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.8.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.8.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.8.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.9.attn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.9.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4864, 896}\n", + "INFO:hf-to-gguf:blk.9.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.9.ffn_up.weight, torch.bfloat16 --> BF16, shape = {896, 4864}\n", + "INFO:hf-to-gguf:blk.9.ffn_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.9.attn_k.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.9.attn_k.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:blk.9.attn_output.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.9.attn_q.bias, torch.bfloat16 --> F32, shape = {896}\n", + "INFO:hf-to-gguf:blk.9.attn_q.weight, torch.bfloat16 --> BF16, shape = {896, 896}\n", + "INFO:hf-to-gguf:blk.9.attn_v.bias, torch.bfloat16 --> F32, shape = {128}\n", + "INFO:hf-to-gguf:blk.9.attn_v.weight, torch.bfloat16 --> BF16, shape = {896, 128}\n", + "INFO:hf-to-gguf:output_norm.weight, torch.bfloat16 --> F32, shape = {896}\n", + "Writing: 0%| | 0.00/988M [00:00\n", + " main()\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/convert-hf-to-gguf.py\", line 2881, in main\n", + " model_instance.write()\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/convert-hf-to-gguf.py\", line 331, in write\n", + " self.gguf_writer.write_tensors_to_file(progress=True)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/gguf_writer.py\", line 312, in write_tensors_to_file\n", + " ti.tensor.tofile(self.fout)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/lazy.py\", line 233, in tofile\n", + " eager = LazyNumpyTensor.to_eager(self)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/lazy.py\", line 193, in to_eager\n", + " return cls._recurse_apply(t, simple_to_eager)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/lazy.py\", line 109, in _recurse_apply\n", + " return fn(o)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/lazy.py\", line 185, in simple_to_eager\n", + " lt._data = lt._func(lt._args)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/lazy.py\", line 158, in \n", + " return cls(meta=cls.eager_to_meta(res), lazy=shared_lazy, args=args, func=lambda a: fn(*a, **kwargs))\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/quants.py\", line 52, in __quantize_bf16_array\n", + " return __apply_over_grouped_rows(__compute_fp32_to_bf16, arr=n, otype=np.int16, oshape=n.shape)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/quants.py\", line 47, in __apply_over_grouped_rows\n", + " np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/quants.py\", line 47, in \n", + " np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/quants.py\", line 30, in __compute_fp32_to_bf16\n", + " n = np.where((n & 0x7fffffff) > 0x7f800000, (n & 0xffff0000) | (64 << 16), n)\n", + "OverflowError: Python integer 4294901760 out of bounds for int32\n", + "Writing: 0%| | 0.00/988M [00:00\n", + " save_model(model, tokenizer)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/translation_engine.py\", line 219, in save_model\n", + " model.save_pretrained_gguf(\n", + " File \"/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/unsloth/save.py\", line 1527, in unsloth_save_pretrained_gguf\n", + " all_file_locations = save_to_gguf(model_type, model_dtype, is_sentencepiece_model,\n", + " File \"/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/unsloth/save.py\", line 1113, in save_to_gguf\n", + " raise RuntimeError(\n", + "RuntimeError: Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n", + "You do not need to close this Python program. Run the following commands in a new terminal:\n", + "You must run this in the same folder as you're saving your model.\n", + "git clone --recursive https://github.com/ggerganov/llama.cpp\n", + "cd llama.cpp && make clean && make all -j\n", + "Once that's done, redo the quantization.\n", + "Tuning unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/novel-translation/translation_engine.py\n", + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit True 2048 10 None datasets/mac/mac.tsv results/mac-results.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.516 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "100%|███████████████████████████████████████| 1133/1133 [59:36<00:00, 3.16s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old耿拿起枪，眯着眼睛一搂扳机就响了枪，金麻雀噼里啪啦的往下掉，铁砂子在柳枝间飞溅，发出“...\n", + "\n", + "[1 rows x 5 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.945 GB of memory reserved.\n", + "Unsloth 2024.6 patched 28 layers with 0 QKV layers, 28 O layers and 28 MLP layers.\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.945 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 18,464,768\n", + "{'loss': 1.7416, 'grad_norm': 0.6486354470252991, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.568, 'grad_norm': 0.6254323124885559, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.5159, 'grad_norm': 0.6513530015945435, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.5169, 'grad_norm': 0.5732458233833313, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.4958, 'grad_norm': 0.5724458694458008, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + " 9%|███▎ | 500/5660 [12:05<2:13:34, 1.55s/it]/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/other.py:611: UserWarning: Unable to fetch remote file due to the following error (MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /unsloth/Qwen2-1.5B-Instruct-bnb-4bit/resolve/main/config.json (Caused by NameResolutionError(\": Failed to resolve \\'huggingface.co\\' ([Errno -3] Temporary failure in name resolution)\"))'), '(Request ID: 73fef4ae-41d2-4b61-b3af-92f4996c5ae6)') - silently ignoring the lookup for the file config.json in unsloth/Qwen2-1.5B-Instruct-bnb-4bit.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in unsloth/Qwen2-1.5B-Instruct-bnb-4bit - will assume that the vocabulary was not modified.\n", + " warnings.warn(\n", + "{'loss': 1.4181, 'grad_norm': 0.6020762324333191, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.2977, 'grad_norm': 0.7245773077011108, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.3392, 'grad_norm': 0.6871509552001953, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.2784, 'grad_norm': 0.89788419008255, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.2796, 'grad_norm': 0.8698562383651733, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + " 18%|██████▌ | 1000/5660 [25:03<1:52:33, 1.45s/it]/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/other.py:611: UserWarning: Unable to fetch remote file due to the following error (MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /unsloth/Qwen2-1.5B-Instruct-bnb-4bit/resolve/main/config.json (Caused by NameResolutionError(\": Failed to resolve \\'huggingface.co\\' ([Errno -3] Temporary failure in name resolution)\"))'), '(Request ID: aec2499a-0591-44e8-bbc9-1568ebca28ce)') - silently ignoring the lookup for the file config.json in unsloth/Qwen2-1.5B-Instruct-bnb-4bit.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in unsloth/Qwen2-1.5B-Instruct-bnb-4bit - will assume that the vocabulary was not modified.\n", + " warnings.warn(\n", + "{'loss': 1.3317, 'grad_norm': 0.8551518321037292, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.1279, 'grad_norm': 0.9991661310195923, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 0.9962, 'grad_norm': 1.0851796865463257, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.03, 'grad_norm': 1.223488450050354, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.0346, 'grad_norm': 1.1075948476791382, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.0356, 'grad_norm': 1.0564122200012207, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.0085, 'grad_norm': 0.8879927396774292, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.6982, 'grad_norm': 1.2948524951934814, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.7112, 'grad_norm': 1.411033034324646, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.7143, 'grad_norm': 1.2779042720794678, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.7048, 'grad_norm': 1.4070408344268799, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 0.7431, 'grad_norm': 1.2386316061019897, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.6442, 'grad_norm': 1.4571985006332397, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.4587, 'grad_norm': 1.3495371341705322, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.4463, 'grad_norm': 1.4228668212890625, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.4788, 'grad_norm': 1.4749239683151245, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.4788, 'grad_norm': 1.9813690185546875, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.4853, 'grad_norm': 1.4942362308502197, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.358, 'grad_norm': 1.398263692855835, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.2917, 'grad_norm': 1.148834228515625, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.2974, 'grad_norm': 1.8417447805404663, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.316, 'grad_norm': 1.3501039743423462, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.3081, 'grad_norm': 1.3924249410629272, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.3056, 'grad_norm': 0.7402265071868896, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.1918, 'grad_norm': 1.6596429347991943, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.2038, 'grad_norm': 1.4057574272155762, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.2068, 'grad_norm': 1.0855220556259155, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.2067, 'grad_norm': 1.2501165866851807, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.2132, 'grad_norm': 1.1509685516357422, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.1937, 'grad_norm': 0.7672592401504517, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.1486, 'grad_norm': 0.8496847152709961, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.156, 'grad_norm': 0.8369797468185425, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.1561, 'grad_norm': 0.8741617202758789, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.1574, 'grad_norm': 0.69870924949646, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.1585, 'grad_norm': 0.7949297428131104, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.1358, 'grad_norm': 0.47111940383911133, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.1299, 'grad_norm': 0.6585063934326172, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.1309, 'grad_norm': 0.4326067566871643, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.1325, 'grad_norm': 0.3893156349658966, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.1334, 'grad_norm': 0.968989908695221, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.1335, 'grad_norm': 0.48122939467430115, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.1172, 'grad_norm': 0.4522119462490082, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1181, 'grad_norm': 0.5501106977462769, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1194, 'grad_norm': 0.46891143918037415, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.12, 'grad_norm': 0.5252432823181152, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1228, 'grad_norm': 0.4517185688018799, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 8395.4379, 'train_samples_per_second': 5.393, 'train_steps_per_second': 0.674, 'train_loss': 0.5991904156789342, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [2:19:55<00:00, 1.48s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "8395.4379 seconds used for training.\n", + "139.92 minutes used for training.\n", + "Peak reserved memory = 3.945 GB.\n", + "Peak reserved memory for training = 0.0 GB.\n", + "Peak reserved memory % of max memory = 32.891 %.\n", + "Peak reserved memory for training % of max memory = 0.0 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "100%|███████████████████████████████████████| 1133/1133 [50:44<00:00, 2.69s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his pistol, squinted, and fire...\n", + "\n", + "[1 rows x 6 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.945 GB of memory reserved.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... This might take 10 minutes for Llama-7b... Done.\n", + "Unsloth: Merging 4bit and LoRA weights to 16bit...\n", + "Unsloth: Will use up to 29.87 out of 47.05 RAM for saving.\n", + "100%|███████████████████████████████████████████| 28/28 [00:00<00:00, 42.85it/s]\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... This might take 5 minutes for Llama-7b...\n", + "Done.\n", + "Unsloth: Converting qwen2 model. Can use fast conversion = False.\n", + "==((====))== Unsloth: Conversion from QLoRA to GGUF information\n", + " \\\\ /| [0] Installing llama.cpp will take 3 minutes.\n", + "O^O/ \\_/ \\ [1] Converting HF to GUUF 16bits will take 3 minutes.\n", + "\\ / [2] Converting GGUF 16bits to ['q5_k_m'] will take 10 minutes each.\n", + " \"-____-\" In total, you will have to wait at least 16 minutes.\n", + "\n", + "Unsloth: [0] Installing llama.cpp. This will take 3 minutes...\n", + "Unsloth: [1] Converting model at models/Qwen2-1.5B-Instruct-bnb-4bit-MAC-q5_k_m into bf16 GGUF format.\n", + "The output location will be ./models/Qwen2-1.5B-Instruct-bnb-4bit-MAC-q5_k_m/unsloth.BF16.gguf\n", + "This will take 3 minutes...\n", + "INFO:hf-to-gguf:Loading model: Qwen2-1.5B-Instruct-bnb-4bit-MAC-q5_k_m\n", + "INFO:gguf.gguf_writer:gguf: This GGUF file is for Little Endian only\n", + "INFO:hf-to-gguf:Set model parameters\n", + "INFO:hf-to-gguf:gguf: context length = 32768\n", + "INFO:hf-to-gguf:gguf: embedding length = 1536\n", + "INFO:hf-to-gguf:gguf: feed forward length = 8960\n", + "INFO:hf-to-gguf:gguf: head count = 12\n", + "INFO:hf-to-gguf:gguf: key-value head count = 2\n", + "INFO:hf-to-gguf:gguf: rope theta = 1000000.0\n", + "INFO:hf-to-gguf:gguf: rms norm epsilon = 1e-06\n", + "INFO:hf-to-gguf:gguf: file type = 32\n", + "INFO:hf-to-gguf:Set model tokenizer\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "INFO:gguf.vocab:Adding 151387 merge(s).\n", + "INFO:gguf.vocab:Setting special token type eos to 151645\n", + "INFO:gguf.vocab:Setting special token type pad to 151643\n", + "INFO:gguf.vocab:Setting special token type bos to 151643\n", + "INFO:gguf.vocab:Setting chat_template to {% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\n", + "You are a helpful assistant.<|im_end|>\n", + "' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n", + "' + message['content'] + '<|im_end|>' + '\n", + "'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n", + "' }}{% endif %}\n", + "INFO:hf-to-gguf:Exporting model to 'models/Qwen2-1.5B-Instruct-bnb-4bit-MAC-q5_k_m/unsloth.BF16.gguf'\n", + "INFO:hf-to-gguf:gguf: loading model part 'model.safetensors'\n", + "INFO:hf-to-gguf:token_embd.weight, torch.bfloat16 --> BF16, shape = {1536, 151936}\n", + "INFO:hf-to-gguf:blk.0.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.0.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.0.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.0.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.0.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.0.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.0.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.0.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.0.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.0.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.0.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.0.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.1.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.1.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.1.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.1.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.1.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.1.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.1.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.1.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.1.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.1.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.1.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.1.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.10.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.10.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.10.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.10.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.10.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.10.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.10.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.10.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.10.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.10.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.10.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.10.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.11.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.11.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.11.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.11.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.11.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.11.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.11.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.11.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.11.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.11.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.11.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.11.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.12.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.12.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.12.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.12.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.12.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.12.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.12.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.12.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.12.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.12.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.12.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.12.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.13.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.13.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.13.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.13.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.13.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.13.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.13.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.13.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.13.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.13.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.13.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.13.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.14.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.14.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.14.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.14.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.14.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.14.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.14.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.14.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.14.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.14.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.14.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.14.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.15.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.15.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.15.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.15.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.15.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.15.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.15.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.15.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.15.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.15.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.15.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.15.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.16.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.16.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.16.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.16.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.16.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.16.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.16.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.16.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.16.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.16.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.16.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.16.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.17.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.17.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.17.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.17.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.17.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.17.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.17.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.17.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.17.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.17.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.17.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.17.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.18.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.18.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.18.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.18.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.18.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.18.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.18.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.18.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.18.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.18.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.18.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.18.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.19.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.19.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.19.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.19.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.19.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.19.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.19.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.19.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.19.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.19.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.19.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.19.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.2.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.2.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.2.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.2.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.2.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.2.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.2.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.2.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.2.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.2.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.2.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.2.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.20.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.20.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.20.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.20.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.20.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.20.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.20.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.20.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.20.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.20.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.20.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.20.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.21.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.21.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.21.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.21.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.21.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.21.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.21.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.21.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.21.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.21.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.21.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.21.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.22.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.22.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.22.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.22.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.22.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.22.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.22.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.22.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.22.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.22.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.22.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.22.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.23.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.23.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.23.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.23.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.23.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.23.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.23.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.23.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.23.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.23.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.23.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.23.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.24.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.24.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.24.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.24.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.24.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.24.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.24.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.24.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.24.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.24.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.24.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.24.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.25.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.25.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.25.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.25.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.25.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.25.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.25.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.25.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.25.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.25.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.25.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.25.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.26.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.26.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.26.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.26.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.26.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.26.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.26.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.26.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.26.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.26.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.26.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.26.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.27.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.27.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.27.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.27.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.27.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.27.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.27.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.27.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.27.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.27.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.27.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.27.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.3.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.3.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.3.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.3.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.3.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.3.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.3.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.3.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.3.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.3.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.3.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.3.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.4.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.4.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.4.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.4.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.4.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.4.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.4.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.4.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.4.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.4.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.4.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.4.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.5.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.5.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.5.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.5.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.5.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.5.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.5.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.5.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.5.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.5.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.5.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.5.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.6.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.6.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.6.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.6.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.6.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.6.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.6.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.6.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.6.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.6.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.6.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.6.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.7.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.7.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.7.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.7.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.7.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.7.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.7.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.7.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.7.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.7.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.7.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.7.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.8.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.8.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.8.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.8.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.8.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.8.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.8.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.8.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.8.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.8.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.8.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.8.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.9.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.9.ffn_down.weight, torch.bfloat16 --> BF16, shape = {8960, 1536}\n", + "INFO:hf-to-gguf:blk.9.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.9.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 8960}\n", + "INFO:hf-to-gguf:blk.9.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.9.attn_k.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.9.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:blk.9.attn_output.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.9.attn_q.bias, torch.bfloat16 --> F32, shape = {1536}\n", + "INFO:hf-to-gguf:blk.9.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 1536}\n", + "INFO:hf-to-gguf:blk.9.attn_v.bias, torch.bfloat16 --> F32, shape = {256}\n", + "INFO:hf-to-gguf:blk.9.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 256}\n", + "INFO:hf-to-gguf:output_norm.weight, torch.bfloat16 --> F32, shape = {1536}\n", + "Writing: 0%| | 0.00/3.09G [00:00\n", + " main()\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/convert-hf-to-gguf.py\", line 2881, in main\n", + " model_instance.write()\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/convert-hf-to-gguf.py\", line 331, in write\n", + " self.gguf_writer.write_tensors_to_file(progress=True)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/gguf_writer.py\", line 312, in write_tensors_to_file\n", + " ti.tensor.tofile(self.fout)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/lazy.py\", line 233, in tofile\n", + " eager = LazyNumpyTensor.to_eager(self)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/lazy.py\", line 193, in to_eager\n", + " return cls._recurse_apply(t, simple_to_eager)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/lazy.py\", line 109, in _recurse_apply\n", + " return fn(o)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/lazy.py\", line 185, in simple_to_eager\n", + " lt._data = lt._func(lt._args)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/lazy.py\", line 158, in \n", + " return cls(meta=cls.eager_to_meta(res), lazy=shared_lazy, args=args, func=lambda a: fn(*a, **kwargs))\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/quants.py\", line 52, in __quantize_bf16_array\n", + " return __apply_over_grouped_rows(__compute_fp32_to_bf16, arr=n, otype=np.int16, oshape=n.shape)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/quants.py\", line 47, in __apply_over_grouped_rows\n", + " np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/quants.py\", line 47, in \n", + " np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/llama.cpp/gguf-py/gguf/quants.py\", line 30, in __compute_fp32_to_bf16\n", + " n = np.where((n & 0x7fffffff) > 0x7f800000, (n & 0xffff0000) | (64 << 16), n)\n", + "OverflowError: Python integer 4294901760 out of bounds for int32\n", + "Writing: 0%| | 0.00/3.09G [00:00\n", + " save_model(model, tokenizer)\n", + " File \"/home/inflaton/code/projects/courses/novel-translation/translation_engine.py\", line 219, in save_model\n", + " File \"/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/unsloth/save.py\", line 1527, in unsloth_save_pretrained_gguf\n", + " all_file_locations = save_to_gguf(model_type, model_dtype, is_sentencepiece_model,\n", + " File \"/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/unsloth/save.py\", line 1113, in save_to_gguf\n", + " raise RuntimeError(\n", + "RuntimeError: Unsloth: Quantization failed! You might have to compile llama.cpp yourself, then run this again.\n", + "You do not need to close this Python program. Run the following commands in a new terminal:\n", + "You must run this in the same folder as you're saving your model.\n", + "git clone --recursive https://github.com/ggerganov/llama.cpp\n", + "cd llama.cpp && make clean && make all -j\n", + "Once that's done, redo the quantization.\n" + ] + } + ], + "source": [ + "!./tune-small.sh" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "07_MAC_+_Qwen2-7B-Instructi_Unsloth_train", + "widgets": {} + }, + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "036fc5746f43416db18c19ad8fd36677": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "06e806c82c7b4cbea31c5358dd9c3434": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "087b76a8b7514269b1f0ab29b062e444": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a069d2ab23824f29aa320ac256e2cfe9", + "placeholder": "​", + "style": "IPY_MODEL_06e806c82c7b4cbea31c5358dd9c3434", + "value": "Map (num_proc=2): 100%" + } + }, + "09b76013aa9e45efb6deb23a7a0d0925": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dea41c5260884aa6879b5e1d1697b14f", + "placeholder": "​", + "style": "IPY_MODEL_89965917796a4f81b899fdc7685f33df", + "value": "config.json: 100%" + } + }, + "0a92c56bfa134ef583220d7ef0b13e17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c34be936c8145d3ab41282f30a70713": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f8b6bfe16894500838793f2491d403f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "177c78fce95d4b4ab33057c5a048d693": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1f44c9ce1adf470cbb19784493ed209f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c34be936c8145d3ab41282f30a70713", + "placeholder": "​", + "style": "IPY_MODEL_0a92c56bfa134ef583220d7ef0b13e17", + "value": "model.safetensors: 100%" + } + }, + "201b59ccd9f845e197029b57e424aefc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2157f01726d748f8a9ae4a00664430da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21db8a77b00d4a4e82fdfa608657531f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26e4202cca81496a90d15a0dd4ca9cf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba90fdb8822d47dab7ba203bee297f37", + "IPY_MODEL_61560ff6a36b44f4a9dfdae5c52791d4", + "IPY_MODEL_95fbe66647904c06a20f640630d6dc0e" + ], + "layout": "IPY_MODEL_57182a263d324a3dbf1471c74290a0d5" + } + }, + "27155728b6b84cb199c91c940095d0a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6b91feeed5464877991ac2c207aebe7c", + "IPY_MODEL_cca8113c54c0495daedce1327bf9c68b", + "IPY_MODEL_2e63a29e2f7247bba5beede9a568c99f" + ], + "layout": "IPY_MODEL_5c9d781c28944f3eb86e2a6d44efdf18" + } + }, + "271ddaa553a042d09b6db7b450643d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a58d04b428c46f4b3dbadd3bc6cd529": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d18ddf6482c4d97829ac0e5a7b9868f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "IPY_MODEL_f2df530d22c74977b249dd9fb5f4829b", + "IPY_MODEL_89b2ef0dbfea47ab8e6f8d659e3351d1" + ], + "layout": "IPY_MODEL_3056b148aa9f4e6e8aa3b61d26886255" + } + }, + "2e5087c76f98437cb5dc729230358cba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e63a29e2f7247bba5beede9a568c99f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b993eaec6b224440bf80c0958c6fb536", + "placeholder": "​", + "style": "IPY_MODEL_de868e26e7154f62aa86223a539ad421", + "value": " 464/464 [00:00<00:00, 27.1kB/s]" + } + }, + "2f6c70dd266c4816bfad3fd3d192929a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "30307300bc4e4baf96560e30969a82b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e36a3f9eff0e4cf68834d66b0213ae96", + "placeholder": "​", + "style": "IPY_MODEL_a0037bdccf254159becde630bee3d1db", + "value": "generation_config.json: 100%" + } + }, + "3056b148aa9f4e6e8aa3b61d26886255": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30cdc32298134cb0be4d41615b9e5774": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3572201bd4d74a58b7a665f9bdfdcdba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35b0e8c26d6640e9bd0ed7b242a423d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e5087c76f98437cb5dc729230358cba", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_036fc5746f43416db18c19ad8fd36677", + "value": 51760 + } + }, + "36166c7bcb854b34aca1f41a5d6ea50b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "370692d819df41828b48c4ad446f977b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "39b29a75374b45c0a22506010be2b84e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_30cdc32298134cb0be4d41615b9e5774", + "max": 1179, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_47928317548c454bba6358ab132e8dee", + "value": 1179 + } + }, + "3cf2dd993b5e4d3daecf61e4bab5a404": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_087b76a8b7514269b1f0ab29b062e444", + "IPY_MODEL_35b0e8c26d6640e9bd0ed7b242a423d8", + "IPY_MODEL_54ad89e05fd74576b9b8b5b5a10eaf8d" + ], + "layout": "IPY_MODEL_a41dc44766444a998bec2d777f249d23" + } + }, + "43dec2ede91341f5af60eb522e18e984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4463edd481c1467f914c7dcd6c6e6ffc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47928317548c454bba6358ab132e8dee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "49277aeeac16434a865a4d12308b1abc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4ae7e449e4ea4c729b5f34607c18ebae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b2061b8a73c43ffb0c2f83daf0d0183": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c4c88d4c701450692fa0f6b0c5764b0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c666f4ace3943f8b80ecd20e7503236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ccedf0d93094e63b57a0f8a434fba06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4463edd481c1467f914c7dcd6c6e6ffc", + "max": 44307561, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6d3b9a05db0b4dadb638c686faa0c40a", + "value": 44307561 + } + }, + "4dcf6ff672d24983a1877a8431709aa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5807d5fb827d490fb3bc698f801ffff5", + "placeholder": "​", + "style": "IPY_MODEL_c4f2b06a82fd4987b8b659524a7b503b", + "value": "Generating train split: 100%" + } + }, + "4ea63adfce694725bdba878aef709dd3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5234566b1bfc4655b8d582ea5b46ed9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "54ad89e05fd74576b9b8b5b5a10eaf8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdb1941405ed4e4aa06019933892deb3", + "placeholder": "​", + "style": "IPY_MODEL_668d5377ca56426a99753867e6e24862", + "value": " 51760/51760 [01:02<00:00, 1131.51 examples/s]" + } + }, + "56aee4853b7740e6a977254f5d1fa66d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "57182a263d324a3dbf1471c74290a0d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5807d5fb827d490fb3bc698f801ffff5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c9d781c28944f3eb86e2a6d44efdf18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f40db8173dd4d76b6ef5ed6d9ec8b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61560ff6a36b44f4a9dfdae5c52791d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db19fc8d37db4e45a5790a876836d8c4", + "max": 11610, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36166c7bcb854b34aca1f41a5d6ea50b", + "value": 11610 + } + }, + "6578fd7acdb54c4c93528ea431fd0144": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_370692d819df41828b48c4ad446f977b", + "placeholder": "​", + "style": "IPY_MODEL_a0bf9160eb2647409b3200270914b90f", + "value": " 50.6k/50.6k [00:00<00:00, 2.71MB/s]" + } + }, + "668d5377ca56426a99753867e6e24862": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697f027529b54ee9956bae78a11e0611": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69ac12aec0714318bf2c83d4f4e745f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b2012c3f88547af8884a9ea90e3164b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_938f45f1b3e24118b815d96ae34ba86a", + "placeholder": "​", + "style": "IPY_MODEL_9367047a800747f79c6b225d92397846", + "value": " 44.3M/44.3M [00:01<00:00, 31.0MB/s]" + } + }, + "6b91feeed5464877991ac2c207aebe7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b2061b8a73c43ffb0c2f83daf0d0183", + "placeholder": "​", + "style": "IPY_MODEL_69ac12aec0714318bf2c83d4f4e745f5", + "value": "special_tokens_map.json: 100%" + } + }, + "6d3b9a05db0b4dadb638c686faa0c40a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6dbbedeca9314e66ae50e44ffa31a414": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6e34619b45934040b6092e6fb01ea7fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71ce208e20d6483abb9ed923510c86d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d69dc491b3ab44d7852b21873ed7bb7f", + "placeholder": "​", + "style": "IPY_MODEL_f401d53bf28e44eb906bce6c05412662", + "value": " 51760/51760 [00:01<00:00, 45512.81 examples/s]" + } + }, + "7358cdad832342c983e31efb8754ab78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e352a3404f4c7dad0737f57d29e92f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_988a0e8c1f89446086858da0a891a79c", + "IPY_MODEL_4ccedf0d93094e63b57a0f8a434fba06", + "IPY_MODEL_6b2012c3f88547af8884a9ea90e3164b" + ], + "layout": "IPY_MODEL_7e29cb8dd4df4d5b94407cd8fd3f2011" + } + }, + "74501720ac7e4dbb911a4a99b3633bc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "78e5400bff924a92a4cc61c4ff18b182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9b313fd861948f5aba25b24b1518d30", + "placeholder": "​", + "style": "IPY_MODEL_4c666f4ace3943f8b80ecd20e7503236", + "value": " 1.18k/1.18k [00:00<00:00, 31.3kB/s]" + } + }, + "7975adbc2ec5489ea7fa0167e620d85c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e34619b45934040b6092e6fb01ea7fe", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_271ddaa553a042d09b6db7b450643d8f", + "value": 51760 + } + }, + "7e29cb8dd4df4d5b94407cd8fd3f2011": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "810ff6c0e17d4fa09a30fef27eacff90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89965917796a4f81b899fdc7685f33df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "89b2ef0dbfea47ab8e6f8d659e3351d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8908fa0df3743ecb9d12983a739104f", + "placeholder": "​", + "style": "IPY_MODEL_177c78fce95d4b4ab33057c5a048d693", + "value": " 9.09M/9.09M [00:00<00:00, 32.6MB/s]" + } + }, + "8b3505352a5a42bf910428c40ce40465": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_49277aeeac16434a865a4d12308b1abc", + "placeholder": "​", + "style": "IPY_MODEL_2157f01726d748f8a9ae4a00664430da", + "value": " 5.70G/5.70G [01:02<00:00, 30.1MB/s]" + } + }, + "8fc142b628fb40568730234de1cafde2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ae7e449e4ea4c729b5f34607c18ebae", + "max": 172, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3572201bd4d74a58b7a665f9bdfdcdba", + "value": 172 + } + }, + "9367047a800747f79c6b225d92397846": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "938f45f1b3e24118b815d96ae34ba86a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "95fbe66647904c06a20f640630d6dc0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b0a370dc20654b279b9680692e34418e", + "placeholder": "​", + "style": "IPY_MODEL_cfeb365ddf7548d58b2557f22737fcf5", + "value": " 11.6k/11.6k [00:00<00:00, 716kB/s]" + } + }, + "988a0e8c1f89446086858da0a891a79c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ad2be500fc164c0f86f33e914ef8e6a0", + "placeholder": "​", + "style": "IPY_MODEL_5234566b1bfc4655b8d582ea5b46ed9f", + "value": "Downloading data: 100%" + } + }, + "98c58f23f4d549518832cb2d18f796e8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_09b76013aa9e45efb6deb23a7a0d0925", + "IPY_MODEL_39b29a75374b45c0a22506010be2b84e", + "IPY_MODEL_78e5400bff924a92a4cc61c4ff18b182" + ], + "layout": "IPY_MODEL_2a58d04b428c46f4b3dbadd3bc6cd529" + } + }, + "99fdbb0300c14c139d1937c646f0cfe7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7358cdad832342c983e31efb8754ab78", + "placeholder": "​", + "style": "IPY_MODEL_e9adf418296e436fb48bb9f78885598b", + "value": " 51760/51760 [00:01<00:00, 38665.95 examples/s]" + } + }, + "9f679ad3ec7f4fe8ad0510ffb57bc2ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ea63adfce694725bdba878aef709dd3", + "placeholder": "​", + "style": "IPY_MODEL_74501720ac7e4dbb911a4a99b3633bc6", + "value": "tokenizer.json: 100%" + } + }, + "a0037bdccf254159becde630bee3d1db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a069d2ab23824f29aa320ac256e2cfe9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0bf9160eb2647409b3200270914b90f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a41dc44766444a998bec2d777f249d23": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8464a4c711e4e00aafdfc919b60d07e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fb995c740590427b882572c81d4e848c", + "placeholder": "​", + "style": "IPY_MODEL_201b59ccd9f845e197029b57e424aefc", + "value": " 172/172 [00:00<00:00, 12.0kB/s]" + } + }, + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad2be500fc164c0f86f33e914ef8e6a0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b0240cd9a4554b29ae11f8051984a1c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_edaf890370314a218f138015faa0b05d", + "placeholder": "​", + "style": "IPY_MODEL_697f027529b54ee9956bae78a11e0611", + "value": "Map: 100%" + } + }, + "b0a370dc20654b279b9680692e34418e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b518dcee69074b87be73957cd810e7ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d891f8d0b1fc462f8008d02bb2a15692", + "placeholder": "​", + "style": "IPY_MODEL_cced8fd7e998472794f3f3e3018956a5", + "value": "tokenizer_config.json: 100%" + } + }, + "b8908fa0df3743ecb9d12983a739104f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b993eaec6b224440bf80c0958c6fb536": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9b313fd861948f5aba25b24b1518d30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba90fdb8822d47dab7ba203bee297f37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f8b6bfe16894500838793f2491d403f", + "placeholder": "​", + "style": "IPY_MODEL_bb19f6c747754682a514373a3a0535ba", + "value": "Downloading readme: 100%" + } + }, + "bb19f6c747754682a514373a3a0535ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc883d4cf13e4f8b8a4fe5f410cb6efd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9159e03e61f4f56978ece9c3bca49b2", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_810ff6c0e17d4fa09a30fef27eacff90", + "value": 51760 + } + }, + "c161d94df0f04feba9542237e0856c22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c22f71b1f85843209d7e5321506b9cb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f44c9ce1adf470cbb19784493ed209f", + "IPY_MODEL_f1addc4479d849879e743cf9089e6540", + "IPY_MODEL_8b3505352a5a42bf910428c40ce40465" + ], + "layout": "IPY_MODEL_4c4c88d4c701450692fa0f6b0c5764b0" + } + }, + "c4f2b06a82fd4987b8b659524a7b503b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cca8113c54c0495daedce1327bf9c68b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e02f9b7849c64531835eb77b860d1c93", + "max": 464, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_56aee4853b7740e6a977254f5d1fa66d", + "value": 464 + } + }, + "cced8fd7e998472794f3f3e3018956a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cf245afeb1c04f29a24d291608c3d157": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b518dcee69074b87be73957cd810e7ed", + "IPY_MODEL_e29104486d594b2992d7285e0ef77371", + "IPY_MODEL_6578fd7acdb54c4c93528ea431fd0144" + ], + "layout": "IPY_MODEL_d35db8148a354c56aaac56dbae22536f" + } + }, + "cfe8cae0e22b495bafa221a63d13b283": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfeb365ddf7548d58b2557f22737fcf5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1b47d39450d4019ae85c9b2f943eeaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4dcf6ff672d24983a1877a8431709aa9", + "IPY_MODEL_7975adbc2ec5489ea7fa0167e620d85c", + "IPY_MODEL_71ce208e20d6483abb9ed923510c86d7" + ], + "layout": "IPY_MODEL_cfe8cae0e22b495bafa221a63d13b283" + } + }, + "d35db8148a354c56aaac56dbae22536f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69dc491b3ab44d7852b21873ed7bb7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d891f8d0b1fc462f8008d02bb2a15692": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8e5318cead340c4adbeaccc05d39225": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "daf4cd890b35422683d22fd30bc71e83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b0240cd9a4554b29ae11f8051984a1c6", + "IPY_MODEL_bc883d4cf13e4f8b8a4fe5f410cb6efd", + "IPY_MODEL_99fdbb0300c14c139d1937c646f0cfe7" + ], + "layout": "IPY_MODEL_c161d94df0f04feba9542237e0856c22" + } + }, + "db19fc8d37db4e45a5790a876836d8c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de868e26e7154f62aa86223a539ad421": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dea41c5260884aa6879b5e1d1697b14f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e02f9b7849c64531835eb77b860d1c93": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e29104486d594b2992d7285e0ef77371": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "max": 50641, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2f6c70dd266c4816bfad3fd3d192929a", + "value": 50641 + } + }, + "e36a3f9eff0e4cf68834d66b0213ae96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9159e03e61f4f56978ece9c3bca49b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9adf418296e436fb48bb9f78885598b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "edaf890370314a218f138015faa0b05d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f1addc4479d849879e743cf9089e6540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43dec2ede91341f5af60eb522e18e984", + "max": 5702746405, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d8e5318cead340c4adbeaccc05d39225", + "value": 5702746405 + } + }, + "f2df530d22c74977b249dd9fb5f4829b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_21db8a77b00d4a4e82fdfa608657531f", + "max": 9085698, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6dbbedeca9314e66ae50e44ffa31a414", + "value": 9085698 + } + }, + "f401d53bf28e44eb906bce6c05412662": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fb995c740590427b882572c81d4e848c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fce7a61c25ec4390af43d92b7c473a45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_30307300bc4e4baf96560e30969a82b6", + "IPY_MODEL_8fc142b628fb40568730234de1cafde2", + "IPY_MODEL_a8464a4c711e4e00aafdfc919b60d07e" + ], + "layout": "IPY_MODEL_5f40db8173dd4d76b6ef5ed6d9ec8b6e" + } + }, + "fdb1941405ed4e4aa06019933892deb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/notebooks/04_tune-small-no-flash-attn.ipynb b/notebooks/04_tune-small-no-flash-attn.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f56083454905fe1ba6f9bd3e0538faf3c84faad0 --- /dev/null +++ b/notebooks/04_tune-small-no-flash-attn.ipynb @@ -0,0 +1,4607 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "0ea8b46b-839b-445b-8043-ccdf4e920ace", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6d394937-6c99-4a7c-9d32-7600a280032f", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /home/inflaton/code/projects/courses/novel-translation\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9f67ec60-2f24-411c-84eb-0dd664b44775", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f1597656-8042-4878-9d3b-9ebfb8dd86dc", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('unsloth/Qwen2-0.5B-Instruct-bnb-4bit',\n", + " True,\n", + " None,\n", + " None,\n", + " 2048,\n", + " 10,\n", + " None,\n", + " 'datasets/mac/mac.tsv',\n", + " 'results/mac-results_v3.csv')" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "model_name = os.getenv(\"MODEL_NAME\")\n", + "token = os.getenv(\"HF_TOKEN\") or None\n", + "load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n", + "local_model = os.getenv(\"LOCAL_MODEL\")\n", + "hub_model = os.getenv(\"HUB_MODEL\")\n", + "num_train_epochs = int(os.getenv(\"NUM_TRAIN_EPOCHS\") or 0)\n", + "data_path = os.getenv(\"DATA_PATH\")\n", + "results_path = os.getenv(\"RESULTS_PATH\")\n", + "\n", + "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n", + "dtype = (\n", + " None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n", + ")\n", + "\n", + "model_name, load_in_4bit, local_model, hub_model, max_seq_length, num_train_epochs, dtype, data_path, results_path" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fri Jun 21 18:14:08 2024 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 545.23.07 Driver Version: 546.12 CUDA Version: 12.3 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 NVIDIA GeForce RTX 4080 ... On | 00000000:01:00.0 On | N/A |\n", + "| N/A 54C P5 7W / 150W | 234MiB / 12282MiB | 42% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "| No running processes found |\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "!nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mWARNING: Package(s) not found: flash-attn\u001b[0m\u001b[33m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!pip show flash-attn" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/novel-translation\n", + "Tuning unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/novel-translation/translation_engine_v3.py\n", + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n", + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit True 2048 10 None datasets/mac/mac.tsv results/mac-results_v3.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.633 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "100%|█████████████████████████████████████| 1133/1133 [1:21:58<00:00, 4.34s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... The gun is lifted by Old Teng, with his eyes c...\n", + "\n", + "[1 rows x 7 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.043 GB of memory reserved.\n", + "Unsloth 2024.6 patched 24 layers with 0 QKV layers, 24 O layers and 24 MLP layers.\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.043 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 8,798,208\n", + "{'loss': 1.9401, 'grad_norm': 0.9639493823051453, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.7761, 'grad_norm': 0.8035507798194885, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.7147, 'grad_norm': 0.9184380769729614, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.7157, 'grad_norm': 0.7518640160560608, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.6863, 'grad_norm': 0.8273285031318665, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.608, 'grad_norm': 0.8058149814605713, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.4919, 'grad_norm': 0.913749098777771, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.5404, 'grad_norm': 0.9821408987045288, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.4722, 'grad_norm': 1.0769108533859253, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.4689, 'grad_norm': 0.990247368812561, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.5277, 'grad_norm': 1.0138424634933472, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.3496, 'grad_norm': 1.163719892501831, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 1.2155, 'grad_norm': 1.1911096572875977, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.2574, 'grad_norm': 1.316228985786438, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.2555, 'grad_norm': 1.343137502670288, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.2662, 'grad_norm': 1.177164077758789, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.2342, 'grad_norm': 1.1601923704147339, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.9801, 'grad_norm': 1.4796860218048096, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.9924, 'grad_norm': 1.5130630731582642, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.9984, 'grad_norm': 1.5380527973175049, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.9815, 'grad_norm': 1.570404291152954, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 1.0242, 'grad_norm': 1.5608525276184082, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.9313, 'grad_norm': 1.5546534061431885, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.74, 'grad_norm': 1.7187141180038452, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.7402, 'grad_norm': 1.848803162574768, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.7757, 'grad_norm': 2.113391160964966, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.7729, 'grad_norm': 2.1327595710754395, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.7938, 'grad_norm': 2.1906380653381348, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.631, 'grad_norm': 1.9121012687683105, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.556, 'grad_norm': 1.620622992515564, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.5605, 'grad_norm': 1.618948221206665, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.5889, 'grad_norm': 1.811829686164856, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.5825, 'grad_norm': 2.1219239234924316, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.5787, 'grad_norm': 2.149881601333618, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.3971, 'grad_norm': 1.961249589920044, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.4116, 'grad_norm': 1.9004943370819092, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.429, 'grad_norm': 2.7660951614379883, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.4209, 'grad_norm': 1.6345818042755127, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.4275, 'grad_norm': 1.6811296939849854, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.3852, 'grad_norm': 1.5187770128250122, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.2924, 'grad_norm': 1.5174031257629395, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.3134, 'grad_norm': 1.481117606163025, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.3139, 'grad_norm': 1.8529402017593384, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.303, 'grad_norm': 1.9317971467971802, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.3176, 'grad_norm': 1.7100918292999268, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.2473, 'grad_norm': 1.5257225036621094, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.2267, 'grad_norm': 1.563258171081543, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.2282, 'grad_norm': 1.091956377029419, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.2358, 'grad_norm': 1.2666215896606445, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.2354, 'grad_norm': 2.033458948135376, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.2375, 'grad_norm': 1.1279031038284302, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.181, 'grad_norm': 1.1606773138046265, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1902, 'grad_norm': 1.120147466659546, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1874, 'grad_norm': 1.1482255458831787, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.1878, 'grad_norm': 1.6668341159820557, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + " 97%|███████████████████████████████████▉ | 5500/5660 [3:50:58<06:29, 2.43s/it]/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/other.py:611: UserWarning: Unable to fetch remote file due to the following error (MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /unsloth/Qwen2-0.5B-Instruct-bnb-4bit/resolve/main/config.json (Caused by NameResolutionError(\": Failed to resolve \\'huggingface.co\\' ([Errno -3] Temporary failure in name resolution)\"))'), '(Request ID: c47e2483-6a5c-4a75-8fdb-c35160bdbb0d)') - silently ignoring the lookup for the file config.json in unsloth/Qwen2-0.5B-Instruct-bnb-4bit.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in unsloth/Qwen2-0.5B-Instruct-bnb-4bit - will assume that the vocabulary was not modified.\n", + " warnings.warn(\n", + "{'loss': 0.1887, 'grad_norm': 1.131452202796936, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 14270.4989, 'train_samples_per_second': 3.173, 'train_steps_per_second': 0.397, 'train_loss': 0.7989168851198661, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [3:57:50<00:00, 2.52s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "14270.4989 seconds used for training.\n", + "237.84 minutes used for training.\n", + "Peak reserved memory = 3.043 GB.\n", + "Peak reserved memory for training = 0.0 GB.\n", + "Peak reserved memory % of max memory = 25.371 %.\n", + "Peak reserved memory for training % of max memory = 0.0 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "100%|█████████████████████████████████████| 1133/1133 [1:52:28<00:00, 5.96s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his rifle, squinting, and aimi...\n", + "\n", + "[1 rows x 8 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.895 GB of memory reserved.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... This might take 10 minutes for Llama-7b... Done.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving 4bit Bitsandbytes model. Please wait...\n", + "README.md: 100%|███████████████████████████████| 593/593 [00:00<00:00, 3.29MB/s]\n", + "model.safetensors: 100%|█████████████████████| 493M/493M [01:07<00:00, 7.32MB/s]\n", + "README.md: 100%|███████████████████████████████| 599/599 [00:00<00:00, 2.60MB/s]\n", + "Saved merged_4bit model to https://huggingface.co/Qwen2-0.5B-Instruct-bnb-4bit-MAC-merged_4bit_forced\n", + "Tuning unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/novel-translation/translation_engine_v3.py\n", + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit True 2048 10 None datasets/mac/mac.tsv results/mac-results_v3.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.516 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "100%|█████████████████████████████████████| 1133/1133 [1:32:31<00:00, 4.90s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geer lifted his gun and squinted at it thr...\n", + "\n", + "[1 rows x 9 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.945 GB of memory reserved.\n", + "Unsloth 2024.6 patched 28 layers with 0 QKV layers, 28 O layers and 28 MLP layers.\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.945 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 18,464,768\n", + "{'loss': 1.7413, 'grad_norm': 0.6446139216423035, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.5677, 'grad_norm': 0.6344625353813171, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.5159, 'grad_norm': 0.6517081260681152, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.5171, 'grad_norm': 0.5709907412528992, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.4959, 'grad_norm': 0.568977952003479, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.418, 'grad_norm': 0.6047976016998291, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.2972, 'grad_norm': 0.7236104607582092, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.3396, 'grad_norm': 0.6862805485725403, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.2789, 'grad_norm': 0.8805968165397644, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.2793, 'grad_norm': 0.8751853108406067, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.3315, 'grad_norm': 0.8396161794662476, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.1276, 'grad_norm': 0.9836744666099548, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 0.9962, 'grad_norm': 1.0947535037994385, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.0297, 'grad_norm': 1.1813771724700928, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.0349, 'grad_norm': 1.0732464790344238, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.0351, 'grad_norm': 1.4764126539230347, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.0081, 'grad_norm': 0.9122354388237, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.698, 'grad_norm': 1.328641414642334, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.7099, 'grad_norm': 1.3143669366836548, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.7166, 'grad_norm': 1.3019964694976807, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.7053, 'grad_norm': 1.4416537284851074, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 0.7448, 'grad_norm': 1.191597819328308, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.6435, 'grad_norm': 1.6522774696350098, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.4617, 'grad_norm': 1.5974637269973755, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.4463, 'grad_norm': 2.0055346488952637, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.4806, 'grad_norm': 1.7279341220855713, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.4799, 'grad_norm': 1.6954944133758545, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.4874, 'grad_norm': 1.6931037902832031, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.3582, 'grad_norm': 1.3343307971954346, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.2918, 'grad_norm': 1.6368076801300049, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.2989, 'grad_norm': 1.4428879022598267, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.3153, 'grad_norm': 1.721048355102539, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.307, 'grad_norm': 1.464351773262024, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.3065, 'grad_norm': 1.1433079242706299, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.1922, 'grad_norm': 1.671250343322754, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.2034, 'grad_norm': 1.0947823524475098, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.2064, 'grad_norm': 1.121410608291626, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.208, 'grad_norm': 1.0732439756393433, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.2129, 'grad_norm': 1.4153015613555908, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.1935, 'grad_norm': 0.7308939099311829, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.1485, 'grad_norm': 0.49827346205711365, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.157, 'grad_norm': 0.9691917300224304, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.1574, 'grad_norm': 0.8331239819526672, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.1584, 'grad_norm': 0.6648879051208496, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.1577, 'grad_norm': 1.7691469192504883, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.1366, 'grad_norm': 0.4894603192806244, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.1303, 'grad_norm': 0.7218558192253113, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.1311, 'grad_norm': 0.512703537940979, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.1333, 'grad_norm': 0.49423089623451233, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.1338, 'grad_norm': 0.8092319965362549, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + " 88%|████████████████████████████████▋ | 5000/5660 [3:52:46<30:36, 2.78s/it]/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/other.py:611: UserWarning: Unable to fetch remote file due to the following error (MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /unsloth/Qwen2-1.5B-Instruct-bnb-4bit/resolve/main/config.json (Caused by NameResolutionError(\": Failed to resolve \\'huggingface.co\\' ([Errno -3] Temporary failure in name resolution)\"))'), '(Request ID: 8549560f-a295-478f-bfcb-44574f659439)') - silently ignoring the lookup for the file config.json in unsloth/Qwen2-1.5B-Instruct-bnb-4bit.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in unsloth/Qwen2-1.5B-Instruct-bnb-4bit - will assume that the vocabulary was not modified.\n", + " warnings.warn(\n", + "{'loss': 0.1326, 'grad_norm': 0.4783310890197754, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.1177, 'grad_norm': 0.44080662727355957, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1184, 'grad_norm': 0.5362476706504822, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1191, 'grad_norm': 0.47040414810180664, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.12, 'grad_norm': 0.5143936276435852, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1228, 'grad_norm': 0.433252215385437, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 15909.7187, 'train_samples_per_second': 2.846, 'train_steps_per_second': 0.356, 'train_loss': 0.5994435462850564, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [4:25:09<00:00, 2.81s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "15909.7187 seconds used for training.\n", + "265.16 minutes used for training.\n", + "Peak reserved memory = 3.945 GB.\n", + "Peak reserved memory for training = 0.0 GB.\n", + "Peak reserved memory % of max memory = 32.891 %.\n", + "Peak reserved memory for training % of max memory = 0.0 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "100%|█████████████████████████████████████| 1133/1133 [2:17:51<00:00, 7.30s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised the rifle to his eye, squinted...\n", + "\n", + "[1 rows x 10 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5.33 GB of memory reserved.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... This might take 10 minutes for Llama-7b... Done.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving 4bit Bitsandbytes model. Please wait...\n", + "README.md: 100%|███████████████████████████████| 593/593 [00:00<00:00, 3.33MB/s]\n", + "model.safetensors: 100%|███████████████████| 1.22G/1.22G [01:41<00:00, 12.0MB/s]\n", + "README.md: 100%|███████████████████████████████| 599/599 [00:00<00:00, 3.39MB/s]\n", + "Saved merged_4bit model to https://huggingface.co/Qwen2-1.5B-Instruct-bnb-4bit-MAC-merged_4bit_forced\n" + ] + } + ], + "source": [ + "!./tune-small.sh" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/novel-translation\n", + "Tuning unsloth/Qwen2-0.5B-Instruct\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/novel-translation/translation_engine_v3.py\n", + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n", + "unsloth/Qwen2-0.5B-Instruct True 2048 10 None datasets/mac/mac.tsv results/mac-results_v3.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-0.5B-Instruct\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.633 GB of memory reserved.\n", + "loading train/test data files\n", + "Map: 100%|█████████████████████████| 4528/4528 [00:00<00:00, 8820.36 examples/s]\n", + "Map: 100%|█████████████████████████| 1133/1133 [00:00<00:00, 4632.29 examples/s]\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-0.5B-Instruct\n", + "100%|█████████████████████████████████████| 1133/1133 [1:18:16<00:00, 4.15s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Teng raises his gun, closing his eyes with...\n", + "\n", + "[1 rows x 11 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.912 GB of memory reserved.\n", + "Unsloth 2024.6 patched 24 layers with 0 QKV layers, 24 O layers and 24 MLP layers.\n", + "Map (num_proc=2): 100%|████████████| 4528/4528 [00:02<00:00, 2104.12 examples/s]\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.912 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 8,798,208\n", + "{'loss': 1.9401, 'grad_norm': 0.9639493823051453, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.7762, 'grad_norm': 0.8044034838676453, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.7147, 'grad_norm': 0.9299402832984924, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.7153, 'grad_norm': 0.7542973160743713, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.6864, 'grad_norm': 0.8260002136230469, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.6077, 'grad_norm': 0.8066288828849792, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.4918, 'grad_norm': 0.9108980894088745, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.5405, 'grad_norm': 0.9814554452896118, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.4721, 'grad_norm': 1.0819430351257324, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.4689, 'grad_norm': 0.9869484901428223, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.5278, 'grad_norm': 1.0101701021194458, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.3491, 'grad_norm': 1.1646143198013306, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 1.2154, 'grad_norm': 1.185487985610962, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.2576, 'grad_norm': 1.3092553615570068, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.2554, 'grad_norm': 1.3459289073944092, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.2662, 'grad_norm': 1.1843770742416382, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.2348, 'grad_norm': 1.1434661149978638, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.9799, 'grad_norm': 1.5520676374435425, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.9922, 'grad_norm': 1.5200737714767456, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.9983, 'grad_norm': 1.5352424383163452, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.9806, 'grad_norm': 1.5652823448181152, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 1.0233, 'grad_norm': 1.6117287874221802, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.9315, 'grad_norm': 1.5618888139724731, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.7403, 'grad_norm': 1.6472711563110352, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.7402, 'grad_norm': 1.7707853317260742, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.7748, 'grad_norm': 2.0032260417938232, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.773, 'grad_norm': 2.1941237449645996, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.7942, 'grad_norm': 1.99174165725708, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.6302, 'grad_norm': 2.141026496887207, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.555, 'grad_norm': 1.5508298873901367, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.5604, 'grad_norm': 1.7151756286621094, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.5907, 'grad_norm': 1.838254690170288, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.5818, 'grad_norm': 2.1077213287353516, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.5784, 'grad_norm': 1.6738381385803223, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.3977, 'grad_norm': 3.3859152793884277, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.4097, 'grad_norm': 1.9067131280899048, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.4299, 'grad_norm': 2.464520215988159, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.4206, 'grad_norm': 1.9985911846160889, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.4272, 'grad_norm': 1.5266529321670532, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.3845, 'grad_norm': 1.98494553565979, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.2941, 'grad_norm': 1.8065193891525269, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.3126, 'grad_norm': 1.2746621370315552, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.3137, 'grad_norm': 2.0079100131988525, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.3043, 'grad_norm': 1.8258264064788818, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.3171, 'grad_norm': 1.5290628671646118, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.2472, 'grad_norm': 1.458840250968933, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.2266, 'grad_norm': 1.4306132793426514, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.2287, 'grad_norm': 1.590943455696106, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.2362, 'grad_norm': 1.136732816696167, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.2357, 'grad_norm': 2.2578623294830322, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.2376, 'grad_norm': 1.0956445932388306, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.1812, 'grad_norm': 1.0962462425231934, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1901, 'grad_norm': 1.0591093301773071, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1872, 'grad_norm': 1.2788742780685425, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.1885, 'grad_norm': 1.6012808084487915, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1893, 'grad_norm': 1.0120514631271362, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 14483.8236, 'train_samples_per_second': 3.126, 'train_steps_per_second': 0.391, 'train_loss': 0.7988819386849555, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [4:01:23<00:00, 2.56s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "14483.8236 seconds used for training.\n", + "241.4 minutes used for training.\n", + "Peak reserved memory = 1.371 GB.\n", + "Peak reserved memory for training = 0.459 GB.\n", + "Peak reserved memory % of max memory = 11.431 %.\n", + "Peak reserved memory for training % of max memory = 3.827 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-0.5B-Instruct\n", + "100%|█████████████████████████████████████| 1133/1133 [1:40:58<00:00, 5.35s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his rifle and made a twist eye...\n", + "\n", + "[1 rows x 12 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.371 GB of memory reserved.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... This might take 10 minutes for Llama-7b... Done.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving 4bit Bitsandbytes model. Please wait...\n", + "model.safetensors: 100%|█████████████████████| 493M/493M [00:38<00:00, 12.7MB/s]\n", + "README.md: 100%|███████████████████████████████| 581/581 [00:00<00:00, 3.37MB/s]\n", + "Saved merged_4bit model to https://huggingface.co/Qwen2-0.5B-Instruct-MAC-merged_4bit_forced\n", + "Tuning unsloth/Qwen2-1.5B-Instruct\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/novel-translation/translation_engine_v3.py\n", + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n", + "unsloth/Qwen2-1.5B-Instruct True 2048 10 None datasets/mac/mac.tsv results/mac-results_v3.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-1.5B-Instruct\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.516 GB of memory reserved.\n", + "loading train/test data files\n", + "Map: 100%|████████████████████████| 4528/4528 [00:00<00:00, 18414.78 examples/s]\n", + "Map: 100%|█████████████████████████| 1133/1133 [00:00<00:00, 9253.25 examples/s]\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-1.5B-Instruct\n", + "100%|█████████████████████████████████████| 1133/1133 [1:13:14<00:00, 3.88s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝...\n", + "\n", + "[1 rows x 13 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.777 GB of memory reserved.\n", + "Unsloth 2024.6 patched 28 layers with 0 QKV layers, 28 O layers and 28 MLP layers.\n", + "Map (num_proc=2): 100%|████████████| 4528/4528 [00:02<00:00, 2257.38 examples/s]\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.777 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 18,464,768\n", + "{'loss': 1.7417, 'grad_norm': 0.6452760696411133, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.5681, 'grad_norm': 0.6196172833442688, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.516, 'grad_norm': 0.6484832763671875, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.5172, 'grad_norm': 0.5743487477302551, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.4959, 'grad_norm': 0.5681684017181396, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.418, 'grad_norm': 0.6087610125541687, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.2977, 'grad_norm': 0.7200368642807007, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.3391, 'grad_norm': 0.6828812956809998, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.2787, 'grad_norm': 0.8858041763305664, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.2797, 'grad_norm': 0.8660985231399536, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.3315, 'grad_norm': 1.1288585662841797, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.1279, 'grad_norm': 1.004281759262085, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 0.9951, 'grad_norm': 1.0934711694717407, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.0299, 'grad_norm': 1.2658041715621948, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.0337, 'grad_norm': 1.0768382549285889, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.0343, 'grad_norm': 1.031988501548767, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.0086, 'grad_norm': 0.9097064733505249, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.6964, 'grad_norm': 1.314468502998352, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.7091, 'grad_norm': 1.35807204246521, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.7151, 'grad_norm': 1.3476978540420532, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + " 35%|████████████▎ | 2000/5660 [1:37:50<2:55:23, 2.88s/it]/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/other.py:611: UserWarning: Unable to fetch remote file due to the following error (MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /unsloth/qwen2-1.5b-instruct-bnb-4bit/resolve/main/config.json (Caused by NameResolutionError(\": Failed to resolve \\'huggingface.co\\' ([Errno -3] Temporary failure in name resolution)\"))'), '(Request ID: 628a4ff6-a882-4c3c-8607-702c8596b3a3)') - silently ignoring the lookup for the file config.json in unsloth/qwen2-1.5b-instruct-bnb-4bit.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in unsloth/qwen2-1.5b-instruct-bnb-4bit - will assume that the vocabulary was not modified.\n", + " warnings.warn(\n", + "{'loss': 0.7047, 'grad_norm': 1.4935038089752197, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 0.7433, 'grad_norm': 1.3313759565353394, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.6411, 'grad_norm': 1.1943646669387817, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.4593, 'grad_norm': 1.4492411613464355, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.4449, 'grad_norm': 1.6588672399520874, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + " 44%|███████████████▍ | 2500/5660 [2:02:16<2:35:11, 2.95s/it]/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/other.py:611: UserWarning: Unable to fetch remote file due to the following error (MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /unsloth/qwen2-1.5b-instruct-bnb-4bit/resolve/main/config.json (Caused by NameResolutionError(\": Failed to resolve \\'huggingface.co\\' ([Errno -3] Temporary failure in name resolution)\"))'), '(Request ID: 5665c6e3-3fe8-43f6-b849-73a605e379e9)') - silently ignoring the lookup for the file config.json in unsloth/qwen2-1.5b-instruct-bnb-4bit.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in unsloth/qwen2-1.5b-instruct-bnb-4bit - will assume that the vocabulary was not modified.\n", + " warnings.warn(\n", + "{'loss': 0.4778, 'grad_norm': 1.644708514213562, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.4785, 'grad_norm': 1.7970712184906006, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.487, 'grad_norm': 1.5495588779449463, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.3612, 'grad_norm': 1.3270775079727173, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.2908, 'grad_norm': 1.0414644479751587, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + " 53%|██████████████████▌ | 3000/5660 [2:27:09<2:06:06, 2.84s/it]/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/other.py:611: UserWarning: Unable to fetch remote file due to the following error (MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /unsloth/qwen2-1.5b-instruct-bnb-4bit/resolve/main/config.json (Caused by NameResolutionError(\": Failed to resolve \\'huggingface.co\\' ([Errno -3] Temporary failure in name resolution)\"))'), '(Request ID: 46085388-05e0-410c-b629-34f2cbb41b46)') - silently ignoring the lookup for the file config.json in unsloth/qwen2-1.5b-instruct-bnb-4bit.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in unsloth/qwen2-1.5b-instruct-bnb-4bit - will assume that the vocabulary was not modified.\n", + " warnings.warn(\n", + "{'loss': 0.2978, 'grad_norm': 1.3449839353561401, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.3143, 'grad_norm': 1.344139814376831, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.3089, 'grad_norm': 1.6479969024658203, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.3053, 'grad_norm': 0.8114176392555237, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.1932, 'grad_norm': 1.7617920637130737, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.2036, 'grad_norm': 1.1698989868164062, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.205, 'grad_norm': 1.1838085651397705, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.2078, 'grad_norm': 1.3557209968566895, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.2152, 'grad_norm': 1.0370357036590576, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.1935, 'grad_norm': 0.6839048862457275, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.1503, 'grad_norm': 0.8074870705604553, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.1571, 'grad_norm': 0.7514998912811279, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.1571, 'grad_norm': 0.7462531328201294, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.157, 'grad_norm': 0.773760199546814, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.1584, 'grad_norm': 1.2061128616333008, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.1365, 'grad_norm': 0.5050584077835083, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.1301, 'grad_norm': 0.6061640381813049, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.1314, 'grad_norm': 0.4381011724472046, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.1336, 'grad_norm': 0.4109954535961151, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.1334, 'grad_norm': 0.7153268456459045, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.1324, 'grad_norm': 0.5012986063957214, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.1179, 'grad_norm': 0.47853657603263855, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1182, 'grad_norm': 0.5196290612220764, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1194, 'grad_norm': 0.47395309805870056, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.1205, 'grad_norm': 0.8086031675338745, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1225, 'grad_norm': 0.43366023898124695, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 16717.8313, 'train_samples_per_second': 2.708, 'train_steps_per_second': 0.339, 'train_loss': 0.5991969135540535, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [4:38:37<00:00, 2.95s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "16717.8313 seconds used for training.\n", + "278.63 minutes used for training.\n", + "Peak reserved memory = 2.367 GB.\n", + "Peak reserved memory for training = 0.59 GB.\n", + "Peak reserved memory % of max memory = 19.735 %.\n", + "Peak reserved memory for training % of max memory = 4.919 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-1.5B-Instruct\n", + "100%|█████████████████████████████████████| 1133/1133 [2:14:26<00:00, 7.12s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his pistol, squinted through t...\n", + "\n", + "[1 rows x 14 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5.33 GB of memory reserved.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... This might take 10 minutes for Llama-7b... Done.\n", + "Unsloth: Merging 4bit and LoRA weights to 4bit...\n", + "This might take 5 minutes...\n", + "Done.\n", + "Unsloth: Saving 4bit Bitsandbytes model. Please wait...\n", + "model.safetensors: 100%|███████████████████| 1.22G/1.22G [02:18<00:00, 8.81MB/s]\n", + "README.md: 100%|███████████████████████████████| 581/581 [00:00<00:00, 3.67MB/s]\n", + "Saved merged_4bit model to https://huggingface.co/Qwen2-1.5B-Instruct-MAC-merged_4bit_forced\n" + ] + } + ], + "source": [ + "!./tune-small-2.sh" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "07_MAC_+_Qwen2-7B-Instructi_Unsloth_train", + "widgets": {} + }, + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "036fc5746f43416db18c19ad8fd36677": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "06e806c82c7b4cbea31c5358dd9c3434": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "087b76a8b7514269b1f0ab29b062e444": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a069d2ab23824f29aa320ac256e2cfe9", + "placeholder": "​", + "style": "IPY_MODEL_06e806c82c7b4cbea31c5358dd9c3434", + "value": "Map (num_proc=2): 100%" + } + }, + "09b76013aa9e45efb6deb23a7a0d0925": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dea41c5260884aa6879b5e1d1697b14f", + "placeholder": "​", + "style": "IPY_MODEL_89965917796a4f81b899fdc7685f33df", + "value": "config.json: 100%" + } + }, + "0a92c56bfa134ef583220d7ef0b13e17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c34be936c8145d3ab41282f30a70713": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f8b6bfe16894500838793f2491d403f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "177c78fce95d4b4ab33057c5a048d693": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1f44c9ce1adf470cbb19784493ed209f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c34be936c8145d3ab41282f30a70713", + "placeholder": "​", + "style": "IPY_MODEL_0a92c56bfa134ef583220d7ef0b13e17", + "value": "model.safetensors: 100%" + } + }, + "201b59ccd9f845e197029b57e424aefc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2157f01726d748f8a9ae4a00664430da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21db8a77b00d4a4e82fdfa608657531f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26e4202cca81496a90d15a0dd4ca9cf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba90fdb8822d47dab7ba203bee297f37", + "IPY_MODEL_61560ff6a36b44f4a9dfdae5c52791d4", + "IPY_MODEL_95fbe66647904c06a20f640630d6dc0e" + ], + "layout": "IPY_MODEL_57182a263d324a3dbf1471c74290a0d5" + } + }, + "27155728b6b84cb199c91c940095d0a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6b91feeed5464877991ac2c207aebe7c", + "IPY_MODEL_cca8113c54c0495daedce1327bf9c68b", + "IPY_MODEL_2e63a29e2f7247bba5beede9a568c99f" + ], + "layout": "IPY_MODEL_5c9d781c28944f3eb86e2a6d44efdf18" + } + }, + "271ddaa553a042d09b6db7b450643d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a58d04b428c46f4b3dbadd3bc6cd529": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d18ddf6482c4d97829ac0e5a7b9868f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "IPY_MODEL_f2df530d22c74977b249dd9fb5f4829b", + "IPY_MODEL_89b2ef0dbfea47ab8e6f8d659e3351d1" + ], + "layout": "IPY_MODEL_3056b148aa9f4e6e8aa3b61d26886255" + } + }, + "2e5087c76f98437cb5dc729230358cba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e63a29e2f7247bba5beede9a568c99f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b993eaec6b224440bf80c0958c6fb536", + "placeholder": "​", + "style": "IPY_MODEL_de868e26e7154f62aa86223a539ad421", + "value": " 464/464 [00:00<00:00, 27.1kB/s]" + } + }, + "2f6c70dd266c4816bfad3fd3d192929a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "30307300bc4e4baf96560e30969a82b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e36a3f9eff0e4cf68834d66b0213ae96", + "placeholder": "​", + "style": "IPY_MODEL_a0037bdccf254159becde630bee3d1db", + "value": "generation_config.json: 100%" + } + }, + "3056b148aa9f4e6e8aa3b61d26886255": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30cdc32298134cb0be4d41615b9e5774": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3572201bd4d74a58b7a665f9bdfdcdba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35b0e8c26d6640e9bd0ed7b242a423d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e5087c76f98437cb5dc729230358cba", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_036fc5746f43416db18c19ad8fd36677", + "value": 51760 + } + }, + "36166c7bcb854b34aca1f41a5d6ea50b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "370692d819df41828b48c4ad446f977b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "39b29a75374b45c0a22506010be2b84e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_30cdc32298134cb0be4d41615b9e5774", + "max": 1179, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_47928317548c454bba6358ab132e8dee", + "value": 1179 + } + }, + "3cf2dd993b5e4d3daecf61e4bab5a404": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_087b76a8b7514269b1f0ab29b062e444", + "IPY_MODEL_35b0e8c26d6640e9bd0ed7b242a423d8", + "IPY_MODEL_54ad89e05fd74576b9b8b5b5a10eaf8d" + ], + "layout": "IPY_MODEL_a41dc44766444a998bec2d777f249d23" + } + }, + "43dec2ede91341f5af60eb522e18e984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4463edd481c1467f914c7dcd6c6e6ffc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47928317548c454bba6358ab132e8dee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "49277aeeac16434a865a4d12308b1abc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4ae7e449e4ea4c729b5f34607c18ebae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b2061b8a73c43ffb0c2f83daf0d0183": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c4c88d4c701450692fa0f6b0c5764b0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c666f4ace3943f8b80ecd20e7503236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ccedf0d93094e63b57a0f8a434fba06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4463edd481c1467f914c7dcd6c6e6ffc", + "max": 44307561, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6d3b9a05db0b4dadb638c686faa0c40a", + "value": 44307561 + } + }, + "4dcf6ff672d24983a1877a8431709aa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5807d5fb827d490fb3bc698f801ffff5", + "placeholder": "​", + "style": "IPY_MODEL_c4f2b06a82fd4987b8b659524a7b503b", + "value": "Generating train split: 100%" + } + }, + "4ea63adfce694725bdba878aef709dd3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5234566b1bfc4655b8d582ea5b46ed9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "54ad89e05fd74576b9b8b5b5a10eaf8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdb1941405ed4e4aa06019933892deb3", + "placeholder": "​", + "style": "IPY_MODEL_668d5377ca56426a99753867e6e24862", + "value": " 51760/51760 [01:02<00:00, 1131.51 examples/s]" + } + }, + "56aee4853b7740e6a977254f5d1fa66d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "57182a263d324a3dbf1471c74290a0d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5807d5fb827d490fb3bc698f801ffff5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c9d781c28944f3eb86e2a6d44efdf18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f40db8173dd4d76b6ef5ed6d9ec8b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61560ff6a36b44f4a9dfdae5c52791d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db19fc8d37db4e45a5790a876836d8c4", + "max": 11610, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36166c7bcb854b34aca1f41a5d6ea50b", + "value": 11610 + } + }, + "6578fd7acdb54c4c93528ea431fd0144": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_370692d819df41828b48c4ad446f977b", + "placeholder": "​", + "style": "IPY_MODEL_a0bf9160eb2647409b3200270914b90f", + "value": " 50.6k/50.6k [00:00<00:00, 2.71MB/s]" + } + }, + "668d5377ca56426a99753867e6e24862": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697f027529b54ee9956bae78a11e0611": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69ac12aec0714318bf2c83d4f4e745f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b2012c3f88547af8884a9ea90e3164b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_938f45f1b3e24118b815d96ae34ba86a", + "placeholder": "​", + "style": "IPY_MODEL_9367047a800747f79c6b225d92397846", + "value": " 44.3M/44.3M [00:01<00:00, 31.0MB/s]" + } + }, + "6b91feeed5464877991ac2c207aebe7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b2061b8a73c43ffb0c2f83daf0d0183", + "placeholder": "​", + "style": "IPY_MODEL_69ac12aec0714318bf2c83d4f4e745f5", + "value": "special_tokens_map.json: 100%" + } + }, + "6d3b9a05db0b4dadb638c686faa0c40a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6dbbedeca9314e66ae50e44ffa31a414": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6e34619b45934040b6092e6fb01ea7fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71ce208e20d6483abb9ed923510c86d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d69dc491b3ab44d7852b21873ed7bb7f", + "placeholder": "​", + "style": "IPY_MODEL_f401d53bf28e44eb906bce6c05412662", + "value": " 51760/51760 [00:01<00:00, 45512.81 examples/s]" + } + }, + "7358cdad832342c983e31efb8754ab78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e352a3404f4c7dad0737f57d29e92f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_988a0e8c1f89446086858da0a891a79c", + "IPY_MODEL_4ccedf0d93094e63b57a0f8a434fba06", + "IPY_MODEL_6b2012c3f88547af8884a9ea90e3164b" + ], + "layout": "IPY_MODEL_7e29cb8dd4df4d5b94407cd8fd3f2011" + } + }, + "74501720ac7e4dbb911a4a99b3633bc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "78e5400bff924a92a4cc61c4ff18b182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9b313fd861948f5aba25b24b1518d30", + "placeholder": "​", + "style": "IPY_MODEL_4c666f4ace3943f8b80ecd20e7503236", + "value": " 1.18k/1.18k [00:00<00:00, 31.3kB/s]" + } + }, + "7975adbc2ec5489ea7fa0167e620d85c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e34619b45934040b6092e6fb01ea7fe", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_271ddaa553a042d09b6db7b450643d8f", + "value": 51760 + } + }, + "7e29cb8dd4df4d5b94407cd8fd3f2011": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "810ff6c0e17d4fa09a30fef27eacff90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89965917796a4f81b899fdc7685f33df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "89b2ef0dbfea47ab8e6f8d659e3351d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8908fa0df3743ecb9d12983a739104f", + "placeholder": "​", + "style": "IPY_MODEL_177c78fce95d4b4ab33057c5a048d693", + "value": " 9.09M/9.09M [00:00<00:00, 32.6MB/s]" + } + }, + "8b3505352a5a42bf910428c40ce40465": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_49277aeeac16434a865a4d12308b1abc", + "placeholder": "​", + "style": "IPY_MODEL_2157f01726d748f8a9ae4a00664430da", + "value": " 5.70G/5.70G [01:02<00:00, 30.1MB/s]" + } + }, + "8fc142b628fb40568730234de1cafde2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ae7e449e4ea4c729b5f34607c18ebae", + "max": 172, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3572201bd4d74a58b7a665f9bdfdcdba", + "value": 172 + } + }, + "9367047a800747f79c6b225d92397846": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "938f45f1b3e24118b815d96ae34ba86a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "95fbe66647904c06a20f640630d6dc0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b0a370dc20654b279b9680692e34418e", + "placeholder": "​", + "style": "IPY_MODEL_cfeb365ddf7548d58b2557f22737fcf5", + "value": " 11.6k/11.6k [00:00<00:00, 716kB/s]" + } + }, + "988a0e8c1f89446086858da0a891a79c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ad2be500fc164c0f86f33e914ef8e6a0", + "placeholder": "​", + "style": "IPY_MODEL_5234566b1bfc4655b8d582ea5b46ed9f", + "value": "Downloading data: 100%" + } + }, + "98c58f23f4d549518832cb2d18f796e8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_09b76013aa9e45efb6deb23a7a0d0925", + "IPY_MODEL_39b29a75374b45c0a22506010be2b84e", + "IPY_MODEL_78e5400bff924a92a4cc61c4ff18b182" + ], + "layout": "IPY_MODEL_2a58d04b428c46f4b3dbadd3bc6cd529" + } + }, + "99fdbb0300c14c139d1937c646f0cfe7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7358cdad832342c983e31efb8754ab78", + "placeholder": "​", + "style": "IPY_MODEL_e9adf418296e436fb48bb9f78885598b", + "value": " 51760/51760 [00:01<00:00, 38665.95 examples/s]" + } + }, + "9f679ad3ec7f4fe8ad0510ffb57bc2ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ea63adfce694725bdba878aef709dd3", + "placeholder": "​", + "style": "IPY_MODEL_74501720ac7e4dbb911a4a99b3633bc6", + "value": "tokenizer.json: 100%" + } + }, + "a0037bdccf254159becde630bee3d1db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a069d2ab23824f29aa320ac256e2cfe9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0bf9160eb2647409b3200270914b90f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a41dc44766444a998bec2d777f249d23": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8464a4c711e4e00aafdfc919b60d07e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fb995c740590427b882572c81d4e848c", + "placeholder": "​", + "style": "IPY_MODEL_201b59ccd9f845e197029b57e424aefc", + "value": " 172/172 [00:00<00:00, 12.0kB/s]" + } + }, + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad2be500fc164c0f86f33e914ef8e6a0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b0240cd9a4554b29ae11f8051984a1c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_edaf890370314a218f138015faa0b05d", + "placeholder": "​", + "style": "IPY_MODEL_697f027529b54ee9956bae78a11e0611", + "value": "Map: 100%" + } + }, + "b0a370dc20654b279b9680692e34418e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b518dcee69074b87be73957cd810e7ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d891f8d0b1fc462f8008d02bb2a15692", + "placeholder": "​", + "style": "IPY_MODEL_cced8fd7e998472794f3f3e3018956a5", + "value": "tokenizer_config.json: 100%" + } + }, + "b8908fa0df3743ecb9d12983a739104f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b993eaec6b224440bf80c0958c6fb536": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9b313fd861948f5aba25b24b1518d30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba90fdb8822d47dab7ba203bee297f37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f8b6bfe16894500838793f2491d403f", + "placeholder": "​", + "style": "IPY_MODEL_bb19f6c747754682a514373a3a0535ba", + "value": "Downloading readme: 100%" + } + }, + "bb19f6c747754682a514373a3a0535ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc883d4cf13e4f8b8a4fe5f410cb6efd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9159e03e61f4f56978ece9c3bca49b2", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_810ff6c0e17d4fa09a30fef27eacff90", + "value": 51760 + } + }, + "c161d94df0f04feba9542237e0856c22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c22f71b1f85843209d7e5321506b9cb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f44c9ce1adf470cbb19784493ed209f", + "IPY_MODEL_f1addc4479d849879e743cf9089e6540", + "IPY_MODEL_8b3505352a5a42bf910428c40ce40465" + ], + "layout": "IPY_MODEL_4c4c88d4c701450692fa0f6b0c5764b0" + } + }, + "c4f2b06a82fd4987b8b659524a7b503b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cca8113c54c0495daedce1327bf9c68b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e02f9b7849c64531835eb77b860d1c93", + "max": 464, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_56aee4853b7740e6a977254f5d1fa66d", + "value": 464 + } + }, + "cced8fd7e998472794f3f3e3018956a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cf245afeb1c04f29a24d291608c3d157": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b518dcee69074b87be73957cd810e7ed", + "IPY_MODEL_e29104486d594b2992d7285e0ef77371", + "IPY_MODEL_6578fd7acdb54c4c93528ea431fd0144" + ], + "layout": "IPY_MODEL_d35db8148a354c56aaac56dbae22536f" + } + }, + "cfe8cae0e22b495bafa221a63d13b283": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfeb365ddf7548d58b2557f22737fcf5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1b47d39450d4019ae85c9b2f943eeaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4dcf6ff672d24983a1877a8431709aa9", + "IPY_MODEL_7975adbc2ec5489ea7fa0167e620d85c", + "IPY_MODEL_71ce208e20d6483abb9ed923510c86d7" + ], + "layout": "IPY_MODEL_cfe8cae0e22b495bafa221a63d13b283" + } + }, + "d35db8148a354c56aaac56dbae22536f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69dc491b3ab44d7852b21873ed7bb7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d891f8d0b1fc462f8008d02bb2a15692": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8e5318cead340c4adbeaccc05d39225": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "daf4cd890b35422683d22fd30bc71e83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b0240cd9a4554b29ae11f8051984a1c6", + "IPY_MODEL_bc883d4cf13e4f8b8a4fe5f410cb6efd", + "IPY_MODEL_99fdbb0300c14c139d1937c646f0cfe7" + ], + "layout": "IPY_MODEL_c161d94df0f04feba9542237e0856c22" + } + }, + "db19fc8d37db4e45a5790a876836d8c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de868e26e7154f62aa86223a539ad421": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dea41c5260884aa6879b5e1d1697b14f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e02f9b7849c64531835eb77b860d1c93": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e29104486d594b2992d7285e0ef77371": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "max": 50641, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2f6c70dd266c4816bfad3fd3d192929a", + "value": 50641 + } + }, + "e36a3f9eff0e4cf68834d66b0213ae96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9159e03e61f4f56978ece9c3bca49b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9adf418296e436fb48bb9f78885598b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "edaf890370314a218f138015faa0b05d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f1addc4479d849879e743cf9089e6540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43dec2ede91341f5af60eb522e18e984", + "max": 5702746405, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d8e5318cead340c4adbeaccc05d39225", + "value": 5702746405 + } + }, + "f2df530d22c74977b249dd9fb5f4829b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_21db8a77b00d4a4e82fdfa608657531f", + "max": 9085698, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6dbbedeca9314e66ae50e44ffa31a414", + "value": 9085698 + } + }, + "f401d53bf28e44eb906bce6c05412662": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fb995c740590427b882572c81d4e848c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fce7a61c25ec4390af43d92b7c473a45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_30307300bc4e4baf96560e30969a82b6", + "IPY_MODEL_8fc142b628fb40568730234de1cafde2", + "IPY_MODEL_a8464a4c711e4e00aafdfc919b60d07e" + ], + "layout": "IPY_MODEL_5f40db8173dd4d76b6ef5ed6d9ec8b6e" + } + }, + "fdb1941405ed4e4aa06019933892deb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/05_tune-small-with-flash-attn.ipynb b/notebooks/05_tune-small-with-flash-attn.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..3a3e52a632a46eee834d5e0e6f85ece3e34594dc --- /dev/null +++ b/notebooks/05_tune-small-with-flash-attn.ipynb @@ -0,0 +1,4665 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "0ea8b46b-839b-445b-8043-ccdf4e920ace", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6d394937-6c99-4a7c-9d32-7600a280032f", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /home/inflaton/code/projects/courses/novel-translation\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9f67ec60-2f24-411c-84eb-0dd664b44775", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f1597656-8042-4878-9d3b-9ebfb8dd86dc", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('unsloth/Qwen2-0.5B-Instruct-bnb-4bit',\n", + " True,\n", + " None,\n", + " None,\n", + " 2048,\n", + " 10,\n", + " None,\n", + " 'datasets/mac/mac.tsv',\n", + " 'results/mac-results_v3.csv')" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "model_name = os.getenv(\"MODEL_NAME\")\n", + "token = os.getenv(\"HF_TOKEN\") or None\n", + "load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n", + "local_model = os.getenv(\"LOCAL_MODEL\")\n", + "hub_model = os.getenv(\"HUB_MODEL\")\n", + "num_train_epochs = int(os.getenv(\"NUM_TRAIN_EPOCHS\") or 0)\n", + "data_path = os.getenv(\"DATA_PATH\")\n", + "results_path = os.getenv(\"RESULTS_PATH\")\n", + "\n", + "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n", + "dtype = (\n", + " None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n", + ")\n", + "\n", + "model_name, load_in_4bit, local_model, hub_model, max_seq_length, num_train_epochs, dtype, data_path, results_path" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sun Jun 23 12:46:16 2024 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 545.23.07 Driver Version: 546.12 CUDA Version: 12.3 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 NVIDIA GeForce RTX 4080 ... On | 00000000:01:00.0 On | N/A |\n", + "| N/A 53C P8 5W / 150W | 452MiB / 12282MiB | 11% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "| No running processes found |\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "!nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 3 μs, sys: 1 μs, total: 4 μs\n", + "Wall time: 6.91 μs\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "# !pip install flash-attn --no-build-isolation" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: flash-attn\n", + "Version: 2.5.9.post1\n", + "Summary: Flash Attention: Fast and Memory-Efficient Exact Attention\n", + "Home-page: https://github.com/Dao-AILab/flash-attention\n", + "Author: Tri Dao\n", + "Author-email: trid@cs.stanford.edu\n", + "License: \n", + "Location: /home/inflaton/miniconda3/envs/unsloth_env/lib/python3.10/site-packages\n", + "Requires: einops, torch\n", + "Required-by: \n" + ] + } + ], + "source": [ + "!pip show flash-attn" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/novel-translation\n", + "Tuning unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/novel-translation/translation_engine_v3.py\n", + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n", + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit True 2048 10 None datasets/mac/mac.tsv results/mac-results_v3.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = True.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.633 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Teng raised his gun, closing his eyes and gripping a triangular eye. A boom of bullets rang out as he fired one, like thunder crashing down. The hammering sound of steel stones echoed through the branches of the trees. \n", + "\n", + "The noise was so loud that it made my heart beat faster.\n", + "--------\n", + "step 3: Old Teng raised his gun, closing his eyes and gripping a triangular eye. A boom of bullets rang out as he fired one, like thunder crashing down. The hammering sound of steel stones echoed through the branches of the trees. \n", + "\n", + "The noise was so loud that it made my heart beat faster.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:06:42<00:00, 3.53s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Teng raised his gun, closing his eyes and ...\n", + "\n", + "[1 rows x 3 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.023 GB of memory reserved.\n", + "Unsloth 2024.6 patched 24 layers with 0 QKV layers, 24 O layers and 24 MLP layers.\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.023 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 8,798,208\n", + "{'loss': 1.9401, 'grad_norm': 0.9639493823051453, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.7762, 'grad_norm': 0.8060959577560425, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.7146, 'grad_norm': 0.9296559691429138, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.7155, 'grad_norm': 0.7544056177139282, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.6861, 'grad_norm': 0.8301573395729065, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.6078, 'grad_norm': 0.8045125007629395, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.4921, 'grad_norm': 0.9031914472579956, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.5401, 'grad_norm': 0.9808986186981201, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.4721, 'grad_norm': 1.0783536434173584, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.4686, 'grad_norm': 0.988545835018158, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.5278, 'grad_norm': 1.0253351926803589, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.3493, 'grad_norm': 1.1621149778366089, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 1.2156, 'grad_norm': 1.1786366701126099, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.2572, 'grad_norm': 1.3017158508300781, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.2552, 'grad_norm': 1.2804787158966064, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.2664, 'grad_norm': 1.1679364442825317, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.2346, 'grad_norm': 1.171284794807434, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.9801, 'grad_norm': 1.5025601387023926, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.9924, 'grad_norm': 1.5331358909606934, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.9986, 'grad_norm': 1.5144548416137695, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.9813, 'grad_norm': 1.5257072448730469, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 1.0233, 'grad_norm': 1.5738170146942139, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.9316, 'grad_norm': 1.5346697568893433, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.7391, 'grad_norm': 1.7242717742919922, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.7396, 'grad_norm': 1.7575305700302124, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.7765, 'grad_norm': 2.111323833465576, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.7738, 'grad_norm': 2.2442333698272705, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.7942, 'grad_norm': 2.135150909423828, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.6302, 'grad_norm': 1.8503246307373047, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.5572, 'grad_norm': 1.59291410446167, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.5616, 'grad_norm': 1.7540444135665894, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.5898, 'grad_norm': 2.267761707305908, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.5818, 'grad_norm': 2.345642328262329, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.5804, 'grad_norm': 2.3560101985931396, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.3973, 'grad_norm': 3.8223047256469727, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.4132, 'grad_norm': 1.865454912185669, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.4299, 'grad_norm': 2.473954200744629, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.422, 'grad_norm': 1.6875197887420654, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.4284, 'grad_norm': 1.5689966678619385, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.3869, 'grad_norm': 1.6680887937545776, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.2934, 'grad_norm': 1.7025184631347656, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.3139, 'grad_norm': 1.5031529664993286, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.3133, 'grad_norm': 1.9866334199905396, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.306, 'grad_norm': 2.1866486072540283, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.3183, 'grad_norm': 1.4326164722442627, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.2477, 'grad_norm': 1.3497223854064941, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.2262, 'grad_norm': 1.6162991523742676, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.2302, 'grad_norm': 1.0059006214141846, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.236, 'grad_norm': 1.3581494092941284, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.2366, 'grad_norm': 1.891054391860962, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.238, 'grad_norm': 1.0669790506362915, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.1814, 'grad_norm': 1.2125357389450073, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1897, 'grad_norm': 1.044737696647644, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1873, 'grad_norm': 1.1148860454559326, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.189, 'grad_norm': 1.8938679695129395, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1892, 'grad_norm': 1.0202747583389282, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 13804.2064, 'train_samples_per_second': 3.28, 'train_steps_per_second': 0.41, 'train_loss': 0.7992495260474539, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [3:50:04<00:00, 2.44s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "13804.2064 seconds used for training.\n", + "230.07 minutes used for training.\n", + "Peak reserved memory = 3.023 GB.\n", + "Peak reserved memory for training = 0.0 GB.\n", + "Peak reserved memory % of max memory = 25.204 %.\n", + "Peak reserved memory for training % of max memory = 0.0 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his rifle, squinted his triangular eye, and fired – a gun, like a ladle, crackled as the shot fell down towards him.\n", + "--------\n", + "step 3: Old Geng raised his rifle, squinted his triangular eye, and fired – a gun, like a ladle, crackled as the shot fell down towards him.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:39:47<00:00, 5.28s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his rifle, squinted his triang...\n", + "\n", + "[1 rows x 4 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.023 GB of memory reserved.\n", + "README.md: 100%|███████████████████████████████| 593/593 [00:00<00:00, 3.39MB/s]\n", + "model.safetensors: 100%|█████████████████████| 493M/493M [00:41<00:00, 11.9MB/s]\n", + "Saved model to https://huggingface.co/Qwen2-0.5B-Instruct-bnb-4bit-MAC-lora\n", + "README.md: 100%|███████████████████████████████| 599/599 [00:00<00:00, 2.91MB/s]\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... Done.\n", + "Unsloth: Saving LoRA adapters. Please wait...\n", + "401 Client Error: Unauthorized for url: https://huggingface.co/api/repos/create (Request ID: Root=1-667805f5-6581fe263332f4220001e82f;27494d10-63ec-4751-8bd3-059f4a0c16c7)\n", + "\n", + "Invalid username or password.\n", + "Tuning unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/novel-translation/translation_engine_v3.py\n", + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit True 2048 10 None datasets/mac/mac.tsv results/mac-results_v3.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = True.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.516 GB of memory reserved.\n", + "loading train/test data files\n", + "Map: 100%|█████████████████████████| 4528/4528 [00:00<00:00, 6537.59 examples/s]\n", + "Map: 100%|█████████████████████████| 1133/1133 [00:00<00:00, 7281.32 examples/s]\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geer lifted his gun, squinting one eye as he pulled the trigger. A hail of bullets rained down from his rifle. Golden sparrows plopped down from the trees, sandstones flying through the willows, making a clattering sound.\n", + "--------\n", + "step 3: Old Geer lifted his gun, squinting one eye as he pulled the trigger. A hail of bullets rained down from his rifle. Golden sparrows plopped down from the trees, sandstones flying through the willows, making a clattering sound.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:33:33<00:00, 4.95s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geer lifted his gun, squinting one eye as ...\n", + "\n", + "[1 rows x 5 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.945 GB of memory reserved.\n", + "Unsloth 2024.6 patched 28 layers with 0 QKV layers, 28 O layers and 28 MLP layers.\n", + "Map (num_proc=2): 100%|████████████| 4528/4528 [00:01<00:00, 2276.92 examples/s]\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.945 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 18,464,768\n", + "{'loss': 1.7411, 'grad_norm': 0.658531665802002, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.5679, 'grad_norm': 0.6339693665504456, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.5154, 'grad_norm': 0.6460090279579163, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.5171, 'grad_norm': 0.5758596658706665, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.4961, 'grad_norm': 0.5699881315231323, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.4185, 'grad_norm': 0.6029691100120544, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.2981, 'grad_norm': 0.7669411301612854, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.3393, 'grad_norm': 0.6863419413566589, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.2785, 'grad_norm': 0.9242411255836487, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.2799, 'grad_norm': 0.8570270538330078, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.3319, 'grad_norm': 1.2672839164733887, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.1278, 'grad_norm': 0.9834404587745667, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 0.9963, 'grad_norm': 1.082615852355957, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.0299, 'grad_norm': 1.1531076431274414, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.0355, 'grad_norm': 1.0619866847991943, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.0358, 'grad_norm': 1.0393147468566895, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.0088, 'grad_norm': 0.8888176679611206, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.6966, 'grad_norm': 1.2902939319610596, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.7113, 'grad_norm': 1.3367533683776855, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.7152, 'grad_norm': 1.3983240127563477, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.7054, 'grad_norm': 1.503348469734192, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 0.7441, 'grad_norm': 1.2397220134735107, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.644, 'grad_norm': 1.1092045307159424, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.46, 'grad_norm': 1.3230023384094238, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.4453, 'grad_norm': 1.4391876459121704, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.4831, 'grad_norm': 1.8462566137313843, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.4777, 'grad_norm': 2.09181547164917, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.4871, 'grad_norm': 1.7411134243011475, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.3573, 'grad_norm': 1.5780448913574219, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.2919, 'grad_norm': 1.4988445043563843, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.2988, 'grad_norm': 0.9282442927360535, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.3139, 'grad_norm': 1.3400734663009644, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.3079, 'grad_norm': 1.5739268064498901, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.3078, 'grad_norm': 0.6474704742431641, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.194, 'grad_norm': 1.3646390438079834, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.2034, 'grad_norm': 1.275925636291504, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.2082, 'grad_norm': 1.2210203409194946, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.205, 'grad_norm': 0.8089584112167358, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.2143, 'grad_norm': 1.1388084888458252, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.195, 'grad_norm': 0.7365647554397583, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.148, 'grad_norm': 0.5169339179992676, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.1574, 'grad_norm': 0.9203131198883057, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.1573, 'grad_norm': 0.7963287830352783, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.1567, 'grad_norm': 0.6291562914848328, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.1591, 'grad_norm': 1.3627387285232544, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.1367, 'grad_norm': 0.6390063166618347, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.1307, 'grad_norm': 0.6079268455505371, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.1313, 'grad_norm': 0.47809019684791565, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.1333, 'grad_norm': 0.41499361395835876, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.1335, 'grad_norm': 0.6223477125167847, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.1331, 'grad_norm': 0.4558337330818176, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.118, 'grad_norm': 0.4607541561126709, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1186, 'grad_norm': 0.49737748503685, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1197, 'grad_norm': 0.46176978945732117, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.1203, 'grad_norm': 0.48998674750328064, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1231, 'grad_norm': 0.443093866109848, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 15713.8663, 'train_samples_per_second': 2.882, 'train_steps_per_second': 0.36, 'train_loss': 0.5995376785736624, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [4:21:53<00:00, 2.78s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "15713.8663 seconds used for training.\n", + "261.9 minutes used for training.\n", + "Peak reserved memory = 3.945 GB.\n", + "Peak reserved memory for training = 0.0 GB.\n", + "Peak reserved memory % of max memory = 32.891 %.\n", + "Peak reserved memory for training % of max memory = 0.0 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised the pistol to his eye, squeezed the trigger, and some of the shot flew straight into the sky, like ice pellets, as spattered tin shells burst against the willows.\n", + "--------\n", + "step 3: Old Geng raised the pistol to his eye, squeezed the trigger, and some of the shot flew straight into the sky, like ice pellets, as spattered tin shells burst against the willows.\n", + "100%|█████████████████████████████████████| 1133/1133 [2:11:23<00:00, 6.96s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised the pistol to his eye, squeeze...\n", + "\n", + "[1 rows x 6 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5.33 GB of memory reserved.\n", + "README.md: 100%|███████████████████████████████| 593/593 [00:00<00:00, 3.08MB/s]\n", + "model.safetensors: 100%|███████████████████| 1.22G/1.22G [02:18<00:00, 8.82MB/s]\n", + "Saved model to https://huggingface.co/Qwen2-1.5B-Instruct-bnb-4bit-MAC-lora\n", + "README.md: 100%|███████████████████████████████| 599/599 [00:00<00:00, 2.99MB/s]\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... Done.\n", + "Unsloth: Saving LoRA adapters. Please wait...\n", + "401 Client Error: Unauthorized for url: https://huggingface.co/api/repos/create (Request ID: Root=1-66787955-4b5759766262340722a532d6;dcdc16ae-e45d-406a-81c9-13d22426edcb)\n", + "\n", + "Invalid username or password.\n", + "CPU times: user 23min 32s, sys: 8min 51s, total: 32min 24s\n", + "Wall time: 14h 50min 41s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./tune-small.sh" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/novel-translation\n", + "Tuning unsloth/Qwen2-0.5B-Instruct\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/novel-translation/translation_engine_v3.py\n", + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n", + "unsloth/Qwen2-0.5B-Instruct True 2048 10 None datasets/mac/mac.tsv results/mac-results_v3.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-0.5B-Instruct\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = True.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.633 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old耿举起枪，眯着眼睛，枪声轰鸣，子弹砰砰砰地落在地上，一颗颗冰雹般的大鸟扑棱棱地落在柳树之间，咔嚓一声。\n", + "--------\n", + "step 3: Old耿举起枪，眯着眼睛，枪声轰鸣，子弹砰砰砰地落在地上，一颗颗冰雹般的大鸟扑棱棱地落在柳树之间，咔嚓一声。\n", + "100%|█████████████████████████████████████| 1133/1133 [1:07:20<00:00, 3.57s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old耿举起枪，眯着眼睛，枪声轰鸣，子弹砰砰砰地落在地上，一颗颗冰雹般的大鸟扑棱棱地落在柳树...\n", + "\n", + "[1 rows x 7 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.873 GB of memory reserved.\n", + "Unsloth 2024.6 patched 24 layers with 0 QKV layers, 24 O layers and 24 MLP layers.\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.873 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 8,798,208\n", + "{'loss': 1.9401, 'grad_norm': 0.9639493823051453, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.7763, 'grad_norm': 0.8041688799858093, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.7147, 'grad_norm': 0.93106609582901, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.7156, 'grad_norm': 0.753624677658081, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.6862, 'grad_norm': 0.823365330696106, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.6076, 'grad_norm': 0.807159423828125, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.492, 'grad_norm': 0.9032222032546997, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.5407, 'grad_norm': 0.9780230522155762, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.4722, 'grad_norm': 1.0792107582092285, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.4688, 'grad_norm': 0.9899805784225464, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.5279, 'grad_norm': 1.0187550783157349, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.3492, 'grad_norm': 1.1653039455413818, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 1.2155, 'grad_norm': 1.1759636402130127, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.258, 'grad_norm': 1.3161606788635254, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.2556, 'grad_norm': 1.345459222793579, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.2664, 'grad_norm': 1.1774756908416748, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.2348, 'grad_norm': 1.1240969896316528, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.9803, 'grad_norm': 1.504067301750183, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.9925, 'grad_norm': 1.5067857503890991, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.9989, 'grad_norm': 1.520134449005127, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.9807, 'grad_norm': 1.5608190298080444, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 1.0231, 'grad_norm': 1.6115481853485107, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.9304, 'grad_norm': 1.5296086072921753, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.7399, 'grad_norm': 1.6573024988174438, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.7408, 'grad_norm': 1.7512829303741455, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.7755, 'grad_norm': 2.100616216659546, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.7736, 'grad_norm': 2.121138572692871, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.7939, 'grad_norm': 2.1208739280700684, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.6306, 'grad_norm': 1.7874706983566284, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.5555, 'grad_norm': 1.7197502851486206, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.5615, 'grad_norm': 1.6627233028411865, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.5916, 'grad_norm': 1.8348921537399292, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.5824, 'grad_norm': 2.2089450359344482, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.5792, 'grad_norm': 2.447774648666382, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.3998, 'grad_norm': 1.8090907335281372, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.4113, 'grad_norm': 1.8496599197387695, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.4296, 'grad_norm': 2.046454429626465, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.4214, 'grad_norm': 1.8460564613342285, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.4279, 'grad_norm': 1.7839864492416382, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.3849, 'grad_norm': 1.6806727647781372, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.2936, 'grad_norm': 1.7148785591125488, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.3127, 'grad_norm': 1.2444647550582886, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.3143, 'grad_norm': 2.0613274574279785, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.3047, 'grad_norm': 1.9974720478057861, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.3181, 'grad_norm': 1.7139118909835815, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.247, 'grad_norm': 1.3952387571334839, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.226, 'grad_norm': 1.4544029235839844, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.2295, 'grad_norm': 1.3040739297866821, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.2367, 'grad_norm': 1.0945595502853394, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.236, 'grad_norm': 2.110018491744995, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.2384, 'grad_norm': 1.0737488269805908, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.181, 'grad_norm': 1.0586763620376587, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.19, 'grad_norm': 1.11255943775177, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1869, 'grad_norm': 1.0752365589141846, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.1881, 'grad_norm': 1.592451810836792, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1889, 'grad_norm': 1.070407748222351, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 13633.0365, 'train_samples_per_second': 3.321, 'train_steps_per_second': 0.415, 'train_loss': 0.7991274287759625, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [3:47:13<00:00, 2.41s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "13633.0365 seconds used for training.\n", + "227.22 minutes used for training.\n", + "Peak reserved memory = 1.369 GB.\n", + "Peak reserved memory for training = 0.496 GB.\n", + "Peak reserved memory % of max memory = 11.414 %.\n", + "Peak reserved memory for training % of max memory = 4.135 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his rifle and tilted his head, clasping the trigger, and a crash of iron shrapnel fell beside him, splashing over time boundaries and scattering like ice rain.\n", + "--------\n", + "step 3: Old Geng raised his rifle and tilted his head, clasping the trigger, and a crash of iron shrapnel fell beside him, splashing over time boundaries and scattering like ice rain.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:36:22<00:00, 5.10s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his rifle and tilted his head,...\n", + "\n", + "[1 rows x 8 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.369 GB of memory reserved.\n", + "README.md: 100%|███████████████████████████████| 599/599 [00:00<00:00, 3.47MB/s]\n", + "model.safetensors: 100%|█████████████████████| 493M/493M [01:42<00:00, 4.83MB/s]\n", + "Saved model to https://huggingface.co/Qwen2-0.5B-Instruct-MAC-lora\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... Done.\n", + "Unsloth: Saving LoRA adapters. Please wait...\n", + "401 Client Error: Unauthorized for url: https://huggingface.co/api/repos/create (Request ID: Root=1-6678d5ab-7ebfba970b06941d330c774c;eef78700-1bc6-4a4b-82dc-3a0513f85a69)\n", + "\n", + "Invalid username or password.\n", + "Tuning unsloth/Qwen2-1.5B-Instruct\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/novel-translation/translation_engine_v3.py\n", + "loading env vars from: /home/inflaton/code/projects/courses/novel-translation/.env\n", + "unsloth/Qwen2-1.5B-Instruct True 2048 10 None datasets/mac/mac.tsv results/mac-results_v3.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-1.5B-Instruct\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = True.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.516 GB of memory reserved.\n", + "loading train/test data files\n", + "Map: 100%|█████████████████████████| 4528/4528 [00:00<00:00, 9716.63 examples/s]\n", + "Map: 100%|█████████████████████████| 1133/1133 [00:00<00:00, 5762.27 examples/s]\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geer lifted his gun, squinted one of his eyes, clutched it, and fired off a shot like hail of golden sparrows, sand grains flying from the willows, making a sound.\n", + "--------\n", + "step 3: Old Geer lifted his gun, squinted one of his eyes, clutched it, and fired off a shot like hail of golden sparrows, sand grains flying from the willows, making a sound.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:08:04<00:00, 3.61s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geer lifted his gun, squinted one of his e...\n", + "\n", + "[1 rows x 9 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.758 GB of memory reserved.\n", + "Unsloth 2024.6 patched 28 layers with 0 QKV layers, 28 O layers and 28 MLP layers.\n", + "Map (num_proc=2): 100%|████████████| 4528/4528 [00:02<00:00, 2227.71 examples/s]\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.758 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 18,464,768\n", + "{'loss': 1.7417, 'grad_norm': 0.6435620188713074, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.5681, 'grad_norm': 0.6191736459732056, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.516, 'grad_norm': 0.6416980028152466, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.5173, 'grad_norm': 0.5749340057373047, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.4962, 'grad_norm': 0.5725200176239014, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.4184, 'grad_norm': 0.6020503640174866, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.2973, 'grad_norm': 0.7252504229545593, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.3397, 'grad_norm': 0.6848253011703491, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.2787, 'grad_norm': 0.9209019541740417, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.2797, 'grad_norm': 0.8531824946403503, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.3313, 'grad_norm': 0.8282666206359863, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.1277, 'grad_norm': 0.9999461770057678, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 0.9964, 'grad_norm': 1.1321799755096436, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.0302, 'grad_norm': 1.2508618831634521, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.0345, 'grad_norm': 1.0669715404510498, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.0366, 'grad_norm': 1.0575922727584839, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.0088, 'grad_norm': 0.8964874148368835, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.697, 'grad_norm': 1.2907732725143433, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.7093, 'grad_norm': 1.3694052696228027, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.7154, 'grad_norm': 1.3360122442245483, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.7039, 'grad_norm': 1.3949358463287354, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 0.7428, 'grad_norm': 1.2718554735183716, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.643, 'grad_norm': 1.135117769241333, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.4595, 'grad_norm': 1.454145908355713, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.444, 'grad_norm': 1.6696418523788452, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.4826, 'grad_norm': 1.7666652202606201, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.4795, 'grad_norm': 1.742395281791687, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.4856, 'grad_norm': 1.5103402137756348, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.3593, 'grad_norm': 1.3078151941299438, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.2904, 'grad_norm': 1.2514950037002563, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.2991, 'grad_norm': 1.658071756362915, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.3144, 'grad_norm': 1.6055220365524292, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.3093, 'grad_norm': 1.5153316259384155, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.3058, 'grad_norm': 0.9724624752998352, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.1934, 'grad_norm': 0.954325795173645, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.2036, 'grad_norm': 1.537642240524292, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.2079, 'grad_norm': 1.2404054403305054, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.2049, 'grad_norm': 1.0820835828781128, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.215, 'grad_norm': 1.32541024684906, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.1946, 'grad_norm': 0.7059425711631775, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.1489, 'grad_norm': 1.0154873132705688, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.1581, 'grad_norm': 0.7846829891204834, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.1593, 'grad_norm': 0.9743372201919556, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.1568, 'grad_norm': 0.8308056592941284, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.1587, 'grad_norm': 0.8165437579154968, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.137, 'grad_norm': 1.081515908241272, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.1305, 'grad_norm': 0.6178275942802429, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.1318, 'grad_norm': 0.4706704914569855, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.1331, 'grad_norm': 0.41128799319267273, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.1349, 'grad_norm': 0.674155056476593, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.1327, 'grad_norm': 0.49978604912757874, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.1177, 'grad_norm': 0.44771721959114075, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1187, 'grad_norm': 0.587639331817627, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1198, 'grad_norm': 0.47292351722717285, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.1203, 'grad_norm': 0.5652945041656494, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1232, 'grad_norm': 0.93674236536026, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 15791.9321, 'train_samples_per_second': 2.867, 'train_steps_per_second': 0.358, 'train_loss': 0.5995039068767966, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [4:23:11<00:00, 2.79s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "15791.9321 seconds used for training.\n", + "263.2 minutes used for training.\n", + "Peak reserved memory = 2.365 GB.\n", + "Peak reserved memory for training = 0.607 GB.\n", + "Peak reserved memory % of max memory = 19.718 %.\n", + "Peak reserved memory for training % of max memory = 5.061 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his gun, squinted, and emptied it. The cocoon of bullets split open as they flew, like ice pellets, until it was gone, sending chiseling sounds tumbling through the air as iron seeds smashed pruneflower stems.\n", + "--------\n", + "step 3: Old Geng raised his gun, squinted, and emptied it. The cocoon of bullets split open as they flew, like ice pellets, until it was gone, sending chiseling sounds tumbling through the air as iron seeds smashed pruneflower stems.\n", + "100%|█████████████████████████████████████| 1133/1133 [2:10:28<00:00, 6.91s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his gun, squinted, and emptied...\n", + "\n", + "[1 rows x 10 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5.33 GB of memory reserved.\n", + "README.md: 100%|███████████████████████████████| 599/599 [00:00<00:00, 3.36MB/s]\n", + "model.safetensors: 100%|███████████████████| 1.22G/1.22G [04:57<00:00, 4.09MB/s]\n", + "Saved model to https://huggingface.co/Qwen2-1.5B-Instruct-MAC-lora\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... Done.\n", + "Unsloth: Saving LoRA adapters. Please wait...\n", + "401 Client Error: Unauthorized for url: https://huggingface.co/api/repos/create (Request ID: Root=1-667943a2-3b0230ba1811ed550b585d53;2d0f1d17-f232-41f6-9eab-add0e87114f8)\n", + "\n", + "Invalid username or password.\n", + "CPU times: user 23min 30s, sys: 8min 24s, total: 31min 55s\n", + "Wall time: 14h 23min 14s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./tune-small-2.sh" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "07_MAC_+_Qwen2-7B-Instructi_Unsloth_train", + "widgets": {} + }, + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "036fc5746f43416db18c19ad8fd36677": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "06e806c82c7b4cbea31c5358dd9c3434": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "087b76a8b7514269b1f0ab29b062e444": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a069d2ab23824f29aa320ac256e2cfe9", + "placeholder": "​", + "style": "IPY_MODEL_06e806c82c7b4cbea31c5358dd9c3434", + "value": "Map (num_proc=2): 100%" + } + }, + "09b76013aa9e45efb6deb23a7a0d0925": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dea41c5260884aa6879b5e1d1697b14f", + "placeholder": "​", + "style": "IPY_MODEL_89965917796a4f81b899fdc7685f33df", + "value": "config.json: 100%" + } + }, + "0a92c56bfa134ef583220d7ef0b13e17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c34be936c8145d3ab41282f30a70713": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f8b6bfe16894500838793f2491d403f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "177c78fce95d4b4ab33057c5a048d693": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1f44c9ce1adf470cbb19784493ed209f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c34be936c8145d3ab41282f30a70713", + "placeholder": "​", + "style": "IPY_MODEL_0a92c56bfa134ef583220d7ef0b13e17", + "value": "model.safetensors: 100%" + } + }, + "201b59ccd9f845e197029b57e424aefc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2157f01726d748f8a9ae4a00664430da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21db8a77b00d4a4e82fdfa608657531f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26e4202cca81496a90d15a0dd4ca9cf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba90fdb8822d47dab7ba203bee297f37", + "IPY_MODEL_61560ff6a36b44f4a9dfdae5c52791d4", + "IPY_MODEL_95fbe66647904c06a20f640630d6dc0e" + ], + "layout": "IPY_MODEL_57182a263d324a3dbf1471c74290a0d5" + } + }, + "27155728b6b84cb199c91c940095d0a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6b91feeed5464877991ac2c207aebe7c", + "IPY_MODEL_cca8113c54c0495daedce1327bf9c68b", + "IPY_MODEL_2e63a29e2f7247bba5beede9a568c99f" + ], + "layout": "IPY_MODEL_5c9d781c28944f3eb86e2a6d44efdf18" + } + }, + "271ddaa553a042d09b6db7b450643d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a58d04b428c46f4b3dbadd3bc6cd529": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d18ddf6482c4d97829ac0e5a7b9868f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "IPY_MODEL_f2df530d22c74977b249dd9fb5f4829b", + "IPY_MODEL_89b2ef0dbfea47ab8e6f8d659e3351d1" + ], + "layout": "IPY_MODEL_3056b148aa9f4e6e8aa3b61d26886255" + } + }, + "2e5087c76f98437cb5dc729230358cba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e63a29e2f7247bba5beede9a568c99f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b993eaec6b224440bf80c0958c6fb536", + "placeholder": "​", + "style": "IPY_MODEL_de868e26e7154f62aa86223a539ad421", + "value": " 464/464 [00:00<00:00, 27.1kB/s]" + } + }, + "2f6c70dd266c4816bfad3fd3d192929a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "30307300bc4e4baf96560e30969a82b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e36a3f9eff0e4cf68834d66b0213ae96", + "placeholder": "​", + "style": "IPY_MODEL_a0037bdccf254159becde630bee3d1db", + "value": "generation_config.json: 100%" + } + }, + "3056b148aa9f4e6e8aa3b61d26886255": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30cdc32298134cb0be4d41615b9e5774": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3572201bd4d74a58b7a665f9bdfdcdba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35b0e8c26d6640e9bd0ed7b242a423d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e5087c76f98437cb5dc729230358cba", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_036fc5746f43416db18c19ad8fd36677", + "value": 51760 + } + }, + "36166c7bcb854b34aca1f41a5d6ea50b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "370692d819df41828b48c4ad446f977b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "39b29a75374b45c0a22506010be2b84e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_30cdc32298134cb0be4d41615b9e5774", + "max": 1179, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_47928317548c454bba6358ab132e8dee", + "value": 1179 + } + }, + "3cf2dd993b5e4d3daecf61e4bab5a404": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_087b76a8b7514269b1f0ab29b062e444", + "IPY_MODEL_35b0e8c26d6640e9bd0ed7b242a423d8", + "IPY_MODEL_54ad89e05fd74576b9b8b5b5a10eaf8d" + ], + "layout": "IPY_MODEL_a41dc44766444a998bec2d777f249d23" + } + }, + "43dec2ede91341f5af60eb522e18e984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4463edd481c1467f914c7dcd6c6e6ffc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47928317548c454bba6358ab132e8dee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "49277aeeac16434a865a4d12308b1abc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4ae7e449e4ea4c729b5f34607c18ebae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b2061b8a73c43ffb0c2f83daf0d0183": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c4c88d4c701450692fa0f6b0c5764b0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c666f4ace3943f8b80ecd20e7503236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ccedf0d93094e63b57a0f8a434fba06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4463edd481c1467f914c7dcd6c6e6ffc", + "max": 44307561, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6d3b9a05db0b4dadb638c686faa0c40a", + "value": 44307561 + } + }, + "4dcf6ff672d24983a1877a8431709aa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5807d5fb827d490fb3bc698f801ffff5", + "placeholder": "​", + "style": "IPY_MODEL_c4f2b06a82fd4987b8b659524a7b503b", + "value": "Generating train split: 100%" + } + }, + "4ea63adfce694725bdba878aef709dd3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5234566b1bfc4655b8d582ea5b46ed9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "54ad89e05fd74576b9b8b5b5a10eaf8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdb1941405ed4e4aa06019933892deb3", + "placeholder": "​", + "style": "IPY_MODEL_668d5377ca56426a99753867e6e24862", + "value": " 51760/51760 [01:02<00:00, 1131.51 examples/s]" + } + }, + "56aee4853b7740e6a977254f5d1fa66d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "57182a263d324a3dbf1471c74290a0d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5807d5fb827d490fb3bc698f801ffff5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c9d781c28944f3eb86e2a6d44efdf18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f40db8173dd4d76b6ef5ed6d9ec8b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61560ff6a36b44f4a9dfdae5c52791d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db19fc8d37db4e45a5790a876836d8c4", + "max": 11610, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36166c7bcb854b34aca1f41a5d6ea50b", + "value": 11610 + } + }, + "6578fd7acdb54c4c93528ea431fd0144": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_370692d819df41828b48c4ad446f977b", + "placeholder": "​", + "style": "IPY_MODEL_a0bf9160eb2647409b3200270914b90f", + "value": " 50.6k/50.6k [00:00<00:00, 2.71MB/s]" + } + }, + "668d5377ca56426a99753867e6e24862": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697f027529b54ee9956bae78a11e0611": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69ac12aec0714318bf2c83d4f4e745f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b2012c3f88547af8884a9ea90e3164b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_938f45f1b3e24118b815d96ae34ba86a", + "placeholder": "​", + "style": "IPY_MODEL_9367047a800747f79c6b225d92397846", + "value": " 44.3M/44.3M [00:01<00:00, 31.0MB/s]" + } + }, + "6b91feeed5464877991ac2c207aebe7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b2061b8a73c43ffb0c2f83daf0d0183", + "placeholder": "​", + "style": "IPY_MODEL_69ac12aec0714318bf2c83d4f4e745f5", + "value": "special_tokens_map.json: 100%" + } + }, + "6d3b9a05db0b4dadb638c686faa0c40a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6dbbedeca9314e66ae50e44ffa31a414": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6e34619b45934040b6092e6fb01ea7fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71ce208e20d6483abb9ed923510c86d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d69dc491b3ab44d7852b21873ed7bb7f", + "placeholder": "​", + "style": "IPY_MODEL_f401d53bf28e44eb906bce6c05412662", + "value": " 51760/51760 [00:01<00:00, 45512.81 examples/s]" + } + }, + "7358cdad832342c983e31efb8754ab78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e352a3404f4c7dad0737f57d29e92f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_988a0e8c1f89446086858da0a891a79c", + "IPY_MODEL_4ccedf0d93094e63b57a0f8a434fba06", + "IPY_MODEL_6b2012c3f88547af8884a9ea90e3164b" + ], + "layout": "IPY_MODEL_7e29cb8dd4df4d5b94407cd8fd3f2011" + } + }, + "74501720ac7e4dbb911a4a99b3633bc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "78e5400bff924a92a4cc61c4ff18b182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9b313fd861948f5aba25b24b1518d30", + "placeholder": "​", + "style": "IPY_MODEL_4c666f4ace3943f8b80ecd20e7503236", + "value": " 1.18k/1.18k [00:00<00:00, 31.3kB/s]" + } + }, + "7975adbc2ec5489ea7fa0167e620d85c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e34619b45934040b6092e6fb01ea7fe", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_271ddaa553a042d09b6db7b450643d8f", + "value": 51760 + } + }, + "7e29cb8dd4df4d5b94407cd8fd3f2011": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "810ff6c0e17d4fa09a30fef27eacff90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89965917796a4f81b899fdc7685f33df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "89b2ef0dbfea47ab8e6f8d659e3351d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8908fa0df3743ecb9d12983a739104f", + "placeholder": "​", + "style": "IPY_MODEL_177c78fce95d4b4ab33057c5a048d693", + "value": " 9.09M/9.09M [00:00<00:00, 32.6MB/s]" + } + }, + "8b3505352a5a42bf910428c40ce40465": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_49277aeeac16434a865a4d12308b1abc", + "placeholder": "​", + "style": "IPY_MODEL_2157f01726d748f8a9ae4a00664430da", + "value": " 5.70G/5.70G [01:02<00:00, 30.1MB/s]" + } + }, + "8fc142b628fb40568730234de1cafde2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ae7e449e4ea4c729b5f34607c18ebae", + "max": 172, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3572201bd4d74a58b7a665f9bdfdcdba", + "value": 172 + } + }, + "9367047a800747f79c6b225d92397846": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "938f45f1b3e24118b815d96ae34ba86a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "95fbe66647904c06a20f640630d6dc0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b0a370dc20654b279b9680692e34418e", + "placeholder": "​", + "style": "IPY_MODEL_cfeb365ddf7548d58b2557f22737fcf5", + "value": " 11.6k/11.6k [00:00<00:00, 716kB/s]" + } + }, + "988a0e8c1f89446086858da0a891a79c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ad2be500fc164c0f86f33e914ef8e6a0", + "placeholder": "​", + "style": "IPY_MODEL_5234566b1bfc4655b8d582ea5b46ed9f", + "value": "Downloading data: 100%" + } + }, + "98c58f23f4d549518832cb2d18f796e8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_09b76013aa9e45efb6deb23a7a0d0925", + "IPY_MODEL_39b29a75374b45c0a22506010be2b84e", + "IPY_MODEL_78e5400bff924a92a4cc61c4ff18b182" + ], + "layout": "IPY_MODEL_2a58d04b428c46f4b3dbadd3bc6cd529" + } + }, + "99fdbb0300c14c139d1937c646f0cfe7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7358cdad832342c983e31efb8754ab78", + "placeholder": "​", + "style": "IPY_MODEL_e9adf418296e436fb48bb9f78885598b", + "value": " 51760/51760 [00:01<00:00, 38665.95 examples/s]" + } + }, + "9f679ad3ec7f4fe8ad0510ffb57bc2ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ea63adfce694725bdba878aef709dd3", + "placeholder": "​", + "style": "IPY_MODEL_74501720ac7e4dbb911a4a99b3633bc6", + "value": "tokenizer.json: 100%" + } + }, + "a0037bdccf254159becde630bee3d1db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a069d2ab23824f29aa320ac256e2cfe9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0bf9160eb2647409b3200270914b90f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a41dc44766444a998bec2d777f249d23": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8464a4c711e4e00aafdfc919b60d07e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fb995c740590427b882572c81d4e848c", + "placeholder": "​", + "style": "IPY_MODEL_201b59ccd9f845e197029b57e424aefc", + "value": " 172/172 [00:00<00:00, 12.0kB/s]" + } + }, + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad2be500fc164c0f86f33e914ef8e6a0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b0240cd9a4554b29ae11f8051984a1c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_edaf890370314a218f138015faa0b05d", + "placeholder": "​", + "style": "IPY_MODEL_697f027529b54ee9956bae78a11e0611", + "value": "Map: 100%" + } + }, + "b0a370dc20654b279b9680692e34418e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b518dcee69074b87be73957cd810e7ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d891f8d0b1fc462f8008d02bb2a15692", + "placeholder": "​", + "style": "IPY_MODEL_cced8fd7e998472794f3f3e3018956a5", + "value": "tokenizer_config.json: 100%" + } + }, + "b8908fa0df3743ecb9d12983a739104f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b993eaec6b224440bf80c0958c6fb536": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9b313fd861948f5aba25b24b1518d30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba90fdb8822d47dab7ba203bee297f37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f8b6bfe16894500838793f2491d403f", + "placeholder": "​", + "style": "IPY_MODEL_bb19f6c747754682a514373a3a0535ba", + "value": "Downloading readme: 100%" + } + }, + "bb19f6c747754682a514373a3a0535ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc883d4cf13e4f8b8a4fe5f410cb6efd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9159e03e61f4f56978ece9c3bca49b2", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_810ff6c0e17d4fa09a30fef27eacff90", + "value": 51760 + } + }, + "c161d94df0f04feba9542237e0856c22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c22f71b1f85843209d7e5321506b9cb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f44c9ce1adf470cbb19784493ed209f", + "IPY_MODEL_f1addc4479d849879e743cf9089e6540", + "IPY_MODEL_8b3505352a5a42bf910428c40ce40465" + ], + "layout": "IPY_MODEL_4c4c88d4c701450692fa0f6b0c5764b0" + } + }, + "c4f2b06a82fd4987b8b659524a7b503b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cca8113c54c0495daedce1327bf9c68b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e02f9b7849c64531835eb77b860d1c93", + "max": 464, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_56aee4853b7740e6a977254f5d1fa66d", + "value": 464 + } + }, + "cced8fd7e998472794f3f3e3018956a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cf245afeb1c04f29a24d291608c3d157": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b518dcee69074b87be73957cd810e7ed", + "IPY_MODEL_e29104486d594b2992d7285e0ef77371", + "IPY_MODEL_6578fd7acdb54c4c93528ea431fd0144" + ], + "layout": "IPY_MODEL_d35db8148a354c56aaac56dbae22536f" + } + }, + "cfe8cae0e22b495bafa221a63d13b283": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfeb365ddf7548d58b2557f22737fcf5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1b47d39450d4019ae85c9b2f943eeaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4dcf6ff672d24983a1877a8431709aa9", + "IPY_MODEL_7975adbc2ec5489ea7fa0167e620d85c", + "IPY_MODEL_71ce208e20d6483abb9ed923510c86d7" + ], + "layout": "IPY_MODEL_cfe8cae0e22b495bafa221a63d13b283" + } + }, + "d35db8148a354c56aaac56dbae22536f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69dc491b3ab44d7852b21873ed7bb7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d891f8d0b1fc462f8008d02bb2a15692": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8e5318cead340c4adbeaccc05d39225": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "daf4cd890b35422683d22fd30bc71e83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b0240cd9a4554b29ae11f8051984a1c6", + "IPY_MODEL_bc883d4cf13e4f8b8a4fe5f410cb6efd", + "IPY_MODEL_99fdbb0300c14c139d1937c646f0cfe7" + ], + "layout": "IPY_MODEL_c161d94df0f04feba9542237e0856c22" + } + }, + "db19fc8d37db4e45a5790a876836d8c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de868e26e7154f62aa86223a539ad421": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dea41c5260884aa6879b5e1d1697b14f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e02f9b7849c64531835eb77b860d1c93": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e29104486d594b2992d7285e0ef77371": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "max": 50641, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2f6c70dd266c4816bfad3fd3d192929a", + "value": 50641 + } + }, + "e36a3f9eff0e4cf68834d66b0213ae96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9159e03e61f4f56978ece9c3bca49b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9adf418296e436fb48bb9f78885598b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "edaf890370314a218f138015faa0b05d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f1addc4479d849879e743cf9089e6540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43dec2ede91341f5af60eb522e18e984", + "max": 5702746405, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d8e5318cead340c4adbeaccc05d39225", + "value": 5702746405 + } + }, + "f2df530d22c74977b249dd9fb5f4829b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_21db8a77b00d4a4e82fdfa608657531f", + "max": 9085698, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6dbbedeca9314e66ae50e44ffa31a414", + "value": 9085698 + } + }, + "f401d53bf28e44eb906bce6c05412662": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fb995c740590427b882572c81d4e848c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fce7a61c25ec4390af43d92b7c473a45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_30307300bc4e4baf96560e30969a82b6", + "IPY_MODEL_8fc142b628fb40568730234de1cafde2", + "IPY_MODEL_a8464a4c711e4e00aafdfc919b60d07e" + ], + "layout": "IPY_MODEL_5f40db8173dd4d76b6ef5ed6d9ec8b6e" + } + }, + "fdb1941405ed4e4aa06019933892deb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/06_tune-small-py3.11.ipynb b/notebooks/06_tune-small-py3.11.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..41257cb1d936dc55f300f33e3ff0eda78bc352f6 --- /dev/null +++ b/notebooks/06_tune-small-py3.11.ipynb @@ -0,0 +1,4673 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "0ea8b46b-839b-445b-8043-ccdf4e920ace", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6d394937-6c99-4a7c-9d32-7600a280032f", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /home/inflaton/code/projects/courses/llm-finetuning\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9f67ec60-2f24-411c-84eb-0dd664b44775", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f1597656-8042-4878-9d3b-9ebfb8dd86dc", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('unsloth/Qwen2-0.5B-Instruct-bnb-4bit',\n", + " True,\n", + " None,\n", + " None,\n", + " 2048,\n", + " 10,\n", + " None,\n", + " 'datasets/mac/mac.tsv',\n", + " 'results/mac-results_py3.11.csv')" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "model_name = os.getenv(\"MODEL_NAME\")\n", + "token = os.getenv(\"HF_TOKEN\") or None\n", + "load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n", + "local_model = os.getenv(\"LOCAL_MODEL\")\n", + "hub_model = os.getenv(\"HUB_MODEL\")\n", + "num_train_epochs = int(os.getenv(\"NUM_TRAIN_EPOCHS\") or 0)\n", + "data_path = os.getenv(\"DATA_PATH\")\n", + "results_path = os.getenv(\"RESULTS_PATH\")\n", + "\n", + "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n", + "dtype = (\n", + " None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n", + ")\n", + "\n", + "model_name, load_in_4bit, local_model, hub_model, max_seq_length, num_train_epochs, dtype, data_path, results_path" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tue Jun 25 23:53:09 2024 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 545.23.07 Driver Version: 546.12 CUDA Version: 12.3 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 NVIDIA GeForce RTX 4080 ... On | 00000000:01:00.0 On | N/A |\n", + "| N/A 54C P8 5W / 150W | 483MiB / 12282MiB | 9% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "| No running processes found |\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "!nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Python 3.11.9\n", + "\u001b[33mWARNING: Package(s) not found: flash-attn\u001b[0m\u001b[33m\n", + "\u001b[0mCPU times: user 7.35 ms, sys: 1.47 ms, total: 8.81 ms\n", + "Wall time: 509 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "!python --version\n", + "!pip show flash-attn" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/llm-finetuning\n", + "Tuning unsloth/Qwen2-0.5B-Instruct\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "unsloth/Qwen2-0.5B-Instruct True 2048 10 None datasets/mac/mac.tsv results/mac-results_py3.11.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-0.5B-Instruct\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0+cu121. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.633 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old耿举枪，斜着眼睛眯起来，扣动扳机，砰的一声响，子弹一颗颗落在地上，像是打在棉花团上一样，砰的一声又一声，声音很清脆。\n", + "--------\n", + "step 3: Old耿举枪，斜着眼睛眯起来，扣动扳机，砰的一声响，子弹一颗颗落在地上，像是打在棉花团上一样，砰的一声又一声，声音很清脆。\n", + "100%|███████████████████████████████████████| 1133/1133 [21:27<00:00, 1.14s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old耿举枪，斜着眼睛眯起来，扣动扳机，砰的一声响，子弹一颗颗落在地上，像是打在棉花团上一样...\n", + "\n", + "[1 rows x 3 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.893 GB of memory reserved.\n", + "Unsloth 2024.6 patched 24 layers with 0 QKV layers, 24 O layers and 24 MLP layers.\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/transformers/training_args.py:1965: FutureWarning: `--push_to_hub_token` is deprecated and will be removed in version 5 of 🤗 Transformers. Use `--hub_token` instead.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:269: UserWarning: You passed a `max_seq_length` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:283: UserWarning: You passed a `dataset_num_proc` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:307: UserWarning: You passed a `dataset_text_field` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "Map (num_proc=2): 100%|████████████| 4528/4528 [00:00<00:00, 5300.69 examples/s]\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.893 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 8,798,208\n", + "{'loss': 1.9402, 'grad_norm': 0.9608356952667236, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.7757, 'grad_norm': 0.8031952977180481, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.7142, 'grad_norm': 0.9293338656425476, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.7148, 'grad_norm': 0.7500923275947571, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.6853, 'grad_norm': 0.8244311809539795, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.607, 'grad_norm': 0.7938973903656006, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.491, 'grad_norm': 0.8929483890533447, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.5396, 'grad_norm': 0.9767352342605591, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.4711, 'grad_norm': 1.0663349628448486, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.4675, 'grad_norm': 0.9794139862060547, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.5265, 'grad_norm': 1.0100280046463013, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.3482, 'grad_norm': 1.1571651697158813, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 1.2146, 'grad_norm': 1.173205018043518, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.2565, 'grad_norm': 1.2972688674926758, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.2542, 'grad_norm': 1.3474286794662476, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.2652, 'grad_norm': 1.1654258966445923, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.2327, 'grad_norm': 1.0851577520370483, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.9784, 'grad_norm': 1.477825403213501, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.9903, 'grad_norm': 1.5403794050216675, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.9977, 'grad_norm': 1.510469913482666, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.9786, 'grad_norm': 1.5427402257919312, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 1.0211, 'grad_norm': 1.569674015045166, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.9282, 'grad_norm': 1.5466079711914062, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.7374, 'grad_norm': 1.6999439001083374, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.7371, 'grad_norm': 1.7169603109359741, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.7729, 'grad_norm': 2.1618764400482178, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.7718, 'grad_norm': 2.412935972213745, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.7913, 'grad_norm': 1.8272628784179688, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.6267, 'grad_norm': 1.8343967199325562, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.5543, 'grad_norm': 1.6036676168441772, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.5589, 'grad_norm': 1.6545743942260742, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.5872, 'grad_norm': 1.8736636638641357, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.5794, 'grad_norm': 2.7655928134918213, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.5763, 'grad_norm': 2.0712125301361084, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.3967, 'grad_norm': 1.8133138418197632, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.4092, 'grad_norm': 1.7030404806137085, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.4278, 'grad_norm': 2.3163561820983887, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.4194, 'grad_norm': 1.8104350566864014, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.4248, 'grad_norm': 1.6338876485824585, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.3832, 'grad_norm': 1.7445095777511597, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.2912, 'grad_norm': 1.7364373207092285, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.3119, 'grad_norm': 1.3449686765670776, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.3127, 'grad_norm': 1.8036069869995117, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.3028, 'grad_norm': 2.0399818420410156, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.318, 'grad_norm': 1.544445514678955, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.2465, 'grad_norm': 1.361892819404602, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.2259, 'grad_norm': 2.45100736618042, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.2285, 'grad_norm': 1.014116883277893, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.2358, 'grad_norm': 1.2610195875167847, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.2351, 'grad_norm': 1.6966944932937622, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.2387, 'grad_norm': 1.0741287469863892, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.1809, 'grad_norm': 1.14618980884552, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1911, 'grad_norm': 1.1346627473831177, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1869, 'grad_norm': 1.1694375276565552, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.1878, 'grad_norm': 1.8589993715286255, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1889, 'grad_norm': 0.9836981892585754, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 4526.9366, 'train_samples_per_second': 10.002, 'train_steps_per_second': 1.25, 'train_loss': 0.7976709857846317, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [1:15:26<00:00, 1.25it/s]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "4526.9366 seconds used for training.\n", + "75.45 minutes used for training.\n", + "Peak reserved memory = 1.371 GB.\n", + "Peak reserved memory for training = 0.478 GB.\n", + "Peak reserved memory % of max memory = 11.431 %.\n", + "Peak reserved memory for training % of max memory = 3.985 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his rifle and tilted his head to one side; as the shotgun blast issued, crinkles formed in his eye, and ice-shells splattered through the air like hailstones.\n", + "--------\n", + "step 3: Old Geng raised his rifle and tilted his head to one side; as the shotgun blast issued, crinkles formed in his eye, and ice-shells splattered through the air like hailstones.\n", + "100%|███████████████████████████████████████| 1133/1133 [29:54<00:00, 1.58s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his rifle and tilted his head ...\n", + "\n", + "[1 rows x 4 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.371 GB of memory reserved.\n", + "model.safetensors: 100%|█████████████████████| 493M/493M [00:50<00:00, 9.77MB/s]\n", + "Saved model to https://huggingface.co/Qwen2-0.5B-Instruct-MAC-lora\n", + "README.md: 100%|███████████████████████████████| 599/599 [00:00<00:00, 11.4MB/s]\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... Done.\n", + "Unsloth: Saving LoRA adapters. Please wait...\n", + "401 Client Error: Unauthorized for url: https://huggingface.co/api/repos/create (Request ID: Root=1-667b060d-6b558b430a940ed15545a0b3;57af47e9-c736-4f24-80b8-1d4983acc0e0)\n", + "\n", + "Invalid username or password.\n", + "Tuning unsloth/Qwen2-1.5B-Instruct\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "unsloth/Qwen2-1.5B-Instruct True 2048 10 None datasets/mac/mac.tsv results/mac-results_py3.11.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-1.5B-Instruct\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0+cu121. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.516 GB of memory reserved.\n", + "loading train/test data files\n", + "Map: 100%|████████████████████████| 4528/4528 [00:00<00:00, 31812.02 examples/s]\n", + "Map: 100%|████████████████████████| 1133/1133 [00:00<00:00, 14717.68 examples/s]\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Lao Jing lifted up his gun, squinted one of his two triangular eyes, and fired a shot with his gun. A hail of golden sparrows rained down from above, striking the willow branches with a loud clatter, making a sound.\n", + "--------\n", + "step 3: Lao Jing lifted up his gun, squinted one of his two triangular eyes, and fired a shot with his gun. A hail of golden sparrows rained down from above, striking the willow branches with a loud clatter, making a sound.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:34:06<00:00, 4.98s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Lao Jing lifted up his gun, squinted one of hi...\n", + "\n", + "[1 rows x 5 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.945 GB of memory reserved.\n", + "Unsloth 2024.6 patched 28 layers with 0 QKV layers, 28 O layers and 28 MLP layers.\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/transformers/training_args.py:1965: FutureWarning: `--push_to_hub_token` is deprecated and will be removed in version 5 of 🤗 Transformers. Use `--hub_token` instead.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:269: UserWarning: You passed a `max_seq_length` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:283: UserWarning: You passed a `dataset_num_proc` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:307: UserWarning: You passed a `dataset_text_field` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "Map (num_proc=2): 100%|████████████| 4528/4528 [00:02<00:00, 2177.21 examples/s]\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.945 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 18,464,768\n", + "{'loss': 1.7414, 'grad_norm': 0.645324170589447, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.5681, 'grad_norm': 0.6221398115158081, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.5156, 'grad_norm': 0.6400603652000427, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.5174, 'grad_norm': 0.5692432522773743, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.496, 'grad_norm': 0.5720127820968628, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.4181, 'grad_norm': 0.5995662808418274, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.2976, 'grad_norm': 0.7103854417800903, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.3392, 'grad_norm': 0.6867524981498718, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.2788, 'grad_norm': 0.8870283961296082, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.2799, 'grad_norm': 0.8677213788032532, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.3311, 'grad_norm': 0.824891984462738, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.1283, 'grad_norm': 0.9994134902954102, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 0.9958, 'grad_norm': 1.1032615900039673, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.0305, 'grad_norm': 1.2964789867401123, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.0345, 'grad_norm': 1.1391383409500122, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.0352, 'grad_norm': 1.033879041671753, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.0078, 'grad_norm': 0.893123209476471, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.6974, 'grad_norm': 1.3191255331039429, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.7094, 'grad_norm': 1.3912413120269775, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.7159, 'grad_norm': 1.262569785118103, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.7052, 'grad_norm': 1.4299038648605347, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 0.7441, 'grad_norm': 1.2525451183319092, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.6426, 'grad_norm': 1.1614264249801636, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.4613, 'grad_norm': 1.4634888172149658, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.4463, 'grad_norm': 1.479321002960205, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.4803, 'grad_norm': 1.524327278137207, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.4775, 'grad_norm': 1.8287591934204102, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.4858, 'grad_norm': 1.5343736410140991, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.3607, 'grad_norm': 1.509474277496338, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.291, 'grad_norm': 0.9709746837615967, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.297, 'grad_norm': 1.1549153327941895, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.3137, 'grad_norm': 1.5136038064956665, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.3086, 'grad_norm': 1.5126714706420898, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.3054, 'grad_norm': 1.2357054948806763, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.1908, 'grad_norm': 0.998966634273529, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.2016, 'grad_norm': 1.4552719593048096, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.2063, 'grad_norm': 1.193115234375, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.2068, 'grad_norm': 1.1482857465744019, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.2145, 'grad_norm': 1.2252038717269897, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.194, 'grad_norm': 0.8770560026168823, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.1487, 'grad_norm': 0.7064879536628723, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.1571, 'grad_norm': 0.8916934132575989, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.1576, 'grad_norm': 0.910902738571167, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.1576, 'grad_norm': 1.3216708898544312, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.1596, 'grad_norm': 0.9173678755760193, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.1368, 'grad_norm': 0.7557782530784607, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.1305, 'grad_norm': 1.3723442554473877, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.1315, 'grad_norm': 0.4736746847629547, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.1328, 'grad_norm': 0.4113923907279968, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.1343, 'grad_norm': 0.628764271736145, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.1329, 'grad_norm': 0.5089218020439148, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.1179, 'grad_norm': 0.45263954997062683, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1185, 'grad_norm': 0.552614152431488, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1193, 'grad_norm': 0.46947821974754333, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.1202, 'grad_norm': 0.5204765200614929, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1232, 'grad_norm': 0.4254581928253174, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 16066.3343, 'train_samples_per_second': 2.818, 'train_steps_per_second': 0.352, 'train_loss': 0.5993303972082509, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [4:27:46<00:00, 2.84s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "16066.3343 seconds used for training.\n", + "267.77 minutes used for training.\n", + "Peak reserved memory = 3.945 GB.\n", + "Peak reserved memory for training = 0.0 GB.\n", + "Peak reserved memory % of max memory = 32.891 %.\n", + "Peak reserved memory for training % of max memory = 0.0 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised the pistol to his eye, squeezed the trigger, and rainafterdowned birds landed: Golden sparrows, dead or alive, screeched and dropped from the treetops, sending sticks flying everywhere.\n", + "--------\n", + "step 3: Old Geng raised the pistol to his eye, squeezed the trigger, and rainafterdowned birds landed: Golden sparrows, dead or alive, screeched and dropped from the treetops, sending sticks flying everywhere.\n", + "100%|█████████████████████████████████████| 1133/1133 [2:11:15<00:00, 6.95s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised the pistol to his eye, squeeze...\n", + "\n", + "[1 rows x 6 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5.33 GB of memory reserved.\n", + "model.safetensors: 100%|███████████████████| 1.22G/1.22G [01:25<00:00, 14.3MB/s]\n", + "Saved model to https://huggingface.co/Qwen2-1.5B-Instruct-MAC-lora\n", + "README.md: 100%|███████████████████████████████| 599/599 [00:00<00:00, 3.80MB/s]\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... Done.\n", + "Unsloth: Saving LoRA adapters. Please wait...\n", + "401 Client Error: Unauthorized for url: https://huggingface.co/api/repos/create (Request ID: Root=1-667b7a4d-4dd9df470d5a01b733cc9060;fa501a9a-6bbd-4865-bbdc-ddcc5f272821)\n", + "\n", + "Invalid username or password.\n", + "CPU times: user 13min 32s, sys: 5min 37s, total: 19min 10s\n", + "Wall time: 10h 24min 39s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./scripts/tune-small-2.sh" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/llm-finetuning\n", + "Tuning unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "unsloth/Qwen2-0.5B-Instruct-bnb-4bit True 2048 10 None datasets/mac/mac.tsv results/mac-results_py3.11.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0+cu121. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.633 GB of memory reserved.\n", + "loading train/test data files\n", + "Map: 100%|████████████████████████| 4528/4528 [00:00<00:00, 12373.02 examples/s]\n", + "Map: 100%|█████████████████████████| 1133/1133 [00:00<00:00, 9100.92 examples/s]\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Tang raises his gun, half-closed eyes, firing the trigger of the revolver. The bullet crackles as the hammer rings, a thunderous bang that echoes through the branches of the tree. The sand grains hit the ground with a thud, echoing like a drum.\n", + "--------\n", + "step 3: Old Tang raises his gun, half-closed eyes, firing the trigger of the revolver. The bullet crackles as the hammer rings, a thunderous bang that echoes through the branches of the tree. The sand grains hit the ground with a thud, echoing like a drum.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:20:24<00:00, 4.26s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Tang raises his gun, half-closed eyes, fir...\n", + "\n", + "[1 rows x 7 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.023 GB of memory reserved.\n", + "Unsloth 2024.6 patched 24 layers with 0 QKV layers, 24 O layers and 24 MLP layers.\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/transformers/training_args.py:1965: FutureWarning: `--push_to_hub_token` is deprecated and will be removed in version 5 of 🤗 Transformers. Use `--hub_token` instead.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:269: UserWarning: You passed a `max_seq_length` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:283: UserWarning: You passed a `dataset_num_proc` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:307: UserWarning: You passed a `dataset_text_field` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "Map (num_proc=2): 100%|████████████| 4528/4528 [00:02<00:00, 2107.23 examples/s]\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "3.023 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 8,798,208\n", + "{'loss': 1.9402, 'grad_norm': 0.9608356952667236, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.7759, 'grad_norm': 0.7993724346160889, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.7139, 'grad_norm': 0.9072545766830444, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.715, 'grad_norm': 0.7505761384963989, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.6855, 'grad_norm': 0.819739580154419, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.6071, 'grad_norm': 0.7929678559303284, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.491, 'grad_norm': 0.8940563797950745, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.5395, 'grad_norm': 0.9731950163841248, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.4715, 'grad_norm': 1.0640537738800049, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.4674, 'grad_norm': 0.9785354137420654, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.5265, 'grad_norm': 1.007393479347229, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.3483, 'grad_norm': 1.1656651496887207, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 1.2144, 'grad_norm': 1.1790250539779663, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.2563, 'grad_norm': 1.3019884824752808, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.2541, 'grad_norm': 1.2913496494293213, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.2652, 'grad_norm': 1.15500009059906, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.233, 'grad_norm': 1.1158246994018555, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.9784, 'grad_norm': 1.5188616514205933, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.9898, 'grad_norm': 1.5216989517211914, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.9973, 'grad_norm': 1.5246195793151855, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.9778, 'grad_norm': 1.5367473363876343, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 1.0212, 'grad_norm': 1.583097219467163, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.9286, 'grad_norm': 1.4903535842895508, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.7365, 'grad_norm': 1.7766040563583374, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.7392, 'grad_norm': 1.7640079259872437, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.772, 'grad_norm': 2.152620315551758, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.7712, 'grad_norm': 2.200035333633423, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.7922, 'grad_norm': 1.8203985691070557, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.6284, 'grad_norm': 2.1231496334075928, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.5538, 'grad_norm': 1.633864402770996, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.5586, 'grad_norm': 1.8590402603149414, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.588, 'grad_norm': 1.8585463762283325, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.5816, 'grad_norm': 2.0347542762756348, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.5778, 'grad_norm': 1.9615962505340576, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.398, 'grad_norm': 1.861493468284607, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.4113, 'grad_norm': 1.8733363151550293, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.4295, 'grad_norm': 2.361309289932251, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.42, 'grad_norm': 1.871649980545044, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.4263, 'grad_norm': 1.567151427268982, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.3841, 'grad_norm': 1.7655881643295288, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.2916, 'grad_norm': 1.5300374031066895, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.3137, 'grad_norm': 1.399330973625183, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.3127, 'grad_norm': 2.009575605392456, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.3035, 'grad_norm': 1.74385404586792, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.3187, 'grad_norm': 1.5835634469985962, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.2469, 'grad_norm': 1.3062812089920044, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.2274, 'grad_norm': 1.4867122173309326, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.229, 'grad_norm': 1.0662896633148193, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.2378, 'grad_norm': 1.3190470933914185, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.2362, 'grad_norm': 1.9712920188903809, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.238, 'grad_norm': 1.1038957834243774, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.1807, 'grad_norm': 1.2336972951889038, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1896, 'grad_norm': 1.145556926727295, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1878, 'grad_norm': 1.1361063718795776, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.189, 'grad_norm': 1.5798214673995972, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.19, 'grad_norm': 0.9943256974220276, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 13819.9366, 'train_samples_per_second': 3.276, 'train_steps_per_second': 0.41, 'train_loss': 0.7980813258949523, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [3:50:19<00:00, 2.44s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "13819.9366 seconds used for training.\n", + "230.33 minutes used for training.\n", + "Peak reserved memory = 3.023 GB.\n", + "Peak reserved memory for training = 0.0 GB.\n", + "Peak reserved memory % of max memory = 25.204 %.\n", + "Peak reserved memory for training % of max memory = 0.0 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-0.5B-Instruct-bnb-4bit\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his rifle, clenching one of its triangular sights, and fired. A hailstone blast tore through the field, scattering large icy crickets (the cricket-like sounds of the gun wasp were like the crackles of钢铁的碎块) that fell powerfully in the willows between them.\n", + "--------\n", + "step 3: Old Geng raised his rifle, clenching one of its triangular sights, and fired. A hailstone blast tore through the field, scattering large icy crickets (the cricket-like sounds of the gun wasp were like the crackles of钢铁的碎块) that fell powerfully in the willows between them.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:48:52<00:00, 5.77s/it]\n", + " chinese ... unsloth/Qwen2-0.5B-Instruct-bnb-4bit(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his rifle, clenching one of it...\n", + "\n", + "[1 rows x 8 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "4.344 GB of memory reserved.\n", + "model.safetensors: 100%|█████████████████████| 493M/493M [01:44<00:00, 4.71MB/s]\n", + "Saved model to https://huggingface.co/Qwen2-0.5B-Instruct-bnb-4bit-MAC-lora\n", + "README.md: 100%|███████████████████████████████| 599/599 [00:00<00:00, 3.50MB/s]\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... Done.\n", + "Unsloth: Saving LoRA adapters. Please wait...\n", + "401 Client Error: Unauthorized for url: https://huggingface.co/api/repos/create (Request ID: Root=1-667bdd5e-773cdcf53c89ad5261301688;d08e235b-a133-4e46-b5db-3e4f74133461)\n", + "\n", + "Invalid username or password.\n", + "Tuning unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "unsloth/Qwen2-1.5B-Instruct-bnb-4bit True 2048 10 None datasets/mac/mac.tsv results/mac-results_py3.11.csv True True True\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0+cu121. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.516 GB of memory reserved.\n", + "loading train/test data files\n", + "Map: 100%|████████████████████████| 4528/4528 [00:00<00:00, 10965.65 examples/s]\n", + "Map: 100%|█████████████████████████| 1133/1133 [00:00<00:00, 9010.28 examples/s]\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating base model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old耿拿起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。\n", + "--------\n", + "step 3: Old耿拿起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞迸着，嚓嚓有声。\n", + "100%|█████████████████████████████████████| 1133/1133 [1:09:28<00:00, 3.68s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old耿拿起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝...\n", + "\n", + "[1 rows x 9 columns]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.758 GB of memory reserved.\n", + "Unsloth 2024.6 patched 28 layers with 0 QKV layers, 28 O layers and 28 MLP layers.\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/transformers/training_args.py:1965: FutureWarning: `--push_to_hub_token` is deprecated and will be removed in version 5 of 🤗 Transformers. Use `--hub_token` instead.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:269: UserWarning: You passed a `max_seq_length` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:283: UserWarning: You passed a `dataset_num_proc` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "/home/inflaton/miniconda3/envs/llm-fine-tune/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:307: UserWarning: You passed a `dataset_text_field` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", + " warnings.warn(\n", + "Map (num_proc=2): 100%|████████████| 4528/4528 [00:02<00:00, 2245.59 examples/s]\n", + "(4) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.758 GB of memory reserved.\n", + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,528 | Num Epochs = 10\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", + "\\ / Total batch size = 8 | Total steps = 5,660\n", + " \"-____-\" Number of trainable parameters = 18,464,768\n", + "{'loss': 1.7415, 'grad_norm': 0.6513338088989258, 'learning_rate': 0.00019664014146772768, 'epoch': 0.18}\n", + "{'loss': 1.5683, 'grad_norm': 0.6299685835838318, 'learning_rate': 0.0001931034482758621, 'epoch': 0.35}\n", + "{'loss': 1.5159, 'grad_norm': 0.6519908905029297, 'learning_rate': 0.00018956675508399648, 'epoch': 0.53}\n", + "{'loss': 1.5172, 'grad_norm': 0.580685019493103, 'learning_rate': 0.00018603006189213086, 'epoch': 0.71}\n", + "{'loss': 1.496, 'grad_norm': 0.5678043365478516, 'learning_rate': 0.00018249336870026527, 'epoch': 0.88}\n", + "{'loss': 1.4184, 'grad_norm': 0.6064684987068176, 'learning_rate': 0.00017895667550839965, 'epoch': 1.06}\n", + "{'loss': 1.2976, 'grad_norm': 0.7216887474060059, 'learning_rate': 0.00017541998231653406, 'epoch': 1.24}\n", + "{'loss': 1.3393, 'grad_norm': 0.6848570108413696, 'learning_rate': 0.00017188328912466844, 'epoch': 1.41}\n", + "{'loss': 1.2784, 'grad_norm': 0.9106780886650085, 'learning_rate': 0.00016834659593280285, 'epoch': 1.59}\n", + "{'loss': 1.2801, 'grad_norm': 0.8649025559425354, 'learning_rate': 0.00016480990274093723, 'epoch': 1.77}\n", + "{'loss': 1.3313, 'grad_norm': 0.880467414855957, 'learning_rate': 0.00016127320954907164, 'epoch': 1.94}\n", + "{'loss': 1.1276, 'grad_norm': 1.0269683599472046, 'learning_rate': 0.000157736516357206, 'epoch': 2.12}\n", + "{'loss': 0.9956, 'grad_norm': 1.0964370965957642, 'learning_rate': 0.0001541998231653404, 'epoch': 2.3}\n", + "{'loss': 1.0299, 'grad_norm': 1.298215389251709, 'learning_rate': 0.0001506631299734748, 'epoch': 2.47}\n", + "{'loss': 1.0348, 'grad_norm': 1.1091656684875488, 'learning_rate': 0.0001471264367816092, 'epoch': 2.65}\n", + "{'loss': 1.0346, 'grad_norm': 1.0557124614715576, 'learning_rate': 0.0001435897435897436, 'epoch': 2.83}\n", + "{'loss': 1.0081, 'grad_norm': 0.8857123255729675, 'learning_rate': 0.000140053050397878, 'epoch': 3.0}\n", + "{'loss': 0.6977, 'grad_norm': 1.3270587921142578, 'learning_rate': 0.0001365163572060124, 'epoch': 3.18}\n", + "{'loss': 0.7104, 'grad_norm': 1.3964347839355469, 'learning_rate': 0.00013297966401414678, 'epoch': 3.36}\n", + "{'loss': 0.7135, 'grad_norm': 1.324993371963501, 'learning_rate': 0.0001294429708222812, 'epoch': 3.53}\n", + "{'loss': 0.7054, 'grad_norm': 1.4066098928451538, 'learning_rate': 0.00012590627763041555, 'epoch': 3.71}\n", + "{'loss': 0.7424, 'grad_norm': 1.287760615348816, 'learning_rate': 0.00012236958443854996, 'epoch': 3.89}\n", + "{'loss': 0.6421, 'grad_norm': 1.2553821802139282, 'learning_rate': 0.00011883289124668435, 'epoch': 4.06}\n", + "{'loss': 0.4585, 'grad_norm': 1.6327370405197144, 'learning_rate': 0.00011529619805481875, 'epoch': 4.24}\n", + "{'loss': 0.4444, 'grad_norm': 1.6593241691589355, 'learning_rate': 0.00011175950486295315, 'epoch': 4.42}\n", + "{'loss': 0.4802, 'grad_norm': 1.5733627080917358, 'learning_rate': 0.00010822281167108754, 'epoch': 4.59}\n", + "{'loss': 0.4779, 'grad_norm': 1.6328530311584473, 'learning_rate': 0.00010468611847922194, 'epoch': 4.77}\n", + "{'loss': 0.4873, 'grad_norm': 1.664158582687378, 'learning_rate': 0.00010114942528735633, 'epoch': 4.95}\n", + "{'loss': 0.3602, 'grad_norm': 1.923504114151001, 'learning_rate': 9.761273209549072e-05, 'epoch': 5.12}\n", + "{'loss': 0.2924, 'grad_norm': 1.2101478576660156, 'learning_rate': 9.407603890362513e-05, 'epoch': 5.3}\n", + "{'loss': 0.2975, 'grad_norm': 1.3352692127227783, 'learning_rate': 9.053934571175951e-05, 'epoch': 5.48}\n", + "{'loss': 0.3142, 'grad_norm': 1.528309941291809, 'learning_rate': 8.70026525198939e-05, 'epoch': 5.65}\n", + "{'loss': 0.306, 'grad_norm': 1.3299399614334106, 'learning_rate': 8.34659593280283e-05, 'epoch': 5.83}\n", + "{'loss': 0.3081, 'grad_norm': 0.5754266381263733, 'learning_rate': 7.99292661361627e-05, 'epoch': 6.01}\n", + "{'loss': 0.1935, 'grad_norm': 1.24480402469635, 'learning_rate': 7.639257294429708e-05, 'epoch': 6.18}\n", + "{'loss': 0.2045, 'grad_norm': 1.5091683864593506, 'learning_rate': 7.285587975243147e-05, 'epoch': 6.36}\n", + "{'loss': 0.2071, 'grad_norm': 0.9175063371658325, 'learning_rate': 6.931918656056587e-05, 'epoch': 6.54}\n", + "{'loss': 0.2055, 'grad_norm': 1.013519525527954, 'learning_rate': 6.578249336870027e-05, 'epoch': 6.71}\n", + "{'loss': 0.2133, 'grad_norm': 2.126910448074341, 'learning_rate': 6.224580017683466e-05, 'epoch': 6.89}\n", + "{'loss': 0.194, 'grad_norm': 0.8879603147506714, 'learning_rate': 5.870910698496905e-05, 'epoch': 7.07}\n", + "{'loss': 0.149, 'grad_norm': 0.7381364107131958, 'learning_rate': 5.517241379310345e-05, 'epoch': 7.24}\n", + "{'loss': 0.1564, 'grad_norm': 0.7718709111213684, 'learning_rate': 5.163572060123785e-05, 'epoch': 7.42}\n", + "{'loss': 0.1573, 'grad_norm': 0.6855681538581848, 'learning_rate': 4.809902740937224e-05, 'epoch': 7.6}\n", + "{'loss': 0.1578, 'grad_norm': 0.9972281455993652, 'learning_rate': 4.4562334217506634e-05, 'epoch': 7.77}\n", + "{'loss': 0.1588, 'grad_norm': 0.818317174911499, 'learning_rate': 4.1025641025641023e-05, 'epoch': 7.95}\n", + "{'loss': 0.1375, 'grad_norm': 0.46658992767333984, 'learning_rate': 3.7488947833775426e-05, 'epoch': 8.13}\n", + "{'loss': 0.1304, 'grad_norm': 0.5777143836021423, 'learning_rate': 3.3952254641909815e-05, 'epoch': 8.3}\n", + "{'loss': 0.1315, 'grad_norm': 0.4499460756778717, 'learning_rate': 3.041556145004421e-05, 'epoch': 8.48}\n", + "{'loss': 0.1327, 'grad_norm': 0.4142455756664276, 'learning_rate': 2.6878868258178604e-05, 'epoch': 8.66}\n", + "{'loss': 0.1337, 'grad_norm': 0.6300601959228516, 'learning_rate': 2.3342175066313e-05, 'epoch': 8.83}\n", + "{'loss': 0.1327, 'grad_norm': 0.4785163998603821, 'learning_rate': 1.9805481874447392e-05, 'epoch': 9.01}\n", + "{'loss': 0.1178, 'grad_norm': 0.4512840211391449, 'learning_rate': 1.6268788682581788e-05, 'epoch': 9.19}\n", + "{'loss': 0.1186, 'grad_norm': 0.5308012366294861, 'learning_rate': 1.273209549071618e-05, 'epoch': 9.36}\n", + "{'loss': 0.1195, 'grad_norm': 0.4567421078681946, 'learning_rate': 9.195402298850575e-06, 'epoch': 9.54}\n", + "{'loss': 0.1203, 'grad_norm': 0.68560391664505, 'learning_rate': 5.658709106984969e-06, 'epoch': 9.72}\n", + "{'loss': 0.1231, 'grad_norm': 0.41683992743492126, 'learning_rate': 2.1220159151193635e-06, 'epoch': 9.89}\n", + "{'train_runtime': 16113.2955, 'train_samples_per_second': 2.81, 'train_steps_per_second': 0.351, 'train_loss': 0.5992827712857681, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5660/5660 [4:28:33<00:00, 2.85s/it]\n", + "(5) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "16113.2955 seconds used for training.\n", + "268.55 minutes used for training.\n", + "Peak reserved memory = 2.367 GB.\n", + "Peak reserved memory for training = 0.609 GB.\n", + "Peak reserved memory % of max memory = 19.735 %.\n", + "Peak reserved memory for training % of max memory = 5.078 %.\n", + "Evaluating fine-tuned model: unsloth/Qwen2-1.5B-Instruct-bnb-4bit\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his pistol, cocked it, and fired – a string of pocky-gun shots sent the shooting birds skittering to the ground.\n", + "--------\n", + "step 3: Old Geng raised his pistol, cocked it, and fired – a string of pocky-gun shots sent the shooting birds skittering to the ground.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:44:47<00:00, 5.55s/it]\n", + " chinese ... unsloth/Qwen2-1.5B-Instruct-bnb-4bit(finetuned)\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his pistol, cocked it, and fir...\n", + "\n", + "[1 rows x 10 columns]\n", + "(6) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "2.367 GB of memory reserved.\n", + "model.safetensors: 100%|███████████████████| 1.22G/1.22G [04:55<00:00, 4.12MB/s]\n", + "Saved model to https://huggingface.co/Qwen2-1.5B-Instruct-bnb-4bit-MAC-lora\n", + "README.md: 100%|███████████████████████████████| 599/599 [00:00<00:00, 2.91MB/s]\n", + "Unsloth: Saving tokenizer... Done.\n", + "Unsloth: Saving model... Done.\n", + "Unsloth: Saving LoRA adapters. Please wait...\n", + "401 Client Error: Unauthorized for url: https://huggingface.co/api/repos/create (Request ID: Root=1-667c46cc-179d7b0b46deeb6521c1ce66;676583ba-2144-4b1c-928b-aafd3708580e)\n", + "\n", + "Invalid username or password.\n", + "CPU times: user 20min 59s, sys: 8min 13s, total: 29min 13s\n", + "Wall time: 14h 32min 29s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./scripts/tune-small.sh" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "07_MAC_+_Qwen2-7B-Instructi_Unsloth_train", + "widgets": {} + }, + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.1.-1" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "036fc5746f43416db18c19ad8fd36677": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "06e806c82c7b4cbea31c5358dd9c3434": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "087b76a8b7514269b1f0ab29b062e444": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a069d2ab23824f29aa320ac256e2cfe9", + "placeholder": "​", + "style": "IPY_MODEL_06e806c82c7b4cbea31c5358dd9c3434", + "value": "Map (num_proc=2): 100%" + } + }, + "09b76013aa9e45efb6deb23a7a0d0925": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dea41c5260884aa6879b5e1d1697b14f", + "placeholder": "​", + "style": "IPY_MODEL_89965917796a4f81b899fdc7685f33df", + "value": "config.json: 100%" + } + }, + "0a92c56bfa134ef583220d7ef0b13e17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c34be936c8145d3ab41282f30a70713": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f8b6bfe16894500838793f2491d403f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "177c78fce95d4b4ab33057c5a048d693": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1f44c9ce1adf470cbb19784493ed209f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c34be936c8145d3ab41282f30a70713", + "placeholder": "​", + "style": "IPY_MODEL_0a92c56bfa134ef583220d7ef0b13e17", + "value": "model.safetensors: 100%" + } + }, + "201b59ccd9f845e197029b57e424aefc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2157f01726d748f8a9ae4a00664430da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21db8a77b00d4a4e82fdfa608657531f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26e4202cca81496a90d15a0dd4ca9cf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba90fdb8822d47dab7ba203bee297f37", + "IPY_MODEL_61560ff6a36b44f4a9dfdae5c52791d4", + "IPY_MODEL_95fbe66647904c06a20f640630d6dc0e" + ], + "layout": "IPY_MODEL_57182a263d324a3dbf1471c74290a0d5" + } + }, + "27155728b6b84cb199c91c940095d0a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6b91feeed5464877991ac2c207aebe7c", + "IPY_MODEL_cca8113c54c0495daedce1327bf9c68b", + "IPY_MODEL_2e63a29e2f7247bba5beede9a568c99f" + ], + "layout": "IPY_MODEL_5c9d781c28944f3eb86e2a6d44efdf18" + } + }, + "271ddaa553a042d09b6db7b450643d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a58d04b428c46f4b3dbadd3bc6cd529": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d18ddf6482c4d97829ac0e5a7b9868f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "IPY_MODEL_f2df530d22c74977b249dd9fb5f4829b", + "IPY_MODEL_89b2ef0dbfea47ab8e6f8d659e3351d1" + ], + "layout": "IPY_MODEL_3056b148aa9f4e6e8aa3b61d26886255" + } + }, + "2e5087c76f98437cb5dc729230358cba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e63a29e2f7247bba5beede9a568c99f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b993eaec6b224440bf80c0958c6fb536", + "placeholder": "​", + "style": "IPY_MODEL_de868e26e7154f62aa86223a539ad421", + "value": " 464/464 [00:00<00:00, 27.1kB/s]" + } + }, + "2f6c70dd266c4816bfad3fd3d192929a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "30307300bc4e4baf96560e30969a82b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e36a3f9eff0e4cf68834d66b0213ae96", + "placeholder": "​", + "style": "IPY_MODEL_a0037bdccf254159becde630bee3d1db", + "value": "generation_config.json: 100%" + } + }, + "3056b148aa9f4e6e8aa3b61d26886255": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30cdc32298134cb0be4d41615b9e5774": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3572201bd4d74a58b7a665f9bdfdcdba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35b0e8c26d6640e9bd0ed7b242a423d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e5087c76f98437cb5dc729230358cba", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_036fc5746f43416db18c19ad8fd36677", + "value": 51760 + } + }, + "36166c7bcb854b34aca1f41a5d6ea50b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "370692d819df41828b48c4ad446f977b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "39b29a75374b45c0a22506010be2b84e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_30cdc32298134cb0be4d41615b9e5774", + "max": 1179, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_47928317548c454bba6358ab132e8dee", + "value": 1179 + } + }, + "3cf2dd993b5e4d3daecf61e4bab5a404": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_087b76a8b7514269b1f0ab29b062e444", + "IPY_MODEL_35b0e8c26d6640e9bd0ed7b242a423d8", + "IPY_MODEL_54ad89e05fd74576b9b8b5b5a10eaf8d" + ], + "layout": "IPY_MODEL_a41dc44766444a998bec2d777f249d23" + } + }, + "43dec2ede91341f5af60eb522e18e984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4463edd481c1467f914c7dcd6c6e6ffc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47928317548c454bba6358ab132e8dee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "49277aeeac16434a865a4d12308b1abc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4ae7e449e4ea4c729b5f34607c18ebae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b2061b8a73c43ffb0c2f83daf0d0183": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c4c88d4c701450692fa0f6b0c5764b0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c666f4ace3943f8b80ecd20e7503236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ccedf0d93094e63b57a0f8a434fba06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4463edd481c1467f914c7dcd6c6e6ffc", + "max": 44307561, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6d3b9a05db0b4dadb638c686faa0c40a", + "value": 44307561 + } + }, + "4dcf6ff672d24983a1877a8431709aa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5807d5fb827d490fb3bc698f801ffff5", + "placeholder": "​", + "style": "IPY_MODEL_c4f2b06a82fd4987b8b659524a7b503b", + "value": "Generating train split: 100%" + } + }, + "4ea63adfce694725bdba878aef709dd3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5234566b1bfc4655b8d582ea5b46ed9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "54ad89e05fd74576b9b8b5b5a10eaf8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdb1941405ed4e4aa06019933892deb3", + "placeholder": "​", + "style": "IPY_MODEL_668d5377ca56426a99753867e6e24862", + "value": " 51760/51760 [01:02<00:00, 1131.51 examples/s]" + } + }, + "56aee4853b7740e6a977254f5d1fa66d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "57182a263d324a3dbf1471c74290a0d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5807d5fb827d490fb3bc698f801ffff5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c9d781c28944f3eb86e2a6d44efdf18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f40db8173dd4d76b6ef5ed6d9ec8b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61560ff6a36b44f4a9dfdae5c52791d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db19fc8d37db4e45a5790a876836d8c4", + "max": 11610, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36166c7bcb854b34aca1f41a5d6ea50b", + "value": 11610 + } + }, + "6578fd7acdb54c4c93528ea431fd0144": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_370692d819df41828b48c4ad446f977b", + "placeholder": "​", + "style": "IPY_MODEL_a0bf9160eb2647409b3200270914b90f", + "value": " 50.6k/50.6k [00:00<00:00, 2.71MB/s]" + } + }, + "668d5377ca56426a99753867e6e24862": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697f027529b54ee9956bae78a11e0611": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69ac12aec0714318bf2c83d4f4e745f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b2012c3f88547af8884a9ea90e3164b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_938f45f1b3e24118b815d96ae34ba86a", + "placeholder": "​", + "style": "IPY_MODEL_9367047a800747f79c6b225d92397846", + "value": " 44.3M/44.3M [00:01<00:00, 31.0MB/s]" + } + }, + "6b91feeed5464877991ac2c207aebe7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b2061b8a73c43ffb0c2f83daf0d0183", + "placeholder": "​", + "style": "IPY_MODEL_69ac12aec0714318bf2c83d4f4e745f5", + "value": "special_tokens_map.json: 100%" + } + }, + "6d3b9a05db0b4dadb638c686faa0c40a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6dbbedeca9314e66ae50e44ffa31a414": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6e34619b45934040b6092e6fb01ea7fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71ce208e20d6483abb9ed923510c86d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d69dc491b3ab44d7852b21873ed7bb7f", + "placeholder": "​", + "style": "IPY_MODEL_f401d53bf28e44eb906bce6c05412662", + "value": " 51760/51760 [00:01<00:00, 45512.81 examples/s]" + } + }, + "7358cdad832342c983e31efb8754ab78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e352a3404f4c7dad0737f57d29e92f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_988a0e8c1f89446086858da0a891a79c", + "IPY_MODEL_4ccedf0d93094e63b57a0f8a434fba06", + "IPY_MODEL_6b2012c3f88547af8884a9ea90e3164b" + ], + "layout": "IPY_MODEL_7e29cb8dd4df4d5b94407cd8fd3f2011" + } + }, + "74501720ac7e4dbb911a4a99b3633bc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "78e5400bff924a92a4cc61c4ff18b182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9b313fd861948f5aba25b24b1518d30", + "placeholder": "​", + "style": "IPY_MODEL_4c666f4ace3943f8b80ecd20e7503236", + "value": " 1.18k/1.18k [00:00<00:00, 31.3kB/s]" + } + }, + "7975adbc2ec5489ea7fa0167e620d85c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e34619b45934040b6092e6fb01ea7fe", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_271ddaa553a042d09b6db7b450643d8f", + "value": 51760 + } + }, + "7e29cb8dd4df4d5b94407cd8fd3f2011": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "810ff6c0e17d4fa09a30fef27eacff90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89965917796a4f81b899fdc7685f33df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "89b2ef0dbfea47ab8e6f8d659e3351d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8908fa0df3743ecb9d12983a739104f", + "placeholder": "​", + "style": "IPY_MODEL_177c78fce95d4b4ab33057c5a048d693", + "value": " 9.09M/9.09M [00:00<00:00, 32.6MB/s]" + } + }, + "8b3505352a5a42bf910428c40ce40465": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_49277aeeac16434a865a4d12308b1abc", + "placeholder": "​", + "style": "IPY_MODEL_2157f01726d748f8a9ae4a00664430da", + "value": " 5.70G/5.70G [01:02<00:00, 30.1MB/s]" + } + }, + "8fc142b628fb40568730234de1cafde2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ae7e449e4ea4c729b5f34607c18ebae", + "max": 172, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3572201bd4d74a58b7a665f9bdfdcdba", + "value": 172 + } + }, + "9367047a800747f79c6b225d92397846": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "938f45f1b3e24118b815d96ae34ba86a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "95fbe66647904c06a20f640630d6dc0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b0a370dc20654b279b9680692e34418e", + "placeholder": "​", + "style": "IPY_MODEL_cfeb365ddf7548d58b2557f22737fcf5", + "value": " 11.6k/11.6k [00:00<00:00, 716kB/s]" + } + }, + "988a0e8c1f89446086858da0a891a79c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ad2be500fc164c0f86f33e914ef8e6a0", + "placeholder": "​", + "style": "IPY_MODEL_5234566b1bfc4655b8d582ea5b46ed9f", + "value": "Downloading data: 100%" + } + }, + "98c58f23f4d549518832cb2d18f796e8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_09b76013aa9e45efb6deb23a7a0d0925", + "IPY_MODEL_39b29a75374b45c0a22506010be2b84e", + "IPY_MODEL_78e5400bff924a92a4cc61c4ff18b182" + ], + "layout": "IPY_MODEL_2a58d04b428c46f4b3dbadd3bc6cd529" + } + }, + "99fdbb0300c14c139d1937c646f0cfe7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7358cdad832342c983e31efb8754ab78", + "placeholder": "​", + "style": "IPY_MODEL_e9adf418296e436fb48bb9f78885598b", + "value": " 51760/51760 [00:01<00:00, 38665.95 examples/s]" + } + }, + "9f679ad3ec7f4fe8ad0510ffb57bc2ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ea63adfce694725bdba878aef709dd3", + "placeholder": "​", + "style": "IPY_MODEL_74501720ac7e4dbb911a4a99b3633bc6", + "value": "tokenizer.json: 100%" + } + }, + "a0037bdccf254159becde630bee3d1db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a069d2ab23824f29aa320ac256e2cfe9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0bf9160eb2647409b3200270914b90f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a41dc44766444a998bec2d777f249d23": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8464a4c711e4e00aafdfc919b60d07e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fb995c740590427b882572c81d4e848c", + "placeholder": "​", + "style": "IPY_MODEL_201b59ccd9f845e197029b57e424aefc", + "value": " 172/172 [00:00<00:00, 12.0kB/s]" + } + }, + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad2be500fc164c0f86f33e914ef8e6a0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b0240cd9a4554b29ae11f8051984a1c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_edaf890370314a218f138015faa0b05d", + "placeholder": "​", + "style": "IPY_MODEL_697f027529b54ee9956bae78a11e0611", + "value": "Map: 100%" + } + }, + "b0a370dc20654b279b9680692e34418e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b518dcee69074b87be73957cd810e7ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d891f8d0b1fc462f8008d02bb2a15692", + "placeholder": "​", + "style": "IPY_MODEL_cced8fd7e998472794f3f3e3018956a5", + "value": "tokenizer_config.json: 100%" + } + }, + "b8908fa0df3743ecb9d12983a739104f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b993eaec6b224440bf80c0958c6fb536": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9b313fd861948f5aba25b24b1518d30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba90fdb8822d47dab7ba203bee297f37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f8b6bfe16894500838793f2491d403f", + "placeholder": "​", + "style": "IPY_MODEL_bb19f6c747754682a514373a3a0535ba", + "value": "Downloading readme: 100%" + } + }, + "bb19f6c747754682a514373a3a0535ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc883d4cf13e4f8b8a4fe5f410cb6efd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9159e03e61f4f56978ece9c3bca49b2", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_810ff6c0e17d4fa09a30fef27eacff90", + "value": 51760 + } + }, + "c161d94df0f04feba9542237e0856c22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c22f71b1f85843209d7e5321506b9cb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f44c9ce1adf470cbb19784493ed209f", + "IPY_MODEL_f1addc4479d849879e743cf9089e6540", + "IPY_MODEL_8b3505352a5a42bf910428c40ce40465" + ], + "layout": "IPY_MODEL_4c4c88d4c701450692fa0f6b0c5764b0" + } + }, + "c4f2b06a82fd4987b8b659524a7b503b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cca8113c54c0495daedce1327bf9c68b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e02f9b7849c64531835eb77b860d1c93", + "max": 464, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_56aee4853b7740e6a977254f5d1fa66d", + "value": 464 + } + }, + "cced8fd7e998472794f3f3e3018956a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cf245afeb1c04f29a24d291608c3d157": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b518dcee69074b87be73957cd810e7ed", + "IPY_MODEL_e29104486d594b2992d7285e0ef77371", + "IPY_MODEL_6578fd7acdb54c4c93528ea431fd0144" + ], + "layout": "IPY_MODEL_d35db8148a354c56aaac56dbae22536f" + } + }, + "cfe8cae0e22b495bafa221a63d13b283": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfeb365ddf7548d58b2557f22737fcf5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1b47d39450d4019ae85c9b2f943eeaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4dcf6ff672d24983a1877a8431709aa9", + "IPY_MODEL_7975adbc2ec5489ea7fa0167e620d85c", + "IPY_MODEL_71ce208e20d6483abb9ed923510c86d7" + ], + "layout": "IPY_MODEL_cfe8cae0e22b495bafa221a63d13b283" + } + }, + "d35db8148a354c56aaac56dbae22536f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69dc491b3ab44d7852b21873ed7bb7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d891f8d0b1fc462f8008d02bb2a15692": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8e5318cead340c4adbeaccc05d39225": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "daf4cd890b35422683d22fd30bc71e83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b0240cd9a4554b29ae11f8051984a1c6", + "IPY_MODEL_bc883d4cf13e4f8b8a4fe5f410cb6efd", + "IPY_MODEL_99fdbb0300c14c139d1937c646f0cfe7" + ], + "layout": "IPY_MODEL_c161d94df0f04feba9542237e0856c22" + } + }, + "db19fc8d37db4e45a5790a876836d8c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de868e26e7154f62aa86223a539ad421": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dea41c5260884aa6879b5e1d1697b14f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e02f9b7849c64531835eb77b860d1c93": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e29104486d594b2992d7285e0ef77371": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "max": 50641, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2f6c70dd266c4816bfad3fd3d192929a", + "value": 50641 + } + }, + "e36a3f9eff0e4cf68834d66b0213ae96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9159e03e61f4f56978ece9c3bca49b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9adf418296e436fb48bb9f78885598b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "edaf890370314a218f138015faa0b05d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f1addc4479d849879e743cf9089e6540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43dec2ede91341f5af60eb522e18e984", + "max": 5702746405, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d8e5318cead340c4adbeaccc05d39225", + "value": 5702746405 + } + }, + "f2df530d22c74977b249dd9fb5f4829b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_21db8a77b00d4a4e82fdfa608657531f", + "max": 9085698, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6dbbedeca9314e66ae50e44ffa31a414", + "value": 9085698 + } + }, + "f401d53bf28e44eb906bce6c05412662": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fb995c740590427b882572c81d4e848c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fce7a61c25ec4390af43d92b7c473a45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_30307300bc4e4baf96560e30969a82b6", + "IPY_MODEL_8fc142b628fb40568730234de1cafde2", + "IPY_MODEL_a8464a4c711e4e00aafdfc919b60d07e" + ], + "layout": "IPY_MODEL_5f40db8173dd4d76b6ef5ed6d9ec8b6e" + } + }, + "fdb1941405ed4e4aa06019933892deb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/07_tune-lf-py3.11.ipynb b/notebooks/07_tune-lf-py3.11.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..57ffa0c4a882e10d986f5b1189156de26e3837e2 --- /dev/null +++ b/notebooks/07_tune-lf-py3.11.ipynb @@ -0,0 +1,7022 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "0ea8b46b-839b-445b-8043-ccdf4e920ace", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6d394937-6c99-4a7c-9d32-7600a280032f", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /home/inflaton/code/projects/courses/llm-finetuning\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9f67ec60-2f24-411c-84eb-0dd664b44775", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f1597656-8042-4878-9d3b-9ebfb8dd86dc", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('unsloth/Qwen2-0.5B-Instruct-bnb-4bit',\n", + " True,\n", + " None,\n", + " None,\n", + " 2048,\n", + " 10,\n", + " None,\n", + " 'datasets/mac/mac.tsv',\n", + " 'results/mac-results_lf.csv')" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "model_name = os.getenv(\"MODEL_NAME\")\n", + "token = os.getenv(\"HF_TOKEN\") or None\n", + "load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n", + "local_model = os.getenv(\"LOCAL_MODEL\")\n", + "hub_model = os.getenv(\"HUB_MODEL\")\n", + "num_train_epochs = int(os.getenv(\"NUM_TRAIN_EPOCHS\") or 0)\n", + "data_path = os.getenv(\"DATA_PATH\")\n", + "results_path = os.getenv(\"RESULTS_PATH\")\n", + "\n", + "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n", + "dtype = (\n", + " None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n", + ")\n", + "\n", + "model_name, load_in_4bit, local_model, hub_model, max_seq_length, num_train_epochs, dtype, data_path, results_path" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sat Jun 29 17:26:00 2024 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 545.23.07 Driver Version: 546.12 CUDA Version: 12.3 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 NVIDIA GeForce RTX 4080 ... On | 00000000:01:00.0 Off | N/A |\n", + "| N/A 50C P8 4W / 150W | 129MiB / 12282MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "| No running processes found |\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "!nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "loading train/test data files\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fabc731ff8e5499a9c842ef6833f3e98", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Generating train split: 0 examples [00:00, ? examples/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2e186baa65dc4dd1956fa2db0d83b4a1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Generating test split: 0 examples [00:00, ? examples/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english'],\n", + " num_rows: 1133\n", + " })\n", + "})\n" + ] + } + ], + "source": [ + "from llm_toolkit.translation_engine import load_translation_dataset\n", + "\n", + "dataset = load_translation_dataset(data_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "df = dataset[\"train\"].to_pandas()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "df_alpaca = pd.DataFrame({\"instruction\": [\"Please translate the following Chinese text into English and provide only the translated content, nothing else.\"]*len(df)})" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "

\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
instructioninputoutput
0Please translate the following Chinese text in...全仗着狐仙搭救。Because I was protected by a fox fairy.
1Please translate the following Chinese text in...过后，表哥告诉她俩，这人是导演，在外国留过学的，还会编剧，今天拍的这戏，就是他自编自导的。He was the director, the cousin later told the...
2Please translate the following Chinese text in...这凤姐忽然想起一件事来，便向窗外叫：“蓉儿回来！”Xi-feng suddenly seemed to remember something,...
3Please translate the following Chinese text in...三个老红卫兵走到叶文洁面前，面对着她站成了一排——当年，她们也是这样面对叶哲泰的——试图再现...The three old Red Guards stood in front of Ye ...
4Please translate the following Chinese text in...程先生照单全收，都是一个“谢”字，然后问王琦瑶有什么话说。Mr. Cheng accepted their toast with equanimity...
............
4523Please translate the following Chinese text in...外边有两张腿歪面裂的八仙桌子，桌旁胡乱搡着几条狭窄的木凳。Two rickety tables with scarred tops and a few...
4524Please translate the following Chinese text in...贾瑞听了，喜的抓耳挠腮。At this last remark Jia Rui positively scratch...
4525Please translate the following Chinese text in...听了这样的评价，我们心情激动，和大家一起振臂高呼：打倒王二！Hearing comments like this, our emotions were ...
4526Please translate the following Chinese text in...海老公道：“记住了吗？”'Can you remember that?'
4527Please translate the following Chinese text in...上面说，这样写缺少细节。This time the opinions from above said it need...
\n", + "

4528 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " instruction \\\n", + "0 Please translate the following Chinese text in... \n", + "1 Please translate the following Chinese text in... \n", + "2 Please translate the following Chinese text in... \n", + "3 Please translate the following Chinese text in... \n", + "4 Please translate the following Chinese text in... \n", + "... ... \n", + "4523 Please translate the following Chinese text in... \n", + "4524 Please translate the following Chinese text in... \n", + "4525 Please translate the following Chinese text in... \n", + "4526 Please translate the following Chinese text in... \n", + "4527 Please translate the following Chinese text in... \n", + "\n", + " input \\\n", + "0 全仗着狐仙搭救。 \n", + "1 过后，表哥告诉她俩，这人是导演，在外国留过学的，还会编剧，今天拍的这戏，就是他自编自导的。 \n", + "2 这凤姐忽然想起一件事来，便向窗外叫：“蓉儿回来！” \n", + "3 三个老红卫兵走到叶文洁面前，面对着她站成了一排——当年，她们也是这样面对叶哲泰的——试图再现... \n", + "4 程先生照单全收，都是一个“谢”字，然后问王琦瑶有什么话说。 \n", + "... ... \n", + "4523 外边有两张腿歪面裂的八仙桌子，桌旁胡乱搡着几条狭窄的木凳。 \n", + "4524 贾瑞听了，喜的抓耳挠腮。 \n", + "4525 听了这样的评价，我们心情激动，和大家一起振臂高呼：打倒王二！ \n", + "4526 海老公道：“记住了吗？” \n", + "4527 上面说，这样写缺少细节。 \n", + "\n", + " output \n", + "0 Because I was protected by a fox fairy. \n", + "1 He was the director, the cousin later told the... \n", + "2 Xi-feng suddenly seemed to remember something,... \n", + "3 The three old Red Guards stood in front of Ye ... \n", + "4 Mr. Cheng accepted their toast with equanimity... \n", + "... ... \n", + "4523 Two rickety tables with scarred tops and a few... \n", + "4524 At this last remark Jia Rui positively scratch... \n", + "4525 Hearing comments like this, our emotions were ... \n", + "4526 'Can you remember that?' \n", + "4527 This time the opinions from above said it need... \n", + "\n", + "[4528 rows x 3 columns]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_alpaca[\"input\"] = df[\"chinese\"]\n", + "df_alpaca[\"output\"] = df[\"english\"]\n", + "df_alpaca" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "df_alpaca.to_json(\n", + " \"llama-factory/data/alpaca_mac.json\", orient=\"records\", lines=False, indent=2\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_json(\"llama-factory/data/alpaca_mac.json\", orient=\"records\", lines=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
instructioninputoutput
0Please translate the following Chinese text in...全仗着狐仙搭救。Because I was protected by a fox fairy.
1Please translate the following Chinese text in...过后，表哥告诉她俩，这人是导演，在外国留过学的，还会编剧，今天拍的这戏，就是他自编自导的。He was the director, the cousin later told the...
2Please translate the following Chinese text in...这凤姐忽然想起一件事来，便向窗外叫：“蓉儿回来！”Xi-feng suddenly seemed to remember something,...
3Please translate the following Chinese text in...三个老红卫兵走到叶文洁面前，面对着她站成了一排——当年，她们也是这样面对叶哲泰的——试图再现...The three old Red Guards stood in front of Ye ...
4Please translate the following Chinese text in...程先生照单全收，都是一个“谢”字，然后问王琦瑶有什么话说。Mr. Cheng accepted their toast with equanimity...
\n", + "
" + ], + "text/plain": [ + " instruction \\\n", + "0 Please translate the following Chinese text in... \n", + "1 Please translate the following Chinese text in... \n", + "2 Please translate the following Chinese text in... \n", + "3 Please translate the following Chinese text in... \n", + "4 Please translate the following Chinese text in... \n", + "\n", + " input \\\n", + "0 全仗着狐仙搭救。 \n", + "1 过后，表哥告诉她俩，这人是导演，在外国留过学的，还会编剧，今天拍的这戏，就是他自编自导的。 \n", + "2 这凤姐忽然想起一件事来，便向窗外叫：“蓉儿回来！” \n", + "3 三个老红卫兵走到叶文洁面前，面对着她站成了一排——当年，她们也是这样面对叶哲泰的——试图再现... \n", + "4 程先生照单全收，都是一个“谢”字，然后问王琦瑶有什么话说。 \n", + "\n", + " output \n", + "0 Because I was protected by a fox fairy. \n", + "1 He was the director, the cousin later told the... \n", + "2 Xi-feng suddenly seemed to remember something,... \n", + "3 The three old Red Guards stood in front of Ye ... \n", + "4 Mr. Cheng accepted their toast with equanimity... " + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Python 3.11.9\n", + "\u001b[33mWARNING: Package(s) not found: flash-attn\u001b[0m\u001b[33m\n", + "\u001b[0mCPU times: user 23.2 ms, sys: 3.38 ms, total: 26.6 ms\n", + "Wall time: 518 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "!python --version\n", + "!pip show flash-attn" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/llm-finetuning/llama-factory\n", + "06/29/2024 21:58:18 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: torch.bfloat16\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-29 21:58:18,444 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-29 21:58:18,444 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-29 21:58:18,444 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-29 21:58:18,444 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-29 21:58:18,444 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-29 21:58:18,444 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-29 21:58:18,572 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/29/2024 21:58:18 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/29/2024 21:58:18 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "06/29/2024 21:58:18 - INFO - llamafactory.data.loader - Loading dataset alpaca_mac.json...\n", + "Converting format of dataset (num_proc=16): 100%|█| 4528/4528 [00:00<00:00, 1613\n", + "Running tokenizer on dataset (num_proc=16): 100%|█| 4528/4528 [00:01<00:00, 3159\n", + "input_ids:\n", + "[151644, 872, 198, 5501, 14683, 279, 2701, 8453, 1467, 1119, 6364, 323, 3410, 1172, 279, 24531, 2213, 11, 4302, 770, 624, 35987, 102895, 99164, 100324, 100717, 100095, 99509, 1773, 151645, 198, 151644, 77091, 198, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "inputs:\n", + "<|im_start|>user\n", + "Please translate the following Chinese text into English and provide only the translated content, nothing else.\n", + "全仗着狐仙搭救。<|im_end|>\n", + "<|im_start|>assistant\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "label_ids:\n", + "[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "labels:\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "[INFO|configuration_utils.py:733] 2024-06-29 21:58:21,872 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 21:58:21,873 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:3556] 2024-06-29 21:58:21,942 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-29 21:58:24,477 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-29 21:58:24,480 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-29 21:58:59,030 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-29 21:58:59,030 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-29 21:58:59,317 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-29 21:58:59,317 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/29/2024 21:58:59 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.\n", + "06/29/2024 21:58:59 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/29/2024 21:58:59 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.\n", + "06/29/2024 21:58:59 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA\n", + "06/29/2024 21:58:59 - INFO - llamafactory.model.model_utils.misc - Found linear modules: q_proj,up_proj,k_proj,v_proj,gate_proj,down_proj,o_proj\n", + "06/29/2024 21:58:59 - INFO - llamafactory.model.loader - trainable params: 4,399,104 || all params: 498,431,872 || trainable%: 0.8826\n", + "[INFO|trainer.py:642] 2024-06-29 21:58:59,830 >> Using auto half precision backend\n", + "06/29/2024 21:58:59 - WARNING - llamafactory.train.callbacks - Previous trainer log in this folder will be deleted.\n", + "[INFO|trainer.py:2128] 2024-06-29 21:58:59,963 >> ***** Running training *****\n", + "[INFO|trainer.py:2129] 2024-06-29 21:58:59,963 >> Num examples = 4,482\n", + "[INFO|trainer.py:2130] 2024-06-29 21:58:59,963 >> Num Epochs = 10\n", + "[INFO|trainer.py:2131] 2024-06-29 21:58:59,963 >> Instantaneous batch size per device = 1\n", + "[INFO|trainer.py:2134] 2024-06-29 21:58:59,963 >> Total train batch size (w. parallel, distributed & accumulation) = 8\n", + "[INFO|trainer.py:2135] 2024-06-29 21:58:59,963 >> Gradient Accumulation steps = 8\n", + "[INFO|trainer.py:2136] 2024-06-29 21:58:59,963 >> Total optimization steps = 5,600\n", + "[INFO|trainer.py:2137] 2024-06-29 21:58:59,964 >> Number of trainable parameters = 4,399,104\n", + "{'loss': 2.5824, 'grad_norm': 3.00181245803833, 'learning_rate': 1.7857142857142857e-06, 'epoch': 0.02}\n", + "{'loss': 2.7043, 'grad_norm': 3.7918665409088135, 'learning_rate': 3.5714285714285714e-06, 'epoch': 0.04}\n", + "{'loss': 2.5845, 'grad_norm': 2.4548499584198, 'learning_rate': 5.357142857142857e-06, 'epoch': 0.05}\n", + "{'loss': 2.5238, 'grad_norm': 5.136275291442871, 'learning_rate': 7.142857142857143e-06, 'epoch': 0.07}\n", + "{'loss': 2.7407, 'grad_norm': 2.911478281021118, 'learning_rate': 8.92857142857143e-06, 'epoch': 0.09}\n", + "{'loss': 2.4438, 'grad_norm': 2.7009449005126953, 'learning_rate': 1.0714285714285714e-05, 'epoch': 0.11}\n", + "{'loss': 2.619, 'grad_norm': 2.6438188552856445, 'learning_rate': 1.25e-05, 'epoch': 0.12}\n", + "{'loss': 2.3602, 'grad_norm': 2.3748607635498047, 'learning_rate': 1.4285714285714285e-05, 'epoch': 0.14}\n", + "{'loss': 2.5023, 'grad_norm': 2.8664743900299072, 'learning_rate': 1.6071428571428572e-05, 'epoch': 0.16}\n", + "{'loss': 2.3225, 'grad_norm': 2.3505067825317383, 'learning_rate': 1.785714285714286e-05, 'epoch': 0.18}\n", + "{'loss': 2.3869, 'grad_norm': 3.261944532394409, 'learning_rate': 1.9642857142857145e-05, 'epoch': 0.2}\n", + "{'loss': 2.3922, 'grad_norm': 2.6836485862731934, 'learning_rate': 2.1428571428571428e-05, 'epoch': 0.21}\n", + "{'loss': 2.3024, 'grad_norm': 2.848069667816162, 'learning_rate': 2.3214285714285715e-05, 'epoch': 0.23}\n", + "{'loss': 2.3501, 'grad_norm': 3.22798752784729, 'learning_rate': 2.5e-05, 'epoch': 0.25}\n", + "{'loss': 2.2154, 'grad_norm': 2.441416025161743, 'learning_rate': 2.6785714285714288e-05, 'epoch': 0.27}\n", + "{'loss': 2.2651, 'grad_norm': 2.3891408443450928, 'learning_rate': 2.857142857142857e-05, 'epoch': 0.29}\n", + "{'loss': 2.333, 'grad_norm': 2.3359410762786865, 'learning_rate': 3.0357142857142857e-05, 'epoch': 0.3}\n", + "{'loss': 2.1135, 'grad_norm': 2.6461141109466553, 'learning_rate': 3.2142857142857144e-05, 'epoch': 0.32}\n", + "{'loss': 2.2379, 'grad_norm': 3.4454798698425293, 'learning_rate': 3.392857142857143e-05, 'epoch': 0.34}\n", + "{'loss': 2.4006, 'grad_norm': 2.9662983417510986, 'learning_rate': 3.571428571428572e-05, 'epoch': 0.36}\n", + "{'loss': 2.3065, 'grad_norm': 2.796970844268799, 'learning_rate': 3.7500000000000003e-05, 'epoch': 0.37}\n", + "{'loss': 2.2302, 'grad_norm': 3.6208152770996094, 'learning_rate': 3.928571428571429e-05, 'epoch': 0.39}\n", + "{'loss': 2.1966, 'grad_norm': 3.335953950881958, 'learning_rate': 4.107142857142857e-05, 'epoch': 0.41}\n", + "{'loss': 2.3829, 'grad_norm': 4.235249042510986, 'learning_rate': 4.2857142857142856e-05, 'epoch': 0.43}\n", + "{'loss': 2.2592, 'grad_norm': 3.228585720062256, 'learning_rate': 4.464285714285715e-05, 'epoch': 0.45}\n", + "{'loss': 2.2236, 'grad_norm': 3.2165491580963135, 'learning_rate': 4.642857142857143e-05, 'epoch': 0.46}\n", + "{'loss': 2.2113, 'grad_norm': 4.193121433258057, 'learning_rate': 4.8214285714285716e-05, 'epoch': 0.48}\n", + "{'loss': 2.3292, 'grad_norm': 4.554675579071045, 'learning_rate': 5e-05, 'epoch': 0.5}\n", + "{'loss': 2.1239, 'grad_norm': 2.7911994457244873, 'learning_rate': 5.1785714285714296e-05, 'epoch': 0.52}\n", + "{'loss': 2.2483, 'grad_norm': 3.6781301498413086, 'learning_rate': 5.3571428571428575e-05, 'epoch': 0.54}\n", + "{'loss': 2.2574, 'grad_norm': 4.210690021514893, 'learning_rate': 5.535714285714286e-05, 'epoch': 0.55}\n", + "{'loss': 2.0374, 'grad_norm': 6.651491165161133, 'learning_rate': 5.714285714285714e-05, 'epoch': 0.57}\n", + "{'loss': 2.1021, 'grad_norm': 5.034158706665039, 'learning_rate': 5.8928571428571435e-05, 'epoch': 0.59}\n", + "{'loss': 2.1575, 'grad_norm': 4.4245381355285645, 'learning_rate': 6.0714285714285715e-05, 'epoch': 0.61}\n", + "{'loss': 2.1584, 'grad_norm': 4.884017467498779, 'learning_rate': 6.25e-05, 'epoch': 0.62}\n", + "{'loss': 2.0592, 'grad_norm': 3.4757015705108643, 'learning_rate': 6.428571428571429e-05, 'epoch': 0.64}\n", + "{'loss': 2.2959, 'grad_norm': 4.756143093109131, 'learning_rate': 6.607142857142857e-05, 'epoch': 0.66}\n", + "{'loss': 2.2236, 'grad_norm': 3.61995005607605, 'learning_rate': 6.785714285714286e-05, 'epoch': 0.68}\n", + "{'loss': 1.9521, 'grad_norm': 3.775660991668701, 'learning_rate': 6.964285714285715e-05, 'epoch': 0.7}\n", + "{'loss': 2.1048, 'grad_norm': 3.84194016456604, 'learning_rate': 7.142857142857143e-05, 'epoch': 0.71}\n", + "{'loss': 2.2049, 'grad_norm': 3.697145462036133, 'learning_rate': 7.321428571428571e-05, 'epoch': 0.73}\n", + "{'loss': 2.2091, 'grad_norm': 3.071280002593994, 'learning_rate': 7.500000000000001e-05, 'epoch': 0.75}\n", + "{'loss': 2.1879, 'grad_norm': 3.8867111206054688, 'learning_rate': 7.67857142857143e-05, 'epoch': 0.77}\n", + "{'loss': 2.0959, 'grad_norm': 4.871102333068848, 'learning_rate': 7.857142857142858e-05, 'epoch': 0.79}\n", + "{'loss': 2.0237, 'grad_norm': 2.9602854251861572, 'learning_rate': 8.035714285714287e-05, 'epoch': 0.8}\n", + "{'loss': 2.12, 'grad_norm': 3.3257362842559814, 'learning_rate': 8.214285714285714e-05, 'epoch': 0.82}\n", + "{'loss': 2.1227, 'grad_norm': 5.4583024978637695, 'learning_rate': 8.392857142857144e-05, 'epoch': 0.84}\n", + "{'loss': 2.1448, 'grad_norm': 3.455509901046753, 'learning_rate': 8.571428571428571e-05, 'epoch': 0.86}\n", + "{'loss': 2.138, 'grad_norm': 2.953312397003174, 'learning_rate': 8.75e-05, 'epoch': 0.87}\n", + "{'loss': 2.3248, 'grad_norm': 3.1288394927978516, 'learning_rate': 8.92857142857143e-05, 'epoch': 0.89}\n", + "{'loss': 2.2541, 'grad_norm': 3.630788803100586, 'learning_rate': 9.107142857142857e-05, 'epoch': 0.91}\n", + "{'loss': 2.1579, 'grad_norm': 4.1369805335998535, 'learning_rate': 9.285714285714286e-05, 'epoch': 0.93}\n", + "{'loss': 2.1881, 'grad_norm': 3.945438861846924, 'learning_rate': 9.464285714285715e-05, 'epoch': 0.95}\n", + "{'loss': 2.1433, 'grad_norm': 3.308486223220825, 'learning_rate': 9.642857142857143e-05, 'epoch': 0.96}\n", + "{'loss': 2.1414, 'grad_norm': 3.59633207321167, 'learning_rate': 9.821428571428572e-05, 'epoch': 0.98}\n", + "{'loss': 2.1674, 'grad_norm': 3.1946074962615967, 'learning_rate': 0.0001, 'epoch': 1.0}\n", + " 10%|███▊ | 560/5600 [11:54<1:46:21, 1.27s/it][INFO|trainer.py:3788] 2024-06-29 22:10:54,528 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 22:10:54,528 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 22:10:54,528 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-560\n", + "[INFO|configuration_utils.py:733] 2024-06-29 22:10:57,646 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 22:10:57,646 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 22:10:57,680 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-560/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 22:10:57,680 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-560/special_tokens_map.json\n", + "{'loss': 2.0604, 'grad_norm': 3.1620354652404785, 'learning_rate': 9.999902864657691e-05, 'epoch': 1.02}\n", + "{'loss': 1.8643, 'grad_norm': 3.8117380142211914, 'learning_rate': 9.999611462404875e-05, 'epoch': 1.04}\n", + "{'loss': 2.0455, 'grad_norm': 3.2619926929473877, 'learning_rate': 9.999125804563732e-05, 'epoch': 1.05}\n", + "{'loss': 1.9864, 'grad_norm': 4.930575370788574, 'learning_rate': 9.998445910004082e-05, 'epoch': 1.07}\n", + "{'loss': 1.9508, 'grad_norm': 3.7913410663604736, 'learning_rate': 9.997571805142639e-05, 'epoch': 1.09}\n", + "{'loss': 1.9803, 'grad_norm': 4.443136215209961, 'learning_rate': 9.996503523941994e-05, 'epoch': 1.11}\n", + "{'loss': 1.9275, 'grad_norm': 3.6109349727630615, 'learning_rate': 9.99524110790929e-05, 'epoch': 1.12}\n", + "{'loss': 1.9954, 'grad_norm': 5.655592918395996, 'learning_rate': 9.993784606094612e-05, 'epoch': 1.14}\n", + "{'loss': 1.9877, 'grad_norm': 3.884321928024292, 'learning_rate': 9.992134075089084e-05, 'epoch': 1.16}\n", + "{'loss': 1.8812, 'grad_norm': 3.8242244720458984, 'learning_rate': 9.99028957902266e-05, 'epoch': 1.18}\n", + "{'loss': 1.8684, 'grad_norm': 2.90846586227417, 'learning_rate': 9.988251189561645e-05, 'epoch': 1.2}\n", + "{'loss': 1.993, 'grad_norm': 3.7888333797454834, 'learning_rate': 9.986018985905901e-05, 'epoch': 1.21}\n", + "{'loss': 1.8779, 'grad_norm': 4.632900714874268, 'learning_rate': 9.983593054785776e-05, 'epoch': 1.23}\n", + "{'loss': 1.999, 'grad_norm': 4.890506267547607, 'learning_rate': 9.980973490458728e-05, 'epoch': 1.25}\n", + "{'loss': 1.9202, 'grad_norm': 4.923672676086426, 'learning_rate': 9.978160394705668e-05, 'epoch': 1.27}\n", + "{'loss': 2.1275, 'grad_norm': 4.535311222076416, 'learning_rate': 9.975153876827008e-05, 'epoch': 1.29}\n", + "{'loss': 2.0239, 'grad_norm': 3.3138980865478516, 'learning_rate': 9.971954053638399e-05, 'epoch': 1.3}\n", + "{'loss': 2.0188, 'grad_norm': 3.4345853328704834, 'learning_rate': 9.968561049466214e-05, 'epoch': 1.32}\n", + "{'loss': 2.0142, 'grad_norm': 8.660140991210938, 'learning_rate': 9.964974996142698e-05, 'epoch': 1.34}\n", + "{'loss': 1.7658, 'grad_norm': 4.349238872528076, 'learning_rate': 9.961196033000861e-05, 'epoch': 1.36}\n", + "{'loss': 1.9781, 'grad_norm': 4.784688949584961, 'learning_rate': 9.957224306869053e-05, 'epoch': 1.37}\n", + "{'loss': 1.8478, 'grad_norm': 5.488619327545166, 'learning_rate': 9.953059972065265e-05, 'epoch': 1.39}\n", + "{'loss': 1.719, 'grad_norm': 3.712329387664795, 'learning_rate': 9.948703190391131e-05, 'epoch': 1.41}\n", + "{'loss': 2.0146, 'grad_norm': 4.957381248474121, 'learning_rate': 9.944154131125642e-05, 'epoch': 1.43}\n", + "{'loss': 1.8996, 'grad_norm': 4.802273273468018, 'learning_rate': 9.939412971018574e-05, 'epoch': 1.45}\n", + "{'loss': 1.9122, 'grad_norm': 3.6675291061401367, 'learning_rate': 9.934479894283606e-05, 'epoch': 1.46}\n", + "{'loss': 2.0716, 'grad_norm': 3.885627031326294, 'learning_rate': 9.92935509259118e-05, 'epoch': 1.48}\n", + "{'loss': 1.8866, 'grad_norm': 5.027438640594482, 'learning_rate': 9.924038765061042e-05, 'epoch': 1.5}\n", + "{'loss': 1.8188, 'grad_norm': 3.39078426361084, 'learning_rate': 9.918531118254507e-05, 'epoch': 1.52}\n", + "{'loss': 1.948, 'grad_norm': 4.390409469604492, 'learning_rate': 9.912832366166442e-05, 'epoch': 1.54}\n", + "{'loss': 1.9499, 'grad_norm': 5.019458770751953, 'learning_rate': 9.906942730216939e-05, 'epoch': 1.55}\n", + "{'loss': 1.8564, 'grad_norm': 3.9593818187713623, 'learning_rate': 9.900862439242719e-05, 'epoch': 1.57}\n", + "{'loss': 1.9739, 'grad_norm': 4.117242336273193, 'learning_rate': 9.894591729488242e-05, 'epoch': 1.59}\n", + "{'loss': 2.0614, 'grad_norm': 3.597482204437256, 'learning_rate': 9.888130844596524e-05, 'epoch': 1.61}\n", + "{'loss': 1.8292, 'grad_norm': 3.4714455604553223, 'learning_rate': 9.881480035599667e-05, 'epoch': 1.62}\n", + "{'loss': 1.8707, 'grad_norm': 3.4483628273010254, 'learning_rate': 9.874639560909117e-05, 'epoch': 1.64}\n", + "{'loss': 1.8787, 'grad_norm': 3.199208974838257, 'learning_rate': 9.867609686305617e-05, 'epoch': 1.66}\n", + "{'loss': 1.8856, 'grad_norm': 3.4779880046844482, 'learning_rate': 9.860390684928873e-05, 'epoch': 1.68}\n", + "{'loss': 1.8618, 'grad_norm': 5.559018135070801, 'learning_rate': 9.852982837266955e-05, 'epoch': 1.7}\n", + "{'loss': 1.864, 'grad_norm': 4.512182235717773, 'learning_rate': 9.84538643114539e-05, 'epoch': 1.71}\n", + "{'loss': 1.9054, 'grad_norm': 3.1477646827697754, 'learning_rate': 9.837601761715983e-05, 'epoch': 1.73}\n", + "{'loss': 2.0045, 'grad_norm': 3.805159091949463, 'learning_rate': 9.829629131445342e-05, 'epoch': 1.75}\n", + "{'loss': 1.9549, 'grad_norm': 3.356356143951416, 'learning_rate': 9.82146885010314e-05, 'epoch': 1.77}\n", + "{'loss': 1.8738, 'grad_norm': 4.890620231628418, 'learning_rate': 9.81312123475006e-05, 'epoch': 1.78}\n", + "{'loss': 1.906, 'grad_norm': 3.6688284873962402, 'learning_rate': 9.804586609725499e-05, 'epoch': 1.8}\n", + "{'loss': 1.8104, 'grad_norm': 3.987600564956665, 'learning_rate': 9.79586530663494e-05, 'epoch': 1.82}\n", + "{'loss': 1.7931, 'grad_norm': 3.517052173614502, 'learning_rate': 9.78695766433709e-05, 'epoch': 1.84}\n", + "{'loss': 1.984, 'grad_norm': 3.507730722427368, 'learning_rate': 9.777864028930705e-05, 'epoch': 1.86}\n", + "{'loss': 1.8427, 'grad_norm': 4.782810211181641, 'learning_rate': 9.768584753741134e-05, 'epoch': 1.87}\n", + "{'loss': 1.8765, 'grad_norm': 4.302423000335693, 'learning_rate': 9.759120199306613e-05, 'epoch': 1.89}\n", + "{'loss': 2.0702, 'grad_norm': 4.296674728393555, 'learning_rate': 9.74947073336423e-05, 'epoch': 1.91}\n", + "{'loss': 2.0158, 'grad_norm': 4.246646881103516, 'learning_rate': 9.73963673083566e-05, 'epoch': 1.93}\n", + "{'loss': 1.9104, 'grad_norm': 3.5928955078125, 'learning_rate': 9.72961857381258e-05, 'epoch': 1.95}\n", + "{'loss': 1.7656, 'grad_norm': 3.674893379211426, 'learning_rate': 9.719416651541839e-05, 'epoch': 1.96}\n", + "{'loss': 1.8906, 'grad_norm': 3.089376211166382, 'learning_rate': 9.709031360410318e-05, 'epoch': 1.98}\n", + "{'loss': 1.9109, 'grad_norm': 4.134565830230713, 'learning_rate': 9.698463103929542e-05, 'epoch': 2.0}\n", + " 20%|███████▍ | 1120/5600 [23:38<1:33:12, 1.25s/it][INFO|trainer.py:3788] 2024-06-29 22:22:38,708 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 22:22:38,709 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 22:22:38,709 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-1120\n", + "[INFO|configuration_utils.py:733] 2024-06-29 22:22:41,066 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 22:22:41,066 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 22:22:41,093 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-1120/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 22:22:41,093 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-1120/special_tokens_map.json\n", + "{'loss': 1.6353, 'grad_norm': 4.750072479248047, 'learning_rate': 9.687712292719997e-05, 'epoch': 2.02}\n", + "{'loss': 1.4907, 'grad_norm': 3.403005838394165, 'learning_rate': 9.67677934449517e-05, 'epoch': 2.03}\n", + "{'loss': 1.5814, 'grad_norm': 4.471570014953613, 'learning_rate': 9.665664684045333e-05, 'epoch': 2.05}\n", + "{'loss': 1.5284, 'grad_norm': 5.069768905639648, 'learning_rate': 9.654368743221022e-05, 'epoch': 2.07}\n", + "{'loss': 1.424, 'grad_norm': 3.740079641342163, 'learning_rate': 9.642891960916268e-05, 'epoch': 2.09}\n", + "{'loss': 1.5123, 'grad_norm': 3.6874232292175293, 'learning_rate': 9.631234783051544e-05, 'epoch': 2.11}\n", + "{'loss': 1.7386, 'grad_norm': 4.291646480560303, 'learning_rate': 9.619397662556435e-05, 'epoch': 2.12}\n", + "{'loss': 1.5025, 'grad_norm': 4.665364742279053, 'learning_rate': 9.607381059352038e-05, 'epoch': 2.14}\n", + "{'loss': 1.5494, 'grad_norm': 4.6409173011779785, 'learning_rate': 9.595185440333103e-05, 'epoch': 2.16}\n", + "{'loss': 1.596, 'grad_norm': 5.967792987823486, 'learning_rate': 9.582811279349882e-05, 'epoch': 2.18}\n", + "{'loss': 1.5333, 'grad_norm': 3.9247050285339355, 'learning_rate': 9.570259057189717e-05, 'epoch': 2.2}\n", + "{'loss': 1.5773, 'grad_norm': 5.318151950836182, 'learning_rate': 9.557529261558367e-05, 'epoch': 2.21}\n", + "{'loss': 1.6159, 'grad_norm': 5.290398120880127, 'learning_rate': 9.544622387061055e-05, 'epoch': 2.23}\n", + "{'loss': 1.6438, 'grad_norm': 4.89390230178833, 'learning_rate': 9.53153893518325e-05, 'epoch': 2.25}\n", + "{'loss': 1.486, 'grad_norm': 4.651273250579834, 'learning_rate': 9.518279414271183e-05, 'epoch': 2.27}\n", + "{'loss': 1.531, 'grad_norm': 5.672192573547363, 'learning_rate': 9.504844339512095e-05, 'epoch': 2.28}\n", + "{'loss': 1.5734, 'grad_norm': 3.6605958938598633, 'learning_rate': 9.491234232914221e-05, 'epoch': 2.3}\n", + "{'loss': 1.6449, 'grad_norm': 4.812197685241699, 'learning_rate': 9.477449623286505e-05, 'epoch': 2.32}\n", + "{'loss': 1.659, 'grad_norm': 4.542179584503174, 'learning_rate': 9.463491046218058e-05, 'epoch': 2.34}\n", + "{'loss': 1.6723, 'grad_norm': 4.588232517242432, 'learning_rate': 9.449359044057345e-05, 'epoch': 2.36}\n", + "{'loss': 1.619, 'grad_norm': 6.938955783843994, 'learning_rate': 9.435054165891109e-05, 'epoch': 2.37}\n", + "{'loss': 1.7966, 'grad_norm': 4.723308563232422, 'learning_rate': 9.420576967523049e-05, 'epoch': 2.39}\n", + "{'loss': 1.6444, 'grad_norm': 4.64656925201416, 'learning_rate': 9.405928011452211e-05, 'epoch': 2.41}\n", + "{'loss': 1.5322, 'grad_norm': 5.396662712097168, 'learning_rate': 9.391107866851143e-05, 'epoch': 2.43}\n", + "{'loss': 1.6074, 'grad_norm': 4.109992027282715, 'learning_rate': 9.376117109543769e-05, 'epoch': 2.45}\n", + "{'loss': 1.5129, 'grad_norm': 4.073942184448242, 'learning_rate': 9.360956321983028e-05, 'epoch': 2.46}\n", + "{'loss': 1.5839, 'grad_norm': 5.0658698081970215, 'learning_rate': 9.345626093228233e-05, 'epoch': 2.48}\n", + "{'loss': 1.726, 'grad_norm': 4.494250297546387, 'learning_rate': 9.330127018922194e-05, 'epoch': 2.5}\n", + "{'loss': 1.6129, 'grad_norm': 5.197183609008789, 'learning_rate': 9.314459701268065e-05, 'epoch': 2.52}\n", + "{'loss': 1.5691, 'grad_norm': 4.414649486541748, 'learning_rate': 9.298624749005951e-05, 'epoch': 2.53}\n", + "{'loss': 1.6516, 'grad_norm': 6.023291110992432, 'learning_rate': 9.282622777389258e-05, 'epoch': 2.55}\n", + "{'loss': 1.4793, 'grad_norm': 5.750635147094727, 'learning_rate': 9.266454408160779e-05, 'epoch': 2.57}\n", + "{'loss': 1.761, 'grad_norm': 6.335220813751221, 'learning_rate': 9.250120269528546e-05, 'epoch': 2.59}\n", + "{'loss': 1.5627, 'grad_norm': 6.77303409576416, 'learning_rate': 9.233620996141421e-05, 'epoch': 2.61}\n", + "{'loss': 1.656, 'grad_norm': 3.9022696018218994, 'learning_rate': 9.21695722906443e-05, 'epoch': 2.62}\n", + "{'loss': 1.5537, 'grad_norm': 3.297802209854126, 'learning_rate': 9.200129615753859e-05, 'epoch': 2.64}\n", + "{'loss': 1.5451, 'grad_norm': 4.561464309692383, 'learning_rate': 9.183138810032099e-05, 'epoch': 2.66}\n", + "{'loss': 1.7119, 'grad_norm': 5.242650508880615, 'learning_rate': 9.165985472062246e-05, 'epoch': 2.68}\n", + "{'loss': 1.499, 'grad_norm': 5.535559177398682, 'learning_rate': 9.148670268322438e-05, 'epoch': 2.7}\n", + "{'loss': 1.4735, 'grad_norm': 5.1633100509643555, 'learning_rate': 9.131193871579975e-05, 'epoch': 2.71}\n", + "{'loss': 1.7502, 'grad_norm': 5.2197184562683105, 'learning_rate': 9.113556960865167e-05, 'epoch': 2.73}\n", + "{'loss': 1.6312, 'grad_norm': 4.655239105224609, 'learning_rate': 9.09576022144496e-05, 'epoch': 2.75}\n", + "{'loss': 1.6455, 'grad_norm': 4.8979644775390625, 'learning_rate': 9.077804344796302e-05, 'epoch': 2.77}\n", + "{'loss': 1.6535, 'grad_norm': 4.097564220428467, 'learning_rate': 9.059690028579284e-05, 'epoch': 2.78}\n", + "{'loss': 1.5067, 'grad_norm': 3.5800154209136963, 'learning_rate': 9.041417976610027e-05, 'epoch': 2.8}\n", + "{'loss': 1.5709, 'grad_norm': 4.4109787940979, 'learning_rate': 9.022988898833342e-05, 'epoch': 2.82}\n", + "{'loss': 1.5517, 'grad_norm': 4.352450370788574, 'learning_rate': 9.004403511295141e-05, 'epoch': 2.84}\n", + "{'loss': 1.5259, 'grad_norm': 6.1658525466918945, 'learning_rate': 8.985662536114613e-05, 'epoch': 2.86}\n", + "{'loss': 1.7768, 'grad_norm': 4.468559265136719, 'learning_rate': 8.966766701456177e-05, 'epoch': 2.87}\n", + "{'loss': 1.5683, 'grad_norm': 4.119050979614258, 'learning_rate': 8.947716741501177e-05, 'epoch': 2.89}\n", + "{'loss': 1.5665, 'grad_norm': 5.216476917266846, 'learning_rate': 8.928513396419368e-05, 'epoch': 2.91}\n", + "{'loss': 1.524, 'grad_norm': 4.42177152633667, 'learning_rate': 8.90915741234015e-05, 'epoch': 2.93}\n", + "{'loss': 1.7169, 'grad_norm': 4.006609916687012, 'learning_rate': 8.889649541323574e-05, 'epoch': 2.95}\n", + "{'loss': 1.7248, 'grad_norm': 4.3928914070129395, 'learning_rate': 8.869990541331138e-05, 'epoch': 2.96}\n", + "{'loss': 1.5582, 'grad_norm': 5.347744464874268, 'learning_rate': 8.850181176196315e-05, 'epoch': 2.98}\n", + "{'loss': 1.6926, 'grad_norm': 4.4702019691467285, 'learning_rate': 8.83022221559489e-05, 'epoch': 3.0}\n", + " 30%|███████████ | 1680/5600 [35:12<1:20:28, 1.23s/it][INFO|trainer.py:3788] 2024-06-29 22:34:12,572 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 22:34:12,573 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 22:34:12,573 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-1680\n", + "[INFO|configuration_utils.py:733] 2024-06-29 22:34:15,015 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 22:34:15,015 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 22:34:15,046 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-1680/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 22:34:15,047 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-1680/special_tokens_map.json\n", + "{'loss': 1.4704, 'grad_norm': 4.0504021644592285, 'learning_rate': 8.810114435015054e-05, 'epoch': 3.02}\n", + "{'loss': 1.1325, 'grad_norm': 4.55043363571167, 'learning_rate': 8.789858615727265e-05, 'epoch': 3.03}\n", + "{'loss': 1.3183, 'grad_norm': 5.89686393737793, 'learning_rate': 8.7694555447539e-05, 'epoch': 3.05}\n", + "{'loss': 1.2266, 'grad_norm': 6.354063510894775, 'learning_rate': 8.748906014838672e-05, 'epoch': 3.07}\n", + "{'loss': 1.2011, 'grad_norm': 5.328189849853516, 'learning_rate': 8.728210824415827e-05, 'epoch': 3.09}\n", + "{'loss': 1.3191, 'grad_norm': 5.733210563659668, 'learning_rate': 8.707370777579133e-05, 'epoch': 3.11}\n", + "{'loss': 1.242, 'grad_norm': 4.455051422119141, 'learning_rate': 8.68638668405062e-05, 'epoch': 3.12}\n", + "{'loss': 1.3565, 'grad_norm': 5.194347381591797, 'learning_rate': 8.665259359149132e-05, 'epoch': 3.14}\n", + "{'loss': 1.2447, 'grad_norm': 5.317370414733887, 'learning_rate': 8.643989623758643e-05, 'epoch': 3.16}\n", + "{'loss': 1.3055, 'grad_norm': 6.149751663208008, 'learning_rate': 8.622578304296364e-05, 'epoch': 3.18}\n", + "{'loss': 1.2915, 'grad_norm': 5.196756839752197, 'learning_rate': 8.601026232680634e-05, 'epoch': 3.2}\n", + "{'loss': 1.2714, 'grad_norm': 6.269302845001221, 'learning_rate': 8.579334246298593e-05, 'epoch': 3.21}\n", + "{'loss': 1.4645, 'grad_norm': 4.260131359100342, 'learning_rate': 8.557503187973651e-05, 'epoch': 3.23}\n", + "{'loss': 1.266, 'grad_norm': 4.698756217956543, 'learning_rate': 8.535533905932738e-05, 'epoch': 3.25}\n", + "{'loss': 1.242, 'grad_norm': 5.149835109710693, 'learning_rate': 8.513427253773346e-05, 'epoch': 3.27}\n", + "{'loss': 1.2025, 'grad_norm': 5.154805660247803, 'learning_rate': 8.491184090430364e-05, 'epoch': 3.28}\n", + "{'loss': 1.3668, 'grad_norm': 6.301427364349365, 'learning_rate': 8.468805280142709e-05, 'epoch': 3.3}\n", + "{'loss': 1.3626, 'grad_norm': 4.72573709487915, 'learning_rate': 8.446291692419736e-05, 'epoch': 3.32}\n", + "{'loss': 1.3326, 'grad_norm': 4.458547592163086, 'learning_rate': 8.423644202007467e-05, 'epoch': 3.34}\n", + "{'loss': 1.324, 'grad_norm': 4.596677303314209, 'learning_rate': 8.400863688854597e-05, 'epoch': 3.36}\n", + "{'loss': 1.3217, 'grad_norm': 5.230796813964844, 'learning_rate': 8.377951038078302e-05, 'epoch': 3.37}\n", + "{'loss': 1.2399, 'grad_norm': 4.330605983734131, 'learning_rate': 8.354907139929851e-05, 'epoch': 3.39}\n", + "{'loss': 1.2958, 'grad_norm': 5.9636945724487305, 'learning_rate': 8.33173288976002e-05, 'epoch': 3.41}\n", + "{'loss': 1.3935, 'grad_norm': 4.622984409332275, 'learning_rate': 8.308429187984297e-05, 'epoch': 3.43}\n", + "{'loss': 1.3421, 'grad_norm': 4.806463718414307, 'learning_rate': 8.284996940047903e-05, 'epoch': 3.44}\n", + "{'loss': 1.3621, 'grad_norm': 4.162802219390869, 'learning_rate': 8.261437056390606e-05, 'epoch': 3.46}\n", + "{'loss': 1.2849, 'grad_norm': 5.431687355041504, 'learning_rate': 8.237750452411353e-05, 'epoch': 3.48}\n", + "{'loss': 1.2827, 'grad_norm': 6.106764316558838, 'learning_rate': 8.213938048432697e-05, 'epoch': 3.5}\n", + "{'loss': 1.2381, 'grad_norm': 5.98523473739624, 'learning_rate': 8.190000769665044e-05, 'epoch': 3.52}\n", + "{'loss': 1.3037, 'grad_norm': 4.923933029174805, 'learning_rate': 8.1659395461707e-05, 'epoch': 3.53}\n", + "{'loss': 1.2708, 'grad_norm': 6.869691371917725, 'learning_rate': 8.141755312827736e-05, 'epoch': 3.55}\n", + "{'loss': 1.414, 'grad_norm': 4.601339340209961, 'learning_rate': 8.117449009293668e-05, 'epoch': 3.57}\n", + "{'loss': 1.1949, 'grad_norm': 4.767725944519043, 'learning_rate': 8.093021579968941e-05, 'epoch': 3.59}\n", + "{'loss': 1.2801, 'grad_norm': 4.9436211585998535, 'learning_rate': 8.068473973960238e-05, 'epoch': 3.61}\n", + "{'loss': 1.3493, 'grad_norm': 5.783080577850342, 'learning_rate': 8.043807145043604e-05, 'epoch': 3.62}\n", + "{'loss': 1.3132, 'grad_norm': 4.968575477600098, 'learning_rate': 8.019022051627388e-05, 'epoch': 3.64}\n", + "{'loss': 1.2486, 'grad_norm': 5.723098278045654, 'learning_rate': 7.994119656715002e-05, 'epoch': 3.66}\n", + "{'loss': 1.4033, 'grad_norm': 7.168787956237793, 'learning_rate': 7.969100927867507e-05, 'epoch': 3.68}\n", + "{'loss': 1.3969, 'grad_norm': 5.891693592071533, 'learning_rate': 7.943966837166023e-05, 'epoch': 3.69}\n", + "{'loss': 1.4086, 'grad_norm': 4.852097511291504, 'learning_rate': 7.91871836117395e-05, 'epoch': 3.71}\n", + "{'loss': 1.208, 'grad_norm': 5.643867015838623, 'learning_rate': 7.89335648089903e-05, 'epoch': 3.73}\n", + "{'loss': 1.3448, 'grad_norm': 5.375209808349609, 'learning_rate': 7.86788218175523e-05, 'epoch': 3.75}\n", + "{'loss': 1.316, 'grad_norm': 5.470929145812988, 'learning_rate': 7.842296453524463e-05, 'epoch': 3.77}\n", + "{'loss': 1.2369, 'grad_norm': 4.993719577789307, 'learning_rate': 7.81660029031811e-05, 'epoch': 3.78}\n", + "{'loss': 1.3265, 'grad_norm': 5.081270217895508, 'learning_rate': 7.79079469053842e-05, 'epoch': 3.8}\n", + "{'loss': 1.3366, 'grad_norm': 5.608216285705566, 'learning_rate': 7.764880656839696e-05, 'epoch': 3.82}\n", + "{'loss': 1.2876, 'grad_norm': 5.217581272125244, 'learning_rate': 7.738859196089358e-05, 'epoch': 3.84}\n", + "{'loss': 1.1134, 'grad_norm': 5.468497276306152, 'learning_rate': 7.712731319328798e-05, 'epoch': 3.86}\n", + "{'loss': 1.2106, 'grad_norm': 5.239170074462891, 'learning_rate': 7.68649804173412e-05, 'epoch': 3.87}\n", + "{'loss': 1.3169, 'grad_norm': 4.908669471740723, 'learning_rate': 7.660160382576683e-05, 'epoch': 3.89}\n", + "{'loss': 1.3074, 'grad_norm': 6.217924118041992, 'learning_rate': 7.633719365183504e-05, 'epoch': 3.91}\n", + "{'loss': 1.2273, 'grad_norm': 5.6632513999938965, 'learning_rate': 7.60717601689749e-05, 'epoch': 3.93}\n", + "{'loss': 1.0764, 'grad_norm': 4.552252769470215, 'learning_rate': 7.580531369037533e-05, 'epoch': 3.94}\n", + "{'loss': 1.5051, 'grad_norm': 5.1463823318481445, 'learning_rate': 7.553786456858429e-05, 'epoch': 3.96}\n", + "{'loss': 1.3593, 'grad_norm': 4.828197956085205, 'learning_rate': 7.526942319510655e-05, 'epoch': 3.98}\n", + "{'loss': 1.1506, 'grad_norm': 3.7453665733337402, 'learning_rate': 7.500000000000001e-05, 'epoch': 4.0}\n", + " 40%|██████████████▊ | 2240/5600 [46:40<1:08:55, 1.23s/it][INFO|trainer.py:3788] 2024-06-29 22:45:40,494 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 22:45:40,495 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 22:45:40,495 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-2240\n", + "[INFO|configuration_utils.py:733] 2024-06-29 22:45:42,919 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 22:45:42,919 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 22:45:42,944 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-2240/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 22:45:42,944 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-2240/special_tokens_map.json\n", + "{'loss': 1.0956, 'grad_norm': 3.8211286067962646, 'learning_rate': 7.472960545147038e-05, 'epoch': 4.02}\n", + "{'loss': 0.9704, 'grad_norm': 4.844677448272705, 'learning_rate': 7.445825005546448e-05, 'epoch': 4.03}\n", + "{'loss': 1.0473, 'grad_norm': 5.256546497344971, 'learning_rate': 7.4185944355262e-05, 'epoch': 4.05}\n", + "{'loss': 0.9475, 'grad_norm': 5.321354389190674, 'learning_rate': 7.391269893106592e-05, 'epoch': 4.07}\n", + "{'loss': 0.963, 'grad_norm': 5.462662220001221, 'learning_rate': 7.363852439959135e-05, 'epoch': 4.09}\n", + "{'loss': 1.1535, 'grad_norm': 4.1576056480407715, 'learning_rate': 7.33634314136531e-05, 'epoch': 4.11}\n", + "{'loss': 0.868, 'grad_norm': 5.811347484588623, 'learning_rate': 7.308743066175172e-05, 'epoch': 4.12}\n", + "{'loss': 0.9822, 'grad_norm': 4.920297145843506, 'learning_rate': 7.281053286765815e-05, 'epoch': 4.14}\n", + "{'loss': 1.0763, 'grad_norm': 4.794600963592529, 'learning_rate': 7.253274878999727e-05, 'epoch': 4.16}\n", + "{'loss': 0.9458, 'grad_norm': 5.9534010887146, 'learning_rate': 7.225408922182961e-05, 'epoch': 4.18}\n", + "{'loss': 0.9607, 'grad_norm': 5.4286298751831055, 'learning_rate': 7.197456499023225e-05, 'epoch': 4.19}\n", + "{'loss': 1.0148, 'grad_norm': 5.160229206085205, 'learning_rate': 7.169418695587791e-05, 'epoch': 4.21}\n", + "{'loss': 0.9381, 'grad_norm': 5.876075267791748, 'learning_rate': 7.141296601261314e-05, 'epoch': 4.23}\n", + "{'loss': 1.0924, 'grad_norm': 4.944575786590576, 'learning_rate': 7.113091308703498e-05, 'epoch': 4.25}\n", + "{'loss': 0.9265, 'grad_norm': 5.3727545738220215, 'learning_rate': 7.084803913806641e-05, 'epoch': 4.27}\n", + "{'loss': 0.9944, 'grad_norm': 7.851329803466797, 'learning_rate': 7.056435515653059e-05, 'epoch': 4.28}\n", + "{'loss': 1.0168, 'grad_norm': 4.633089542388916, 'learning_rate': 7.027987216472377e-05, 'epoch': 4.3}\n", + "{'loss': 0.8804, 'grad_norm': 5.784977912902832, 'learning_rate': 6.999460121598704e-05, 'epoch': 4.32}\n", + "{'loss': 1.0768, 'grad_norm': 8.059889793395996, 'learning_rate': 6.970855339427698e-05, 'epoch': 4.34}\n", + "{'loss': 0.9443, 'grad_norm': 6.03590726852417, 'learning_rate': 6.942173981373474e-05, 'epoch': 4.36}\n", + "{'loss': 0.9489, 'grad_norm': 12.74445915222168, 'learning_rate': 6.91341716182545e-05, 'epoch': 4.37}\n", + "{'loss': 1.049, 'grad_norm': 5.12004280090332, 'learning_rate': 6.884585998105026e-05, 'epoch': 4.39}\n", + "{'loss': 1.0488, 'grad_norm': 10.477398872375488, 'learning_rate': 6.855681610422189e-05, 'epoch': 4.41}\n", + "{'loss': 1.1279, 'grad_norm': 5.085097789764404, 'learning_rate': 6.826705121831976e-05, 'epoch': 4.43}\n", + "{'loss': 1.088, 'grad_norm': 5.412323474884033, 'learning_rate': 6.797657658190839e-05, 'epoch': 4.44}\n", + "{'loss': 0.9124, 'grad_norm': 11.975234985351562, 'learning_rate': 6.768540348112907e-05, 'epoch': 4.46}\n", + "{'loss': 1.1052, 'grad_norm': 4.589102745056152, 'learning_rate': 6.739354322926136e-05, 'epoch': 4.48}\n", + "{'loss': 1.0427, 'grad_norm': 5.322690963745117, 'learning_rate': 6.710100716628344e-05, 'epoch': 4.5}\n", + "{'loss': 1.0325, 'grad_norm': 5.0710577964782715, 'learning_rate': 6.680780665843155e-05, 'epoch': 4.52}\n", + "{'loss': 1.059, 'grad_norm': 5.193735122680664, 'learning_rate': 6.651395309775837e-05, 'epoch': 4.53}\n", + "{'loss': 1.0252, 'grad_norm': 6.584225177764893, 'learning_rate': 6.621945790169036e-05, 'epoch': 4.55}\n", + "{'loss': 0.9911, 'grad_norm': 6.5015435218811035, 'learning_rate': 6.592433251258423e-05, 'epoch': 4.57}\n", + "{'loss': 1.0211, 'grad_norm': 5.898025035858154, 'learning_rate': 6.562858839728223e-05, 'epoch': 4.59}\n", + "{'loss': 0.9131, 'grad_norm': 5.380829811096191, 'learning_rate': 6.533223704666672e-05, 'epoch': 4.61}\n", + "{'loss': 1.0094, 'grad_norm': 5.253726959228516, 'learning_rate': 6.503528997521366e-05, 'epoch': 4.62}\n", + "{'loss': 1.0349, 'grad_norm': 4.567104339599609, 'learning_rate': 6.473775872054521e-05, 'epoch': 4.64}\n", + "{'loss': 1.0492, 'grad_norm': 5.842156410217285, 'learning_rate': 6.44396548429815e-05, 'epoch': 4.66}\n", + "{'loss': 1.0739, 'grad_norm': 5.842441082000732, 'learning_rate': 6.414098992509138e-05, 'epoch': 4.68}\n", + "{'loss': 0.9745, 'grad_norm': 5.929434299468994, 'learning_rate': 6.384177557124247e-05, 'epoch': 4.69}\n", + "{'loss': 0.9974, 'grad_norm': 6.804376125335693, 'learning_rate': 6.354202340715026e-05, 'epoch': 4.71}\n", + "{'loss': 0.9628, 'grad_norm': 4.4478020668029785, 'learning_rate': 6.324174507942637e-05, 'epoch': 4.73}\n", + "{'loss': 0.9861, 'grad_norm': 4.888654708862305, 'learning_rate': 6.294095225512603e-05, 'epoch': 4.75}\n", + "{'loss': 1.0063, 'grad_norm': 5.920362949371338, 'learning_rate': 6.263965662129487e-05, 'epoch': 4.77}\n", + "{'loss': 1.0548, 'grad_norm': 5.524910926818848, 'learning_rate': 6.233786988451468e-05, 'epoch': 4.78}\n", + "{'loss': 1.0929, 'grad_norm': 4.255885601043701, 'learning_rate': 6.203560377044866e-05, 'epoch': 4.8}\n", + "{'loss': 0.9848, 'grad_norm': 6.053053379058838, 'learning_rate': 6.173287002338577e-05, 'epoch': 4.82}\n", + "{'loss': 0.9432, 'grad_norm': 5.0641655921936035, 'learning_rate': 6.142968040578449e-05, 'epoch': 4.84}\n", + "{'loss': 1.0374, 'grad_norm': 7.123205661773682, 'learning_rate': 6.112604669781572e-05, 'epoch': 4.85}\n", + "{'loss': 1.0961, 'grad_norm': 5.436131954193115, 'learning_rate': 6.0821980696905146e-05, 'epoch': 4.87}\n", + "{'loss': 1.0741, 'grad_norm': 5.909348964691162, 'learning_rate': 6.0517494217274794e-05, 'epoch': 4.89}\n", + "{'loss': 1.1491, 'grad_norm': 5.223842144012451, 'learning_rate': 6.021259908948402e-05, 'epoch': 4.91}\n", + "{'loss': 1.0061, 'grad_norm': 5.396011829376221, 'learning_rate': 5.9907307159969884e-05, 'epoch': 4.93}\n", + "{'loss': 1.1121, 'grad_norm': 5.92130184173584, 'learning_rate': 5.960163029058682e-05, 'epoch': 4.94}\n", + "{'loss': 1.1207, 'grad_norm': 8.12635326385498, 'learning_rate': 5.9295580358145744e-05, 'epoch': 4.96}\n", + "{'loss': 1.141, 'grad_norm': 6.187139511108398, 'learning_rate': 5.898916925395264e-05, 'epoch': 4.98}\n", + "{'loss': 1.0886, 'grad_norm': 5.036999702453613, 'learning_rate': 5.868240888334653e-05, 'epoch': 5.0}\n", + " 50%|██████████████████▌ | 2800/5600 [59:01<1:00:22, 1.29s/it][INFO|trainer.py:3788] 2024-06-29 22:58:01,067 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 22:58:01,067 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 22:58:01,067 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-2800\n", + "[INFO|configuration_utils.py:733] 2024-06-29 22:58:03,475 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 22:58:03,475 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 22:58:03,501 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-2800/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 22:58:03,501 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-2800/special_tokens_map.json\n", + "{'loss': 0.7949, 'grad_norm': 5.331892490386963, 'learning_rate': 5.837531116523682e-05, 'epoch': 5.02}\n", + "{'loss': 0.7411, 'grad_norm': 6.530398368835449, 'learning_rate': 5.806788803164034e-05, 'epoch': 5.03}\n", + "{'loss': 0.7875, 'grad_norm': 5.302547454833984, 'learning_rate': 5.7760151427217576e-05, 'epoch': 5.05}\n", + "{'loss': 0.7917, 'grad_norm': 5.69765567779541, 'learning_rate': 5.745211330880872e-05, 'epoch': 5.07}\n", + "{'loss': 0.8076, 'grad_norm': 4.74371862411499, 'learning_rate': 5.714378564496901e-05, 'epoch': 5.09}\n", + "{'loss': 0.9093, 'grad_norm': 5.205414772033691, 'learning_rate': 5.683518041550368e-05, 'epoch': 5.1}\n", + "{'loss': 0.6875, 'grad_norm': 5.478267192840576, 'learning_rate': 5.6526309611002594e-05, 'epoch': 5.12}\n", + "{'loss': 0.8528, 'grad_norm': 6.004168510437012, 'learning_rate': 5.621718523237427e-05, 'epoch': 5.14}\n", + "{'loss': 0.7398, 'grad_norm': 6.278602600097656, 'learning_rate': 5.590781929037965e-05, 'epoch': 5.16}\n", + "{'loss': 0.6957, 'grad_norm': 4.800085544586182, 'learning_rate': 5.559822380516539e-05, 'epoch': 5.18}\n", + "{'loss': 0.8193, 'grad_norm': 5.28222131729126, 'learning_rate': 5.5288410805796895e-05, 'epoch': 5.19}\n", + "{'loss': 0.825, 'grad_norm': 5.969717502593994, 'learning_rate': 5.497839232979084e-05, 'epoch': 5.21}\n", + "{'loss': 0.8286, 'grad_norm': 7.2066216468811035, 'learning_rate': 5.466818042264753e-05, 'epoch': 5.23}\n", + "{'loss': 0.8905, 'grad_norm': 5.272522449493408, 'learning_rate': 5.435778713738292e-05, 'epoch': 5.25}\n", + "{'loss': 0.7673, 'grad_norm': 4.872743606567383, 'learning_rate': 5.404722453406017e-05, 'epoch': 5.27}\n", + "{'loss': 0.7346, 'grad_norm': 7.293342113494873, 'learning_rate': 5.373650467932122e-05, 'epoch': 5.28}\n", + "{'loss': 0.737, 'grad_norm': 5.229869365692139, 'learning_rate': 5.3425639645917834e-05, 'epoch': 5.3}\n", + "{'loss': 0.7656, 'grad_norm': 4.550146579742432, 'learning_rate': 5.311464151224261e-05, 'epoch': 5.32}\n", + "{'loss': 0.7256, 'grad_norm': 4.5456223487854, 'learning_rate': 5.2803522361859594e-05, 'epoch': 5.34}\n", + "{'loss': 0.7886, 'grad_norm': 5.301971912384033, 'learning_rate': 5.249229428303486e-05, 'epoch': 5.35}\n", + "{'loss': 0.7801, 'grad_norm': 7.148138523101807, 'learning_rate': 5.218096936826681e-05, 'epoch': 5.37}\n", + "{'loss': 0.8076, 'grad_norm': 6.13567590713501, 'learning_rate': 5.18695597138163e-05, 'epoch': 5.39}\n", + "{'loss': 0.6323, 'grad_norm': 5.02504825592041, 'learning_rate': 5.155807741923666e-05, 'epoch': 5.41}\n", + "{'loss': 0.707, 'grad_norm': 6.629558563232422, 'learning_rate': 5.124653458690365e-05, 'epoch': 5.43}\n", + "{'loss': 0.7033, 'grad_norm': 6.333116054534912, 'learning_rate': 5.0934943321545115e-05, 'epoch': 5.44}\n", + "{'loss': 0.797, 'grad_norm': 10.160740852355957, 'learning_rate': 5.062331572977076e-05, 'epoch': 5.46}\n", + "{'loss': 0.9649, 'grad_norm': 5.644074440002441, 'learning_rate': 5.031166391960168e-05, 'epoch': 5.48}\n", + "{'loss': 0.7452, 'grad_norm': 5.57203483581543, 'learning_rate': 5e-05, 'epoch': 5.5}\n", + "{'loss': 0.7909, 'grad_norm': 6.231649875640869, 'learning_rate': 4.968833608039832e-05, 'epoch': 5.52}\n", + "{'loss': 0.8125, 'grad_norm': 6.369471073150635, 'learning_rate': 4.9376684270229254e-05, 'epoch': 5.53}\n", + "{'loss': 0.7572, 'grad_norm': 14.980217933654785, 'learning_rate': 4.9065056678454904e-05, 'epoch': 5.55}\n", + "{'loss': 0.9001, 'grad_norm': 6.943779468536377, 'learning_rate': 4.875346541309637e-05, 'epoch': 5.57}\n", + "{'loss': 0.8092, 'grad_norm': 6.565555572509766, 'learning_rate': 4.844192258076336e-05, 'epoch': 5.59}\n", + "{'loss': 0.9073, 'grad_norm': 8.527596473693848, 'learning_rate': 4.813044028618373e-05, 'epoch': 5.6}\n", + "{'loss': 0.8484, 'grad_norm': 5.995067119598389, 'learning_rate': 4.781903063173321e-05, 'epoch': 5.62}\n", + "{'loss': 0.7387, 'grad_norm': 16.719541549682617, 'learning_rate': 4.750770571696514e-05, 'epoch': 5.64}\n", + "{'loss': 0.8061, 'grad_norm': 5.842343807220459, 'learning_rate': 4.7196477638140404e-05, 'epoch': 5.66}\n", + "{'loss': 0.7958, 'grad_norm': 7.201180458068848, 'learning_rate': 4.68853584877574e-05, 'epoch': 5.68}\n", + "{'loss': 0.8338, 'grad_norm': 6.153838634490967, 'learning_rate': 4.657436035408217e-05, 'epoch': 5.69}\n", + "{'loss': 0.7411, 'grad_norm': 5.899301528930664, 'learning_rate': 4.626349532067879e-05, 'epoch': 5.71}\n", + "{'loss': 0.8199, 'grad_norm': 5.865950107574463, 'learning_rate': 4.595277546593984e-05, 'epoch': 5.73}\n", + "{'loss': 0.7367, 'grad_norm': 4.905264377593994, 'learning_rate': 4.564221286261709e-05, 'epoch': 5.75}\n", + "{'loss': 0.9049, 'grad_norm': 6.099426746368408, 'learning_rate': 4.5331819577352474e-05, 'epoch': 5.77}\n", + "{'loss': 0.7343, 'grad_norm': 7.31098747253418, 'learning_rate': 4.502160767020918e-05, 'epoch': 5.78}\n", + "{'loss': 0.7381, 'grad_norm': 5.501935958862305, 'learning_rate': 4.471158919420312e-05, 'epoch': 5.8}\n", + "{'loss': 0.7133, 'grad_norm': 7.434685707092285, 'learning_rate': 4.4401776194834613e-05, 'epoch': 5.82}\n", + "{'loss': 0.7352, 'grad_norm': 9.345376968383789, 'learning_rate': 4.409218070962036e-05, 'epoch': 5.84}\n", + "{'loss': 0.8973, 'grad_norm': 6.876387119293213, 'learning_rate': 4.378281476762576e-05, 'epoch': 5.85}\n", + "{'loss': 0.6906, 'grad_norm': 14.176045417785645, 'learning_rate': 4.347369038899744e-05, 'epoch': 5.87}\n", + "{'loss': 0.9098, 'grad_norm': 4.8011040687561035, 'learning_rate': 4.316481958449634e-05, 'epoch': 5.89}\n", + "{'loss': 0.7331, 'grad_norm': 5.1314697265625, 'learning_rate': 4.285621435503101e-05, 'epoch': 5.91}\n", + "{'loss': 0.8579, 'grad_norm': 7.106369495391846, 'learning_rate': 4.254788669119127e-05, 'epoch': 5.93}\n", + "{'loss': 0.748, 'grad_norm': 4.865246295928955, 'learning_rate': 4.223984857278242e-05, 'epoch': 5.94}\n", + "{'loss': 0.8392, 'grad_norm': 5.906892776489258, 'learning_rate': 4.1932111968359664e-05, 'epoch': 5.96}\n", + "{'loss': 0.8015, 'grad_norm': 5.705036163330078, 'learning_rate': 4.162468883476319e-05, 'epoch': 5.98}\n", + "{'loss': 0.8302, 'grad_norm': 5.603642463684082, 'learning_rate': 4.131759111665349e-05, 'epoch': 6.0}\n", + " 60%|██████████████████████▏ | 3360/5600 [1:11:19<47:06, 1.26s/it][INFO|trainer.py:3788] 2024-06-29 23:10:19,379 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 23:10:19,379 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 23:10:19,379 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-3360\n", + "[INFO|configuration_utils.py:733] 2024-06-29 23:10:21,875 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 23:10:21,876 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 23:10:21,913 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-3360/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 23:10:21,914 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-3360/special_tokens_map.json\n", + "{'loss': 0.6066, 'grad_norm': 5.037189960479736, 'learning_rate': 4.101083074604737e-05, 'epoch': 6.02}\n", + "{'loss': 0.5565, 'grad_norm': 15.377517700195312, 'learning_rate': 4.0704419641854274e-05, 'epoch': 6.03}\n", + "{'loss': 0.5423, 'grad_norm': 5.809545516967773, 'learning_rate': 4.03983697094132e-05, 'epoch': 6.05}\n", + "{'loss': 0.6718, 'grad_norm': 6.553403377532959, 'learning_rate': 4.0092692840030134e-05, 'epoch': 6.07}\n", + "{'loss': 0.6391, 'grad_norm': 5.6180901527404785, 'learning_rate': 3.978740091051599e-05, 'epoch': 6.09}\n", + "{'loss': 0.6823, 'grad_norm': 5.028639793395996, 'learning_rate': 3.9482505782725224e-05, 'epoch': 6.1}\n", + "{'loss': 0.5271, 'grad_norm': 5.877006530761719, 'learning_rate': 3.917801930309486e-05, 'epoch': 6.12}\n", + "{'loss': 0.6207, 'grad_norm': 5.783533096313477, 'learning_rate': 3.887395330218429e-05, 'epoch': 6.14}\n", + "{'loss': 0.6231, 'grad_norm': 7.259427547454834, 'learning_rate': 3.857031959421553e-05, 'epoch': 6.16}\n", + "{'loss': 0.6113, 'grad_norm': 5.142063140869141, 'learning_rate': 3.8267129976614254e-05, 'epoch': 6.18}\n", + "{'loss': 0.6245, 'grad_norm': 6.205945014953613, 'learning_rate': 3.7964396229551364e-05, 'epoch': 6.19}\n", + "{'loss': 0.6885, 'grad_norm': 5.446501731872559, 'learning_rate': 3.7662130115485314e-05, 'epoch': 6.21}\n", + "{'loss': 0.6127, 'grad_norm': 5.06493616104126, 'learning_rate': 3.7360343378705124e-05, 'epoch': 6.23}\n", + "{'loss': 0.6773, 'grad_norm': 6.783751964569092, 'learning_rate': 3.705904774487396e-05, 'epoch': 6.25}\n", + "{'loss': 0.5281, 'grad_norm': 9.579876899719238, 'learning_rate': 3.675825492057364e-05, 'epoch': 6.27}\n", + "{'loss': 0.642, 'grad_norm': 5.472534656524658, 'learning_rate': 3.6457976592849754e-05, 'epoch': 6.28}\n", + "{'loss': 0.5956, 'grad_norm': 4.942157745361328, 'learning_rate': 3.6158224428757535e-05, 'epoch': 6.3}\n", + "{'loss': 0.6226, 'grad_norm': 4.671095848083496, 'learning_rate': 3.585901007490863e-05, 'epoch': 6.32}\n", + "{'loss': 0.6398, 'grad_norm': 7.523804664611816, 'learning_rate': 3.556034515701852e-05, 'epoch': 6.34}\n", + "{'loss': 0.5765, 'grad_norm': 5.049912929534912, 'learning_rate': 3.5262241279454785e-05, 'epoch': 6.35}\n", + "{'loss': 0.5686, 'grad_norm': 4.921656608581543, 'learning_rate': 3.4964710024786354e-05, 'epoch': 6.37}\n", + "{'loss': 0.5776, 'grad_norm': 4.165037155151367, 'learning_rate': 3.4667762953333295e-05, 'epoch': 6.39}\n", + "{'loss': 0.5287, 'grad_norm': 14.588775634765625, 'learning_rate': 3.4371411602717784e-05, 'epoch': 6.41}\n", + "{'loss': 0.6236, 'grad_norm': 4.320646286010742, 'learning_rate': 3.4075667487415785e-05, 'epoch': 6.43}\n", + "{'loss': 0.6174, 'grad_norm': 4.169257164001465, 'learning_rate': 3.3780542098309654e-05, 'epoch': 6.44}\n", + "{'loss': 0.6235, 'grad_norm': 3.5882270336151123, 'learning_rate': 3.3486046902241664e-05, 'epoch': 6.46}\n", + "{'loss': 0.6929, 'grad_norm': 5.139246940612793, 'learning_rate': 3.319219334156847e-05, 'epoch': 6.48}\n", + "{'loss': 0.6181, 'grad_norm': 6.403084754943848, 'learning_rate': 3.289899283371657e-05, 'epoch': 6.5}\n", + "{'loss': 0.702, 'grad_norm': 5.330471038818359, 'learning_rate': 3.2606456770738636e-05, 'epoch': 6.51}\n", + "{'loss': 0.558, 'grad_norm': 6.444238662719727, 'learning_rate': 3.231459651887093e-05, 'epoch': 6.53}\n", + "{'loss': 0.6338, 'grad_norm': 5.4946417808532715, 'learning_rate': 3.2023423418091626e-05, 'epoch': 6.55}\n", + "{'loss': 0.5606, 'grad_norm': 5.147060871124268, 'learning_rate': 3.173294878168025e-05, 'epoch': 6.57}\n", + "{'loss': 0.7059, 'grad_norm': 5.5029754638671875, 'learning_rate': 3.1443183895778105e-05, 'epoch': 6.59}\n", + "{'loss': 0.647, 'grad_norm': 5.451030731201172, 'learning_rate': 3.115414001894974e-05, 'epoch': 6.6}\n", + "{'loss': 0.6321, 'grad_norm': 5.880076885223389, 'learning_rate': 3.086582838174551e-05, 'epoch': 6.62}\n", + "{'loss': 0.6221, 'grad_norm': 12.090547561645508, 'learning_rate': 3.0578260186265265e-05, 'epoch': 6.64}\n", + "{'loss': 0.5515, 'grad_norm': 4.961390495300293, 'learning_rate': 3.029144660572304e-05, 'epoch': 6.66}\n", + "{'loss': 0.6303, 'grad_norm': 8.35487174987793, 'learning_rate': 3.000539878401296e-05, 'epoch': 6.68}\n", + "{'loss': 0.6175, 'grad_norm': 5.784793376922607, 'learning_rate': 2.9720127835276256e-05, 'epoch': 6.69}\n", + "{'loss': 0.6365, 'grad_norm': 5.296642780303955, 'learning_rate': 2.9435644843469436e-05, 'epoch': 6.71}\n", + "{'loss': 0.6053, 'grad_norm': 5.430149078369141, 'learning_rate': 2.9151960861933614e-05, 'epoch': 6.73}\n", + "{'loss': 0.6135, 'grad_norm': 5.0150980949401855, 'learning_rate': 2.886908691296504e-05, 'epoch': 6.75}\n", + "{'loss': 0.6041, 'grad_norm': 5.136585235595703, 'learning_rate': 2.858703398738686e-05, 'epoch': 6.76}\n", + "{'loss': 0.5429, 'grad_norm': 4.231466293334961, 'learning_rate': 2.8305813044122097e-05, 'epoch': 6.78}\n", + "{'loss': 0.5956, 'grad_norm': 5.151216983795166, 'learning_rate': 2.8025435009767747e-05, 'epoch': 6.8}\n", + "{'loss': 0.5732, 'grad_norm': 3.7542734146118164, 'learning_rate': 2.774591077817038e-05, 'epoch': 6.82}\n", + "{'loss': 0.6358, 'grad_norm': 6.12777042388916, 'learning_rate': 2.746725121000273e-05, 'epoch': 6.84}\n", + "{'loss': 0.5031, 'grad_norm': 11.638378143310547, 'learning_rate': 2.718946713234185e-05, 'epoch': 6.85}\n", + "{'loss': 0.6171, 'grad_norm': 9.199576377868652, 'learning_rate': 2.6912569338248315e-05, 'epoch': 6.87}\n", + "{'loss': 0.6104, 'grad_norm': 10.14255428314209, 'learning_rate': 2.66365685863469e-05, 'epoch': 6.89}\n", + "{'loss': 0.7077, 'grad_norm': 9.090829849243164, 'learning_rate': 2.636147560040866e-05, 'epoch': 6.91}\n", + "{'loss': 0.5531, 'grad_norm': 9.668030738830566, 'learning_rate': 2.6087301068934106e-05, 'epoch': 6.93}\n", + "{'loss': 0.6159, 'grad_norm': 6.352726936340332, 'learning_rate': 2.581405564473801e-05, 'epoch': 6.94}\n", + "{'loss': 0.6046, 'grad_norm': 5.168361663818359, 'learning_rate': 2.5541749944535554e-05, 'epoch': 6.96}\n", + "{'loss': 0.7733, 'grad_norm': 7.233384132385254, 'learning_rate': 2.527039454852963e-05, 'epoch': 6.98}\n", + "{'loss': 0.6154, 'grad_norm': 9.114374160766602, 'learning_rate': 2.500000000000001e-05, 'epoch': 7.0}\n", + " 70%|█████████████████████████▉ | 3920/5600 [1:23:31<34:46, 1.24s/it][INFO|trainer.py:3788] 2024-06-29 23:22:31,824 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 23:22:31,824 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 23:22:31,824 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-3920\n", + "[INFO|configuration_utils.py:733] 2024-06-29 23:22:34,201 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 23:22:34,202 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 23:22:34,235 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-3920/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 23:22:34,235 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-3920/special_tokens_map.json\n", + "{'loss': 0.4505, 'grad_norm': 4.652220726013184, 'learning_rate': 2.473057680489348e-05, 'epoch': 7.01}\n", + "{'loss': 0.385, 'grad_norm': 2.926722526550293, 'learning_rate': 2.4462135431415733e-05, 'epoch': 7.03}\n", + "{'loss': 0.4096, 'grad_norm': 6.222466468811035, 'learning_rate': 2.4194686309624663e-05, 'epoch': 7.05}\n", + "{'loss': 0.553, 'grad_norm': 3.829651117324829, 'learning_rate': 2.39282398310251e-05, 'epoch': 7.07}\n", + "{'loss': 0.403, 'grad_norm': 5.209712982177734, 'learning_rate': 2.366280634816496e-05, 'epoch': 7.09}\n", + "{'loss': 0.5494, 'grad_norm': 4.316225051879883, 'learning_rate': 2.3398396174233178e-05, 'epoch': 7.1}\n", + "{'loss': 0.4251, 'grad_norm': 5.665122985839844, 'learning_rate': 2.3135019582658802e-05, 'epoch': 7.12}\n", + "{'loss': 0.4833, 'grad_norm': 5.162817478179932, 'learning_rate': 2.2872686806712035e-05, 'epoch': 7.14}\n", + "{'loss': 0.4832, 'grad_norm': 4.767073631286621, 'learning_rate': 2.261140803910644e-05, 'epoch': 7.16}\n", + "{'loss': 0.4627, 'grad_norm': 6.984405994415283, 'learning_rate': 2.235119343160303e-05, 'epoch': 7.18}\n", + "{'loss': 0.48, 'grad_norm': 5.248043060302734, 'learning_rate': 2.2092053094615813e-05, 'epoch': 7.19}\n", + "{'loss': 0.4851, 'grad_norm': 5.531778812408447, 'learning_rate': 2.1833997096818898e-05, 'epoch': 7.21}\n", + "{'loss': 0.4751, 'grad_norm': 5.573154926300049, 'learning_rate': 2.157703546475539e-05, 'epoch': 7.23}\n", + "{'loss': 0.5816, 'grad_norm': 4.959446430206299, 'learning_rate': 2.132117818244771e-05, 'epoch': 7.25}\n", + "{'loss': 0.4175, 'grad_norm': 4.046441078186035, 'learning_rate': 2.1066435191009715e-05, 'epoch': 7.26}\n", + "{'loss': 0.5647, 'grad_norm': 7.062335968017578, 'learning_rate': 2.0812816388260518e-05, 'epoch': 7.28}\n", + "{'loss': 0.479, 'grad_norm': 4.6393914222717285, 'learning_rate': 2.056033162833977e-05, 'epoch': 7.3}\n", + "{'loss': 0.471, 'grad_norm': 5.455317497253418, 'learning_rate': 2.0308990721324927e-05, 'epoch': 7.32}\n", + "{'loss': 0.5885, 'grad_norm': 4.32041597366333, 'learning_rate': 2.0058803432849987e-05, 'epoch': 7.34}\n", + "{'loss': 0.4522, 'grad_norm': 4.541329383850098, 'learning_rate': 1.980977948372612e-05, 'epoch': 7.35}\n", + "{'loss': 0.4865, 'grad_norm': 5.104362964630127, 'learning_rate': 1.9561928549563968e-05, 'epoch': 7.37}\n", + "{'loss': 0.554, 'grad_norm': 5.151457786560059, 'learning_rate': 1.931526026039764e-05, 'epoch': 7.39}\n", + "{'loss': 0.4381, 'grad_norm': 5.234814643859863, 'learning_rate': 1.906978420031059e-05, 'epoch': 7.41}\n", + "{'loss': 0.5294, 'grad_norm': 6.009786128997803, 'learning_rate': 1.8825509907063327e-05, 'epoch': 7.43}\n", + "{'loss': 0.4886, 'grad_norm': 6.153667449951172, 'learning_rate': 1.8582446871722636e-05, 'epoch': 7.44}\n", + "{'loss': 0.5583, 'grad_norm': 5.528926849365234, 'learning_rate': 1.8340604538293015e-05, 'epoch': 7.46}\n", + "{'loss': 0.5186, 'grad_norm': 6.47043514251709, 'learning_rate': 1.8099992303349577e-05, 'epoch': 7.48}\n", + "{'loss': 0.4369, 'grad_norm': 4.640471458435059, 'learning_rate': 1.7860619515673033e-05, 'epoch': 7.5}\n", + "{'loss': 0.485, 'grad_norm': 4.996728420257568, 'learning_rate': 1.7622495475886487e-05, 'epoch': 7.51}\n", + "{'loss': 0.5824, 'grad_norm': 7.510169982910156, 'learning_rate': 1.738562943609396e-05, 'epoch': 7.53}\n", + "{'loss': 0.5401, 'grad_norm': 5.8573503494262695, 'learning_rate': 1.7150030599520984e-05, 'epoch': 7.55}\n", + "{'loss': 0.4099, 'grad_norm': 4.604180335998535, 'learning_rate': 1.691570812015704e-05, 'epoch': 7.57}\n", + "{'loss': 0.4631, 'grad_norm': 9.454184532165527, 'learning_rate': 1.6682671102399805e-05, 'epoch': 7.59}\n", + "{'loss': 0.4046, 'grad_norm': 3.9995360374450684, 'learning_rate': 1.6450928600701504e-05, 'epoch': 7.6}\n", + "{'loss': 0.3599, 'grad_norm': 5.843255043029785, 'learning_rate': 1.622048961921699e-05, 'epoch': 7.62}\n", + "{'loss': 0.5126, 'grad_norm': 5.647862434387207, 'learning_rate': 1.599136311145402e-05, 'epoch': 7.64}\n", + "{'loss': 0.6103, 'grad_norm': 6.46891450881958, 'learning_rate': 1.5763557979925324e-05, 'epoch': 7.66}\n", + "{'loss': 0.5807, 'grad_norm': 6.223480224609375, 'learning_rate': 1.553708307580265e-05, 'epoch': 7.68}\n", + "{'loss': 0.5201, 'grad_norm': 4.753687381744385, 'learning_rate': 1.531194719857292e-05, 'epoch': 7.69}\n", + "{'loss': 0.3547, 'grad_norm': 5.846710681915283, 'learning_rate': 1.5088159095696363e-05, 'epoch': 7.71}\n", + "{'loss': 0.5295, 'grad_norm': 5.13261079788208, 'learning_rate': 1.4865727462266543e-05, 'epoch': 7.73}\n", + "{'loss': 0.5368, 'grad_norm': 4.849207401275635, 'learning_rate': 1.4644660940672627e-05, 'epoch': 7.75}\n", + "{'loss': 0.5151, 'grad_norm': 4.458810806274414, 'learning_rate': 1.4424968120263504e-05, 'epoch': 7.76}\n", + "{'loss': 0.4958, 'grad_norm': 7.0515360832214355, 'learning_rate': 1.4206657537014079e-05, 'epoch': 7.78}\n", + "{'loss': 0.5166, 'grad_norm': 6.9797258377075195, 'learning_rate': 1.398973767319368e-05, 'epoch': 7.8}\n", + "{'loss': 0.5007, 'grad_norm': 8.272122383117676, 'learning_rate': 1.3774216957036367e-05, 'epoch': 7.82}\n", + "{'loss': 0.4178, 'grad_norm': 5.713352203369141, 'learning_rate': 1.3560103762413584e-05, 'epoch': 7.84}\n", + "{'loss': 0.4001, 'grad_norm': 7.498878479003906, 'learning_rate': 1.3347406408508695e-05, 'epoch': 7.85}\n", + "{'loss': 0.5782, 'grad_norm': 6.81415319442749, 'learning_rate': 1.3136133159493802e-05, 'epoch': 7.87}\n", + "{'loss': 0.493, 'grad_norm': 5.0307936668396, 'learning_rate': 1.2926292224208664e-05, 'epoch': 7.89}\n", + "{'loss': 0.4523, 'grad_norm': 4.477788925170898, 'learning_rate': 1.2717891755841722e-05, 'epoch': 7.91}\n", + "{'loss': 0.496, 'grad_norm': 5.846407413482666, 'learning_rate': 1.2510939851613285e-05, 'epoch': 7.93}\n", + "{'loss': 0.5292, 'grad_norm': 7.384892463684082, 'learning_rate': 1.230544455246101e-05, 'epoch': 7.94}\n", + "{'loss': 0.425, 'grad_norm': 6.020524978637695, 'learning_rate': 1.2101413842727345e-05, 'epoch': 7.96}\n", + "{'loss': 0.5331, 'grad_norm': 5.7436699867248535, 'learning_rate': 1.1898855649849461e-05, 'epoch': 7.98}\n", + "{'loss': 0.3988, 'grad_norm': 4.166412353515625, 'learning_rate': 1.1697777844051105e-05, 'epoch': 8.0}\n", + " 80%|█████████████████████████████▌ | 4480/5600 [1:35:21<23:03, 1.24s/it][INFO|trainer.py:3788] 2024-06-29 23:34:21,043 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 23:34:21,043 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 23:34:21,043 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-4480\n", + "[INFO|configuration_utils.py:733] 2024-06-29 23:34:23,861 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 23:34:23,861 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 23:34:23,897 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-4480/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 23:34:23,898 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-4480/special_tokens_map.json\n", + "{'loss': 0.3943, 'grad_norm': 3.8637747764587402, 'learning_rate': 1.1498188238036861e-05, 'epoch': 8.01}\n", + "{'loss': 0.3449, 'grad_norm': 4.5357465744018555, 'learning_rate': 1.130009458668863e-05, 'epoch': 8.03}\n", + "{'loss': 0.4843, 'grad_norm': 4.340099334716797, 'learning_rate': 1.1103504586764263e-05, 'epoch': 8.05}\n", + "{'loss': 0.3624, 'grad_norm': 5.39348030090332, 'learning_rate': 1.090842587659851e-05, 'epoch': 8.07}\n", + "{'loss': 0.3457, 'grad_norm': 5.173300743103027, 'learning_rate': 1.0714866035806326e-05, 'epoch': 8.09}\n", + "{'loss': 0.4419, 'grad_norm': 3.9911515712738037, 'learning_rate': 1.0522832584988234e-05, 'epoch': 8.1}\n", + "{'loss': 0.344, 'grad_norm': 4.810797214508057, 'learning_rate': 1.0332332985438248e-05, 'epoch': 8.12}\n", + "{'loss': 0.4852, 'grad_norm': 4.6731462478637695, 'learning_rate': 1.0143374638853891e-05, 'epoch': 8.14}\n", + "{'loss': 0.3417, 'grad_norm': 4.519662857055664, 'learning_rate': 9.955964887048607e-06, 'epoch': 8.16}\n", + "{'loss': 0.3707, 'grad_norm': 6.008825302124023, 'learning_rate': 9.770111011666583e-06, 'epoch': 8.17}\n", + "{'loss': 0.4661, 'grad_norm': 4.404787540435791, 'learning_rate': 9.58582023389974e-06, 'epoch': 8.19}\n", + "{'loss': 0.4659, 'grad_norm': 3.959002733230591, 'learning_rate': 9.403099714207175e-06, 'epoch': 8.21}\n", + "{'loss': 0.4714, 'grad_norm': 5.200716972351074, 'learning_rate': 9.221956552036992e-06, 'epoch': 8.23}\n", + "{'loss': 0.4455, 'grad_norm': 4.942255973815918, 'learning_rate': 9.042397785550405e-06, 'epoch': 8.25}\n", + "{'loss': 0.4403, 'grad_norm': 5.782726764678955, 'learning_rate': 8.864430391348332e-06, 'epoch': 8.26}\n", + "{'loss': 0.3306, 'grad_norm': 3.7129645347595215, 'learning_rate': 8.688061284200266e-06, 'epoch': 8.28}\n", + "{'loss': 0.407, 'grad_norm': 8.037576675415039, 'learning_rate': 8.513297316775625e-06, 'epoch': 8.3}\n", + "{'loss': 0.3798, 'grad_norm': 5.382339000701904, 'learning_rate': 8.34014527937756e-06, 'epoch': 8.32}\n", + "{'loss': 0.4849, 'grad_norm': 4.840189456939697, 'learning_rate': 8.168611899679013e-06, 'epoch': 8.34}\n", + "{'loss': 0.417, 'grad_norm': 5.7303619384765625, 'learning_rate': 7.998703842461431e-06, 'epoch': 8.35}\n", + "{'loss': 0.4589, 'grad_norm': 6.085827350616455, 'learning_rate': 7.830427709355725e-06, 'epoch': 8.37}\n", + "{'loss': 0.4225, 'grad_norm': 4.722183704376221, 'learning_rate': 7.663790038585793e-06, 'epoch': 8.39}\n", + "{'loss': 0.3773, 'grad_norm': 5.256956100463867, 'learning_rate': 7.498797304714544e-06, 'epoch': 8.41}\n", + "{'loss': 0.4246, 'grad_norm': 5.674898624420166, 'learning_rate': 7.33545591839222e-06, 'epoch': 8.42}\n", + "{'loss': 0.4219, 'grad_norm': 4.489896774291992, 'learning_rate': 7.173772226107434e-06, 'epoch': 8.44}\n", + "{'loss': 0.4486, 'grad_norm': 5.115447521209717, 'learning_rate': 7.013752509940485e-06, 'epoch': 8.46}\n", + "{'loss': 0.4512, 'grad_norm': 4.548392295837402, 'learning_rate': 6.855402987319348e-06, 'epoch': 8.48}\n", + "{'loss': 0.3836, 'grad_norm': 6.2048258781433105, 'learning_rate': 6.698729810778065e-06, 'epoch': 8.5}\n", + "{'loss': 0.3778, 'grad_norm': 4.5989766120910645, 'learning_rate': 6.54373906771768e-06, 'epoch': 8.51}\n", + "{'loss': 0.4007, 'grad_norm': 5.147210121154785, 'learning_rate': 6.390436780169734e-06, 'epoch': 8.53}\n", + "{'loss': 0.4504, 'grad_norm': 4.499249458312988, 'learning_rate': 6.238828904562316e-06, 'epoch': 8.55}\n", + "{'loss': 0.4176, 'grad_norm': 4.788080215454102, 'learning_rate': 6.088921331488568e-06, 'epoch': 8.57}\n", + "{'loss': 0.2845, 'grad_norm': 3.5535483360290527, 'learning_rate': 5.94071988547788e-06, 'epoch': 8.59}\n", + "{'loss': 0.3807, 'grad_norm': 4.653518199920654, 'learning_rate': 5.794230324769517e-06, 'epoch': 8.6}\n", + "{'loss': 0.3348, 'grad_norm': 4.7170915603637695, 'learning_rate': 5.649458341088915e-06, 'epoch': 8.62}\n", + "{'loss': 0.3807, 'grad_norm': 17.9665584564209, 'learning_rate': 5.506409559426573e-06, 'epoch': 8.64}\n", + "{'loss': 0.4922, 'grad_norm': 4.38849401473999, 'learning_rate': 5.365089537819434e-06, 'epoch': 8.66}\n", + "{'loss': 0.4016, 'grad_norm': 4.990530967712402, 'learning_rate': 5.2255037671349535e-06, 'epoch': 8.67}\n", + "{'loss': 0.4209, 'grad_norm': 4.245598793029785, 'learning_rate': 5.087657670857798e-06, 'epoch': 8.69}\n", + "{'loss': 0.3529, 'grad_norm': 3.6876637935638428, 'learning_rate': 4.951556604879048e-06, 'epoch': 8.71}\n", + "{'loss': 0.4205, 'grad_norm': 6.267766952514648, 'learning_rate': 4.8172058572881765e-06, 'epoch': 8.73}\n", + "{'loss': 0.391, 'grad_norm': 4.628519535064697, 'learning_rate': 4.684610648167503e-06, 'epoch': 8.75}\n", + "{'loss': 0.4038, 'grad_norm': 5.335127353668213, 'learning_rate': 4.5537761293894535e-06, 'epoch': 8.76}\n", + "{'loss': 0.4519, 'grad_norm': 5.06191349029541, 'learning_rate': 4.424707384416344e-06, 'epoch': 8.78}\n", + "{'loss': 0.4043, 'grad_norm': 3.3718318939208984, 'learning_rate': 4.29740942810285e-06, 'epoch': 8.8}\n", + "{'loss': 0.4329, 'grad_norm': 5.270512104034424, 'learning_rate': 4.1718872065011904e-06, 'epoch': 8.82}\n", + "{'loss': 0.4345, 'grad_norm': 4.938543796539307, 'learning_rate': 4.048145596668967e-06, 'epoch': 8.84}\n", + "{'loss': 0.4661, 'grad_norm': 4.726830005645752, 'learning_rate': 3.9261894064796135e-06, 'epoch': 8.85}\n", + "{'loss': 0.4037, 'grad_norm': 4.747579574584961, 'learning_rate': 3.8060233744356633e-06, 'epoch': 8.87}\n", + "{'loss': 0.3594, 'grad_norm': 3.65122652053833, 'learning_rate': 3.687652169484568e-06, 'epoch': 8.89}\n", + "{'loss': 0.3756, 'grad_norm': 3.7553329467773438, 'learning_rate': 3.5710803908373224e-06, 'epoch': 8.91}\n", + "{'loss': 0.4363, 'grad_norm': 6.1218132972717285, 'learning_rate': 3.4563125677897932e-06, 'epoch': 8.92}\n", + "{'loss': 0.5039, 'grad_norm': 6.221901893615723, 'learning_rate': 3.343353159546675e-06, 'epoch': 8.94}\n", + "{'loss': 0.4145, 'grad_norm': 4.449114799499512, 'learning_rate': 3.2322065550483007e-06, 'epoch': 8.96}\n", + "{'loss': 0.3358, 'grad_norm': 3.244713306427002, 'learning_rate': 3.1228770728000455e-06, 'epoch': 8.98}\n", + "{'loss': 0.3726, 'grad_norm': 5.383361339569092, 'learning_rate': 3.0153689607045845e-06, 'epoch': 9.0}\n", + " 90%|█████████████████████████████████▎ | 5040/5600 [1:47:11<11:48, 1.26s/it][INFO|trainer.py:3788] 2024-06-29 23:46:11,764 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 23:46:11,764 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 23:46:11,764 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-5040\n", + "[INFO|configuration_utils.py:733] 2024-06-29 23:46:14,139 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 23:46:14,139 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 23:46:14,173 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-5040/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 23:46:14,173 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-5040/special_tokens_map.json\n", + "{'loss': 0.3451, 'grad_norm': 5.190840244293213, 'learning_rate': 2.9096863958968268e-06, 'epoch': 9.01}\n", + "{'loss': 0.33, 'grad_norm': 3.5857107639312744, 'learning_rate': 2.8058334845816213e-06, 'epoch': 9.03}\n", + "{'loss': 0.3675, 'grad_norm': 4.7077860832214355, 'learning_rate': 2.7038142618741992e-06, 'epoch': 9.05}\n", + "{'loss': 0.4356, 'grad_norm': 4.774041175842285, 'learning_rate': 2.603632691643415e-06, 'epoch': 9.07}\n", + "{'loss': 0.3459, 'grad_norm': 3.2734451293945312, 'learning_rate': 2.5052926663577e-06, 'epoch': 9.09}\n", + "{'loss': 0.3926, 'grad_norm': 5.444535732269287, 'learning_rate': 2.408798006933882e-06, 'epoch': 9.1}\n", + "{'loss': 0.295, 'grad_norm': 4.564394474029541, 'learning_rate': 2.314152462588659e-06, 'epoch': 9.12}\n", + "{'loss': 0.3274, 'grad_norm': 3.5276427268981934, 'learning_rate': 2.221359710692961e-06, 'epoch': 9.14}\n", + "{'loss': 0.3454, 'grad_norm': 4.8225603103637695, 'learning_rate': 2.1304233566290964e-06, 'epoch': 9.16}\n", + "{'loss': 0.2982, 'grad_norm': 3.1064751148223877, 'learning_rate': 2.041346933650612e-06, 'epoch': 9.17}\n", + "{'loss': 0.3529, 'grad_norm': 3.431065082550049, 'learning_rate': 1.9541339027450256e-06, 'epoch': 9.19}\n", + "{'loss': 0.4354, 'grad_norm': 4.004822254180908, 'learning_rate': 1.8687876524993987e-06, 'epoch': 9.21}\n", + "{'loss': 0.3608, 'grad_norm': 5.244897842407227, 'learning_rate': 1.785311498968617e-06, 'epoch': 9.23}\n", + "{'loss': 0.3693, 'grad_norm': 4.393815517425537, 'learning_rate': 1.70370868554659e-06, 'epoch': 9.25}\n", + "{'loss': 0.3802, 'grad_norm': 4.819892883300781, 'learning_rate': 1.6239823828401945e-06, 'epoch': 9.26}\n", + "{'loss': 0.3838, 'grad_norm': 3.781949996948242, 'learning_rate': 1.5461356885461075e-06, 'epoch': 9.28}\n", + "{'loss': 0.4715, 'grad_norm': 4.076176166534424, 'learning_rate': 1.4701716273304521e-06, 'epoch': 9.3}\n", + "{'loss': 0.3256, 'grad_norm': 4.226771354675293, 'learning_rate': 1.3960931507112752e-06, 'epoch': 9.32}\n", + "{'loss': 0.3638, 'grad_norm': 3.562203884124756, 'learning_rate': 1.3239031369438326e-06, 'epoch': 9.34}\n", + "{'loss': 0.3687, 'grad_norm': 4.55058479309082, 'learning_rate': 1.2536043909088191e-06, 'epoch': 9.35}\n", + "{'loss': 0.3869, 'grad_norm': 4.373401165008545, 'learning_rate': 1.1851996440033319e-06, 'epoch': 9.37}\n", + "{'loss': 0.3151, 'grad_norm': 4.085133075714111, 'learning_rate': 1.118691554034773e-06, 'epoch': 9.39}\n", + "{'loss': 0.3557, 'grad_norm': 4.491430282592773, 'learning_rate': 1.0540827051175818e-06, 'epoch': 9.41}\n", + "{'loss': 0.405, 'grad_norm': 4.82833194732666, 'learning_rate': 9.913756075728087e-07, 'epoch': 9.42}\n", + "{'loss': 0.2972, 'grad_norm': 2.666112184524536, 'learning_rate': 9.305726978306173e-07, 'epoch': 9.44}\n", + "{'loss': 0.3194, 'grad_norm': 4.029996871948242, 'learning_rate': 8.716763383355864e-07, 'epoch': 9.46}\n", + "{'loss': 0.3984, 'grad_norm': 3.864152193069458, 'learning_rate': 8.146888174549339e-07, 'epoch': 9.48}\n", + "{'loss': 0.3483, 'grad_norm': 4.201892375946045, 'learning_rate': 7.596123493895991e-07, 'epoch': 9.5}\n", + "{'loss': 0.4642, 'grad_norm': 4.560868740081787, 'learning_rate': 7.064490740882057e-07, 'epoch': 9.51}\n", + "{'loss': 0.379, 'grad_norm': 4.305575370788574, 'learning_rate': 6.552010571639456e-07, 'epoch': 9.53}\n", + "{'loss': 0.445, 'grad_norm': 5.4909772872924805, 'learning_rate': 6.058702898142643e-07, 'epoch': 9.55}\n", + "{'loss': 0.3116, 'grad_norm': 4.831486225128174, 'learning_rate': 5.584586887435739e-07, 'epoch': 9.57}\n", + "{'loss': 0.3896, 'grad_norm': 4.905820846557617, 'learning_rate': 5.129680960887007e-07, 'epoch': 9.59}\n", + "{'loss': 0.3798, 'grad_norm': 3.7179861068725586, 'learning_rate': 4.6940027934735954e-07, 'epoch': 9.6}\n", + "{'loss': 0.3401, 'grad_norm': 4.62000036239624, 'learning_rate': 4.277569313094809e-07, 'epoch': 9.62}\n", + "{'loss': 0.4521, 'grad_norm': 4.725619792938232, 'learning_rate': 3.8803966999139684e-07, 'epoch': 9.64}\n", + "{'loss': 0.4075, 'grad_norm': 3.523742914199829, 'learning_rate': 3.50250038573019e-07, 'epoch': 9.66}\n", + "{'loss': 0.3438, 'grad_norm': 3.7823429107666016, 'learning_rate': 3.143895053378698e-07, 'epoch': 9.67}\n", + "{'loss': 0.2996, 'grad_norm': 3.2718749046325684, 'learning_rate': 2.8045946361601183e-07, 'epoch': 9.69}\n", + "{'loss': 0.4503, 'grad_norm': 5.158358097076416, 'learning_rate': 2.4846123172992954e-07, 'epoch': 9.71}\n", + "{'loss': 0.3938, 'grad_norm': 3.8553905487060547, 'learning_rate': 2.1839605294330933e-07, 'epoch': 9.73}\n", + "{'loss': 0.4459, 'grad_norm': 4.788202285766602, 'learning_rate': 1.9026509541272275e-07, 'epoch': 9.75}\n", + "{'loss': 0.3762, 'grad_norm': 4.024471759796143, 'learning_rate': 1.640694521422459e-07, 'epoch': 9.76}\n", + "{'loss': 0.4065, 'grad_norm': 5.944757461547852, 'learning_rate': 1.3981014094099353e-07, 'epoch': 9.78}\n", + "{'loss': 0.3105, 'grad_norm': 3.0800580978393555, 'learning_rate': 1.1748810438355628e-07, 'epoch': 9.8}\n", + "{'loss': 0.4782, 'grad_norm': 3.273432731628418, 'learning_rate': 9.710420977340762e-08, 'epoch': 9.82}\n", + "{'loss': 0.3914, 'grad_norm': 4.411673069000244, 'learning_rate': 7.865924910916977e-08, 'epoch': 9.83}\n", + "{'loss': 0.3274, 'grad_norm': 4.555184364318848, 'learning_rate': 6.215393905388278e-08, 'epoch': 9.85}\n", + "{'loss': 0.289, 'grad_norm': 5.107693672180176, 'learning_rate': 4.7588920907110094e-08, 'epoch': 9.87}\n", + "{'loss': 0.3202, 'grad_norm': 4.9626617431640625, 'learning_rate': 3.496476058006959e-08, 'epoch': 9.89}\n", + "{'loss': 0.433, 'grad_norm': 5.598171234130859, 'learning_rate': 2.4281948573617874e-08, 'epoch': 9.91}\n", + "{'loss': 0.4018, 'grad_norm': 4.289453029632568, 'learning_rate': 1.5540899959187727e-08, 'epoch': 9.92}\n", + "{'loss': 0.3691, 'grad_norm': 4.765395641326904, 'learning_rate': 8.741954362678772e-09, 'epoch': 9.94}\n", + "{'loss': 0.3645, 'grad_norm': 5.474503993988037, 'learning_rate': 3.885375951256931e-09, 'epoch': 9.96}\n", + "{'loss': 0.4003, 'grad_norm': 3.922280788421631, 'learning_rate': 9.713534230904041e-10, 'epoch': 9.98}\n", + "{'loss': 0.382, 'grad_norm': 4.276446342468262, 'learning_rate': 0.0, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5600/5600 [1:59:00<00:00, 1.26s/it][INFO|trainer.py:3788] 2024-06-29 23:58:00,034 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 23:58:00,034 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 23:58:00,034 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-5600\n", + "[INFO|configuration_utils.py:733] 2024-06-29 23:58:02,446 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 23:58:02,446 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 23:58:02,476 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-5600/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 23:58:02,476 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-5600/special_tokens_map.json\n", + "[INFO|trainer.py:2383] 2024-06-29 23:58:02,637 >> \n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "{'train_runtime': 7142.6727, 'train_samples_per_second': 6.275, 'train_steps_per_second': 0.784, 'train_loss': 1.0784291120512144, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5600/5600 [1:59:02<00:00, 1.28s/it]\n", + "[INFO|trainer.py:3478] 2024-06-29 23:58:02,640 >> Saving model checkpoint to saves/qwen2-0.5b/lora/sft\n", + "[INFO|configuration_utils.py:733] 2024-06-29 23:58:03,159 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-29 23:58:03,160 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-29 23:58:03,220 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-29 23:58:03,220 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/special_tokens_map.json\n", + "***** train metrics *****\n", + " epoch = 9.9955\n", + " total_flos = 7657006GF\n", + " train_loss = 1.0784\n", + " train_runtime = 1:59:02.67\n", + " train_samples_per_second = 6.275\n", + " train_steps_per_second = 0.784\n", + "Figure saved at: saves/qwen2-0.5b/lora/sft/training_loss.png\n", + "Figure saved at: saves/qwen2-0.5b/lora/sft/training_eval_loss.png\n", + "[INFO|trainer.py:3788] 2024-06-29 23:58:03,541 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-29 23:58:03,541 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-29 23:58:03,541 >> Batch size = 1\n", + "100%|███████████████████████████████████████████| 46/46 [00:01<00:00, 25.51it/s]\n", + "***** eval metrics *****\n", + " epoch = 9.9955\n", + " eval_loss = 3.4661\n", + " eval_runtime = 0:00:01.85\n", + " eval_samples_per_second = 24.833\n", + " eval_steps_per_second = 24.833\n", + "[INFO|modelcard.py:449] 2024-06-29 23:58:05,395 >> Dropping the following result as it does not have all the necessary fields:\n", + "{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}\n", + "CPU times: user 1min 32s, sys: 30.2 s, total: 2min 2s\n", + "Wall time: 1h 59min 52s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./scripts/tune-lf.sh config/qwen2_0.5b_lora_sft.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/llm-finetuning/llama-factory\n", + "06/30/2024 06:14:31 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: torch.bfloat16\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 06:14:31,888 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 06:14:31,888 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 06:14:31,888 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 06:14:31,888 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 06:14:31,888 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 06:14:31,888 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 06:14:32,031 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 06:14:32 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 06:14:32 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "06/30/2024 06:14:32 - INFO - llamafactory.data.loader - Loading dataset alpaca_mac.json...\n", + "Converting format of dataset (num_proc=16): 100%|█| 4528/4528 [00:00<00:00, 1488\n", + "Running tokenizer on dataset (num_proc=16): 100%|█| 4528/4528 [00:01<00:00, 3433\n", + "input_ids:\n", + "[151644, 872, 198, 5501, 14683, 279, 2701, 8453, 1467, 1119, 6364, 323, 3410, 1172, 279, 24531, 2213, 11, 4302, 770, 624, 35987, 102895, 99164, 100324, 100717, 100095, 99509, 1773, 151645, 198, 151644, 77091, 198, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "inputs:\n", + "<|im_start|>user\n", + "Please translate the following Chinese text into English and provide only the translated content, nothing else.\n", + "全仗着狐仙搭救。<|im_end|>\n", + "<|im_start|>assistant\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "label_ids:\n", + "[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "labels:\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "[INFO|configuration_utils.py:733] 2024-06-30 06:14:35,044 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 06:14:35,045 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 06:14:35,702 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 06:14:37,609 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 06:14:37,613 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 06:16:33,749 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 06:16:33,749 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 06:16:34,027 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 06:16:34,027 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 06:16:34 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.\n", + "06/30/2024 06:16:34 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 06:16:34 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.\n", + "06/30/2024 06:16:34 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA\n", + "06/30/2024 06:16:34 - INFO - llamafactory.model.model_utils.misc - Found linear modules: k_proj,q_proj,v_proj,gate_proj,up_proj,o_proj,down_proj\n", + "06/30/2024 06:16:34 - INFO - llamafactory.model.loader - trainable params: 9,232,384 || all params: 1,552,946,688 || trainable%: 0.5945\n", + "[INFO|trainer.py:642] 2024-06-30 06:16:34,928 >> Using auto half precision backend\n", + "[INFO|trainer.py:2128] 2024-06-30 06:16:35,081 >> ***** Running training *****\n", + "[INFO|trainer.py:2129] 2024-06-30 06:16:35,081 >> Num examples = 4,482\n", + "[INFO|trainer.py:2130] 2024-06-30 06:16:35,081 >> Num Epochs = 10\n", + "[INFO|trainer.py:2131] 2024-06-30 06:16:35,081 >> Instantaneous batch size per device = 1\n", + "[INFO|trainer.py:2134] 2024-06-30 06:16:35,081 >> Total train batch size (w. parallel, distributed & accumulation) = 8\n", + "[INFO|trainer.py:2135] 2024-06-30 06:16:35,081 >> Gradient Accumulation steps = 8\n", + "[INFO|trainer.py:2136] 2024-06-30 06:16:35,081 >> Total optimization steps = 5,600\n", + "[INFO|trainer.py:2137] 2024-06-30 06:16:35,083 >> Number of trainable parameters = 9,232,384\n", + "{'loss': 2.1598, 'grad_norm': 1.7301031351089478, 'learning_rate': 1.7857142857142857e-06, 'epoch': 0.02}\n", + "{'loss': 2.2894, 'grad_norm': 1.9221487045288086, 'learning_rate': 3.5714285714285714e-06, 'epoch': 0.04}\n", + "{'loss': 2.152, 'grad_norm': 1.5344856977462769, 'learning_rate': 5.357142857142857e-06, 'epoch': 0.05}\n", + "{'loss': 2.1602, 'grad_norm': 3.0139236450195312, 'learning_rate': 7.142857142857143e-06, 'epoch': 0.07}\n", + "{'loss': 2.3519, 'grad_norm': 1.501677393913269, 'learning_rate': 8.92857142857143e-06, 'epoch': 0.09}\n", + "{'loss': 2.05, 'grad_norm': 1.680209994316101, 'learning_rate': 1.0714285714285714e-05, 'epoch': 0.11}\n", + "{'loss': 2.1875, 'grad_norm': 1.694694995880127, 'learning_rate': 1.25e-05, 'epoch': 0.12}\n", + "{'loss': 1.9546, 'grad_norm': 1.5895333290100098, 'learning_rate': 1.4285714285714285e-05, 'epoch': 0.14}\n", + "{'loss': 2.075, 'grad_norm': 2.115245819091797, 'learning_rate': 1.6071428571428572e-05, 'epoch': 0.16}\n", + "{'loss': 1.9713, 'grad_norm': 1.3625324964523315, 'learning_rate': 1.785714285714286e-05, 'epoch': 0.18}\n", + "{'loss': 2.0099, 'grad_norm': 1.7853630781173706, 'learning_rate': 1.9642857142857145e-05, 'epoch': 0.2}\n", + "{'loss': 1.9603, 'grad_norm': 1.3131749629974365, 'learning_rate': 2.1428571428571428e-05, 'epoch': 0.21}\n", + "{'loss': 1.9619, 'grad_norm': 1.6807270050048828, 'learning_rate': 2.3214285714285715e-05, 'epoch': 0.23}\n", + "{'loss': 1.9889, 'grad_norm': 2.136683464050293, 'learning_rate': 2.5e-05, 'epoch': 0.25}\n", + "{'loss': 1.8445, 'grad_norm': 1.5379092693328857, 'learning_rate': 2.6785714285714288e-05, 'epoch': 0.27}\n", + "{'loss': 1.8162, 'grad_norm': 1.4818131923675537, 'learning_rate': 2.857142857142857e-05, 'epoch': 0.29}\n", + "{'loss': 1.9681, 'grad_norm': 1.3765653371810913, 'learning_rate': 3.0357142857142857e-05, 'epoch': 0.3}\n", + "{'loss': 1.7704, 'grad_norm': 1.7519148588180542, 'learning_rate': 3.2142857142857144e-05, 'epoch': 0.32}\n", + "{'loss': 1.8997, 'grad_norm': 2.2547669410705566, 'learning_rate': 3.392857142857143e-05, 'epoch': 0.34}\n", + "{'loss': 2.0083, 'grad_norm': 1.9038093090057373, 'learning_rate': 3.571428571428572e-05, 'epoch': 0.36}\n", + "{'loss': 1.9641, 'grad_norm': 1.864136815071106, 'learning_rate': 3.7500000000000003e-05, 'epoch': 0.37}\n", + "{'loss': 1.8745, 'grad_norm': 2.456977605819702, 'learning_rate': 3.928571428571429e-05, 'epoch': 0.39}\n", + "{'loss': 1.8564, 'grad_norm': 2.0037779808044434, 'learning_rate': 4.107142857142857e-05, 'epoch': 0.41}\n", + "{'loss': 2.0248, 'grad_norm': 2.459550619125366, 'learning_rate': 4.2857142857142856e-05, 'epoch': 0.43}\n", + "{'loss': 1.9225, 'grad_norm': 2.4255712032318115, 'learning_rate': 4.464285714285715e-05, 'epoch': 0.45}\n", + "{'loss': 1.8559, 'grad_norm': 2.2272531986236572, 'learning_rate': 4.642857142857143e-05, 'epoch': 0.46}\n", + "{'loss': 1.916, 'grad_norm': 3.067957878112793, 'learning_rate': 4.8214285714285716e-05, 'epoch': 0.48}\n", + "{'loss': 1.9695, 'grad_norm': 2.689528226852417, 'learning_rate': 5e-05, 'epoch': 0.5}\n", + "{'loss': 1.7267, 'grad_norm': 1.640542984008789, 'learning_rate': 5.1785714285714296e-05, 'epoch': 0.52}\n", + "{'loss': 1.8751, 'grad_norm': 2.6767070293426514, 'learning_rate': 5.3571428571428575e-05, 'epoch': 0.54}\n", + "{'loss': 1.8821, 'grad_norm': 2.2540671825408936, 'learning_rate': 5.535714285714286e-05, 'epoch': 0.55}\n", + "{'loss': 1.7133, 'grad_norm': 3.7877705097198486, 'learning_rate': 5.714285714285714e-05, 'epoch': 0.57}\n", + "{'loss': 1.7552, 'grad_norm': 2.7244925498962402, 'learning_rate': 5.8928571428571435e-05, 'epoch': 0.59}\n", + "{'loss': 1.8089, 'grad_norm': 2.4050076007843018, 'learning_rate': 6.0714285714285715e-05, 'epoch': 0.61}\n", + "{'loss': 1.8102, 'grad_norm': 3.4505980014801025, 'learning_rate': 6.25e-05, 'epoch': 0.62}\n", + "{'loss': 1.7452, 'grad_norm': 1.8404840230941772, 'learning_rate': 6.428571428571429e-05, 'epoch': 0.64}\n", + "{'loss': 1.9171, 'grad_norm': 2.923614025115967, 'learning_rate': 6.607142857142857e-05, 'epoch': 0.66}\n", + "{'loss': 1.8893, 'grad_norm': 2.2417802810668945, 'learning_rate': 6.785714285714286e-05, 'epoch': 0.68}\n", + "{'loss': 1.6041, 'grad_norm': 1.8358319997787476, 'learning_rate': 6.964285714285715e-05, 'epoch': 0.7}\n", + "{'loss': 1.7782, 'grad_norm': 2.7531838417053223, 'learning_rate': 7.142857142857143e-05, 'epoch': 0.71}\n", + "{'loss': 1.8365, 'grad_norm': 2.2503859996795654, 'learning_rate': 7.321428571428571e-05, 'epoch': 0.73}\n", + "{'loss': 1.863, 'grad_norm': 1.8987295627593994, 'learning_rate': 7.500000000000001e-05, 'epoch': 0.75}\n", + "{'loss': 1.8407, 'grad_norm': 2.950441598892212, 'learning_rate': 7.67857142857143e-05, 'epoch': 0.77}\n", + "{'loss': 1.7695, 'grad_norm': 3.3668158054351807, 'learning_rate': 7.857142857142858e-05, 'epoch': 0.79}\n", + "{'loss': 1.6759, 'grad_norm': 1.843374252319336, 'learning_rate': 8.035714285714287e-05, 'epoch': 0.8}\n", + "{'loss': 1.7465, 'grad_norm': 2.3402576446533203, 'learning_rate': 8.214285714285714e-05, 'epoch': 0.82}\n", + "{'loss': 1.7852, 'grad_norm': 3.2396647930145264, 'learning_rate': 8.392857142857144e-05, 'epoch': 0.84}\n", + "{'loss': 1.7626, 'grad_norm': 2.432474136352539, 'learning_rate': 8.571428571428571e-05, 'epoch': 0.86}\n", + "{'loss': 1.8173, 'grad_norm': 1.9021589756011963, 'learning_rate': 8.75e-05, 'epoch': 0.87}\n", + "{'loss': 1.9716, 'grad_norm': 1.968782901763916, 'learning_rate': 8.92857142857143e-05, 'epoch': 0.89}\n", + "{'loss': 1.8814, 'grad_norm': 2.0488665103912354, 'learning_rate': 9.107142857142857e-05, 'epoch': 0.91}\n", + "{'loss': 1.7689, 'grad_norm': 2.5687661170959473, 'learning_rate': 9.285714285714286e-05, 'epoch': 0.93}\n", + "{'loss': 1.8, 'grad_norm': 3.141063690185547, 'learning_rate': 9.464285714285715e-05, 'epoch': 0.95}\n", + "{'loss': 1.8067, 'grad_norm': 2.3366873264312744, 'learning_rate': 9.642857142857143e-05, 'epoch': 0.96}\n", + "{'loss': 1.7689, 'grad_norm': 2.356125831604004, 'learning_rate': 9.821428571428572e-05, 'epoch': 0.98}\n", + "{'loss': 1.7444, 'grad_norm': 1.962470293045044, 'learning_rate': 0.0001, 'epoch': 1.0}\n", + " 10%|███▊ | 560/5600 [15:11<2:19:14, 1.66s/it][INFO|trainer.py:3788] 2024-06-30 06:31:46,942 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 06:31:46,942 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 06:31:46,942 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-560\n", + "[INFO|configuration_utils.py:733] 2024-06-30 06:31:50,591 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 06:31:50,592 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 06:31:50,659 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-560/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 06:31:50,660 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-560/special_tokens_map.json\n", + "{'loss': 1.6815, 'grad_norm': 2.0423543453216553, 'learning_rate': 9.999902864657691e-05, 'epoch': 1.02}\n", + "{'loss': 1.5142, 'grad_norm': 3.1950676441192627, 'learning_rate': 9.999611462404875e-05, 'epoch': 1.04}\n", + "{'loss': 1.6976, 'grad_norm': 2.1450624465942383, 'learning_rate': 9.999125804563732e-05, 'epoch': 1.05}\n", + "{'loss': 1.636, 'grad_norm': 2.6176905632019043, 'learning_rate': 9.998445910004082e-05, 'epoch': 1.07}\n", + "{'loss': 1.6114, 'grad_norm': 2.9323713779449463, 'learning_rate': 9.997571805142639e-05, 'epoch': 1.09}\n", + "{'loss': 1.6291, 'grad_norm': 2.9673070907592773, 'learning_rate': 9.996503523941994e-05, 'epoch': 1.11}\n", + "{'loss': 1.6302, 'grad_norm': 2.463287591934204, 'learning_rate': 9.99524110790929e-05, 'epoch': 1.12}\n", + "{'loss': 1.6341, 'grad_norm': 4.124421119689941, 'learning_rate': 9.993784606094612e-05, 'epoch': 1.14}\n", + "{'loss': 1.6547, 'grad_norm': 2.851663589477539, 'learning_rate': 9.992134075089084e-05, 'epoch': 1.16}\n", + "{'loss': 1.5286, 'grad_norm': 2.5066659450531006, 'learning_rate': 9.99028957902266e-05, 'epoch': 1.18}\n", + "{'loss': 1.4801, 'grad_norm': 2.078930139541626, 'learning_rate': 9.988251189561645e-05, 'epoch': 1.2}\n", + "{'loss': 1.6318, 'grad_norm': 3.086003065109253, 'learning_rate': 9.986018985905901e-05, 'epoch': 1.21}\n", + "{'loss': 1.5591, 'grad_norm': 3.057227849960327, 'learning_rate': 9.983593054785776e-05, 'epoch': 1.23}\n", + "{'loss': 1.6401, 'grad_norm': 3.679922342300415, 'learning_rate': 9.980973490458728e-05, 'epoch': 1.25}\n", + "{'loss': 1.6262, 'grad_norm': 3.8075058460235596, 'learning_rate': 9.978160394705668e-05, 'epoch': 1.27}\n", + "{'loss': 1.7599, 'grad_norm': 3.5445713996887207, 'learning_rate': 9.975153876827008e-05, 'epoch': 1.29}\n", + "{'loss': 1.6814, 'grad_norm': 2.6588189601898193, 'learning_rate': 9.971954053638399e-05, 'epoch': 1.3}\n", + "{'loss': 1.6972, 'grad_norm': 2.6084141731262207, 'learning_rate': 9.968561049466214e-05, 'epoch': 1.32}\n", + "{'loss': 1.6675, 'grad_norm': 3.312152147293091, 'learning_rate': 9.964974996142698e-05, 'epoch': 1.34}\n", + "{'loss': 1.4381, 'grad_norm': 3.4132375717163086, 'learning_rate': 9.961196033000861e-05, 'epoch': 1.36}\n", + "{'loss': 1.6732, 'grad_norm': 3.6682002544403076, 'learning_rate': 9.957224306869053e-05, 'epoch': 1.37}\n", + "{'loss': 1.5185, 'grad_norm': 4.421182155609131, 'learning_rate': 9.953059972065265e-05, 'epoch': 1.39}\n", + "{'loss': 1.3911, 'grad_norm': 2.5544440746307373, 'learning_rate': 9.948703190391131e-05, 'epoch': 1.41}\n", + "{'loss': 1.6939, 'grad_norm': 3.4235222339630127, 'learning_rate': 9.944154131125642e-05, 'epoch': 1.43}\n", + "{'loss': 1.5821, 'grad_norm': 3.2818450927734375, 'learning_rate': 9.939412971018574e-05, 'epoch': 1.45}\n", + "{'loss': 1.551, 'grad_norm': 3.252692461013794, 'learning_rate': 9.934479894283606e-05, 'epoch': 1.46}\n", + "{'loss': 1.7187, 'grad_norm': 2.9500677585601807, 'learning_rate': 9.92935509259118e-05, 'epoch': 1.48}\n", + "{'loss': 1.5578, 'grad_norm': 3.451415538787842, 'learning_rate': 9.924038765061042e-05, 'epoch': 1.5}\n", + "{'loss': 1.4891, 'grad_norm': 2.3982598781585693, 'learning_rate': 9.918531118254507e-05, 'epoch': 1.52}\n", + "{'loss': 1.6728, 'grad_norm': 3.524627685546875, 'learning_rate': 9.912832366166442e-05, 'epoch': 1.54}\n", + "{'loss': 1.6112, 'grad_norm': 3.316537857055664, 'learning_rate': 9.906942730216939e-05, 'epoch': 1.55}\n", + "{'loss': 1.547, 'grad_norm': 2.789212465286255, 'learning_rate': 9.900862439242719e-05, 'epoch': 1.57}\n", + "{'loss': 1.6212, 'grad_norm': 3.1522133350372314, 'learning_rate': 9.894591729488242e-05, 'epoch': 1.59}\n", + "{'loss': 1.7589, 'grad_norm': 2.6350767612457275, 'learning_rate': 9.888130844596524e-05, 'epoch': 1.61}\n", + "{'loss': 1.5101, 'grad_norm': 2.931504487991333, 'learning_rate': 9.881480035599667e-05, 'epoch': 1.62}\n", + "{'loss': 1.6024, 'grad_norm': 2.5779600143432617, 'learning_rate': 9.874639560909117e-05, 'epoch': 1.64}\n", + "{'loss': 1.5994, 'grad_norm': 3.0192410945892334, 'learning_rate': 9.867609686305617e-05, 'epoch': 1.66}\n", + "{'loss': 1.5899, 'grad_norm': 2.50893497467041, 'learning_rate': 9.860390684928873e-05, 'epoch': 1.68}\n", + "{'loss': 1.5526, 'grad_norm': 3.570330858230591, 'learning_rate': 9.852982837266955e-05, 'epoch': 1.7}\n", + "{'loss': 1.5617, 'grad_norm': 4.337871074676514, 'learning_rate': 9.84538643114539e-05, 'epoch': 1.71}\n", + "{'loss': 1.5299, 'grad_norm': 2.3411428928375244, 'learning_rate': 9.837601761715983e-05, 'epoch': 1.73}\n", + "{'loss': 1.6652, 'grad_norm': 2.955780029296875, 'learning_rate': 9.829629131445342e-05, 'epoch': 1.75}\n", + "{'loss': 1.651, 'grad_norm': 2.441587209701538, 'learning_rate': 9.82146885010314e-05, 'epoch': 1.77}\n", + "{'loss': 1.5477, 'grad_norm': 2.947199821472168, 'learning_rate': 9.81312123475006e-05, 'epoch': 1.78}\n", + "{'loss': 1.5604, 'grad_norm': 2.740534543991089, 'learning_rate': 9.804586609725499e-05, 'epoch': 1.8}\n", + "{'loss': 1.5216, 'grad_norm': 2.7406256198883057, 'learning_rate': 9.79586530663494e-05, 'epoch': 1.82}\n", + "{'loss': 1.4901, 'grad_norm': 2.576497793197632, 'learning_rate': 9.78695766433709e-05, 'epoch': 1.84}\n", + "{'loss': 1.6326, 'grad_norm': 2.4222359657287598, 'learning_rate': 9.777864028930705e-05, 'epoch': 1.86}\n", + "{'loss': 1.4982, 'grad_norm': 3.2682604789733887, 'learning_rate': 9.768584753741134e-05, 'epoch': 1.87}\n", + "{'loss': 1.5688, 'grad_norm': 2.756934642791748, 'learning_rate': 9.759120199306613e-05, 'epoch': 1.89}\n", + "{'loss': 1.6835, 'grad_norm': 3.1586759090423584, 'learning_rate': 9.74947073336423e-05, 'epoch': 1.91}\n", + "{'loss': 1.7065, 'grad_norm': 3.218165874481201, 'learning_rate': 9.73963673083566e-05, 'epoch': 1.93}\n", + "{'loss': 1.6155, 'grad_norm': 2.732252836227417, 'learning_rate': 9.72961857381258e-05, 'epoch': 1.95}\n", + "{'loss': 1.5021, 'grad_norm': 2.702173948287964, 'learning_rate': 9.719416651541839e-05, 'epoch': 1.96}\n", + "{'loss': 1.6002, 'grad_norm': 2.3407227993011475, 'learning_rate': 9.709031360410318e-05, 'epoch': 1.98}\n", + "{'loss': 1.5955, 'grad_norm': 3.0833232402801514, 'learning_rate': 9.698463103929542e-05, 'epoch': 2.0}\n", + " 20%|███████▍ | 1120/5600 [30:57<2:03:57, 1.66s/it][INFO|trainer.py:3788] 2024-06-30 06:47:32,631 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 06:47:32,631 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 06:47:32,631 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-1120\n", + "[INFO|configuration_utils.py:733] 2024-06-30 06:47:35,643 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 06:47:35,644 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 06:47:35,688 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1120/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 06:47:35,688 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1120/special_tokens_map.json\n", + "{'loss': 1.2986, 'grad_norm': 6.459508895874023, 'learning_rate': 9.687712292719997e-05, 'epoch': 2.02}\n", + "{'loss': 1.1686, 'grad_norm': 2.6047580242156982, 'learning_rate': 9.67677934449517e-05, 'epoch': 2.03}\n", + "{'loss': 1.2613, 'grad_norm': 4.400974273681641, 'learning_rate': 9.665664684045333e-05, 'epoch': 2.05}\n", + "{'loss': 1.1817, 'grad_norm': 3.368881940841675, 'learning_rate': 9.654368743221022e-05, 'epoch': 2.07}\n", + "{'loss': 1.1255, 'grad_norm': 2.8357393741607666, 'learning_rate': 9.642891960916268e-05, 'epoch': 2.09}\n", + "{'loss': 1.2003, 'grad_norm': 2.8627848625183105, 'learning_rate': 9.631234783051544e-05, 'epoch': 2.11}\n", + "{'loss': 1.3588, 'grad_norm': 4.006772041320801, 'learning_rate': 9.619397662556435e-05, 'epoch': 2.12}\n", + "{'loss': 1.1791, 'grad_norm': 3.8697452545166016, 'learning_rate': 9.607381059352038e-05, 'epoch': 2.14}\n", + "{'loss': 1.1847, 'grad_norm': 4.039665222167969, 'learning_rate': 9.595185440333103e-05, 'epoch': 2.16}\n", + "{'loss': 1.1875, 'grad_norm': 4.559266567230225, 'learning_rate': 9.582811279349882e-05, 'epoch': 2.18}\n", + "{'loss': 1.2245, 'grad_norm': 3.3498127460479736, 'learning_rate': 9.570259057189717e-05, 'epoch': 2.2}\n", + "{'loss': 1.2361, 'grad_norm': 4.742955684661865, 'learning_rate': 9.557529261558367e-05, 'epoch': 2.21}\n", + "{'loss': 1.2929, 'grad_norm': 5.568743705749512, 'learning_rate': 9.544622387061055e-05, 'epoch': 2.23}\n", + "{'loss': 1.2945, 'grad_norm': 5.399260997772217, 'learning_rate': 9.53153893518325e-05, 'epoch': 2.25}\n", + "{'loss': 1.1725, 'grad_norm': 3.5391037464141846, 'learning_rate': 9.518279414271183e-05, 'epoch': 2.27}\n", + "{'loss': 1.1956, 'grad_norm': 4.30355978012085, 'learning_rate': 9.504844339512095e-05, 'epoch': 2.28}\n", + "{'loss': 1.2378, 'grad_norm': 3.2837555408477783, 'learning_rate': 9.491234232914221e-05, 'epoch': 2.3}\n", + "{'loss': 1.2634, 'grad_norm': 4.105693340301514, 'learning_rate': 9.477449623286505e-05, 'epoch': 2.32}\n", + "{'loss': 1.3589, 'grad_norm': 3.694589614868164, 'learning_rate': 9.463491046218058e-05, 'epoch': 2.34}\n", + "{'loss': 1.3134, 'grad_norm': 3.689924716949463, 'learning_rate': 9.449359044057345e-05, 'epoch': 2.36}\n", + "{'loss': 1.2572, 'grad_norm': 4.2927374839782715, 'learning_rate': 9.435054165891109e-05, 'epoch': 2.37}\n", + "{'loss': 1.4522, 'grad_norm': 4.005749225616455, 'learning_rate': 9.420576967523049e-05, 'epoch': 2.39}\n", + "{'loss': 1.343, 'grad_norm': 4.006478309631348, 'learning_rate': 9.405928011452211e-05, 'epoch': 2.41}\n", + "{'loss': 1.181, 'grad_norm': 4.455829620361328, 'learning_rate': 9.391107866851143e-05, 'epoch': 2.43}\n", + "{'loss': 1.2442, 'grad_norm': 3.436230421066284, 'learning_rate': 9.376117109543769e-05, 'epoch': 2.45}\n", + "{'loss': 1.2157, 'grad_norm': 3.515488386154175, 'learning_rate': 9.360956321983028e-05, 'epoch': 2.46}\n", + "{'loss': 1.2723, 'grad_norm': 3.4698567390441895, 'learning_rate': 9.345626093228233e-05, 'epoch': 2.48}\n", + "{'loss': 1.3747, 'grad_norm': 4.542730808258057, 'learning_rate': 9.330127018922194e-05, 'epoch': 2.5}\n", + "{'loss': 1.2685, 'grad_norm': 3.6365323066711426, 'learning_rate': 9.314459701268065e-05, 'epoch': 2.52}\n", + "{'loss': 1.2574, 'grad_norm': 3.8041131496429443, 'learning_rate': 9.298624749005951e-05, 'epoch': 2.53}\n", + "{'loss': 1.3031, 'grad_norm': 3.81734037399292, 'learning_rate': 9.282622777389258e-05, 'epoch': 2.55}\n", + "{'loss': 1.168, 'grad_norm': 4.677352428436279, 'learning_rate': 9.266454408160779e-05, 'epoch': 2.57}\n", + "{'loss': 1.3771, 'grad_norm': 5.038273811340332, 'learning_rate': 9.250120269528546e-05, 'epoch': 2.59}\n", + "{'loss': 1.2421, 'grad_norm': 5.5514702796936035, 'learning_rate': 9.233620996141421e-05, 'epoch': 2.61}\n", + "{'loss': 1.2833, 'grad_norm': 3.1367263793945312, 'learning_rate': 9.21695722906443e-05, 'epoch': 2.62}\n", + "{'loss': 1.2831, 'grad_norm': 2.603522539138794, 'learning_rate': 9.200129615753859e-05, 'epoch': 2.64}\n", + "{'loss': 1.2421, 'grad_norm': 3.707820177078247, 'learning_rate': 9.183138810032099e-05, 'epoch': 2.66}\n", + "{'loss': 1.3674, 'grad_norm': 3.9344961643218994, 'learning_rate': 9.165985472062246e-05, 'epoch': 2.68}\n", + "{'loss': 1.1452, 'grad_norm': 4.652283668518066, 'learning_rate': 9.148670268322438e-05, 'epoch': 2.7}\n", + "{'loss': 1.1737, 'grad_norm': 4.732541084289551, 'learning_rate': 9.131193871579975e-05, 'epoch': 2.71}\n", + "{'loss': 1.4043, 'grad_norm': 3.7013778686523438, 'learning_rate': 9.113556960865167e-05, 'epoch': 2.73}\n", + "{'loss': 1.334, 'grad_norm': 3.8859188556671143, 'learning_rate': 9.09576022144496e-05, 'epoch': 2.75}\n", + "{'loss': 1.2964, 'grad_norm': 3.6818110942840576, 'learning_rate': 9.077804344796302e-05, 'epoch': 2.77}\n", + "{'loss': 1.3015, 'grad_norm': 3.5502216815948486, 'learning_rate': 9.059690028579284e-05, 'epoch': 2.78}\n", + "{'loss': 1.1433, 'grad_norm': 3.0337369441986084, 'learning_rate': 9.041417976610027e-05, 'epoch': 2.8}\n", + "{'loss': 1.2503, 'grad_norm': 3.4227890968322754, 'learning_rate': 9.022988898833342e-05, 'epoch': 2.82}\n", + "{'loss': 1.2781, 'grad_norm': 3.566080093383789, 'learning_rate': 9.004403511295141e-05, 'epoch': 2.84}\n", + "{'loss': 1.2557, 'grad_norm': 4.064306735992432, 'learning_rate': 8.985662536114613e-05, 'epoch': 2.86}\n", + "{'loss': 1.4121, 'grad_norm': 3.106153726577759, 'learning_rate': 8.966766701456177e-05, 'epoch': 2.87}\n", + "{'loss': 1.2789, 'grad_norm': 3.873041868209839, 'learning_rate': 8.947716741501177e-05, 'epoch': 2.89}\n", + "{'loss': 1.2759, 'grad_norm': 3.9415042400360107, 'learning_rate': 8.928513396419368e-05, 'epoch': 2.91}\n", + "{'loss': 1.2078, 'grad_norm': 3.456357002258301, 'learning_rate': 8.90915741234015e-05, 'epoch': 2.93}\n", + "{'loss': 1.3886, 'grad_norm': 3.5346779823303223, 'learning_rate': 8.889649541323574e-05, 'epoch': 2.95}\n", + "{'loss': 1.33, 'grad_norm': 3.6706087589263916, 'learning_rate': 8.869990541331138e-05, 'epoch': 2.96}\n", + "{'loss': 1.2564, 'grad_norm': 4.235021591186523, 'learning_rate': 8.850181176196315e-05, 'epoch': 2.98}\n", + "{'loss': 1.3518, 'grad_norm': 3.6379354000091553, 'learning_rate': 8.83022221559489e-05, 'epoch': 3.0}\n", + " 30%|███████████ | 1680/5600 [46:41<1:48:23, 1.66s/it][INFO|trainer.py:3788] 2024-06-30 07:03:16,574 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 07:03:16,574 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 07:03:16,574 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-1680\n", + "[INFO|configuration_utils.py:733] 2024-06-30 07:03:19,590 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 07:03:19,590 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 07:03:19,633 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1680/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 07:03:19,633 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1680/special_tokens_map.json\n", + "{'loss': 1.15, 'grad_norm': 3.3373215198516846, 'learning_rate': 8.810114435015054e-05, 'epoch': 3.02}\n", + "{'loss': 0.8154, 'grad_norm': 4.1678571701049805, 'learning_rate': 8.789858615727265e-05, 'epoch': 3.03}\n", + "{'loss': 1.0096, 'grad_norm': 7.9504194259643555, 'learning_rate': 8.7694555447539e-05, 'epoch': 3.05}\n", + "{'loss': 0.9017, 'grad_norm': 3.666703462600708, 'learning_rate': 8.748906014838672e-05, 'epoch': 3.07}\n", + "{'loss': 0.8374, 'grad_norm': 3.569261312484741, 'learning_rate': 8.728210824415827e-05, 'epoch': 3.09}\n", + "{'loss': 0.9234, 'grad_norm': 5.156108856201172, 'learning_rate': 8.707370777579133e-05, 'epoch': 3.11}\n", + "{'loss': 0.8894, 'grad_norm': 3.83931827545166, 'learning_rate': 8.68638668405062e-05, 'epoch': 3.12}\n", + "{'loss': 0.9901, 'grad_norm': 4.340090274810791, 'learning_rate': 8.665259359149132e-05, 'epoch': 3.14}\n", + "{'loss': 0.8987, 'grad_norm': 5.530636310577393, 'learning_rate': 8.643989623758643e-05, 'epoch': 3.16}\n", + "{'loss': 0.94, 'grad_norm': 4.701400279998779, 'learning_rate': 8.622578304296364e-05, 'epoch': 3.18}\n", + "{'loss': 0.9471, 'grad_norm': 5.912676811218262, 'learning_rate': 8.601026232680634e-05, 'epoch': 3.2}\n", + "{'loss': 0.8883, 'grad_norm': 5.244345188140869, 'learning_rate': 8.579334246298593e-05, 'epoch': 3.21}\n", + "{'loss': 1.1187, 'grad_norm': 3.5720531940460205, 'learning_rate': 8.557503187973651e-05, 'epoch': 3.23}\n", + "{'loss': 0.8993, 'grad_norm': 4.100275993347168, 'learning_rate': 8.535533905932738e-05, 'epoch': 3.25}\n", + "{'loss': 0.9287, 'grad_norm': 3.9435741901397705, 'learning_rate': 8.513427253773346e-05, 'epoch': 3.27}\n", + "{'loss': 0.8239, 'grad_norm': 4.083703994750977, 'learning_rate': 8.491184090430364e-05, 'epoch': 3.28}\n", + "{'loss': 1.0248, 'grad_norm': 4.739283084869385, 'learning_rate': 8.468805280142709e-05, 'epoch': 3.3}\n", + "{'loss': 1.0099, 'grad_norm': 4.6722493171691895, 'learning_rate': 8.446291692419736e-05, 'epoch': 3.32}\n", + "{'loss': 0.9861, 'grad_norm': 3.677231550216675, 'learning_rate': 8.423644202007467e-05, 'epoch': 3.34}\n", + "{'loss': 0.9352, 'grad_norm': 3.738945245742798, 'learning_rate': 8.400863688854597e-05, 'epoch': 3.36}\n", + "{'loss': 1.0084, 'grad_norm': 4.615973949432373, 'learning_rate': 8.377951038078302e-05, 'epoch': 3.37}\n", + "{'loss': 0.8946, 'grad_norm': 4.280567169189453, 'learning_rate': 8.354907139929851e-05, 'epoch': 3.39}\n", + "{'loss': 0.9536, 'grad_norm': 5.548139572143555, 'learning_rate': 8.33173288976002e-05, 'epoch': 3.41}\n", + "{'loss': 1.032, 'grad_norm': 4.183009147644043, 'learning_rate': 8.308429187984297e-05, 'epoch': 3.43}\n", + "{'loss': 0.9905, 'grad_norm': 4.598621368408203, 'learning_rate': 8.284996940047903e-05, 'epoch': 3.44}\n", + "{'loss': 1.0168, 'grad_norm': 3.7102458477020264, 'learning_rate': 8.261437056390606e-05, 'epoch': 3.46}\n", + "{'loss': 0.9419, 'grad_norm': 3.9970738887786865, 'learning_rate': 8.237750452411353e-05, 'epoch': 3.48}\n", + "{'loss': 0.945, 'grad_norm': 5.531300067901611, 'learning_rate': 8.213938048432697e-05, 'epoch': 3.5}\n", + "{'loss': 0.8867, 'grad_norm': 5.528501510620117, 'learning_rate': 8.190000769665044e-05, 'epoch': 3.52}\n", + "{'loss': 0.9773, 'grad_norm': 5.0458807945251465, 'learning_rate': 8.1659395461707e-05, 'epoch': 3.53}\n", + "{'loss': 0.9484, 'grad_norm': 7.089639663696289, 'learning_rate': 8.141755312827736e-05, 'epoch': 3.55}\n", + "{'loss': 1.0592, 'grad_norm': 5.28053617477417, 'learning_rate': 8.117449009293668e-05, 'epoch': 3.57}\n", + "{'loss': 0.8727, 'grad_norm': 3.750885009765625, 'learning_rate': 8.093021579968941e-05, 'epoch': 3.59}\n", + "{'loss': 0.9438, 'grad_norm': 3.9763479232788086, 'learning_rate': 8.068473973960238e-05, 'epoch': 3.61}\n", + "{'loss': 1.0057, 'grad_norm': 8.926958084106445, 'learning_rate': 8.043807145043604e-05, 'epoch': 3.62}\n", + "{'loss': 0.9272, 'grad_norm': 4.707141399383545, 'learning_rate': 8.019022051627388e-05, 'epoch': 3.64}\n", + "{'loss': 0.9354, 'grad_norm': 4.845958232879639, 'learning_rate': 7.994119656715002e-05, 'epoch': 3.66}\n", + "{'loss': 1.0041, 'grad_norm': 6.272175312042236, 'learning_rate': 7.969100927867507e-05, 'epoch': 3.68}\n", + "{'loss': 1.0257, 'grad_norm': 5.634955883026123, 'learning_rate': 7.943966837166023e-05, 'epoch': 3.69}\n", + "{'loss': 1.0411, 'grad_norm': 4.726901054382324, 'learning_rate': 7.91871836117395e-05, 'epoch': 3.71}\n", + "{'loss': 0.8919, 'grad_norm': 5.341351509094238, 'learning_rate': 7.89335648089903e-05, 'epoch': 3.73}\n", + "{'loss': 0.9918, 'grad_norm': 4.697306156158447, 'learning_rate': 7.86788218175523e-05, 'epoch': 3.75}\n", + "{'loss': 1.0214, 'grad_norm': 7.20255708694458, 'learning_rate': 7.842296453524463e-05, 'epoch': 3.77}\n", + "{'loss': 0.8907, 'grad_norm': 4.981348037719727, 'learning_rate': 7.81660029031811e-05, 'epoch': 3.78}\n", + "{'loss': 0.9927, 'grad_norm': 4.630974292755127, 'learning_rate': 7.79079469053842e-05, 'epoch': 3.8}\n", + "{'loss': 0.9723, 'grad_norm': 4.9225921630859375, 'learning_rate': 7.764880656839696e-05, 'epoch': 3.82}\n", + "{'loss': 0.9968, 'grad_norm': 5.320995807647705, 'learning_rate': 7.738859196089358e-05, 'epoch': 3.84}\n", + "{'loss': 0.8093, 'grad_norm': 4.394636154174805, 'learning_rate': 7.712731319328798e-05, 'epoch': 3.86}\n", + "{'loss': 0.9058, 'grad_norm': 4.045576572418213, 'learning_rate': 7.68649804173412e-05, 'epoch': 3.87}\n", + "{'loss': 1.0048, 'grad_norm': 3.463576316833496, 'learning_rate': 7.660160382576683e-05, 'epoch': 3.89}\n", + "{'loss': 0.9774, 'grad_norm': 6.120863914489746, 'learning_rate': 7.633719365183504e-05, 'epoch': 3.91}\n", + "{'loss': 0.8715, 'grad_norm': 4.576050758361816, 'learning_rate': 7.60717601689749e-05, 'epoch': 3.93}\n", + "{'loss': 0.7799, 'grad_norm': 3.344226360321045, 'learning_rate': 7.580531369037533e-05, 'epoch': 3.94}\n", + "{'loss': 1.1199, 'grad_norm': 4.684515476226807, 'learning_rate': 7.553786456858429e-05, 'epoch': 3.96}\n", + "{'loss': 1.0056, 'grad_norm': 3.8074159622192383, 'learning_rate': 7.526942319510655e-05, 'epoch': 3.98}\n", + "{'loss': 0.8473, 'grad_norm': 3.2416229248046875, 'learning_rate': 7.500000000000001e-05, 'epoch': 4.0}\n", + " 40%|██████████████ | 2240/5600 [1:02:28<1:32:15, 1.65s/it][INFO|trainer.py:3788] 2024-06-30 07:19:03,748 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 07:19:03,748 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 07:19:03,748 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-2240\n", + "[INFO|configuration_utils.py:733] 2024-06-30 07:19:06,736 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 07:19:06,736 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 07:19:06,775 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2240/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 07:19:06,776 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2240/special_tokens_map.json\n", + "{'loss': 0.7957, 'grad_norm': 4.7848052978515625, 'learning_rate': 7.472960545147038e-05, 'epoch': 4.02}\n", + "{'loss': 0.631, 'grad_norm': 4.0633649826049805, 'learning_rate': 7.445825005546448e-05, 'epoch': 4.03}\n", + "{'loss': 0.7215, 'grad_norm': 4.579054832458496, 'learning_rate': 7.4185944355262e-05, 'epoch': 4.05}\n", + "{'loss': 0.6143, 'grad_norm': 4.206972122192383, 'learning_rate': 7.391269893106592e-05, 'epoch': 4.07}\n", + "{'loss': 0.6206, 'grad_norm': 4.568854808807373, 'learning_rate': 7.363852439959135e-05, 'epoch': 4.09}\n", + "{'loss': 0.7532, 'grad_norm': 4.556376934051514, 'learning_rate': 7.33634314136531e-05, 'epoch': 4.11}\n", + "{'loss': 0.5763, 'grad_norm': 5.614088535308838, 'learning_rate': 7.308743066175172e-05, 'epoch': 4.12}\n", + "{'loss': 0.6721, 'grad_norm': 9.883162498474121, 'learning_rate': 7.281053286765815e-05, 'epoch': 4.14}\n", + "{'loss': 0.7292, 'grad_norm': 4.348223686218262, 'learning_rate': 7.253274878999727e-05, 'epoch': 4.16}\n", + "{'loss': 0.6231, 'grad_norm': 3.8863847255706787, 'learning_rate': 7.225408922182961e-05, 'epoch': 4.18}\n", + "{'loss': 0.6226, 'grad_norm': 8.888066291809082, 'learning_rate': 7.197456499023225e-05, 'epoch': 4.19}\n", + "{'loss': 0.6884, 'grad_norm': 4.336313247680664, 'learning_rate': 7.169418695587791e-05, 'epoch': 4.21}\n", + "{'loss': 0.6207, 'grad_norm': 4.8345112800598145, 'learning_rate': 7.141296601261314e-05, 'epoch': 4.23}\n", + "{'loss': 0.7374, 'grad_norm': 5.926130771636963, 'learning_rate': 7.113091308703498e-05, 'epoch': 4.25}\n", + "{'loss': 0.6514, 'grad_norm': 5.482864856719971, 'learning_rate': 7.084803913806641e-05, 'epoch': 4.27}\n", + "{'loss': 0.6823, 'grad_norm': 6.955780029296875, 'learning_rate': 7.056435515653059e-05, 'epoch': 4.28}\n", + "{'loss': 0.6645, 'grad_norm': 10.101131439208984, 'learning_rate': 7.027987216472377e-05, 'epoch': 4.3}\n", + "{'loss': 0.5592, 'grad_norm': 5.433172225952148, 'learning_rate': 6.999460121598704e-05, 'epoch': 4.32}\n", + "{'loss': 0.7167, 'grad_norm': 6.796948432922363, 'learning_rate': 6.970855339427698e-05, 'epoch': 4.34}\n", + "{'loss': 0.6169, 'grad_norm': 3.863734006881714, 'learning_rate': 6.942173981373474e-05, 'epoch': 4.36}\n", + "{'loss': 0.6398, 'grad_norm': 3.5763421058654785, 'learning_rate': 6.91341716182545e-05, 'epoch': 4.37}\n", + "{'loss': 0.7066, 'grad_norm': 4.580504894256592, 'learning_rate': 6.884585998105026e-05, 'epoch': 4.39}\n", + "{'loss': 0.647, 'grad_norm': 5.605465412139893, 'learning_rate': 6.855681610422189e-05, 'epoch': 4.41}\n", + "{'loss': 0.7305, 'grad_norm': 3.584751605987549, 'learning_rate': 6.826705121831976e-05, 'epoch': 4.43}\n", + "{'loss': 0.7089, 'grad_norm': 6.16217041015625, 'learning_rate': 6.797657658190839e-05, 'epoch': 4.44}\n", + "{'loss': 0.5937, 'grad_norm': 3.5875444412231445, 'learning_rate': 6.768540348112907e-05, 'epoch': 4.46}\n", + "{'loss': 0.7547, 'grad_norm': 4.757628917694092, 'learning_rate': 6.739354322926136e-05, 'epoch': 4.48}\n", + "{'loss': 0.6766, 'grad_norm': 5.012269020080566, 'learning_rate': 6.710100716628344e-05, 'epoch': 4.5}\n", + "{'loss': 0.6572, 'grad_norm': 5.2274861335754395, 'learning_rate': 6.680780665843155e-05, 'epoch': 4.52}\n", + "{'loss': 0.7324, 'grad_norm': 5.329851150512695, 'learning_rate': 6.651395309775837e-05, 'epoch': 4.53}\n", + "{'loss': 0.7048, 'grad_norm': 6.628935813903809, 'learning_rate': 6.621945790169036e-05, 'epoch': 4.55}\n", + "{'loss': 0.683, 'grad_norm': 5.611133575439453, 'learning_rate': 6.592433251258423e-05, 'epoch': 4.57}\n", + "{'loss': 0.678, 'grad_norm': 5.939394950866699, 'learning_rate': 6.562858839728223e-05, 'epoch': 4.59}\n", + "{'loss': 0.5917, 'grad_norm': 10.7606201171875, 'learning_rate': 6.533223704666672e-05, 'epoch': 4.61}\n", + "{'loss': 0.6841, 'grad_norm': 6.9346089363098145, 'learning_rate': 6.503528997521366e-05, 'epoch': 4.62}\n", + "{'loss': 0.7123, 'grad_norm': 5.5321364402771, 'learning_rate': 6.473775872054521e-05, 'epoch': 4.64}\n", + "{'loss': 0.6863, 'grad_norm': 4.588550567626953, 'learning_rate': 6.44396548429815e-05, 'epoch': 4.66}\n", + "{'loss': 0.6828, 'grad_norm': 4.912098407745361, 'learning_rate': 6.414098992509138e-05, 'epoch': 4.68}\n", + "{'loss': 0.6467, 'grad_norm': 7.303658485412598, 'learning_rate': 6.384177557124247e-05, 'epoch': 4.69}\n", + "{'loss': 0.6986, 'grad_norm': 4.651421546936035, 'learning_rate': 6.354202340715026e-05, 'epoch': 4.71}\n", + "{'loss': 0.6532, 'grad_norm': 4.812668800354004, 'learning_rate': 6.324174507942637e-05, 'epoch': 4.73}\n", + "{'loss': 0.6688, 'grad_norm': 4.208662509918213, 'learning_rate': 6.294095225512603e-05, 'epoch': 4.75}\n", + "{'loss': 0.674, 'grad_norm': 5.573670387268066, 'learning_rate': 6.263965662129487e-05, 'epoch': 4.77}\n", + "{'loss': 0.7383, 'grad_norm': 4.292681694030762, 'learning_rate': 6.233786988451468e-05, 'epoch': 4.78}\n", + "{'loss': 0.7485, 'grad_norm': 4.01066255569458, 'learning_rate': 6.203560377044866e-05, 'epoch': 4.8}\n", + "{'loss': 0.6504, 'grad_norm': 4.865781307220459, 'learning_rate': 6.173287002338577e-05, 'epoch': 4.82}\n", + "{'loss': 0.6004, 'grad_norm': 3.7839431762695312, 'learning_rate': 6.142968040578449e-05, 'epoch': 4.84}\n", + "{'loss': 0.6536, 'grad_norm': 7.742762565612793, 'learning_rate': 6.112604669781572e-05, 'epoch': 4.85}\n", + "{'loss': 0.7322, 'grad_norm': 5.1467719078063965, 'learning_rate': 6.0821980696905146e-05, 'epoch': 4.87}\n", + "{'loss': 0.7393, 'grad_norm': 6.317329406738281, 'learning_rate': 6.0517494217274794e-05, 'epoch': 4.89}\n", + "{'loss': 0.7548, 'grad_norm': 5.456260681152344, 'learning_rate': 6.021259908948402e-05, 'epoch': 4.91}\n", + "{'loss': 0.6722, 'grad_norm': 5.673567771911621, 'learning_rate': 5.9907307159969884e-05, 'epoch': 4.93}\n", + "{'loss': 0.7384, 'grad_norm': 4.8718366622924805, 'learning_rate': 5.960163029058682e-05, 'epoch': 4.94}\n", + "{'loss': 0.7634, 'grad_norm': 5.771657943725586, 'learning_rate': 5.9295580358145744e-05, 'epoch': 4.96}\n", + "{'loss': 0.7871, 'grad_norm': 4.919590473175049, 'learning_rate': 5.898916925395264e-05, 'epoch': 4.98}\n", + "{'loss': 0.7701, 'grad_norm': 4.445159912109375, 'learning_rate': 5.868240888334653e-05, 'epoch': 5.0}\n", + " 50%|█████████████████▌ | 2800/5600 [1:18:09<1:20:41, 1.73s/it][INFO|trainer.py:3788] 2024-06-30 07:34:44,634 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 07:34:44,634 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 07:34:44,634 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-2800\n", + "[INFO|configuration_utils.py:733] 2024-06-30 07:34:47,846 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 07:34:47,846 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 07:34:47,887 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2800/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 07:34:47,887 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2800/special_tokens_map.json\n", + "{'loss': 0.5161, 'grad_norm': 3.8078582286834717, 'learning_rate': 5.837531116523682e-05, 'epoch': 5.02}\n", + "{'loss': 0.4045, 'grad_norm': 4.426279544830322, 'learning_rate': 5.806788803164034e-05, 'epoch': 5.03}\n", + "{'loss': 0.4832, 'grad_norm': 6.388131618499756, 'learning_rate': 5.7760151427217576e-05, 'epoch': 5.05}\n", + "{'loss': 0.4554, 'grad_norm': 4.689113616943359, 'learning_rate': 5.745211330880872e-05, 'epoch': 5.07}\n", + "{'loss': 0.4532, 'grad_norm': 4.104332447052002, 'learning_rate': 5.714378564496901e-05, 'epoch': 5.09}\n", + "{'loss': 0.5509, 'grad_norm': 4.345515727996826, 'learning_rate': 5.683518041550368e-05, 'epoch': 5.1}\n", + "{'loss': 0.4002, 'grad_norm': 6.301547527313232, 'learning_rate': 5.6526309611002594e-05, 'epoch': 5.12}\n", + "{'loss': 0.4822, 'grad_norm': 5.300792217254639, 'learning_rate': 5.621718523237427e-05, 'epoch': 5.14}\n", + "{'loss': 0.4324, 'grad_norm': 4.0373311042785645, 'learning_rate': 5.590781929037965e-05, 'epoch': 5.16}\n", + "{'loss': 0.4274, 'grad_norm': 6.742273330688477, 'learning_rate': 5.559822380516539e-05, 'epoch': 5.18}\n", + "{'loss': 0.494, 'grad_norm': 6.803271293640137, 'learning_rate': 5.5288410805796895e-05, 'epoch': 5.19}\n", + "{'loss': 0.4682, 'grad_norm': 3.1775426864624023, 'learning_rate': 5.497839232979084e-05, 'epoch': 5.21}\n", + "{'loss': 0.4614, 'grad_norm': 3.7366745471954346, 'learning_rate': 5.466818042264753e-05, 'epoch': 5.23}\n", + "{'loss': 0.5448, 'grad_norm': 5.096468448638916, 'learning_rate': 5.435778713738292e-05, 'epoch': 5.25}\n", + "{'loss': 0.4847, 'grad_norm': 4.3523712158203125, 'learning_rate': 5.404722453406017e-05, 'epoch': 5.27}\n", + "{'loss': 0.4473, 'grad_norm': 4.652655601501465, 'learning_rate': 5.373650467932122e-05, 'epoch': 5.28}\n", + "{'loss': 0.4453, 'grad_norm': 4.760082244873047, 'learning_rate': 5.3425639645917834e-05, 'epoch': 5.3}\n", + "{'loss': 0.4814, 'grad_norm': 5.638540267944336, 'learning_rate': 5.311464151224261e-05, 'epoch': 5.32}\n", + "{'loss': 0.4013, 'grad_norm': 3.9371888637542725, 'learning_rate': 5.2803522361859594e-05, 'epoch': 5.34}\n", + "{'loss': 0.4984, 'grad_norm': 4.2124152183532715, 'learning_rate': 5.249229428303486e-05, 'epoch': 5.35}\n", + "{'loss': 0.4914, 'grad_norm': 6.735795974731445, 'learning_rate': 5.218096936826681e-05, 'epoch': 5.37}\n", + "{'loss': 0.464, 'grad_norm': 4.825798988342285, 'learning_rate': 5.18695597138163e-05, 'epoch': 5.39}\n", + "{'loss': 0.3474, 'grad_norm': 4.686152458190918, 'learning_rate': 5.155807741923666e-05, 'epoch': 5.41}\n", + "{'loss': 0.3999, 'grad_norm': 4.344501972198486, 'learning_rate': 5.124653458690365e-05, 'epoch': 5.43}\n", + "{'loss': 0.3818, 'grad_norm': 3.8981587886810303, 'learning_rate': 5.0934943321545115e-05, 'epoch': 5.44}\n", + "{'loss': 0.4732, 'grad_norm': 8.811891555786133, 'learning_rate': 5.062331572977076e-05, 'epoch': 5.46}\n", + "{'loss': 0.609, 'grad_norm': 4.967749118804932, 'learning_rate': 5.031166391960168e-05, 'epoch': 5.48}\n", + "{'loss': 0.4442, 'grad_norm': 4.958866596221924, 'learning_rate': 5e-05, 'epoch': 5.5}\n", + "{'loss': 0.4474, 'grad_norm': 4.941844940185547, 'learning_rate': 4.968833608039832e-05, 'epoch': 5.52}\n", + "{'loss': 0.5222, 'grad_norm': 4.754947662353516, 'learning_rate': 4.9376684270229254e-05, 'epoch': 5.53}\n", + "{'loss': 0.4465, 'grad_norm': 4.058730125427246, 'learning_rate': 4.9065056678454904e-05, 'epoch': 5.55}\n", + "{'loss': 0.5767, 'grad_norm': 5.571474552154541, 'learning_rate': 4.875346541309637e-05, 'epoch': 5.57}\n", + "{'loss': 0.4737, 'grad_norm': 4.0056939125061035, 'learning_rate': 4.844192258076336e-05, 'epoch': 5.59}\n", + "{'loss': 0.5223, 'grad_norm': 5.950839042663574, 'learning_rate': 4.813044028618373e-05, 'epoch': 5.6}\n", + "{'loss': 0.5301, 'grad_norm': 4.6719255447387695, 'learning_rate': 4.781903063173321e-05, 'epoch': 5.62}\n", + "{'loss': 0.4188, 'grad_norm': 4.333907127380371, 'learning_rate': 4.750770571696514e-05, 'epoch': 5.64}\n", + "{'loss': 0.4934, 'grad_norm': 6.121321678161621, 'learning_rate': 4.7196477638140404e-05, 'epoch': 5.66}\n", + "{'loss': 0.4343, 'grad_norm': 5.436617374420166, 'learning_rate': 4.68853584877574e-05, 'epoch': 5.68}\n", + "{'loss': 0.4969, 'grad_norm': 5.086023330688477, 'learning_rate': 4.657436035408217e-05, 'epoch': 5.69}\n", + "{'loss': 0.4571, 'grad_norm': 5.212259769439697, 'learning_rate': 4.626349532067879e-05, 'epoch': 5.71}\n", + "{'loss': 0.5086, 'grad_norm': 4.355545997619629, 'learning_rate': 4.595277546593984e-05, 'epoch': 5.73}\n", + "{'loss': 0.4502, 'grad_norm': 3.553330421447754, 'learning_rate': 4.564221286261709e-05, 'epoch': 5.75}\n", + "{'loss': 0.5377, 'grad_norm': 4.984807014465332, 'learning_rate': 4.5331819577352474e-05, 'epoch': 5.77}\n", + "{'loss': 0.4203, 'grad_norm': 10.004477500915527, 'learning_rate': 4.502160767020918e-05, 'epoch': 5.78}\n", + "{'loss': 0.4515, 'grad_norm': 4.771313190460205, 'learning_rate': 4.471158919420312e-05, 'epoch': 5.8}\n", + "{'loss': 0.4102, 'grad_norm': 3.963116407394409, 'learning_rate': 4.4401776194834613e-05, 'epoch': 5.82}\n", + "{'loss': 0.4378, 'grad_norm': 5.920322895050049, 'learning_rate': 4.409218070962036e-05, 'epoch': 5.84}\n", + "{'loss': 0.5433, 'grad_norm': 5.597177505493164, 'learning_rate': 4.378281476762576e-05, 'epoch': 5.85}\n", + "{'loss': 0.3711, 'grad_norm': 10.070011138916016, 'learning_rate': 4.347369038899744e-05, 'epoch': 5.87}\n", + "{'loss': 0.5813, 'grad_norm': 3.5711491107940674, 'learning_rate': 4.316481958449634e-05, 'epoch': 5.89}\n", + "{'loss': 0.4328, 'grad_norm': 4.168658256530762, 'learning_rate': 4.285621435503101e-05, 'epoch': 5.91}\n", + "{'loss': 0.5556, 'grad_norm': 10.734298706054688, 'learning_rate': 4.254788669119127e-05, 'epoch': 5.93}\n", + "{'loss': 0.4497, 'grad_norm': 4.482186794281006, 'learning_rate': 4.223984857278242e-05, 'epoch': 5.94}\n", + "{'loss': 0.5173, 'grad_norm': 5.7400054931640625, 'learning_rate': 4.1932111968359664e-05, 'epoch': 5.96}\n", + "{'loss': 0.5062, 'grad_norm': 4.264299392700195, 'learning_rate': 4.162468883476319e-05, 'epoch': 5.98}\n", + "{'loss': 0.4793, 'grad_norm': 9.265963554382324, 'learning_rate': 4.131759111665349e-05, 'epoch': 6.0}\n", + " 60%|█████████████████████ | 3360/5600 [1:33:57<1:02:06, 1.66s/it][INFO|trainer.py:3788] 2024-06-30 07:50:32,520 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 07:50:32,520 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 07:50:32,520 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-3360\n", + "[INFO|configuration_utils.py:733] 2024-06-30 07:50:35,897 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 07:50:35,897 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 07:50:35,949 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-3360/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 07:50:35,949 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-3360/special_tokens_map.json\n", + "{'loss': 0.3304, 'grad_norm': 3.7002038955688477, 'learning_rate': 4.101083074604737e-05, 'epoch': 6.02}\n", + "{'loss': 0.2771, 'grad_norm': 4.872511863708496, 'learning_rate': 4.0704419641854274e-05, 'epoch': 6.03}\n", + "{'loss': 0.297, 'grad_norm': 3.524137020111084, 'learning_rate': 4.03983697094132e-05, 'epoch': 6.05}\n", + "{'loss': 0.3305, 'grad_norm': 1.7784379720687866, 'learning_rate': 4.0092692840030134e-05, 'epoch': 6.07}\n", + "{'loss': 0.3112, 'grad_norm': 4.346820831298828, 'learning_rate': 3.978740091051599e-05, 'epoch': 6.09}\n", + "{'loss': 0.349, 'grad_norm': 3.382376194000244, 'learning_rate': 3.9482505782725224e-05, 'epoch': 6.1}\n", + "{'loss': 0.256, 'grad_norm': 5.693515300750732, 'learning_rate': 3.917801930309486e-05, 'epoch': 6.12}\n", + "{'loss': 0.3141, 'grad_norm': 4.295924186706543, 'learning_rate': 3.887395330218429e-05, 'epoch': 6.14}\n", + "{'loss': 0.3338, 'grad_norm': 6.968880653381348, 'learning_rate': 3.857031959421553e-05, 'epoch': 6.16}\n", + "{'loss': 0.3056, 'grad_norm': 5.941532135009766, 'learning_rate': 3.8267129976614254e-05, 'epoch': 6.18}\n", + "{'loss': 0.3486, 'grad_norm': 3.909396171569824, 'learning_rate': 3.7964396229551364e-05, 'epoch': 6.19}\n", + "{'loss': 0.3718, 'grad_norm': 3.2766306400299072, 'learning_rate': 3.7662130115485314e-05, 'epoch': 6.21}\n", + "{'loss': 0.3308, 'grad_norm': 2.8996589183807373, 'learning_rate': 3.7360343378705124e-05, 'epoch': 6.23}\n", + "{'loss': 0.3928, 'grad_norm': 3.5176424980163574, 'learning_rate': 3.705904774487396e-05, 'epoch': 6.25}\n", + "{'loss': 0.2392, 'grad_norm': 10.349823951721191, 'learning_rate': 3.675825492057364e-05, 'epoch': 6.27}\n", + "{'loss': 0.3343, 'grad_norm': 7.081973552703857, 'learning_rate': 3.6457976592849754e-05, 'epoch': 6.28}\n", + "{'loss': 0.3097, 'grad_norm': 4.772485733032227, 'learning_rate': 3.6158224428757535e-05, 'epoch': 6.3}\n", + "{'loss': 0.3416, 'grad_norm': 3.539324998855591, 'learning_rate': 3.585901007490863e-05, 'epoch': 6.32}\n", + "{'loss': 0.34, 'grad_norm': 3.7091081142425537, 'learning_rate': 3.556034515701852e-05, 'epoch': 6.34}\n", + "{'loss': 0.3241, 'grad_norm': 5.218664646148682, 'learning_rate': 3.5262241279454785e-05, 'epoch': 6.35}\n", + "{'loss': 0.297, 'grad_norm': 3.1589152812957764, 'learning_rate': 3.4964710024786354e-05, 'epoch': 6.37}\n", + "{'loss': 0.3073, 'grad_norm': 2.8222711086273193, 'learning_rate': 3.4667762953333295e-05, 'epoch': 6.39}\n", + "{'loss': 0.2632, 'grad_norm': 5.614787578582764, 'learning_rate': 3.4371411602717784e-05, 'epoch': 6.41}\n", + "{'loss': 0.3375, 'grad_norm': 3.49419903755188, 'learning_rate': 3.4075667487415785e-05, 'epoch': 6.43}\n", + "{'loss': 0.323, 'grad_norm': 3.6888363361358643, 'learning_rate': 3.3780542098309654e-05, 'epoch': 6.44}\n", + "{'loss': 0.384, 'grad_norm': 2.8714163303375244, 'learning_rate': 3.3486046902241664e-05, 'epoch': 6.46}\n", + "{'loss': 0.36, 'grad_norm': 3.664397716522217, 'learning_rate': 3.319219334156847e-05, 'epoch': 6.48}\n", + "{'loss': 0.3239, 'grad_norm': 6.702901840209961, 'learning_rate': 3.289899283371657e-05, 'epoch': 6.5}\n", + "{'loss': 0.4023, 'grad_norm': 4.371044158935547, 'learning_rate': 3.2606456770738636e-05, 'epoch': 6.51}\n", + "{'loss': 0.2967, 'grad_norm': 7.265868663787842, 'learning_rate': 3.231459651887093e-05, 'epoch': 6.53}\n", + "{'loss': 0.3192, 'grad_norm': 4.020201683044434, 'learning_rate': 3.2023423418091626e-05, 'epoch': 6.55}\n", + "{'loss': 0.2575, 'grad_norm': 4.1831374168396, 'learning_rate': 3.173294878168025e-05, 'epoch': 6.57}\n", + "{'loss': 0.4101, 'grad_norm': 3.9656155109405518, 'learning_rate': 3.1443183895778105e-05, 'epoch': 6.59}\n", + "{'loss': 0.3168, 'grad_norm': 4.220931053161621, 'learning_rate': 3.115414001894974e-05, 'epoch': 6.6}\n", + "{'loss': 0.3144, 'grad_norm': 5.018192768096924, 'learning_rate': 3.086582838174551e-05, 'epoch': 6.62}\n", + "{'loss': 0.3115, 'grad_norm': 5.038303852081299, 'learning_rate': 3.0578260186265265e-05, 'epoch': 6.64}\n", + "{'loss': 0.2704, 'grad_norm': 2.8466811180114746, 'learning_rate': 3.029144660572304e-05, 'epoch': 6.66}\n", + "{'loss': 0.3401, 'grad_norm': 6.789051055908203, 'learning_rate': 3.000539878401296e-05, 'epoch': 6.68}\n", + "{'loss': 0.3225, 'grad_norm': 4.522548198699951, 'learning_rate': 2.9720127835276256e-05, 'epoch': 6.69}\n", + "{'loss': 0.3306, 'grad_norm': 3.3021113872528076, 'learning_rate': 2.9435644843469436e-05, 'epoch': 6.71}\n", + "{'loss': 0.3113, 'grad_norm': 6.549985885620117, 'learning_rate': 2.9151960861933614e-05, 'epoch': 6.73}\n", + "{'loss': 0.3258, 'grad_norm': 5.234971523284912, 'learning_rate': 2.886908691296504e-05, 'epoch': 6.75}\n", + "{'loss': 0.3151, 'grad_norm': 11.139360427856445, 'learning_rate': 2.858703398738686e-05, 'epoch': 6.76}\n", + "{'loss': 0.2697, 'grad_norm': 4.061713695526123, 'learning_rate': 2.8305813044122097e-05, 'epoch': 6.78}\n", + "{'loss': 0.2933, 'grad_norm': 5.231247425079346, 'learning_rate': 2.8025435009767747e-05, 'epoch': 6.8}\n", + "{'loss': 0.3019, 'grad_norm': 4.183421611785889, 'learning_rate': 2.774591077817038e-05, 'epoch': 6.82}\n", + "{'loss': 0.3599, 'grad_norm': 3.7946386337280273, 'learning_rate': 2.746725121000273e-05, 'epoch': 6.84}\n", + "{'loss': 0.2529, 'grad_norm': 4.079517364501953, 'learning_rate': 2.718946713234185e-05, 'epoch': 6.85}\n", + "{'loss': 0.3049, 'grad_norm': 4.9091033935546875, 'learning_rate': 2.6912569338248315e-05, 'epoch': 6.87}\n", + "{'loss': 0.3128, 'grad_norm': 4.097899436950684, 'learning_rate': 2.66365685863469e-05, 'epoch': 6.89}\n", + "{'loss': 0.3901, 'grad_norm': 5.156275749206543, 'learning_rate': 2.636147560040866e-05, 'epoch': 6.91}\n", + "{'loss': 0.2444, 'grad_norm': 4.529239177703857, 'learning_rate': 2.6087301068934106e-05, 'epoch': 6.93}\n", + "{'loss': 0.3085, 'grad_norm': 2.783036470413208, 'learning_rate': 2.581405564473801e-05, 'epoch': 6.94}\n", + "{'loss': 0.3202, 'grad_norm': 5.854332447052002, 'learning_rate': 2.5541749944535554e-05, 'epoch': 6.96}\n", + "{'loss': 0.3932, 'grad_norm': 5.028692722320557, 'learning_rate': 2.527039454852963e-05, 'epoch': 6.98}\n", + "{'loss': 0.2907, 'grad_norm': 7.342068672180176, 'learning_rate': 2.500000000000001e-05, 'epoch': 7.0}\n", + " 70%|█████████████████████████▉ | 3920/5600 [1:49:46<47:02, 1.68s/it][INFO|trainer.py:3788] 2024-06-30 08:06:21,782 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 08:06:21,782 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 08:06:21,782 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-3920\n", + "[INFO|configuration_utils.py:733] 2024-06-30 08:06:24,921 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 08:06:24,921 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 08:06:24,969 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-3920/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 08:06:24,969 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-3920/special_tokens_map.json\n", + "{'loss': 0.173, 'grad_norm': 2.5581254959106445, 'learning_rate': 2.473057680489348e-05, 'epoch': 7.01}\n", + "{'loss': 0.155, 'grad_norm': 1.571964144706726, 'learning_rate': 2.4462135431415733e-05, 'epoch': 7.03}\n", + "{'loss': 0.1707, 'grad_norm': 4.100607872009277, 'learning_rate': 2.4194686309624663e-05, 'epoch': 7.05}\n", + "{'loss': 0.2566, 'grad_norm': 3.124080181121826, 'learning_rate': 2.39282398310251e-05, 'epoch': 7.07}\n", + "{'loss': 0.1927, 'grad_norm': 2.3269150257110596, 'learning_rate': 2.366280634816496e-05, 'epoch': 7.09}\n", + "{'loss': 0.2814, 'grad_norm': 2.976987838745117, 'learning_rate': 2.3398396174233178e-05, 'epoch': 7.1}\n", + "{'loss': 0.1816, 'grad_norm': 7.45166015625, 'learning_rate': 2.3135019582658802e-05, 'epoch': 7.12}\n", + "{'loss': 0.2094, 'grad_norm': 3.6462252140045166, 'learning_rate': 2.2872686806712035e-05, 'epoch': 7.14}\n", + "{'loss': 0.1964, 'grad_norm': 4.667559623718262, 'learning_rate': 2.261140803910644e-05, 'epoch': 7.16}\n", + "{'loss': 0.2011, 'grad_norm': 1.900875449180603, 'learning_rate': 2.235119343160303e-05, 'epoch': 7.18}\n", + "{'loss': 0.1984, 'grad_norm': 3.4627413749694824, 'learning_rate': 2.2092053094615813e-05, 'epoch': 7.19}\n", + "{'loss': 0.2214, 'grad_norm': 3.783445358276367, 'learning_rate': 2.1833997096818898e-05, 'epoch': 7.21}\n", + "{'loss': 0.2047, 'grad_norm': 3.812368154525757, 'learning_rate': 2.157703546475539e-05, 'epoch': 7.23}\n", + "{'loss': 0.2822, 'grad_norm': 4.192997455596924, 'learning_rate': 2.132117818244771e-05, 'epoch': 7.25}\n", + "{'loss': 0.1728, 'grad_norm': 2.5023624897003174, 'learning_rate': 2.1066435191009715e-05, 'epoch': 7.26}\n", + "{'loss': 0.257, 'grad_norm': 7.863531589508057, 'learning_rate': 2.0812816388260518e-05, 'epoch': 7.28}\n", + "{'loss': 0.1866, 'grad_norm': 3.7875170707702637, 'learning_rate': 2.056033162833977e-05, 'epoch': 7.3}\n", + "{'loss': 0.1975, 'grad_norm': 1.9177666902542114, 'learning_rate': 2.0308990721324927e-05, 'epoch': 7.32}\n", + "{'loss': 0.2784, 'grad_norm': 3.343583345413208, 'learning_rate': 2.0058803432849987e-05, 'epoch': 7.34}\n", + "{'loss': 0.1964, 'grad_norm': 4.66720724105835, 'learning_rate': 1.980977948372612e-05, 'epoch': 7.35}\n", + "{'loss': 0.2273, 'grad_norm': 4.310459136962891, 'learning_rate': 1.9561928549563968e-05, 'epoch': 7.37}\n", + "{'loss': 0.2671, 'grad_norm': 3.40097975730896, 'learning_rate': 1.931526026039764e-05, 'epoch': 7.39}\n", + "{'loss': 0.1796, 'grad_norm': 4.316131591796875, 'learning_rate': 1.906978420031059e-05, 'epoch': 7.41}\n", + "{'loss': 0.2422, 'grad_norm': 3.5017640590667725, 'learning_rate': 1.8825509907063327e-05, 'epoch': 7.43}\n", + "{'loss': 0.2054, 'grad_norm': 3.1226840019226074, 'learning_rate': 1.8582446871722636e-05, 'epoch': 7.44}\n", + "{'loss': 0.2701, 'grad_norm': 3.492358684539795, 'learning_rate': 1.8340604538293015e-05, 'epoch': 7.46}\n", + "{'loss': 0.2063, 'grad_norm': 3.7369136810302734, 'learning_rate': 1.8099992303349577e-05, 'epoch': 7.48}\n", + "{'loss': 0.2125, 'grad_norm': 2.5832254886627197, 'learning_rate': 1.7860619515673033e-05, 'epoch': 7.5}\n", + "{'loss': 0.2224, 'grad_norm': 4.129978179931641, 'learning_rate': 1.7622495475886487e-05, 'epoch': 7.51}\n", + "{'loss': 0.2496, 'grad_norm': 3.395150661468506, 'learning_rate': 1.738562943609396e-05, 'epoch': 7.53}\n", + "{'loss': 0.2337, 'grad_norm': 3.5454814434051514, 'learning_rate': 1.7150030599520984e-05, 'epoch': 7.55}\n", + "{'loss': 0.166, 'grad_norm': 4.08375883102417, 'learning_rate': 1.691570812015704e-05, 'epoch': 7.57}\n", + "{'loss': 0.1812, 'grad_norm': 2.692218542098999, 'learning_rate': 1.6682671102399805e-05, 'epoch': 7.59}\n", + "{'loss': 0.1598, 'grad_norm': 4.0160231590271, 'learning_rate': 1.6450928600701504e-05, 'epoch': 7.6}\n", + "{'loss': 0.1428, 'grad_norm': 3.891842842102051, 'learning_rate': 1.622048961921699e-05, 'epoch': 7.62}\n", + "{'loss': 0.2096, 'grad_norm': 4.69844913482666, 'learning_rate': 1.599136311145402e-05, 'epoch': 7.64}\n", + "{'loss': 0.3205, 'grad_norm': 3.1264488697052, 'learning_rate': 1.5763557979925324e-05, 'epoch': 7.66}\n", + "{'loss': 0.2708, 'grad_norm': 4.53989315032959, 'learning_rate': 1.553708307580265e-05, 'epoch': 7.68}\n", + "{'loss': 0.2095, 'grad_norm': 2.3542563915252686, 'learning_rate': 1.531194719857292e-05, 'epoch': 7.69}\n", + "{'loss': 0.1645, 'grad_norm': 5.031445503234863, 'learning_rate': 1.5088159095696363e-05, 'epoch': 7.71}\n", + "{'loss': 0.2475, 'grad_norm': 5.543506145477295, 'learning_rate': 1.4865727462266543e-05, 'epoch': 7.73}\n", + "{'loss': 0.2831, 'grad_norm': 3.0280065536499023, 'learning_rate': 1.4644660940672627e-05, 'epoch': 7.75}\n", + "{'loss': 0.2326, 'grad_norm': 3.493994951248169, 'learning_rate': 1.4424968120263504e-05, 'epoch': 7.76}\n", + "{'loss': 0.2206, 'grad_norm': 3.2602357864379883, 'learning_rate': 1.4206657537014079e-05, 'epoch': 7.78}\n", + "{'loss': 0.2249, 'grad_norm': 5.095388412475586, 'learning_rate': 1.398973767319368e-05, 'epoch': 7.8}\n", + "{'loss': 0.2642, 'grad_norm': 13.810650825500488, 'learning_rate': 1.3774216957036367e-05, 'epoch': 7.82}\n", + "{'loss': 0.1592, 'grad_norm': 11.616129875183105, 'learning_rate': 1.3560103762413584e-05, 'epoch': 7.84}\n", + "{'loss': 0.1754, 'grad_norm': 4.300165176391602, 'learning_rate': 1.3347406408508695e-05, 'epoch': 7.85}\n", + "{'loss': 0.2721, 'grad_norm': 3.653315782546997, 'learning_rate': 1.3136133159493802e-05, 'epoch': 7.87}\n", + "{'loss': 0.1998, 'grad_norm': 3.611405611038208, 'learning_rate': 1.2926292224208664e-05, 'epoch': 7.89}\n", + "{'loss': 0.2173, 'grad_norm': 12.638509750366211, 'learning_rate': 1.2717891755841722e-05, 'epoch': 7.91}\n", + "{'loss': 0.2315, 'grad_norm': 12.878602981567383, 'learning_rate': 1.2510939851613285e-05, 'epoch': 7.93}\n", + "{'loss': 0.2456, 'grad_norm': 3.8157997131347656, 'learning_rate': 1.230544455246101e-05, 'epoch': 7.94}\n", + "{'loss': 0.1785, 'grad_norm': 4.757344722747803, 'learning_rate': 1.2101413842727345e-05, 'epoch': 7.96}\n", + "{'loss': 0.2599, 'grad_norm': 9.757575035095215, 'learning_rate': 1.1898855649849461e-05, 'epoch': 7.98}\n", + "{'loss': 0.1662, 'grad_norm': 2.5260682106018066, 'learning_rate': 1.1697777844051105e-05, 'epoch': 8.0}\n", + " 80%|█████████████████████████████▌ | 4480/5600 [2:05:14<30:43, 1.65s/it][INFO|trainer.py:3788] 2024-06-30 08:21:50,010 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 08:21:50,010 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 08:21:50,010 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-4480\n", + "[INFO|configuration_utils.py:733] 2024-06-30 08:21:52,924 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 08:21:52,925 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 08:21:52,971 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-4480/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 08:21:52,971 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-4480/special_tokens_map.json\n", + "{'loss': 0.1445, 'grad_norm': 1.7341903448104858, 'learning_rate': 1.1498188238036861e-05, 'epoch': 8.01}\n", + "{'loss': 0.113, 'grad_norm': 2.666015863418579, 'learning_rate': 1.130009458668863e-05, 'epoch': 8.03}\n", + "{'loss': 0.2103, 'grad_norm': 3.4498844146728516, 'learning_rate': 1.1103504586764263e-05, 'epoch': 8.05}\n", + "{'loss': 0.1438, 'grad_norm': 3.1626198291778564, 'learning_rate': 1.090842587659851e-05, 'epoch': 8.07}\n", + "{'loss': 0.1212, 'grad_norm': 2.0630691051483154, 'learning_rate': 1.0714866035806326e-05, 'epoch': 8.09}\n", + "{'loss': 0.1799, 'grad_norm': 3.088937282562256, 'learning_rate': 1.0522832584988234e-05, 'epoch': 8.1}\n", + "{'loss': 0.1107, 'grad_norm': 2.786867380142212, 'learning_rate': 1.0332332985438248e-05, 'epoch': 8.12}\n", + "{'loss': 0.1752, 'grad_norm': 1.7368676662445068, 'learning_rate': 1.0143374638853891e-05, 'epoch': 8.14}\n", + "{'loss': 0.1186, 'grad_norm': 2.039095878601074, 'learning_rate': 9.955964887048607e-06, 'epoch': 8.16}\n", + "{'loss': 0.133, 'grad_norm': 3.3849267959594727, 'learning_rate': 9.770111011666583e-06, 'epoch': 8.17}\n", + "{'loss': 0.1982, 'grad_norm': 3.557002305984497, 'learning_rate': 9.58582023389974e-06, 'epoch': 8.19}\n", + "{'loss': 0.1974, 'grad_norm': 2.3108747005462646, 'learning_rate': 9.403099714207175e-06, 'epoch': 8.21}\n", + "{'loss': 0.1912, 'grad_norm': 3.278822183609009, 'learning_rate': 9.221956552036992e-06, 'epoch': 8.23}\n", + "{'loss': 0.1803, 'grad_norm': 3.1523773670196533, 'learning_rate': 9.042397785550405e-06, 'epoch': 8.25}\n", + "{'loss': 0.1569, 'grad_norm': 4.3890862464904785, 'learning_rate': 8.864430391348332e-06, 'epoch': 8.26}\n", + "{'loss': 0.1246, 'grad_norm': 2.280132532119751, 'learning_rate': 8.688061284200266e-06, 'epoch': 8.28}\n", + "{'loss': 0.1357, 'grad_norm': 5.60243034362793, 'learning_rate': 8.513297316775625e-06, 'epoch': 8.3}\n", + "{'loss': 0.148, 'grad_norm': 4.402873992919922, 'learning_rate': 8.34014527937756e-06, 'epoch': 8.32}\n", + "{'loss': 0.1943, 'grad_norm': 3.0184195041656494, 'learning_rate': 8.168611899679013e-06, 'epoch': 8.34}\n", + "{'loss': 0.1651, 'grad_norm': 3.244899034500122, 'learning_rate': 7.998703842461431e-06, 'epoch': 8.35}\n", + "{'loss': 0.1725, 'grad_norm': 2.839618444442749, 'learning_rate': 7.830427709355725e-06, 'epoch': 8.37}\n", + "{'loss': 0.1764, 'grad_norm': 2.485934257507324, 'learning_rate': 7.663790038585793e-06, 'epoch': 8.39}\n", + "{'loss': 0.1308, 'grad_norm': 1.899274230003357, 'learning_rate': 7.498797304714544e-06, 'epoch': 8.41}\n", + "{'loss': 0.1754, 'grad_norm': 2.954799175262451, 'learning_rate': 7.33545591839222e-06, 'epoch': 8.42}\n", + "{'loss': 0.1637, 'grad_norm': 2.595350980758667, 'learning_rate': 7.173772226107434e-06, 'epoch': 8.44}\n", + "{'loss': 0.1736, 'grad_norm': 3.245035409927368, 'learning_rate': 7.013752509940485e-06, 'epoch': 8.46}\n", + "{'loss': 0.1975, 'grad_norm': 2.79209303855896, 'learning_rate': 6.855402987319348e-06, 'epoch': 8.48}\n", + "{'loss': 0.1608, 'grad_norm': 3.176992177963257, 'learning_rate': 6.698729810778065e-06, 'epoch': 8.5}\n", + "{'loss': 0.1425, 'grad_norm': 3.4199535846710205, 'learning_rate': 6.54373906771768e-06, 'epoch': 8.51}\n", + "{'loss': 0.1397, 'grad_norm': 2.7271015644073486, 'learning_rate': 6.390436780169734e-06, 'epoch': 8.53}\n", + "{'loss': 0.2058, 'grad_norm': 2.070603847503662, 'learning_rate': 6.238828904562316e-06, 'epoch': 8.55}\n", + "{'loss': 0.1612, 'grad_norm': 2.0566606521606445, 'learning_rate': 6.088921331488568e-06, 'epoch': 8.57}\n", + "{'loss': 0.0814, 'grad_norm': 2.8068063259124756, 'learning_rate': 5.94071988547788e-06, 'epoch': 8.59}\n", + "{'loss': 0.1446, 'grad_norm': 2.784498691558838, 'learning_rate': 5.794230324769517e-06, 'epoch': 8.6}\n", + "{'loss': 0.1159, 'grad_norm': 1.9032204151153564, 'learning_rate': 5.649458341088915e-06, 'epoch': 8.62}\n", + "{'loss': 0.1592, 'grad_norm': 3.4346718788146973, 'learning_rate': 5.506409559426573e-06, 'epoch': 8.64}\n", + "{'loss': 0.22, 'grad_norm': 1.5027986764907837, 'learning_rate': 5.365089537819434e-06, 'epoch': 8.66}\n", + "{'loss': 0.1448, 'grad_norm': 2.700094223022461, 'learning_rate': 5.2255037671349535e-06, 'epoch': 8.67}\n", + "{'loss': 0.173, 'grad_norm': 1.7306227684020996, 'learning_rate': 5.087657670857798e-06, 'epoch': 8.69}\n", + "{'loss': 0.1248, 'grad_norm': 2.1537575721740723, 'learning_rate': 4.951556604879048e-06, 'epoch': 8.71}\n", + "{'loss': 0.1908, 'grad_norm': 3.676980972290039, 'learning_rate': 4.8172058572881765e-06, 'epoch': 8.73}\n", + "{'loss': 0.1352, 'grad_norm': 3.5679290294647217, 'learning_rate': 4.684610648167503e-06, 'epoch': 8.75}\n", + "{'loss': 0.1403, 'grad_norm': 3.0709311962127686, 'learning_rate': 4.5537761293894535e-06, 'epoch': 8.76}\n", + "{'loss': 0.157, 'grad_norm': 4.296623706817627, 'learning_rate': 4.424707384416344e-06, 'epoch': 8.78}\n", + "{'loss': 0.1893, 'grad_norm': 2.2179601192474365, 'learning_rate': 4.29740942810285e-06, 'epoch': 8.8}\n", + "{'loss': 0.1651, 'grad_norm': 3.256356716156006, 'learning_rate': 4.1718872065011904e-06, 'epoch': 8.82}\n", + "{'loss': 0.1567, 'grad_norm': 2.9854118824005127, 'learning_rate': 4.048145596668967e-06, 'epoch': 8.84}\n", + "{'loss': 0.1463, 'grad_norm': 1.8327380418777466, 'learning_rate': 3.9261894064796135e-06, 'epoch': 8.85}\n", + "{'loss': 0.1511, 'grad_norm': 3.7827322483062744, 'learning_rate': 3.8060233744356633e-06, 'epoch': 8.87}\n", + "{'loss': 0.1404, 'grad_norm': 1.0582958459854126, 'learning_rate': 3.687652169484568e-06, 'epoch': 8.89}\n", + "{'loss': 0.1494, 'grad_norm': 2.8895883560180664, 'learning_rate': 3.5710803908373224e-06, 'epoch': 8.91}\n", + "{'loss': 0.1691, 'grad_norm': 4.131629467010498, 'learning_rate': 3.4563125677897932e-06, 'epoch': 8.92}\n", + "{'loss': 0.2159, 'grad_norm': 3.1218199729919434, 'learning_rate': 3.343353159546675e-06, 'epoch': 8.94}\n", + "{'loss': 0.1488, 'grad_norm': 1.3627033233642578, 'learning_rate': 3.2322065550483007e-06, 'epoch': 8.96}\n", + "{'loss': 0.1338, 'grad_norm': 0.7474280595779419, 'learning_rate': 3.1228770728000455e-06, 'epoch': 8.98}\n", + "{'loss': 0.1358, 'grad_norm': 7.197608947753906, 'learning_rate': 3.0153689607045845e-06, 'epoch': 9.0}\n", + " 90%|█████████████████████████████████▎ | 5040/5600 [2:20:27<14:49, 1.59s/it][INFO|trainer.py:3788] 2024-06-30 08:37:02,463 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 08:37:02,463 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 08:37:02,463 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-5040\n", + "[INFO|configuration_utils.py:733] 2024-06-30 08:37:05,334 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 08:37:05,334 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 08:37:05,376 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-5040/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 08:37:05,376 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-5040/special_tokens_map.json\n", + "{'loss': 0.127, 'grad_norm': 2.002713441848755, 'learning_rate': 2.9096863958968268e-06, 'epoch': 9.01}\n", + "{'loss': 0.1209, 'grad_norm': 1.9945132732391357, 'learning_rate': 2.8058334845816213e-06, 'epoch': 9.03}\n", + "{'loss': 0.1432, 'grad_norm': 1.8608288764953613, 'learning_rate': 2.7038142618741992e-06, 'epoch': 9.05}\n", + "{'loss': 0.1707, 'grad_norm': 3.0099308490753174, 'learning_rate': 2.603632691643415e-06, 'epoch': 9.07}\n", + "{'loss': 0.1253, 'grad_norm': 2.0166702270507812, 'learning_rate': 2.5052926663577e-06, 'epoch': 9.09}\n", + "{'loss': 0.1316, 'grad_norm': 2.971637487411499, 'learning_rate': 2.408798006933882e-06, 'epoch': 9.1}\n", + "{'loss': 0.0958, 'grad_norm': 3.2099649906158447, 'learning_rate': 2.314152462588659e-06, 'epoch': 9.12}\n", + "{'loss': 0.1209, 'grad_norm': 3.63370418548584, 'learning_rate': 2.221359710692961e-06, 'epoch': 9.14}\n", + "{'loss': 0.0965, 'grad_norm': 1.7885162830352783, 'learning_rate': 2.1304233566290964e-06, 'epoch': 9.16}\n", + "{'loss': 0.0858, 'grad_norm': 1.2611875534057617, 'learning_rate': 2.041346933650612e-06, 'epoch': 9.17}\n", + "{'loss': 0.1183, 'grad_norm': 1.9412541389465332, 'learning_rate': 1.9541339027450256e-06, 'epoch': 9.19}\n", + "{'loss': 0.1395, 'grad_norm': 1.758565068244934, 'learning_rate': 1.8687876524993987e-06, 'epoch': 9.21}\n", + "{'loss': 0.0943, 'grad_norm': 2.854973316192627, 'learning_rate': 1.785311498968617e-06, 'epoch': 9.23}\n", + "{'loss': 0.1427, 'grad_norm': 6.644575119018555, 'learning_rate': 1.70370868554659e-06, 'epoch': 9.25}\n", + "{'loss': 0.1434, 'grad_norm': 3.3001134395599365, 'learning_rate': 1.6239823828401945e-06, 'epoch': 9.26}\n", + "{'loss': 0.1218, 'grad_norm': 0.8132327795028687, 'learning_rate': 1.5461356885461075e-06, 'epoch': 9.28}\n", + "{'loss': 0.1723, 'grad_norm': 1.5899766683578491, 'learning_rate': 1.4701716273304521e-06, 'epoch': 9.3}\n", + "{'loss': 0.1067, 'grad_norm': 1.619358777999878, 'learning_rate': 1.3960931507112752e-06, 'epoch': 9.32}\n", + "{'loss': 0.1708, 'grad_norm': 1.985873818397522, 'learning_rate': 1.3239031369438326e-06, 'epoch': 9.34}\n", + "{'loss': 0.1221, 'grad_norm': 2.568528175354004, 'learning_rate': 1.2536043909088191e-06, 'epoch': 9.35}\n", + "{'loss': 0.1285, 'grad_norm': 2.3798413276672363, 'learning_rate': 1.1851996440033319e-06, 'epoch': 9.37}\n", + "{'loss': 0.1118, 'grad_norm': 0.7661011815071106, 'learning_rate': 1.118691554034773e-06, 'epoch': 9.39}\n", + "{'loss': 0.1178, 'grad_norm': 2.5488016605377197, 'learning_rate': 1.0540827051175818e-06, 'epoch': 9.41}\n", + "{'loss': 0.1357, 'grad_norm': 3.0472471714019775, 'learning_rate': 9.913756075728087e-07, 'epoch': 9.42}\n", + "{'loss': 0.0948, 'grad_norm': 0.8541691899299622, 'learning_rate': 9.305726978306173e-07, 'epoch': 9.44}\n", + "{'loss': 0.1502, 'grad_norm': 0.9478998780250549, 'learning_rate': 8.716763383355864e-07, 'epoch': 9.46}\n", + "{'loss': 0.1266, 'grad_norm': 1.7219117879867554, 'learning_rate': 8.146888174549339e-07, 'epoch': 9.48}\n", + "{'loss': 0.0963, 'grad_norm': 2.673491954803467, 'learning_rate': 7.596123493895991e-07, 'epoch': 9.5}\n", + "{'loss': 0.186, 'grad_norm': 2.8655078411102295, 'learning_rate': 7.064490740882057e-07, 'epoch': 9.51}\n", + "{'loss': 0.1359, 'grad_norm': 2.7357897758483887, 'learning_rate': 6.552010571639456e-07, 'epoch': 9.53}\n", + "{'loss': 0.1508, 'grad_norm': 2.8306162357330322, 'learning_rate': 6.058702898142643e-07, 'epoch': 9.55}\n", + "{'loss': 0.1036, 'grad_norm': 3.270542621612549, 'learning_rate': 5.584586887435739e-07, 'epoch': 9.57}\n", + "{'loss': 0.1504, 'grad_norm': 2.821152925491333, 'learning_rate': 5.129680960887007e-07, 'epoch': 9.59}\n", + "{'loss': 0.1388, 'grad_norm': 1.7769047021865845, 'learning_rate': 4.6940027934735954e-07, 'epoch': 9.6}\n", + "{'loss': 0.1101, 'grad_norm': 2.468860387802124, 'learning_rate': 4.277569313094809e-07, 'epoch': 9.62}\n", + "{'loss': 0.1552, 'grad_norm': 2.018123149871826, 'learning_rate': 3.8803966999139684e-07, 'epoch': 9.64}\n", + "{'loss': 0.124, 'grad_norm': 1.8176459074020386, 'learning_rate': 3.50250038573019e-07, 'epoch': 9.66}\n", + "{'loss': 0.1084, 'grad_norm': 1.4483444690704346, 'learning_rate': 3.143895053378698e-07, 'epoch': 9.67}\n", + "{'loss': 0.1122, 'grad_norm': 2.107964038848877, 'learning_rate': 2.8045946361601183e-07, 'epoch': 9.69}\n", + "{'loss': 0.167, 'grad_norm': 3.6413228511810303, 'learning_rate': 2.4846123172992954e-07, 'epoch': 9.71}\n", + "{'loss': 0.1457, 'grad_norm': 3.200455904006958, 'learning_rate': 2.1839605294330933e-07, 'epoch': 9.73}\n", + "{'loss': 0.1466, 'grad_norm': 2.517279863357544, 'learning_rate': 1.9026509541272275e-07, 'epoch': 9.75}\n", + "{'loss': 0.1311, 'grad_norm': 1.4828776121139526, 'learning_rate': 1.640694521422459e-07, 'epoch': 9.76}\n", + "{'loss': 0.1393, 'grad_norm': 2.981771945953369, 'learning_rate': 1.3981014094099353e-07, 'epoch': 9.78}\n", + "{'loss': 0.0979, 'grad_norm': 1.9358062744140625, 'learning_rate': 1.1748810438355628e-07, 'epoch': 9.8}\n", + "{'loss': 0.1863, 'grad_norm': 1.4688208103179932, 'learning_rate': 9.710420977340762e-08, 'epoch': 9.82}\n", + "{'loss': 0.1427, 'grad_norm': 2.011298418045044, 'learning_rate': 7.865924910916977e-08, 'epoch': 9.83}\n", + "{'loss': 0.0873, 'grad_norm': 3.8186533451080322, 'learning_rate': 6.215393905388278e-08, 'epoch': 9.85}\n", + "{'loss': 0.099, 'grad_norm': 2.1099774837493896, 'learning_rate': 4.7588920907110094e-08, 'epoch': 9.87}\n", + "{'loss': 0.1162, 'grad_norm': 2.2421796321868896, 'learning_rate': 3.496476058006959e-08, 'epoch': 9.89}\n", + "{'loss': 0.1642, 'grad_norm': 3.2422990798950195, 'learning_rate': 2.4281948573617874e-08, 'epoch': 9.91}\n", + "{'loss': 0.1631, 'grad_norm': 2.7475242614746094, 'learning_rate': 1.5540899959187727e-08, 'epoch': 9.92}\n", + "{'loss': 0.145, 'grad_norm': 2.781863212585449, 'learning_rate': 8.741954362678772e-09, 'epoch': 9.94}\n", + "{'loss': 0.1242, 'grad_norm': 3.5185129642486572, 'learning_rate': 3.885375951256931e-09, 'epoch': 9.96}\n", + "{'loss': 0.1676, 'grad_norm': 2.617418050765991, 'learning_rate': 9.713534230904041e-10, 'epoch': 9.98}\n", + "{'loss': 0.1304, 'grad_norm': 2.882068395614624, 'learning_rate': 0.0, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5600/5600 [2:35:33<00:00, 1.59s/it][INFO|trainer.py:3788] 2024-06-30 08:52:08,248 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 08:52:08,248 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 08:52:08,248 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-5600\n", + "[INFO|configuration_utils.py:733] 2024-06-30 08:52:11,263 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 08:52:11,263 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 08:52:11,334 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-5600/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 08:52:11,334 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-5600/special_tokens_map.json\n", + "[INFO|trainer.py:2383] 2024-06-30 08:52:11,559 >> \n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "{'train_runtime': 9336.476, 'train_samples_per_second': 4.801, 'train_steps_per_second': 0.6, 'train_loss': 0.7698830796884639, 'epoch': 10.0}\n", + "100%|█████████████████████████████████████| 5600/5600 [2:35:36<00:00, 1.67s/it]\n", + "[INFO|trainer.py:3478] 2024-06-30 08:52:11,560 >> Saving model checkpoint to saves/qwen2-1.5b/lora/sft\n", + "[INFO|configuration_utils.py:733] 2024-06-30 08:52:12,070 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 08:52:12,070 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-06-30 08:52:12,110 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-06-30 08:52:12,110 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/special_tokens_map.json\n", + "***** train metrics *****\n", + " epoch = 9.9955\n", + " total_flos = 27888647GF\n", + " train_loss = 0.7699\n", + " train_runtime = 2:35:36.47\n", + " train_samples_per_second = 4.801\n", + " train_steps_per_second = 0.6\n", + "Figure saved at: saves/qwen2-1.5b/lora/sft/training_loss.png\n", + "Figure saved at: saves/qwen2-1.5b/lora/sft/training_eval_loss.png\n", + "[INFO|trainer.py:3788] 2024-06-30 08:52:12,411 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-06-30 08:52:12,411 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-06-30 08:52:12,411 >> Batch size = 1\n", + "100%|███████████████████████████████████████████| 46/46 [00:02<00:00, 19.87it/s]\n", + "***** eval metrics *****\n", + " epoch = 9.9955\n", + " eval_loss = 3.5357\n", + " eval_runtime = 0:00:02.39\n", + " eval_samples_per_second = 19.224\n", + " eval_steps_per_second = 19.224\n", + "[INFO|modelcard.py:449] 2024-06-30 08:52:14,805 >> Dropping the following result as it does not have all the necessary fields:\n", + "{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}\n", + "CPU times: user 2min 11s, sys: 44.5 s, total: 2min 56s\n", + "Wall time: 2h 37min 48s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./scripts/tune-lf.sh config/qwen2_1.5b_lora_sft.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "!./scripts/tune-lf.sh config/qwen2_7b_lora_sft.yaml" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "07_MAC_+_Qwen2-7B-Instructi_Unsloth_train", + "widgets": {} + }, + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "036fc5746f43416db18c19ad8fd36677": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "06e806c82c7b4cbea31c5358dd9c3434": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "087b76a8b7514269b1f0ab29b062e444": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a069d2ab23824f29aa320ac256e2cfe9", + "placeholder": "​", + "style": "IPY_MODEL_06e806c82c7b4cbea31c5358dd9c3434", + "value": "Map (num_proc=2): 100%" + } + }, + "09b76013aa9e45efb6deb23a7a0d0925": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dea41c5260884aa6879b5e1d1697b14f", + "placeholder": "​", + "style": "IPY_MODEL_89965917796a4f81b899fdc7685f33df", + "value": "config.json: 100%" + } + }, + "0a92c56bfa134ef583220d7ef0b13e17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c34be936c8145d3ab41282f30a70713": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f8b6bfe16894500838793f2491d403f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "177c78fce95d4b4ab33057c5a048d693": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1f44c9ce1adf470cbb19784493ed209f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c34be936c8145d3ab41282f30a70713", + "placeholder": "​", + "style": "IPY_MODEL_0a92c56bfa134ef583220d7ef0b13e17", + "value": "model.safetensors: 100%" + } + }, + "201b59ccd9f845e197029b57e424aefc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2157f01726d748f8a9ae4a00664430da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21db8a77b00d4a4e82fdfa608657531f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26e4202cca81496a90d15a0dd4ca9cf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba90fdb8822d47dab7ba203bee297f37", + "IPY_MODEL_61560ff6a36b44f4a9dfdae5c52791d4", + "IPY_MODEL_95fbe66647904c06a20f640630d6dc0e" + ], + "layout": "IPY_MODEL_57182a263d324a3dbf1471c74290a0d5" + } + }, + "27155728b6b84cb199c91c940095d0a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6b91feeed5464877991ac2c207aebe7c", + "IPY_MODEL_cca8113c54c0495daedce1327bf9c68b", + "IPY_MODEL_2e63a29e2f7247bba5beede9a568c99f" + ], + "layout": "IPY_MODEL_5c9d781c28944f3eb86e2a6d44efdf18" + } + }, + "271ddaa553a042d09b6db7b450643d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a58d04b428c46f4b3dbadd3bc6cd529": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d18ddf6482c4d97829ac0e5a7b9868f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "IPY_MODEL_f2df530d22c74977b249dd9fb5f4829b", + "IPY_MODEL_89b2ef0dbfea47ab8e6f8d659e3351d1" + ], + "layout": "IPY_MODEL_3056b148aa9f4e6e8aa3b61d26886255" + } + }, + "2e5087c76f98437cb5dc729230358cba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e63a29e2f7247bba5beede9a568c99f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b993eaec6b224440bf80c0958c6fb536", + "placeholder": "​", + "style": "IPY_MODEL_de868e26e7154f62aa86223a539ad421", + "value": " 464/464 [00:00<00:00, 27.1kB/s]" + } + }, + "2f6c70dd266c4816bfad3fd3d192929a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "30307300bc4e4baf96560e30969a82b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e36a3f9eff0e4cf68834d66b0213ae96", + "placeholder": "​", + "style": "IPY_MODEL_a0037bdccf254159becde630bee3d1db", + "value": "generation_config.json: 100%" + } + }, + "3056b148aa9f4e6e8aa3b61d26886255": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30cdc32298134cb0be4d41615b9e5774": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3572201bd4d74a58b7a665f9bdfdcdba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35b0e8c26d6640e9bd0ed7b242a423d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e5087c76f98437cb5dc729230358cba", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_036fc5746f43416db18c19ad8fd36677", + "value": 51760 + } + }, + "36166c7bcb854b34aca1f41a5d6ea50b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "370692d819df41828b48c4ad446f977b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "39b29a75374b45c0a22506010be2b84e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_30cdc32298134cb0be4d41615b9e5774", + "max": 1179, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_47928317548c454bba6358ab132e8dee", + "value": 1179 + } + }, + "3cf2dd993b5e4d3daecf61e4bab5a404": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_087b76a8b7514269b1f0ab29b062e444", + "IPY_MODEL_35b0e8c26d6640e9bd0ed7b242a423d8", + "IPY_MODEL_54ad89e05fd74576b9b8b5b5a10eaf8d" + ], + "layout": "IPY_MODEL_a41dc44766444a998bec2d777f249d23" + } + }, + "43dec2ede91341f5af60eb522e18e984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4463edd481c1467f914c7dcd6c6e6ffc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47928317548c454bba6358ab132e8dee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "49277aeeac16434a865a4d12308b1abc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4ae7e449e4ea4c729b5f34607c18ebae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b2061b8a73c43ffb0c2f83daf0d0183": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c4c88d4c701450692fa0f6b0c5764b0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c666f4ace3943f8b80ecd20e7503236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ccedf0d93094e63b57a0f8a434fba06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4463edd481c1467f914c7dcd6c6e6ffc", + "max": 44307561, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6d3b9a05db0b4dadb638c686faa0c40a", + "value": 44307561 + } + }, + "4dcf6ff672d24983a1877a8431709aa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5807d5fb827d490fb3bc698f801ffff5", + "placeholder": "​", + "style": "IPY_MODEL_c4f2b06a82fd4987b8b659524a7b503b", + "value": "Generating train split: 100%" + } + }, + "4ea63adfce694725bdba878aef709dd3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5234566b1bfc4655b8d582ea5b46ed9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "54ad89e05fd74576b9b8b5b5a10eaf8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdb1941405ed4e4aa06019933892deb3", + "placeholder": "​", + "style": "IPY_MODEL_668d5377ca56426a99753867e6e24862", + "value": " 51760/51760 [01:02<00:00, 1131.51 examples/s]" + } + }, + "56aee4853b7740e6a977254f5d1fa66d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "57182a263d324a3dbf1471c74290a0d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5807d5fb827d490fb3bc698f801ffff5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c9d781c28944f3eb86e2a6d44efdf18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f40db8173dd4d76b6ef5ed6d9ec8b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61560ff6a36b44f4a9dfdae5c52791d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db19fc8d37db4e45a5790a876836d8c4", + "max": 11610, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36166c7bcb854b34aca1f41a5d6ea50b", + "value": 11610 + } + }, + "6578fd7acdb54c4c93528ea431fd0144": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_370692d819df41828b48c4ad446f977b", + "placeholder": "​", + "style": "IPY_MODEL_a0bf9160eb2647409b3200270914b90f", + "value": " 50.6k/50.6k [00:00<00:00, 2.71MB/s]" + } + }, + "668d5377ca56426a99753867e6e24862": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697f027529b54ee9956bae78a11e0611": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69ac12aec0714318bf2c83d4f4e745f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b2012c3f88547af8884a9ea90e3164b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_938f45f1b3e24118b815d96ae34ba86a", + "placeholder": "​", + "style": "IPY_MODEL_9367047a800747f79c6b225d92397846", + "value": " 44.3M/44.3M [00:01<00:00, 31.0MB/s]" + } + }, + "6b91feeed5464877991ac2c207aebe7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b2061b8a73c43ffb0c2f83daf0d0183", + "placeholder": "​", + "style": "IPY_MODEL_69ac12aec0714318bf2c83d4f4e745f5", + "value": "special_tokens_map.json: 100%" + } + }, + "6d3b9a05db0b4dadb638c686faa0c40a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6dbbedeca9314e66ae50e44ffa31a414": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6e34619b45934040b6092e6fb01ea7fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71ce208e20d6483abb9ed923510c86d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d69dc491b3ab44d7852b21873ed7bb7f", + "placeholder": "​", + "style": "IPY_MODEL_f401d53bf28e44eb906bce6c05412662", + "value": " 51760/51760 [00:01<00:00, 45512.81 examples/s]" + } + }, + "7358cdad832342c983e31efb8754ab78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e352a3404f4c7dad0737f57d29e92f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_988a0e8c1f89446086858da0a891a79c", + "IPY_MODEL_4ccedf0d93094e63b57a0f8a434fba06", + "IPY_MODEL_6b2012c3f88547af8884a9ea90e3164b" + ], + "layout": "IPY_MODEL_7e29cb8dd4df4d5b94407cd8fd3f2011" + } + }, + "74501720ac7e4dbb911a4a99b3633bc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "78e5400bff924a92a4cc61c4ff18b182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9b313fd861948f5aba25b24b1518d30", + "placeholder": "​", + "style": "IPY_MODEL_4c666f4ace3943f8b80ecd20e7503236", + "value": " 1.18k/1.18k [00:00<00:00, 31.3kB/s]" + } + }, + "7975adbc2ec5489ea7fa0167e620d85c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e34619b45934040b6092e6fb01ea7fe", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_271ddaa553a042d09b6db7b450643d8f", + "value": 51760 + } + }, + "7e29cb8dd4df4d5b94407cd8fd3f2011": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "810ff6c0e17d4fa09a30fef27eacff90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89965917796a4f81b899fdc7685f33df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "89b2ef0dbfea47ab8e6f8d659e3351d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8908fa0df3743ecb9d12983a739104f", + "placeholder": "​", + "style": "IPY_MODEL_177c78fce95d4b4ab33057c5a048d693", + "value": " 9.09M/9.09M [00:00<00:00, 32.6MB/s]" + } + }, + "8b3505352a5a42bf910428c40ce40465": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_49277aeeac16434a865a4d12308b1abc", + "placeholder": "​", + "style": "IPY_MODEL_2157f01726d748f8a9ae4a00664430da", + "value": " 5.70G/5.70G [01:02<00:00, 30.1MB/s]" + } + }, + "8fc142b628fb40568730234de1cafde2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ae7e449e4ea4c729b5f34607c18ebae", + "max": 172, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3572201bd4d74a58b7a665f9bdfdcdba", + "value": 172 + } + }, + "9367047a800747f79c6b225d92397846": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "938f45f1b3e24118b815d96ae34ba86a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "95fbe66647904c06a20f640630d6dc0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b0a370dc20654b279b9680692e34418e", + "placeholder": "​", + "style": "IPY_MODEL_cfeb365ddf7548d58b2557f22737fcf5", + "value": " 11.6k/11.6k [00:00<00:00, 716kB/s]" + } + }, + "988a0e8c1f89446086858da0a891a79c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ad2be500fc164c0f86f33e914ef8e6a0", + "placeholder": "​", + "style": "IPY_MODEL_5234566b1bfc4655b8d582ea5b46ed9f", + "value": "Downloading data: 100%" + } + }, + "98c58f23f4d549518832cb2d18f796e8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_09b76013aa9e45efb6deb23a7a0d0925", + "IPY_MODEL_39b29a75374b45c0a22506010be2b84e", + "IPY_MODEL_78e5400bff924a92a4cc61c4ff18b182" + ], + "layout": "IPY_MODEL_2a58d04b428c46f4b3dbadd3bc6cd529" + } + }, + "99fdbb0300c14c139d1937c646f0cfe7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7358cdad832342c983e31efb8754ab78", + "placeholder": "​", + "style": "IPY_MODEL_e9adf418296e436fb48bb9f78885598b", + "value": " 51760/51760 [00:01<00:00, 38665.95 examples/s]" + } + }, + "9f679ad3ec7f4fe8ad0510ffb57bc2ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ea63adfce694725bdba878aef709dd3", + "placeholder": "​", + "style": "IPY_MODEL_74501720ac7e4dbb911a4a99b3633bc6", + "value": "tokenizer.json: 100%" + } + }, + "a0037bdccf254159becde630bee3d1db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a069d2ab23824f29aa320ac256e2cfe9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0bf9160eb2647409b3200270914b90f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a41dc44766444a998bec2d777f249d23": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8464a4c711e4e00aafdfc919b60d07e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fb995c740590427b882572c81d4e848c", + "placeholder": "​", + "style": "IPY_MODEL_201b59ccd9f845e197029b57e424aefc", + "value": " 172/172 [00:00<00:00, 12.0kB/s]" + } + }, + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad2be500fc164c0f86f33e914ef8e6a0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b0240cd9a4554b29ae11f8051984a1c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_edaf890370314a218f138015faa0b05d", + "placeholder": "​", + "style": "IPY_MODEL_697f027529b54ee9956bae78a11e0611", + "value": "Map: 100%" + } + }, + "b0a370dc20654b279b9680692e34418e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b518dcee69074b87be73957cd810e7ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d891f8d0b1fc462f8008d02bb2a15692", + "placeholder": "​", + "style": "IPY_MODEL_cced8fd7e998472794f3f3e3018956a5", + "value": "tokenizer_config.json: 100%" + } + }, + "b8908fa0df3743ecb9d12983a739104f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b993eaec6b224440bf80c0958c6fb536": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9b313fd861948f5aba25b24b1518d30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba90fdb8822d47dab7ba203bee297f37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f8b6bfe16894500838793f2491d403f", + "placeholder": "​", + "style": "IPY_MODEL_bb19f6c747754682a514373a3a0535ba", + "value": "Downloading readme: 100%" + } + }, + "bb19f6c747754682a514373a3a0535ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc883d4cf13e4f8b8a4fe5f410cb6efd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9159e03e61f4f56978ece9c3bca49b2", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_810ff6c0e17d4fa09a30fef27eacff90", + "value": 51760 + } + }, + "c161d94df0f04feba9542237e0856c22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c22f71b1f85843209d7e5321506b9cb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f44c9ce1adf470cbb19784493ed209f", + "IPY_MODEL_f1addc4479d849879e743cf9089e6540", + "IPY_MODEL_8b3505352a5a42bf910428c40ce40465" + ], + "layout": "IPY_MODEL_4c4c88d4c701450692fa0f6b0c5764b0" + } + }, + "c4f2b06a82fd4987b8b659524a7b503b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cca8113c54c0495daedce1327bf9c68b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e02f9b7849c64531835eb77b860d1c93", + "max": 464, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_56aee4853b7740e6a977254f5d1fa66d", + "value": 464 + } + }, + "cced8fd7e998472794f3f3e3018956a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cf245afeb1c04f29a24d291608c3d157": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b518dcee69074b87be73957cd810e7ed", + "IPY_MODEL_e29104486d594b2992d7285e0ef77371", + "IPY_MODEL_6578fd7acdb54c4c93528ea431fd0144" + ], + "layout": "IPY_MODEL_d35db8148a354c56aaac56dbae22536f" + } + }, + "cfe8cae0e22b495bafa221a63d13b283": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfeb365ddf7548d58b2557f22737fcf5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1b47d39450d4019ae85c9b2f943eeaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4dcf6ff672d24983a1877a8431709aa9", + "IPY_MODEL_7975adbc2ec5489ea7fa0167e620d85c", + "IPY_MODEL_71ce208e20d6483abb9ed923510c86d7" + ], + "layout": "IPY_MODEL_cfe8cae0e22b495bafa221a63d13b283" + } + }, + "d35db8148a354c56aaac56dbae22536f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69dc491b3ab44d7852b21873ed7bb7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d891f8d0b1fc462f8008d02bb2a15692": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8e5318cead340c4adbeaccc05d39225": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "daf4cd890b35422683d22fd30bc71e83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b0240cd9a4554b29ae11f8051984a1c6", + "IPY_MODEL_bc883d4cf13e4f8b8a4fe5f410cb6efd", + "IPY_MODEL_99fdbb0300c14c139d1937c646f0cfe7" + ], + "layout": "IPY_MODEL_c161d94df0f04feba9542237e0856c22" + } + }, + "db19fc8d37db4e45a5790a876836d8c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de868e26e7154f62aa86223a539ad421": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dea41c5260884aa6879b5e1d1697b14f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e02f9b7849c64531835eb77b860d1c93": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e29104486d594b2992d7285e0ef77371": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "max": 50641, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2f6c70dd266c4816bfad3fd3d192929a", + "value": 50641 + } + }, + "e36a3f9eff0e4cf68834d66b0213ae96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9159e03e61f4f56978ece9c3bca49b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9adf418296e436fb48bb9f78885598b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "edaf890370314a218f138015faa0b05d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f1addc4479d849879e743cf9089e6540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43dec2ede91341f5af60eb522e18e984", + "max": 5702746405, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d8e5318cead340c4adbeaccc05d39225", + "value": 5702746405 + } + }, + "f2df530d22c74977b249dd9fb5f4829b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_21db8a77b00d4a4e82fdfa608657531f", + "max": 9085698, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6dbbedeca9314e66ae50e44ffa31a414", + "value": 9085698 + } + }, + "f401d53bf28e44eb906bce6c05412662": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fb995c740590427b882572c81d4e848c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fce7a61c25ec4390af43d92b7c473a45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_30307300bc4e4baf96560e30969a82b6", + "IPY_MODEL_8fc142b628fb40568730234de1cafde2", + "IPY_MODEL_a8464a4c711e4e00aafdfc919b60d07e" + ], + "layout": "IPY_MODEL_5f40db8173dd4d76b6ef5ed6d9ec8b6e" + } + }, + "fdb1941405ed4e4aa06019933892deb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/07r2_tune-lf-py3.11.ipynb b/notebooks/07r2_tune-lf-py3.11.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..7d1c5ea9813aade98e46209dc20f2008610ac75e --- /dev/null +++ b/notebooks/07r2_tune-lf-py3.11.ipynb @@ -0,0 +1,9938 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "0ea8b46b-839b-445b-8043-ccdf4e920ace", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6d394937-6c99-4a7c-9d32-7600a280032f", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /home/inflaton/code/projects/courses/llm-finetuning\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9f67ec60-2f24-411c-84eb-0dd664b44775", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f1597656-8042-4878-9d3b-9ebfb8dd86dc", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('unsloth/Qwen2-0.5B-Instruct',\n", + " True,\n", + " None,\n", + " None,\n", + " 2048,\n", + " 6,\n", + " None,\n", + " 'datasets/mac/mac.tsv',\n", + " 'results/mac-results_lf-r2.csv')" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "model_name = os.getenv(\"MODEL_NAME\")\n", + "token = os.getenv(\"HF_TOKEN\") or None\n", + "load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n", + "local_model = os.getenv(\"LOCAL_MODEL\")\n", + "hub_model = os.getenv(\"HUB_MODEL\")\n", + "num_train_epochs = int(os.getenv(\"NUM_TRAIN_EPOCHS\") or 0)\n", + "data_path = os.getenv(\"DATA_PATH\")\n", + "results_path = os.getenv(\"RESULTS_PATH\")\n", + "\n", + "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n", + "dtype = (\n", + " None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n", + ")\n", + "\n", + "model_name, load_in_4bit, local_model, hub_model, max_seq_length, num_train_epochs, dtype, data_path, results_path" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Thu Jul 4 11:06:16 2024 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 545.23.07 Driver Version: 546.12 CUDA Version: 12.3 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 NVIDIA GeForce RTX 4080 ... On | 00000000:01:00.0 Off | N/A |\n", + "| N/A 52C P8 3W / 150W | 355MiB / 12282MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "| No running processes found |\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "!nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english'],\n", + " num_rows: 1133\n", + " })\n", + "})\n" + ] + } + ], + "source": [ + "from llm_toolkit.translation_engine import load_translation_dataset\n", + "\n", + "dataset = load_translation_dataset(data_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "df = dataset[\"train\"].to_pandas()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "df_alpaca = pd.DataFrame({\"instruction\": [\"Please translate the following Chinese text into English and provide only the translated content, nothing else.\"]*len(df)})" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
instructioninputoutput
0Please translate the following Chinese text in...全仗着狐仙搭救。Because I was protected by a fox fairy.
1Please translate the following Chinese text in...过后，表哥告诉她俩，这人是导演，在外国留过学的，还会编剧，今天拍的这戏，就是他自编自导的。He was the director, the cousin later told the...
2Please translate the following Chinese text in...这凤姐忽然想起一件事来，便向窗外叫：“蓉儿回来！”Xi-feng suddenly seemed to remember something,...
3Please translate the following Chinese text in...三个老红卫兵走到叶文洁面前，面对着她站成了一排——当年，她们也是这样面对叶哲泰的——试图再现...The three old Red Guards stood in front of Ye ...
4Please translate the following Chinese text in...程先生照单全收，都是一个“谢”字，然后问王琦瑶有什么话说。Mr. Cheng accepted their toast with equanimity...
............
4523Please translate the following Chinese text in...外边有两张腿歪面裂的八仙桌子，桌旁胡乱搡着几条狭窄的木凳。Two rickety tables with scarred tops and a few...
4524Please translate the following Chinese text in...贾瑞听了，喜的抓耳挠腮。At this last remark Jia Rui positively scratch...
4525Please translate the following Chinese text in...听了这样的评价，我们心情激动，和大家一起振臂高呼：打倒王二！Hearing comments like this, our emotions were ...
4526Please translate the following Chinese text in...海老公道：“记住了吗？”'Can you remember that?'
4527Please translate the following Chinese text in...上面说，这样写缺少细节。This time the opinions from above said it need...
\n", + "

4528 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " instruction \\\n", + "0 Please translate the following Chinese text in... \n", + "1 Please translate the following Chinese text in... \n", + "2 Please translate the following Chinese text in... \n", + "3 Please translate the following Chinese text in... \n", + "4 Please translate the following Chinese text in... \n", + "... ... \n", + "4523 Please translate the following Chinese text in... \n", + "4524 Please translate the following Chinese text in... \n", + "4525 Please translate the following Chinese text in... \n", + "4526 Please translate the following Chinese text in... \n", + "4527 Please translate the following Chinese text in... \n", + "\n", + " input \\\n", + "0 全仗着狐仙搭救。 \n", + "1 过后，表哥告诉她俩，这人是导演，在外国留过学的，还会编剧，今天拍的这戏，就是他自编自导的。 \n", + "2 这凤姐忽然想起一件事来，便向窗外叫：“蓉儿回来！” \n", + "3 三个老红卫兵走到叶文洁面前，面对着她站成了一排——当年，她们也是这样面对叶哲泰的——试图再现... \n", + "4 程先生照单全收，都是一个“谢”字，然后问王琦瑶有什么话说。 \n", + "... ... \n", + "4523 外边有两张腿歪面裂的八仙桌子，桌旁胡乱搡着几条狭窄的木凳。 \n", + "4524 贾瑞听了，喜的抓耳挠腮。 \n", + "4525 听了这样的评价，我们心情激动，和大家一起振臂高呼：打倒王二！ \n", + "4526 海老公道：“记住了吗？” \n", + "4527 上面说，这样写缺少细节。 \n", + "\n", + " output \n", + "0 Because I was protected by a fox fairy. \n", + "1 He was the director, the cousin later told the... \n", + "2 Xi-feng suddenly seemed to remember something,... \n", + "3 The three old Red Guards stood in front of Ye ... \n", + "4 Mr. Cheng accepted their toast with equanimity... \n", + "... ... \n", + "4523 Two rickety tables with scarred tops and a few... \n", + "4524 At this last remark Jia Rui positively scratch... \n", + "4525 Hearing comments like this, our emotions were ... \n", + "4526 'Can you remember that?' \n", + "4527 This time the opinions from above said it need... \n", + "\n", + "[4528 rows x 3 columns]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_alpaca[\"input\"] = df[\"chinese\"]\n", + "df_alpaca[\"output\"] = df[\"english\"]\n", + "df_alpaca" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "df_alpaca.to_json(\n", + " \"llama-factory/data/alpaca_mac.json\", orient=\"records\", lines=False, indent=2\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_json(\"llama-factory/data/alpaca_mac.json\", orient=\"records\", lines=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
instructioninputoutput
0Please translate the following Chinese text in...全仗着狐仙搭救。Because I was protected by a fox fairy.
1Please translate the following Chinese text in...过后，表哥告诉她俩，这人是导演，在外国留过学的，还会编剧，今天拍的这戏，就是他自编自导的。He was the director, the cousin later told the...
2Please translate the following Chinese text in...这凤姐忽然想起一件事来，便向窗外叫：“蓉儿回来！”Xi-feng suddenly seemed to remember something,...
3Please translate the following Chinese text in...三个老红卫兵走到叶文洁面前，面对着她站成了一排——当年，她们也是这样面对叶哲泰的——试图再现...The three old Red Guards stood in front of Ye ...
4Please translate the following Chinese text in...程先生照单全收，都是一个“谢”字，然后问王琦瑶有什么话说。Mr. Cheng accepted their toast with equanimity...
\n", + "
" + ], + "text/plain": [ + " instruction \\\n", + "0 Please translate the following Chinese text in... \n", + "1 Please translate the following Chinese text in... \n", + "2 Please translate the following Chinese text in... \n", + "3 Please translate the following Chinese text in... \n", + "4 Please translate the following Chinese text in... \n", + "\n", + " input \\\n", + "0 全仗着狐仙搭救。 \n", + "1 过后，表哥告诉她俩，这人是导演，在外国留过学的，还会编剧，今天拍的这戏，就是他自编自导的。 \n", + "2 这凤姐忽然想起一件事来，便向窗外叫：“蓉儿回来！” \n", + "3 三个老红卫兵走到叶文洁面前，面对着她站成了一排——当年，她们也是这样面对叶哲泰的——试图再现... \n", + "4 程先生照单全收，都是一个“谢”字，然后问王琦瑶有什么话说。 \n", + "\n", + " output \n", + "0 Because I was protected by a fox fairy. \n", + "1 He was the director, the cousin later told the... \n", + "2 Xi-feng suddenly seemed to remember something,... \n", + "3 The three old Red Guards stood in front of Ye ... \n", + "4 Mr. Cheng accepted their toast with equanimity... " + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Python 3.11.9\n", + "\u001b[33mWARNING: Package(s) not found: flash-attn\u001b[0m\u001b[33m\n", + "\u001b[0mCPU times: user 5.39 ms, sys: 19.5 ms, total: 24.9 ms\n", + "Wall time: 527 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "!python --version\n", + "!pip show flash-attn" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/llm-finetuning/llama-factory\n", + "07/04/2024 11:09:05 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: torch.bfloat16\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 11:09:06,545 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 11:09:06,545 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 11:09:06,545 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 11:09:06,545 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 11:09:06,545 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 11:09:06,545 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-04 11:09:06,662 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/04/2024 11:09:06 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/04/2024 11:09:06 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "07/04/2024 11:09:06 - INFO - llamafactory.data.loader - Loading dataset alpaca_mac.json...\n", + "Converting format of dataset (num_proc=16): 100%|█| 4528/4528 [00:00<00:00, 1685\n", + "Running tokenizer on dataset (num_proc=16): 100%|█| 4528/4528 [00:01<00:00, 3476\n", + "input_ids:\n", + "[151644, 872, 198, 5501, 14683, 279, 2701, 8453, 1467, 1119, 6364, 323, 3410, 1172, 279, 24531, 2213, 11, 4302, 770, 624, 35987, 102895, 99164, 100324, 100717, 100095, 99509, 1773, 151645, 198, 151644, 77091, 198, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "inputs:\n", + "<|im_start|>user\n", + "Please translate the following Chinese text into English and provide only the translated content, nothing else.\n", + "全仗着狐仙搭救。<|im_end|>\n", + "<|im_start|>assistant\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "label_ids:\n", + "[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "labels:\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "[INFO|configuration_utils.py:733] 2024-07-04 11:09:09,749 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 11:09:09,750 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:3556] 2024-07-04 11:09:09,841 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-04 11:09:13,066 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-04 11:09:13,069 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-04 11:10:03,269 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-04 11:10:03,270 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-04 11:10:03,578 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-04 11:10:03,578 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "07/04/2024 11:10:03 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.\n", + "07/04/2024 11:10:03 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/04/2024 11:10:03 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.\n", + "07/04/2024 11:10:03 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA\n", + "07/04/2024 11:10:03 - INFO - llamafactory.model.model_utils.misc - Found linear modules: up_proj,down_proj,k_proj,q_proj,v_proj,o_proj,gate_proj\n", + "07/04/2024 11:10:04 - INFO - llamafactory.model.loader - trainable params: 4,399,104 || all params: 498,431,872 || trainable%: 0.8826\n", + "[INFO|trainer.py:642] 2024-07-04 11:10:04,049 >> Using auto half precision backend\n", + "07/04/2024 11:10:04 - WARNING - llamafactory.train.callbacks - Previous trainer log in this folder will be deleted.\n", + "[INFO|trainer.py:2128] 2024-07-04 11:10:04,194 >> ***** Running training *****\n", + "[INFO|trainer.py:2129] 2024-07-04 11:10:04,194 >> Num examples = 4,482\n", + "[INFO|trainer.py:2130] 2024-07-04 11:10:04,194 >> Num Epochs = 6\n", + "[INFO|trainer.py:2131] 2024-07-04 11:10:04,194 >> Instantaneous batch size per device = 1\n", + "[INFO|trainer.py:2134] 2024-07-04 11:10:04,194 >> Total train batch size (w. parallel, distributed & accumulation) = 8\n", + "[INFO|trainer.py:2135] 2024-07-04 11:10:04,194 >> Gradient Accumulation steps = 8\n", + "[INFO|trainer.py:2136] 2024-07-04 11:10:04,195 >> Total optimization steps = 3,360\n", + "[INFO|trainer.py:2137] 2024-07-04 11:10:04,196 >> Number of trainable parameters = 4,399,104\n", + "[INFO|integration_utils.py:750] 2024-07-04 11:10:04,198 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33minflaton-sg\u001b[0m (\u001b[33minflaton-ai\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.4\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/home/inflaton/code/projects/courses/llm-finetuning/llama-factory/wandb/run-20240704_111005-u8sqhi0x\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mqwen2_0.5b_lora_sft\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/u8sqhi0x\u001b[0m\n", + "{'loss': 2.581, 'grad_norm': 2.9743993282318115, 'learning_rate': 2.9761904761904763e-06, 'epoch': 0.02}\n", + "{'loss': 2.704, 'grad_norm': 3.803558826446533, 'learning_rate': 5.9523809523809525e-06, 'epoch': 0.04}\n", + "{'loss': 2.5764, 'grad_norm': 2.419433116912842, 'learning_rate': 8.92857142857143e-06, 'epoch': 0.05}\n", + "{'loss': 2.4994, 'grad_norm': 4.8528876304626465, 'learning_rate': 1.1904761904761905e-05, 'epoch': 0.07}\n", + "{'loss': 2.6881, 'grad_norm': 2.5375239849090576, 'learning_rate': 1.4880952380952381e-05, 'epoch': 0.09}\n", + "{'loss': 2.3869, 'grad_norm': 2.810744524002075, 'learning_rate': 1.785714285714286e-05, 'epoch': 0.11}\n", + "{'loss': 2.5728, 'grad_norm': 2.6387815475463867, 'learning_rate': 2.0833333333333336e-05, 'epoch': 0.12}\n", + "{'loss': 2.3077, 'grad_norm': 2.4742910861968994, 'learning_rate': 2.380952380952381e-05, 'epoch': 0.14}\n", + "{'loss': 2.4318, 'grad_norm': 3.0079479217529297, 'learning_rate': 2.6785714285714288e-05, 'epoch': 0.16}\n", + "{'loss': 2.29, 'grad_norm': 2.584622859954834, 'learning_rate': 2.9761904761904762e-05, 'epoch': 0.18}\n", + "{'loss': 2.3407, 'grad_norm': 3.3264784812927246, 'learning_rate': 3.273809523809524e-05, 'epoch': 0.2}\n", + "{'loss': 2.3577, 'grad_norm': 2.667269468307495, 'learning_rate': 3.571428571428572e-05, 'epoch': 0.21}\n", + "{'loss': 2.2612, 'grad_norm': 2.8811182975769043, 'learning_rate': 3.8690476190476195e-05, 'epoch': 0.23}\n", + "{'loss': 2.3096, 'grad_norm': 3.249279499053955, 'learning_rate': 4.166666666666667e-05, 'epoch': 0.25}\n", + "{'loss': 2.183, 'grad_norm': 2.5008630752563477, 'learning_rate': 4.464285714285715e-05, 'epoch': 0.27}\n", + "{'loss': 2.23, 'grad_norm': 2.457791328430176, 'learning_rate': 4.761904761904762e-05, 'epoch': 0.29}\n", + "{'loss': 2.3025, 'grad_norm': 2.4453022480010986, 'learning_rate': 5.05952380952381e-05, 'epoch': 0.3}\n", + "{'loss': 2.0884, 'grad_norm': 2.7773451805114746, 'learning_rate': 5.3571428571428575e-05, 'epoch': 0.32}\n", + "{'loss': 2.2048, 'grad_norm': 3.600346565246582, 'learning_rate': 5.6547619047619046e-05, 'epoch': 0.34}\n", + "{'loss': 2.3676, 'grad_norm': 2.939140796661377, 'learning_rate': 5.9523809523809524e-05, 'epoch': 0.36}\n", + "{'loss': 2.2684, 'grad_norm': 2.7832212448120117, 'learning_rate': 6.25e-05, 'epoch': 0.37}\n", + "{'loss': 2.2021, 'grad_norm': 3.7691140174865723, 'learning_rate': 6.547619047619048e-05, 'epoch': 0.39}\n", + "{'loss': 2.1625, 'grad_norm': 3.3338756561279297, 'learning_rate': 6.845238095238096e-05, 'epoch': 0.41}\n", + "{'loss': 2.3564, 'grad_norm': 4.061848163604736, 'learning_rate': 7.142857142857143e-05, 'epoch': 0.43}\n", + "{'loss': 2.2266, 'grad_norm': 3.3382863998413086, 'learning_rate': 7.440476190476191e-05, 'epoch': 0.45}\n", + "{'loss': 2.1837, 'grad_norm': 3.208007335662842, 'learning_rate': 7.738095238095239e-05, 'epoch': 0.46}\n", + "{'loss': 2.1765, 'grad_norm': 4.045449733734131, 'learning_rate': 8.035714285714287e-05, 'epoch': 0.48}\n", + "{'loss': 2.2863, 'grad_norm': 4.37124490737915, 'learning_rate': 8.333333333333334e-05, 'epoch': 0.5}\n", + "{'loss': 2.0807, 'grad_norm': 2.6629326343536377, 'learning_rate': 8.630952380952382e-05, 'epoch': 0.52}\n", + "{'loss': 2.2086, 'grad_norm': 3.6005942821502686, 'learning_rate': 8.92857142857143e-05, 'epoch': 0.54}\n", + "{'loss': 2.2231, 'grad_norm': 4.065690040588379, 'learning_rate': 9.226190476190478e-05, 'epoch': 0.55}\n", + "{'loss': 1.9875, 'grad_norm': 6.6260294914245605, 'learning_rate': 9.523809523809524e-05, 'epoch': 0.57}\n", + "{'loss': 2.0721, 'grad_norm': 4.6804656982421875, 'learning_rate': 9.821428571428572e-05, 'epoch': 0.59}\n", + "{'loss': 2.1194, 'grad_norm': 4.226340293884277, 'learning_rate': 9.999956828659095e-05, 'epoch': 0.61}\n", + "{'loss': 2.1256, 'grad_norm': 4.530922889709473, 'learning_rate': 9.999471159635539e-05, 'epoch': 0.62}\n", + "{'loss': 2.0243, 'grad_norm': 3.235328197479248, 'learning_rate': 9.998445910004082e-05, 'epoch': 0.64}\n", + "{'loss': 2.2819, 'grad_norm': 4.247537136077881, 'learning_rate': 9.996881190417393e-05, 'epoch': 0.66}\n", + "{'loss': 2.1964, 'grad_norm': 3.339164972305298, 'learning_rate': 9.994777169751806e-05, 'epoch': 0.68}\n", + "{'loss': 1.9102, 'grad_norm': 2.744009494781494, 'learning_rate': 9.992134075089084e-05, 'epoch': 0.7}\n", + "{'loss': 2.0751, 'grad_norm': 3.513111114501953, 'learning_rate': 9.988952191691925e-05, 'epoch': 0.71}\n", + "{'loss': 2.1697, 'grad_norm': 3.301513433456421, 'learning_rate': 9.985231862973168e-05, 'epoch': 0.73}\n", + "{'loss': 2.1742, 'grad_norm': 2.8456363677978516, 'learning_rate': 9.980973490458728e-05, 'epoch': 0.75}\n", + "{'loss': 2.1497, 'grad_norm': 3.499181032180786, 'learning_rate': 9.976177533744261e-05, 'epoch': 0.77}\n", + "{'loss': 2.0643, 'grad_norm': 4.2905964851379395, 'learning_rate': 9.97084451044556e-05, 'epoch': 0.79}\n", + "{'loss': 1.9934, 'grad_norm': 2.706711769104004, 'learning_rate': 9.964974996142698e-05, 'epoch': 0.8}\n", + "{'loss': 2.0795, 'grad_norm': 3.038059949874878, 'learning_rate': 9.958569624317893e-05, 'epoch': 0.82}\n", + "{'loss': 2.0908, 'grad_norm': 4.291042804718018, 'learning_rate': 9.951629086287151e-05, 'epoch': 0.84}\n", + "{'loss': 2.105, 'grad_norm': 3.027702808380127, 'learning_rate': 9.944154131125642e-05, 'epoch': 0.86}\n", + "{'loss': 2.112, 'grad_norm': 2.6875832080841064, 'learning_rate': 9.936145565586871e-05, 'epoch': 0.87}\n", + "{'loss': 2.2824, 'grad_norm': 2.8110086917877197, 'learning_rate': 9.927604254015585e-05, 'epoch': 0.89}\n", + "{'loss': 2.2181, 'grad_norm': 3.3072471618652344, 'learning_rate': 9.918531118254507e-05, 'epoch': 0.91}\n", + "{'loss': 2.1132, 'grad_norm': 3.8883237838745117, 'learning_rate': 9.90892713754483e-05, 'epoch': 0.93}\n", + "{'loss': 2.1513, 'grad_norm': 3.775455951690674, 'learning_rate': 9.898793348420536e-05, 'epoch': 0.95}\n", + "{'loss': 2.1119, 'grad_norm': 3.0280404090881348, 'learning_rate': 9.888130844596524e-05, 'epoch': 0.96}\n", + "{'loss': 2.1126, 'grad_norm': 3.2323291301727295, 'learning_rate': 9.876940776850569e-05, 'epoch': 0.98}\n", + "{'loss': 2.1328, 'grad_norm': 2.91339373588562, 'learning_rate': 9.865224352899119e-05, 'epoch': 1.0}\n", + " 17%|██████▎ | 560/3360 [12:30<1:02:53, 1.35s/it][INFO|trainer.py:3788] 2024-07-04 11:22:39,524 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 11:22:39,524 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 11:22:39,524 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-560\n", + "[INFO|configuration_utils.py:733] 2024-07-04 11:22:42,026 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 11:22:42,027 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 11:22:42,060 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-560/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 11:22:42,060 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-560/special_tokens_map.json\n", + "{'loss': 1.996, 'grad_norm': 2.9073429107666016, 'learning_rate': 9.852982837266955e-05, 'epoch': 1.02}\n", + "{'loss': 1.7941, 'grad_norm': 3.4045894145965576, 'learning_rate': 9.840217551150706e-05, 'epoch': 1.04}\n", + "{'loss': 1.9779, 'grad_norm': 2.8464860916137695, 'learning_rate': 9.826929872276255e-05, 'epoch': 1.05}\n", + "{'loss': 1.92, 'grad_norm': 3.770984411239624, 'learning_rate': 9.81312123475006e-05, 'epoch': 1.07}\n", + "{'loss': 1.8683, 'grad_norm': 3.4236226081848145, 'learning_rate': 9.798793128904356e-05, 'epoch': 1.09}\n", + "{'loss': 1.9201, 'grad_norm': 4.08709716796875, 'learning_rate': 9.78394710113631e-05, 'epoch': 1.11}\n", + "{'loss': 1.8563, 'grad_norm': 3.362687349319458, 'learning_rate': 9.768584753741134e-05, 'epoch': 1.12}\n", + "{'loss': 1.913, 'grad_norm': 5.210264682769775, 'learning_rate': 9.752707744739145e-05, 'epoch': 1.14}\n", + "{'loss': 1.9273, 'grad_norm': 3.515490770339966, 'learning_rate': 9.736317787696816e-05, 'epoch': 1.16}\n", + "{'loss': 1.8016, 'grad_norm': 3.4942610263824463, 'learning_rate': 9.719416651541839e-05, 'epoch': 1.18}\n", + "{'loss': 1.7993, 'grad_norm': 2.7268266677856445, 'learning_rate': 9.702006160372209e-05, 'epoch': 1.2}\n", + "{'loss': 1.9155, 'grad_norm': 3.6193785667419434, 'learning_rate': 9.684088193259355e-05, 'epoch': 1.21}\n", + "{'loss': 1.8261, 'grad_norm': 4.29509973526001, 'learning_rate': 9.665664684045333e-05, 'epoch': 1.23}\n", + "{'loss': 1.9301, 'grad_norm': 4.692563056945801, 'learning_rate': 9.646737621134112e-05, 'epoch': 1.25}\n", + "{'loss': 1.8418, 'grad_norm': 4.545106410980225, 'learning_rate': 9.627309047276974e-05, 'epoch': 1.27}\n", + "{'loss': 2.0611, 'grad_norm': 4.3200860023498535, 'learning_rate': 9.607381059352038e-05, 'epoch': 1.29}\n", + "{'loss': 1.9531, 'grad_norm': 3.2151238918304443, 'learning_rate': 9.586955808137958e-05, 'epoch': 1.3}\n", + "{'loss': 1.9447, 'grad_norm': 3.385021209716797, 'learning_rate': 9.566035498081784e-05, 'epoch': 1.32}\n", + "{'loss': 1.9424, 'grad_norm': 8.94682502746582, 'learning_rate': 9.544622387061055e-05, 'epoch': 1.34}\n", + "{'loss': 1.706, 'grad_norm': 4.064428806304932, 'learning_rate': 9.522718786140097e-05, 'epoch': 1.36}\n", + "{'loss': 1.9165, 'grad_norm': 4.604166507720947, 'learning_rate': 9.500327059320606e-05, 'epoch': 1.37}\n", + "{'loss': 1.7816, 'grad_norm': 5.32956600189209, 'learning_rate': 9.477449623286505e-05, 'epoch': 1.39}\n", + "{'loss': 1.6637, 'grad_norm': 3.613009214401245, 'learning_rate': 9.454088947143116e-05, 'epoch': 1.41}\n", + "{'loss': 1.9416, 'grad_norm': 4.8296799659729, 'learning_rate': 9.430247552150673e-05, 'epoch': 1.43}\n", + "{'loss': 1.8371, 'grad_norm': 4.565757751464844, 'learning_rate': 9.405928011452211e-05, 'epoch': 1.45}\n", + "{'loss': 1.846, 'grad_norm': 3.5512914657592773, 'learning_rate': 9.381132949795861e-05, 'epoch': 1.46}\n", + "{'loss': 2.0069, 'grad_norm': 3.9040660858154297, 'learning_rate': 9.35586504325155e-05, 'epoch': 1.48}\n", + "{'loss': 1.8083, 'grad_norm': 3.609498977661133, 'learning_rate': 9.330127018922194e-05, 'epoch': 1.5}\n", + "{'loss': 1.7487, 'grad_norm': 3.3245325088500977, 'learning_rate': 9.303921654649362e-05, 'epoch': 1.52}\n", + "{'loss': 1.8764, 'grad_norm': 4.417221546173096, 'learning_rate': 9.277251778713474e-05, 'epoch': 1.54}\n", + "{'loss': 1.8843, 'grad_norm': 4.959105014801025, 'learning_rate': 9.250120269528546e-05, 'epoch': 1.55}\n", + "{'loss': 1.793, 'grad_norm': 3.7974698543548584, 'learning_rate': 9.22253005533154e-05, 'epoch': 1.57}\n", + "{'loss': 1.9039, 'grad_norm': 3.882502555847168, 'learning_rate': 9.194484113866313e-05, 'epoch': 1.59}\n", + "{'loss': 1.9854, 'grad_norm': 3.416905164718628, 'learning_rate': 9.165985472062246e-05, 'epoch': 1.61}\n", + "{'loss': 1.7529, 'grad_norm': 3.456245183944702, 'learning_rate': 9.137037205707552e-05, 'epoch': 1.62}\n", + "{'loss': 1.8017, 'grad_norm': 3.490054130554199, 'learning_rate': 9.107642439117321e-05, 'epoch': 1.64}\n", + "{'loss': 1.8225, 'grad_norm': 3.2115142345428467, 'learning_rate': 9.077804344796302e-05, 'epoch': 1.66}\n", + "{'loss': 1.8333, 'grad_norm': 3.5726113319396973, 'learning_rate': 9.04752614309652e-05, 'epoch': 1.68}\n", + "{'loss': 1.7861, 'grad_norm': 3.9323503971099854, 'learning_rate': 9.01681110186971e-05, 'epoch': 1.7}\n", + "{'loss': 1.8067, 'grad_norm': 4.4842352867126465, 'learning_rate': 8.985662536114613e-05, 'epoch': 1.71}\n", + "{'loss': 1.8397, 'grad_norm': 3.1608762741088867, 'learning_rate': 8.954083807619208e-05, 'epoch': 1.73}\n", + "{'loss': 1.9411, 'grad_norm': 3.920475959777832, 'learning_rate': 8.922078324597879e-05, 'epoch': 1.75}\n", + "{'loss': 1.8974, 'grad_norm': 3.438220739364624, 'learning_rate': 8.889649541323574e-05, 'epoch': 1.77}\n", + "{'loss': 1.8202, 'grad_norm': 4.780834674835205, 'learning_rate': 8.856800957755e-05, 'epoch': 1.78}\n", + "{'loss': 1.8528, 'grad_norm': 3.768432378768921, 'learning_rate': 8.823536119158864e-05, 'epoch': 1.8}\n", + "{'loss': 1.753, 'grad_norm': 4.07826042175293, 'learning_rate': 8.789858615727265e-05, 'epoch': 1.82}\n", + "{'loss': 1.7389, 'grad_norm': 3.5676631927490234, 'learning_rate': 8.755772082190194e-05, 'epoch': 1.84}\n", + "{'loss': 1.9198, 'grad_norm': 3.463003635406494, 'learning_rate': 8.721280197423258e-05, 'epoch': 1.86}\n", + "{'loss': 1.7722, 'grad_norm': 4.634316921234131, 'learning_rate': 8.68638668405062e-05, 'epoch': 1.87}\n", + "{'loss': 1.8237, 'grad_norm': 4.284477710723877, 'learning_rate': 8.651095308043232e-05, 'epoch': 1.89}\n", + "{'loss': 2.0051, 'grad_norm': 4.610734462738037, 'learning_rate': 8.61540987831238e-05, 'epoch': 1.91}\n", + "{'loss': 1.9493, 'grad_norm': 4.1395392417907715, 'learning_rate': 8.579334246298593e-05, 'epoch': 1.93}\n", + "{'loss': 1.8477, 'grad_norm': 3.6301958560943604, 'learning_rate': 8.542872305555978e-05, 'epoch': 1.95}\n", + "{'loss': 1.7, 'grad_norm': 3.8048858642578125, 'learning_rate': 8.50602799133199e-05, 'epoch': 1.96}\n", + "{'loss': 1.8371, 'grad_norm': 3.2337429523468018, 'learning_rate': 8.468805280142709e-05, 'epoch': 1.98}\n", + "{'loss': 1.8531, 'grad_norm': 4.216500282287598, 'learning_rate': 8.43120818934367e-05, 'epoch': 2.0}\n", + " 33%|█████████████ | 1120/3360 [25:00<49:13, 1.32s/it][INFO|trainer.py:3788] 2024-07-04 11:35:10,200 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 11:35:10,200 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 11:35:10,200 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-1120\n", + "[INFO|configuration_utils.py:733] 2024-07-04 11:35:13,176 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 11:35:13,177 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 11:35:13,210 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-1120/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 11:35:13,211 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-1120/special_tokens_map.json\n", + "{'loss': 1.5674, 'grad_norm': 4.559268474578857, 'learning_rate': 8.393240776696274e-05, 'epoch': 2.02}\n", + "{'loss': 1.4393, 'grad_norm': 3.3662822246551514, 'learning_rate': 8.354907139929851e-05, 'epoch': 2.03}\n", + "{'loss': 1.5166, 'grad_norm': 4.587384223937988, 'learning_rate': 8.316211416299397e-05, 'epoch': 2.05}\n", + "{'loss': 1.4818, 'grad_norm': 5.713983535766602, 'learning_rate': 8.27715778213905e-05, 'epoch': 2.07}\n", + "{'loss': 1.3679, 'grad_norm': 3.7478792667388916, 'learning_rate': 8.237750452411353e-05, 'epoch': 2.09}\n", + "{'loss': 1.4682, 'grad_norm': 3.7805116176605225, 'learning_rate': 8.197993680252334e-05, 'epoch': 2.11}\n", + "{'loss': 1.6848, 'grad_norm': 4.318390846252441, 'learning_rate': 8.157891756512488e-05, 'epoch': 2.12}\n", + "{'loss': 1.447, 'grad_norm': 4.625955581665039, 'learning_rate': 8.117449009293668e-05, 'epoch': 2.14}\n", + "{'loss': 1.4888, 'grad_norm': 4.70202112197876, 'learning_rate': 8.076669803481965e-05, 'epoch': 2.16}\n", + "{'loss': 1.5405, 'grad_norm': 6.126914978027344, 'learning_rate': 8.035558540276618e-05, 'epoch': 2.18}\n", + "{'loss': 1.4751, 'grad_norm': 3.867528200149536, 'learning_rate': 7.994119656715002e-05, 'epoch': 2.2}\n", + "{'loss': 1.5175, 'grad_norm': 4.935867786407471, 'learning_rate': 7.952357625193749e-05, 'epoch': 2.21}\n", + "{'loss': 1.5586, 'grad_norm': 5.28302001953125, 'learning_rate': 7.91027695298606e-05, 'epoch': 2.23}\n", + "{'loss': 1.5798, 'grad_norm': 4.9564738273620605, 'learning_rate': 7.86788218175523e-05, 'epoch': 2.25}\n", + "{'loss': 1.4184, 'grad_norm': 4.7498779296875, 'learning_rate': 7.8251778870645e-05, 'epoch': 2.27}\n", + "{'loss': 1.4736, 'grad_norm': 5.780045032501221, 'learning_rate': 7.782168677883206e-05, 'epoch': 2.28}\n", + "{'loss': 1.5192, 'grad_norm': 3.647230625152588, 'learning_rate': 7.738859196089358e-05, 'epoch': 2.3}\n", + "{'loss': 1.5836, 'grad_norm': 4.818410396575928, 'learning_rate': 7.695254115968648e-05, 'epoch': 2.32}\n", + "{'loss': 1.6111, 'grad_norm': 4.5074286460876465, 'learning_rate': 7.651358143709972e-05, 'epoch': 2.34}\n", + "{'loss': 1.6122, 'grad_norm': 4.6216816902160645, 'learning_rate': 7.60717601689749e-05, 'epoch': 2.36}\n", + "{'loss': 1.5633, 'grad_norm': 9.873260498046875, 'learning_rate': 7.562712503999327e-05, 'epoch': 2.37}\n", + "{'loss': 1.7444, 'grad_norm': 4.795359134674072, 'learning_rate': 7.517972403852905e-05, 'epoch': 2.39}\n", + "{'loss': 1.5804, 'grad_norm': 4.818080902099609, 'learning_rate': 7.472960545147038e-05, 'epoch': 2.41}\n", + "{'loss': 1.4748, 'grad_norm': 5.576250076293945, 'learning_rate': 7.427681785900761e-05, 'epoch': 2.43}\n", + "{'loss': 1.5531, 'grad_norm': 4.261260509490967, 'learning_rate': 7.382141012939034e-05, 'epoch': 2.45}\n", + "{'loss': 1.4554, 'grad_norm': 4.23293399810791, 'learning_rate': 7.33634314136531e-05, 'epoch': 2.46}\n", + "{'loss': 1.5272, 'grad_norm': 4.627878665924072, 'learning_rate': 7.290293114031061e-05, 'epoch': 2.48}\n", + "{'loss': 1.6616, 'grad_norm': 4.36018705368042, 'learning_rate': 7.243995901002312e-05, 'epoch': 2.5}\n", + "{'loss': 1.5503, 'grad_norm': 5.698966026306152, 'learning_rate': 7.197456499023225e-05, 'epoch': 2.52}\n", + "{'loss': 1.5043, 'grad_norm': 4.486359119415283, 'learning_rate': 7.150679930976825e-05, 'epoch': 2.53}\n", + "{'loss': 1.5796, 'grad_norm': 8.031678199768066, 'learning_rate': 7.103671245342887e-05, 'epoch': 2.55}\n", + "{'loss': 1.4317, 'grad_norm': 5.806405544281006, 'learning_rate': 7.056435515653059e-05, 'epoch': 2.57}\n", + "{'loss': 1.696, 'grad_norm': 6.584068298339844, 'learning_rate': 7.008977839943299e-05, 'epoch': 2.59}\n", + "{'loss': 1.4768, 'grad_norm': 4.871330261230469, 'learning_rate': 6.961303340203653e-05, 'epoch': 2.61}\n", + "{'loss': 1.587, 'grad_norm': 3.9512643814086914, 'learning_rate': 6.91341716182545e-05, 'epoch': 2.62}\n", + "{'loss': 1.4991, 'grad_norm': 3.4907033443450928, 'learning_rate': 6.86532447304597e-05, 'epoch': 2.64}\n", + "{'loss': 1.4822, 'grad_norm': 4.603860855102539, 'learning_rate': 6.817030464390656e-05, 'epoch': 2.66}\n", + "{'loss': 1.6408, 'grad_norm': 5.737949371337891, 'learning_rate': 6.768540348112907e-05, 'epoch': 2.68}\n", + "{'loss': 1.4316, 'grad_norm': 5.838085174560547, 'learning_rate': 6.719859357631535e-05, 'epoch': 2.7}\n", + "{'loss': 1.414, 'grad_norm': 5.460419654846191, 'learning_rate': 6.670992746965938e-05, 'epoch': 2.71}\n", + "{'loss': 1.6858, 'grad_norm': 5.311679363250732, 'learning_rate': 6.621945790169036e-05, 'epoch': 2.73}\n", + "{'loss': 1.5802, 'grad_norm': 4.987999439239502, 'learning_rate': 6.572723780758069e-05, 'epoch': 2.75}\n", + "{'loss': 1.5672, 'grad_norm': 5.01920223236084, 'learning_rate': 6.523332031143272e-05, 'epoch': 2.77}\n", + "{'loss': 1.5914, 'grad_norm': 4.382671356201172, 'learning_rate': 6.473775872054521e-05, 'epoch': 2.78}\n", + "{'loss': 1.4284, 'grad_norm': 3.818115711212158, 'learning_rate': 6.424060651966007e-05, 'epoch': 2.8}\n", + "{'loss': 1.499, 'grad_norm': 4.427730560302734, 'learning_rate': 6.374191736518974e-05, 'epoch': 2.82}\n", + "{'loss': 1.4914, 'grad_norm': 4.508190631866455, 'learning_rate': 6.324174507942637e-05, 'epoch': 2.84}\n", + "{'loss': 1.4629, 'grad_norm': 6.055968284606934, 'learning_rate': 6.274014364473274e-05, 'epoch': 2.86}\n", + "{'loss': 1.717, 'grad_norm': 4.5250678062438965, 'learning_rate': 6.22371671977162e-05, 'epoch': 2.87}\n", + "{'loss': 1.5103, 'grad_norm': 4.378949165344238, 'learning_rate': 6.173287002338577e-05, 'epoch': 2.89}\n", + "{'loss': 1.511, 'grad_norm': 5.3176751136779785, 'learning_rate': 6.122730654929334e-05, 'epoch': 2.91}\n", + "{'loss': 1.4656, 'grad_norm': 4.5037994384765625, 'learning_rate': 6.072053133965938e-05, 'epoch': 2.93}\n", + "{'loss': 1.6443, 'grad_norm': 4.189935684204102, 'learning_rate': 6.021259908948402e-05, 'epoch': 2.95}\n", + "{'loss': 1.6633, 'grad_norm': 4.525129795074463, 'learning_rate': 5.970356461864391e-05, 'epoch': 2.96}\n", + "{'loss': 1.4935, 'grad_norm': 5.440227508544922, 'learning_rate': 5.919348286597569e-05, 'epoch': 2.98}\n", + "{'loss': 1.6304, 'grad_norm': 4.765013694763184, 'learning_rate': 5.868240888334653e-05, 'epoch': 3.0}\n", + " 50%|███████████████████▌ | 1680/3360 [37:13<35:24, 1.26s/it][INFO|trainer.py:3788] 2024-07-04 11:47:23,337 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 11:47:23,337 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 11:47:23,337 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-1680\n", + "[INFO|configuration_utils.py:733] 2024-07-04 11:47:25,920 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 11:47:25,920 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 11:47:25,956 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-1680/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 11:47:25,956 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-1680/special_tokens_map.json\n", + "{'loss': 1.4346, 'grad_norm': 4.2551727294921875, 'learning_rate': 5.8170397829712485e-05, 'epoch': 3.02}\n", + "{'loss': 1.1148, 'grad_norm': 4.442202568054199, 'learning_rate': 5.765750496516547e-05, 'epoch': 3.03}\n", + "{'loss': 1.2852, 'grad_norm': 5.140079021453857, 'learning_rate': 5.714378564496901e-05, 'epoch': 3.05}\n", + "{'loss': 1.2086, 'grad_norm': 5.270480632781982, 'learning_rate': 5.6629295313583974e-05, 'epoch': 3.07}\n", + "{'loss': 1.1824, 'grad_norm': 5.192230224609375, 'learning_rate': 5.611408949868457e-05, 'epoch': 3.09}\n", + "{'loss': 1.2875, 'grad_norm': 5.830446720123291, 'learning_rate': 5.559822380516539e-05, 'epoch': 3.11}\n", + "{'loss': 1.2162, 'grad_norm': 4.606627464294434, 'learning_rate': 5.5081753909140096e-05, 'epoch': 3.12}\n", + "{'loss': 1.3341, 'grad_norm': 5.547798156738281, 'learning_rate': 5.456473555193242e-05, 'epoch': 3.14}\n", + "{'loss': 1.2143, 'grad_norm': 5.579686641693115, 'learning_rate': 5.404722453406017e-05, 'epoch': 3.16}\n", + "{'loss': 1.2823, 'grad_norm': 6.129615783691406, 'learning_rate': 5.3529276709212816e-05, 'epoch': 3.18}\n", + "{'loss': 1.2582, 'grad_norm': 5.295398712158203, 'learning_rate': 5.30109479782233e-05, 'epoch': 3.2}\n", + "{'loss': 1.2438, 'grad_norm': 6.145551681518555, 'learning_rate': 5.249229428303486e-05, 'epoch': 3.21}\n", + "{'loss': 1.4372, 'grad_norm': 4.52131986618042, 'learning_rate': 5.197337160066331e-05, 'epoch': 3.23}\n", + "{'loss': 1.2346, 'grad_norm': 4.7772955894470215, 'learning_rate': 5.145423593715557e-05, 'epoch': 3.25}\n", + "{'loss': 1.1929, 'grad_norm': 5.328940391540527, 'learning_rate': 5.0934943321545115e-05, 'epoch': 3.27}\n", + "{'loss': 1.1731, 'grad_norm': 5.733246803283691, 'learning_rate': 5.041554979980486e-05, 'epoch': 3.28}\n", + "{'loss': 1.3325, 'grad_norm': 6.418582439422607, 'learning_rate': 4.9896111428798254e-05, 'epoch': 3.3}\n", + "{'loss': 1.3305, 'grad_norm': 4.787232398986816, 'learning_rate': 4.9376684270229254e-05, 'epoch': 3.32}\n", + "{'loss': 1.2982, 'grad_norm': 4.655210971832275, 'learning_rate': 4.8857324384591653e-05, 'epoch': 3.34}\n", + "{'loss': 1.2833, 'grad_norm': 4.85659122467041, 'learning_rate': 4.8338087825118675e-05, 'epoch': 3.36}\n", + "{'loss': 1.2835, 'grad_norm': 5.313413143157959, 'learning_rate': 4.781903063173321e-05, 'epoch': 3.37}\n", + "{'loss': 1.2001, 'grad_norm': 4.640489101409912, 'learning_rate': 4.730020882499964e-05, 'epoch': 3.39}\n", + "{'loss': 1.2597, 'grad_norm': 6.197988033294678, 'learning_rate': 4.678167840007767e-05, 'epoch': 3.41}\n", + "{'loss': 1.3514, 'grad_norm': 4.942805290222168, 'learning_rate': 4.626349532067879e-05, 'epoch': 3.43}\n", + "{'loss': 1.3118, 'grad_norm': 5.112833499908447, 'learning_rate': 4.574571551302647e-05, 'epoch': 3.44}\n", + "{'loss': 1.3232, 'grad_norm': 4.470940113067627, 'learning_rate': 4.522839485981994e-05, 'epoch': 3.46}\n", + "{'loss': 1.2533, 'grad_norm': 5.801645755767822, 'learning_rate': 4.471158919420312e-05, 'epoch': 3.48}\n", + "{'loss': 1.2343, 'grad_norm': 6.3296709060668945, 'learning_rate': 4.4195354293738484e-05, 'epoch': 3.5}\n", + "{'loss': 1.1995, 'grad_norm': 6.262467384338379, 'learning_rate': 4.367974587438733e-05, 'epoch': 3.52}\n", + "{'loss': 1.2744, 'grad_norm': 5.313882827758789, 'learning_rate': 4.316481958449634e-05, 'epoch': 3.53}\n", + "{'loss': 1.2366, 'grad_norm': 7.450092792510986, 'learning_rate': 4.2650630998791615e-05, 'epoch': 3.55}\n", + "{'loss': 1.3738, 'grad_norm': 4.7678680419921875, 'learning_rate': 4.213723561238074e-05, 'epoch': 3.57}\n", + "{'loss': 1.1538, 'grad_norm': 4.40903377532959, 'learning_rate': 4.162468883476319e-05, 'epoch': 3.59}\n", + "{'loss': 1.2502, 'grad_norm': 5.227618217468262, 'learning_rate': 4.111304598385018e-05, 'epoch': 3.61}\n", + "{'loss': 1.3061, 'grad_norm': 6.307828903198242, 'learning_rate': 4.060236227999441e-05, 'epoch': 3.62}\n", + "{'loss': 1.2667, 'grad_norm': 5.422544002532959, 'learning_rate': 4.0092692840030134e-05, 'epoch': 3.64}\n", + "{'loss': 1.2039, 'grad_norm': 7.9964141845703125, 'learning_rate': 3.9584092671324606e-05, 'epoch': 3.66}\n", + "{'loss': 1.3509, 'grad_norm': 7.364163875579834, 'learning_rate': 3.907661666584131e-05, 'epoch': 3.68}\n", + "{'loss': 1.3427, 'grad_norm': 6.175056457519531, 'learning_rate': 3.857031959421553e-05, 'epoch': 3.69}\n", + "{'loss': 1.345, 'grad_norm': 5.0636725425720215, 'learning_rate': 3.806525609984312e-05, 'epoch': 3.71}\n", + "{'loss': 1.1779, 'grad_norm': 5.742904186248779, 'learning_rate': 3.7561480692983006e-05, 'epoch': 3.73}\n", + "{'loss': 1.29, 'grad_norm': 5.6552276611328125, 'learning_rate': 3.705904774487396e-05, 'epoch': 3.75}\n", + "{'loss': 1.2704, 'grad_norm': 5.890940189361572, 'learning_rate': 3.655801148186655e-05, 'epoch': 3.77}\n", + "{'loss': 1.1811, 'grad_norm': 5.2217583656311035, 'learning_rate': 3.6058425979570485e-05, 'epoch': 3.78}\n", + "{'loss': 1.2768, 'grad_norm': 5.42200231552124, 'learning_rate': 3.556034515701852e-05, 'epoch': 3.8}\n", + "{'loss': 1.2891, 'grad_norm': 5.615239143371582, 'learning_rate': 3.506382277084696e-05, 'epoch': 3.82}\n", + "{'loss': 1.2401, 'grad_norm': 5.646175861358643, 'learning_rate': 3.4568912409493945e-05, 'epoch': 3.84}\n", + "{'loss': 1.0597, 'grad_norm': 5.7333197593688965, 'learning_rate': 3.4075667487415785e-05, 'epoch': 3.86}\n", + "{'loss': 1.1621, 'grad_norm': 5.321319580078125, 'learning_rate': 3.358414123932195e-05, 'epoch': 3.87}\n", + "{'loss': 1.2736, 'grad_norm': 4.852396011352539, 'learning_rate': 3.3094386714429724e-05, 'epoch': 3.89}\n", + "{'loss': 1.2597, 'grad_norm': 7.163392066955566, 'learning_rate': 3.2606456770738636e-05, 'epoch': 3.91}\n", + "{'loss': 1.1871, 'grad_norm': 5.611868381500244, 'learning_rate': 3.212040406932569e-05, 'epoch': 3.93}\n", + "{'loss': 1.0307, 'grad_norm': 4.783786296844482, 'learning_rate': 3.163628106866172e-05, 'epoch': 3.94}\n", + "{'loss': 1.4526, 'grad_norm': 5.691711902618408, 'learning_rate': 3.115414001894974e-05, 'epoch': 3.96}\n", + "{'loss': 1.3101, 'grad_norm': 5.280589580535889, 'learning_rate': 3.067403295648566e-05, 'epoch': 3.98}\n", + "{'loss': 1.0932, 'grad_norm': 4.22761869430542, 'learning_rate': 3.019601169804216e-05, 'epoch': 4.0}\n", + " 67%|██████████████████████████ | 2240/3360 [49:01<23:24, 1.25s/it][INFO|trainer.py:3788] 2024-07-04 11:59:10,533 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 11:59:10,533 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 11:59:10,533 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-2240\n", + "[INFO|configuration_utils.py:733] 2024-07-04 11:59:13,447 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 11:59:13,448 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 11:59:13,487 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-2240/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 11:59:13,487 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-2240/special_tokens_map.json\n", + "{'loss': 1.1326, 'grad_norm': 3.8860654830932617, 'learning_rate': 2.9720127835276256e-05, 'epoch': 4.02}\n", + "{'loss': 1.0202, 'grad_norm': 5.21559476852417, 'learning_rate': 2.9246432729161055e-05, 'epoch': 4.03}\n", + "{'loss': 1.0934, 'grad_norm': 5.658751964569092, 'learning_rate': 2.8774977504442647e-05, 'epoch': 4.05}\n", + "{'loss': 0.9934, 'grad_norm': 5.090124130249023, 'learning_rate': 2.8305813044122097e-05, 'epoch': 4.07}\n", + "{'loss': 1.0309, 'grad_norm': 5.851395606994629, 'learning_rate': 2.7838989983964065e-05, 'epoch': 4.09}\n", + "{'loss': 1.187, 'grad_norm': 4.703646659851074, 'learning_rate': 2.737455870703155e-05, 'epoch': 4.11}\n", + "{'loss': 0.9171, 'grad_norm': 4.95070219039917, 'learning_rate': 2.6912569338248315e-05, 'epoch': 4.12}\n", + "{'loss': 1.0232, 'grad_norm': 4.933461666107178, 'learning_rate': 2.645307173898901e-05, 'epoch': 4.14}\n", + "{'loss': 1.1005, 'grad_norm': 5.395535469055176, 'learning_rate': 2.5996115501697694e-05, 'epoch': 4.16}\n", + "{'loss': 0.9827, 'grad_norm': 4.670980453491211, 'learning_rate': 2.5541749944535554e-05, 'epoch': 4.18}\n", + "{'loss': 0.9969, 'grad_norm': 9.713501930236816, 'learning_rate': 2.5090024106057962e-05, 'epoch': 4.19}\n", + "{'loss': 1.0557, 'grad_norm': 5.423773288726807, 'learning_rate': 2.464098673992205e-05, 'epoch': 4.21}\n", + "{'loss': 0.9903, 'grad_norm': 5.628043174743652, 'learning_rate': 2.4194686309624663e-05, 'epoch': 4.23}\n", + "{'loss': 1.1397, 'grad_norm': 5.057712554931641, 'learning_rate': 2.3751170983272e-05, 'epoch': 4.25}\n", + "{'loss': 0.962, 'grad_norm': 4.844544410705566, 'learning_rate': 2.3310488628380757e-05, 'epoch': 4.27}\n", + "{'loss': 1.0187, 'grad_norm': 7.445083141326904, 'learning_rate': 2.2872686806712035e-05, 'epoch': 4.28}\n", + "{'loss': 1.0618, 'grad_norm': 4.87847900390625, 'learning_rate': 2.243781276913811e-05, 'epoch': 4.3}\n", + "{'loss': 0.9125, 'grad_norm': 5.181140899658203, 'learning_rate': 2.200591345054267e-05, 'epoch': 4.32}\n", + "{'loss': 1.125, 'grad_norm': 8.97202205657959, 'learning_rate': 2.157703546475539e-05, 'epoch': 4.34}\n", + "{'loss': 0.9747, 'grad_norm': 6.134432792663574, 'learning_rate': 2.115122509952085e-05, 'epoch': 4.36}\n", + "{'loss': 0.9803, 'grad_norm': 9.630309104919434, 'learning_rate': 2.0728528311502976e-05, 'epoch': 4.37}\n", + "{'loss': 1.0843, 'grad_norm': 5.363273620605469, 'learning_rate': 2.0308990721324927e-05, 'epoch': 4.39}\n", + "{'loss': 1.0764, 'grad_norm': 7.712973117828369, 'learning_rate': 1.989265760864542e-05, 'epoch': 4.41}\n", + "{'loss': 1.1397, 'grad_norm': 5.690403938293457, 'learning_rate': 1.947957390727185e-05, 'epoch': 4.43}\n", + "{'loss': 1.1258, 'grad_norm': 5.744186878204346, 'learning_rate': 1.906978420031059e-05, 'epoch': 4.44}\n", + "{'loss': 0.9438, 'grad_norm': 8.820874214172363, 'learning_rate': 1.8663332715355396e-05, 'epoch': 4.46}\n", + "{'loss': 1.132, 'grad_norm': 4.420164108276367, 'learning_rate': 1.8260263319713844e-05, 'epoch': 4.48}\n", + "{'loss': 1.0819, 'grad_norm': 5.586333751678467, 'learning_rate': 1.7860619515673033e-05, 'epoch': 4.5}\n", + "{'loss': 1.0571, 'grad_norm': 5.625140190124512, 'learning_rate': 1.746444443580433e-05, 'epoch': 4.52}\n", + "{'loss': 1.1021, 'grad_norm': 5.7560577392578125, 'learning_rate': 1.7071780838308288e-05, 'epoch': 4.53}\n", + "{'loss': 1.0531, 'grad_norm': 5.3450727462768555, 'learning_rate': 1.6682671102399805e-05, 'epoch': 4.55}\n", + "{'loss': 1.017, 'grad_norm': 6.27817440032959, 'learning_rate': 1.629715722373423e-05, 'epoch': 4.57}\n", + "{'loss': 1.0471, 'grad_norm': 5.72844934463501, 'learning_rate': 1.5915280809874932e-05, 'epoch': 4.59}\n", + "{'loss': 0.9309, 'grad_norm': 5.988643169403076, 'learning_rate': 1.553708307580265e-05, 'epoch': 4.61}\n", + "{'loss': 1.0538, 'grad_norm': 5.950584411621094, 'learning_rate': 1.5162604839467265e-05, 'epoch': 4.62}\n", + "{'loss': 1.0554, 'grad_norm': 4.944731712341309, 'learning_rate': 1.4791886517382413e-05, 'epoch': 4.64}\n", + "{'loss': 1.0857, 'grad_norm': 6.031637191772461, 'learning_rate': 1.4424968120263504e-05, 'epoch': 4.66}\n", + "{'loss': 1.0667, 'grad_norm': 5.933581352233887, 'learning_rate': 1.4061889248709343e-05, 'epoch': 4.68}\n", + "{'loss': 0.9942, 'grad_norm': 6.697149276733398, 'learning_rate': 1.370268908892825e-05, 'epoch': 4.69}\n", + "{'loss': 1.0146, 'grad_norm': 7.122743129730225, 'learning_rate': 1.3347406408508695e-05, 'epoch': 4.71}\n", + "{'loss': 0.9921, 'grad_norm': 4.69237756729126, 'learning_rate': 1.2996079552235263e-05, 'epoch': 4.73}\n", + "{'loss': 1.0017, 'grad_norm': 5.421998977661133, 'learning_rate': 1.264874643795021e-05, 'epoch': 4.75}\n", + "{'loss': 1.0102, 'grad_norm': 6.121133804321289, 'learning_rate': 1.230544455246101e-05, 'epoch': 4.77}\n", + "{'loss': 1.062, 'grad_norm': 5.060891151428223, 'learning_rate': 1.1966210947494583e-05, 'epoch': 4.78}\n", + "{'loss': 1.125, 'grad_norm': 4.1661529541015625, 'learning_rate': 1.1631082235698316e-05, 'epoch': 4.8}\n", + "{'loss': 0.9848, 'grad_norm': 6.484502792358398, 'learning_rate': 1.130009458668863e-05, 'epoch': 4.82}\n", + "{'loss': 0.9632, 'grad_norm': 5.1096086502075195, 'learning_rate': 1.097328372314721e-05, 'epoch': 4.84}\n", + "{'loss': 1.0435, 'grad_norm': 7.69472074508667, 'learning_rate': 1.0650684916965559e-05, 'epoch': 4.85}\n", + "{'loss': 1.1122, 'grad_norm': 6.654355525970459, 'learning_rate': 1.0332332985438248e-05, 'epoch': 4.87}\n", + "{'loss': 1.0846, 'grad_norm': 6.456166744232178, 'learning_rate': 1.0018262287505086e-05, 'epoch': 4.89}\n", + "{'loss': 1.1491, 'grad_norm': 5.556300163269043, 'learning_rate': 9.708506720042932e-06, 'epoch': 4.91}\n", + "{'loss': 1.0227, 'grad_norm': 5.921450614929199, 'learning_rate': 9.403099714207175e-06, 'epoch': 4.93}\n", + "{'loss': 1.13, 'grad_norm': 5.2472052574157715, 'learning_rate': 9.102074231823727e-06, 'epoch': 4.94}\n", + "{'loss': 1.1184, 'grad_norm': 6.798206806182861, 'learning_rate': 8.805462761831418e-06, 'epoch': 4.96}\n", + "{'loss': 1.1483, 'grad_norm': 6.1544647216796875, 'learning_rate': 8.513297316775625e-06, 'epoch': 4.98}\n", + "{'loss': 1.0966, 'grad_norm': 5.619192600250244, 'learning_rate': 8.225609429353187e-06, 'epoch': 5.0}\n", + " 83%|██████████████████████████████▊ | 2800/3360 [1:00:45<11:43, 1.26s/it][INFO|trainer.py:3788] 2024-07-04 12:10:55,158 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 12:10:55,158 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 12:10:55,158 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-2800\n", + "[INFO|configuration_utils.py:733] 2024-07-04 12:10:57,881 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 12:10:57,882 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 12:10:57,908 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-2800/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 12:10:57,908 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-2800/special_tokens_map.json\n", + "{'loss': 0.9401, 'grad_norm': 5.465145587921143, 'learning_rate': 7.942430149009161e-06, 'epoch': 5.02}\n", + "{'loss': 0.9053, 'grad_norm': 5.05084228515625, 'learning_rate': 7.663790038585793e-06, 'epoch': 5.03}\n", + "{'loss': 0.9659, 'grad_norm': 6.576834678649902, 'learning_rate': 7.389719171023857e-06, 'epoch': 5.05}\n", + "{'loss': 0.9701, 'grad_norm': 6.316474437713623, 'learning_rate': 7.1202471261170245e-06, 'epoch': 5.07}\n", + "{'loss': 0.9986, 'grad_norm': 5.003658771514893, 'learning_rate': 6.855402987319348e-06, 'epoch': 5.09}\n", + "{'loss': 1.0909, 'grad_norm': 5.66398286819458, 'learning_rate': 6.595215338606397e-06, 'epoch': 5.1}\n", + "{'loss': 0.8373, 'grad_norm': 5.239981174468994, 'learning_rate': 6.339712261390213e-06, 'epoch': 5.12}\n", + "{'loss': 1.0119, 'grad_norm': 5.830501079559326, 'learning_rate': 6.088921331488568e-06, 'epoch': 5.14}\n", + "{'loss': 0.89, 'grad_norm': 5.683416366577148, 'learning_rate': 5.8428696161488215e-06, 'epoch': 5.16}\n", + "{'loss': 0.8321, 'grad_norm': 5.024005889892578, 'learning_rate': 5.601583671126531e-06, 'epoch': 5.18}\n", + "{'loss': 0.9924, 'grad_norm': 5.65994930267334, 'learning_rate': 5.365089537819434e-06, 'epoch': 5.19}\n", + "{'loss': 0.982, 'grad_norm': 5.285236835479736, 'learning_rate': 5.133412740456806e-06, 'epoch': 5.21}\n", + "{'loss': 0.988, 'grad_norm': 6.087540149688721, 'learning_rate': 4.906578283344759e-06, 'epoch': 5.23}\n", + "{'loss': 1.0628, 'grad_norm': 5.564962863922119, 'learning_rate': 4.684610648167503e-06, 'epoch': 5.25}\n", + "{'loss': 0.9339, 'grad_norm': 5.311854362487793, 'learning_rate': 4.467533791345191e-06, 'epoch': 5.27}\n", + "{'loss': 0.9112, 'grad_norm': 6.383027076721191, 'learning_rate': 4.255371141448272e-06, 'epoch': 5.28}\n", + "{'loss': 0.8851, 'grad_norm': 5.323634147644043, 'learning_rate': 4.048145596668967e-06, 'epoch': 5.3}\n", + "{'loss': 0.9129, 'grad_norm': 5.474393844604492, 'learning_rate': 3.84587952234991e-06, 'epoch': 5.32}\n", + "{'loss': 0.8769, 'grad_norm': 6.140456676483154, 'learning_rate': 3.6485947485702832e-06, 'epoch': 5.34}\n", + "{'loss': 0.9177, 'grad_norm': 5.710687637329102, 'learning_rate': 3.4563125677897932e-06, 'epoch': 5.35}\n", + "{'loss': 0.9235, 'grad_norm': 6.829979419708252, 'learning_rate': 3.269053732550581e-06, 'epoch': 5.37}\n", + "{'loss': 0.9744, 'grad_norm': 6.83032751083374, 'learning_rate': 3.086838453237506e-06, 'epoch': 5.39}\n", + "{'loss': 0.7769, 'grad_norm': 5.491135597229004, 'learning_rate': 2.9096863958968268e-06, 'epoch': 5.41}\n", + "{'loss': 0.8412, 'grad_norm': 6.708963394165039, 'learning_rate': 2.737616680113758e-06, 'epoch': 5.43}\n", + "{'loss': 0.8511, 'grad_norm': 6.676459312438965, 'learning_rate': 2.570647876948895e-06, 'epoch': 5.44}\n", + "{'loss': 0.9549, 'grad_norm': 7.339512825012207, 'learning_rate': 2.408798006933882e-06, 'epoch': 5.46}\n", + "{'loss': 1.1274, 'grad_norm': 6.678201198577881, 'learning_rate': 2.252084538126542e-06, 'epoch': 5.48}\n", + "{'loss': 0.9263, 'grad_norm': 6.124770641326904, 'learning_rate': 2.100524384225555e-06, 'epoch': 5.5}\n", + "{'loss': 0.943, 'grad_norm': 7.016269207000732, 'learning_rate': 1.9541339027450256e-06, 'epoch': 5.52}\n", + "{'loss': 0.9571, 'grad_norm': 5.896731853485107, 'learning_rate': 1.8129288932490274e-06, 'epoch': 5.53}\n", + "{'loss': 0.8802, 'grad_norm': 5.532138347625732, 'learning_rate': 1.6769245956464396e-06, 'epoch': 5.55}\n", + "{'loss': 1.0613, 'grad_norm': 6.437876224517822, 'learning_rate': 1.5461356885461075e-06, 'epoch': 5.57}\n", + "{'loss': 0.957, 'grad_norm': 5.419349670410156, 'learning_rate': 1.4205762876726092e-06, 'epoch': 5.59}\n", + "{'loss': 1.0672, 'grad_norm': 6.222854137420654, 'learning_rate': 1.3002599443428243e-06, 'epoch': 5.6}\n", + "{'loss': 1.0228, 'grad_norm': 6.305788993835449, 'learning_rate': 1.1851996440033319e-06, 'epoch': 5.62}\n", + "{'loss': 0.8494, 'grad_norm': 6.640852928161621, 'learning_rate': 1.0754078048289374e-06, 'epoch': 5.64}\n", + "{'loss': 0.9589, 'grad_norm': 5.630051612854004, 'learning_rate': 9.708962763824048e-07, 'epoch': 5.66}\n", + "{'loss': 0.9514, 'grad_norm': 5.754588603973389, 'learning_rate': 8.716763383355864e-07, 'epoch': 5.68}\n", + "{'loss': 0.9896, 'grad_norm': 6.073591232299805, 'learning_rate': 7.777586992519959e-07, 'epoch': 5.69}\n", + "{'loss': 0.8798, 'grad_norm': 6.883085250854492, 'learning_rate': 6.891534954310885e-07, 'epoch': 5.71}\n", + "{'loss': 0.9749, 'grad_norm': 5.874994277954102, 'learning_rate': 6.058702898142643e-07, 'epoch': 5.73}\n", + "{'loss': 0.862, 'grad_norm': 5.205725193023682, 'learning_rate': 5.279180709527765e-07, 'epoch': 5.75}\n", + "{'loss': 1.0397, 'grad_norm': 6.112522602081299, 'learning_rate': 4.553052520375911e-07, 'epoch': 5.77}\n", + "{'loss': 0.8691, 'grad_norm': 6.450985431671143, 'learning_rate': 3.8803966999139684e-07, 'epoch': 5.78}\n", + "{'loss': 0.884, 'grad_norm': 5.139239311218262, 'learning_rate': 3.261285846227868e-07, 'epoch': 5.8}\n", + "{'loss': 0.8508, 'grad_norm': 6.213397979736328, 'learning_rate': 2.6957867784270787e-07, 'epoch': 5.82}\n", + "{'loss': 0.8554, 'grad_norm': 27.320371627807617, 'learning_rate': 2.1839605294330933e-07, 'epoch': 5.84}\n", + "{'loss': 1.036, 'grad_norm': 7.248013973236084, 'learning_rate': 1.725862339392259e-07, 'epoch': 5.85}\n", + "{'loss': 0.8262, 'grad_norm': 7.534704685211182, 'learning_rate': 1.3215416497138754e-07, 'epoch': 5.87}\n", + "{'loss': 1.0454, 'grad_norm': 5.765580654144287, 'learning_rate': 9.710420977340762e-08, 'epoch': 5.89}\n", + "{'loss': 0.8484, 'grad_norm': 5.267190456390381, 'learning_rate': 6.744015120061509e-08, 'epoch': 5.91}\n", + "{'loss': 0.9818, 'grad_norm': 6.66579008102417, 'learning_rate': 4.316519082179227e-08, 'epoch': 5.93}\n", + "{'loss': 0.8825, 'grad_norm': 4.743204593658447, 'learning_rate': 2.4281948573617874e-08, 'epoch': 5.94}\n", + "{'loss': 0.9975, 'grad_norm': 6.015940189361572, 'learning_rate': 1.0792462477909882e-08, 'epoch': 5.96}\n", + "{'loss': 0.9418, 'grad_norm': 5.236660957336426, 'learning_rate': 2.6981884216847884e-09, 'epoch': 5.98}\n", + "{'loss': 0.9678, 'grad_norm': 5.222324371337891, 'learning_rate': 0.0, 'epoch': 6.0}\n", + "100%|█████████████████████████████████████| 3360/3360 [1:12:30<00:00, 1.25s/it][INFO|trainer.py:3788] 2024-07-04 12:22:39,963 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 12:22:39,963 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 12:22:39,964 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-3360\n", + "[INFO|configuration_utils.py:733] 2024-07-04 12:22:42,459 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 12:22:42,460 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 12:22:42,487 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/checkpoint-3360/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 12:22:42,487 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/checkpoint-3360/special_tokens_map.json\n", + "[INFO|trainer.py:2383] 2024-07-04 12:22:42,628 >> \n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "{'train_runtime': 4358.4327, 'train_samples_per_second': 6.17, 'train_steps_per_second': 0.771, 'train_loss': 1.4797242326395852, 'epoch': 6.0}\n", + "100%|█████████████████████████████████████| 3360/3360 [1:12:33<00:00, 1.30s/it]\n", + "[INFO|trainer.py:3478] 2024-07-04 12:22:42,631 >> Saving model checkpoint to saves/qwen2-0.5b/lora/sft\n", + "[INFO|configuration_utils.py:733] 2024-07-04 12:22:43,255 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 12:22:43,256 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 12:22:43,285 >> tokenizer config file saved in saves/qwen2-0.5b/lora/sft/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 12:22:43,285 >> Special tokens file saved in saves/qwen2-0.5b/lora/sft/special_tokens_map.json\n", + "***** train metrics *****\n", + " epoch = 5.9973\n", + " total_flos = 4594110GF\n", + " train_loss = 1.4797\n", + " train_runtime = 1:12:38.43\n", + " train_samples_per_second = 6.17\n", + " train_steps_per_second = 0.771\n", + "Figure saved at: saves/qwen2-0.5b/lora/sft/training_loss.png\n", + "Figure saved at: saves/qwen2-0.5b/lora/sft/training_eval_loss.png\n", + "[INFO|trainer.py:3788] 2024-07-04 12:22:43,568 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 12:22:43,568 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 12:22:43,568 >> Batch size = 1\n", + "100%|███████████████████████████████████████████| 46/46 [00:01<00:00, 25.60it/s]\n", + "***** eval metrics *****\n", + " epoch = 5.9973\n", + " eval_loss = 2.5474\n", + " eval_runtime = 0:00:01.84\n", + " eval_samples_per_second = 24.959\n", + " eval_steps_per_second = 24.959\n", + "[INFO|modelcard.py:449] 2024-07-04 12:22:45,413 >> Dropping the following result as it does not have all the necessary fields:\n", + "{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: | 0.085 MB of 0.085 MB uploaded\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss ▂▁▂▄▇██\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime ▇█▃▁▇▄▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second ▂▁▆█▂▄▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second ▂▁▆█▂▄▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm ▄▁▁▃▂▃▃▃▃▄▃▄▂▃▄▆▃▂▄▄▅▆▄▆▅▅▃▄█▅▆█▆▆▅▅▆▇▇▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate ▂▄▅▇██████▇▇▇▇▇▆▆▆▆▅▅▅▄▄▄▃▃▃▃▂▂▂▂▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss █▇▇▇▆▆▆▆▆▆▅▅▆▅▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss 2.5474\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime 1.843\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second 24.959\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second 24.959\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: total_flos 4932888177414144.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch 5.99732\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step 3360\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm 5.22232\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss 0.9678\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 1.47972\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_runtime 4358.4327\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_samples_per_second 6.17\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_steps_per_second 0.771\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mqwen2_0.5b_lora_sft\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/u8sqhi0x\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 6 W&B file(s), 0 media file(s), 1 artifact file(s) and 0 other file(s)\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240704_111005-u8sqhi0x/logs\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n", + "CPU times: user 59.8 s, sys: 18.1 s, total: 1min 17s\n", + "Wall time: 1h 13min 51s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./scripts/tune-lf.sh config/qwen2_0.5b_lora_sft.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Current Directory:\n", + "/home/inflaton/code/projects/courses/llm-finetuning/llama-factory\n", + "07/04/2024 12:22:59 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: torch.bfloat16\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 12:23:00,122 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 12:23:00,122 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 12:23:00,122 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 12:23:00,122 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 12:23:00,122 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 12:23:00,122 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-04 12:23:00,234 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/04/2024 12:23:00 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/04/2024 12:23:00 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "07/04/2024 12:23:00 - INFO - llamafactory.data.loader - Loading dataset alpaca_mac.json...\n", + "Converting format of dataset (num_proc=16): 100%|█| 4528/4528 [00:00<00:00, 1573\n", + "Running tokenizer on dataset (num_proc=16): 100%|█| 4528/4528 [00:01<00:00, 3491\n", + "input_ids:\n", + "[151644, 872, 198, 5501, 14683, 279, 2701, 8453, 1467, 1119, 6364, 323, 3410, 1172, 279, 24531, 2213, 11, 4302, 770, 624, 35987, 102895, 99164, 100324, 100717, 100095, 99509, 1773, 151645, 198, 151644, 77091, 198, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "inputs:\n", + "<|im_start|>user\n", + "Please translate the following Chinese text into English and provide only the translated content, nothing else.\n", + "全仗着狐仙搭救。<|im_end|>\n", + "<|im_start|>assistant\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "label_ids:\n", + "[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "labels:\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "[INFO|configuration_utils.py:733] 2024-07-04 12:23:03,981 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 12:23:03,982 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:3556] 2024-07-04 12:23:04,016 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-04 12:23:06,701 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-04 12:23:06,704 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-04 12:26:42,040 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-04 12:26:42,040 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-04 12:26:42,765 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-04 12:26:42,766 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "07/04/2024 12:26:43 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.\n", + "07/04/2024 12:26:43 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/04/2024 12:26:43 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.\n", + "07/04/2024 12:26:43 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA\n", + "07/04/2024 12:26:43 - INFO - llamafactory.model.model_utils.misc - Found linear modules: q_proj,gate_proj,down_proj,k_proj,v_proj,up_proj,o_proj\n", + "07/04/2024 12:26:43 - INFO - llamafactory.model.loader - trainable params: 9,232,384 || all params: 1,552,946,688 || trainable%: 0.5945\n", + "[INFO|trainer.py:642] 2024-07-04 12:26:43,511 >> Using auto half precision backend\n", + "[INFO|trainer.py:2128] 2024-07-04 12:26:43,666 >> ***** Running training *****\n", + "[INFO|trainer.py:2129] 2024-07-04 12:26:43,666 >> Num examples = 4,482\n", + "[INFO|trainer.py:2130] 2024-07-04 12:26:43,666 >> Num Epochs = 6\n", + "[INFO|trainer.py:2131] 2024-07-04 12:26:43,666 >> Instantaneous batch size per device = 1\n", + "[INFO|trainer.py:2134] 2024-07-04 12:26:43,666 >> Total train batch size (w. parallel, distributed & accumulation) = 8\n", + "[INFO|trainer.py:2135] 2024-07-04 12:26:43,666 >> Gradient Accumulation steps = 8\n", + "[INFO|trainer.py:2136] 2024-07-04 12:26:43,666 >> Total optimization steps = 3,360\n", + "[INFO|trainer.py:2137] 2024-07-04 12:26:43,668 >> Number of trainable parameters = 9,232,384\n", + "[INFO|integration_utils.py:750] 2024-07-04 12:26:43,670 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33minflaton-sg\u001b[0m (\u001b[33minflaton-ai\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.4\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/home/inflaton/code/projects/courses/llm-finetuning/llama-factory/wandb/run-20240704_122645-mpc5sxtf\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mqwen2_1.5b_lora_sft\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/mpc5sxtf\u001b[0m\n", + "{'loss': 2.1612, 'grad_norm': 1.7288845777511597, 'learning_rate': 2.9761904761904763e-06, 'epoch': 0.02}\n", + "{'loss': 2.2871, 'grad_norm': 1.9337925910949707, 'learning_rate': 5.9523809523809525e-06, 'epoch': 0.04}\n", + "{'loss': 2.1455, 'grad_norm': 1.5129448175430298, 'learning_rate': 8.92857142857143e-06, 'epoch': 0.05}\n", + "{'loss': 2.1376, 'grad_norm': 2.9766852855682373, 'learning_rate': 1.1904761904761905e-05, 'epoch': 0.07}\n", + "{'loss': 2.2937, 'grad_norm': 1.413576602935791, 'learning_rate': 1.4880952380952381e-05, 'epoch': 0.09}\n", + "{'loss': 2.0076, 'grad_norm': 1.7012724876403809, 'learning_rate': 1.785714285714286e-05, 'epoch': 0.11}\n", + "{'loss': 2.1399, 'grad_norm': 1.679208517074585, 'learning_rate': 2.0833333333333336e-05, 'epoch': 0.12}\n", + "{'loss': 1.9036, 'grad_norm': 1.6296344995498657, 'learning_rate': 2.380952380952381e-05, 'epoch': 0.14}\n", + "{'loss': 2.0186, 'grad_norm': 2.1293675899505615, 'learning_rate': 2.6785714285714288e-05, 'epoch': 0.16}\n", + "{'loss': 1.9517, 'grad_norm': 1.4419277906417847, 'learning_rate': 2.9761904761904762e-05, 'epoch': 0.18}\n", + "{'loss': 1.979, 'grad_norm': 1.8672434091567993, 'learning_rate': 3.273809523809524e-05, 'epoch': 0.2}\n", + "{'loss': 1.9362, 'grad_norm': 1.3589439392089844, 'learning_rate': 3.571428571428572e-05, 'epoch': 0.21}\n", + "{'loss': 1.9264, 'grad_norm': 1.71873140335083, 'learning_rate': 3.8690476190476195e-05, 'epoch': 0.23}\n", + "{'loss': 1.9515, 'grad_norm': 2.2398152351379395, 'learning_rate': 4.166666666666667e-05, 'epoch': 0.25}\n", + "{'loss': 1.8163, 'grad_norm': 1.5651923418045044, 'learning_rate': 4.464285714285715e-05, 'epoch': 0.27}\n", + "{'loss': 1.792, 'grad_norm': 1.5333657264709473, 'learning_rate': 4.761904761904762e-05, 'epoch': 0.29}\n", + "{'loss': 1.9457, 'grad_norm': 1.448676347732544, 'learning_rate': 5.05952380952381e-05, 'epoch': 0.3}\n", + "{'loss': 1.7585, 'grad_norm': 1.8496005535125732, 'learning_rate': 5.3571428571428575e-05, 'epoch': 0.32}\n", + "{'loss': 1.8682, 'grad_norm': 2.332167387008667, 'learning_rate': 5.6547619047619046e-05, 'epoch': 0.34}\n", + "{'loss': 1.9775, 'grad_norm': 1.9075323343276978, 'learning_rate': 5.9523809523809524e-05, 'epoch': 0.36}\n", + "{'loss': 1.9233, 'grad_norm': 1.8132203817367554, 'learning_rate': 6.25e-05, 'epoch': 0.37}\n", + "{'loss': 1.8469, 'grad_norm': 2.514983892440796, 'learning_rate': 6.547619047619048e-05, 'epoch': 0.39}\n", + "{'loss': 1.8242, 'grad_norm': 2.0344440937042236, 'learning_rate': 6.845238095238096e-05, 'epoch': 0.41}\n", + "{'loss': 1.9965, 'grad_norm': 2.310185194015503, 'learning_rate': 7.142857142857143e-05, 'epoch': 0.43}\n", + "{'loss': 1.9004, 'grad_norm': 2.3513343334198, 'learning_rate': 7.440476190476191e-05, 'epoch': 0.45}\n", + "{'loss': 1.8188, 'grad_norm': 2.2934393882751465, 'learning_rate': 7.738095238095239e-05, 'epoch': 0.46}\n", + "{'loss': 1.8803, 'grad_norm': 2.8724184036254883, 'learning_rate': 8.035714285714287e-05, 'epoch': 0.48}\n", + "{'loss': 1.9181, 'grad_norm': 2.4238462448120117, 'learning_rate': 8.333333333333334e-05, 'epoch': 0.5}\n", + "{'loss': 1.6932, 'grad_norm': 1.5286414623260498, 'learning_rate': 8.630952380952382e-05, 'epoch': 0.52}\n", + "{'loss': 1.8331, 'grad_norm': 2.563647985458374, 'learning_rate': 8.92857142857143e-05, 'epoch': 0.54}\n", + "{'loss': 1.8539, 'grad_norm': 2.127699613571167, 'learning_rate': 9.226190476190478e-05, 'epoch': 0.55}\n", + "{'loss': 1.6796, 'grad_norm': 3.4179396629333496, 'learning_rate': 9.523809523809524e-05, 'epoch': 0.57}\n", + "{'loss': 1.7209, 'grad_norm': 2.492151975631714, 'learning_rate': 9.821428571428572e-05, 'epoch': 0.59}\n", + "{'loss': 1.7723, 'grad_norm': 2.3568859100341797, 'learning_rate': 9.999956828659095e-05, 'epoch': 0.61}\n", + "{'loss': 1.7839, 'grad_norm': 3.5560832023620605, 'learning_rate': 9.999471159635539e-05, 'epoch': 0.62}\n", + "{'loss': 1.7146, 'grad_norm': 1.712493658065796, 'learning_rate': 9.998445910004082e-05, 'epoch': 0.64}\n", + "{'loss': 1.8911, 'grad_norm': 2.824240207672119, 'learning_rate': 9.996881190417393e-05, 'epoch': 0.66}\n", + "{'loss': 1.8631, 'grad_norm': 2.2122113704681396, 'learning_rate': 9.994777169751806e-05, 'epoch': 0.68}\n", + "{'loss': 1.5738, 'grad_norm': 1.7466025352478027, 'learning_rate': 9.992134075089084e-05, 'epoch': 0.7}\n", + "{'loss': 1.7552, 'grad_norm': 2.581709623336792, 'learning_rate': 9.988952191691925e-05, 'epoch': 0.71}\n", + "{'loss': 1.808, 'grad_norm': 2.1387972831726074, 'learning_rate': 9.985231862973168e-05, 'epoch': 0.73}\n", + "{'loss': 1.8303, 'grad_norm': 1.7675608396530151, 'learning_rate': 9.980973490458728e-05, 'epoch': 0.75}\n", + "{'loss': 1.8013, 'grad_norm': 2.706218719482422, 'learning_rate': 9.976177533744261e-05, 'epoch': 0.77}\n", + "{'loss': 1.7443, 'grad_norm': 2.9387295246124268, 'learning_rate': 9.97084451044556e-05, 'epoch': 0.79}\n", + "{'loss': 1.6509, 'grad_norm': 1.6503076553344727, 'learning_rate': 9.964974996142698e-05, 'epoch': 0.8}\n", + "{'loss': 1.722, 'grad_norm': 2.0305140018463135, 'learning_rate': 9.958569624317893e-05, 'epoch': 0.82}\n", + "{'loss': 1.7625, 'grad_norm': 2.8122429847717285, 'learning_rate': 9.951629086287151e-05, 'epoch': 0.84}\n", + "{'loss': 1.7194, 'grad_norm': 2.0110862255096436, 'learning_rate': 9.944154131125642e-05, 'epoch': 0.86}\n", + "{'loss': 1.7894, 'grad_norm': 1.7363322973251343, 'learning_rate': 9.936145565586871e-05, 'epoch': 0.87}\n", + "{'loss': 1.9447, 'grad_norm': 1.8065259456634521, 'learning_rate': 9.927604254015585e-05, 'epoch': 0.89}\n", + "{'loss': 1.8639, 'grad_norm': 1.8963510990142822, 'learning_rate': 9.918531118254507e-05, 'epoch': 0.91}\n", + "{'loss': 1.7336, 'grad_norm': 2.30542254447937, 'learning_rate': 9.90892713754483e-05, 'epoch': 0.93}\n", + "{'loss': 1.7705, 'grad_norm': 2.9846692085266113, 'learning_rate': 9.898793348420536e-05, 'epoch': 0.95}\n", + "{'loss': 1.7884, 'grad_norm': 2.1550045013427734, 'learning_rate': 9.888130844596524e-05, 'epoch': 0.96}\n", + "{'loss': 1.7428, 'grad_norm': 2.1323790550231934, 'learning_rate': 9.876940776850569e-05, 'epoch': 0.98}\n", + "{'loss': 1.7183, 'grad_norm': 1.8198726177215576, 'learning_rate': 9.865224352899119e-05, 'epoch': 1.0}\n", + " 17%|██████▎ | 560/3360 [15:31<1:20:24, 1.72s/it][INFO|trainer.py:3788] 2024-07-04 12:42:20,584 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 12:42:20,584 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 12:42:20,585 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-560\n", + "[INFO|configuration_utils.py:733] 2024-07-04 12:42:23,808 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 12:42:23,809 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 12:42:23,882 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-560/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 12:42:23,883 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-560/special_tokens_map.json\n", + "{'loss': 1.6305, 'grad_norm': 1.8726240396499634, 'learning_rate': 9.852982837266955e-05, 'epoch': 1.02}\n", + "{'loss': 1.4509, 'grad_norm': 2.8097503185272217, 'learning_rate': 9.840217551150706e-05, 'epoch': 1.04}\n", + "{'loss': 1.6345, 'grad_norm': 2.0100064277648926, 'learning_rate': 9.826929872276255e-05, 'epoch': 1.05}\n", + "{'loss': 1.5736, 'grad_norm': 2.456465482711792, 'learning_rate': 9.81312123475006e-05, 'epoch': 1.07}\n", + "{'loss': 1.5363, 'grad_norm': 2.7739548683166504, 'learning_rate': 9.798793128904356e-05, 'epoch': 1.09}\n", + "{'loss': 1.5754, 'grad_norm': 2.8599655628204346, 'learning_rate': 9.78394710113631e-05, 'epoch': 1.11}\n", + "{'loss': 1.5728, 'grad_norm': 2.2901456356048584, 'learning_rate': 9.768584753741134e-05, 'epoch': 1.12}\n", + "{'loss': 1.5632, 'grad_norm': 3.6802914142608643, 'learning_rate': 9.752707744739145e-05, 'epoch': 1.14}\n", + "{'loss': 1.5927, 'grad_norm': 2.5885791778564453, 'learning_rate': 9.736317787696816e-05, 'epoch': 1.16}\n", + "{'loss': 1.4571, 'grad_norm': 2.383814573287964, 'learning_rate': 9.719416651541839e-05, 'epoch': 1.18}\n", + "{'loss': 1.424, 'grad_norm': 2.032453775405884, 'learning_rate': 9.702006160372209e-05, 'epoch': 1.2}\n", + "{'loss': 1.5577, 'grad_norm': 2.8879408836364746, 'learning_rate': 9.684088193259355e-05, 'epoch': 1.21}\n", + "{'loss': 1.5083, 'grad_norm': 2.9004592895507812, 'learning_rate': 9.665664684045333e-05, 'epoch': 1.23}\n", + "{'loss': 1.5696, 'grad_norm': 3.4651644229888916, 'learning_rate': 9.646737621134112e-05, 'epoch': 1.25}\n", + "{'loss': 1.542, 'grad_norm': 3.6657605171203613, 'learning_rate': 9.627309047276974e-05, 'epoch': 1.27}\n", + "{'loss': 1.6975, 'grad_norm': 3.4882619380950928, 'learning_rate': 9.607381059352038e-05, 'epoch': 1.29}\n", + "{'loss': 1.6179, 'grad_norm': 2.73240327835083, 'learning_rate': 9.586955808137958e-05, 'epoch': 1.3}\n", + "{'loss': 1.6236, 'grad_norm': 2.60489559173584, 'learning_rate': 9.566035498081784e-05, 'epoch': 1.32}\n", + "{'loss': 1.5901, 'grad_norm': 3.45670223236084, 'learning_rate': 9.544622387061055e-05, 'epoch': 1.34}\n", + "{'loss': 1.3816, 'grad_norm': 3.3906328678131104, 'learning_rate': 9.522718786140097e-05, 'epoch': 1.36}\n", + "{'loss': 1.6149, 'grad_norm': 3.6723110675811768, 'learning_rate': 9.500327059320606e-05, 'epoch': 1.37}\n", + "{'loss': 1.4588, 'grad_norm': 4.5224103927612305, 'learning_rate': 9.477449623286505e-05, 'epoch': 1.39}\n", + "{'loss': 1.3431, 'grad_norm': 2.5576796531677246, 'learning_rate': 9.454088947143116e-05, 'epoch': 1.41}\n", + "{'loss': 1.6278, 'grad_norm': 3.344188690185547, 'learning_rate': 9.430247552150673e-05, 'epoch': 1.43}\n", + "{'loss': 1.5137, 'grad_norm': 3.4474005699157715, 'learning_rate': 9.405928011452211e-05, 'epoch': 1.45}\n", + "{'loss': 1.4911, 'grad_norm': 2.6104114055633545, 'learning_rate': 9.381132949795861e-05, 'epoch': 1.46}\n", + "{'loss': 1.6567, 'grad_norm': 3.090139150619507, 'learning_rate': 9.35586504325155e-05, 'epoch': 1.48}\n", + "{'loss': 1.5008, 'grad_norm': 3.6463866233825684, 'learning_rate': 9.330127018922194e-05, 'epoch': 1.5}\n", + "{'loss': 1.4248, 'grad_norm': 2.3963379859924316, 'learning_rate': 9.303921654649362e-05, 'epoch': 1.52}\n", + "{'loss': 1.6043, 'grad_norm': 3.4818763732910156, 'learning_rate': 9.277251778713474e-05, 'epoch': 1.54}\n", + "{'loss': 1.5517, 'grad_norm': 3.180640697479248, 'learning_rate': 9.250120269528546e-05, 'epoch': 1.55}\n", + "{'loss': 1.4711, 'grad_norm': 2.7267000675201416, 'learning_rate': 9.22253005533154e-05, 'epoch': 1.57}\n", + "{'loss': 1.5511, 'grad_norm': 3.386282444000244, 'learning_rate': 9.194484113866313e-05, 'epoch': 1.59}\n", + "{'loss': 1.6975, 'grad_norm': 2.707632064819336, 'learning_rate': 9.165985472062246e-05, 'epoch': 1.61}\n", + "{'loss': 1.4396, 'grad_norm': 2.970285177230835, 'learning_rate': 9.137037205707552e-05, 'epoch': 1.62}\n", + "{'loss': 1.5347, 'grad_norm': 2.7082931995391846, 'learning_rate': 9.107642439117321e-05, 'epoch': 1.64}\n", + "{'loss': 1.5446, 'grad_norm': 2.947016716003418, 'learning_rate': 9.077804344796302e-05, 'epoch': 1.66}\n", + "{'loss': 1.5401, 'grad_norm': 2.4926042556762695, 'learning_rate': 9.04752614309652e-05, 'epoch': 1.68}\n", + "{'loss': 1.479, 'grad_norm': 3.50626802444458, 'learning_rate': 9.01681110186971e-05, 'epoch': 1.7}\n", + "{'loss': 1.5107, 'grad_norm': 4.556169509887695, 'learning_rate': 8.985662536114613e-05, 'epoch': 1.71}\n", + "{'loss': 1.473, 'grad_norm': 2.4575538635253906, 'learning_rate': 8.954083807619208e-05, 'epoch': 1.73}\n", + "{'loss': 1.6125, 'grad_norm': 3.063415765762329, 'learning_rate': 8.922078324597879e-05, 'epoch': 1.75}\n", + "{'loss': 1.5893, 'grad_norm': 2.45483660697937, 'learning_rate': 8.889649541323574e-05, 'epoch': 1.77}\n", + "{'loss': 1.4993, 'grad_norm': 3.031142473220825, 'learning_rate': 8.856800957755e-05, 'epoch': 1.78}\n", + "{'loss': 1.5025, 'grad_norm': 2.9005496501922607, 'learning_rate': 8.823536119158864e-05, 'epoch': 1.8}\n", + "{'loss': 1.4725, 'grad_norm': 2.9155054092407227, 'learning_rate': 8.789858615727265e-05, 'epoch': 1.82}\n", + "{'loss': 1.4313, 'grad_norm': 2.5998966693878174, 'learning_rate': 8.755772082190194e-05, 'epoch': 1.84}\n", + "{'loss': 1.5647, 'grad_norm': 2.5580039024353027, 'learning_rate': 8.721280197423258e-05, 'epoch': 1.86}\n", + "{'loss': 1.4349, 'grad_norm': 3.395029067993164, 'learning_rate': 8.68638668405062e-05, 'epoch': 1.87}\n", + "{'loss': 1.5214, 'grad_norm': 2.8961341381073, 'learning_rate': 8.651095308043232e-05, 'epoch': 1.89}\n", + "{'loss': 1.6206, 'grad_norm': 3.4450645446777344, 'learning_rate': 8.61540987831238e-05, 'epoch': 1.91}\n", + "{'loss': 1.6429, 'grad_norm': 3.4198362827301025, 'learning_rate': 8.579334246298593e-05, 'epoch': 1.93}\n", + "{'loss': 1.5473, 'grad_norm': 2.9955196380615234, 'learning_rate': 8.542872305555978e-05, 'epoch': 1.95}\n", + "{'loss': 1.4405, 'grad_norm': 2.7997260093688965, 'learning_rate': 8.50602799133199e-05, 'epoch': 1.96}\n", + "{'loss': 1.5382, 'grad_norm': 2.4689786434173584, 'learning_rate': 8.468805280142709e-05, 'epoch': 1.98}\n", + "{'loss': 1.5378, 'grad_norm': 3.09759783744812, 'learning_rate': 8.43120818934367e-05, 'epoch': 2.0}\n", + " 33%|████████████▎ | 1120/3360 [32:07<1:05:51, 1.76s/it][INFO|trainer.py:3788] 2024-07-04 12:58:56,606 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 12:58:56,606 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 12:58:56,606 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-1120\n", + "[INFO|configuration_utils.py:733] 2024-07-04 12:58:59,895 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 12:58:59,896 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 12:58:59,945 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1120/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 12:58:59,945 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1120/special_tokens_map.json\n", + "{'loss': 1.2363, 'grad_norm': 3.1393024921417236, 'learning_rate': 8.393240776696274e-05, 'epoch': 2.02}\n", + "{'loss': 1.1161, 'grad_norm': 2.708930253982544, 'learning_rate': 8.354907139929851e-05, 'epoch': 2.03}\n", + "{'loss': 1.1975, 'grad_norm': 4.3620429039001465, 'learning_rate': 8.316211416299397e-05, 'epoch': 2.05}\n", + "{'loss': 1.1225, 'grad_norm': 3.3463101387023926, 'learning_rate': 8.27715778213905e-05, 'epoch': 2.07}\n", + "{'loss': 1.0548, 'grad_norm': 2.8970718383789062, 'learning_rate': 8.237750452411353e-05, 'epoch': 2.09}\n", + "{'loss': 1.1526, 'grad_norm': 2.99774432182312, 'learning_rate': 8.197993680252334e-05, 'epoch': 2.11}\n", + "{'loss': 1.3093, 'grad_norm': 4.249015808105469, 'learning_rate': 8.157891756512488e-05, 'epoch': 2.12}\n", + "{'loss': 1.1306, 'grad_norm': 3.889763593673706, 'learning_rate': 8.117449009293668e-05, 'epoch': 2.14}\n", + "{'loss': 1.1286, 'grad_norm': 4.101832866668701, 'learning_rate': 8.076669803481965e-05, 'epoch': 2.16}\n", + "{'loss': 1.1271, 'grad_norm': 4.3527703285217285, 'learning_rate': 8.035558540276618e-05, 'epoch': 2.18}\n", + "{'loss': 1.1593, 'grad_norm': 3.5413858890533447, 'learning_rate': 7.994119656715002e-05, 'epoch': 2.2}\n", + "{'loss': 1.1824, 'grad_norm': 5.094357490539551, 'learning_rate': 7.952357625193749e-05, 'epoch': 2.21}\n", + "{'loss': 1.2347, 'grad_norm': 5.2239089012146, 'learning_rate': 7.91027695298606e-05, 'epoch': 2.23}\n", + "{'loss': 1.2285, 'grad_norm': 5.532718658447266, 'learning_rate': 7.86788218175523e-05, 'epoch': 2.25}\n", + "{'loss': 1.1147, 'grad_norm': 3.8143270015716553, 'learning_rate': 7.8251778870645e-05, 'epoch': 2.27}\n", + "{'loss': 1.1478, 'grad_norm': 4.406189441680908, 'learning_rate': 7.782168677883206e-05, 'epoch': 2.28}\n", + "{'loss': 1.1846, 'grad_norm': 3.269481658935547, 'learning_rate': 7.738859196089358e-05, 'epoch': 2.3}\n", + "{'loss': 1.2015, 'grad_norm': 4.366032123565674, 'learning_rate': 7.695254115968648e-05, 'epoch': 2.32}\n", + "{'loss': 1.3038, 'grad_norm': 3.7871077060699463, 'learning_rate': 7.651358143709972e-05, 'epoch': 2.34}\n", + "{'loss': 1.2532, 'grad_norm': 3.805539846420288, 'learning_rate': 7.60717601689749e-05, 'epoch': 2.36}\n", + "{'loss': 1.2044, 'grad_norm': 4.302929401397705, 'learning_rate': 7.562712503999327e-05, 'epoch': 2.37}\n", + "{'loss': 1.3852, 'grad_norm': 4.319093227386475, 'learning_rate': 7.517972403852905e-05, 'epoch': 2.39}\n", + "{'loss': 1.2647, 'grad_norm': 3.8114326000213623, 'learning_rate': 7.472960545147038e-05, 'epoch': 2.41}\n", + "{'loss': 1.1138, 'grad_norm': 4.816274166107178, 'learning_rate': 7.427681785900761e-05, 'epoch': 2.43}\n", + "{'loss': 1.1797, 'grad_norm': 3.7659311294555664, 'learning_rate': 7.382141012939034e-05, 'epoch': 2.45}\n", + "{'loss': 1.1566, 'grad_norm': 3.777496337890625, 'learning_rate': 7.33634314136531e-05, 'epoch': 2.46}\n", + "{'loss': 1.2235, 'grad_norm': 3.779813051223755, 'learning_rate': 7.290293114031061e-05, 'epoch': 2.48}\n", + "{'loss': 1.3044, 'grad_norm': 4.243238925933838, 'learning_rate': 7.243995901002312e-05, 'epoch': 2.5}\n", + "{'loss': 1.1993, 'grad_norm': 3.7302756309509277, 'learning_rate': 7.197456499023225e-05, 'epoch': 2.52}\n", + "{'loss': 1.1955, 'grad_norm': 3.837207555770874, 'learning_rate': 7.150679930976825e-05, 'epoch': 2.53}\n", + "{'loss': 1.2282, 'grad_norm': 4.182308673858643, 'learning_rate': 7.103671245342887e-05, 'epoch': 2.55}\n", + "{'loss': 1.1068, 'grad_norm': 4.697420120239258, 'learning_rate': 7.056435515653059e-05, 'epoch': 2.57}\n", + "{'loss': 1.3001, 'grad_norm': 5.241019248962402, 'learning_rate': 7.008977839943299e-05, 'epoch': 2.59}\n", + "{'loss': 1.1734, 'grad_norm': 5.618649959564209, 'learning_rate': 6.961303340203653e-05, 'epoch': 2.61}\n", + "{'loss': 1.2205, 'grad_norm': 3.501143455505371, 'learning_rate': 6.91341716182545e-05, 'epoch': 2.62}\n", + "{'loss': 1.2196, 'grad_norm': 2.823162317276001, 'learning_rate': 6.86532447304597e-05, 'epoch': 2.64}\n", + "{'loss': 1.1884, 'grad_norm': 3.8134286403656006, 'learning_rate': 6.817030464390656e-05, 'epoch': 2.66}\n", + "{'loss': 1.296, 'grad_norm': 3.9806973934173584, 'learning_rate': 6.768540348112907e-05, 'epoch': 2.68}\n", + "{'loss': 1.0861, 'grad_norm': 5.336892604827881, 'learning_rate': 6.719859357631535e-05, 'epoch': 2.7}\n", + "{'loss': 1.1123, 'grad_norm': 5.413362503051758, 'learning_rate': 6.670992746965938e-05, 'epoch': 2.71}\n", + "{'loss': 1.3405, 'grad_norm': 3.942927122116089, 'learning_rate': 6.621945790169036e-05, 'epoch': 2.73}\n", + "{'loss': 1.2739, 'grad_norm': 3.9731507301330566, 'learning_rate': 6.572723780758069e-05, 'epoch': 2.75}\n", + "{'loss': 1.2215, 'grad_norm': 3.9058139324188232, 'learning_rate': 6.523332031143272e-05, 'epoch': 2.77}\n", + "{'loss': 1.231, 'grad_norm': 3.7157390117645264, 'learning_rate': 6.473775872054521e-05, 'epoch': 2.78}\n", + "{'loss': 1.0667, 'grad_norm': 3.2383055686950684, 'learning_rate': 6.424060651966007e-05, 'epoch': 2.8}\n", + "{'loss': 1.1742, 'grad_norm': 3.6972646713256836, 'learning_rate': 6.374191736518974e-05, 'epoch': 2.82}\n", + "{'loss': 1.2108, 'grad_norm': 3.783498764038086, 'learning_rate': 6.324174507942637e-05, 'epoch': 2.84}\n", + "{'loss': 1.1861, 'grad_norm': 4.8546037673950195, 'learning_rate': 6.274014364473274e-05, 'epoch': 2.86}\n", + "{'loss': 1.364, 'grad_norm': 3.1121954917907715, 'learning_rate': 6.22371671977162e-05, 'epoch': 2.87}\n", + "{'loss': 1.2202, 'grad_norm': 4.141942024230957, 'learning_rate': 6.173287002338577e-05, 'epoch': 2.89}\n", + "{'loss': 1.2125, 'grad_norm': 4.195278167724609, 'learning_rate': 6.122730654929334e-05, 'epoch': 2.91}\n", + "{'loss': 1.1392, 'grad_norm': 3.6065282821655273, 'learning_rate': 6.072053133965938e-05, 'epoch': 2.93}\n", + "{'loss': 1.3093, 'grad_norm': 3.8997342586517334, 'learning_rate': 6.021259908948402e-05, 'epoch': 2.95}\n", + "{'loss': 1.258, 'grad_norm': 4.212363243103027, 'learning_rate': 5.970356461864391e-05, 'epoch': 2.96}\n", + "{'loss': 1.1774, 'grad_norm': 4.735218524932861, 'learning_rate': 5.919348286597569e-05, 'epoch': 2.98}\n", + "{'loss': 1.2808, 'grad_norm': 3.88008713722229, 'learning_rate': 5.868240888334653e-05, 'epoch': 3.0}\n", + " 50%|███████████████████▌ | 1680/3360 [48:42<49:15, 1.76s/it][INFO|trainer.py:3788] 2024-07-04 13:15:31,424 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 13:15:31,425 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 13:15:31,425 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-1680\n", + "[INFO|configuration_utils.py:733] 2024-07-04 13:15:34,788 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 13:15:34,789 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 13:15:34,839 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1680/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 13:15:34,839 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1680/special_tokens_map.json\n", + "{'loss': 1.1006, 'grad_norm': 3.581298589706421, 'learning_rate': 5.8170397829712485e-05, 'epoch': 3.02}\n", + "{'loss': 0.7853, 'grad_norm': 4.149472713470459, 'learning_rate': 5.765750496516547e-05, 'epoch': 3.03}\n", + "{'loss': 0.9606, 'grad_norm': 4.768033027648926, 'learning_rate': 5.714378564496901e-05, 'epoch': 3.05}\n", + "{'loss': 0.8799, 'grad_norm': 3.7473530769348145, 'learning_rate': 5.6629295313583974e-05, 'epoch': 3.07}\n", + "{'loss': 0.8164, 'grad_norm': 3.66397762298584, 'learning_rate': 5.611408949868457e-05, 'epoch': 3.09}\n", + "{'loss': 0.8902, 'grad_norm': 5.061825275421143, 'learning_rate': 5.559822380516539e-05, 'epoch': 3.11}\n", + "{'loss': 0.8534, 'grad_norm': 4.06561803817749, 'learning_rate': 5.5081753909140096e-05, 'epoch': 3.12}\n", + "{'loss': 0.9668, 'grad_norm': 4.875536918640137, 'learning_rate': 5.456473555193242e-05, 'epoch': 3.14}\n", + "{'loss': 0.8607, 'grad_norm': 6.051300048828125, 'learning_rate': 5.404722453406017e-05, 'epoch': 3.16}\n", + "{'loss': 0.9096, 'grad_norm': 5.530092716217041, 'learning_rate': 5.3529276709212816e-05, 'epoch': 3.18}\n", + "{'loss': 0.9201, 'grad_norm': 6.524964809417725, 'learning_rate': 5.30109479782233e-05, 'epoch': 3.2}\n", + "{'loss': 0.856, 'grad_norm': 4.842297554016113, 'learning_rate': 5.249229428303486e-05, 'epoch': 3.21}\n", + "{'loss': 1.0534, 'grad_norm': 3.963986396789551, 'learning_rate': 5.197337160066331e-05, 'epoch': 3.23}\n", + "{'loss': 0.8642, 'grad_norm': 4.481607437133789, 'learning_rate': 5.145423593715557e-05, 'epoch': 3.25}\n", + "{'loss': 0.8856, 'grad_norm': 3.9990179538726807, 'learning_rate': 5.0934943321545115e-05, 'epoch': 3.27}\n", + "{'loss': 0.7925, 'grad_norm': 4.209486484527588, 'learning_rate': 5.041554979980486e-05, 'epoch': 3.28}\n", + "{'loss': 0.9874, 'grad_norm': 4.624832630157471, 'learning_rate': 4.9896111428798254e-05, 'epoch': 3.3}\n", + "{'loss': 0.9581, 'grad_norm': 5.0234785079956055, 'learning_rate': 4.9376684270229254e-05, 'epoch': 3.32}\n", + "{'loss': 0.9273, 'grad_norm': 4.156904220581055, 'learning_rate': 4.8857324384591653e-05, 'epoch': 3.34}\n", + "{'loss': 0.8929, 'grad_norm': 4.292726516723633, 'learning_rate': 4.8338087825118675e-05, 'epoch': 3.36}\n", + "{'loss': 0.9584, 'grad_norm': 5.206954002380371, 'learning_rate': 4.781903063173321e-05, 'epoch': 3.37}\n", + "{'loss': 0.8548, 'grad_norm': 4.075423717498779, 'learning_rate': 4.730020882499964e-05, 'epoch': 3.39}\n", + "{'loss': 0.9083, 'grad_norm': 5.6302008628845215, 'learning_rate': 4.678167840007767e-05, 'epoch': 3.41}\n", + "{'loss': 0.9967, 'grad_norm': 4.765602111816406, 'learning_rate': 4.626349532067879e-05, 'epoch': 3.43}\n", + "{'loss': 0.9439, 'grad_norm': 4.012918949127197, 'learning_rate': 4.574571551302647e-05, 'epoch': 3.44}\n", + "{'loss': 0.9758, 'grad_norm': 3.995499849319458, 'learning_rate': 4.522839485981994e-05, 'epoch': 3.46}\n", + "{'loss': 0.9056, 'grad_norm': 4.168616771697998, 'learning_rate': 4.471158919420312e-05, 'epoch': 3.48}\n", + "{'loss': 0.8866, 'grad_norm': 5.359450817108154, 'learning_rate': 4.4195354293738484e-05, 'epoch': 3.5}\n", + "{'loss': 0.8474, 'grad_norm': 5.709634304046631, 'learning_rate': 4.367974587438733e-05, 'epoch': 3.52}\n", + "{'loss': 0.9325, 'grad_norm': 6.928687572479248, 'learning_rate': 4.316481958449634e-05, 'epoch': 3.53}\n", + "{'loss': 0.8947, 'grad_norm': 9.245586395263672, 'learning_rate': 4.2650630998791615e-05, 'epoch': 3.55}\n", + "{'loss': 1.0068, 'grad_norm': 7.456272602081299, 'learning_rate': 4.213723561238074e-05, 'epoch': 3.57}\n", + "{'loss': 0.8202, 'grad_norm': 3.894721746444702, 'learning_rate': 4.162468883476319e-05, 'epoch': 3.59}\n", + "{'loss': 0.8858, 'grad_norm': 4.249356269836426, 'learning_rate': 4.111304598385018e-05, 'epoch': 3.61}\n", + "{'loss': 0.9275, 'grad_norm': 6.780489921569824, 'learning_rate': 4.060236227999441e-05, 'epoch': 3.62}\n", + "{'loss': 0.8648, 'grad_norm': 5.042501449584961, 'learning_rate': 4.0092692840030134e-05, 'epoch': 3.64}\n", + "{'loss': 0.8905, 'grad_norm': 4.697298526763916, 'learning_rate': 3.9584092671324606e-05, 'epoch': 3.66}\n", + "{'loss': 0.9487, 'grad_norm': 8.913374900817871, 'learning_rate': 3.907661666584131e-05, 'epoch': 3.68}\n", + "{'loss': 0.9711, 'grad_norm': 6.1330885887146, 'learning_rate': 3.857031959421553e-05, 'epoch': 3.69}\n", + "{'loss': 0.9569, 'grad_norm': 4.3145599365234375, 'learning_rate': 3.806525609984312e-05, 'epoch': 3.71}\n", + "{'loss': 0.8455, 'grad_norm': 5.199124813079834, 'learning_rate': 3.7561480692983006e-05, 'epoch': 3.73}\n", + "{'loss': 0.9282, 'grad_norm': 4.841589450836182, 'learning_rate': 3.705904774487396e-05, 'epoch': 3.75}\n", + "{'loss': 0.9613, 'grad_norm': 5.256564140319824, 'learning_rate': 3.655801148186655e-05, 'epoch': 3.77}\n", + "{'loss': 0.832, 'grad_norm': 4.737905025482178, 'learning_rate': 3.6058425979570485e-05, 'epoch': 3.78}\n", + "{'loss': 0.9194, 'grad_norm': 4.6155524253845215, 'learning_rate': 3.556034515701852e-05, 'epoch': 3.8}\n", + "{'loss': 0.9149, 'grad_norm': 5.484898090362549, 'learning_rate': 3.506382277084696e-05, 'epoch': 3.82}\n", + "{'loss': 0.9029, 'grad_norm': 4.691559314727783, 'learning_rate': 3.4568912409493945e-05, 'epoch': 3.84}\n", + "{'loss': 0.7487, 'grad_norm': 4.6990132331848145, 'learning_rate': 3.4075667487415785e-05, 'epoch': 3.86}\n", + "{'loss': 0.8389, 'grad_norm': 3.9886608123779297, 'learning_rate': 3.358414123932195e-05, 'epoch': 3.87}\n", + "{'loss': 0.9443, 'grad_norm': 3.797034978866577, 'learning_rate': 3.3094386714429724e-05, 'epoch': 3.89}\n", + "{'loss': 0.9102, 'grad_norm': 9.836748123168945, 'learning_rate': 3.2606456770738636e-05, 'epoch': 3.91}\n", + "{'loss': 0.8031, 'grad_norm': 6.517895221710205, 'learning_rate': 3.212040406932569e-05, 'epoch': 3.93}\n", + "{'loss': 0.7276, 'grad_norm': 3.757455825805664, 'learning_rate': 3.163628106866172e-05, 'epoch': 3.94}\n", + "{'loss': 1.0437, 'grad_norm': 5.128631591796875, 'learning_rate': 3.115414001894974e-05, 'epoch': 3.96}\n", + "{'loss': 0.9261, 'grad_norm': 4.2124457359313965, 'learning_rate': 3.067403295648566e-05, 'epoch': 3.98}\n", + "{'loss': 0.7864, 'grad_norm': 3.609720230102539, 'learning_rate': 3.019601169804216e-05, 'epoch': 4.0}\n", + " 67%|████████████████████████▋ | 2240/3360 [1:05:16<32:59, 1.77s/it][INFO|trainer.py:3788] 2024-07-04 13:32:05,670 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 13:32:05,670 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 13:32:05,670 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-2240\n", + "[INFO|configuration_utils.py:733] 2024-07-04 13:32:08,839 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 13:32:08,839 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 13:32:08,917 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2240/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 13:32:08,917 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2240/special_tokens_map.json\n", + "{'loss': 0.7877, 'grad_norm': 3.5848188400268555, 'learning_rate': 2.9720127835276256e-05, 'epoch': 4.02}\n", + "{'loss': 0.6372, 'grad_norm': 4.3321661949157715, 'learning_rate': 2.9246432729161055e-05, 'epoch': 4.03}\n", + "{'loss': 0.7208, 'grad_norm': 5.079366207122803, 'learning_rate': 2.8774977504442647e-05, 'epoch': 4.05}\n", + "{'loss': 0.6386, 'grad_norm': 4.389534950256348, 'learning_rate': 2.8305813044122097e-05, 'epoch': 4.07}\n", + "{'loss': 0.6416, 'grad_norm': 3.866262197494507, 'learning_rate': 2.7838989983964065e-05, 'epoch': 4.09}\n", + "{'loss': 0.7771, 'grad_norm': 4.4248528480529785, 'learning_rate': 2.737455870703155e-05, 'epoch': 4.11}\n", + "{'loss': 0.5944, 'grad_norm': 5.1537370681762695, 'learning_rate': 2.6912569338248315e-05, 'epoch': 4.12}\n", + "{'loss': 0.6846, 'grad_norm': 4.410754203796387, 'learning_rate': 2.645307173898901e-05, 'epoch': 4.14}\n", + "{'loss': 0.7499, 'grad_norm': 3.9317386150360107, 'learning_rate': 2.5996115501697694e-05, 'epoch': 4.16}\n", + "{'loss': 0.6378, 'grad_norm': 3.794434070587158, 'learning_rate': 2.5541749944535554e-05, 'epoch': 4.18}\n", + "{'loss': 0.6153, 'grad_norm': 4.012321472167969, 'learning_rate': 2.5090024106057962e-05, 'epoch': 4.19}\n", + "{'loss': 0.6922, 'grad_norm': 4.712143898010254, 'learning_rate': 2.464098673992205e-05, 'epoch': 4.21}\n", + "{'loss': 0.6079, 'grad_norm': 5.002867698669434, 'learning_rate': 2.4194686309624663e-05, 'epoch': 4.23}\n", + "{'loss': 0.7554, 'grad_norm': 6.034168720245361, 'learning_rate': 2.3751170983272e-05, 'epoch': 4.25}\n", + "{'loss': 0.6634, 'grad_norm': 5.4491376876831055, 'learning_rate': 2.3310488628380757e-05, 'epoch': 4.27}\n", + "{'loss': 0.6635, 'grad_norm': 6.335705280303955, 'learning_rate': 2.2872686806712035e-05, 'epoch': 4.28}\n", + "{'loss': 0.6732, 'grad_norm': 4.363458633422852, 'learning_rate': 2.243781276913811e-05, 'epoch': 4.3}\n", + "{'loss': 0.5751, 'grad_norm': 4.058308124542236, 'learning_rate': 2.200591345054267e-05, 'epoch': 4.32}\n", + "{'loss': 0.7378, 'grad_norm': 5.493106365203857, 'learning_rate': 2.157703546475539e-05, 'epoch': 4.34}\n", + "{'loss': 0.6231, 'grad_norm': 4.587257385253906, 'learning_rate': 2.115122509952085e-05, 'epoch': 4.36}\n", + "{'loss': 0.6361, 'grad_norm': 4.070307731628418, 'learning_rate': 2.0728528311502976e-05, 'epoch': 4.37}\n", + "{'loss': 0.7245, 'grad_norm': 5.507742404937744, 'learning_rate': 2.0308990721324927e-05, 'epoch': 4.39}\n", + "{'loss': 0.6516, 'grad_norm': 4.98870849609375, 'learning_rate': 1.989265760864542e-05, 'epoch': 4.41}\n", + "{'loss': 0.7311, 'grad_norm': 4.5378618240356445, 'learning_rate': 1.947957390727185e-05, 'epoch': 4.43}\n", + "{'loss': 0.713, 'grad_norm': 6.595687389373779, 'learning_rate': 1.906978420031059e-05, 'epoch': 4.44}\n", + "{'loss': 0.5884, 'grad_norm': 3.995753765106201, 'learning_rate': 1.8663332715355396e-05, 'epoch': 4.46}\n", + "{'loss': 0.7598, 'grad_norm': 3.745181083679199, 'learning_rate': 1.8260263319713844e-05, 'epoch': 4.48}\n", + "{'loss': 0.673, 'grad_norm': 5.82590389251709, 'learning_rate': 1.7860619515673033e-05, 'epoch': 4.5}\n", + "{'loss': 0.6552, 'grad_norm': 5.151037216186523, 'learning_rate': 1.746444443580433e-05, 'epoch': 4.52}\n", + "{'loss': 0.7091, 'grad_norm': 5.6730499267578125, 'learning_rate': 1.7071780838308288e-05, 'epoch': 4.53}\n", + "{'loss': 0.7061, 'grad_norm': 5.2298502922058105, 'learning_rate': 1.6682671102399805e-05, 'epoch': 4.55}\n", + "{'loss': 0.654, 'grad_norm': 5.273619651794434, 'learning_rate': 1.629715722373423e-05, 'epoch': 4.57}\n", + "{'loss': 0.6822, 'grad_norm': 6.466513633728027, 'learning_rate': 1.5915280809874932e-05, 'epoch': 4.59}\n", + "{'loss': 0.5785, 'grad_norm': 6.050833702087402, 'learning_rate': 1.553708307580265e-05, 'epoch': 4.61}\n", + "{'loss': 0.6877, 'grad_norm': 6.502690315246582, 'learning_rate': 1.5162604839467265e-05, 'epoch': 4.62}\n", + "{'loss': 0.6984, 'grad_norm': 4.606260299682617, 'learning_rate': 1.4791886517382413e-05, 'epoch': 4.64}\n", + "{'loss': 0.6909, 'grad_norm': 4.80437707901001, 'learning_rate': 1.4424968120263504e-05, 'epoch': 4.66}\n", + "{'loss': 0.6827, 'grad_norm': 5.20365047454834, 'learning_rate': 1.4061889248709343e-05, 'epoch': 4.68}\n", + "{'loss': 0.6361, 'grad_norm': 6.29000997543335, 'learning_rate': 1.370268908892825e-05, 'epoch': 4.69}\n", + "{'loss': 0.6747, 'grad_norm': 4.9368438720703125, 'learning_rate': 1.3347406408508695e-05, 'epoch': 4.71}\n", + "{'loss': 0.6435, 'grad_norm': 5.528055667877197, 'learning_rate': 1.2996079552235263e-05, 'epoch': 4.73}\n", + "{'loss': 0.6501, 'grad_norm': 4.367548942565918, 'learning_rate': 1.264874643795021e-05, 'epoch': 4.75}\n", + "{'loss': 0.6376, 'grad_norm': 4.568158149719238, 'learning_rate': 1.230544455246101e-05, 'epoch': 4.77}\n", + "{'loss': 0.7034, 'grad_norm': 5.3214287757873535, 'learning_rate': 1.1966210947494583e-05, 'epoch': 4.78}\n", + "{'loss': 0.7303, 'grad_norm': 3.9356067180633545, 'learning_rate': 1.1631082235698316e-05, 'epoch': 4.8}\n", + "{'loss': 0.6436, 'grad_norm': 5.198613166809082, 'learning_rate': 1.130009458668863e-05, 'epoch': 4.82}\n", + "{'loss': 0.595, 'grad_norm': 4.129484176635742, 'learning_rate': 1.097328372314721e-05, 'epoch': 4.84}\n", + "{'loss': 0.6302, 'grad_norm': 8.324830055236816, 'learning_rate': 1.0650684916965559e-05, 'epoch': 4.85}\n", + "{'loss': 0.7296, 'grad_norm': 5.789163589477539, 'learning_rate': 1.0332332985438248e-05, 'epoch': 4.87}\n", + "{'loss': 0.7077, 'grad_norm': 5.632966995239258, 'learning_rate': 1.0018262287505086e-05, 'epoch': 4.89}\n", + "{'loss': 0.7339, 'grad_norm': 4.699968338012695, 'learning_rate': 9.708506720042932e-06, 'epoch': 4.91}\n", + "{'loss': 0.6334, 'grad_norm': 3.969327926635742, 'learning_rate': 9.403099714207175e-06, 'epoch': 4.93}\n", + "{'loss': 0.7298, 'grad_norm': 4.980201244354248, 'learning_rate': 9.102074231823727e-06, 'epoch': 4.94}\n", + "{'loss': 0.7236, 'grad_norm': 6.4100565910339355, 'learning_rate': 8.805462761831418e-06, 'epoch': 4.96}\n", + "{'loss': 0.7751, 'grad_norm': 5.446720600128174, 'learning_rate': 8.513297316775625e-06, 'epoch': 4.98}\n", + "{'loss': 0.7407, 'grad_norm': 4.180345058441162, 'learning_rate': 8.225609429353187e-06, 'epoch': 5.0}\n", + " 83%|██████████████████████████████▊ | 2800/3360 [1:21:52<16:57, 1.82s/it][INFO|trainer.py:3788] 2024-07-04 13:48:40,919 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 13:48:40,919 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 13:48:40,919 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-2800\n", + "[INFO|configuration_utils.py:733] 2024-07-04 13:48:44,254 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 13:48:44,254 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 13:48:44,307 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2800/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 13:48:44,307 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2800/special_tokens_map.json\n", + "{'loss': 0.5816, 'grad_norm': 3.6714768409729004, 'learning_rate': 7.942430149009161e-06, 'epoch': 5.02}\n", + "{'loss': 0.516, 'grad_norm': 3.9655864238739014, 'learning_rate': 7.663790038585793e-06, 'epoch': 5.03}\n", + "{'loss': 0.5876, 'grad_norm': 4.2771453857421875, 'learning_rate': 7.389719171023857e-06, 'epoch': 5.05}\n", + "{'loss': 0.5746, 'grad_norm': 5.545507431030273, 'learning_rate': 7.1202471261170245e-06, 'epoch': 5.07}\n", + "{'loss': 0.5789, 'grad_norm': 4.685436248779297, 'learning_rate': 6.855402987319348e-06, 'epoch': 5.09}\n", + "{'loss': 0.6558, 'grad_norm': 5.384147644042969, 'learning_rate': 6.595215338606397e-06, 'epoch': 5.1}\n", + "{'loss': 0.4955, 'grad_norm': 5.300227642059326, 'learning_rate': 6.339712261390213e-06, 'epoch': 5.12}\n", + "{'loss': 0.6284, 'grad_norm': 5.341045379638672, 'learning_rate': 6.088921331488568e-06, 'epoch': 5.14}\n", + "{'loss': 0.5285, 'grad_norm': 4.509070873260498, 'learning_rate': 5.8428696161488215e-06, 'epoch': 5.16}\n", + "{'loss': 0.5073, 'grad_norm': 4.6753339767456055, 'learning_rate': 5.601583671126531e-06, 'epoch': 5.18}\n", + "{'loss': 0.5849, 'grad_norm': 3.836711883544922, 'learning_rate': 5.365089537819434e-06, 'epoch': 5.19}\n", + "{'loss': 0.5714, 'grad_norm': 4.124776840209961, 'learning_rate': 5.133412740456806e-06, 'epoch': 5.21}\n", + "{'loss': 0.5811, 'grad_norm': 4.734057903289795, 'learning_rate': 4.906578283344759e-06, 'epoch': 5.23}\n", + "{'loss': 0.6351, 'grad_norm': 5.501781463623047, 'learning_rate': 4.684610648167503e-06, 'epoch': 5.25}\n", + "{'loss': 0.5622, 'grad_norm': 4.912986755371094, 'learning_rate': 4.467533791345191e-06, 'epoch': 5.27}\n", + "{'loss': 0.5305, 'grad_norm': 6.3503899574279785, 'learning_rate': 4.255371141448272e-06, 'epoch': 5.28}\n", + "{'loss': 0.5406, 'grad_norm': 4.923576354980469, 'learning_rate': 4.048145596668967e-06, 'epoch': 5.3}\n", + "{'loss': 0.5534, 'grad_norm': 4.20800256729126, 'learning_rate': 3.84587952234991e-06, 'epoch': 5.32}\n", + "{'loss': 0.4831, 'grad_norm': 4.633558750152588, 'learning_rate': 3.6485947485702832e-06, 'epoch': 5.34}\n", + "{'loss': 0.5587, 'grad_norm': 5.120583534240723, 'learning_rate': 3.4563125677897932e-06, 'epoch': 5.35}\n", + "{'loss': 0.5696, 'grad_norm': 5.966647148132324, 'learning_rate': 3.269053732550581e-06, 'epoch': 5.37}\n", + "{'loss': 0.5767, 'grad_norm': 5.047117233276367, 'learning_rate': 3.086838453237506e-06, 'epoch': 5.39}\n", + "{'loss': 0.4262, 'grad_norm': 4.478403091430664, 'learning_rate': 2.9096863958968268e-06, 'epoch': 5.41}\n", + "{'loss': 0.4798, 'grad_norm': 4.455025672912598, 'learning_rate': 2.737616680113758e-06, 'epoch': 5.43}\n", + "{'loss': 0.4574, 'grad_norm': 3.7917206287384033, 'learning_rate': 2.570647876948895e-06, 'epoch': 5.44}\n", + "{'loss': 0.5635, 'grad_norm': 7.098059177398682, 'learning_rate': 2.408798006933882e-06, 'epoch': 5.46}\n", + "{'loss': 0.7231, 'grad_norm': 4.642895698547363, 'learning_rate': 2.252084538126542e-06, 'epoch': 5.48}\n", + "{'loss': 0.5122, 'grad_norm': 5.233055591583252, 'learning_rate': 2.100524384225555e-06, 'epoch': 5.5}\n", + "{'loss': 0.524, 'grad_norm': 4.6845173835754395, 'learning_rate': 1.9541339027450256e-06, 'epoch': 5.52}\n", + "{'loss': 0.5816, 'grad_norm': 5.447011470794678, 'learning_rate': 1.8129288932490274e-06, 'epoch': 5.53}\n", + "{'loss': 0.5329, 'grad_norm': 3.755023717880249, 'learning_rate': 1.6769245956464396e-06, 'epoch': 5.55}\n", + "{'loss': 0.6767, 'grad_norm': 5.255481719970703, 'learning_rate': 1.5461356885461075e-06, 'epoch': 5.57}\n", + "{'loss': 0.5529, 'grad_norm': 4.8336567878723145, 'learning_rate': 1.4205762876726092e-06, 'epoch': 5.59}\n", + "{'loss': 0.6372, 'grad_norm': 5.332770824432373, 'learning_rate': 1.3002599443428243e-06, 'epoch': 5.6}\n", + "{'loss': 0.634, 'grad_norm': 5.157808780670166, 'learning_rate': 1.1851996440033319e-06, 'epoch': 5.62}\n", + "{'loss': 0.5033, 'grad_norm': 4.826900005340576, 'learning_rate': 1.0754078048289374e-06, 'epoch': 5.64}\n", + "{'loss': 0.5681, 'grad_norm': 3.9047048091888428, 'learning_rate': 9.708962763824048e-07, 'epoch': 5.66}\n", + "{'loss': 0.5432, 'grad_norm': 6.038053512573242, 'learning_rate': 8.716763383355864e-07, 'epoch': 5.68}\n", + "{'loss': 0.6018, 'grad_norm': 5.233924388885498, 'learning_rate': 7.777586992519959e-07, 'epoch': 5.69}\n", + "{'loss': 0.5367, 'grad_norm': 6.929383277893066, 'learning_rate': 6.891534954310885e-07, 'epoch': 5.71}\n", + "{'loss': 0.6039, 'grad_norm': 4.509579181671143, 'learning_rate': 6.058702898142643e-07, 'epoch': 5.73}\n", + "{'loss': 0.5292, 'grad_norm': 4.131773948669434, 'learning_rate': 5.279180709527765e-07, 'epoch': 5.75}\n", + "{'loss': 0.6327, 'grad_norm': 4.368628025054932, 'learning_rate': 4.553052520375911e-07, 'epoch': 5.77}\n", + "{'loss': 0.4859, 'grad_norm': 4.966446399688721, 'learning_rate': 3.8803966999139684e-07, 'epoch': 5.78}\n", + "{'loss': 0.5397, 'grad_norm': 5.083605766296387, 'learning_rate': 3.261285846227868e-07, 'epoch': 5.8}\n", + "{'loss': 0.4758, 'grad_norm': 4.257706165313721, 'learning_rate': 2.6957867784270787e-07, 'epoch': 5.82}\n", + "{'loss': 0.492, 'grad_norm': 5.183888912200928, 'learning_rate': 2.1839605294330933e-07, 'epoch': 5.84}\n", + "{'loss': 0.6466, 'grad_norm': 7.4429707527160645, 'learning_rate': 1.725862339392259e-07, 'epoch': 5.85}\n", + "{'loss': 0.4461, 'grad_norm': 6.51588249206543, 'learning_rate': 1.3215416497138754e-07, 'epoch': 5.87}\n", + "{'loss': 0.6614, 'grad_norm': 4.2303786277771, 'learning_rate': 9.710420977340762e-08, 'epoch': 5.89}\n", + "{'loss': 0.4817, 'grad_norm': 6.3713908195495605, 'learning_rate': 6.744015120061509e-08, 'epoch': 5.91}\n", + "{'loss': 0.6231, 'grad_norm': 10.188394546508789, 'learning_rate': 4.316519082179227e-08, 'epoch': 5.93}\n", + "{'loss': 0.5204, 'grad_norm': 4.387541770935059, 'learning_rate': 2.4281948573617874e-08, 'epoch': 5.94}\n", + "{'loss': 0.5938, 'grad_norm': 5.084804534912109, 'learning_rate': 1.0792462477909882e-08, 'epoch': 5.96}\n", + "{'loss': 0.576, 'grad_norm': 4.955512523651123, 'learning_rate': 2.6981884216847884e-09, 'epoch': 5.98}\n", + "{'loss': 0.5752, 'grad_norm': 4.1065449714660645, 'learning_rate': 0.0, 'epoch': 6.0}\n", + "100%|█████████████████████████████████████| 3360/3360 [1:38:23<00:00, 1.77s/it][INFO|trainer.py:3788] 2024-07-04 14:05:12,056 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 14:05:12,056 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 14:05:12,056 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-3360\n", + "[INFO|configuration_utils.py:733] 2024-07-04 14:05:15,110 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 14:05:15,111 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 14:05:15,155 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-3360/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 14:05:15,155 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-3360/special_tokens_map.json\n", + "[INFO|trainer.py:2383] 2024-07-04 14:05:15,382 >> \n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "{'train_runtime': 5911.7152, 'train_samples_per_second': 4.549, 'train_steps_per_second': 0.568, 'train_loss': 1.1251599807114827, 'epoch': 6.0}\n", + "100%|█████████████████████████████████████| 3360/3360 [1:38:26<00:00, 1.76s/it]\n", + "[INFO|trainer.py:3478] 2024-07-04 14:05:15,386 >> Saving model checkpoint to saves/qwen2-1.5b/lora/sft\n", + "[INFO|configuration_utils.py:733] 2024-07-04 14:05:16,251 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 14:05:16,251 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 14:05:16,306 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 14:05:16,306 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/special_tokens_map.json\n", + "***** train metrics *****\n", + " epoch = 5.9973\n", + " total_flos = 16732846GF\n", + " train_loss = 1.1252\n", + " train_runtime = 1:38:31.71\n", + " train_samples_per_second = 4.549\n", + " train_steps_per_second = 0.568\n", + "Figure saved at: saves/qwen2-1.5b/lora/sft/training_loss.png\n", + "Figure saved at: saves/qwen2-1.5b/lora/sft/training_eval_loss.png\n", + "[INFO|trainer.py:3788] 2024-07-04 14:05:16,625 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 14:05:16,625 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 14:05:16,625 >> Batch size = 1\n", + "100%|███████████████████████████████████████████| 46/46 [00:02<00:00, 18.96it/s]\n", + "***** eval metrics *****\n", + " epoch = 5.9973\n", + " eval_loss = 2.4373\n", + " eval_runtime = 0:00:02.50\n", + " eval_samples_per_second = 18.363\n", + " eval_steps_per_second = 18.363\n", + "[INFO|modelcard.py:449] 2024-07-04 14:05:19,133 >> Dropping the following result as it does not have all the necessary fields:\n", + "{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \\ 0.086 MB of 0.086 MB uploaded\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss ▁▁▂▄▆██\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime ▃▄▁▅█▁▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second ▆▅█▄▁█▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second ▆▅█▄▁█▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm ▃▁▂▂▁▃▂▃▄▄▃▅▃▃▅▅▅▃▄▄▄▆▅▇▅▆▄▅█▅▇▆▅▇▅▆▆▆▆▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate ▂▄▅▇██████▇▇▇▇▇▆▆▆▆▅▅▅▄▄▄▃▃▃▃▂▂▂▂▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss █▇▇▇▆▆▆▆▆▆▅▅▆▅▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss 2.43734\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime 2.5051\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second 18.363\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second 18.363\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: total_flos 1.7966756916707328e+16\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch 5.99732\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step 3360\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm 4.10654\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss 0.5752\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 1.12516\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_runtime 5911.7152\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_samples_per_second 4.549\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_steps_per_second 0.568\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mqwen2_1.5b_lora_sft\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/mpc5sxtf\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 6 W&B file(s), 0 media file(s), 1 artifact file(s) and 0 other file(s)\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240704_122645-mpc5sxtf/logs\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n", + "CPU times: user 1min 28s, sys: 26.5 s, total: 1min 54s\n", + "Wall time: 1h 42min 32s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./scripts/tune-lf.sh config/qwen2_1.5b_lora_sft.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/llm-finetuning/llama-factory\n", + "07/04/2024 14:50:13 - WARNING - llamafactory.hparams.parser - We recommend enable `upcast_layernorm` in quantized training.\n", + "07/04/2024 14:50:13 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: torch.bfloat16\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 14:50:14,466 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-7B-Instruct/snapshots/41c66b0be1c3081f13defc6bdf946c2ef240d6a6/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 14:50:14,466 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-7B-Instruct/snapshots/41c66b0be1c3081f13defc6bdf946c2ef240d6a6/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 14:50:14,466 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-7B-Instruct/snapshots/41c66b0be1c3081f13defc6bdf946c2ef240d6a6/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 14:50:14,466 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 14:50:14,466 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 14:50:14,467 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-7B-Instruct/snapshots/41c66b0be1c3081f13defc6bdf946c2ef240d6a6/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-04 14:50:14,635 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/04/2024 14:50:14 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/04/2024 14:50:14 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "07/04/2024 14:50:14 - INFO - llamafactory.data.loader - Loading dataset alpaca_mac.json...\n", + "Converting format of dataset (num_proc=16): 100%|█| 4528/4528 [00:00<00:00, 1650\n", + "Running tokenizer on dataset (num_proc=16): 100%|█| 4528/4528 [00:01<00:00, 3163\n", + "input_ids:\n", + "[151644, 872, 198, 5501, 14683, 279, 2701, 8453, 1467, 1119, 6364, 323, 3410, 1172, 279, 24531, 2213, 11, 4302, 770, 624, 35987, 102895, 99164, 100324, 100717, 100095, 99509, 1773, 151645, 198, 151644, 77091, 198, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "inputs:\n", + "<|im_start|>user\n", + "Please translate the following Chinese text into English and provide only the translated content, nothing else.\n", + "全仗着狐仙搭救。<|im_end|>\n", + "<|im_start|>assistant\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "label_ids:\n", + "[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "labels:\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "[INFO|configuration_utils.py:733] 2024-07-04 14:50:17,794 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-7B-Instruct/snapshots/41c66b0be1c3081f13defc6bdf946c2ef240d6a6/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 14:50:17,795 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "07/04/2024 14:50:17 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "config.json: 100%|█████████████████████████| 1.19k/1.19k [00:00<00:00, 12.3MB/s]\n", + "[INFO|configuration_utils.py:733] 2024-07-04 14:50:19,202 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 14:50:19,203 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"unsloth/qwen2-7b-instruct-bnb-4bit\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0+cu121. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "[INFO|configuration_utils.py:733] 2024-07-04 14:50:20,339 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 14:50:20,340 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"unsloth/qwen2-7b-instruct-bnb-4bit\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|configuration_utils.py:733] 2024-07-04 14:50:20,992 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 14:50:20,993 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"unsloth/qwen2-7b-instruct-bnb-4bit\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "model.safetensors: 100%|███████████████████| 5.55G/5.55G [31:00<00:00, 2.98MB/s]\n", + "[INFO|modeling_utils.py:3556] 2024-07-04 15:21:22,487 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-04 15:21:26,212 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-04 15:21:26,219 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-04 15:26:00,017 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-04 15:26:00,018 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at unsloth/qwen2-7b-instruct-bnb-4bit.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "generation_config.json: 100%|██████████████████| 243/243 [00:00<00:00, 3.75MB/s]\n", + "[INFO|configuration_utils.py:955] 2024-07-04 15:26:01,541 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-04 15:26:01,542 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.05,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "tokenizer_config.json: 100%|███████████████| 1.33k/1.33k [00:00<00:00, 19.0MB/s]\n", + "vocab.json: 100%|██████████████████████████| 2.78M/2.78M [00:01<00:00, 1.75MB/s]\n", + "merges.txt: 100%|██████████████████████████| 1.67M/1.67M [00:00<00:00, 1.89MB/s]\n", + "added_tokens.json: 100%|█████████████████████| 80.0/80.0 [00:00<00:00, 1.29MB/s]\n", + "special_tokens_map.json: 100%|█████████████████| 367/367 [00:00<00:00, 6.11MB/s]\n", + "tokenizer.json: 100%|██████████████████████| 7.03M/7.03M [00:02<00:00, 3.09MB/s]\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:12,737 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:12,737 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:12,737 >> loading file added_tokens.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/added_tokens.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:12,737 >> loading file special_tokens_map.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/special_tokens_map.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:12,737 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:12,737 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/tokenizer.json\n", + "[WARNING|logging.py:313] 2024-07-04 15:26:12,946 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:13,696 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:13,696 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:13,696 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:13,696 >> loading file added_tokens.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/added_tokens.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:13,696 >> loading file special_tokens_map.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/special_tokens_map.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 15:26:13,696 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-04 15:26:13,877 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/04/2024 15:26:14 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.\n", + "07/04/2024 15:26:14 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.\n", + "07/04/2024 15:26:14 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA\n", + "07/04/2024 15:26:14 - INFO - llamafactory.model.model_utils.misc - Found linear modules: gate_proj,o_proj,v_proj,k_proj,up_proj,q_proj,down_proj\n", + "[WARNING|logging.py:328] 2024-07-04 15:26:15,372 >> Unsloth 2024.6 patched 28 layers with 0 QKV layers, 28 O layers and 28 MLP layers.\n", + "07/04/2024 15:26:16 - INFO - llamafactory.model.loader - trainable params: 20,185,088 || all params: 7,635,801,600 || trainable%: 0.2643\n", + "[INFO|trainer.py:642] 2024-07-04 15:26:16,270 >> Using auto half precision backend\n", + "07/04/2024 15:26:16 - INFO - llamafactory.train.trainer_utils - Using LoRA+ optimizer with loraplus lr ratio 16.00.\n", + "[WARNING|:223] 2024-07-04 15:26:16,423 >> ==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,482 | Num Epochs = 6\n", + "O^O/ \\_/ \\ Batch size per device = 1 | Gradient Accumulation steps = 8\n", + "\\ / Total batch size = 8 | Total steps = 3,360\n", + " \"-____-\" Number of trainable parameters = 20,185,088\n", + "[INFO|integration_utils.py:750] 2024-07-04 15:26:16,929 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33minflaton-sg\u001b[0m (\u001b[33minflaton-ai\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.4\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/home/inflaton/code/projects/courses/llm-finetuning/llama-factory/wandb/run-20240704_152618-o710838e\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mqwen2_7b_lora_sft\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/o710838e\u001b[0m\n", + "{'loss': 2.1957, 'grad_norm': 2.977725028991699, 'learning_rate': 2.9761904761904763e-06, 'epoch': 0.02}\n", + "{'loss': 1.9984, 'grad_norm': 1.17664635181427, 'learning_rate': 5.9523809523809525e-06, 'epoch': 0.04}\n", + "{'loss': 1.7375, 'grad_norm': 0.7683635354042053, 'learning_rate': 8.92857142857143e-06, 'epoch': 0.05}\n", + "{'loss': 1.7268, 'grad_norm': 1.5277972221374512, 'learning_rate': 1.1904761904761905e-05, 'epoch': 0.07}\n", + "{'loss': 1.7873, 'grad_norm': 0.7151318788528442, 'learning_rate': 1.4880952380952381e-05, 'epoch': 0.09}\n", + "{'loss': 1.6224, 'grad_norm': 0.7458081841468811, 'learning_rate': 1.785714285714286e-05, 'epoch': 0.11}\n", + "{'loss': 1.7345, 'grad_norm': 0.7242929339408875, 'learning_rate': 2.0833333333333336e-05, 'epoch': 0.12}\n", + "{'loss': 1.57, 'grad_norm': 0.8281179666519165, 'learning_rate': 2.380952380952381e-05, 'epoch': 0.14}\n", + "{'loss': 1.6718, 'grad_norm': 1.0110186338424683, 'learning_rate': 2.6785714285714288e-05, 'epoch': 0.16}\n", + "{'loss': 1.6219, 'grad_norm': 0.8258731961250305, 'learning_rate': 2.9761904761904762e-05, 'epoch': 0.18}\n", + "{'loss': 1.6115, 'grad_norm': 0.9346244931221008, 'learning_rate': 3.273809523809524e-05, 'epoch': 0.2}\n", + "{'loss': 1.6081, 'grad_norm': 1.0503712892532349, 'learning_rate': 3.571428571428572e-05, 'epoch': 0.21}\n", + "{'loss': 1.5874, 'grad_norm': 1.1157383918762207, 'learning_rate': 3.8690476190476195e-05, 'epoch': 0.23}\n", + "{'loss': 1.5825, 'grad_norm': 1.212875485420227, 'learning_rate': 4.166666666666667e-05, 'epoch': 0.25}\n", + "{'loss': 1.512, 'grad_norm': 1.073009967803955, 'learning_rate': 4.464285714285715e-05, 'epoch': 0.27}\n", + "{'loss': 1.5074, 'grad_norm': 0.8304378390312195, 'learning_rate': 4.761904761904762e-05, 'epoch': 0.29}\n", + "{'loss': 1.6019, 'grad_norm': 0.8581129908561707, 'learning_rate': 5.05952380952381e-05, 'epoch': 0.3}\n", + "{'loss': 1.4319, 'grad_norm': 1.027076244354248, 'learning_rate': 5.3571428571428575e-05, 'epoch': 0.32}\n", + "{'loss': 1.555, 'grad_norm': 1.3464545011520386, 'learning_rate': 5.6547619047619046e-05, 'epoch': 0.34}\n", + "{'loss': 1.6637, 'grad_norm': 1.2714892625808716, 'learning_rate': 5.9523809523809524e-05, 'epoch': 0.36}\n", + "{'loss': 1.6159, 'grad_norm': 1.0014649629592896, 'learning_rate': 6.25e-05, 'epoch': 0.37}\n", + "{'loss': 1.5019, 'grad_norm': 1.4355653524398804, 'learning_rate': 6.547619047619048e-05, 'epoch': 0.39}\n", + "{'loss': 1.5167, 'grad_norm': 1.2876572608947754, 'learning_rate': 6.845238095238096e-05, 'epoch': 0.41}\n", + "{'loss': 1.6807, 'grad_norm': 1.4459688663482666, 'learning_rate': 7.142857142857143e-05, 'epoch': 0.43}\n", + "{'loss': 1.6053, 'grad_norm': 1.7381216287612915, 'learning_rate': 7.440476190476191e-05, 'epoch': 0.45}\n", + "{'loss': 1.4993, 'grad_norm': 1.516874074935913, 'learning_rate': 7.738095238095239e-05, 'epoch': 0.46}\n", + "{'loss': 1.58, 'grad_norm': 1.7755393981933594, 'learning_rate': 8.035714285714287e-05, 'epoch': 0.48}\n", + "{'loss': 1.5699, 'grad_norm': 1.7302135229110718, 'learning_rate': 8.333333333333334e-05, 'epoch': 0.5}\n", + "{'loss': 1.419, 'grad_norm': 1.172330617904663, 'learning_rate': 8.630952380952382e-05, 'epoch': 0.52}\n", + "{'loss': 1.5505, 'grad_norm': 1.676744818687439, 'learning_rate': 8.92857142857143e-05, 'epoch': 0.54}\n", + "{'loss': 1.5749, 'grad_norm': 1.8019312620162964, 'learning_rate': 9.226190476190478e-05, 'epoch': 0.55}\n", + "{'loss': 1.4185, 'grad_norm': 2.2339751720428467, 'learning_rate': 9.523809523809524e-05, 'epoch': 0.57}\n", + "{'loss': 1.4871, 'grad_norm': 1.8845446109771729, 'learning_rate': 9.821428571428572e-05, 'epoch': 0.59}\n", + "{'loss': 1.4547, 'grad_norm': 1.5382771492004395, 'learning_rate': 9.999956828659095e-05, 'epoch': 0.61}\n", + "{'loss': 1.5409, 'grad_norm': 2.5924744606018066, 'learning_rate': 9.999471159635539e-05, 'epoch': 0.62}\n", + "{'loss': 1.4544, 'grad_norm': 1.6850535869598389, 'learning_rate': 9.998445910004082e-05, 'epoch': 0.64}\n", + "{'loss': 1.6716, 'grad_norm': 2.093435287475586, 'learning_rate': 9.996881190417393e-05, 'epoch': 0.66}\n", + "{'loss': 1.5389, 'grad_norm': 1.9192240238189697, 'learning_rate': 9.994777169751806e-05, 'epoch': 0.68}\n", + "{'loss': 1.3255, 'grad_norm': 1.820000410079956, 'learning_rate': 9.992134075089084e-05, 'epoch': 0.7}\n", + "{'loss': 1.4784, 'grad_norm': 1.8777908086776733, 'learning_rate': 9.988952191691925e-05, 'epoch': 0.71}\n", + "{'loss': 1.5354, 'grad_norm': 1.7081478834152222, 'learning_rate': 9.985231862973168e-05, 'epoch': 0.73}\n", + "{'loss': 1.5822, 'grad_norm': 1.6461598873138428, 'learning_rate': 9.980973490458728e-05, 'epoch': 0.75}\n", + "{'loss': 1.5233, 'grad_norm': 2.1327311992645264, 'learning_rate': 9.976177533744261e-05, 'epoch': 0.77}\n", + "{'loss': 1.4739, 'grad_norm': 2.4746365547180176, 'learning_rate': 9.97084451044556e-05, 'epoch': 0.79}\n", + "{'loss': 1.4276, 'grad_norm': 1.7821303606033325, 'learning_rate': 9.964974996142698e-05, 'epoch': 0.8}\n", + "{'loss': 1.4803, 'grad_norm': 1.551522970199585, 'learning_rate': 9.958569624317893e-05, 'epoch': 0.82}\n", + "{'loss': 1.5314, 'grad_norm': 2.6767489910125732, 'learning_rate': 9.951629086287151e-05, 'epoch': 0.84}\n", + "{'loss': 1.4844, 'grad_norm': 1.7266111373901367, 'learning_rate': 9.944154131125642e-05, 'epoch': 0.86}\n", + "{'loss': 1.5248, 'grad_norm': 1.7948070764541626, 'learning_rate': 9.936145565586871e-05, 'epoch': 0.87}\n", + "{'loss': 1.6563, 'grad_norm': 1.6244261264801025, 'learning_rate': 9.927604254015585e-05, 'epoch': 0.89}\n", + "{'loss': 1.5928, 'grad_norm': 1.7924832105636597, 'learning_rate': 9.918531118254507e-05, 'epoch': 0.91}\n", + "{'loss': 1.4955, 'grad_norm': 2.337216377258301, 'learning_rate': 9.90892713754483e-05, 'epoch': 0.93}\n", + "{'loss': 1.558, 'grad_norm': 2.165968179702759, 'learning_rate': 9.898793348420536e-05, 'epoch': 0.95}\n", + "{'loss': 1.5148, 'grad_norm': 1.7740817070007324, 'learning_rate': 9.888130844596524e-05, 'epoch': 0.96}\n", + "{'loss': 1.5339, 'grad_norm': 2.276500940322876, 'learning_rate': 9.876940776850569e-05, 'epoch': 0.98}\n", + "{'loss': 1.4748, 'grad_norm': 1.852982521057129, 'learning_rate': 9.865224352899119e-05, 'epoch': 1.0}\n", + " 17%|██████ | 560/3360 [1:04:27<5:48:51, 7.48s/it][INFO|trainer.py:3788] 2024-07-04 16:30:50,001 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 16:30:50,003 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 16:30:50,003 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-7b/lora/sft/checkpoint-560\n", + "[INFO|configuration_utils.py:733] 2024-07-04 16:31:06,164 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 16:31:06,165 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 16:31:06,584 >> tokenizer config file saved in saves/qwen2-7b/lora/sft/checkpoint-560/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 16:31:06,585 >> Special tokens file saved in saves/qwen2-7b/lora/sft/checkpoint-560/special_tokens_map.json\n", + "{'loss': 1.0693, 'grad_norm': 2.4884133338928223, 'learning_rate': 9.852982837266955e-05, 'epoch': 1.02}\n", + "{'loss': 0.9156, 'grad_norm': 2.0713613033294678, 'learning_rate': 9.840217551150706e-05, 'epoch': 1.04}\n", + "{'loss': 1.0533, 'grad_norm': 2.160870313644409, 'learning_rate': 9.826929872276255e-05, 'epoch': 1.05}\n", + "{'loss': 0.9734, 'grad_norm': 2.251491069793701, 'learning_rate': 9.81312123475006e-05, 'epoch': 1.07}\n", + "{'loss': 0.925, 'grad_norm': 2.160745859146118, 'learning_rate': 9.798793128904356e-05, 'epoch': 1.09}\n", + "{'loss': 1.0312, 'grad_norm': 2.9455161094665527, 'learning_rate': 9.78394710113631e-05, 'epoch': 1.11}\n", + "{'loss': 0.9867, 'grad_norm': 1.834627628326416, 'learning_rate': 9.768584753741134e-05, 'epoch': 1.12}\n", + "{'loss': 0.9962, 'grad_norm': 2.891728401184082, 'learning_rate': 9.752707744739145e-05, 'epoch': 1.14}\n", + "{'loss': 1.0046, 'grad_norm': 2.459664821624756, 'learning_rate': 9.736317787696816e-05, 'epoch': 1.16}\n", + "{'loss': 0.872, 'grad_norm': 2.503146171569824, 'learning_rate': 9.719416651541839e-05, 'epoch': 1.18}\n", + "{'loss': 0.9536, 'grad_norm': 1.9054204225540161, 'learning_rate': 9.702006160372209e-05, 'epoch': 1.2}\n", + "{'loss': 0.9768, 'grad_norm': 2.08803129196167, 'learning_rate': 9.684088193259355e-05, 'epoch': 1.21}\n", + "{'loss': 0.9448, 'grad_norm': 2.4227285385131836, 'learning_rate': 9.665664684045333e-05, 'epoch': 1.23}\n", + "{'loss': 1.0078, 'grad_norm': 2.396881103515625, 'learning_rate': 9.646737621134112e-05, 'epoch': 1.25}\n", + "{'loss': 0.9285, 'grad_norm': 4.0550384521484375, 'learning_rate': 9.627309047276974e-05, 'epoch': 1.27}\n", + "{'loss': 1.0518, 'grad_norm': 3.4381208419799805, 'learning_rate': 9.607381059352038e-05, 'epoch': 1.29}\n", + "{'loss': 1.0221, 'grad_norm': 2.341543674468994, 'learning_rate': 9.586955808137958e-05, 'epoch': 1.3}\n", + "{'loss': 1.0084, 'grad_norm': 2.660717725753784, 'learning_rate': 9.566035498081784e-05, 'epoch': 1.32}\n", + "{'loss': 1.0374, 'grad_norm': 2.4253923892974854, 'learning_rate': 9.544622387061055e-05, 'epoch': 1.34}\n", + "{'loss': 0.8872, 'grad_norm': 3.2932205200195312, 'learning_rate': 9.522718786140097e-05, 'epoch': 1.36}\n", + "{'loss': 1.0013, 'grad_norm': 3.3068909645080566, 'learning_rate': 9.500327059320606e-05, 'epoch': 1.37}\n", + "{'loss': 0.9135, 'grad_norm': 3.9048690795898438, 'learning_rate': 9.477449623286505e-05, 'epoch': 1.39}\n", + "{'loss': 0.8808, 'grad_norm': 2.9740893840789795, 'learning_rate': 9.454088947143116e-05, 'epoch': 1.41}\n", + "{'loss': 1.0511, 'grad_norm': 3.2612483501434326, 'learning_rate': 9.430247552150673e-05, 'epoch': 1.43}\n", + "{'loss': 0.9457, 'grad_norm': 2.8854198455810547, 'learning_rate': 9.405928011452211e-05, 'epoch': 1.45}\n", + "{'loss': 0.9401, 'grad_norm': 2.1029069423675537, 'learning_rate': 9.381132949795861e-05, 'epoch': 1.46}\n", + "{'loss': 1.0803, 'grad_norm': 3.1445486545562744, 'learning_rate': 9.35586504325155e-05, 'epoch': 1.48}\n", + "{'loss': 0.9944, 'grad_norm': 3.0867714881896973, 'learning_rate': 9.330127018922194e-05, 'epoch': 1.5}\n", + "{'loss': 0.8916, 'grad_norm': 2.589761257171631, 'learning_rate': 9.303921654649362e-05, 'epoch': 1.52}\n", + "{'loss': 0.988, 'grad_norm': 2.9633171558380127, 'learning_rate': 9.277251778713474e-05, 'epoch': 1.54}\n", + "{'loss': 0.9376, 'grad_norm': 3.082129716873169, 'learning_rate': 9.250120269528546e-05, 'epoch': 1.55}\n", + "{'loss': 0.9333, 'grad_norm': 2.1602373123168945, 'learning_rate': 9.22253005533154e-05, 'epoch': 1.57}\n", + "{'loss': 1.0027, 'grad_norm': 2.900174617767334, 'learning_rate': 9.194484113866313e-05, 'epoch': 1.59}\n", + "{'loss': 1.1305, 'grad_norm': 3.4030845165252686, 'learning_rate': 9.165985472062246e-05, 'epoch': 1.61}\n", + "{'loss': 0.8973, 'grad_norm': 2.5629944801330566, 'learning_rate': 9.137037205707552e-05, 'epoch': 1.62}\n", + "{'loss': 0.9483, 'grad_norm': 3.2390940189361572, 'learning_rate': 9.107642439117321e-05, 'epoch': 1.64}\n", + "{'loss': 0.9879, 'grad_norm': 2.5794193744659424, 'learning_rate': 9.077804344796302e-05, 'epoch': 1.66}\n", + "{'loss': 0.9668, 'grad_norm': 2.389864444732666, 'learning_rate': 9.04752614309652e-05, 'epoch': 1.68}\n", + "{'loss': 0.8852, 'grad_norm': 3.5650432109832764, 'learning_rate': 9.01681110186971e-05, 'epoch': 1.7}\n", + "{'loss': 0.9984, 'grad_norm': 3.166510581970215, 'learning_rate': 8.985662536114613e-05, 'epoch': 1.71}\n", + "{'loss': 0.9526, 'grad_norm': 2.176490306854248, 'learning_rate': 8.954083807619208e-05, 'epoch': 1.73}\n", + "{'loss': 1.0377, 'grad_norm': 3.0157470703125, 'learning_rate': 8.922078324597879e-05, 'epoch': 1.75}\n", + "{'loss': 1.1106, 'grad_norm': 2.781142234802246, 'learning_rate': 8.889649541323574e-05, 'epoch': 1.77}\n", + "{'loss': 1.0373, 'grad_norm': 3.456441879272461, 'learning_rate': 8.856800957755e-05, 'epoch': 1.78}\n", + "{'loss': 1.0307, 'grad_norm': 3.646578311920166, 'learning_rate': 8.823536119158864e-05, 'epoch': 1.8}\n", + "{'loss': 0.9769, 'grad_norm': 3.4664463996887207, 'learning_rate': 8.789858615727265e-05, 'epoch': 1.82}\n", + "{'loss': 0.9524, 'grad_norm': 2.52860951423645, 'learning_rate': 8.755772082190194e-05, 'epoch': 1.84}\n", + "{'loss': 1.0686, 'grad_norm': 3.0946435928344727, 'learning_rate': 8.721280197423258e-05, 'epoch': 1.86}\n", + "{'loss': 0.9359, 'grad_norm': 3.146989583969116, 'learning_rate': 8.68638668405062e-05, 'epoch': 1.87}\n", + "{'loss': 1.0035, 'grad_norm': 3.2309892177581787, 'learning_rate': 8.651095308043232e-05, 'epoch': 1.89}\n", + "{'loss': 1.0669, 'grad_norm': 3.8748905658721924, 'learning_rate': 8.61540987831238e-05, 'epoch': 1.91}\n", + "{'loss': 1.0676, 'grad_norm': 3.329939603805542, 'learning_rate': 8.579334246298593e-05, 'epoch': 1.93}\n", + "{'loss': 0.9976, 'grad_norm': 3.7491514682769775, 'learning_rate': 8.542872305555978e-05, 'epoch': 1.95}\n", + "{'loss': 0.9471, 'grad_norm': 3.245119571685791, 'learning_rate': 8.50602799133199e-05, 'epoch': 1.96}\n", + "{'loss': 0.9998, 'grad_norm': 2.7840590476989746, 'learning_rate': 8.468805280142709e-05, 'epoch': 1.98}\n", + "{'loss': 1.0361, 'grad_norm': 3.2855234146118164, 'learning_rate': 8.43120818934367e-05, 'epoch': 2.0}\n", + " 33%|███████████▋ | 1120/3360 [2:09:53<4:22:41, 7.04s/it][INFO|trainer.py:3788] 2024-07-04 17:36:15,576 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 17:36:15,578 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 17:36:15,580 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-7b/lora/sft/checkpoint-1120\n", + "[INFO|configuration_utils.py:733] 2024-07-04 17:36:31,166 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 17:36:31,166 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 17:36:31,345 >> tokenizer config file saved in saves/qwen2-7b/lora/sft/checkpoint-1120/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 17:36:31,345 >> Special tokens file saved in saves/qwen2-7b/lora/sft/checkpoint-1120/special_tokens_map.json\n", + "{'loss': 0.4204, 'grad_norm': 3.7729196548461914, 'learning_rate': 8.393240776696274e-05, 'epoch': 2.02}\n", + "{'loss': 0.3656, 'grad_norm': 2.241997718811035, 'learning_rate': 8.354907139929851e-05, 'epoch': 2.03}\n", + "{'loss': 0.3554, 'grad_norm': 3.0927772521972656, 'learning_rate': 8.316211416299397e-05, 'epoch': 2.05}\n", + "{'loss': 0.3576, 'grad_norm': 4.350724697113037, 'learning_rate': 8.27715778213905e-05, 'epoch': 2.07}\n", + "{'loss': 0.2409, 'grad_norm': 2.0694334506988525, 'learning_rate': 8.237750452411353e-05, 'epoch': 2.09}\n", + "{'loss': 0.3769, 'grad_norm': 2.3954668045043945, 'learning_rate': 8.197993680252334e-05, 'epoch': 2.11}\n", + "{'loss': 0.4252, 'grad_norm': 2.997573137283325, 'learning_rate': 8.157891756512488e-05, 'epoch': 2.12}\n", + "{'loss': 0.3827, 'grad_norm': 3.1807985305786133, 'learning_rate': 8.117449009293668e-05, 'epoch': 2.14}\n", + "{'loss': 0.3291, 'grad_norm': 4.341946125030518, 'learning_rate': 8.076669803481965e-05, 'epoch': 2.16}\n", + "{'loss': 0.3049, 'grad_norm': 4.446887493133545, 'learning_rate': 8.035558540276618e-05, 'epoch': 2.18}\n", + "{'loss': 0.3638, 'grad_norm': 2.7504091262817383, 'learning_rate': 7.994119656715002e-05, 'epoch': 2.2}\n", + "{'loss': 0.4044, 'grad_norm': 2.769212484359741, 'learning_rate': 7.952357625193749e-05, 'epoch': 2.21}\n", + "{'loss': 0.3703, 'grad_norm': 5.1489362716674805, 'learning_rate': 7.91027695298606e-05, 'epoch': 2.23}\n", + "{'loss': 0.3866, 'grad_norm': 4.870989799499512, 'learning_rate': 7.86788218175523e-05, 'epoch': 2.25}\n", + "{'loss': 0.383, 'grad_norm': 2.3800389766693115, 'learning_rate': 7.8251778870645e-05, 'epoch': 2.27}\n", + "{'loss': 0.3855, 'grad_norm': 3.800349473953247, 'learning_rate': 7.782168677883206e-05, 'epoch': 2.28}\n", + "{'loss': 0.4051, 'grad_norm': 2.723214864730835, 'learning_rate': 7.738859196089358e-05, 'epoch': 2.3}\n", + "{'loss': 0.4282, 'grad_norm': 3.5306265354156494, 'learning_rate': 7.695254115968648e-05, 'epoch': 2.32}\n", + "{'loss': 0.4128, 'grad_norm': 2.6264665126800537, 'learning_rate': 7.651358143709972e-05, 'epoch': 2.34}\n", + "{'loss': 0.4174, 'grad_norm': 3.427201747894287, 'learning_rate': 7.60717601689749e-05, 'epoch': 2.36}\n", + "{'loss': 0.3553, 'grad_norm': 3.8674330711364746, 'learning_rate': 7.562712503999327e-05, 'epoch': 2.37}\n", + "{'loss': 0.4509, 'grad_norm': 3.253030776977539, 'learning_rate': 7.517972403852905e-05, 'epoch': 2.39}\n", + "{'loss': 0.3599, 'grad_norm': 3.4824795722961426, 'learning_rate': 7.472960545147038e-05, 'epoch': 2.41}\n", + "{'loss': 0.3248, 'grad_norm': 4.311473369598389, 'learning_rate': 7.427681785900761e-05, 'epoch': 2.43}\n", + "{'loss': 0.3835, 'grad_norm': 3.2026665210723877, 'learning_rate': 7.382141012939034e-05, 'epoch': 2.45}\n", + "{'loss': 0.3631, 'grad_norm': 3.4886059761047363, 'learning_rate': 7.33634314136531e-05, 'epoch': 2.46}\n", + "{'loss': 0.4001, 'grad_norm': 2.1931118965148926, 'learning_rate': 7.290293114031061e-05, 'epoch': 2.48}\n", + "{'loss': 0.4094, 'grad_norm': 3.082930564880371, 'learning_rate': 7.243995901002312e-05, 'epoch': 2.5}\n", + "{'loss': 0.3916, 'grad_norm': 2.8144562244415283, 'learning_rate': 7.197456499023225e-05, 'epoch': 2.52}\n", + "{'loss': 0.4212, 'grad_norm': 3.546799898147583, 'learning_rate': 7.150679930976825e-05, 'epoch': 2.53}\n", + "{'loss': 0.3852, 'grad_norm': 3.623589038848877, 'learning_rate': 7.103671245342887e-05, 'epoch': 2.55}\n", + "{'loss': 0.3294, 'grad_norm': 3.896050214767456, 'learning_rate': 7.056435515653059e-05, 'epoch': 2.57}\n", + "{'loss': 0.4378, 'grad_norm': 2.8549437522888184, 'learning_rate': 7.008977839943299e-05, 'epoch': 2.59}\n", + "{'loss': 0.3744, 'grad_norm': 2.963679313659668, 'learning_rate': 6.961303340203653e-05, 'epoch': 2.61}\n", + "{'loss': 0.4083, 'grad_norm': 3.584379196166992, 'learning_rate': 6.91341716182545e-05, 'epoch': 2.62}\n", + "{'loss': 0.3875, 'grad_norm': 3.231067180633545, 'learning_rate': 6.86532447304597e-05, 'epoch': 2.64}\n", + "{'loss': 0.3555, 'grad_norm': 3.2355687618255615, 'learning_rate': 6.817030464390656e-05, 'epoch': 2.66}\n", + "{'loss': 0.3962, 'grad_norm': 4.36820125579834, 'learning_rate': 6.768540348112907e-05, 'epoch': 2.68}\n", + "{'loss': 0.3224, 'grad_norm': 2.6882545948028564, 'learning_rate': 6.719859357631535e-05, 'epoch': 2.7}\n", + "{'loss': 0.3478, 'grad_norm': 3.5584182739257812, 'learning_rate': 6.670992746965938e-05, 'epoch': 2.71}\n", + "{'loss': 0.4298, 'grad_norm': 4.19834041595459, 'learning_rate': 6.621945790169036e-05, 'epoch': 2.73}\n", + "{'loss': 0.4304, 'grad_norm': 4.770883083343506, 'learning_rate': 6.572723780758069e-05, 'epoch': 2.75}\n", + "{'loss': 0.3657, 'grad_norm': 4.010149955749512, 'learning_rate': 6.523332031143272e-05, 'epoch': 2.77}\n", + "{'loss': 0.3699, 'grad_norm': 3.2105469703674316, 'learning_rate': 6.473775872054521e-05, 'epoch': 2.78}\n", + "{'loss': 0.3342, 'grad_norm': 3.494490146636963, 'learning_rate': 6.424060651966007e-05, 'epoch': 2.8}\n", + "{'loss': 0.327, 'grad_norm': 3.291541814804077, 'learning_rate': 6.374191736518974e-05, 'epoch': 2.82}\n", + "{'loss': 0.3928, 'grad_norm': 3.125520706176758, 'learning_rate': 6.324174507942637e-05, 'epoch': 2.84}\n", + "{'loss': 0.3776, 'grad_norm': 4.660810470581055, 'learning_rate': 6.274014364473274e-05, 'epoch': 2.86}\n", + "{'loss': 0.4623, 'grad_norm': 2.8751118183135986, 'learning_rate': 6.22371671977162e-05, 'epoch': 2.87}\n", + "{'loss': 0.4122, 'grad_norm': 4.0637078285217285, 'learning_rate': 6.173287002338577e-05, 'epoch': 2.89}\n", + "{'loss': 0.4056, 'grad_norm': 3.7399301528930664, 'learning_rate': 6.122730654929334e-05, 'epoch': 2.91}\n", + "{'loss': 0.3351, 'grad_norm': 4.581759452819824, 'learning_rate': 6.072053133965938e-05, 'epoch': 2.93}\n", + "{'loss': 0.3849, 'grad_norm': 3.381431818008423, 'learning_rate': 6.021259908948402e-05, 'epoch': 2.95}\n", + "{'loss': 0.3947, 'grad_norm': 4.740965366363525, 'learning_rate': 5.970356461864391e-05, 'epoch': 2.96}\n", + "{'loss': 0.3945, 'grad_norm': 5.124401569366455, 'learning_rate': 5.919348286597569e-05, 'epoch': 2.98}\n", + "{'loss': 0.4098, 'grad_norm': 3.3869075775146484, 'learning_rate': 5.868240888334653e-05, 'epoch': 3.0}\n", + " 50%|█████████████████▌ | 1680/3360 [3:13:07<3:03:37, 6.56s/it][INFO|trainer.py:3788] 2024-07-04 18:39:30,098 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 18:39:30,098 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 18:39:30,098 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-7b/lora/sft/checkpoint-1680\n", + "[INFO|configuration_utils.py:733] 2024-07-04 18:39:46,491 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 18:39:46,492 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 18:39:46,762 >> tokenizer config file saved in saves/qwen2-7b/lora/sft/checkpoint-1680/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 18:39:46,762 >> Special tokens file saved in saves/qwen2-7b/lora/sft/checkpoint-1680/special_tokens_map.json\n", + "{'loss': 0.1954, 'grad_norm': 2.6332297325134277, 'learning_rate': 5.8170397829712485e-05, 'epoch': 3.02}\n", + "{'loss': 0.0883, 'grad_norm': 3.9817214012145996, 'learning_rate': 5.765750496516547e-05, 'epoch': 3.03}\n", + "{'loss': 0.1392, 'grad_norm': 1.9517065286636353, 'learning_rate': 5.714378564496901e-05, 'epoch': 3.05}\n", + "{'loss': 0.1231, 'grad_norm': 1.8976528644561768, 'learning_rate': 5.6629295313583974e-05, 'epoch': 3.07}\n", + "{'loss': 0.1299, 'grad_norm': 1.4403581619262695, 'learning_rate': 5.611408949868457e-05, 'epoch': 3.09}\n", + "{'loss': 0.1068, 'grad_norm': 3.757260322570801, 'learning_rate': 5.559822380516539e-05, 'epoch': 3.11}\n", + "{'loss': 0.0975, 'grad_norm': 1.9946837425231934, 'learning_rate': 5.5081753909140096e-05, 'epoch': 3.12}\n", + "{'loss': 0.1205, 'grad_norm': 3.075326442718506, 'learning_rate': 5.456473555193242e-05, 'epoch': 3.14}\n", + "{'loss': 0.1226, 'grad_norm': 2.4876396656036377, 'learning_rate': 5.404722453406017e-05, 'epoch': 3.16}\n", + "{'loss': 0.1144, 'grad_norm': 2.430744171142578, 'learning_rate': 5.3529276709212816e-05, 'epoch': 3.18}\n", + "{'loss': 0.1399, 'grad_norm': 3.6195318698883057, 'learning_rate': 5.30109479782233e-05, 'epoch': 3.2}\n", + "{'loss': 0.1156, 'grad_norm': 3.914135217666626, 'learning_rate': 5.249229428303486e-05, 'epoch': 3.21}\n", + "{'loss': 0.1372, 'grad_norm': 1.994607925415039, 'learning_rate': 5.197337160066331e-05, 'epoch': 3.23}\n", + "{'loss': 0.1138, 'grad_norm': 1.6210600137710571, 'learning_rate': 5.145423593715557e-05, 'epoch': 3.25}\n", + "{'loss': 0.14, 'grad_norm': 2.50508713722229, 'learning_rate': 5.0934943321545115e-05, 'epoch': 3.27}\n", + "{'loss': 0.1152, 'grad_norm': 4.362739562988281, 'learning_rate': 5.041554979980486e-05, 'epoch': 3.28}\n", + "{'loss': 0.1549, 'grad_norm': 3.601013422012329, 'learning_rate': 4.9896111428798254e-05, 'epoch': 3.3}\n", + "{'loss': 0.1429, 'grad_norm': 2.076098680496216, 'learning_rate': 4.9376684270229254e-05, 'epoch': 3.32}\n", + "{'loss': 0.1353, 'grad_norm': 1.633200764656067, 'learning_rate': 4.8857324384591653e-05, 'epoch': 3.34}\n", + "{'loss': 0.1284, 'grad_norm': 4.053235054016113, 'learning_rate': 4.8338087825118675e-05, 'epoch': 3.36}\n", + "{'loss': 0.1526, 'grad_norm': 2.4892356395721436, 'learning_rate': 4.781903063173321e-05, 'epoch': 3.37}\n", + "{'loss': 0.1042, 'grad_norm': 1.8938469886779785, 'learning_rate': 4.730020882499964e-05, 'epoch': 3.39}\n", + "{'loss': 0.1569, 'grad_norm': 1.758270502090454, 'learning_rate': 4.678167840007767e-05, 'epoch': 3.41}\n", + "{'loss': 0.117, 'grad_norm': 1.9446786642074585, 'learning_rate': 4.626349532067879e-05, 'epoch': 3.43}\n", + "{'loss': 0.1603, 'grad_norm': 2.5028741359710693, 'learning_rate': 4.574571551302647e-05, 'epoch': 3.44}\n", + "{'loss': 0.1528, 'grad_norm': 3.524077892303467, 'learning_rate': 4.522839485981994e-05, 'epoch': 3.46}\n", + "{'loss': 0.1366, 'grad_norm': 2.425860643386841, 'learning_rate': 4.471158919420312e-05, 'epoch': 3.48}\n", + "{'loss': 0.1231, 'grad_norm': 2.6059088706970215, 'learning_rate': 4.4195354293738484e-05, 'epoch': 3.5}\n", + "{'loss': 0.1479, 'grad_norm': 3.934004783630371, 'learning_rate': 4.367974587438733e-05, 'epoch': 3.52}\n", + "{'loss': 0.1466, 'grad_norm': 2.3225414752960205, 'learning_rate': 4.316481958449634e-05, 'epoch': 3.53}\n", + "{'loss': 0.1161, 'grad_norm': 3.3421878814697266, 'learning_rate': 4.2650630998791615e-05, 'epoch': 3.55}\n", + "{'loss': 0.1312, 'grad_norm': 2.411162853240967, 'learning_rate': 4.213723561238074e-05, 'epoch': 3.57}\n", + "{'loss': 0.1144, 'grad_norm': 2.74504017829895, 'learning_rate': 4.162468883476319e-05, 'epoch': 3.59}\n", + "{'loss': 0.1303, 'grad_norm': 3.3871073722839355, 'learning_rate': 4.111304598385018e-05, 'epoch': 3.61}\n", + "{'loss': 0.1272, 'grad_norm': 2.4120686054229736, 'learning_rate': 4.060236227999441e-05, 'epoch': 3.62}\n", + "{'loss': 0.1127, 'grad_norm': 2.2959489822387695, 'learning_rate': 4.0092692840030134e-05, 'epoch': 3.64}\n", + "{'loss': 0.131, 'grad_norm': 2.5716683864593506, 'learning_rate': 3.9584092671324606e-05, 'epoch': 3.66}\n", + "{'loss': 0.1512, 'grad_norm': 3.035562753677368, 'learning_rate': 3.907661666584131e-05, 'epoch': 3.68}\n", + "{'loss': 0.1253, 'grad_norm': 2.897613048553467, 'learning_rate': 3.857031959421553e-05, 'epoch': 3.69}\n", + "{'loss': 0.1084, 'grad_norm': 2.2627975940704346, 'learning_rate': 3.806525609984312e-05, 'epoch': 3.71}\n", + "{'loss': 0.105, 'grad_norm': 2.2742927074432373, 'learning_rate': 3.7561480692983006e-05, 'epoch': 3.73}\n", + "{'loss': 0.1489, 'grad_norm': 1.9651683568954468, 'learning_rate': 3.705904774487396e-05, 'epoch': 3.75}\n", + "{'loss': 0.1448, 'grad_norm': 4.107623100280762, 'learning_rate': 3.655801148186655e-05, 'epoch': 3.77}\n", + "{'loss': 0.0998, 'grad_norm': 2.270852565765381, 'learning_rate': 3.6058425979570485e-05, 'epoch': 3.78}\n", + "{'loss': 0.1176, 'grad_norm': 3.770810842514038, 'learning_rate': 3.556034515701852e-05, 'epoch': 3.8}\n", + "{'loss': 0.1175, 'grad_norm': 4.139482498168945, 'learning_rate': 3.506382277084696e-05, 'epoch': 3.82}\n", + "{'loss': 0.152, 'grad_norm': 2.7534141540527344, 'learning_rate': 3.4568912409493945e-05, 'epoch': 3.84}\n", + "{'loss': 0.0974, 'grad_norm': 2.224083423614502, 'learning_rate': 3.4075667487415785e-05, 'epoch': 3.86}\n", + "{'loss': 0.1133, 'grad_norm': 1.7634135484695435, 'learning_rate': 3.358414123932195e-05, 'epoch': 3.87}\n", + "{'loss': 0.1311, 'grad_norm': 2.7758963108062744, 'learning_rate': 3.3094386714429724e-05, 'epoch': 3.89}\n", + "{'loss': 0.1341, 'grad_norm': 2.842358350753784, 'learning_rate': 3.2606456770738636e-05, 'epoch': 3.91}\n", + "{'loss': 0.0884, 'grad_norm': 1.71796452999115, 'learning_rate': 3.212040406932569e-05, 'epoch': 3.93}\n", + "{'loss': 0.0956, 'grad_norm': 2.689420461654663, 'learning_rate': 3.163628106866172e-05, 'epoch': 3.94}\n", + "{'loss': 0.1731, 'grad_norm': 2.630415439605713, 'learning_rate': 3.115414001894974e-05, 'epoch': 3.96}\n", + "{'loss': 0.1458, 'grad_norm': 2.928737163543701, 'learning_rate': 3.067403295648566e-05, 'epoch': 3.98}\n", + "{'loss': 0.1278, 'grad_norm': 2.467090129852295, 'learning_rate': 3.019601169804216e-05, 'epoch': 4.0}\n", + " 67%|███████████████████████▎ | 2240/3360 [4:14:45<2:03:53, 6.64s/it][INFO|trainer.py:3788] 2024-07-04 19:41:08,043 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 19:41:08,044 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 19:41:08,044 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-7b/lora/sft/checkpoint-2240\n", + "[INFO|configuration_utils.py:733] 2024-07-04 19:41:22,728 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 19:41:22,729 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 19:41:22,849 >> tokenizer config file saved in saves/qwen2-7b/lora/sft/checkpoint-2240/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 19:41:22,850 >> Special tokens file saved in saves/qwen2-7b/lora/sft/checkpoint-2240/special_tokens_map.json\n", + "{'loss': 0.0535, 'grad_norm': 0.6558727025985718, 'learning_rate': 2.9720127835276256e-05, 'epoch': 4.02}\n", + "{'loss': 0.0402, 'grad_norm': 2.1889960765838623, 'learning_rate': 2.9246432729161055e-05, 'epoch': 4.03}\n", + "{'loss': 0.0377, 'grad_norm': 0.8100994229316711, 'learning_rate': 2.8774977504442647e-05, 'epoch': 4.05}\n", + "{'loss': 0.0398, 'grad_norm': 2.8209896087646484, 'learning_rate': 2.8305813044122097e-05, 'epoch': 4.07}\n", + "{'loss': 0.0308, 'grad_norm': 1.5516138076782227, 'learning_rate': 2.7838989983964065e-05, 'epoch': 4.09}\n", + "{'loss': 0.0597, 'grad_norm': 4.609562873840332, 'learning_rate': 2.737455870703155e-05, 'epoch': 4.11}\n", + "{'loss': 0.0231, 'grad_norm': 2.7549400329589844, 'learning_rate': 2.6912569338248315e-05, 'epoch': 4.12}\n", + "{'loss': 0.0448, 'grad_norm': 5.040008068084717, 'learning_rate': 2.645307173898901e-05, 'epoch': 4.14}\n", + "{'loss': 0.0253, 'grad_norm': 1.6336179971694946, 'learning_rate': 2.5996115501697694e-05, 'epoch': 4.16}\n", + "{'loss': 0.0289, 'grad_norm': 0.8074469566345215, 'learning_rate': 2.5541749944535554e-05, 'epoch': 4.18}\n", + "{'loss': 0.0338, 'grad_norm': 1.710808277130127, 'learning_rate': 2.5090024106057962e-05, 'epoch': 4.19}\n", + "{'loss': 0.0379, 'grad_norm': 2.1768016815185547, 'learning_rate': 2.464098673992205e-05, 'epoch': 4.21}\n", + "{'loss': 0.0682, 'grad_norm': 3.6282131671905518, 'learning_rate': 2.4194686309624663e-05, 'epoch': 4.23}\n", + "{'loss': 0.0515, 'grad_norm': 1.100537896156311, 'learning_rate': 2.3751170983272e-05, 'epoch': 4.25}\n", + "{'loss': 0.0354, 'grad_norm': 0.6081830859184265, 'learning_rate': 2.3310488628380757e-05, 'epoch': 4.27}\n", + "{'loss': 0.0334, 'grad_norm': 1.5605361461639404, 'learning_rate': 2.2872686806712035e-05, 'epoch': 4.28}\n", + "{'loss': 0.0492, 'grad_norm': 2.5406620502471924, 'learning_rate': 2.243781276913811e-05, 'epoch': 4.3}\n", + "{'loss': 0.0279, 'grad_norm': 2.160897970199585, 'learning_rate': 2.200591345054267e-05, 'epoch': 4.32}\n", + "{'loss': 0.0342, 'grad_norm': 2.3391342163085938, 'learning_rate': 2.157703546475539e-05, 'epoch': 4.34}\n", + "{'loss': 0.0332, 'grad_norm': 1.3248311281204224, 'learning_rate': 2.115122509952085e-05, 'epoch': 4.36}\n", + "{'loss': 0.0334, 'grad_norm': 2.741152763366699, 'learning_rate': 2.0728528311502976e-05, 'epoch': 4.37}\n", + "{'loss': 0.0542, 'grad_norm': 2.237809419631958, 'learning_rate': 2.0308990721324927e-05, 'epoch': 4.39}\n", + "{'loss': 0.0344, 'grad_norm': 3.8997409343719482, 'learning_rate': 1.989265760864542e-05, 'epoch': 4.41}\n", + "{'loss': 0.0439, 'grad_norm': 0.6022194623947144, 'learning_rate': 1.947957390727185e-05, 'epoch': 4.43}\n", + "{'loss': 0.0346, 'grad_norm': 1.2296243906021118, 'learning_rate': 1.906978420031059e-05, 'epoch': 4.44}\n", + "{'loss': 0.0209, 'grad_norm': 0.28131213784217834, 'learning_rate': 1.8663332715355396e-05, 'epoch': 4.46}\n", + "{'loss': 0.0271, 'grad_norm': 2.75640606880188, 'learning_rate': 1.8260263319713844e-05, 'epoch': 4.48}\n", + "{'loss': 0.0408, 'grad_norm': 3.289303779602051, 'learning_rate': 1.7860619515673033e-05, 'epoch': 4.5}\n", + "{'loss': 0.0344, 'grad_norm': 1.2157098054885864, 'learning_rate': 1.746444443580433e-05, 'epoch': 4.52}\n", + "{'loss': 0.0272, 'grad_norm': 1.5058122873306274, 'learning_rate': 1.7071780838308288e-05, 'epoch': 4.53}\n", + "{'loss': 0.0283, 'grad_norm': 1.8522496223449707, 'learning_rate': 1.6682671102399805e-05, 'epoch': 4.55}\n", + "{'loss': 0.027, 'grad_norm': 2.126176595687866, 'learning_rate': 1.629715722373423e-05, 'epoch': 4.57}\n", + "{'loss': 0.0434, 'grad_norm': 2.065514326095581, 'learning_rate': 1.5915280809874932e-05, 'epoch': 4.59}\n", + "{'loss': 0.0427, 'grad_norm': 2.2047812938690186, 'learning_rate': 1.553708307580265e-05, 'epoch': 4.61}\n", + "{'loss': 0.0266, 'grad_norm': 2.1723501682281494, 'learning_rate': 1.5162604839467265e-05, 'epoch': 4.62}\n", + "{'loss': 0.0201, 'grad_norm': 1.7166253328323364, 'learning_rate': 1.4791886517382413e-05, 'epoch': 4.64}\n", + "{'loss': 0.0306, 'grad_norm': 0.5556966066360474, 'learning_rate': 1.4424968120263504e-05, 'epoch': 4.66}\n", + "{'loss': 0.0249, 'grad_norm': 1.101198434829712, 'learning_rate': 1.4061889248709343e-05, 'epoch': 4.68}\n", + "{'loss': 0.0324, 'grad_norm': 0.6396570801734924, 'learning_rate': 1.370268908892825e-05, 'epoch': 4.69}\n", + "{'loss': 0.0303, 'grad_norm': 2.5093636512756348, 'learning_rate': 1.3347406408508695e-05, 'epoch': 4.71}\n", + "{'loss': 0.0522, 'grad_norm': 1.5739742517471313, 'learning_rate': 1.2996079552235263e-05, 'epoch': 4.73}\n", + "{'loss': 0.0293, 'grad_norm': 0.9539183974266052, 'learning_rate': 1.264874643795021e-05, 'epoch': 4.75}\n", + "{'loss': 0.0289, 'grad_norm': 0.5063753724098206, 'learning_rate': 1.230544455246101e-05, 'epoch': 4.77}\n", + "{'loss': 0.0457, 'grad_norm': 1.6972631216049194, 'learning_rate': 1.1966210947494583e-05, 'epoch': 4.78}\n", + "{'loss': 0.0228, 'grad_norm': 0.8949175477027893, 'learning_rate': 1.1631082235698316e-05, 'epoch': 4.8}\n", + "{'loss': 0.0345, 'grad_norm': 1.8337916135787964, 'learning_rate': 1.130009458668863e-05, 'epoch': 4.82}\n", + "{'loss': 0.0221, 'grad_norm': 2.356985569000244, 'learning_rate': 1.097328372314721e-05, 'epoch': 4.84}\n", + "{'loss': 0.0328, 'grad_norm': 2.9775609970092773, 'learning_rate': 1.0650684916965559e-05, 'epoch': 4.85}\n", + "{'loss': 0.0298, 'grad_norm': 2.2749829292297363, 'learning_rate': 1.0332332985438248e-05, 'epoch': 4.87}\n", + "{'loss': 0.0411, 'grad_norm': 1.9781012535095215, 'learning_rate': 1.0018262287505086e-05, 'epoch': 4.89}\n", + "{'loss': 0.0461, 'grad_norm': 1.8106870651245117, 'learning_rate': 9.708506720042932e-06, 'epoch': 4.91}\n", + "{'loss': 0.0354, 'grad_norm': 1.3991378545761108, 'learning_rate': 9.403099714207175e-06, 'epoch': 4.93}\n", + "{'loss': 0.0269, 'grad_norm': 0.6455625891685486, 'learning_rate': 9.102074231823727e-06, 'epoch': 4.94}\n", + "{'loss': 0.0339, 'grad_norm': 1.2710880041122437, 'learning_rate': 8.805462761831418e-06, 'epoch': 4.96}\n", + "{'loss': 0.0334, 'grad_norm': 1.1816545724868774, 'learning_rate': 8.513297316775625e-06, 'epoch': 4.98}\n", + "{'loss': 0.0301, 'grad_norm': 1.668415904045105, 'learning_rate': 8.225609429353187e-06, 'epoch': 5.0}\n", + " 83%|█████████████████████████████▏ | 2800/3360 [5:16:56<1:03:45, 6.83s/it][INFO|trainer.py:3788] 2024-07-04 20:43:18,672 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 20:43:18,672 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 20:43:18,673 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-7b/lora/sft/checkpoint-2800\n", + "[INFO|configuration_utils.py:733] 2024-07-04 20:43:32,430 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 20:43:32,431 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 20:43:32,551 >> tokenizer config file saved in saves/qwen2-7b/lora/sft/checkpoint-2800/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 20:43:32,551 >> Special tokens file saved in saves/qwen2-7b/lora/sft/checkpoint-2800/special_tokens_map.json\n", + "{'loss': 0.0078, 'grad_norm': 1.6351463794708252, 'learning_rate': 7.942430149009161e-06, 'epoch': 5.02}\n", + "{'loss': 0.0062, 'grad_norm': 0.11965573579072952, 'learning_rate': 7.663790038585793e-06, 'epoch': 5.03}\n", + "{'loss': 0.0105, 'grad_norm': 0.05803072825074196, 'learning_rate': 7.389719171023857e-06, 'epoch': 5.05}\n", + "{'loss': 0.0058, 'grad_norm': 0.143271803855896, 'learning_rate': 7.1202471261170245e-06, 'epoch': 5.07}\n", + "{'loss': 0.0035, 'grad_norm': 0.17391343414783478, 'learning_rate': 6.855402987319348e-06, 'epoch': 5.09}\n", + "{'loss': 0.0072, 'grad_norm': 0.19679808616638184, 'learning_rate': 6.595215338606397e-06, 'epoch': 5.1}\n", + "{'loss': 0.0123, 'grad_norm': 0.09687481820583344, 'learning_rate': 6.339712261390213e-06, 'epoch': 5.12}\n", + "{'loss': 0.0057, 'grad_norm': 0.069660983979702, 'learning_rate': 6.088921331488568e-06, 'epoch': 5.14}\n", + "{'loss': 0.0072, 'grad_norm': 1.3626017570495605, 'learning_rate': 5.8428696161488215e-06, 'epoch': 5.16}\n", + "{'loss': 0.0047, 'grad_norm': 2.0419363975524902, 'learning_rate': 5.601583671126531e-06, 'epoch': 5.18}\n", + "{'loss': 0.0097, 'grad_norm': 0.2337513566017151, 'learning_rate': 5.365089537819434e-06, 'epoch': 5.19}\n", + "{'loss': 0.0042, 'grad_norm': 0.05815720558166504, 'learning_rate': 5.133412740456806e-06, 'epoch': 5.21}\n", + "{'loss': 0.008, 'grad_norm': 1.3515617847442627, 'learning_rate': 4.906578283344759e-06, 'epoch': 5.23}\n", + "{'loss': 0.0087, 'grad_norm': 0.37659117579460144, 'learning_rate': 4.684610648167503e-06, 'epoch': 5.25}\n", + "{'loss': 0.0031, 'grad_norm': 0.33385252952575684, 'learning_rate': 4.467533791345191e-06, 'epoch': 5.27}\n", + "{'loss': 0.0067, 'grad_norm': 0.15747712552547455, 'learning_rate': 4.255371141448272e-06, 'epoch': 5.28}\n", + "{'loss': 0.007, 'grad_norm': 1.2530337572097778, 'learning_rate': 4.048145596668967e-06, 'epoch': 5.3}\n", + "{'loss': 0.0136, 'grad_norm': 2.182263135910034, 'learning_rate': 3.84587952234991e-06, 'epoch': 5.32}\n", + "{'loss': 0.0035, 'grad_norm': 1.1545133590698242, 'learning_rate': 3.6485947485702832e-06, 'epoch': 5.34}\n", + "{'loss': 0.0061, 'grad_norm': 0.33282843232154846, 'learning_rate': 3.4563125677897932e-06, 'epoch': 5.35}\n", + "{'loss': 0.004, 'grad_norm': 0.2662621736526489, 'learning_rate': 3.269053732550581e-06, 'epoch': 5.37}\n", + "{'loss': 0.0071, 'grad_norm': 1.1687767505645752, 'learning_rate': 3.086838453237506e-06, 'epoch': 5.39}\n", + "{'loss': 0.0082, 'grad_norm': 0.12040398269891739, 'learning_rate': 2.9096863958968268e-06, 'epoch': 5.41}\n", + "{'loss': 0.0042, 'grad_norm': 0.22544123232364655, 'learning_rate': 2.737616680113758e-06, 'epoch': 5.43}\n", + "{'loss': 0.0056, 'grad_norm': 0.3548804521560669, 'learning_rate': 2.570647876948895e-06, 'epoch': 5.44}\n", + "{'loss': 0.0133, 'grad_norm': 0.7295147180557251, 'learning_rate': 2.408798006933882e-06, 'epoch': 5.46}\n", + "{'loss': 0.0125, 'grad_norm': 0.05939454585313797, 'learning_rate': 2.252084538126542e-06, 'epoch': 5.48}\n", + "{'loss': 0.0064, 'grad_norm': 0.5182624459266663, 'learning_rate': 2.100524384225555e-06, 'epoch': 5.5}\n", + "{'loss': 0.0043, 'grad_norm': 0.13460208475589752, 'learning_rate': 1.9541339027450256e-06, 'epoch': 5.52}\n", + "{'loss': 0.0066, 'grad_norm': 0.8837604522705078, 'learning_rate': 1.8129288932490274e-06, 'epoch': 5.53}\n", + "{'loss': 0.0092, 'grad_norm': 0.332492858171463, 'learning_rate': 1.6769245956464396e-06, 'epoch': 5.55}\n", + "{'loss': 0.0048, 'grad_norm': 0.2933903634548187, 'learning_rate': 1.5461356885461075e-06, 'epoch': 5.57}\n", + "{'loss': 0.0054, 'grad_norm': 0.371267706155777, 'learning_rate': 1.4205762876726092e-06, 'epoch': 5.59}\n", + "{'loss': 0.0083, 'grad_norm': 0.14521144330501556, 'learning_rate': 1.3002599443428243e-06, 'epoch': 5.6}\n", + "{'loss': 0.0073, 'grad_norm': 1.345499038696289, 'learning_rate': 1.1851996440033319e-06, 'epoch': 5.62}\n", + "{'loss': 0.0064, 'grad_norm': 0.025303443893790245, 'learning_rate': 1.0754078048289374e-06, 'epoch': 5.64}\n", + "{'loss': 0.0049, 'grad_norm': 1.9373172521591187, 'learning_rate': 9.708962763824048e-07, 'epoch': 5.66}\n", + "{'loss': 0.0063, 'grad_norm': 0.6459546685218811, 'learning_rate': 8.716763383355864e-07, 'epoch': 5.68}\n", + "{'loss': 0.005, 'grad_norm': 1.4349000453948975, 'learning_rate': 7.777586992519959e-07, 'epoch': 5.69}\n", + "{'loss': 0.0103, 'grad_norm': 0.5553787350654602, 'learning_rate': 6.891534954310885e-07, 'epoch': 5.71}\n", + "{'loss': 0.0054, 'grad_norm': 0.19051159918308258, 'learning_rate': 6.058702898142643e-07, 'epoch': 5.73}\n", + "{'loss': 0.0059, 'grad_norm': 0.36273324489593506, 'learning_rate': 5.279180709527765e-07, 'epoch': 5.75}\n", + "{'loss': 0.0084, 'grad_norm': 0.4064849019050598, 'learning_rate': 4.553052520375911e-07, 'epoch': 5.77}\n", + "{'loss': 0.0033, 'grad_norm': 0.2132396250963211, 'learning_rate': 3.8803966999139684e-07, 'epoch': 5.78}\n", + "{'loss': 0.0176, 'grad_norm': 2.6782572269439697, 'learning_rate': 3.261285846227868e-07, 'epoch': 5.8}\n", + "{'loss': 0.0064, 'grad_norm': 0.27686187624931335, 'learning_rate': 2.6957867784270787e-07, 'epoch': 5.82}\n", + "{'loss': 0.0041, 'grad_norm': 0.86066734790802, 'learning_rate': 2.1839605294330933e-07, 'epoch': 5.84}\n", + "{'loss': 0.0082, 'grad_norm': 0.16934335231781006, 'learning_rate': 1.725862339392259e-07, 'epoch': 5.85}\n", + "{'loss': 0.0047, 'grad_norm': 0.6522320508956909, 'learning_rate': 1.3215416497138754e-07, 'epoch': 5.87}\n", + "{'loss': 0.0063, 'grad_norm': 0.5966488718986511, 'learning_rate': 9.710420977340762e-08, 'epoch': 5.89}\n", + "{'loss': 0.0038, 'grad_norm': 0.1901843547821045, 'learning_rate': 6.744015120061509e-08, 'epoch': 5.91}\n", + "{'loss': 0.0123, 'grad_norm': 2.4536399841308594, 'learning_rate': 4.316519082179227e-08, 'epoch': 5.93}\n", + "{'loss': 0.0048, 'grad_norm': 0.5865656733512878, 'learning_rate': 2.4281948573617874e-08, 'epoch': 5.94}\n", + "{'loss': 0.006, 'grad_norm': 0.9566450715065002, 'learning_rate': 1.0792462477909882e-08, 'epoch': 5.96}\n", + "{'loss': 0.0043, 'grad_norm': 1.3847167491912842, 'learning_rate': 2.6981884216847884e-09, 'epoch': 5.98}\n", + "{'loss': 0.0049, 'grad_norm': 1.5407752990722656, 'learning_rate': 0.0, 'epoch': 6.0}\n", + "100%|█████████████████████████████████████| 3360/3360 [6:19:44<00:00, 6.09s/it][INFO|trainer.py:3788] 2024-07-04 21:46:06,786 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 21:46:06,786 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 21:46:06,786 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-7b/lora/sft/checkpoint-3360\n", + "[INFO|configuration_utils.py:733] 2024-07-04 21:46:23,425 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 21:46:23,426 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 21:46:23,565 >> tokenizer config file saved in saves/qwen2-7b/lora/sft/checkpoint-3360/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 21:46:23,565 >> Special tokens file saved in saves/qwen2-7b/lora/sft/checkpoint-3360/special_tokens_map.json\n", + "[INFO|:482] 2024-07-04 21:46:23,978 >> \n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "{'train_runtime': 22807.0531, 'train_samples_per_second': 1.179, 'train_steps_per_second': 0.147, 'train_loss': 0.5189488330479002, 'epoch': 6.0}\n", + "100%|█████████████████████████████████████| 3360/3360 [6:20:01<00:00, 6.79s/it]\n", + "[INFO|trainer.py:3478] 2024-07-04 21:46:23,983 >> Saving model checkpoint to saves/qwen2-7b/lora/sft\n", + "[INFO|configuration_utils.py:733] 2024-07-04 21:46:25,525 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-7b-instruct-bnb-4bit/snapshots/8d8ce83e5c9fc23482eeae78027d1fc87bc2edad/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 21:46:25,525 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 21:46:25,650 >> tokenizer config file saved in saves/qwen2-7b/lora/sft/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 21:46:25,650 >> Special tokens file saved in saves/qwen2-7b/lora/sft/special_tokens_map.json\n", + "***** train metrics *****\n", + " epoch = 5.9973\n", + " total_flos = 89914948GF\n", + " train_loss = 0.5189\n", + " train_runtime = 6:20:07.05\n", + " train_samples_per_second = 1.179\n", + " train_steps_per_second = 0.147\n", + "Figure saved at: saves/qwen2-7b/lora/sft/training_loss.png\n", + "Figure saved at: saves/qwen2-7b/lora/sft/training_eval_loss.png\n", + "[INFO|trainer.py:3788] 2024-07-04 21:46:26,044 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 21:46:26,044 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 21:46:26,045 >> Batch size = 1\n", + "100%|███████████████████████████████████████████| 46/46 [00:08<00:00, 5.41it/s]\n", + "***** eval metrics *****\n", + " epoch = 5.9973\n", + " eval_loss = 2.9878\n", + " eval_runtime = 0:00:08.78\n", + " eval_samples_per_second = 5.234\n", + " eval_steps_per_second = 5.234\n", + "[INFO|modelcard.py:449] 2024-07-04 21:46:34,837 >> Dropping the following result as it does not have all the necessary fields:\n", + "{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: / 0.092 MB of 0.092 MB uploaded\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss ▁▂▃▅▇██\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime █▇█▆▅█▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second ▁▂▁▂▃▁█\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second ▁▂▁▂▃▁█\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm ▃▂▃▃▃▄▄▄▆▆▄▅▅▆▅▆▅▅▅▇▄▆▇▅▄▄▄█▃▂▄▄▃▁▁▁▂▁▁▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate ▂▄▅▇██████▇▇▇▇▇▆▆▆▆▅▅▅▄▄▄▃▃▃▃▂▂▂▂▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss ███▇▇▇▇▅▅▅▅▅▅▅▃▃▃▃▂▂▁▁▂▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss 2.9878\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime 8.7891\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second 5.234\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second 5.234\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: total_flos 9.654544053942682e+16\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch 5.99732\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step 3360\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm 1.54078\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss 0.0049\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 0.51895\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_runtime 22807.0531\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_samples_per_second 1.179\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_steps_per_second 0.147\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mqwen2_7b_lora_sft\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/o710838e\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 6 W&B file(s), 0 media file(s), 1 artifact file(s) and 0 other file(s)\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240704_152618-o710838e/logs\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n", + "CPU times: user 23min 50s, sys: 8min 47s, total: 32min 37s\n", + "Wall time: 6h 56min 32s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./scripts/tune-lf.sh config/qwen2_7b_lora_sft_unsloth.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/llm-finetuning/llama-factory\n", + "07/04/2024 21:56:42 - WARNING - llamafactory.hparams.parser - We recommend enable `upcast_layernorm` in quantized training.\n", + "07/04/2024 21:56:42 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: torch.bfloat16\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:56:42,789 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:56:42,789 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:56:42,789 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:56:42,789 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:56:42,789 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:56:42,789 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-04 21:56:42,918 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/04/2024 21:56:42 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/04/2024 21:56:42 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "07/04/2024 21:56:42 - INFO - llamafactory.data.loader - Loading dataset alpaca_mac.json...\n", + "Converting format of dataset (num_proc=16): 100%|█| 4528/4528 [00:00<00:00, 1521\n", + "Running tokenizer on dataset (num_proc=16): 100%|█| 4528/4528 [00:01<00:00, 2757\n", + "input_ids:\n", + "[151644, 872, 198, 5501, 14683, 279, 2701, 8453, 1467, 1119, 6364, 323, 3410, 1172, 279, 24531, 2213, 11, 4302, 770, 624, 35987, 102895, 99164, 100324, 100717, 100095, 99509, 1773, 151645, 198, 151644, 77091, 198, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "inputs:\n", + "<|im_start|>user\n", + "Please translate the following Chinese text into English and provide only the translated content, nothing else.\n", + "全仗着狐仙搭救。<|im_end|>\n", + "<|im_start|>assistant\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "label_ids:\n", + "[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "labels:\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "[INFO|configuration_utils.py:733] 2024-07-04 21:56:47,196 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 21:56:47,197 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "07/04/2024 21:56:47 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[INFO|configuration_utils.py:733] 2024-07-04 21:56:48,123 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 21:56:48,123 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"unsloth/qwen2-1.5b-instruct-bnb-4bit\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.6\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0+cu121. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. Xformers = 0.0.26.post1. FA = False.\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "[INFO|configuration_utils.py:733] 2024-07-04 21:56:49,865 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 21:56:49,865 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"unsloth/qwen2-1.5b-instruct-bnb-4bit\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|configuration_utils.py:733] 2024-07-04 21:56:50,495 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 21:56:50,496 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"unsloth/qwen2-1.5b-instruct-bnb-4bit\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:3556] 2024-07-04 21:56:50,707 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-04 21:56:56,626 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-04 21:56:56,631 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-04 21:58:31,535 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-04 21:58:31,535 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at unsloth/qwen2-1.5b-instruct-bnb-4bit.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-04 21:58:32,073 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-04 21:58:32,073 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:33,489 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:33,489 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:33,489 >> loading file added_tokens.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/added_tokens.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:33,490 >> loading file special_tokens_map.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/special_tokens_map.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:33,490 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:33,490 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/tokenizer.json\n", + "[WARNING|logging.py:313] 2024-07-04 21:58:33,937 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:34,912 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:34,912 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:34,912 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:34,912 >> loading file added_tokens.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/added_tokens.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:34,912 >> loading file special_tokens_map.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/special_tokens_map.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-04 21:58:34,912 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-04 21:58:35,100 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/04/2024 21:58:35 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.\n", + "07/04/2024 21:58:35 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.\n", + "07/04/2024 21:58:35 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA\n", + "07/04/2024 21:58:35 - INFO - llamafactory.model.model_utils.misc - Found linear modules: v_proj,k_proj,down_proj,gate_proj,q_proj,o_proj,up_proj\n", + "[WARNING|logging.py:328] 2024-07-04 21:58:36,612 >> Unsloth 2024.6 patched 28 layers with 0 QKV layers, 28 O layers and 28 MLP layers.\n", + "07/04/2024 21:58:37 - INFO - llamafactory.model.loader - trainable params: 9,232,384 || all params: 1,786,320,384 || trainable%: 0.5168\n", + "[INFO|trainer.py:642] 2024-07-04 21:58:37,463 >> Using auto half precision backend\n", + "07/04/2024 21:58:37 - WARNING - llamafactory.train.callbacks - Previous trainer log in this folder will be deleted.\n", + "07/04/2024 21:58:37 - INFO - llamafactory.train.trainer_utils - Using LoRA+ optimizer with loraplus lr ratio 16.00.\n", + "[WARNING|:223] 2024-07-04 21:58:37,613 >> ==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,482 | Num Epochs = 6\n", + "O^O/ \\_/ \\ Batch size per device = 1 | Gradient Accumulation steps = 8\n", + "\\ / Total batch size = 8 | Total steps = 3,360\n", + " \"-____-\" Number of trainable parameters = 9,232,384\n", + "[INFO|integration_utils.py:750] 2024-07-04 21:58:38,026 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33minflaton-sg\u001b[0m (\u001b[33minflaton-ai\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.4\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/home/inflaton/code/projects/courses/llm-finetuning/llama-factory/wandb/run-20240704_215839-4fbnqsea\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mqwen2_1.5b_lora_sft\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/4fbnqsea\u001b[0m\n", + "{'loss': 2.2167, 'grad_norm': 1.7105902433395386, 'learning_rate': 2.9761904761904763e-06, 'epoch': 0.02}\n", + "{'loss': 2.2613, 'grad_norm': 2.196908712387085, 'learning_rate': 5.9523809523809525e-06, 'epoch': 0.04}\n", + "{'loss': 2.0707, 'grad_norm': 0.9740070104598999, 'learning_rate': 8.92857142857143e-06, 'epoch': 0.05}\n", + "{'loss': 1.9514, 'grad_norm': 1.8389497995376587, 'learning_rate': 1.1904761904761905e-05, 'epoch': 0.07}\n", + "{'loss': 2.1174, 'grad_norm': 1.03471839427948, 'learning_rate': 1.4880952380952381e-05, 'epoch': 0.09}\n", + "{'loss': 1.8992, 'grad_norm': 1.198785662651062, 'learning_rate': 1.785714285714286e-05, 'epoch': 0.11}\n", + "{'loss': 2.0404, 'grad_norm': 1.114922285079956, 'learning_rate': 2.0833333333333336e-05, 'epoch': 0.12}\n", + "{'loss': 1.8447, 'grad_norm': 1.1239877939224243, 'learning_rate': 2.380952380952381e-05, 'epoch': 0.14}\n", + "{'loss': 1.9283, 'grad_norm': 1.5919139385223389, 'learning_rate': 2.6785714285714288e-05, 'epoch': 0.16}\n", + "{'loss': 1.9026, 'grad_norm': 0.998127818107605, 'learning_rate': 2.9761904761904762e-05, 'epoch': 0.18}\n", + "{'loss': 1.8932, 'grad_norm': 1.1430412530899048, 'learning_rate': 3.273809523809524e-05, 'epoch': 0.2}\n", + "{'loss': 1.8906, 'grad_norm': 1.0670546293258667, 'learning_rate': 3.571428571428572e-05, 'epoch': 0.21}\n", + "{'loss': 1.8343, 'grad_norm': 1.4356828927993774, 'learning_rate': 3.8690476190476195e-05, 'epoch': 0.23}\n", + "{'loss': 1.8725, 'grad_norm': 1.7043449878692627, 'learning_rate': 4.166666666666667e-05, 'epoch': 0.25}\n", + "{'loss': 1.7689, 'grad_norm': 1.09099543094635, 'learning_rate': 4.464285714285715e-05, 'epoch': 0.27}\n", + "{'loss': 1.7491, 'grad_norm': 0.9564052224159241, 'learning_rate': 4.761904761904762e-05, 'epoch': 0.29}\n", + "{'loss': 1.8979, 'grad_norm': 1.0268529653549194, 'learning_rate': 5.05952380952381e-05, 'epoch': 0.3}\n", + "{'loss': 1.695, 'grad_norm': 1.2585980892181396, 'learning_rate': 5.3571428571428575e-05, 'epoch': 0.32}\n", + "{'loss': 1.803, 'grad_norm': 2.146714448928833, 'learning_rate': 5.6547619047619046e-05, 'epoch': 0.34}\n", + "{'loss': 1.9439, 'grad_norm': 1.163086175918579, 'learning_rate': 5.9523809523809524e-05, 'epoch': 0.36}\n", + "{'loss': 1.8679, 'grad_norm': 1.2789676189422607, 'learning_rate': 6.25e-05, 'epoch': 0.37}\n", + "{'loss': 1.7942, 'grad_norm': 1.5350133180618286, 'learning_rate': 6.547619047619048e-05, 'epoch': 0.39}\n", + "{'loss': 1.7723, 'grad_norm': 1.333762526512146, 'learning_rate': 6.845238095238096e-05, 'epoch': 0.41}\n", + "{'loss': 1.9781, 'grad_norm': 1.342468500137329, 'learning_rate': 7.142857142857143e-05, 'epoch': 0.43}\n", + "{'loss': 1.8381, 'grad_norm': 1.785408616065979, 'learning_rate': 7.440476190476191e-05, 'epoch': 0.45}\n", + "{'loss': 1.77, 'grad_norm': 1.5936214923858643, 'learning_rate': 7.738095238095239e-05, 'epoch': 0.46}\n", + "{'loss': 1.8368, 'grad_norm': 1.7655868530273438, 'learning_rate': 8.035714285714287e-05, 'epoch': 0.48}\n", + "{'loss': 1.838, 'grad_norm': 1.5333795547485352, 'learning_rate': 8.333333333333334e-05, 'epoch': 0.5}\n", + "{'loss': 1.6551, 'grad_norm': 1.4578733444213867, 'learning_rate': 8.630952380952382e-05, 'epoch': 0.52}\n", + "{'loss': 1.8046, 'grad_norm': 1.649754524230957, 'learning_rate': 8.92857142857143e-05, 'epoch': 0.54}\n", + "{'loss': 1.8364, 'grad_norm': 1.618801236152649, 'learning_rate': 9.226190476190478e-05, 'epoch': 0.55}\n", + "{'loss': 1.6749, 'grad_norm': 2.321563243865967, 'learning_rate': 9.523809523809524e-05, 'epoch': 0.57}\n", + "{'loss': 1.7095, 'grad_norm': 1.7713825702667236, 'learning_rate': 9.821428571428572e-05, 'epoch': 0.59}\n", + "{'loss': 1.7458, 'grad_norm': 2.338412046432495, 'learning_rate': 9.999956828659095e-05, 'epoch': 0.61}\n", + "{'loss': 1.7693, 'grad_norm': 2.676462173461914, 'learning_rate': 9.999471159635539e-05, 'epoch': 0.62}\n", + "{'loss': 1.702, 'grad_norm': 1.777328610420227, 'learning_rate': 9.998445910004082e-05, 'epoch': 0.64}\n", + "{'loss': 1.8997, 'grad_norm': 2.657947063446045, 'learning_rate': 9.996881190417393e-05, 'epoch': 0.66}\n", + "{'loss': 1.8264, 'grad_norm': 1.7980377674102783, 'learning_rate': 9.994777169751806e-05, 'epoch': 0.68}\n", + "{'loss': 1.5464, 'grad_norm': 1.6675528287887573, 'learning_rate': 9.992134075089084e-05, 'epoch': 0.7}\n", + "{'loss': 1.7621, 'grad_norm': 2.088773727416992, 'learning_rate': 9.988952191691925e-05, 'epoch': 0.71}\n", + "{'loss': 1.7907, 'grad_norm': 1.8195936679840088, 'learning_rate': 9.985231862973168e-05, 'epoch': 0.73}\n", + "{'loss': 1.8215, 'grad_norm': 1.8611762523651123, 'learning_rate': 9.980973490458728e-05, 'epoch': 0.75}\n", + "{'loss': 1.7694, 'grad_norm': 2.018522024154663, 'learning_rate': 9.976177533744261e-05, 'epoch': 0.77}\n", + "{'loss': 1.741, 'grad_norm': 2.393932342529297, 'learning_rate': 9.97084451044556e-05, 'epoch': 0.79}\n", + "{'loss': 1.6568, 'grad_norm': 1.8010462522506714, 'learning_rate': 9.964974996142698e-05, 'epoch': 0.8}\n", + "{'loss': 1.7109, 'grad_norm': 1.6937175989151, 'learning_rate': 9.958569624317893e-05, 'epoch': 0.82}\n", + "{'loss': 1.7973, 'grad_norm': 2.7904672622680664, 'learning_rate': 9.951629086287151e-05, 'epoch': 0.84}\n", + "{'loss': 1.7033, 'grad_norm': 1.759727954864502, 'learning_rate': 9.944154131125642e-05, 'epoch': 0.86}\n", + "{'loss': 1.7797, 'grad_norm': 1.7603638172149658, 'learning_rate': 9.936145565586871e-05, 'epoch': 0.87}\n", + "{'loss': 1.9387, 'grad_norm': 1.8501616716384888, 'learning_rate': 9.927604254015585e-05, 'epoch': 0.89}\n", + "{'loss': 1.8734, 'grad_norm': 1.8340226411819458, 'learning_rate': 9.918531118254507e-05, 'epoch': 0.91}\n", + "{'loss': 1.7725, 'grad_norm': 2.32716703414917, 'learning_rate': 9.90892713754483e-05, 'epoch': 0.93}\n", + "{'loss': 1.7641, 'grad_norm': 2.673140048980713, 'learning_rate': 9.898793348420536e-05, 'epoch': 0.95}\n", + "{'loss': 1.7813, 'grad_norm': 1.9481444358825684, 'learning_rate': 9.888130844596524e-05, 'epoch': 0.96}\n", + "{'loss': 1.7363, 'grad_norm': 2.068895101547241, 'learning_rate': 9.876940776850569e-05, 'epoch': 0.98}\n", + "{'loss': 1.725, 'grad_norm': 1.8741337060928345, 'learning_rate': 9.865224352899119e-05, 'epoch': 1.0}\n", + " 17%|██████▎ | 560/3360 [20:10<1:47:08, 2.30s/it][INFO|trainer.py:3788] 2024-07-04 22:18:54,222 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 22:18:54,223 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 22:18:54,223 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-560\n", + "[INFO|configuration_utils.py:733] 2024-07-04 22:18:59,836 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 22:18:59,838 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 22:18:59,966 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-560/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 22:18:59,967 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-560/special_tokens_map.json\n", + "{'loss': 1.3163, 'grad_norm': 2.1074018478393555, 'learning_rate': 9.852982837266955e-05, 'epoch': 1.02}\n", + "{'loss': 1.1057, 'grad_norm': 2.284273147583008, 'learning_rate': 9.840217551150706e-05, 'epoch': 1.04}\n", + "{'loss': 1.3071, 'grad_norm': 1.8365180492401123, 'learning_rate': 9.826929872276255e-05, 'epoch': 1.05}\n", + "{'loss': 1.2093, 'grad_norm': 2.285874843597412, 'learning_rate': 9.81312123475006e-05, 'epoch': 1.07}\n", + "{'loss': 1.1653, 'grad_norm': 2.244819402694702, 'learning_rate': 9.798793128904356e-05, 'epoch': 1.09}\n", + "{'loss': 1.276, 'grad_norm': 2.5032386779785156, 'learning_rate': 9.78394710113631e-05, 'epoch': 1.11}\n", + "{'loss': 1.2116, 'grad_norm': 2.2835264205932617, 'learning_rate': 9.768584753741134e-05, 'epoch': 1.12}\n", + "{'loss': 1.1953, 'grad_norm': 3.390573740005493, 'learning_rate': 9.752707744739145e-05, 'epoch': 1.14}\n", + "{'loss': 1.2537, 'grad_norm': 2.312870502471924, 'learning_rate': 9.736317787696816e-05, 'epoch': 1.16}\n", + "{'loss': 1.1042, 'grad_norm': 2.252488851547241, 'learning_rate': 9.719416651541839e-05, 'epoch': 1.18}\n", + "{'loss': 1.1397, 'grad_norm': 1.93602454662323, 'learning_rate': 9.702006160372209e-05, 'epoch': 1.2}\n", + "{'loss': 1.1822, 'grad_norm': 2.3258895874023438, 'learning_rate': 9.684088193259355e-05, 'epoch': 1.21}\n", + "{'loss': 1.1777, 'grad_norm': 2.362091064453125, 'learning_rate': 9.665664684045333e-05, 'epoch': 1.23}\n", + "{'loss': 1.2182, 'grad_norm': 2.36861515045166, 'learning_rate': 9.646737621134112e-05, 'epoch': 1.25}\n", + "{'loss': 1.181, 'grad_norm': 3.928402900695801, 'learning_rate': 9.627309047276974e-05, 'epoch': 1.27}\n", + "{'loss': 1.3375, 'grad_norm': 3.1305952072143555, 'learning_rate': 9.607381059352038e-05, 'epoch': 1.29}\n", + "{'loss': 1.2559, 'grad_norm': 2.16672682762146, 'learning_rate': 9.586955808137958e-05, 'epoch': 1.3}\n", + "{'loss': 1.26, 'grad_norm': 2.531378984451294, 'learning_rate': 9.566035498081784e-05, 'epoch': 1.32}\n", + "{'loss': 1.2656, 'grad_norm': 2.2649450302124023, 'learning_rate': 9.544622387061055e-05, 'epoch': 1.34}\n", + "{'loss': 1.0581, 'grad_norm': 2.7688372135162354, 'learning_rate': 9.522718786140097e-05, 'epoch': 1.36}\n", + "{'loss': 1.2188, 'grad_norm': 3.3669986724853516, 'learning_rate': 9.500327059320606e-05, 'epoch': 1.37}\n", + "{'loss': 1.1538, 'grad_norm': 3.8478970527648926, 'learning_rate': 9.477449623286505e-05, 'epoch': 1.39}\n", + "{'loss': 1.0648, 'grad_norm': 2.5197343826293945, 'learning_rate': 9.454088947143116e-05, 'epoch': 1.41}\n", + "{'loss': 1.2997, 'grad_norm': 3.149819850921631, 'learning_rate': 9.430247552150673e-05, 'epoch': 1.43}\n", + "{'loss': 1.1716, 'grad_norm': 2.626891851425171, 'learning_rate': 9.405928011452211e-05, 'epoch': 1.45}\n", + "{'loss': 1.2123, 'grad_norm': 2.029723644256592, 'learning_rate': 9.381132949795861e-05, 'epoch': 1.46}\n", + "{'loss': 1.3143, 'grad_norm': 2.6693994998931885, 'learning_rate': 9.35586504325155e-05, 'epoch': 1.48}\n", + "{'loss': 1.2098, 'grad_norm': 2.4133574962615967, 'learning_rate': 9.330127018922194e-05, 'epoch': 1.5}\n", + "{'loss': 1.1153, 'grad_norm': 2.2110259532928467, 'learning_rate': 9.303921654649362e-05, 'epoch': 1.52}\n", + "{'loss': 1.2865, 'grad_norm': 2.425077438354492, 'learning_rate': 9.277251778713474e-05, 'epoch': 1.54}\n", + "{'loss': 1.2322, 'grad_norm': 2.287026882171631, 'learning_rate': 9.250120269528546e-05, 'epoch': 1.55}\n", + "{'loss': 1.1479, 'grad_norm': 2.3768105506896973, 'learning_rate': 9.22253005533154e-05, 'epoch': 1.57}\n", + "{'loss': 1.2783, 'grad_norm': 3.2799324989318848, 'learning_rate': 9.194484113866313e-05, 'epoch': 1.59}\n", + "{'loss': 1.3401, 'grad_norm': 2.8332979679107666, 'learning_rate': 9.165985472062246e-05, 'epoch': 1.61}\n", + "{'loss': 1.0948, 'grad_norm': 2.450061321258545, 'learning_rate': 9.137037205707552e-05, 'epoch': 1.62}\n", + "{'loss': 1.1901, 'grad_norm': 2.617992401123047, 'learning_rate': 9.107642439117321e-05, 'epoch': 1.64}\n", + "{'loss': 1.2412, 'grad_norm': 2.531679391860962, 'learning_rate': 9.077804344796302e-05, 'epoch': 1.66}\n", + "{'loss': 1.1875, 'grad_norm': 2.6147513389587402, 'learning_rate': 9.04752614309652e-05, 'epoch': 1.68}\n", + "{'loss': 1.1308, 'grad_norm': 3.1184866428375244, 'learning_rate': 9.01681110186971e-05, 'epoch': 1.7}\n", + "{'loss': 1.2466, 'grad_norm': 2.7524633407592773, 'learning_rate': 8.985662536114613e-05, 'epoch': 1.71}\n", + "{'loss': 1.1582, 'grad_norm': 2.410403251647949, 'learning_rate': 8.954083807619208e-05, 'epoch': 1.73}\n", + "{'loss': 1.2996, 'grad_norm': 3.132530927658081, 'learning_rate': 8.922078324597879e-05, 'epoch': 1.75}\n", + "{'loss': 1.3292, 'grad_norm': 3.093569755554199, 'learning_rate': 8.889649541323574e-05, 'epoch': 1.77}\n", + "{'loss': 1.2493, 'grad_norm': 2.8685665130615234, 'learning_rate': 8.856800957755e-05, 'epoch': 1.78}\n", + "{'loss': 1.2413, 'grad_norm': 3.4880857467651367, 'learning_rate': 8.823536119158864e-05, 'epoch': 1.8}\n", + "{'loss': 1.2145, 'grad_norm': 3.321408271789551, 'learning_rate': 8.789858615727265e-05, 'epoch': 1.82}\n", + "{'loss': 1.1431, 'grad_norm': 2.608922243118286, 'learning_rate': 8.755772082190194e-05, 'epoch': 1.84}\n", + "{'loss': 1.2395, 'grad_norm': 2.6196181774139404, 'learning_rate': 8.721280197423258e-05, 'epoch': 1.86}\n", + "{'loss': 1.0924, 'grad_norm': 3.0364978313446045, 'learning_rate': 8.68638668405062e-05, 'epoch': 1.87}\n", + "{'loss': 1.2218, 'grad_norm': 3.5102291107177734, 'learning_rate': 8.651095308043232e-05, 'epoch': 1.89}\n", + "{'loss': 1.2639, 'grad_norm': 4.278683662414551, 'learning_rate': 8.61540987831238e-05, 'epoch': 1.91}\n", + "{'loss': 1.2978, 'grad_norm': 3.729332208633423, 'learning_rate': 8.579334246298593e-05, 'epoch': 1.93}\n", + "{'loss': 1.1956, 'grad_norm': 3.6756839752197266, 'learning_rate': 8.542872305555978e-05, 'epoch': 1.95}\n", + "{'loss': 1.1345, 'grad_norm': 2.913640022277832, 'learning_rate': 8.50602799133199e-05, 'epoch': 1.96}\n", + "{'loss': 1.217, 'grad_norm': 2.75384259223938, 'learning_rate': 8.468805280142709e-05, 'epoch': 1.98}\n", + "{'loss': 1.2316, 'grad_norm': 3.1801509857177734, 'learning_rate': 8.43120818934367e-05, 'epoch': 2.0}\n", + " 33%|████████████▎ | 1120/3360 [41:10<1:24:27, 2.26s/it][INFO|trainer.py:3788] 2024-07-04 22:39:54,830 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 22:39:54,830 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 22:39:54,830 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-1120\n", + "[INFO|configuration_utils.py:733] 2024-07-04 22:39:59,689 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 22:39:59,690 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 22:39:59,739 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1120/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 22:39:59,740 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1120/special_tokens_map.json\n", + "{'loss': 0.6282, 'grad_norm': 2.980609893798828, 'learning_rate': 8.393240776696274e-05, 'epoch': 2.02}\n", + "{'loss': 0.516, 'grad_norm': 2.145615577697754, 'learning_rate': 8.354907139929851e-05, 'epoch': 2.03}\n", + "{'loss': 0.5637, 'grad_norm': 2.9827773571014404, 'learning_rate': 8.316211416299397e-05, 'epoch': 2.05}\n", + "{'loss': 0.5459, 'grad_norm': 6.992089748382568, 'learning_rate': 8.27715778213905e-05, 'epoch': 2.07}\n", + "{'loss': 0.4226, 'grad_norm': 2.0110838413238525, 'learning_rate': 8.237750452411353e-05, 'epoch': 2.09}\n", + "{'loss': 0.5595, 'grad_norm': 1.9566326141357422, 'learning_rate': 8.197993680252334e-05, 'epoch': 2.11}\n", + "{'loss': 0.6434, 'grad_norm': 3.0069830417633057, 'learning_rate': 8.157891756512488e-05, 'epoch': 2.12}\n", + "{'loss': 0.5572, 'grad_norm': 2.4360501766204834, 'learning_rate': 8.117449009293668e-05, 'epoch': 2.14}\n", + "{'loss': 0.5111, 'grad_norm': 3.1125354766845703, 'learning_rate': 8.076669803481965e-05, 'epoch': 2.16}\n", + "{'loss': 0.5006, 'grad_norm': 3.5583136081695557, 'learning_rate': 8.035558540276618e-05, 'epoch': 2.18}\n", + "{'loss': 0.5521, 'grad_norm': 2.597862482070923, 'learning_rate': 7.994119656715002e-05, 'epoch': 2.2}\n", + "{'loss': 0.6284, 'grad_norm': 3.2273318767547607, 'learning_rate': 7.952357625193749e-05, 'epoch': 2.21}\n", + "{'loss': 0.6074, 'grad_norm': 3.255476474761963, 'learning_rate': 7.91027695298606e-05, 'epoch': 2.23}\n", + "{'loss': 0.5721, 'grad_norm': 2.2420713901519775, 'learning_rate': 7.86788218175523e-05, 'epoch': 2.25}\n", + "{'loss': 0.5287, 'grad_norm': 3.241563558578491, 'learning_rate': 7.8251778870645e-05, 'epoch': 2.27}\n", + "{'loss': 0.5853, 'grad_norm': 3.7581467628479004, 'learning_rate': 7.782168677883206e-05, 'epoch': 2.28}\n", + "{'loss': 0.6142, 'grad_norm': 4.938629627227783, 'learning_rate': 7.738859196089358e-05, 'epoch': 2.3}\n", + "{'loss': 0.6064, 'grad_norm': 3.4490935802459717, 'learning_rate': 7.695254115968648e-05, 'epoch': 2.32}\n", + "{'loss': 0.6328, 'grad_norm': 3.473822832107544, 'learning_rate': 7.651358143709972e-05, 'epoch': 2.34}\n", + "{'loss': 0.6386, 'grad_norm': 3.5730648040771484, 'learning_rate': 7.60717601689749e-05, 'epoch': 2.36}\n", + "{'loss': 0.5591, 'grad_norm': 3.024034023284912, 'learning_rate': 7.562712503999327e-05, 'epoch': 2.37}\n", + "{'loss': 0.7168, 'grad_norm': 3.799771547317505, 'learning_rate': 7.517972403852905e-05, 'epoch': 2.39}\n", + "{'loss': 0.5869, 'grad_norm': 3.3111960887908936, 'learning_rate': 7.472960545147038e-05, 'epoch': 2.41}\n", + "{'loss': 0.5025, 'grad_norm': 3.5263280868530273, 'learning_rate': 7.427681785900761e-05, 'epoch': 2.43}\n", + "{'loss': 0.5964, 'grad_norm': 3.572462797164917, 'learning_rate': 7.382141012939034e-05, 'epoch': 2.45}\n", + "{'loss': 0.5491, 'grad_norm': 3.038294792175293, 'learning_rate': 7.33634314136531e-05, 'epoch': 2.46}\n", + "{'loss': 0.6004, 'grad_norm': 7.641390800476074, 'learning_rate': 7.290293114031061e-05, 'epoch': 2.48}\n", + "{'loss': 0.6356, 'grad_norm': 3.8366777896881104, 'learning_rate': 7.243995901002312e-05, 'epoch': 2.5}\n", + "{'loss': 0.5982, 'grad_norm': 3.146303176879883, 'learning_rate': 7.197456499023225e-05, 'epoch': 2.52}\n", + "{'loss': 0.6127, 'grad_norm': 4.154056072235107, 'learning_rate': 7.150679930976825e-05, 'epoch': 2.53}\n", + "{'loss': 0.5952, 'grad_norm': 2.470127582550049, 'learning_rate': 7.103671245342887e-05, 'epoch': 2.55}\n", + "{'loss': 0.4994, 'grad_norm': 5.2111053466796875, 'learning_rate': 7.056435515653059e-05, 'epoch': 2.57}\n", + "{'loss': 0.6969, 'grad_norm': 3.3999710083007812, 'learning_rate': 7.008977839943299e-05, 'epoch': 2.59}\n", + "{'loss': 0.6066, 'grad_norm': 3.942821979522705, 'learning_rate': 6.961303340203653e-05, 'epoch': 2.61}\n", + "{'loss': 0.6744, 'grad_norm': 3.511596918106079, 'learning_rate': 6.91341716182545e-05, 'epoch': 2.62}\n", + "{'loss': 0.5972, 'grad_norm': 2.605888605117798, 'learning_rate': 6.86532447304597e-05, 'epoch': 2.64}\n", + "{'loss': 0.6058, 'grad_norm': 3.500854730606079, 'learning_rate': 6.817030464390656e-05, 'epoch': 2.66}\n", + "{'loss': 0.6422, 'grad_norm': 2.9531426429748535, 'learning_rate': 6.768540348112907e-05, 'epoch': 2.68}\n", + "{'loss': 0.5311, 'grad_norm': 4.346620559692383, 'learning_rate': 6.719859357631535e-05, 'epoch': 2.7}\n", + "{'loss': 0.4986, 'grad_norm': 4.6267900466918945, 'learning_rate': 6.670992746965938e-05, 'epoch': 2.71}\n", + "{'loss': 0.6576, 'grad_norm': 4.185196876525879, 'learning_rate': 6.621945790169036e-05, 'epoch': 2.73}\n", + "{'loss': 0.6437, 'grad_norm': 3.265991687774658, 'learning_rate': 6.572723780758069e-05, 'epoch': 2.75}\n", + "{'loss': 0.574, 'grad_norm': 4.036723613739014, 'learning_rate': 6.523332031143272e-05, 'epoch': 2.77}\n", + "{'loss': 0.5839, 'grad_norm': 3.2608094215393066, 'learning_rate': 6.473775872054521e-05, 'epoch': 2.78}\n", + "{'loss': 0.5044, 'grad_norm': 3.3588390350341797, 'learning_rate': 6.424060651966007e-05, 'epoch': 2.8}\n", + "{'loss': 0.5707, 'grad_norm': 3.363955020904541, 'learning_rate': 6.374191736518974e-05, 'epoch': 2.82}\n", + "{'loss': 0.5785, 'grad_norm': 3.4573071002960205, 'learning_rate': 6.324174507942637e-05, 'epoch': 2.84}\n", + "{'loss': 0.5755, 'grad_norm': 4.1820855140686035, 'learning_rate': 6.274014364473274e-05, 'epoch': 2.86}\n", + "{'loss': 0.7532, 'grad_norm': 2.9372756481170654, 'learning_rate': 6.22371671977162e-05, 'epoch': 2.87}\n", + "{'loss': 0.6447, 'grad_norm': 4.2755632400512695, 'learning_rate': 6.173287002338577e-05, 'epoch': 2.89}\n", + "{'loss': 0.6018, 'grad_norm': 4.274354934692383, 'learning_rate': 6.122730654929334e-05, 'epoch': 2.91}\n", + "{'loss': 0.5677, 'grad_norm': 4.0272393226623535, 'learning_rate': 6.072053133965938e-05, 'epoch': 2.93}\n", + "{'loss': 0.6344, 'grad_norm': 3.0991122722625732, 'learning_rate': 6.021259908948402e-05, 'epoch': 2.95}\n", + "{'loss': 0.6559, 'grad_norm': 3.816575527191162, 'learning_rate': 5.970356461864391e-05, 'epoch': 2.96}\n", + "{'loss': 0.5647, 'grad_norm': 3.187918186187744, 'learning_rate': 5.919348286597569e-05, 'epoch': 2.98}\n", + "{'loss': 0.6381, 'grad_norm': 3.6101670265197754, 'learning_rate': 5.868240888334653e-05, 'epoch': 3.0}\n", + " 50%|█████████████████▌ | 1680/3360 [1:12:00<2:09:10, 4.61s/it][INFO|trainer.py:3788] 2024-07-04 23:10:44,677 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 23:10:44,677 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 23:10:44,677 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-1680\n", + "[INFO|configuration_utils.py:733] 2024-07-04 23:10:52,385 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 23:10:52,387 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 23:10:52,534 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1680/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 23:10:52,535 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-1680/special_tokens_map.json\n", + "{'loss': 0.3795, 'grad_norm': 3.6095104217529297, 'learning_rate': 5.8170397829712485e-05, 'epoch': 3.02}\n", + "{'loss': 0.1885, 'grad_norm': 2.648378610610962, 'learning_rate': 5.765750496516547e-05, 'epoch': 3.03}\n", + "{'loss': 0.2431, 'grad_norm': 3.102599859237671, 'learning_rate': 5.714378564496901e-05, 'epoch': 3.05}\n", + "{'loss': 0.2407, 'grad_norm': 1.7132669687271118, 'learning_rate': 5.6629295313583974e-05, 'epoch': 3.07}\n", + "{'loss': 0.1859, 'grad_norm': 2.363086462020874, 'learning_rate': 5.611408949868457e-05, 'epoch': 3.09}\n", + "{'loss': 0.2814, 'grad_norm': 2.7699074745178223, 'learning_rate': 5.559822380516539e-05, 'epoch': 3.11}\n", + "{'loss': 0.2066, 'grad_norm': 2.309485912322998, 'learning_rate': 5.5081753909140096e-05, 'epoch': 3.12}\n", + "{'loss': 0.2561, 'grad_norm': 3.8177757263183594, 'learning_rate': 5.456473555193242e-05, 'epoch': 3.14}\n", + "{'loss': 0.2839, 'grad_norm': 5.046483039855957, 'learning_rate': 5.404722453406017e-05, 'epoch': 3.16}\n", + "{'loss': 0.2309, 'grad_norm': 3.3046510219573975, 'learning_rate': 5.3529276709212816e-05, 'epoch': 3.18}\n", + "{'loss': 0.2678, 'grad_norm': 3.739877939224243, 'learning_rate': 5.30109479782233e-05, 'epoch': 3.2}\n", + "{'loss': 0.2305, 'grad_norm': 3.0891871452331543, 'learning_rate': 5.249229428303486e-05, 'epoch': 3.21}\n", + "{'loss': 0.3009, 'grad_norm': 2.0775339603424072, 'learning_rate': 5.197337160066331e-05, 'epoch': 3.23}\n", + "{'loss': 0.1974, 'grad_norm': 4.094172477722168, 'learning_rate': 5.145423593715557e-05, 'epoch': 3.25}\n", + "{'loss': 0.2613, 'grad_norm': 3.4857871532440186, 'learning_rate': 5.0934943321545115e-05, 'epoch': 3.27}\n", + "{'loss': 0.1759, 'grad_norm': 5.555017948150635, 'learning_rate': 5.041554979980486e-05, 'epoch': 3.28}\n", + "{'loss': 0.2755, 'grad_norm': 5.37070894241333, 'learning_rate': 4.9896111428798254e-05, 'epoch': 3.3}\n", + "{'loss': 0.3013, 'grad_norm': 3.0473411083221436, 'learning_rate': 4.9376684270229254e-05, 'epoch': 3.32}\n", + "{'loss': 0.2713, 'grad_norm': 2.421534299850464, 'learning_rate': 4.8857324384591653e-05, 'epoch': 3.34}\n", + "{'loss': 0.2342, 'grad_norm': 3.430769205093384, 'learning_rate': 4.8338087825118675e-05, 'epoch': 3.36}\n", + "{'loss': 0.2836, 'grad_norm': 3.117511510848999, 'learning_rate': 4.781903063173321e-05, 'epoch': 3.37}\n", + "{'loss': 0.2305, 'grad_norm': 2.2710249423980713, 'learning_rate': 4.730020882499964e-05, 'epoch': 3.39}\n", + "{'loss': 0.2707, 'grad_norm': 2.8062386512756348, 'learning_rate': 4.678167840007767e-05, 'epoch': 3.41}\n", + "{'loss': 0.2347, 'grad_norm': 3.199958324432373, 'learning_rate': 4.626349532067879e-05, 'epoch': 3.43}\n", + "{'loss': 0.2987, 'grad_norm': 2.9405529499053955, 'learning_rate': 4.574571551302647e-05, 'epoch': 3.44}\n", + "{'loss': 0.2748, 'grad_norm': 2.3248393535614014, 'learning_rate': 4.522839485981994e-05, 'epoch': 3.46}\n", + "{'loss': 0.2595, 'grad_norm': 2.7082927227020264, 'learning_rate': 4.471158919420312e-05, 'epoch': 3.48}\n", + "{'loss': 0.2452, 'grad_norm': 2.636992931365967, 'learning_rate': 4.4195354293738484e-05, 'epoch': 3.5}\n", + "{'loss': 0.2322, 'grad_norm': 2.870598554611206, 'learning_rate': 4.367974587438733e-05, 'epoch': 3.52}\n", + "{'loss': 0.2822, 'grad_norm': 2.3464884757995605, 'learning_rate': 4.316481958449634e-05, 'epoch': 3.53}\n", + "{'loss': 0.2228, 'grad_norm': 4.499746322631836, 'learning_rate': 4.2650630998791615e-05, 'epoch': 3.55}\n", + "{'loss': 0.2826, 'grad_norm': 3.5622456073760986, 'learning_rate': 4.213723561238074e-05, 'epoch': 3.57}\n", + "{'loss': 0.2505, 'grad_norm': 2.92927622795105, 'learning_rate': 4.162468883476319e-05, 'epoch': 3.59}\n", + "{'loss': 0.2715, 'grad_norm': 4.32992696762085, 'learning_rate': 4.111304598385018e-05, 'epoch': 3.61}\n", + "{'loss': 0.2382, 'grad_norm': 3.33722186088562, 'learning_rate': 4.060236227999441e-05, 'epoch': 3.62}\n", + "{'loss': 0.2219, 'grad_norm': 3.15584135055542, 'learning_rate': 4.0092692840030134e-05, 'epoch': 3.64}\n", + "{'loss': 0.2593, 'grad_norm': 2.6653778553009033, 'learning_rate': 3.9584092671324606e-05, 'epoch': 3.66}\n", + "{'loss': 0.2825, 'grad_norm': 2.261251449584961, 'learning_rate': 3.907661666584131e-05, 'epoch': 3.68}\n", + "{'loss': 0.2472, 'grad_norm': 2.40474796295166, 'learning_rate': 3.857031959421553e-05, 'epoch': 3.69}\n", + "{'loss': 0.2667, 'grad_norm': 3.5820109844207764, 'learning_rate': 3.806525609984312e-05, 'epoch': 3.71}\n", + "{'loss': 0.2426, 'grad_norm': 4.061399459838867, 'learning_rate': 3.7561480692983006e-05, 'epoch': 3.73}\n", + "{'loss': 0.3113, 'grad_norm': 3.3326733112335205, 'learning_rate': 3.705904774487396e-05, 'epoch': 3.75}\n", + "{'loss': 0.2658, 'grad_norm': 3.946682929992676, 'learning_rate': 3.655801148186655e-05, 'epoch': 3.77}\n", + "{'loss': 0.2051, 'grad_norm': 1.952369213104248, 'learning_rate': 3.6058425979570485e-05, 'epoch': 3.78}\n", + "{'loss': 0.249, 'grad_norm': 3.139533519744873, 'learning_rate': 3.556034515701852e-05, 'epoch': 3.8}\n", + "{'loss': 0.2398, 'grad_norm': 2.4751150608062744, 'learning_rate': 3.506382277084696e-05, 'epoch': 3.82}\n", + "{'loss': 0.2659, 'grad_norm': 1.9120585918426514, 'learning_rate': 3.4568912409493945e-05, 'epoch': 3.84}\n", + "{'loss': 0.1847, 'grad_norm': 2.8865163326263428, 'learning_rate': 3.4075667487415785e-05, 'epoch': 3.86}\n", + "{'loss': 0.2245, 'grad_norm': 3.2274515628814697, 'learning_rate': 3.358414123932195e-05, 'epoch': 3.87}\n", + "{'loss': 0.2643, 'grad_norm': 2.924294948577881, 'learning_rate': 3.3094386714429724e-05, 'epoch': 3.89}\n", + "{'loss': 0.252, 'grad_norm': 3.187256336212158, 'learning_rate': 3.2606456770738636e-05, 'epoch': 3.91}\n", + "{'loss': 0.1969, 'grad_norm': 2.353398084640503, 'learning_rate': 3.212040406932569e-05, 'epoch': 3.93}\n", + "{'loss': 0.2, 'grad_norm': 2.357897996902466, 'learning_rate': 3.163628106866172e-05, 'epoch': 3.94}\n", + "{'loss': 0.2773, 'grad_norm': 3.165809392929077, 'learning_rate': 3.115414001894974e-05, 'epoch': 3.96}\n", + "{'loss': 0.2495, 'grad_norm': 3.546583414077759, 'learning_rate': 3.067403295648566e-05, 'epoch': 3.98}\n", + "{'loss': 0.2513, 'grad_norm': 3.0604918003082275, 'learning_rate': 3.019601169804216e-05, 'epoch': 4.0}\n", + " 67%|███████████████████████▎ | 2240/3360 [1:48:51<1:17:16, 4.14s/it][INFO|trainer.py:3788] 2024-07-04 23:47:35,277 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-04 23:47:35,278 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-04 23:47:35,278 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-2240\n", + "[INFO|configuration_utils.py:733] 2024-07-04 23:47:44,213 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-04 23:47:44,213 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-04 23:47:44,277 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2240/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-04 23:47:44,277 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2240/special_tokens_map.json\n", + "{'loss': 0.1188, 'grad_norm': 1.1784201860427856, 'learning_rate': 2.9720127835276256e-05, 'epoch': 4.02}\n", + "{'loss': 0.0602, 'grad_norm': 1.9491609334945679, 'learning_rate': 2.9246432729161055e-05, 'epoch': 4.03}\n", + "{'loss': 0.1191, 'grad_norm': 8.893132209777832, 'learning_rate': 2.8774977504442647e-05, 'epoch': 4.05}\n", + "{'loss': 0.0814, 'grad_norm': 2.4567410945892334, 'learning_rate': 2.8305813044122097e-05, 'epoch': 4.07}\n", + "{'loss': 0.0717, 'grad_norm': 1.0190716981887817, 'learning_rate': 2.7838989983964065e-05, 'epoch': 4.09}\n", + "{'loss': 0.1036, 'grad_norm': 2.9603097438812256, 'learning_rate': 2.737455870703155e-05, 'epoch': 4.11}\n", + "{'loss': 0.0639, 'grad_norm': 0.43291687965393066, 'learning_rate': 2.6912569338248315e-05, 'epoch': 4.12}\n", + "{'loss': 0.1147, 'grad_norm': 16.320343017578125, 'learning_rate': 2.645307173898901e-05, 'epoch': 4.14}\n", + "{'loss': 0.083, 'grad_norm': 2.4415814876556396, 'learning_rate': 2.5996115501697694e-05, 'epoch': 4.16}\n", + "{'loss': 0.0894, 'grad_norm': 2.5519323348999023, 'learning_rate': 2.5541749944535554e-05, 'epoch': 4.18}\n", + "{'loss': 0.0634, 'grad_norm': 1.0128456354141235, 'learning_rate': 2.5090024106057962e-05, 'epoch': 4.19}\n", + "{'loss': 0.0978, 'grad_norm': 0.7708680629730225, 'learning_rate': 2.464098673992205e-05, 'epoch': 4.21}\n", + "{'loss': 0.0895, 'grad_norm': 2.129037618637085, 'learning_rate': 2.4194686309624663e-05, 'epoch': 4.23}\n", + "{'loss': 0.0986, 'grad_norm': 2.0388691425323486, 'learning_rate': 2.3751170983272e-05, 'epoch': 4.25}\n", + "{'loss': 0.1058, 'grad_norm': 2.9288082122802734, 'learning_rate': 2.3310488628380757e-05, 'epoch': 4.27}\n", + "{'loss': 0.1175, 'grad_norm': 4.13016414642334, 'learning_rate': 2.2872686806712035e-05, 'epoch': 4.28}\n", + "{'loss': 0.1101, 'grad_norm': 2.0640783309936523, 'learning_rate': 2.243781276913811e-05, 'epoch': 4.3}\n", + "{'loss': 0.0602, 'grad_norm': 2.8615546226501465, 'learning_rate': 2.200591345054267e-05, 'epoch': 4.32}\n", + "{'loss': 0.1019, 'grad_norm': 3.2558248043060303, 'learning_rate': 2.157703546475539e-05, 'epoch': 4.34}\n", + "{'loss': 0.0819, 'grad_norm': 2.1427247524261475, 'learning_rate': 2.115122509952085e-05, 'epoch': 4.36}\n", + "{'loss': 0.0767, 'grad_norm': 7.249903202056885, 'learning_rate': 2.0728528311502976e-05, 'epoch': 4.37}\n", + "{'loss': 0.0823, 'grad_norm': 2.022773027420044, 'learning_rate': 2.0308990721324927e-05, 'epoch': 4.39}\n", + "{'loss': 0.0797, 'grad_norm': 3.4550766944885254, 'learning_rate': 1.989265760864542e-05, 'epoch': 4.41}\n", + "{'loss': 0.0927, 'grad_norm': 1.1615883111953735, 'learning_rate': 1.947957390727185e-05, 'epoch': 4.43}\n", + "{'loss': 0.0782, 'grad_norm': 3.103994607925415, 'learning_rate': 1.906978420031059e-05, 'epoch': 4.44}\n", + "{'loss': 0.0575, 'grad_norm': 1.6370556354522705, 'learning_rate': 1.8663332715355396e-05, 'epoch': 4.46}\n", + "{'loss': 0.1022, 'grad_norm': 1.106717824935913, 'learning_rate': 1.8260263319713844e-05, 'epoch': 4.48}\n", + "{'loss': 0.1071, 'grad_norm': 3.171022415161133, 'learning_rate': 1.7860619515673033e-05, 'epoch': 4.5}\n", + "{'loss': 0.1038, 'grad_norm': 1.9004364013671875, 'learning_rate': 1.746444443580433e-05, 'epoch': 4.52}\n", + "{'loss': 0.0836, 'grad_norm': 1.7966681718826294, 'learning_rate': 1.7071780838308288e-05, 'epoch': 4.53}\n", + "{'loss': 0.0773, 'grad_norm': 2.2593512535095215, 'learning_rate': 1.6682671102399805e-05, 'epoch': 4.55}\n", + "{'loss': 0.0671, 'grad_norm': 2.4209578037261963, 'learning_rate': 1.629715722373423e-05, 'epoch': 4.57}\n", + "{'loss': 0.0869, 'grad_norm': 3.6910362243652344, 'learning_rate': 1.5915280809874932e-05, 'epoch': 4.59}\n", + "{'loss': 0.0713, 'grad_norm': 2.8420000076293945, 'learning_rate': 1.553708307580265e-05, 'epoch': 4.61}\n", + "{'loss': 0.0886, 'grad_norm': 1.897133469581604, 'learning_rate': 1.5162604839467265e-05, 'epoch': 4.62}\n", + "{'loss': 0.0804, 'grad_norm': 2.0078957080841064, 'learning_rate': 1.4791886517382413e-05, 'epoch': 4.64}\n", + "{'loss': 0.0828, 'grad_norm': 2.6949617862701416, 'learning_rate': 1.4424968120263504e-05, 'epoch': 4.66}\n", + "{'loss': 0.0906, 'grad_norm': 2.1701433658599854, 'learning_rate': 1.4061889248709343e-05, 'epoch': 4.68}\n", + "{'loss': 0.0854, 'grad_norm': 2.741436004638672, 'learning_rate': 1.370268908892825e-05, 'epoch': 4.69}\n", + "{'loss': 0.0847, 'grad_norm': 1.9649664163589478, 'learning_rate': 1.3347406408508695e-05, 'epoch': 4.71}\n", + "{'loss': 0.1074, 'grad_norm': 2.995682716369629, 'learning_rate': 1.2996079552235263e-05, 'epoch': 4.73}\n", + "{'loss': 0.0675, 'grad_norm': 1.7899149656295776, 'learning_rate': 1.264874643795021e-05, 'epoch': 4.75}\n", + "{'loss': 0.0736, 'grad_norm': 3.165422201156616, 'learning_rate': 1.230544455246101e-05, 'epoch': 4.77}\n", + "{'loss': 0.0949, 'grad_norm': 3.376789093017578, 'learning_rate': 1.1966210947494583e-05, 'epoch': 4.78}\n", + "{'loss': 0.0774, 'grad_norm': 0.7393803000450134, 'learning_rate': 1.1631082235698316e-05, 'epoch': 4.8}\n", + "{'loss': 0.0685, 'grad_norm': 4.275796890258789, 'learning_rate': 1.130009458668863e-05, 'epoch': 4.82}\n", + "{'loss': 0.0642, 'grad_norm': 1.65122652053833, 'learning_rate': 1.097328372314721e-05, 'epoch': 4.84}\n", + "{'loss': 0.0855, 'grad_norm': 1.4425795078277588, 'learning_rate': 1.0650684916965559e-05, 'epoch': 4.85}\n", + "{'loss': 0.0883, 'grad_norm': 2.1447832584381104, 'learning_rate': 1.0332332985438248e-05, 'epoch': 4.87}\n", + "{'loss': 0.1137, 'grad_norm': 2.644052743911743, 'learning_rate': 1.0018262287505086e-05, 'epoch': 4.89}\n", + "{'loss': 0.1026, 'grad_norm': 0.3625916838645935, 'learning_rate': 9.708506720042932e-06, 'epoch': 4.91}\n", + "{'loss': 0.0708, 'grad_norm': 0.9670233130455017, 'learning_rate': 9.403099714207175e-06, 'epoch': 4.93}\n", + "{'loss': 0.0886, 'grad_norm': 1.222226619720459, 'learning_rate': 9.102074231823727e-06, 'epoch': 4.94}\n", + "{'loss': 0.0913, 'grad_norm': 1.5419262647628784, 'learning_rate': 8.805462761831418e-06, 'epoch': 4.96}\n", + "{'loss': 0.105, 'grad_norm': 1.7759844064712524, 'learning_rate': 8.513297316775625e-06, 'epoch': 4.98}\n", + "{'loss': 0.0818, 'grad_norm': 1.2991019487380981, 'learning_rate': 8.225609429353187e-06, 'epoch': 5.0}\n", + " 83%|██████████████████████████████▊ | 2800/3360 [2:24:53<36:03, 3.86s/it][INFO|trainer.py:3788] 2024-07-05 00:23:37,381 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-05 00:23:37,382 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-05 00:23:37,382 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-2800\n", + "[INFO|configuration_utils.py:733] 2024-07-05 00:23:45,000 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 00:23:45,001 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-05 00:23:45,087 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2800/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-05 00:23:45,087 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-2800/special_tokens_map.json\n", + "{'loss': 0.0391, 'grad_norm': 1.8985695838928223, 'learning_rate': 7.942430149009161e-06, 'epoch': 5.02}\n", + "{'loss': 0.0262, 'grad_norm': 0.18104498088359833, 'learning_rate': 7.663790038585793e-06, 'epoch': 5.03}\n", + "{'loss': 0.0369, 'grad_norm': 0.4857228696346283, 'learning_rate': 7.389719171023857e-06, 'epoch': 5.05}\n", + "{'loss': 0.0285, 'grad_norm': 0.5048622488975525, 'learning_rate': 7.1202471261170245e-06, 'epoch': 5.07}\n", + "{'loss': 0.0239, 'grad_norm': 1.3091479539871216, 'learning_rate': 6.855402987319348e-06, 'epoch': 5.09}\n", + "{'loss': 0.0315, 'grad_norm': 0.7383649945259094, 'learning_rate': 6.595215338606397e-06, 'epoch': 5.1}\n", + "{'loss': 0.0227, 'grad_norm': 0.46847808361053467, 'learning_rate': 6.339712261390213e-06, 'epoch': 5.12}\n", + "{'loss': 0.0286, 'grad_norm': 2.871511936187744, 'learning_rate': 6.088921331488568e-06, 'epoch': 5.14}\n", + "{'loss': 0.0215, 'grad_norm': 0.5253076553344727, 'learning_rate': 5.8428696161488215e-06, 'epoch': 5.16}\n", + "{'loss': 0.0212, 'grad_norm': 0.7373698949813843, 'learning_rate': 5.601583671126531e-06, 'epoch': 5.18}\n", + "{'loss': 0.0468, 'grad_norm': 1.2003121376037598, 'learning_rate': 5.365089537819434e-06, 'epoch': 5.19}\n", + "{'loss': 0.0269, 'grad_norm': 0.1384514421224594, 'learning_rate': 5.133412740456806e-06, 'epoch': 5.21}\n", + "{'loss': 0.016, 'grad_norm': 0.6597172617912292, 'learning_rate': 4.906578283344759e-06, 'epoch': 5.23}\n", + "{'loss': 0.0273, 'grad_norm': 1.3373147249221802, 'learning_rate': 4.684610648167503e-06, 'epoch': 5.25}\n", + "{'loss': 0.022, 'grad_norm': 1.9218050241470337, 'learning_rate': 4.467533791345191e-06, 'epoch': 5.27}\n", + "{'loss': 0.0266, 'grad_norm': 0.33371880650520325, 'learning_rate': 4.255371141448272e-06, 'epoch': 5.28}\n", + "{'loss': 0.0246, 'grad_norm': 0.3639131486415863, 'learning_rate': 4.048145596668967e-06, 'epoch': 5.3}\n", + "{'loss': 0.04, 'grad_norm': 0.7324997186660767, 'learning_rate': 3.84587952234991e-06, 'epoch': 5.32}\n", + "{'loss': 0.02, 'grad_norm': 1.7712045907974243, 'learning_rate': 3.6485947485702832e-06, 'epoch': 5.34}\n", + "{'loss': 0.0304, 'grad_norm': 1.001847267150879, 'learning_rate': 3.4563125677897932e-06, 'epoch': 5.35}\n", + "{'loss': 0.0251, 'grad_norm': 1.4244178533554077, 'learning_rate': 3.269053732550581e-06, 'epoch': 5.37}\n", + "{'loss': 0.0201, 'grad_norm': 0.938901960849762, 'learning_rate': 3.086838453237506e-06, 'epoch': 5.39}\n", + "{'loss': 0.017, 'grad_norm': 0.722439706325531, 'learning_rate': 2.9096863958968268e-06, 'epoch': 5.41}\n", + "{'loss': 0.0278, 'grad_norm': 0.9856802225112915, 'learning_rate': 2.737616680113758e-06, 'epoch': 5.43}\n", + "{'loss': 0.0275, 'grad_norm': 1.7459590435028076, 'learning_rate': 2.570647876948895e-06, 'epoch': 5.44}\n", + "{'loss': 0.0419, 'grad_norm': 15.734712600708008, 'learning_rate': 2.408798006933882e-06, 'epoch': 5.46}\n", + "{'loss': 0.0498, 'grad_norm': 0.5652347207069397, 'learning_rate': 2.252084538126542e-06, 'epoch': 5.48}\n", + "{'loss': 0.0281, 'grad_norm': 0.6292805075645447, 'learning_rate': 2.100524384225555e-06, 'epoch': 5.5}\n", + "{'loss': 0.025, 'grad_norm': 1.3762198686599731, 'learning_rate': 1.9541339027450256e-06, 'epoch': 5.52}\n", + "{'loss': 0.0228, 'grad_norm': 0.6231855154037476, 'learning_rate': 1.8129288932490274e-06, 'epoch': 5.53}\n", + "{'loss': 0.021, 'grad_norm': 0.2345045506954193, 'learning_rate': 1.6769245956464396e-06, 'epoch': 5.55}\n", + "{'loss': 0.0314, 'grad_norm': 0.8907411694526672, 'learning_rate': 1.5461356885461075e-06, 'epoch': 5.57}\n", + "{'loss': 0.0324, 'grad_norm': 0.8636724948883057, 'learning_rate': 1.4205762876726092e-06, 'epoch': 5.59}\n", + "{'loss': 0.0306, 'grad_norm': 1.4055633544921875, 'learning_rate': 1.3002599443428243e-06, 'epoch': 5.6}\n", + "{'loss': 0.0276, 'grad_norm': 0.9670897722244263, 'learning_rate': 1.1851996440033319e-06, 'epoch': 5.62}\n", + "{'loss': 0.0328, 'grad_norm': 0.16922369599342346, 'learning_rate': 1.0754078048289374e-06, 'epoch': 5.64}\n", + "{'loss': 0.031, 'grad_norm': 1.8827847242355347, 'learning_rate': 9.708962763824048e-07, 'epoch': 5.66}\n", + "{'loss': 0.0214, 'grad_norm': 0.40066924691200256, 'learning_rate': 8.716763383355864e-07, 'epoch': 5.68}\n", + "{'loss': 0.0272, 'grad_norm': 0.28809547424316406, 'learning_rate': 7.777586992519959e-07, 'epoch': 5.69}\n", + "{'loss': 0.0253, 'grad_norm': 1.053158164024353, 'learning_rate': 6.891534954310885e-07, 'epoch': 5.71}\n", + "{'loss': 0.025, 'grad_norm': 0.2853540778160095, 'learning_rate': 6.058702898142643e-07, 'epoch': 5.73}\n", + "{'loss': 0.0354, 'grad_norm': 1.2035536766052246, 'learning_rate': 5.279180709527765e-07, 'epoch': 5.75}\n", + "{'loss': 0.0276, 'grad_norm': 0.9827560782432556, 'learning_rate': 4.553052520375911e-07, 'epoch': 5.77}\n", + "{'loss': 0.0209, 'grad_norm': 0.42196208238601685, 'learning_rate': 3.8803966999139684e-07, 'epoch': 5.78}\n", + "{'loss': 0.0265, 'grad_norm': 1.0920729637145996, 'learning_rate': 3.261285846227868e-07, 'epoch': 5.8}\n", + "{'loss': 0.0218, 'grad_norm': 0.4562773108482361, 'learning_rate': 2.6957867784270787e-07, 'epoch': 5.82}\n", + "{'loss': 0.0229, 'grad_norm': 1.235041618347168, 'learning_rate': 2.1839605294330933e-07, 'epoch': 5.84}\n", + "{'loss': 0.0371, 'grad_norm': 0.8272603154182434, 'learning_rate': 1.725862339392259e-07, 'epoch': 5.85}\n", + "{'loss': 0.0187, 'grad_norm': 0.5107071399688721, 'learning_rate': 1.3215416497138754e-07, 'epoch': 5.87}\n", + "{'loss': 0.0347, 'grad_norm': 1.0998457670211792, 'learning_rate': 9.710420977340762e-08, 'epoch': 5.89}\n", + "{'loss': 0.027, 'grad_norm': 1.8781795501708984, 'learning_rate': 6.744015120061509e-08, 'epoch': 5.91}\n", + "{'loss': 0.0351, 'grad_norm': 0.9750437140464783, 'learning_rate': 4.316519082179227e-08, 'epoch': 5.93}\n", + "{'loss': 0.0209, 'grad_norm': 1.2990669012069702, 'learning_rate': 2.4281948573617874e-08, 'epoch': 5.94}\n", + "{'loss': 0.0354, 'grad_norm': 1.9354966878890991, 'learning_rate': 1.0792462477909882e-08, 'epoch': 5.96}\n", + "{'loss': 0.0381, 'grad_norm': 1.044374704360962, 'learning_rate': 2.6981884216847884e-09, 'epoch': 5.98}\n", + "{'loss': 0.0228, 'grad_norm': 0.6751245856285095, 'learning_rate': 0.0, 'epoch': 6.0}\n", + "100%|█████████████████████████████████████| 3360/3360 [3:00:43<00:00, 3.75s/it][INFO|trainer.py:3788] 2024-07-05 00:59:27,574 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-05 00:59:27,574 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-05 00:59:27,574 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-1.5b/lora/sft/checkpoint-3360\n", + "[INFO|configuration_utils.py:733] 2024-07-05 00:59:35,314 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 00:59:35,316 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-05 00:59:35,381 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/checkpoint-3360/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-05 00:59:35,382 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/checkpoint-3360/special_tokens_map.json\n", + "[INFO|:482] 2024-07-05 00:59:35,695 >> \n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "{'train_runtime': 10857.6726, 'train_samples_per_second': 2.477, 'train_steps_per_second': 0.309, 'train_loss': 0.6667878782021858, 'epoch': 6.0}\n", + "100%|█████████████████████████████████████| 3360/3360 [3:00:51<00:00, 3.23s/it]\n", + "[INFO|trainer.py:3478] 2024-07-05 00:59:35,700 >> Saving model checkpoint to saves/qwen2-1.5b/lora/sft\n", + "[INFO|configuration_utils.py:733] 2024-07-05 00:59:36,890 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-1.5b-instruct-bnb-4bit/snapshots/9f10684b3a26fbf25e50921655353e2e3e599d70/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 00:59:36,891 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2574] 2024-07-05 00:59:36,947 >> tokenizer config file saved in saves/qwen2-1.5b/lora/sft/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2583] 2024-07-05 00:59:36,947 >> Special tokens file saved in saves/qwen2-1.5b/lora/sft/special_tokens_map.json\n", + "***** train metrics *****\n", + " epoch = 5.9973\n", + " total_flos = 19692141GF\n", + " train_loss = 0.6668\n", + " train_runtime = 3:00:57.67\n", + " train_samples_per_second = 2.477\n", + " train_steps_per_second = 0.309\n", + "Figure saved at: saves/qwen2-1.5b/lora/sft/training_loss.png\n", + "Figure saved at: saves/qwen2-1.5b/lora/sft/training_eval_loss.png\n", + "[INFO|trainer.py:3788] 2024-07-05 00:59:37,341 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-05 00:59:37,341 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-05 00:59:37,341 >> Batch size = 1\n", + "100%|███████████████████████████████████████████| 46/46 [00:05<00:00, 7.96it/s]\n", + "***** eval metrics *****\n", + " epoch = 5.9973\n", + " eval_loss = 3.4014\n", + " eval_runtime = 0:00:05.94\n", + " eval_samples_per_second = 7.742\n", + " eval_steps_per_second = 7.742\n", + "[INFO|modelcard.py:449] 2024-07-05 00:59:43,285 >> Dropping the following result as it does not have all the necessary fields:\n", + "{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: | 0.091 MB of 0.091 MB uploaded\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss ▁▁▃▄▆██\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime ▂▁▆█▆▆▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second ▆█▂▁▂▂▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second ▆█▂▁▂▂▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm ▂▁▁▂▂▂▂▂▂▂▂▂▂▂▂▃▃▂▂▃▂▂▂▂▂▂▂█▃▁▂▂▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate ▂▄▅▇██████▇▇▇▇▇▆▆▆▆▅▅▅▄▄▄▃▃▃▃▂▂▂▂▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss ████▇▇▇▅▆▆▅▅▅▅▃▃▃▃▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss 3.40137\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime 5.9413\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second 7.742\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second 7.742\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: total_flos 2.114427607798579e+16\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch 5.99732\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step 3360\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm 0.67512\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss 0.0228\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 0.66679\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_runtime 10857.6726\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_samples_per_second 2.477\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_steps_per_second 0.309\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mqwen2_1.5b_lora_sft\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/4fbnqsea\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 6 W&B file(s), 0 media file(s), 1 artifact file(s) and 0 other file(s)\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240704_215839-4fbnqsea/logs\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n", + "CPU times: user 3min 32s, sys: 1min 10s, total: 4min 43s\n", + "Wall time: 3h 3min 14s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./scripts/tune-lf.sh config/qwen2_1.5b_lora_sft_unsloth.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current Directory:\n", + "/home/inflaton/code/projects/courses/llm-finetuning/llama-factory\n", + "07/05/2024 06:15:40 - WARNING - llamafactory.hparams.parser - We recommend enable `upcast_layernorm` in quantized training.\n", + "07/05/2024 06:15:40 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: torch.bfloat16\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-05 06:15:40,695 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-05 06:15:40,695 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-05 06:15:40,695 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-05 06:15:40,695 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-05 06:15:40,695 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-05 06:15:40,695 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-05 06:15:40,871 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/05/2024 06:15:40 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/05/2024 06:15:40 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "07/05/2024 06:15:40 - INFO - llamafactory.data.loader - Loading dataset alpaca_mac.json...\n", + "Converting format of dataset (num_proc=16): 100%|█| 4528/4528 [00:00<00:00, 1717\n", + "Running tokenizer on dataset (num_proc=16): 100%|█| 4528/4528 [00:01<00:00, 2570\n", + "input_ids:\n", + "[151644, 872, 198, 5501, 14683, 279, 2701, 8453, 1467, 1119, 6364, 323, 3410, 1172, 279, 24531, 2213, 11, 4302, 770, 624, 35987, 102895, 99164, 100324, 100717, 100095, 99509, 1773, 151645, 198, 151644, 77091, 198, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "inputs:\n", + "<|im_start|>user\n", + "Please translate the following Chinese text into English and provide only the translated content, nothing else.\n", + "全仗着狐仙搭救。<|im_end|>\n", + "<|im_start|>assistant\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "label_ids:\n", + "[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 17949, 358, 572, 2617, 553, 264, 38835, 44486, 13, 151645]\n", + "labels:\n", + "Because I was protected by a fox fairy.<|im_end|>\n", + "[INFO|configuration_utils.py:733] 2024-07-05 06:15:44,437 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 06:15:44,438 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "07/05/2024 06:15:44 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[INFO|configuration_utils.py:733] 2024-07-05 06:15:45,429 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 06:15:45,430 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"unsloth/qwen2-0.5b-instruct-bnb-4bit\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "==((====))== Unsloth: Fast Qwen2 patching release 2024.7\n", + " \\\\ /| GPU: NVIDIA GeForce RTX 4080 Laptop GPU. Max memory: 11.994 GB. Platform = Linux.\n", + "O^O/ \\_/ \\ Pytorch: 2.3.0+cu121. CUDA = 8.9. CUDA Toolkit = 12.1.\n", + "\\ / Bfloat16 = TRUE. FA [Xformers = 0.0.26.post1. FA2 = False]\n", + " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", + "[INFO|configuration_utils.py:733] 2024-07-05 06:15:46,517 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 06:15:46,517 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"unsloth/qwen2-0.5b-instruct-bnb-4bit\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|configuration_utils.py:733] 2024-07-05 06:15:47,071 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 06:15:47,071 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"unsloth/qwen2-0.5b-instruct-bnb-4bit\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:3556] 2024-07-05 06:15:47,115 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-05 06:15:48,951 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-05 06:15:48,969 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-05 06:16:14,443 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-05 06:16:14,443 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at unsloth/qwen2-0.5b-instruct-bnb-4bit.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-05 06:16:14,971 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-05 06:16:14,971 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "07/05/2024 06:16:18 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.\n", + "07/05/2024 06:16:18 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.\n", + "07/05/2024 06:16:18 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA\n", + "07/05/2024 06:16:18 - INFO - llamafactory.model.model_utils.misc - Found linear modules: gate_proj,q_proj,k_proj,up_proj,down_proj,o_proj,v_proj\n", + "[WARNING|logging.py:328] 2024-07-05 06:16:19,091 >> Unsloth 2024.7 patched 24 layers with 0 QKV layers, 24 O layers and 24 MLP layers.\n", + "07/05/2024 06:16:19 - INFO - llamafactory.model.loader - trainable params: 4,399,104 || all params: 634,566,528 || trainable%: 0.6932\n", + "[INFO|trainer.py:642] 2024-07-05 06:16:19,940 >> Using auto half precision backend\n", + "07/05/2024 06:16:19 - WARNING - llamafactory.train.callbacks - Previous trainer log in this folder will be deleted.\n", + "07/05/2024 06:16:20 - INFO - llamafactory.train.trainer_utils - Using LoRA+ optimizer with loraplus lr ratio 16.00.\n", + "[WARNING|:223] 2024-07-05 06:16:20,129 >> ==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", + " \\\\ /| Num examples = 4,482 | Num Epochs = 6\n", + "O^O/ \\_/ \\ Batch size per device = 1 | Gradient Accumulation steps = 8\n", + "\\ / Total batch size = 8 | Total steps = 3,360\n", + " \"-____-\" Number of trainable parameters = 4,399,104\n", + "[INFO|integration_utils.py:750] 2024-07-05 06:16:20,818 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33minflaton-sg\u001b[0m (\u001b[33minflaton-ai\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.4\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/home/inflaton/code/projects/courses/llm-finetuning/llama-factory/wandb/run-20240705_061623-3amepb0m\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mqwen2_0.5b_lora_sft\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/3amepb0m\u001b[0m\n", + "{'loss': 2.6325, 'grad_norm': 2.6052567958831787, 'learning_rate': 2.9761904761904763e-06, 'epoch': 0.02}\n", + "{'loss': 2.6514, 'grad_norm': 2.433773994445801, 'learning_rate': 5.9523809523809525e-06, 'epoch': 0.04}\n", + "{'loss': 2.474, 'grad_norm': 2.1471617221832275, 'learning_rate': 8.92857142857143e-06, 'epoch': 0.05}\n", + "{'loss': 2.3031, 'grad_norm': 4.300695419311523, 'learning_rate': 1.1904761904761905e-05, 'epoch': 0.07}\n", + "{'loss': 2.4774, 'grad_norm': 1.8105831146240234, 'learning_rate': 1.4880952380952381e-05, 'epoch': 0.09}\n", + "{'loss': 2.2519, 'grad_norm': 2.077115297317505, 'learning_rate': 1.785714285714286e-05, 'epoch': 0.11}\n", + "{'loss': 2.4309, 'grad_norm': 1.9538270235061646, 'learning_rate': 2.0833333333333336e-05, 'epoch': 0.12}\n", + "{'loss': 2.22, 'grad_norm': 2.1473119258880615, 'learning_rate': 2.380952380952381e-05, 'epoch': 0.14}\n", + "{'loss': 2.3228, 'grad_norm': 2.819317579269409, 'learning_rate': 2.6785714285714288e-05, 'epoch': 0.16}\n", + "{'loss': 2.238, 'grad_norm': 1.9084508419036865, 'learning_rate': 2.9761904761904762e-05, 'epoch': 0.18}\n", + "{'loss': 2.2707, 'grad_norm': 2.1343274116516113, 'learning_rate': 3.273809523809524e-05, 'epoch': 0.2}\n", + "{'loss': 2.286, 'grad_norm': 2.273739814758301, 'learning_rate': 3.571428571428572e-05, 'epoch': 0.21}\n", + "{'loss': 2.1805, 'grad_norm': 2.505805253982544, 'learning_rate': 3.8690476190476195e-05, 'epoch': 0.23}\n", + "{'loss': 2.2527, 'grad_norm': 2.4992618560791016, 'learning_rate': 4.166666666666667e-05, 'epoch': 0.25}\n", + "{'loss': 2.1387, 'grad_norm': 1.9521129131317139, 'learning_rate': 4.464285714285715e-05, 'epoch': 0.27}\n", + "{'loss': 2.1733, 'grad_norm': 1.7223074436187744, 'learning_rate': 4.761904761904762e-05, 'epoch': 0.29}\n", + "{'loss': 2.2774, 'grad_norm': 1.8748223781585693, 'learning_rate': 5.05952380952381e-05, 'epoch': 0.3}\n", + "{'loss': 2.0726, 'grad_norm': 2.039461135864258, 'learning_rate': 5.3571428571428575e-05, 'epoch': 0.32}\n", + "{'loss': 2.1471, 'grad_norm': 2.512571096420288, 'learning_rate': 5.6547619047619046e-05, 'epoch': 0.34}\n", + "{'loss': 2.3088, 'grad_norm': 2.0730302333831787, 'learning_rate': 5.9523809523809524e-05, 'epoch': 0.36}\n", + "{'loss': 2.2315, 'grad_norm': 1.9101688861846924, 'learning_rate': 6.25e-05, 'epoch': 0.37}\n", + "{'loss': 2.1767, 'grad_norm': 2.6846179962158203, 'learning_rate': 6.547619047619048e-05, 'epoch': 0.39}\n", + "{'loss': 2.1396, 'grad_norm': 2.3576760292053223, 'learning_rate': 6.845238095238096e-05, 'epoch': 0.41}\n", + "{'loss': 2.3496, 'grad_norm': 2.5166685581207275, 'learning_rate': 7.142857142857143e-05, 'epoch': 0.43}\n", + "{'loss': 2.1899, 'grad_norm': 2.326274871826172, 'learning_rate': 7.440476190476191e-05, 'epoch': 0.45}\n", + "{'loss': 2.1658, 'grad_norm': 2.342203140258789, 'learning_rate': 7.738095238095239e-05, 'epoch': 0.46}\n", + "{'loss': 2.1561, 'grad_norm': 2.895669937133789, 'learning_rate': 8.035714285714287e-05, 'epoch': 0.48}\n", + "{'loss': 2.264, 'grad_norm': 3.2078170776367188, 'learning_rate': 8.333333333333334e-05, 'epoch': 0.5}\n", + "{'loss': 2.088, 'grad_norm': 2.282803773880005, 'learning_rate': 8.630952380952382e-05, 'epoch': 0.52}\n", + "{'loss': 2.1821, 'grad_norm': 2.5930910110473633, 'learning_rate': 8.92857142857143e-05, 'epoch': 0.54}\n", + "{'loss': 2.2382, 'grad_norm': 2.7073450088500977, 'learning_rate': 9.226190476190478e-05, 'epoch': 0.55}\n", + "{'loss': 2.0117, 'grad_norm': 3.457638740539551, 'learning_rate': 9.523809523809524e-05, 'epoch': 0.57}\n", + "{'loss': 2.0526, 'grad_norm': 3.453278064727783, 'learning_rate': 9.821428571428572e-05, 'epoch': 0.59}\n", + "{'loss': 2.1403, 'grad_norm': 2.7960667610168457, 'learning_rate': 9.999956828659095e-05, 'epoch': 0.61}\n", + "{'loss': 2.161, 'grad_norm': 3.307030439376831, 'learning_rate': 9.999471159635539e-05, 'epoch': 0.62}\n", + "{'loss': 2.0478, 'grad_norm': 2.788396120071411, 'learning_rate': 9.998445910004082e-05, 'epoch': 0.64}\n", + "{'loss': 2.3267, 'grad_norm': 4.489534378051758, 'learning_rate': 9.996881190417393e-05, 'epoch': 0.66}\n", + "{'loss': 2.2085, 'grad_norm': 2.93642520904541, 'learning_rate': 9.994777169751806e-05, 'epoch': 0.68}\n", + "{'loss': 1.8982, 'grad_norm': 2.470207929611206, 'learning_rate': 9.992134075089084e-05, 'epoch': 0.7}\n", + "{'loss': 2.1388, 'grad_norm': 2.992520809173584, 'learning_rate': 9.988952191691925e-05, 'epoch': 0.71}\n", + "{'loss': 2.1675, 'grad_norm': 2.986842155456543, 'learning_rate': 9.985231862973168e-05, 'epoch': 0.73}\n", + "{'loss': 2.1914, 'grad_norm': 2.8504011631011963, 'learning_rate': 9.980973490458728e-05, 'epoch': 0.75}\n", + "{'loss': 2.1588, 'grad_norm': 3.4979565143585205, 'learning_rate': 9.976177533744261e-05, 'epoch': 0.77}\n", + "{'loss': 2.0952, 'grad_norm': 3.6922664642333984, 'learning_rate': 9.97084451044556e-05, 'epoch': 0.79}\n", + "{'loss': 2.0288, 'grad_norm': 2.895118236541748, 'learning_rate': 9.964974996142698e-05, 'epoch': 0.8}\n", + "{'loss': 2.1275, 'grad_norm': 3.1226203441619873, 'learning_rate': 9.958569624317893e-05, 'epoch': 0.82}\n", + "{'loss': 2.1303, 'grad_norm': 4.210818767547607, 'learning_rate': 9.951629086287151e-05, 'epoch': 0.84}\n", + "{'loss': 2.1294, 'grad_norm': 2.9749433994293213, 'learning_rate': 9.944154131125642e-05, 'epoch': 0.86}\n", + "{'loss': 2.1612, 'grad_norm': 2.9232656955718994, 'learning_rate': 9.936145565586871e-05, 'epoch': 0.87}\n", + "{'loss': 2.3294, 'grad_norm': 2.8355772495269775, 'learning_rate': 9.927604254015585e-05, 'epoch': 0.89}\n", + "{'loss': 2.274, 'grad_norm': 3.1120338439941406, 'learning_rate': 9.918531118254507e-05, 'epoch': 0.91}\n", + "{'loss': 2.1442, 'grad_norm': 4.310208797454834, 'learning_rate': 9.90892713754483e-05, 'epoch': 0.93}\n", + "{'loss': 2.1595, 'grad_norm': 3.8621461391448975, 'learning_rate': 9.898793348420536e-05, 'epoch': 0.95}\n", + "{'loss': 2.1399, 'grad_norm': 2.8605706691741943, 'learning_rate': 9.888130844596524e-05, 'epoch': 0.96}\n", + "{'loss': 2.1673, 'grad_norm': 3.161895275115967, 'learning_rate': 9.876940776850569e-05, 'epoch': 0.98}\n", + "{'loss': 2.1621, 'grad_norm': 3.304511785507202, 'learning_rate': 9.865224352899119e-05, 'epoch': 1.0}\n", + " 17%|██████▎ | 560/3360 [15:54<1:17:19, 1.66s/it][INFO|trainer.py:3788] 2024-07-05 06:32:21,677 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-05 06:32:21,677 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-05 06:32:21,677 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-560\n", + "[INFO|configuration_utils.py:733] 2024-07-05 06:32:25,696 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 06:32:25,697 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "{'loss': 1.6151, 'grad_norm': 3.5378007888793945, 'learning_rate': 9.852982837266955e-05, 'epoch': 1.02}\n", + "{'loss': 1.3799, 'grad_norm': 3.1997132301330566, 'learning_rate': 9.840217551150706e-05, 'epoch': 1.04}\n", + "{'loss': 1.6132, 'grad_norm': 2.48860502243042, 'learning_rate': 9.826929872276255e-05, 'epoch': 1.05}\n", + "{'loss': 1.4984, 'grad_norm': 3.7188329696655273, 'learning_rate': 9.81312123475006e-05, 'epoch': 1.07}\n", + "{'loss': 1.4967, 'grad_norm': 3.4040935039520264, 'learning_rate': 9.798793128904356e-05, 'epoch': 1.09}\n", + "{'loss': 1.5688, 'grad_norm': 3.603771924972534, 'learning_rate': 9.78394710113631e-05, 'epoch': 1.11}\n", + "{'loss': 1.4902, 'grad_norm': 3.248730421066284, 'learning_rate': 9.768584753741134e-05, 'epoch': 1.12}\n", + "{'loss': 1.4788, 'grad_norm': 4.081541538238525, 'learning_rate': 9.752707744739145e-05, 'epoch': 1.14}\n", + "{'loss': 1.5933, 'grad_norm': 3.348815441131592, 'learning_rate': 9.736317787696816e-05, 'epoch': 1.16}\n", + "{'loss': 1.4597, 'grad_norm': 5.059058666229248, 'learning_rate': 9.719416651541839e-05, 'epoch': 1.18}\n", + "{'loss': 1.5088, 'grad_norm': 2.929900646209717, 'learning_rate': 9.702006160372209e-05, 'epoch': 1.2}\n", + "{'loss': 1.5122, 'grad_norm': 3.9229655265808105, 'learning_rate': 9.684088193259355e-05, 'epoch': 1.21}\n", + "{'loss': 1.4982, 'grad_norm': 4.456009864807129, 'learning_rate': 9.665664684045333e-05, 'epoch': 1.23}\n", + "{'loss': 1.5631, 'grad_norm': 6.255136966705322, 'learning_rate': 9.646737621134112e-05, 'epoch': 1.25}\n", + "{'loss': 1.5067, 'grad_norm': 4.147162914276123, 'learning_rate': 9.627309047276974e-05, 'epoch': 1.27}\n", + "{'loss': 1.6788, 'grad_norm': 4.083860874176025, 'learning_rate': 9.607381059352038e-05, 'epoch': 1.29}\n", + "{'loss': 1.6006, 'grad_norm': 3.7379791736602783, 'learning_rate': 9.586955808137958e-05, 'epoch': 1.3}\n", + "{'loss': 1.6328, 'grad_norm': 3.6500179767608643, 'learning_rate': 9.566035498081784e-05, 'epoch': 1.32}\n", + "{'loss': 1.6155, 'grad_norm': 3.455841302871704, 'learning_rate': 9.544622387061055e-05, 'epoch': 1.34}\n", + "{'loss': 1.3868, 'grad_norm': 3.636683702468872, 'learning_rate': 9.522718786140097e-05, 'epoch': 1.36}\n", + "{'loss': 1.5776, 'grad_norm': 4.494875431060791, 'learning_rate': 9.500327059320606e-05, 'epoch': 1.37}\n", + "{'loss': 1.4877, 'grad_norm': 4.710891246795654, 'learning_rate': 9.477449623286505e-05, 'epoch': 1.39}\n", + "{'loss': 1.401, 'grad_norm': 3.5016818046569824, 'learning_rate': 9.454088947143116e-05, 'epoch': 1.41}\n", + "{'loss': 1.628, 'grad_norm': 4.40405797958374, 'learning_rate': 9.430247552150673e-05, 'epoch': 1.43}\n", + "{'loss': 1.4999, 'grad_norm': 3.74572491645813, 'learning_rate': 9.405928011452211e-05, 'epoch': 1.45}\n", + "{'loss': 1.5602, 'grad_norm': 4.144255638122559, 'learning_rate': 9.381132949795861e-05, 'epoch': 1.46}\n", + "{'loss': 1.6872, 'grad_norm': 4.109062671661377, 'learning_rate': 9.35586504325155e-05, 'epoch': 1.48}\n", + "{'loss': 1.5494, 'grad_norm': 7.194815635681152, 'learning_rate': 9.330127018922194e-05, 'epoch': 1.5}\n", + "{'loss': 1.4354, 'grad_norm': 3.779526948928833, 'learning_rate': 9.303921654649362e-05, 'epoch': 1.52}\n", + "{'loss': 1.593, 'grad_norm': 3.863893508911133, 'learning_rate': 9.277251778713474e-05, 'epoch': 1.54}\n", + "{'loss': 1.5795, 'grad_norm': 3.684547185897827, 'learning_rate': 9.250120269528546e-05, 'epoch': 1.55}\n", + "{'loss': 1.5245, 'grad_norm': 3.9775428771972656, 'learning_rate': 9.22253005533154e-05, 'epoch': 1.57}\n", + "{'loss': 1.631, 'grad_norm': 4.817204475402832, 'learning_rate': 9.194484113866313e-05, 'epoch': 1.59}\n", + "{'loss': 1.658, 'grad_norm': 3.928107738494873, 'learning_rate': 9.165985472062246e-05, 'epoch': 1.61}\n", + "{'loss': 1.464, 'grad_norm': 4.099756240844727, 'learning_rate': 9.137037205707552e-05, 'epoch': 1.62}\n", + "{'loss': 1.5206, 'grad_norm': 3.9024410247802734, 'learning_rate': 9.107642439117321e-05, 'epoch': 1.64}\n", + "{'loss': 1.6011, 'grad_norm': 3.7552289962768555, 'learning_rate': 9.077804344796302e-05, 'epoch': 1.66}\n", + "{'loss': 1.4891, 'grad_norm': 3.713045835494995, 'learning_rate': 9.04752614309652e-05, 'epoch': 1.68}\n", + "{'loss': 1.5139, 'grad_norm': 3.589451313018799, 'learning_rate': 9.01681110186971e-05, 'epoch': 1.7}\n", + "{'loss': 1.5901, 'grad_norm': 3.9955010414123535, 'learning_rate': 8.985662536114613e-05, 'epoch': 1.71}\n", + "{'loss': 1.5646, 'grad_norm': 3.6160426139831543, 'learning_rate': 8.954083807619208e-05, 'epoch': 1.73}\n", + "{'loss': 1.6884, 'grad_norm': 4.0372796058654785, 'learning_rate': 8.922078324597879e-05, 'epoch': 1.75}\n", + "{'loss': 1.6813, 'grad_norm': 4.466279983520508, 'learning_rate': 8.889649541323574e-05, 'epoch': 1.77}\n", + "{'loss': 1.5947, 'grad_norm': 5.11010217666626, 'learning_rate': 8.856800957755e-05, 'epoch': 1.78}\n", + "{'loss': 1.6637, 'grad_norm': 5.363622188568115, 'learning_rate': 8.823536119158864e-05, 'epoch': 1.8}\n", + "{'loss': 1.5541, 'grad_norm': 4.0909223556518555, 'learning_rate': 8.789858615727265e-05, 'epoch': 1.82}\n", + "{'loss': 1.523, 'grad_norm': 3.796602249145508, 'learning_rate': 8.755772082190194e-05, 'epoch': 1.84}\n", + "{'loss': 1.6437, 'grad_norm': 4.511483669281006, 'learning_rate': 8.721280197423258e-05, 'epoch': 1.86}\n", + "{'loss': 1.4852, 'grad_norm': 4.5722246170043945, 'learning_rate': 8.68638668405062e-05, 'epoch': 1.87}\n", + "{'loss': 1.5986, 'grad_norm': 4.731987953186035, 'learning_rate': 8.651095308043232e-05, 'epoch': 1.89}\n", + "{'loss': 1.7502, 'grad_norm': 6.07273530960083, 'learning_rate': 8.61540987831238e-05, 'epoch': 1.91}\n", + "{'loss': 1.6979, 'grad_norm': 5.418001651763916, 'learning_rate': 8.579334246298593e-05, 'epoch': 1.93}\n", + "{'loss': 1.5625, 'grad_norm': 4.6554341316223145, 'learning_rate': 8.542872305555978e-05, 'epoch': 1.95}\n", + "{'loss': 1.4509, 'grad_norm': 3.8252899646759033, 'learning_rate': 8.50602799133199e-05, 'epoch': 1.96}\n", + "{'loss': 1.5915, 'grad_norm': 4.251583099365234, 'learning_rate': 8.468805280142709e-05, 'epoch': 1.98}\n", + "{'loss': 1.6074, 'grad_norm': 4.587167739868164, 'learning_rate': 8.43120818934367e-05, 'epoch': 2.0}\n", + " 33%|████████████▎ | 1120/3360 [31:19<1:00:24, 1.62s/it][INFO|trainer.py:3788] 2024-07-05 06:47:46,319 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-05 06:47:46,319 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-05 06:47:46,319 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-1120\n", + "[INFO|configuration_utils.py:733] 2024-07-05 06:47:50,281 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 06:47:50,282 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "{'loss': 0.9111, 'grad_norm': 5.008914470672607, 'learning_rate': 8.393240776696274e-05, 'epoch': 2.02}\n", + "{'loss': 0.863, 'grad_norm': 6.299067974090576, 'learning_rate': 8.354907139929851e-05, 'epoch': 2.03}\n", + "{'loss': 0.8515, 'grad_norm': 4.728297233581543, 'learning_rate': 8.316211416299397e-05, 'epoch': 2.05}\n", + "{'loss': 0.8316, 'grad_norm': 4.379367351531982, 'learning_rate': 8.27715778213905e-05, 'epoch': 2.07}\n", + "{'loss': 0.6608, 'grad_norm': 4.311402320861816, 'learning_rate': 8.237750452411353e-05, 'epoch': 2.09}\n", + "{'loss': 0.8713, 'grad_norm': 3.783642530441284, 'learning_rate': 8.197993680252334e-05, 'epoch': 2.11}\n", + "{'loss': 1.0158, 'grad_norm': 4.141658782958984, 'learning_rate': 8.157891756512488e-05, 'epoch': 2.12}\n", + "{'loss': 0.9155, 'grad_norm': 4.355412483215332, 'learning_rate': 8.117449009293668e-05, 'epoch': 2.14}\n", + "{'loss': 0.8407, 'grad_norm': 5.703305721282959, 'learning_rate': 8.076669803481965e-05, 'epoch': 2.16}\n", + "{'loss': 0.8494, 'grad_norm': 5.374706745147705, 'learning_rate': 8.035558540276618e-05, 'epoch': 2.18}\n", + "{'loss': 0.8743, 'grad_norm': 4.037242889404297, 'learning_rate': 7.994119656715002e-05, 'epoch': 2.2}\n", + "{'loss': 0.9841, 'grad_norm': 4.615417957305908, 'learning_rate': 7.952357625193749e-05, 'epoch': 2.21}\n", + "{'loss': 0.9296, 'grad_norm': 4.376211643218994, 'learning_rate': 7.91027695298606e-05, 'epoch': 2.23}\n", + "{'loss': 0.9142, 'grad_norm': 4.084548473358154, 'learning_rate': 7.86788218175523e-05, 'epoch': 2.25}\n", + "{'loss': 0.8517, 'grad_norm': 4.527939796447754, 'learning_rate': 7.8251778870645e-05, 'epoch': 2.27}\n", + "{'loss': 0.9113, 'grad_norm': 5.170512676239014, 'learning_rate': 7.782168677883206e-05, 'epoch': 2.28}\n", + "{'loss': 0.9332, 'grad_norm': 4.342284202575684, 'learning_rate': 7.738859196089358e-05, 'epoch': 2.3}\n", + "{'loss': 0.9759, 'grad_norm': 4.931323051452637, 'learning_rate': 7.695254115968648e-05, 'epoch': 2.32}\n", + "{'loss': 1.0079, 'grad_norm': 3.684819459915161, 'learning_rate': 7.651358143709972e-05, 'epoch': 2.34}\n", + "{'loss': 0.9958, 'grad_norm': 5.162328720092773, 'learning_rate': 7.60717601689749e-05, 'epoch': 2.36}\n", + "{'loss': 0.9528, 'grad_norm': 4.386671543121338, 'learning_rate': 7.562712503999327e-05, 'epoch': 2.37}\n", + "{'loss': 1.1468, 'grad_norm': 5.785244464874268, 'learning_rate': 7.517972403852905e-05, 'epoch': 2.39}\n", + "{'loss': 0.9291, 'grad_norm': 4.308371543884277, 'learning_rate': 7.472960545147038e-05, 'epoch': 2.41}\n", + "{'loss': 0.8408, 'grad_norm': 5.942112922668457, 'learning_rate': 7.427681785900761e-05, 'epoch': 2.43}\n", + "{'loss': 0.9693, 'grad_norm': 4.682136535644531, 'learning_rate': 7.382141012939034e-05, 'epoch': 2.45}\n", + "{'loss': 0.8726, 'grad_norm': 4.883449077606201, 'learning_rate': 7.33634314136531e-05, 'epoch': 2.46}\n", + "{'loss': 0.9426, 'grad_norm': 4.833103656768799, 'learning_rate': 7.290293114031061e-05, 'epoch': 2.48}\n", + "{'loss': 1.0333, 'grad_norm': 4.8503289222717285, 'learning_rate': 7.243995901002312e-05, 'epoch': 2.5}\n", + "{'loss': 0.9984, 'grad_norm': 4.3091230392456055, 'learning_rate': 7.197456499023225e-05, 'epoch': 2.52}\n", + "{'loss': 1.0019, 'grad_norm': 4.726260662078857, 'learning_rate': 7.150679930976825e-05, 'epoch': 2.53}\n", + "{'loss': 0.9594, 'grad_norm': 3.850511312484741, 'learning_rate': 7.103671245342887e-05, 'epoch': 2.55}\n", + "{'loss': 0.8701, 'grad_norm': 5.5012030601501465, 'learning_rate': 7.056435515653059e-05, 'epoch': 2.57}\n", + "{'loss': 1.0956, 'grad_norm': 5.610720157623291, 'learning_rate': 7.008977839943299e-05, 'epoch': 2.59}\n", + "{'loss': 0.9175, 'grad_norm': 3.8002779483795166, 'learning_rate': 6.961303340203653e-05, 'epoch': 2.61}\n", + "{'loss': 1.0243, 'grad_norm': 5.210932731628418, 'learning_rate': 6.91341716182545e-05, 'epoch': 2.62}\n", + "{'loss': 0.9902, 'grad_norm': 3.9311327934265137, 'learning_rate': 6.86532447304597e-05, 'epoch': 2.64}\n", + "{'loss': 0.9589, 'grad_norm': 4.984393119812012, 'learning_rate': 6.817030464390656e-05, 'epoch': 2.66}\n", + "{'loss': 0.9985, 'grad_norm': 4.881758689880371, 'learning_rate': 6.768540348112907e-05, 'epoch': 2.68}\n", + "{'loss': 0.8961, 'grad_norm': 6.465915203094482, 'learning_rate': 6.719859357631535e-05, 'epoch': 2.7}\n", + "{'loss': 0.8434, 'grad_norm': 5.6094183921813965, 'learning_rate': 6.670992746965938e-05, 'epoch': 2.71}\n", + "{'loss': 1.0485, 'grad_norm': 5.219779968261719, 'learning_rate': 6.621945790169036e-05, 'epoch': 2.73}\n", + "{'loss': 1.0165, 'grad_norm': 5.263071060180664, 'learning_rate': 6.572723780758069e-05, 'epoch': 2.75}\n", + "{'loss': 0.9104, 'grad_norm': 4.919801235198975, 'learning_rate': 6.523332031143272e-05, 'epoch': 2.77}\n", + "{'loss': 0.9633, 'grad_norm': 4.69899320602417, 'learning_rate': 6.473775872054521e-05, 'epoch': 2.78}\n", + "{'loss': 0.8483, 'grad_norm': 4.0923285484313965, 'learning_rate': 6.424060651966007e-05, 'epoch': 2.8}\n", + "{'loss': 0.8888, 'grad_norm': 5.461803436279297, 'learning_rate': 6.374191736518974e-05, 'epoch': 2.82}\n", + "{'loss': 0.9598, 'grad_norm': 4.758564472198486, 'learning_rate': 6.324174507942637e-05, 'epoch': 2.84}\n", + "{'loss': 0.9436, 'grad_norm': 6.395792007446289, 'learning_rate': 6.274014364473274e-05, 'epoch': 2.86}\n", + "{'loss': 1.1634, 'grad_norm': 6.077510356903076, 'learning_rate': 6.22371671977162e-05, 'epoch': 2.87}\n", + "{'loss': 1.0049, 'grad_norm': 5.1858720779418945, 'learning_rate': 6.173287002338577e-05, 'epoch': 2.89}\n", + "{'loss': 0.9795, 'grad_norm': 6.103806972503662, 'learning_rate': 6.122730654929334e-05, 'epoch': 2.91}\n", + "{'loss': 0.9422, 'grad_norm': 5.469768524169922, 'learning_rate': 6.072053133965938e-05, 'epoch': 2.93}\n", + "{'loss': 1.0349, 'grad_norm': 4.436359405517578, 'learning_rate': 6.021259908948402e-05, 'epoch': 2.95}\n", + "{'loss': 1.1161, 'grad_norm': 5.872861862182617, 'learning_rate': 5.970356461864391e-05, 'epoch': 2.96}\n", + "{'loss': 0.9069, 'grad_norm': 5.360676288604736, 'learning_rate': 5.919348286597569e-05, 'epoch': 2.98}\n", + "{'loss': 1.0593, 'grad_norm': 4.815310001373291, 'learning_rate': 5.868240888334653e-05, 'epoch': 3.0}\n", + " 50%|███████████████████▌ | 1680/3360 [46:46<45:31, 1.63s/it][INFO|trainer.py:3788] 2024-07-05 07:03:13,485 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-05 07:03:13,485 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-05 07:03:13,485 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-1680\n", + "[INFO|configuration_utils.py:733] 2024-07-05 07:03:17,790 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 07:03:17,790 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "{'loss': 0.6987, 'grad_norm': 4.28726863861084, 'learning_rate': 5.8170397829712485e-05, 'epoch': 3.02}\n", + "{'loss': 0.3462, 'grad_norm': 5.342904567718506, 'learning_rate': 5.765750496516547e-05, 'epoch': 3.03}\n", + "{'loss': 0.4899, 'grad_norm': 3.8532354831695557, 'learning_rate': 5.714378564496901e-05, 'epoch': 3.05}\n", + "{'loss': 0.4609, 'grad_norm': 4.3072590827941895, 'learning_rate': 5.6629295313583974e-05, 'epoch': 3.07}\n", + "{'loss': 0.4106, 'grad_norm': 4.2518463134765625, 'learning_rate': 5.611408949868457e-05, 'epoch': 3.09}\n", + "{'loss': 0.5169, 'grad_norm': 4.579401016235352, 'learning_rate': 5.559822380516539e-05, 'epoch': 3.11}\n", + "{'loss': 0.4794, 'grad_norm': 3.6858370304107666, 'learning_rate': 5.5081753909140096e-05, 'epoch': 3.12}\n", + "{'loss': 0.5473, 'grad_norm': 8.67149543762207, 'learning_rate': 5.456473555193242e-05, 'epoch': 3.14}\n", + "{'loss': 0.4638, 'grad_norm': 6.095928192138672, 'learning_rate': 5.404722453406017e-05, 'epoch': 3.16}\n", + "{'loss': 0.4697, 'grad_norm': 6.712044715881348, 'learning_rate': 5.3529276709212816e-05, 'epoch': 3.18}\n", + "{'loss': 0.4869, 'grad_norm': 4.1765336990356445, 'learning_rate': 5.30109479782233e-05, 'epoch': 3.2}\n", + "{'loss': 0.4821, 'grad_norm': 4.068556308746338, 'learning_rate': 5.249229428303486e-05, 'epoch': 3.21}\n", + "{'loss': 0.6011, 'grad_norm': 3.6553525924682617, 'learning_rate': 5.197337160066331e-05, 'epoch': 3.23}\n", + "{'loss': 0.4558, 'grad_norm': 4.888422012329102, 'learning_rate': 5.145423593715557e-05, 'epoch': 3.25}\n", + "{'loss': 0.5203, 'grad_norm': 4.138525009155273, 'learning_rate': 5.0934943321545115e-05, 'epoch': 3.27}\n", + "{'loss': 0.3826, 'grad_norm': 4.2213358879089355, 'learning_rate': 5.041554979980486e-05, 'epoch': 3.28}\n", + "{'loss': 0.5895, 'grad_norm': 4.9374260902404785, 'learning_rate': 4.9896111428798254e-05, 'epoch': 3.3}\n", + "{'loss': 0.5609, 'grad_norm': 4.482494831085205, 'learning_rate': 4.9376684270229254e-05, 'epoch': 3.32}\n", + "{'loss': 0.5478, 'grad_norm': 3.9575753211975098, 'learning_rate': 4.8857324384591653e-05, 'epoch': 3.34}\n", + "{'loss': 0.4865, 'grad_norm': 5.01925802230835, 'learning_rate': 4.8338087825118675e-05, 'epoch': 3.36}\n", + "{'loss': 0.5365, 'grad_norm': 4.109598636627197, 'learning_rate': 4.781903063173321e-05, 'epoch': 3.37}\n", + "{'loss': 0.4814, 'grad_norm': 3.7702512741088867, 'learning_rate': 4.730020882499964e-05, 'epoch': 3.39}\n", + "{'loss': 0.5355, 'grad_norm': 6.243114948272705, 'learning_rate': 4.678167840007767e-05, 'epoch': 3.41}\n", + "{'loss': 0.5361, 'grad_norm': 4.488025188446045, 'learning_rate': 4.626349532067879e-05, 'epoch': 3.43}\n", + "{'loss': 0.5952, 'grad_norm': 4.389721870422363, 'learning_rate': 4.574571551302647e-05, 'epoch': 3.44}\n", + "{'loss': 0.6049, 'grad_norm': 4.847557067871094, 'learning_rate': 4.522839485981994e-05, 'epoch': 3.46}\n", + "{'loss': 0.5697, 'grad_norm': 3.9925057888031006, 'learning_rate': 4.471158919420312e-05, 'epoch': 3.48}\n", + "{'loss': 0.5018, 'grad_norm': 5.327306747436523, 'learning_rate': 4.4195354293738484e-05, 'epoch': 3.5}\n", + "{'loss': 0.4745, 'grad_norm': 5.380455493927002, 'learning_rate': 4.367974587438733e-05, 'epoch': 3.52}\n", + "{'loss': 0.5421, 'grad_norm': 3.978426694869995, 'learning_rate': 4.316481958449634e-05, 'epoch': 3.53}\n", + "{'loss': 0.5091, 'grad_norm': 8.685088157653809, 'learning_rate': 4.2650630998791615e-05, 'epoch': 3.55}\n", + "{'loss': 0.6102, 'grad_norm': 4.471510887145996, 'learning_rate': 4.213723561238074e-05, 'epoch': 3.57}\n", + "{'loss': 0.4623, 'grad_norm': 4.236584663391113, 'learning_rate': 4.162468883476319e-05, 'epoch': 3.59}\n", + "{'loss': 0.5203, 'grad_norm': 5.698358535766602, 'learning_rate': 4.111304598385018e-05, 'epoch': 3.61}\n", + "{'loss': 0.5314, 'grad_norm': 5.975699424743652, 'learning_rate': 4.060236227999441e-05, 'epoch': 3.62}\n", + "{'loss': 0.484, 'grad_norm': 5.300996780395508, 'learning_rate': 4.0092692840030134e-05, 'epoch': 3.64}\n", + "{'loss': 0.4564, 'grad_norm': 7.857934474945068, 'learning_rate': 3.9584092671324606e-05, 'epoch': 3.66}\n", + "{'loss': 0.5715, 'grad_norm': 3.796581268310547, 'learning_rate': 3.907661666584131e-05, 'epoch': 3.68}\n", + "{'loss': 0.539, 'grad_norm': 4.170958995819092, 'learning_rate': 3.857031959421553e-05, 'epoch': 3.69}\n", + "{'loss': 0.5249, 'grad_norm': 6.283390045166016, 'learning_rate': 3.806525609984312e-05, 'epoch': 3.71}\n", + "{'loss': 0.4406, 'grad_norm': 6.235040664672852, 'learning_rate': 3.7561480692983006e-05, 'epoch': 3.73}\n", + "{'loss': 0.553, 'grad_norm': 3.715141534805298, 'learning_rate': 3.705904774487396e-05, 'epoch': 3.75}\n", + "{'loss': 0.5154, 'grad_norm': 6.352488040924072, 'learning_rate': 3.655801148186655e-05, 'epoch': 3.77}\n", + "{'loss': 0.4681, 'grad_norm': 4.480152130126953, 'learning_rate': 3.6058425979570485e-05, 'epoch': 3.78}\n", + "{'loss': 0.4915, 'grad_norm': 5.1917219161987305, 'learning_rate': 3.556034515701852e-05, 'epoch': 3.8}\n", + "{'loss': 0.5371, 'grad_norm': 4.501936912536621, 'learning_rate': 3.506382277084696e-05, 'epoch': 3.82}\n", + "{'loss': 0.5273, 'grad_norm': 3.53322434425354, 'learning_rate': 3.4568912409493945e-05, 'epoch': 3.84}\n", + "{'loss': 0.4405, 'grad_norm': 4.688470840454102, 'learning_rate': 3.4075667487415785e-05, 'epoch': 3.86}\n", + "{'loss': 0.5048, 'grad_norm': 6.739779949188232, 'learning_rate': 3.358414123932195e-05, 'epoch': 3.87}\n", + "{'loss': 0.5572, 'grad_norm': 4.120084762573242, 'learning_rate': 3.3094386714429724e-05, 'epoch': 3.89}\n", + "{'loss': 0.5498, 'grad_norm': 7.938605785369873, 'learning_rate': 3.2606456770738636e-05, 'epoch': 3.91}\n", + "{'loss': 0.3955, 'grad_norm': 4.132835865020752, 'learning_rate': 3.212040406932569e-05, 'epoch': 3.93}\n", + "{'loss': 0.4149, 'grad_norm': 3.777303457260132, 'learning_rate': 3.163628106866172e-05, 'epoch': 3.94}\n", + "{'loss': 0.6278, 'grad_norm': 5.201406955718994, 'learning_rate': 3.115414001894974e-05, 'epoch': 3.96}\n", + "{'loss': 0.5186, 'grad_norm': 7.258588790893555, 'learning_rate': 3.067403295648566e-05, 'epoch': 3.98}\n", + "{'loss': 0.4754, 'grad_norm': 3.8839337825775146, 'learning_rate': 3.019601169804216e-05, 'epoch': 4.0}\n", + " 67%|████████████████████████▋ | 2240/3360 [1:01:49<29:29, 1.58s/it][INFO|trainer.py:3788] 2024-07-05 07:18:16,440 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-05 07:18:16,441 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-05 07:18:16,441 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-2240\n", + "[INFO|configuration_utils.py:733] 2024-07-05 07:18:20,481 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 07:18:20,481 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "{'loss': 0.3004, 'grad_norm': 3.442147731781006, 'learning_rate': 2.9720127835276256e-05, 'epoch': 4.02}\n", + "{'loss': 0.2198, 'grad_norm': 3.406682252883911, 'learning_rate': 2.9246432729161055e-05, 'epoch': 4.03}\n", + "{'loss': 0.2312, 'grad_norm': 3.0993845462799072, 'learning_rate': 2.8774977504442647e-05, 'epoch': 4.05}\n", + "{'loss': 0.2038, 'grad_norm': 4.1203694343566895, 'learning_rate': 2.8305813044122097e-05, 'epoch': 4.07}\n", + "{'loss': 0.1819, 'grad_norm': 2.116147756576538, 'learning_rate': 2.7838989983964065e-05, 'epoch': 4.09}\n", + "{'loss': 0.2672, 'grad_norm': 3.2091379165649414, 'learning_rate': 2.737455870703155e-05, 'epoch': 4.11}\n", + "{'loss': 0.1779, 'grad_norm': 2.386085033416748, 'learning_rate': 2.6912569338248315e-05, 'epoch': 4.12}\n", + "{'loss': 0.2997, 'grad_norm': 28.640592575073242, 'learning_rate': 2.645307173898901e-05, 'epoch': 4.14}\n", + "{'loss': 0.227, 'grad_norm': 2.2596945762634277, 'learning_rate': 2.5996115501697694e-05, 'epoch': 4.16}\n", + "{'loss': 0.2184, 'grad_norm': 4.521151065826416, 'learning_rate': 2.5541749944535554e-05, 'epoch': 4.18}\n", + "{'loss': 0.2038, 'grad_norm': 5.861654281616211, 'learning_rate': 2.5090024106057962e-05, 'epoch': 4.19}\n", + "{'loss': 0.2423, 'grad_norm': 4.1528639793396, 'learning_rate': 2.464098673992205e-05, 'epoch': 4.21}\n", + "{'loss': 0.2367, 'grad_norm': 2.1180801391601562, 'learning_rate': 2.4194686309624663e-05, 'epoch': 4.23}\n", + "{'loss': 0.2627, 'grad_norm': 4.030113697052002, 'learning_rate': 2.3751170983272e-05, 'epoch': 4.25}\n", + "{'loss': 0.2329, 'grad_norm': 4.907358646392822, 'learning_rate': 2.3310488628380757e-05, 'epoch': 4.27}\n", + "{'loss': 0.2542, 'grad_norm': 4.652915000915527, 'learning_rate': 2.2872686806712035e-05, 'epoch': 4.28}\n", + "{'loss': 0.2495, 'grad_norm': 5.10890531539917, 'learning_rate': 2.243781276913811e-05, 'epoch': 4.3}\n", + "{'loss': 0.1662, 'grad_norm': 3.823878288269043, 'learning_rate': 2.200591345054267e-05, 'epoch': 4.32}\n", + "{'loss': 0.2812, 'grad_norm': 3.004128932952881, 'learning_rate': 2.157703546475539e-05, 'epoch': 4.34}\n", + "{'loss': 0.2202, 'grad_norm': 4.443856716156006, 'learning_rate': 2.115122509952085e-05, 'epoch': 4.36}\n", + "{'loss': 0.2244, 'grad_norm': 2.996962070465088, 'learning_rate': 2.0728528311502976e-05, 'epoch': 4.37}\n", + "{'loss': 0.2474, 'grad_norm': 4.116214752197266, 'learning_rate': 2.0308990721324927e-05, 'epoch': 4.39}\n", + "{'loss': 0.1881, 'grad_norm': 4.773007392883301, 'learning_rate': 1.989265760864542e-05, 'epoch': 4.41}\n", + "{'loss': 0.2721, 'grad_norm': 3.045060873031616, 'learning_rate': 1.947957390727185e-05, 'epoch': 4.43}\n", + "{'loss': 0.2474, 'grad_norm': 5.480595111846924, 'learning_rate': 1.906978420031059e-05, 'epoch': 4.44}\n", + "{'loss': 0.1786, 'grad_norm': 2.452791929244995, 'learning_rate': 1.8663332715355396e-05, 'epoch': 4.46}\n", + "{'loss': 0.2655, 'grad_norm': 1.6951186656951904, 'learning_rate': 1.8260263319713844e-05, 'epoch': 4.48}\n", + "{'loss': 0.2307, 'grad_norm': 4.780274868011475, 'learning_rate': 1.7860619515673033e-05, 'epoch': 4.5}\n", + "{'loss': 0.2661, 'grad_norm': 4.14153528213501, 'learning_rate': 1.746444443580433e-05, 'epoch': 4.52}\n", + "{'loss': 0.2482, 'grad_norm': 4.6406989097595215, 'learning_rate': 1.7071780838308288e-05, 'epoch': 4.53}\n", + "{'loss': 0.2268, 'grad_norm': 3.8813576698303223, 'learning_rate': 1.6682671102399805e-05, 'epoch': 4.55}\n", + "{'loss': 0.2058, 'grad_norm': 3.428504467010498, 'learning_rate': 1.629715722373423e-05, 'epoch': 4.57}\n", + "{'loss': 0.2407, 'grad_norm': 4.3143415451049805, 'learning_rate': 1.5915280809874932e-05, 'epoch': 4.59}\n", + "{'loss': 0.2153, 'grad_norm': 4.420351505279541, 'learning_rate': 1.553708307580265e-05, 'epoch': 4.61}\n", + "{'loss': 0.2423, 'grad_norm': 3.758807897567749, 'learning_rate': 1.5162604839467265e-05, 'epoch': 4.62}\n", + "{'loss': 0.2376, 'grad_norm': 3.164726734161377, 'learning_rate': 1.4791886517382413e-05, 'epoch': 4.64}\n", + "{'loss': 0.223, 'grad_norm': 2.6924712657928467, 'learning_rate': 1.4424968120263504e-05, 'epoch': 4.66}\n", + "{'loss': 0.224, 'grad_norm': 3.1326253414154053, 'learning_rate': 1.4061889248709343e-05, 'epoch': 4.68}\n", + "{'loss': 0.2097, 'grad_norm': 3.1166789531707764, 'learning_rate': 1.370268908892825e-05, 'epoch': 4.69}\n", + "{'loss': 0.2607, 'grad_norm': 4.3387651443481445, 'learning_rate': 1.3347406408508695e-05, 'epoch': 4.71}\n", + "{'loss': 0.2217, 'grad_norm': 2.9194934368133545, 'learning_rate': 1.2996079552235263e-05, 'epoch': 4.73}\n", + "{'loss': 0.175, 'grad_norm': 2.6297366619110107, 'learning_rate': 1.264874643795021e-05, 'epoch': 4.75}\n", + "{'loss': 0.2148, 'grad_norm': 3.174553632736206, 'learning_rate': 1.230544455246101e-05, 'epoch': 4.77}\n", + "{'loss': 0.246, 'grad_norm': 3.611652374267578, 'learning_rate': 1.1966210947494583e-05, 'epoch': 4.78}\n", + "{'loss': 0.2477, 'grad_norm': 3.13002610206604, 'learning_rate': 1.1631082235698316e-05, 'epoch': 4.8}\n", + "{'loss': 0.2034, 'grad_norm': 3.1411221027374268, 'learning_rate': 1.130009458668863e-05, 'epoch': 4.82}\n", + "{'loss': 0.1899, 'grad_norm': 3.8253543376922607, 'learning_rate': 1.097328372314721e-05, 'epoch': 4.84}\n", + "{'loss': 0.2432, 'grad_norm': 4.582285404205322, 'learning_rate': 1.0650684916965559e-05, 'epoch': 4.85}\n", + "{'loss': 0.2412, 'grad_norm': 3.9309003353118896, 'learning_rate': 1.0332332985438248e-05, 'epoch': 4.87}\n", + "{'loss': 0.2543, 'grad_norm': 4.181048393249512, 'learning_rate': 1.0018262287505086e-05, 'epoch': 4.89}\n", + "{'loss': 0.2759, 'grad_norm': 1.8343684673309326, 'learning_rate': 9.708506720042932e-06, 'epoch': 4.91}\n", + "{'loss': 0.189, 'grad_norm': 2.335709571838379, 'learning_rate': 9.403099714207175e-06, 'epoch': 4.93}\n", + "{'loss': 0.2495, 'grad_norm': 4.065298080444336, 'learning_rate': 9.102074231823727e-06, 'epoch': 4.94}\n", + "{'loss': 0.2557, 'grad_norm': 2.8378493785858154, 'learning_rate': 8.805462761831418e-06, 'epoch': 4.96}\n", + "{'loss': 0.2784, 'grad_norm': 3.395693063735962, 'learning_rate': 8.513297316775625e-06, 'epoch': 4.98}\n", + "{'loss': 0.2621, 'grad_norm': 4.082712173461914, 'learning_rate': 8.225609429353187e-06, 'epoch': 5.0}\n", + " 83%|██████████████████████████████▊ | 2800/3360 [1:16:55<15:13, 1.63s/it][INFO|trainer.py:3788] 2024-07-05 07:33:22,309 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-05 07:33:22,309 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-05 07:33:22,310 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-2800\n", + "[INFO|configuration_utils.py:733] 2024-07-05 07:33:26,470 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 07:33:26,471 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "{'loss': 0.1274, 'grad_norm': 1.5695966482162476, 'learning_rate': 7.942430149009161e-06, 'epoch': 5.02}\n", + "{'loss': 0.1052, 'grad_norm': 0.5931769609451294, 'learning_rate': 7.663790038585793e-06, 'epoch': 5.03}\n", + "{'loss': 0.1188, 'grad_norm': 4.6362762451171875, 'learning_rate': 7.389719171023857e-06, 'epoch': 5.05}\n", + "{'loss': 0.1032, 'grad_norm': 2.54799485206604, 'learning_rate': 7.1202471261170245e-06, 'epoch': 5.07}\n", + "{'loss': 0.0781, 'grad_norm': 2.0741422176361084, 'learning_rate': 6.855402987319348e-06, 'epoch': 5.09}\n", + "{'loss': 0.1428, 'grad_norm': 2.47188401222229, 'learning_rate': 6.595215338606397e-06, 'epoch': 5.1}\n", + "{'loss': 0.0839, 'grad_norm': 1.993886947631836, 'learning_rate': 6.339712261390213e-06, 'epoch': 5.12}\n", + "{'loss': 0.1086, 'grad_norm': 1.442935824394226, 'learning_rate': 6.088921331488568e-06, 'epoch': 5.14}\n", + "{'loss': 0.0851, 'grad_norm': 2.1466658115386963, 'learning_rate': 5.8428696161488215e-06, 'epoch': 5.16}\n", + "{'loss': 0.0964, 'grad_norm': 1.960119366645813, 'learning_rate': 5.601583671126531e-06, 'epoch': 5.18}\n", + "{'loss': 0.1149, 'grad_norm': 1.3245364427566528, 'learning_rate': 5.365089537819434e-06, 'epoch': 5.19}\n", + "{'loss': 0.1074, 'grad_norm': 0.817304253578186, 'learning_rate': 5.133412740456806e-06, 'epoch': 5.21}\n", + "{'loss': 0.0966, 'grad_norm': 1.4587805271148682, 'learning_rate': 4.906578283344759e-06, 'epoch': 5.23}\n", + "{'loss': 0.1326, 'grad_norm': 5.115628719329834, 'learning_rate': 4.684610648167503e-06, 'epoch': 5.25}\n", + "{'loss': 0.1112, 'grad_norm': 2.1370065212249756, 'learning_rate': 4.467533791345191e-06, 'epoch': 5.27}\n", + "{'loss': 0.0918, 'grad_norm': 1.5177031755447388, 'learning_rate': 4.255371141448272e-06, 'epoch': 5.28}\n", + "{'loss': 0.0911, 'grad_norm': 3.415386199951172, 'learning_rate': 4.048145596668967e-06, 'epoch': 5.3}\n", + "{'loss': 0.1295, 'grad_norm': 9.106415748596191, 'learning_rate': 3.84587952234991e-06, 'epoch': 5.32}\n", + "{'loss': 0.0753, 'grad_norm': 1.1960046291351318, 'learning_rate': 3.6485947485702832e-06, 'epoch': 5.34}\n", + "{'loss': 0.1281, 'grad_norm': 3.4662070274353027, 'learning_rate': 3.4563125677897932e-06, 'epoch': 5.35}\n", + "{'loss': 0.1045, 'grad_norm': 1.4903005361557007, 'learning_rate': 3.269053732550581e-06, 'epoch': 5.37}\n", + "{'loss': 0.094, 'grad_norm': 2.3145623207092285, 'learning_rate': 3.086838453237506e-06, 'epoch': 5.39}\n", + "{'loss': 0.056, 'grad_norm': 1.6177632808685303, 'learning_rate': 2.9096863958968268e-06, 'epoch': 5.41}\n", + "{'loss': 0.093, 'grad_norm': 1.7712160348892212, 'learning_rate': 2.737616680113758e-06, 'epoch': 5.43}\n", + "{'loss': 0.0846, 'grad_norm': 2.1207849979400635, 'learning_rate': 2.570647876948895e-06, 'epoch': 5.44}\n", + "{'loss': 0.1257, 'grad_norm': 1.7891684770584106, 'learning_rate': 2.408798006933882e-06, 'epoch': 5.46}\n", + "{'loss': 0.1472, 'grad_norm': 1.305862545967102, 'learning_rate': 2.252084538126542e-06, 'epoch': 5.48}\n", + "{'loss': 0.0784, 'grad_norm': 2.511289596557617, 'learning_rate': 2.100524384225555e-06, 'epoch': 5.5}\n", + "{'loss': 0.1159, 'grad_norm': 2.205674886703491, 'learning_rate': 1.9541339027450256e-06, 'epoch': 5.52}\n", + "{'loss': 0.1057, 'grad_norm': 2.3121867179870605, 'learning_rate': 1.8129288932490274e-06, 'epoch': 5.53}\n", + "{'loss': 0.1044, 'grad_norm': 0.5653843283653259, 'learning_rate': 1.6769245956464396e-06, 'epoch': 5.55}\n", + "{'loss': 0.1248, 'grad_norm': 2.8058314323425293, 'learning_rate': 1.5461356885461075e-06, 'epoch': 5.57}\n", + "{'loss': 0.1108, 'grad_norm': 1.7656151056289673, 'learning_rate': 1.4205762876726092e-06, 'epoch': 5.59}\n", + "{'loss': 0.103, 'grad_norm': 1.4396343231201172, 'learning_rate': 1.3002599443428243e-06, 'epoch': 5.6}\n", + "{'loss': 0.1239, 'grad_norm': 2.5784292221069336, 'learning_rate': 1.1851996440033319e-06, 'epoch': 5.62}\n", + "{'loss': 0.0961, 'grad_norm': 0.813414990901947, 'learning_rate': 1.0754078048289374e-06, 'epoch': 5.64}\n", + "{'loss': 0.1176, 'grad_norm': 2.7768945693969727, 'learning_rate': 9.708962763824048e-07, 'epoch': 5.66}\n", + "{'loss': 0.0784, 'grad_norm': 1.4548313617706299, 'learning_rate': 8.716763383355864e-07, 'epoch': 5.68}\n", + "{'loss': 0.0995, 'grad_norm': 1.4250032901763916, 'learning_rate': 7.777586992519959e-07, 'epoch': 5.69}\n", + "{'loss': 0.1014, 'grad_norm': 3.0032870769500732, 'learning_rate': 6.891534954310885e-07, 'epoch': 5.71}\n", + "{'loss': 0.0993, 'grad_norm': 3.392124891281128, 'learning_rate': 6.058702898142643e-07, 'epoch': 5.73}\n", + "{'loss': 0.0962, 'grad_norm': 2.9156267642974854, 'learning_rate': 5.279180709527765e-07, 'epoch': 5.75}\n", + "{'loss': 0.1407, 'grad_norm': 2.50022292137146, 'learning_rate': 4.553052520375911e-07, 'epoch': 5.77}\n", + "{'loss': 0.0832, 'grad_norm': 1.8522708415985107, 'learning_rate': 3.8803966999139684e-07, 'epoch': 5.78}\n", + "{'loss': 0.1036, 'grad_norm': 2.559648275375366, 'learning_rate': 3.261285846227868e-07, 'epoch': 5.8}\n", + "{'loss': 0.0851, 'grad_norm': 6.124639987945557, 'learning_rate': 2.6957867784270787e-07, 'epoch': 5.82}\n", + "{'loss': 0.0861, 'grad_norm': 1.8628261089324951, 'learning_rate': 2.1839605294330933e-07, 'epoch': 5.84}\n", + "{'loss': 0.1153, 'grad_norm': 2.0182836055755615, 'learning_rate': 1.725862339392259e-07, 'epoch': 5.85}\n", + "{'loss': 0.0913, 'grad_norm': 2.184485912322998, 'learning_rate': 1.3215416497138754e-07, 'epoch': 5.87}\n", + "{'loss': 0.132, 'grad_norm': 2.652066707611084, 'learning_rate': 9.710420977340762e-08, 'epoch': 5.89}\n", + "{'loss': 0.0822, 'grad_norm': 2.054509401321411, 'learning_rate': 6.744015120061509e-08, 'epoch': 5.91}\n", + "{'loss': 0.1632, 'grad_norm': 2.1160929203033447, 'learning_rate': 4.316519082179227e-08, 'epoch': 5.93}\n", + "{'loss': 0.0715, 'grad_norm': 3.3849403858184814, 'learning_rate': 2.4281948573617874e-08, 'epoch': 5.94}\n", + "{'loss': 0.1134, 'grad_norm': 3.3306052684783936, 'learning_rate': 1.0792462477909882e-08, 'epoch': 5.96}\n", + "{'loss': 0.1273, 'grad_norm': 2.356410026550293, 'learning_rate': 2.6981884216847884e-09, 'epoch': 5.98}\n", + "{'loss': 0.1189, 'grad_norm': 2.4627721309661865, 'learning_rate': 0.0, 'epoch': 6.0}\n", + "100%|█████████████████████████████████████| 3360/3360 [1:31:57<00:00, 1.60s/it][INFO|trainer.py:3788] 2024-07-05 07:48:24,113 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-05 07:48:24,113 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-05 07:48:24,113 >> Batch size = 1\n", + "\n", + " 0%| | 0/46 [00:00> Saving model checkpoint to saves/qwen2-0.5b/lora/sft/checkpoint-3360\n", + "[INFO|configuration_utils.py:733] 2024-07-05 07:48:28,128 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 07:48:28,128 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "[INFO|:482] 2024-07-05 07:48:28,348 >> \n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "{'train_runtime': 5527.5332, 'train_samples_per_second': 4.865, 'train_steps_per_second': 0.608, 'train_loss': 0.927943646074051, 'epoch': 6.0}\n", + "100%|█████████████████████████████████████| 3360/3360 [1:32:01<00:00, 1.64s/it]\n", + "[INFO|trainer.py:3478] 2024-07-05 07:48:28,351 >> Saving model checkpoint to saves/qwen2-0.5b/lora/sft\n", + "[INFO|configuration_utils.py:733] 2024-07-05 07:48:29,375 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--unsloth--qwen2-0.5b-instruct-bnb-4bit/snapshots/c3b24ce4827d69f5c3bde9aba00047774069ab72/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-05 07:48:29,376 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"quantization_config\": {\n", + " \"_load_in_4bit\": true,\n", + " \"_load_in_8bit\": false,\n", + " \"bnb_4bit_compute_dtype\": \"bfloat16\",\n", + " \"bnb_4bit_quant_storage\": \"uint8\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_enable_fp32_cpu_offload\": false,\n", + " \"llm_int8_has_fp16_weight\": false,\n", + " \"llm_int8_skip_modules\": null,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"load_in_4bit\": true,\n", + " \"load_in_8bit\": false,\n", + " \"quant_method\": \"bitsandbytes\"\n", + " },\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "***** train metrics *****\n", + " epoch = 5.9973\n", + " total_flos = 6320365GF\n", + " train_loss = 0.9279\n", + " train_runtime = 1:32:07.53\n", + " train_samples_per_second = 4.865\n", + " train_steps_per_second = 0.608\n", + "Figure saved at: saves/qwen2-0.5b/lora/sft/training_loss.png\n", + "Figure saved at: saves/qwen2-0.5b/lora/sft/training_eval_loss.png\n", + "[INFO|trainer.py:3788] 2024-07-05 07:48:29,751 >> \n", + "***** Running Evaluation *****\n", + "[INFO|trainer.py:3790] 2024-07-05 07:48:29,752 >> Num examples = 46\n", + "[INFO|trainer.py:3793] 2024-07-05 07:48:29,752 >> Batch size = 1\n", + "100%|███████████████████████████████████████████| 46/46 [00:03<00:00, 15.10it/s]\n", + "***** eval metrics *****\n", + " epoch = 5.9973\n", + " eval_loss = 3.5429\n", + " eval_runtime = 0:00:03.16\n", + " eval_samples_per_second = 14.532\n", + " eval_steps_per_second = 14.532\n", + "[INFO|modelcard.py:449] 2024-07-05 07:48:32,920 >> Dropping the following result as it does not have all the necessary fields:\n", + "{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: / 0.561 MB of 0.561 MB uploaded\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss ▁▁▃▄▆██\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime ▁▂▃▂▄▂█\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second █▇▆▇▅▇▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second █▇▆▇▅▇▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm ▂▁▁▂▁▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂█▂▂▂▂▁▁▁▂▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate ▂▄▅▇██████▇▇▇▇▇▆▆▆▆▅▅▅▄▄▄▃▃▃▃▂▂▂▂▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss ████▇▇▇▅▆▆▆▆▆▆▄▄▄▄▄▄▂▂▂▂▂▂▂▂▂▂▁▂▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss 3.54292\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime 3.1655\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second 14.532\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second 14.532\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: total_flos 6786441021493248.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch 5.99732\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step 3360\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm 2.46277\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss 0.1189\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 0.92794\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_runtime 5527.5332\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_samples_per_second 4.865\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_steps_per_second 0.608\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mqwen2_0.5b_lora_sft\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/3amepb0m\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240705_061623-3amepb0m/logs\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n", + "CPU times: user 1min 4s, sys: 21.9 s, total: 1min 26s\n", + "Wall time: 1h 33min 9s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "!./scripts/tune-lf.sh config/qwen2_0.5b_lora_sft_unsloth.yaml" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "07_MAC_+_Qwen2-7B-Instructi_Unsloth_train", + "widgets": {} + }, + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "036fc5746f43416db18c19ad8fd36677": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "06e806c82c7b4cbea31c5358dd9c3434": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "087b76a8b7514269b1f0ab29b062e444": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a069d2ab23824f29aa320ac256e2cfe9", + "placeholder": "​", + "style": "IPY_MODEL_06e806c82c7b4cbea31c5358dd9c3434", + "value": "Map (num_proc=2): 100%" + } + }, + "09b76013aa9e45efb6deb23a7a0d0925": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dea41c5260884aa6879b5e1d1697b14f", + "placeholder": "​", + "style": "IPY_MODEL_89965917796a4f81b899fdc7685f33df", + "value": "config.json: 100%" + } + }, + "0a92c56bfa134ef583220d7ef0b13e17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c34be936c8145d3ab41282f30a70713": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f8b6bfe16894500838793f2491d403f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "177c78fce95d4b4ab33057c5a048d693": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1f44c9ce1adf470cbb19784493ed209f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c34be936c8145d3ab41282f30a70713", + "placeholder": "​", + "style": "IPY_MODEL_0a92c56bfa134ef583220d7ef0b13e17", + "value": "model.safetensors: 100%" + } + }, + "201b59ccd9f845e197029b57e424aefc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2157f01726d748f8a9ae4a00664430da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21db8a77b00d4a4e82fdfa608657531f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26e4202cca81496a90d15a0dd4ca9cf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba90fdb8822d47dab7ba203bee297f37", + "IPY_MODEL_61560ff6a36b44f4a9dfdae5c52791d4", + "IPY_MODEL_95fbe66647904c06a20f640630d6dc0e" + ], + "layout": "IPY_MODEL_57182a263d324a3dbf1471c74290a0d5" + } + }, + "27155728b6b84cb199c91c940095d0a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6b91feeed5464877991ac2c207aebe7c", + "IPY_MODEL_cca8113c54c0495daedce1327bf9c68b", + "IPY_MODEL_2e63a29e2f7247bba5beede9a568c99f" + ], + "layout": "IPY_MODEL_5c9d781c28944f3eb86e2a6d44efdf18" + } + }, + "271ddaa553a042d09b6db7b450643d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a58d04b428c46f4b3dbadd3bc6cd529": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d18ddf6482c4d97829ac0e5a7b9868f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "IPY_MODEL_f2df530d22c74977b249dd9fb5f4829b", + "IPY_MODEL_89b2ef0dbfea47ab8e6f8d659e3351d1" + ], + "layout": "IPY_MODEL_3056b148aa9f4e6e8aa3b61d26886255" + } + }, + "2e5087c76f98437cb5dc729230358cba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e63a29e2f7247bba5beede9a568c99f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b993eaec6b224440bf80c0958c6fb536", + "placeholder": "​", + "style": "IPY_MODEL_de868e26e7154f62aa86223a539ad421", + "value": " 464/464 [00:00<00:00, 27.1kB/s]" + } + }, + "2f6c70dd266c4816bfad3fd3d192929a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "30307300bc4e4baf96560e30969a82b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e36a3f9eff0e4cf68834d66b0213ae96", + "placeholder": "​", + "style": "IPY_MODEL_a0037bdccf254159becde630bee3d1db", + "value": "generation_config.json: 100%" + } + }, + "3056b148aa9f4e6e8aa3b61d26886255": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30cdc32298134cb0be4d41615b9e5774": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3572201bd4d74a58b7a665f9bdfdcdba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35b0e8c26d6640e9bd0ed7b242a423d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e5087c76f98437cb5dc729230358cba", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_036fc5746f43416db18c19ad8fd36677", + "value": 51760 + } + }, + "36166c7bcb854b34aca1f41a5d6ea50b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "370692d819df41828b48c4ad446f977b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "39b29a75374b45c0a22506010be2b84e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_30cdc32298134cb0be4d41615b9e5774", + "max": 1179, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_47928317548c454bba6358ab132e8dee", + "value": 1179 + } + }, + "3cf2dd993b5e4d3daecf61e4bab5a404": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_087b76a8b7514269b1f0ab29b062e444", + "IPY_MODEL_35b0e8c26d6640e9bd0ed7b242a423d8", + "IPY_MODEL_54ad89e05fd74576b9b8b5b5a10eaf8d" + ], + "layout": "IPY_MODEL_a41dc44766444a998bec2d777f249d23" + } + }, + "43dec2ede91341f5af60eb522e18e984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4463edd481c1467f914c7dcd6c6e6ffc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47928317548c454bba6358ab132e8dee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "49277aeeac16434a865a4d12308b1abc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4ae7e449e4ea4c729b5f34607c18ebae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b2061b8a73c43ffb0c2f83daf0d0183": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c4c88d4c701450692fa0f6b0c5764b0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c666f4ace3943f8b80ecd20e7503236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ccedf0d93094e63b57a0f8a434fba06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4463edd481c1467f914c7dcd6c6e6ffc", + "max": 44307561, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6d3b9a05db0b4dadb638c686faa0c40a", + "value": 44307561 + } + }, + "4dcf6ff672d24983a1877a8431709aa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5807d5fb827d490fb3bc698f801ffff5", + "placeholder": "​", + "style": "IPY_MODEL_c4f2b06a82fd4987b8b659524a7b503b", + "value": "Generating train split: 100%" + } + }, + "4ea63adfce694725bdba878aef709dd3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5234566b1bfc4655b8d582ea5b46ed9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "54ad89e05fd74576b9b8b5b5a10eaf8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdb1941405ed4e4aa06019933892deb3", + "placeholder": "​", + "style": "IPY_MODEL_668d5377ca56426a99753867e6e24862", + "value": " 51760/51760 [01:02<00:00, 1131.51 examples/s]" + } + }, + "56aee4853b7740e6a977254f5d1fa66d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "57182a263d324a3dbf1471c74290a0d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5807d5fb827d490fb3bc698f801ffff5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c9d781c28944f3eb86e2a6d44efdf18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f40db8173dd4d76b6ef5ed6d9ec8b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61560ff6a36b44f4a9dfdae5c52791d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db19fc8d37db4e45a5790a876836d8c4", + "max": 11610, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36166c7bcb854b34aca1f41a5d6ea50b", + "value": 11610 + } + }, + "6578fd7acdb54c4c93528ea431fd0144": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_370692d819df41828b48c4ad446f977b", + "placeholder": "​", + "style": "IPY_MODEL_a0bf9160eb2647409b3200270914b90f", + "value": " 50.6k/50.6k [00:00<00:00, 2.71MB/s]" + } + }, + "668d5377ca56426a99753867e6e24862": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697f027529b54ee9956bae78a11e0611": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69ac12aec0714318bf2c83d4f4e745f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b2012c3f88547af8884a9ea90e3164b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_938f45f1b3e24118b815d96ae34ba86a", + "placeholder": "​", + "style": "IPY_MODEL_9367047a800747f79c6b225d92397846", + "value": " 44.3M/44.3M [00:01<00:00, 31.0MB/s]" + } + }, + "6b91feeed5464877991ac2c207aebe7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b2061b8a73c43ffb0c2f83daf0d0183", + "placeholder": "​", + "style": "IPY_MODEL_69ac12aec0714318bf2c83d4f4e745f5", + "value": "special_tokens_map.json: 100%" + } + }, + "6d3b9a05db0b4dadb638c686faa0c40a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6dbbedeca9314e66ae50e44ffa31a414": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6e34619b45934040b6092e6fb01ea7fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71ce208e20d6483abb9ed923510c86d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d69dc491b3ab44d7852b21873ed7bb7f", + "placeholder": "​", + "style": "IPY_MODEL_f401d53bf28e44eb906bce6c05412662", + "value": " 51760/51760 [00:01<00:00, 45512.81 examples/s]" + } + }, + "7358cdad832342c983e31efb8754ab78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e352a3404f4c7dad0737f57d29e92f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_988a0e8c1f89446086858da0a891a79c", + "IPY_MODEL_4ccedf0d93094e63b57a0f8a434fba06", + "IPY_MODEL_6b2012c3f88547af8884a9ea90e3164b" + ], + "layout": "IPY_MODEL_7e29cb8dd4df4d5b94407cd8fd3f2011" + } + }, + "74501720ac7e4dbb911a4a99b3633bc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "78e5400bff924a92a4cc61c4ff18b182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9b313fd861948f5aba25b24b1518d30", + "placeholder": "​", + "style": "IPY_MODEL_4c666f4ace3943f8b80ecd20e7503236", + "value": " 1.18k/1.18k [00:00<00:00, 31.3kB/s]" + } + }, + "7975adbc2ec5489ea7fa0167e620d85c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e34619b45934040b6092e6fb01ea7fe", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_271ddaa553a042d09b6db7b450643d8f", + "value": 51760 + } + }, + "7e29cb8dd4df4d5b94407cd8fd3f2011": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "810ff6c0e17d4fa09a30fef27eacff90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89965917796a4f81b899fdc7685f33df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "89b2ef0dbfea47ab8e6f8d659e3351d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8908fa0df3743ecb9d12983a739104f", + "placeholder": "​", + "style": "IPY_MODEL_177c78fce95d4b4ab33057c5a048d693", + "value": " 9.09M/9.09M [00:00<00:00, 32.6MB/s]" + } + }, + "8b3505352a5a42bf910428c40ce40465": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_49277aeeac16434a865a4d12308b1abc", + "placeholder": "​", + "style": "IPY_MODEL_2157f01726d748f8a9ae4a00664430da", + "value": " 5.70G/5.70G [01:02<00:00, 30.1MB/s]" + } + }, + "8fc142b628fb40568730234de1cafde2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ae7e449e4ea4c729b5f34607c18ebae", + "max": 172, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3572201bd4d74a58b7a665f9bdfdcdba", + "value": 172 + } + }, + "9367047a800747f79c6b225d92397846": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "938f45f1b3e24118b815d96ae34ba86a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "95fbe66647904c06a20f640630d6dc0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b0a370dc20654b279b9680692e34418e", + "placeholder": "​", + "style": "IPY_MODEL_cfeb365ddf7548d58b2557f22737fcf5", + "value": " 11.6k/11.6k [00:00<00:00, 716kB/s]" + } + }, + "988a0e8c1f89446086858da0a891a79c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ad2be500fc164c0f86f33e914ef8e6a0", + "placeholder": "​", + "style": "IPY_MODEL_5234566b1bfc4655b8d582ea5b46ed9f", + "value": "Downloading data: 100%" + } + }, + "98c58f23f4d549518832cb2d18f796e8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_09b76013aa9e45efb6deb23a7a0d0925", + "IPY_MODEL_39b29a75374b45c0a22506010be2b84e", + "IPY_MODEL_78e5400bff924a92a4cc61c4ff18b182" + ], + "layout": "IPY_MODEL_2a58d04b428c46f4b3dbadd3bc6cd529" + } + }, + "99fdbb0300c14c139d1937c646f0cfe7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7358cdad832342c983e31efb8754ab78", + "placeholder": "​", + "style": "IPY_MODEL_e9adf418296e436fb48bb9f78885598b", + "value": " 51760/51760 [00:01<00:00, 38665.95 examples/s]" + } + }, + "9f679ad3ec7f4fe8ad0510ffb57bc2ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ea63adfce694725bdba878aef709dd3", + "placeholder": "​", + "style": "IPY_MODEL_74501720ac7e4dbb911a4a99b3633bc6", + "value": "tokenizer.json: 100%" + } + }, + "a0037bdccf254159becde630bee3d1db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a069d2ab23824f29aa320ac256e2cfe9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0bf9160eb2647409b3200270914b90f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a41dc44766444a998bec2d777f249d23": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8464a4c711e4e00aafdfc919b60d07e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fb995c740590427b882572c81d4e848c", + "placeholder": "​", + "style": "IPY_MODEL_201b59ccd9f845e197029b57e424aefc", + "value": " 172/172 [00:00<00:00, 12.0kB/s]" + } + }, + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad2be500fc164c0f86f33e914ef8e6a0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b0240cd9a4554b29ae11f8051984a1c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_edaf890370314a218f138015faa0b05d", + "placeholder": "​", + "style": "IPY_MODEL_697f027529b54ee9956bae78a11e0611", + "value": "Map: 100%" + } + }, + "b0a370dc20654b279b9680692e34418e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b518dcee69074b87be73957cd810e7ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d891f8d0b1fc462f8008d02bb2a15692", + "placeholder": "​", + "style": "IPY_MODEL_cced8fd7e998472794f3f3e3018956a5", + "value": "tokenizer_config.json: 100%" + } + }, + "b8908fa0df3743ecb9d12983a739104f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b993eaec6b224440bf80c0958c6fb536": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9b313fd861948f5aba25b24b1518d30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba90fdb8822d47dab7ba203bee297f37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f8b6bfe16894500838793f2491d403f", + "placeholder": "​", + "style": "IPY_MODEL_bb19f6c747754682a514373a3a0535ba", + "value": "Downloading readme: 100%" + } + }, + "bb19f6c747754682a514373a3a0535ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc883d4cf13e4f8b8a4fe5f410cb6efd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9159e03e61f4f56978ece9c3bca49b2", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_810ff6c0e17d4fa09a30fef27eacff90", + "value": 51760 + } + }, + "c161d94df0f04feba9542237e0856c22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c22f71b1f85843209d7e5321506b9cb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f44c9ce1adf470cbb19784493ed209f", + "IPY_MODEL_f1addc4479d849879e743cf9089e6540", + "IPY_MODEL_8b3505352a5a42bf910428c40ce40465" + ], + "layout": "IPY_MODEL_4c4c88d4c701450692fa0f6b0c5764b0" + } + }, + "c4f2b06a82fd4987b8b659524a7b503b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cca8113c54c0495daedce1327bf9c68b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e02f9b7849c64531835eb77b860d1c93", + "max": 464, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_56aee4853b7740e6a977254f5d1fa66d", + "value": 464 + } + }, + "cced8fd7e998472794f3f3e3018956a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cf245afeb1c04f29a24d291608c3d157": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b518dcee69074b87be73957cd810e7ed", + "IPY_MODEL_e29104486d594b2992d7285e0ef77371", + "IPY_MODEL_6578fd7acdb54c4c93528ea431fd0144" + ], + "layout": "IPY_MODEL_d35db8148a354c56aaac56dbae22536f" + } + }, + "cfe8cae0e22b495bafa221a63d13b283": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfeb365ddf7548d58b2557f22737fcf5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1b47d39450d4019ae85c9b2f943eeaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4dcf6ff672d24983a1877a8431709aa9", + "IPY_MODEL_7975adbc2ec5489ea7fa0167e620d85c", + "IPY_MODEL_71ce208e20d6483abb9ed923510c86d7" + ], + "layout": "IPY_MODEL_cfe8cae0e22b495bafa221a63d13b283" + } + }, + "d35db8148a354c56aaac56dbae22536f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69dc491b3ab44d7852b21873ed7bb7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d891f8d0b1fc462f8008d02bb2a15692": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8e5318cead340c4adbeaccc05d39225": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "daf4cd890b35422683d22fd30bc71e83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b0240cd9a4554b29ae11f8051984a1c6", + "IPY_MODEL_bc883d4cf13e4f8b8a4fe5f410cb6efd", + "IPY_MODEL_99fdbb0300c14c139d1937c646f0cfe7" + ], + "layout": "IPY_MODEL_c161d94df0f04feba9542237e0856c22" + } + }, + "db19fc8d37db4e45a5790a876836d8c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de868e26e7154f62aa86223a539ad421": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dea41c5260884aa6879b5e1d1697b14f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e02f9b7849c64531835eb77b860d1c93": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e29104486d594b2992d7285e0ef77371": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "max": 50641, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2f6c70dd266c4816bfad3fd3d192929a", + "value": 50641 + } + }, + "e36a3f9eff0e4cf68834d66b0213ae96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9159e03e61f4f56978ece9c3bca49b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9adf418296e436fb48bb9f78885598b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "edaf890370314a218f138015faa0b05d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f1addc4479d849879e743cf9089e6540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43dec2ede91341f5af60eb522e18e984", + "max": 5702746405, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d8e5318cead340c4adbeaccc05d39225", + "value": 5702746405 + } + }, + "f2df530d22c74977b249dd9fb5f4829b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_21db8a77b00d4a4e82fdfa608657531f", + "max": 9085698, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6dbbedeca9314e66ae50e44ffa31a414", + "value": 9085698 + } + }, + "f401d53bf28e44eb906bce6c05412662": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fb995c740590427b882572c81d4e848c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fce7a61c25ec4390af43d92b7c473a45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_30307300bc4e4baf96560e30969a82b6", + "IPY_MODEL_8fc142b628fb40568730234de1cafde2", + "IPY_MODEL_a8464a4c711e4e00aafdfc919b60d07e" + ], + "layout": "IPY_MODEL_5f40db8173dd4d76b6ef5ed6d9ec8b6e" + } + }, + "fdb1941405ed4e4aa06019933892deb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/08_eval-lf-py3.11.ipynb b/notebooks/08_eval-lf-py3.11.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..7d84f46baefc150de3a9123435a97c562bbf6b74 --- /dev/null +++ b/notebooks/08_eval-lf-py3.11.ipynb @@ -0,0 +1,6437 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "0ea8b46b-839b-445b-8043-ccdf4e920ace", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6d394937-6c99-4a7c-9d32-7600a280032f", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: /home/inflaton/code/projects/courses/llm-finetuning\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"workding dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9f67ec60-2f24-411c-84eb-0dd664b44775", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f1597656-8042-4878-9d3b-9ebfb8dd86dc", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('unsloth/Qwen2-0.5B-Instruct',\n", + " True,\n", + " None,\n", + " None,\n", + " 2048,\n", + " 10,\n", + " None,\n", + " 'datasets/mac/mac.tsv',\n", + " 'results/mac-results_lf.csv')" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "model_name = os.getenv(\"MODEL_NAME\")\n", + "token = os.getenv(\"HF_TOKEN\") or None\n", + "load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n", + "local_model = os.getenv(\"LOCAL_MODEL\")\n", + "hub_model = os.getenv(\"HUB_MODEL\")\n", + "num_train_epochs = int(os.getenv(\"NUM_TRAIN_EPOCHS\") or 0)\n", + "data_path = os.getenv(\"DATA_PATH\")\n", + "results_path = os.getenv(\"RESULTS_PATH\")\n", + "\n", + "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n", + "dtype = (\n", + " None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n", + ")\n", + "\n", + "model_name, load_in_4bit, local_model, hub_model, max_seq_length, num_train_epochs, dtype, data_path, results_path" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sun Jun 30 13:21:10 2024 \n", + "+---------------------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 545.23.07 Driver Version: 546.12 CUDA Version: 12.3 |\n", + "|-----------------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|=========================================+======================+======================|\n", + "| 0 NVIDIA GeForce RTX 4080 ... On | 00000000:01:00.0 Off | N/A |\n", + "| N/A 49C P8 3W / 150W | 194MiB / 12282MiB | 0% Default |\n", + "| | | N/A |\n", + "+-----------------------------------------+----------------------+----------------------+\n", + " \n", + "+---------------------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=======================================================================================|\n", + "| No running processes found |\n", + "+---------------------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "!nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Python 3.11.9\n", + "\u001b[33mWARNING: Package(s) not found: flash-attn\u001b[0m\u001b[33m\n", + "\u001b[0mCPU times: user 32 ms, sys: 10.6 ms, total: 42.6 ms\n", + "Wall time: 1.23 s\n" + ] + } + ], + "source": [ + "%%time\n", + "!python --version\n", + "!pip show flash-attn" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-0.5B-Instruct llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-560 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-0.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 13:21:34,519 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 13:21:34,519 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 13:21:34,519 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 13:21:34,519 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 13:21:34,519 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 13:21:34,520 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 13:21:34,863 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 13:21:34 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 13:21:34 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 13:21:35,179 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 13:21:35,181 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 13:21:35 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 13:21:35 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 13:21:35,287 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 13:21:37,852 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 13:21:37,860 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 13:22:34,747 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 13:22:34,747 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 13:22:35,055 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 13:22:35,055 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 13:22:35 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 13:22:36 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-560\n", + "06/30/2024 13:22:36 - INFO - llamafactory.model.loader - all params: 498,431,872\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.666 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Trinket raised his gun and squinted his triangular eye. The trigger sounded as if a bullet had been shot; the bullets ricocheted off of the branches like hailstones. The sound was so loud that it could be heard from miles away.\n", + "--------\n", + "step 3: Old Trinket raised his gun and squinted his triangular eye. The trigger sounded as if a bullet had been shot; the bullets ricocheted off of the branches like hailstones. The sound was so loud that it could be heard from miles away.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:43:14<00:00, 5.47s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.9 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-0.5B-Instruct_checkpoint-560\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Trinket raised his gun and squinted his tr...\n", + "\n", + "[1 rows x 3 columns]\n", + "{'accuracy': 0.00088261253309797, 'correct_ids': [272], 'meteor': 0.28906766286950575, 'bleu_scores': {'bleu': 0.05350226890847294, 'precisions': [0.34546985517009093, 0.08439261827222748, 0.02716499544211486, 0.011066742726754135], 'brevity_penalty': 0.9833003245834433, 'length_ratio': 0.9834382245776747, 'translation_length': 29690, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.32218455635719456, 'rouge2': 0.09323903991316618, 'rougeL': 0.26091815189986767, 'rougeLsum': 0.2609816275457334}}\n", + "Epoch 2\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-0.5B-Instruct llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-1120 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-0.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:06:25,573 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:06:25,574 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:06:25,574 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:06:25,574 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:06:25,574 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:06:25,574 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 15:06:25,971 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 15:06:25 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 15:06:25 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 15:06:26,308 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 15:06:26,309 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 15:06:26 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 15:06:26 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 15:06:26,450 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 15:06:28,647 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 15:06:28,655 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 15:07:24,483 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 15:07:24,484 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 15:07:24,816 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 15:07:24,816 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 15:07:25 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 15:07:25 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-1120\n", + "06/30/2024 15:07:25 - INFO - llamafactory.model.loader - all params: 498,431,872\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.666 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his gun, his eyes narrowed. The shotgun fired; a deafening boom of gunfire followed, crickets chirping in the air, the sound like the cracking of ice chips on branches.\n", + "--------\n", + "step 3: Old Geng raised his gun, his eyes narrowed. The shotgun fired; a deafening boom of gunfire followed, crickets chirping in the air, the sound like the cracking of ice chips on branches.\n", + "100%|███████████████████████████████████████| 1133/1133 [46:43<00:00, 2.47s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.9 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-0.5B-Instruct_checkpoint-1120\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his gun, his eyes narrowed. Th...\n", + "\n", + "[1 rows x 4 columns]\n", + "{'accuracy': 0.00088261253309797, 'correct_ids': [659], 'meteor': 0.3075388134142166, 'bleu_scores': {'bleu': 0.06482340202869877, 'precisions': [0.36907098754416645, 0.10273004537677602, 0.038322655794991264, 0.01656785511248274], 'brevity_penalty': 0.9254426305194808, 'length_ratio': 0.9280887711162636, 'translation_length': 28019, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.34374303845779386, 'rouge2': 0.11899790599832506, 'rougeL': 0.2851818971023854, 'rougeLsum': 0.285674896233578}}\n", + "Epoch 3\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-0.5B-Instruct llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-1680 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-0.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:54:26,677 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:54:26,678 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:54:26,678 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:54:26,678 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:54:26,678 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 15:54:26,678 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 15:54:26,803 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 15:54:26 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 15:54:26 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 15:54:27,176 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 15:54:27,177 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 15:54:27 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 15:54:27 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 15:54:27,212 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 15:54:27,943 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 15:54:27,946 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 15:54:50,953 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 15:54:50,954 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 15:54:51,228 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 15:54:51,228 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 15:54:51 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 15:54:51 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-1680\n", + "06/30/2024 15:54:51 - INFO - llamafactory.model.loader - all params: 498,431,872\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.666 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng held his gun up, half-closed, and cocked it as if he was aiming for a bullet. The barrel cracked, and bullets flew down like ice nuggets; the leaves on the riverbank splashed like cannonballs.\n", + "--------\n", + "step 3: Old Geng held his gun up, half-closed, and cocked it as if he was aiming for a bullet. The barrel cracked, and bullets flew down like ice nuggets; the leaves on the riverbank splashed like cannonballs.\n", + "100%|███████████████████████████████████████| 1133/1133 [44:44<00:00, 2.37s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "4.24 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-0.5B-Instruct_checkpoint-1680\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng held his gun up, half-closed, and coc...\n", + "\n", + "[1 rows x 5 columns]\n", + "{'accuracy': 0.00353045013239188, 'correct_ids': [147, 194, 202, 364], 'meteor': 0.3232125016634757, 'bleu_scores': {'bleu': 0.06687635711488571, 'precisions': [0.33171058236475387, 0.0956102480068068, 0.03666427030913012, 0.017202185050724392], 'brevity_penalty': 1.0, 'length_ratio': 1.0886386220602848, 'translation_length': 32866, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.352664915385991, 'rouge2': 0.1232869942455126, 'rougeL': 0.2909052156293055, 'rougeLsum': 0.2907588163008441}}\n", + "Epoch 4\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-0.5B-Instruct llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-2240 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-0.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 16:39:54,252 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 16:39:54,252 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 16:39:54,252 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 16:39:54,252 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 16:39:54,252 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 16:39:54,252 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 16:39:54,394 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 16:39:54 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 16:39:54 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 16:39:54,662 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 16:39:54,663 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 16:39:54 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 16:39:54 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 16:39:54,705 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 16:39:55,523 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 16:39:55,526 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 16:40:17,339 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 16:40:17,339 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 16:40:17,617 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 16:40:17,617 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 16:40:17 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 16:40:18 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-2240\n", + "06/30/2024 16:40:18 - INFO - llamafactory.model.loader - all params: 498,431,872\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.666 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his gun, his triangular eye half-lit. The trigger sounded as he fired, bullets raining down like a cold stinker from skyward. The metal chattering sounded as if it had broken glass in the branches of the willows.\n", + "--------\n", + "step 3: Old Geng raised his gun, his triangular eye half-lit. The trigger sounded as he fired, bullets raining down like a cold stinker from skyward. The metal chattering sounded as if it had broken glass in the branches of the willows.\n", + "100%|███████████████████████████████████████| 1133/1133 [57:01<00:00, 3.02s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "4.221 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-0.5B-Instruct_checkpoint-2240\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his gun, his triangular eye ha...\n", + "\n", + "[1 rows x 6 columns]\n", + "{'accuracy': 0.00441306266548985, 'correct_ids': [147, 202, 364, 533, 850], 'meteor': 0.3141676906431015, 'bleu_scores': {'bleu': 0.05981782718505817, 'precisions': [0.2922991381706978, 0.08376151792634268, 0.033080163769061886, 0.01580821413223648], 'brevity_penalty': 1.0, 'length_ratio': 1.1914541238820802, 'translation_length': 35970, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3493638878638674, 'rouge2': 0.1255400870123861, 'rougeL': 0.2910327113370838, 'rougeLsum': 0.2905461546619883}}\n", + "Epoch 5\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-0.5B-Instruct llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-2800 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-0.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 17:37:38,874 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 17:37:38,874 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 17:37:38,875 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 17:37:38,875 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 17:37:38,875 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 17:37:38,875 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 17:37:39,004 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 17:37:39 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 17:37:39 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 17:37:39,272 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 17:37:39,272 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 17:37:39 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 17:37:39 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 17:37:39,347 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 17:37:41,000 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 17:37:41,003 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 17:38:03,532 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 17:38:03,532 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 17:38:03,825 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 17:38:03,825 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 17:38:04 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 17:38:04 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-2800\n", + "06/30/2024 17:38:04 - INFO - llamafactory.model.loader - all params: 498,431,872\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.666 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng took out his pistol, squinted over a triangular brow, then fired. A hail of bullets fell like ice-crystals from the sky: crisscrossing branches crackled with sounds like a bucketful of cold metal sparrows flying through the air.\n", + "--------\n", + "step 3: Old Geng took out his pistol, squinted over a triangular brow, then fired. A hail of bullets fell like ice-crystals from the sky: crisscrossing branches crackled with sounds like a bucketful of cold metal sparrows flying through the air.\n", + "100%|███████████████████████████████████████| 1133/1133 [44:54<00:00, 2.38s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "4.201 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-0.5B-Instruct_checkpoint-2800\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng took out his pistol, squinted over a ...\n", + "\n", + "[1 rows x 7 columns]\n", + "{'accuracy': 0.00264783759929391, 'correct_ids': [147, 194, 364], 'meteor': 0.31468732087511564, 'bleu_scores': {'bleu': 0.06531154622295796, 'precisions': [0.31492039110270875, 0.09110635696821516, 0.03624564735675847, 0.017496635262449527], 'brevity_penalty': 1.0, 'length_ratio': 1.121331566743955, 'translation_length': 33853, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3477119790584883, 'rouge2': 0.12383470549112005, 'rougeL': 0.28723768855041154, 'rougeLsum': 0.287515203604385}}\n", + "Epoch 6\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-0.5B-Instruct llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-3360 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-0.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 18:23:20,733 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 18:23:20,733 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 18:23:20,733 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 18:23:20,733 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 18:23:20,733 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 18:23:20,733 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 18:23:20,880 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 18:23:20 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 18:23:20 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 18:23:21,195 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 18:23:21,195 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 18:23:21 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 18:23:21 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 18:23:21,271 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 18:23:23,604 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 18:23:23,608 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 18:23:50,830 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 18:23:50,830 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 18:23:51,197 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 18:23:51,197 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 18:23:51 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 18:23:51 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-3360\n", + "06/30/2024 18:23:51 - INFO - llamafactory.model.loader - all params: 498,431,872\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.666 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng held his rifle up and cocked it over his right eye. Then the shotgun fired, loud as a bolt of golden sparrows that fell like ice-cold quakes down on the field. The chattering bits of iron were heard splashing across the field, cracking and crunching.\n", + "--------\n", + "step 3: Old Geng held his rifle up and cocked it over his right eye. Then the shotgun fired, loud as a bolt of golden sparrows that fell like ice-cold quakes down on the field. The chattering bits of iron were heard splashing across the field, cracking and crunching.\n", + "100%|███████████████████████████████████████| 1133/1133 [45:34<00:00, 2.41s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "4.26 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-0.5B-Instruct_checkpoint-3360\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng held his rifle up and cocked it over ...\n", + "\n", + "[1 rows x 8 columns]\n", + "{'accuracy': 0.00176522506619594, 'correct_ids': [272, 364], 'meteor': 0.3060953047058868, 'bleu_scores': {'bleu': 0.06197290227987762, 'precisions': [0.30625790139064474, 0.08672151109263164, 0.03420510771689357, 0.01623686723973257], 'brevity_penalty': 1.0, 'length_ratio': 1.1528320635972176, 'translation_length': 34804, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.33981449502350625, 'rouge2': 0.11735200363049994, 'rougeL': 0.2798705836787463, 'rougeLsum': 0.27962230715315634}}\n", + "Epoch 7\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-0.5B-Instruct llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-3920 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-0.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:09:46,741 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:09:46,741 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:09:46,741 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:09:46,741 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:09:46,741 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:09:46,741 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 19:09:46,876 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 19:09:46 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 19:09:46 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 19:09:47,204 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 19:09:47,204 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 19:09:47 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 19:09:47 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 19:09:47,246 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 19:09:48,444 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 19:09:48,446 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 19:10:17,136 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 19:10:17,136 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 19:10:17,747 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 19:10:17,747 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 19:10:18 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 19:10:18 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-3920\n", + "06/30/2024 19:10:18 - INFO - llamafactory.model.loader - all params: 498,431,872\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.666 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng held his gun to his chest, eyes on a triangle shape, but the trigger sounded when he cocked it and fired: shot after shot of golden sparrows thundered down the slope, and shattering stones clattered as they fell through the air.\n", + "--------\n", + "step 3: Old Geng held his gun to his chest, eyes on a triangle shape, but the trigger sounded when he cocked it and fired: shot after shot of golden sparrows thundered down the slope, and shattering stones clattered as they fell through the air.\n", + "100%|███████████████████████████████████████| 1133/1133 [33:14<00:00, 1.76s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.057 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-0.5B-Instruct_checkpoint-3920\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng held his gun to his chest, eyes on a ...\n", + "\n", + "[1 rows x 9 columns]\n", + "{'accuracy': 0.00088261253309797, 'correct_ids': [364], 'meteor': 0.29569751947150547, 'bleu_scores': {'bleu': 0.06290335358107121, 'precisions': [0.33640226628895187, 0.09157729444388761, 0.033815921952574386, 0.015028901734104046], 'brevity_penalty': 1.0, 'length_ratio': 1.0289499834382245, 'translation_length': 31064, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.328871406524778, 'rouge2': 0.10887785000250436, 'rougeL': 0.2694111761024649, 'rougeLsum': 0.2691332869747859}}\n", + "Epoch 8\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-0.5B-Instruct llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-4480 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-0.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:43:50,488 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:43:50,488 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:43:50,488 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:43:50,488 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:43:50,488 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 19:43:50,488 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 19:43:50,640 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 19:43:50 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 19:43:50 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 19:43:50,918 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 19:43:50,918 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 19:43:50 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 19:43:50 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 19:43:50,973 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 19:43:51,791 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 19:43:51,794 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 19:44:16,853 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 19:44:16,853 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 19:44:17,214 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 19:44:17,214 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 19:44:17 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 19:44:17 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-4480\n", + "06/30/2024 19:44:17 - INFO - llamafactory.model.loader - all params: 498,431,872\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.666 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng took up his gun and raised a triangular brow – the cocking of the revolver started as soon as he lifted it. Bang! Bang! Bang! – hail was raining down from the heavens in a torrent of gold sparrows that sounded like hail as they whizzed down the path, rustling leaves as they passed by.\n", + "--------\n", + "step 3: Old Geng took up his gun and raised a triangular brow – the cocking of the revolver started as soon as he lifted it. Bang! Bang! Bang! – hail was raining down from the heavens in a torrent of gold sparrows that sounded like hail as they whizzed down the path, rustling leaves as they passed by.\n", + "100%|███████████████████████████████████████| 1133/1133 [34:03<00:00, 1.80s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.9 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-0.5B-Instruct_checkpoint-4480\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng took up his gun and raised a triangul...\n", + "\n", + "[1 rows x 10 columns]\n", + "{'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.29297589531864165, 'bleu_scores': {'bleu': 0.066715036654756, 'precisions': [0.33156006043817676, 0.0917577933735923, 0.03666926492018843, 0.017757733774927772], 'brevity_penalty': 1.0, 'length_ratio': 1.0522689632328586, 'translation_length': 31768, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3235260151085271, 'rouge2': 0.10613228641914846, 'rougeL': 0.2654728857129883, 'rougeLsum': 0.26595119389766264}}\n", + "Epoch 9\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-0.5B-Instruct llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-5040 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-0.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 20:18:42,543 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 20:18:42,544 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 20:18:42,544 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 20:18:42,544 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 20:18:42,544 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 20:18:42,544 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 20:18:42,670 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 20:18:42 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 20:18:42 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 20:18:42,995 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 20:18:42,995 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 20:18:42 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 20:18:42 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 20:18:43,139 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 20:18:44,397 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 20:18:44,400 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 20:19:10,704 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 20:19:10,704 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 20:19:11,065 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 20:19:11,065 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 20:19:11 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 20:19:11 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-5040\n", + "06/30/2024 20:19:11 - INFO - llamafactory.model.loader - all params: 498,431,872\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.666 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng took out his pistol, but it was too late. Shot after shot of shotgun went off as he held the trigger, a bang-bang-gong-tingling fall from the din of bullets falling from the air.\n", + "--------\n", + "step 3: Old Geng took out his pistol, but it was too late. Shot after shot of shotgun went off as he held the trigger, a bang-bang-gong-tingling fall from the din of bullets falling from the air.\n", + "100%|███████████████████████████████████████| 1133/1133 [47:05<00:00, 2.49s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "4.221 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-0.5B-Instruct_checkpoint-5040\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng took out his pistol, but it was too l...\n", + "\n", + "[1 rows x 11 columns]\n", + "{'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.2833319356953958, 'bleu_scores': {'bleu': 0.05430760022077538, 'precisions': [0.28200039135660976, 0.0749133949191686, 0.029243256147051803, 0.01408015809300315], 'brevity_penalty': 1.0, 'length_ratio': 1.184928784365684, 'translation_length': 35773, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3123182638202295, 'rouge2': 0.1006380742528073, 'rougeL': 0.25624416362806557, 'rougeLsum': 0.25609208337653155}}\n", + "Epoch 10\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-0.5B-Instruct llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-5600 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-0.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:06:41,264 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:06:41,264 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:06:41,264 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:06:41,264 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:06:41,264 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:06:41,264 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 21:06:41,413 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 21:06:41 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 21:06:41 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 21:06:41,679 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 21:06:41,680 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-0.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 896,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 4864,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 24,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 14,\n", + " \"num_hidden_layers\": 24,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 21:06:41 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 21:06:41 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 21:06:41,746 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 21:06:42,649 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 21:06:42,653 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 21:07:13,550 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 21:07:13,550 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-0.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 21:07:13,853 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct/snapshots/c291d6fce4804a1d39305f388dd32897d1f7acc4/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 21:07:13,853 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 21:07:14 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 21:07:14 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-5600\n", + "06/30/2024 21:07:14 - INFO - llamafactory.model.loader - all params: 498,431,872\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.666 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-0.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng took up his gun and fired – hammering rain! Yellow streaks flew as he fired a goose-pat of gold – and then there was the sound of gravel between his bullets – crunchy.\n", + "--------\n", + "step 3: Old Geng took up his gun and fired – hammering rain! Yellow streaks flew as he fired a goose-pat of gold – and then there was the sound of gravel between his bullets – crunchy.\n", + "100%|███████████████████████████████████████| 1133/1133 [48:43<00:00, 2.58s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "4.221 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-0.5B-Instruct_checkpoint-5600\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng took up his gun and fired – hammering...\n", + "\n", + "[1 rows x 12 columns]\n", + "{'accuracy': 0.0, 'correct_ids': [], 'meteor': 0.28432663251720675, 'bleu_scores': {'bleu': 0.052792420940353475, 'precisions': [0.29167024596970476, 0.07445989937851435, 0.0279223562549752, 0.012809131261889664], 'brevity_penalty': 1.0, 'length_ratio': 1.156773766147731, 'translation_length': 34923, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.31243618674019946, 'rouge2': 0.09792736995151512, 'rougeL': 0.25604383226456534, 'rougeLsum': 0.2555907570933199}}\n", + "CPU times: user 12min 29s, sys: 4min 19s, total: 16min 48s\n", + "Wall time: 8h 34min 52s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "os.environ[\"MODEL_NAME\"] = \"Qwen/Qwen2-0.5B-Instruct\" \n", + "for i in range(1, num_train_epochs + 1):\n", + " print(f\"Epoch {i}\")\n", + " adapter_path = f\"llama-factory/saves/qwen2-0.5b/lora/sft/checkpoint-{560 * i}\"\n", + " os.environ[\"ADAPTER_NAME_OR_PATH\"] = adapter_path\n", + " !python llm_toolkit/eval.py " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-1.5B-Instruct llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-560 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-1.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:56:19,887 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:56:19,887 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:56:19,887 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:56:19,887 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:56:19,887 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 21:56:19,887 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 21:56:20,070 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 21:56:20 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 21:56:20 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 21:56:20,393 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 21:56:20,393 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 21:56:20 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 21:56:20 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 21:56:20,457 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 21:56:22,769 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 21:56:22,772 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 21:58:45,740 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 21:58:45,740 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 21:58:46,024 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 21:58:46,024 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 21:58:46 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 21:58:46 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-560\n", + "06/30/2024 21:58:46 - INFO - llamafactory.model.loader - all params: 1,552,946,688\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.604 GB of memory reserved.\n", + "loading train/test data files\n", + "Map: 100%|████████████████████████| 4528/4528 [00:00<00:00, 35828.73 examples/s]\n", + "Map: 100%|████████████████████████| 1133/1133 [00:00<00:00, 12322.75 examples/s]\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Grannie Geng held up his gun with one eye, narrowed it, raised the barrel of the rifle, fired a hail of bullets at the target.\n", + "--------\n", + "step 3: Grannie Geng held up his gun with one eye, narrowed it, raised the barrel of the rifle, fired a hail of bullets at the target.\n", + "100%|███████████████████████████████████████| 1133/1133 [44:42<00:00, 2.37s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.857 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-1.5B-Instruct_checkpoint-560\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Grannie Geng held up his gun with one eye, nar...\n", + "\n", + "[1 rows x 13 columns]\n", + "{'accuracy': 0.00264783759929391, 'correct_ids': [240, 738, 1026], 'meteor': 0.3555548051770412, 'bleu_scores': {'bleu': 0.08837370077365968, 'precisions': [0.4154119950169069, 0.13452266152362585, 0.055553404823661494, 0.026475589021131892], 'brevity_penalty': 0.9281439603442432, 'length_ratio': 0.9306061609804571, 'translation_length': 28095, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.39474926445540526, 'rouge2': 0.14909336721544575, 'rougeL': 0.3340601663307491, 'rougeLsum': 0.33415584663948783}}\n", + "Epoch 2\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-1.5B-Instruct llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-1120 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-1.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 22:43:48,381 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 22:43:48,381 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 22:43:48,381 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 22:43:48,381 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 22:43:48,381 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 22:43:48,381 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 22:43:48,549 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 22:43:48 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 22:43:48 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 22:43:48,826 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 22:43:48,826 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 22:43:48 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 22:43:48 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 22:43:48,853 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 22:43:49,950 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 22:43:49,954 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 22:46:48,562 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 22:46:48,562 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 22:46:48,846 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 22:46:48,846 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 22:46:51 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 22:46:54 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-1120\n", + "06/30/2024 22:46:55 - INFO - llamafactory.model.loader - all params: 1,552,946,688\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.604 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his rifle and squinted at it through a slit in his eye. He squeezed the trigger and gold sparrows began to fall like rain. Iron sand scattered among the willow branches crackled.\n", + "--------\n", + "step 3: Old Geng raised his rifle and squinted at it through a slit in his eye. He squeezed the trigger and gold sparrows began to fall like rain. Iron sand scattered among the willow branches crackled.\n", + "100%|███████████████████████████████████████| 1133/1133 [54:52<00:00, 2.91s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.818 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-1.5B-Instruct_checkpoint-1120\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his rifle and squinted at it t...\n", + "\n", + "[1 rows x 14 columns]\n", + "{'accuracy': 0.00353045013239188, 'correct_ids': [77, 272, 381, 659], 'meteor': 0.364551066769633, 'bleu_scores': {'bleu': 0.09512979475404361, 'precisions': [0.41979252665206934, 0.1427074758661977, 0.06224115026959444, 0.03069440470838272], 'brevity_penalty': 0.9197334814475309, 'length_ratio': 0.9227890029811195, 'translation_length': 27859, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.40366781962223464, 'rouge2': 0.1631594243449107, 'rougeL': 0.34288741533227174, 'rougeLsum': 0.34268506193513737}}\n", + "Epoch 3\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-1.5B-Instruct llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-1680 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-1.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 23:42:11,002 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 23:42:11,002 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 23:42:11,002 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 23:42:11,002 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 23:42:11,002 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-06-30 23:42:11,002 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-06-30 23:42:11,240 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "06/30/2024 23:42:11 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "06/30/2024 23:42:11 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-06-30 23:42:11,554 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-06-30 23:42:11,554 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "06/30/2024 23:42:11 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "06/30/2024 23:42:11 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-06-30 23:42:11,668 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-06-30 23:42:13,979 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 23:42:13,983 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-06-30 23:43:46,052 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-06-30 23:43:46,052 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-06-30 23:43:47,155 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-06-30 23:43:47,155 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "06/30/2024 23:43:47 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "06/30/2024 23:43:48 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-1680\n", + "06/30/2024 23:43:48 - INFO - llamafactory.model.loader - all params: 1,552,946,688\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.604 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng took his gun off the table and raised it to his eye. He squeezed the trigger and a hail of bullets fell from the sky, golden sparrows falling like rain as shrapnel flew through the air among the willows.\n", + "--------\n", + "step 3: Old Geng took his gun off the table and raised it to his eye. He squeezed the trigger and a hail of bullets fell from the sky, golden sparrows falling like rain as shrapnel flew through the air among the willows.\n", + "100%|███████████████████████████████████████| 1133/1133 [42:11<00:00, 2.23s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.838 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-1.5B-Instruct_checkpoint-1680\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng took his gun off the table and raised...\n", + "\n", + "[1 rows x 15 columns]\n", + "{'accuracy': 0.00529567519858782, 'correct_ids': [77, 147, 199, 452, 738, 918], 'meteor': 0.3723931629938662, 'bleu_scores': {'bleu': 0.1007710645770402, 'precisions': [0.4158811367698076, 0.14392059553349876, 0.0641747868453106, 0.03384639860000795], 'brevity_penalty': 0.9437209131631352, 'length_ratio': 0.9452467704537927, 'translation_length': 28537, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.40370214820886885, 'rouge2': 0.1641473385689542, 'rougeL': 0.3423335232392143, 'rougeLsum': 0.3424044524649077}}\n", + "Epoch 4\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-1.5B-Instruct llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-2240 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-1.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 00:26:19,392 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 00:26:19,392 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 00:26:19,392 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 00:26:19,392 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 00:26:19,392 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 00:26:19,392 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-01 00:26:19,534 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/01/2024 00:26:19 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/01/2024 00:26:19 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-07-01 00:26:19,883 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-01 00:26:19,883 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "07/01/2024 00:26:19 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "07/01/2024 00:26:19 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-07-01 00:26:19,958 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-01 00:26:21,213 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 00:26:21,216 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-01 00:27:43,020 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-01 00:27:43,020 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-01 00:27:43,422 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 00:27:43,422 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "07/01/2024 00:27:43 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/01/2024 00:27:44 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-2240\n", + "07/01/2024 00:27:44 - INFO - llamafactory.model.loader - all params: 1,552,946,688\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.604 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his rifle and squeezed the trigger. The sound of gunfire joined the chattering rain as hundreds of sparrows fell from the sky, the pellets flying through the air between the willow twigs.\n", + "--------\n", + "step 3: Old Geng raised his rifle and squeezed the trigger. The sound of gunfire joined the chattering rain as hundreds of sparrows fell from the sky, the pellets flying through the air between the willow twigs.\n", + "100%|███████████████████████████████████████| 1133/1133 [40:27<00:00, 2.14s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.838 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-1.5B-Instruct_checkpoint-2240\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his rifle and squeezed the tri...\n", + "\n", + "[1 rows x 16 columns]\n", + "{'accuracy': 0.00264783759929391, 'correct_ids': [147, 199, 738], 'meteor': 0.35847259317675817, 'bleu_scores': {'bleu': 0.09681182585608442, 'precisions': [0.4169993042077123, 0.14579353556964927, 0.06572957431515054, 0.03353403579193845], 'brevity_penalty': 0.8998048931972519, 'length_ratio': 0.9045048029148725, 'translation_length': 27307, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3940152835057211, 'rouge2': 0.16326412776493693, 'rougeL': 0.33702749255447373, 'rougeLsum': 0.3369782380738291}}\n", + "Epoch 5\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-1.5B-Instruct llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-2800 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-1.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 01:08:35,227 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 01:08:35,227 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 01:08:35,227 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 01:08:35,227 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 01:08:35,227 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 01:08:35,227 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-01 01:08:35,401 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/01/2024 01:08:35 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/01/2024 01:08:35 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-07-01 01:08:35,697 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-01 01:08:35,697 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "07/01/2024 01:08:35 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "07/01/2024 01:08:35 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-07-01 01:08:35,772 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-01 01:08:37,565 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 01:08:37,570 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-01 01:10:00,800 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-01 01:10:00,800 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-01 01:10:01,095 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 01:10:01,096 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "07/01/2024 01:10:01 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/01/2024 01:10:02 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-2800\n", + "07/01/2024 01:10:02 - INFO - llamafactory.model.loader - all params: 1,552,946,688\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.604 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng took aim and squeezed the trigger; dozens of gold-winged sparrows fell in a drenching rain, iron-shrapnel crackled among the willows, and a chorus of tiny explosions sounded over their heads.\n", + "--------\n", + "step 3: Old Geng took aim and squeezed the trigger; dozens of gold-winged sparrows fell in a drenching rain, iron-shrapnel crackled among the willows, and a chorus of tiny explosions sounded over their heads.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:17:25<00:00, 4.10s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5.197 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-1.5B-Instruct_checkpoint-2800\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng took aim and squeezed the trigger; do...\n", + "\n", + "[1 rows x 17 columns]\n", + "{'accuracy': 0.00176522506619594, 'correct_ids': [147, 199], 'meteor': 0.35988930837184085, 'bleu_scores': {'bleu': 0.09029975816152737, 'precisions': [0.36273504273504276, 0.12144836028606404, 0.05442995653627549, 0.02772855206921714], 'brevity_penalty': 1.0, 'length_ratio': 1.0657502484266312, 'translation_length': 32175, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3917385628494343, 'rouge2': 0.158275578220186, 'rougeL': 0.33145202576141436, 'rougeLsum': 0.331550843392171}}\n", + "Epoch 6\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-1.5B-Instruct llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-3360 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-1.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 02:27:49,309 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 02:27:49,309 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 02:27:49,309 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 02:27:49,309 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 02:27:49,309 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 02:27:49,309 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-01 02:27:49,467 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/01/2024 02:27:49 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/01/2024 02:27:49 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-07-01 02:27:49,780 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-01 02:27:49,781 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "07/01/2024 02:27:49 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "07/01/2024 02:27:49 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-07-01 02:27:49,851 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-01 02:27:51,890 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 02:27:51,895 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-01 02:29:12,004 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-01 02:29:12,004 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-01 02:29:12,299 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 02:29:12,299 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "07/01/2024 02:29:12 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/01/2024 02:29:13 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-3360\n", + "07/01/2024 02:29:13 - INFO - llamafactory.model.loader - all params: 1,552,946,688\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.604 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng took a step forward, raised his pistol, and squeezed the trigger. The pellets of lead raining down from above exploded against the snow-covered ground. They flew through the air as sparks of iron grit crackled among the willows.\n", + "--------\n", + "step 3: Old Geng took a step forward, raised his pistol, and squeezed the trigger. The pellets of lead raining down from above exploded against the snow-covered ground. They flew through the air as sparks of iron grit crackled among the willows.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:14:59<00:00, 3.97s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5.178 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-1.5B-Instruct_checkpoint-3360\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng took a step forward, raised his pisto...\n", + "\n", + "[1 rows x 18 columns]\n", + "{'accuracy': 0.00176522506619594, 'correct_ids': [147, 199], 'meteor': 0.3460642024871934, 'bleu_scores': {'bleu': 0.09384985027759411, 'precisions': [0.39390243902439026, 0.1306634744440817, 0.059353130319651975, 0.031256174181056626], 'brevity_penalty': 0.949408256548351, 'length_ratio': 0.9506459092414706, 'translation_length': 28700, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.37889769060024026, 'rouge2': 0.14962195702951014, 'rougeL': 0.32301072520504354, 'rougeLsum': 0.3229695536364973}}\n", + "Epoch 7\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-1.5B-Instruct llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-3920 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-1.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 03:44:33,600 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 03:44:33,600 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 03:44:33,600 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 03:44:33,601 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 03:44:33,601 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 03:44:33,601 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-01 03:44:34,047 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/01/2024 03:44:34 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/01/2024 03:44:34 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-07-01 03:44:34,340 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-01 03:44:34,341 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "07/01/2024 03:44:34 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "07/01/2024 03:44:34 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-07-01 03:44:34,397 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-01 03:44:35,481 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 03:44:35,484 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-01 03:45:57,180 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-01 03:45:57,180 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-01 03:45:57,530 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 03:45:57,530 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "07/01/2024 03:45:57 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/01/2024 03:45:58 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-3920\n", + "07/01/2024 03:45:58 - INFO - llamafactory.model.loader - all params: 1,552,946,688\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.604 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng raised his pistol, opened it up, and a few bullets flew out, like hailstones. Golden sparrows fell, and grit exploded among the willows, making a tinkling sound.\n", + "--------\n", + "step 3: Old Geng raised his pistol, opened it up, and a few bullets flew out, like hailstones. Golden sparrows fell, and grit exploded among the willows, making a tinkling sound.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:15:29<00:00, 4.00s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5.197 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-1.5B-Instruct_checkpoint-3920\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng raised his pistol, opened it up, and ...\n", + "\n", + "[1 rows x 19 columns]\n", + "{'accuracy': 0.00176522506619594, 'correct_ids': [147, 199], 'meteor': 0.3479480952549209, 'bleu_scores': {'bleu': 0.08568897530454278, 'precisions': [0.34471041533934044, 0.11467889908256881, 0.051635392233515764, 0.02641279718624235], 'brevity_penalty': 1.0, 'length_ratio': 1.0917853593905267, 'translation_length': 32961, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3755109220918789, 'rouge2': 0.14664341233690792, 'rougeL': 0.3172964023166135, 'rougeLsum': 0.31738234724622777}}\n", + "Epoch 8\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-1.5B-Instruct llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-4480 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-1.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 05:01:48,632 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 05:01:48,632 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 05:01:48,632 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 05:01:48,632 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 05:01:48,632 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 05:01:48,632 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-01 05:01:48,913 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/01/2024 05:01:48 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/01/2024 05:01:48 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-07-01 05:01:49,230 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-01 05:01:49,230 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "07/01/2024 05:01:49 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "07/01/2024 05:01:49 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-07-01 05:01:49,319 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-01 05:01:51,629 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 05:01:51,633 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-01 05:03:12,246 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-01 05:03:12,246 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-01 05:03:12,762 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 05:03:12,762 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "07/01/2024 05:03:13 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/01/2024 05:03:13 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-4480\n", + "07/01/2024 05:03:13 - INFO - llamafactory.model.loader - all params: 1,552,946,688\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.604 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng took a shot with his rifle. A spray of bullets flew forth, like ice pellets, and a cloud of sparrows fell to the ground. Shot after shot, each one accompanied by a crack, exploded against the willows.\n", + "--------\n", + "step 3: Old Geng took a shot with his rifle. A spray of bullets flew forth, like ice pellets, and a cloud of sparrows fell to the ground. Shot after shot, each one accompanied by a crack, exploded against the willows.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:16:24<00:00, 4.05s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5.236 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-1.5B-Instruct_checkpoint-4480\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng took a shot with his rifle. A spray o...\n", + "\n", + "[1 rows x 20 columns]\n", + "{'accuracy': 0.00176522506619594, 'correct_ids': [147, 199], 'meteor': 0.33844145976530193, 'bleu_scores': {'bleu': 0.08009132331873689, 'precisions': [0.33483795251421866, 0.10704716804785346, 0.047180778918814184, 0.024331389503317917], 'brevity_penalty': 1.0, 'length_ratio': 1.1007287181185823, 'translation_length': 33231, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.37147413848910177, 'rouge2': 0.14173580477944275, 'rougeL': 0.31332200211175076, 'rougeLsum': 0.3132659362806373}}\n", + "Epoch 9\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-1.5B-Instruct llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-5040 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-1.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 06:20:02,103 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 06:20:02,103 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 06:20:02,103 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 06:20:02,103 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 06:20:02,103 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 06:20:02,103 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-01 06:20:02,237 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/01/2024 06:20:02 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/01/2024 06:20:02 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-07-01 06:20:02,540 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-01 06:20:02,540 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "07/01/2024 06:20:02 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "07/01/2024 06:20:02 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-07-01 06:20:02,582 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-01 06:20:04,218 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 06:20:04,222 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-01 06:21:26,114 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-01 06:21:26,115 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-01 06:21:26,406 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 06:21:26,406 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "07/01/2024 06:21:26 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/01/2024 06:21:27 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-5040\n", + "07/01/2024 06:21:27 - INFO - llamafactory.model.loader - all params: 1,552,946,688\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.604 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng took a step forward, raised his rifle to his eye, and squeezed the trigger. Rifle pellets, bayoneted sparrows, rained down around him. Shotouts of iron sand flew everywhere, crackling as they went.\n", + "--------\n", + "step 3: Old Geng took a step forward, raised his rifle to his eye, and squeezed the trigger. Rifle pellets, bayoneted sparrows, rained down around him. Shotouts of iron sand flew everywhere, crackling as they went.\n", + "100%|█████████████████████████████████████| 1133/1133 [1:15:49<00:00, 4.02s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "5.197 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-1.5B-Instruct_checkpoint-5040\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng took a step forward, raised his rifle...\n", + "\n", + "[1 rows x 21 columns]\n", + "{'accuracy': 0.00176522506619594, 'correct_ids': [147, 199], 'meteor': 0.3380289789419591, 'bleu_scores': {'bleu': 0.08738865032530332, 'precisions': [0.36355344170440107, 0.11703423082126911, 0.052124366910523356, 0.026296513331380018], 'brevity_penalty': 1.0, 'length_ratio': 1.0167936402782378, 'translation_length': 30697, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3674967504985488, 'rouge2': 0.14110284985778096, 'rougeL': 0.3092157882639477, 'rougeLsum': 0.30969047388276916}}\n", + "Epoch 10\n", + "loading env vars from: /home/inflaton/code/projects/courses/llm-finetuning/.env\n", + "Adding /home/inflaton/code/projects/courses/llm-finetuning to sys.path\n", + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n", + "loading /home/inflaton/code/projects/courses/llm-finetuning/llm_toolkit/translation_engine.py\n", + "Qwen/Qwen2-1.5B-Instruct llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-5600 True datasets/mac/mac.tsv results/mac-results_lf.csv\n", + "(1) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-1.5B-Instruct\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 07:37:36,379 >> loading file vocab.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 07:37:36,379 >> loading file merges.txt from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 07:37:36,379 >> loading file tokenizer.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 07:37:36,379 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 07:37:36,379 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2161] 2024-07-01 07:37:36,379 >> loading file tokenizer_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json\n", + "[WARNING|logging.py:313] 2024-07-01 07:37:36,515 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/01/2024 07:37:36 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/01/2024 07:37:36 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "[INFO|configuration_utils.py:733] 2024-07-01 07:37:36,942 >> loading configuration file config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json\n", + "[INFO|configuration_utils.py:800] 2024-07-01 07:37:36,943 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-1.5B-Instruct\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 1536,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 8960,\n", + " \"max_position_embeddings\": 32768,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 12,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 2,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 32768,\n", + " \"tie_word_embeddings\": true,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.42.3\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 151936\n", + "}\n", + "\n", + "07/01/2024 07:37:36 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "07/01/2024 07:37:36 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3556] 2024-07-01 07:37:36,987 >> loading weights file model.safetensors from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors\n", + "[INFO|modeling_utils.py:1531] 2024-07-01 07:37:38,446 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 07:37:38,450 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151645\n", + "}\n", + "\n", + "[INFO|modeling_utils.py:4364] 2024-07-01 07:39:01,352 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4372] 2024-07-01 07:39:01,352 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:955] 2024-07-01 07:39:01,658 >> loading configuration file generation_config.json from cache at /home/inflaton/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json\n", + "[INFO|configuration_utils.py:1000] 2024-07-01 07:39:01,658 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"do_sample\": true,\n", + " \"eos_token_id\": [\n", + " 151645,\n", + " 151643\n", + " ],\n", + " \"pad_token_id\": 151643,\n", + " \"repetition_penalty\": 1.1,\n", + " \"temperature\": 0.7,\n", + " \"top_k\": 20,\n", + " \"top_p\": 0.8\n", + "}\n", + "\n", + "07/01/2024 07:39:02 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/01/2024 07:39:02 - INFO - llamafactory.model.adapter - Loaded adapter(s): llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-5600\n", + "07/01/2024 07:39:02 - INFO - llamafactory.model.loader - all params: 1,552,946,688\n", + "(2) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.604 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 4528\n", + " })\n", + " test: Dataset({\n", + " features: ['chinese', 'english', 'text', 'prompt'],\n", + " num_rows: 1133\n", + " })\n", + "})\n", + "Evaluating model: Qwen/Qwen2-1.5B-Instruct\n", + " 0%| | 0/1133 [00:00\n", + "--------\n", + "step 2: Old Geng reached for his rifle, wedged it to his eye, took a squeeze, and fired—a shower of lead pellets flew from the barrel, crackering through the air as they hit.\n", + "--------\n", + "step 3: Old Geng reached for his rifle, wedged it to his eye, took a squeeze, and fired—a shower of lead pellets flew from the barrel, crackering through the air as they hit.\n", + "100%|███████████████████████████████████████| 1133/1133 [43:28<00:00, 2.30s/it]\n", + "(3) GPU = NVIDIA GeForce RTX 4080 Laptop GPU. Max memory = 11.994 GB.\n", + "1.877 GB of memory reserved.\n", + " chinese ... Qwen/Qwen2-1.5B-Instruct_checkpoint-5600\n", + "0 老耿端起枪，眯缝起一只三角眼，一搂扳机响了枪，冰雹般的金麻雀劈哩啪啦往下落，铁砂子在柳枝间飞... ... Old Geng reached for his rifle, wedged it to h...\n", + "\n", + "[1 rows x 22 columns]\n", + "{'accuracy': 0.00176522506619594, 'correct_ids': [147, 199], 'meteor': 0.3339867178782917, 'bleu_scores': {'bleu': 0.08544000315753703, 'precisions': [0.3757308441891476, 0.11972682649213914, 0.05255355422133274, 0.025644000928289626], 'brevity_penalty': 0.9682716284409708, 'length_ratio': 0.9687644915534945, 'translation_length': 29247, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3631078400113029, 'rouge2': 0.13850862500702893, 'rougeL': 0.3081859195764205, 'rougeLsum': 0.30821718216431304}}\n", + "CPU times: user 21min 38s, sys: 7min 47s, total: 29min 25s\n", + "Wall time: 10h 26min 31s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "os.environ[\"MODEL_NAME\"] = \"Qwen/Qwen2-1.5B-Instruct\" \n", + "for i in range(1, num_train_epochs + 1):\n", + " print(f\"Epoch {i}\")\n", + " adapter_path = f\"llama-factory/saves/qwen2-1.5b/lora/sft/checkpoint-{560 * i}\"\n", + " os.environ[\"ADAPTER_NAME_OR_PATH\"] = adapter_path\n", + " !python llm_toolkit/eval.py " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "os.environ[\"MODEL_NAME\"] = \"Qwen/Qwen2-7B-Instruct\" \n", + "for i in range(1, num_train_epochs + 1):\n", + " print(f\"Epoch {i}\")\n", + " adapter_path = f\"llama-factory/saves/qwen2-7b/lora/sft/checkpoint-{560 * i}\"\n", + " os.environ[\"ADAPTER_NAME_OR_PATH\"] = adapter_path\n", + " !python llm_toolkit/eval.py " + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_openai import ChatOpenAI\n", + "\n", + "llm = ChatOpenAI(\n", + " model=\"gpt-4o\",\n", + " temperature=0,\n", + " max_tokens=None,\n", + " timeout=None,\n", + " max_retries=2,\n", + " # api_key=\"...\", # if you prefer to pass api key in directly instaed of using env vars\n", + " base_url=\"http://localhost:8000/v1\",\n", + " # organization=\"...\",\n", + " # other params...\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package wordnet to /home/inflaton/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to /home/inflaton/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to /home/inflaton/nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n" + ] + } + ], + "source": [ + "from llm_toolkit.translation_utils import *" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'The body was found on the morning beach'" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "translate_via_llm(\"死者凌晨去的沙滩\")" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "cache_dict = {}" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'死者凌晨去的沙滩': 'The body was found on the morning beach'}" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "translate(\n", + " \"死者凌晨去的沙滩\",\n", + " cache_dict=cache_dict,\n", + ")\n", + "cache_dict" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "07_MAC_+_Qwen2-7B-Instructi_Unsloth_train", + "widgets": {} + }, + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "036fc5746f43416db18c19ad8fd36677": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "06e806c82c7b4cbea31c5358dd9c3434": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "087b76a8b7514269b1f0ab29b062e444": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a069d2ab23824f29aa320ac256e2cfe9", + "placeholder": "​", + "style": "IPY_MODEL_06e806c82c7b4cbea31c5358dd9c3434", + "value": "Map (num_proc=2): 100%" + } + }, + "09b76013aa9e45efb6deb23a7a0d0925": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dea41c5260884aa6879b5e1d1697b14f", + "placeholder": "​", + "style": "IPY_MODEL_89965917796a4f81b899fdc7685f33df", + "value": "config.json: 100%" + } + }, + "0a92c56bfa134ef583220d7ef0b13e17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c34be936c8145d3ab41282f30a70713": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f8b6bfe16894500838793f2491d403f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "177c78fce95d4b4ab33057c5a048d693": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1f44c9ce1adf470cbb19784493ed209f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c34be936c8145d3ab41282f30a70713", + "placeholder": "​", + "style": "IPY_MODEL_0a92c56bfa134ef583220d7ef0b13e17", + "value": "model.safetensors: 100%" + } + }, + "201b59ccd9f845e197029b57e424aefc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2157f01726d748f8a9ae4a00664430da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "21db8a77b00d4a4e82fdfa608657531f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26e4202cca81496a90d15a0dd4ca9cf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba90fdb8822d47dab7ba203bee297f37", + "IPY_MODEL_61560ff6a36b44f4a9dfdae5c52791d4", + "IPY_MODEL_95fbe66647904c06a20f640630d6dc0e" + ], + "layout": "IPY_MODEL_57182a263d324a3dbf1471c74290a0d5" + } + }, + "27155728b6b84cb199c91c940095d0a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6b91feeed5464877991ac2c207aebe7c", + "IPY_MODEL_cca8113c54c0495daedce1327bf9c68b", + "IPY_MODEL_2e63a29e2f7247bba5beede9a568c99f" + ], + "layout": "IPY_MODEL_5c9d781c28944f3eb86e2a6d44efdf18" + } + }, + "271ddaa553a042d09b6db7b450643d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a58d04b428c46f4b3dbadd3bc6cd529": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d18ddf6482c4d97829ac0e5a7b9868f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9f679ad3ec7f4fe8ad0510ffb57bc2ab", + "IPY_MODEL_f2df530d22c74977b249dd9fb5f4829b", + "IPY_MODEL_89b2ef0dbfea47ab8e6f8d659e3351d1" + ], + "layout": "IPY_MODEL_3056b148aa9f4e6e8aa3b61d26886255" + } + }, + "2e5087c76f98437cb5dc729230358cba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e63a29e2f7247bba5beede9a568c99f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b993eaec6b224440bf80c0958c6fb536", + "placeholder": "​", + "style": "IPY_MODEL_de868e26e7154f62aa86223a539ad421", + "value": " 464/464 [00:00<00:00, 27.1kB/s]" + } + }, + "2f6c70dd266c4816bfad3fd3d192929a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "30307300bc4e4baf96560e30969a82b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e36a3f9eff0e4cf68834d66b0213ae96", + "placeholder": "​", + "style": "IPY_MODEL_a0037bdccf254159becde630bee3d1db", + "value": "generation_config.json: 100%" + } + }, + "3056b148aa9f4e6e8aa3b61d26886255": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30cdc32298134cb0be4d41615b9e5774": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3572201bd4d74a58b7a665f9bdfdcdba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35b0e8c26d6640e9bd0ed7b242a423d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e5087c76f98437cb5dc729230358cba", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_036fc5746f43416db18c19ad8fd36677", + "value": 51760 + } + }, + "36166c7bcb854b34aca1f41a5d6ea50b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "370692d819df41828b48c4ad446f977b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "39b29a75374b45c0a22506010be2b84e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_30cdc32298134cb0be4d41615b9e5774", + "max": 1179, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_47928317548c454bba6358ab132e8dee", + "value": 1179 + } + }, + "3cf2dd993b5e4d3daecf61e4bab5a404": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_087b76a8b7514269b1f0ab29b062e444", + "IPY_MODEL_35b0e8c26d6640e9bd0ed7b242a423d8", + "IPY_MODEL_54ad89e05fd74576b9b8b5b5a10eaf8d" + ], + "layout": "IPY_MODEL_a41dc44766444a998bec2d777f249d23" + } + }, + "43dec2ede91341f5af60eb522e18e984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4463edd481c1467f914c7dcd6c6e6ffc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47928317548c454bba6358ab132e8dee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "49277aeeac16434a865a4d12308b1abc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4ae7e449e4ea4c729b5f34607c18ebae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b2061b8a73c43ffb0c2f83daf0d0183": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c4c88d4c701450692fa0f6b0c5764b0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c666f4ace3943f8b80ecd20e7503236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ccedf0d93094e63b57a0f8a434fba06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4463edd481c1467f914c7dcd6c6e6ffc", + "max": 44307561, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6d3b9a05db0b4dadb638c686faa0c40a", + "value": 44307561 + } + }, + "4dcf6ff672d24983a1877a8431709aa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5807d5fb827d490fb3bc698f801ffff5", + "placeholder": "​", + "style": "IPY_MODEL_c4f2b06a82fd4987b8b659524a7b503b", + "value": "Generating train split: 100%" + } + }, + "4ea63adfce694725bdba878aef709dd3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5234566b1bfc4655b8d582ea5b46ed9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "54ad89e05fd74576b9b8b5b5a10eaf8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdb1941405ed4e4aa06019933892deb3", + "placeholder": "​", + "style": "IPY_MODEL_668d5377ca56426a99753867e6e24862", + "value": " 51760/51760 [01:02<00:00, 1131.51 examples/s]" + } + }, + "56aee4853b7740e6a977254f5d1fa66d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "57182a263d324a3dbf1471c74290a0d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5807d5fb827d490fb3bc698f801ffff5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c9d781c28944f3eb86e2a6d44efdf18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f40db8173dd4d76b6ef5ed6d9ec8b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61560ff6a36b44f4a9dfdae5c52791d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db19fc8d37db4e45a5790a876836d8c4", + "max": 11610, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36166c7bcb854b34aca1f41a5d6ea50b", + "value": 11610 + } + }, + "6578fd7acdb54c4c93528ea431fd0144": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_370692d819df41828b48c4ad446f977b", + "placeholder": "​", + "style": "IPY_MODEL_a0bf9160eb2647409b3200270914b90f", + "value": " 50.6k/50.6k [00:00<00:00, 2.71MB/s]" + } + }, + "668d5377ca56426a99753867e6e24862": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697f027529b54ee9956bae78a11e0611": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69ac12aec0714318bf2c83d4f4e745f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b2012c3f88547af8884a9ea90e3164b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_938f45f1b3e24118b815d96ae34ba86a", + "placeholder": "​", + "style": "IPY_MODEL_9367047a800747f79c6b225d92397846", + "value": " 44.3M/44.3M [00:01<00:00, 31.0MB/s]" + } + }, + "6b91feeed5464877991ac2c207aebe7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b2061b8a73c43ffb0c2f83daf0d0183", + "placeholder": "​", + "style": "IPY_MODEL_69ac12aec0714318bf2c83d4f4e745f5", + "value": "special_tokens_map.json: 100%" + } + }, + "6d3b9a05db0b4dadb638c686faa0c40a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6dbbedeca9314e66ae50e44ffa31a414": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6e34619b45934040b6092e6fb01ea7fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71ce208e20d6483abb9ed923510c86d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d69dc491b3ab44d7852b21873ed7bb7f", + "placeholder": "​", + "style": "IPY_MODEL_f401d53bf28e44eb906bce6c05412662", + "value": " 51760/51760 [00:01<00:00, 45512.81 examples/s]" + } + }, + "7358cdad832342c983e31efb8754ab78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e352a3404f4c7dad0737f57d29e92f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_988a0e8c1f89446086858da0a891a79c", + "IPY_MODEL_4ccedf0d93094e63b57a0f8a434fba06", + "IPY_MODEL_6b2012c3f88547af8884a9ea90e3164b" + ], + "layout": "IPY_MODEL_7e29cb8dd4df4d5b94407cd8fd3f2011" + } + }, + "74501720ac7e4dbb911a4a99b3633bc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "78e5400bff924a92a4cc61c4ff18b182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9b313fd861948f5aba25b24b1518d30", + "placeholder": "​", + "style": "IPY_MODEL_4c666f4ace3943f8b80ecd20e7503236", + "value": " 1.18k/1.18k [00:00<00:00, 31.3kB/s]" + } + }, + "7975adbc2ec5489ea7fa0167e620d85c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e34619b45934040b6092e6fb01ea7fe", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_271ddaa553a042d09b6db7b450643d8f", + "value": 51760 + } + }, + "7e29cb8dd4df4d5b94407cd8fd3f2011": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "810ff6c0e17d4fa09a30fef27eacff90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89965917796a4f81b899fdc7685f33df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "89b2ef0dbfea47ab8e6f8d659e3351d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8908fa0df3743ecb9d12983a739104f", + "placeholder": "​", + "style": "IPY_MODEL_177c78fce95d4b4ab33057c5a048d693", + "value": " 9.09M/9.09M [00:00<00:00, 32.6MB/s]" + } + }, + "8b3505352a5a42bf910428c40ce40465": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_49277aeeac16434a865a4d12308b1abc", + "placeholder": "​", + "style": "IPY_MODEL_2157f01726d748f8a9ae4a00664430da", + "value": " 5.70G/5.70G [01:02<00:00, 30.1MB/s]" + } + }, + "8fc142b628fb40568730234de1cafde2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ae7e449e4ea4c729b5f34607c18ebae", + "max": 172, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3572201bd4d74a58b7a665f9bdfdcdba", + "value": 172 + } + }, + "9367047a800747f79c6b225d92397846": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "938f45f1b3e24118b815d96ae34ba86a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "95fbe66647904c06a20f640630d6dc0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b0a370dc20654b279b9680692e34418e", + "placeholder": "​", + "style": "IPY_MODEL_cfeb365ddf7548d58b2557f22737fcf5", + "value": " 11.6k/11.6k [00:00<00:00, 716kB/s]" + } + }, + "988a0e8c1f89446086858da0a891a79c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ad2be500fc164c0f86f33e914ef8e6a0", + "placeholder": "​", + "style": "IPY_MODEL_5234566b1bfc4655b8d582ea5b46ed9f", + "value": "Downloading data: 100%" + } + }, + "98c58f23f4d549518832cb2d18f796e8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_09b76013aa9e45efb6deb23a7a0d0925", + "IPY_MODEL_39b29a75374b45c0a22506010be2b84e", + "IPY_MODEL_78e5400bff924a92a4cc61c4ff18b182" + ], + "layout": "IPY_MODEL_2a58d04b428c46f4b3dbadd3bc6cd529" + } + }, + "99fdbb0300c14c139d1937c646f0cfe7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7358cdad832342c983e31efb8754ab78", + "placeholder": "​", + "style": "IPY_MODEL_e9adf418296e436fb48bb9f78885598b", + "value": " 51760/51760 [00:01<00:00, 38665.95 examples/s]" + } + }, + "9f679ad3ec7f4fe8ad0510ffb57bc2ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ea63adfce694725bdba878aef709dd3", + "placeholder": "​", + "style": "IPY_MODEL_74501720ac7e4dbb911a4a99b3633bc6", + "value": "tokenizer.json: 100%" + } + }, + "a0037bdccf254159becde630bee3d1db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a069d2ab23824f29aa320ac256e2cfe9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0bf9160eb2647409b3200270914b90f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a41dc44766444a998bec2d777f249d23": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8464a4c711e4e00aafdfc919b60d07e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fb995c740590427b882572c81d4e848c", + "placeholder": "​", + "style": "IPY_MODEL_201b59ccd9f845e197029b57e424aefc", + "value": " 172/172 [00:00<00:00, 12.0kB/s]" + } + }, + "a9f0cc51fc3d4d7b874c32dcf1c5bdf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad2be500fc164c0f86f33e914ef8e6a0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b0240cd9a4554b29ae11f8051984a1c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_edaf890370314a218f138015faa0b05d", + "placeholder": "​", + "style": "IPY_MODEL_697f027529b54ee9956bae78a11e0611", + "value": "Map: 100%" + } + }, + "b0a370dc20654b279b9680692e34418e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b518dcee69074b87be73957cd810e7ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d891f8d0b1fc462f8008d02bb2a15692", + "placeholder": "​", + "style": "IPY_MODEL_cced8fd7e998472794f3f3e3018956a5", + "value": "tokenizer_config.json: 100%" + } + }, + "b8908fa0df3743ecb9d12983a739104f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b993eaec6b224440bf80c0958c6fb536": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9b313fd861948f5aba25b24b1518d30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba90fdb8822d47dab7ba203bee297f37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f8b6bfe16894500838793f2491d403f", + "placeholder": "​", + "style": "IPY_MODEL_bb19f6c747754682a514373a3a0535ba", + "value": "Downloading readme: 100%" + } + }, + "bb19f6c747754682a514373a3a0535ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc883d4cf13e4f8b8a4fe5f410cb6efd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9159e03e61f4f56978ece9c3bca49b2", + "max": 51760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_810ff6c0e17d4fa09a30fef27eacff90", + "value": 51760 + } + }, + "c161d94df0f04feba9542237e0856c22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c22f71b1f85843209d7e5321506b9cb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f44c9ce1adf470cbb19784493ed209f", + "IPY_MODEL_f1addc4479d849879e743cf9089e6540", + "IPY_MODEL_8b3505352a5a42bf910428c40ce40465" + ], + "layout": "IPY_MODEL_4c4c88d4c701450692fa0f6b0c5764b0" + } + }, + "c4f2b06a82fd4987b8b659524a7b503b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cca8113c54c0495daedce1327bf9c68b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e02f9b7849c64531835eb77b860d1c93", + "max": 464, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_56aee4853b7740e6a977254f5d1fa66d", + "value": 464 + } + }, + "cced8fd7e998472794f3f3e3018956a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cf245afeb1c04f29a24d291608c3d157": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b518dcee69074b87be73957cd810e7ed", + "IPY_MODEL_e29104486d594b2992d7285e0ef77371", + "IPY_MODEL_6578fd7acdb54c4c93528ea431fd0144" + ], + "layout": "IPY_MODEL_d35db8148a354c56aaac56dbae22536f" + } + }, + "cfe8cae0e22b495bafa221a63d13b283": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfeb365ddf7548d58b2557f22737fcf5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1b47d39450d4019ae85c9b2f943eeaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4dcf6ff672d24983a1877a8431709aa9", + "IPY_MODEL_7975adbc2ec5489ea7fa0167e620d85c", + "IPY_MODEL_71ce208e20d6483abb9ed923510c86d7" + ], + "layout": "IPY_MODEL_cfe8cae0e22b495bafa221a63d13b283" + } + }, + "d35db8148a354c56aaac56dbae22536f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69dc491b3ab44d7852b21873ed7bb7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d891f8d0b1fc462f8008d02bb2a15692": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8e5318cead340c4adbeaccc05d39225": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "daf4cd890b35422683d22fd30bc71e83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b0240cd9a4554b29ae11f8051984a1c6", + "IPY_MODEL_bc883d4cf13e4f8b8a4fe5f410cb6efd", + "IPY_MODEL_99fdbb0300c14c139d1937c646f0cfe7" + ], + "layout": "IPY_MODEL_c161d94df0f04feba9542237e0856c22" + } + }, + "db19fc8d37db4e45a5790a876836d8c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de868e26e7154f62aa86223a539ad421": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dea41c5260884aa6879b5e1d1697b14f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e02f9b7849c64531835eb77b860d1c93": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e29104486d594b2992d7285e0ef77371": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a9f0cc51fc3d4d7b874c32dcf1c5bdf2", + "max": 50641, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2f6c70dd266c4816bfad3fd3d192929a", + "value": 50641 + } + }, + "e36a3f9eff0e4cf68834d66b0213ae96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9159e03e61f4f56978ece9c3bca49b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9adf418296e436fb48bb9f78885598b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "edaf890370314a218f138015faa0b05d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f1addc4479d849879e743cf9089e6540": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43dec2ede91341f5af60eb522e18e984", + "max": 5702746405, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d8e5318cead340c4adbeaccc05d39225", + "value": 5702746405 + } + }, + "f2df530d22c74977b249dd9fb5f4829b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_21db8a77b00d4a4e82fdfa608657531f", + "max": 9085698, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6dbbedeca9314e66ae50e44ffa31a414", + "value": 9085698 + } + }, + "f401d53bf28e44eb906bce6c05412662": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fb995c740590427b882572c81d4e848c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fce7a61c25ec4390af43d92b7c473a45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_30307300bc4e4baf96560e30969a82b6", + "IPY_MODEL_8fc142b628fb40568730234de1cafde2", + "IPY_MODEL_a8464a4c711e4e00aafdfc919b60d07e" + ], + "layout": "IPY_MODEL_5f40db8173dd4d76b6ef5ed6d9ec8b6e" + } + }, + "fdb1941405ed4e4aa06019933892deb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/requirements.txt b/requirements.txt index bc5e1604e100b08051f6aa31735e0caf820b8a91..88b4471380a748806c78820270adacec1f5ad824 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,15 @@ -huggingface_hub==0.22.2 \ No newline at end of file +nltk==3.8.1 +python-dotenv==1.0.1 +black==24.4.0 +evaluate==0.4.2 +rouge_score==0.1.2 +pytest==8.2.1 +seaborn==0.13.2 +scikit-learn==1.5.0 +jupyter +ipywidgets +packaging +# triton +# xformers +langchain_openai==0.1.13 +wandb==0.17.4 \ No newline at end of file diff --git a/results/experiment-1-results.csv b/results/experiment-1-results.csv new file mode 100644 index 0000000000000000000000000000000000000000..c4faac54ce32c8000e989a73c49d7ae296c8528a --- /dev/null +++ b/results/experiment-1-results.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfb0c7a3813e9c98c9245c9303b2fb95c1fd7d6a92dd4e0d9d3fe4e4d29a8849 +size 2072299 diff --git a/results/experiment-2-results.csv b/results/experiment-2-results.csv new file mode 100644 index 0000000000000000000000000000000000000000..14e26f3b21a462767d5668ae196f7708df50c840 --- /dev/null +++ b/results/experiment-2-results.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1c99b9bb0c6539a9ff3c9198d730f110c5b6371cba803e1992802beb13e3600 +size 2038783 diff --git a/results/experiment-3-results.csv b/results/experiment-3-results.csv new file mode 100644 index 0000000000000000000000000000000000000000..00a38db0521e984f761bb4641942326b54a92999 --- /dev/null +++ b/results/experiment-3-results.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0b8dcb783ed847422ca4f2000b5106742b992537f4b84da6b5ca0b4c22bf0dd +size 1427300 diff --git a/results/mac-results-no-flash-attn.csv b/results/mac-results-no-flash-attn.csv new file mode 100644 index 0000000000000000000000000000000000000000..fa6b56dd5472dc6857df51d9620365ab59e5a882 --- /dev/null +++ b/results/mac-results-no-flash-attn.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89144b0a3e727b326be559637312e353208a7e506b7c0c701ce8e4392e4cbb5e +size 2129451 diff --git a/results/mac-results-with-flash-attn.csv b/results/mac-results-with-flash-attn.csv new file mode 100644 index 0000000000000000000000000000000000000000..03c87b8c4109476831d4e61103c3f7a252b9053d --- /dev/null +++ b/results/mac-results-with-flash-attn.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c73be2c390511d0a59090b57c53f0a66c0d4c4648c209ef7155aa97ff73c0b9 +size 1461478 diff --git a/results/mac-results.csv b/results/mac-results.csv new file mode 100644 index 0000000000000000000000000000000000000000..c539fdaa38a9b0d4f7756eb8291d29117fab5a1f --- /dev/null +++ b/results/mac-results.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7eb1c66dd7162f27a969599ddb3695c3ac82a88bff15cd57d7ed00ca86ab19cd +size 2072299 diff --git a/results/mac-results_final.csv b/results/mac-results_final.csv new file mode 100644 index 0000000000000000000000000000000000000000..9e5e7801a13b500be3b8d2faeceeb90b221311ed --- /dev/null +++ b/results/mac-results_final.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aacf61087ae3b1fd622407c75d0a969b232517c7489841da722e0228bb69a310 +size 2334006 diff --git a/results/mac-results_lf-r2.csv b/results/mac-results_lf-r2.csv new file mode 100644 index 0000000000000000000000000000000000000000..8b931681a28971d8b5f7ba674d1f7a4c24df55e4 --- /dev/null +++ b/results/mac-results_lf-r2.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25c14d76c8d71ecbce6bc83d641ec4f54f6c0e188fccfcfd8536758a12ed456a +size 2442353 diff --git a/results/mac-results_lf-r3.csv b/results/mac-results_lf-r3.csv new file mode 100644 index 0000000000000000000000000000000000000000..4c6c42fc624a7b0ab9958a23136cb4b5799c8664 --- /dev/null +++ b/results/mac-results_lf-r3.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ea9402ad5c87e3b7dcb570cf0a3c0bf33bef093c522d4d2ba6dbf633e21f035 +size 531603 diff --git a/results/mac-results_lf.csv b/results/mac-results_lf.csv new file mode 100644 index 0000000000000000000000000000000000000000..a196bab99c61126a9093c872fd70d36fb62dfa3c --- /dev/null +++ b/results/mac-results_lf.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5acc087808de5df6839cbf7b170094c6e63445aab4bea15e4be9564b905eb51 +size 3236072 diff --git a/results/mac-results_py3.11.csv b/results/mac-results_py3.11.csv new file mode 100644 index 0000000000000000000000000000000000000000..42e2b1fa940709ed288fa4ac86cb053456340d5f --- /dev/null +++ b/results/mac-results_py3.11.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4adb0922c02cc435858b4ba44b4cdaaee4afe6fcc8721a795d740c36d8d94c2c +size 1463058 diff --git a/results/mac-results_v3.csv b/results/mac-results_v3.csv new file mode 100644 index 0000000000000000000000000000000000000000..5c11486f1305574d95d92adbfefd5b8f75ba86bb --- /dev/null +++ b/results/mac-results_v3.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bfe9ce9720d0cf67ba118d8b2d82f8f6c0bd0f763a8aa00fc1f43f58e544157 +size 1683953 diff --git a/results/model_training_evaluation_times.csv b/results/model_training_evaluation_times.csv new file mode 100644 index 0000000000000000000000000000000000000000..e22a30aed88fc9cc41dda1225bce6f129313a705 --- /dev/null +++ b/results/model_training_evaluation_times.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5691ccd7fafb765772c2e5da0ada82bd2f3532459dcfed8517565e7cc0d9f1a8 +size 441 diff --git a/scripts/lf-api.sh b/scripts/lf-api.sh new file mode 100755 index 0000000000000000000000000000000000000000..d0ccb5e02427288d0570acbf788679dc6ea32dba --- /dev/null +++ b/scripts/lf-api.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +BASEDIR=$(dirname "$0") +cd $BASEDIR/../llama-factory +echo Current Directory: +pwd + +API_PORT=8000 llamafactory-cli api $1 diff --git a/scripts/tune-large.sh b/scripts/tune-large.sh new file mode 100755 index 0000000000000000000000000000000000000000..04fe8f15221634a57547af5c8bdab0ba3972391b --- /dev/null +++ b/scripts/tune-large.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +BASEDIR=$(dirname "$0") +cd $BASEDIR +echo Current Directory: +pwd + +nvidia-smi +uname -a +cat /etc/os-release +lscpu +grep MemTotal /proc/meminfo + +# pip install -r requirements.txt +# FLASH_ATTENTION_FORCE_BUILD=TRUE pip install --upgrade flash-attn + +# export MODEL_NAME=unsloth/Qwen2-72B-Instruct-bnb-4bit +# echo Tuning $MODEL_NAME +# python tune.py + +export MODEL_NAME=unsloth/llama-3-70b-Instruct-bnb-4bit +echo Tuning $MODEL_NAME +python tune.py + diff --git a/scripts/tune-lf.sh b/scripts/tune-lf.sh new file mode 100644 index 0000000000000000000000000000000000000000..0b722d60fe1b09cbc086ca4e1aa49265f0932b23 --- /dev/null +++ b/scripts/tune-lf.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +BASEDIR=$(dirname "$0") +cd $BASEDIR/../llama-factory +echo Current Directory: +pwd + +YAML=$1 python -c 'import os, json, sys, yaml; filename=os.getenv("YAML"); y=yaml.safe_load(open(filename)) ; print(f"{filename}:\n", json.dumps(y, indent=2))' +llamafactory-cli train $1 \ No newline at end of file diff --git a/scripts/tune-medium.sh b/scripts/tune-medium.sh new file mode 100755 index 0000000000000000000000000000000000000000..fc27178be736afc77c866b41d4a22ab603894b80 --- /dev/null +++ b/scripts/tune-medium.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +BASEDIR=$(dirname "$0") +cd $BASEDIR +echo Current Directory: +pwd + +nvidia-smi +uname -a +cat /etc/os-release +lscpu +grep MemTotal /proc/meminfo + +# pip install -r requirements.txt +# FLASH_ATTENTION_FORCE_BUILD=TRUE pip install --upgrade flash-attn + +export MODEL_NAME=unsloth/Qwen2-7B-Instruct +echo Tuning $MODEL_NAME +python llm_toolkit/tune.py + +export MODEL_NAME=unsloth/mistral-7b-instruct-v0.3 +echo Tuning $MODEL_NAME +python llm_toolkit/tune.py + +export MODEL_NAME=gradientai/Llama-3-8B-Instruct-Gradient-1048k +echo Tuning $MODEL_NAME +python llm_toolkit/tune.py diff --git a/scripts/tune-small-2.sh b/scripts/tune-small-2.sh new file mode 100755 index 0000000000000000000000000000000000000000..0dd6a2e4a8f74e150563dc5cadceabd992f45888 --- /dev/null +++ b/scripts/tune-small-2.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +BASEDIR=$(dirname "$0") +cd $BASEDIR/.. +echo Current Directory: +pwd + +export MODEL_NAME=unsloth/Qwen2-0.5B-Instruct +echo Tuning $MODEL_NAME +python llm_toolkit/tune.py + +export MODEL_NAME=unsloth/Qwen2-1.5B-Instruct +echo Tuning $MODEL_NAME +python llm_toolkit/tune.py diff --git a/scripts/tune-small.sh b/scripts/tune-small.sh new file mode 100755 index 0000000000000000000000000000000000000000..aa77dd81174e7e518d9b91a2e4d0b42244ebf438 --- /dev/null +++ b/scripts/tune-small.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +BASEDIR=$(dirname "$0") +cd $BASEDIR/.. +echo Current Directory: +pwd + +export MODEL_NAME=unsloth/Qwen2-0.5B-Instruct-bnb-4bit +echo Tuning $MODEL_NAME +python llm_toolkit/tune.py + +export MODEL_NAME=unsloth/Qwen2-1.5B-Instruct-bnb-4bit +echo Tuning $MODEL_NAME +python llm_toolkit/tune.py