File size: 78,812 Bytes
942627f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 |
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "VhQSb7PdZznG"
},
"source": [
"# Sequence-to-sequence activity recognition"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "n2y0GYTdc-nY",
"outputId": "8a4c97ee-752b-4ef3-a83d-e6da46c5f019"
},
"outputs": [],
"source": [
"!pip3 install wandb"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "gSxOaWIFSBM-",
"outputId": "475bc447-6414-46af-c5ec-49526e2808f8"
},
"outputs": [],
"source": [
"!pip3 install git+https://github.com/tensorflow/addons.git"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "VSncNSHtZznI"
},
"outputs": [],
"source": [
"from tensorflow.keras.layers import Add, Dense, Dropout, MultiHeadAttention, LayerNormalization, Layer, Normalization\n",
"from tensorflow.keras.optimizers import Adam\n",
"from tensorflow.keras import Model\n",
"from tensorflow.keras.initializers import TruncatedNormal\n",
"from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler, Callback\n",
"from tensorflow_addons.optimizers import AdamW\n",
"from wandb.keras import WandbCallback\n",
"from sklearn.model_selection import train_test_split \n",
"\n",
"import math\n",
"import wandb\n",
"import numpy as np\n",
"import pandas as pd\n",
"import tensorflow as tf\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "_kdFgpMxZznJ"
},
"source": [
"## Init logger"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Z6DnpqLPZznK",
"outputId": "078f5861-e753-4525-fe92-0516ac23f007"
},
"outputs": [],
"source": [
"wandb.login()\n",
"\n",
"sweep_config = {\n",
" 'method': 'grid',\n",
" 'metric': {\n",
" 'goal': 'maximize',\n",
" 'name': 'val_accuracy'\n",
" },\n",
" 'parameters': {\n",
" 'epochs': {\n",
" 'value': 50\n",
" },\n",
" 'num_layers': {\n",
" 'value': 3\n",
" },\n",
" 'embed_layer_size': {\n",
" 'value': 128\n",
" },\n",
" 'fc_layer_size': {\n",
" 'value': 256\n",
" },\n",
" 'num_heads': {\n",
" 'value': 6\n",
" },\n",
" 'dropout': {\n",
" 'value': 0.1\n",
" },\n",
" 'attention_dropout': {\n",
" 'value': 0.1\n",
" },\n",
" 'optimizer': {\n",
" 'value': 'adam'\n",
" },\n",
" 'amsgrad': {\n",
" 'value': False\n",
" },\n",
" 'label_smoothing': {\n",
" 'value': 0.1\n",
" },\n",
" 'learning_rate': {\n",
" 'value': 1e-3\n",
" },\n",
" #'weight_decay': {\n",
" # 'values': [2.5e-4, 1e-4, 5e-5, 1e-5]\n",
" #},\n",
" 'warmup_steps': {\n",
" 'value': 10\n",
" },\n",
" 'batch_size': {\n",
" 'value': 64\n",
" },\n",
" 'global_clipnorm': {\n",
" 'value': 3.0\n",
" },\n",
" }\n",
"}\n",
"\n",
"sweep_id = wandb.sweep(sweep_config, project=\"HAR-Transformer\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "-mGp0L3_ZznL"
},
"source": [
"## Layer"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "0lFGhNtyZznL"
},
"outputs": [],
"source": [
"class PositionalEmbedding(Layer):\n",
" def __init__(self, units, dropout_rate, **kwargs):\n",
" super(PositionalEmbedding, self).__init__(**kwargs)\n",
"\n",
" self.units = units\n",
"\n",
" self.projection = Dense(units, kernel_initializer=TruncatedNormal(stddev=0.02))\n",
"\n",
" self.dropout = Dropout(rate=dropout_rate)\n",
"\n",
" def build(self, input_shape):\n",
" super(PositionalEmbedding, self).build(input_shape)\n",
"\n",
" self.position = self.add_weight(\n",
" name=\"position\",\n",
" shape=(1, input_shape[1], self.units),\n",
" initializer=TruncatedNormal(stddev=0.02),\n",
" trainable=True,\n",
" )\n",
"\n",
" def call(self, inputs, training):\n",
" x = self.projection(inputs)\n",
" x = x + self.position\n",
"\n",
" return self.dropout(x, training=training)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "PIwd6GlIZznM"
},
"outputs": [],
"source": [
"class Encoder(Layer):\n",
" def __init__(\n",
" self, embed_dim, mlp_dim, num_heads, dropout_rate, attention_dropout_rate, **kwargs\n",
" ):\n",
" super(Encoder, self).__init__(**kwargs)\n",
"\n",
" self.mha = MultiHeadAttention(\n",
" num_heads=num_heads,\n",
" key_dim=embed_dim,\n",
" dropout=attention_dropout_rate,\n",
" kernel_initializer=TruncatedNormal(stddev=0.02),\n",
" )\n",
"\n",
" self.dense_0 = Dense(\n",
" units=mlp_dim,\n",
" activation=\"gelu\",\n",
" kernel_initializer=TruncatedNormal(stddev=0.02),\n",
" )\n",
" self.dense_1 = Dense(\n",
" units=embed_dim, kernel_initializer=TruncatedNormal(stddev=0.02)\n",
" )\n",
"\n",
" self.dropout_0 = Dropout(rate=dropout_rate)\n",
" self.dropout_1 = Dropout(rate=dropout_rate)\n",
"\n",
" self.norm_0 = LayerNormalization(epsilon=1e-5)\n",
" self.norm_1 = LayerNormalization(epsilon=1e-5)\n",
"\n",
" self.add_0 = Add()\n",
" self.add_1 = Add()\n",
"\n",
" def call(self, inputs, training):\n",
" # Attention block\n",
" x = self.norm_0(inputs)\n",
" x = self.mha(\n",
" query=x,\n",
" value=x,\n",
" key=x,\n",
" training=training,\n",
" )\n",
" x = self.dropout_0(x, training=training)\n",
" x = self.add_0([x, inputs])\n",
"\n",
" # MLP block\n",
" y = self.norm_1(x)\n",
" y = self.dense_0(y)\n",
" y = self.dense_1(y)\n",
" y = self.dropout_1(y, training=training)\n",
"\n",
" return self.add_1([x, y])\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "YRQTRP60ZznN"
},
"source": [
"## Model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "UYEKK7pYZznN"
},
"outputs": [],
"source": [
"class Transformer(Model):\n",
" def __init__(\n",
" self,\n",
" num_layers,\n",
" embed_dim,\n",
" mlp_dim,\n",
" num_heads,\n",
" num_classes,\n",
" dropout_rate,\n",
" attention_dropout_rate,\n",
" **kwargs\n",
" ):\n",
" super(Transformer, self).__init__(**kwargs)\n",
"\n",
" # Input (normalization of RAW measurements)\n",
" self.input_norm = Normalization()\n",
"\n",
" # Input\n",
" self.pos_embs = PositionalEmbedding(embed_dim, dropout_rate)\n",
"\n",
" # Encoder\n",
" self.e_layers = [\n",
" Encoder(embed_dim, mlp_dim, num_heads, dropout_rate, attention_dropout_rate)\n",
" for _ in range(num_layers)\n",
" ]\n",
"\n",
" # Output\n",
" self.norm = LayerNormalization(epsilon=1e-5)\n",
" self.final_layer = Dense(num_classes, kernel_initializer=\"zeros\")\n",
"\n",
" def call(self, inputs, training):\n",
" x = self.input_norm(inputs)\n",
" x = self.pos_embs(x, training=training)\n",
"\n",
" for layer in self.e_layers:\n",
" x = layer(x, training=training)\n",
"\n",
" x = self.norm(x)\n",
" x = self.final_layer(x)\n",
"\n",
" return x\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "j42cze_qiAIb"
},
"source": [
"## Loss"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "NK6QapYViAIb"
},
"outputs": [],
"source": [
"def smoothed_sparse_categorical_crossentropy(label_smoothing: float = 0.0):\n",
" def loss_fn(y_true, y_pred):\n",
" num_classes = tf.shape(y_pred)[-1]\n",
" y_true = tf.one_hot(y_true, num_classes)\n",
"\n",
" loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits=True, label_smoothing=label_smoothing)\n",
" return tf.reduce_mean(loss)\n",
"\n",
" return loss_fn"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "PxmZ1ZWBAgLX"
},
"source": [
"## LR scheduler"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "GEtbF3TdAjDU"
},
"outputs": [],
"source": [
"def cosine_schedule(base_lr, total_steps, warmup_steps):\n",
" def step_fn(epoch):\n",
" lr = base_lr\n",
" epoch += 1\n",
"\n",
" progress = (epoch - warmup_steps) / float(total_steps - warmup_steps)\n",
" progress = tf.clip_by_value(progress, 0.0, 1.0)\n",
" \n",
" lr = lr * 0.5 * (1.0 + tf.cos(math.pi * progress))\n",
"\n",
" if warmup_steps:\n",
" lr = lr * tf.minimum(1.0, epoch / warmup_steps)\n",
"\n",
" return lr\n",
"\n",
" return step_fn\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "MBlu9AxBHG09"
},
"outputs": [],
"source": [
"class PrintLR(Callback):\n",
" def on_epoch_end(self, epoch, logs=None):\n",
" wandb.log({\"lr\": self.model.optimizer.lr.numpy()}, commit=False)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7dIynjZAZznP"
},
"source": [
"## Dataset"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "4GkimkgOZznP",
"outputId": "c43e079b-f8b2-4d51-f9b0-a5339a3c9b77"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(60060, 300, 6) (60060, 300)\n",
"(12470, 300, 6) (12470, 300)\n",
"(10599, 300, 6) (10599, 300)\n"
]
}
],
"source": [
"CLASS_LABELS = np.array(\n",
" [\n",
" \"Stand\",\n",
" \"Sit\",\n",
" \"Talk-sit\",\n",
" \"Talk-stand\",\n",
" \"Stand-sit\",\n",
" \"Lay\",\n",
" \"Lay-stand\",\n",
" \"Pick\",\n",
" \"Jump\",\n",
" \"Push-up\",\n",
" \"Sit-up\",\n",
" \"Walk\",\n",
" \"Walk-backward\",\n",
" \"Walk-circle\",\n",
" \"Run\",\n",
" \"Stair-up\",\n",
" \"Stair-down\",\n",
" \"Table-tennis\"\n",
" ]\n",
")\n",
"\n",
"# load dataset\n",
"f = np.load('./new_dataset.npz')\n",
"signals = f['signals']\n",
"labels = f['labels']\n",
"\n",
"# split to train-test\n",
"X_train, X_test, y_train, y_test = train_test_split(\n",
" signals, labels, test_size=0.15, random_state=9, stratify=labels\n",
")\n",
"X_train, X_val, y_train, y_val = train_test_split(\n",
" X_train, y_train, test_size=0.15, random_state=9, stratify=y_train\n",
")\n",
"print(X_train.shape, y_train.shape)\n",
"print(X_test.shape, y_test.shape)\n",
"print(X_val.shape, y_val.shape)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 739
},
"id": "RSXjG7qHZznQ",
"outputId": "c83fae89-3e09-4f05-eb7f-6c4b6ab76db4"
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAApUAAALSCAYAAACMDCpzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdfbxtVV0v/s+XA3q8HqAEFfSIBz2CCMhBCPMBFZ8FJfMhExMrLMwwCyswTf2Z1yiNuMY1SEDMRMGHS3hBSa+QWYIiED4gSHqEgxAKQpAiT+P3x1wbN3jg7MNYa+192O/367Vfe6+15l5jrLnWmvMzxxhzzGqtBQAAemw03xUAAGDDJ1QCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAui2oUFlV76mq1VXVqmrVHP/n56vqQ1V1cVV9vaoOm3Q9AQC4owUVKpN8LMmTk3x3Pf7nuCTntda2a63tmOSIidQMAIC7tKBCZWvt8621NXe+v6p+oao+V1XnVNV5VfXS0f0rk+ye5PBZz3Hl9GoMAECSbDzfFViXqvq5JH+XZO/W2hVVtWWSc6vq35I8JsmaJH9bVbsnuTrJIa218+avxgAAi8+CD5VJnpjkEUk+VVWz798+Q/33SPInrbUDq+p5Sf5vVa1ord08/aoCACxOG0KorCRfb6098WceGFonL2+tnZEkrbVPVdV9kjw8ySXTrSYAwOK1oMZU3oV/S7JtVT1z5o6qWjUKj19J8l9V9djR/XtkCKGXzUtNAQAWqWqtzXcdbldVRyfZJ8lWGcZHXt9aW1lVj0vy7iRbJNkkyaVJXthau7Gqdkvy3iT3S/KTJH/YWvvneXkBAACL1IIKlQAAbJg2hO5vAAAWOKESAIBuC+bs7/ve977tgQ984HxXAwCAu3D55Zff1Fq779oeWzCh8oEPfGDWrPmZi+kAALBAVNX37+ox3d8AAHQTKgEA6CZUAgDQbcGMqQQAWJvbbrst5tWejqrKRhvdszZHoRIAWJBuuummXHrppbn55pvnuyqLyiabbJJtttkm97nPfdbr/4RKAGBBuvTSS7Pppptmiy22SFXNd3UWhdZarr766lx66aVZuXLlev2vUAkALDi33XZbbr755myxxRbZeGNxZZq22GKLXHPNNbntttvWqyvciToAwIIzM4ZSC+X0zazz9R3HOufoX1X/lGSrJLcluT7J77XWzlvLcgckOTRDYP1ckte21gyGAAC6rDj01Ik87+rD9pnzstdff3223nrrvOxlL8uxxx6bJDn++ONz8skn5+STT86ZZ56Z3//938/5558/kbouZOvTUvkrrbXHttZWJTk8yfF3XqCqtk3yZ0n2TLIyyYOT/PYY6gkAMO9OPPHE7LbbbvnEJz6RG264Yb6rs6DMOVS21q6ddXPzJGtrE31JklNaa1e2oc30qCQv76siAMDCcOyxx+aQQw7JU57ylJx44onzXZ0FZb1GvlbV3yfZa3Rz77Ussk2S7866vXp0HwDABu0b3/hGLrvssjznOc/JLbfcksMOOywHHHDAfFdrwVivE3Vaa/u31h6W5M1J/qKn4Ko6uKrWzPxoQgYAFrJjjz02+++/f5YsWZK999473/nOd3LhhRfOd7UWjHt09ndr7QNJ9qqqLe700KVJHj7r9orRfWt7jsNba8tnfpYtW3ZPqgIAMHE333xzPvjBD+YDH/hAVqxYkZUrV+ZHP/rR7SfrMMdQWVU/V1UPmXX7hUmuTnLNnRb9eJJ9q2qrGs5Hf02Sj4yrsgAA8+GUU07JIx7xiFx++eVZvXp1Vq9enbPOOisf/OAHXfFnZK5jKjdP8tGqul+GKYW+n+T5rbVWVcdkODnnlNbat6vqrUn+dfR/ZyY5etyVBgCYpmOPPTaveMUr7nDfDjvskIc+9KG5/vrr56lWC0stlAu0L1++vK1Zs2a+qwEALAC33nprLr744my33XZZsmTJfFdnUbm7dV9Vl7fWlq/t/1xRBwCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoNtcJz8HAJhfb9t8Qs973ToXWbVqVZLkpptuykUXXZSdd945SbL99tvnxBNPnFMxp5xySs4444z89V//9T2v6wImVAIArMP555+fJFm9enVWrVp1++3Zbrnllmy88V1Hq3333Tf77rvvxOo433R/AwDcQytWrMghhxySPfbYI6961aty5ZVXZq+99spuu+2WHXfcMQcddFBuu+22JMnxxx+fF77whUmSM888MzvttFNe+9rXZpdddsmOO+6Yc845Zz5fSrdF21K54tBTJ17G6sP2mXgZAMD8uvrqq3P22WenqnLjjTfmk5/8ZJYtW5Zbb701v/RLv5STTjopv/qrv/oz//fNb34zxx57bN773vfmqKOOypve9Kacfvrp8/AKxmPRhkrmj0APwL3Jr//6r6eqkiS33XZbDjnkkHzhC19Iay1XXXVVdtppp7WGypUrV+bxj398kuQJT3hC3v3ud0+13uMmVAIAdFi2bNntfx9++OG56qqrcvbZZ2fp0qU5+OCDc+ONN671/5YuXXr730uWLMktt9wy8bpOkjGVAABj8sMf/jBbbbVVli5dmiuvvDIf/ehH57tKU6OlEgBgTF7/+tfnJS95SXbcccc85CEPyTOf+cz5rtLUVGttvuuQJFm+fHlbs2bN1Mozrm/+WPcArMutt96aiy++ONttt12WLFky39VZVO5u3VfV5a215Wv7P93fAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG7mqQQANgg7f2DniTzvV1/11XUus2rVqiTJTTfdlIsuuig77zzUZfvtt8+JJ54457LOPPPM3HjjjXnuc597zyq7gAmVAADrcP755ydJVq9enVWrVt1+e32deeaZufbaa++VoVL3NwDAPXT66afnyU9+cnbbbbfsscceOeOMM5Ik3/rWt/KkJz0pu+yyS3beeee8+c1vzvnnn5+jjjoqH/rQh7Jq1aq8/e1vn+faj5eWSgCAe+Db3/523va2t+X000/PZpttlksuuSR77rlnVq9enSOPPDLPf/7z88Y3vjFJcs011+QBD3hAXvOa1+Taa6/NEUccMc+1Hz+hEgDgHvj0pz+dSy65JE95ylNuv2+jjTbKpZdemqc85Sn5oz/6o9xwww156lOfuiiuAS5UAgDcA621POtZz8oJJ5zwM4896lGPyhOf+MR85jOfyZFHHpkjjjgip5122jzUcnqMqQQAuAee85zn5LOf/WwuuOCC2+/70pe+lGQYU/ngBz84+++/f/7yL/8yZ511VpJks802y3XXXTcv9Z00oRIA4B5YuXJlTjjhhBx44IHZZZddssMOO9w+VvJjH/tYdt555+y666552ctelqOOOipJ8su//Ms5//zz75Un6lRrbb7rkCRZvnx5W7NmzdTKW3HoqRMvY/Vh+0y8jA2RdQ/Autx66625+OKLs91222XJkiXzXZ1F5e7WfVVd3lpbvrb/01IJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQzWUaAYANwoWP3mEiz7vDNy9c5zJ777139t577xx00EF3uH+XXXbJW9/61rzoRS/6mf85/vjjc/LJJ+fkk0/OOeeck3e961058cQTf2a5G264IZtuumnWNXf4tddem6OOOiqHHnro7fe9+tWvzite8Yrstdde63wNk6alEgBgHQ444IC8//3vv8N955xzTq644oq84AUvWOf/77777msNlOvj2muvzWGHHXaH+4455pgFESgToRIAYJ323XffXHbZZXe4zvdxxx2XfffdN89+9rOz2267Zccdd8xBBx2U22677Wf+/8wzz8yqVatuv3300UfnUY96VHbdddf89V//9R2WfcUrXpHdd989j33sY7PPPvvkyiuvTJK85jWvyfXXX59Vq1Zl9913T5I87WlPy8knn5wkueqqq/KiF70oO++8c3baaaccffTRtz/nihUr8pa3vCVPeMITsu222+Yd73jH+FbOiFAJALAOm2yySV75ylfmuOOOS5LceOON+fCHP5w3vOEN+eQnP5mvfOUrueCCC7J69eqcdNJJd/tcX/va1/LWt741n//853Peeeflxz/+8R0eP+KII3LOOefkggsuyJ577pm3ve1tSZKjjjoqm266ac4///ycc845P/O8r3vd67L99tvnq1/9aj73uc/lHe94R84666zbH7/22mvzxS9+MV/+8pfzrne9K5dffnnnWrkjoRIAYA4OOOCAfOhDH8pNN92UT3ziE9lhhx3y8Ic/PIccckh22WWX7LrrrjnnnHNy/vnn3+3zfO5zn8vznve8bL311kmS3/md37nD4yeccEJ233337LTTTjnmmGPW+XwzPvvZz+bAAw9MkjzoQQ/Ki170onz2s5+9/fH99tsvSbLlllvmEY94RL7zne/M+bXPhVAJADAHj3nMY7Jy5cp88pOfzHHHHZcDDjgghx9+eK666qqcffbZueCCC7LffvvlxhtvXK/nrarb//7CF76Q97znPTnttNPyta99LYcffvh6P9/anjdJli5devvfS5YsyS233HKPnveuCJUAAHN0wAEH5J3vfGe+9KUv5WUve1l++MMfZquttsrSpUtz5ZVX5qMf/eg6n+PpT396Pv3pT98+VvKoo466/bEf/vCH2XTTTbPFFlvkpptuusO4yM022yw//vGPc9NNN631eZ/5zGfmfe97X5Lk+9//fj7xiU/kWc96Vs/LXS9CJQDAHL3sZS/LRRddlJe+9KVZtmxZXv/61+fss8/OjjvumFe+8pV55jOfuc7n2GmnnfK2t70te+65Z3bdddfc9773vf2x5z73udl+++2z/fbbZ88997zDyT0PeMADsv/+++exj33s7SfqzPae97wnF154YXbeeefstddeedOb3pTHP/7x43nhc1DrmhNpWpYvX97WrFkztfJWHHrqxMtYfdg+Ey9jQ2TdA7Aut956ay6++OJst912WbJkyXxXZ1G5u3VfVZe31pav7f+0VAIA0E2oBACgm1AJAEA3oRIAWHBmpsNZKOd+LCYz6/zOUxKty8aTqAwAQI+NNtoom2yySa6++upsscUW6x1wuGdaa7n66quzySabZKON1q/tUagEABakbbbZJpdeemmuueaa+a7KorLJJptkm222We//EyoBgAXpPve5T1auXJnbbrtNN/iUVNV6t1DOECoBgAXtnoYcpsu7BABAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdNp7vCgCLw4pDT51KOasP22cq5QBwR1oqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3eYUKqtqaVWdXFUXV9W/V9VnqmrlWpZbUVW3VtX5s34eOf5qAwCwkGy8Hsv+XZJPtdZaVR2U5JgkT1vLcte31laNo3IAAGwY5tRS2Vq7sbV2Wmutje46K8mKidUKAIANyj0dU/n6JP94F4/dv6q+XFXnVtVbqmrJPSwDAIANxHqHyqr6kyQrk7xxLQ9fkeShrbVfSPLMJHsmecNdPM/BVbVm5ueGG25Y36oAALBArFeorKo/TPKiJM9rrf3ozo+31n7SWrtq9Pc1SY7LECx/Rmvt8Nba8pmfZcuWrX/tAQBYEOYcKqvq4CQvT/Ks1tq1d7HMg6pqk9Hf980QQM8bR0UBAFi45jql0PIkf5Xk55KcMZoq6OzRY2+vqteMFn1ykvOq6t+TnJvkyiT/c/zVBgBgIZnTlEKttTVJ6i4ee8usvz+R5BPjqRoAABsKV9QBAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQbeP5rsB8Wb10vymUct0UytjwWPcAcO+jpRIAgG5CJQAA3RZt9zcA07Hi0FMnXsbqw/aZeBnA3dNSCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6DanUFlVS6vq5Kq6uKr+vao+U1Ur72LZ51fVN6vqW1X1iarabLxVBgBgoVmflsq/S7J9a22XJP+Y5Jg7L1BVy5Icm+SFrbVHJflekj8dR0UBAFi45hQqW2s3ttZOa6210V1nJVmxlkWfl+S81to3R7ffm+Tl3bUEAGBBu6djKl+fobXyzrZJ8t1Zt1cn2bqqNr6H5QAAsAFY77BXVX+SZGWSZ/QUXFUHJzl45vbmm2/e83QAAMyj9WqprKo/TPKiJM9rrf1oLYtcmuThs26vSHJFa+2WOy/YWju8tbZ85mfZsmXrUxUAABaQOYfKUcviy5M8q7V27V0s9ukkj6uqR49uvzbJR/qqCADAQjen7u+qWp7kr5J8O8kZVZUkP2mtPb6q3p7ke621o1pr11fVq5OcPBpH+bUkr5pQ3QEAWCDmFCpba2uS1F089pY73T4lySn9VQMAYEPhijoAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0G3j+a4AsDisXrrflEq6bkrlADCblkoAALoJlQAAdNP9DSwKKw49deJlrD5sn4mXARsS37vFRUslAADdhEoAALoJlQAAdBMqAQDo5kQdAOBeZxonCSVOFJpNSyUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQbU6hsqreU1Wrq6pV1aq7WOZpVfXjqjp/1s/9xltdAAAWoo3nuNzHkvxlki+sY7mLWmtrDZ0ALE6rl+43hVKum0IZwN2ZU6hsrX0+SapqsrUBAGCDNO4xlY+sqnOr6stV9doxPzcAAAvUXLu/5+LcJMtba9dV1fIkp1XVD1prJ61t4ao6OMnBM7c333zzMVYFAIBpGltLZWvtv1pr143+XpPkw0n2vJvlD2+tLZ/5WbZs2biqAgDAlI0tVFbV1lW10ejvTZM8P8l543p+AAAWrrlOKXR0Va1JsjzJ6VV1yej+Y6pq39FiL07y1ar69yRnJflMkvdPoM4AACwwcz37+8C7uP/Vs/4+MsmRY6oXAAAbEFfUAQCgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdNp7vCgAAk7Hi0FMnXsbqw/aZeBlsGLRUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN02nu8KAEzD6qX7TaGU66ZQBmw4fO8WFy2VAAB0EyoBAOgmVAIA0M2YSgDgXmc64zkTYzp/SkslAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOi28XxXAADg3mbFoadOvIzVh+0z8TLWh5ZKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbs7/nwTTOCEsW3llhAMC9l5ZKAAC6CZUAAHQTKgEA6CZUAgDQzYk6i9BivHQUADBZWioBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQzWUaAeBeavXS/aZQynVTKIMNwZxaKqvqPVW1uqpaVa26m+UOqKpvVdV/VNX7qmqT8VUVAICFaq7d3x9L8uQk372rBapq2yR/lmTPJCuTPDjJb/dWEACAhW9OobK19vnW2pp1LPaSJKe01q5srbUkRyV5eW8FAQBY+MZ5os42uWNL5urRfWtVVQdX1ZqZnxtuuGGMVQEAYJrm7ezv1trhrbXlMz/Lli2br6oAANBpnKHy0iQPn3V7xeg+AADu5cYZKj+eZN+q2qqqKslrknxkjM8PAMACNdcphY6uqjVJlic5vaouGd1/TFXtmySttW8neWuSf01ySZLvJzl6IrUGAGBBmdPk5621A+/i/lff6fb7krxvDPUCAGAD4jKNAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB023i+KwCLyYpDT514GasP22fiZQDAnWmpBACgm1AJAEA3oRIAgG5CJQAA3ZyoAwAwZquX7jeFUq6bQhlzp6USAIBuQiUAAN2ESgAAugmVAAB0c6LOPJjO4N1koQ3gBQDuvbRUAgDQTagEAKCb7u9FaDHOnQUwH1YceupUyll92D5TKQfujpZKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3Tae7wrANK049NSplLP6sH2mUg4ALBRaKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALqZpxKmaPXS/aZQynVTKAMA7khLJQAA3bRUAtzLuZIUMA1aKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbs7+BuBebRpnvzvzHbRUAgAwBloqAWBCpnMVrcSVtFgItFQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2c/Q2LiPn6AJgULZUAAHSbc6isqkdV1b9V1cVV9eWq2nEtyzytqn5cVefP+rnfeKsMAMBCsz7d30cn+bvW2vFV9ZIkxyf5hbUsd1FrbdU4KgcAwIZhTi2VVfWgJLsn+YfRXR9P8rCqWjmpigEAsOGYa/f3w5Jc0Vq7JUlaay3JpUm2Wcuyj6yqc0dd5K8dUz0BAFjAxn3297lJlrfWrquq5UlOq6oftNZOuvOCVXVwkoNnbm+++eZjrgoAANMy15bKy5JsXVUbJ0lVVYZWyktnL9Ra+6/W2nWjv9ck+XCSPdf2hK21w1try2d+li1bdk9fAwAA82xOobK1dlWGVshfG9314iRrWmuXzF6uqrauqo1Gf2+a5PlJzhtfdQEAWIjWZ57KA5McWFUXJzk0yW8kSVUdU1X7jpZ5cZKvVtW/JzkryWeSvH+M9QUAYAGa85jK1tpFSZ6wlvtfPevvI5McOZ6qAQCwoXCZRhaV1Uv3m1JJ102pHABYGFymEQCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBt0U4ptPO220y8jK9OvIQNk3UP02UqLWAatFQCANBNqAQAoJtQCQBAN6ESAIBui/ZEHQAWh+mcqOQkJdBSCQBAN6ESAIBuQiUAAN2MqQSmYhqT3icmvgeYL1oqAQDopqUSFhFnwQIwKVoqAQDotmhbKk/681smX8irJl/Ehsi6B4B7Hy2VAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3RbtPJXAdE1lftLEHKUA80RLJQAA3YRKAAC66f4GYKJ23nabiZfx1YmXAKyLlkoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN1MKAYuCaW1g+nzvFhehEgC415lGoE2E2tl0fwMA0E1LJQATddKf3zL5Ql41+SKAu6elEgCAbkIlAADdhEoAALoJlQAAdHOiDgAwEU7SWlyESmBRsHODxWUq3/nE934W3d8AAHQTKgEA6CZUAgDQzZhKALiXmsb1r137mhlaKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdTCkEAPdSLk/KNGmpBACgm5ZKAIAxW4wTzwuV82AaH7Tkrj9si/GDDgBMlu5vAAC6CZUAAHQTKgEA6GZMJcAUrDj01FhtiXQAACAASURBVImXsfqwfSZeBjA3i3E6J6FyHkzlg5bc5YdtMX7QAYDJ0v0NAEA3LZUwRaZzAuDeSkslAADdtFQCwITM98UuYJq0VAIA0E1LJUyRM+8BuLcSKgGmYPXS/aZQynVTKANg7YRKFhXjm4Bpmu95iWGahEqAezkHU8A0OFEHAIBuWipZVHRFweLjogMwHUIlLCJ2rouTgylgGoRKAO7VTOUF0yFUwiJi5wrApDhRBwCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAtzlPfl5Vj0rygSRbJrkuya+31r6+luUOSHJohsD6uSSvba3dPJ7qAmyYXCITuLdbnyvqHJ3k71prx1fVS5Icn+QXZi9QVdsm+bMkj0vyn0n+MclvJ/nfY6ktwAbK1YyAe7s5dX9X1YOS7J7kH0Z3fTzJw6pq5Z0WfUmSU1prV7bWWpKjkrx8XJUFAGBhmuuYyocluaK1dkuSjALjpUnu3J+zTZLvzrq9ei3LAABwL7M+3d9jVVUHJzl41l23VtWV81WfOViW5Ib1+o+qxVv+Yn7t813+Yn7t813+Yn7t813+Yn7t813+Yn7t813+eMueqwfe1QNzDZWXJdm6qjZurd1SVZWhBfLSOy13aZJHzrq9Yi3LJElaa4cnOXyO5c+7qlrTWluu/MVV9mIvfzG/9vkufzG/9vkufzG/9vkufzG/9oVQfq85dX+31q5Kcm6SXxvd9eIka1prl9xp0Y8n2beqthoFz9ck+ci4KgsAwMK0PvNUHpjkwKq6OMOUQb+RJFV1TFXtmySttW8neWuSf01ySZLvZzhrHACAe7E5j6lsrV2U5Alruf/Vd7r9viTv66/agjPfXfWLufzF/Nrnu/zF/Nrnu/zF/Nrnu/zF/Nrnu/zF/NoXQvldajiRGwAA7jmXaQQAoJtQCQBAN6ESYB1Gs1nAHVSVfSjM4gvBnMzsVKtqp9E13uezLo+tqudU1X3msx4sHs3gc8F6LVprtyXDurF+JmPWvmejxb6ON4SDmAVfwYWiqpbOY9k/P19lz7LF6Pd7kmya3OHLft8p1+WpSQ5K8t6qOrCqHjPl8lNVD5tyeQvuu7oQ6zQOsz7X96mqbatq36p6aVU9aPbj82k+drAzwXpa73tVbTKNcu6JqnpcVX2uqn67qrZqI6PHpvLeLITP4TTMrNfW2m0O7vKrVbV5klTVkvmuzNrcK3cK4zYKdR+oqj+uqj2mVOaS0e9tkrx2GmXeTV22TPKqqjomydZJrq6q/zHrC75/VW02xSr9nyRHJrkxyZuSHFZVf1tVL55kK+rMTm60Pg4d/T2R79Cs93/r5KctIvOtql5QVbsl02mlmWmNnvIOdOY9/b0kJ2S4iMNzk7y9qrZdCDu2Se9gZwXrpVX11Ko6qaoOGb3+iX0WZ5W7fZLjJlXOGPwoyelJ9kpyclX979H2536TeG9mf/6raoeqetg0P4fTDrCztn+7VNVBVXVqVR03s+2Zr0BdVZtPu4ds1rr41ST7t9auq6pHJnljVb1hmnWZC6FybjZP8pUMgeqtVXXU6Aj1URMsc7uq2jXDZPJbzH6gqn5uyi1lP0zyb0l2TnL/JIcl+b2q+sWq2i/J61pr/zWNilTVRq21NUnuk2G9HJrkQ0luTfLuJH8xwRaOx1TVb2SY0P+W5A7Bauuqety4Cmqt3Tr688TRxnSLUTnzfXT67CT/q6o+VlX7V9VDZrfSjNMosL81uUMr2cRf/6x1/5okv5LkVUnem2TLJAdPu9di1k7liVX11qq6sKo+VVVPm2CxM/uGNyR5Y5JvZXjvz62qt0yq0Fmfo5uSbFpVW84cuFXVQ+ajV2JtWmvfbK39RYbLEN+QYR+xf5JPVtWRVfXwMZc38/n/qySvT/L1qnpMVW1SVXOeb/ruzFrP9x+t6ydU1YrZ5U/LrO/gkUl2SfLRJNcnObKqnjSt+sxaJ4+uqqMyzCH5J1X1q1W13TTqkGTmtb4yyeFV9eAkhyR5UpIdq+rxU6rHnAiVc9BaW91a+8sMwWXjJA9NsluGN/i9VfWkCRS7bZJ3Jdkvw4HZk0ZHJ0nyxxk+UBM3+lI9oLX2xQw7l+cm+WyS3ZP8RYZLdh41jbrcyW8mOaG1dkJr7cQkf5rkC0lOb63dPKEyK8n2SV6Q5CFV9ftV9czRY7+VYZ2M2ysyBNj9kztsbKduFG7ekuRtSX6S5M8yhN7jq+p5VXX/MRf5mCS7VdUDZ+5ord1aVTuNuZzbzdqJ7JDknNbaZa2177fWvpLhPXhShgOaqZn1nv+vDNuf1yb5epJjq+qXJ1zmE5K8pbX2ptbaM5Lsk+SxVTX2axOPGrwfXlVLW2vfSfKfSZ7QWrutql6X5O+T/NK4y11fsz4je2ao3zOT/H6SdybZJMkjM/SiTKK8HZJ8IMlFrbVvJFme8R/o/FWSTyb5oyR/WFUHV9U+NaXeqFmt1Tsl+Ulr7bcyvOY/THJKkt+qqvtNoy6z/HGG7/2FSZYmeVaSN1fVr0+64NHnf2abszrJnyc5r7X2vCRbJVkIw+NuN5YjnHuzqqrWWquqByR5dmvtsVW1aZLtMrSi7JDk2nGX21o7raq+l+GL9IAMrYOXVtUFGVpOfubqRhPyggw7kaOTLEvy7dHG7ANVtTJD0L50SnWZ+YJtnOSaJI+vqjOS/Li19sNRqFkziXKrarPW2vlVdWOSJUm+k+EIeveqemWGjczY35PW2mVV9b4MBzBPT3JAa+2qmc/luMtbh9tG6/nZGTZun8jwPrw6ybFJ/inJr4+rsNba16rq6iTPS/L3VfXE0fPfkOTgcZVz52JHv1+S5Beq6p1JjsnwOp+W5Puttf+qqiXTCPiztj+PTnJFa+1PRw+dUVVnJ3llVX2mtXbDBMpcmeS2JFfMPNZa+7eq+ocM24Jxe26GIQenVdW/JTkzyVuq6tAk387wnl84gXLXy6zu/72S/Pvovh8k+UFVHZFkj9baf46zyNHvl2c4sHhIki+N7tspydNHjR59hQzb1i2TPDnD9uzRSR6X5BFJ9kzyH0mm0SNVGV7zbkmur2FIwY+T3FxVn0uyz+j2xI3WSSV5+OigKqNA+9gkz0hy0STLr6r7ttZ+0lq7qapOTXJehlbb91XVVkke3Fr79CTrsL6EynWYteN+RoYj57TWrk/ylap6bZL3tNa+Ps4yq+rNGbrbvpvkt1prPx61Uv5SkgclObC1NvEgN9q5/GNV/VOSvTOMcbpkFOQ+nuRL89Fy1lq7par+JkN39x8k+VpV/UKSbVtrp0+o2N+oqg9n2Ngd1Vr7j1E3xG5Jfi7J8a21746zwKr6+dbaD5NcnuRXM5wk9Y6qesPoMzhVs74LeyfZvbV24yjgr8lovYzqfY8D16hL6eQMgfUjSf4kySGj93fnJKdl2KhOxKzX+I0kn07ylAwHkMuTXJLhPUiGsDUNMzvYpyW5fw0nhVw5euzSJA8ZZ6BM7rAOds0QLs6sqsOSnJFhXfxza+2bVbVxa+2WMRZ9fpJ/TvL0DAezN49+PpHk6NbaDTOtWAvER5IcU8MQoH9N8r0MPTdfutv/ugdGr/vbSVZkaFQ4aPTQyzKG70MNw4puS7JHkjNaa1dkOJg4o4Zx3bu01r7ZW85czArtOyd5fJKPVNVpGbaDz0ly/KjOEz2wm7VOdk5yY1X9VpKPt9auSXL26GfSfrOq/i7Jlq2191bVsUk2Gu0D/yjJOVOow3pxmcY5qqqHJvmbDDu1kzIMHfj9DEcKvzPGcjZK8muttb8ftUR8ZVTev06wW3fOqmrfJL+WYYdzS5KXtta+Nk91+cUMXYFbJvlykn9prX12AuVsnOQlrbWPVNUHk/x3hhaKc5J8bRJHzVW1KsP7/oMM4eHJGca1bZHkm0l+o7X23+Mudw71emCS92dolfz71tq1o/svyNBi8oMxPP8vZehm3iVDS/huSd7bWjvo7v53Ekb12TvJMzN0a16YITQcN83vY1W9McnvJLkqw7q/JMmqJP82+lyOfQc7CjK75Kev/0kZgsZxrbW3j7ms+yS5deY1jA4injH6+UmGA5c/H/eB2/qaCRpV9dQk98vwmXhehvflkRl6rfabxHezhnH078rQO/ZnGbbBeyV51rjKq6r/meSFSf4uyaeSfGe+9jujg/bHZHiN22X4LHwhw4ma/6+1dvmU6vGqDMObfpLkcxm2wxcn+fok100Ns028OMkHMzTinJHkXzL0En0/Q2NGa619f1J1uCeEyvUwOiJ9Q4ausB9l2Kj8/qg7eFxlPDtDN9dXq+qXkuyb4Uv1nQwfqlNaaxeMq7x11GXJaAzb05Msb639/azH7pvhiPkDrbWfTKEuM11y98+woXlohi/Xv2f4HE+l5aiGkyOePKrDTCvdBRnWw9i+TKOW6Y0zjJf5boad1pZJrstwcHNJG04UmLqqen6SAzO05N03w5H8z7fWXjjr6L7n+WdOxtkpw5CCX0yyY4aegv/ZhvG9YzfrM7bPqNyLkpzfWvvq6PHtM3zm79da+4NJ1OFu6rYkQyvV0zKEuxdkaE09Lck/jqsVaVZo2iFDt+dpo3Vynwzr5KkZukK3TfK0Npw0N45yX5thqM/7M7yeC2bqk6F16vlJ/qC1dtM4yrunZn1GTs4Qrk+pqt0zbA8uSvLN1tp1YyzvtRm2c2e31q6u4WTA12UYgvODJB9qw3jfcZS1cZLfyBBWH5xhm/PNDN3ekxyrPrsOM+v3hRkO3s8eDTfZJkOr5Ux3/OrW2iFTqM8mGRqQlmYYT7xnhpOyliR5Y2vt21Oow/2T/G6Gg+utM4Tac5N8trU20e73e0KovBuzx62NNuq3ZRiI/dTR319srf1ozGWen2H85EmjjftDM+xUn57hRJDzWmuvGWeZd1OXmR3Mp5J8sLV2QlUdlOHL/fHW2smz19GE6zITcP80w3i3K5JcmSFkXZLkrNbalydU9saj7oZnZPSe1zBX2A4ZTqS6tLX27kmUfad6zLwfH0hyVWvtjyZd5t3U5XlJXpqhJfHiDAc7F/WEylmv76FJdm6tfXq0Ud8yw47kl5P8bWvtP8b0MmaXPbMz2yZDULtgVG4y7FjPa629f9zlzrFOB2boAv9UG8bYzgT5xyd5YoYxxa8ec9m/liG8/DBDy+xJMz0SNUzb9YjW2v8bY3lLMvSAvDTDwcTFSf4xw+fqsnGVMw6j9X9Eko+Ncx2spZwtkvxlhoPLG5N8LUOQuHCcLdOzvncPSnL1aDu7S4bGjCcl+V5r7fXjKGuO9VmSYZjJ5hkacC7M0KDy7Qz73yckuW5S2/s71WXvDCfpHNxaO3d03+4Zxs2+d8Jl/8y2tIax1ftm+J78zeyGnoVCqJyDqnp3hlaCKzJ8wP8lyWWttWvHGapGR6FHtNaeMvpivSDDPHlnZWjufnWb8li6Ufffma21HavqJRm6Ab6YoeXo90Zjb6ZZny9l6J65IUOL4a6jnw+01j45wXK3SPJ/k+zdhpNVXp7kltbaR+unA8l7y5gJzk/IcFT86xlaA09orX1u1nJbJfnvaX0WZtXroRm6oVZmGE/01TG3zs6U85cZWgNfN+ux7hbQOdbhtRkC0x+OWsmemqEL6sGttZdOqx6z6rNRhulDts/QM3JZhtB7bobvwHZJbmqtXTLmcrfIMI50pwzB4gkZDuB+c9KtMzWM4XtVkhdlCBbfSfKqNt6TX+6xUSva/8rwXrwjw0HtFRPq8n54hm71PTIcWC3L0IL41QwH0mMbz1/DCYHXZpg+7pzRAcz9Moznm0qwn3Ug9aAMrZK7ZmihW5Hkn1prfzKNesyqz/0y9Mo8OcMBzlRC3Kyg/+AMB1tbZOh6P3tmu19V95nvlvu1ESrvwqw3da8M41j+JsNZd4/IMLXAdUkOHWdLZVUdnuTa1trbR93gv5lhTOXho/Kvaa29eVzlzbFOOyX5/zLMBfnKDCdOfC/DYO6xzcu4jjrMbGi2zdCK+6rW2o2zHn9UkjXjCHZrKXsm6Lw+yYrW2h+M/v6VDBvgd7bW/nVMZc28zjMyTOvxuxlayx6doQvmgNbaR2YvO45y16Ne/5hksww7+fsl+XGG1oOPjDPUjFrrn99aW1Ojsx9rGOv1T621fx5XOXdR9soMXa3/0GaND53ZgE8zVM5a75tn2O5sn+HEoacl+Upr7ZWTKO9O9903w1CTQzOE2D+c5Ouvqs1ndx/XMLb4V6YdJu5ODdP3PD/DvJ2PznACyTeSfHjc4X5U3tattStG27nHZegh2SXDMIHjx1TGkgzbtF/MEOB+kqGb9V8ydH1Pc6L1TdqsrvYahmIcnqFl+Nga/wlidy5/5nt33ySbtda+X1UvyHAg8aUMJ4h+ZZLrZNZ+5/gk/2P0s0WGnoOvZNjmjvUE4XFx9vddm/nAPD3J4a21E5LbB0vvmWTZOAPlyLlJnjFqJXhThjMKPzL6cH0vP+2Om6ZvZehmfkeSN4+6Xt6YYcD0tD0nw1QOx1XV+5Nc3Fr7bmvtW5MqcFYX015JPjQKlA/N0GL74gw7l7GEytGG7GEZWun+b1W9tbX2C6MW4l/L8F5M/KzHu6jXVkm2aK09edR6sl2GHeoeGbqkxmLUGvq9DGOW0n46XnefJH87rnLuVObMAeQjM3zOn5PkAVV1boZ1flkbTe4/zVbK0XpfOgpZ5yU5r6o+n+EyqaeN6j6JE3T+OMN7cGobZh/4dg0nDW49Wk9jPaCZtRN/dZIXjj5fM2MWz89wVviCMTqg/ViSj40+ry/I0B05tlkJZq2T7ZIcUVUvHW3nvlXD+NZtM7RYjsXoM/ThJB8ebYNemWFmjZvbFKesGR1A/U4NJ/5dmOS7o33OtRlaZ5NhyM0kbTQq4/cyzPpxeYbP47EZTpDarLW29yQrMNrnV5IdWmuPT5IapjLcN8O+5ysZ5qpdcITKuzD6QleGM6yeW1XnZZhw9rIMXdKT8JkMX+ZzM5yc8Rezjtqfm59OIzE1o536787crqqfy7DTndrloWbtwM7JMNXS7kkOSPK90Rf+H6bQNfa+UZkPyTAM4RtV9b8znFwwTntkmFB8VX46/+nXkvxnGw3In2agnNUy9/iM5ghswxm4362qf84wzm9srTOttctH37V31zBH5MYZusB+0MZ0UshazExTc0iGYSavy9Dl+6oMrbGfzjDx9lSNvmuHjVpuz0pyQWvtezVMQr16tNjYQu5MiM3QIrJ7kl+pqm9m2IEdmOFAN8nt0xyNs9ydRmW8JsNZx8/OcBm6S5I8qbV29bjKuydmtRw9KsM2erMMw2H+ubV2VMZ/AYiZYHNAhq7o/57V3bldkvu3MZ6kUcPlD9dkmIf1siTvrGHOyrEcMK+H7TIMtdgpw3fxe6N67NRa+1Iy+av7zNq+tgzb4P/IcCW5b2S4GMVEZ92YdaC4c5Iv1nBlvQtG3d4fGv0sWLq/78boKPEvkjwwQ0vBRRk+WN9srX1vguU+MMN4vR+ONvIvTvL61tpUrjt+p7ps0lq7eTQM4MFtmL7k59poKpn5MFone2SY5mS7DNdDnejYklHrwNMzTP5+cVU9J8mbWmtPGXM5W2UYXrFRhiD79xlOxti4tfZb026lnFWvt2eY5PxTGeYN/HJrbSwtJaOQ9MjW2nmj21tmaBF4WIaN+XcyzFM40Xnhqur/ZJgO5sej29tl+O6d21o7fZpd36Pyt88QtO6f4TNxY4azUFe11nadYLlbJnl4hh37rhm6Wr84iS7oqnp4a+27NYxb/3aG6bP2aa39TlUdmSRtHqaSuitV9ZUMB7ZHZggXP0xyaoYGgLGPL6+q05O8rbX2xar6H204SfCvM7TgHTGmMp6WYVjRF0Y/38kwq8P7kjy+zRpqNA01jGN8YoYD2W0znKxzVmvt/0x7+zc62Hl6hs/laZPez9yp7PdkOMg6M0OX+/kZhsBNrOt/HITKdRiFiV0zBJjtMxzFn9imN2D350dl39gmeCLK3ZQ/c4R+UpKPttYmNvH0XZQ/0w20c4au5isytNydN6rXsjbmyZ/XUZ9HZtj5PSnJ0jaBeTFH5WyU4Qoir8twFuw72nAlnakGm1l1eUqGEwZ2zdB6/98ZWjbe3TrHstZwqcuWofXt+Rm6XS+pqkdkmLtwYnMTzvp87Zhh3NbXMlzb/ZJpr+e1GW1/dsww1OBRGVqvzm6tfXbSO9gapjK5td1x/PI4T0zcJEML3/syzIX5rgxn/X6mDTNNvDvJ51trp4yjvHtq1vCIPTNcuvBFGcaU71nDvLVPTfKLk2hoqKrfTbJ9a+33ZuqSoYFj397vxah18skZxoTOTHz+ixm61W/NcCb4VAP97M/0KFwumfL2fea9fn2GA/tzM/QSvjJDuD20tXbaFOvzuAzTPD07w7XPv5xhKqN5a9RZF93fd2MU6P47Q6vM2aMWlWdkimMZRmOaphbkZu1k79Nau2kU3DbPcAbey6dVjxmjumyZYUf/lSRvz3DEdmUNV1n44KTKHg1/qFE1Zupxamvt0RnjmNK1BcXR7XdX1ZczjGW9atb9UzUq88wMV1bZPEPL1VMybPDHcXLURhlOPjggw7jhnarqmgzr+OvJ5E5MmvWcD81wCbrHZhjucXH9/+2dZ7hV1bWG3wECNsQawGDB2FDE3oggUkSIvRsRe8GoIWrMjTW5KYZoLNg1YuyI1wZcRLEECyqWa8WCgopGBMUWjXKRcX98c3kWBL0qa6192Ge8z3Meztn7cObce68155ijfMPsNXTv1yT0mq6L2SiX8nVUqPBVAUMZBmXu/l8C5ZH1yz9f8GfQBnXMuQUd1G5C0jH7mtl01EXqwgLH+17k7rk+wHVI0uz19NgIpIJQmEGZM2x6oJzeQ0w5rY8g7/2jBRiUm6Diz1boM3jY3U8wFUP+Eun//m1hxviezE3zOwDomjzWlR2kc+M0R2lemV7tI6gpQ5E91v+N3P23WJrPU8iwxdR4ZI/GbFBCeCr/jZxnbkfUAqs3KuV/CG2sr3oj6GxTFrkF7WSkiTbRVBnYydWLucqq43zl9QooFHwWqka/EQng9vumv7EQYy8+f9gneQ02SmHoQuQc0kHlCrQ5PeQpNzT32m9Em9Yfaxj67ocMyVlIs/GJFIbLwnHf+5pI+UJ/BNqhQ+4KKOVkZdSStA1wkpdQVbuAuRjyxG6PxKzbprGfKXvsr5lPdg0cDmzvkjQq7RqYz0s0ANjV3feswCO6CTJk5gJ3oc27HaqmrrlRmWHKc56JohRdUV75SHTQLFSzMF2Lj3lKeTJpla6HCoSe8VQ4thB//zwkgzTEJN9zMTAb7XcTkKxWl4UZ43vMaRAqzvrCpIBxjruPqkV0JjcnA1q5WtIuj/Qxy7wXMqPy5yjd6Au07w33gtKNSsfd42sBX8jNvC4NOWRTUJhgp1rPreTX3Rxt7pcg6R5oOHxsAbSowZxGIU/W+cCg9Nhg4IiSxlsO9fU9Cdg89/iA7GfkpSvq/d4TuAdVN56PKs2XQh68N4CV8p9DRe95s/Rv//T+j0Th7qxN6dFFvAdIRPrX6ft2yPO8NZLQ2AYVRZX1GrPrugdwLqowb58eWxroVdX7/XVzS9/fk607RV13XzPmfmgDBTgT6Ju/Fop+fekaXzL9vBQ6XNyGqqlLe53fcZ7ZfbBm7tpYGRWtvZnuhaUKHK95+ndd4M8lvq4ngVVyP09AnsstgGVr8P72RXrIIHWDC2vwWWfrQc/0XgxL6/FWuXmWtgbnxl8vXVsboH3uCaTG8PcqP5vv+xXh7xzz5Vd9jkIPbd19E5NW1rk00jL+IkivP/NUDAeON7PrgTZmNhiFf3qgcElVc2qJbuw3UCVuqxSCPRQYWNKwbVDIoT3wGzObhm7sOzz1WfWCTqvp72TyJMuTOiWgvKbZSDZpZg1P6/ujrh5rIy/lA8BFqHjlywI8192QmD3uPj3l8a3masWYFQ6UQm7eWdeew4APzewVVG39aFljf4e5gfL3RqXHi5YPWi592w8VvN2Yxvl1er6snOXB6PD0rqnKfTSqNN4Q5RDfaWY71eia/4rc+LsAXc3sTHd/AuhkKuJ4y4sVPc+UCM4HupvZ2+5+/ldPFhApSrmUGyPpnjtRYU4L1DWm6ihc9no7I0cOrirnY9JcK4nO5Pb+LFJyJ8qvbo8UPo5Fnt0yo3SZqsKayKh+DjkazjOpDuzmjTz0DZFTOQ+5C6YDCkluhPqrgk5PH3oFvT5ryONmNhO42BV26IkMiFXQwnOMV1QJmFtMugAvuMKso1Bv4B7AZE/VwkXj7q8DfzYVCiyG8u2aA7uZ2ZtIwqhwqQ13n4VyRy9LuU1H0mBU2df+xxJwpUC0QsVpTyD9xoPdfYpJCHhibl7fN/S9oM1tVRQZWGCuaRm4+zhgXFq4+6FNZFeU/lKqfEieXLi7F3rfmyO1iaElDrspOjh0A8YkI7OFqyisO6q+vbqEcZshQfex6HPvBXyGjMslkLRNzQulclyK1oI/mJoAXOapbWWReENl76+AI4DfpVSk64Ar3X1SAWM8mfLD90QRqSWRsP1KZvapF9i7/FvM5cuUP9gZ2NHMlkYpEI+5+ztVGJSJbB3bD5jo7qcDmDraDEEOjCFlTiB3vR8BzDCz1snAxqVR+ucyxy+KyKlcAKY2YXPQRTYU9QHeFqnoV9rRpkpM7QH3Q6dyQ8b131Fy8AcVz2UZd//YzJ5DlY5T0+OroZNc4X3X09/PTqzLo3BMF5Po7NrAGSgseqw30m4GRZJe93rIO38JuhZeRAeuLkUs+Ol93hMJDS+JKhz7AR+X5CHLxs0+57boPp+ZHSrNrCtwuLsfXNb43zCvJVFo8nmkkdcKydbMRIbMzBLG3BmFuz9AHvKnUCeV81F1/3VF51KbCoH2QZv1o2msGSj8O6dW+cPzzTEz8jsgpYdX03v1F1S48RcvS/eA+QAAHK9JREFUMN/W1FRiJDowz849vjPKOV3P3Vcoarzc318FCWrvje6/w12i86ViEo6fiSJfG6SvLYDl0f47yd3PKnseaS7ZevBrpHLxcyT8PjvlN3Z098EVzKM10qHshg5ZVwKXe3kavYUTRmUit4B0BXq4CiMM9V/eB1Uo3uQLmSDdWDGzFX3etnT9kbzLjugUe427/6miuayK2sKtRYMQcz40/zPgr97QbaWMOeyFcjb75B7rAAx1993LGrcxka6Bie7+Xno/BqN+x2+7KkUL3fhrsbmZ2V+RMfUASneZjA4Pn6c1oKqe46u4ei3vDfR096OSN2kTYHOUx3dMwYbdlii89jnyFE1Ch8r+yMCfCvyizJCfSZf1cFQUNdor7N7ybTGzccjgeQvtAx2AQahFa2FOBlNP8bEo/N8OONdzbRiLNuwXMH4zdK29XEWY1czOQN63jVFoeWpKf+mCPNdvu/tVZb/u+ebUHrgcRYjuQc6E45Aj4Ykq5pCbS3bY3gZFx8pK9yqUMCoTZrYNOh0NRhvpObnTS5ZQXriXoLFgZg+inMWbkXTJB+nxNsiwdncvMww3/3zWQJ2LVkGL+Xh0guuKemBvVvL4P0S5jVlhSjN0bbR190Fljl1LTMLySyIj5lrPiWyb2VaoGvG55E0qZbEve3NLYd3uKJc10+pbD3gXGbPdkHf8pao2NFM17j9pCMOdnjdmzWx5d59V5HzM7FLgZNQ56B0k7vxxNh7wqasSt2gv5anoOhqPFAXaIaOtMyqIHIqaP9R0c8qt/yeiXM/n0KHjdZSD+La7v13CuOsg0f39UE7f34Gr3P2/qzSwysSUK78N8ojfhu69V1CqzdPu/r6lHt8VGNO7IeWBJ1wdvTI90hVRN51HvODq/m+Yy4/RQXKWu1+fHmuN8swLT7cogzAqE8movByFVoehPKLXUm7RCOSpG13LOZZFymnJhG97I4PieWRMjarVImZmPwHuRdV4ByGP5bNIdqJ0QWQz+ymSDZmFQhFLAIO9gLymxopJ4mcAqsCegUR/WySDphUy6CtZYMsiXVd/QKHlv6OQ43TkITGk2Te5QoOyJRI47ozyd9dH6gMTUXHUlKLnYSqO+VcyGi9BRjboEDUKpfoUnk9qZqsDpyG5qK2Q5uwMtJGuk35tg1qnl8z/2afUoD7IoBzp7h8WbOBnkbLOqNf8R+nxzVGO3bru3q2IsRobZrYF2n82RSoAH6N78KoKxm6OUi+WQaH4SagwbopJ7u0zL7mDTe6z3xPYDXlH33f3HdL98k6ZUbmiCaOSrzaZp1GoNesg0BudUh5F4biNvYQcvsZEMhraoM1tB7Torwjc4O6/r2gO2Q3WA1jO3W/LPdcCeQpLyS/JbxJpsZmLPBLbpu9LyeNsTKTX3QfljX2BjJpnkVbricA/3H1wVWHhMjF1RzoUSdj8Lwo9jnP3+2s8p17o3muN8iofcfdhBY9zFdpET3G1YTV0gDgMHayfdvf+RY6ZG3uxbKM2sy7IM/4+ymVbuejXujCY2a3Ig/Y0qlY/AKUFDHT3J0sY72ngAFflb/7xmueYFsmCDPLk3NgMFck94+43VnGwS/vevsBP0/jjkITPg8CbZXzO842faUPfijSYeyKZqt8nr34Ldz+jzDkUSRiVgJn9D5ILuj4ZNMuijbUXqv6+290fqOUcq8JSgUz6vi16H2Z5Ra2pcjfYWBR+vd7MjkF9YG9391sqmMPZwOooJPgiWlymFe2daMyY2Q6oGGEvdB+shLx5p6dT/CJrVKbwus/nifox8hb2RGkgu7oqLiuZT7rm90eRgY+Tcb8GKmSZVPQGm4yXPi65qh8ifdAPkadmGNrIXq43Y+a7YCom2o2GyvSpKEy/DdDdC+r1nfv8twVOc/feudBvKxSlubJsj1lV5F7vD5Axtx3KYZyAOliVHvLOzSV7n89DTqQH0aHqYOQ5/bO7n1PBPFqjiElP9D4McPfXTL3fL1iUoqRN3qg0dXM4z927p9P6LiiX71F0kj+hLM9YYyB3U62JxJ83QAvoY8BYr0GLOjNbCVVer59CAocjA2c94Dh3n17CmNlCtx3q2nMBSgNYA2gJfIT6vtatpzLnJT4YndDvzT3XBq0XjV4n7buwIOM4HSo+rMI7n7vuNkeV1tumQ+2xqHvH0KINeDPbB9jf3Xc2s5VRUVxHJNc1CLjXKyrKm29ejeLAlvtMBgB3JcPbgKXd/ZOyDG2TFvDynuRs0mM7I/3IHkWPVyty68zlKJ2gN7r+ZqMGIxfkI1RVzAele3XzeYtVb0fFUuNLHj+73k5EaSBLp3tzSyTvt2mZ4xdNs1pPoBEwAFV5gbxy+6Dqzz7IS3lEjeZVCbnT76ko1L0qOqkdAoxNC13VtAVeMrPdkYbeYORJXrMMgzKRbWY9UXuwq939TNRr/C7g+Xo2KGEeYe1fo97Xzc1siElDsnO9GZTQoA1nItPtnU51Av+Z/ugA1Psa9P5vBmxtZl1L8AgviTzwoHDuysAQd78VdbXp/nX/sUwag0GZkbzZB6CDLKijyidm1iEZRGXoxt4H7Gdmh5kUMEAh2etKGKtm5NaZTZH24wfImXMcqqwHdE9WNKVlkIzXT3JjN0f3xcSv+09FkDMomyPB9RWALU2FsyegCMIiRRiV0mTraFLSPwWFOIanC/8ddMHVJWa2opn1SBd0Z3c/DQn8DkG9td+nxI4m38BktLH/HoXAX0RtAUvtrpIWsWWBHcysU7rhp7n7De5+eVljNwayBdzU8/4ld5+GvFYbALcDB+aMrrrDRXbAuhIl71cxbrbBrgNMMbOLUKHCQFQNvjEUvsHeC+xhZi8jPb6rUSQAVHE8No3ZvMAxFxncfW4y5K9FhiWAm+SerjCzVmUYwO7+LPAbpHDxWzN7Ca3BdWVUwlcFSM8i3d813f1Zd78LddW5B6o7ZLiUTkYCZ5jZCDP7BSqUe8nd/1XWuMkzP9ckq3WdqzhtdyTjdw0qCj2vrPHLom43ie/AOLRwPIVaAQ7xho4CO9BQuFOPHI5Cu/8EJqfT8eLJ3T/ezAaiG79SXJVuP8t+TuHAvujkViZroZPySsiIfdnMJqHF5R8lj11Tcgv4YsBrJg25Diinclugf5W5TrXEKxb6T1yCru/ZqHjmI1PHof8oeiB3fzP97T4oxD4OvpLR6o5khgpvCbkoYGbXouK04S7R9w3MbBB6rwzl3ZdZiXsDUptogaSVXvSKuphVzFtIo3Ip4Bkz64s62P0geYQrXWfcfYSZPYUiY11pSIErk61NahubkiIHLsWFx01NPwoXuq+CJm9Uuvu7QN+UxzcnLeaLI52wll5BZ4Ea0he13ptqZqcBiwMfp5PaD5CUwexv/AslYGYtXBWp26Fq7+FmtmvZ4Vd3fyXlm22M8ny2QN1dbkInx6bAeOQl6wD8p7t/mnIsh6fnmwFNztgog2zjNLM13f0OU8Hge66WpIOAV116fYUXRaWD83/l5tIarQd3ucTuF9lCrO+LqShmIirE2cvMXkX3/4vAIcmTWCrpelgDFebUozGZqXhsC4xInrpX0aHqARpaEVa2zliDGsGnyGP4WhXjorB/J7TePmFmnwD/7e6vII/1uyjta5GiyRfqLAhT/9veqKvGqFrPpwxMBUoPA9u5+6O5xw9BFY9zgLPcfUIN5pYlco8Abnb3mysadzm0sMxJi90yqPLzhXSjNxnMrKWrRdmmwNlA76bouSqbtMFehwrBHkA9jyeb9OlauSqwSzfwUni9FWqT+GlT8EgviPR5tEJaod2R6sSaKD1glJekgmHzVoAPdfcN6+0zSGHe95FsVW933yP3XHtgbnLyVDkng6+M+cuQZ/i8qg5VZrYe0sy9GUkprY1UGNohdYZpZc+haMKobKKYJBR6ozDLUihv7mJ3n2Rmi1d5Ss55bFpmnlFTtfFTwNplGjM5A3ZHVKTVGyXMP4SEsV9196qKNmpOyhf7wtQK8xF3f8rM2rr7u03Re1U26eDSH8h62i+L8rrHZWHpoBpy69BWqLPLYqh4cR0kffOyu59dwrgtsjUmpZ286+6X5jxodUFaY3+DikEvQ0WQrdz9nylftbm731jBPLLPeX1UWX9oMi5fBzbzGnXOS0b3KuhA85FXWAFfJGFUNlHM7FGUJzfLJFsxAOmFvQ6MRuLXn1ZxUs6d0k8G7nH3ialIoJO7P1/Fid3MHke5tecincKNkMfiqHr1Vi+I3IL7DLCbu0+p9ZyaAin8vAa6D/sCF7n7ZfXmrWqs5K77dYFbUbOLL9Jz2fpUmJGXG28d4FR3PyA9vh7q5LbIdFD5NqT0svdRp64LUS7/XLTXjEHFcce4+91lX/O5z/OXqP3yb9PhbruUhlLZ4Tntc3PTtbAHSj/5ZxVjl0VUfzdBTC0pv0wGZTN3H+nue6P8jkuRG75jhZuZmSqLV0lzAN1oz5taeJWS+5uFPtKJ9XNUdd7W3XdH8hLPATVtF1c2aVHLwk9ZGKgn0mmckr1HQfGY2ZJm1s3MlnL3T9z9GXf/Jcrhq1lXnyZKthcehHL9vki59QCbmtkBRXoNc2vrbKC1ma2QjKlJwIrJuKwn/oTyJScBP3H39ZHjojtS+bjB3e+G8qu+cwbjj4C1TDJRH7v7HfM9XwWe1txOwJnJa7tIr7lNvlCnKeLuD5mKYKBBJw+X8OuV6asS0kL6Zfp+OHC8mV0PtDFpZPYBelCCbmBu8eoAXIG8k5n4bWuSYVX0uI2JXGrBTWY2GVUbP0SDPmsU5hSMqVPL4sgruSvwrEne5xGkl7p2lsMbXspqyN0H7YER6bEsBWgg8rIVQjIaVkVh7qlm9i7Q1d1HmdlxwM5I9mlSUWM2AjZFOYIfmFlLM7sP5RHfB1zh7m9CdQL4KdTcCkmm/dGk8vEY8IS7f1L2+Bk5A7Y/kvGDRXzNDaOyiZLlLpaZr/gtedzMZqJ8zlHJS3YR8lpORSGRsvM7n0WFSQ68Z2b/gaoTS+352sjYHzgNtQc7F3gZGsX1UY8MQTl6n6NUiwnoQLM7Ul24Duqv3/MiwhjgVDP7AHgTfT49kHZgUeyAhL7HmNkElLt9elp3pgDH0yBOv8hjUtR409WZqANwEiqIvBZp4X5uZme69EHLTnPqDkx09+lmdjSKjPVAXsutUJvGv5Q5h9xc2iL5ujdR/UAWFVukD5JhVAa15lgkuHyhSfi5A1pk9/AS9QJzBTpdgR7u/sfkQRiHCnZuoaHDSd3j7tPM7ArgHDPrhSRUZkROX7GkDXZt1JauB+re9TDS7WuDjIpZ6dejKKp6RiED/yQUHWkP3OTubxQ4xtNIuqsnsFMa539RLudl9RACnY98B6f9aejg9FAy3n9ZRcjZpO7R190fMLMTUHrTk+5+jpmtiKSkppY8h2zf2Q/oBhyFWkGeYFIeqDr8XjhRqBPUDDNb0efttdofeQR2RILs13hJPYhTXukc1AJyYlpYsuT5pVACd02qAKvGzJZLYamVUTrEUBTuO6HKUFBTIKV43Ozut6SfT0NR7tL7jAf/P9YgpbU5sDwyhv5RYIFOS5TPnqX8bI5ky3oBX6DDxZkFG7E1xdRU4z4U0m0NHAmM8Yb+38+7etyX7plPBUMtUO3AZ+gA9wLq5FNZK14zG4+klX4FPOzuV5nZmcB4dx9bxRzKIozKoGaY+pv+C2l0/VfmmTTJCR2ENtuhJY29DXA5knEZhlrVvZa8cyOQQTu6jLEbE2a2Ecohew+FYbZBBUsroM4iB7u6PAQLiakz1D+AA4EHXDJN9wCnu/uEKqtOA5GrBF4P5W93RKLTDwNPu/vHBY93NHAicBVwhydBdVOv8b7oQP0Lr0HTiTJJa/qCOjjdC2yTdy5UNJ9WwGYozakTCkMPd/e/VTD2WsDl7r6dqeFBt+Sdfhw4wt3/p+w5lEkYlUFNSNXeW6A8lt4oJPI8MnBGlRlyNbOfoBDUCjS04ewNvIZac+2NJEUqObXWEjP7EUqDWQ61KZ2BtPk+Ai5AOp1DajfD+iFV2vdH3vGVUPhtY2BDb0JaqI2JnFF5G8qh3heYhkK2nwBXe4HNF9I1MAC1P+0MvALcAYz0RVDo+vtiktDaC137P69YxqcmTS5y11oHtO/MAX7o7geb2ZbA2e7erazxqyKMyqCmpBNjG7TA7oCMzBWRxEQpIcF0OjwX9fH9MnmQ+qCF5T3gbnd/oIyxFwVyi9/VwAyXzE1QIGa2PKryPR5dcxOR96Ku1QYaI2a2AgrHbmnS7x2EPIa9gCPd/eWSxm2PvNa7ozVwKnCgV9xVphaknNHKOjhZI2pykV77McAJqEhvCoqYPenuZ1UxhzIJozKoOWa2TBZmShVxfYBZXkJLNFN7yvPcvXu6uXcBbkAeypkoj/CtosdtTOQW2K2RHudBwFhkyN+X+712SAA/8ipLxMw6IuNyjLvfGcVR1WJmA5HHeBhaG3qZ2dIo97VfSWO2cfVfz37eCNjb3U8uY7xAWI2aXJhZP1SY89fs4GhmuwCbo8rzK4AH6yFiEUZlUDmWOlOY2ZrIqNkAJU0/Box198I04RYw9jlIf/I/zWx74GAk53AOqUDF3U8va/zGQK4g6X4kn/Ez5B1eF2mkHeruw/O/W7vZBkG5pEhFG5SScAZwDarMnuPuxxY4TnbfHYb0SVdD7XGHuXupVcdNmdz7vj4q0OkBPO7um5hEx88Fji4zSmBmO6ECpXbAdFRHcHM9plhFR52gcnKVlKciY2ZV5P4/BBhrEj0vi6eAjinkdQoKOQ1PVYfvAMuUOHajIC2wqwBLpGKkFd19c2Rgj0OFOplHMwzKoK5x9w/d/Q13fwKlIQxEm/9lBY/jZtYZGRdnAG2B7YHJZvZSWpOCgsmtYbVscjEGXVeDUMj7p8AEMxtm6nteN4ROZVApSQ+sMxKZ7ezuB5m6K/wBWAvYE+W5lMU4FP54ChWmDMmFoXagoXCn3tkCddHZCPgwPfY86vLxJITweVC/5FJAVkcFI5sCtwHnIzmx6QWPt1qSCToIVX63B25x90FmdiFAmRGaAKhhk4u0ls5KX4+b2SUo5WJH4Bgzu8PrpN97GJVB1RwOtEQ6lJOThtni7j4eGJ/ym54ta/CUBN836ZXNcfePTD1+9wBauvvTZY3dyHgYabM1AzCzA4CuuZ+jm0tQz2SVxrehg+YM4HfAb9HBtjCj0iRqfbqpucBnqJPM0DQuqLNSky0MLBNrBE0ucoWPWwKHoq5Zt6BC0fvMrG29GJQQRmVQPX2R9uFUk/Dz4sDHZvYLdLO9U4VGm88rbL4EMBttKk2CzBOT9PHuAn6PpE3OyH6lRlMLglIxs6WTLuAawHR3Pyn33OnAHmb2QoGpH21Qx5xbUKXvTcD9wL5mNh3JGF1Y0FjBvGxtZnNQW8yJ2YMusfERyCtdqBbp/CSDsj3q7X0u2udOBk4xs971VhgaOZVBZaTK6y1RLhHu/mrSBRuBEuPXReGnSnH3D9z95jKr/xoDyYCcB1e/3bNRvk8nd5+RPV71/IKgIi40s7tQN5P5oyIvAT2LzCV29/fc/SjUkvEtZFCADM3fAX9y99eLGi/4N4Yh2aa1k+LFSunxq9B+VBpJzQSUbjTa3S9w99PcvRMwGkXI6orwVAZVMhAJjF9taoV4O3Cxuw8zsxvc/fPaTq9+SQK/V6TT+UOZFl4uzH0U8p5E6DuoW1Lo8wqUS7cJcKiZrYuE/l9DXVYuLXi8JVG9yFOp8vsUYDdk7IyJe60crKHJxb4oV74ZcD3wWtIj3RBpVZbJzWb2GUp7mDjfc3OQI6WuCEmhoDLSjdzf3WeZ2c6os8R2wOvo1PYXpIsYF2XBmDp57IaMx7ZoMb0dLXT/QlXwm7n7zJARCuqd1HRhJWB9oB+KlHRGecbdi8pxS2k926HWj8uide494Og07p3AThEZKB6rcZMLa+gatwVqf7sbMBLl1L6N1uKz3f2FsuZQC8KoDCrB1Gt7iLv/2HItuVI1+C7oJDnQ3Z+r5TybAqZuLnsBx6IChdmos0Ufi/7TQRMjefGXR95L3P3qAv/2CUiq62J0cOuFvFbvoHvwTXc/qKjxAtGYmlyY2RLA0kAXpMu8A/JQ3urue1YxhyoJozKoDDNr6e6zI7zaeEjdXI5EIfHR8dkETYG8N97MfgVc4O6fZY0ZChxnCVRlPBAZNOejg1zz1AAi7rcSaGxNLvIdlJIeaT/gbXe/v6o5VEUYlUEQBEGTIifz0h043903Lnm8dkhOrS0q2Bhb5nhNHTMbgLzCJwK3Igm1S9x9mpmdiho+lNZk4xu6xk0E7qxnTdIwKoMgCIImg5m18NRj2czOQIL/l5bgpTwV+AIYj0Sv2yHjojOSFhqKtHJjEy6YVHV9DQozv4HyVjNP4UPAMVVoEpvZ34BpqMp8DuozvgzK8zyv7PFrQVR/B0EQBHVNrv/zOqg97AHpqZtR1TcFG5SrAx2RJuFJqAp5Bqo4Xyf92th6K9JoLNSyyUUj6BpXU8KoDIIgCOqanDdwNtA65bXNcvdJZvbDlPM2qcDxXjezIzND1cy6IGmh94EfASuHQVk+NWpyUdOucbUmjMogCIKgbknVv6uiMPdUM3sX6Oruo8zsOGBn4F6gMKMS5vV8unveiJhc5DjBt8PdP0Ce6bJpFF3jakXkVAZBEAR1i5n1Q236xgATgLWB45HXagpwFvBilmdZwXxCB7ZOSVJGDwPbufujuccPQTqVc4Cz3H1CjaZYOmFUBkEQBHVL6rt8ICqWWAq1R1wO9eK+LPUBD0MvWGjM7DygN9ACXWtZ17hJZrZ4U+gaF0ZlEARBUJeYWUvgy0wL0sw2R1IzvVBl9lvAme7+Ru1mGdQL0TVOvTCDIAiCoB45DBVLnGZmXdz9cXf/E8p7uwj4EnW3CYKFInWN+zIZlM3cfaS77w10Qv3kdwU61rNBCeGpDIIgCOqU1PN+AGqJ2Bl4BbgDGOnu02o5t6D+iK5xYVQGQRAETYBcbuXuQBvUi/vApGkYBEEBhFEZBEEQ1DX53svp542Avd395BpOKwjqjjAqgyAIgroj10XnMJTPthqqxh3m7lNrO7sgqE+iUCcIgiCoO5JB2Rk4EjgDaAtsjwp3XkpddYIgKJAwKoMgCIK6wsxWS98eBFwFtAducfctUSXuPe7+fo2mFwR1SxiVQRAEQd1gZi2A081sK+Az4FpgD+DB9CufA3fXaHpBUNeEURkEQRDUE21Q15xbkPD0qsD9wL5m1hPYF3j26/97EATflyjUCYIgCOqO1If5l8Bc4C5gF6AdcKO7X1jLuQVBvRJGZRAEQVAXmJkBS6I6nc/MbCngFNTVZBgwpqmKUgdBFSxW6wkEQRAEQUEMRiHvd81sWdRv+WFgQ9RJ504z28nd59ZwjkFQt4RRGQRBENQLzYA1gLGoY04vVKwzGlgCmBkGZRCUR4S/gyAIgrrAzJYA9gEGAo8C5wMzgObuPqcp92QOgioIozIIgiCoK8ysHXA4Ejwf7e5jazylIGgShFEZBEEQLPKY2anAF8B4YBaq9N4A6AxMAYYCczw2vSAojcipDIIgCBZpzGx1oCPwA+Ak4GkU9t4EWCf92lh3f6EW8wuCpkJ4KoMgCIJFHjNbzN3npO+7IGmh94EfASu7+7Bazi8ImgJhVAZBEARBEAQLTbRpDIIgCOqWJIgeBEEFhKcyCIIgCIIgWGjCUxkEQRAEQRAsNGFUBkEQBEEQBAtNGJVBEARBEATBQhNGZRAEQRAEQbDQhFEZBEEQBEEQLDRhVAZBEARBEAQLTRiVQRAEQRAEwULzf6WV+hNgIEr/AAAAAElFTkSuQmCC",
"text/plain": [
"<Figure size 800x800 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plt.figure(figsize=(10, 10), dpi=80)\n",
"\n",
"unique, counts = np.unique(labels, return_counts=True)\n",
"plt.bar(CLASS_LABELS[unique], counts)\n",
"plt.xticks(rotation=70)\n",
"\n",
"unique, counts = np.unique(y_train, return_counts=True)\n",
"plt.bar(CLASS_LABELS[unique], counts)\n",
"plt.xticks(rotation=70)\n",
"\n",
"unique, counts = np.unique(y_test, return_counts=True)\n",
"plt.bar(CLASS_LABELS[unique], counts)\n",
"plt.xticks(rotation=70)\n",
"\n",
"unique, counts = np.unique(y_val, return_counts=True)\n",
"plt.bar(CLASS_LABELS[unique], counts)\n",
"plt.xticks(rotation=70)\n",
"\n",
"plt.legend([\"All\", \"Train\", \"Test\", \"Validation\"])\n",
"\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "QaZxGMgKZznS"
},
"outputs": [],
"source": [
"def train(config=None):\n",
" with wandb.init(config=config):\n",
" config = wandb.config\n",
" \n",
" # Generate new model\n",
" model = Transformer(\n",
" num_layers=config.num_layers,\n",
" embed_dim=config.embed_layer_size,\n",
" mlp_dim=config.fc_layer_size,\n",
" num_heads=config.num_heads,\n",
" num_classes=18,\n",
" dropout_rate=config.dropout,\n",
" attention_dropout_rate=config.attention_dropout,\n",
" )\n",
"\n",
" # adapt on training dataset - must be before model.compile !!!\n",
" model.input_norm.adapt(X_train, batch_size=config.batch_size)\n",
" print(model.input_norm.variables)\n",
"\n",
" # Select optimizer\n",
" if config.optimizer == \"adam\":\n",
" optim = Adam(\n",
" global_clipnorm=config.global_clipnorm,\n",
" amsgrad=config.amsgrad,\n",
" )\n",
" elif config.optimizer == \"adamw\":\n",
" optim = AdamW(\n",
" weight_decay=config.weight_decay,\n",
" amsgrad=config.amsgrad,\n",
" global_clipnorm=config.global_clipnorm,\n",
" exclude_from_weight_decay=[\"position\"]\n",
" )\n",
" else:\n",
" raise ValueError(\"The used optimizer is not in list of available\")\n",
"\n",
" model.compile(\n",
" loss=smoothed_sparse_categorical_crossentropy(label_smoothing=config.label_smoothing),\n",
" optimizer=optim,\n",
" metrics=[\"accuracy\"],\n",
" )\n",
"\n",
" # Train model\n",
" model.fit(\n",
" X_train,\n",
" y_train,\n",
" batch_size=config.batch_size,\n",
" epochs=config.epochs,\n",
" validation_data=(X_val, y_val),\n",
" callbacks=[\n",
" LearningRateScheduler(cosine_schedule(base_lr=config.learning_rate, total_steps=config.epochs, warmup_steps=config.warmup_steps)),\n",
" PrintLR(),\n",
" WandbCallback(monitor=\"val_accuracy\", mode='max', save_weights_only=True),\n",
" EarlyStopping(monitor=\"val_accuracy\", mode='max', min_delta=0.001, patience=5),\n",
" ],\n",
" verbose=1\n",
" )\n",
"\n",
" model.summary()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000,
"referenced_widgets": [
"c2f96abecad54565be62d18c1b5c1e68",
"fa0aba1429524429af176534638122db",
"0054c6582b4c45faa323add90dd34770",
"634c65b48e0b40359f6158364fb54ad7",
"ebdfa2b37e5540e39bd6624f22eeb19a",
"7198820fcadc4aaf875ee617dd605fbc",
"18adf181e391491a9426d17e69a8c574",
"dc1f7e46eaf643999d482c3b17e3bee6"
]
},
"id": "J743-OTSSsZy",
"outputId": "1ea18bc2-7308-4e10-cabf-7c94643fab4a"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: lwikvs2y with config:\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tamsgrad: False\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tattention_dropout: 0.1\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tbatch_size: 64\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tdropout: 0.1\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tembed_layer_size: 128\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tepochs: 50\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tfc_layer_size: 256\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tglobal_clipnorm: 3\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tlabel_smoothing: 0.1\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tlearning_rate: 0.001\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tnum_heads: 6\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \tnum_layers: 3\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \toptimizer: adam\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \twarmup_steps: 10\n"
]
},
{
"data": {
"text/html": [
"\n",
" Syncing run <strong><a href=\"https://wandb.ai/markub/imu-transformer/runs/lwikvs2y\" target=\"_blank\">earnest-sweep-1</a></strong> to <a href=\"https://wandb.ai/markub/imu-transformer\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">docs</a>).<br/>\n",
"Sweep page: <a href=\"https://wandb.ai/markub/imu-transformer/sweeps/cfdl7wcr\" target=\"_blank\">https://wandb.ai/markub/imu-transformer/sweeps/cfdl7wcr</a><br/>\n",
"\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[<tf.Variable 'mean:0' shape=(6,) dtype=float32, numpy=\n",
"array([ 0.10979815, -0.07648689, -0.08781412, 0.03063406, 0.00979011,\n",
" 0.00650088], dtype=float32)>, <tf.Variable 'variance:0' shape=(6,) dtype=float32, numpy=\n",
"array([28.058855 , 9.622038 , 16.08444 , 3.0307784, 1.9513103,\n",
" 2.0505407], dtype=float32)>, <tf.Variable 'count:0' shape=() dtype=int64, numpy=18018000>]\n",
"Epoch 1/50\n",
" 6/939 [..............................] - ETA: 3:08 - loss: 2.8825 - accuracy: 0.1228WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0914s vs `on_train_batch_end` time: 0.0915s). Check your callbacks.\n",
"939/939 [==============================] - 226s 234ms/step - loss: 1.8975 - accuracy: 0.4512 - val_loss: 1.4693 - val_accuracy: 0.6309 - lr: 1.0000e-04\n",
"Epoch 2/50\n",
"939/939 [==============================] - 224s 239ms/step - loss: 1.2502 - accuracy: 0.7161 - val_loss: 1.0970 - val_accuracy: 0.7766 - lr: 2.0000e-04\n",
"Epoch 3/50\n",
"939/939 [==============================] - 224s 238ms/step - loss: 1.0634 - accuracy: 0.7823 - val_loss: 1.0177 - val_accuracy: 0.8070 - lr: 3.0000e-04\n",
"Epoch 4/50\n",
"939/939 [==============================] - 225s 240ms/step - loss: 0.9973 - accuracy: 0.8063 - val_loss: 0.9463 - val_accuracy: 0.8246 - lr: 4.0000e-04\n",
"Epoch 5/50\n",
"939/939 [==============================] - 224s 239ms/step - loss: 0.9527 - accuracy: 0.8252 - val_loss: 0.9526 - val_accuracy: 0.8252 - lr: 5.0000e-04\n",
"Epoch 6/50\n",
"939/939 [==============================] - 233s 248ms/step - loss: 0.9277 - accuracy: 0.8355 - val_loss: 0.9304 - val_accuracy: 0.8317 - lr: 6.0000e-04\n",
"Epoch 7/50\n",
"939/939 [==============================] - 226s 240ms/step - loss: 0.9065 - accuracy: 0.8444 - val_loss: 0.8776 - val_accuracy: 0.8602 - lr: 7.0000e-04\n",
"Epoch 8/50\n",
"939/939 [==============================] - 224s 239ms/step - loss: 0.8888 - accuracy: 0.8529 - val_loss: 0.8554 - val_accuracy: 0.8703 - lr: 8.0000e-04\n",
"Epoch 9/50\n",
"939/939 [==============================] - 224s 239ms/step - loss: 0.8734 - accuracy: 0.8596 - val_loss: 0.9027 - val_accuracy: 0.8493 - lr: 9.0000e-04\n",
"Epoch 10/50\n",
"939/939 [==============================] - 232s 247ms/step - loss: 0.8616 - accuracy: 0.8657 - val_loss: 0.8845 - val_accuracy: 0.8542 - lr: 0.0010\n",
"Epoch 11/50\n",
"939/939 [==============================] - 227s 242ms/step - loss: 0.8363 - accuracy: 0.8779 - val_loss: 0.8222 - val_accuracy: 0.8856 - lr: 9.9846e-04\n",
"Epoch 12/50\n",
"939/939 [==============================] - 232s 248ms/step - loss: 0.8288 - accuracy: 0.8815 - val_loss: 0.8512 - val_accuracy: 0.8751 - lr: 9.9384e-04\n",
"Epoch 13/50\n",
"939/939 [==============================] - 234s 249ms/step - loss: 0.8128 - accuracy: 0.8888 - val_loss: 0.8171 - val_accuracy: 0.8859 - lr: 9.8619e-04\n",
"Epoch 14/50\n",
"939/939 [==============================] - 226s 241ms/step - loss: 0.8014 - accuracy: 0.8944 - val_loss: 0.7949 - val_accuracy: 0.8972 - lr: 9.7553e-04\n",
"Epoch 15/50\n",
"939/939 [==============================] - 232s 247ms/step - loss: 0.7910 - accuracy: 0.8990 - val_loss: 0.8334 - val_accuracy: 0.8826 - lr: 9.6194e-04\n",
"Epoch 16/50\n",
"939/939 [==============================] - 226s 241ms/step - loss: 0.7824 - accuracy: 0.9037 - val_loss: 0.8004 - val_accuracy: 0.8956 - lr: 9.4550e-04\n",
"Epoch 17/50\n",
"939/939 [==============================] - 225s 239ms/step - loss: 0.7732 - accuracy: 0.9078 - val_loss: 0.7767 - val_accuracy: 0.9084 - lr: 9.2632e-04\n",
"Epoch 18/50\n",
"939/939 [==============================] - 232s 247ms/step - loss: 0.7614 - accuracy: 0.9136 - val_loss: 0.7578 - val_accuracy: 0.9181 - lr: 9.0451e-04\n",
"Epoch 19/50\n",
"939/939 [==============================] - 226s 241ms/step - loss: 0.7565 - accuracy: 0.9163 - val_loss: 0.7581 - val_accuracy: 0.9151 - lr: 8.8020e-04\n",
"Epoch 20/50\n",
"939/939 [==============================] - 225s 240ms/step - loss: 0.7475 - accuracy: 0.9202 - val_loss: 0.7378 - val_accuracy: 0.9265 - lr: 8.5355e-04\n",
"Epoch 21/50\n",
"939/939 [==============================] - 225s 239ms/step - loss: 0.7413 - accuracy: 0.9231 - val_loss: 0.7479 - val_accuracy: 0.9215 - lr: 8.2472e-04\n",
"Epoch 22/50\n",
"939/939 [==============================] - 225s 240ms/step - loss: 0.7364 - accuracy: 0.9255 - val_loss: 0.7346 - val_accuracy: 0.9281 - lr: 7.9389e-04\n",
"Epoch 23/50\n",
"939/939 [==============================] - 225s 240ms/step - loss: 0.7311 - accuracy: 0.9279 - val_loss: 0.7497 - val_accuracy: 0.9220 - lr: 7.6125e-04\n",
"Epoch 24/50\n",
"939/939 [==============================] - 225s 240ms/step - loss: 0.7251 - accuracy: 0.9307 - val_loss: 0.7317 - val_accuracy: 0.9298 - lr: 7.2700e-04\n",
"Epoch 25/50\n",
"939/939 [==============================] - 224s 238ms/step - loss: 0.7216 - accuracy: 0.9324 - val_loss: 0.7182 - val_accuracy: 0.9356 - lr: 6.9134e-04\n",
"Epoch 26/50\n",
"939/939 [==============================] - 226s 241ms/step - loss: 0.7163 - accuracy: 0.9348 - val_loss: 0.7221 - val_accuracy: 0.9340 - lr: 6.5451e-04\n",
"Epoch 27/50\n",
"939/939 [==============================] - 233s 249ms/step - loss: 0.7107 - accuracy: 0.9373 - val_loss: 0.7117 - val_accuracy: 0.9390 - lr: 6.1672e-04\n",
"Epoch 28/50\n",
"939/939 [==============================] - 227s 242ms/step - loss: 0.7077 - accuracy: 0.9391 - val_loss: 0.7110 - val_accuracy: 0.9397 - lr: 5.7822e-04\n",
"Epoch 29/50\n",
"939/939 [==============================] - 225s 240ms/step - loss: 0.7030 - accuracy: 0.9409 - val_loss: 0.7051 - val_accuracy: 0.9416 - lr: 5.3923e-04\n",
"Epoch 30/50\n",
"939/939 [==============================] - 225s 240ms/step - loss: 0.6987 - accuracy: 0.9429 - val_loss: 0.6998 - val_accuracy: 0.9432 - lr: 5.0000e-04\n",
"Epoch 31/50\n",
"939/939 [==============================] - 233s 248ms/step - loss: 0.6951 - accuracy: 0.9448 - val_loss: 0.6992 - val_accuracy: 0.9447 - lr: 4.6077e-04\n",
"Epoch 32/50\n",
"939/939 [==============================] - 227s 241ms/step - loss: 0.6931 - accuracy: 0.9459 - val_loss: 0.6999 - val_accuracy: 0.9443 - lr: 4.2178e-04\n",
"Epoch 33/50\n",
"939/939 [==============================] - 225s 239ms/step - loss: 0.6892 - accuracy: 0.9474 - val_loss: 0.6952 - val_accuracy: 0.9458 - lr: 3.8328e-04\n",
"Epoch 34/50\n",
"939/939 [==============================] - 224s 239ms/step - loss: 0.6837 - accuracy: 0.9505 - val_loss: 0.6854 - val_accuracy: 0.9508 - lr: 3.4549e-04\n",
"Epoch 35/50\n",
"939/939 [==============================] - 232s 247ms/step - loss: 0.6752 - accuracy: 0.9549 - val_loss: 0.6496 - val_accuracy: 0.9714 - lr: 3.0866e-04\n",
"Epoch 36/50\n",
"939/939 [==============================] - 234s 249ms/step - loss: 0.6197 - accuracy: 0.9840 - val_loss: 0.6156 - val_accuracy: 0.9858 - lr: 2.7300e-04\n",
"Epoch 37/50\n",
"939/939 [==============================] - 234s 249ms/step - loss: 0.6016 - accuracy: 0.9919 - val_loss: 0.6100 - val_accuracy: 0.9891 - lr: 2.3875e-04\n",
"Epoch 38/50\n",
"939/939 [==============================] - 226s 241ms/step - loss: 0.5956 - accuracy: 0.9943 - val_loss: 0.6126 - val_accuracy: 0.9877 - lr: 2.0611e-04\n",
"Epoch 39/50\n",
"939/939 [==============================] - 233s 248ms/step - loss: 0.5924 - accuracy: 0.9957 - val_loss: 0.6096 - val_accuracy: 0.9894 - lr: 1.7528e-04\n",
"Epoch 40/50\n",
"939/939 [==============================] - 227s 241ms/step - loss: 0.5902 - accuracy: 0.9965 - val_loss: 0.6141 - val_accuracy: 0.9880 - lr: 1.4645e-04\n",
"Epoch 41/50\n",
"939/939 [==============================] - 225s 240ms/step - loss: 0.5881 - accuracy: 0.9973 - val_loss: 0.6074 - val_accuracy: 0.9908 - lr: 1.1980e-04\n",
"Epoch 42/50\n",
"939/939 [==============================] - 226s 240ms/step - loss: 0.5868 - accuracy: 0.9979 - val_loss: 0.6050 - val_accuracy: 0.9915 - lr: 9.5491e-05\n",
"Epoch 43/50\n",
"939/939 [==============================] - 233s 248ms/step - loss: 0.5866 - accuracy: 0.9979 - val_loss: 0.6042 - val_accuracy: 0.9914 - lr: 7.3680e-05\n",
"Epoch 44/50\n",
"939/939 [==============================] - 227s 241ms/step - loss: 0.5851 - accuracy: 0.9986 - val_loss: 0.6060 - val_accuracy: 0.9910 - lr: 5.4497e-05\n",
"Epoch 45/50\n",
"939/939 [==============================] - 226s 240ms/step - loss: 0.5845 - accuracy: 0.9988 - val_loss: 0.6055 - val_accuracy: 0.9914 - lr: 3.8060e-05\n",
"Epoch 46/50\n",
"939/939 [==============================] - 225s 239ms/step - loss: 0.5837 - accuracy: 0.9991 - val_loss: 0.6056 - val_accuracy: 0.9918 - lr: 2.4472e-05\n",
"Model: \"transformer\"\n",
"_________________________________________________________________\n",
" Layer (type) Output Shape Param # \n",
"=================================================================\n",
" normalization (Normalizatio multiple 13 \n",
" n) \n",
" \n",
" positional_embedding (Posit multiple 39296 \n",
" ionalEmbedding) \n",
" \n",
" encoder (Encoder) multiple 462080 \n",
" \n",
" encoder_1 (Encoder) multiple 462080 \n",
" \n",
" encoder_2 (Encoder) multiple 462080 \n",
" \n",
" layer_normalization_6 (Laye multiple 256 \n",
" rNormalization) \n",
" \n",
" dense_7 (Dense) multiple 2322 \n",
" \n",
"=================================================================\n",
"Total params: 1,428,127\n",
"Trainable params: 1,428,114\n",
"Non-trainable params: 13\n",
"_________________________________________________________________\n"
]
},
{
"data": {
"text/html": [
"<br/>Waiting for W&B process to finish, PID 498... <strong style=\"color:green\">(success).</strong>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c2f96abecad54565be62d18c1b5c1e68",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(Label(value=' 5.52MB of 5.52MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<style>\n",
" table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
" .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; width: 100% }\n",
" .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
" </style>\n",
"<div class=\"wandb-row\"><div class=\"wandb-col\">\n",
"<h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>accuracy</td><td>▁▄▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇█████████</td></tr><tr><td>epoch</td><td>▁▁▁▁▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇████</td></tr><tr><td>loss</td><td>█▅▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>lr</td><td>▁▂▃▄▄▅▆▇██████▇▇▇▇▇▇▆▆▅▅▅▅▄▄▄▃▃▃▂▂▂▂▁▁▁▁</td></tr><tr><td>val_accuracy</td><td>▁▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇██████████</td></tr><tr><td>val_loss</td><td>█▅▄▄▄▄▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁</td></tr></table><br/></div><div class=\"wandb-col\">\n",
"<h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>accuracy</td><td>0.99912</td></tr><tr><td>best_epoch</td><td>45</td></tr><tr><td>best_val_accuracy</td><td>0.99177</td></tr><tr><td>epoch</td><td>45</td></tr><tr><td>loss</td><td>0.58374</td></tr><tr><td>lr</td><td>2e-05</td></tr><tr><td>val_accuracy</td><td>0.99177</td></tr><tr><td>val_loss</td><td>0.60557</td></tr></table>\n",
"</div></div>\n",
"Synced 5 W&B file(s), 1 media file(s), 0 artifact file(s) and 1 other file(s)\n",
"<br/>Synced <strong style=\"color:#cdcd00\">earnest-sweep-1</strong>: <a href=\"https://wandb.ai/markub/imu-transformer/runs/lwikvs2y\" target=\"_blank\">https://wandb.ai/markub/imu-transformer/runs/lwikvs2y</a><br/>\n",
"Find logs at: <code>./wandb/run-20220201_003202-lwikvs2y/logs</code><br/>\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Sweep Agent: Waiting for job.\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: Sweep Agent: Exiting.\n"
]
}
],
"source": [
"wandb.agent(sweep_id, train, count=32)"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"collapsed_sections": [],
"name": "Training_posledna_verzia_3-3.ipynb",
"provenance": []
},
"interpreter": {
"hash": "9185113d2128201d66faecd4f34fb34e89a635073a034991399523e584519355"
},
"kernelspec": {
"display_name": "Python 3.9.6 64-bit ('base': conda)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
},
"orig_nbformat": 4,
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"0054c6582b4c45faa323add90dd34770": {
"model_module": "@jupyter-widgets/controls",
"model_module_version": "1.5.0",
"model_name": "LabelModel",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "LabelModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "LabelView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_7198820fcadc4aaf875ee617dd605fbc",
"placeholder": "",
"style": "IPY_MODEL_ebdfa2b37e5540e39bd6624f22eeb19a",
"value": " 5.55MB of 5.55MB uploaded (0.00MB deduped)\r"
}
},
"18adf181e391491a9426d17e69a8c574": {
"model_module": "@jupyter-widgets/controls",
"model_module_version": "1.5.0",
"model_name": "ProgressStyleModel",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"634c65b48e0b40359f6158364fb54ad7": {
"model_module": "@jupyter-widgets/controls",
"model_module_version": "1.5.0",
"model_name": "FloatProgressModel",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_dc1f7e46eaf643999d482c3b17e3bee6",
"max": 1,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_18adf181e391491a9426d17e69a8c574",
"value": 1
}
},
"7198820fcadc4aaf875ee617dd605fbc": {
"model_module": "@jupyter-widgets/base",
"model_module_version": "1.2.0",
"model_name": "LayoutModel",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"c2f96abecad54565be62d18c1b5c1e68": {
"model_module": "@jupyter-widgets/controls",
"model_module_version": "1.5.0",
"model_name": "VBoxModel",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "VBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "VBoxView",
"box_style": "",
"children": [
"IPY_MODEL_0054c6582b4c45faa323add90dd34770",
"IPY_MODEL_634c65b48e0b40359f6158364fb54ad7"
],
"layout": "IPY_MODEL_fa0aba1429524429af176534638122db"
}
},
"dc1f7e46eaf643999d482c3b17e3bee6": {
"model_module": "@jupyter-widgets/base",
"model_module_version": "1.2.0",
"model_name": "LayoutModel",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"ebdfa2b37e5540e39bd6624f22eeb19a": {
"model_module": "@jupyter-widgets/controls",
"model_module_version": "1.5.0",
"model_name": "DescriptionStyleModel",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"fa0aba1429524429af176534638122db": {
"model_module": "@jupyter-widgets/base",
"model_module_version": "1.2.0",
"model_name": "LayoutModel",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
}
}
}
},
"nbformat": 4,
"nbformat_minor": 0
}
|