Update To Sklearn interface

TabPFN/PrepareDatasets.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -14,7 +14,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -25,18 +25,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The autoreload extension is already loaded. To reload it, use:\n",
-      "  %reload_ext autoreload\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "%load_ext autoreload\n",
     "\n",
@@ -54,7 +45,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -63,42 +54,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "OrderedDict([(99,\n",
-       "              {'id': 99,\n",
-       "               'alias': 'OpenML-CC18',\n",
-       "               'main_entity_type': 'task',\n",
-       "               'name': 'OpenML-CC18 Curated Classification benchmark',\n",
-       "               'status': 'active',\n",
-       "               'creation_date': '2019-02-21 18:47:13',\n",
-       "               'creator': 1}),\n",
-       "             (225,\n",
-       "              {'id': 225,\n",
-       "               'alias': 'OpenML-friendly',\n",
-       "               'main_entity_type': 'task',\n",
-       "               'name': 'OpenML100-friendly',\n",
-       "               'status': 'active',\n",
-       "               'creation_date': '2019-09-16 19:41:46',\n",
-       "               'creator': 1})])"
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "openml.study.list_suites()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -108,7 +73,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -120,7 +85,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -130,27 +95,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "30"
-      ]
-     },
-     "execution_count": 12,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "len(tids)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -159,7 +113,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -169,20 +123,23 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "outputs": [],
-   "source": [
-    "open_ml_datasets, open_ml_datasets_df = load_openml_list(test_dids_classification, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 100000, num_feats=100, return_capped=True)\n"
-   ],
    "metadata": {
     "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "open_ml_datasets, open_ml_datasets_df = load_openml_list(test_dids_classification, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 100000, num_feats=100, return_capped=True)\n"
+   ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -191,41 +148,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\\begin{tabular}{lrrrrrrr}\n",
-      "\\toprule\n",
-      "                                  Name &  \\# Features &  \\# Categorical Features &  \\# Instances &  \\# Classes &  \\# NaNs &  Minority Class Size &    id \\\\\n",
-      "\\midrule\n",
-      "                    KDDCup09\\_appetency &         231 &                      39 &        50000 &          2 & 8024152 &                  890 &  1111 \\\\\n",
-      "                              airlines &           8 &                       5 &       539383 &          2 &       0 &               240264 &  1169 \\\\\n",
-      "                        bank-marketing &          17 &                      10 &        45211 &          2 &       0 &                 5289 &  1461 \\\\\n",
-      "                                 nomao &         119 &                      30 &        34465 &          2 &       0 &                 9844 &  1486 \\\\\n",
-      "                                 adult &          15 &                       9 &        48842 &          2 &    6465 &                11687 &  1590 \\\\\n",
-      "                             covertype &          55 &                      45 &       581012 &          7 &       0 &                 2747 &  1596 \\\\\n",
-      "                           numerai28.6 &          22 &                       1 &        96320 &          2 &       0 &                47662 & 23517 \\\\\n",
-      "                             connect-4 &          43 &                      43 &        67557 &          3 &       0 &                 6449 & 40668 \\\\\n",
-      "jungle\\_chess\\_2pcs\\_raw\\_endgame\\_complete &           7 &                       1 &        44819 &          3 &       0 &                 4335 & 41027 \\\\\n",
-      "                            APSFailure &         171 &                       1 &        76000 &          2 & 1078695 &                 1375 & 41138 \\\\\n",
-      "                                albert &          79 &                      53 &       425240 &          2 & 2734000 &               212620 & 41147 \\\\\n",
-      "                             MiniBooNE &          51 &                       1 &       130064 &          2 &       0 &                36499 & 41150 \\\\\n",
-      "                             guillermo &        4297 &                       1 &        20000 &          2 &       0 &                 8003 & 41159 \\\\\n",
-      "                              riccardo &        4297 &                       1 &        20000 &          2 &       0 &                 5000 & 41161 \\\\\n",
-      "                               volkert &         181 &                       1 &        58310 &         10 &       0 &                 1361 & 41166 \\\\\n",
-      "                                dionis &          61 &                       1 &       416188 &        355 &       0 &                  878 & 41167 \\\\\n",
-      "                                jannis &          55 &                       1 &        83733 &          4 &       0 &                 1687 & 41168 \\\\\n",
-      "                                helena &          28 &                       1 &        65196 &        100 &       0 &                  111 & 41169 \\\\\n",
-      "\\bottomrule\n",
-      "\\end{tabular}\n",
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "print_table = open_ml_datasets_df\n",
     "print_table = print_table[['name', 'NumberOfFeatures', 'NumberOfSymbolicFeatures', 'NumberOfInstances', 'NumberOfClasses', 'NumberOfMissingValues', 'MinorityClassSize']].copy()\n",
@@ -247,6 +172,15 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    },
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "open_cc_datasets, open_cc_datasets_df = load_openml_list(open_cc_dids, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 2000, num_feats=100, return_capped=True)\n",
@@ -319,13 +253,7 @@
     "\n",
     "# Remove too easy\n",
     "openml_list = openml_list[openml_list.CfsSubsetEval_DecisionStumpAUC != 1]"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "code",
@@ -365,9 +293,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.13"
+   "version": "3.9.6"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

TabPFN/PriorFittingCustomPrior.ipynb

@@ -0,0 +1,353 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "## Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import random\n",
+    "import time\n",
+    "import warnings\n",
+    "from datetime import datetime\n",
+    "\n",
+    "import torch\n",
+    "\n",
+    "import numpy as np\n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
+    "from scripts.differentiable_pfn_evaluation import eval_model_range\n",
+    "from scripts.model_builder import get_model, get_default_spec, save_model, load_model\n",
+    "from scripts.transformer_prediction_interface import transformer_predict, get_params_from_config, load_model_workflow\n",
+    "\n",
+    "from scripts.model_configs import *\n",
+    "\n",
+    "from datasets import load_openml_list, open_cc_dids, open_cc_valid_dids\n",
+    "from priors.utils import plot_prior, plot_features\n",
+    "from priors.utils import uniform_int_sampler_f\n",
+    "\n",
+    "from scripts.tabular_metrics import calculate_score_per_method, calculate_score\n",
+    "from scripts.tabular_evaluation import evaluate\n",
+    "\n",
+    "from priors.differentiable_prior import DifferentiableHyperparameterList, draw_random_style, merge_style_with_info\n",
+    "from scripts import tabular_metrics\n",
+    "from notebook_utils import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "large_datasets = True\n",
+    "max_samples = 10000 if large_datasets else 5000\n",
+    "bptt = 10000 if large_datasets else 3000\n",
+    "suite='cc'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "device = 'cpu'\n",
+    "base_path = '.'\n",
+    "max_features = 100"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def print_models(model_string):\n",
+    "    print(model_string)\n",
+    "\n",
+    "    for i in range(80):\n",
+    "        for e in range(50):\n",
+    "            exists = Path(os.path.join(base_path, f'models_diff/prior_diff_real_checkpoint{model_string}_n_{i}_epoch_{e}.cpkt')).is_file()\n",
+    "            if exists:\n",
+    "                print(os.path.join(base_path, f'models_diff/prior_diff_real_checkpoint{model_string}_n_{i}_epoch_{e}.cpkt'))\n",
+    "        print()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def train_function(config_sample, i, add_name=''):\n",
+    "    start_time = time.time()\n",
+    "    N_epochs_to_save = 50\n",
+    "    \n",
+    "    def save_callback(model, epoch):\n",
+    "        if not hasattr(model, 'last_saved_epoch'):\n",
+    "            model.last_saved_epoch = 0\n",
+    "        if ((time.time() - start_time) / (maximum_runtime * 60 / N_epochs_to_save)) > model.last_saved_epoch:\n",
+    "            print('Saving model..')\n",
+    "            config_sample['epoch_in_training'] = epoch\n",
+    "            save_model(model, base_path, f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{model.last_saved_epoch}.cpkt',\n",
+    "                           config_sample)\n",
+    "            model.last_saved_epoch = model.last_saved_epoch + 1 # TODO: Rename to checkpoint\n",
+    "    \n",
+    "    model = get_model(config_sample\n",
+    "                      , device\n",
+    "                      , should_train=True\n",
+    "                      , verbose=1\n",
+    "                      , epoch_callback = save_callback)\n",
+    "    \n",
+    "    return"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "## Define prior settings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "def reload_config(config_type='causal', task_type='multiclass', longer=0):\n",
+    "    config = get_prior_config(config_type=config_type)\n",
+    "    \n",
+    "    config['prior_type'], config['differentiable'], config['flexible'] = 'prior_bag', True, True\n",
+    "    \n",
+    "    model_string = ''\n",
+    "    \n",
+    "    config['epochs'] = 12000\n",
+    "    config['recompute_attn'] = True\n",
+    "\n",
+    "    config['max_num_classes'] = 10\n",
+    "    config['num_classes'] = uniform_int_sampler_f(2, config['max_num_classes'])\n",
+    "    config['balanced'] = False\n",
+    "    model_string = model_string + '_multiclass'\n",
+    "    \n",
+    "    model_string = model_string + '_'+datetime.now().strftime(\"%m_%d_%Y_%H_%M_%S\")\n",
+    "    \n",
+    "    return config, model_string"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "## Visualize Prior samples"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "config, model_string = reload_config(longer=1)\n",
+    "\n",
+    "config['bptt_extra_samples'] = None\n",
+    "\n",
+    "# diff\n",
+    "config['output_multiclass_ordered_p'] = 0.\n",
+    "del config['differentiable_hyperparameters']['output_multiclass_ordered_p']\n",
+    "\n",
+    "config['multiclass_type'] = 'rank'\n",
+    "del config['differentiable_hyperparameters']['multiclass_type']\n",
+    "\n",
+    "config['sampling'] = 'normal' # vielleicht schlecht?\n",
+    "del config['differentiable_hyperparameters']['sampling']\n",
+    "\n",
+    "config['pre_sample_causes'] = True\n",
+    "# end diff\n",
+    "\n",
+    "config['multiclass_loss_type'] = 'nono' # 'compatible'\n",
+    "config['normalize_to_ranking'] = False # False\n",
+    "\n",
+    "config['categorical_feature_p'] = .2 # diff: .0\n",
+    "\n",
+    "# turn this back on in a random search!?\n",
+    "config['nan_prob_no_reason'] = .0\n",
+    "config['nan_prob_unknown_reason'] = .0 # diff: .0\n",
+    "config['set_value_to_nan'] = .1 # diff: 1.\n",
+    "\n",
+    "config['normalize_with_sqrt'] = False\n",
+    "\n",
+    "config['new_mlp_per_example'] = True\n",
+    "config['prior_mlp_scale_weights_sqrt'] = True\n",
+    "config['batch_size_per_gp_sample'] = None\n",
+    "\n",
+    "config['normalize_ignore_label_too'] = False\n",
+    "\n",
+    "config['differentiable_hps_as_style'] = False\n",
+    "config['max_eval_pos'] = 1000\n",
+    "\n",
+    "config['random_feature_rotation'] = True\n",
+    "config['rotate_normalized_labels'] = True\n",
+    "\n",
+    "config[\"mix_activations\"] = False # False heisst eig True\n",
+    "\n",
+    "config['emsize'] = 512\n",
+    "config['nhead'] = config['emsize'] // 128\n",
+    "config['bptt'] = 1024+128\n",
+    "config['canonical_y_encoder'] = False\n",
+    "\n",
+    "    \n",
+    "config['aggregate_k_gradients'] = 8\n",
+    "config['batch_size'] = 8*config['aggregate_k_gradients']\n",
+    "config['num_steps'] = 1024//config['aggregate_k_gradients']\n",
+    "config['epochs'] = 400\n",
+    "config['total_available_time_in_s'] = None #60*60*22 # 22 hours for some safety...\n",
+    "\n",
+    "config['train_mixed_precision'] = True\n",
+    "config['efficient_eval_masking'] = True\n",
+    "\n",
+    "config_sample = evaluate_hypers(config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Using style prior: True\n",
+      "MODEL BUILDER <module 'priors.differentiable_prior' from '/home/hollmann/TabPFN/priors/differentiable_prior.py'> <function get_model.<locals>.make_get_batch.<locals>.new_get_batch at 0x7f24bd339af0>\n",
+      "Using cpu:0 device\n",
+      "init dist\n",
+      "Not using distributed\n",
+      "DataLoader.__dict__ {'num_steps': 33554432, 'get_batch_kwargs': {'batch_size': 1, 'eval_pos_seq_len_sampler': <function train.<locals>.eval_pos_seq_len_sampler at 0x7f24bd493ee0>, 'seq_len_maximum': 1152, 'device': 'cpu:0', 'num_features': 100, 'hyperparameters': {'lr': 0.00011555441385381896, 'dropout': 0.0, 'emsize': 512, 'batch_size': 1, 'nlayers': 12, 'num_features': 100, 'nhead': 4, 'nhid_factor': 2, 'bptt': 1152, 'eval_positions': [1094], 'seq_len_used': 50, 'sampling': 'normal', 'epochs': 400, 'num_steps': 33554432, 'verbose': True, 'mix_activations': False, 'pre_sample_causes': True, 'multiclass_type': 'rank', 'nan_prob_unknown_reason_reason_prior': 0.5, 'categorical_feature_p': 0.2, 'nan_prob_no_reason': 0.0, 'nan_prob_unknown_reason': 0.0, 'nan_prob_a_reason': 0.0, 'max_num_classes': 10, 'num_classes': <function <lambda>.<locals>.<lambda> at 0x7f24c2d03ee0>, 'noise_type': 'Gaussian', 'balanced': False, 'normalize_to_ranking': False, 'set_value_to_nan': 0.1, 'normalize_by_used_features': True, 'num_features_used': <function <lambda>.<locals>.<lambda> at 0x7f24c2d03e50>, 'num_categorical_features_sampler_a': -1.0, 'differentiable_hyperparameters': {'distribution': 'uniform', 'min': 2.0, 'max': 10.0}, 'prior_type': 'prior_bag', 'differentiable': True, 'flexible': True, 'recompute_attn': True, 'bptt_extra_samples': None, 'output_multiclass_ordered_p': 0.0, 'multiclass_loss_type': 'nono', 'normalize_with_sqrt': False, 'new_mlp_per_example': True, 'prior_mlp_scale_weights_sqrt': True, 'batch_size_per_gp_sample': None, 'normalize_ignore_label_too': False, 'differentiable_hps_as_style': False, 'max_eval_pos': 1000, 'random_feature_rotation': True, 'rotate_normalized_labels': True, 'canonical_y_encoder': False, 'aggregate_k_gradients': 8, 'total_available_time_in_s': None, 'train_mixed_precision': True, 'efficient_eval_masking': True, 'prior_bag_get_batch': (<function get_model.<locals>.make_get_batch.<locals>.new_get_batch at 0x7f24bf3e8550>, <function get_model.<locals>.make_get_batch.<locals>.new_get_batch at 0x7f24bd339e50>), 'prior_bag_exp_weights_1': 2.0, 'normalize_labels': True, 'check_is_compatible': True}, 'batch_size_per_gp_sample': None, 'get_batch': <function get_model.<locals>.make_get_batch.<locals>.new_get_batch at 0x7f24bd339af0>, 'differentiable_hyperparameters': {'prior_bag_exp_weights_1': {'distribution': 'uniform', 'min': 2.0, 'max': 10.0}, 'num_layers': {'distribution': 'meta_gamma', 'max_alpha': 2, 'max_scale': 3, 'round': True, 'lower_bound': 2}, 'prior_mlp_hidden_dim': {'distribution': 'meta_gamma', 'max_alpha': 3, 'max_scale': 100, 'round': True, 'lower_bound': 4}, 'prior_mlp_dropout_prob': {'distribution': 'meta_beta', 'scale': 0.6, 'min': 0.1, 'max': 5.0}, 'noise_std': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 0.3, 'min_mean': 0.0001, 'round': False, 'lower_bound': 0.0}, 'init_std': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 0.01, 'round': False, 'lower_bound': 0.0}, 'num_causes': {'distribution': 'meta_gamma', 'max_alpha': 3, 'max_scale': 7, 'round': True, 'lower_bound': 2}, 'is_causal': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'pre_sample_weights': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'y_is_effect': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'prior_mlp_activations': {'distribution': 'meta_choice_mixed', 'choice_values': [<class 'torch.nn.modules.activation.Tanh'>, <class 'torch.nn.modules.linear.Identity'>, <class 'torch.nn.modules.activation.ReLU'>]}, 'block_wise_dropout': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'sort_features': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'in_clique': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'outputscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 1e-05, 'round': False, 'lower_bound': 0}, 'lengthscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 1e-05, 'round': False, 'lower_bound': 0}, 'noise': {'distribution': 'meta_choice', 'choice_values': [1e-05, 0.0001, 0.01]}}}, 'num_features': 100, 'epoch_count': 0}\n",
+      "PRIOR_BAG: tensor([1.0000, 2.3162]) [1]\n",
+      "{'is_causal': False, 'num_causes': 4, 'prior_mlp_hidden_dim': 6, 'num_layers': 2, 'noise_std': 0.0021951181710037487, 'y_is_effect': True, 'pre_sample_weights': True, 'prior_mlp_dropout_prob': 0.11217365522242403, 'pre_sample_causes': True}\n",
+      "Hparams dict_keys(['prior_bag_exp_weights_1', 'num_layers_alpha', 'num_layers_scale', 'prior_mlp_hidden_dim_alpha', 'prior_mlp_hidden_dim_scale', 'prior_mlp_dropout_prob_b', 'prior_mlp_dropout_prob_k', 'noise_std_log_mean', 'noise_std_log_std', 'init_std_log_mean', 'init_std_log_std', 'num_causes_alpha', 'num_causes_scale', 'is_causal_choice_1_weight', 'pre_sample_weights_choice_1_weight', 'y_is_effect_choice_1_weight', 'prior_mlp_activations_choice_1_weight', 'prior_mlp_activations_choice_2_weight', 'block_wise_dropout_choice_1_weight', 'sort_features_choice_1_weight', 'in_clique_choice_1_weight', 'outputscale_log_mean', 'outputscale_log_std', 'lengthscale_log_mean', 'lengthscale_log_std', 'noise_choice_1_weight', 'noise_choice_2_weight'])\n",
+      "Style definition of first 3 examples: None\n",
+      "Using a Transformer with 25.82 M parameters\n",
+      "PRIOR_BAG: tensor([1.0000, 7.0192]) [1]\n",
+      "{'is_causal': True, 'num_causes': 2, 'prior_mlp_hidden_dim': 10, 'num_layers': 2, 'noise_std': 0.0031679113358953426, 'y_is_effect': False, 'pre_sample_weights': True, 'prior_mlp_dropout_prob': 0.009754962364049987, 'pre_sample_causes': True}\n",
+      "Hparams dict_keys(['prior_bag_exp_weights_1', 'num_layers_alpha', 'num_layers_scale', 'prior_mlp_hidden_dim_alpha', 'prior_mlp_hidden_dim_scale', 'prior_mlp_dropout_prob_b', 'prior_mlp_dropout_prob_k', 'noise_std_log_mean', 'noise_std_log_std', 'init_std_log_mean', 'init_std_log_std', 'num_causes_alpha', 'num_causes_scale', 'is_causal_choice_1_weight', 'pre_sample_weights_choice_1_weight', 'y_is_effect_choice_1_weight', 'prior_mlp_activations_choice_1_weight', 'prior_mlp_activations_choice_2_weight', 'block_wise_dropout_choice_1_weight', 'sort_features_choice_1_weight', 'in_clique_choice_1_weight', 'outputscale_log_mean', 'outputscale_log_std', 'lengthscale_log_mean', 'lengthscale_log_std', 'noise_choice_1_weight', 'noise_choice_2_weight'])\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdAAAAHECAYAAACJGnuNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAACbOElEQVR4nOzddXhc15n48e+9w8ySRizbsszMdsBh5obTNsV0S7/iFrew293ilnELaZumDTbcMNuJY7ZjtixZzBrmuff3x9hjy5It2RbY0vk8T54nvnPhaO7MvPfQeyRVVREEQRAE4eTIY10AQRAEQTgbiQAqCIIgCKdABFBBEARBOAUigAqCIAjCKRABVBAEQRBOgfZkdvZ6vWplZeUIFUUYThs3buxSVdV37HZxD88e4h6e/cQ9PPsd7x7CSQbQyspKNmzYMDylEkaUJEkHB9ou7uHZQ9zDs5+4h2e/491DOMkAKowf0UAP8eb9pLub0Dj9GPxTiKgm9jT0EkukqS5zMqXUiUYztFb+5s4Iext6MRu0BCJJUukskwuMlOp6SAV7Qc2gMxhJBzrImj20SgVEMWGQVRraI5QW2tDo9PSEkpgMWtp7Y9jMOgpcZupagmhkiUl+K24pzL7OLN1RlckVXtq7YwQjCaaUuWjtjhKNp6nyO9BqJBLpLIFwklA0RaHLiFmTpdgOSDK6tnfRFlTRonipbQky2SNTIneS7m5FshfQrimkrldiWoUbo1HD9v3dxBMZygptlBRYScTTaGKdWCKNZJMxsq4KMs4yKopdJ3UfOnpj7D3YS1cwwaQSO9VlLkwG8bU8WdlokERrLemuJnSeYgzFU9BanGNdrEFlwj0kW/aT7m1H7yvDUDwZjck2bOdXUgmSrbWk2uvQWN0kHZWooTZ02ShKsBONzY3W7iXZvAeNyYbePwmjf0qfczS1h+nuDdPQGsSgheoSOwlFwppoRxdoQGe2ETSVsqUNirxWTHoNB9vClPosdPXGSCUSTCt3UKLpIdVWi6TRoveVoUoaMr1taO1uUl2tZPQWeg0l9Kg2SnwW2rpjNHeEKXKZSKUzdPVEKC6woTcaqGsJ4zDJTPVJeKQQ6e4mkGSy3sls6TbS3ZtgcpmTGVWeIX2fWroi7DnYSySWYkqpkyllTnRazaDHiW/qBJRJJQm+9RiJTU8f2TbjPB6JLuKFzZ0AyBJ86yPLmTe1YNDzNXdE+Npv13D+gjJe3dRIVyABwO3nFGLreQw1EcFSvYjud57MH+Momc1buotJakw0tkfY++x+3nd5DZKs4WcPbsnvV1pgZX5NAU++cQCjXsMHrp7Jrx7ZyRUrKnn5se0cbAtz2yU1/O/9mwhEkgBoZInP3LaAh17ey8HWcP5cd1w6jRc3dHD3uV5CxmLq62P89Jm3mOS3MHt6C71bHs/va6lcxL7USv7wxA4+eM0sfv/4dhQVtBqJD14zi2p7As0rPyYc6ckdIMmkVn+a3ekZTKtwD+k+dAfjfP+vG9hzsDe/7eM3zeWy5ZVDOl7IUdJJetc8Qmj9kc+zbd5FuC9+Pxq9aQxLdmLZeISu5/9AbPfb+W3Oc27GtfJGJM3w/DRHd79F55O/AEB74ceh7jlsTjvdr96f30fnKcVUNYfeV+9H5ynBd82nMBbngmh9S5CG1gD/+4+tZJVc0h2v08h/X20n/vSPSJPbJtt9GKa8n+/9eRfvv2omyXSWnz6wlUAkiSxL/PrOQtqf/iGoCgAaiwPPpR9GY7HT/sB/58uid/ppKb+DSKyEXz68lfk1PrYoKht3d+T3uXRZBfsaAxxoDlLkMfOFpQm0b98LgKTV4175aX7ybDsA99wwhytXVp3wPWrtivKN371NW3c0dw4Jvnb3UpbMLBr0/RWDiA5p74mxZlsLwUM/wuNZtK2RxKZn+mxL7HyNhX4l/29Fhfuf2008kR70fLvqu+kNJdHIUj54AiwryZBu3Yd1+jJCm57rc4zavJ3F/jSvbGhk4bRckO4KJnnijQN99mvqiGA363NlTGXZVd+D32vBaTNwsC2MyaAlkcrmgydAVlF57u16XDZjn3M9teYALreDpt4UqtXLva/nHhaunmOFbU/1LV/9BlZW5M711vZWZk7yApDJqmzc3Y4z1oByOHgCqArmXU+zd38rQ83uVd8S6hM8Ae59agcdvbEhHS/kpLtbCK3v+3kOb3mRTFfzGJVoaFKdjX2CJ0BgzSOke1uH5fyZUDfdL96b/7dscWKSU4Q29H2v0t1NaMz2Q//fTKqtLv9aXWuQ59Y15oMnwPxKK+l1DwBHtimhTiYbc5/llzc04rIZ8t/JhVNcqFufygdPyLUYJFv39yuzEmhlTpHEm1ubyWQVqsucfYInwAvvNLBkRiEAbd0xGtXC/GtqJoW7dS1Vxbm/5y/P7KSj58Tfp/2NvfngCaCq8KendhCOpk54HIgaKABvbW/l+3/dQCarYDXp+M97VjCl1DnWxRoxSjrJ0R/+w7T0DZY9oSTprMJgz/CRWBqtRiaVyfbZrlPTpABkDWqm/4dRq6ZRVPIBx2TUERrgQ5tVjnzxQtEUVpMO5dAX2qDXDBjke8NJKv32PtvCsTRmo5ZkKoGCRCSWO04nZUHJ9juHRs29HowmKfVZ89tjiQxqMtL/jYgFULNpFEVFo5H6v36MRKr/NWPJDOm0MsDeZ6ZgJEldS5BQNEWx10plsR3tEJv9BxKKJmnvjqHXayj2WobUjKZkBv485z7no6O9O0o8lcXnNGEx6YZ0jJpJ9N+oZFHSg/9wD+n82TRK4lDwkGTUTArZYCEb6//ZVY/6/CvpeP7/48ksvZG+5XGaNajdoX7n0GbiyHLuO3z0M6TLokEJB/rtryRjSFL/z4pWKxGK5u5dNjvAfVX6bktkJJDkI7XbWC92k/ZQ+TOksyf+PsWSmX7bAuFkv9+zgUz4Gmhje5jv/3UDk0sc/OdHl2MyavnhfRvIDvKmn82MXj8aT0mfbRqbm4PRvqHyypVV2C2GQc83tcJFKpPFZtYjHRU3mlN2JK2eVFsdxrIZfY6RDGZaM3ZKC6x0Hqq1BsJxzlvQt1xajdTnR3RmlYcDzUE0GhmdViYQTlLosfQr0+qFZew40N1n24rZfnbWdVPsNSFn4lwwxwPAtnbQFE7us69sttOUyAXNJTOK2Lq/60gZJnnIeiYDfYNkuno1vqKCIfcblxZaMej6BohVc0vwuozHOeLMEowk+d1j2/n6b9/iB/dt5LM/fY13drb12y8bCxOv30743TdINO87boBoaAvx9d+s5bM/fZ1P/ehV/v78nvwP6YnoXEVo3cV9tmntPrSuwZvgTlcyleHFdw7yqf99lU/+8BW+9fu3aWgLD34goHMXI5v7PuTp/dXonIXHOeLkaGwerLPPy/1DVZB0RhLtDVinL++7o6w90mQsa9H7KvIvlfmsXLiw73v72q4Q+lkX9z2HJNOjLySTVVgxx9+nxvr23iD6WRf0K5++sApVPeZ3VtbQFlZZND33HiTTWZzWvr9BlX47rd25BwNZglJTrE/tNlKxkl0HAwAsn12M13Hi71NVsQP5mOfdK1dW4bYP/j2c0AFUVVV+9chWTAYNX/vAUuZNLeAj182muTPKi+sbx7p4I8bkcOO66jPop65ANtnQTlqE49ovUlVdSaXfhttu5H1XzuDc+SWDnwyoLnPyHx9Yyu76bj507Syqiu24bAZaEkZc13+RTCKGacp8LLPOQzbZ0JXPJnrOp9jTo+X68yaxaXc7lywuYfksPz6HiUuXVeC0Gpha5uQzty1g/c42Ct1m7rlhNtNLDEwrs/P65iY+dfM8aipcbNrdzsdvmktZYa7sd142jUq/jdsvncbUchcOq56Ll5QzrcLJe88vxmLUoX33aa6eLnPDeZN4Z3+Irlm3YZqZK5++aj6RVZ/gX9vCfPCamcyc5MZpNeB1GnnPhdXMmeLlYMaN5tJPo/OVo7G6kBa9h0TpQmZN9gz5PlQU2fn2R5czc5IHu0XP1auquOvyaRh0Z0fDUH1riNc3H2kmVVX49cPb6AocqcFkE1F6Xv0brX/7Jp2P/4SWe79EdNfafufKZLI89NI+DrTkajaKovLQS/vY3xgYtBxai5OiGz+PZcZKZJMNS81SCm/+Ejr70O/FqTrQEuSnD2whlsjVYnbV9/Dnp3eQSPWv1RxL5yzEf+vXME1ZiGyyYZ2zmoJrPoHGZB302KGQtTpcq27CsfQaZLMdee+r6JfciLZ4KrZ5F6GxONAXV+O7+uNEdr2F3j+Zwhs+i6Fkav4c1eUuJpe6ef/lU/E4jJQVWHj/FTX0Fi1CXnA9GqsTXUEl6oWf5tFtGW5cXU1FkZ21W1v48HWzKCu0odXItFtrcJ1/OxqbG523FM/lH0XnLiHZVo998VVoLA50RZNJr/40G1p0zK32cuvFU1m/s53bL61h0TQfdoue8+YVc+tFU9m2v5PJJXa+dvtMKlwSWlcRWrsP20UfYmPQi9Gg5YoVlbz/yhkY9Cf+Pk0ucfDNDy9ncqkDp83AbZdM5dJllUjS4K1I0smsxrJo0SJ1PA29fre2iy//ag0fvX42V62aBOSC6ud++jqJVIZffuGCIb2JZyJJkjaqqrro2O1H38NMKkk8FERntmI0mwGIxFJksgpO28nXgiLxFOmMgk4rk84ouGxGMpEAKhCNJUFvwapJEVO0hFMyOq2ENhMni4zJaiEcSaPV5mqWoWgKo14mlVZQVRWtpOA2SiQzadJZiSR6zEZd7tyJDGajBgmZSCyFJIFGklBUhawqodVI6EmhVTOkNWa0soKRDEa7HVXKjRo2aiX0mQjZbJaEZEKRdUgS+fchHEsRiaewm/VYTLk+2UA4gV5NomYzZPXWIdXWBxJPpImnsjisBjRHPQoP5R6OpTe3NvO9v/Qvx6++eAFlhbmRpPHGXbT+5Wt9Xpf0Jko/9CN0riM1rZ5gnE/88BXCsb7N8R+8ZhbXnde3deB4lEwKJR5BNlmRtfqT/XNOyUsbGvjJ3zf32SZJ8LsvX0SRxzKke6ikkyiJKBqTHUk7/A9PqqqQjQaRdAY0BjOxaBwlGcGgAdloRdYbSPd2IGn16OwDD4CLxFJEowmMWgXzoRbq3oSEkSQGk4lAQgYJDFoNJpOWcDQNqoqSSUE2jaw3YTTq0CV6UDNpMlozWhm0Wg2y2Y4SC6HKOsJpDRaTHoNeQzSeJhRNYjFo0GTixDKgM5pJpbOoqoJZzmCxWVCScdRMCkmnRzLa6QnFURUVj9Pc5/s0mGg8TSqdxWkz9PndP949hAneB/roq/tztZOlR5osJEnikqUV/PLhrdQ2B8d1X6hWb8Dm7TvK1mo+9R8eq6n/sVqrEwDnUQ/V9kP/5RxpfrWajgSg4wXwgUKUy370/59c4JfgqKYaA9rjXMNm1mM75r3JlfH0m1tNRh0m49D6zc4kxV4rsiz16ZOaPdmDx3nkPVHiA/S3peIoqb4DOywmPTUVLjbs6jtgxO81D7k8slaPbBvaCOjh4rL2/7SU+qyYT+J+yjoDsu7UHr6GQpJktNYj06vMFhNY+nbX6N0nbu62mvX9fhsKjvpOFx7Ti2JwHg4tx9w/s3/A88uHynf03bOYdEf1J5vo31Fz6NhjHpZ8zqF/Zo7W93pDM2GbcLuDcTbuaueyZZUD9EMVo9XIvLqxaYxKJwhnvooiG199/5L8A8icKV4+esMczIYjP0JaVxGSpu+Pkq6gkqTO0WebQa/hzsum47QdCSTnzS9hatnJzasdbZNKHJwz70gfoU4rc88Nc7BbRqcGLIytCVsDfWVjE4oKFywu6/ea1axnbrWX9Tvb+NC1s8agdIJw5tNoZJbMLGJKmZN4IoPbbuhXk9Z7Sym8+ct0PfNbMsF2NMXTaK2+nm/+agOfv3MR0yuP1Dkmlzr50afOpaUrglGvpazQdtI1gtHmtBm554Y5XL68ikg8RbHPSnnh8CVCEM5sEzaAvrapiemVboq9A3fYL6gp4P92v0tbd5SiAUZ5CoKQ47Ybj26T70OSJMyT5mK64T/YuP0g21tSvPlYM4oK/3v/Rn7wyXP6NNcXuM0UuE+tCW6s2C0GZk8ZuSZY4cw1IZtw23ti1LeGWDFn4PZ4gAWHJvdv2tNx3H0EYaJJZxR21nVz37928vDLe6ltDg7puP3dKr98vo3X3+3hcJdpW3eMntD4T1wijF8Tsgb6zo7cXLUlM47fcV7is+J1mti2r4srVpw4FZQgTBQ76rr5j9+uzU+Uf+CFvXz346uYPMhgO88Ac/EcVj02y5ndRCsIJzIha6Dv7GyjtMBKse/4860kSWJGlZtd9T1DTs0mCONZKpPl4Zf29ckyk0hlh9RKU15o472XT8//W6eV+fQt8095xKQgnAkmXA00lkjzbm0X15wz+Nyy6ZVuXt/cTGdv/KzrlxGE4aYoKtF4/yxC0fjg+ZKNBi3XnDuJeTUFhCJJCtxmSk7wACsIZ4MJVwPdvr+LTPZIqqgTmXZohODO+p5B9hSE8c+o13LdeX2XupIkWDxjaKnnDHot1WVOFk4vpKzQhjzIJPdsVmF/Y4BXNjSybkdrnwxHgnAmmHA10G21Xei1MtMqB59fVuW3Y9Rr2F3fw/kLSkehdIJwZls4rYDP3b6Ax16rxWTQ8p6LpjK1/PTnaiqK2i+gbtnXybf/sC6fqGFapZt/v2sRXueZu0SZMLFMuAC6fX8X0yrdQ1rlQaORmVruYpeogQoTSH1riK37Ookl0syt9lFd5kKnzTVWWc16zl9YxrJZfmRJQq8f/Ht0IvsbAzy37iDNHWEuWVrB/JoCHFYDoWguUf3RWY521/ewvykgAqhwxphQATQcS1HfGuKOS6cN+ZjplW4eemkviWQG4xBWNheEs1l9S5Av/fJNooeSo//9+T1888PLWVDTN+XjqXwXVFUhG+4FrRat2UFDa4iv/HoN8UPLSW2v7c7nvk2ksnT09G+yjcZO3N8aCCfYfbCXupYQFUU2plW6cNtFwBVGxoSKCO/WdqGqMGuyd8jHTK1woahQ2xxk5qSRX91BEMbS1v1d+eAJuRVWHnhhDzOq3BgHWdXiRDKhbkKbnie44Rk0RgvuC9/LgVhZPnge9uCLezhnXjFum5ELFpby2uZmtFqZaDyNJOWWgDueZCrDAy/s5ak1RxaEPm9+CR+7ce4Zn9FIODtNsADajV6nOak+m+oyJwB7G3pFABXGvdgAi5On0lnURIR0PI3W6uyX2/ZEwrEUHT0xpEAbml1voSZjZJIxOh79EVz8/X77y7KEBGi1Mpcur6TAbSYQSVLksVBaYGFSiaPfMemMQjCSJBJP8ezb9X1ee21zM1efM4maitFNMi9MDBMqgO4+2MPUcme+P2coXDYjPpeJvQ29I1gyQTgzzK328ffn9+TnehZ7THz5YjNdf/s6mWAn1tnn4VxxPbpBFqtWVZW6lhC/eGgz+xqD6LQyt626g4XWF6Aht/xXmTmBzazrs4TZ7ZdOw+0w0dge5j//sI5AJJepSJLgPz64FJ1WQySWoieUxGrWEU9mePSVfbyxpYXSAit3XzWTh17alz8OIJk+ZtFmQRgmEyaAptJZDjQHufbcoa0teLSpZS72DmFhX0E4m7V0RohEU3zxzkU8+eYBIrE0X72mgMij3wI1F4TCW14k4Z5CmwN6wgn8HgvVpc4+SeSbOsI0dUR47u169jXmUv2lMwp/ebWNiqtX4z0UQLNd9dxzw1zqW0O0dEY5d0EJsyd7yWSybK/t6hMEVRXue3Y3XqeJn/5jM/ubgqyaW0wynWX9znYA9jUGONga4sYLqvn783sAKHSbKPGJXNbCyJgwAfRAS5BMVqWm4uSH3E8td7JmWwvBSBLHAOv/CcLZrrE9zNd/u5buYAKNLLGgxseHrp2FLbCNhHpUDW7ycv6xy8DrO9blN33w6plcfe5kQpEkr29p5vHXa7lsWSXb93f3u05HQocXCdnupTbh4VdPbuTzdyzgvVfMAKCzN84DL+wZsIyhaIpXNjaxvykXlEt8Vh54cW+ffVIZBYtJh8dhZPZkLzeunoLHIQYRCSNjwiRS2HMw1wR7KnPWqg8ds0/UQoVxauu+TrqDCQCyisr6XR3c/9xuJH3fDFy9hYt4fUff7oy//GsXrV0R1m5v4fePv0tnb5zdB3uo8PdfosXlccL136F51ZcJaVxcvrySF945mH/9re0tPP76AZw2I8fmWbhqZRUvr2/M/zuZzmIx9q8DlBVY+elnz+fTt8yjsrh/n6kgDJcJE0D3HuzF6zSd0tPolFInsoToBxXGre5Aot+2tu4Ysq8Cnb86vy2e6f+Tkc4oRGJpnnj9QH7bxt0dnDe/pE+AO2deCT6PnV8918x379vCfc/uZtv+Ti5eUsmm3e30hOK8cmgR+xfWHeQDV8+iusyJ32PhI9fNZtH0QlKZbP58b2xp5qpVk/qUZc4UD1XFDhxWA9oB5non01kyWdEnKgyPCdOEu7uh95SabwFMBi2lhTYRQIVxa85ULw+/sq/PtsuWV2J2F7Bt5l1YJ7WgVxLYfKWUFKSYX2GmJ5pl3e5eKotsuDURXDYDLV1RIJdZ6P7ndvOpW+bTFYwjSxLzyw2s336QPU3h/DWaO6Psa+zlre2tLJ/lp7rcyf6mAC1dUe59egfzpxZw0wXVrJhTDMDHbpjD//59E6oK3cEERr2G/7pnBS2dEZw2I9VlDlz2/iu/BMIJ1u9s55m19fg9Zq45bzLTxMhc4TRNiADaG07Q0RPjqpWnvizZ1DIX63a0oaoqknTiHJ6CcLaZVuHms7cv4M9P7ySezHDduZNZNTcXtEJY+M6TMQBuW9XBN5YFSG9/GMwubrv1WjJZhcjfvsiN5/07u+qPrPep1co0tIf527O70WpkfnNXIXva+0+T2VXfS7HPyuNvHOC//20la7e1EIykyGRVmrsifZqCV84tprTARmt3FJfNQKXfgdWsY26174R/38sbGvnTUzsB2N8UYN2ONn7wqXMHnBYjCEM1IQLo3kP9n6daA4XcQKIX1zfQ3hOjyCNG9Qnji8mgZfXCMuZP9ZHJqngcxvyD4txqHzdfWI3VpGNJdgPxNx/OHRTshLYf4TnvVnqzaXyb/sB33vtJagNaTAYtNpOOH92/6dAVVDKpJHOKdby5o++1F9V4eeatBiCXQP4HnzyHg21hNLJEpd+Oz3WkH1an1TClzMmUQ/Ozh6I7FOeRV/b32ZbKKBxoDogAKpyWCRFA9zUGkGVp0EV/TyQ/kKghIAKoMG45bf2bPxPJDFv2dbKw3EC67rm+L6oKaioBkowSaMW7+5/MuvlLADz95gFuWD0FFZAlyNp1VHduYPXsEl7Znhuhu3iaF1mjIRBJ4rQZKPZaKXCb8XuPZByKJdKk0lkcVgPdwQSqquJ1mobcEqSRZPS6/n23Ws2EGQIijJAJEUBrm4OUFVgx6E498XWl345OK7O3sZdz5pcMY+kE4cwVS6T57WPvsrchwJQCH5LBCvFwn30kjZbDmRcs01fkt5cX2fjPP76TT9e3tsjG125ayK3eZi6dVYZkdpJUdfzwgXdZOrOI2y6pocBtJpNVqG8N0dUbAyQefGkvXYE45y8oJauokAhz2VSwywkM3hIilhJ21AXYcaCbqeVO5kzx9Vm/12kzcNfl0/nx3zfnt9ktejwOIwdbQ5QUWEUwFU7JhAigB5qDzK0eev7bgWg1MpNLHGIgkTChBMJJtu/vAuDlbT1ceOX1aF/7Vf51jc2DqmSQNFocy67FVDUHyA0ienlDU59ct/VtYd44oHDJsuWEOiNoNTLlbjM//7wPq1GXX9llw652/ufed7jz8un87dnduaAJ/PO1Wq49p4rLbTtJ/utfdANy+Vwe01zOSxtbAHh+ncTKuX7uuWEuNrM+f+1ls4v5ptXIOzvacNr0aGSZb/1hHZmMwifeM5fzF5aJICqctHEfQAPhJD2hxLD0dVSXu3h+3UGyWQWN+LIJE4DZqKXIY6atO0YileU361TuOuczlNCGyeHGUFKDpNFgm3shWoePVBY0qkpGUTjYFup3PpfdyLd+/3b+QfS8+aXcfdWMfPDsDsb59SPbUFRIprL54HnY8+80svpcK4cbbyOly3jpqVzwvGaRh5VFMYzhbaT2xclMmonWlstfbTZoWTitgNICC1/51Ro6eo+s9PKLh7ZSXeYacN6qIJzIuI8CB5pzWUuGI4BOLXOSTGVpaA8PvrMgjANOm5F/u3Fuvna2ryXKa81mjAuvxjb3AvTeEnSuIroVGw+/UssXf/4G//fYdtq6oly8tKLPuYo8Zg60BPu04ry2uYlt+7vYWdfNKxsbqWsOUlWcC2QD1QjtFj1S8khgzqi5fRZOcXCRdgPGtb+F7c8QfOqndL9wL9lEtM/xPaFEn+AJucQRPaH+82AFYTDjvgZa2xwAYNIwZCQ5nMVob0OAKpHhRJgg5lb7+MlnzqOlK4LNrKfCb+/TPJpIZbj3qR2s2dYK5B5a3363lW9+ZDm3XjyVf75ai8mg4e6rZvK3Z3f3O/+22i7efb6Ltu7cVJnLllewfHYRiVQmX/s97LaLJiOvuZ/DqRAcPTuYWTGXC6r1qOve6HPe6K61OJZeg6bkSCIIl82Ixajts2SbViOLRbqFUzLuA+iB5iAFbjPWo77wp8rvtWAx6djX2MulyyoGP0AQxgFZlqjw24/bxNnWHc0Hz8M6Awm6euPcenEN82sKONAcRFEU5lb7+rXguG1Geg6lEbxoSfmhNH4Ser2GWy+uIZnKkkhlkGWJVza34F/2ISxbHiDb24IhG+Pj108n1dU0YNnUbKrPv4s8Fr5w1yJ+8NcNRBMZjHoNn751PsW+468zKgjHMyEC6ORhmuslSRJTy5xiIJEgHEWWJWQJjumuRKOR2dsY4Ku/XptPn/e+K6dTUWTjYFsuiC6ZUUgoliKVUVg4rYBAOMmL7zTkz7Fyjp9Ll1Xgd1voCMa5/7k9fL1O4eJ5t3LeKg9Fk8owW81k3CZaN1aSbq/PH6t1F6NzF/cr79xiDb/98GQSshnV7KHQYxbJUYRTMq4DaCyRpqUryupFZcN2zupyFw+/vI9EKoNRP67fPkEYkmKPhatWTeKJN47kwp1S6qC8yMbv/rm9T+7Zvz6zi8/dsRCPw4hGlrFZ9Hzjd28BuUQn9z/XdyWWNdtaWb2wjL+/uIcl04v40afPpbM3hs2ip6zAhtGQ+w5qLXYKr/sMoY3PEtu/CVPlbOxLrkRr7Zs8JV6/nY7Hfkw2GkQ2WvFd/QnwLAREABVO3riOAHUtucEGw5ltZGqZE0VROdAcZEaVZ9jOKwjDobUrQm1zkExGodJvH5XVSLRaDTddUM20Sjeb9nRQXeZk/tQCLCYdXYG+A3YUFdp7YrjsRg42B4nE03zg6hn889VaFEXF5zSh1ci0dh8Z/LO/OciB5hBVfgd+n4UF0woHLIfeW4rn4rtxnXMLstGMJPed950OdNL+6I9QDs1jVRIROv75v5R88AfovaXD/K4IE8G4DqCHR+AOVxMuHMlItLchIAKocEZpag/z9d+tpevQyioGnYbvfGwFNaOQNN1lN3LOvBLOmdc3ycgVK6vYffBIl8fi6YV09cb5yjNrALCZddxzwxyuPqcKh9VIbzhJIpXl8hWVuaZcKbdGaH1riDXbWlhcqScdTKFzFAxYDknWoDHbBnwtG+nOB8/D1EyKg7X1KFHzqLxPwvgyrqex1LUEcVj1uAdYneFUue1GvE4Tuw/2DNs5BWE4bNnbmQ+ekFu669FX95PJDN/yXdl4mFRnI5lI33EAnb0x1m5r4ak3D7C9tovEoQQKC2sK+NiNc/C5TBR7LaxeVMYzb9XnjwvH0jzyyn5SaYVv/t/bPPf2QV7b1MSfntrB9edP5tx5JbyyIdcnuvtgL+HabTT//guE330DJdM/Mf2JyGY7ks7Qd6Mk0xzW8JVfraG2KXDS74cwsY3rGmhtc5BJxY5hHyAwZ4qX9TvbURQV+dhVfwVhjLT1RPtta+6IkM5m0WpP/1k50bKfzqd+SbqzAY3di++qj6MvnEy8p439B0P89Y0umjpzU04+ftNcLlteid1q4IoVVaycU4wsS7yzo63feUPRJAeag336SlUVXt3URCar5gcnFblN6KNNKIkInY//BJ37uxiLq/ud73h0Lj/eKz5G55M/ByULkoy65DYe3RwllVHYc7D3tPJlCxPPuA2g6YxCQ1uIa8+dPOznnjfVx8sbGjnQHDypVSEEYSQtqCng8aMWtQa4dFklJoPutM+difTS/sgPyIZyaf2yoS7aH/wfXKvvpOeFP1Kk0fH5+dfzt7oC1u8Lce9TO1hQU4DTaqA7lKC5M4JGlij2WZGkfOpcIDe/OpHK9rtmOqOg1cpUFNk4Z14JbotMIG3EWTobmraT7mk7qQAqSRLW6cuRPaXU7qkjqJh4ZFOUurbcg4d4GBZO1rgNoI3tYTJZlcklzmE/97xDaw9u3tshAqhwxphW6eaTN8/jz0/vJJnOct25k1kxxz8s584EO/PB8zA1k0KJ5cYZqNk00oYHuXrVp1m/D2LJDM1dETbubucfL+zNZ/qp9Nv50nsX8/2/biCrqBS6zcyo9KCoar/Aeu25k/E6DOxrCvKbR7fnt79nxWWcU5xBYzl+6r3mjggHWoJkFYUqvyM/h1XSaLH4q+hp0eaXWnNY9ayaU0JNuXM43iphAhm3AfTwAKKqkuHPb+myG6n029myt5P3XDh12M8vCEOVDnSQjQXRWlyYHV4uWVrBoukFZLMqHodp2GpVstGKpNWjZvomJpA0fWu3tnQ3sqxhyfRCXlrfgCxJfdLk1beG6A4m+NnnzycSTdMZjPPTf2ym0G3mA1fPZMveTlLpLNeeO5nZ1T56Q4n8QtiHPbS2nYW33oCk1aNk0sjavmVobAuxbmc7yXSuVtveHSOTzTK59MiUlqUzi/j6B5ZS2xxAq5F56s063t7Ryu2XTGPFHP+wJF4Rxr9xG0DrWoPodZo+6woOp3lTfTz1Zh2JZCY/F00QRouqqsRrN9HxxM9Q4hE0Fge+a/8f5qo5uO3Dk5ZOSSVINO8lXr8NnacU98V30/2v3+Zfty+8jFjt5j7HpPQOLlvmobTAxoZd7YSiqWNPy/6mXq4+ZxIA8WSG//m3lfSGkxS4zFyxohJZlvN5cBvjaZIDNO+Gkiqtf/kaRbd+DfPk+X1ea+yI8OCLe/MrwVhMOnwuE1XFzvwDhcmoY8nMIlKZLN/7y4b8sT9/aAs2i57ls4en5i6Mb+N2FG59S4hKvw3NCPVrLJ5RSCarsHFPx4icXxBOJN3TemhOYwSAbDRIx6M/JB0Yvs9jdM862u7/FsG1/6TryZ8T2fU2/ju/TcENn6Porv9CP+Nc0l2N+f1NNUspnT6TD14zk017Og7Nle4/NWTmUdO/TAYtNRVuls3yM6nEgV6n7ZNE3ucw4jsmT61Bp6HAmtund80jKOlkn9ff2dnWZxm1aDzNrroeskr/0cjPrzvYb9vLGxv7bROEgYzLAKqqKnUtwRFN+D6zyoPNrOetY3KACsJoSAY6UY8JHEoiSqynfVjOn4kE6Hn5r32vWb+VbDSIbvISnqvV8dHf7WfnrE8QWv4xLNd/jYIrPobdV4hep+X68ycTiiZR1VxzqSTlBulcuLiM9p4YiVTmOFeGYCRJT2+YZFsdpnADn7+xmhJvboFst93Il26eik8OIxutqKkEqnL06F21X/IGgEAkiU6r6be92Gvpt83v6b9NEAYyLtseuwIJwrH0iAZQjUZm2awi1mxrIZ3JDvjlFISRktJYQNbkpmMcImn1xDEzLJ96JYuS6h+I1GyaxvYwf3xqBwC/fiE3LcVmTvHjz0ylMBfnmFbh5gefPIfdB3vxOIysXlhGU0eYt7a3Ut8S4qYLq8lEAkgaLRpTrpslGk+zbkcrf3t2N8tnF1NZaEKTyTCzIM2XF/eScFZizERQX/suiSnzMdcsxVQ5G43hSA1VkiQuWlzO1n19BzydN3/gTEMXLq7os/C3yaDlvPklA+4rCMcalwG0rvXQAKLikV0gd8WcYl54p4Gt+7pYNH3g9GKCMBISJi/q0juQ3r4PVAUkmezSOwlpnBQNw/k1NheOJVcTePOh/DZJo0NXNJlkWMFk0PZpJg3H0oSiaQrd0NoV5em1dby2sYmSAivnzivh149uJRjJ9Yf+6MNzSG56io53nkI2WHCvvhPTpHnsrOvhx3/fzFWrqthe28Xjr+e+xz+7owiHLou8/q9IWh3WpVcT2f027vPvwFDcf5ragmkFfPT62fzjhT1IksTtl9Ywd6pvwL+zuszJ9z+xitqmIEgwpdQpFtYWhmx8BtCW3BevcoS/CHOrvViMWt7Y0iwCqDCqCjw2npam4V76WSxqlIhkZXeXgdsXD8+gOUmSsS+4BNloIbz5BXS+CoJzb+N3L7fT0BbmqlVVdPTGeW1Tbhkxj8OI224glcny9+d388rG3PZAJMn+xgAfunYWL29o4MbV1fiCG+l5+T4AspEA7Q9/D/+d3+b1zQlkCWxmfX4UvUGvwZbpJfDmw/my9XQcxH3BXWjtHjTG/s2tdouBq1ZNYuWc3EosrkEykVUWO0YlZ7Aw/ozTABqiyGPGbDz9CeQnotNqWD67mLXbW0ils+h1ohlXGB06rYbViyvZc9DBvpYgpT4rNy10Y7MYBj94iLQ2N86lV2ObewHNvRm+/vM387XOPQ29XLmyiiKPmWQqyxfuXITHYaKlM5IPqocl01ksJh3/9bGVaNJxmv/0bL9rJRp3Ul40F51OQyp9pFl6bpWTxI4n++2f6m7GseSqE5Z/sMAJkE3ESPc0o2Yz6NzFaC0ikApDNz4DaPPIDiA62rnzS3hxfQMbdrWzYk7/tQcFYaQ4rAaWzCxiyczhaLQ9Po3RQn1bU58mW4CX1jfwnY+txOMw4nHk+iG1WhmTUUc03jdPrcmgQa/VoChaNDYXmd6+g+80ZgeLJhXy2Gv7qTyq6yWSzKJY+3+XMwYnDZ0xKopO3Mqkqgqhrm7CSQWH3YLNfqSGngl10/3ivUR3rQVAV1BB4fWfFSuzCEM27kbhJpIZWrujoxZA50zx4rQaeG1z0+A7C8JZaqCEDBqNjNt+JHgCFLjM3H3VjD77TS1z5r+Pst6I+5ybQTry06OxOjGWz6Cq2MH3PnEO2/Z1cctFU7GYdOxuCJCZdnFuwNQhkt5EvW4yawcZAZ9NxNi1s55v/mUbH/vx2/zH799hz4Ejo5TjDTvzwRMg3XGQ8OYXUNXhS74vjG/jrgZa3xZCVUd+ANFhGo3MqrnFPLfuILFEesSbjQVhLEwqduC0GQiEj0ydufOyaXid/ZM2nDu/FL/HQm1zEI/DyLQKd58gayyfQfH7/5tkay2y3oTBPwW9Nzfy1eswUdcSZOPuDi5aXIZBr+XBbUFuvuLL0LaXrKSlTVfKL1/oYtW8EzfRtrd18T8P7iUQyZV5f3OY//7rFn74yVX43BaSzXv6HRM7sAVn6hY0BvMpvU/CxDLuAujhRbRHqwYKcN6CUp5aU8fb77ZywaLyUbuuIIyWYp+V/7pnBe/saKOpI8Ly2X5mThp4PVyTQUtpoQ2zSYvdbKDA3TcYSbIGY3H1gIngjQYtN66u5nt/3ZBPjC/LEucvWsyPX29BUVQSqdzUmeWzT9xl0t4TywfPw3pCCVo6gvjcFgwlNbDhX31eN0+ej6wfvuUPhfFtHAbQIBajlgLX8KQzG4qaChcFbjOvbW4WAVQYtyqK7P36HBOpDNmsisV0pOVlV103P7hvI52BOFaTjk/dMo8lM/1Dzgq2YFoBX37/Yp584wB2s56rz51MeYGV9105g4df2ofLLnPnZdMGzHJ0NKvF2C9BvSxBTyRNU0eYovIZWGasIrrzTQB0hVXY5l2EJI27ni1hhIy/ANocpHIE1gA9EUmSOHdeCY++up9gJInDOnwjIQXhTJTJKrxb28Xfn99DKJri2nMns3y2n0xW4fv3bcgv7B2Jp/n+Xzfwk8+eP+iAn8PMRh0rZhezZEYRkiTlA29+XVFJwmYZPNl7aaGdm1aV8NAbzflt71k9icdfr2PJzDi3XzoN7xUfxbn0apRsBp3HT1IyI4nEKMIQjasAqigq9a0hLloy+rXAc+eX8PDL+1izrYUrVlSN+vUFYTTtbejlP373Vr5298uHtwJQXe7MB8/DMlmVjp7BR8we6+icuIcN9eG0J5Qgns7VXqdXuukKpbBaDLy6pZ3a5iBarcwtF01FYzCjKZ5CVyDGG++08Py6g5T4LNx0wVSmVZ64hisI4yqAtvVESaSyo9r/eVil305ZoY3XNzeLACqMeztqu/s0jQI89lot//HBpZiNWmKJvlNenLaRbZVJpjOEIimMeg0763r41SPb6AklWD67iHlTC/j1430HDK2Y7UdzKEBnFZUn36zj0Vf2A9DUEWHL3i5++KlzRIIF4YTGVWP/kQFEo5+KS5Ikzptfwo4D3XT29s8hKgjjidnUf7S5zazDZTfyqZvn9+nvfN8V0ykrtI1IOTKRXnr3b2Pflq386oH1fOP/3qa9J0Y6k0vG8Nb2NhLJDOfMO5LfduG0gj5ztrsDcZ5840Cf8ybTWQ62hUekzML4Ma5qoHXNQWRZovwkm4qGyznzS7jv2d28saWJG1b3H2EoCOPFrEkerCYdkUMJEyQJbr2kBpNBy7JZRfzks+fT0RPDZTNQVmTDqB/+n5pkx0HaH/oemUA7ZuC90y/g6egc7nt2N1efM4kHX9wLwJ+e2slv/v0Cblw9BVVV8XutfQY9aTQSJoOW9DGLheu046p+IYyA8RVAW0KU+KwYxiilXrHXSnWZk9c2N4sAKoxrFX47//PxVWzf30U0nmLOFB/V5U4gNze60m8f0VzUSiZF4I2HyASOJEZQd73M+Sun8dyWDEcPIfQ4jJiMOkoKBq4Fexwm3nflDH7+4Jb8tiK3iUklovlWOLHxFUBbg0wf447/8xaU8vvH36WpI0zpcb6wgjAejHSQPBElHiF+cHu/7cZkFwa9JT8ASZbgYzfOwT1IXtxVc4vxOU1s299FgcvEnCk+isS6oMIgxk0ADUVTdPbGuXLF2D41rppbzB+eeJfXNzdz+6XTxrQsgjBeySYrpsrZRHe91Wd73OBl8XQ7s6Z4+IJvIcU+65CCvNmoY35NAfNrCkaqyMI4NG4CaG1TAMit5zeWPA4Tsyd7eX1zE7ddUjOq81EFYaKQtXqcq95DsvVAvhnXNPcS1NKpfGShD5dNZBMSRt64CaD7DwXQyaVj32+xal4Jv3p4K43t4TEb0CQI452hoILi932HdE8rss6IzlMs0vAJo2pcBVC/x4LVPHiGkpG2cFquGWjTng4RQAXhFKQDHSQO7iDZVouxbAbG8ulora5++2mtrgG3C8JoGEcBNEhN+ZnxRSpwmSkrtLFxVwfXnTdlrIsjCGeVbCxE51O/InFokFBow7+wLbgUz0XvQ9aJNJnCmWNcTHQKRVN09MSYcgY03x62cFoB7x7oJnHMIsSCIJxYqqspHzwPC296nnRv2xiVSBAGNi5qoEf6P53Del41kya0+QUi218l1d2CxmzDPHUJzuXXDdpstHBaAY+9Vsu22i6WzCga1nIJwnimZtMDbUXNiodR4cwyLmqg+xsDwPAG0HRvG833fpnu5/8ASNjmrkZfUElow79o+t1niNdtO+HxMyd5MOg1bNrdMWxlEoSJQOcpRess7LPNWDEbnUs8iApnlnFTA/V7LVgHyM95KpLt9bT+7RugQuFNX8Q8dUl+Okqqq4n2R39I6wPfoejmr2CeNHfAc+i0GuZM8bJxdzuqqorpLIIwRDq7h8L3fInQpudIHNyBpWYJ1jmr0RhFYgPhzDIuaqD7GgNUD1PtM9XVROv930LS6im5+7tYapb2CX56bynFd/0nek8JHY/+kFRX03HPtXBaIW3dMVq7osNSNkGYKAwF5Xgv+QDF7/8fXOfdht7tH+siCUI/Z30A7QrE6QrEqak8/RG46d42Wv/2LSRJxn/HN9Ed50urMdkovPlLSFod7Q9/HyWdHHC/w9NZNuxuH/B1QRCOT5I1aAwm0XojnLHO+gC6q74H4LRz4KaDHbTe9w3UbBr/7d9A7yk54f46RwG+az9NuruZnlfvH3CfIo+FEp+FjaIfVBDOOGomQ6qrmVR3ixigJJySs74PdHd9D3qd5rQW0c6Eumj927dQkjH8d3wLfUH5kI4zV83FvuhyQu88hXX6coyl/XPfLpxWyLNv1ZNMZ8dslRhBEPrKhLrpXfsI4U0vgCThWHIljiXXoLWdGXPJhbPDuKiBTi135ldfOFnpQActf/062ViIotu+jsE/6aSOd6++E43NQ9ezv0dVsv1eXzitkFRGYfv+rlMqnyAIw0dVVbqDcXpam4lsfw1UBZQswbefIF5/4pH1gnCsszqAJlIZDjSf+hJmydZaWv7yNZREFP/t38BYMvWkzyHrjXguvptUex2hjc/1e33WZA96nYaNoh9UEMZUVyDO/c/t5uM/eIWvPNxOw+LPIhUeyRQW3bNuDEsnnI3O6gC6ryFAVlGZdpIBVFUVguufpuUvX0OSJPx3fhtj8amn3LNMW4apag69r/2dbCzc5zW97vB0FtEPKghj6dWNjfzjhb1E42nae2L88MkmeqZem3/d4BdpN4WTc1YH0K37OpElmFnlGdL+qpIl8u4bNP/+C3Q//0eM5TMp+cD3MRRWnlY5JEnCc/HdKKkEgbWP9Ht94bQCWruitHRFTus6giCcPDWbIRhO8q+36vu9tq9XRjaY0TqLsNQsGf3CCWe1s3oQ0ZZ9nVSXu7AMkkBBSSUIb32Z4LonyAQ70XlKKLj2/2GZuWrYhsjrfeVYZ59PcMO/sC+6Ap3zyMK8Cw5NZ9m4q4Pic6zDcj1BEE4sHewitncdkXdfRzfzQrxOEx298T77uD1uCmZ+Fr2vDK3DO0YlFc5WZ20NNBpPs6+hl3nVvuPuo6QSBN56jIZffozu5/+AxpbLcFL60Z9gnXXOsM8vc593K5Ik0/va3/tsL/Za8Xstoh9UEEaJkkkTWPMw3c//kWTLfiIv/Z73LLSikY985wtcJmbVFGOeMl8ET+GUnLU10O21XSgqzJ06cACN12+n86lfkQl2YJo0D9eq92As6z/NZDhp7R7si68g+NbjOJZeg6GoKv/awmkFPP/2QTGdRRBGQSbQQXjLS0c2KFm8G37Hdz/4JRqDYNRrmFzmpNgrWoSEU3fW1kC37O3EoNcwraLvvC1VVQmsezKXUUijwX/Xt/Hf9vURD56HOVfcgGyy0PPKfX22L55eRCqjsGWPGEwkCCNOlkDq+/OmhDoo13Zz8dIKzplfKoKncNrOygCqqirrdrQxd4oPnbZvba73tb/T8+K9mGuWUPLBH2IqnzmqZdMYLThX3kj8wBZidVvz22dP8WI16XhzW8uolkcQJiKdsxDHsmv6bNO6/Oh9Q0uSIghDcVY24e5vCtAViHPHpX1rlYG3HiOw5hFs8y7Ce8VHkaSxeT6wL7yM0Ppn6Hnpr5g+OBtJktFpZZbP9vPm1hZS6Sx60YwrCCNGkjU4Fl+JvqCC+P6N6AurMFcvFH2dwrA6K2uga7a2IMsSS2YeWR8wtn8TPS/fh2XGSryXf2TMgieArNXjPv8OUu11RN59I7991dwS4skMm0UzriCMOK3ViW3mKgqu/TTOZdcMmt9aEE7WWRdAs1mFVzY2sqCmALtFD+TS8XU88VP0BeX4rvo4kjz2tTvLzJXoiybR++r9KJkUAHOqvdjMel7ZePwl0ARBEISzw1kXQDfu6aAnlOTiJbm+DCWTov2RH6IqCoU3fgFZZxjjEuZIkoznwveSCXURWv8MAFqNzIWLy3j73VZ6QokxLqEgCIJwOs66APr4a7W4bAYWz8g133Y//ydSbbUUXP3J467fOVZMlbMxTV5AYM0j+RR/ly+vJKuoPL/u4BiXThAEQTgdZ1UA3VnXzbb9Xdywego6rUx42yuENz+PY/l1Z2waLs+Fd6GkEvS+/g8Ain1W5k318exb9aQzyhiXThAEQThVZ00AzWQVfvPoNtx2A5ctqyTZdoCuf/0OY8VM3OffPtbFOy69rxz7ossIbXyORMt+AK47bzLdwQQvrm8Y49IJgiAIp+qsCKCqqvL7x9+lriXEPTfMQZeN0/7wD5BNVgqu++wZMWjoRNzn3YbG6qTrX79FVbIsqClgWoWLfzy/h1giPdbFEwRBEE7BGR9AW7oifP+vG3h6TR3Xnz+FZdN9tP/zR2QiPRTe+EW0VudYF3FQssGcWzO07QCBtx5DkiQ+eM0sesMJ/vbs7rEuniAIgnAKzqhECllF5d39Xeys76G2KcD+pgDdwQQaWeKuy6dz0+pJdDz+ExL12/Fd80mMJdVjXeQhs0xfgWXPOnpf+wemytlMq5zKFSuqeOKNA8yc5GHFnOKxLuKISiQzNHeEMWbCWNLdGM0mdJ5SNKaTS6fW1h0hEejBlOhEb9AR1HhQdBZKCq0YdLmPc28wSqanBWK9mN0FmAvLiHa2k+ppQqvVIusNGJ0FqJJMqreN9oydtjDYbCbKCix0B5N0BuJYdCoGrUQwFMXtsmF3WGlu7UVS0hTaJEzZGFopi8FXRndST0t3hGg8TYnPSnmRvU/icuHMl0xnaG6PEIgkKXCbKfFZh33BiWNlswpNHRG6g3HcDhOlBVai8TSNHWFURaW0wIbLbhzy+YKRJE0dEWQZkikFWYLSAisOg0KouZ5ELIpqK0Q1uekMxglFUzhtBlRFJZHOYjXpULIqiXQGvU5LOJbCatKRzmRRVDAZNBh0GjJZlUgsTSqj4HMZc8eksngdJrpCceLJLBajFrNRSyKZIZ7MklVUrCYdGUVBAjSyjM2ip8RnobM3TmcgjpJJU2mKQDxI1uSmNWVF0mgpK7ThtB2ZYZHJKjR3RghHU0TiKdx2I9F4Br1Ok/t7raMzG+OMCKCqqvLa5mb+/NQOuoIJJCm3gsmsSV6qy52smluMyyTR8fD3ie3fiPui92Gbff5YF/ukSJKE9/KPkmzaQ8djP6HkA9/ng9fMZH9TgB//fRNuh5FpFSe3MPjZIhpP88gr+6ixRSnY9DuSkR4AzDXL8F76AbS2oa3nWtsUoKfhAL7NfyQaaCUK6Mtn8bb9MrbYvVyxsoqeQIzknjWob/4JlCxhWYPrsnuIbHiKdEdu5LNl+grUmqUE1v6Txprb+NHTB0hlFCQJPn/HQh5/vZauQIKrVlVx/3N7yGQVZFni5gunsmZbC43tYeZV2fnwSjPypodRbQWs917Dn5/ZjaLmpit9/o4FrJwrJu6fLRLJDE+vqePPz+xEVcGg0/Dl9y9m4bTCEbumoqi8saWZn/xjM1lFRSNLfO0DS3nwxT3squ8FoLzQxpffv5jSAtug52vrjvLTBzYzqdjBnoO97GnIneOu1cWck3mL+I7XAJAnL2GN4yoeeGl/7m/Va7jrsunc//xuJEni7qtmEk+mue9fu0mms0gSXLVyEnsbeyn2Wpg1ycOWfV28saUZAIdVz3sunEognGTj7nbqWkIAVPpt3HJxDS+tb2DDrlzyGLfdyPXnTyaWyNDcGcZtNzGjyk19a4i1W5r49KIYgbf/CqoCshbTyg/w36+By2bkc3cspMRnJZtVeG1TE9FEmgde2MvV50zij0/upLUrCsDsyR4+fet8Ct2W4b1hAxjzJtxgJMl3/7KeH/1tI26HkS+9dzEPfOdKfvOlC/n8nQu59tzJWCKNNP/xC8RqN+O97CM4l14z+InPQBqjhYLrP0Mm1E37w99Hi8KX37cYl83I136zlvU728a6iCOirjXEjn3tFDa/gnooeALE9rxNomnvkM+zZW87nvZ3UAKt+W2phndZ4Ojm/uf30NIZJdzaiLrmXlCyuR2ULIHn/w9LzbL8MdFda1HiYRLlS/nli52kDo2GNht17G8KsLchwAWLynjopX1ksrnXFEXlwZf2cs68XEvBlroQOztVLNOXEfTNywdPyD0d/+zBLbR2R0/l7RLGQEN7mHufzgVPgGQ6y0/+vpmuQPzEB56Glq4IP39wC9lDHxyjXsO2/V354Hm4XK8OMfHKpj0d1DYF0es0+eAJsMyfygdPgN6ipfzjxf1H/tZUln++tp9z55cSjafpCcZ59JX9JNO575CqwpNvHmDx9EJe2diEViPngydAMJJi+75OVFXNB0+A+tYwtU1BapuC+W09oQSb93Tybm0XxV4r1WVOGtvDPPjiPq5fYEFed18ueAIoGbRv/5n3LHKwrzHA29tz3/vmzghvbW/h9U3NWIw62rpj+eAJsL22my17O4f0np2uMQ2gb21v4eM/eJl3drTz/itn8P1PnsvKucWYDFqUVILI7rdoe+C/abn3yyjpFP7bvo594aVjWeTTZiydhu/qj5No2EH7oz/EZdHwvU+sosRr5dt/WMfvH3+XdCY71sUcVj3BONVFBqSOff1eS3UN7cchq6jEY3H0Xf0Drj548FATTppEqAeymT6vq5kUkqZvY0s2FiKmsROKpvLbClwmmjoiAMiyRDzZ9zyKopLJqPl/13UpSLKWYEafD56HxRIZQuHkkP42YewNlNgkEEkSio7cPQxEkvmHNwCfy0xTR7jfflv2duYf5E5kd30PHoeRlq5IfpssS+iTgb7XTfVveOwOJrCbc5ndMopK7wCf3VQ6V4bgUd+Zw2LJbJ/gmS/TwR783r41wQPNQfxeK4qS+54k0wqZrIKF+JEH30PUdBK7NleWrftyQTEQTlLss3GgJUhJgZW6liDH2n2wt9+2kTAmATQcS/Gj+zfy3/eux+Mw8b8fX8oVkxKE1z1Ox+M/pen/Pkv9j95HxyM/JNGyD9e5t1L2kR9jqpozFsUddrZZ5+K97MPE9m2g7f5vYyPK9z91DleurOLx12v5+PdfYd27raiqOvjJzgIFLjPbGuMoJf3vn6Gwckjn0MgSNpuVpL//ORKuKfQE49gtekwuH9Ix2agkvQkl3fcHUmtzY0124nEc6V9q7YpS6bcDkEpnsZl1fY7RaWXko74xNQUa1EwalzaOVtO3r8xh1eN2DL3vShhbPpeJY7s7C9ymk+p/PFkeuxGT4Ugwa+uOUnXo83e05bP9aDWD/1TPrfbR0ROj7KjmXkVRiev7dg159P1H/vu9FroPPURoZYlCt7nP65IEel2uDC5b//5Fu0XH1DLngGVqbO/7UDC9yk1Dey7Ymo1aDDoZg15DSDEjafV99pWNFrpSuesdzn3ucZg42BZkepWb+tYgNeV9l7QEmDNldBYNGNUAqigqL66r457/fp43NjVxTWWQz5gfQ73v32j92zfpefmvxA/uQGPz4Fx2Nf47vknFp3+P65z3IBvMg1/gLGJfeBkF132GZNsBmn7/OZLbX+Kj183k2x9ZjlYr8V9/eoev/notm/Z0nPWBtLLYzqXLq2hwL0NTMCm3UZJxLLkKQ8nUIZ9n3tQCujzz0JRMz28zzVrN2g4b/3bTXIp9FuzFZcir78l/XmSjBfdVnyJetz13gKzBvuhyJJ0BY88+PnWRG+ehAQcqUF3mZNmsIl5a38AtF9Xkg6jJoOX9V87gpQ2NSBJcPM9LjQ8i21/DFT7Ax66flf8xtFv0fO6Ohfhc4+szO56VF9r4f7fOzy9277IZ+NztC3HZRi6A+r1WvnjXIiym3GdMI0vMmeLlvAWl+X0WTisY8gDDudU+Vs4tpjuUYPnsI1nZ1ncYsS6/Kb8+qr1tAx+7fgZ6be7fbruRq1ZV8cbmJoo8ZnwuE9ecMykfKA06DbdcXMOarc1ce+5k0pks1547CfnQILnyIhs1FW5C8TQLpxXkr7ugpoDSAivLZvk5PJ6u0m9nWoWL5bP8hGNJttd2MaXUyZ2XTeOxLRHSKz+EdPi7a7KSWP5hHlrXy4rZfpYcyj5X7LNwxYoqls4owqDTYjHpmFF15CHhwsVlzB6lACqdzI/zokWL1A0bNgx5fyUZJ9nZyMH99azf08lr9RraUyYqtR3cbF5HmT2LsbgaQ/EUDMXVGPyT0Zj7P4GNZ6muJjqf/hXJpj1o7F5ss89DVz6HVxr1PPLqAXpCSUp8Vs6dX8LCaQVMKnH0WwN1IJIkbVRVddGx20/2Hg6XrKLS0hVBk45hywbQG43o3H7kY544BxOKJOnt6cWU7MFg0BOQnWj1Rop9lvyIyWg8TbyrFTkVxuR0Y/L4iQV6SHa3otXIaAwGDA4fSBLpQAfdaRMdMQmrxUBFkZ2O3hjdwQQmrYpJoxCKJHC67DiduVG4ZFO4zaBV0hi1KjqXn0hGpqUrSiKZxe+1UOQ5/QEMZ9o9HO9UVc31o8dSeJ0mvE7TaZ9zKPewvTtGbziB02agyGMhnszQ2hVBUXI1w8MBdigSqQwtnREkJDKKioSK32vFrINwWyPxaIysxYPR7qSjJ0YknsZq0qGqKooKBp2WVCZDVgG9ViYSS2E26nJNzaqK0aDFaNCQTGWJJ7OkMwoeu4GsCpmMgttuIBBJ5kbEamVMRi2pdJZkWkE9dHwmq6KRVTSyjNWsp9hrpTsYp6MnRjaTodQYRUpFSRucdGcsaLUyxV4LJmPf96GtO0o4miKezOK06Umksui0Mn6vBaN++MbHHu8ewmkE0Ey4l8Bbj6KmU6iZ3H/NIYmX2xzEkgqxlEIko6NDcRBXcz+SleYIl0+FVQsqMJVMRevwjfgw8bOBqqrE9m0gtPFfxA9sI1cXklBsPjYrNawLFLA3bEZFQquRKXDlvtx2ix6jXotRr+HWS2r6DN0WP75nP3EPz37iHp79hi2ASpLUCYgs6GeHClVVfcduFPfwrCLu4dlP3MOz34D3EE4ygAqCIAiCkDPm80AFQRAE4WwkAqggCIIgnAIRQAVBEAThFJwRuXCF0eP1etXKysqxLoYwBBs3buwaaPCCuIdnD3EPz37Hu4cgAuiEU1lZiRg+f3aQJGnAUZriHp49xD08+x3vHoIIoIIgCKclHewk2VqLEguj85VhKJqErBtagpD2xiboaUSJ9iDbC8FdQmGxf/ADhWHVVN9EbVuMWDJLRYGZ4iIrTtfgq0SJACoIgnCK0qEu2h/+Aam22vy2ghs+j3X68kGP7WxtI/n2A2T2rs1vMy5/DyHr5djtjhEpr9Bf08FmvvP3XTR1xQCQJfjqnXNYMoQAKgYRCcIhqqqy40A33cGRW8JKGF9SbXV9gidA9wt/JBMJDHqsEmzvEzwBEuv+SbJndJbiEnJqW6P54AmgqHDvc7V0t7UPeqyogQrCIQ++uJf7nt2Nzazj559fjcdx+rlQhfFNScaRl95Cq2cJ0aRCsSWD6fWfoWb6L/k10LH9N2ZQUmIZvNEUTWT6besJp0gPYQk5EUAFAUgkMzz08j4mlThoaAvxyCv7+ch1s8e6WMIZLuSexl/W6Vj/r60AaDUy/37XV6i09V9i61halx/ZZEWJH1m/U1NQhdYxeNOhMHwqi6xIEhydlO/ShUVYHbbjH3SIaMIVBGD9znaSqSwfumYWS2f5eW1TE8qxq2QLwjFqO1Ks39WR/3cmq/Cnp3bSFey/5uaxfOUV2K/9d7SlM5F0RrRTlmK95B48hUUjWWThGMWFFr5652xKCyxYjFquX1nKJQsKsVqtgx4raqCCAKzZ1oLLZmDGJA+dgThrtrZQ3xpiUokYzCEcXyDcv7m1tTtKIJLA5x58PVj35Bng+H8oySh6sxW7a/CaqzC8nC4PS10ephRZSGcUrE7bkIIniAAqCKiqys66buZN9eUXNQbYtr9LBFDhhArd/fvJ5031YTcOvXHP7XUD7kH3E0aWp6jwpI8RTbjChNfRG6c3nKSmIvcj5nWa8HstvFvbNcYlE850JS4tH7h6JtZDi17PqHJz0+op+FyD1z6Fs5+ogQoT3p6DPQBMqzjSfFZT7hIBVBiU12VjcXmMyXfNJ5FWcZpUCi3JISdSEM5uogYqTHj7GgPotDKVfnt+W1Wxg65gglB08OkIwsR1oD3Gl/+8mz17DhJsaeCRV2r58RNNJJL9p0YI44+ogQoTXkN7mLICGxrNkefJquJcMK1rCTK3esA80oJwaMBQkr+81nfSfSCSpMggfl7HO1EDFSa8hrYw5UV953xVFecGD9W1BMeiSMJZwmbq31RbXmTDcqhPVBjfRAAVJrRYIk1XIN4vgDptBhxWPY3tkeMcKQig00osm3Uk+btRr+GKFZWkM9kxLJUwWkQbgzChNbSHASgv7J91pNhrpaVLBFDh+IKRNJmswh2XTSOTUVCBB1/cx+LpIhnCRCACqDChtXZFASj29Z847fda2LJXJPYWjq/Cb2Pbg51s2HWkD/T68ybjcYo8yhOBCKDChNbZm0vo7XP1/8Er9lp4eUMjiWQGoxgQIgzAadNzz41z2LavC0kCu0XP4hmFaGRprIsmjALRBypMaJ2BOA6rHqO+f4As9uZqpa3d0dEulnCWONAcxqjXYjXrCEaS+L1WdhzoJhgRK6pMBOKxWpjQOnpj+I7T3Ob3WQBo6YrmR+UKwtF0GonfPLotP194055ObrukRgwimiBEDVSY0Dp748dNu1bsPRRAO8VAImFgHb2xfsk2HnutlrhIpDAhiAAqTFiqqtIVOH4N1GzU4bQZ8gONBOFYGgauaSpDWIxZOPuJACpMWNF4mngye8LE336PRfSBCsdV7LFgM/dNmnDdeZNx6UQf6EQg+kCFCavjBCNwDytwmdl9KNm8IByrPZjmhtXVNHdG6OyNM73KTWdvnJhahH3ww4WznKiBChNWZ28M4LhNuJALrt3BOFlFHa1iCWeRUCzFn5/eyfb9XSRSGR59eR+vbmoknBBNuBOBqIEKE1ZnYCg1UBOZrEognMDjOLMmx2ezCk2dEXqCCbxOI8U+m5h/OMoKXGa0Gpn2nhjtPbkHstULSzHqRN1kIhABVJiwOnvj6LQyDovhuPsc7h/t7I2fUQE0m1V4ZWMjv3hoK1lFRauR+X+3zefceSVIkgiio8Vi1PCR62bxwjsNdAbiLJlRyNKZRWREg8WEIAKoMGEdngMqn6DWdrh5t6M3xrRK92gVbVAN7WF++fBWygst1PhN7GyO8fMHt1Dlt1NeJHrfRouaSaGoKjeuriaTVdBpJeqaeyn2HX9gmjB+iAAqTFidgfgJm2/hSPPu4ZR/Z4r27iifvaKI0u51aNp2cUX1LOrti+joiYkAOooK000Esxr+9FQvoWiSxdUOrq2KYsMJiOQb450IoMKE1dkbZ0FNwQn3MRt1WEy6fH/pmaLcoZJ+/S9kA61kAALtVPlq0dV8fqyLNqFkkwmK1/2Or8y+gKzFi/7gMygbmshUzx7rogmjQPR0CxNSOqPQG04MWgOF3ECijkMjds8U1nQP2UBrn21KZz32bGBsCjRBZRwlyHojmnArmlArRHswLrmRhMYy1kUTRoGogQoTUncwjqrmguNgfE7zGRdA9caBBz7pDccfECUMvx7sxM//Bk+uOUhrXZzV81dQabWz0CHuw0QgaqDChJRfxsw5+GCPApcpP2f0TJE0+dBNWdpnm37G+aRM3jEq0cQUS2T53v1b2VEfIJrI8MDLdazf00N3KDX4wcJZT9RAhQmpM3AoicJQaqAuE9FEhmg8jcWkG3T/0VDXlUaefj36wrnoQ02k7GWEXFUEu1LMFGNXRk1vKMHd5/moNnShS/YStpTxyPYQyZRYjWUiEAFUmJAOp/HzniAL0WH5uaCB+BkTQFVV5dv37yGrKNgtpQQjCQz6vXzl/UvGumgTSo0ni7z5PpRD/dF24O5zPoRWK+biTgSiCVeYkDp74zhtBvQ6zaD7HpnKcuY04ybTCsl0lkxWpSeUIKuoxBIZsQ7lKHOkOvLB8zDNpodwakUT7kQgAqgwIXWeYCHtYx3er+sMmsri91jQavrWcgw6DUUeMfpzNGWTif7bElGUbHoMSiOMNhFAhQlpKEkUDnPajGhk6YyaC1paYOXjN83N577VaiQ+dcs8ir3WMS7ZxCK5S5A0fZv1TXMuJoJtjEokjCbRBypMOKqq0tEbZ+G0wiHtr5ElPA7jGRVAZVliWUGU4musBBIyLqNCuTdywrSEwvDbFbDgOvdT2Pf+C0LtZCev5B2mM08V92EiEAFUmHBC0RSpdHZIc0AP8zpNZ1QTbrq3jc6HvoMxlaDo0LbO9Vb0H/gBOueJsysJwyccT/PkxgzXrvogNr3KrpYUj72+nzlza8a6aMIoEAFUmHCGsozZsXzOM2th7UyoCzXVt/9NiUfIhLpEAB1FVX4bxuWV7G4MEYgkqfLb+eztC7FbzozR2sLIEgFUmHBOJonCYV6nke5tcRRFPSOaSbVmO0gyqEcWbpY0OjRmkUh+NKUyCvc9u4uuQO5h5jXg9ktq8LvNeMa2aMIoEIOIhAnn8HSUk6uB5hbWDkaSI1WskyLpDDiWXQscDuYSjhXXI+nOnDVLJ4LO3lg+eB725JsHSKbFdKKJQNRAhQmnMxBHr9Ngt+iHfMzhhAudgTguu3GkijZk6WAn8QNbcJ13C2omjaTVEdm5BlPlbHQOUfcZLRq1f6BMprJoRNVkQhABVJhwOnvj+JwmJGnoTbFHZyOaWu4aqaINmcZkJdVeT6q97shGWYNsEtMnRlORx4xRryFxVOq+q1ZV4dbGgbH/nAgjSwRQYcLpDMROqvkWjtRAz5SRuDp3CaYrPk1DUKYnqcVjSFPpltG7/WNdtAklGM9y52XT2bqvk85AnPk1PpAkEqpBLKc9AYgAKkw4Hb1xFk8/ucE2NrMOvU5zxgTQRDLF0wfMPL62Kb/tlvPKeU91CoNJfK1HS1cwye+feJfqMieFbjMvvtNAOJZm1Rw/Q5tlLJzNxDdNmFBS6SyBcDLfJDtUkiThc545yRSaWgN9gifAQ683sGJWIZMmndzfJpy6QleuP3xfYyC/bW61F4tRdIJOBOIuCxNKVzAXAE8micJhPqeZrt4zI4DGkpl+2xR14O3CyCnV9vCBq6ZjMuTqIpNLHNx+bgmWbHiMSyaMBlEDFSaU/BzQUwigXqeJTXs6hrtIp8Rf4MBpNVDsNjDNb+Td5ji90QxFPtHzNpqsaoxFjQ8x+/13k1J12MJ1sPYXcM3nxrpowigQAVSYUPJzQE8iicJhXqeJ3nCCTFZBO8bzFAoLXPz07klE3nkCtXUXF5bPxrboKlxeEUBHk6o30TPrVh56oYHW3iSrZzpYfe6HURVl8IOFs54IoMKE0plfSPvk53J6nSZUFbqDCQrdY9vPmAn3EHnqx2SCh2rEu98g0lWP7c5vobWIIDpa2rMOvvXANtKZXMB84M04sayGm891jm3BhFEh+kCFCaUzEMdtN6DTDr6Q9rEON/ueCSNx453NR4LnIemuRhJdLWNUoompsSuRD56H/eudVsLJsU/3KIw8UQMVJpRcEoVTqz36jspGNNZSqgYkGXnyMtKeyeg696LUrSelnPyDgXDqDPr+P6EWkxbjANuF8UfUQIUJpaM3hvcUBhDBmZVMIaj1EL/im/yqdQGfe1bid11LSF71LQIa51gXbUKpcMkUe/p+nt67uhiTmIs7IYi7LEwYqqrSGYizZGbR4DsPwGTQYjXp8gORxpKMynf/2UjgUHL77fVBfhBM8Y33ijy4o0nfXctnF8eoU/x0xxSq3Sq+ukdQpn8U7GJlnPFOBFBhwghGUqQzyilNYTkst7B2YvAdR1hPKJ4Pnod19sYJhce+bBOJ3ulB98KvqdHokPQGlJ0RtEVTMFrFQK6JQDThChNGZyBXcyw4ySxER8sF0LFvwrWaDf22yRJYBtgujBzZW4Fh/lWo2TRKPIJstmM5/33oLdaxLpowCkQAFSaMjvxC2qdeA/U5TWfEIKLSYg/Xrijts+0951dSWuweoxJNTLtbkvyqtpKW5Z+jc+kn2FR9Dz94roeAaAmYEEQTrjBhHMlCdOo1UJ/LRDiWIpHKjOlIS7PJwKUrJjFnioeeUAKvw4S/wIHeMPQ1ToXT1xVMsK0+xLb6w1tCAAQiSZy2sV83VhhZIoAKE0ZHbwyTQYPNrDvlcxw9Ere0YOzW3gyEE+w8GGTPwV6Mei17W+JMj4PdZsZ2EguFC6fHbun/WZpa7kInVtSeEMRdFiaM9u4YhW7LSS2kfawzZSpLW08UjQSLyyTm2HtZXJb7m9q6o2NaronGqNdyxYpK5EMfKY/DyOqFpRgNom4yEYi7LEwYHb2x0xpABEclUxjjVVnUrEJVphZ5zR9R0wkkvQnlnI+QUWeOabkmmtJCG0+tOcDNF9egKirheBq9ToPHcer97MLZQ9RAhQlBVVXae2IUuE/vh83jMCJJY18DdagB5Dd/j5rODVZRU3HkN36HLRsc03JNNH6PhQ9fOxuHRU8ilWHx9AJWzike62IJo0TUQIUJIRxLE09mKHRbTus8Oq0Gp9Uw5iNx9akwiUyqzzY1FceQiYxRiSau0gLbmPaHC2NH1ECFCaGjJzcHtPA0a6BwZswFNbu9oOn7/CvpDFhcYhqLIIwWEUCFCaG993AAPb0aKOSmsnSMcR+oyVeC5/KPgZwLopJGh+fKj2PwloxpuQRhIhFNuMKE0N59KAvRMKzjWei2sH5nO4qiIstjs2yVJGuwzz4HY/FklHgYjdmOzu0/rRHGgiCcHFEDFSaEjt4YFmMuGfzp8nstpDMK3cGxzTaTjQZIdzYQ27+RVGcj2WhoTMsjCBONqIEKE0J7T2xYmm8Bij2587R2R04rMf3pyCaidL9wL9Fda/PbbHMvxHPJB5D1IgOOIIwGUQMVJoSO3tOfwnKY33sogHaNXdKCdHdzn+AJEN76EumeljEqkSBMPCKACuOeqqp0DGMN1OM0odVIYxpA1Wx64O2ZgbcLgjD8RAAVxr1AOEkilaVwGAYQAWhkiUK3hdYxTJuncxejdfedsK8vnITW7R+jEgnCxCP6QIVxr6kzl1ygpGD41mj0ey1jWgPVWl0U3vQFQu88Tbx+O+YpC7AvvAyt2T5mZRKEiUYEUGHcazkUQEt9wxdAi70W3q3tQlXVMZs6YvCV4738IyiJGLLRjCRrxqQcgjBRiSZcYdxr7oyi08r5lVSGQ5HHQiKVJRBODts5T4Uka9CYbSJ4CsIYEAFUGPeaOyIUey3DmvQgPxJ3jJcPy8SCpDqbyMbEHFBBGG2iCVcYVYqisrexl3RaYVqlG5125J/hmjsjVPiHN9l38aEA2tIZZUaVZ1jPPVTxxp10PvVrMj0t6Lxl+K78GMbSmjEpiyBMRCKACqMmGEnynT+9w676HiAXhL569xLKi0Zu4Esmq9DWHWXFnOEdnVrgNqPVSDR3js3qJ+neNtof/C6SbxJK+VKUzj20P/x9Su7+HlqHd0zKJAgTjWjCFUZFKp3lG//3FrVNAf7tprl86b2LiSUz/Mfv3qInNHIp8Tp6YmQVlWLv8A0gAtBqZEp8Vg62jU3TaTrURWzZR/hL4gK+/pabv2cvIbrovaSDHWNSHkGYiEQAFUbF/c/tprYpyBfuWsTlyytZObeYb39kOeFoit/+c9uIXfdgWxiAssLhDaAAFUX2/PlHW1jn4YcvR3lnTy+ReJo1O3v42RspovqxaU4WhIlIBFBhxLV0RXj89VouXlLOsllHmlKrih3ccnENa7e1smFX+4hcu74liCTlgt1wK/fb6OiJEUuMfvaf9qiGzkDfmntzV4yOuBiNKwijRQRQYcT95ZldaDUyd14+vd9r158/hWKvhT8/vRNFUYf92nWtIfweC0bD8Hf3Hw7Kje2jXws1GvqvKiNJYBpguyAII0MEUGFENXdGWLuthWvOnYzb3n+VEJ1W5paLa6hvDbFuR9uwX7++NURVsWPQ/VRVIdleT3T3OmIHtpCNDR4UDwfQsWjGLXbIXDTf12fbVUv8FNnFuEBBGC3i2yaMqMdfr0Ujy1y1quq4+5w3v4R/vLCHB1/ay7JZRcOW2SeezNDWHeWCRWXH3UdVsoS3vkxg7aNkAkcNwJFkLNOW4r7gveicBQMeW+g2o9dpxmQgkSbUwlXGzSy6agGtERm/VaGoYy1SyAXW6lEvjyBMRCKACiMmFE3x0vpGVi8sxWU7/hqVGo3MtedO5jePbmN3fS/Tq9zDcv2DbSFUFar8A/d/ZqNBOh7/CfG6bRhKpuJa9R70BZUoqRix/ZsIbXyO2P5NeC/9ELa5F/Q7XpYlqort1DYFh6W8J0OSJKQ9L+PnZYq1etRMKrf9nKtGvSyCMFGJACqMmGffqieVznLtuZMH3feCRWX89ZmdPPFG7bAF0APNucBWOUATbibcS8tfvkI2EsB7xcewzbuwT83XVDELx6LL6Xzql3Q+9UvSPa24zr+9X+14armL59cdJJtV0GhGr0dE5y7GWD6TRMOOfPA0TVkkVmMRhFEk+kCFEZHOZHnqzQPMn+qj4jg1wKOZDFouXlrB2u2tdPbGh6UMu+p7cNoMFLj65sBVElHa/vGfZKMh/Hd+C/v8iwZsNtY6fBTd+jVs8y4isPZRel/7e799ppa7SKayo94PqjHb8V31cTyXfBBz9SK8l30Y76UfRGMcnjVPBUEYnAigwoh4Y0szveEk150/ZcjHXLVqEqgqz6ytG5Yy7KrrYXqlu09wVDIp2h76HqmuZgpv+gLGkqknPIek0eK94p5cEF3zCIG3Huvzek25C4A9Db3DUuaToXMV4lh8BUU3fxn7wsuO21crCMLIEAFUGHaqqvLYa7WUF9mYP9U3+AGHFLrNLJ3l57m360mkMqdVhp5QgvaeGDOOag5WlSwdj/2ERMMOCq75BOZJ84Z0LkmS8F7+ESwzVtLz8l+J7Fqbf63IY8Zu0bP34OgHUEEQxpYIoMKw27a/i7qWENeeO/mkR9Rec84kwrE0r2xsOq0y7KrL5dudVpkLoKqq0vWv3xHbsw7PJR/AOvOckzqfJGsouPqTGEqn0fnEz0m21ua2SxJTy13squ8+rfIKgnD2EQFUGHaPv16Lw6rn/AWl/V5T0knSvW2kuppQkv37OmdO8jCl1MFjr+4nexqJFXbWdaPXykwucQLQ+9o/CG95EeeKG3AsvvKUzilpdRTd9EU0FgdtD32XTDhX65w31UdzZ5T2ntgpl1cQhLOPGIUrDKumjjDrd7Zz2yU16HW5tHLZaJDwtleI7FxLqr0OVCW/v9ZVhKV6Ebb5F6P3liJJEjesrub7f93A2++2snJO8SmVY9OeDmZO8qDTygTXP0NgzcPY5l6I6/zbT+vv01gcFL7nS7T8+au0P/Rd/Hd9mwU1BflrXr688rTOLwjC2UMEUGFYPfZaLTqtzOUrKskmogTfeozg+qdR00kMJVNxrrgenduPJGvJhDqJN+witPE5gu88hXnqEtwX3MmKOcX4PRYeeXkfK2b7T7oZuK07SlNHhMuXVxLZuYbu5/+IeepivFd8dFiSNBgKKym49tO0P/x9Op/4GSXXfxafy8RmEUAFYUIRAVQYNm3dUV58p4FLl1Vg6NhF01O/IhvpwTJzVS5Jgbd/k65zRa6GGtr4HIF1T9D0u89gX3gZ165cyW+e2M22/V3MrR76QCQgnxJwprGdjsd/irFsGgXXfQZJHr5E65aaJbgveh89L96L9uX7WFAzl9c3N5NKZ/M1b0EQxjfRByoMm78/vweNLHGRZj1t//gvZKOZ4ru/R+F1nxkweB6msThwnXszZR/7Bba5FxBa/ww1m36Iyyzxl1NIMv/apiaqvDqkl36KoWgSRbd8BVlnON0/rx/HkquwL7qC4LonmGdsIp7MjEg+X0EQzkwigArD4mBriFc2NrLKcgDNjmdxLL2Gkg/+AGPx0OeBaq1OfFfcQ8nd38XkcHO59CZ7GwO88vrWIZ+jsT3MvsYAc2Jvoy+ooOi2ryMbzKfyJw1KkiQ8F78f89TFFG39Mx6LzIvrG0bkWgPJZhUa2kJs2dtBU3t4RFazEUZWMpVhV103a7Y2s6+xF0VRBj9IGHZtHT28u7uRDVvraGgc+qL0oglXOG2ZVJKf/uEFzGS41FGL//pvYSqfecrnMxRPofju/+HyzS/yxoMt3Pt0lJr4ZoovuOWEwTAbj/DAXx9Hi5lzq00U3/JlZIPpuPsPB0nWUHDdZ1D+8V8s3LOVF/bMpqM3RoFrZIL2YdmswutbmvnZA1vIZBX0WpnP3r6QFXNOvs9YGBvxZJrn3j7IvU/tJKuoGPQaPnXLPM6dd/zWGmH4NTS28+SbB3l2QysAbruRL98xm2lTBh/AKGqgwmlJtuzn/v/9Gft6tdxcE2faPd87reB5mCTJOBdcwsfffxFBxcyfXu+h4Vcfp/vlv5Jo2Y+qZIHc/M5UZyM9r/2DrT/7HGtbTayqgOo7vzjiwfMwWWeg6JavsnqyhKxmuf/vz6OqI1sbbO6M5IMnQCqj8ON/bKKlKzqi1xWGz/6mIH96ckd+ulYyleU3j2yjrnX0FyeYyFo6o/ngCbkkLH9/cT+d3YFBjxU1UOGUZOMRAm8+xNa17/Bo6BIWVBi5/kPvG/baz8xppdx8UZgHXpSo1lpY8vYTBN96DGQtGpMVJRlDzaRQVXhcuh5Jq+Ouuy4a1gFDQyHrjcy48/Os+sk/eLXWxrl/+wXz3vPBEWs+7g0l88HzsGQqSyCcoMRnHZFrCsOrOxDn2Fb3cCxNIJQA/+Br2ArDozOQ6LdtV0OYeCwNnhMfKwKocFKUVJzQ5hcJrHmYhrCe38WuwOu28PkPnT9iTYe3XlJDbXOQ+3eD/epvc66vh1R7HdlYGNlgRuf280JnEZuer+N9V04f8ebT45F1Bj50zy1s/O9n+b9tFj7b9RmKL/sA5polw/7eeBxG9FqZVOZIEDUZtLgGWLRcODP5XCZkWerTd+2w6gdceF4YOQXu/u/37EoHJotu0GNFE64wKFVVSbYeoPvFP9Pw84/S/cK9bNQt5OfxazBZLXznY6uwmfUjdn2tRubf37uI+VML+PUTe/nZOzo6Jl2B67KPkp5/E/fXF3Hv83Usn+3nhpNIXj8SnDYjn7lzGc1ZD7/tXk7dQz+m5U9fIrJrLWomPWzXKfZZ+eztCzHqczVtk0HL5+9YSLFX1D7PFlNKnXz0+tnotbmfYYtJx8dvmkuFqH2OqmKvletXliIfesYt9pi55cLJ+NzOQY+VRrqvRjizLFq0SN2wYcMJ98nGI6R7Wkl1NpBo3EXi4A4ywQ5CWNjnWsWaSAW17QlmVLn54l2L8DhGp68xq6g8/PJeHnl5H/FkNr9dkuDKlVV88JpZaEdxTc4TeWl9A794aAsOA9xo38TM1HYkgxnLlIUYy6ZhKK5G5y094fQaSZI2qqq66Njth++hqqq0dkXpDSdw2434RfA84wx2DzMZhQMtAXpDSXwuM5NKRPAcC+3dvfR0R0mkMvhcFkpLjsw9P949BNGEO6GlulsIvPkQ2XgYJRFFSUTIxkIo8Uh+H9lk43HlPDbEPQTiKvRAiU/Dp2+Zx+pF5Wjk0RvxqZElbrmohitWVLFlTydNnRFsZh2LphdS5Dmz1sG8cHE5JT4rv3x4K+mFt1NUFiG6+y1i+zcS2fFGfj/ZZEVr86KxOii47jNoTLYhX0OSJIp9VopFn+dZS6uVmVo+PAvIC6eu0OOi0OM66eNEDXSCkSSpEzg41uUQhqRCVdV+aZjEPTyriHt49hvwHoIIoIIgCIJwSs6MDiNBEARBOMuIACoIgiAIp0AEUEEQBEE4BWIUriCcZbxer1pZWTnWxRCGYOPGjV0DDUAR9/Dscbx7CCKACsJZp7KyksHm8gpnBkmSBhxpK+7h2eN49xBEABUEQRgzwXCS2uYA3cEERR4Lk0sdmI2Dp5ADyIS7SbYeQImHyXomcTBupqMnjsssUa4PYCGOvmgyOoe3z3HZRJRk2wEygQ60di+GoklozLn5x5lQN5GmfaQiYdIWH93aQiSNllgiQyiaYlKhkY6eGOFYmmKHTJk9ixRqQ4lH0DoLUbV6urJ26rtSZKRcaslELE6pJYVH6UJjdZNVZeKdzUg6A1HnFGq70lhNOkKRFJKSptyh4jVl0Wsg2dmMrNURNhayuxOKC+xUV/kw6LSke9tIdhykHS8NQYksWpxWPQowqdiB13n8BC+hjjb2dyk0d8WwmPRMKXNSXmQ/6fsnAqggCMIYiMbT/PmZnbzwzpE1ZO++agbXnjsZzSAZtTKRXjqe+BmJ+nfReCtYW3QL979+ZDH3S+Z5uIpX0Wceo+CGz+WDqJJJE3znSQJvPJTf177kKtzn3YaSjNH++M9INrybf8162b/zbL2Bf62t56NX1/C7J+rYeTC3WowkweeuraLinT+gJHKrADmv/CTfebqd9p4YkEtPeNMF1fzhmYN8cZWCc+sryGYH0R2vk51/I798KcmKueX88YkdhKKp/DG/vrOAjn/+AJQMALKziFTVXXz5n7v57K1zWTnZQNv93yY473b+61+7iSVy+9nMOq4/fwqPvLyPL965CM8AQTTZcZAN9So/eXArh2dxTi5x8Pk7F1JaMPREJiAGEQmCcBYIRpJs3ddJJJYa66IMm4b2cJ/gCfDXf+2mpXvwJelS7XUk6nOBLlp9MQ+82d7n9ee3dBMsXUGyZS+p9gP57ZmeFgJvPtJn39A7T5HubibVXt8neAJEY3H+tbYeAJtJmw+eAKoKf3y5FWXaRflt4Vf/wuoZR4JQNJ6mqSOMVqthT9xDonkveo8fgBbDJHrCGboC8XzwBJhbZSPy1j/zwRNACbRRbexCq5H4/ZO76OwMIGl1vH5Qkw+ekFvNpr0nRkdvjNqWgZeFawuk+dvzezk6BUJtc5D9jYEB9z8RUQMVBOGM9sQbtfz56V2k0lnMRi1fvXsJc6YMOKbjrBJL9F9cIJNVSCQzA+zdl5KI5f8/qWrJKv0fLOJZGTvka4cA2WQcVKXfvtlkjGyyf+COJY/sm0hl+70eCCfIGp0cXjwwGw3hNvdN79kdTOCwGuiKZJEN5vxavrGsjN2ipzec7LO/z6pBae/sdy1NKoxeZycUTZFMZdGbHTQF+r9XPcEEDouBWLz/+6uqKsmsTHew/xJm0QHux2BEDVQQhDPWo6/s4/8ee5e51V7+44NL8TpN/M+96wkc86N7Nir2WLAY+9ZhKvx2Ct2DL8en85bAoTVvHdEGSrx9j7GZdXjUHpBkdN6yI8e5CtE6Cvvsq7E40LmK0HtK8+c8zGPV4LTlFjxw2gzIx+S+XjnDg77xnfy/DVOX8ua+eJ99ZlS5OdAcZFahhBILIUm5sOM3JGjqjFBd5uyz/9p9EQwzV/f7m8PmMmKJDEumF+BxGkm37mf11P6LMUytcNHQHh6wOVaSJLzGDCvn+PtslyUoKzz5PlARQAVBOCNt29/Jn5/eyaq5xXz17qUsnlHEl967mFgizWOv7R/r4p02v8/KNz+ynJoKF7IssXhGIV+4YwF2y/FX6DlMX1BB0S1fRecthV0v8Zkr/Cys8SHLEtPKHXzpCg+mg2spuvlLGAor88dprS4Kb/oCpqq5IMkYy2dQdPNX0Dl86AsrKLj5K2g9JbkF66euIm3x8Yn3zGVmlZsHXqnny3fMpbTAilYjc8FcH7esLEBOhEDWYp6+AvuSq1lU7cRm1mE16bj+/Mm0dUX56MV+ioOb8V7zSbKqgqQz4D74Il+8ZSZNHWFuvrAau0WPxajl0jlOdP4pWJdcg6QzoLF54Px7eHBLkvPm+bn76pnYCkvxXvkxqhPv8t7VfiwmHXaLnpsuqKazJ8pX715C1XFWtnEUFHLtihLOX1CCViPj91j4wp2LmFEpkskLwrg3lCXpznapdJZP/OAVZBl+8pnzMRqO1NT+58/vsH1/F3/6j0sx6DTHP8kZYLDlzCDXTxhNpLFb9Bj1J9erlo2FUdIJtBYXSQVC0RQWgwZ9OoSkM6A1D1yryqYSKPEwstGCxmDud854NEJUMpNVZUwGLQaDlkgsjc2sJxmPk0iksGgzpFQ9RikJ6QToTWjVDGi0uZWbNDq0egNKJolDToIkoTFZQW8m1tMJGi1mhzs34EgCVVEhm8ahS6Mx2zEa9SR62olnJTKqlgwafF4HOu2Re56JBEBVCGSNqGpuYJNep8VuGXx94miwl84ImM0GClzHr/WL5cwEQTirPPrqflq7o/znR5f3CZ4AV62cxNptrax7t5Vz55eOUQmHj8Wkw2Ia2tSVY2nMNjTkmiqNcFQAPnEfsUZvRKM3HvecVrONYxfJMxtyZTQZBh+pWtinMtd/uT2bryj//ydajs/kLeZEqw1rrU4AvCfY53gsDheW01x+VTThCoJwRglGkjz88j5Wzilm3tSCfq/PmOTBbtGzfmf7AEcLwugRAVQQhDPKU2/WkUxluf3SmgFf18gSC6cVsHF3B1lFdEEJY0cEUEEQzhixRJon3zzAsllFJ8wMs3h6EeFYiv2NvaNYOkHoSwRQQRDOGK9taiIaT3PjBdUn3G/GJDcAu+pFABXGjgiggiCcMV54p4FKv52a8hNPKfA4TBS4TOw+2DNKJROE/kQAFQThjFDfGmJfY4CLl5QjSdKg+0+rcLOnXgRQYeyIACoIwhnhpfUNaDUS5y0Y2tSUmkoXXcEE3cH44DsLwggQAVQQhDGnqiprtrUwv6YAh3XwTDwAk0ucANS1hEawZIJwfCKACoIw5mqbg3T2xlkx2z/4zodU+HOjdOuOs+qGIIw0EUAFQRhzb21vRZZg8YyiwXc+xGrSUeAyUd8qaqDC2BABVBCEMffW9hZmTvIOufn2sEq/QwRQYcyIACoIwpjq6InR2B5hycyh1z4Pq/DbaOqIkM70X+NSEEaaCKCCIIypzXtziyfPrzn5RbLLCm0oikpbd//FoAVhpIkAKgjCmNqytwO33Uh54eCrfByr5NBKHk0dkeEuliAMSgRQQRDGjKKobN3XxbypviElTzjW4QDa0ikCqDD6RAAVBGHMHGgOEo6lmD/15JtvIbeWptNmoFkEUGEMiAAqCMKYefdAFwBzqk8tgEKuFioCqDAWRAAVBGHM7Kzrwe+x4LYbT/kcIoAKY0UEUEEQxoSqquys62Z6lfu0zlPisxCMpIjEUsNUMkEYGhFABUEYE82dEYKRFDOqPKd1nsMDiUQtVBhtIoAKgjAmdtblliKbOek0a6AFIoAKY0MEUEEQxsTOum7sFn2+BnmqCt1mZAlau2LDVDJBGBoRQAVBGBM763qYUeU+pfmfR9NpNXhdZlq7RDYiYXSJACoIwqjrDSdo7YoyvfL0+j8PK/ZYaO0WTbjC6BIBVBCEUbf3YC8A0ypdw3I+v9ciaqDCqBMBVBCEUbevMYAsS0wqcQzL+fxeC+FYWkxlEUaVCKCCIIy6vQ29VBTZMOq1w3I+v9cCQKtYlUUYRSKACoIwqlRVZV9jgKnlw9N8C0cFUNGMK4wiEUAFQRhVrd1RIvE01WXOYTtnkUcEUGH0iQAqCMKo2tsQABjWGqhBp8HjMNIiAqgwikQAFQRhVO1r7EWv05zSAtonIkbiCqNNBFBBEEbVvoYAk0scaDTD+/Pj91jEICJhVIkAKgjCqMlkFWqbg1SXO4f93H6vhUA4STyZGfZzC8JARAAVBGHUNLSFSaWzTC0bvv7Pw4q9uZy6baIWKowSEUAFQRg1+xpzGYhGqgYKiIFEwqgRAVQQhFFT2xzEbNRS5LYM+7mLPGZATGURRo8IoIIgjJoDTUGqih3I8umtwDIQs1GH02oQAVQYNSKACoIwKrKKSl1riMmlw5P/diBiKoswmkQAFQRhVDR35AYQTR6mBPIDyQVQsayZMDpEABUEYVQcaA4CMKnEOWLX8HstdAUTJNPZEbuGIBwmAqggCKOitjmITitTWmAdsWv4D+XEbRdTWYRRIAKoIAij4kBzkAq/He0wZyA6mliVRRhNIoAKgjDiVFWltjk4ov2fAMViXVBhFIkAKgjCiOvojRONp0c8gFrNemxmnUimIIwKEUAFQRhxB5oDAEwa4QAKubVBRROuMBpEABUEYcTVNgeRJajw20f8Wn6vReTDFUaFCKCCIIy4A81BSgpsGPXaEb+W32uhoydGOqOM+LWEiU0EUEEQRtyBURhAdFix14KiQmdvbFSuJ0xcIoAKgjCiAuEk3cHEiKbwO5rfk5tnKgYSCSNNBFBBEEbUkQxEoxRAxVxQYZSIACoIwoiqPTwCt3h0AqjDqsdk0Iq5oMKIEwFUEIQRdaA5SIHbjNWsH5XrSZKEX0xlEUaBCKCCIIyo0RxAdJhY1kwYDSKACoIwYmKJNC1d0VHr/zzM77XQ3hMlq6ijel1hYhEBVBCEEVPXEgIYkxpoJqvSFYiP6nWFiUUEUEEQRkztKKbwO9qRkbhicW1h5IgAKgjCiDnQHMRpNeC2G0f1uqW+3FzQhrbwqF5XmFhEABUEYcQcaA4yqcSBJEmjel2X3YjLZqD20BxUQRgJIoAKgjAi0pksDW3hUW++PWxyqTOfxEEQRoIIoIIgjIiDrWGyisqUUueYXH9SiYOG9jCpdHZMri+MfyO/NIIgCBPS4QFEJ8qBm+pqIrTpOTLBTvS+CuwLL0Vrcw/L9SeVOFAUlfrWEFPLXcNyTkE4mgiggiCMiNqmIBajlkK3ecDXw1tfpvOZ3yDJGrTOAmL7NhJc/xS+K/8N64yVp339w1NnDjQHRQAVRoQIoIIgjIja5gCTSpwDDiCK7l5H51O/xFQ1l4JrP43G4iDd20bHEz+n458/Rk0nsc294LSuX+g2YzHpxEAiYcSIPlBBEIZdNqtQ3xIasPk2E+yk46lfYCiupvDmL6Gx5PbRuYrw3/ENTFVz6HzqV0T3vHNaZZAkiepSJ3sO9pzWeQTheEQAFQRh2DV1REhllAEzEHW/eC9kMxRc/xlkbd8E87JWT+F7/h2DfzIdT/yUVFfTaZVj5mQP9a0hwrHUaZ1HEAYiAqggCMPuyAAiZ5/tsbqtRHe/jXPljeichQMeK+sMFN70RWSdgfaHvoeSjJ1yOWZP9qKqsONA9ymfQxCORwRQQRCGXW1TEINeQ/GhjEAAqqrS+8r9aB0FOJZdc8LjtXYPBTd8nnRvG53P/AZVPbWk8FPLnei1Mu/WigAqDD8RQAVBGHa1zUEmFTvQyEcGEMXrt5Fs3Y9z5Q39mm4HYiqfgeu8W4nuXEN4y0unVA6dVsO0Sjfba7tO6XhBOBERQAVBGFaKonKgOdCv/zOw5lE0Vje22ecP+VzOFddjqppD9/N/INXZcErlmTXJQ11LkFBU9IMKw0sEUEEQhlVrd5R4MtsnhV+iaQ+Jg+/iWHY1klY35HNJkozvmk8hG0y0P/ojlHTypMuzcHohqgrrd7ad9LGCcCIigAqCMKz2HOwF6JO8ILD2UWSTFfv8i0/6fFqrC981nybd1Uz383886eOry5x4HUbe2t560scKwomIACoIwrDac7AHk0FDaaENgFTHQWL7NuBYfCWy3nRK5zRPmotzxfWEt7xIZMebJ3WsJEksm+Vn854OEsnMKV1fEAYiAqggCMNqb0Mv1WWu/ACiwNp/IumN2BddflrndZ13K4bSGjqf+Q3pnpOrTS6b7SeVUdi4p+O0yiAIRxMBVBCEYZNIZahrCVFTkWu+Tfe2Edm5BvuCS9CYbKd1bknWUHjdZ5BkDe3//DFqJj3kY2dN8uC0GXhlQ+NplUEQjiYCqCAIw6a2KUhWUak51P8ZWPMokqzBseTE8z6HSuvw4bvq30i11dL9yn1DPk6jkblgYRnrd7XTG0oMS1kEQQRQQRCGTX4AUYWLdKCD8PZXsc2/CK1t+FZDsdQsxb7oCkLvPEV07/ohH3fRknIUReWVjaeXHlAQDhMBVBCEYbPjQDd+rwWXzUhg7aMgSTiXXz/s1/Fc+F70hVV0PvULMqGhJUkoK7QxvdLNC+8cPOXMRoJwNBFABUEYFtmswrsHupgzxUs62EF46yvY512E1u4Z9mtJWh2FN3wWNZuh47GfoCrZIR138ZJymjoi7KwTK7QIp08EUEEQhkVtc5BYIsPcKT56Xr4PSZZxrhj+2udhOncxvsvvIdG4i55X7x/SMefMK8Fi0vHUmwdGrFzCxCECqCAIw2Lb/lxTarWpl+jONTiWXYPW7h3Ra1pnnYN9waUE33qM6O51g+5vNGi5ZGkFa7e30hWIj2jZhPFPBFBBEIbFtn2dVBRZybzxBzQ294j0fQ7Ec/HdGPxT6HjqF6R7Wgbd/8qVVaCqPLO2bhRKJ4xnIoAKgnDaEskMOw50M9XYTaqjAd/l9yDrjaNybUmro/DGzyPJGtoe/gFKInrC/QvdZpbO8vPsWwdJpofWdyoIAxEBVBCE07ZpTwepjEJN92vY5l+MuXrhqF5f6/BRcP1nSHc30/7oD1GzJ06ycPWqSYRjKV7fJKa0CKdOBFBBEE7bm+t2Y5GSTK+w47n47jEpg7lqLr4rP0a8bhsdT/7ihCNzZ032UOm388QbB8SUFuGUiQAqCMJp6dzyBut29zDX1kXJzV9C1hnGrCy2OatxX3AX0R1v0v7ID1AyA68BKkkS1503mfrWEBt3i/y4wqkRAVQQhFOSCXXT8cTPeO6Rp0mh49pbr0FjcQx+4AhzLr8OzyUfJLZ3Pa1/++ZxEy2ct6AUn8vEgy/uFbVQ4ZSIACoIwklJdRyk48mf0/DLfyO0/U3e1CylosjK9KklY120PMfiKyi44XOkOg7S9H+fI7jh2X5NulqNzI3nT2FXfQ/vHugeo5IKZzPtWBdAEIQzn6qqJA6+S+Ctx4kf2IykM2BfcAlbzctofmw/X7x2GpIkjXUx+7BOX4GhsJLOZ35D93P/R2jTsziXX491xkokTe6n76KlFfzjxb38/bk9zPqY54z7G8ZaJJ6iqT1CIpXBZNDishkpcJsJhBO0dEUxGbRksyqBSBKTQQuqigoYdBqS6SzReBqzUYvNrCcUSxGJpbCb9dgseiLxNOFYGr/bhCXVjTYdIW1wUh82oNPKmIxain1WUuks3YE4WVUlm1XRalSyWYlgNInTagAJ0hkVu0WP32PBaMjd20gsRXNnBI0soddrCUWS2C16SnxWNBqZcCxFS2cEAL/bgCHagZJOoHMWorW5h/T+iAAqCMJxqapCbO96AmseJdm6H43Fgeu827AvuJRQVse9P3yVSSUOVs4tHuuiDkjnLsZ/x7eI7n6b3tf/QecTP6Pn1ftxLL4S+7wLMRgt3HrRVH7zz+2s29HGsln+sS7yGaOtO8qvHt7K5r2dANRUuJg/1ce8qQX88uGtuGwGKvw2nn6zDkUFl83ALRfXUOy1UN8a4vl1B6lrCTGjys20CjePvV6Loqg4rQbuvnoGew728u7+dj63LEPorT9DNoNkMGNb8WH+89kIK+eW4LYZcNmNNHdGKC2wkkhmsJj1/PmpnfSGk8gS3HTBVPY09OBzmakosnL+gnISqQy/fGgL+xoD3HbJNP7+/G6iiQxajcw9N8xm9mQPv3pkG1v35Zr350xy8b4pnWg33I/WUUDhe/4dQ2HloO+RaMIVBKEfVVWI7HiTpv/7LO0Pf59sPIz38o9S9onf4Fp1E90JDf/x27eIJdJ89vYFyPKZW3OTJAnr9OWUfuTHFN38FXT/v707D27zvu88/n5wnwR4iRd4kzqoywd1ObYcOZajxqkTx2nixGk7TdrJ7mymnclMutud7m4yOzttp+102zTdI20O7yZO08bxJnZiO7Gj2JJ8iKQuUiTFS7xJ8AIIgLifZ/+gKFsyJZEgABLw9/Un8BxfijP68Pd7vr/f4y5j/pXvMvz1LzL3i2/zoRY71WUOvvXTLmKyLvS6tu7p6+EJy2/aicSSvNo2wuh0gH1NJfz09eXwBFgIRHn5rWH8oQiD436GJhYBuGfHNp492Y967UBfMMqPXu2nubqQJw8UwOlvQzIBgBZdwvLmt3i8tYifn7lKscvKvD+CXqfwg19cwe208JPXBlkIRAFQNfjhK1f44D3V/PLtEWJxjSsjC7zVNcn5vlmOtVbzr7/qIxRZvn4iqfL0z7o5c2nqengCXBxcoDNWhc7qJOH3Mn/yGdR49I7/RjICFUJcp2kaS31tLPz6GWLeYYwlHtwf/UPmi/by9mSAwed7GRz30zeygE6n8KefP0RtecFml70miqLD1nwvtuZ7iU4O4H/7efxtP8d/9md8svwYf9NVxbef7+KLj+/b7FK3hI5VupP7Rn247CYAYgn1Pd8PjvvR63T0jfquf7bacSPTARJJlTLdEmg3fq+GA5Ral9fx+kMxLl+d48CucqxmA5FoksFx/3uut/KHTzSWZMwboHNg+Zm2zWLEF7gxCCtL7Zy/8t6frWMszqGiCqLjASIjnajhwB07yiVAhXif0DSNi/2znL4wQffVeWZ9YTSgwGaiyGWhyBjBOd+LLuglZq7G7z7OhN/ExP/1k1RfB8Bi0lNf6eLEkTo+9mAj2wptm/tDpchc0ci2j/0RRcc+h7/tZzR0vMyD5haePwXN9gDHjt/3vn8eeveOUs52T9/wWVO1i3BkOdxMhvdOYNZXFpBMajRVu7k6uXjL46rLnBgNOvwRG3ZFd0OI6qxOZiJGAFx2E7vqivEFoyxF4ljMeuorC66PbleYjHoAzCY9nm1OdDqFs93ThCMJ3A4zvuA7ITo5G+LjDzbdMAIFuMtjJN4/BYClpgWdxXnHfyMJUCHy3GIoxqtto7z4xhDjMyGsZj276oppqS9C0VQWpibwTk5wKWrAr1ah4UEXgTK9kZpyB4f3VFBf6aKxykV5sX1LT9eul6GgmOKHfpvCDzzB77b9guHnF/i7l5IEL/05xx4+hH3nYXTm3PwjYaMOtJTzZtcUF68FTXO1G6vJwJE9FXQNLXBpYI6PfqCeF84MoWngdpj58OFaXA4zjVUu+isKuDq5yPkrM3z8wUZ+8toAqgYFdhNPHGuif9THpf4AX77vd+GN/wNqAsVkIXLo9/jxSwucOFzLnH+JEpeVUW+AJ4/vwB+M8rGjjXzn+cv4glEUBZ441sRr58f40IFqjAaF7TVuqsudvH15ilfbR3ny+Ha+/1Iv4WgCg17hcyd2sbexhPae6esj1d31bvaZJ1CXFtEXlFD04GfRme68nlmR9U9C5JbW1latra3ttseEwnEu9s9w6sIEZy5Okkiq7Kor4sThGg5Uamgz/YQHzrHU34GWiGEsrcZ14KNYWu5H0Rsx6HXvyxFYYHGJ//yNV+ifVbnHNMRxRw+N2xuw1e/DWrMLY4kHRadf8/UURWnXNK315s/X8jvcCgJLMUanA8vNOxYDbqeVsmIbC4sRxmeD7+rCjWAzG0BTUACjSUc0phKKxLCZjTjtRhaDMQLhOC67GafdQCCUIBiOUVFkwRGfxxALEjO7GQqYMBr02CwGqrY5icaSzPqWSKoaqqph1OlIqCq+UJRChxlQiCdVnDYzlaX25W7ga7WPe5e7cM0mPb5gFJfDTFWpA8O1Ltxx73IXbmWRBXNoCjUexuAux/iud9je6ncIEqBC5Jx3/+c7PhPklbMjBMNxlsIJAsElxidm8YYUNBRsBpXDpYvc7xqnLD5OcnHu+j6xersb+87D2HcdwVKz+30ZmKuJJ5L84BdXeO5kH7GERqE+TJluniJdkAetV/AUmzEUlKCz2NGZ7ejMVhSDEUVnoPDop24I2FwPUCEBKkReURRlBhje7DrEmtRqmlZ684fyO8wpq/4OQQJUCCGESImsAxVCCCFSIAEqhBBCpEACVAghhEiBBKgQQgiRAtlIQQghsqykpESrq6vb7DLEGrS3t8/eqgtXAlQIIbKsrq4OWQeaGxRFueVyI5nCFUIIIVIgASqEEO8TS5E4/aM+kqqs/08HmcIVQoj3gTFvgP/4D6dZCETZ21jCV//g8PW3mIjUyAhUCCHeB77+w/MkkhpPHt/BpYFZnj3Zv9kl5TwJUCGEyHO9w/NcHprnM4/s4KkTOzm8p5znfj1AOJrY7NJymgSoEELkuZ++PoTdYuDhgzUAPHa0kVA4zttdU5tcWW6TABVCiDwWTyR5+/Ik999Vdf1dmS31xRQ6zZy+OLHJ1eU2CVAhhMhjnQNzhKNJDu4uv/6ZXqdw375K2runZRp3AyRAhRAij53tnsZk1LO/+cbNdA7tLieWULk8NLdJleU+CVAhhMhj56/MsKexGPNNS1Z21RWh1ylc6p/dpMpynwSoEELkqcBSjNHpALvri9/zncVsoLnaTeegjEBTJQEqhBB5qvvqPAC76otW/X5vUwn9oz55DpoiCVAhhMhT3UPz6HUKzdXuVb/fVVdEUtUYGPNlta58IQEqhBB5qnd4gYYqFxbT6ru2Nl0L1n4J0JRIgAohRB7SNI3BCT+NHvctjyl0WihxW+kb9WWtrnwiASqEEHloZiFMKBynobLgtsc1V7vplwBNiQSoEELkocEJPwD1Va7bHtfkcTMxGyIYjmejrLwiASqEEHloaNyPokBd+e1HoA3XAnZ4cjEbZeUVCVAhhMhDgxN+KkscWMy3f+1zTbkTgOEpCdD1kgAVQog8NDixeH10eTulbis2i0FGoCmQABVCiDwTDMfxzi9Rf4cGIgBFUagtL2B4KpCFyvKLBKgQQuSZldFkfeWdR6CwPI07PLmIpmmZLCvvSIAKIUSeGfMGAfBsc6zp+LqKAoLhOPOLkUyWlXckQIUQIs+MeQMYDTpKC21rOr72WqeuTOOujwSoEELkmfGZIJUldvQ6ZU3Hr3Tijkgn7rpIgAohRJ4Z9wbxbHOu+XiXw4zbaWZ4Ukag6yEBKoQQeSSeUJmaX6Jqjc8/V3i2ORifCWaoqvwkASqEEHlkai6EqmpUla4vQKtKHdebj8TaSIAKIUQeWW8H7grPNgeBpRiLoVgmyspLEqBCCJFHVqZh1xugKyPWcRmFrpkEqBBC5JExb4BCpxmbxbiu81aemY7PSCPRWkmACiFEHhn3BtfdQARQVmjDoFfkOeg6SIAKIUQeGZ9Z3xKWFXq9jooSu3TiroMEqBBC5Al/MEpgKb7uDtwVVaWylGU9JECFECJPpNqBu8KzzcnkbIhkUk1nWXlLAlQIIfJEqh24K6pKHSSSGtPzS+ksK29JgAohRJ4Y9wbXtYn8zVaCd0ymcddEAlQIIfLE+EyQinVsIn+z60tZpBN3TSRAhRAiT4x5AylP3wI4bSYK7KYt00gU902zNHAOLRnf7FJWZdjsAoQQQmxcIqkyNbfEffsqN3SdrbIn7tLgeaZ++GeQTGD27KDiqa+iM5g2u6wbyAhUCCHywNRciKSqbWgEClvjrSxqIsbMT/8eY1ElxY98nuhYL/43f7KpNa1GAlQIIfLAynPLVNeArqgqdeALRAmGN2/aNHjp1ySDC5Q88nlcBx7F1tyK/+wLqImttdG9BKgQQuSBlWnXqhR2IXq3lRHsxCaOQv1nf4aprB5L7R4AXAc/irq0SKjr1KbVtBoJUCGEyAPjM0HcTjMO6/o2kb/ZSifumHdzNpWPzY4RnxnBuf8hFGW5m9hSuweDexvB7jObUtOtSIAKIUQeGPMGNzx9C1BevLwMZrMaiUI9bwJg33Ho+meKomDfcYjw1Uuo0a2zyYMEqBBC5IExb3DDDUQABr2O8uLN21R+qa8Nc2UzhoLiGz637zgEyQRLg+c3pa7VSIAKIUSOWwzFCCzF0hKgsPwcdDNGoGo0THRyAGvdXkLhOM+83MvfPNNBW/c05qrtKCYrkaudWa/rVmQdqBBC5Lh0deCuqCp10NHrJalqKe9qlIrIWA9oKmpFC3/8968zOh3AYTXyatsoX3x8L/d6dhIe6cpaPXciI1AhhMhx4zPLDT+pvAd0NZ5tDuIJlZmF7D5vjIx0gU7Pd87GGfMG+dofHOG7/+UEh3aX883/18mYo4X47BjJkD+rdd2KBKgQQuS4MW8Qg17HtqLUNpG/2TuduNmdxg0PdzHiuptfn5/kyYe3c/eObRgNOr782XsodJp5pteOpkFktDurdd2KBKgQQuS4Me/GNpG/2cpUcDYDVI1HiU4O8IJvO4VOM48fa7r+nc1i5DOP7KRvMkKXWrdlpnElQIUQIseNeQNUl6Xn+SeAy2HGacvupvKx6SHGYi565gx84lgzFtONLToPH6imxG3llHoX0bErWavrdiRAhRAih8UTKpNzS2l7/rliuRM3e5spRCcHOR1txmTQ8fDBmvd8r9frOHGklu5AASMTc1viDS0SoEIIkcOm5kKoadhE/mZVpY6svhc0MD5ER6yB+++quuVuSo8cqkVRoC1cS8w7mrXabkUCVAghctjo9PIosToDI9CFQJRQljaVPz/oI6IZOXav55bHFDot7K0roCNWR2SiLyt13Y4EqBBC5LB3NpFP8wj02vWy8RxUjUc5O2PHadLY21hy22OPttYzqxbQ1ycjUCGEEBsw5g1Q4rJgNad3X5xsduIGxofoink40GBDr799LB3ZW4lO0XhjMJLxuu5EAlQIIXLYqDeIpyy907ewvKm8TqdkpZGo7fwAUYw80Npwx2ML7CZ2lyTpWHCTjG1uiEqACiFEjtI0jXFvIO0NRABGg46KYltWpnDb+/1YlTj799av6fgD24uYVZ0MX9nc56ASoEIIkaPmFyOEo8m0L2FZUVXqzHgnrqZpdM4Y2OUOYTTo13TOwXu3A3D2/NUMVnZnEqBCCJGjVjpwMzECXbnu+EyIpKpl5PoAwxPzLCTM7PdY1nxOZU0lZYYA566GMlbXWkiACiFEjlpp8KnOwDNQgJpyJ4mkyuRs5kahbR3L07D37Kla8zmKorCvOEKvz0wkmshUaXckASqEEDlqdDqAzWKg0GnOyPVrywsAGJ7KXCNRR88MZTofVU3b13Xe3XUOEpqO872TGarsziRAhRAiRw1PBagtL0BRMvPOTk+ZA0WBkcnFjFw/Gk/S402yyzqLwVW6rnP37q7FRJyz54cyUttaSIAKIUQO0jSNq5OL1FYUZOweFpOBimI7V6cyE6BdA3PEVYV9Fcq6/wiwVzWy3TjFuX4fmpa5Z7S3IwEqhBA5aM4fIRSOU1eemeefK2orChiezMwUbkf3JAaS7G5a3+gTwOAqZZdtjpmQxsTs5jQTSYAKIUQOunptWrWu0pXR+9SWFzA5GyQaT6b92u3dkzQapinwNK77XEVR2F9tAuBcrzfdpa2JBKgQQuSglQCtzfgI1Imqwdh0ekehMwthxuai7DROYC6/8w5Eq/HUeijRBejomU5rbWslASqEEDloeHKREpcFh82U0ftkqhO349qoscWxgMFdltI1zOUN7DBOcKl/lnhCTWd5ayIBKoQQOSjTDUQrKkvsmAw6Bsf9ab3uuV4vbkOUWk9Jyl3E5vIGdhkniMRVuq/OpbW+tZAAFUKIHJNIqox5A9RlIUD1eh31VS76x3xpu2YyqXK+z8sOwziWirXtf7saQ2E52+1+9IrGud6ZtNW3VhKgQgiRY0anAySSGvUZbiBa0eRxMzjuR03Tln59oz5C4QQ79WOYK5pSvo6iKLgqq2mwBq5PCWeTBKgQQuSYvlEfAM017qzcr8njIhxNMJGmLf3ae7woCuwwTmKuWH8H7ruZyxvZzhCD434WAtl9vZkEqBBC5Jj+UR92y/ImB9nQ6HEv33csPc9B23umaXDGcNpNKTcQrTCXN7DTMAbAhSvZncaVABVCiBzTN+ajqdqdsS38blZT5sRk0NF/beS7Ef5glP4xH7tME5grGjf8M5gqGvDo53CYlaxP40qACiFEDoknklyd8NN0bVSYDelsJOro9aJpsD3evaHnnyuMRRXozRZ2F0Y4d2Umbc9p10ICVAghcsjw5HIDUXN1YVbv2+xxMzjuI5nc2HrLjh4vBVY91boZzJUbD1BF0WEuq2eHYQxfIMpwhvbtXY0EqBBC5JC+a6PARk92OnBX7KovIhxNMjSRekCpqkZHr5e925LoFNIyAgUwlTfQHOkElgM6WyRAhRAih/RcnafAbqKsyJbV++5uKAagczD1DQv6x3wshmK0WKbRO4swONMzirZUbadA9VNTYs7qc1AJUCGEyCFdg3PsbijOWgPRimKXlfJiG5eHUg/Qt7qm0CnQFOnEXNmcttos1TsB2FMc4/LQPJFoIm3Xvh0JUCGEyBEzC2Gm55fYc200mG27G4rpHJhL+f2bb1yaoKXWhSUwjrWmJW11GQpK0BeUsNMwRiKpcr4vO8tZJECFECJHdA3OAu9Mp2bb7vpiAksxRlN4M8vodIDR6SCt5XEALNW70lqbxbOD2sB57FYjb1yaTOu1b0UCVAghckTn4Bw2iyHj7wC9ld2Ny8F9aWD907grobbXMIxismAqq0tnaVg8OyE4R2uzi7OXp0hssFt4LSRAhRAiR3QNztFSX4xel93nnysqiu2UFdloT+H9m2cuTbC9xo11pguLZyeKTp/W2lZGtPcULxFYitOVQsivlwSoEELkgDl/mDFvcNOef8Ly5u0HdpVxoW+WaDy55vOGJxcZGPPzwJ5S4jMjaZ++BTCV1aKzFdAc78Fk1HP60kTa73EzCVAhhMgBbd3Lo77WXRvbO3ajWlvKiMWTXOqfXfM5r7SNotcpHHQtn2Ot35f2uhRFh7VuL+rIBQ62lHH6wkTGX7ItASqEEDmgrXuaEreVmnLnptaxt7EEm8XAqQvjazo+mVT5VfsoB1rKMExcQGd1bvgNLLdird9PMrjA0WYLi6EYHSlMNa+HBKgQQmxxsXiSC30ztO4qy/r6z5uZjHqO7K3gjUuTxNYwjXu2expfIMpDrdWEB89hbdif9uefK2wN+wFoZhC3w8yr7aMZuc8KCVAhhNji2numCUeTHNlbsdmlAPDBezwsRRK81TV1x2Of+/UApYVW9hctkQz5sTXenbG6DAUlGEuriQ20c/SeKt7ummYxFMvY/SRAhRBiizt1foICu4n9TSWbXQoAe5tKKSuy8fypwdse1zM8T9fgHI890ECk7ywoOmwNmQtQAPuOw0RGunloTyGJpMrLbw1n7F4SoEIIsYUtReK8dXmK+/ZVotdvjf+y9TqF33yggctD81wZWVj1GE3T+O4Ll3E5TBw/WEuw63WsdXvQ2zO7htXRch9oKiULnexvLuGFU4MZWxO6NX4bQgghVnWyY4xoLMnxgzWbXcoNjh+socBu4tvPd626td+ZS5N0Dszx5PEd6OcGSfimcex+ION1mUprMJZ4CPWc4bGjjcz6I5y5mJklLRKgQgixRWmaxotvXKWh0kVztXuzy7mBzWLkcyd20jkwxy/eHrnhu5mFMN/4lws0eVycOFJHsPM1FIMJ+87DWanN0XI/keEu9pclqSp18MNfXiGZgRdtS4AKIcQW1dHrZWhikUfvr99w920yHCA80sVSXzvRyQHUxMabax45XMddzaX8jx9d5NSFcTRNY3hqkT/9n6dJqipf/uy96BJRAp2vYdtxEJ05O69gc979MOgMBNpf4nO/sZPhqQC/ahu584nrZEj7FYUQQmyYpmn84OVeSgutHLu3OrVrqElCvW+zePYFIqM9wDujMMVoxtbcivvwxzFXNKR0fb1O4Y9/p5WvffNN/uLpNpw2I8FwHIfVyFd//wjVZU78Z19Aiy7hOvDRlO6RCoOjEEfLfQQu/oojX3qS7TVuvvdiDx/YX4XVnL7YkwAVQogt6LVz4/QML/Cl39qP0bD+ycLwyGVmX/wm8ZkRDIXlFD7wKcxVzegsdhKBOSJDlwh2vU7o8mlsOw5Rcvz3MLhK130fp83En3/pfl45O0Lv8AJlxTZOHK7D5TCjxiL4Tj+LpaYFS1X63v+5Fq4DjxLsfI3Ftp/xhceO8R++cYp/+kknX/qtu9J2DwlQIYTYYnyBKP/0k06aPC4ePli7rnOTIT9zrz5N8OJJDAUlbHv8y9h3Hn7P5gWOnUcoOvYU/raf4zv9I0YHz1N49NO4DjyKol9fNBj0Oj58uI4PH6678ed44zmSIR9ln/zKuq6XDubKJmw7DuF748ds/zcP8fiDTTx7sp/WXWUc3pOe9bTyDFQIIbaQRFLlr7/fTjAc5w8/ffea37yiaRqBiycZ/V9/RLDzFO77PoHni3+Lo+UDt9z5R2exU3j/J/F88W+x1u5h/pWnGf/Wvycy3rfhnyM6OYjvzLM4dj+w/KqxTVD80G+DqjLzwj/w1IntNHlc/NX32um5Op+W60uACiHEFhFPJPnr77Vz/soM//YT+6hf43s/Y95hpr7/NWZ++nWMxZV4fv8vKTr2FDqTZU3nG93bKPvUn1D2xFdILi0y8Z0/YfalfyQZDqb0cyQWZ5n6179Ab3dT/MgXUrpGOhiLKij60O8QHjhH4PUf8J++cIiiAgtf/cc36ejxbvj6ymrrd4QQQmROa2ur1tbWdsNnI1OL/N0/n6d3ZIHP/+ZuHv9g0x2vE52+iu+NHxPqOo1itlJ87Cmc9zyCoqQ+NlKjS8yf/D6LbS+imCy4DnyEgtaPYHC413R+ZLSb6ef+O2p0icrPfQ1zeWoNSumiaRqzL/5vAh0v47zrYZIHPsN/e/ocw1OLfOxoI58+vgOH1XjL8xVFadc0rXXV7yRAhRAiu94doMOTi3zvpR7e7JzEYTXy7z55Fx/YX7nqeWo8Smz6KuGhC4SutBGbGkAxmnEdeBTX4cfQW9P3ppaYd5iFU/9CqPtNUBSsDfuxNd6DxbMTY3EFOpMVWA4oNRIiOtZL4NKvCHW/icFVStkTX0m5uzfdNE1l4eQz+M48i6GgBHPrY/zz1VJ+2T6JzWzg6D0eDu+poKWuCMtNXboSoEIIsYW8O0AvXJnhz54+y0fuq+NjRxtxOczEfV4W219EDQdJRoKokSAJ/ywJn5flpSgK5ooGHHsfxLH7KHpb5l5xFpsbJ3jxJMHLp67df5litqEzGFFjEbR4FACdxYHz7ocpvP+T1wN2KwkPdzH/ytNEJ/vROwpJPPGXPPfaIGcuTBBLqJiMer7/X38Ds/GdZ8YSoEIIsYUoijIDZG6Xc5FOtZqmrbq+RwJUCCGESIF04QohhBApkAAVQgghUiABKoQQQqRAAlQIIYRIgQSoEEIIkQIJUCGEECIFEqBCCCFECiRAhRBCiBRIgAohhBAp+P8+Vpp3WLeClQAAAABJRU5ErkJggg==\n",
+      "text/plain": [
+       "<Figure size 576x576 with 10 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAEDCAYAAAD9SFsgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAUbElEQVR4nO3df5BdZX3H8fc3u5vfptklm7AkCEHWEH4I2JWfWhkiGsCStFNacLAZm5qOVURGhya1FjvVKbaOYseCZoiaKo1FYEykCMZVRrQaWYTBhCQkgvnFkmwoJJhEdjf77R/POfeuYTcbvifrPRs+rxnm3nvuc8/9niX3M8855znPMXdHRCRiVK0LEJGRSwEiImEKEBEJU4CISJgCRETCFCAiElaKADGzuWa20cw2m9niWtdzKDM70cx+aGbrzWydmd2QLW8ys9Vmtil7bKx1rf2ZWZ2ZPWZm92WvS1uvmU02s7vNbEP2d76w5PXemP1bWGtmK8xsbNnqNbOvmNkuM1vbb9mgNZrZkuw3uNHM3nUk31HzADGzOuA/gMuB04Frzez02lb1Cr3AR919NnAB8MGsxsVAu7u3Au3Z6zK5AVjf73WZ6/0C8IC7nwacTaq7lPWa2XTgw0Cbu58J1AHXUL56vwbMPWTZgDVm/56vAc7IPnNb9ts8PHev6X/AhcCD/V4vAZbUuq4hal4JXAZsBFqyZS3AxlrX1q/GGdk/kEuB+7JlpawXmAQ8A9ghy8ta73RgG9AE1AP3Ae8sY73AycDaof6mh/7ugAeBC4daf817IFT/Z+S2Z8tKycxOBs4F1gDT3L0TIHucWsPSDnUrcBPQ129ZWes9BegCvprtct1hZhMoab3uvgP4LLAV6AT2uPv3KGm9hxisxtDvsAwBYgMsK+X4ejObCNwDfMTd99a6nsGY2buBXe7+aK1rOUL1wJuB2939XGAfte/+Dyo7bjAPmAmcAEwws+tqW1Vhod9hGQJkO3Biv9czgGdrVMugzKyBFB53uvu92eKdZtaSvd8C7KpVfYe4GLjKzH4NfBO41My+QXnr3Q5sd/c12eu7SYFS1nrfATzj7l3u3gPcC1xEeevtb7AaQ7/DMgTII0Crmc00s9GkAzmralzT7zAzA5YB6939c/3eWgUsyJ4vIB0bqTl3X+LuM9z9ZNLf8wfufh3lrfc5YJuZzcoWzQGepKT1knZdLjCz8dm/jTmkg75lrbe/wWpcBVxjZmPMbCbQCvx8yLXV+iBPdsDmCuAp4FfAx2tdzwD1vZXUnXsCeDz77wrgONKByk3ZY1Otax2g9kuoHkQtbb3AOUBH9jf+NtBY8nr/CdgArAW+DowpW73ACtIxmh5SD2Ph4WoEPp79BjcClx/Jd1j2QRGRV60MuzAiMkIpQEQkTAEiImEKEBEJU4CISNiwBcirvcLWzBYNVy3DZaTVrHqH12ux3mEJkOAVtiPqj58ZaTWr3uH1mqt3uHog5wGb3f1pd+8mDaeeN0zfJSI1MiwDyczsz4C57v7X2ev3Aue7+4cGaj+lqc4njB/Frt7jARjVU32vryFbZ3ZNacOebgC8ob76fb0H07L6NH1B74SUiwfHpPdHdVfX59nHGn6TttstXUPUMzFbvq/atm7PAQB+e8I4AM6aNu136u7q6qK5uXmgTSol1Tu8Rlq96x7fyOhR49jb07Xb3UOF1w/dJGTIK/uy/a9FAA2va+T4az/BMx+9DYA3ffZvK+32nZiSY8zuFAqvf2DPK1bcNzptRvfk0QB0nZtS57i3dwKwbcuUSttJU38DQOOXU2K88MbUdvRlu1Nd3zqu0rZx+U8B2P2lNwLQceWnB91gkZFm7uwlADy44ZYt0XUM1y7MkFf2uftSd29z97a68ROGqQwRGU7D1QOpXGEL7CBdEfqewRqP6oHxO/sqPY8nPnZb5b22T3wAgLqe1IEZteW59EbjH1Ta1B1MvRSvmwzAhB1ps7Y/leZKmbyhOjPbgalpCkjztJ/UtDE9Pjsl9TxaunpfUd9La7NeyZWDb7DISOPbis+aMSwB4u69ZvYh0rRodcBX3H3dcHyXiNTOcPVAcPf7gfuHa/0iUnvDFiCvRl8D7J86qnLANN9tAej459sBmL8pzTK/9qx0QLPxyernd5+fzsKc/sl0LOiq29LjskfeCsDs9zxVaXvKhHSwdNXetwEw7Yo0DeT8pvR41/S2StvZj6QD037SgWIbKFJCz77/nPTk1v8Mr0ND2UUkrBQ9EOuDhn1eOVWbHzCFas/j260PAjDr4Q+8oo29nD7nPemA6EsHxwJQvzudou3cP6nStrsvHVDNx5e8eCCN8dj0UuptWH11EnMbnRr17h1dbANFSqiuu/gYMPVARCSsFD2Qhj3dHP/Adg5OST2Fyqlaqsc88p7Hxr9Kx0TmzntvpU3TwucB2HfRqQB86ydpHNusmx8DYNOy2ZW2W15Io11nrE+na19/VZqU+mc/SW3GvVAdA9c9M50GPu32l9KCkXalg8hh7LskG3Z92+HbHY56ICISVooeiDfU09vSiPWksyn9B4nlZ1vyYx55z+OBlV+vtLns2vcBMP6RdA/hpuPPBqD7ojMAaLmnupm9Y1MPY1znfgC2fib1cJrHp/Xnx0YA6takL//tpW8qsnkipXTqTS8AsLnAOtQDEZEwBYiIhJViF8Z6D1K/ay8909KuS35tC1QHieWnavMDpvluC8DqFV8F4G0f+hsAnj8vHSCdeu8zADR8a1yl7bYXJqcn97wOgBs/uQKAJWv+FIBRO8ZW2k54+1kA1O8/WGj7RMpo663pN8D8+DrUAxGRsFL0QLy+jt6pkyrzeeRX1UJ1eHo+SCw/VZsfMIVqz+PhL34ZgCvOuhSAbe87DYDp76ze4vOkN2S9kV3pSsRlm68CoLUnzTpk69dX2toprweg6/zqHCEix4qpX0y/hSeHaHc46oGISFgpeiC9E0ax8y0T6cl2yfL5PKB6YVw+PD0fJJafqoXqMY+853H/L38AwBv+O52infKjiZW2TaPT+r7/7bcAcOplTwMwY/yLADy05dRK25k3pmW7L64OhRc5Vjx/Rjbn5/fj61APRETCStEDOTgG9rYeZMasNKw8n0kMqpfk5xfG5cPT80FiUD3bkh/zyHsev/qLLwFwyvcWVtqe15rajkljaNi5bCYAM65P67VH+/U2etOl/x+76MFswd9FN1GkdFrueByAXxZYh3ogIhJWih7IqG6YsLWObWPT7On95zDNJwPKL8nPL4zrPzw9H+eRn23Jj3nkPY+n37ms0nZdd5oc6E+mzwLgU3+expA016UL5r7/plmVtgePT2dfbt/wRwBcf1qRrRQpl42fSeOcGPBmK0dGPRARCVOAiEhYKXZhvB5ePs4rN33Kb70A1TlM85nE8vk88qtqoTo8PR8klp+qzQ+Y5rstAGeMTm08i84dPU0AzB2Xrs4dM7Z6W7y6PWnZ+SfsLrR9ImU0Znfd0I2GoB6IiISVogfS8Bun5ScHsezgZ37TJ6jOnp7P05HPJJbP5wFULozLh6fng8TyU7X5AVOo9jw2/WWa2ezMf083s/q3yZ59ptqz6Xvu1wD84uvnpAXnRbZOpJxmtKff0FNDtDsc9UBEJKwUPRA3o6/B2PuGVE5+u0mo3rclnz09n8M0n0kMqpfk5xfG5cPT80Fi+alaqB7zyHseaz+cJoT81O50jnb52gsqbbsvTKeMuzWSXY5BLy7OevEPx9ehHoiIhJWiB9IzEZ67cBSTT09nO/IbXUP1jnH5fVvy2dPzOUyhOhlQfkl+fmFcPjw9HyQG1bMt+TGPvOfxD1M2ALBizB9W2o7dnCYvmvuvO4ptoEgJHXh4SuF1qAciImGl6IE07IOpHU7PutTzaOnqrbyX36s2v2Ncft+W/rOn59MQ5pMB5Zfk5xfG9R+eno/zyM+25Mc88p7HugvvrLS9kj8G4P7vpDafP7fARoqUzKQtfUM3GoJ6ICISpgARkbBS7MLU7TnApPueoG///le8N/uR7KbX2Y2u89tN5jd9gurs6fkcpvlMYvl8HvlVtem70nfkg8TyU7X5AdN8twXgf376HQAuP/WitOAfAxsnUlJuQ7cZinogIhJWih7Ib08Yx+YbzmbS7NQLeGlttcfgJ6UL4Xr3phnb8xtd97/dZH7flnz29HwO03wmsXw+D6heGJcPT88HieWnavMDplDteXx38/8W2j6RMmr68fbC61APRETCzN2HbjXM2travKOjo9ZliLwmmdmj7t4W+Wy4B2JmJ5rZD81svZmtM7MbsuVNZrbazDZlj41DrUtERqYiuzC9wEfdfTZwAfBBMzsdWAy0u3sr0J69FpFjUDhA3L3T3X+RPX8JWA9MB+YBy7Nmyyl0614RKbOjchDVzE4GzgXWANPcvRNSyABTB/nMIjPrMLOOrq6uo1GGiPyeFQ4QM5sI3AN8xN33Hunn3H2pu7e5e1tzc3PRMkSkBgoFiJk1kMLjTne/N1u808xasvdbgF3FShSRsipyFsaAZcB6d/9cv7dWAQuy5wuAlfHyRKTMioxEvRh4L/BLM3s8W/b3wC3AXWa2ENgKXF2oQhEprXCAuPuPgcEux5kTXa+IjBwayi4iYQoQEQlTgIhImAJERMIUICISpgARkTAFiIiEKUBEJEwBIiJhChARCVOAiEiYAkREwhQgIhKmABGRMAWIiIQpQEQkTAEiImEKEBEJU4CISJgCRETCFCAiEqYAEZEwBYiIhClARCRMASIiYQoQEQlTgIhImAJERMIUICISpgARkTAFiIiEKUBEJEwBIiJhChARCVOAiEhY4QAxszoze8zM7steN5nZajPblD02Fi9TRMroaPRAbgDW93u9GGh391agPXstIsegQgFiZjOAK4E7+i2eByzPni8H5hf5DhEpr6I9kFuBm4C+fsumuXsnQPY4daAPmtkiM+sws46urq6CZYhILYQDxMzeDexy90cjn3f3pe7e5u5tzc3N0TJEpIbqC3z2YuAqM7sCGAtMMrNvADvNrMXdO82sBdh1NAoVkfIJ90DcfYm7z3D3k4FrgB+4+3XAKmBB1mwBsLJwlSJSSsMxDuQW4DIz2wRclr0WkWNQkV2YCnd/CHgoe/48MOdorFdEyk0jUUUkTAEiImEKEBEJU4CISJgCRETCFCAiEqYAEZEwBYiIhClARCRMASIiYQoQEQlTgIhImAJERMIUICISpgARkTAFiIiEKUBEJEwBIiJhChARCVOAiEiYAkREwhQgIhKmABGRMAWIiIQpQEQkTAEiImEKEBEJU4CISJgCRETCFCAiEqYAEZEwBYiIhClARCRMASIiYQoQEQkrFCBmNtnM7jazDWa23swuNLMmM1ttZpuyx8ajVayIlEvRHsgXgAfc/TTgbGA9sBhod/dWoD17LSLHoHCAmNkk4I+AZQDu3u3uLwLzgOVZs+XA/GIlikhZFemBnAJ0AV81s8fM7A4zmwBMc/dOgOxx6kAfNrNFZtZhZh1dXV0FyhCRWikSIPXAm4Hb3f1cYB+vYnfF3Ze6e5u7tzU3NxcoQ0RqpUiAbAe2u/ua7PXdpEDZaWYtANnjrmIlikhZhQPE3Z8DtpnZrGzRHOBJYBWwIFu2AFhZqEIRKa36gp+/HrjTzEYDTwPvI4XSXWa2ENgKXF3wO0SkpAoFiLs/DrQN8NacIusVkZFBI1FFJEwBIiJhChARCVOAiEiYAkREwhQgIhKmABGRMAWIiIQpQEQkTAEiImEKEBEJU4CISJgCRETCFCAiEqYAEZEwBYiIhClARCRMASIiYQoQEQlTgIhImAJERMIUICISpgARkTAFiIiEKUBEJEwBIiJhChARCVOAiEiYAkREwhQgIhKmABGRMAWIiIQpQEQkTAEiImEKEBEJKxQgZnajma0zs7VmtsLMxppZk5mtNrNN2WPj0SpWRMolHCBmNh34MNDm7mcCdcA1wGKg3d1bgfbstYgcg4ruwtQD48ysHhgPPAvMA5Zn7y8H5hf8DhEpqXCAuPsO4LPAVqAT2OPu3wOmuXtn1qYTmHo0ChWR8imyC9NI6m3MBE4AJpjZda/i84vMrMPMOrq6uqJliEgNFdmFeQfwjLt3uXsPcC9wEbDTzFoAssddA33Y3Ze6e5u7tzU3NxcoQ0RqpUiAbAUuMLPxZmbAHGA9sApYkLVZAKwsVqKIlFV99IPuvsbM7gZ+AfQCjwFLgYnAXWa2kBQyVx+NQkWkfMIBAuDuNwM3H7L4ZVJvRESOcRqJKiJhChARCVOAiEiYAkREwhQgIhKmABGRMAWIiIQpQEQkTAEiImEKEBEJU4CISJgCRETCFCAiEqYAEZEwBYiIhClARCRMASIiYQoQEQlTgIhImAJERMIUICISpgARkTAFiIiEKUBEJEwBIiJhChARCVOAiEiYAkREwhQgIhKmABGRMAWIiIQpQEQkTAEiImEKEBEJU4CISNiQAWJmXzGzXWa2tt+yJjNbbWabssfGfu8tMbPNZrbRzN41XIWLSO0dSQ/ka8DcQ5YtBtrdvRVoz15jZqcD1wBnZJ+5zczqjlq1IlIqQwaIu/8I+L9DFs8DlmfPlwPz+y3/pru/7O7PAJuB845OqSJSNvXBz01z904Ad+80s6nZ8unAz/q1254tO6xNT2zl8unX45MmAuDbnq289+z7zwGgrtsB2HfJPgBOvemFSputt74OgKlfHAfA82eMAaDljscB2PiZsyptx+xOHaIZ7fsBeHFxejzw8BQAJm3pq7R1S49NP94OwHe3fH6oTREZMfqeay28jmiADMYGWOYDNjRbBCwCGFs38SiXISK/D+Y+4O/7dxuZnQzc5+5nZq83ApdkvY8W4CF3n2VmSwDc/V+ydg8Cn3T3nw6x/i5gH7C7yMbUwBRGVs2qd3iN1HpPcvfmyAqiPZBVwALgluxxZb/l/2VmnwNOAFqBnw+1MndvNrMOd28L1lMTI61m1Tu8Xov1DhkgZrYCuASYYmbbgZtJwXGXmS0EtgJXA7j7OjO7C3gS6AU+6O4HixQoIuU1ZIC4+7WDvDVnkPafBj5dpCgRGRnKNBJ1aa0LCBhpNave4fWaq/eIDqKKiAykTD0QERlhFCAiEqYAEZEwBYiIhClARCRMASIiYf8Pmu3ntIpnOt4AAAAASUVORK5CYII=\n",
+      "text/plain": [
+       "<Figure size 288x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "config_sample['batch_size'] = 4\n",
+    "model = get_model(config_sample, device, should_train=False, verbose=2) # , state_dict=model[2].state_dict()\n",
+    "(hp_embedding, data, _), targets, single_eval_pos = next(iter(model[3]))\n",
+    "\n",
+    "from utils import normalize_data\n",
+    "fig = plt.figure(figsize=(8, 8))\n",
+    "N = 100\n",
+    "plot_features(data[0:N, 0, 0:4], targets[0:N, 0], fig=fig)\n",
+    "\n",
+    "d = np.concatenate([data[:, 0, :].T, np.expand_dims(targets[:, 0], -1).T])\n",
+    "d[np.isnan(d)] = 0\n",
+    "c = np.corrcoef(d)\n",
+    "plt.matshow(np.abs(c), vmin=0, vmax=1)\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "## Training"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = get_model(config_sample, device, should_train=True, verbose=0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

TabPFN/{TabPFNPredictionOnly.ipynb → QuickPredictionDemo.ipynb}

@@ -44,14 +44,15 @@
     "import torch\n",
     "import numpy as np\n",
     "import os\n",
-    "import random\n",
     "\n",
-    "from model_builder import get_model, get_default_spec, save_model, load_model\n",
+    "from scripts.model_builder import get_model, get_default_spec, save_model, load_model\n",
     "from scripts.transformer_prediction_interface import transformer_predict, get_params_from_config, TabPFNClassifier\n",
+    "from scripts.differentiable_pfn_evaluation import eval_model, eval_model_range\n",
     "\n",
-    "from datasets import load_openml_list, open_cc_dids, open_cc_valid_dids\n",
+    "from datasets import load_openml_list, open_cc_dids, open_cc_valid_dids, test_dids_classification\n",
     "\n",
-    "from scripts import tabular_metrics"
+    "from scripts import tabular_metrics\n",
+    "import random"
    ]
   },
   {
@@ -66,6 +67,7 @@
   {
    "cell_type": "markdown",
    "metadata": {
+    "jp-MarkdownHeadingCollapsed": true,
     "tags": []
    },
    "source": [
@@ -76,9 +78,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "jupyter": {
-     "outputs_hidden": true
-    },
     "tags": []
    },
    "outputs": [],
@@ -96,27 +95,6 @@
     "random.shuffle(cc_valid_datasets_multiclass)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datasets import get_openml_classification"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dataset = openml.datasets.get_dataset(31)\n",
-    "X, y, categorical_indicator, attribute_names = dataset.get_data(\n",
-    "        dataset_format=\"array\", target=dataset.default_target_attribute\n",
-    "    )"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -156,7 +134,7 @@
     "tags": []
    },
    "source": [
-    "### Select a dataset for prediction"
+    "### Run on a single dataset"
    ]
   },
   {
@@ -174,7 +152,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "evaluation_dataset_index = 4 # Index of the dataset to predict\n",
+    "evaluation_dataset_index = 0 # Index of the dataset to predict\n",
     "ds = test_datasets[evaluation_dataset_index]\n",
     "print(f'Evaluation dataset name: {ds[0]} shape {ds[1].shape}')"
    ]
@@ -191,13 +169,36 @@
     "test_xs, test_ys = xs[eval_position:], ys[eval_position:]"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "classifier = TabPFNClassifier(device='cpu')\n",
+    "classifier.fit(train_xs, train_ys)\n",
+    "prediction_ = classifier.predict_proba(test_xs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "roc, ce = tabular_metrics.auc_metric(test_ys, prediction_), tabular_metrics.cross_entropy(test_ys, prediction_)\n",
+    "'AUC', float(roc), 'Cross Entropy', float(ce)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
+    "jp-MarkdownHeadingCollapsed": true,
     "tags": []
    },
    "source": [
-    "### Predict using a Fitted and Tuned Model"
+    "### Run on all datasets\n",
+    "This section runs a differentiable hyperparameter tuning run and saves the results to a results file, which can be inserted in TabularEval.ipynb to compare to other baselines."
    ]
   },
   {
@@ -206,9 +207,35 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "classifier = TabPFNClassifier(device='cpu')\n",
-    "classifier.fit(train_xs, train_ys)\n",
-    "prediction_ = classifier.predict_proba(test_xs)"
+    "eval_positions=[1000]\n",
+    "bptt=2000\n",
+    "\n",
+    "N_models = 3\n",
+    "models_per_block = 1\n",
+    "\n",
+    "eval_addition = 'user_run'\n",
+    "device = 'cpu'\n",
+    "\n",
+    "eval_model_range(i_range=[0], e=-1\n",
+    "                          , valid_datasets=[]#cc_valid_datasets_multiclass\n",
+    "                          , test_datasets=cc_test_datasets_multiclass\n",
+    "                          , train_datasets=[]\n",
+    "                          , eval_positions_test=eval_positions\n",
+    "                          , bptt_test=bptt\n",
+    "                          , add_name=model_string\n",
+    "                          , base_path=base_path\n",
+    "                          , selection_metric='auc'\n",
+    "                          , best_grad_steps=0\n",
+    "                          , eval_addition=eval_addition\n",
+    "                          , N_ensemble_configurations_list = [32]\n",
+    "                          , device=device)#range(0, 10)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Run generalization experiments"
    ]
   },
   {
@@ -217,8 +244,110 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "roc, ce = tabular_metrics.auc_metric(test_ys, prediction_), tabular_metrics.cross_entropy(test_ys, prediction_)\n",
-    "'AUC', float(roc), 'Cross Entropy', float(ce)"
+    "# Loading longer OpenML Datasets for generalization experiments (optional)\n",
+    "test_datasets_multiclass, test_datasets_multiclass_df = load_openml_list(test_dids_classification, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 10000, num_feats=100, return_capped=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_datasets_longer_generalization = [ds for ds in test_datasets_multiclass if ds[1].shape[0] >= 10000]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def test_gen(classifier_key, split):\n",
+    "    if classifier_key == 'tabpfn':\n",
+    "        model = TabPFNClassifier(device='cuda', base_path='/work/dlclarge1/hollmann-PFN_Tabular/',\n",
+    "                                model_string=model_string, N_ensemble_configurations=4\n",
+    "                          , no_preprocess_mode=False, i=i, feature_shift_decoder=False)\n",
+    "    else:\n",
+    "        model = classifier_dict[classifier_key]\n",
+    "    \n",
+    "    ces = []\n",
+    "    for k in tqdm(range(0, len(test_datasets_longer_generalization))):\n",
+    "        x, y = test_datasets_longer_generalization[k][1], test_datasets_longer_generalization[k][2].numpy()\n",
+    "        x = normalize_data(x).numpy()\n",
+    "        x[np.isnan(x)] = 0.0\n",
+    "        print(x.shape[0])\n",
+    "        \n",
+    "        if x.shape[0] < 10000:\n",
+    "            continue\n",
+    "        if len(np.unique(y)) > 2:\n",
+    "            continue\n",
+    "\n",
+    "        for bptt_ in [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000, 5500, 6000, 6500, 7000, 7500, 8000, 8500, 9000, 9500, 10000]:\n",
+    "            bptt_ = bptt_ // 2\n",
+    "            #model = classifier_dict[classifier_key]\n",
+    "            x_, y_ = x.copy(), y.copy()\n",
+    "            x_train, x_test, y_train, y_test = train_test_split(x_, y_, test_size=0.5, random_state=split)\n",
+    "            x_train, y_train = x_train[0:bptt_], y_train[0:bptt_]\n",
+    "            model.fit(x_train, y_train) # ranking[0:j]\n",
+    "            pred = model.predict_proba(x_test) # ranking[0:j]\n",
+    "            ce = tabular_metrics.auc_metric(y_test, pred)\n",
+    "            ces += [{'bptt': bptt_, 'k': k, 'm': float(ce), 'method': classifier_key, 'split': split}]\n",
+    "            print(x_train.shape, ce)\n",
+    "    with open(f'generalization_{classifier_key}_{split}.obj',\"wb\") as fh:\n",
+    "        pickle.dump(ces,fh)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_gen('tabpfn', 0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ces = []\n",
+    "for classifier_key in classifier_dict:\n",
+    "    for split in range(0,5):\n",
+    "        try:\n",
+    "            with open(f'generalization_{classifier_key}_{split}.obj',\"rb\") as fh:\n",
+    "                ces += pickle.load(fh)\n",
+    "        except:\n",
+    "            pass\n",
+    "df = pd.DataFrame(ces)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = df.groupby(['bptt', 'split', 'method']).mean().reset_index()\n",
+    "fig, ax = plt.subplots(1,1, figsize=(8, 6)) # , sharey=True\n",
+    "\n",
+    "colors = iter(sns.color_palette(\"tab10\"))\n",
+    "for classifier_key in ['tabpfn']:#df.method.unique():\n",
+    "    c = next(colors)\n",
+    "    sns.lineplot(x='bptt', y='m', data=df[df.method==classifier_key], label=relabeler[classifier_key], color=c, ax = ax)\n",
+    "    #ax.text(x = df[df.method==classifier_key].iloc[50].bptt, # x-coordinate position of data label\n",
+    "    # y = df[df.method==classifier_key].iloc[50].m, # y-coordinate position of data label, adjusted to be 150 below the data point\n",
+    "    # s = classifier_key, # data label, formatted to ignore decimals\n",
+    "    # color = c, size=12) # set colour of line\n",
+    "    \n",
+    "ax.get_legend().remove()\n",
+    "ax.set(xlabel='Number of training samples')\n",
+    "ax.set(ylabel='ROC AUC')\n",
+    "plt.axvline(x=1024, linestyle='dashed', color='red')\n",
+    "plt.ylim((0.73,0.79))\n",
+    "plt.xlim((250,5000))"
    ]
   },
   {
@@ -245,7 +374,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.13"
+   "version": "3.9.6"
   }
  },
  "nbformat": 4,

TabPFN/README.md

@@ -2,11 +2,8 @@
 
 ## Installation
 ```
-git clone git@github.com:automl/TabPFN.git
-cd TabPFN
 conda create -n TabPFN python=3.7
-conda activate TabPFN
-pip install -r requirements.txt
+$environment_path$/pip install -r requirements.txt
 ```
 
 To run the autogluon baseline please create a separate environment and install autogluon==0.4.0, installation in the same environment as our other baselines is not possible.

TabPFN/RunFullDatasetAnalyses.ipynb

@@ -0,0 +1,833 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "from scripts import tabular_baselines\n",
+    "\n",
+    "import seaborn as sns\n",
+    "import numpy as np\n",
+    "\n",
+    "from datasets import load_openml_list, valid_dids_classification, test_dids_classification, open_cc_dids\n",
+    "from scripts.tabular_baselines import *\n",
+    "from scripts.tabular_evaluation import evaluate\n",
+    "from scripts.tabular_metrics import calculate_score, make_ranks_and_wins_table, make_metric_matrix\n",
+    "from scripts import tabular_metrics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from notebook_utils import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "# Datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "cc_test_datasets_multiclass, cc_test_datasets_multiclass_df = load_openml_list(open_cc_dids, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 10000, num_feats=100, return_capped=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_datasets(selector, task_type, suite='openml'):\n",
+    "    if task_type == 'binary':\n",
+    "        ds = valid_datasets_binary if selector == 'valid' else test_datasets_binary\n",
+    "    else:\n",
+    "        if suite == 'openml':\n",
+    "            ds = valid_datasets_multiclass if selector == 'valid' else test_datasets_multiclass\n",
+    "        elif suite == 'cc':\n",
+    "            ds = valid_datasets_multiclass if selector == 'valid' else cc_test_datasets_multiclass\n",
+    "        else:\n",
+    "            raise Exception(\"Unknown suite\")\n",
+    "    return ds"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "# Setting params"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eval_positions = [1000]\n",
+    "max_features = 100\n",
+    "bptt = 2000\n",
+    "selector = 'test'\n",
+    "base_path = os.path.join('.')\n",
+    "overwrite=False\n",
+    "max_times = [0.5, 1, 15, 30, 60, 60*5, 60*15, 60*60]\n",
+    "metric_used = tabular_metrics.auc_metric\n",
+    "methods = ['transformer', 'logistic', 'gp', 'knn', 'catboost', 'xgb', 'autosklearn2', 'autogluon']\n",
+    "task_type = 'multiclass'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "suite = 'cc'\n",
+    "test_datasets = get_datasets('test',task_type, suite=suite)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "clf_dict= {'gp': gp_metric\n",
+    "                , 'knn': knn_metric\n",
+    "                , 'catboost': catboost_metric\n",
+    "                , 'xgb': xgb_metric\n",
+    "           , 'transformer': transformer_metric\n",
+    "                , 'logistic': logistic_metric\n",
+    "           , 'autosklearn': autosklearn_metric\n",
+    "             , 'autosklearn2': autosklearn2_metric\n",
+    "            , 'autogluon': autogluon_metric}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "device = 'cpu'\n",
+    "\n",
+    "def eval_method(task_type, method, dids, selector, eval_positions, max_time, metric_used, split_number, append_metric=True, fetch_only=False, verbose=False):\n",
+    "    \n",
+    "    dids = dids if type(dids) is list else [dids]\n",
+    "    \n",
+    "    for did in dids:\n",
+    "\n",
+    "        ds = get_datasets(selector, task_type, suite=suite)\n",
+    "\n",
+    "        ds = ds if did is None else ds[did:did+1]\n",
+    "\n",
+    "        clf = clf_dict[method]\n",
+    "\n",
+    "        time_string = '_time_'+str(max_time) if max_time else ''\n",
+    "        metric_used_string = '_'+tabular_baselines.get_scoring_string(metric_used, usage='') if append_metric else ''\n",
+    "\n",
+    "        result = evaluate(datasets=ds\n",
+    "                          , model=clf\n",
+    "                          , method=method+time_string+metric_used_string\n",
+    "                          , bptt=bptt, base_path=base_path\n",
+    "                          , eval_positions=eval_positions\n",
+    "                          , device=device, max_splits=1\n",
+    "                          , overwrite=overwrite\n",
+    "                          , save=True\n",
+    "                          , metric_used=metric_used\n",
+    "                          , path_interfix=task_type\n",
+    "                          , fetch_only=fetch_only\n",
+    "                          , split_number=split_number\n",
+    "                          , verbose=verbose\n",
+    "                          , max_time=max_time)\n",
+    "    \n",
+    "    return result"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "# Baseline Evaluation\n",
+    "This section runs baselines and saves results locally."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!mkdir {base_path}/results\n",
+    "!mkdir {base_path}/results/tabular/\n",
+    "!mkdir {base_path}/results/tabular/multiclass/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# RUN ONE METHOD ON ONE DATASET AND SPLIT\n",
+    "overwrite=True\n",
+    "dataset_id = 0\n",
+    "split_number = 1\n",
+    "maximum_runtime = 30\n",
+    "r = eval_method(task_type, 'transformer', dataset_id, 'test', eval_positions, maximum_runtime, metric_used, split_number)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# RUN ALL METHODS, SPLITS AND DATASETS\n",
+    "test_datasets = get_datasets('test',task_type, suite=suite)\n",
+    "\n",
+    "overwrite=True\n",
+    "jobs = [\n",
+    "    eval_method(task_type, m, did, selector, eval_positions, max_time, metric_used, split_number)\n",
+    "    for did in range(0, len(test_datasets))\n",
+    "    for selector in ['test']\n",
+    "    for m in methods\n",
+    "    for max_time in max_times\n",
+    "    for split_number in [1, 2, 3, 4, 5]\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "# Comparison"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "pos = str(eval_positions[0])\n",
+    "\n",
+    "global_results = {}\n",
+    "overwrite=False\n",
+    "\n",
+    "for method in baseline_methods:\n",
+    "    for max_time in max_times:\n",
+    "        for split_number in range(1,5+1):\n",
+    "            global_results[method+'_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_split_'+str(split_number)] = eval_method(task_type, method,  None, selector, \n",
+    "                                                                                                                                    eval_positions, fetch_only=True, \n",
+    "                                                                                                                                    verbose=False, max_time=max_time,\n",
+    "                                                                                                                                    metric_used=metric_used, split_number=split_number)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path_ = 'prior_tuning_result.pkl'\n",
+    "\n",
+    "try:\n",
+    "    output = open(path_, 'rb')\n",
+    "    _, metrics, _, _, _, _ = CustomUnpickler(output).load()\n",
+    "except:\n",
+    "    output = open(path_, 'rb')\n",
+    "    _, metrics, _, _, _ = CustomUnpickler(output).load()\n",
+    "if isinstance(metrics, list):\n",
+    "    for i in range(1, len(metrics[1])+1):\n",
+    "        global_results['transformer_split_'+str(i)] = metrics[2][i-1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Verify integrity of results\n",
+    "for bl in set(global_results.keys()):\n",
+    "    if 'split_1' in bl:\n",
+    "        for ds in test_datasets:\n",
+    "            if f'{ds[0]}_ys_at_1000' not in global_results[bl]:\n",
+    "                continue\n",
+    "            match = (global_results[bl][f'{ds[0]}_ys_at_1000'] == global_results['transformer_split_1'][f'{ds[0]}_ys_at_1000']).float().mean()\n",
+    "            if not match:\n",
+    "                raise Exception(\"Not the same labels used\")\n",
+    "            "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "limit_to = ''\n",
+    "calculate_score(tabular_metrics.auc_metric, 'roc', global_results, test_datasets, eval_positions + [-1], limit_to=limit_to)\n",
+    "calculate_score(tabular_metrics.cross_entropy, 'cross_entropy', global_results, test_datasets, eval_positions + [-1], limit_to=limit_to)\n",
+    "calculate_score(tabular_metrics.accuracy_metric, 'acc', global_results, test_datasets, eval_positions + [-1])\n",
+    "calculate_score(tabular_metrics.time_metric, 'time', global_results, test_datasets, eval_positions + [-1], aggregator='sum', limit_to=limit_to)\n",
+    "calculate_score(tabular_metrics.time_metric, 'time', global_results, test_datasets, eval_positions + [-1], aggregator='mean', limit_to=limit_to)\n",
+    "calculate_score(tabular_metrics.count_metric, 'count', global_results, test_datasets, eval_positions + [-1], aggregator='sum', limit_to=limit_to)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "#### ROC and AUC plots from TabPFN Paper"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def generate_ranks_and_wins_table(global_results_filtered, metric_key, max_time, split_number, time_matrix):\n",
+    "    global_results_filtered_split = {**global_results_filtered}\n",
+    "    global_results_filtered_split = {k: global_results_filtered_split[k] for k in global_results_filtered_split.keys() if '_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_split_'+str(split_number) in k or 'transformer_split_'+str(split_number) in k}\n",
+    "\n",
+    "    matrix, matrix_stds = make_metric_matrix(global_results_filtered_split, methods, pos, metric_key, test_datasets)\n",
+    "    for method in methods:\n",
+    "        if time_matrix[method] > max_time * 2:\n",
+    "            matrix[method] = np.nan\n",
+    "        # = np.nan\n",
+    "\n",
+    "    if metric_key == 'cross_entropy':\n",
+    "        matrix = -(matrix.fillna(-100))\n",
+    "    else:\n",
+    "        matrix = matrix.fillna(-1)\n",
+    "    return make_ranks_and_wins_table(matrix.copy())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%matplotlib inline\n",
+    "\n",
+    "df_ = []\n",
+    "metric_keys = ['roc', 'cross_entropy', 'time']\n",
+    "\n",
+    "for max_time in max_times:\n",
+    "    global_results_filtered = {**global_results}\n",
+    "    global_results_filtered = {k: global_results_filtered[k] for k in global_results_filtered.keys() if '_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_' in k or 'transformer' in k}\n",
+    "    \n",
+    "    time_matrix, _ = make_metric_matrix(global_results_filtered, methods, pos, 'time', test_datasets)\n",
+    "    time_matrix = time_matrix.mean()\n",
+    "    \n",
+    "    if len(global_results_filtered) == 0:\n",
+    "        continue\n",
+    "        \n",
+    "    # Calculate ranks and wins per split\n",
+    "    for metric_key in metric_keys:\n",
+    "        for split_number in range(1,6):\n",
+    "            ranks, wins = generate_ranks_and_wins_table(global_results_filtered, metric_key, max_time, split_number, time_matrix)\n",
+    "\n",
+    "            for method in methods:\n",
+    "                method_ = method+'_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='') if method != 'transformer' else method\n",
+    "                global_results[method_+'_split_'+str(split_number)]['mean_rank_'+metric_key+f'_at_{pos}'] = ranks[method]\n",
+    "                global_results[method_+'_split_'+str(split_number)]['mean_wins_'+metric_key+f'_at_{pos}'] = wins[method]\n",
+    "    \n",
+    "    #for method in global_results.keys():\n",
+    "    #    global_results[method]['mean_rank_'+metric_key+f'_at_{pos}'] = ranks[]\n",
+    "    \n",
+    "    avg_times = {}\n",
+    "    for method_ in methods:\n",
+    "        avg_times[method_] = []\n",
+    "        for split_number in range(1,6):\n",
+    "            if method_ != 'transformer':\n",
+    "                method = method_+'_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_split_'+str(split_number)\n",
+    "            else:\n",
+    "                method = method_+'_split_'+str(split_number)\n",
+    "            avg_times[method_] += [global_results[method][f'mean_time_at_{pos}']]\n",
+    "    avg_times = pd.DataFrame(avg_times).mean()\n",
+    "    \n",
+    "    for metric_key in metric_keys:\n",
+    "        for ranking in ['', 'rank_', 'wins_']:\n",
+    "            for method_ in methods:\n",
+    "                for split_number in range(1,6):\n",
+    "                    method = method_\n",
+    "                    if method_ != 'transformer':\n",
+    "                        method = method_+'_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_split_'+str(split_number)\n",
+    "                    else:\n",
+    "                        method = method_+'_split_'+str(split_number)\n",
+    "\n",
+    "                    if global_results[method][f'sum_count_at_{pos}'] <= 29:\n",
+    "                        print('Warning not all datasets generated for '+method+' '+ str(global_results[method][f'sum_count_at_{pos}']))\n",
+    "                        \n",
+    "                    time = global_results[method]['mean_time'] if ranking == '' else max_time\n",
+    "                    time = max_time # Todo: This is not the real time\n",
+    "                    df_ += [{'metric'+ranking+metric_key: global_results[method]['mean_'+ranking+metric_key+f'_at_{pos}'], 'real_time': avg_times[method_], 'time': time, 'method': method_, 'split_number': split_number}]\n",
+    "                    # For Roc AUC Plots\n",
+    "                    #if 'transformer' in method:\n",
+    "                    #    df_ += [{'metric'+ranking+metric_key: global_results[method]['mean_'+ranking+metric_key+f'_at_{pos}'], 'real_time': avg_times[method_], 'time': time, 'method': method_, 'split_number': split_number}]\n",
+    "                    #    df_ += [{'metric'+ranking+metric_key: global_results[method]['mean_'+ranking+metric_key+f'_at_{pos}'], 'real_time': max(avg_times), 'time': max(max_times), 'method': method_, 'split_number': split_number}]\n",
+    "                            \n",
+    "            \n",
+    "df_ = pd.DataFrame(df_)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "metric_renamer = {'roc': 'ROC AUC', 'cross_entropy': 'Cross entropy'\n",
+    "                  , 'rank_roc': 'Mean ROC AUC Rank', 'rank_cross_entropy': 'Mean Cross entropy Rank'\n",
+    "                  , 'wins_roc': 'Mean ROC AUC Wins', 'wins_cross_entropy': 'Mean Cross entropy Wins'\n",
+    "                  , 'time': 'actual time taken'}\n",
+    "max_times_renamer = {0.5: \"0.5s\", 1: \"1s\", 5: \"5s\", 15: \"15s\", 30: \"30s\", 60: \"1min\", 300: \"5min\", 900: \"15min\", 3600: \"1h\", 14400: \"4h\"}\n",
+    "\n",
+    "def make_tabular_results_plot(metric_key, exclude, max_times, df_, grouping=True):\n",
+    "    f, ax = plt.subplots(figsize=(7, 7))\n",
+    "    #ax.set(xscale=\"log\")\n",
+    "    \n",
+    "    df_.loc[:, 'time_log'] = np.log10(df_.time)\n",
+    "    df_.loc[:, 'real_time_log'] = np.log10(df_.real_time)\n",
+    "    time_column = 'time_log' if grouping else 'real_time_log'\n",
+    "\n",
+    "    sns.set_palette(\"tab10\")\n",
+    "    for method in methods:\n",
+    "        if method in exclude or method=='transformer':\n",
+    "            continue\n",
+    "        df_method = df_[df_.method==method].copy()\n",
+    "        ax = sns.lineplot(time_column, 'metric'+metric_key, data=df_method, marker='o', label=method, ax=ax)\n",
+    "    #sns.scatterplot(data=df_, x='time', y='metric', hue='method', ax=ax, style='method') #\n",
+    "    df_trans = df_[df_.method=='transformer']\n",
+    "    if time_column == 'real_time_log':\n",
+    "        # Removing dots for line for transformers\n",
+    "        df_trans = df_trans[np.logical_or(df_trans.real_time == df_trans.real_time.min(), df_trans.real_time == df_trans.real_time.max())]\n",
+    "        df_trans.loc[:, 'metric'+metric_key] = df_trans['metric'+metric_key].mean()\n",
+    "        df_trans.loc[:, time_column] = np.log(1) # Hacky code to get the right time from our measurements\n",
+    "    ax = sns.lineplot(time_column, 'metric'+metric_key, data=df_trans, linestyle='--', marker='o', ci=\"sd\", ax=ax)\n",
+    "    \n",
+    "    #ax = sns.scatterplot(data = df_trans, x=time_column, y='metric'+metric_key, s=800, marker='*', color='grey') #\n",
+    "    #ax = plt.scatter(df_trans[time_column], df_trans['metric'+metric_key], s=600, marker=['*']) #\n",
+    "    \n",
+    "    if grouping:\n",
+    "        ax.set_xlabel(\"Time (s, requested, not actual)\")\n",
+    "    else:\n",
+    "        ax.set_xlabel(\"Time taken\")\n",
+    "    ax.set_ylabel(metric_renamer[metric_key])\n",
+    "\n",
+    "    #ax.legend()\n",
+    "    \n",
+    "    times = np.log10(max_times)\n",
+    "    ax.set_xticks(times)\n",
+    "    ax.set_xticklabels([max_times_renamer[t] for t in max_times])\n",
+    "    \n",
+    "    #ax.legend([],[], frameon=False)\n",
+    "    \n",
+    "    return ax"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_absolute = df_.copy()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_absolute = df_.copy()\n",
+    "df_absolute = df_absolute[np.logical_or(df_.method != 'autogluon', df_.time >= 30)] # Autogluon did not yield any useful results before 30s\n",
+    "\n",
+    "knn_extend = df_absolute[np.logical_and(df_absolute.method=='knn', df_absolute.time == 3600)].copy()\n",
+    "knn_extend['real_time'] = 14400\n",
+    "knn_extend['time'] = 14400\n",
+    "df_absolute = df_absolute.append(knn_extend, ignore_index=True).reindex()\n",
+    "\n",
+    "knn_extend = df_absolute[np.logical_and(df_absolute.method=='logistic', df_absolute.time == 3600)].copy()\n",
+    "knn_extend['real_time'] = 14400\n",
+    "knn_extend['time'] = 14400\n",
+    "\n",
+    "df_absolute = df_absolute.append(knn_extend, ignore_index=True).reindex()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "exclude=['']\n",
+    "#ax = make_tabular_results_plot('time', exclude=exclude)\n",
+    "ax = make_tabular_results_plot('roc', df_=df_absolute, exclude=exclude, grouping=False, max_times=[1, 5, 30, 60*5, 60*60])\n",
+    "ax.set_ylim([0.84, 0.9])\n",
+    "ax.set_xlim([np.log10(0.7), np.log10(3600)])\n",
+    "ax.legend([],[], frameon=False)\n",
+    "\n",
+    "#tikzplotlib.save(f'roc_over_time.tex', axis_height='5cm', axis_width='6cm', strict=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ax = make_tabular_results_plot('rank_roc', df_=df_[df_.time >= 1].copy(), exclude=['tabnet'], max_times=[1, 5, 30, 60*5, 60*60])\n",
+    "ax.invert_yaxis()\n",
+    "ax.set_xlim([np.log10(1.0), np.log10(3600)])\n",
+    "ax.legend([],[], frameon=False)\n",
+    "tikzplotlib.save(f'roc_raks_tabular.tex', axis_height='5cm', axis_width='6cm', strict=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ax = make_tabular_results_plot('wins_roc', df_=df_[df_.time >= 1].copy(), exclude=exclude, max_times=[1, 5, 30, 60*5, 60*60])\n",
+    "ax.set_xlim([np.log10(1.0), np.log10(3600)])\n",
+    "ax.legend([],[], frameon=False)\n",
+    "tikzplotlib.save(f'roc_wins_tabular.tex', axis_height='5cm', axis_width='6cm', strict=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "#### Big Table metrics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "max_time = '3600'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "global_results_filtered = {**global_results}\n",
+    "global_results_filtered = {k: global_results_filtered[k] for k in global_results_filtered.keys() if '_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_' in k or 'transformer' in k}\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "roc_matrix, roc_matrix_stds = make_metric_matrix(global_results_filtered, methods, pos, 'roc', test_datasets_multiclass_filtered)\n",
+    "acc_matrix, acc_matrix_stds = make_metric_matrix(global_results_filtered, methods, pos, 'acc', test_datasets_multiclass_filtered)\n",
+    "cross_entropy_matrix, cross_entropy_matrix_stds = make_metric_matrix(global_results_filtered, methods, pos, 'cross_entropy', test_datasets_multiclass_filtered)\n",
+    "time_matrix, time_matrix_stds = make_metric_matrix(global_results_filtered, methods, pos, 'time', test_datasets_multiclass_filtered)\n",
+    "\n",
+    "roc_rank, rocs_wins = make_ranks_and_wins_table(roc_matrix.copy())\n",
+    "acc_rank, acc_wins = make_ranks_and_wins_table(acc_matrix.copy())\n",
+    "cross_entropy_rank, cross_entropy_wins = make_ranks_and_wins_table(-cross_entropy_matrix.copy())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def wins_vs_idx(matrix, idx):\n",
+    "    wins_auc = np.array([[(matrix.values[:, j] < matrix.values[:, i]).sum() if i != j else 0 for i,method in enumerate(methods)] for j in [idx]])\n",
+    "    ties_auc = np.array([[(matrix.values[:, j] == matrix.values[:, i]).sum() if i != j else 0 for i,method in enumerate(methods)] for j in [idx]])\n",
+    "    losses_auc = np.array([[(matrix.values[:, j] > matrix.values[:, i]).sum() if i != j else 0 for i,method in enumerate(methods)] for j in [idx]])\n",
+    "    \n",
+    "    return wins_auc, ties_auc, losses_auc\n",
+    "\n",
+    "transformer_idx = np.where(roc_matrix.columns == 'transformer')[0][0]\n",
+    "\n",
+    "wins_roc_vs_us, ties_roc_vs_us, losses_roc_vs_us = wins_vs_idx(roc_matrix, transformer_idx)\n",
+    "wins_acc_vs_us, ties_acc_vs_us, losses_acc_vs_us = wins_vs_idx(acc_matrix, transformer_idx)\n",
+    "wins_ce_vs_us, ties_ce_vs_us, losses_ce_vs_us = wins_vs_idx(-cross_entropy_matrix, transformer_idx)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def rename(table):\n",
+    "    return table.rename(columns=relabeler).T.rename(columns={'blood-transfusion-service-center': 'blood-transfus..'\n",
+    "                                                                , 'jungle_chess_2pcs_raw_endgame_complete': 'jungle\\_chess..', 'bank-marketing': 'bank-market..'}).T\n",
+    "\n",
+    "def get_suffix(i, k):\n",
+    "    suffix = ''\n",
+    "    suffix = suffix+'s' if test_datasets[i][5]['samples_capped'] == True else suffix\n",
+    "    suffix = suffix+'f' if test_datasets[i][5]['feats_capped'] == True else suffix\n",
+    "    suffix = suffix+'c' if test_datasets[i][5]['classes_capped'] == True else suffix\n",
+    "    suffix = '' if len(suffix) == 0 else f' [{suffix}]'\n",
+    "    \n",
+    "    return k + suffix"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "relabeler = {'transformer': 'Tabular PFN'\n",
+    "             , 'autogluon': 'Autogluon'\n",
+    "             , 'autosklearn2': 'Autosklearn2'\n",
+    "             , 'gp': 'GP (RBF)'\n",
+    "             , 'logistic': 'Log. Regr.'\n",
+    "             , 'knn': 'KNN'\n",
+    "             , 'catboost': 'Catboost'\n",
+    "            , 'xgb': 'XGB'}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "table = roc_matrix.copy()\n",
+    "#table = roc_ovr_matrix.copy()\n",
+    "#table = acc_matrix.copy()\n",
+    "#table = cross_entropy_matrix.copy()\n",
+    "\n",
+    "#table = table_acc\n",
+    "table.index = [get_suffix(i, k) for i, k in enumerate(table.index[0:table.shape[0]])]\n",
+    "\n",
+    "table.loc['Wins AUC OVO'] = rocs_wins.values\n",
+    "#table.loc['Mean AUC OVR'] = roc_ovr_matrix.mean(skipna=True)\n",
+    "table.loc['Wins Acc.'] = acc_wins.values\n",
+    "#table.loc['Mean Bal. Acc.'] = balanced_acc_matrix.mean()\n",
+    "table.loc['Wins CE'] = cross_entropy_wins.values\n",
+    "\n",
+    "table.loc['Win/T/L AUC vs Us'] = [\"{:d}/{:d}/{:d}\".format(w, t, l) for w,t,l in zip(wins_roc_vs_us[-1, :], ties_roc_vs_us[-1, :], losses_roc_vs_us[-1, :])]\n",
+    "table.loc['Win/T/L Acc vs Us'] = [\"{:d}/{:d}/{:d}\".format(w, t, l) for w,t,l in zip(wins_acc_vs_us[-1, :], ties_acc_vs_us[-1, :], losses_acc_vs_us[-1, :])]\n",
+    "table.loc['Win/T/L CE vs Us'] = [\"{:d}/{:d}/{:d}\".format(w, t, l) for w,t,l in zip(wins_ce_vs_us[-1, :], ties_ce_vs_us[-1, :], losses_ce_vs_us[-1, :])]\n",
+    "\n",
+    "table.loc['Mean AUC OVO'] = roc_matrix.mean(skipna=True)\n",
+    "table.loc['Mean AUC OVO Stds'] = roc_matrix_stds.mean(skipna=True)\n",
+    "\n",
+    "#table.loc['Mean AUC OVR'] = roc_ovr_matrix.mean(skipna=True)\n",
+    "table.loc['Mean Acc.'] = acc_matrix.mean()\n",
+    "table.loc['Mean Acc. Stds'] = acc_matrix_stds.mean(skipna=True)\n",
+    "\n",
+    "#table.loc['Mean Bal. Acc.'] = balanced_acc_matrix.mean()\n",
+    "table.loc['Mean CE'] = cross_entropy_matrix.mean()\n",
+    "table.loc['Mean CE Stds'] = cross_entropy_matrix_stds.mean()\n",
+    "\n",
+    "table.loc['M. rank AUC OVO'] = roc_rank.values\n",
+    "#table.loc['Mean rank AUC OVR'] = roc_ovr_rank.values\n",
+    "table.loc['Mean rank Acc.'] = acc_rank.values\n",
+    "#table.loc['Mean rank Bal. Acc.'] = balanced_acc_rank.values\n",
+    "table.loc['Mean rank CE'] = cross_entropy_rank.values\n",
+    "\n",
+    "table.loc['Mean time (s)'] = time_matrix.mean()\n",
+    "table.loc['Mean time (s)', 'knn'] = 0.5\n",
+    "table.loc['Mean time (s)', 'logistic'] = 60\n",
+    "\n",
+    "table = table[['knn', 'logistic', 'gp', 'catboost', 'xgb', 'autosklearn2', 'autogluon', 'transformer']]\n",
+    "rename(table).round(decimals=3).style.highlight_max(axis = 1, props= 'font-weight: bold;').format(precision=3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def bold_extreme_values(data, format_string=\"%.3g\", max_=True):\n",
+    "    data = data.astype(float).round(3)\n",
+    "    if max_:\n",
+    "        extrema = data != data.max()\n",
+    "    else:\n",
+    "        extrema = data != data.min()\n",
+    "    bolded = data.apply(lambda x : \"\\\\textbf{%s}\" % format_string % x)\n",
+    "    formatted = data.apply(lambda x : format_string % x)\n",
+    "    return formatted.where(extrema, bolded) \n",
+    "\n",
+    "def to_str(data, format_string=\"%.3g\"):\n",
+    "    formatted = data.apply(lambda x : format_string % x)\n",
+    "    return formatted"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "keys_max = [\"Mean rank CE\", \"Mean rank Acc.\", \"Mean rank AUC OVO\", \"Mean rank AUC OVR\", \"Mean rank Bal. Acc.\", \"Mean AUC OVO\", \"Mean Acc.\"]\n",
+    "keys_max = [\"Mean AUC OVO\", \"Mean Acc.\", \"Wins AUC OVO\", \"Wins Acc.\", \"Wins CE\"]\n",
+    "\n",
+    "keys_min = [\"Mean rank CE\", \"Mean rank Acc.\", \"M. rank AUC OVO\", \"Mean CE\"]\n",
+    "\n",
+    "table_latex = rename(table).copy()\n",
+    "\n",
+    "table_latex.iloc[0:30] = table_latex.iloc[0:30].apply(lambda data : bold_extreme_values(data),axis=1)\n",
+    "table_latex.loc[[\"Mean time (s)\"]] = table_latex.loc[[\"Mean time (s)\"]].apply(lambda data : bold_extreme_values(data, format_string=\"%.4g\", max_=False), axis=1)\n",
+    "table_latex.loc[keys_max] = table_latex.loc[keys_max].apply(lambda data : bold_extreme_values(data),axis=1)\n",
+    "table_latex.loc[keys_min] = table_latex.loc[keys_min].apply(lambda data : bold_extreme_values(data, max_=False),axis=1)\n",
+    "\n",
+    "table_latex.loc[['Mean CE Stds']] = table_latex.loc[['Mean CE Stds']].apply(lambda data : to_str(data, format_string=\"%.2g\"),axis=1)\n",
+    "table_latex.loc['Mean CE'] = table_latex.loc['Mean CE'] + '$\\pm$' + table_latex.loc['Mean CE Stds']\n",
+    "table_latex = table_latex.drop(['Mean CE Stds'])\n",
+    "\n",
+    "table_latex.loc[['Mean Acc. Stds']] = table_latex.loc[['Mean Acc. Stds']].apply(lambda data : to_str(data, format_string=\"%.2g\"),axis=1)\n",
+    "table_latex.loc['Mean Acc.'] = table_latex.loc['Mean Acc.'] + '$\\pm$' + table_latex.loc['Mean Acc. Stds']\n",
+    "table_latex = table_latex.drop(['Mean Acc. Stds'])\n",
+    "\n",
+    "table_latex.loc[['Mean AUC OVO Stds']] = table_latex.loc[['Mean AUC OVO Stds']].apply(lambda data : to_str(data, format_string=\"%.2g\"),axis=1)\n",
+    "table_latex.loc['Mean AUC OVO'] = table_latex.loc['Mean AUC OVO'] + '$\\pm$' + table_latex.loc['Mean AUC OVO Stds']\n",
+    "table_latex = table_latex.drop(['Mean AUC OVO Stds'])\n",
+    "\n",
+    "table_latex\n",
+    "#print(table_latex.to_latex(escape=False))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "print(table_latex.to_latex(escape=False))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "table_latex_small = table_latex.iloc[-len(keys_min+keys_max)-1-3:]\n",
+    "table_latex_small"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(table_latex_small.to_latex(escape=False))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "table_latex = table.copy()\n",
+    "\n",
+    "table_latex.iloc[:-5] = table_latex.iloc[:-5].apply(lambda data : bold_extreme_values(data),axis=1)\n",
+    "table_latex.iloc[-5:-5] = table_latex.iloc[-5:-5].apply(lambda data : bold_extreme_values(data, max_=False),axis=1)\n",
+    "\n",
+    "table_latex\n",
+    "#print(table_latex.to_latex(escape=False))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rename(table[-7:]).round(decimals=3).style.highlight_min(axis = 1, props= 'font-weight: bold;').format(precision=3)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

TabPFN/SyntheticGPAblation.ipynb

@@ -1,392 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%load_ext autoreload\n",
-    "\n",
-    "%autoreload 2"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import time\n",
-    "\n",
-    "import torch\n",
-    "\n",
-    "import numpy as np\n",
-    "\n",
-    "import matplotlib.pyplot as plt\n",
-    "\n",
-    "from model_builder import get_model, get_default_spec, save_model, load_model\n",
-    "\n",
-    "from scripts.model_configs import *"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "# Setting params"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "device = 'cuda'\n",
-    "base_path = os.path.join('.')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def train_function(config_sample, i, add_name=''):\n",
-    "    start_time = time.time()\n",
-    "    N_epochs_to_save = 50\n",
-    "    \n",
-    "    def save_callback(model, epoch):\n",
-    "        if not hasattr(model, 'last_saved_epoch'):\n",
-    "            model.last_saved_epoch = 0\n",
-    "        if ((time.time() - start_time) / (maximum_runtime * 60 / N_epochs_to_save)) > model.last_saved_epoch:\n",
-    "            print('Saving model..')\n",
-    "            config_sample['epoch_in_training'] = epoch\n",
-    "            save_model(model, base_path, f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{model.last_saved_epoch}.cpkt',\n",
-    "                           config_sample)\n",
-    "            model.last_saved_epoch = model.last_saved_epoch + 1 # TODO: Rename to checkpoint\n",
-    "    \n",
-    "    model = get_model(config_sample\n",
-    "                      , device\n",
-    "                      , should_train=True\n",
-    "                      , verbose=1\n",
-    "                      , epoch_callback = save_callback)\n",
-    "    \n",
-    "    return"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "heading_collapsed": true,
-    "tags": []
-   },
-   "source": [
-    "# Check synthetic data fitting"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "#### Workflow functions"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {
-    "hidden": true,
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "def generate_test_data(test_gp_params):\n",
-    "    # Generate test data\n",
-    "    config = {**test_gp_params}\n",
-    "\n",
-    "    config['verbose'] = False\n",
-    "    config['differentiable'] = False\n",
-    "    #config['bptt'] = config['bptt_in_training']\n",
-    "\n",
-    "    model_test_data = get_model(config, device, should_train=False, verbose=True)\n",
-    "    (hp_embedding, data, targets_), targets = next(iter(model_test_data[3]))\n",
-    "    (hp_embedding, data, targets_), targets = (hp_embedding, data.to(device), targets_.to(device)), targets.to(device)\n",
-    "    \n",
-    "    return (hp_embedding, data, targets_), targets\n",
-    "\n",
-    "def evaluate_hp_range(model, hparam_true, vary_hparam_ind, data, targets, eval_pos, plot_step_size):\n",
-    "    losses, hparams = [], []\n",
-    "    for l in np.arange(-1.74, 1.74, plot_step_size):\n",
-    "        hparam = [*hparam_true]\n",
-    "        hparam[vary_hparam_ind] = l\n",
-    "        hp_embedding_used = torch.tensor(hparam).to(device).float()\n",
-    "        with torch.inference_mode():\n",
-    "            outputs = torch.sigmoid(model[2]((hp_embedding_used.repeat(data.shape[1], 1), data, targets.float()), single_eval_pos=eval_pos)).squeeze(-1)\n",
-    "        \n",
-    "        loss = torch.nn.BCELoss()(outputs.flatten(), targets[eval_pos:].flatten()).detach().cpu()\n",
-    "        losses += [loss]\n",
-    "        hparam_real = [diff_hparams_f[i][1](hp) for i, hp in enumerate(hparam)]\n",
-    "        hparams += [hparam_real]\n",
-    "        \n",
-    "        print(loss, hparam_real, hparam, outputs.shape)\n",
-    "    return np.array(losses), np.array(hparams)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def differentiable_hparam_tuning_workflow(config_sample, hparam_label, batch_size=4, N_grad_steps=50, plot_step_size=0.1):\n",
-    "    test_gp_params = {\n",
-    "        \"lengthscale\": 1.0,\n",
-    "        #\"lengthscale_mean\": true_lengthscale,\n",
-    "        #\"lengthscale_std\": 0.5,\n",
-    "        \"noise\": 0.2,\n",
-    "        \"outputscale\": 1.0,\n",
-    "        'batch_size': batch_size\n",
-    "    }\n",
-    "    config_sample.update(test_gp_params)\n",
-    "    (hp_embedding, data, targets_), targets = generate_test_data(config_sample)\n",
-    "    hparam_true = [diff_hparams_f[i][0](test_gp_params[hp]) for i, hp in enumerate(diff_hparams_keys)]\n",
-    "    #hparam_true = [test_gp_params[hp] for i, hp in enumerate(diff_hparams_keys)]\n",
-    "\n",
-    "    for vary_hparam_ind, vary_hparam_name in hparam_label:\n",
-    "        print(vary_hparam_name)\n",
-    "\n",
-    "        losses, hparams = evaluate_hp_range(model, hparam_true, vary_hparam_ind, data, targets, eval_pos, plot_step_size=plot_step_size)\n",
-    "\n",
-    "        # TODO: Make only one parameter diffable\n",
-    "        hparam = torch.tensor([*hparam_true]).to(device).float()\n",
-    "        hparam[vary_hparam_ind] = hparam[vary_hparam_ind] + 0.1 #random.random() * 2 - 1\n",
-    "        hparam = torch.nn.Parameter(hparam, requires_grad=True)\n",
-    "        hparam_grad_mask = torch.zeros_like(hparam)\n",
-    "        hparam_grad_mask[vary_hparam_ind] = 1\n",
-    "\n",
-    "        optimizer = torch.optim.Adam([hparam], lr=0.1)\n",
-    "    \n",
-    "        for t in range(N_grad_steps):\n",
-    "            style = hparam.repeat(data.shape[1], 1)\n",
-    "            outputs = torch.sigmoid(model[2]((style, data, targets.float()), single_eval_pos=eval_pos)).squeeze(-1)\n",
-    "            loss = torch.nn.BCELoss()(outputs.flatten(), targets[eval_pos:].flatten())\n",
-    "            optimizer.zero_grad()\n",
-    "            loss.backward()\n",
-    "            with torch.no_grad():\n",
-    "                hparam.grad *= hparam_grad_mask\n",
-    "            optimizer.step()\n",
-    "            print('loss:', loss, 'hparams', diff_hparams_f[vary_hparam_ind][1](hparam[vary_hparam_ind]), 'true', diff_hparams_f[vary_hparam_ind][1](hparam_true[vary_hparam_ind]))\n",
-    "        inferred_param = diff_hparams_f[vary_hparam_ind][1](hparam[vary_hparam_ind].cpu().detach().numpy())\n",
-    "        return hparams, losses, inferred_param, vary_hparam_ind, hparam_true\n",
-    "        "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "#### Fitting a PFN with HP-Diffable GP Prior"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {
-    "hidden": true,
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "num_features = 5\n",
-    "bptt = 200\n",
-    "eval_positions = [100]\n",
-    "\n",
-    "config_general = get_general_config(num_features, bptt, eval_positions)\n",
-    "config_flexible_categorical = get_flexible_categorical_config(num_features)\n",
-    "\n",
-    "config_gp = {'noise': 0.2,  \"lengthscale\": 1.0, \"outputscale\": 1.0}\n",
-    "config_diff_gp = {'differentiable_hyperparameters': {\n",
-    "        'outputscale': {'distribution': 'uniform', 'min': 0., 'max': 10.0},\n",
-    "        'lengthscale': {'distribution': 'uniform', 'min': 0., 'max': 10.0},\n",
-    "        'noise': {'distribution': 'uniform', 'min': 0.0000001, 'max': 0.5},\n",
-    "    }\n",
-    "}\n",
-    "\n",
-    "config = {**config_general, **config_flexible_categorical, **config_diff_gp, **config_gp}\n",
-    "\n",
-    "config['prior_type'], config['differentiable'], config['flexible'] = 'gp', True, True\n",
-    "config['num_features'], config['num_features_used'] = num_features, num_features\n",
-    "config['epochs'], config['num_steps'], config['verbose'] = 500, 100, False\n",
-    "config[\"lr\"] = 0.00001\n",
-    "config[\"dropout\"] = 0\n",
-    "config[\"emsize\"] = 512\n",
-    "config[\"batch_size\"] = 128\n",
-    "config[\"aggregate_k_gradients\"] = 1\n",
-    "config['set_value_to_nan'] = 0.0\n",
-    "config['output_multiclass_ordered_p'] = 1.0\n",
-    "config['categorical_feature_p'] = 0.0\n",
-    "config['nan_prob_a_reason'] = 0.0\n",
-    "config['nan_prob_no_reason'] = 0.0\n",
-    "config['nan_prob_unknown_reason'] = 0.0\n",
-    "config[\"nlayers\"] = 8\n",
-    "\n",
-    "# TODO: This should not be sampled, but be one config\n",
-    "# TODO: This uses old hyperparam sampler throws error\n",
-    "config_sample = evaluate_hypers(config)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {
-    "hidden": true,
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Using style prior: True\n",
-      "Using cpu:0 device\n",
-      "Not using distributed\n",
-      "DataLoader.__dict__ {'num_steps': 100, 'fuse_x_y': False, 'get_batch_kwargs': {'batch_size': 128, 'seq_len': 200, 'seq_len_maximum': 200, 'device': 'cpu:0', 'num_features': 5, 'hyperparameters': {'lr': 1e-05, 'dropout': 0, 'emsize': 512, 'batch_size': 128, 'nlayers': 8, 'num_features': 5, 'nhead': 4, 'nhid_factor': 2, 'bptt': 200, 'eval_positions': None, 'seq_len_used': 200, 'sampling': 'normal', 'epochs': 500, 'num_steps': 100, 'verbose': False, 'pre_sample_causes': True, 'mix_activations': False, 'nan_prob_unknown_reason_reason_prior': 1.0, 'categorical_feature_p': 0.0, 'nan_prob_no_reason': 0.0, 'nan_prob_unknown_reason': 0.0, 'nan_prob_a_reason': 0.0, 'max_num_classes': 2, 'num_classes': 2, 'noise_type': 'Gaussian', 'balanced': True, 'normalize_to_ranking': False, 'set_value_to_nan': 0.0, 'normalize_by_used_features': True, 'num_features_used': 5, 'differentiable_hyperparameters': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'noise': 0.2, 'lengthscale': 1.0, 'outputscale': 1.0, 'prior_type': 'gp', 'differentiable': True, 'flexible': True, 'aggregate_k_gradients': 1, 'output_multiclass_ordered_p': 1.0, 'recompute_attn': False}, 'num_outputs': 1, 'dynamic_batch_size': 2, 'get_batch': <function get_model.<locals>.make_get_batch.<locals>.<lambda> at 0x7f39ad8dcf80>, 'differentiable_hyperparameters': {'outputscale': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'lengthscale': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'noise': {'distribution': 'uniform', 'min': 1e-07, 'max': 0.5}}}, 'num_features': 5, 'num_outputs': 1}\n",
-      "Using a Transformer with 17.35 M parameters\n"
-     ]
-    }
-   ],
-   "source": [
-    "device = 'cuda'\n",
-    "train_function(config_sample, 0, add_name='gp_experiments_diff_with_noise_no_meta_new')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "#### Evaluating a PFN (with pretrained model)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {
-    "hidden": true,
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Using style prior: True\n",
-      "Using cpu:0 device\n",
-      "Not using distributed\n",
-      "DataLoader.__dict__ {'num_steps': 100, 'fuse_x_y': False, 'get_batch_kwargs': {'batch_size': 1, 'seq_len': 10, 'seq_len_maximum': 10, 'device': 'cpu:0', 'num_features': 5, 'hyperparameters': {'lr': 1e-05, 'dropout': 0, 'emsize': 512, 'batch_size': 1, 'nlayers': 8, 'num_features': 5, 'nhead': 4, 'nhid_factor': 2, 'bptt': 10, 'eval_positions': [190], 'seq_len_used': 200, 'sampling': 'normal', 'epochs': 500, 'num_steps': 100, 'verbose': False, 'pre_sample_causes': True, 'mix_activations': False, 'nan_prob_unknown_reason_reason_prior': 1.0, 'output_multiclass_ordered_p': 1.0, 'categorical_feature_p': 0.0, 'nan_prob_no_reason': 0.0, 'nan_prob_unknown_reason': 0.0, 'nan_prob_a_reason': 0.0, 'max_num_classes': 2, 'num_classes': 2, 'noise_type': 'Gaussian', 'balanced': True, 'multiclass_type': 'rank', 'normalize_to_ranking': False, 'set_value_to_nan': 0.0, 'normalize_by_used_features': True, 'num_features_used': <function load_model.<locals>.<lambda> at 0x7f39ad8534d0>, 'differentiable_hyperparameters': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'noise': 0.03, 'lengthscale': 1.0, 'outputscale': 1.0, 'prior_type': 'gp', 'differentiable': True, 'flexible': True, 'aggregate_k_gradients': 1, 'recompute_attn': False, 'bptt_extra_samples': None, 'epoch_in_training': 0.998, 'categorical_features_sampler': <function load_model.<locals>.<lambda> at 0x7f39ad853680>, 'num_features_used_in_training': 5, 'num_classes_in_training': 2, 'batch_size_in_training': 128, 'bptt_in_training': 200, 'bptt_extra_samples_in_training': None}, 'num_outputs': 1, 'dynamic_batch_size': 2, 'get_batch': <function get_model.<locals>.make_get_batch.<locals>.<lambda> at 0x7f39ad81ab90>, 'differentiable_hyperparameters': {'outputscale': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'lengthscale': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'noise': {'distribution': 'uniform', 'min': 1e-07, 'max': 0.5}}}, 'num_features': 5, 'num_outputs': 1}\n",
-      "Using a Transformer with 17.35 M parameters\n"
-     ]
-    }
-   ],
-   "source": [
-    "device = 'cpu'\n",
-    "model, c = load_model(base_path, f'models_diff/gp_ablation_model.cpkt', device, eval_positions, verbose=False)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from priors.differentiable_prior import DifferentiableHyperparameterList\n",
-    "diff_list = DifferentiableHyperparameterList(c['differentiable_hyperparameters'], 512, device)\n",
-    "diff_hparams_keys, diff_hparams_f = diff_list.get_hyperparameter_info()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "model[2].eval()\n",
-    "eval_pos = 100\n",
-    "\n",
-    "hparam_label = [(1, 'outputscale')]\n",
-    "hparam_label = [(0, 'lengthscale')]\n",
-    "hparam_label = [(2, 'noise')]\n",
-    "hparam_labels = [[(1, 'outputscale')], [(2, 'noise')], [(0, 'lengthscale')]]\n",
-    "#hparam_labels = [[(2, 'noise')]]\n",
-    "\n",
-    "hparams, losses, inferred_param, vary_hparam_ind, hparam_true = {}, {}, {}, {}, {}\n",
-    "\n",
-    "for hparam_label in hparam_labels:\n",
-    "    (hparams[hparam_label[0][1]], losses[hparam_label[0][1]], inferred_param[hparam_label[0][1]], vary_hparam_ind[hparam_label[0][1]], \n",
-    "     hparam_true[hparam_label[0][1]]) = differentiable_hparam_tuning_workflow(config_sample, \n",
-    "                                                                                                          hparam_label=hparam_label, \n",
-    "                                                                                                          batch_size=256, \n",
-    "                                                                                                          N_grad_steps=50,\n",
-    "                                                                             plot_step_size=0.05)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "label = 'lengthscale'\n",
-    "\n",
-    "#import tikzplotlib\n",
-    "\n",
-    "inferred = losses[label]\n",
-    "\n",
-    "plt.plot(hparams[label][:, vary_hparam_ind[label]], losses[label])\n",
-    "true = diff_hparams_f[vary_hparam_ind[label]][1](hparam_true[label][vary_hparam_ind[label]])\n",
-    "plt.axvline(x=inferred_param[label], linestyle='solid', color='red')\n",
-    "plt.axvline(x=true, linestyle='dashed')\n",
-    "\n",
-    "plt.ylabel('Cross entropy Loss')\n",
-    "plt.xlabel(label)\n",
-    "\n",
-    "#tikzplotlib.save(f'diff_inferred_params_{label}.tex', axis_height='5.2cm', axis_width='5.2cm', strict=True)\n",
-    "\n",
-    "plt.show()"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

TabPFN/differentiable_pfn_evaluation.py

@@ -1,345 +0,0 @@
-import os
-import torch
-import numpy as np
-import time
-import pickle
-from scripts import  tabular_metrics
-from scripts.tabular_metrics import calculate_score_per_method
-from scripts.tabular_evaluation import evaluate
-from priors.differentiable_prior import draw_random_style
-from tqdm import tqdm
-import random
-from scripts.transformer_prediction_interface import get_params_from_config, load_model_workflow
-
-"""
-===============================
-PUBLIC FUNCTIONS FOR EVALUATION
-===============================
-"""
-
-
-def eval_model_range(i_range, *args, **kwargs):
-    for i in i_range:
-        eval_model(i, *args, **kwargs)
-
-
-
-def eval_model(i, e, valid_datasets, test_datasets, train_datasets, eval_positions_valid, eval_positions_test,
-               bptt_valid,
-               bptt_test, add_name, base_path, device='cpu', eval_addition='', **extra_tuning_args):
-    """
-    Differentiable model evaliation workflow. Evaluates and saves results to disk.
-
-    :param i:
-    :param e:
-    :param valid_datasets:
-    :param test_datasets:
-    :param train_datasets:
-    :param eval_positions_valid:
-    :param eval_positions_test:
-    :param bptt_valid:
-    :param bptt_test:
-    :param add_name:
-    :param base_path:
-    :param device:
-    :param eval_addition:
-    :param extra_tuning_args:
-    :return:
-    """
-    model, c, results_file = load_model_workflow(i, e, add_name, base_path, device, eval_addition)
-    params = {'bptt': bptt_valid
-        , 'bptt_final': bptt_test
-        , 'eval_positions': eval_positions_valid
-        , 'eval_positions_test': eval_positions_test
-        , 'valid_datasets': valid_datasets
-        , 'test_datasets': test_datasets
-        , 'train_datasets': train_datasets
-        , 'verbose': True
-        , 'device': device
-              }
-
-    params.update(get_params_from_config(c))
-
-    start = time.time()
-    metrics, metrics_valid, style, temperature, optimization_route = evaluate_differentiable_model(model, **params,
-                                                                                                   **extra_tuning_args)
-    print('Evaluation time: ', time.time() - start)
-
-    print(results_file)
-    r = [c.copy(), metrics, metrics_valid, style.to('cpu'), temperature.to('cpu'), optimization_route]
-    with open(results_file, 'wb') as output:
-        del r[0]['num_features_used']
-        del r[0]['categorical_features_sampler']
-        pickle.dump(r, output)
-
-    _, _, _, style, temperature, _ = r
-
-    return r, model
-
-"""
-===============================
-INTERNAL HELPER FUNCTIONS
-===============================
-"""
-
-def evaluate_differentiable_model(model
-                                  , valid_datasets
-                                  , test_datasets
-                                  , train_datasets
-                                  , N_draws=100
-                                  , N_grad_steps=10
-                                  , eval_positions=None
-                                  , eval_positions_test=None
-                                  , bptt=100
-                                  , bptt_final=200
-                                  , style=None
-                                  , n_parallel_configurations=1
-                                  , device='cpu'
-                                  , selection_metric='auc'
-                                  , final_splits=[1, 2, 3, 4, 5]
-                                  , N_ensemble_configurations_list=[1, 5, 10, 20, 50, 100]
-                                  , **kwargs):
-    """
-    Evaluation function for diffable model evaluation. Returns a list of results.
-
-    :param model:
-    :param valid_datasets:
-    :param test_datasets:
-    :param train_datasets:
-    :param N_draws:
-    :param N_grad_steps:
-    :param eval_positions:
-    :param eval_positions_test:
-    :param bptt:
-    :param bptt_final:
-    :param style:
-    :param n_parallel_configurations:
-    :param device:
-    :param selection_metric:
-    :param final_splits:
-    :param N_ensemble_configurations_list:
-    :param kwargs:
-    :return:
-    """
-    torch.manual_seed(0)
-    np.random.seed(0)
-    random.seed(0)
-
-    diffable_metric = tabular_metrics.cross_entropy
-    evaluation_metric = tabular_metrics.auc_metric
-    if selection_metric in ('auc', 'roc'):
-        selection_metric_min_max = 'max'
-        selection_metric = tabular_metrics.auc_metric
-        evaluation_metric = selection_metric
-    elif selection_metric in ('ce', 'selection_metric'):
-        selection_metric_min_max = 'min'
-        selection_metric = tabular_metrics.cross_entropy
-        evaluation_metric = selection_metric
-
-    print('Diffable metric', diffable_metric, ' Selection metric', selection_metric, ' Evaluation metric',
-          evaluation_metric)
-    print('N PARALLEL CONFIGURATIONS', n_parallel_configurations)
-    print('eval_positions', eval_positions)
-
-    def evaluate_valid(style, softmax_temperature, results, results_tracked):
-        result_valid = eval_step(valid_datasets, style, softmax_temperature=softmax_temperature,
-                                 return_tensor=False, inference_mode=True, selection_metric=selection_metric,
-                                 evaluation_metric=evaluation_metric, eval_positions=eval_positions, bptt=bptt, model=model[2])
-        result_valid = [float(result_valid[f'mean_select_at_{pos}']) for pos in eval_positions]
-        results += [result_valid]
-        results_tracked += [np.nanmean(result_valid)]
-
-    model[2].to(device)
-    model[2].eval()
-
-    results_on_valid, results_on_valid_tracked = [], []
-    best_style, best_softmax_temperature = style, torch.cat(
-        [torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)], 0)
-    optimization_routes = []
-
-    best_style = torch.cat([draw_random_style(model[3], device).detach() for n in range(0, n_parallel_configurations)],
-                      0)
-    best_softmax_temperature = torch.cat([torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)],
-                                    0)
-
-
-    for _ in tqdm(range(0, N_draws), desc='Iterate over Optimization initializations'): # Evaluates N hparam draws
-        style = torch.cat([draw_random_style(model[3], device).detach() for n in range(0, n_parallel_configurations)],
-                          0)
-        softmax_temperature = torch.cat([torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)],
-                                        0)
-
-        evaluate_valid(style, softmax_temperature, results_on_valid, results_on_valid_tracked)
-
-        print(f'Draw --> Valid Selection metric: {results_on_valid[-1]}')
-
-        if N_grad_steps > 0:
-            gradient_optimize_result = gradient_optimize_style(model, style, N_grad_steps
-                                                               , softmax_temperature=softmax_temperature
-                                                               , model=model[2]
-                                                               , train_datasets=train_datasets
-                                                               , valid_datasets=valid_datasets
-                                                               , selection_metric_min_max=selection_metric_min_max
-                                                               , **kwargs)
-            optimization_routes += [gradient_optimize_result['optimization_route']]
-
-            evaluate_valid(gradient_optimize_result['best_style']
-                                          , gradient_optimize_result['best_temperature']
-                                          , results_on_valid, results_on_valid_tracked)
-
-            print(f'After diff --> Valid Selection metric: {results_on_valid[-1]}')
-
-        if selection_metric_min_max == 'min':
-            is_best = (results_on_valid_tracked[-1] <= min(results_on_valid_tracked))
-        else:
-            is_best = (results_on_valid_tracked[-1] >= max(results_on_valid_tracked))
-
-        if is_best or best_style is None:
-            best_style = gradient_optimize_result['best_style'].clone()
-            best_softmax_temperature = gradient_optimize_result['best_temperature'].clone()
-    torch.cuda.empty_cache()
-
-    def final_evaluation():
-        print('Running eval dataset with final params (no gradients)..')
-        print(best_style, best_softmax_temperature)
-        result_test = []
-        for N_ensemble_configurations in N_ensemble_configurations_list:
-            print(f'Running with {N_ensemble_configurations} ensemble_configurations')
-            kwargs['N_ensemble_configurations'] = N_ensemble_configurations
-            splits = []
-            for split in final_splits:
-                splits += [eval_step(test_datasets, best_style, softmax_temperature=best_softmax_temperature
-                                     , return_tensor=False, eval_positions=eval_positions_test,
-                                     bptt=bptt_final, inference_mode=True, split_number=split, model=model[2]
-                                     , selection_metric=selection_metric, evaluation_metric=evaluation_metric)]
-            result_test += [splits]
-
-        print('Running valid dataset with final params (no gradients)..')
-        result_valid = eval_step(valid_datasets, best_style, softmax_temperature=best_softmax_temperature
-                                 , return_tensor=False, eval_positions=eval_positions_test,
-                                 bptt=bptt_final, inference_mode=True, model=model[2]
-                                 , selection_metric=selection_metric, evaluation_metric=evaluation_metric)
-
-        return result_test, result_valid
-
-    result_test, result_valid = final_evaluation()
-
-    return result_test, result_valid, best_style, best_softmax_temperature, optimization_routes
-
-
-def eval_step(ds, used_style, selection_metric, evaluation_metric, eval_positions, return_tensor=True, **kwargs):
-    def step():
-        return evaluate(datasets=ds,
-                        method='transformer'
-                        , overwrite=True
-                        , style=used_style
-                        , eval_positions=eval_positions
-                        , metric_used=selection_metric
-                        , save=False
-                        , path_interfix=None
-                        , base_path=None
-                        , verbose=True
-                        , **kwargs)
-
-    if return_tensor:
-        r = step()
-    else:
-        with torch.no_grad():
-            r = step()
-
-    calculate_score_per_method(selection_metric, 'select', r, ds, eval_positions, aggregator='mean')
-    calculate_score_per_method(evaluation_metric, 'eval', r, ds, eval_positions, aggregator='mean')
-
-    return r
-
-
-def gradient_optimize_style(model, init_style, steps, softmax_temperature, train_datasets, valid_datasets, learning_rate=0.03, optimize_all=False,
-                            limit_style=True, N_datasets_sampled=90, optimize_softmax_temperature=True, selection_metric_min_max='max', **kwargs):
-    """
-    Uses gradient based methods to optimize 'style' on the 'train_datasets' and uses stopping with 'valid_datasets'.
-
-    :param model:
-    :param init_style:
-    :param steps:
-    :param learning_rate:
-    :param softmax_temperature:
-    :param train_datasets:
-    :param valid_datasets:
-    :param optimize_all:
-    :param limit_style:
-    :param N_datasets_sampled:
-    :param optimize_softmax_temperature:
-    :param selection_metric_min_max:
-    :param kwargs:
-    :return:
-    """
-    grad_style = torch.nn.Parameter(init_style.detach(), requires_grad=True)
-
-    best_style, best_temperature, best_selection_metric, best_diffable_metric = grad_style.detach(), softmax_temperature.detach(), None, None
-    softmax_temperature = torch.nn.Parameter(softmax_temperature.detach(), requires_grad=optimize_softmax_temperature)
-    variables_to_optimize = model[2].parameters() if optimize_all else [grad_style, softmax_temperature]
-    optimizer = torch.optim.Adam(variables_to_optimize, lr=learning_rate)
-
-    optimization_route_selection, optimization_route_diffable = [], []
-    optimization_route_selection_valid, optimization_route_diffable_valid = [], []
-
-    def eval_opt(ds, return_tensor=True, inference_mode=False):
-        result = eval_step(ds, grad_style, softmax_temperature=softmax_temperature, return_tensor=return_tensor
-                           , inference_mode=inference_mode, model=model[2], **kwargs)
-
-        diffable_metric = result['mean_metric']
-        selection_metric = result['mean_select']
-
-        return diffable_metric, selection_metric
-
-    def eval_all_datasets(datasets, propagate=True):
-        selection_metrics_this_step, diffable_metrics_this_step = [], []
-        for ds in datasets:
-            diffable_metric_train, selection_metric_train = eval_opt([ds], inference_mode=(not propagate))
-            if not torch.isnan(diffable_metric_train).any():
-                if propagate and diffable_metric_train.requires_grad == True:
-                    diffable_metric_train.backward()
-                selection_metrics_this_step += [selection_metric_train]
-                diffable_metrics_this_step += [float(diffable_metric_train.detach().cpu().numpy())]
-        diffable_metric_train = np.nanmean(diffable_metrics_this_step)
-        selection_metric_train = np.nanmean(selection_metrics_this_step)
-
-        return diffable_metric_train, selection_metric_train
-
-    for t in tqdm(range(steps), desc='Iterate over Optimization steps'):
-        optimizer.zero_grad()
-
-        # Select subset of datasets
-        random.seed(t)
-        train_datasets_ = random.sample(train_datasets, N_datasets_sampled)
-
-        # Get score on train
-        diffable_metric_train, selection_metric_train = eval_all_datasets(train_datasets_, propagate=True)
-        optimization_route_selection += [float(selection_metric_train)]
-        optimization_route_diffable += [float(diffable_metric_train)]
-
-        # Get score on valid
-        diffable_metric_valid, selection_metric_valid = eval_all_datasets(valid_datasets, propagate=False)
-        optimization_route_selection_valid += [float(selection_metric_valid)]
-        optimization_route_diffable_valid += [float(diffable_metric_valid)]
-
-        is_best = (selection_metric_min_max == 'min' and best_selection_metric > selection_metric_valid)
-        is_best = is_best or (selection_metric_min_max == 'max' and best_selection_metric < selection_metric_valid)
-        if (best_selection_metric is None) or (not np.isnan(selection_metric_valid) and is_best):
-            print('New best', best_selection_metric, selection_metric_valid)
-            best_style = grad_style.detach().clone()
-            best_temperature = softmax_temperature.detach().clone()
-            best_selection_metric, best_diffable_metric = selection_metric_valid, diffable_metric_valid
-
-        optimizer.step()
-
-        if limit_style:
-            grad_style = grad_style.detach().clamp(-1.74, 1.74)
-
-        print(f'Valid: Diffable metric={diffable_metric_valid} Selection metric={selection_metric_valid};' +
-            f'Train: Diffable metric={diffable_metric_train} Selection metric={selection_metric_train}')
-
-    print(f'Return best:{best_style} {best_selection_metric}')
-    return {'best_style': best_style, 'best_temperature': best_temperature
-            , 'optimization_route': {'select': optimization_route_selection, 'loss': optimization_route_diffable,
-               'test_select': optimization_route_selection_valid, 'test_loss': optimization_route_diffable_valid}}

TabPFN/layer.py

@@ -103,6 +103,12 @@ class TransformerEncoderLayer(Module):
 
             src2 = torch.cat([global_tokens_src2, train_tokens_src2, eval_tokens_src2], dim=0)
 
+        elif isinstance(src_mask, int):
+            assert src_key_padding_mask is None
+            single_eval_position = src_mask
+            src_left = self.self_attn(src_[:single_eval_position], src_[:single_eval_position], src_[:single_eval_position])[0]
+            src_right = self.self_attn(src_[single_eval_position:], src_[:single_eval_position], src_[:single_eval_position])[0]
+            src2 = torch.cat([src_left, src_right], dim=0)
         else:
             if self.recompute_attn:
                 src2 = checkpoint(self.self_attn, src_, src_, src_, src_key_padding_mask, True, src_mask)[0]

TabPFN/model_builder.py

@@ -1,273 +0,0 @@
-from train import train, Losses
-import priors
-import encoders
-
-from collections import defaultdict
-
-from priors.utils import trunc_norm_sampler_f, gamma_sampler_f
-from utils import get_uniform_single_eval_pos_sampler
-import torch
-import math
-
-def save_model(model, path, filename, config_sample):
-    config_sample = {**config_sample}
-
-    def make_serializable(config_sample):
-        if isinstance(config_sample, dict):
-            config_sample = {k: make_serializable(config_sample[k]) for k in config_sample}
-        if isinstance(config_sample, list):
-            config_sample = [make_serializable(v) for v in config_sample]
-        if callable(config_sample):
-            config_sample = str(config_sample)
-        return config_sample
-
-    #if 'num_features_used' in config_sample:
-    #    del config_sample['num_features_used']
-
-    #config_sample['num_classes_as_str'] = str(config_sample['num_classes'])
-    #del config_sample['num_classes']
-
-    config_sample = make_serializable(config_sample)
-
-    torch.save((model.state_dict(), None, config_sample), os.path.join(path, filename))
-
-
-import subprocess as sp
-import os
-
-def get_gpu_memory():
-    command = "nvidia-smi"
-    memory_free_info = sp.check_output(command.split()).decode('ascii')
-    return memory_free_info
-
-
-def load_model(path, filename, device, eval_positions, verbose):
-    # TODO: This function only restores evaluation functionality but training canät be continued. It is also not flexible.
-
-    model_state, optimizer_state, config_sample = torch.load(
-        os.path.join(path, filename), map_location='cpu')
-    if ('differentiable_hyperparameters' in config_sample
-            and 'prior_mlp_activations' in config_sample['differentiable_hyperparameters']):
-        config_sample['differentiable_hyperparameters']['prior_mlp_activations']['choice_values_used'] = config_sample[
-                                                                                                         'differentiable_hyperparameters'][
-                                                                                                         'prior_mlp_activations'][
-                                                                                                         'choice_values']
-        config_sample['differentiable_hyperparameters']['prior_mlp_activations']['choice_values'] = [
-            torch.nn.Tanh for k in config_sample['differentiable_hyperparameters']['prior_mlp_activations']['choice_values']]
-
-    config_sample['categorical_features_sampler'] = lambda: lambda x: ([], [], [])
-    config_sample['num_features_used_in_training'] = config_sample['num_features_used']
-    config_sample['num_features_used'] = lambda: config_sample['num_features']
-    config_sample['num_classes_in_training'] = config_sample['num_classes']
-    config_sample['num_classes'] = 2
-    config_sample['batch_size_in_training'] = config_sample['batch_size']
-    config_sample['batch_size'] = 1
-    config_sample['bptt_in_training'] = config_sample['bptt']
-    config_sample['bptt'] = 10
-    config_sample['bptt_extra_samples_in_training'] = config_sample['bptt_extra_samples']
-    config_sample['bptt_extra_samples'] = None
-
-    #print('Memory', str(get_gpu_memory()))
-
-    model = get_model(config_sample, device=device, should_train=False, verbose=verbose)
-    module_prefix = 'module.'
-    model_state = {k.replace(module_prefix, ''): v for k, v in model_state.items()}
-    model[2].load_state_dict(model_state)
-    model[2].to(device)
-
-    return model, config_sample
-
-def fix_loaded_config_sample(loaded_config_sample, config):
-    def copy_to_sample(*k):
-        t,s = loaded_config_sample, config
-        for k_ in k[:-1]:
-            t = t[k_]
-            s = s[k_]
-        t[k[-1]] = s[k[-1]]
-    copy_to_sample('num_features_used')
-    copy_to_sample('num_classes')
-    copy_to_sample('differentiable_hyperparameters','prior_mlp_activations','choice_values')
-
-def load_config_sample(path, template_config):
-    model_state, optimizer_state, loaded_config_sample = torch.load(path, map_location='cpu')
-    fix_loaded_config_sample(loaded_config_sample, template_config)
-    return loaded_config_sample
-
-def get_default_spec(test_datasets, valid_datasets):
-    bptt = 10000
-    eval_positions = [1000, 2000, 3000, 4000, 5000] # list(2 ** np.array([4, 5, 6, 7, 8, 9, 10, 11, 12]))
-    max_features = max([X.shape[1] for (_, X, _, _, _, _) in test_datasets] + [X.shape[1] for (_, X, _, _, _, _) in valid_datasets])
-    max_splits = 5
-
-    return bptt, eval_positions, max_features, max_splits
-
-def get_mlp_prior_hyperparameters(config):
-    config = {hp: (list(config[hp].values())[0]) if type(config[hp]) is dict else config[hp] for hp in config}
-
-    if "prior_sigma_gamma_k" in config:
-        sigma_sampler = gamma_sampler_f(config["prior_sigma_gamma_k"], config["prior_sigma_gamma_theta"])
-        config['init_std'] = sigma_sampler
-    if "prior_noise_std_gamma_k" in config:
-        noise_std_sampler = gamma_sampler_f(config["prior_noise_std_gamma_k"], config["prior_noise_std_gamma_theta"])
-        config['noise_std'] = noise_std_sampler
-
-    return config
-
-
-def get_gp_mix_prior_hyperparameters(config):
-    return {'lengthscale_concentration': config["prior_lengthscale_concentration"],
-            'nu': config["prior_nu"],
-            'outputscale_concentration': config["prior_outputscale_concentration"],
-            'categorical_data': config["prior_y_minmax_norm"],
-            'y_minmax_norm': config["prior_lengthscale_concentration"],
-            'noise_concentration': config["prior_noise_concentration"],
-            'noise_rate': config["prior_noise_rate"]}
-
-def get_gp_prior_hyperparameters(config):
-    return {hp: (list(config[hp].values())[0]) if type(config[hp]) is dict else config[hp] for hp in config}
-
-
-def get_meta_gp_prior_hyperparameters(config):
-    config = {hp: (list(config[hp].values())[0]) if type(config[hp]) is dict else config[hp] for hp in config}
-
-    if "outputscale_mean" in config:
-        outputscale_sampler = trunc_norm_sampler_f(config["outputscale_mean"]
-                                                   , config["outputscale_mean"] * config["outputscale_std_f"])
-        config['outputscale'] = outputscale_sampler
-    if "lengthscale_mean" in config:
-        lengthscale_sampler = trunc_norm_sampler_f(config["lengthscale_mean"],
-                                                   config["lengthscale_mean"] * config["lengthscale_std_f"])
-        config['lengthscale'] = lengthscale_sampler
-
-    return config
-
-
-def get_model(config, device, should_train=True, verbose=False, state_dict=None, epoch_callback=None):
-    extra_kwargs = {}
-    verbose_train, verbose_prior = verbose >= 1, verbose >= 2
-    config['verbose'] = verbose_prior
-
-    if 'aggregate_k_gradients' not in config or config['aggregate_k_gradients'] is None:
-        config['aggregate_k_gradients'] = math.ceil(config['batch_size'] * ((config['nlayers'] * config['emsize'] * config['bptt'] * config['bptt']) / 10824640000))
-
-    config['num_steps'] = math.ceil(config['num_steps'] * config['aggregate_k_gradients'])
-    config['batch_size'] = math.ceil(config['batch_size'] / config['aggregate_k_gradients'])
-    config['recompute_attn'] = config['recompute_attn'] if 'recompute_attn' in config else False
-
-    def make_get_batch(model_proto, **extra_kwargs):
-        extra_kwargs = defaultdict(lambda: None, **extra_kwargs)
-        return (lambda batch_size, seq_len, num_features, hyperparameters
-                , device, model_proto=model_proto, get_batch=extra_kwargs['get_batch']
-                       , prior_bag_priors=extra_kwargs['prior_bag_priors']: model_proto.get_batch(
-            batch_size=batch_size
-            , seq_len=seq_len
-            , device=device
-            , get_batch=get_batch
-            , hyperparameters=hyperparameters
-            , num_features=num_features))
-
-    if config['prior_type'] == 'prior_bag':
-        # Prior bag combines priors
-        get_batch_gp = make_get_batch(priors.fast_gp)
-        get_batch_mlp = make_get_batch(priors.mlp)
-        if 'flexible' in config and config['flexible']:
-            get_batch_gp = make_get_batch(priors.flexible_categorical, **{'get_batch': get_batch_gp})
-            get_batch_mlp = make_get_batch(priors.flexible_categorical, **{'get_batch': get_batch_mlp})
-        prior_bag_hyperparameters = {'prior_bag_get_batch': (get_batch_gp, get_batch_mlp)
-            , 'prior_bag_exp_weights_1': 2.0}
-        prior_hyperparameters = {**get_mlp_prior_hyperparameters(config), **get_gp_prior_hyperparameters(config)
-            , **prior_bag_hyperparameters}
-        model_proto = priors.prior_bag
-    else:
-        if config['prior_type'] == 'mlp':
-            prior_hyperparameters = get_mlp_prior_hyperparameters(config)
-            model_proto = priors.mlp
-        elif config['prior_type'] == 'gp':
-            prior_hyperparameters = get_gp_prior_hyperparameters(config)
-            model_proto = priors.fast_gp
-        elif config['prior_type'] == 'gp_mix':
-            prior_hyperparameters = get_gp_mix_prior_hyperparameters(config)
-            model_proto = priors.fast_gp_mix
-        else:
-            raise Exception()
-
-        if 'flexible' in config and config['flexible']:
-            get_batch_base = make_get_batch(model_proto)
-            extra_kwargs['get_batch'] = get_batch_base
-            model_proto = priors.flexible_categorical
-
-    use_style = False
-
-    if 'differentiable' in config and config['differentiable']:
-        get_batch_base = make_get_batch(model_proto, **extra_kwargs)
-        extra_kwargs = {'get_batch': get_batch_base, 'differentiable_hyperparameters': config['differentiable_hyperparameters']}
-        model_proto = priors.differentiable_prior
-        use_style = True
-    print(f"Using style prior: {use_style}")
-
-    if (('nan_prob_no_reason' in config and config['nan_prob_no_reason'] > 0.0) or
-        ('nan_prob_a_reason' in config and config['nan_prob_a_reason'] > 0.0) or
-        ('nan_prob_unknown_reason' in config and config['nan_prob_unknown_reason'] > 0.0)):
-        encoder = encoders.NanHandlingEncoder
-    else:
-        encoder = encoders.Linear
-
-    num_outputs = config['num_outputs'] if 'num_outputs' in config else 1
-    if config['max_num_classes'] == 2:
-        if 'joint_loss' in config and config['joint_loss']:
-            loss = JointBCELossWithLogits
-        else:
-            loss = Losses.bce
-    elif config['max_num_classes'] > 2:
-        loss = Losses.ce(torch.ones((config['max_num_classes'])))
-    else:
-        loss = BarDistribution(borders=get_bucket_limits(500, full_range=(-10, 10)))
-
-    aggregate_k_gradients = 1 if 'aggregate_k_gradients' not in config else config['aggregate_k_gradients']
-    check_is_compatible = False if 'multiclass_loss_type' not in config else (config['multiclass_loss_type'] == 'compatible')
-    config['multiclass_type'] = config['multiclass_type'] if 'multiclass_type' in config else 'rank'
-    config['mix_activations'] = config['mix_activations'] if 'mix_activations' in config else False
-
-    config['bptt_extra_samples'] = config['bptt_extra_samples'] if 'bptt_extra_samples' in config else None
-    config['eval_positions'] = [int(config['bptt'] * 0.95)] if config['bptt_extra_samples'] is None else [int(config['bptt'])]
-
-    epochs = 0 if not should_train else config['epochs']
-    model = train(model_proto.DataLoader
-                  , loss
-                  , encoder
-                  , style_encoder_generator = encoders.StyleEncoder if use_style else None
-                  , emsize=config['emsize']
-                  , nhead=config['nhead']
-                  , y_encoder_generator= encoders.get_Canonical(config['max_num_classes']) if config.get('canonical_y_encoder', False) else encoders.Linear
-                  , pos_encoder_generator=None
-                  , batch_size=config['batch_size']
-                  , nlayers=config['nlayers']
-                  , nhid=config['emsize'] * config['nhid_factor']
-                  , epochs=epochs
-                  , total_available_time_in_s=config.get('total_available_time_in_s', None)
-                  , warmup_epochs=20
-                  , bptt=config['bptt']
-                  , gpu_device=device
-                  , dropout=config['dropout']
-                  , steps_per_epoch=config['num_steps']
-                  , single_eval_pos_gen=get_uniform_single_eval_pos_sampler(config['bptt'])
-                  , load_weights_from_this_state_dict=state_dict
-                  , aggregate_k_gradients=aggregate_k_gradients
-                  , check_is_compatible=check_is_compatible
-                  , recompute_attn=config['recompute_attn']
-                  , epoch_callback=epoch_callback
-                  , bptt_extra_samples = config['bptt_extra_samples']
-                  , extra_prior_kwargs_dict={
-            'num_features': config['num_features']
-            , 'fuse_x_y': False
-            , 'hyperparameters': prior_hyperparameters
-            , 'num_outputs':num_outputs
-            , 'dynamic_batch_size': 1 if ('num_global_att_tokens' in config and config['num_global_att_tokens']) else 2
-            , **extra_kwargs
-        }
-                  , lr=config['lr']
-                  , verbose=verbose_train,
-                  weight_decay=config.get('weight_decay', 0.0),
-                  normalize_labels=True)
-
-    return model

TabPFN/models_diff/gp_ablation_model.cpkt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c7b0c8febc553cca3fdee265b5a1cd7567dbf83da855969940be4707a9218ffb
-size 69460013

TabPFN/models_diff/prior_diff_real_checkpoint_n_8x_lr0.0003_epoch_49.cpkt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dae97f45bd53d719fc2b23fac4ec55eab16d63892196d939b1bb1c3b408be242
-size 103616779

TabPFN/prior_tuning_result.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:24d2189bbc836aeea888cf6c540f2c1b45b5351822931189e8bf10a0bc80a0b6
-size 18668851

TabPFN/scripts/differentiable_pfn_evaluation.py

@@ -10,8 +10,9 @@ from priors.differentiable_prior import draw_random_style
 from tqdm import tqdm
 from pathlib import Path
 import random
-from model_builder import load_model
+from scripts.model_builder import load_model
 from scripts.transformer_prediction_interface import get_params_from_config
+from scripts.transformer_prediction_interface import load_model_workflow
 
 """
 ===============================
@@ -24,55 +25,9 @@ def eval_model_range(i_range, *args, **kwargs):
     for i in i_range:
         eval_model(i, *args, **kwargs)
 
-
-def load_model_workflow(i, e, add_name, base_path, device='cpu', eval_addition=''):
-    """
-    Workflow for loading a model and setting appropriate parameters for diffable hparam tuning.
-
-    :param i:
-    :param e:
-    :param eval_positions_valid:
-    :param add_name:
-    :param base_path:
-    :param device:
-    :param eval_addition:
-    :return:
-    """
-    def check_file(e):
-        model_file = f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{e}.cpkt'
-        model_path = os.path.join(base_path, model_file)
-        # print('Evaluate ', model_path)
-        results_file = os.path.join(base_path,
-                                    f'models_diff/prior_diff_real_results{add_name}_n_{i}_epoch_{e}_{eval_addition}.pkl')
-        if not Path(model_path).is_file():  # or Path(results_file).is_file():
-            return None, None, None
-        return model_file, model_path, results_file
-
-    model_file = None
-    if e == -1:
-        for e_ in range(100, -1, -1):
-            model_file_, model_path_, results_file_ = check_file(e_)
-            if model_file_ is not None:
-                e = e_
-                model_file, model_path, results_file = model_file_, model_path_, results_file_
-                break
-    else:
-        model_file, model_path, results_file = check_file(e)
-
-    if model_file is None:
-        print('No checkpoint found')
-        return None
-
-    print(f'Loading {model_file}')
-
-    model, c = load_model(base_path, model_file, device, eval_positions=[], verbose=False)
-
-    return model, c, results_file
-
-
-def eval_model(i, e, valid_datasets, test_datasets, train_datasets, eval_positions_valid, eval_positions_test,
-               bptt_valid,
-               bptt_test, add_name, base_path, device='cpu', eval_addition='', **extra_tuning_args):
+def eval_model(i, e, valid_datasets, test_datasets, train_datasets, add_name, base_path,  eval_positions_valid=[1000], eval_positions_test=[1000],
+               bptt_valid=2000,
+               bptt_test=2000, device='cpu', eval_addition='', differentiable=False, **extra_tuning_args):
     """
     Differentiable model evaliation workflow. Evaluates and saves results to disk.
 
@@ -107,12 +62,12 @@ def eval_model(i, e, valid_datasets, test_datasets, train_datasets, eval_positio
     params.update(get_params_from_config(c))
 
     start = time.time()
-    metrics, metrics_valid, style, temperature, optimization_route = evaluate_differentiable_model(model, **params,
-                                                                                                   **extra_tuning_args)
+    metrics, metrics_valid, style, temperature, optimization_route = evaluate_point_model(model, **params,
+                                                                                                       **extra_tuning_args)
     print('Evaluation time: ', time.time() - start)
 
     print(results_file)
-    r = [c.copy(), metrics, metrics_valid, style.to('cpu'), temperature.to('cpu'), optimization_route]
+    r = [c.copy(), metrics, metrics_valid, style.to('cpu') if style else style, temperature.to('cpu') if temperature else temperature, optimization_route]
     with open(results_file, 'wb') as output:
         del r[0]['num_features_used']
         del r[0]['categorical_features_sampler']
@@ -128,22 +83,18 @@ INTERNAL HELPER FUNCTIONS
 ===============================
 """
 
-def evaluate_differentiable_model(model
+def evaluate_point_model(model
                                   , valid_datasets
                                   , test_datasets
                                   , train_datasets
-                                  , N_draws=100
-                                  , N_grad_steps=10
-                                  , eval_positions=None
                                   , eval_positions_test=None
-                                  , bptt=100
                                   , bptt_final=200
-                                  , style=None
-                                  , n_parallel_configurations=1
                                   , device='cpu'
                                   , selection_metric='auc'
                                   , final_splits=[1, 2, 3, 4, 5]
                                   , N_ensemble_configurations_list=[1, 5, 10, 20, 50, 100]
+                         , bptt=None
+                         , eval_positions=None
                                   , **kwargs):
     """
     Evaluation function for diffable model evaluation. Returns a list of results.
@@ -171,107 +122,38 @@ def evaluate_differentiable_model(model
     np.random.seed(0)
     random.seed(0)
 
-    diffable_metric = tabular_metrics.cross_entropy
     evaluation_metric = tabular_metrics.auc_metric
-    if selection_metric in ('auc', 'roc'):
-        selection_metric_min_max = 'max'
-        selection_metric = tabular_metrics.auc_metric
-        evaluation_metric = selection_metric
-    elif selection_metric in ('ce', 'selection_metric'):
-        selection_metric_min_max = 'min'
-        selection_metric = tabular_metrics.cross_entropy
-        evaluation_metric = selection_metric
-
-    print('Diffable metric', diffable_metric, ' Selection metric', selection_metric, ' Evaluation metric',
-          evaluation_metric)
-    print('N PARALLEL CONFIGURATIONS', n_parallel_configurations)
-    print('eval_positions', eval_positions)
-
-    def evaluate_valid(style, softmax_temperature, results, results_tracked):
-        result_valid = eval_step(valid_datasets, style, softmax_temperature=softmax_temperature,
-                                 return_tensor=False, inference_mode=True, selection_metric=selection_metric,
-                                 evaluation_metric=evaluation_metric, eval_positions=eval_positions, bptt=bptt, model=model[2])
-        result_valid = [float(result_valid[f'mean_select_at_{pos}']) for pos in eval_positions]
-        results += [result_valid]
-        results_tracked += [np.nanmean(result_valid)]
+    selection_metric = tabular_metrics.auc_metric
 
     model[2].to(device)
     model[2].eval()
 
-    results_on_valid, results_on_valid_tracked = [], []
-    best_style, best_softmax_temperature = style, torch.cat(
-        [torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)], 0)
-    optimization_routes = []
-
-    best_style = torch.cat([draw_random_style(model[3], device).detach() for n in range(0, n_parallel_configurations)],
-                      0)
-    best_softmax_temperature = torch.cat([torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)],
-                                    0)
-
-
-    for _ in tqdm(range(0, N_draws), desc='Iterate over Optimization initializations'): # Evaluates N hparam draws
-        style = torch.cat([draw_random_style(model[3], device).detach() for n in range(0, n_parallel_configurations)],
-                          0)
-        softmax_temperature = torch.cat([torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)],
-                                        0)
-
-        evaluate_valid(style, softmax_temperature, results_on_valid, results_on_valid_tracked)
-
-        print(f'Draw --> Valid Selection metric: {results_on_valid[-1]}')
-
-        if N_grad_steps > 0:
-            gradient_optimize_result = gradient_optimize_style(model, style, N_grad_steps
-                                                               , softmax_temperature=softmax_temperature
-                                                               , model=model[2]
-                                                               , train_datasets=train_datasets
-                                                               , valid_datasets=valid_datasets
-                                                               , selection_metric_min_max=selection_metric_min_max
-                                                               , **kwargs)
-            optimization_routes += [gradient_optimize_result['optimization_route']]
-
-            evaluate_valid(gradient_optimize_result['best_style']
-                                          , gradient_optimize_result['best_temperature']
-                                          , results_on_valid, results_on_valid_tracked)
-
-            print(f'After diff --> Valid Selection metric: {results_on_valid[-1]}')
-
-        if selection_metric_min_max == 'min':
-            is_best = (results_on_valid_tracked[-1] <= min(results_on_valid_tracked))
-        else:
-            is_best = (results_on_valid_tracked[-1] >= max(results_on_valid_tracked))
-
-        if is_best or best_style is None:
-            best_style = gradient_optimize_result['best_style'].clone()
-            best_softmax_temperature = gradient_optimize_result['best_temperature'].clone()
-    torch.cuda.empty_cache()
-
     def final_evaluation():
         print('Running eval dataset with final params (no gradients)..')
-        print(best_style, best_softmax_temperature)
         result_test = []
         for N_ensemble_configurations in N_ensemble_configurations_list:
             print(f'Running with {N_ensemble_configurations} ensemble_configurations')
             kwargs['N_ensemble_configurations'] = N_ensemble_configurations
             splits = []
             for split in final_splits:
-                splits += [eval_step(test_datasets, best_style, softmax_temperature=best_softmax_temperature
+                splits += [eval_step(test_datasets, None, softmax_temperature=torch.tensor([0])
                                      , return_tensor=False, eval_positions=eval_positions_test,
-                                     bptt=bptt_final, inference_mode=True, split_number=split, model=model[2]
-                                     , selection_metric=selection_metric, evaluation_metric=evaluation_metric)]
+                                     bptt=bptt_final, split_number=split, model=model[2], device=device
+                                     , selection_metric=selection_metric, evaluation_metric=evaluation_metric
+                                     , **kwargs)]
             result_test += [splits]
 
         print('Running valid dataset with final params (no gradients)..')
-        result_valid = eval_step(valid_datasets, best_style, softmax_temperature=best_softmax_temperature
+        result_valid = eval_step(valid_datasets, None, softmax_temperature=torch.tensor([0])
                                  , return_tensor=False, eval_positions=eval_positions_test,
-                                 bptt=bptt_final, inference_mode=True, model=model[2]
-                                 , selection_metric=selection_metric, evaluation_metric=evaluation_metric)
+                                 bptt=bptt_final, model=model[2], device=device
+                                 , selection_metric=selection_metric, evaluation_metric=evaluation_metric,**kwargs)
 
         return result_test, result_valid
 
     result_test, result_valid = final_evaluation()
 
-    return result_test, result_valid, best_style, best_softmax_temperature, optimization_routes
-
+    return result_test, result_valid, None, None, None
 
 def eval_step(ds, used_style, selection_metric, evaluation_metric, eval_positions, return_tensor=True, **kwargs):
     def step():
@@ -284,7 +166,6 @@ def eval_step(ds, used_style, selection_metric, evaluation_metric, eval_position
                         , save=False
                         , path_interfix=None
                         , base_path=None
-                        , verbose=True
                         , **kwargs)
 
     if return_tensor:
@@ -299,7 +180,7 @@ def eval_step(ds, used_style, selection_metric, evaluation_metric, eval_position
     return r
 
 
-def gradient_optimize_style(model, init_style, steps, softmax_temperature, train_datasets, valid_datasets, learning_rate=0.03, optimize_all=False,
+def gradient_optimize_style(model, init_style, steps, softmax_temperature, train_datasets, valid_datasets, bptt, learning_rate=0.03, optimize_all=False,
                             limit_style=True, N_datasets_sampled=90, optimize_softmax_temperature=True, selection_metric_min_max='max', **kwargs):
     """
     Uses gradient based methods to optimize 'style' on the 'train_datasets' and uses stopping with 'valid_datasets'.
@@ -331,7 +212,7 @@ def gradient_optimize_style(model, init_style, steps, softmax_temperature, train
 
     def eval_opt(ds, return_tensor=True, inference_mode=False):
         result = eval_step(ds, grad_style, softmax_temperature=softmax_temperature, return_tensor=return_tensor
-                           , inference_mode=inference_mode, model=model[2], **kwargs)
+                           , inference_mode=inference_mode, model=model[2], bptt=bptt, **kwargs)
 
         diffable_metric = result['mean_metric']
         selection_metric = result['mean_select']
@@ -369,9 +250,10 @@ def gradient_optimize_style(model, init_style, steps, softmax_temperature, train
         optimization_route_selection_valid += [float(selection_metric_valid)]
         optimization_route_diffable_valid += [float(diffable_metric_valid)]
 
-        is_best = (selection_metric_min_max == 'min' and best_selection_metric > selection_metric_valid)
+        is_best = (best_selection_metric is None)
+        is_best = is_best or (selection_metric_min_max == 'min' and best_selection_metric > selection_metric_valid)
         is_best = is_best or (selection_metric_min_max == 'max' and best_selection_metric < selection_metric_valid)
-        if (best_selection_metric is None) or (not np.isnan(selection_metric_valid) and is_best):
+        if (not np.isnan(selection_metric_valid) and is_best):
             print('New best', best_selection_metric, selection_metric_valid)
             best_style = grad_style.detach().clone()
             best_temperature = softmax_temperature.detach().clone()

TabPFN/scripts/model_configs.py

@@ -12,10 +12,10 @@ def get_general_config(max_features, bptt, eval_positions=None):
     Returns the general PFN training hyperparameters.
     """
     config_general = {
-        "lr": CSH.UniformFloatHyperparameter('lr', lower=0.00002, upper=0.0002, log=True),
+        "lr": CSH.UniformFloatHyperparameter('lr', lower=0.0001, upper=0.00015, log=True),
         "dropout": CSH.CategoricalHyperparameter('dropout', [0.0]),
         "emsize": CSH.CategoricalHyperparameter('emsize', [2 ** i for i in range(8, 9)]), ## upper bound is -1
-        "batch_size": CSH.CategoricalHyperparameter('batch_size', [2 ** i for i in range(8, 9)]),
+        "batch_size": CSH.CategoricalHyperparameter('batch_size', [2 ** i for i in range(6, 8)]),
         "nlayers": CSH.CategoricalHyperparameter('nlayers', [12]),
         "num_features": max_features,
         "nhead": CSH.CategoricalHyperparameter('nhead', [4]),
@@ -27,8 +27,9 @@ def get_general_config(max_features, bptt, eval_positions=None):
         "epochs": 80,
         "num_steps": 100,
         "verbose": False,
-        "pre_sample_causes": True, # This is MLP
-        "mix_activations": False,#hp.choice('mix_activations', [True, False]),
+        "mix_activations": False,
+        "pre_sample_causes": True,
+        "multiclass_type": 'rank'
     }
 
     return config_general
@@ -38,9 +39,9 @@ def get_flexible_categorical_config(max_features):
     Returns the configuration parameters for the tabular multiclass wrapper.
     """
     config_flexible_categorical = {
-        "nan_prob_unknown_reason_reason_prior": CSH.CategoricalHyperparameter('nan_prob_unknown_reason_reason_prior', [1.0]),
-        "categorical_feature_p": CSH.CategoricalHyperparameter('categorical_feature_p', [0.0]),
-        "nan_prob_no_reason": CSH.CategoricalHyperparameter('nan_prob_no_reason', [0.0, 0.1, 0.2]),
+        "nan_prob_unknown_reason_reason_prior": CSH.CategoricalHyperparameter('nan_prob_unknown_reason_reason_prior', [0.5]),
+        "categorical_feature_p": CSH.CategoricalHyperparameter('categorical_feature_p', [0.0, 0.1, 0.2]),
+        "nan_prob_no_reason": CSH.CategoricalHyperparameter('nan_prob_no_reason', [0.0, 0.1]),
         "nan_prob_unknown_reason": CSH.CategoricalHyperparameter('nan_prob_unknown_reason', [0.0]),
         "nan_prob_a_reason": CSH.CategoricalHyperparameter('nan_prob_a_reason', [0.0]),
         # "num_classes": lambda : random.randint(2, 10), "balanced": False,
@@ -66,6 +67,7 @@ def get_diff_flex():
         # "num_categorical_features_sampler_a": hp.choice('num_categorical_features_sampler_a',
         #                                                 [{'distribution': 'uniform', 'min': 0.3, 'max': 0.9}, None]),
         # "num_categorical_features_sampler_b": {'distribution': 'uniform', 'min': 0.3, 'max': 0.9},
+
         "output_multiclass_ordered_p": {'distribution': 'uniform', 'min': 0.0, 'max': 0.5}, #CSH.CategoricalHyperparameter('output_multiclass_ordered_p', [0.0, 0.1, 0.2]),
         "multiclass_type": {'distribution': 'meta_choice', 'choice_values': ['value', 'rank']},
     }
@@ -91,34 +93,41 @@ def get_diff_causal():
     Returns the configuration parameters for a differentiable wrapper around MLP / Causal mixture.
     """
     diff_causal = {
-        "num_layers": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 6, 'min_mean': 1, 'round': True,
+        #"mix_activations": {'distribution': 'meta_choice', 'choice_values': [True, False]},
+        #"num_layers": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 6, 'min_mean': 1, 'round': True,
+        #               'lower_bound': 2},
+        "num_layers": {'distribution': 'meta_gamma', 'max_alpha': 2, 'max_scale': 3, 'round': True,
                        'lower_bound': 2},
         # Better beta?
-        "prior_mlp_hidden_dim": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 130, 'min_mean': 5,
-                                 'round': True, 'lower_bound': 4},
+        #"prior_mlp_hidden_dim": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 130, 'min_mean': 5,
+        #                         'round': True, 'lower_bound': 4},
+        "prior_mlp_hidden_dim": {'distribution': 'meta_gamma', 'max_alpha': 3, 'max_scale': 100, 'round': True, 'lower_bound': 4},
 
-        "prior_mlp_dropout_prob": {'distribution': 'meta_beta', 'scale': 0.9, 'min': 0.1, 'max': 5.0},
+        "prior_mlp_dropout_prob": {'distribution': 'meta_beta', 'scale': 0.6, 'min': 0.1, 'max': 5.0},
     # This mustn't be too high since activations get too large otherwise
 
         "noise_std": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': .3, 'min_mean': 0.0001, 'round': False,
                       'lower_bound': 0.0},
         "init_std": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 0.01, 'round': False,
                      'lower_bound': 0.0},
-        "num_causes": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 12, 'min_mean': 1, 'round': True,
-                       'lower_bound': 1},
+        #"num_causes": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 12, 'min_mean': 1, 'round': True,
+        #               'lower_bound': 1},
+        "num_causes": {'distribution': 'meta_gamma', 'max_alpha': 3, 'max_scale': 7, 'round': True,
+                                 'lower_bound': 2},
+
         "is_causal": {'distribution': 'meta_choice', 'choice_values': [True, False]},
         "pre_sample_weights": {'distribution': 'meta_choice', 'choice_values': [True, False]},
         "y_is_effect": {'distribution': 'meta_choice', 'choice_values': [True, False]},
+        "sampling": {'distribution': 'meta_choice', 'choice_values': ['normal', 'mixed']},
         "prior_mlp_activations": {'distribution': 'meta_choice_mixed', 'choice_values': [
             torch.nn.Tanh
-            , torch.nn.ReLU
             , torch.nn.Identity
-            , lambda : torch.nn.LeakyReLU(negative_slope=0.1)
-            , torch.nn.ELU
+            , torch.nn.ReLU
         ]},
         "block_wise_dropout": {'distribution': 'meta_choice', 'choice_values': [True, False]},
         "sort_features": {'distribution': 'meta_choice', 'choice_values': [True, False]},
         "in_clique": {'distribution': 'meta_choice', 'choice_values': [True, False]},
+        #'pre_sample_causes': {'distribution': 'meta_choice', 'choice_values': [True, False]},
     }
 
     return diff_causal
@@ -128,7 +137,7 @@ def get_diff_prior_bag():
     Returns the configuration parameters for a GP and MLP / Causal mixture.
     """
     diff_prior_bag = {
-        'prior_bag_exp_weights_1': {'distribution': 'uniform', 'min': 100000., 'max': 100001.},
+        'prior_bag_exp_weights_1': {'distribution': 'uniform', 'min': 2.0, 'max': 10.0},
         # MLP Weight (Biased, since MLP works better, 1.0 is weight for prior number 0)
     }
 
@@ -148,6 +157,72 @@ def get_diff_config():
     return config_diff
 
 
+def get_prior_config(config_type):
+    if config_type == 'causal':
+        return get_prior_config_causal()
+    elif config_type == 'gp':
+        return get_prior_config_gp()
+    elif config_type == 'bnn':
+        return get_prior_config_bnn()
+
+
+def get_prior_config_gp(max_features=100):
+    config_general = get_general_config(max_features, 50, eval_positions=[30])
+    config_general_real_world = {**config_general}
+
+    config_flexible_categorical = get_flexible_categorical_config(max_features)
+    config_flexible_categorical_real_world = {**config_flexible_categorical}
+
+    config_gp = {}
+
+    config_diff = get_diff_config()
+
+    config = {**config_general_real_world, **config_flexible_categorical_real_world, **config_diff, **config_gp}
+
+    config['differentiable_hyperparameters']['prior_bag_exp_weights_1'] = {'distribution': 'uniform', 'min': 0.0,
+                                                                                  'max': .01}  # Never select MLP
+
+
+def get_prior_config_bnn(max_features=100):
+    config_general = get_general_config(max_features, 50, eval_positions=[30])
+    config_general_real_world = {**config_general}
+
+    config_flexible_categorical = get_flexible_categorical_config(max_features)
+    config_flexible_categorical_real_world = {**config_flexible_categorical}
+
+    config_gp = {}
+    config_mlp = {}
+
+    config_diff = get_diff_config()
+
+    config = {**config_general_real_world, **config_flexible_categorical_real_world, **config_diff, **config_gp,
+              **config_mlp}
+
+    config['differentiable_hyperparameters']['prior_bag_exp_weights_1'] = {'distribution': 'uniform',
+                                                                                  'min': 1000.0,
+                                                                                  'max': 1001.0}  # Always select MLP
+
+
+def get_prior_config_causal(max_features=100):
+    config_general = get_general_config(max_features, 50, eval_positions=[30])
+    config_general_real_world = {**config_general}
+
+    config_flexible_categorical = get_flexible_categorical_config(max_features)
+    config_flexible_categorical_real_world = {**config_flexible_categorical}
+    config_flexible_categorical_real_world[
+        'num_categorical_features_sampler_a'] = -1.0  # Categorical features disabled by default
+
+    config_gp = {}
+    config_mlp = {}
+
+    config_diff = get_diff_config()
+
+    config = {**config_general_real_world, **config_flexible_categorical_real_world, **config_diff, **config_gp,
+              **config_mlp}
+
+    return config
+
+
 def sample_differentiable(config):
     """"
     Returns sampled hyperparameters from a differentiable wrapper, that is it makes a non-differentiable out of

TabPFN/scripts/tabular_baselines.py

@@ -1,19 +1,34 @@
+import pandas
 from catboost import CatBoostClassifier, Pool
+from sklearn.model_selection import GridSearchCV
+from sklearn.model_selection import KFold
+from sklearn.model_selection import ParameterGrid
 
+import tempfile
+import random
 import math
-
+import os
+#from pytorch_tabnet.tab_model import TabNetClassifier, TabNetRegressor
+from sklearn import preprocessing
+from torch import nn
+from sklearn.metrics import make_scorer
 from sklearn.impute import SimpleImputer
 
+
+from sklearn.base import BaseEstimator, ClassifierMixin
 import xgboost as xgb
 from sklearn import neighbors
 from sklearn.gaussian_process import GaussianProcessClassifier
 from sklearn.gaussian_process.kernels import RBF
 import numpy as np
-
+import torch
+import itertools
 from scripts import tabular_metrics
 import pandas as pd
+from tqdm import tqdm
+from utils import remove_outliers
 
-from sklearn.linear_model import LogisticRegression
+from sklearn.linear_model import LogisticRegression, Ridge
 from sklearn.model_selection import cross_val_score
 import time
 
@@ -37,18 +52,28 @@ def get_scoring_direction(metric_used):
     else:
         raise Exception('No scoring string found for metric')
 
+def is_classification(metric_used):
+    if metric_used == tabular_metrics.auc_metric or metric_used == tabular_metrics.cross_entropy:
+        return 'classification'
+    elif metric_used == tabular_metrics.auc_metric:
+        return -1
+
+# Loss
+
 def get_scoring_string(metric_used, multiclass=True, usage="sklearn_cv"):
     if metric_used == tabular_metrics.auc_metric:
         if usage == 'sklearn_cv':
             return 'roc_auc_ovo'
         elif usage == 'autogluon':
-            return 'log_loss' # Autogluon crashes when using 'roc_auc' with some datasets usning logloss gives better scores;
+            #return 'log_loss' # Autogluon crashes when using 'roc_auc' with some datasets usning logloss gives better scores;
                               # We might be able to fix this, but doesn't work out of box.
                               # File bug report? Error happens with dataset robert and fabert
             if multiclass:
                 return 'roc_auc_ovo_macro'
             else:
                 return 'roc_auc'
+        elif usage == 'tabnet':
+            return 'logloss' if multiclass else 'auc'
         elif usage == 'autosklearn':
             if multiclass:
                 return autosklearn.metrics.log_loss # roc_auc only works for binary, use logloss instead
@@ -58,25 +83,72 @@ def get_scoring_string(metric_used, multiclass=True, usage="sklearn_cv"):
             return 'MultiClass' # Effectively LogLoss, ROC not available
         elif usage == 'xgb':
             return 'logloss'
+        elif usage == 'lightgbm':
+            if multiclass:
+                return 'auc'
+            else:
+                return 'binary'
         return 'roc_auc'
     elif metric_used == tabular_metrics.cross_entropy:
         if usage == 'sklearn_cv':
             return 'neg_log_loss'
         elif usage == 'autogluon':
             return 'log_loss'
+        elif usage == 'tabnet':
+            return 'logloss'
         elif usage == 'autosklearn':
             return autosklearn.metrics.log_loss
         elif usage == 'catboost':
             return 'MultiClass' # Effectively LogLoss
         return 'logloss'
+    elif metric_used == tabular_metrics.r2_metric:
+        if usage == 'autosklearn':
+            return autosklearn.metrics.r2
+        elif usage == 'sklearn_cv':
+            return 'r2' # tabular_metrics.neg_r2
+        elif usage == 'autogluon':
+            return 'r2'
+        elif usage == 'xgb': # XGB cannot directly optimize r2
+            return 'rmse'
+        elif usage == 'catboost': # Catboost cannot directly optimize r2 ("Can't be used for optimization." - docu)
+            return 'RMSE'
+        else:
+            return 'r2'
+    elif metric_used == tabular_metrics.root_mean_squared_error_metric:
+        if usage == 'autosklearn':
+            return autosklearn.metrics.root_mean_squared_error
+        elif usage == 'sklearn_cv':
+            return 'neg_root_mean_squared_error' # tabular_metrics.neg_r2
+        elif usage == 'autogluon':
+            return 'rmse'
+        elif usage == 'xgb':
+            return 'rmse'
+        elif usage == 'catboost':
+            return 'RMSE'
+        else:
+            return 'neg_root_mean_squared_error'
+    elif metric_used == tabular_metrics.mean_absolute_error_metric:
+        if usage == 'autosklearn':
+            return autosklearn.metrics.mean_absolute_error
+        elif usage == 'sklearn_cv':
+            return 'neg_mean_absolute_error' # tabular_metrics.neg_r2
+        elif usage == 'autogluon':
+            return 'mae'
+        elif usage == 'xgb':
+            return 'mae'
+        elif usage == 'catboost':
+            return 'MAE'
+        else:
+            return 'neg_mean_absolute_error'
     else:
         raise Exception('No scoring string found for metric')
 
 def eval_f(params, clf_, x, y, metric_used, start_time, max_time):
     if time.time() - start_time > max_time:
         return np.nan
-    scores = cross_val_score(clf_(**params), x, y, cv=CV, scoring=get_scoring_string(metric_used))
-
+    scores = cross_val_score(clf_(**params), x, y, cv=CV, scoring=get_scoring_string(metric_used, usage='sklearn_cv'))
+    if get_scoring_string(metric_used, usage='sklearn_cv') == 'r2':
+        return np.nanmean(scores)
     return -np.nanmean(scores)
 
 def preprocess_impute(x, y, test_x, test_y, impute, one_hot, standardize, cat_features=[]):
@@ -110,10 +182,26 @@ def preprocess_impute(x, y, test_x, test_y, impute, one_hot, standardize, cat_fe
         x, test_x = scaler.transform(x), scaler.transform(test_x)
 
     return x, y, test_x, test_y
+import torch
+import random
+from tqdm import tqdm
+def transformer_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
+    from scripts.transformer_prediction_interface import TabPFNClassifier
+
+    classifier = TabPFNClassifier(device='cpu', base_path='.',
+                                  model_string='')
+    classifier.fit(x, y)
+    print('Train data shape', x.shape, ' Test data shape', test_x.shape)
+    pred = classifier.predict_proba(test_x)
+
+    metric = metric_used(test_y, pred)
+
+    return metric, pred, None
 
 ## Auto Gluon
+# WARNING: Crashes for some predictors for regression
 def autogluon_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
-    from autogluon.tabular import TabularPredictor # Inside function so package can be sued without installation
+    from autogluon.tabular import TabularPredictor
     x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y
                                              , one_hot=False
                                              , cat_features=cat_features
@@ -121,12 +209,15 @@ def autogluon_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=3
                                              , standardize=False)
     train_data = pd.DataFrame(np.concatenate([x, y[:, np.newaxis]], 1))
     test_data = pd.DataFrame(np.concatenate([test_x, test_y[:, np.newaxis]], 1))
-
+    if is_classification(metric_used):
+        problem_type = 'multiclass' if len(np.unique(y)) > 2 else 'binary'
+    else:
+        problem_type = 'regression'
     # AutoGluon automatically infers datatypes, we don't specify the categorical labels
     predictor = TabularPredictor(
         label=train_data.columns[-1],
         eval_metric=get_scoring_string(metric_used, usage='autogluon', multiclass=(len(np.unique(y)) > 2)),
-        problem_type='multiclass' if len(np.unique(y)) > 2 else 'binary'
+        problem_type=problem_type
         ## seed=int(y[:].sum()) doesn't accept seed
     ).fit(
         train_data=train_data,
@@ -135,19 +226,717 @@ def autogluon_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=3
         # The seed is deterministic but varies for each dataset and each split of it
     )
 
-    pred = predictor.predict_proba(test_data, as_multiclass=True).values
+    if is_classification(metric_used):
+        pred = predictor.predict_proba(test_data, as_multiclass=True).values
+    else:
+        pred = predictor.predict(test_data).values
+
+    metric = metric_used(test_y, pred)
+
+    return metric, pred, predictor.fit_summary()
+
+
+from autogluon.core.models import AbstractModel
+from scripts.transformer_prediction_interface import TabPFNClassifier
+
+
+class TabPFNModel(AbstractModel):
+    def __init__(self, **kwargs):
+        # Simply pass along kwargs to parent, and init our internal `_feature_generator` variable to None
+        super().__init__(**kwargs)
+
+    # The `_preprocess` method takes the input data and transforms it to the internal representation usable by the model.
+    # `_preprocess` is called by `preprocess` and is used during model fit and model inference.
+    def _preprocess(self, X: pd.DataFrame, is_train=False, **kwargs) -> np.ndarray:
+        return X
+
+    # The `_fit` method takes the input training data (and optionally the validation data) and trains the model.
+    def _fit(self,
+             X: pd.DataFrame,  # training data
+             y: pd.Series,  # training labels
+             # X_val=None,  # val data (unused in RF model)
+             # y_val=None,  # val labels (unused in RF model)
+             # time_limit=None,  # time limit in seconds (ignored in tutorial)
+             **kwargs):  # kwargs includes many other potential inputs, refer to AbstractModel documentation for details
+
+        self.model = TabPFNClassifier(device='cpu', base_path='/work/dlclarge1/hollmann-PFN_Tabular/',
+                                      model_string='_longer_multiclass_causal_05_02_2022_12_49_44_sams',
+                                      N_ensemble_configurations=10)
+        self.model.fit(X.to_numpy(), y.to_numpy())
+
+    def _predict_proba(self, X, **kwargs):
+        X = self.preprocess(X, **kwargs)
+
+        #if self.problem_type in [REGRESSION, QUANTILE]:
+        #    y_pred = self.model.predict(X)
+        #    return y_pred
+
+        y_pred_proba = self.model.predict_proba(X.to_numpy())
+        return super()._convert_proba_to_unified_form(y_pred_proba)
+
+    # The `_set_default_params` method defines the default hyperparameters of the model.
+    # User-specified parameters will override these values on a key-by-key basis.
+    def _set_default_params(self):
+        default_params = {
+        }
+from autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config
+def autogluon_tabpfn_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
+    from autogluon.tabular import TabularPredictor
+    x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y
+                                             , one_hot=False
+                                             , cat_features=cat_features
+                                             , impute=False
+                                             , standardize=False)
+    train_data = pd.DataFrame(np.concatenate([x, y[:, np.newaxis]], 1))
+    test_data = pd.DataFrame(np.concatenate([test_x, test_y[:, np.newaxis]], 1))
+    if is_classification(metric_used):
+        problem_type = 'multiclass' if len(np.unique(y)) > 2 else 'binary'
+    else:
+        problem_type = 'regression'
+    # AutoGluon automatically infers datatypes, we don't specify the categorical labels
+    custom_hyperparameters = {}#get_hyperparameter_config('default')
+    custom_hyperparameters[TabPFNModel] = {}
+    predictor = TabularPredictor(
+        label=train_data.columns[-1],
+        eval_metric=get_scoring_string(metric_used, usage='autogluon', multiclass=(len(np.unique(y)) > 2)),
+        problem_type=problem_type
+        ## seed=int(y[:].sum()) doesn't accept seed
+    ).fit(
+        train_data=train_data,
+        time_limit=max_time,
+        presets=['best_quality'],
+        hyperparameters=custom_hyperparameters
+        # The seed is deterministic but varies for each dataset and each split of it
+    )
+
+    if is_classification(metric_used):
+        pred = predictor.predict_proba(test_data, as_multiclass=True).values
+    else:
+        pred = predictor.predict(test_data).values
 
     metric = metric_used(test_y, pred)
 
     return metric, pred, predictor.fit_summary()
 
+def get_updates_for_regularization_cocktails(
+    categorical_indicator: np.ndarray):
+    """
+    These updates replicate the regularization cocktail paper search space.
+    Args:
+        categorical_indicator (np.ndarray)
+            An array that indicates whether a feature is categorical or not.
+        args (Namespace):
+            The different updates for the setup of the run, mostly updates
+            for the different regularization ingredients.
+    Returns:
+    ________
+        pipeline_update, search_space_updates, include_updates (Tuple[dict, HyperparameterSearchSpaceUpdates, dict]):
+            The pipeline updates like number of epochs, budget, seed etc.
+            The search space updates like setting different hps to different values or ranges.
+            Lastly include updates, which can be used to include different features.
+    """
+    from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
+    import argparse
+
+    augmentation_names_to_trainers = {
+        'mixup': 'MixUpTrainer',
+        'cutout': 'RowCutOutTrainer',
+        'cutmix': 'RowCutMixTrainer',
+        'standard': 'StandardTrainer',
+        'adversarial': 'AdversarialTrainer',
+    }
+
+    include_updates = dict()
+    include_updates['network_embedding'] = ['NoEmbedding']
+    include_updates['network_init'] = ['NoInit']
+
+    has_cat_features = any(categorical_indicator)
+    has_numerical_features = not all(categorical_indicator)
+
+    def str2bool(v):
+        if isinstance(v, bool):
+            return [v, ]
+        if v.lower() in ('yes', 'true', 't', 'y', '1'):
+            return [True, ]
+        elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+            return [False, ]
+        elif v.lower() == 'conditional':
+            return [True, False]
+        else:
+            raise ValueError('No valid value given.')
+    search_space_updates = HyperparameterSearchSpaceUpdates()
+
+    # architecture head
+    search_space_updates.append(
+        node_name='network_head',
+        hyperparameter='__choice__',
+        value_range=['no_head'],
+        default_value='no_head',
+    )
+    search_space_updates.append(
+        node_name='network_head',
+        hyperparameter='no_head:activation',
+        value_range=['relu'],
+        default_value='relu',
+    )
+
+    # backbone architecture
+    search_space_updates.append(
+        node_name='network_backbone',
+        hyperparameter='__choice__',
+        value_range=['ShapedResNetBackbone'],
+        default_value='ShapedResNetBackbone',
+    )
+    search_space_updates.append(
+        node_name='network_backbone',
+        hyperparameter='ShapedResNetBackbone:resnet_shape',
+        value_range=['brick'],
+        default_value='brick',
+    )
+    search_space_updates.append(
+        node_name='network_backbone',
+        hyperparameter='ShapedResNetBackbone:num_groups',
+        value_range=[2],
+        default_value=2,
+    )
+    search_space_updates.append(
+        node_name='network_backbone',
+        hyperparameter='ShapedResNetBackbone:blocks_per_group',
+        value_range=[2],
+        default_value=2,
+    )
+    search_space_updates.append(
+        node_name='network_backbone',
+        hyperparameter='ShapedResNetBackbone:output_dim',
+        value_range=[512],
+        default_value=512,
+    )
+    search_space_updates.append(
+        node_name='network_backbone',
+        hyperparameter='ShapedResNetBackbone:max_units',
+        value_range=[512],
+        default_value=512,
+    )
+    search_space_updates.append(
+        node_name='network_backbone',
+        hyperparameter='ShapedResNetBackbone:activation',
+        value_range=['relu'],
+        default_value='relu',
+    )
+    search_space_updates.append(
+        node_name='network_backbone',
+        hyperparameter='ShapedResNetBackbone:shake_shake_update_func',
+        value_range=['even-even'],
+        default_value='even-even',
+    )
+
+    # training updates
+    search_space_updates.append(
+        node_name='lr_scheduler',
+        hyperparameter='__choice__',
+        value_range=['CosineAnnealingWarmRestarts'],
+        default_value='CosineAnnealingWarmRestarts',
+    )
+    search_space_updates.append(
+        node_name='lr_scheduler',
+        hyperparameter='CosineAnnealingWarmRestarts:n_restarts',
+        value_range=[3],
+        default_value=3,
+    )
+    search_space_updates.append(
+        node_name='optimizer',
+        hyperparameter='__choice__',
+        value_range=['AdamWOptimizer'],
+        default_value='AdamWOptimizer',
+    )
+    search_space_updates.append(
+        node_name='optimizer',
+        hyperparameter='AdamWOptimizer:lr',
+        value_range=[1e-3],
+        default_value=1e-3,
+    )
+    search_space_updates.append(
+        node_name='data_loader',
+        hyperparameter='batch_size',
+        value_range=[128],
+        default_value=128,
+    )
+
+    # preprocessing
+    search_space_updates.append(
+        node_name='feature_preprocessor',
+        hyperparameter='__choice__',
+        value_range=['NoFeaturePreprocessor'],
+        default_value='NoFeaturePreprocessor',
+    )
+
+    if has_numerical_features:
+        print('has numerical features')
+        search_space_updates.append(
+            node_name='imputer',
+            hyperparameter='numerical_strategy',
+            value_range=['median'],
+            default_value='median',
+        )
+        search_space_updates.append(
+            node_name='scaler',
+            hyperparameter='__choice__',
+            value_range=['StandardScaler'],
+            default_value='StandardScaler',
+        )
+
+    if has_cat_features:
+        print('has cat features')
+        search_space_updates.append(
+            node_name='imputer',
+            hyperparameter='categorical_strategy',
+            value_range=['constant_!missing!'],
+            default_value='constant_!missing!',
+        )
+        search_space_updates.append(
+            node_name='encoder',
+            hyperparameter='__choice__',
+            value_range=['OneHotEncoder'],
+            default_value='OneHotEncoder',
+        )
+
+    search_space_updates.append(
+        node_name='optimizer',
+        hyperparameter='AdamWOptimizer:beta1',
+        value_range=[0.9],
+        default_value=0.9,
+    )
+    search_space_updates.append(
+        node_name='optimizer',
+        hyperparameter='AdamWOptimizer:beta2',
+        value_range=[0.999],
+        default_value=0.999,
+    )
+
+
+    parser = argparse.ArgumentParser(
+        description='Run AutoPyTorch on a benchmark.',
+    )
+    # experiment setup arguments
+    parser.add_argument(
+        '--task_id',
+        type=int,
+        default=233088,
+    )
+    parser.add_argument(
+        '--wall_time',
+        type=int,
+        default=9000,
+    )
+    parser.add_argument(
+        '--func_eval_time',
+        type=int,
+        default=1000,
+    )
+    parser.add_argument(
+        '--epochs',
+        type=int,
+        default=105,
+    )
+    parser.add_argument(
+        '--seed',
+        type=int,
+        default=11,
+    )
+    parser.add_argument(
+        '--tmp_dir',
+        type=str,
+        default='./runs/autoPyTorch_cocktails',
+    )
+    parser.add_argument(
+        '--output_dir',
+        type=str,
+        default='./runs/autoPyTorch_cocktails',
+    )
+    parser.add_argument(
+        '--nr_workers',
+        type=int,
+        default=1,
+    )
+    parser.add_argument(
+        '--nr_threads',
+        type=int,
+        default=1,
+    )
+    parser.add_argument(
+        '--cash_cocktail',
+        help='If the regularization cocktail should be used.',
+        type=bool,
+        default=False,
+    )
+
+    # regularization ingredient arguments
+    parser.add_argument(
+        '--use_swa',
+        help='If stochastic weight averaging should be used.',
+        type=str2bool,
+        nargs='?',
+        const=[True],
+        default=[False],
+    )
+    parser.add_argument(
+        '--use_se',
+        help='If snapshot ensembling should be used.',
+        type=str2bool,
+        nargs='?',
+        const=[True],
+        default=[False],
+    )
+    parser.add_argument(
+        '--use_lookahead',
+        help='If the lookahead optimizing technique should be used.',
+        type=str2bool,
+        nargs='?',
+        const=[True],
+        default=[False],
+    )
+    parser.add_argument(
+        '--use_weight_decay',
+        help='If weight decay regularization should be used.',
+        type=str2bool,
+        nargs='?',
+        const=[True],
+        default=[False],
+    )
+    parser.add_argument(
+        '--use_batch_normalization',
+        help='If batch normalization regularization should be used.',
+        type=str2bool,
+        nargs='?',
+        const=[True],
+        default=[False],
+    )
+    parser.add_argument(
+        '--use_skip_connection',
+        help='If skip connections should be used. '
+             'Turns the network into a residual network.',
+        type=str2bool,
+        nargs='?',
+        const=[True],
+        default=[False],
+    )
+    parser.add_argument(
+        '--use_dropout',
+        help='If dropout regularization should be used.',
+        type=str2bool,
+        nargs='?',
+        const=[True],
+        default=[False],
+    )
+    parser.add_argument(
+        '--mb_choice',
+        help='Multibranch network regularization. '
+             'Only active when skip_connection is active.',
+        type=str,
+        choices=['none', 'shake-shake', 'shake-drop'],
+        default='none',
+    )
+    parser.add_argument(
+        '--augmentation',
+        help='If methods that augment examples should be used',
+        type=str,
+        choices=['mixup', 'cutout', 'cutmix', 'standard', 'adversarial'],
+        default='standard',
+    )
+
+    args = parser.parse_args([]) # just get default values
+
+
+
+    # if the cash formulation of the cocktail is not activated,
+    # otherwise the methods activation will be chosen by the SMBO optimizer.
+
+
+    # No early stopping and train on gpu
+    pipeline_update = {
+        'early_stopping': -1,
+        'min_epochs': args.epochs,
+        'epochs': args.epochs,
+        "device": 'cpu',
+    }
+
+    return pipeline_update, search_space_updates, include_updates
+
+def get_smac_object(
+    scenario_dict,
+    seed: int,
+    ta,
+    ta_kwargs,
+    n_jobs: int,
+    initial_budget: int,
+    max_budget: int,
+    dask_client,
+):
+    """
+    This function returns an SMAC object that is gonna be used as
+    optimizer of pipelines.
+    Args:
+        scenario_dict (typing.Dict[str, typing.Any]): constrain on how to run
+            the jobs.
+        seed (int): to make the job deterministic.
+        ta (typing.Callable): the function to be intensified by smac.
+        ta_kwargs (typing.Dict[str, typing.Any]): Arguments to the above ta.
+        n_jobs (int): Amount of cores to use for this task.
+        initial_budget (int):
+            The initial budget for a configuration.
+        max_budget (int):
+            The maximal budget for a configuration.
+        dask_client (dask.distributed.Client): User provided scheduler.
+    Returns:
+        (SMAC4AC): sequential model algorithm configuration object
+    """
+    from smac.intensification.simple_intensifier import SimpleIntensifier
+    from smac.runhistory.runhistory2epm import RunHistory2EPM4LogCost
+    from smac.scenario.scenario import Scenario
+    from smac.facade.smac_ac_facade import SMAC4AC
+    # multi-fidelity is disabled, that is why initial_budget and max_budget
+    # are not used.
+    rh2EPM = RunHistory2EPM4LogCost
+
+    return SMAC4AC(
+        scenario=Scenario(scenario_dict),
+        rng=seed,
+        runhistory2epm=rh2EPM,
+        tae_runner=ta,
+        tae_runner_kwargs=ta_kwargs,
+        initial_configurations=None,
+        run_id=seed,
+        intensifier=SimpleIntensifier,
+        dask_client=dask_client,
+        n_jobs=n_jobs,
+    )
+
+
+def get_incumbent_results(
+    run_history_file: str,
+    search_space
+):
+    """
+    Get the incumbent configuration and performance from the previous run HPO
+    search with AutoPytorch.
+    Args:
+        run_history_file (str):
+            The path where the AutoPyTorch search data is located.
+        search_space (ConfigSpace.ConfigurationSpace):
+            The ConfigurationSpace that was previously used for the HPO
+            search space.
+    Returns:
+        config, incumbent_run_value (Tuple[ConfigSpace.Configuration, float]):
+            The incumbent configuration found from HPO search and the validation
+            performance it achieved.
+    """
+    from smac.runhistory.runhistory import RunHistory
+    run_history = RunHistory()
+    run_history.load_json(
+        run_history_file,
+        search_space,
+    )
+
+    run_history_data = run_history.data
+    sorted_runvalue_by_cost = sorted(run_history_data.items(), key=lambda item: item[1].cost)
+    incumbent_run_key, incumbent_run_value = sorted_runvalue_by_cost[0]
+    config = run_history.ids_config[incumbent_run_key.config_id]
+    return config, incumbent_run_value
+
+
+def well_tuned_simple_nets_metric(X_train, y_train, X_test, y_test, categorical_indicator, metric_used, max_time=300, nr_workers=1):
+    """Install:
+    git clone https://github.com/automl/Auto-PyTorch.git
+    cd Auto-PyTorch
+    git checkout regularization_cocktails
+    From the page, not needed for me at least: conda install gxx_linux-64 gcc_linux-64 swig
+    conda create --clone CONDANAME --name CLONENAME
+    conda activate CLONENAME
+    pip install -r requirements.txt (I checked looks like nothing should break functionality of our project not sure about baselines, thus a copied env is likely good :))
+    pip install -e .
+    """
+    #os.environ.get('SLURM_JOBID', '')
+    categorical_indicator = np.array([i in categorical_indicator for i in range(X_train.shape[1])])
+    with tempfile.TemporaryDirectory(prefix=f"{len(X_train)}_{len(X_test)}_{max_time}") as temp_dir:
+        from autoPyTorch.api.tabular_classification import TabularClassificationTask
+        from autoPyTorch.datasets.resampling_strategy import HoldoutValTypes, NoResamplingStrategyTypes
+        from autoPyTorch.data.tabular_validator import TabularInputValidator
+        from autoPyTorch.datasets.tabular_dataset import TabularDataset
+        from autoPyTorch import metrics
+        # append random folder to temp_dir to avoid collisions
+        rand_int = str(random.randint(1,1000))
+        temp_dir = os.path.join(temp_dir, 'temp_'+rand_int)
+        out_dir = os.path.join(temp_dir, 'out_'+rand_int)
+
+        start_time = time.time()
+
+        X_train, y_train, X_test, y_test = X_train.cpu().numpy(), y_train.cpu().long().numpy(), X_test.cpu().numpy(), y_test.cpu().long().numpy()
+
+        def safe_int(x):
+            assert np.all(x.astype('int64') == x) or np.any(x != x), np.unique(x) # second condition for ignoring nans
+            return pd.Series(x, dtype='category')
+
+        X_train = pd.DataFrame({i: safe_int(X_train[:,i]) if c else X_train[:,i] for i, c in enumerate(categorical_indicator)})
+        X_test = pd.DataFrame({i: safe_int(X_test[:,i]) if c else X_test[:,i] for i, c in enumerate(categorical_indicator)})
+
+
+        if isinstance(y_train[1], bool):
+            y_train = y_train.astype('bool')
+        if isinstance(y_test[1], bool):
+            y_test = y_test.astype('bool')
+
+        number_of_configurations_limit = 840 # hard coded in the paper
+        epochs = 105
+        func_eval_time = min(1000, max_time/2)
+        seed = int(y_train[:].sum())
+
+
+        resampling_strategy_args = {
+            'val_share': len(y_test)/(len(y_test)+len(y_train)),
+        }
+
+        pipeline_update, search_space_updates, include_updates = get_updates_for_regularization_cocktails(
+            categorical_indicator,
+        )
+        print(search_space_updates)
+
+
+
+        ############################################################################
+        # Build and fit a classifier
+        # ==========================
+        # if we use HPO, we can use multiple workers in parallel
+        if number_of_configurations_limit == 0:
+            nr_workers = 1
+
+        api = TabularClassificationTask(
+            temporary_directory=temp_dir,
+            output_directory=out_dir,
+            delete_tmp_folder_after_terminate=False,
+            delete_output_folder_after_terminate=False,
+            resampling_strategy=HoldoutValTypes.stratified_holdout_validation,
+            resampling_strategy_args=resampling_strategy_args,
+            ensemble_size=1,
+            ensemble_nbest=1,
+            max_models_on_disc=10,
+            include_components=include_updates,
+            search_space_updates=search_space_updates,
+            seed=seed,
+            n_jobs=nr_workers,
+            n_threads=1,
+        )
+
+        api.set_pipeline_config(**pipeline_update)
+        ############################################################################
+        # Search for the best hp configuration
+        # ====================================
+        # We search for the best hp configuration only in the case of a cocktail ingredient
+        # that has hyperparameters.
+        print(X_train, X_test)
+        print('temp_dir',temp_dir)
+        print(max_time, min(func_eval_time, max_time, number_of_configurations_limit))
+
+        if number_of_configurations_limit != 0:
+            api.search(
+                X_train=X_train.copy(),
+                y_train=y_train.copy(),
+                X_test=X_test.copy(),
+                y_test=y_test.copy(),
+                optimize_metric='balanced_accuracy',
+                total_walltime_limit=max_time,
+                memory_limit=12000,
+                func_eval_time_limit_secs=min(func_eval_time, max_time),
+                enable_traditional_pipeline=False,
+                get_smac_object_callback=get_smac_object,
+                smac_scenario_args={
+                    'runcount_limit': number_of_configurations_limit,
+                },
+            )
+
+        ############################################################################
+        # Refit on the best hp configuration
+        # ==================================
+        input_validator = TabularInputValidator(
+            is_classification=True,
+        )
+        input_validator.fit(
+            X_train=X_train.copy(),
+            y_train=y_train.copy(),
+            X_test=X_test.copy(),
+            y_test=y_test.copy(),
+        )
+
+        dataset = TabularDataset(
+            X=X_train,
+            Y=y_train,
+            X_test=X_test,
+            Y_test=y_test,
+            seed=seed,
+            validator=input_validator,
+            resampling_strategy=NoResamplingStrategyTypes.no_resampling,
+        )
+        dataset.is_small_preprocess = False
+        print(f"Fitting pipeline with {epochs} epochs")
+
+        search_space = api.get_search_space(dataset)
+        # only when we perform hpo will there be an incumbent configuration
+        # otherwise take a default configuration.
+        if number_of_configurations_limit != 0:
+            configuration, incumbent_run_value = get_incumbent_results(
+                os.path.join(
+                    temp_dir,
+                    'smac3-output',
+                    'run_{}'.format(seed),
+                    'runhistory.json'),
+                search_space,
+            )
+            print(f"Incumbent configuration: {configuration}")
+            print(f"Incumbent trajectory: {api.trajectory}")
+        else:
+            # default configuration
+            configuration = search_space.get_default_configuration()
+            print(f"Default configuration: {configuration}")
+
+        fitted_pipeline, run_info, run_value, dataset = api.fit_pipeline(
+            configuration=configuration,
+            budget_type='epochs',
+            budget=epochs,
+            dataset=dataset,
+            run_time_limit_secs=func_eval_time,
+            eval_metric='balanced_accuracy',
+            memory_limit=12000,
+        )
+
+        X_train = dataset.train_tensors[0]
+        y_train = dataset.train_tensors[1]
+        X_test = dataset.test_tensors[0]
+        y_test = dataset.test_tensors[1]
+
+        if fitted_pipeline is None:
+            api.get_incumbent_config
+
+
+        train_predictions = fitted_pipeline.predict(X_train)
+        test_predictions = fitted_pipeline.predict(X_test)
+
+        metric = metric_used(y_test, test_predictions.squeeze())
+        duration = time.time() - start_time
+
+        print(f'Time taken: {duration} for {metric} metric')
+        print(test_predictions[:10])
+        return metric, test_predictions, None
+
+
+
 ## AUTO Sklearn
 def autosklearn_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
+    import autosklearn.classification
     return autosklearn2_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=max_time, version=1)
 
-from autosklearn.experimental.askl2 import AutoSklearn2Classifier
-from autosklearn.classification import AutoSklearnClassifier
 def autosklearn2_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300, version=2):
+    from autosklearn.experimental.askl2 import AutoSklearn2Classifier
+    from autosklearn.classification import AutoSklearnClassifier
+    from autosklearn.regression import AutoSklearnRegressor
     x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y
                                              , one_hot=False
                                              , cat_features=cat_features
@@ -163,7 +952,12 @@ def autosklearn2_metric(x, y, test_x, test_y, cat_features, metric_used, max_tim
     x = make_pd_from_np(x)
     test_x = make_pd_from_np(test_x)
 
-    clf_ = AutoSklearn2Classifier if version == 2 else AutoSklearnClassifier
+    if is_classification(metric_used):
+        clf_ = AutoSklearn2Classifier if version == 2 else AutoSklearnClassifier
+    else:
+        if version == 2:
+            raise Exception("AutoSklearn 2 doesn't do regression.")
+        clf_ = AutoSklearnRegressor
     clf = clf_(time_left_for_this_task=max_time,
                                                            memory_limit=4000,
                                                            n_jobs=MULTITHREAD,
@@ -174,17 +968,141 @@ def autosklearn2_metric(x, y, test_x, test_y, cat_features, metric_used, max_tim
     # fit model to data
     clf.fit(x, y)
 
-    pred = clf.predict_proba(test_x)
+    if is_classification(metric_used):
+        pred = clf.predict_proba(test_x)
+    else:
+        pred = clf.predict(test_x)
     metric = metric_used(test_y, pred)
 
     return metric, pred, None
 
+param_grid_hyperopt['ridge'] = {
+    'max_iter': hp.randint('max_iter', 50, 500)
+    , 'fit_intercept': hp.choice('fit_intercept', [True, False])
+    , 'alpha': hp.loguniform('alpha', -5, math.log(5.0))}  # 'normalize': [False],
+
+def ridge_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
+    if is_classification(metric_used):
+        raise Exception("Ridge is only applicable to pointwise Regression.")
+
+    x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y
+                                             , one_hot=True, impute=True, standardize=True
+                                             , cat_features=cat_features)
+    def clf_(**params):
+        return Ridge(tol=1e-4, **params)
+
+    start_time = time.time()
+
+    def stop(trial):
+        return time.time() - start_time > max_time, []
+
+    best = fmin(
+        fn=lambda params: eval_f(params, clf_, x, y, metric_used, start_time, max_time),
+        space=param_grid_hyperopt['ridge'],
+        algo=rand.suggest,
+        rstate=np.random.RandomState(int(y[:].sum())),
+        early_stop_fn=stop,
+        # The seed is deterministic but varies for each dataset and each split of it
+        max_evals=10000)
+    best = space_eval(param_grid_hyperopt['ridge'], best)
+
+    clf = clf_(**best)
+    clf.fit(x, y)
+
+    pred = clf.predict(test_x)
+    metric = metric_used(test_y, pred)
+
+    return metric, pred, best
+
+from lightautoml.automl.presets.tabular_presets import TabularAutoML, TabularUtilizedAutoML
+from lightautoml.tasks import Task
+
+def lightautoml_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
+    x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y
+                                             , one_hot=False, impute=False, standardize=False
+                                             , cat_features=cat_features)
+
+    roles = {'target': str(x.shape[-1])}
+    task = Task('multiclass', metric = lambda x, y : metric_used(x, y, numpy=True))
+    automl = TabularUtilizedAutoML(task=task,
+                           timeout=max_time,
+                           cpu_limit=4,  # Optimal for Kaggle kernels
+                           general_params={'use_algos': [['linear_l2',
+                                                          'lgb', 'lgb_tuned']]})
+
+    tr_data = np.concatenate([x, np.expand_dims(y, -1)], -1)
+    tr_data = pd.DataFrame(tr_data, columns=[str(k) for k in range(0, x.shape[-1] + 1)])
+    oof_pred = automl.fit_predict(tr_data, roles=roles)
+    te_data = pd.DataFrame(test_x, columns=[str(k) for k in range(0, x.shape[-1])])
+
+    probabilities = automl.predict(te_data).data
+    probabilities_mapped = probabilities.copy()
+
+    class_map = automl.outer_pipes[0].ml_algos[0].models[0][0].reader.class_mapping
+    if class_map:
+        column_to_class = {col: class_ for class_, col in class_map.items()}
+        for i in range(0, len(column_to_class)):
+            probabilities_mapped[:, int(column_to_class[int(i)])] = probabilities[:, int(i)]
+
+    metric = metric_used(test_y, probabilities_mapped)
+
+    return metric, probabilities_mapped, None
+
+param_grid_hyperopt['lightgbm'] = {
+    'num_leaves': hp.randint('num_leaves', 5, 50)
+    , 'max_depth': hp.randint('max_depth', 3, 20)
+    , 'learning_rate': hp.loguniform('learning_rate', -3, math.log(1.0))
+    , 'n_estimators': hp.randint('n_estimators', 50, 2000)
+    #, 'feature_fraction': 0.8,
+    #, 'subsample': 0.2
+    , 'min_child_weight': hp.choice('min_child_weight', [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4])
+    , 'subsample': hp.uniform('subsample', 0.2, 0.8)
+    , 'colsample_bytree': hp.uniform('colsample_bytree', 0.2, 0.8)
+    , 'reg_alpha': hp.choice('reg_alpha', [0, 1e-1, 1, 2, 5, 7, 10, 50, 100])
+    , 'reg_lambda': hp.choice('reg_lambda', [0, 1e-1, 1, 5, 10, 20, 50, 100])
+}  # 'normalize': [False],
+
+from lightgbm import LGBMClassifier
+
+def lightgbm_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
+    x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y
+                                             , one_hot=False, impute=False, standardize=False
+                                             , cat_features=cat_features)
+
+    def clf_(**params):
+        return LGBMClassifier(categorical_feature=cat_features, use_missing=True
+                              , objective=get_scoring_string(metric_used, usage='lightgbm', multiclass=len(np.unique(y)) > 2), **params)
+
+    start_time = time.time()
+
+    def stop(trial):
+        return time.time() - start_time > max_time, []
+
+    best = fmin(
+        fn=lambda params: eval_f(params, clf_, x, y, metric_used, start_time, max_time),
+        space=param_grid_hyperopt['lightgbm'],
+        algo=rand.suggest,
+        rstate=np.random.RandomState(int(y[:].sum())),
+        early_stop_fn=stop,
+        # The seed is deterministic but varies for each dataset and each split of it
+        max_evals=10000)
+    best = space_eval(param_grid_hyperopt['lightgbm'], best)
+
+    clf = clf_(**best)
+    clf.fit(x, y)
+
+    pred = clf.predict_proba(test_x)
+    metric = metric_used(test_y, pred)
+
+    return metric, pred, best
+
 param_grid_hyperopt['logistic'] = {
     'penalty': hp.choice('penalty', ['l1', 'l2', 'none'])
-    , 'max_iter': hp.randint('max_iter', [50, 500])
+    , 'max_iter': hp.randint('max_iter', 50, 500)
     , 'fit_intercept': hp.choice('fit_intercept', [True, False])
     , 'C': hp.loguniform('C', -5, math.log(5.0))}  # 'normalize': [False],
 
+
 def logistic_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
     x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y
                                              , one_hot=True, impute=True, standardize=True
@@ -225,7 +1143,9 @@ def knn_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
                                              cat_features=cat_features)
 
     def clf_(**params):
-        return neighbors.KNeighborsClassifier(n_jobs=1, **params)
+        if is_classification(metric_used):
+            return neighbors.KNeighborsClassifier(n_jobs=1, **params)
+        return neighbors.KNeighborsRegressor(n_jobs=1, **params)
 
     start_time = time.time()
 
@@ -245,7 +1165,10 @@ def knn_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
     clf = clf_(**best)
     clf.fit(x, y)
 
-    pred = clf.predict_proba(test_x)
+    if is_classification(metric_used):
+        pred = clf.predict_proba(test_x)
+    else:
+        pred = clf.predict(test_x)
     metric = metric_used(test_y, pred)
 
     return metric, pred, best
@@ -253,8 +1176,7 @@ def knn_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
 ## GP
 param_grid_hyperopt['gp'] = {
     'params_y_scale': hp.loguniform('params_y_scale', math.log(0.05), math.log(5.0)),
-    'params_length_scale': hp.loguniform('params_length_scale', math.log(0.1), math.log(1.0)),
-    'n_jobs': hp.choice('njobs', [1])
+    'params_length_scale': hp.loguniform('params_length_scale', math.log(0.1), math.log(1.0))
 }
 def gp_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
     x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y,
@@ -262,7 +1184,10 @@ def gp_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
                                              cat_features=cat_features)
 
     def clf_(params_y_scale,params_length_scale, **params):
-        return GaussianProcessClassifier(kernel= params_y_scale * RBF(params_length_scale), **params)
+        if is_classification(metric_used):
+            return GaussianProcessClassifier(kernel= params_y_scale * RBF(params_length_scale), **params)
+        else:
+            return GaussianProcessRegressor(kernel= params_y_scale * RBF(params_length_scale), **params)
 
     start_time = time.time()
     def stop(trial):
@@ -282,11 +1207,89 @@ def gp_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
     clf = clf_(**best)
     clf.fit(x, y)
 
+    if is_classification(metric_used):
+        pred = clf.predict_proba(test_x)
+    else:
+        pred = clf.predict(test_x)
+    metric = metric_used(test_y, pred)
+
+    return metric, pred, best
+
+## Tabnet
+# https://github.com/dreamquark-ai/tabnet
+#param_grid['tabnet'] = {'n_d': [2, 4], 'n_steps': [2,4,6], 'gamma': [1.3], 'optimizer_params': [{'lr': 2e-2}, {'lr': 2e-1}]}
+
+# Hyperparameter space from dreamquarks implementation recommendations
+param_grid_hyperopt['tabnet'] = {
+    'n_d': hp.randint('n_d', 8, 64),
+    'n_steps': hp.randint('n_steps', 3, 10),
+    'max_epochs': hp.randint('max_epochs', 50, 200),
+    'gamma': hp.uniform('relax', 1.0, 2.0),
+    'momentum': hp.uniform('momentum', 0.01, 0.4),
+}
+
+def tabnet_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
+    from pytorch_tabnet.tab_model import TabNetClassifier
+    # TabNet inputs raw tabular data without any preprocessing and is trained using gradient descent-based optimisation.
+    # However Tabnet cannot handle nans so we impute with mean
+
+    x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y, impute=True, one_hot=False, standardize=False)
+
+    def clf_(**params):
+        return TabNetClassifier(cat_idxs=cat_features, verbose=True, n_a=params['n_d'], seed=int(y[:].sum()), **params)
+
+    def tabnet_eval_f(params, clf_, x, y, metric_used, start_time, max_time):
+        if time.time() - start_time > max_time:
+            return np.nan
+
+        kf = KFold(n_splits=min(CV, x.shape[0] // 2), random_state=None, shuffle=True)
+        metrics = []
+
+        params = {**params}
+        max_epochs = params['max_epochs']
+        del params['max_epochs']
+
+        for train_index, test_index in kf.split(x):
+            X_train, X_valid, y_train, y_valid = x[train_index], x[test_index], y[train_index], y[test_index]
+
+            clf = clf_(**params)
+
+            clf.fit(
+                X_train, y_train,
+                # eval_metric=[get_scoring_string(metric_used, multiclass=len(np.unique(y_train)) > 2, usage='tabnet')],
+                # eval_set=[(X_valid, y_valid)],
+                # patience=15,
+                max_epochs=max_epochs
+            )
+            metrics += [metric_used(y_valid, clf.predict_proba(X_valid))]
+
+        return -np.nanmean(np.array(metrics))
+
+    start_time = time.time()
+    def stop(trial):
+        return time.time() - start_time > max_time, []
+
+    best = fmin(
+        fn=lambda params: tabnet_eval_f(params, clf_, x, y, metric_used, start_time, max_time),
+        space=param_grid_hyperopt['tabnet'],
+        algo=rand.suggest,
+        rstate=np.random.RandomState(int(y[:].sum())),
+        early_stop_fn=stop,
+        max_evals=1000)
+    best = space_eval(param_grid_hyperopt['tabnet'], best)
+    max_epochs = best['max_epochs']
+    del best['max_epochs']
+
+    clf = clf_(**best)
+    clf.fit(x, y, max_epochs=max_epochs) # , max_epochs=mean_best_epochs[best_idx]
+
     pred = clf.predict_proba(test_x)
     metric = metric_used(test_y, pred)
 
     return metric, pred, best
 
+    return metric, pred, params_used[best_idx]
+
 
 # Catboost
 # Hyperparameter space: https://arxiv.org/pdf/2106.03253.pdf
@@ -301,8 +1304,6 @@ param_grid_hyperopt['catboost'] = {
 }
 
 def catboost_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
-    print(x)
-
     x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y
                                              , one_hot=False
                                              , cat_features=cat_features
@@ -323,14 +1324,24 @@ def catboost_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=30
     test_x = make_pd_from_np(test_x)
 
     def clf_(**params):
-        return CatBoostClassifier(
-                               loss_function=get_scoring_string(metric_used, usage='catboost'),
-                               thread_count = MULTITHREAD,
-                               used_ram_limit='4gb',
-            random_seed=int(y[:].sum()),
-                               logging_level='Silent',
-                                cat_features=cat_features,
-                                  **params)
+        if is_classification(metric_used):
+            return CatBoostClassifier(
+                                   loss_function=get_scoring_string(metric_used, usage='catboost'),
+                                   thread_count = MULTITHREAD,
+                                   used_ram_limit='4gb',
+                random_seed=int(y[:].sum()),
+                                   logging_level='Silent',
+                                    cat_features=cat_features,
+                                      **params)
+        else:
+            return CatBoostRegressor(
+                loss_function=get_scoring_string(metric_used, usage='catboost'),
+                thread_count=MULTITHREAD,
+                used_ram_limit='4gb',
+                random_seed=int(y[:].sum()),
+                logging_level='Silent',
+                cat_features=cat_features,
+                **params)
 
     start_time = time.time()
     def stop(trial):
@@ -348,8 +1359,10 @@ def catboost_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=30
 
     clf = clf_(**best)
     clf.fit(x, y)
-
-    pred = clf.predict_proba(test_x)
+    if is_classification(metric_used):
+        pred = clf.predict_proba(test_x)
+    else:
+        pred = clf.predict(test_x)
     metric = metric_used(test_y, pred)
 
     return metric, pred, best
@@ -371,6 +1384,7 @@ param_grid_hyperopt['xgb'] = {
 }
 
 def xgb_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
+    import xgboost as xgb
     # XGB Documentation:
     # XGB handles categorical data appropriately without using One Hot Encoding, categorical features are experimetal
     # XGB handles missing values appropriately without imputation
@@ -382,11 +1396,18 @@ def xgb_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
                                              , standardize=False)
 
     def clf_(**params):
-        return xgb.XGBClassifier(use_label_encoder=False
-                                 , nthread=1
-                                 , **params
-                                 , eval_metric=get_scoring_string(metric_used, usage='xgb') # AUC not implemented
-        )
+        if is_classification(metric_used):
+            return xgb.XGBClassifier(use_label_encoder=False
+                                     , nthread=1
+                                     , **params
+                                     , eval_metric=get_scoring_string(metric_used, usage='xgb') # AUC not implemented
+            )
+        else:
+            return xgb.XGBRegressor(use_label_encoder=False
+                                     , nthread=1
+                                     , **params
+                                     , eval_metric=get_scoring_string(metric_used, usage='xgb')  # AUC not implemented
+                                     )
 
     start_time = time.time()
     def stop(trial):
@@ -405,17 +1426,97 @@ def xgb_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300):
     clf = clf_(**best)
     clf.fit(x, y)
 
-    pred = clf.predict_proba(test_x)
+    if is_classification(metric_used):
+        pred = clf.predict_proba(test_x)
+    else:
+        pred = clf.predict(test_x)
     metric = metric_used(test_y, pred)
 
     return metric, pred, best
 
+"""
+LEGACY UNUSED
+"""
+
+## Ridge
+from sklearn.linear_model import RidgeClassifier
+param_grid['ridge'] = {'alpha': [0, 0.1, .5, 1.0, 2.0], 'fit_intercept': [True, False]} # 'normalize': [False],
+def ridge_metric(x, y, test_x, test_y, cat_features, metric_used):
+    import warnings
+    def warn(*args, **kwargs):
+        pass
+
+    warnings.warn = warn
+
+    x, y, test_x, test_y = x.cpu(), y.cpu(), test_x.cpu(), test_y.cpu()
+    x, test_x = torch.nan_to_num(x), torch.nan_to_num(test_x)
+
+    clf = RidgeClassifier(n_jobs=1)
+
+    # create a dictionary of all values we want to test for n_neighbors
+    # use gridsearch to test all values for n_neighbors
+    clf = GridSearchCV(clf, param_grid['ridge'], cv=min(CV, x.shape[0]//2)
+                       , scoring=get_scoring_string(metric_used)
+                       , n_jobs=MULTITHREAD)
+    # fit model to data
+    clf.fit(x, y.long())
+
+    pred = clf.decision_function(test_x)
+    metric = metric_used(test_y, pred)
+
+    return metric, pred
+
+def mlp_acc(x, y, test_x, test_y, hyperparameters):
+    num_layers, hidden_dim, activation_module, fixed_dropout_prob, is_binary_classification, epochs, lr, weight_decay = hyperparameters
+    num_features = x.shape[1]
+
+    x, y = x.to(device), y.to(device)
+    test_x, test_y = test_x.to(device), test_y.to(device)
+
+    def get_model():
+        model = nn.Sequential(*[
+            module for layer_idx in range(num_layers) for module in [
+                nn.Linear(hidden_dim if layer_idx > 0 else num_features,
+                          2 if layer_idx == num_layers - 1 else hidden_dim),
+                torch.nn.Identity() if layer_idx == num_layers - 1 else activation_module(),
+                torch.nn.Identity() if layer_idx == num_layers - 1 else torch.nn.Dropout(p=fixed_dropout_prob,
+                                                                                         inplace=False)]
+        ])
+        if is_binary_classification:
+            model.add_module(str(len(model)), torch.nn.Softmax(dim=1))  # TODO might also just do an round!?
+        return model
+
+    model = get_model().to(device)
+    criterion = torch.nn.BCELoss()
+    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)
+
+    model.train()
+    for epoch in range(epochs):
+        optimizer.zero_grad()
+        # Forward pass
+        y_pred = model(x)[:, 1]
+        # Compute Loss
+
+        loss = criterion(y_pred.squeeze(), y.float())
+
+        # print('Epoch {}: train loss: {}'.format(epoch, loss.item()))
+        # Backward pass
+        loss.backward()
+        optimizer.step()
+
+    model.eval()
+    pred_y = model(test_x)[:, 1] > 0.5
+    acc = (pred_y == test_y).float().mean()
+    return acc
 
 clf_dict = {'gp': gp_metric
                 , 'knn': knn_metric
                 , 'catboost': catboost_metric
+                , 'tabnet': tabnet_metric
                 , 'xgb': xgb_metric
+            , 'ridge': ridge_metric
                 , 'logistic': logistic_metric
            , 'autosklearn': autosklearn_metric
              , 'autosklearn2': autosklearn2_metric
-            , 'autogluon': autogluon_metric}
+            , 'autogluon': autogluon_metric,
+            'cocktail': well_tuned_simple_nets_metric}

TabPFN/scripts/tabular_baselines_deep.py

@@ -0,0 +1,74 @@
+import os
+import pathlib
+from argparse import Namespace
+
+from sklearn.model_selection import GridSearchCV
+import sys
+
+CV = 5
+param_grid = {}
+
+param_grid['saint'] = {
+    # as in https://github.com/kathrinse/TabSurvey/blob/main/models/saint.py#L268
+    "dim": [32, 64, 128, 256],
+    "depth": [1, 2, 3, 6, 12],
+    "heads": [2, 4, 8],
+    "dropout": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8],
+}
+
+def saint_metric(x, y, test_x, test_y, cat_features, metric_used):
+    ## Original Implementation https://github.com/somepago/saint
+    ## Reimplementation from https://github.com/kathrinse/TabSurvey
+    ## HowTo install
+    # git clone git@github.com:kathrinse/TabSurvey.git
+    # cd TabSurvey
+    # requirements
+    # optuna
+    # scikit-learn
+    # pandas
+    # configargparse
+    # torch
+    # einops
+    pre_cwd = os.getcwd()
+
+    # TODO: Make sure that we change to TabSurvey in here
+    # Assume it is in ../../TabSurvey
+    dest_wd = pathlib.Path(__file__).absolute().parent.parent.joinpath("../TabSurvey")
+    print(f"Change from {pre_cwd} to {dest_wd}")
+    sys.chdir(dest_wd)
+
+    try:
+        from models.saint import SAINT
+
+        import warnings
+        def warn(*args, **kwargs):
+            pass
+
+        # get cat dims
+        # assume cat_features is a list of idx
+        # TODO: FIX this if wrong
+        cat_dims = []
+        for idx in cat_features:
+            cat_dims.append(len(set(x[idx, :])))
+        model_args = Namespace(
+            num_features=x.shape[1],
+            cat_idx=cat_features,
+            cat_dims=cat_dims,
+        )
+        warnings.warn = warn
+
+        x, y, test_x, test_y = x.cpu(), y.cpu(), test_x.cpu(), test_y.cpu()
+
+        clf = SAINT(model_args)
+
+        clf = GridSearchCV(clf, param_grid['saint'], cv=min(CV, x.shape[0]//2))
+        # fit model to data
+        clf.fit(x, y.long())
+
+        pred = clf.decision_function(test_x)
+        metric = metric_used(test_y.cpu().numpy(), pred)
+    except:
+        raise
+    finally:
+        os.chdir(pre_cwd)
+    return metric, pred

TabPFN/scripts/tabular_evaluation.py

@@ -1,16 +1,17 @@
 import time
 import os
 from pathlib import Path
+from contextlib import nullcontext
 
+import torch
 from tqdm import tqdm
 import random
 import numpy as np
 
 from torch import nn
 
-from utils import torch_nanmean
-from datasets import *
-from model_builder import load_model
+from torch.utils.checkpoint import checkpoint
+from utils import normalize_data, torch_nanmean, to_ranking_low_mem, remove_outliers
 from scripts.tabular_baselines import get_scoring_string
 from scripts import tabular_metrics
 from scripts.transformer_prediction_interface import *
@@ -52,7 +53,6 @@ def eval_model_on_ds(i, e, valid_datasets, eval_positions, bptt, add_name, base_
         model_file, model_path, results_file = check_file(e)
 
     model, config_sample = load_model(base_path, model_file, device, None, verbose=False)
-    print(model[2].style_encoder)
 
     params = {'max_features': config_sample['num_features']
         , 'rescale_features': config_sample["normalize_by_used_features"]
@@ -79,7 +79,7 @@ def eval_model_on_ds(i, e, valid_datasets, eval_positions, bptt, add_name, base_
     return metrics_valid, config_sample, model_path
 
 
-def evaluate(datasets, bptt, eval_positions, metric_used, model
+def evaluate(datasets, bptt, eval_positions, metric_used, model, device='cpu'
              , verbose=False
              , return_tensor=False
              , **kwargs):
@@ -102,10 +102,10 @@ def evaluate(datasets, bptt, eval_positions, metric_used, model
     aggregated_metric_datasets, num_datasets = torch.tensor(0.0), 0
 
     # For each dataset
-    for [ds_name, X, y, categorical_feats, _, _] in tqdm.tqdm(datasets, desc='Iterate over datasets') if verbose else datasets:
+    for [ds_name, X, y, categorical_feats, _, _] in datasets:
         dataset_bptt = min(len(X), bptt)
-        # if verbose and dataset_bptt < bptt:
-        #    print(f'Dataset too small for given sequence length, reducing to {len(X)} ({bptt})')
+        #if verbose and dataset_bptt < bptt:
+        #    print(f'Dataset too small for given bptt, reducing to {len(X)} ({bptt})')
 
         aggregated_metric, num = torch.tensor(0.0), 0
         ds_result = {}
@@ -121,9 +121,11 @@ def evaluate(datasets, bptt, eval_positions, metric_used, model
                         , ds_name=ds_name
                         , eval_position = eval_position_real
                         , metric_used = metric_used
+                                  , device=device
                         ,**kwargs)
 
             if r is None:
+                print('Execution failed')
                 continue
 
             _, outputs, ys, best_configs, time_used = r
@@ -132,6 +134,17 @@ def evaluate(datasets, bptt, eval_positions, metric_used, model
                 outputs = outputs.to(outputs.device)
                 ys = ys.to(outputs.device)
 
+            # WARNING: This leaks information on the scaling of the labels
+            if isinstance(model, nn.Module) and "BarDistribution" in str(type(model.criterion)):
+                ys = (ys - torch.min(ys, axis=0)[0]) / (torch.max(ys, axis=0)[0] - torch.min(ys, axis=0)[0])
+
+            # If we use the bar distribution and the metric_used is r2 -> convert buckets
+            #  metric used is prob -> keep
+            if isinstance(model, nn.Module) and "BarDistribution" in str(type(model.criterion)) and (
+                    metric_used == tabular_metrics.r2_metric or metric_used == tabular_metrics.root_mean_squared_error_metric):
+                ds_result[f'{ds_name}_bar_dist_at_{eval_position}'] = outputs
+                outputs = model.criterion.mean(outputs)
+
             ys = ys.T
             ds_result[f'{ds_name}_best_configs_at_{eval_position}'] = best_configs
             ds_result[f'{ds_name}_outputs_at_{eval_position}'] = outputs
@@ -171,7 +184,7 @@ def check_file_exists(path):
             return np.load(f, allow_pickle=True).tolist()
     return None
 
-def generate_valid_split(X, y, bptt, eval_position, split_number=1):
+def generate_valid_split(X, y, bptt, eval_position, is_classification, split_number=1):
     """Generates a deteministic train-(test/valid) split. Both splits must contain the same classes and all classes in
     the entire datasets. If no such split can be sampled in 7 passes, returns None.
 
@@ -187,7 +200,6 @@ def generate_valid_split(X, y, bptt, eval_position, split_number=1):
     torch.manual_seed(split_number)
     perm = torch.randperm(X.shape[0]) if split_number > 1 else torch.arange(0, X.shape[0])
     X, y = X[perm], y[perm]
-
     while not done:
         if seed > 20:
             return None, None # No split could be generated in 7 passes, return None
@@ -195,13 +207,16 @@ def generate_valid_split(X, y, bptt, eval_position, split_number=1):
         i = random.randint(0, len(X) - bptt) if len(X) - bptt > 0 else 0
         y_ = y[i:i + bptt]
 
-        # Checks if all classes from dataset are contained and classes in train and test are equal (contain same
-        # classes) and
-        done = len(torch.unique(y_)) == len(torch.unique(y))
-        done = done and torch.all(torch.unique(y_) == torch.unique(y))
-        done = done and len(torch.unique(y_[:eval_position])) == len(torch.unique(y_[eval_position:]))
-        done = done and torch.all(torch.unique(y_[:eval_position]) == torch.unique(y_[eval_position:]))
-        seed = seed + 1
+        if is_classification:
+            # Checks if all classes from dataset are contained and classes in train and test are equal (contain same
+            # classes) and
+            done = len(torch.unique(y_)) == len(torch.unique(y))
+            done = done and torch.all(torch.unique(y_) == torch.unique(y))
+            done = done and len(torch.unique(y_[:eval_position])) == len(torch.unique(y_[eval_position:]))
+            done = done and torch.all(torch.unique(y_[:eval_position]) == torch.unique(y_[eval_position:]))
+            seed = seed + 1
+        else:
+            done = True
 
     eval_xs = torch.stack([X[i:i + bptt].clone()], 1)
     eval_ys = torch.stack([y[i:i + bptt].clone()], 1)
@@ -211,7 +226,7 @@ def generate_valid_split(X, y, bptt, eval_position, split_number=1):
 
 def evaluate_position(X, y, categorical_feats, model, bptt
                       , eval_position, overwrite, save, base_path, path_interfix, method, ds_name, fetch_only=False
-                      , max_time=300, split_number=1
+                      , max_time=300, split_number=1, metric_used=None, device='cpu'
                       , per_step_normalization=False, **kwargs):
     """
     Evaluates a dataset with a 'bptt' number of training samples.
@@ -250,24 +265,37 @@ def evaluate_position(X, y, categorical_feats, model, bptt
             return None
 
     ## Generate data splits
-    eval_xs, eval_ys = generate_valid_split(X, y, bptt, eval_position, split_number=split_number)
+    eval_xs, eval_ys = generate_valid_split(X, y, bptt, eval_position
+                                            , is_classification=tabular_metrics.is_classification(metric_used)
+                                            , split_number=split_number)
     if eval_xs is None:
-        return None
         print(f"No dataset could be generated {ds_name} {bptt}")
+        return None
 
     eval_ys = (eval_ys > torch.unique(eval_ys).unsqueeze(0)).sum(axis=1).unsqueeze(-1)
 
+    if isinstance(model, nn.Module):
+        model = model.to(device)
+        eval_xs = eval_xs.to(device)
+        eval_ys = eval_ys.to(device)
+
     start_time = time.time()
 
     if isinstance(model, nn.Module): # Two separate predict interfaces for transformer and baselines
-        outputs, best_configs = transformer_predict(model, eval_xs, eval_ys, eval_position, categorical_feats=categorical_feats, **kwargs), None
+        outputs, best_configs = transformer_predict(model, eval_xs, eval_ys, eval_position, metric_used=metric_used
+                                                    , categorical_feats=categorical_feats
+                                                    , inference_mode=True
+                                                    , device=device
+                                                    , extend_features=True,
+                                                    **kwargs), None
     else:
         _, outputs, best_configs = baseline_predict(model, eval_xs, eval_ys, categorical_feats
                                                     , eval_pos=eval_position
-                                                    , max_time=max_time, **kwargs)
-
+                                                    , device=device
+                                                    , max_time=max_time, metric_used=metric_used, **kwargs)
     eval_ys = eval_ys[eval_position:]
     if outputs is None:
+        print('Execution failed')
         return None
 
     if torch.is_tensor(outputs): # Transfers data to cpu for saving

TabPFN/scripts/tabular_metrics.py

@@ -10,10 +10,25 @@ Includes a few metric as well as functions composing metrics on results files.
 
 import numpy as np
 import torch
-from sklearn.metrics import roc_auc_score, accuracy_score, balanced_accuracy_score, average_precision_score
+from sklearn.metrics import roc_auc_score, accuracy_score, balanced_accuracy_score, average_precision_score, mean_squared_error, mean_absolute_error, r2_score
 from scipy.stats import rankdata
 import pandas as pd
 
+def root_mean_squared_error_metric(target, pred):
+    target = torch.tensor(target) if not torch.is_tensor(target) else target
+    pred = torch.tensor(pred) if not torch.is_tensor(pred) else pred
+    return torch.sqrt(torch.nn.functional.mse_loss(target, pred))
+
+def mean_squared_error_metric(target, pred):
+    target = torch.tensor(target) if not torch.is_tensor(target) else target
+    pred = torch.tensor(pred) if not torch.is_tensor(pred) else pred
+    return torch.nn.functional.mse_loss(target, pred)
+
+def mean_absolute_error_metric(target, pred):
+    target = torch.tensor(target) if not torch.is_tensor(target) else target
+    pred = torch.tensor(pred) if not torch.is_tensor(pred) else pred
+    return torch.tensor(mean_absolute_error(target, pred))
+
 """
 ===============================
 Metrics calculation
@@ -37,7 +52,7 @@ def auc_metric(target, pred, multi_class='ovo', numpy=False):
             return roc_auc_score(target, pred)
     except ValueError as e:
         print(e)
-        return np.nan
+        return np.nan if numpy else torch.tensor(np.nan)
 
 def accuracy_metric(target, pred):
     target = torch.tensor(target) if not torch.is_tensor(target) else target
@@ -73,6 +88,19 @@ def cross_entropy(target, pred):
         bce = torch.nn.BCELoss()
         return bce(pred[:, 1].float(), target.float())
 
+def r2_metric(target, pred):
+    target = torch.tensor(target) if not torch.is_tensor(target) else target
+    pred = torch.tensor(pred) if not torch.is_tensor(pred) else pred
+    return torch.tensor(neg_r2(target, pred))
+
+def neg_r2(target, pred):
+    return -r2_score(pred.float(), target.float())
+
+def is_classification(metric_used):
+    if metric_used == auc_metric or metric_used == cross_entropy:
+        return True
+    return False
+
 def time_metric():
     """
     Dummy function, will just be used as a handler.
@@ -90,7 +118,7 @@ def count_metric(x, y):
 Metrics composition
 ===============================
 """
-def calculate_score_per_method(metric, name:str, global_results:dict, ds:list, eval_positions:list, aggregator:str='mean'):
+def calculate_score_per_method(metric, name:str, global_results:dict, ds:list, eval_positions:list[int], aggregator:str='mean'):
     """
     Calculates the metric given by 'metric' and saves it under 'name' in the 'global_results'
 
@@ -156,15 +184,18 @@ def calculate_score(metric, name, global_results, ds, eval_positions, aggregator
 def make_metric_matrix(global_results, methods, pos, name, ds):
     result = []
     for m in global_results:
-        result += [[global_results[m][d[0] + '_' + name + '_at_' + str(pos)] for d in ds]]
+        try:
+            result += [[global_results[m][d[0] + '_' + name + '_at_' + str(pos)] for d in ds]]
+        except Exception as e:
+            result += [[np.nan]]
     result = np.array(result)
-    result = pd.DataFrame(result.T, index=[d[0] for d in ds], columns=[k[:-8] for k in list(global_results.keys())])
+    result = pd.DataFrame(result.T, index=[d[0] for d in ds], columns=[k for k in list(global_results.keys())])
 
     matrix_means, matrix_stds = [], []
 
     for method in methods:
-        matrix_means += [result.iloc[:, [(method) in c for c in result.columns]].mean(axis=1)]
-        matrix_stds += [result.iloc[:, [(method) in c for c in result.columns]].std(axis=1)]
+        matrix_means += [result.iloc[:, [c.startswith(method+'_time') for c in result.columns]].mean(axis=1)]
+        matrix_stds += [result.iloc[:, [c.startswith(method+'_time') for c in result.columns]].std(axis=1)]
 
     matrix_means = pd.DataFrame(matrix_means, index=methods).T
     matrix_stds = pd.DataFrame(matrix_stds, index=methods).T

TabPFN/scripts/transformer_prediction_interface.py

@@ -94,7 +94,7 @@ class TabPFNClassifier(BaseEstimator, ClassifierMixin):
         i, e = i, -1
 
         # File which contains result of hyperparameter tuning run: style (i.e. hyperparameters) and a dataframe with results.
-        style_file = 'prior_tuning_result.pkl'
+        #style_file = 'prior_tuning_result.pkl'
 
         model, c, results_file = load_model_workflow(i, e, add_name=model_string, base_path=base_path, device=device,
                                                      eval_addition='')

TabPFN/tabular_evaluation.py

@@ -1,283 +0,0 @@
-import time
-import os
-from pathlib import Path
-
-from tqdm import tqdm
-import random
-import numpy as np
-
-from torch import nn
-
-from utils import torch_nanmean
-from datasets import *
-from model_builder import load_model
-from scripts.tabular_baselines import get_scoring_string
-from scripts import tabular_metrics
-from scripts.transformer_prediction_interface import *
-from scripts.baseline_prediction_interface import *
-"""
-===============================
-PUBLIC FUNCTIONS FOR EVALUATION
-===============================
-"""
-
-
-def eval_model(i, e, valid_datasets, test_datasets, eval_positions, bptt, add_name, base_path, device='cpu', eval_addition='', **kwargs):
-    metrics_test, config_sample, model_path = eval_model_on_ds(i, e, test_datasets, eval_positions, bptt, add_name, base_path, device=device, eval_addition=eval_addition, **kwargs)
-    metrics_valid, _, _ = eval_model_on_ds(i, e, valid_datasets, eval_positions, bptt, add_name, base_path, device=device, eval_addition=eval_addition, **kwargs)
-    return {'mean_auc_test': metrics_test['mean_roc_at_1000'], 'mean_auc_valid': metrics_valid['mean_roc_at_1000'], 'mean_ce_test': metrics_test['mean_ce_at_1000'], 'mean_ce_valid': metrics_valid['mean_ce_at_1000'], 'config_sample': config_sample, 'model_path': model_path}
-
-def eval_model_on_ds(i, e, valid_datasets, eval_positions, bptt, add_name, base_path, device='cpu', eval_addition='', **kwargs):
-
-    # How to use: evaluate_without_fitting(i,0,valid_datasets, [1024], 100000, add_name=model_string, base_path=base_path,)
-    def check_file(e):
-        model_file = f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{e}.cpkt'
-        model_path = os.path.join(base_path, model_file)
-        # print('Evaluate ', model_path)
-        results_file = os.path.join(base_path,
-                                    f'models_diff/prior_diff_real_results{add_name}_n_{i}_epoch_{e}_{eval_addition}.pkl')
-        if not Path(model_path).is_file():  # or Path(results_file).is_file():
-            # print('checkpoint exists: ', Path(model_file).is_file(), ', results are written:', Path(results_file).is_file())
-            return None, None, None
-        return model_file, model_path, results_file
-
-    if e == -1: # use last checkpoint, if e == -1
-        for e_ in range(100, -1, -1):
-            model_file_, model_path_, results_file_ = check_file(e_)
-            if model_file_ is not None:
-                e = e_
-                model_file, model_path, results_file = model_file_, model_path_, results_file_
-                break
-    else:
-        model_file, model_path, results_file = check_file(e)
-
-    model, config_sample = load_model(base_path, model_file, device, None, verbose=False)
-
-    params = {'max_features': config_sample['num_features']
-        , 'rescale_features': config_sample["normalize_by_used_features"]
-        , 'normalize_to_ranking': config_sample["normalize_to_ranking"]
-        , 'normalize_with_sqrt': config_sample.get("normalize_with_sqrt", False)
-              }
-    metrics_valid = evaluate(datasets=valid_datasets, model=model[2], method='transformer', device=device, overwrite=True,
-                             extend_features=True
-                             # just removed the style keyword but transformer is trained with style, just empty
-                             , save=False
-                             , metric_used=tabular_metrics.cross_entropy
-                             , return_tensor=True
-                             , verbose=False
-                             , eval_positions=eval_positions
-                             , bptt=bptt
-                             , base_path=None
-                             , inference_mode=True
-                             , **params
-                             , **kwargs)
-
-    tabular_metrics.calculate_score_per_method(tabular_metrics.auc_metric, 'roc', metrics_valid, valid_datasets, eval_positions)
-    tabular_metrics.calculate_score_per_method(tabular_metrics.cross_entropy, 'ce', metrics_valid, valid_datasets, eval_positions)
-
-    return metrics_valid, config_sample, model_path
-
-
-def evaluate(datasets, bptt, eval_positions, metric_used, model
-             , verbose=False
-             , return_tensor=False
-             , **kwargs):
-    """
-    Evaluates a list of datasets for a model function.
-
-    :param datasets: List of datasets
-    :param bptt: maximum sequence length
-    :param eval_positions: List of positions where to evaluate models
-    :param verbose: If True, is verbose.
-    :param metric_used: Which metric is optimized for.
-    :param return_tensor: Wheater to return results as a pytorch.tensor or numpy, this is only relevant for transformer.
-    :param kwargs:
-    :return:
-    """
-    overall_result = {'metric_used': get_scoring_string(metric_used)
-                      , 'bptt': bptt
-                      , 'eval_positions': eval_positions}
-
-    aggregated_metric_datasets, num_datasets = torch.tensor(0.0), 0
-
-    # For each dataset
-    for [ds_name, X, y, categorical_feats, _, _] in tqdm.tqdm(datasets, desc='Iterate over datasets') if verbose else datasets:
-        dataset_bptt = min(len(X), bptt)
-        # if verbose and dataset_bptt < bptt:
-        #    print(f'Dataset too small for given sequence length, reducing to {len(X)} ({bptt})')
-
-        aggregated_metric, num = torch.tensor(0.0), 0
-        ds_result = {}
-
-        for eval_position in (eval_positions if verbose else eval_positions):
-            eval_position_real = int(dataset_bptt * 0.5) if 2 * eval_position > dataset_bptt else eval_position
-            eval_position_bptt = int(eval_position_real * 2.0)
-
-            r = evaluate_position(X, y, model=model
-                        , num_classes=len(torch.unique(y))
-                        , categorical_feats = categorical_feats
-                        , bptt = eval_position_bptt
-                        , ds_name=ds_name
-                        , eval_position = eval_position_real
-                        , metric_used = metric_used
-                        ,**kwargs)
-
-            if r is None:
-                continue
-
-            _, outputs, ys, best_configs, time_used = r
-
-            if torch.is_tensor(outputs):
-                outputs = outputs.to(outputs.device)
-                ys = ys.to(outputs.device)
-
-            ys = ys.T
-            ds_result[f'{ds_name}_best_configs_at_{eval_position}'] = best_configs
-            ds_result[f'{ds_name}_outputs_at_{eval_position}'] = outputs
-            ds_result[f'{ds_name}_ys_at_{eval_position}'] = ys
-            ds_result[f'{ds_name}_time_at_{eval_position}'] = time_used
-
-            new_metric = torch_nanmean(torch.stack([metric_used(ys[i], outputs[i]) for i in range(ys.shape[0])]))
-
-            if not return_tensor:
-                make_scalar = lambda x: float(x.detach().cpu().numpy()) if (torch.is_tensor(x) and (len(x.shape) == 0)) else x
-                new_metric = make_scalar(new_metric)
-                ds_result = {k: make_scalar(ds_result[k]) for k in ds_result.keys()}
-
-            lib = torch if return_tensor else np
-            if not lib.isnan(new_metric).any():
-                aggregated_metric, num = aggregated_metric + new_metric, num + 1
-
-        overall_result.update(ds_result)
-        if num > 0:
-            aggregated_metric_datasets, num_datasets = (aggregated_metric_datasets + (aggregated_metric / num)), num_datasets + 1
-
-    overall_result['mean_metric'] = aggregated_metric_datasets / num_datasets
-
-    return overall_result
-
-"""
-===============================
-INTERNAL HELPER FUNCTIONS
-===============================
-"""
-
-def check_file_exists(path):
-    """Checks if a pickle file exists. Returns None if not, else returns the unpickled file."""
-    if (os.path.isfile(path)):
-        print(f'loading results from {path}')
-        with open(path, 'rb') as f:
-            return np.load(f, allow_pickle=True).tolist()
-    return None
-
-def generate_valid_split(X, y, bptt, eval_position, split_number=1):
-    """Generates a deteministic train-(test/valid) split. Both splits must contain the same classes and all classes in
-    the entire datasets. If no such split can be sampled in 7 passes, returns None.
-
-    :param X: torch tensor, feature values
-    :param y: torch tensor, class values
-    :param bptt: Number of samples in train + test
-    :param eval_position: Number of samples in train, i.e. from which index values are in test
-    :param split_number: The split id
-    :return:
-    """
-    done, seed = False, 13
-
-    torch.manual_seed(split_number)
-    perm = torch.randperm(X.shape[0]) if split_number > 1 else torch.arange(0, X.shape[0])
-    X, y = X[perm], y[perm]
-
-    while not done:
-        if seed > 20:
-            return None, None # No split could be generated in 7 passes, return None
-        random.seed(seed)
-        i = random.randint(0, len(X) - bptt) if len(X) - bptt > 0 else 0
-        y_ = y[i:i + bptt]
-
-        # Checks if all classes from dataset are contained and classes in train and test are equal (contain same
-        # classes) and
-        done = len(torch.unique(y_)) == len(torch.unique(y))
-        done = done and torch.all(torch.unique(y_) == torch.unique(y))
-        done = done and len(torch.unique(y_[:eval_position])) == len(torch.unique(y_[eval_position:]))
-        done = done and torch.all(torch.unique(y_[:eval_position]) == torch.unique(y_[eval_position:]))
-        seed = seed + 1
-
-    eval_xs = torch.stack([X[i:i + bptt].clone()], 1)
-    eval_ys = torch.stack([y[i:i + bptt].clone()], 1)
-
-    return eval_xs, eval_ys
-
-
-def evaluate_position(X, y, categorical_feats, model, bptt
-                      , eval_position, overwrite, save, base_path, path_interfix, method, ds_name, fetch_only=False
-                      , max_time=300, split_number=1
-                      , per_step_normalization=False, **kwargs):
-    """
-    Evaluates a dataset with a 'bptt' number of training samples.
-
-    :param X: Dataset X
-    :param y: Dataset labels
-    :param categorical_feats: Indices of categorical features.
-    :param model: Model function
-    :param bptt: Sequence length.
-    :param eval_position: Number of training samples.
-    :param overwrite: Wheater to ove
-    :param overwrite: If True, results on disk are overwritten.
-    :param save:
-    :param path_interfix: Used for constructing path to write on disk.
-    :param method: Model name.
-    :param ds_name: Datset name.
-    :param fetch_only: Wheater to calculate or only fetch results.
-    :param per_step_normalization:
-    :param kwargs:
-    :return:
-    """
-
-    if save:
-        path = os.path.join(base_path, f'results/tabular/{path_interfix}/results_{method}_{ds_name}_{eval_position}_{bptt}_{split_number}.npy')
-        #log_path =
-
-    ## Load results if on disk
-    if not overwrite:
-        result = check_file_exists(path)
-        if result is not None:
-            if not fetch_only:
-                print(f'Loaded saved result for {path}')
-            return result
-        elif fetch_only:
-            print(f'Could not load saved result for {path}')
-            return None
-
-    ## Generate data splits
-    eval_xs, eval_ys = generate_valid_split(X, y, bptt, eval_position, split_number=split_number)
-    if eval_xs is None:
-        return None
-        print(f"No dataset could be generated {ds_name} {bptt}")
-
-    eval_ys = (eval_ys > torch.unique(eval_ys).unsqueeze(0)).sum(axis=1).unsqueeze(-1)
-
-    start_time = time.time()
-
-    if isinstance(model, nn.Module): # Two separate predict interfaces for transformer and baselines
-        outputs, best_configs = transformer_predict(model, eval_xs, eval_ys, eval_position, categorical_feats=categorical_feats, **kwargs), None
-    else:
-        _, outputs, best_configs = baseline_predict(model, eval_xs, eval_ys, categorical_feats
-                                                    , eval_pos=eval_position
-                                                    , max_time=max_time, **kwargs)
-
-    eval_ys = eval_ys[eval_position:]
-    if outputs is None:
-        return None
-
-    if torch.is_tensor(outputs): # Transfers data to cpu for saving
-        outputs = outputs.cpu()
-        eval_ys = eval_ys.cpu()
-
-    ds_result = None, outputs, eval_ys, best_configs, time.time() - start_time
-
-    if save:
-        with open(path, 'wb') as f:
-            np.save(f, ds_result)
-            print(f'saved results to {path}')
-
-    return ds_result

encoders.py

@@ -1,243 +0,0 @@
-import math
-
-import torch
-import torch.nn as nn
-from utils import normalize_data
-import torch.nn.functional as F
-from torch.nn import TransformerEncoder, TransformerEncoderLayer
-
-
-class StyleEncoder(nn.Module):
-    def __init__(self, num_hyperparameters, em_size):
-        super().__init__()
-        self.em_size = em_size
-        self.embedding = nn.Linear(num_hyperparameters, self.em_size)
-
-    def forward(self, hyperparameters):  # B x num_hps
-        return self.embedding(hyperparameters)
-
-
-class StyleEmbEncoder(nn.Module):
-    def __init__(self, num_hyperparameters, em_size, num_embeddings=100):
-        super().__init__()
-        assert num_hyperparameters == 1
-        self.em_size = em_size
-        self.embedding = nn.Embedding(num_embeddings, self.em_size)
-
-    def forward(self, hyperparameters):  # B x num_hps
-        return self.embedding(hyperparameters.squeeze(1))
-
-
-class _PositionalEncoding(nn.Module):
-    def __init__(self, d_model, dropout=0.):
-        super().__init__()
-        self.dropout = nn.Dropout(p=dropout)
-        self.d_model = d_model
-        self.device_test_tensor = nn.Parameter(torch.tensor(1.))
-
-    def forward(self, x):# T x B x num_features
-        assert self.d_model % x.shape[-1]*2 == 0
-        d_per_feature = self.d_model // x.shape[-1]
-        pe = torch.zeros(*x.shape, d_per_feature, device=self.device_test_tensor.device)
-        #position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
-        interval_size = 10
-        div_term = (1./interval_size) * 2*math.pi*torch.exp(torch.arange(0, d_per_feature, 2, device=self.device_test_tensor.device).float()*math.log(math.sqrt(2)))
-        #print(div_term/2/math.pi)
-        pe[..., 0::2] = torch.sin(x.unsqueeze(-1) * div_term)
-        pe[..., 1::2] = torch.cos(x.unsqueeze(-1) * div_term)
-        return self.dropout(pe).view(x.shape[0],x.shape[1],self.d_model)
-
-
-Positional = lambda _, emsize: _PositionalEncoding(d_model=emsize)
-
-class EmbeddingEncoder(nn.Module):
-    def __init__(self, num_features, em_size, num_embs=100):
-        super().__init__()
-        self.num_embs = num_embs
-        self.embeddings = nn.Embedding(num_embs * num_features, em_size, max_norm=True)
-        self.init_weights(.1)
-        self.min_max = (-2,+2)
-
-    @property
-    def width(self):
-        return self.min_max[1] - self.min_max[0]
-
-    def init_weights(self, initrange):
-        self.embeddings.weight.data.uniform_(-initrange, initrange)
-
-    def discretize(self, x):
-        split_size = self.width / self.num_embs
-        return (x - self.min_max[0] // split_size).int().clamp(0, self.num_embs - 1)
-
-    def forward(self, x):  # T x B x num_features
-        x_idxs = self.discretize(x)
-        x_idxs += torch.arange(x.shape[-1], device=x.device).view(1, 1, -1) * self.num_embs
-        # print(x_idxs,self.embeddings.weight.shape)
-        return self.embeddings(x_idxs).mean(-2)
-
-
-class Normalize(nn.Module):
-    def __init__(self, mean, std):
-        super().__init__()
-        self.mean = mean
-        self.std = std
-
-    def forward(self, x):
-        return (x-self.mean)/self.std
-
-
-def get_normalized_uniform_encoder(encoder_creator):
-    """
-    This can be used to wrap an encoder that is fed uniform samples in [0,1] and normalizes these to 0 mean and 1 std.
-    For example, it can be used as `encoder_creator = get_normalized_uniform_encoder(encoders.Linear)`, now this can
-    be initialized with `encoder_creator(feature_dim, in_dim)`.
-    :param encoder:
-    :return:
-    """
-    return lambda in_dim, out_dim: nn.Sequential(Normalize(.5, math.sqrt(1/12)), encoder_creator(in_dim, out_dim))
-
-
-def get_normalized_encoder(encoder_creator, data_std):
-    return lambda in_dim, out_dim: nn.Sequential(Normalize(0., data_std), encoder_creator(in_dim, out_dim))
-
-
-class ZNormalize(nn.Module):
-    def forward(self, x):
-        return (x-x.mean(-1,keepdim=True))/x.std(-1,keepdim=True)
-
-
-class AppendEmbeddingEncoder(nn.Module):
-    def __init__(self, base_encoder, num_features, emsize):
-        super().__init__()
-        self.num_features = num_features
-        self.base_encoder = base_encoder
-        self.emb = nn.Parameter(torch.zeros(emsize))
-
-    def forward(self, x):
-        if (x[-1] == 1.).all():
-            append_embedding = True
-        else:
-            assert (x[-1] == 0.).all(), "You need to specify as last position whether to append embedding. " \
-                                        "If you don't want this behavior, please use the wrapped encoder instead."
-            append_embedding = False
-        x = x[:-1]
-        encoded_x = self.base_encoder(x)
-        if append_embedding:
-            encoded_x = torch.cat([encoded_x, self.emb[None, None, :].repeat(1, encoded_x.shape[1], 1)], 0)
-        return encoded_x
-
-def get_append_embedding_encoder(encoder_creator):
-    return lambda num_features, emsize: AppendEmbeddingEncoder(encoder_creator(num_features, emsize), num_features, emsize)
-
-
-class VariableNumFeaturesEncoder(nn.Module):
-    def __init__(self, base_encoder, num_features):
-        super().__init__()
-        self.base_encoder = base_encoder
-        self.num_features = num_features
-
-    def forward(self, x):
-        x = x * (self.num_features/x.shape[-1])
-        x = torch.cat((x, torch.zeros(*x.shape[:-1], self.num_features - x.shape[-1], device=x.device)), -1)
-        return self.base_encoder(x)
-
-
-def get_variable_num_features_encoder(encoder_creator):
-    return lambda num_features, emsize: VariableNumFeaturesEncoder(encoder_creator(num_features, emsize), num_features)
-
-class NoMeanEncoder(nn.Module):
-    """
-    This can be useful for any prior that is translation invariant in x or y.
-    A standard GP for example is translation invariant in x.
-    That is, GP(x_test+const,x_train+const,y_train) = GP(x_test,x_train,y_train).
-    """
-    def __init__(self, base_encoder):
-        super().__init__()
-        self.base_encoder = base_encoder
-
-    def forward(self, x):
-        return self.base_encoder(x - x.mean(0, keepdim=True))
-
-
-def get_no_mean_encoder(encoder_creator):
-    return lambda num_features, emsize: NoMeanEncoder(encoder_creator(num_features, emsize))
-
-Linear = nn.Linear
-MLP = lambda num_features, emsize: nn.Sequential(nn.Linear(num_features+1,emsize*2),
-                                                 nn.ReLU(),
-                                                 nn.Linear(emsize*2,emsize))
-
-class NanHandlingEncoder(nn.Module):
-    def __init__(self, num_features, emsize, keep_nans=True):
-        super().__init__()
-        self.num_features = 2 * num_features if keep_nans else num_features
-        self.emsize = emsize
-        self.keep_nans = keep_nans
-        self.layer = nn.Linear(self.num_features, self.emsize)
-
-    def forward(self, x):
-        if self.keep_nans:
-            x = torch.cat([torch.nan_to_num(x, nan=0.0), normalize_data(torch.isnan(x) * -1
-                                                          + torch.logical_and(torch.isinf(x), torch.sign(x) == 1) * 1
-                                                          + torch.logical_and(torch.isinf(x), torch.sign(x) == -1) * 2
-                                                          )], -1)
-        else:
-            x = torch.nan_to_num(x, nan=0.0)
-        return self.layer(x)
-
-
-class Linear(nn.Linear):
-    def __init__(self, num_features, emsize, replace_nan_by_zero=False):
-        super().__init__(num_features, emsize)
-        self.num_features = num_features
-        self.emsize = emsize
-        self.replace_nan_by_zero = replace_nan_by_zero
-
-    def forward(self, x):
-        if self.replace_nan_by_zero:
-            x = torch.nan_to_num(x, nan=0.0)
-        return super().forward(x)
-
-    def __setstate__(self, state):
-        super().__setstate__(state)
-        self.__dict__.setdefault('replace_nan_by_zero', True)
-
-
-class Conv(nn.Module):
-    def __init__(self, input_size, emsize):
-        super().__init__()
-        self.convs = torch.nn.ModuleList([nn.Conv2d(64 if i else 1, 64, 3) for i in range(5)])
-        self.linear = nn.Linear(64,emsize)
-
-    def forward(self, x):
-        size = math.isqrt(x.shape[-1])
-        assert size*size == x.shape[-1]
-        x = x.reshape(*x.shape[:-1], 1, size, size)
-        for conv in self.convs:
-            if x.shape[-1] < 4:
-                break
-            x = conv(x)
-            x.relu_()
-        x = nn.AdaptiveAvgPool2d((1,1))(x).squeeze(-1).squeeze(-1)
-        return self.linear(x)
-
-
-class CanEmb(nn.Embedding):
-    def __init__(self, num_features, num_embeddings: int, embedding_dim: int, *args, **kwargs):
-        assert embedding_dim % num_features == 0
-        embedding_dim = embedding_dim // num_features
-        super().__init__(num_embeddings, embedding_dim, *args, **kwargs)
-
-    def forward(self, x):
-        lx = x.long()
-        assert (lx == x).all(), "CanEmb only works with tensors of whole numbers"
-        x = super().forward(lx)
-        return x.view(*x.shape[:-2], -1)
-
-
-def get_Canonical(num_classes):
-    return lambda num_features, emsize: CanEmb(num_features, num_classes, emsize)
-
-
-def get_Embedding(num_embs_per_feature=100):
-    return lambda num_features, emsize: EmbeddingEncoder(num_features, emsize, num_embs=num_embs_per_feature)