diff --git a/.gitattributes b/.gitattributes index bf07816c74bac9b682df196e02c6482e474e9b52..540bbfe48ac01b6173fcb8fd4f59612dba4fd436 100644 --- a/.gitattributes +++ b/.gitattributes @@ -29,3 +29,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zstandard filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.cpkt filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 199aaa31d4aca4850b731f68c1d45defe71661a3..adf698400dddc6338a331dfea36c5b5719ff93f4 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ --- -title: TabPFNPrediction -emoji: 📚 -colorFrom: red -colorTo: green +title: TabPFN +emoji: 🐨 +colorFrom: gray +colorTo: blue sdk: gradio -sdk_version: 3.1.2 +sdk_version: 3.1.1 app_file: app.py -pinned: false -license: apache-2.0 +pinned: true --- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference diff --git a/TabPFN/PrepareDatasets.ipynb b/TabPFN/PrepareDatasets.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..ca848e932f708ebfb794e3a046650665f428ab24 --- /dev/null +++ b/TabPFN/PrepareDatasets.ipynb @@ -0,0 +1,373 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "import openml\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import tqdm\n", + "\n", + "from datasets import load_openml_list, test_dids_classification, valid_large_classification, open_cc_dids, open_cc_valid_dids\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], + "source": [ + "%load_ext autoreload\n", + "\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Prepare test datasets" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "renamer = {'name': 'Name', 'NumberOfFeatures': '# Features', 'NumberOfSymbolicFeatures': '# Categorical Features', 'NumberOfInstances': '# Instances', 'NumberOfMissingValues': '# NaNs', 'NumberOfClasses': '# Classes', 'MinorityClassSize': 'Minority Class Size'}\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "OrderedDict([(99,\n", + " {'id': 99,\n", + " 'alias': 'OpenML-CC18',\n", + " 'main_entity_type': 'task',\n", + " 'name': 'OpenML-CC18 Curated Classification benchmark',\n", + " 'status': 'active',\n", + " 'creation_date': '2019-02-21 18:47:13',\n", + " 'creator': 1}),\n", + " (225,\n", + " {'id': 225,\n", + " 'alias': 'OpenML-friendly',\n", + " 'main_entity_type': 'task',\n", + " 'name': 'OpenML100-friendly',\n", + " 'status': 'active',\n", + " 'creation_date': '2019-09-16 19:41:46',\n", + " 'creator': 1})])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "openml.study.list_suites()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "suite = openml.study.get_suite(suite_id=99)\n", + "tasks = openml.tasks.list_tasks(output_format=\"dataframe\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# Using ``@`` in `pd.DataFrame.query <\n", + "# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_\n", + "# accesses variables outside of the current dataframe.\n", + "tasks = tasks.query(\"tid in @suite.tasks\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "tids = list(tasks[np.logical_and(np.logical_and((tasks.NumberOfInstances <= 2000), (tasks.NumberOfFeatures <= 100))\n", + " , (tasks.NumberOfClasses <= 10))].tid)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "30" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(tids)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "tids = list(tasks[tasks.NumberOfInstances <= 2000].tid)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "open_cc_dids = [openml.tasks.get_task(task_id).get_dataset().id for task_id in tids]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "open_ml_datasets, open_ml_datasets_df = load_openml_list(test_dids_classification, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 100000, num_feats=100, return_capped=True)\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "open_ml_datasets_df = open_ml_datasets_df[open_ml_datasets_df.NumberOfInstances > 10000]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\\begin{tabular}{lrrrrrrr}\n", + "\\toprule\n", + " Name & \\# Features & \\# Categorical Features & \\# Instances & \\# Classes & \\# NaNs & Minority Class Size & id \\\\\n", + "\\midrule\n", + " KDDCup09\\_appetency & 231 & 39 & 50000 & 2 & 8024152 & 890 & 1111 \\\\\n", + " airlines & 8 & 5 & 539383 & 2 & 0 & 240264 & 1169 \\\\\n", + " bank-marketing & 17 & 10 & 45211 & 2 & 0 & 5289 & 1461 \\\\\n", + " nomao & 119 & 30 & 34465 & 2 & 0 & 9844 & 1486 \\\\\n", + " adult & 15 & 9 & 48842 & 2 & 6465 & 11687 & 1590 \\\\\n", + " covertype & 55 & 45 & 581012 & 7 & 0 & 2747 & 1596 \\\\\n", + " numerai28.6 & 22 & 1 & 96320 & 2 & 0 & 47662 & 23517 \\\\\n", + " connect-4 & 43 & 43 & 67557 & 3 & 0 & 6449 & 40668 \\\\\n", + "jungle\\_chess\\_2pcs\\_raw\\_endgame\\_complete & 7 & 1 & 44819 & 3 & 0 & 4335 & 41027 \\\\\n", + " APSFailure & 171 & 1 & 76000 & 2 & 1078695 & 1375 & 41138 \\\\\n", + " albert & 79 & 53 & 425240 & 2 & 2734000 & 212620 & 41147 \\\\\n", + " MiniBooNE & 51 & 1 & 130064 & 2 & 0 & 36499 & 41150 \\\\\n", + " guillermo & 4297 & 1 & 20000 & 2 & 0 & 8003 & 41159 \\\\\n", + " riccardo & 4297 & 1 & 20000 & 2 & 0 & 5000 & 41161 \\\\\n", + " volkert & 181 & 1 & 58310 & 10 & 0 & 1361 & 41166 \\\\\n", + " dionis & 61 & 1 & 416188 & 355 & 0 & 878 & 41167 \\\\\n", + " jannis & 55 & 1 & 83733 & 4 & 0 & 1687 & 41168 \\\\\n", + " helena & 28 & 1 & 65196 & 100 & 0 & 111 & 41169 \\\\\n", + "\\bottomrule\n", + "\\end{tabular}\n", + "\n" + ] + } + ], + "source": [ + "print_table = open_ml_datasets_df\n", + "print_table = print_table[['name', 'NumberOfFeatures', 'NumberOfSymbolicFeatures', 'NumberOfInstances', 'NumberOfClasses', 'NumberOfMissingValues', 'MinorityClassSize']].copy()\n", + "print_table['id'] = print_table.index\n", + "print_table[['NumberOfFeatures', 'NumberOfSymbolicFeatures', 'NumberOfInstances', 'NumberOfClasses', 'NumberOfMissingValues', 'MinorityClassSize']] = print_table[['NumberOfFeatures', 'NumberOfSymbolicFeatures', 'NumberOfInstances', 'NumberOfClasses', 'NumberOfMissingValues', 'MinorityClassSize']].astype(int)\n", + "print_table = print_table.rename(columns=renamer)\n", + "print(print_table.to_latex(index=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Prepare Validation datasets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "open_cc_datasets, open_cc_datasets_df = load_openml_list(open_cc_dids, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 2000, num_feats=100, return_capped=True)\n", + "\n", + "def extend_datasets(datasets, filtering = False):\n", + " extended_datasets = {}\n", + " i = 0\n", + " for d in tqdm(datasets):\n", + " if ((not 'NumberOfFeatures' in datasets[d])\n", + " or (not 'NumberOfClasses' in datasets[d])\n", + " or (not 'NumberOfInstances' in datasets[d])\n", + " # or datasets[d]['NumberOfFeatures'] >= num_feats\n", + " or datasets[d]['NumberOfClasses'] <= 0):\n", + " print(datasets[d])\n", + " continue\n", + " ds = openml.datasets.get_dataset(d, download_data=False)\n", + " if filtering and (datasets[d]['NumberOfInstances'] < 150\n", + " or datasets[d]['NumberOfInstances'] > 2000\n", + " or datasets[d]['NumberOfFeatures'] > 100\n", + " or datasets[d]['NumberOfClasses'] > 10):\n", + " continue\n", + " extended_datasets[d] = datasets[d]\n", + " extended_datasets[d].update(ds.qualities)\n", + " \n", + " return extended_datasets\n", + "\n", + "# All datasets\n", + "openml_list = openml.datasets.list_datasets()\n", + "openml_list = pd.DataFrame.from_dict(openml_list, orient=\"index\")\n", + "\n", + "# Select only classification\n", + "openml_list = openml_list[~openml_list['MajorityClassSize'].isna()]\n", + "\n", + "# Remove duplicated datasets\n", + "duplicated = openml_list.duplicated(subset=['MajorityClassSize', 'MaxNominalAttDistinctValues', 'MinorityClassSize',\n", + " 'NumberOfClasses', 'NumberOfFeatures', 'NumberOfInstances',\n", + " 'NumberOfInstancesWithMissingValues', 'NumberOfMissingValues',\n", + " 'NumberOfNumericFeatures', 'NumberOfSymbolicFeatures'], keep='first')\n", + "openml_list = openml_list[~duplicated]\n", + "\n", + "duplicated = openml_list.duplicated(subset=['name'], keep='first')\n", + "openml_list = openml_list[~duplicated]\n", + "\n", + "# Filter out datasets that don't have meta information or Don't fulfill other criteria\n", + "openml_list = openml_list.to_dict(orient='index')\n", + "openml_list = pd.DataFrame.from_dict(extend_datasets(openml_list, filtering=True), orient=\"index\")\n", + "\n", + "# Filter out datasets in Open CC\n", + "openml_list = openml_list[~openml_list.name.apply(lambda x: x in test_datasets_multiclass_df.name.values)]\n", + "openml_list['CFI'] = openml_list.apply(lambda x: str(x.NumberOfClasses) + '_' + str(x.NumberOfFeatures) + '_' + str(x.NumberOfInstances), axis = 1)\n", + "test_datasets_multiclass_df['CFI'] = test_datasets_multiclass_df.apply(lambda x: str(x.NumberOfClasses) + '_' + str(x.NumberOfFeatures) + '_' + str(x.NumberOfInstances), axis = 1)\n", + "openml_list = openml_list[~openml_list.CFI.apply(lambda x: x in test_datasets_multiclass_df.CFI.values)]\n", + "\n", + "# Remove time series and artificial data\n", + "openml_list = openml_list[~openml_list.name.apply(lambda x: 'autoUniv' in x)]\n", + "openml_list = openml_list[~openml_list.name.apply(lambda x: 'fri_' in x)]\n", + "openml_list = openml_list[~openml_list.name.apply(lambda x: 'FOREX' in x)]\n", + "\n", + "# Remove datasets that overlapped with Open CC closely by name\n", + "openml_list = openml_list[~openml_list.name.apply(lambda x: 'ilpd' in x)]\n", + "openml_list = openml_list[~openml_list.name.apply(lambda x: 'car' in x)]\n", + "openml_list = openml_list[~openml_list.name.apply(lambda x: 'pc1' in x)]\n", + "\n", + "# Remove datasets that didn't load\n", + "openml_list = openml_list[~openml_list.did.apply(lambda x: x in {1065, 40589, 41496, 770, 43097, 43148, 43255, 43595, 43786, 41701})]\n", + "\n", + "# Remove class skew\n", + "openml_list = openml_list[(openml_list.MinorityClassSize / openml_list.MajorityClassSize) > 0.05]\n", + "openml_list = openml_list[openml_list.AutoCorrelation != 1]\n", + "\n", + "# Remove too easy\n", + "openml_list = openml_list[openml_list.CfsSubsetEval_DecisionStumpAUC != 1]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_table = openml_list\n", + "print_table = print_table[['name', 'NumberOfFeatures', 'NumberOfSymbolicFeatures', 'NumberOfInstances', 'NumberOfClasses', 'NumberOfMissingValues', 'MinorityClassSize']].copy()\n", + "print_table['id'] = print_table.index\n", + "print_table[['NumberOfFeatures', 'NumberOfSymbolicFeatures', 'NumberOfInstances', 'NumberOfClasses', 'NumberOfMissingValues', 'MinorityClassSize']] = print_table[['NumberOfFeatures', 'NumberOfSymbolicFeatures', 'NumberOfInstances', 'NumberOfClasses', 'NumberOfMissingValues', 'MinorityClassSize']].astype(int)\n", + "print_table = print_table.rename(columns=renamer)\n", + "print(print_table.to_latex(index=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/TabPFN/README.md b/TabPFN/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3607380865bc971af3ea79fb135d84287e68f8a6 --- /dev/null +++ b/TabPFN/README.md @@ -0,0 +1,23 @@ +# TabPFN + +## Installation +``` +git clone git@github.com:automl/TabPFN.git +cd TabPFN +conda create -n TabPFN python=3.7 +conda activate TabPFN +pip install -r requirements.txt +``` + +To run the autogluon baseline please create a separate environment and install autogluon==0.4.0, installation in the same environment as our other baselines is not possible. + +## Usage +TrainingTuningAndPrediction: Train a TabPFN, Prior Tune and predict using a pretrained model. + +TabularEvaluationVisualization: Run Baselines and load Baseline and TabPFN Results for comparison and plotting. + +PrepareDatasets: Notebook used to inspect Datasets (Not needed to run baselines / TabPFN). + +SytheticGPAblation: Ablation experiments for Gaussian Process fitting with differentiable Hyper Parameters. + + diff --git a/TabPFN/SyntheticGPAblation.ipynb b/TabPFN/SyntheticGPAblation.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..a3ab141f461e40a246cb293f39d13319f55ae106 --- /dev/null +++ b/TabPFN/SyntheticGPAblation.ipynb @@ -0,0 +1,392 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import time\n", + "\n", + "import torch\n", + "\n", + "import numpy as np\n", + "\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from model_builder import get_model, get_default_spec, save_model, load_model\n", + "\n", + "from scripts.model_configs import *" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Setting params" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "device = 'cuda'\n", + "base_path = os.path.join('.')" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "def train_function(config_sample, i, add_name=''):\n", + " start_time = time.time()\n", + " N_epochs_to_save = 50\n", + " \n", + " def save_callback(model, epoch):\n", + " if not hasattr(model, 'last_saved_epoch'):\n", + " model.last_saved_epoch = 0\n", + " if ((time.time() - start_time) / (maximum_runtime * 60 / N_epochs_to_save)) > model.last_saved_epoch:\n", + " print('Saving model..')\n", + " config_sample['epoch_in_training'] = epoch\n", + " save_model(model, base_path, f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{model.last_saved_epoch}.cpkt',\n", + " config_sample)\n", + " model.last_saved_epoch = model.last_saved_epoch + 1 # TODO: Rename to checkpoint\n", + " \n", + " model = get_model(config_sample\n", + " , device\n", + " , should_train=True\n", + " , verbose=1\n", + " , epoch_callback = save_callback)\n", + " \n", + " return" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "heading_collapsed": true, + "tags": [] + }, + "source": [ + "# Check synthetic data fitting" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Workflow functions" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "hidden": true, + "tags": [] + }, + "outputs": [], + "source": [ + "def generate_test_data(test_gp_params):\n", + " # Generate test data\n", + " config = {**test_gp_params}\n", + "\n", + " config['verbose'] = False\n", + " config['differentiable'] = False\n", + " #config['bptt'] = config['bptt_in_training']\n", + "\n", + " model_test_data = get_model(config, device, should_train=False, verbose=True)\n", + " (hp_embedding, data, targets_), targets = next(iter(model_test_data[3]))\n", + " (hp_embedding, data, targets_), targets = (hp_embedding, data.to(device), targets_.to(device)), targets.to(device)\n", + " \n", + " return (hp_embedding, data, targets_), targets\n", + "\n", + "def evaluate_hp_range(model, hparam_true, vary_hparam_ind, data, targets, eval_pos, plot_step_size):\n", + " losses, hparams = [], []\n", + " for l in np.arange(-1.74, 1.74, plot_step_size):\n", + " hparam = [*hparam_true]\n", + " hparam[vary_hparam_ind] = l\n", + " hp_embedding_used = torch.tensor(hparam).to(device).float()\n", + " with torch.inference_mode():\n", + " outputs = torch.sigmoid(model[2]((hp_embedding_used.repeat(data.shape[1], 1), data, targets.float()), single_eval_pos=eval_pos)).squeeze(-1)\n", + " \n", + " loss = torch.nn.BCELoss()(outputs.flatten(), targets[eval_pos:].flatten()).detach().cpu()\n", + " losses += [loss]\n", + " hparam_real = [diff_hparams_f[i][1](hp) for i, hp in enumerate(hparam)]\n", + " hparams += [hparam_real]\n", + " \n", + " print(loss, hparam_real, hparam, outputs.shape)\n", + " return np.array(losses), np.array(hparams)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def differentiable_hparam_tuning_workflow(config_sample, hparam_label, batch_size=4, N_grad_steps=50, plot_step_size=0.1):\n", + " test_gp_params = {\n", + " \"lengthscale\": 1.0,\n", + " #\"lengthscale_mean\": true_lengthscale,\n", + " #\"lengthscale_std\": 0.5,\n", + " \"noise\": 0.2,\n", + " \"outputscale\": 1.0,\n", + " 'batch_size': batch_size\n", + " }\n", + " config_sample.update(test_gp_params)\n", + " (hp_embedding, data, targets_), targets = generate_test_data(config_sample)\n", + " hparam_true = [diff_hparams_f[i][0](test_gp_params[hp]) for i, hp in enumerate(diff_hparams_keys)]\n", + " #hparam_true = [test_gp_params[hp] for i, hp in enumerate(diff_hparams_keys)]\n", + "\n", + " for vary_hparam_ind, vary_hparam_name in hparam_label:\n", + " print(vary_hparam_name)\n", + "\n", + " losses, hparams = evaluate_hp_range(model, hparam_true, vary_hparam_ind, data, targets, eval_pos, plot_step_size=plot_step_size)\n", + "\n", + " # TODO: Make only one parameter diffable\n", + " hparam = torch.tensor([*hparam_true]).to(device).float()\n", + " hparam[vary_hparam_ind] = hparam[vary_hparam_ind] + 0.1 #random.random() * 2 - 1\n", + " hparam = torch.nn.Parameter(hparam, requires_grad=True)\n", + " hparam_grad_mask = torch.zeros_like(hparam)\n", + " hparam_grad_mask[vary_hparam_ind] = 1\n", + "\n", + " optimizer = torch.optim.Adam([hparam], lr=0.1)\n", + " \n", + " for t in range(N_grad_steps):\n", + " style = hparam.repeat(data.shape[1], 1)\n", + " outputs = torch.sigmoid(model[2]((style, data, targets.float()), single_eval_pos=eval_pos)).squeeze(-1)\n", + " loss = torch.nn.BCELoss()(outputs.flatten(), targets[eval_pos:].flatten())\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " with torch.no_grad():\n", + " hparam.grad *= hparam_grad_mask\n", + " optimizer.step()\n", + " print('loss:', loss, 'hparams', diff_hparams_f[vary_hparam_ind][1](hparam[vary_hparam_ind]), 'true', diff_hparams_f[vary_hparam_ind][1](hparam_true[vary_hparam_ind]))\n", + " inferred_param = diff_hparams_f[vary_hparam_ind][1](hparam[vary_hparam_ind].cpu().detach().numpy())\n", + " return hparams, losses, inferred_param, vary_hparam_ind, hparam_true\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Fitting a PFN with HP-Diffable GP Prior" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "hidden": true, + "tags": [] + }, + "outputs": [], + "source": [ + "num_features = 5\n", + "bptt = 200\n", + "eval_positions = [100]\n", + "\n", + "config_general = get_general_config(num_features, bptt, eval_positions)\n", + "config_flexible_categorical = get_flexible_categorical_config(num_features)\n", + "\n", + "config_gp = {'noise': 0.2, \"lengthscale\": 1.0, \"outputscale\": 1.0}\n", + "config_diff_gp = {'differentiable_hyperparameters': {\n", + " 'outputscale': {'distribution': 'uniform', 'min': 0., 'max': 10.0},\n", + " 'lengthscale': {'distribution': 'uniform', 'min': 0., 'max': 10.0},\n", + " 'noise': {'distribution': 'uniform', 'min': 0.0000001, 'max': 0.5},\n", + " }\n", + "}\n", + "\n", + "config = {**config_general, **config_flexible_categorical, **config_diff_gp, **config_gp}\n", + "\n", + "config['prior_type'], config['differentiable'], config['flexible'] = 'gp', True, True\n", + "config['num_features'], config['num_features_used'] = num_features, num_features\n", + "config['epochs'], config['num_steps'], config['verbose'] = 500, 100, False\n", + "config[\"lr\"] = 0.00001\n", + "config[\"dropout\"] = 0\n", + "config[\"emsize\"] = 512\n", + "config[\"batch_size\"] = 128\n", + "config[\"aggregate_k_gradients\"] = 1\n", + "config['set_value_to_nan'] = 0.0\n", + "config['output_multiclass_ordered_p'] = 1.0\n", + "config['categorical_feature_p'] = 0.0\n", + "config['nan_prob_a_reason'] = 0.0\n", + "config['nan_prob_no_reason'] = 0.0\n", + "config['nan_prob_unknown_reason'] = 0.0\n", + "config[\"nlayers\"] = 8\n", + "\n", + "# TODO: This should not be sampled, but be one config\n", + "# TODO: This uses old hyperparam sampler throws error\n", + "config_sample = evaluate_hypers(config)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "hidden": true, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using style prior: True\n", + "Using cpu:0 device\n", + "Not using distributed\n", + "DataLoader.__dict__ {'num_steps': 100, 'fuse_x_y': False, 'get_batch_kwargs': {'batch_size': 128, 'seq_len': 200, 'seq_len_maximum': 200, 'device': 'cpu:0', 'num_features': 5, 'hyperparameters': {'lr': 1e-05, 'dropout': 0, 'emsize': 512, 'batch_size': 128, 'nlayers': 8, 'num_features': 5, 'nhead': 4, 'nhid_factor': 2, 'bptt': 200, 'eval_positions': None, 'seq_len_used': 200, 'sampling': 'normal', 'epochs': 500, 'num_steps': 100, 'verbose': False, 'pre_sample_causes': True, 'mix_activations': False, 'nan_prob_unknown_reason_reason_prior': 1.0, 'categorical_feature_p': 0.0, 'nan_prob_no_reason': 0.0, 'nan_prob_unknown_reason': 0.0, 'nan_prob_a_reason': 0.0, 'max_num_classes': 2, 'num_classes': 2, 'noise_type': 'Gaussian', 'balanced': True, 'normalize_to_ranking': False, 'set_value_to_nan': 0.0, 'normalize_by_used_features': True, 'num_features_used': 5, 'differentiable_hyperparameters': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'noise': 0.2, 'lengthscale': 1.0, 'outputscale': 1.0, 'prior_type': 'gp', 'differentiable': True, 'flexible': True, 'aggregate_k_gradients': 1, 'output_multiclass_ordered_p': 1.0, 'recompute_attn': False}, 'num_outputs': 1, 'dynamic_batch_size': 2, 'get_batch': .make_get_batch.. at 0x7f39ad8dcf80>, 'differentiable_hyperparameters': {'outputscale': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'lengthscale': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'noise': {'distribution': 'uniform', 'min': 1e-07, 'max': 0.5}}}, 'num_features': 5, 'num_outputs': 1}\n", + "Using a Transformer with 17.35 M parameters\n" + ] + } + ], + "source": [ + "device = 'cuda'\n", + "train_function(config_sample, 0, add_name='gp_experiments_diff_with_noise_no_meta_new')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Evaluating a PFN (with pretrained model)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "hidden": true, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using style prior: True\n", + "Using cpu:0 device\n", + "Not using distributed\n", + "DataLoader.__dict__ {'num_steps': 100, 'fuse_x_y': False, 'get_batch_kwargs': {'batch_size': 1, 'seq_len': 10, 'seq_len_maximum': 10, 'device': 'cpu:0', 'num_features': 5, 'hyperparameters': {'lr': 1e-05, 'dropout': 0, 'emsize': 512, 'batch_size': 1, 'nlayers': 8, 'num_features': 5, 'nhead': 4, 'nhid_factor': 2, 'bptt': 10, 'eval_positions': [190], 'seq_len_used': 200, 'sampling': 'normal', 'epochs': 500, 'num_steps': 100, 'verbose': False, 'pre_sample_causes': True, 'mix_activations': False, 'nan_prob_unknown_reason_reason_prior': 1.0, 'output_multiclass_ordered_p': 1.0, 'categorical_feature_p': 0.0, 'nan_prob_no_reason': 0.0, 'nan_prob_unknown_reason': 0.0, 'nan_prob_a_reason': 0.0, 'max_num_classes': 2, 'num_classes': 2, 'noise_type': 'Gaussian', 'balanced': True, 'multiclass_type': 'rank', 'normalize_to_ranking': False, 'set_value_to_nan': 0.0, 'normalize_by_used_features': True, 'num_features_used': . at 0x7f39ad8534d0>, 'differentiable_hyperparameters': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'noise': 0.03, 'lengthscale': 1.0, 'outputscale': 1.0, 'prior_type': 'gp', 'differentiable': True, 'flexible': True, 'aggregate_k_gradients': 1, 'recompute_attn': False, 'bptt_extra_samples': None, 'epoch_in_training': 0.998, 'categorical_features_sampler': . at 0x7f39ad853680>, 'num_features_used_in_training': 5, 'num_classes_in_training': 2, 'batch_size_in_training': 128, 'bptt_in_training': 200, 'bptt_extra_samples_in_training': None}, 'num_outputs': 1, 'dynamic_batch_size': 2, 'get_batch': .make_get_batch.. at 0x7f39ad81ab90>, 'differentiable_hyperparameters': {'outputscale': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'lengthscale': {'distribution': 'uniform', 'min': 0.0, 'max': 10.0}, 'noise': {'distribution': 'uniform', 'min': 1e-07, 'max': 0.5}}}, 'num_features': 5, 'num_outputs': 1}\n", + "Using a Transformer with 17.35 M parameters\n" + ] + } + ], + "source": [ + "device = 'cpu'\n", + "model, c = load_model(base_path, f'models_diff/gp_ablation_model.cpkt', device, eval_positions, verbose=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from priors.differentiable_prior import DifferentiableHyperparameterList\n", + "diff_list = DifferentiableHyperparameterList(c['differentiable_hyperparameters'], 512, device)\n", + "diff_hparams_keys, diff_hparams_f = diff_list.get_hyperparameter_info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "model[2].eval()\n", + "eval_pos = 100\n", + "\n", + "hparam_label = [(1, 'outputscale')]\n", + "hparam_label = [(0, 'lengthscale')]\n", + "hparam_label = [(2, 'noise')]\n", + "hparam_labels = [[(1, 'outputscale')], [(2, 'noise')], [(0, 'lengthscale')]]\n", + "#hparam_labels = [[(2, 'noise')]]\n", + "\n", + "hparams, losses, inferred_param, vary_hparam_ind, hparam_true = {}, {}, {}, {}, {}\n", + "\n", + "for hparam_label in hparam_labels:\n", + " (hparams[hparam_label[0][1]], losses[hparam_label[0][1]], inferred_param[hparam_label[0][1]], vary_hparam_ind[hparam_label[0][1]], \n", + " hparam_true[hparam_label[0][1]]) = differentiable_hparam_tuning_workflow(config_sample, \n", + " hparam_label=hparam_label, \n", + " batch_size=256, \n", + " N_grad_steps=50,\n", + " plot_step_size=0.05)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "label = 'lengthscale'\n", + "\n", + "#import tikzplotlib\n", + "\n", + "inferred = losses[label]\n", + "\n", + "plt.plot(hparams[label][:, vary_hparam_ind[label]], losses[label])\n", + "true = diff_hparams_f[vary_hparam_ind[label]][1](hparam_true[label][vary_hparam_ind[label]])\n", + "plt.axvline(x=inferred_param[label], linestyle='solid', color='red')\n", + "plt.axvline(x=true, linestyle='dashed')\n", + "\n", + "plt.ylabel('Cross entropy Loss')\n", + "plt.xlabel(label)\n", + "\n", + "#tikzplotlib.save(f'diff_inferred_params_{label}.tex', axis_height='5.2cm', axis_width='5.2cm', strict=True)\n", + "\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/TabPFN/TabPFNPredictionOnly.ipynb b/TabPFN/TabPFNPredictionOnly.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..2d0f41c8c19e67ac8a1f120c5ff9d4c9a594e0ef --- /dev/null +++ b/TabPFN/TabPFNPredictionOnly.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook shows how to use TabPFN for tabular prediction with a scikit learn wrapper.\n", + "\n", + "classifier = TabPFNClassifier(device='cpu')\n", + "classifier.fit(train_xs, train_ys)\n", + "prediction_ = classifier.predict(test_xs)\n", + "\n", + "The fit function does not perform any computations, but only saves the training data. Computations are only done at inference time, when calling predict.\n", + "Note that the presaved models were trained for up to 100 features, 10 classes and 1000 samples. While the model does not have a hard bound on the number of samples, the features and classes are restricted and larger sizes lead to an error." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "import torch\n", + "import numpy as np\n", + "import os\n", + "import random\n", + "\n", + "from model_builder import get_model, get_default_spec, save_model, load_model\n", + "from scripts.transformer_prediction_interface import transformer_predict, get_params_from_config, TabPFNClassifier\n", + "\n", + "from datasets import load_openml_list, open_cc_dids, open_cc_valid_dids\n", + "\n", + "from scripts import tabular_metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "base_path = '.'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Load datasets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": true + }, + "tags": [] + }, + "outputs": [], + "source": [ + "max_samples = 10000\n", + "bptt = 10000\n", + "\n", + "cc_test_datasets_multiclass, cc_test_datasets_multiclass_df = load_openml_list(open_cc_dids, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = max_samples, num_feats=100, return_capped=True)\n", + "cc_valid_datasets_multiclass, cc_valid_datasets_multiclass_df = load_openml_list(open_cc_valid_dids, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = max_samples, num_feats=100, return_capped=True)\n", + "\n", + "# Loading longer OpenML Datasets for generalization experiments (optional)\n", + "# test_datasets_multiclass, test_datasets_multiclass_df = load_openml_list(test_dids_classification, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 10000, num_feats=100, return_capped=True)\n", + "\n", + "random.seed(0)\n", + "random.shuffle(cc_valid_datasets_multiclass)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import get_openml_classification" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = openml.datasets.get_dataset(31)\n", + "X, y, categorical_indicator, attribute_names = dataset.get_data(\n", + " dataset_format=\"array\", target=dataset.default_target_attribute\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_datasets(selector, task_type, suite='cc'):\n", + " if task_type == 'binary':\n", + " ds = valid_datasets_binary if selector == 'valid' else test_datasets_binary\n", + " else:\n", + " if suite == 'openml':\n", + " ds = valid_datasets_multiclass if selector == 'valid' else test_datasets_multiclass\n", + " elif suite == 'cc':\n", + " ds = cc_valid_datasets_multiclass if selector == 'valid' else cc_test_datasets_multiclass\n", + " else:\n", + " raise Exception(\"Unknown suite\")\n", + " return ds" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_string, longer, task_type = '', 1, 'multiclass'\n", + "eval_positions = [1000]\n", + "bptt = 2000\n", + " \n", + "test_datasets, valid_datasets = get_datasets('test', task_type, suite='cc'), get_datasets('valid', task_type, suite='cc')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### Select a dataset for prediction" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "[(i, test_datasets[i][0]) for i in range(len(test_datasets))]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "evaluation_dataset_index = 4 # Index of the dataset to predict\n", + "ds = test_datasets[evaluation_dataset_index]\n", + "print(f'Evaluation dataset name: {ds[0]} shape {ds[1].shape}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xs, ys = ds[1].clone(), ds[2].clone()\n", + "eval_position = xs.shape[0] // 2\n", + "train_xs, train_ys = xs[0:eval_position], ys[0:eval_position]\n", + "test_xs, test_ys = xs[eval_position:], ys[eval_position:]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Predict using a Fitted and Tuned Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "classifier = TabPFNClassifier(device='cpu')\n", + "classifier.fit(train_xs, train_ys)\n", + "prediction_ = classifier.predict_proba(test_xs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "roc, ce = tabular_metrics.auc_metric(test_ys, prediction_), tabular_metrics.cross_entropy(test_ys, prediction_)\n", + "'AUC', float(roc), 'Cross Entropy', float(ce)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/TabPFN/TabularEvaluationVisualization.ipynb b/TabPFN/TabularEvaluationVisualization.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..c55cdaf2e2426ea6fa1ca696e6f143247380455b --- /dev/null +++ b/TabPFN/TabularEvaluationVisualization.ipynb @@ -0,0 +1,12078 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "from scripts import tabular_baselines\n", + "\n", + "import seaborn as sns\n", + "import numpy as np\n", + "\n", + "from datasets import load_openml_list, valid_dids_classification, test_dids_classification, open_cc_dids\n", + "from scripts.tabular_baselines import *\n", + "from scripts.tabular_evaluation import evaluate\n", + "from scripts.tabular_metrics import calculate_score, make_ranks_and_wins_table, make_metric_matrix\n", + "from scripts import tabular_metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from notebook_utils import *" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Datasets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "cc_test_datasets_multiclass, cc_test_datasets_multiclass_df = load_openml_list(open_cc_dids, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 10000, num_feats=100, return_capped=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def get_datasets(selector, task_type, suite='openml'):\n", + " if task_type == 'binary':\n", + " ds = valid_datasets_binary if selector == 'valid' else test_datasets_binary\n", + " else:\n", + " if suite == 'openml':\n", + " ds = valid_datasets_multiclass if selector == 'valid' else test_datasets_multiclass\n", + " elif suite == 'cc':\n", + " ds = valid_datasets_multiclass if selector == 'valid' else cc_test_datasets_multiclass\n", + " else:\n", + " raise Exception(\"Unknown suite\")\n", + " return ds" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Setting params" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "eval_positions = [1000]\n", + "max_features = 100\n", + "bptt = 2000\n", + "selector = 'test'\n", + "base_path = os.path.join('.')\n", + "overwrite=False\n", + "max_times = [0.5, 1, 15, 30, 60, 60*5, 60*15, 60*60]\n", + "metric_used = tabular_metrics.auc_metric\n", + "baseline_methods = ['logistic', 'gp', 'knn', 'catboost', 'xgb', 'autosklearn2', 'autogluon']\n", + "task_type = 'multiclass'" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "suite = 'cc'\n", + "test_datasets = get_datasets('test',task_type, suite=suite)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "clf_dict= {'gp': gp_metric\n", + " , 'knn': knn_metric\n", + " , 'catboost': catboost_metric\n", + " , 'xgb': xgb_metric\n", + " , 'logistic': logistic_metric\n", + " , 'autosklearn': autosklearn_metric\n", + " , 'autosklearn2': autosklearn2_metric\n", + " , 'autogluon': autogluon_metric}" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "device = 'cpu'\n", + "\n", + "def eval_method(task_type, method, dids, selector, eval_positions, max_time, metric_used, split_number, append_metric=True, fetch_only=False, verbose=False):\n", + " \n", + " dids = dids if type(dids) is list else [dids]\n", + " \n", + " for did in dids:\n", + "\n", + " ds = get_datasets(selector, task_type, suite=suite)\n", + "\n", + " ds = ds if did is None else ds[did:did+1]\n", + "\n", + " clf = clf_dict[method]\n", + "\n", + " time_string = '_time_'+str(max_time) if max_time else ''\n", + " metric_used_string = '_'+tabular_baselines.get_scoring_string(metric_used, usage='') if append_metric else ''\n", + "\n", + " result = evaluate(datasets=ds\n", + " , model=clf\n", + " , method=method+time_string+metric_used_string\n", + " , bptt=bptt, base_path=base_path\n", + " , eval_positions=eval_positions\n", + " , device=device, max_splits=1\n", + " , overwrite=overwrite\n", + " , save=True\n", + " , metric_used=metric_used\n", + " , path_interfix=task_type\n", + " , fetch_only=fetch_only\n", + " , split_number=split_number\n", + " , verbose=verbose\n", + " , max_time=max_time)\n", + " \n", + " return result" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Baseline Evaluation\n", + "This section runs baselines and saves results locally." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "!mkdir {base_path}/results\n", + "!mkdir {base_path}/results/tabular/\n", + "!mkdir {base_path}/results/tabular/multiclass/" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# RUN ONE METHOD ON ONE DATASET AND SPLIT\n", + "overwrite=True\n", + "dataset_id = 1\n", + "split_number = 1\n", + "maximum_runtime = 30\n", + "r = eval_method(task_type, 'logistic', dataset_id, 'test', eval_positions, maximum_runtime, metric_used, split_number)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Calculating splits 312: 0%| | 0/1 [00:00 312: 0%| | 0/1 [00:01 312: 0%| | 0/1 [00:00 312: 0%| | 0/1 [00:01 312: 0%| | 0/1 [00:00 312: 0%| | 0/1 [00:01 312: 0%| | 0/1 [00:00 312: 0%| | 0/1 [00:01 312: 0%| | 0/1 [00:00 312: 0%| | 0/1 [00:00\n", + " jobs = [\n", + " File \"/tmp/ipykernel_3318773/475731429.py\", line 5, in \n", + " eval_method(task, m, did, selector, eval_positions, max_time, metric_used, split_number)\n", + " File \"/tmp/ipykernel_3318773/551782496.py\", line 18, in eval_method\n", + " result = evaluate(datasets=ds\n", + " File \"/home/hollmann/prior-fitting-release/scripts/tabular_evaluation.py\", line 118, in evaluate\n", + " r = evaluate_position(X, y\n", + " File \"/home/hollmann/prior-fitting-release/scripts/tabular_evaluation.py\", line 266, in evaluate_position\n", + " _, outputs, best_configs = baseline_predict(model, eval_xs, eval_ys, categorical_feats\n", + " File \"/home/hollmann/prior-fitting-release/scripts/baseline_prediction_interface.py\", line 24, in baseline_predict\n", + " metric, output, best_config = metric_function(eval_x[:eval_pos],\n", + " File \"/home/hollmann/prior-fitting-release/scripts/tabular_baselines.py\", line 237, in knn_metric\n", + " best = fmin(\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/hyperopt/fmin.py\", line 553, in fmin\n", + " rval.exhaust()\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/hyperopt/fmin.py\", line 356, in exhaust\n", + " self.run(self.max_evals - n_done, block_until_done=self.asynchronous)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/hyperopt/fmin.py\", line 292, in run\n", + " self.serial_evaluate()\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/hyperopt/fmin.py\", line 170, in serial_evaluate\n", + " result = self.domain.evaluate(spec, ctrl)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/hyperopt/base.py\", line 907, in evaluate\n", + " rval = self.fn(pyll_rval)\n", + " File \"/home/hollmann/prior-fitting-release/scripts/tabular_baselines.py\", line 238, in \n", + " fn=lambda params: eval_f(params, clf_, x, y, metric_used, start_time, max_time),\n", + " File \"/home/hollmann/prior-fitting-release/scripts/tabular_baselines.py\", line 80, in eval_f\n", + " scores = cross_val_score(clf_(**params), x, y, cv=CV, scoring=get_scoring_string(metric_used))\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/utils/validation.py\", line 63, in inner_f\n", + " return f(*args, **kwargs)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/model_selection/_validation.py\", line 445, in cross_val_score\n", + " cv_results = cross_validate(estimator=estimator, X=X, y=y, groups=groups,\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/utils/validation.py\", line 63, in inner_f\n", + " return f(*args, **kwargs)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/model_selection/_validation.py\", line 250, in cross_validate\n", + " results = parallel(\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\", line 1044, in __call__\n", + " while self.dispatch_one_batch(iterator):\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\", line 859, in dispatch_one_batch\n", + " self._dispatch(tasks)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\", line 777, in _dispatch\n", + " job = self._backend.apply_async(batch, callback=cb)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/_parallel_backends.py\", line 208, in apply_async\n", + " result = ImmediateResult(func)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/_parallel_backends.py\", line 572, in __init__\n", + " self.results = batch()\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\", line 262, in __call__\n", + " return [func(*args, **kwargs)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\", line 262, in \n", + " return [func(*args, **kwargs)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/utils/fixes.py\", line 222, in __call__\n", + " return self.function(*args, **kwargs)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/model_selection/_validation.py\", line 625, in _fit_and_score\n", + " test_scores = _score(estimator, X_test, y_test, scorer, error_score)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/model_selection/_validation.py\", line 687, in _score\n", + " scores = scorer(estimator, X_test, y_test)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/metrics/_scorer.py\", line 87, in __call__\n", + " score = scorer._score(cached_call, estimator,\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/metrics/_scorer.py\", line 277, in _score\n", + " y_pred = method_caller(clf, \"predict_proba\", X)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/metrics/_scorer.py\", line 53, in _cached_call\n", + " return getattr(estimator, method)(*args, **kwargs)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/neighbors/_classification.py\", line 241, in predict_proba\n", + " neigh_dist, neigh_ind = self.kneighbors(X)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/neighbors/_base.py\", line 722, in kneighbors\n", + " chunked_results = Parallel(n_jobs, **parallel_kwargs)(\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\", line 966, in __call__\n", + " n_jobs = self._initialize_backend()\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\", line 733, in _initialize_backend\n", + " n_jobs = self._backend.configure(n_jobs=self.n_jobs, parallel=self,\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/_parallel_backends.py\", line 489, in configure\n", + " n_jobs = self.effective_n_jobs(n_jobs)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/_parallel_backends.py\", line 509, in effective_n_jobs\n", + " elif mp.current_process().daemon:\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/multiprocessing/process.py\", line 41, in current_process\n", + " return _current_process\n", + "KeyboardInterrupt\n", + "\n", + "During handling of the above exception, another exception occurred:\n", + "\n", + "Traceback (most recent call last):\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/interactiveshell.py\", line 2061, in showtraceback\n", + " stb = value._render_traceback_()\n", + "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n", + "\n", + "During handling of the above exception, another exception occurred:\n", + "\n", + "Traceback (most recent call last):\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/ultratb.py\", line 1101, in get_records\n", + " return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/ultratb.py\", line 248, in wrapped\n", + " return f(*args, **kwargs)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/ultratb.py\", line 281, in _fixed_getinnerframes\n", + " records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/inspect.py\", line 1541, in getinnerframes\n", + " frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/inspect.py\", line 1503, in getframeinfo\n", + " lines, lnum = findsource(frame)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/ultratb.py\", line 182, in findsource\n", + " lines = linecache.getlines(file, globals_dict)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/linecache.py\", line 46, in getlines\n", + " return updatecache(filename, module_globals)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/linecache.py\", line 136, in updatecache\n", + " with tokenize.open(fullname) as fp:\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/tokenize.py\", line 394, in open\n", + " encoding, lines = detect_encoding(buffer.readline)\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/tokenize.py\", line 363, in detect_encoding\n", + " first = read_or_stop()\n", + " File \"/home/hollmann/anaconda3/envs/prior-fitting/lib/python3.9/tokenize.py\", line 321, in read_or_stop\n", + " return readline()\n", + "KeyboardInterrupt\n" + ] + }, + { + "ename": "TypeError", + "evalue": "object of type 'NoneType' has no len()", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)", + " \u001B[0;31m[... skipping hidden 1 frame]\u001B[0m\n", + "\u001B[0;32m/tmp/ipykernel_3318773/475731429.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[1;32m 3\u001B[0m \u001B[0moverwrite\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 4\u001B[0;31m jobs = [\n\u001B[0m\u001B[1;32m 5\u001B[0m \u001B[0meval_method\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtask\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mm\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdid\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mselector\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0meval_positions\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmax_time\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmetric_used\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0msplit_number\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/tmp/ipykernel_3318773/475731429.py\u001B[0m in \u001B[0;36m\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m 4\u001B[0m jobs = [\n\u001B[0;32m----> 5\u001B[0;31m \u001B[0meval_method\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtask\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mm\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdid\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mselector\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0meval_positions\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmax_time\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmetric_used\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0msplit_number\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 6\u001B[0m \u001B[0;32mfor\u001B[0m \u001B[0mdid\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mrange\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtest_datasets\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/tmp/ipykernel_3318773/551782496.py\u001B[0m in \u001B[0;36meval_method\u001B[0;34m(task_type, method, dids, selector, eval_positions, max_time, metric_used, split_number, append_metric, fetch_only, verbose)\u001B[0m\n\u001B[1;32m 17\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 18\u001B[0;31m result = evaluate(datasets=ds\n\u001B[0m\u001B[1;32m 19\u001B[0m \u001B[0;34m,\u001B[0m \u001B[0mmodel\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mclf\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/prior-fitting-release/scripts/tabular_evaluation.py\u001B[0m in \u001B[0;36mevaluate\u001B[0;34m(datasets, bptt, eval_positions, verbose, metric_used, return_tensor, **kwargs)\u001B[0m\n\u001B[1;32m 117\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 118\u001B[0;31m r = evaluate_position(X, y\n\u001B[0m\u001B[1;32m 119\u001B[0m \u001B[0;34m,\u001B[0m \u001B[0mnum_classes\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0munique\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0my\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/prior-fitting-release/scripts/tabular_evaluation.py\u001B[0m in \u001B[0;36mevaluate_position\u001B[0;34m(X, y, categorical_feats, model, bptt, eval_position, overwrite, save, base_path, path_interfix, method, ds_name, fetch_only, max_time, split_number, per_step_normalization, **kwargs)\u001B[0m\n\u001B[1;32m 265\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 266\u001B[0;31m _, outputs, best_configs = baseline_predict(model, eval_xs, eval_ys, categorical_feats\n\u001B[0m\u001B[1;32m 267\u001B[0m \u001B[0;34m,\u001B[0m \u001B[0meval_pos\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0meval_position\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/prior-fitting-release/scripts/baseline_prediction_interface.py\u001B[0m in \u001B[0;36mbaseline_predict\u001B[0;34m(metric_function, eval_xs, eval_ys, categorical_feats, metric_used, eval_pos, max_time, **kwargs)\u001B[0m\n\u001B[1;32m 23\u001B[0m \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 24\u001B[0;31m metric, output, best_config = metric_function(eval_x[:eval_pos],\n\u001B[0m\u001B[1;32m 25\u001B[0m \u001B[0meval_y\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0meval_pos\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/prior-fitting-release/scripts/tabular_baselines.py\u001B[0m in \u001B[0;36mknn_metric\u001B[0;34m(x, y, test_x, test_y, cat_features, metric_used, max_time)\u001B[0m\n\u001B[1;32m 236\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 237\u001B[0;31m best = fmin(\n\u001B[0m\u001B[1;32m 238\u001B[0m \u001B[0mfn\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mlambda\u001B[0m \u001B[0mparams\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0meval_f\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mparams\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mclf_\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mx\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0my\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmetric_used\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstart_time\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmax_time\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/hyperopt/fmin.py\u001B[0m in \u001B[0;36mfmin\u001B[0;34m(fn, space, algo, max_evals, timeout, loss_threshold, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len, show_progressbar, early_stop_fn, trials_save_file)\u001B[0m\n\u001B[1;32m 552\u001B[0m \u001B[0;31m# next line is where the fmin is actually executed\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 553\u001B[0;31m \u001B[0mrval\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mexhaust\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 554\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/hyperopt/fmin.py\u001B[0m in \u001B[0;36mexhaust\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 355\u001B[0m \u001B[0mn_done\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrials\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 356\u001B[0;31m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mmax_evals\u001B[0m \u001B[0;34m-\u001B[0m \u001B[0mn_done\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mblock_until_done\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0masynchronous\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 357\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrials\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrefresh\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/hyperopt/fmin.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(self, N, block_until_done)\u001B[0m\n\u001B[1;32m 291\u001B[0m \u001B[0;31m# -- loop over trials and do the jobs directly\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 292\u001B[0;31m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mserial_evaluate\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 293\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/hyperopt/fmin.py\u001B[0m in \u001B[0;36mserial_evaluate\u001B[0;34m(self, N)\u001B[0m\n\u001B[1;32m 169\u001B[0m \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 170\u001B[0;31m \u001B[0mresult\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdomain\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mevaluate\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mspec\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mctrl\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 171\u001B[0m \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0me\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/hyperopt/base.py\u001B[0m in \u001B[0;36mevaluate\u001B[0;34m(self, config, ctrl, attach_attachments)\u001B[0m\n\u001B[1;32m 906\u001B[0m )\n\u001B[0;32m--> 907\u001B[0;31m \u001B[0mrval\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mfn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mpyll_rval\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 908\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/prior-fitting-release/scripts/tabular_baselines.py\u001B[0m in \u001B[0;36m\u001B[0;34m(params)\u001B[0m\n\u001B[1;32m 237\u001B[0m best = fmin(\n\u001B[0;32m--> 238\u001B[0;31m \u001B[0mfn\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mlambda\u001B[0m \u001B[0mparams\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0meval_f\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mparams\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mclf_\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mx\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0my\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmetric_used\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstart_time\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmax_time\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 239\u001B[0m \u001B[0mspace\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mparam_grid_hyperopt\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m'knn'\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/prior-fitting-release/scripts/tabular_baselines.py\u001B[0m in \u001B[0;36meval_f\u001B[0;34m(params, clf_, x, y, metric_used, start_time, max_time)\u001B[0m\n\u001B[1;32m 79\u001B[0m \u001B[0;32mreturn\u001B[0m \u001B[0mnp\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnan\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 80\u001B[0;31m \u001B[0mscores\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mcross_val_score\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mclf_\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m**\u001B[0m\u001B[0mparams\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mx\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0my\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mcv\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mCV\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mscoring\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mget_scoring_string\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmetric_used\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 81\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/utils/validation.py\u001B[0m in \u001B[0;36minner_f\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m 62\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mextra_args\u001B[0m \u001B[0;34m<=\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 63\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mf\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 64\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/model_selection/_validation.py\u001B[0m in \u001B[0;36mcross_val_score\u001B[0;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, error_score)\u001B[0m\n\u001B[1;32m 444\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 445\u001B[0;31m cv_results = cross_validate(estimator=estimator, X=X, y=y, groups=groups,\n\u001B[0m\u001B[1;32m 446\u001B[0m \u001B[0mscoring\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;34m{\u001B[0m\u001B[0;34m'score'\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mscorer\u001B[0m\u001B[0;34m}\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mcv\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mcv\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/utils/validation.py\u001B[0m in \u001B[0;36minner_f\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m 62\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mextra_args\u001B[0m \u001B[0;34m<=\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 63\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mf\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 64\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/model_selection/_validation.py\u001B[0m in \u001B[0;36mcross_validate\u001B[0;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score, return_estimator, error_score)\u001B[0m\n\u001B[1;32m 249\u001B[0m pre_dispatch=pre_dispatch)\n\u001B[0;32m--> 250\u001B[0;31m results = parallel(\n\u001B[0m\u001B[1;32m 251\u001B[0m delayed(_fit_and_score)(\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, iterable)\u001B[0m\n\u001B[1;32m 1043\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1044\u001B[0;31m \u001B[0;32mwhile\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdispatch_one_batch\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0miterator\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 1045\u001B[0m \u001B[0;32mpass\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\u001B[0m in \u001B[0;36mdispatch_one_batch\u001B[0;34m(self, iterator)\u001B[0m\n\u001B[1;32m 858\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 859\u001B[0;31m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_dispatch\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtasks\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 860\u001B[0m \u001B[0;32mreturn\u001B[0m \u001B[0;32mTrue\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\u001B[0m in \u001B[0;36m_dispatch\u001B[0;34m(self, batch)\u001B[0m\n\u001B[1;32m 776\u001B[0m \u001B[0mjob_idx\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_jobs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 777\u001B[0;31m \u001B[0mjob\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_backend\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mapply_async\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mbatch\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mcallback\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mcb\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 778\u001B[0m \u001B[0;31m# A job can complete so quickly than its callback is\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/_parallel_backends.py\u001B[0m in \u001B[0;36mapply_async\u001B[0;34m(self, func, callback)\u001B[0m\n\u001B[1;32m 207\u001B[0m \u001B[0;34m\"\"\"Schedule a func to be run\"\"\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 208\u001B[0;31m \u001B[0mresult\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mImmediateResult\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfunc\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 209\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mcallback\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/_parallel_backends.py\u001B[0m in \u001B[0;36m__init__\u001B[0;34m(self, batch)\u001B[0m\n\u001B[1;32m 571\u001B[0m \u001B[0;31m# arguments in memory\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 572\u001B[0;31m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mresults\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mbatch\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 573\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 261\u001B[0m \u001B[0;32mwith\u001B[0m \u001B[0mparallel_backend\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_backend\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mn_jobs\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_n_jobs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 262\u001B[0;31m return [func(*args, **kwargs)\n\u001B[0m\u001B[1;32m 263\u001B[0m for func, args, kwargs in self.items]\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\u001B[0m in \u001B[0;36m\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m 261\u001B[0m \u001B[0;32mwith\u001B[0m \u001B[0mparallel_backend\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_backend\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mn_jobs\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_n_jobs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 262\u001B[0;31m return [func(*args, **kwargs)\n\u001B[0m\u001B[1;32m 263\u001B[0m for func, args, kwargs in self.items]\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/utils/fixes.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m 221\u001B[0m \u001B[0;32mwith\u001B[0m \u001B[0mconfig_context\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m**\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mconfig\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 222\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mfunction\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/model_selection/_validation.py\u001B[0m in \u001B[0;36m_fit_and_score\u001B[0;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, split_progress, candidate_progress, error_score)\u001B[0m\n\u001B[1;32m 624\u001B[0m \u001B[0mfit_time\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mtime\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtime\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m-\u001B[0m \u001B[0mstart_time\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 625\u001B[0;31m \u001B[0mtest_scores\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0m_score\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mestimator\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mX_test\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0my_test\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mscorer\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0merror_score\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 626\u001B[0m \u001B[0mscore_time\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mtime\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtime\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m-\u001B[0m \u001B[0mstart_time\u001B[0m \u001B[0;34m-\u001B[0m \u001B[0mfit_time\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/model_selection/_validation.py\u001B[0m in \u001B[0;36m_score\u001B[0;34m(estimator, X_test, y_test, scorer, error_score)\u001B[0m\n\u001B[1;32m 686\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 687\u001B[0;31m \u001B[0mscores\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mscorer\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mestimator\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mX_test\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0my_test\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 688\u001B[0m \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/metrics/_scorer.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, estimator, *args, **kwargs)\u001B[0m\n\u001B[1;32m 86\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0misinstance\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mscorer\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0m_BaseScorer\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 87\u001B[0;31m score = scorer._score(cached_call, estimator,\n\u001B[0m\u001B[1;32m 88\u001B[0m *args, **kwargs)\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/metrics/_scorer.py\u001B[0m in \u001B[0;36m_score\u001B[0;34m(self, method_caller, clf, X, y, sample_weight)\u001B[0m\n\u001B[1;32m 276\u001B[0m \u001B[0my_type\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mtype_of_target\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0my\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 277\u001B[0;31m \u001B[0my_pred\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mmethod_caller\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mclf\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m\"predict_proba\"\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mX\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 278\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0my_type\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;34m\"binary\"\u001B[0m \u001B[0;32mand\u001B[0m \u001B[0my_pred\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mshape\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;36m1\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m<=\u001B[0m \u001B[0;36m2\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/metrics/_scorer.py\u001B[0m in \u001B[0;36m_cached_call\u001B[0;34m(cache, estimator, method, *args, **kwargs)\u001B[0m\n\u001B[1;32m 52\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mcache\u001B[0m \u001B[0;32mis\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 53\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mgetattr\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mestimator\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmethod\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 54\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/neighbors/_classification.py\u001B[0m in \u001B[0;36mpredict_proba\u001B[0;34m(self, X)\u001B[0m\n\u001B[1;32m 240\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 241\u001B[0;31m \u001B[0mneigh_dist\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mneigh_ind\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mkneighbors\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mX\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 242\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/sklearn/neighbors/_base.py\u001B[0m in \u001B[0;36mkneighbors\u001B[0;34m(self, X, n_neighbors, return_distance)\u001B[0m\n\u001B[1;32m 721\u001B[0m \u001B[0mparallel_kwargs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m{\u001B[0m\u001B[0;34m\"prefer\"\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0;34m\"threads\"\u001B[0m\u001B[0;34m}\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 722\u001B[0;31m chunked_results = Parallel(n_jobs, **parallel_kwargs)(\n\u001B[0m\u001B[1;32m 723\u001B[0m delayed(_tree_query_parallel_helper)(\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, iterable)\u001B[0m\n\u001B[1;32m 965\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_managed_backend\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 966\u001B[0;31m \u001B[0mn_jobs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_initialize_backend\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 967\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/parallel.py\u001B[0m in \u001B[0;36m_initialize_backend\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 732\u001B[0m \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 733\u001B[0;31m n_jobs = self._backend.configure(n_jobs=self.n_jobs, parallel=self,\n\u001B[0m\u001B[1;32m 734\u001B[0m **self._backend_args)\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/_parallel_backends.py\u001B[0m in \u001B[0;36mconfigure\u001B[0;34m(self, n_jobs, parallel, prefer, require, idle_worker_timeout, **memmappingexecutor_args)\u001B[0m\n\u001B[1;32m 488\u001B[0m \u001B[0;34m\"\"\"Build a process executor and return the number of workers\"\"\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 489\u001B[0;31m \u001B[0mn_jobs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0meffective_n_jobs\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mn_jobs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 490\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mn_jobs\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;36m1\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/joblib/_parallel_backends.py\u001B[0m in \u001B[0;36meffective_n_jobs\u001B[0;34m(self, n_jobs)\u001B[0m\n\u001B[1;32m 508\u001B[0m \u001B[0;32mreturn\u001B[0m \u001B[0;36m1\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 509\u001B[0;31m \u001B[0;32melif\u001B[0m \u001B[0mmp\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcurrent_process\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdaemon\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 510\u001B[0m \u001B[0;31m# Daemonic processes cannot have children\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/multiprocessing/process.py\u001B[0m in \u001B[0;36mcurrent_process\u001B[0;34m()\u001B[0m\n\u001B[1;32m 40\u001B[0m '''\n\u001B[0;32m---> 41\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0m_current_process\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 42\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;31mKeyboardInterrupt\u001B[0m: ", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001B[0;31mAttributeError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/interactiveshell.py\u001B[0m in \u001B[0;36mshowtraceback\u001B[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001B[0m\n\u001B[1;32m 2060\u001B[0m \u001B[0;31m# in the engines. This should return a list of strings.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 2061\u001B[0;31m \u001B[0mstb\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mvalue\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_render_traceback_\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 2062\u001B[0m \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;31mAttributeError\u001B[0m: 'KeyboardInterrupt' object has no attribute '_render_traceback_'", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001B[0;31mTypeError\u001B[0m Traceback (most recent call last)", + " \u001B[0;31m[... skipping hidden 1 frame]\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/interactiveshell.py\u001B[0m in \u001B[0;36mshowtraceback\u001B[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001B[0m\n\u001B[1;32m 2061\u001B[0m \u001B[0mstb\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mvalue\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_render_traceback_\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 2062\u001B[0m \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 2063\u001B[0;31m stb = self.InteractiveTB.structured_traceback(etype,\n\u001B[0m\u001B[1;32m 2064\u001B[0m value, tb, tb_offset=tb_offset)\n\u001B[1;32m 2065\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/ultratb.py\u001B[0m in \u001B[0;36mstructured_traceback\u001B[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001B[0m\n\u001B[1;32m 1365\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 1366\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtb\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mtb\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1367\u001B[0;31m return FormattedTB.structured_traceback(\n\u001B[0m\u001B[1;32m 1368\u001B[0m self, etype, value, tb, tb_offset, number_of_lines_of_context)\n\u001B[1;32m 1369\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/ultratb.py\u001B[0m in \u001B[0;36mstructured_traceback\u001B[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001B[0m\n\u001B[1;32m 1265\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mmode\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mverbose_modes\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 1266\u001B[0m \u001B[0;31m# Verbose modes need a full traceback\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1267\u001B[0;31m return VerboseTB.structured_traceback(\n\u001B[0m\u001B[1;32m 1268\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0metype\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mvalue\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtb\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtb_offset\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mnumber_of_lines_of_context\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 1269\u001B[0m )\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/ultratb.py\u001B[0m in \u001B[0;36mstructured_traceback\u001B[0;34m(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)\u001B[0m\n\u001B[1;32m 1122\u001B[0m \u001B[0;34m\"\"\"Return a nice text document describing the traceback.\"\"\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 1123\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1124\u001B[0;31m formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,\n\u001B[0m\u001B[1;32m 1125\u001B[0m tb_offset)\n\u001B[1;32m 1126\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/ultratb.py\u001B[0m in \u001B[0;36mformat_exception_as_a_whole\u001B[0;34m(self, etype, evalue, etb, number_of_lines_of_context, tb_offset)\u001B[0m\n\u001B[1;32m 1080\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 1081\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1082\u001B[0;31m \u001B[0mlast_unique\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mrecursion_repeat\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mfind_recursion\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0morig_etype\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mevalue\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mrecords\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 1083\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 1084\u001B[0m \u001B[0mframes\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mformat_records\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mrecords\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mlast_unique\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mrecursion_repeat\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/anaconda3/envs/prior-fitting/lib/python3.9/site-packages/IPython/core/ultratb.py\u001B[0m in \u001B[0;36mfind_recursion\u001B[0;34m(etype, value, records)\u001B[0m\n\u001B[1;32m 380\u001B[0m \u001B[0;31m# first frame (from in to out) that looks different.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 381\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0mis_recursion_error\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0metype\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mvalue\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mrecords\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 382\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mrecords\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 383\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 384\u001B[0m \u001B[0;31m# Select filename, lineno, func_name to track frames with\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;31mTypeError\u001B[0m: object of type 'NoneType' has no len()" + ] + } + ], + "source": [ + "# RUN ALL METHODS, SPLITS AND DATASETS\n", + "test_datasets = get_datasets('test',task_type, suite=suite)\n", + "\n", + "overwrite=True\n", + "jobs = [\n", + " eval_method(task_type, m, did, selector, eval_positions, max_time, metric_used, split_number)\n", + " for did in range(0, len(test_datasets))\n", + " for selector in ['test']\n", + " for m in baseline_methods\n", + " for max_time in max_times\n", + " for split_number in [1, 2, 3, 4, 5]\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Comparison" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "methods = baseline_methods + ['transformer']" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_1_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_15_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_30_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_60_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_300_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_900_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_logistic_time_3600_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_1_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_15_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_30_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_60_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_300_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_900_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_gp_time_3600_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_1_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_15_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_30_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_60_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_300_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_900_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_knn_time_3600_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_1_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_15_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_30_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_60_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_300_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_900_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_catboost_time_3600_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_1_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_15_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_30_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_60_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_300_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_900_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_xgb_time_3600_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_1_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_15_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_30_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_60_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_300_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_900_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autosklearn2_time_3600_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_0.5_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_1_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_15_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_30_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_60_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_300_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_900_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_balance-scale_312_624_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-fourier_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_breast-w_349_698_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-karhunen_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-morphological_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-zernike_1000_2000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_cmc_736_1472_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_credit-approval_345_690_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_credit-g_500_1000_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_diabetes_384_768_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_tic-tac-toe_479_958_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_vehicle_423_846_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_eucalyptus_368_736_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_analcatdata_authorship_420_840_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_analcatdata_dmft_398_796_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc4_729_1458_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc3_781_1562_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_kc2_261_522_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc1_554_1108_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_banknote-authentication_686_1372_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_blood-transfusion-service-center_374_748_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_ilpd_291_582_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_qsar-biodeg_527_1054_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_wdbc_284_568_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_cylinder-bands_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_dresses-sales_250_500_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_MiceProtein_540_1080_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_car_864_1728_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_steel-plates-fault_970_1940_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_climate-model-simulation-crashes_270_540_1.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_balance-scale_312_624_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-fourier_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_breast-w_349_698_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-karhunen_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-morphological_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-zernike_1000_2000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_cmc_736_1472_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_credit-approval_345_690_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_credit-g_500_1000_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_diabetes_384_768_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_tic-tac-toe_479_958_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_vehicle_423_846_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_eucalyptus_368_736_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_analcatdata_authorship_420_840_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_analcatdata_dmft_398_796_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc4_729_1458_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc3_781_1562_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_kc2_261_522_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc1_554_1108_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_banknote-authentication_686_1372_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_blood-transfusion-service-center_374_748_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_ilpd_291_582_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_qsar-biodeg_527_1054_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_wdbc_284_568_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_cylinder-bands_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_dresses-sales_250_500_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_MiceProtein_540_1080_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_car_864_1728_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_steel-plates-fault_970_1940_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_climate-model-simulation-crashes_270_540_2.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_balance-scale_312_624_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-fourier_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_breast-w_349_698_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-karhunen_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-morphological_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-zernike_1000_2000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_cmc_736_1472_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_credit-approval_345_690_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_credit-g_500_1000_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_diabetes_384_768_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_tic-tac-toe_479_958_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_vehicle_423_846_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_eucalyptus_368_736_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_analcatdata_authorship_420_840_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_analcatdata_dmft_398_796_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc4_729_1458_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc3_781_1562_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_kc2_261_522_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc1_554_1108_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_banknote-authentication_686_1372_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_blood-transfusion-service-center_374_748_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_ilpd_291_582_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_qsar-biodeg_527_1054_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_wdbc_284_568_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_cylinder-bands_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_dresses-sales_250_500_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_MiceProtein_540_1080_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_car_864_1728_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_steel-plates-fault_970_1940_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_climate-model-simulation-crashes_270_540_3.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_balance-scale_312_624_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-fourier_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_breast-w_349_698_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-karhunen_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-morphological_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-zernike_1000_2000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_cmc_736_1472_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_credit-approval_345_690_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_credit-g_500_1000_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_diabetes_384_768_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_tic-tac-toe_479_958_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_vehicle_423_846_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_eucalyptus_368_736_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_analcatdata_authorship_420_840_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_analcatdata_dmft_398_796_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc4_729_1458_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc3_781_1562_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_kc2_261_522_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc1_554_1108_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_banknote-authentication_686_1372_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_blood-transfusion-service-center_374_748_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_ilpd_291_582_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_qsar-biodeg_527_1054_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_wdbc_284_568_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_cylinder-bands_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_dresses-sales_250_500_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_MiceProtein_540_1080_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_car_864_1728_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_steel-plates-fault_970_1940_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_climate-model-simulation-crashes_270_540_4.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_balance-scale_312_624_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-fourier_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_breast-w_349_698_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-karhunen_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-morphological_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_mfeat-zernike_1000_2000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_cmc_736_1472_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_credit-approval_345_690_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_credit-g_500_1000_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_diabetes_384_768_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_tic-tac-toe_479_958_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_vehicle_423_846_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_eucalyptus_368_736_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_analcatdata_authorship_420_840_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_analcatdata_dmft_398_796_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc4_729_1458_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc3_781_1562_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_kc2_261_522_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_pc1_554_1108_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_banknote-authentication_686_1372_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_blood-transfusion-service-center_374_748_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_ilpd_291_582_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_qsar-biodeg_527_1054_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_wdbc_284_568_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_cylinder-bands_270_540_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_dresses-sales_250_500_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_MiceProtein_540_1080_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_car_864_1728_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_steel-plates-fault_970_1940_5.npy\n", + "Could not load saved result for ./results/tabular/multiclass/results_autogluon_time_3600_roc_auc_climate-model-simulation-crashes_270_540_5.npy\n" + ] + } + ], + "source": [ + "pos = str(eval_positions[0])\n", + "\n", + "global_results = {}\n", + "overwrite=False\n", + "\n", + "for method in baseline_methods:\n", + " for max_time in max_times:\n", + " for split_number in range(1,5+1):\n", + " global_results[method+'_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_split_'+str(split_number)] = eval_method(task_type, method, None, selector, \n", + " eval_positions, fetch_only=True, \n", + " verbose=False, max_time=max_time,\n", + " metric_used=metric_used, split_number=split_number)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "path_ = 'prior_tuning_result.pkl'\n", + "\n", + "try:\n", + " output = open(path_, 'rb')\n", + " _, metrics, _, _, _, _ = CustomUnpickler(output).load()\n", + "except:\n", + " output = open(path_, 'rb')\n", + " _, metrics, _, _, _ = CustomUnpickler(output).load()\n", + "if isinstance(metrics, list):\n", + " for i in range(1, len(metrics[1])+1):\n", + " global_results['transformer_split_'+str(i)] = metrics[2][i-1]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify integrity of results\n", + "for bl in set(global_results.keys()):\n", + " if 'split_1' in bl:\n", + " for ds in test_datasets:\n", + " if f'{ds[0]}_ys_at_1000' not in global_results[bl]:\n", + " continue\n", + " match = (global_results[bl][f'{ds[0]}_ys_at_1000'] == global_results['transformer_split_1'][f'{ds[0]}_ys_at_1000']).float().mean()\n", + " if not match:\n", + " raise Exception(\"Not the same labels used\")\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "limit_to = ''\n", + "calculate_score(tabular_metrics.auc_metric, 'roc', global_results, test_datasets, eval_positions + [-1], limit_to=limit_to)\n", + "calculate_score(tabular_metrics.cross_entropy, 'cross_entropy', global_results, test_datasets, eval_positions + [-1], limit_to=limit_to)\n", + "calculate_score(tabular_metrics.accuracy_metric, 'acc', global_results, test_datasets, eval_positions + [-1])\n", + "calculate_score(tabular_metrics.time_metric, 'time', global_results, test_datasets, eval_positions + [-1], aggregator='sum', limit_to=limit_to)\n", + "calculate_score(tabular_metrics.time_metric, 'time', global_results, test_datasets, eval_positions + [-1], aggregator='mean', limit_to=limit_to)\n", + "calculate_score(tabular_metrics.count_metric, 'count', global_results, test_datasets, eval_positions + [-1], aggregator='sum', limit_to=limit_to)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### ROC and AUC plots from TabPFN Paper" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_ranks_and_wins_table(global_results_filtered, metric_key, max_time, split_number, time_matrix):\n", + " global_results_filtered_split = {**global_results_filtered}\n", + " global_results_filtered_split = {k: global_results_filtered_split[k] for k in global_results_filtered_split.keys() if '_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_split_'+str(split_number) in k or 'transformer_split_'+str(split_number) in k}\n", + "\n", + " matrix, matrix_stds = make_metric_matrix(global_results_filtered_split, methods, pos, metric_key, test_datasets)\n", + " for method in methods:\n", + " if time_matrix[method] > max_time * 2:\n", + " matrix[method] = np.nan\n", + " # = np.nan\n", + "\n", + " if metric_key == 'cross_entropy':\n", + " matrix = -(matrix.fillna(-100))\n", + " else:\n", + " matrix = matrix.fillna(-1)\n", + " return make_ranks_and_wins_table(matrix.copy())" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "df_ = []\n", + "metric_keys = ['roc', 'cross_entropy', 'time']\n", + "\n", + "for max_time in max_times:\n", + " global_results_filtered = {**global_results}\n", + " global_results_filtered = {k: global_results_filtered[k] for k in global_results_filtered.keys() if '_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_' in k or 'transformer' in k}\n", + " \n", + " time_matrix, _ = make_metric_matrix(global_results_filtered, methods, pos, 'time', test_datasets)\n", + " time_matrix = time_matrix.mean()\n", + " \n", + " if len(global_results_filtered) == 0:\n", + " continue\n", + " \n", + " # Calculate ranks and wins per split\n", + " for metric_key in metric_keys:\n", + " for split_number in range(1,6):\n", + " ranks, wins = generate_ranks_and_wins_table(global_results_filtered, metric_key, max_time, split_number, time_matrix)\n", + "\n", + " for method in methods:\n", + " method_ = method+'_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='') if method != 'transformer' else method\n", + " global_results[method_+'_split_'+str(split_number)]['mean_rank_'+metric_key+f'_at_{pos}'] = ranks[method]\n", + " global_results[method_+'_split_'+str(split_number)]['mean_wins_'+metric_key+f'_at_{pos}'] = wins[method]\n", + " \n", + " #for method in global_results.keys():\n", + " # global_results[method]['mean_rank_'+metric_key+f'_at_{pos}'] = ranks[]\n", + " \n", + " avg_times = {}\n", + " for method_ in methods:\n", + " avg_times[method_] = []\n", + " for split_number in range(1,6):\n", + " if method_ != 'transformer':\n", + " method = method_+'_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_split_'+str(split_number)\n", + " else:\n", + " method = method_+'_split_'+str(split_number)\n", + " avg_times[method_] += [global_results[method][f'mean_time_at_{pos}']]\n", + " avg_times = pd.DataFrame(avg_times).mean()\n", + " \n", + " for metric_key in metric_keys:\n", + " for ranking in ['', 'rank_', 'wins_']:\n", + " for method_ in methods:\n", + " for split_number in range(1,6):\n", + " method = method_\n", + " if method_ != 'transformer':\n", + " method = method_+'_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_split_'+str(split_number)\n", + " else:\n", + " method = method_+'_split_'+str(split_number)\n", + "\n", + " if global_results[method][f'sum_count_at_{pos}'] <= 29:\n", + " print('Warning not all datasets generated for '+method+' '+ str(global_results[method][f'sum_count_at_{pos}']))\n", + " \n", + " time = global_results[method]['mean_time'] if ranking == '' else max_time\n", + " time = max_time # Todo: This is not the real time\n", + " df_ += [{'metric'+ranking+metric_key: global_results[method]['mean_'+ranking+metric_key+f'_at_{pos}'], 'real_time': avg_times[method_], 'time': time, 'method': method_, 'split_number': split_number}]\n", + " # For Roc AUC Plots\n", + " #if 'transformer' in method:\n", + " # df_ += [{'metric'+ranking+metric_key: global_results[method]['mean_'+ranking+metric_key+f'_at_{pos}'], 'real_time': avg_times[method_], 'time': time, 'method': method_, 'split_number': split_number}]\n", + " # df_ += [{'metric'+ranking+metric_key: global_results[method]['mean_'+ranking+metric_key+f'_at_{pos}'], 'real_time': max(avg_times), 'time': max(max_times), 'method': method_, 'split_number': split_number}]\n", + " \n", + " \n", + "df_ = pd.DataFrame(df_)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "metric_renamer = {'roc': 'ROC AUC', 'cross_entropy': 'Cross entropy'\n", + " , 'rank_roc': 'Mean ROC AUC Rank', 'rank_cross_entropy': 'Mean Cross entropy Rank'\n", + " , 'wins_roc': 'Mean ROC AUC Wins', 'wins_cross_entropy': 'Mean Cross entropy Wins'\n", + " , 'time': 'actual time taken'}\n", + "max_times_renamer = {0.5: \"0.5s\", 1: \"1s\", 5: \"5s\", 15: \"15s\", 30: \"30s\", 60: \"1min\", 300: \"5min\", 900: \"15min\", 3600: \"1h\", 14400: \"4h\"}\n", + "\n", + "def make_tabular_results_plot(metric_key, exclude, max_times, df_, grouping=True):\n", + " f, ax = plt.subplots(figsize=(7, 7))\n", + " #ax.set(xscale=\"log\")\n", + " \n", + " df_.loc[:, 'time_log'] = np.log10(df_.time)\n", + " df_.loc[:, 'real_time_log'] = np.log10(df_.real_time)\n", + " time_column = 'time_log' if grouping else 'real_time_log'\n", + "\n", + " sns.set_palette(\"tab10\")\n", + " for method in methods:\n", + " if method in exclude or method=='transformer':\n", + " continue\n", + " df_method = df_[df_.method==method].copy()\n", + " ax = sns.lineplot(time_column, 'metric'+metric_key, data=df_method, marker='o', label=method, ax=ax)\n", + " #sns.scatterplot(data=df_, x='time', y='metric', hue='method', ax=ax, style='method') #\n", + " df_trans = df_[df_.method=='transformer']\n", + " if time_column == 'real_time_log':\n", + " # Removing dots for line for transformers\n", + " df_trans = df_trans[np.logical_or(df_trans.real_time == df_trans.real_time.min(), df_trans.real_time == df_trans.real_time.max())]\n", + " df_trans.loc[:, 'metric'+metric_key] = df_trans['metric'+metric_key].mean()\n", + " df_trans.loc[:, time_column] = np.log(1) # Hacky code to get the right time from our measurements\n", + " ax = sns.lineplot(time_column, 'metric'+metric_key, data=df_trans, linestyle='--', marker='o', ci=\"sd\", ax=ax)\n", + " \n", + " #ax = sns.scatterplot(data = df_trans, x=time_column, y='metric'+metric_key, s=800, marker='*', color='grey') #\n", + " #ax = plt.scatter(df_trans[time_column], df_trans['metric'+metric_key], s=600, marker=['*']) #\n", + " \n", + " if grouping:\n", + " ax.set_xlabel(\"Time (s, requested, not actual)\")\n", + " else:\n", + " ax.set_xlabel(\"Time taken\")\n", + " ax.set_ylabel(metric_renamer[metric_key])\n", + "\n", + " #ax.legend()\n", + " \n", + " times = np.log10(max_times)\n", + " ax.set_xticks(times)\n", + " ax.set_xticklabels([max_times_renamer[t] for t in max_times])\n", + " \n", + " #ax.legend([],[], frameon=False)\n", + " \n", + " return ax" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "df_absolute = df_.copy()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "df_absolute = df_.copy()\n", + "df_absolute = df_absolute[np.logical_or(df_.method != 'autogluon', df_.time >= 30)] # Autogluon did not yield any useful results before 30s\n", + "\n", + "knn_extend = df_absolute[np.logical_and(df_absolute.method=='knn', df_absolute.time == 3600)].copy()\n", + "knn_extend['real_time'] = 14400\n", + "knn_extend['time'] = 14400\n", + "df_absolute = df_absolute.append(knn_extend, ignore_index=True).reindex()\n", + "\n", + "knn_extend = df_absolute[np.logical_and(df_absolute.method=='logistic', df_absolute.time == 3600)].copy()\n", + "knn_extend['real_time'] = 14400\n", + "knn_extend['time'] = 14400\n", + "\n", + "df_absolute = df_absolute.append(knn_extend, ignore_index=True).reindex()" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcYAAAGtCAYAAACMUdSwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAafklEQVR4nO3df5TddX3n8ecrCWk0/JAfU08hIFHTSgQLMmJtq0uhUIgtUH+VVFhxOWBPF9i61C2eRctie9qtba1UbItbxEOpLHXtaY6/oqdGaZVjM1lIMFA0YoEk7BoNbMUKAfLeP+538MM4GTKQ79wh83ycMyf3fu/33vvOzZx55nvne7/fVBWSJGlg3rAHkCRpNjGMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNXoNY5LTktyVZFOSyya5/QVJ/j7JhiRfSLKkue0tSb7efb2lzzklSRqXvj7HmGQ+8DXgFGAzsBZYWVV3NOv8DfCJqvpIkpOAt1bVuUkOAsaAUaCAdcDxVfVAL8NKktTpc4vxBGBTVd1dVTuAG4EzJ6yzHPh8d3lNc/svAJ+rqu1dDD8HnNbjrJIkAbCgx8c+DLivub4ZeOWEddYDrwPeD/wysF+Sg3dx38MmPkGSC4ELARYvXnz8S17ykj02vCTp2W/dunXfrqqR6dynzzDujt8EPpDkPOBmYAvw+O7euaquAa4BGB0drbGxsT5mlCQ9SyW5Z7r36TOMW4DDm+tLumVPqKqtDLYYSbIv8PqqejDJFuDECff9Qo+zSpIE9Ps7xrXAsiRLkywEzgZWtSskOSTJ+AzvBK7tLq8GTk1yYJIDgVO7ZZIk9aq3MFbVY8BFDIJ2J3BTVW1McmWSM7rVTgTuSvI14PnA73b33Q68h0Fc1wJXdsskSepVbx/XmGn+jlGSNFGSdVU1Op37eOQbSZIahlGSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKlhGCVJahhGSZIahlGSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKlhGCVJahhGSZIahlGSpIZhlCSpsWDYA8wGO2sn2x/ezo7Hd7Bw/kIOWnQQ8+L/GSRpLprzYdxZO/n6A1/nks9fwtbvbeXQxYdy1UlXsezAZcZRkuagOf+Tf/vD25+IIsDW723lks9fwvaHtw95MknSMMz5MO54fMcTURy39Xtb2fH4jiFNJEkapjkfxoXzF3Lo4kOftOzQxYeycP7CIU0kSRqmOR/GgxYdxFUnXfVEHMd/x3jQooOGPJkkaRjm/M438zKPZQcu44bX3uBeqZIkwwiDOB7ynEOGPYYkaRZws0iSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKlhGCVJahhGSZIahlGSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhq9hjHJaUnuSrIpyWWT3H5EkjVJbk2yIcmKbvnCJB9OcnuS9UlO7HNOSZLG9RbGJPOBq4HTgeXAyiTLJ6x2OXBTVR0HnA18sFt+AUBVHQOcAvxRErduJUm96zM2JwCbquruqtoB3AicOWGdAvbvLh8AbO0uLwc+D1BV3wIeBEZ7nFWSJKDfMB4G3Ndc39wta10BnJNkM/Ap4OJu+XrgjCQLkiwFjgcOn/gESS5MMpZkbNu2bXt6fknSHDTstydXAtdV1RJgBXB995bptQxCOgb8CfBl4PGJd66qa6pqtKpGR0ZGZm5qSdJea0GPj72FJ2/lLemWtc4HTgOoqluSLAIO6d4+ffv4Skm+DHytx1klSQL63WJcCyxLsjTJQgY716yasM69wMkASY4CFgHbkjw3yeJu+SnAY1V1R4+zSpIE9LjFWFWPJbkIWA3MB66tqo1JrgTGqmoVcCnwoSRvZ7AjznlVVUl+FFidZCeDrcxz+5pTkqRWqmrYM+wRo6OjNTY2NuwxJEmzSJJ1VTWtTzUMe+cbSZJmFcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSo9cwJjktyV1JNiW5bJLbj0iyJsmtSTYkWdEt3yfJR5LcnuTOJO/sc05Jksb1FsYk84GrgdOB5cDKJMsnrHY5cFNVHQecDXywW/5G4Eeq6hjgeOBtSY7sa1ZJksb1ucV4ArCpqu6uqh3AjcCZE9YpYP/u8gHA1mb54iQLgOcAO4B/7XFWSZKAfsN4GHBfc31zt6x1BXBOks3Ap4CLu+UfA74H3A/cC/xhVW2f+ARJLkwylmRs27Zte3h8SdJcNOydb1YC11XVEmAFcH2SeQy2Nh8HDgWWApcmeeHEO1fVNVU1WlWjIyMjMzm3JGkv1WcYtwCHN9eXdMta5wM3AVTVLcAi4BDgV4HPVNWjVfUt4EvAaI+zSpIE9BvGtcCyJEuTLGSwc82qCevcC5wMkOQoBmHc1i0/qVu+GPgp4J97nFWSJKDHMFbVY8BFwGrgTgZ7n25McmWSM7rVLgUuSLIe+ChwXlUVg71Z902ykUFgP1xVG/qaVZKkcRl06NlvdHS0xsbGhj2GJGkWSbKuqqb1q7hh73wjSdKsYhglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKlhGCVJahhGSZIahlGSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKlhGCVJahhGSZIahlGSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKlhGCVJahhGSZIahlGSpMYuw5hkJMnySZYvTzLS71iSJA3HVFuMfwocMsnyg4H39zOOJEnDNVUYX1xVN09cWFX/ALysv5EkSRqeqcK43xS37bOnB5EkaTaYKoybkqyYuDDJ6cDd/Y0kSdLwLJjitt8APpnkTcC6btko8CrgF3ueS5KkodjlFmNVfR04BvgicGT39UXgZVX1tZkYTpKkmTbVFiNV9Qjw4RmaRZKkodtlGJN8F6hmUQHfBtYAv1VV3+l5NkmSZtxUb6XuV1X7N18HMPgd40bgz2dsQkmSZtC0DglXVQ9U1fuAF/U0jyRJQzXtY6Um2Yen+N2kJEnPVlP9jvF1kyw+EPgV4GO9TSRJ0hBNteX3SxOuF/Ad4P1V9cn+RpIkaXh2Gcaqeuuubkvyiqpa289IkiQNz27/rrA7BdXK7utBBnuoSpK0V5kyjEmO5AcxfBR4ATBaVf/S+2SSJA3BVCcqvgX4JIN4vr6qjge+axQlSXuzqT6u8X8ZnHrq+cBIt6x2vfoPS3JakruSbEpy2SS3H5FkTZJbk2wYP5tHkjcnua352pnk2Ok8tyRJT8dUR745i8FBxNcBVyT5JnBgkhN254GTzAeuBk4HlgMru99Tti4Hbqqq44CzgQ92z31DVR1bVccC5wLfrKrbpvH3kiTpaZnyA/5V9f+q6sNVdSrwSuBdwPuS3Lcbj30CsKmq7q6qHcCNwJkTnwLYv7t8ALB1ksdZ2d1XkqTe7fZeqVX1LeADwAeSvGA37nIY0AZ0M4O4tq4APpvkYmAx8POTPM6v8MNBlSSpF9M+JBxAVd2zh55/JXBdVS0BVgDXJ3lipiSvBP6tqr462Z2TXJhkLMnYtm3b9tBIkqS57GmFcTdtAQ5vri/plrXOB24CqKpbgEXAIc3tZwMf3dUTVNU1VTVaVaMjIyO7Wk2SpN3WZxjXAsuSLE2ykEHkVk1Y517gZIAkRzEI47bu+jzgTfj7RUnSDJrqc4zvTfK2SZa/LcnvP9UDV9VjwEXAauBOBnufbkxyZZIzutUuBS5Isp7BluF5VTX+kZDXAPdV1d3T+ytJkvT05QcdmnBDso7BUW5qwvJ5wIaqOnoG5ttto6OjNTY2NuwxJEmzSJJ1VTWtQ5hO9Vbqj0yMIkBV7QQy3eEkSXo2mCqM30+ybOLCbtn3+xtJkqThmepzjO8GPp3kdxgc/QYGZ9R4J/AbPc8lSdJQTHU+xk8nOQt4B3Bxt3gjgwOK3z4Ds0mSNOOmPPJN98H6tyTZt7v+0IxMJUnSkEz5OcYkv57kXuAe4J4k9yT59ZkZTZKkmTfV5xgvB34ROLGqDq6qg4GfA07vbpMkaa8z1RbjucDr2g/Yd5ffBPz7vgeTJGkYpgpjVdXDkyz8PrCzv5EkSRqeqcK4JcnJExcmOQm4v7+RJEkanqn2Sr0E+Lsk/8iTP8f4M3h+REnSXmqXW4xVtRE4GrgZOLL7uhk4urtNkqS9zlN9jvFh4Np2WZJ5Sd5cVTf0OpkkSUMw1cc19k/yziQfSHJKBi4CxvdMlSRprzPVFuP1wAPALcAFwH9lcFaNs6rqtv5HkyRp5k0VxhdW1TEASf4Hgz1Rj5jsIxySJO0tpvq4xqPjF6rqcWCzUZQk7e2m2mL8yST/2l0O8Jzuehh8+H//3qeTJGmGTXXaqfkzOYgkSbPBlGfXkCRprjGMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVKj1zAmOS3JXUk2JblsktuPSLImya1JNiRZ0dz2siS3JNmY5PYki/qcVZIkgAV9PXCS+cDVwCnAZmBtklVVdUez2uXATVX1Z0mWA58CjkyyAPgr4NyqWp/kYODRvmaVJGlcn1uMJwCbquruqtoB3AicOWGdAvbvLh8AbO0unwpsqKr1AFX1nap6vMdZJUkC+g3jYcB9zfXN3bLWFcA5STYz2Fq8uFv+40AlWZ3kfyf5L5M9QZILk4wlGdu2bduenV6SNCcNe+eblcB1VbUEWAFcn2Qeg7d4fxZ4c/fnLyc5eeKdq+qaqhqtqtGRkZGZnFuStJfqM4xbgMOb60u6Za3zgZsAquoWYBFwCIOty5ur6ttV9W8MtiZf3uOskiQB/YZxLbAsydIkC4GzgVUT1rkXOBkgyVEMwrgNWA0ck+S53Y44/w64A0mSetbbXqlV9ViSixhEbj5wbVVtTHIlMFZVq4BLgQ8leTuDHXHOq6oCHkjyxwziWsCnquqTfc0qSdK4DDr07Dc6OlpjY2PDHkOSNIskWVdVo9O5z7B3vpEkaVYxjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDUMoyRJDcMoSVLDMEqS1DCMkiQ1DKMkSQ3DKElSwzBKktQwjJIkNQyjJEkNwyhJUsMwSpLUMIySJDV6DWOS05LclWRTkssmuf2IJGuS3JpkQ5IV3fIjk3w/yW3d15/3OackSeMW9PXASeYDVwOnAJuBtUlWVdUdzWqXAzdV1Z8lWQ58Cjiyu+0bVXVsX/NJkjSZPrcYTwA2VdXdVbUDuBE4c8I6BezfXT4A2NrjPJIkPaU+w3gYcF9zfXO3rHUFcE6SzQy2Fi9ublvavcX6xSSvnuwJklyYZCzJ2LZt2/bg6JKkuWrYO9+sBK6rqiXACuD6JPOA+4Ejquo44D8Df51k/4l3rqprqmq0qkZHRkZmdHBJ0t6pzzBuAQ5vri/plrXOB24CqKpbgEXAIVX1SFV9p1u+DvgG8OM9zipJEtBvGNcCy5IsTbIQOBtYNWGde4GTAZIcxSCM25KMdDvvkOSFwDLg7h5nlSQJ6HGv1Kp6LMlFwGpgPnBtVW1MciUwVlWrgEuBDyV5O4Mdcc6rqkryGuDKJI8CO4Ffq6rtfc0qSdK4VNWwZ9gjRkdHa2xsbNhjSJJmkSTrqmp0OvcZ9s43kiTNKoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKlhGCVJahhGSZIahlGSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKlhGCVJahhGSZIahlGSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKlhGCVJahhGSZIahlGSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKlhGCVJahhGSZIahlGSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZQkqWEYJUlqGEZJkhqGUZKkhmGUJKnRaxiTnJbkriSbklw2ye1HJFmT5NYkG5KsmOT2h5L8Zp9zSpI0rrcwJpkPXA2cDiwHViZZPmG1y4Gbquo44GzggxNu/2Pg033NKEnSRH1uMZ4AbKqqu6tqB3AjcOaEdQrYv7t8ALB1/IYkZwHfBDb2OKMkSU+yoMfHPgy4r7m+GXjlhHWuAD6b5GJgMfDzAEn2BX4LOAXY5duoSS4ELuyuPpTkrmc48yHAt5/hY+ztfI32DF/H2c9/o73DT0z3Dn2GcXesBK6rqj9K8irg+iRHMwjm+6rqoSS7vHNVXQNcs6eGSTJWVaN76vH2Rr5Ge4av4+znv9HeIcnYdO/TZxi3AIc315d0y1rnA6cBVNUtSRYx+F/aK4E3JPkD4HnAziQPV9UHepxXkqRew7gWWJZkKYMgng386oR17gVOBq5LchSwCNhWVa8eXyHJFcBDRlGSNBN62/mmqh4DLgJWA3cy2Pt0Y5Irk5zRrXYpcEGS9cBHgfOqqvqaaTfssbdl92K+RnuGr+Ps57/R3mHa/44ZbockSZpdPPKNJEkNwyhJUmPOhjHJtUm+leSrw55lNkvyL0luT3Lb09ntea5KsijJPyVZn2Rjkv/WLV+a5CvdYRL/Z5KFw551rnkm39NJvtzXXHrmJvu5nuQLSab1sZs5G0bgOrqPiugp/VxVHetnuqblEeCkqvpJ4FjgtCQ/Bfx3Bp/RfTHwAIOPLGnmPa3v6ar66b4G0h5xHXvg5/qcDWNV3Qxsb5cluSTJHd0BzW8c0miznq/TU6uBh7qr+3RfBZwEfKxb/hHgLIAkb0zy1W4L8+aZnneu67Yq3pdkLMmdSV6R5ONJvp7kd5r1Hur+PLG7z8eS/HOSGzLV0Ug0Iyb7ud55Y/cOzteSvHqS259k2Ee+mW0uA5ZW1SNJnjfsYWaJYnDYvgL+ojvakK/TbugOpL8OeDGDA+p/A3iw+ygTDA6TeFh3+d3AL1TVFl/T3k32PQ2wo6pGk/wn4O+A4xn8kP1GkvdV1XcmPM5xwEsZHOP5S8DPAP84I38DTdeCqjqhO4PTb9MdfnRX5uwW4y5sAG5Icg7w2FOtPEf8bFW9nMFZUv5jktfg67RbqurxqjqWwVGfTgBeMsXqX2JwoIsLgPkzMN5cNtn3NMCq7s/bgY1VdX9VPQLczZOP4jXun6pqc1XtBG4Djux3bD0DH+/+XMdu/DsZxid7LYP/2b8cWJtkzm9RV9WW7s9vAX/L4Ae8r9M0VNWDwBrgVcDzmtfricMkVtWvMTgN2+HAuiQHD2HUOWEX39Mw+L0wwM7m8vj1yb7H23Ue38U6mh3G/61269/JMHaSzAMOr6o1DM7scQCw73CnGq4ki5PsN34ZOBW4A1+np5RkZPwt0STPYXCmmDsZBPIN3WpvYfCWHUleVFVfqap3A9uYfAtFz9AuvqfdM11PMmf/h5Pko8CJwCFJNgPvAc5NcgAQ4Kruf/pz2fOBv+32KVgA/DXwOWCNr9NT+jHgI93vGecxOCTiJ5LcAdzY7dBxK/CX3frvTbKMwWv698D6YQw9B/zQ93RVfSbJZcMdS3vCJD/Xf/tpPY6HhJMk6Qd8K1WSpIZhlCSpYRglSWoYRkmSGoZRkqSGYZRmQJKDu7M53Jbk/yTZ0l1+KMkHe3i+s5Is3431rkvyhqdaT5pL5uznGKWZ1B1n81iAJFcAD1XVH/b4lGcBn2BwQAZJ0+AWozRE3VkaPtFdviLJR5L8Q5J7krwuyR905w78TJJ9uvWOT/LFJOuSrE7yYxMe86eBMxgcNOC2JC9KckGStd3ZO/5XkudOMst7ui3I+Une0a2/IT84l+SR3ZknPpTBOSY/2x3VR9qrGEZpdnkRg1NTnQH8FbCmqo4Bvg+8tovjnwJvqKrjgWuB320foKq+zOCA2O/ozjn4DeDjVfWK7vyQdzLhPJBJ3guMAG8FTgaWMTiG6LHA8c2BtpcBV1fVS4EHgdfv2b++NHy+lSrNLp+uqkeT3M7gLBuf6ZbfzuCsAD8BHA18rjus2Xzg/t143KO7w9A9j8GxbVc3t70L+EpVXQiQ5FQGxxC9tbt9XwZBvBf4ZlXd1i3frTMVSM82hlGaXR4BqKqdSR6tHxyzcfwMD2FwSqRXTfNxrwPOqqr1Sc5jcDzJcWsZbBUeVFXbu+f4var6i/YBkhzJD59RwrdStdfxrVTp2eUuYCTJqwCS7JPkpZOs911gv+b6fsD93Vuxb56w7meA3wc+2Z15YjXwH5Ls2z3HYUl+dA//PaRZyy1G6VmkqnZ0H6+4qjvDyQLgT4CNE1a9EfhQkksYnObqXcBXGJzS6is8OZpU1d90UVwFrGBwJpVburdrHwLOYbCFKO31PLuGJEkN30qVJKlhGCVJahhGSZIahlGSpIZhlCSpYRglSWoYRkmSGv8fwrydOVKXkIsAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "exclude=['']\n", + "#ax = make_tabular_results_plot('time', exclude=exclude)\n", + "ax = make_tabular_results_plot('roc', df_=df_absolute, exclude=exclude, grouping=False, max_times=[1, 5, 30, 60*5, 60*60])\n", + "ax.set_ylim([0.84, 0.9])\n", + "ax.set_xlim([np.log10(0.7), np.log10(3600)])\n", + "ax.legend([],[], frameon=False)\n", + "\n", + "#tikzplotlib.save(f'roc_over_time.tex', axis_height='5cm', axis_width='6cm', strict=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 134, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbYAAAGpCAYAAADhiRM+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOz9d3xc53mnjV/PadMreiEKe1EXScmqlm1ZzZZsy5Yd23HsjdebbPZN2eTNJrvJ5k3Pbsom/mWdrLN2ii05lkskd8tdxRIlUaJESeyd6G0wfU57fn+cmSFAgCAAAiRInuvzGQIzc86ZMyBwvnPfz/e+byGlxMfHx8fH51JBudAn4OPj4+Pjs5T4wubj4+Pjc0nhC5uPj4+PzyWFL2w+Pj4+PpcUvrD5+Pj4+FxSaBf6BKbS2Ngoe3p6LvRp+Pj4+PisEHbu3DkqpWxayD4rSth6enp48cUXL/Rp+Pj4+PisEIQQxxa6j5+K9PHx8fG5pFg2YRNCfFYIMSyEeG25XsPHx8fHx+d0ljNi+yfg7mU8vo+Pj4+PzwyWTdiklE8C48t1fB8fHx8fn9m44GtsQohPCCFeFEK8ODIycqFPx8fHx8fnIueCC5uU8tNSyq1Syq1NTQtydPr4+Pj4+Mzgggubj4+Pj4/PUuILm4+Pj4/PJcVy2v2/ADwLbBBCnBRC/PxyvZaPj4+Pj0+NZes8IqX8meU6to+Pj4+Pz5nwU5E+Pj4+PpcUvrD5+Pj4+FxS+MLm4+Pj43NJ4Qubj4+Pj88lhS9sPj4+Pj6XFCtK2FzXpVwuY1kWjuNc6NPx8fHx8bkIWVGDRl3XJZfLUSqVqFQqCCHQdR1N09B1vf69qqqoqoqiKPWvPj4+Pj4+sMKEbXh4mIcffpiHHnqIYDCIZVm4rott25imieM4SCln7KcoyhkFcKr4+QLo4+Pjc+mzooQNIJPJ8Oijj/Kud72L119/nc2bNxMIBLAsC13XZxUn13VxXRfTNCmXy7iuO6sAappWFz9N0zAMY5r41b4XQpyPt+rj4+PjswysOGEDT9yEELzwwgts2rQJgF27drFr1y7C4TDRaJRIJEI0GmXbtm2oqkq5XEZRFAKBwBmFqSaANfGbTQCFEKiqWo/8arfTxU9V1WX/Ofj4+Pj4LJwVKWzJZJJYLMbHPvYxNM07xY6ODoQQ5PN58vk8o6Oj9PX1ccMNNwCwY8cODhw4gKZpRKNRotEoiUSCm2++GYCJiQkURSESiWAYxhlfW0qJ67o4jlNPhbquO2O72vrf1BSov/7n4+Pjc+FZccKWTCZ56KGHsG0bXdfrj7e3t9Pe3j5tWyllPTrbuHEjjY2NdeHL5/OMjY3Vt33mmWfo7+8HIBQKEY1GaW5urgvfwMAAqqoSjUYJhUJnjchqAmhZFpVK5ZzW/1RV9dOfPj4+PkvEihK25uZmPvzhD1MsFikUCsCp1OBsBpCpYtDW1kZbW9sZj719+3Yymcw04bNtu/78k08+yeTkJOCJUTQapaenhxtvvBGAgwcPEggE6tGgruv185oLf/3Px8fH5/yyooRN0zTS6TSJRALHcbBtG8uyME0T0zTrkdHUi/vUlN9ckU9zczPNzc1nfO0777yTXC43TfgCgQDgRWc/+clPptXWBQIBNm/ezLZt25BS8uqrr9bX/aLRKOFwuC7E80lHLnT9zzCMuhD6638+Pj4+p1hRwgbUhWBqGnIqjuPURc9xnLrgWZZFqVSaJgZCiBkX/TORTqdJp9NnfP7973//NNHL5/OkUikAKpUKO3bsmLa9EIJt27ZxzTXXYJomL7/8cl30ajfDMOpCPB8BnLr+VygU/PU/Hx8fn1lYccJ2NmoX59kMIFLKGcJXqVTqEd/U1CPMneY8fbuaGM1GMBjkYx/72Azhq0WI+Xye3bt3zxCh2267jY0bNzI5Ockrr7wyQ/gikci06Gvq+c7F1PW/udKfqqpOS4H6638+Pj6XAhedsM2FEKJ+oa6lEadSi3ampjlr5o9KpTJDeOab5gTQdZ1UKlWP4qaSTqf5+Z//eUql0jTha2lpAaBQKHDs2DFKpdK0/e6++266uroYHBxk9+7d0wQvGo2STqfrrtHTfw41UTpT5Fv7eSx0/a9289f/fHx8ViqXlLCdjfmmOWvCV4v0TNOkVCrhuu60i/dC1rWEEITDYcLh8Iy1vvb2dn72Z38W27YpFAp14WtsbASgXC4zPj7O8ePHp63zvfe97yWdTnPw4EH27t07I+Jra2ub87wWs/5XLBb99T8fH58VzWUlbGdjrjTf1DRnTfimpjnL5fK07eeb5pyKpmkkEgkSicS0x3t6eujp6UFKSaVSqQtfPB6vb+M4Dn19fRSLxbrofPSjH0VVVV5++WUOHz48I825Zs2aeUVZy73+d7oAzhaF+vj4+MwX/woyT6amOWfj9DRnLeKrid/p0woWkuaceg7BYJBgMFiP5gDWrl3L2rVr6+dRKBQoFAr1dchwOEwkEiGXyzEwMIBpmui6zpo1awD48Y9/TH9//zThSyQSbNiwAfBEcz4R6VKt/xmGQSQSIRwO16M/Hx8fn/niXzGWiIWmOWtlDJVKZYabE5jhXpzv+pWiKMRiMWKxWP2xDRs21EUKqKdWa8dsbW2tC+LQ0BCHDh0imUzW9/nWt77F6OjoNOFrbGystzsrFosEAoF5pRnns/5n2zaTk5NMTEwgpSQQCNTTuIFAwBc6Hx+fOfGvEOeJxaQ5a8aWxbo5z4RhGNNcpRs3bmTjxo31+zVDSY1169bR0NBQT4GOjIyQy+Xqwvb444+Ty+XqfTyj0SgdHR3158fHxwmFQgSDwTMKdC0allISDAbrUS94HwpyuRyZTKa+ba1W0DCMOQ0yPj4+lx++sK0AFpLmrPWwXOo05+n7B4PB+v2pojf1nGpcf/3104rbx8bG6vtLKfnqV7+K67r1lmXRaJS1a9eyYcMGpJRkMhmSySSPPPJI/fuHHnoITdOwbXvGhwJf6Hx8fObCF7aLgJWS5jz9nGqsX7/+jNtJKXnb2942o8avFhGWSiUUReFLX/pSXahqo4s+8pGPzIhWa+c/m9BNTEzUPyTU1ugCgYAvdD4+lxm+sF0CnM20UStWP72MYa4051J1J1EUhZ6enjM+bxgG8Xi8Lmo1MpkMUkrGx8d58cUXaW1tpbW1lYaGhhnvdTahy+fz9d6fU4Xu9DSsj4/PpYcvbJcBi0lz1m5LneY807klk8lp4pZMJhFCUCwWGR8f5+jRo4AnYs3Nzdx2220kEolpEx5qnC50rutOEzpVVYlEIvURRrqu+8XlPj6XEGK2ThMXiq1bt8oXX3zxQp+GzxRc150W8U0VPdM0lyTNWVtLe/TRR2ddYwPPeTk4OMjg4CBDQ0Pcd999GIbBSy+9xOHDh+sRXWtr6xlbn019T5Zl1Y+tqmrd+OILnY/PykIIsVNKuXVB+/jC5nMuTO3LeXq3Fsuypm07V1Pqqa5IIcQ0V+RcHDx4kH379jE8PFx/vUQiwUMPPVSP+ILB4Jwp1ZrQWZaFEIJAIEBbW5u/NufjswJYjLD5qUifc+JsZQxTo735pjk1TZv32l6tON11XcbHxxkcHKRSqdQjru9+97tkMhlaWlrqEV1TU9M00VIUhUAgQCQSQdM0XNdlcnKSeDzur8f5+FyE+MLms2xMbZ81G7OlOcvlMuVyuS56NTfo2dbzFEWhsbFxWkcWgCuvvJKBgQEGBwepZQN6e3u58847ATh+/DiNjY3E43Fs255WcvC+972PxsbGWRtq+/j4rFx8YfO5YCiKcsbxQ7W0ZrlcplAoUCwWgenNlucT1U1tN1apVBgaGqq/ZrFY5Dvf+Q4AH/zgB/nWt741reTgS1/6Eh/+8IfrReM+Pj4XB76w+aw4pkZ6kUiEhoaGejcU0zQpFouUSqX6Glxte03T5ozqAoEAXV1d9fvBYJAHHniAwcFBwuHwrCUHACdPnqSjo4NQKLTk79XHx2fp8YXN56Kg1g0lGAzWpxpMjepqYlczQ9WiurON7WlpaaGlpYVgMDhryUEtqjxx4gQdHR1EIpFlfZ8+Pj7nju+K9LlkmOpuLBaLFAqFaVFdzXk5WwpztpKDd73rXfUmzI7jUCqVaG1tnTYuyMfHZ3nxXZE+lzU1d2MgEKjXstVajFUqFYrF4rR5dTUHZk3UNE3jIx/5SH2N7zvf+Q6Dg4Pcd999pNNpQqEQAwMDuK5LMpm8gO/Ux8dnLs6tX5KPzwpH0zRCoRDJZJL29nbWrFlDd3c3bW1txGIxpJT1+XWTk5Nks1lKpRKO47B161aEEHz9619ndHS03rFkaGiI8fHxC/3WfHx8zoAvbD6XFUIIDMMgGo3S1NREd3c3q1evZtWqVTQ3NxMIBCgWi9i2TSqV4p3vfCe6rvONb3yDsbExFEUhEokwMjLC6OjojM4rPj4+Fx5f2Hwue1RVJRgMkkgkaGtrY9WqVfXOKYlEgne+8510d3fX19YURSEajTI+Ps7IyMi0ET4+Pj4XHl/YfHxOIxQK0dnZWTeixGIx7rjjDnRdxzRNBgYGEEIQiUTIZDIMDw/74ubjs4Lwhc3HZxaCweA0cauxY8cOvvnNb3L06NG6uOVyOQYHB2e0CPPx8bkw+MLm43MGauJWc1YCbN++nYaGBr73ve9x6NChurgVi0UGBgZ8cfPxWQH4wubjMwfBYJCOjo56MXggEOC+++6jpaWFH/7wh+zbtw+AcDhMuVymr69vXlMJfHx8lo8VJWym7TI4Wca2/fUKn5VDLXKrNWo2DIN7772X9vZ2XnzxxXo0Fw6HsSyLkydPzhjZ4+Pjc/5YUZ1HAm3r5LZf/T986kPXEdAUwoZGPKgTMlQMbUVpsM9lSKVS4eTJk6iqimEY2LZNqVSq18PV+lSWy2WEEHR0dPhjb3x8zpHFdB5ZcWpxcqLEf3z4JRxXsm8gx64TEzxzaJQdh8c4MJRjJFehaNp+/ZDPeScQCEyL3DRNq4vaM888w0svvVSfBCCE4MSJE1QqlQt92j4+lx0rsqXWyYkSubLNLzy8kzVNUTa2xljXEqUrHSYe0kGCqgiSEYN0WCca1AkbKrq64nTa5xIjEAiwatUqTp48WU9L1lpw1dKS27dvxzAMLMuqN0/2JwP4+Jw/lk3YhBCrgH8BWgEX+LSU8m/ms29nKkQkoPHANR3sHczy7dcGefwVb92tMRpgY2uMDS0xVjdFaI0H0VSBC0QNlVTEIBk2iBgaQV2Zc4yJj89iMAyDzs5OTp48SaVSIRAIcPvtt6NpGq+88gq2bXPTTTeh6zpCCE6ePEl7e7s/GcDH5zyxnBGbDfy6lPIlIUQM2CmE+J6U8o25dupMhfjUh66jbDm897pOgrqCBI6MFtg7mGPvYJa9gzmePjgKgKEqrG32orq1zVG6GyKEDRUhvKguHTZIhQ2iIY2wrqL5UZ3PElATt76+vrq43Xzzzaiqyu7du5FScsstt6Bp3p9YX18f7e3t9ebMPj4+y8d5M48IIR4H/lZK+b0zbXP1tdfJ7z35UyK6RsG0mSiajBdMKraLABQhCOoqAU1hvGDWhW7PQI5DI3ls13svrfEgG1tjrG+NsboxQnMsgKJ4kVs0oJGOGCRCOpGARkDzozqfxWOaJn19fYCXppRSsnPnznofyhq1sTctLS0kEokLdbo+PhcdizGPnBdhE0L0AE8CV0gps6c99wngEwBdXV3XHzt2bMb+ZcuhZDpMlizGCib5ioWUIICAphLUVRxXcmgkX4/o9g7kGC+aAAR1hfXNMTZUo7qudJiQoSIQXlQX1UmHDSIBjbChoSq+0PnMn5rFv2YcmUp/fz8tLS2oqorruhSLRZqbm/2xNz4+82RFCpsQIgr8BPhjKeVX59p2voNGHVdSMG0KZZvxgslE0cR2JVJ6qcmgrqKrgpFchT219OVAjsOjeapBHR3JkBfVtXhRXUPMQKlGbrGgTkPEIB7yTClB/cxTmH18wBO3vr4+XNeti1s2m+WLX/winZ2d3HnnnWiaVhe3hoYG0um0ny3w8TkLK07YhBA68A3gu1LKvzrb9oudoC2lpGy5FEybTNFkPG9StLzWRgrV9KWuYNouB4fz7BnMsm8wx56BLNmy1yUibKhsaPGiujVNXlQX0BUEoKkKDRGDVFgnEtQJ6aof1fnMYDZx27NnD0899RTt7e3cdddd6LpenwGXSqVobGz0xc3HZw5WlLAJ76/1n4FxKeWvzmefxQrbbJi2S8l0yJYtxgsmkyULKSUSCKgqQV1BVQQDk+VT6cvBHMfGCrjVNOeqdJhN1VKD3sYo6YjnchMI4iGNhohBLKgTDqgEND+q8/HErb+/H8dx6uJ24MABfvzjH9Pc3Mw999xTLxEoFAokEgmamppQFN/U5OMzGytN2G4BngJ249n9Af6rlPJbZ9pnKYXtdFxXUrIcCpW5TSkly+HAkBfV1cwphYoX/cUCGhtaY2xsjbG6KcqqVAi92hEloCk0RAMkQzrhgOfAVPyo7rLEtm36+/uxLKtev3b48GF+8IMfcMstt7Bp0yaAurjFYjFaWlp8cfPxmYUVJWyLYTmFbTammlLGiybZktffb6opRQjomyhNEbocJ8aLACgCehoibGyLs645Sm9DhERYQxECIWpRXYBY0DOl+G3BLh9mE7eJiQmSySRCiGktuAqFAuFwmNbWVlTVj/x9fKbiC9s5Mh9TiqEp5Ms2+4Zy9bW6fYM5StU1vWRIPxXVNUbpSIXQVO8CFtRV0hGvri5sqIT8qO6SZjZxAxgfH+epp57irW99a72urVgsEggEaGtrq9e++fj4+MK25NRMKcVqTd1EwaRgzjSlSAnHx4t19+XewSz9k2XAKxJf3RiptgWL0dMQJhpUUYSCKgTxkE5D1CAW8Nbq/LZglxa2bTMwMEClUiEcDgMwODjIt7/9bYLBIPfddx/xeByAUqmEruu0tbWh6/qFPG0fnxWDL2znActxKVYcchWLsbyXvnRPM6VoqsJkyWJftXh872CW/cN5zOo4noaI4bUFa43R2xihPRFCrUZ1YV0lHfXagtWiOt81d3Ezm7gNDw/z7W9/G03TuO++++p1bf5kAB+f6fjCdgGYakrJlCzG8pVZTSmOKzk6Vqx3Stk7mGU453V+11XB2qYoG1q9tbruhjCRgIYAVFWQDBs0hA0iQY2I4bcFuxhxHIf+/v5p4jY2NsY3v/lNhBDcf//99Y4klUoFx3FIp9PE43E/NelzWeML2wqhYjsUK3ObUlRFVNuCeUK3bzDLwZE8luP9fzTHAmxsjbO+JcqapggtiWC9gDwa0EiFDZJhnbDf7PmiwXEcBgYGKJfLdXGbmJhg165d3HrrrdMEzHVdymUvnZ1KpYjH43560ueyxBe2FYrjSoqmTaHimVLGCxa266UlNUUhVDWlWI5bbQuWY+9Alj2DOcYLXlswQ1NY3xxlY2ucNc0RutMRwgG1foxUWCcd8aI6v9nzysVxHAYHBymVSnVxq1GpVLAsi8bGxrpr0rIscrkcUkqSySSJRMJPUfpcVvjCdpFwNlNKQPccmAIYyVfYVy0z2DOQ5fBoAafaF6wtEWRTNarrrTZ7FopACIgaOumoTiJkXFRtwaba4C9VziRur7zyChs2bOBrX/samUyGZDLJQw89hKZpWJZFqVTCdV0SiQTJZJJAIHAB34WPz/nBF7aLmNlMKU71/yageu5LXVWo2A4Hh/N1ods3mCNTTXWGdJUNrTE2tERZ0xylOx0hoCsgQdcUUhGdhrDhFZCvwGbPdt7E7i8ggipK1EANqoiAhlBX1nkuBTVxKxaL9TlthmHw+c9/nkwmU98umUzykY98pJ6WlFLW1+Ci0SipVGpG42Ufn0uJxQibvyq9QtBVhURYIRHW6UyFZzWlZMuegHUkQ6xpivKeazsAGMyW68XjeweyfGnnyXqz51WpEBvb4qxv9hyY6ajuFZAjiIU00mFvhE/oAkd1ru0gSzZqKgACnGwFe6QaxQY1lKiOGtIQARVxCaRZVVWltbWVoaEhCoUCkUgEIcQ0UQPIZDJM/fAphCAYDCKlpFQqkc/nCYfDpNNpf0q3j08VX9hWKIoiiAQ0IgGN5ngQWmIzTCm1sTyGprC9J81t65pQFUHJdNg/fEronjs0xvfeGAIgElDZ0BJnY2uMNU0RVqW8Zs8Sry1YKnyq2fP5agsmXYk9VGTs83twJiqoqQANH94EIQ1pOmBLnLESdvX6rgRUlKiOEtJRAiriIu3oUhO3wcFB8vk8wWCQZDI5I2ITQmDb9jRzSU3gwFubO3HiBKFQiIaGBkKh0CWfzvXxmQs/FXkRM19TipSSvkypXjy+p9oWTOK1BetKh+sOzJ6GCKmIjkCgKF5bsHTYG+ETMpan2bM9WWHk71/BmajUH1NTARo/fiWyZM/YXtou0nKR1bBU6CpqVEeJ6CiGitAvLqFzXZfBwUEsy0LXdR599NFpa2yKovDII4/Q0tLC1q1bicVisx7HNE0syyIQCNQFzu8/6XOx46+xXebM15SiCEG+YrN/8JTQ7R/KUaxuGw9qbGyNV5s9R+hIhTBUBVd663heAblOxNDOuS2YdCX2WImhv9w547mWX7sOt+IgznL8mtDheoXyQldQowZKWPMiuovAOOO6LkNDQ1iWRTQarZtobNumVCrx0ksv8dprryGlZMuWLVx77bVnXFuzLAvTNNF1nYaGBiKRiC9wPhctvrD5zGC+phTHlZycKNaLx/cO5ujLlACvLVhvQ60tmBfVxUMaQggUIUiEvFKDeNCL6hbS7NmeKONaDmP/+PqMiC35jtWMf2k/RnecQHccY1UMJXT27Ll0XKRZjegkCF1UhU731uhWaN2f67oMDw+TzWbra25Tyefz7Ny5k/3796NpGu9+97vnnMRt2zaVSgVN00in08RiMV/gfC46fGHzOStn6pQCoE7plCKEqLYF84Ru32COfUO5+rbpsOE5MFtjrGmM0pYIomsCl2pbsIjXFiwSOHNbMGk5lI9kMYfyBNpiTHxl/6k1tg9twhzIU3ptDPNYFll2QIDeEsHojmF0x9Ga5reWJB2JtBxktfhdqMJLW0Z0lKC2ooRuqriFw+FZhWh8fJx9+/Zx4403IoRgbGyMVCp1RtGybZtyuTxN4PwpAj4XC76w+SyKmiklV7YYLZjkSjZeUg+CUzqleG3BCuwdODXCZzDr2dA1RbCmKVqP6rrTEWIhDaQX8SUjBumwTjSoEza8Zs9mfx5rqEDmqwcJXd1I7M2rqC38SdsFyxNR6Urs4SKVY1nMY1nsYS+SVMIaRnfcu62KoQTmd7GWblXobInAe71TQuelLs+W/lxOXNdlYmKC8fFxVFWd085fqVR45JFHiEQibN++ne7u7jOKtOM4lEolVFWtdzPx23X5rHR8YfNZEqaZUoom43kLx5VI5DRTCsBEwWTvUK4udgeGc/W2YI3RAJvaYqxv9tbqWhNBNMWL6mKOS3rSIrBzFDlYIP3BjQhFQQmquGUboSsogdkvum7RonI8h3ksi3k8h6xUo7m2SD1tqTYE5x2FeULnemIKCE2gpUOoceOClhaYpsno6GjdMTmbCEkpOXr0KM8//zyTk5O0tLRwww030Nraesbj+u26fC4mfGHzWRZON6Vkihb5io0AxGmmFMtxOTJaYM+UqG40762dGarCupYoG5tjbNA0rqpA5+4JJrakEFsaiLdFwVAIqAqJioucKM8pcOCJkjVY8ETuWA57tBrNRfW6yOmdURRj/qk36bi4ZQchQE0H0RLBC+a0lFJSLBYZHh7Gtu0zOh1d12Xfvn3s3LmTYrHIQw89NOf6W22fcrlcb9eVTCZ9gfNZcfjC5nPesByXoumlL8cLJpPFU6aU2lDW2my50XxlWqeUQyN5Qq7kYaKMKfDMtQnu2drFrzy6i5MTJTpTIf7+w9ezNhlCTJRx8xZCV+eVanTyJubxHObRLOaJnOeWVAR6R4RAVxyjJ46aDMxvbc6VuGUHpERLBlCTwXmnO5cax3HIZDKMjY2hadoZ05O2bXP8+HFWr14NwP79+2lvb68PNJ2NWrF3rV1XKpXy+1H6rBh8YfO5YNRMKUXTYaJoMp43Kdte+cA0U4rl4p7I4bw8TONgmb9phnvvXsfvff0NTk6U6sfrTIX4549tB6AtqKNOVnAL8xc48CIva8CL5irHcjjjXvpNiRuey7I7jtERPRWN6YpX7C0liFPrfFJKZNkznygxHT0dmpc7czkwTZORkRHy+TyhUGjONbJyucwjjzyClJIrrriCa665Zs7+klJKyuUyjuMQi8X8dl0+KwJf2HxWFDNMKUULY6RIcLhM0wsjlDaksLakCHXFueOvn5yx//d+7TYGJ8vYrqQ9GaQ9aGBkTdyijTCUBaUXAZysiXk8S+VoFvNkHmwXVIHRGSV4VSOBrgTjj5zW/URTTplYpERWHKTtooR1tAZP4M630URKSaFQYHh4GMdx5izEzuVyvPjiixw4cIBAIMA111zDli1b5hTEWj9K27br/Sj9dl0+Fwq/V6TPiiKgeZ1KUhGDroYIlfES+ZJL6clB3KjOxNooZE0CeBHa6RHb4dEC33x1gIe2djKSM+nPlGiKBehsDhHKWzh5a0ECp8YNQlc0ErqiEWm7WP2FutNSTwbrogbgTFQY+/wer/tJVdiEEIig9yfjmg7mySxCV9EaQ6hR47wJnBCCaDRKKBQ6a3oyFotxxx13cNVVV/H888/zwgsvsHr16jN2L6kd//R2XVP7Ua6U0ggfnzPhR2w+5wVpuZSPTFJ8eYjSyyMk37UGJR3EjehYSYO+TJlf+PzO+hrbpz50Hd98pZ9/ePoIYUPj/dtWce8VrZi2pGTbJEM6PZEg4YKFLDmLiuCmIgyVob+apfvJr1/vuS7P9L5s1+uOogq0hhBqzDjvvSsXkp7MZrPE43EAnnnmGVatWsWqVavOKlamaWKaJsFgkIaGBsLhsC9wPucFPxXps2Ix+/OYx7NkHj9EcFOa2G2dSNsl0JNAaAquKxkrmJQtB8eVDOcqaIqgb6LEZ545wq4TGdoSQT56Uw9vWt1QX8+LGgpdkSDxkgPlxQucCGmM/t/dM7ufvGst9mgJoys254V8ppMycF5beS0kPQne+ttjjz1GNpulra2NG264gebm5rO+jmVZVCqVej/KMxWR+/gsFb6w+axI7KpTMfvtI7gFi9TPbAAH9I4oWmym+05KyXC2woHhHK70ele+fCLDZ585yonxIlva4/y7m3tZ3xKjbDnkTYuAqtATCZIsOQjTQRjqwgROV8B2p00YSH9gI5M/PI65bwK9I0r0pnb0lvCch6k7Kd2qkzIVmLNcYamZr3sSPLv/nj17eOmllyiVSvT29nLzzTfPmOw9G1PbdTU2Nvr9KH2WDV/YfFYc0pFUjmYovTZG4dkB4nd1Y3TGUCI6RvuZLegApu1ybKzAifEikYBGQFN54o1BHt5xnMmSxZvXN/Gzb+qmORbEclyyZQtNQFc4QKriolnuwgRuFlekLNveub8wiCw7BNYlidzYhpaYe3r1NCdlVEdPBxHV/prng1p6slAonLG4u4ZlWbz66qvs37+fBx98EMMw5j3J3G/X5bPc+MLms+KwRotUjmbJPHYIozNK/O5usKSXgpxn0XO2bLF/KEe2ZJEMGViOy5d3nuSxXX0IBA9c0857r+8kbGjYjstk2UJISWcoQJMl0e0FCtwsuKZD8aVhiruGwYXQFQ1EtrWe1fYvpUSaDtKSKCENrSHoNWM+H3PuFpiedF0XRVFwXZfHH3+czs5Orr766nnVtDmOQ7lcRlGUusD57bp8lgJf2HxWFG7Fpnw4Q+6HJ7AHi6Q/uNFrfdUeRYvPHfHMOJYrGc6WOTCSBwmJkM5IrsK/PHeMn+wfIRnS+dAN3dy5uaXe1zJXsbBtl7agTosFQVee8xgbJ29ReH6Q8p4xhKYQvr6Z8NXN8xJp13SQluM5KdNVo4m6/AK3kPQkeE7Ip556isOHDxMMBrn22mvZvHnzvCIx13UplUoIIUilUiQSCV/gfM4JX9h8VgzSlZgncpT2jJH/8Umit3UQXJdChDSvKHqRKbmK7XB0tEBfpkzEUAkbGvuHcvzfp4+wZyBLVzrMz9/cy3XdKe88pCRbtrFshyZVoU0qhKE6kHTxAmePl8k/2495JIsS1ojc0EZwU3pekVjdSakI1IYQWvz8OCkrlQqjo6PzSk8CjIyMsGPHDvr7+4nFYtx33311R+XZmNqPMpFI+O26fBaNL2w+KwY7U66mIA+iJjx3oTRdgqvjS+IWnCxZHBjMkTdtEkEdVRH89NAY//TTowxmy1zXleTf3dxLd0MEqKblKg5lyyYpBJ1SJQKo1bE1i8Xsz5P/aT/2YBE1FSB6UztGT3ze43Tcsg1CoCUDaKnld1IuND0ppaSvr4+9e/fylre8BUVRKBQKRCKReb3e1H6UNYHz23X5LARf2HxWBLWatfzTfVQOZki/fz0ioKG3hNFSS9eiyXUlQ9kyB4fzICAR1LFdyTdfHeBfXzxOyXS4c3MrH7qhi1T41MW0aNoUKzZRR7JKqsRUBS20eIGTUmIeniT/7ABOpoLeFiF6czt66/wu/tKtdjRxJFrSQE0Fl91JOTU9qev6nK22pmKaJl/4whdobGxk+/btNDU1zWu/qe26agI339f0ubzxhc1nRWAO5CnvHSf73WOEt7UQvroJYagYnbFlMU2ULYejYwX6MyWihjfFO1uy+NcXjvOt1wYxVIX3Xt/JA9e0E9DUafsVKhZBy6VTqiQ1BT2sLzotKB1Jec8YhecHcYs2gTUJz0E5TzGf7qTU0FLL37JroelJx3HqJQLlcpk1a9awbdu2eacoT2/XlU6n/X6UPnPiC5vPBccpWFQOZch8/TBCFaQeWo80XQK9iWXvjD9ZtNg/lKVgOiSCOpqq0DdR4h9/eoQdR8ZpjAb4yJu6uX19E8qUVKHluGRLJnrFYZWrkAyoBEL6otOCrulQ2jVC8eVhpO0S2tJAZHsrSnj+a0xutSelUIRX8B03li1NeXp6cj5dRUzT5JVXXmH37t04jsNDDz1EIpFY0GuapollWUQikbrA+d1MfE7HFzafC0qtZq2wY5DSq6Mk37MWNW6gNUXQG87Pp3LXlfRPljg0nEdVBPGgjhCC3SczfOaZIxwaKbC2OcrHb+llS/v0C7HtuEyWTKKOZEs4BBUHoQlEQF3UBdctWhReGKL0+ihCVQhd00T42uaFz4aruCAlSkRHSwWXLYqrpSfHx8fRNG1eqcJiscihQ4e48sorAThx4gStra0LMopMbdfV2Njo96P0mYYvbD4XFGu0SHnfBJPfOExwcwPRm9sRQmB0x897B/yy5XBktMDAZIlYQCeoq7hS8uN9w/zLs8cYK5i8aXUDH72ph/bk9M71o/kK65ojtAcNrPHqPDiFRRdY25kKhWf7qRyaRAlrhLe1EtrcsCCrv1cP5yJtB6EpqKkQWsxYlgGoC01P1iiVSjz88MMEAgGuv/56Nm7cuKBuJDWB89t1+UzFFzafC4ZbsSkfypD91hHcok3qgxuh2gtSCV64OqZM0WTfYI6i6ZAKG6iKoGw5PLarj6+8dBLbkdx7ZRsf2LaKplgAo9q30pXQkQoS1DVc08HJVLAnqvPcgtqi6s+swQL5n/Zj9RdQkwEiN7YRWJNYsFh6fSm9KE6NG2jJACK4tFHc1PSk67rzjqKGh4fZsWMHAwMDxONxtm/fTm9v74LeY60fpa7rfrsun4tf2K6+7mr5g2d+QDqYRhH+L/LFgpRezVrhhUGKLwyRuKcHrTWC1hBEbzx738HlxnEl/ZkSh0byaIpCIuSlycYLJp/fcYzvvzHEjavT/MZdG/mVf3152hTvzW1xlKpgSNvFnjRxxkvIWrH3Ao0mUkrMo1nPQTleRmsJE725/aztxc50LK+riYvQFa/oO7q0Udxi0pNSSk6cOMGOHTvIZDK8//3vn7e5ZCq1fpSqqtLQ0EA0GvXbdV2GXPTCFuoNydv+/DY++ZZPsi61zhe3iwQ7U6G8f5zM44cIdMeJ39mFlBDoTpyXzhrzpWw5HB7JM5itEAtoBKtmjCOjBdIRnd/66u4ZM+G+/As30ZqYvj4oHYlTMLFHS8jFNFzGs/iX945T2DGIW7AweuNE39SO1hKedYr3WY9XnS4AoMaqUdwS9qZcTHrSdV1GRkZoaWkB4OWXX6arq4uGhoYFvfbUdl0NDQ1+P8rLjEtC2Nb+f2tpj7Tz8H0P0xhqvNCn5HMWpOVSPpwh+8Qx7JEiqQ9uREgIdMcX5AI8n0wUTPYN5ShbDsmQl56MBlXu+IufzNj2R79xOz0NkVkFQkqJW7Sxx0reVG9VIIILM5pIy6X4ygjFl4bQWyIk7u1l/NF9Z5zifdbjyVM1cUJX0FLBJZsRt9j0JHjrb48++iiVSoW1a9eybdu2OYedzkZN4IQQpNNp4vG4367rMuCSmaDdX+inZJXAn0a/4rFGi1QOZrD68sTe3IkQClrKWLGiBpCKGGzrSdM3UeTwaAFdUYiHtFmneDuuZLxg0hCdmYITQqBGdNSIjlu2cTJl7IwJikAJqvNa8xK6QmRrC6EtDQhNqYsazD7F+6zHmzLlW9ou1nARa7iIljBQE8EFC+/px65N7h4fH2diYmLexd2hUIgPfOAD9RKBw4cPs2XLFq677rp5F2qrqkokEsF1XcbHxxkfH/fbdfnMyorM9bVH2tmf2c9Hvv0Rvrz/yxTMwoU+JZ9ZcAoW1kCBwvOD6O0RAhtSCBW0xpX/iURVBF0NEW7obSAR1hjOlvm7D11HZ8o799oU74rlcmA4j+POndlQghp6a5TgmgRaOoAs2zgFC+nMT5CUkIYS06cNOgVP3HAWl1URmoIa0VHCGk7eonIsS+XIJHam7KU4F4mqqjQ1NdHd3Y1hGOTzeWzbPut+gUCA7du38/73v59169axf/9+FpMxUhSFcDhMMBhkcnKSo0ePMjw8TLlcntd5+Fz6rLhU5G1/fht/fvuf86PjP+JL+7/EpDlJRI9wa8etPLjuQba2bEVTV2SgeVkhHUnl2CS5HxynciRL6gPrUXQVfVUMLXpx9QKU0ovKTk4UiQR0XCk5MlpgT/8kb93U6tn/W6J0puZvhJGOi5MzscdKSEvOa7L3Gad4v2M1xVdHCF/bgho9t8ik1oAZhBfFJQOLrtMD72eXz+cZHh5GSrmg9KRpmvXZb9/97nfp7u5mw4YNC3ZA1tp1ua4n1kIIAoEAgUAAwzAwDANVVdE07YKuzUkpQUovXVy9efddkCCRSFdC/SsYoRDKZb6eeNGvsdVckVE9ykRlguOTx3l5+GWeG3yOV4ZfwZY2HdEO7uq+i/esew9d8S6/kPMCYY0WKe4aIff940RuaCV0RSNKVMdoW7i7b6VgOy4nJzz35CM7jvOT/SP83YevpzFikK/YbF+dntaSaz5IV+IULJzxMm7Jnrvge7Yp3h/cRPHVYQpP94MiCG1uIHxdM+osk8cXdF5T23cFVNRUEDWmI9TFJXEcx1lwerJGqVTiiSeeYGhoiEQiwfbt2+np6TknsXUcB9u2cWzbEwyJV+SuKBiGQUDX0Q3dEztFOSV40wSnenNd7yZdXMf1jue4uNIF1/sqHYmUjjdB3XVButP29R7zDEFI4PS3Vntu+oMIoRBNNxJNpQmEw4jLsOzhohe20+vYpJTkrTwjxREOTR5ix8AOnh94niPZIyhC4arGq3jn6ndyT+89xAILW4j2WTxuxaZ8YILMY4cQhkrqvWvBgUBv4ryMX1lu9gxkOTCU5//98its703zm3dtJFM0aU0EWdeyuN+zmpDYE2WcbAWhiNlrz2aZ4o3l4mQrFF4cprx3HIDgpjSR65tRFzjXbtZzq0VxArREADURWHTtYaVSYXh4mFKpRCgUmneEJKXk2LFjPP/882QyGZqbm3nb295GNBqtPy9dB9dxcW0b13VwHAfHMnFMG9s2cW0bKV2vqbSUIF1PQKQA4amJWxUeV7o4jjtFTCSaoqLrOoamoWkaiqqgCYGiqiiKUhVa4X0VgBDTHhO1xxBTnqO6vViUULuui1UqYdsWqqYRb2wmkkxihC58Gc354pITtqnYrk2mnGGgMMAbY2/w/ODz7BjcwWTFS1Xe0n4L71n3Hra2bsVQL65U2MVErWYt95OTlF8fI/XgOpSYvqjhoSuVommz4/A433ltkC++eII/f/Aq1rfGGCtU2N7bQPQcO++fKviuANIzmswzUnKyJsWXhii9MQ5IghvThK9vQUssgcDVozgXJaChNgRRIwuP4haSnpwmWI6NbdscOHiQw0eOcNv2G3Bdm0qljCJdpoU5VeEXSlV0VAUhlCmiwoKFxHVdHMepCp+sRlYCkOiaF+EZmo6ueynNaZHeecB1HCrFItJ10INBYo1NhBNJdOPS+Ls7E5e0sE2lZJcYLY1yInuCV0deZcfgDnYN76qnKu/svpP3rHsP3fFuvxZuibEzFYq7R8h+8wihKxuJbG9FhDWM9sUPD12JHBzOc2Qkz29+5VWaY0H+/L1XUag4hAMqV3UuvFvIbEjbxc6aOGOlekpwvsXVTt6k+NIwpdfHwJUEN6Q8gVuisUDSqg1DBTUZRE0YCxqlI6XEMiuMj44xNj6GqihoioI9JcJybAv3dLPHaYLlSMkPnnmWlsYGNq9fRzh0/o1JUnpRnlNNKXopSqriqaDrGoZuYOgamloVPVVZ1m4ptmVilkqAJBiJEWtsIhSLoWqXnjv0shG2Gq50yZk5BguDHMse44XBF3h+8HkOTx5GEQpXNl7JO1a/g3t67iERnH/ncZ/ZkbbrpSC/fhhpOqTevwEkBHuXZnjoSqJsOew4PMYLRyf42x8d5Dfv2sCt65oYzZe5qjM5q/1/sUhX4uRN7NEy0rQRujrvSQhOwaoK3Cg4ksC6FJGtLWjpJRK4KbPilJCGkjQQQQUpXRzHxnUcHMvCNivYFRPLrHjiZVn1Y1iWTSabpWxZBINBdE1DKAqK4pVEzPUhwbIs3jhwkEPHjgGCtT3dbFy7BmOF2PullJ7gOQ6O6yK93CcCzz2qaxqGrnsGlmqEp6qLN+vMhlWpYJZLCAGRZJpouoFgNIqiXBp/kytK2IQQQeBJIIBXL/dlKeXvzbXPufSKtByL8fI4/fl+DkwcYMeQtx6XqWSI6BFuaruJ96x7D9tbt2NofqpyMZgDefJP91HcOUzivl60phB6awQteWnO0zo2WuDQSJ4/+MYblCyHT33welwpcaRkW08adYkbO0spkSUba7zkNV5eQEcTt2hRfHmE4u5Rr0fnuqQncA3zi3AkIB3bSwm6Dq7j4No2tmXj2Ca2aeKULZyy5RUJhQWEFO8vu7YOpaooivdV1NekTr23YqnExOQkrisJBowFXdwLxRJv7D/Asb4+dE3jbbfeTCS8steZ3GqE50V6ntvRQ6CpGoauoRs6uqajqgratLW8hSOlxCqXsE0ToVRNJ+k0gfDsDQYW+xpTNeN8fB+Px3c5jnPtQs5zOYVNABEpZV4IoQNPA78ipXzuTPssVRPkglVgtDjKidwJXht9jeeHnufloZfrqcq3dr2VB9c9SHe8G/US+VSz3LhFi+LuETJfO0ygN+EVYweqw0MvoRTkVCzH5bnDYxwczvP7X3+Dj93Uw3uu62SsUGFdc4yO1PKlxdySjTVY8Fp2LaDpsluyKe4apvTqKNJyMdYkCF/XhJIyZhcsy8KxLJwz1H8pioKiKgihVr9W02umBFdCQIGY4hV+z0PoHcchm8uTzefRNHXBkVcmm+VE/wBXbFiPEIKJyUna2tqINzQihBdJZsfGKBdXbu1rLbVZE73aNbhmljR0Ha1qYqm9p9rz8tQ/1KVS4rkwp9x3XQerXMKxHRRNIxxPEozH0HSjfg5Tr/21Uonac7Nts9RaIYQ48zFdvPftSq7dev3hQrGwZiHHXraCMOmdcb56V6/ezkveM6JHiCQidMY62dK4hVs7b+Vk/iQ7B3eyY3AH//LGv/D5PZ/nioYruLf3Xu7pvYdUMHXJXqDPFelIKgN5Cs8NIjSF6C3tSEditCzdJ8GViK4qrG6MYjuSrd0pHn3xBG/d1EIiqHN4JE9jzFiw/X++KCENozuOk61gD5c8Y9+UriFehOWcEqupgtULtMSQ+4tUjk5iHprEbVJw1mgQ94RpqmBpuo5uBBb2fxmoNYaWMGYjhY2MqYjw3OuEqqqSSiaIRMJMZDIUSmWChj5vE0YyHidZbahcrlTYf/Q4nWvW8ci/fpFMJkMymeR973uf9/wKFTchRD0lebqs10SvUqlQKpWm73fqANMfO/1+7XvdQNM908nk2AiZ0SH0QJBIIkUgEkbTT0XNs0WKU+8vWcTneh+IpCurjQckriXBcZC29DrsOC5T9S6oLdwds6xrbEIIFdgJrAX+t5Tyv8yyzSeATwB0dXVdf+zYsWU5l4pTYaw4Rn+hn0OZQ7ww9AI7BnZ4qUotwo3tN/Lude/mhtYbCGqXZmptsVijJfI/7aPw0wFib1mF0RVHaw6jL9E6zkrGcSU7jowxkq3wa4/u4t4r2vgPt69homjSngyytnl5ykwss4JbrcOySxXM4QL2RAlHOLh47sHZPiYqimdaEIoXYWEBRyq4B0tgSUSLjrIxjEgv7RqVdCVY1SguqCBiKgSUOaO4WnpyPJNBShacnpRSkm7v4Etf/gqZTKb+eDKZ5MMf/CBvvLKLI8ePEzACBAIGwYBBwAiQSibQV3CPyWA4QiIRr0c0k5PZJRNp27awK54b1whGCCdTBCNhlHNsejFDsOQpwXItCfZMwfJWI6u6LABF1G9Tfw+uu21b31h2onMh57Os/7tSSge4RgiRBP5NCHGFlPK107b5NPBp8FKRy3UuATVAe6ydtmgb69Pr2dq6lftX388bY2/wwtAL/OTET/jB8R/QHmnnjq47eM/a97A6uRpNWbl/AOcDt+JgHs9SfHEIvSOKsTqBoqtoyUvbYlxDVQRrm6JUbIe7trTyrdcGuPeqNjqSIU6Ml2hNhM7Z/n86xckMQ0cOTXtMUVSUmEBkJYqjEggannCdjQCwMYxYE0QeLuMeKOH8ZBLRXBW4hqUROKGIU1Gc5SJHvLU4GdMQYWXWKE4IQSQcJhgIeOnJQh5dVaf1fZRSYlcq3tpRpYxVKWOXT31te9+Hpoka4N0XsP+5pxh0FOzTqqHXhjTCmsK47TJYttEUgSYUdFWgC4W2aBBdVTFdF8sFQ1MxNBVVUU/VpCnCS8tW69Pqa4rCe7xuihFKtcbN26a+/Rn2icSThA2dx/7s/yM7Mky8qZl3/NpvAUsTgWqajlZ1TtqmycTASQSCYCxOOJHACE0f7rqkgqWpKKdHhef8js7wPpfpuNOQUmaEED8G7gZeO8vmy4oQgrgRJ27E6Yp3sblhM2/qeBOD+UFeGn6JHYM7eHjPw3xh7xfY0rCFe3rv4Z7ee2gINlzSabfZkFJiDXm9IKUjid7eCbaL3hk77xOxLySN0QBhXePB6zr5yf4R/umZo/zuOzYT1FQOj+S5smNp7P8AVqXM8LHDBEJh1FnWn2RcIvMOZGyveNuYX+Gv0BXEhjBidRB5pCpwT04imqoC17h0EZzQFdCrF8Wcgz1WwhQmlmZiOmWscgmzXMIsFatfve/LhQLFfA6zVMIxK9iVCnalPOdrvem9P0MymZwRsSlA6cgBEq6Li8RFwVUUXFVluFQEx8YORRCxNLamYao6UtOQmk7h1edRHJtKYztmU/upF3MchGMROfIGwnWxokncYBjhWAjbrn9VzPKiL9j3//p/4/nHv8Tt/+4XCcbilHNZdvzbo2x/10PkxkfRg0uzritdiarqqAEd6biYmQKlwQkECsFwjIAR9gRwhQnWfFlO80gTYFVFLQQ8AfwPKeU3zrTPhZygXbSKjJZG6c/3czx7nBeGXuC5gefqrsobWm/ggbUPcEPrDUSMyAU5x/ONPVmhsGOA3A9PELmxjeCm9IoZHnq+GctXePVkhh/vG+Wfnz3KH73rCq7uTC6p/d91HQYO7MexLQJn6SwhbYmctCHvgKEgtAVO4bZlVeCKUJHQoKFs8gTudKGU0vUs5ZWSdyuf+mpVSlQqJU+sTn+++rjjzN2YWNUNjFDIuwVDqEYAV1HQAkGC4TB6MIQeCKIFg9O+6sEQXevWkc3leewb36yvsb3rHfcRj0XpP358fj+LauurU220PENHvlAkl89TMU0qpkm5YmJaFlevWwNI3jh8lGODw9OOpQjBHVdvAWD/yX4m8gXP8q+q6KpCUNdoTyWQrkvJNBGApgiE9H7OW++8h2yxxOPf/Fb9/Txw372EVIWHf/tXiTY2kersJtW5ilRHF8HY9AGu0q0WlrvVm5RIR4Jdjbhc5miqLXBxcWwTKT3TSSiewAiH66aT84kRCmAkQ7z5bXc4u3fvXlAQtpzCdhXwz4CKZxB+VEr5B3PtcyGFrYYrXbKVLIOFQYYKQ+yd2MsLQy/w0tBLWK5Fe6Sd21fdzoPrHmR1YjW6ujLqaZYaabuU9o2T+eoBlKBG4t1rEYog0JO4rKK1GlJKXjo+QbHi8OtfeoVoQOOvHroGy/G6VGxdAvv/6InjVIp5IokUUroIoXg1YpZ55vOquMhxy1vfmmNNy7GtaYJT+94qlQlPhGnINaNLgyzjHHFeZ6h8DLNcxKqUMStl5vJ9CSHQAyGMgCdMxpTv9UAQIxjGCAQxAmF0I4ihBjH0IEYygtEQ9bqbnCamdffkLOnJqQTDERLxGNmJCfRQCKtUIp5KMZnNnRfziJTSE76KSdmsYNsOHa3eYNX9h48wMjZGuVITxgrhUJC733w7AD957nlGxsYA0HWdoGHw4Hvfy2OPPz4jAn3wPe/hW498DjszjnXyCI5lYsXT6OEo4ViKcDxNNJYimWikKZ4CYGhyDFdKlFpqVIGgESBWtf9nCrlqZ7BqahSBrmkEdK8xdcks41g2AomuG4TiSULh2TMJS40RCiAjCo9++Uv8z//5P+nv71/QH9dyuiJfBRZUe7ASUIRCMpgkGUyyOrmaDQ0b2Na6jZHiSL0h8xf2foEv7vsiWxq28Pbut3Nv7700hhsvqS4n9miJ4gtDuEWb+D29YEuM7ssrBTkVIQRrm2LsPD7Oz72phz9/Yh8/2jfM2za1MFaoMDhZPif7f358jFI+hxEI8KU/+m/19ZX7//N/RdMNrOr6klmaksYreWm8SqmIOVnAzBYwK2VMq1QVpFMi5p4latK1IGuT17Iucj1Xq7eSjWyhP3mMcqiAHpwiWMGpohUiEAihGcapMoB5ImU1ihh3kZOV6lqcWo886+7JcIjxyckzuidr4pVIpz2zRSi0pGaLsyGEIBgIEAwESDDdSLR+dS/rV/fW79ccjzU2rV3DqvZWT/jKFSoVE90wZl0zdFyXgbJFc3M3b77pfnITQzx1dA85KckBFHNQzGEcfo1ViiDV0skr+TyW60w7VndTG1vXXQHAj17d4bUOm8Lq1lVcu3ojUkq+vfPpGe93TXMHV63ZhAgYPPbkd1Bq64NCoAjBVRuuZMuajRRKRb799BPVx0+tM165bgurO3vIFXI8/dKz9XVGpfr85jWbaG9qRYnpfOHRf53xs5gvl7cz4iwYqkFrpJWWcAsFq8C61DpuW3UbJ3MneXHoRZ7tf5a/3PmX/P0rf8/2tu08sOYBbmi7gahx8Xa4B69mrbRvnPLecUJXN6HGDLRUECV0ef+6JMI6jdEA165S2dAS43PPHeOWtY3nbP83S0VGjh8l2drGl//od8iOeOmt7MgwX/urP+GOj/4HHv+LP4I5sitCCE+AjCCGEUIPhkhGE7NGTcbU+0EvilKrLkHpSOTxCvH9CvFiGiIayuoQonVhjsWzIYQAXZxai5u0kRkbGVYQUQ0C3oXOMAxaGhvr7knTsme4J8vFwoq19teopQgVR/HMNY6kUYvTGI5DQFLTRM0Rs64ZhlSDd934FpCe6CfbOnh7YxOu9Iq/i9kJJkcGyFplsoMnGD66z/vgoRtE083EUs1E0s2km1rrx71hw1X1WjW3+jUWqi6zCLi6dwOymiL1yuck6Wgc6diUxvL0Nraj6KrXxqtqOElEvdSooig0pRpPO75bd6PWWnG6jlNP/7pS1ufpCUUsWtTgIm+pdSGwXZusmaU/189oaZR9E/tmpCpv67yNd617F2uTawmoF5d7ULqS8qEJMl85iHRc0u9bD2o1BbnIkSaXErmyxQtHxhnJVfgvX93NB7d38TPbu6r2/xBrmxf2ocaxbfr37wUgmkrx2V/9DzO2+dCf/C9e/f63MUJhL2IKhdBDIQIhb/3JCIXRDaM+0mS+6cm5kK5Enqjg7itCwYWEirIhjGhfWoGb9pqyWjLgSG8BI64hQqeiuPmmJy8EUspT61ou3oDZ2nuxq8+dGiTgOTAUvNuUhs2heJRQRwNf+sqXT9XlPfheSn1jlLL52V56VsqFLBODJ5gYPEFm6CT5iREAhKKSaGoj1bKKZGsnyeZO9MDiynaklNhWtd+nKghFYwTCUbRA4JzNI9G2JJ/7wufJZDJ8+tOfXnAq0he2c6BklxgvjdOX72OiPMGukV08N/AcBzMHUYTC5obN3Nl1J/euvpfGUONFUTpgjZXIfu8YpV0jxN/Ri94YQu+8+IaHLid7BrKM5U3+7scHefHYBP/nw9eTihiMF0y296aJzNP+L6Vk5PgRCpkM4VgcRdN47H/+QT1iA4g3NfO+3/ljKguMSKQrkQXPPYkQ83ZPznqckxXcvVWBi6mei7Jj+QQOvMgRyxvASURFRNR6FGeaJuOZSSqmSTBgLGuz4WnnVDNmOHgW+No52pwyZNRnrVVH29SEawEfLkLxKLHOJhRVxXUccidHFiRqs2GWi2SG+pgYOkFm8ATZ0cF6R5NYuplk6ypSLatIta4iEF54xsl1XWzLRLouiqoRisXOyXRyrmtsvrAtAVJKsmaW4eIwQ8UhBvID9VTlRGWCiBZhW9s23rH6HdzYdiNxI74iSwfcikPh5SEmv3aYwNok0Vs60OIGeuvl4QKdL7WxNhXb4T898jJ3bGjml9+6jlzZIhrUuKozOa/jZEeHGT12lHAyhVks8uI3vsrWdz7Id//+r0/VMP3Kb+M6AlWXixOmmnuy4FRnvS2+D6E8WcHdV4KcA1EVZWMI0RFY1nXXaVGcJiCuIkIqKEwp7pYEAwvsnHKm11qCqOtiwLZMJkf6yQyeYGLoJJnhPlzba1wdiiVJtXoil2xZRTi+sK5Mjutgmya4Es0IeCIXCqEssCh+RboiF8PFKmxTsRyLTCVDX76Pycqkl6ocfIGdQzuxXIu2SBu3dNzCu9e+m7WptYS08z+GYzaklFROZMl8+SDOZIX0BzYgNOWSGR661BwczjOQKfGVl/p4fFcff/3+a1jdFGUkX+baVSlSkbk/qZYLefr37yUU9Xpt/uif/4HBQ/t54Dd+h0RLuxcdAJOjebLDeUIxnVhjEG2RLbxk2UVOnFt6EqoC12d6KcqsAxHFS1GuWl6Bg2oUZ1aHh4ZVRFTFUV1y+TzZfAFFqbn7qA78rDLlMVwvpBLVqAuXetQl3NNfcfFR18WG6zrkxoampS+titfSywhFPKFrWUWydRWxVNO8J3k7to1tWSAleihEMBolEAwh5tFCzQgFSCQSvOlttzivvPaqL2wrhaJVZKQ0wkB+gMnKJK+MvsJz/c9xIHPAS1WmN/OWrrdwb++9NEea0ZULt2ZgT1bI/fg4hWcHib21C6Mzit4ZRYtdXGuE54vaWBtVKPziwztZ3RThDx+4gortLYJv60mjnOFCaFsW/fveQNU0NCPA7h89wStPfIvt73of62+4mcxwkUrRQp8yusYse+62ZHOI4FlE80wsVXoSqgLXXxW4yarArQ8jus6DwE2N4nQBMRVLcylbFc+M4HrlKtKVuLaDNL20obS88Ts1I4SUEgRIAa6QVR2rnnu9GnnKV077vvbAlAndVI956kii3spRVIWy9vjU1xNT95n2+IURUyklhcwYE4PHq+nLk5QLWQA0PUCypaOevkw0tZ21JZeUEse2vXFGQhCMRAhGq+txszhqjVCAVDjOxCP7uOuvP8KrA3v9VORKw3EdsqZXGzdWHmMwP8jO4Z080/cME5UJwlqYba3buKf3Hm5uv5l4IH5eSwek7VJ8bZTMVw6gtYaJ39mNEtEvueGhS82x0QJHxgr89OAYn37qMP/9HZvZ1pNmrFBhQ0uMtuTMaFy6LkNHD1HJ5wlGYwwc3M8PP/t3dF99HTc/9GGEEIyezHkOutPShq4jMcsOkYRBNB1EXaSZZ6nSk1AViEETd2/JE8ywgrI+hOgKznsiwWzHPO2RM96TjoSKF2oJXZkedU1LF3pfz9a7sv6eqgeQ1Xb53pdTojh1n6nP1fapOwldSa3vvnRB4k573pvUfeq40+5Xt50msDBTbGtvtiqwglMfWKZHsNOj2anbzIdSbrK+RjcxdIJCxqvBU1SVeGN7NX3pGVLm6lt8ynRiIVS1mqqMeGUj1Qi6aVUrk5/fhzNR4d5//vcLFraV72a4BFAVlVQwRSqYouJUGC+P05Po4e6euzmQOcDzg8/z0/6f8pOTP6Et0sbN7TfzwNoHWJdaR0Rf/vUta6RI/qf9nt33tg5AYDRf2p37l4L2VIjjE0XetqmZb+4e4LPPHOHaVUniQZ2DI3kaogGM09K4k8NDFDMZIskUxckMT//rvxBvauHGdz9Ub3prlipIWamme067qEuYGMwxOaKQaAqiL7ZPpQ6EJExWzQ8G3sV/wUiIA9sUxKiGctiBXQXkngJur4bbocI0gZMzdp/GqfCmeldMvXv6xt7zKp4YWC6udGHah0IJjre31xxaqaYs1Xq/x9Mv8Cvp994TTlkfV1P73vs6RYinPOfKarSKVwrgCaZbF01XujhVwa3tJ6SY/Ycs8aJRKSAQJLVqLemudYDArhSZHOonO9xPcWSYkTf2MLHnILoSIBZvJpFsIRZvIhJtQBc6wgFhU/2qIRzNSwHbZYRT9h6X1TrGT7TiTFQW/XPzhe08E1ADtEXaaA23krfydMY62ZTexINrH+TV0Vd5tv9Zvnzgy3z14FfZnN7Mm1e9mXt676E10oqhLr0z0S1aFF8ZwTqeI3JTO0JX0ZvDc44e8fGojbXZP5Tjozf18Mff2sN33xjivivbkBKOjxen2f+L2UnG+08SiidwHYenvvDPOJbJbR/6WP0Trm07VIoFmnt7MIKnaoqmIhDYloNVcYimw8QbQ9MiETFzh9Pun3pAuhI3Y+KMlb2u6sGZ051nXudPP371/jqQN0qsvjzFF4aw9xbRj0PoumZCWxpOrdXOIhxLJSVeeyzHm3fmuF5k5HjTvh3LxnFspO1NRzg1h25G+ANSVkVQqRYRe+uSiqLMmjpbSrRIkEhDGkUouNKlMDaOXZi7Z+ZCqJtkbC+lKy23eqvet91Tz9lV48zUmyXBAWHH6bTjCLnRq8M7fdCFBYx5NykljrBxFQm6gtQVXBXcIEgVXFXiKhIbG6lIEq6JmgosWtx8YbtACCGIGTFiRozueDeZcoa2aBs3tt3IcHGYnUM7ebrvaf5219/y2dc+y7bWbdzdczc3td9EIpBYkgGp0pWUj2Up7BhEawoR3JhCCWmoCd/aP19aE0GOjRe4dlWSKzsSPLLjGG9e30QipHNivEhbIkgkoGGZFUaOHiYQiaAoCju/+Rgjx45wywc+QqK5pX68YmaScKqJaKppztfVgxCMSIpZC9u2aOyITluTW9ibCCIbIlgjJZysiRIQCH3xv19qV4LAqjhWX57C84MUnu6n9NIw4WubCV3RcE7HPhtCCISqoagwY9jZLHjpQhfpepPDZW2CuOviOhaOVZ1zZ3upM8t06sae6Ufx1tKEmE0M5/9+tUiQaDzNxL94aTg1FSD1MxvIM46VLU0XF3uK8FQ77cspz9Vu0pIz9ptrMuZUiRea8NyomldQLzQFgqfuowlvG33KdppA6AJHuGQygwwPHmVo4BAj/UfrhpRQJE5jezcN7T00tvcQSzchEPWoc2xskJafWU/mC/vn/bOb9h78NbaVRcku1Zsxl+0yBzMH2TG4gxcHX6y7Km9uv5n7Vt/HhvQGovri18GssRKZxw9ROTBB8r3rUKMGgd4EiuFPFV8Iw9kyrw9MkinY/OdHd/Ge6zr56E09dfv/Fe0xBg8ewK5UCEQiHH/tFZ58+B/Z8KZb2Xb/g/XjVIoFLFOgam2EE/M37ZglG8eRNLRFiKTOzfbuFCysoQLSclFC2pIYQcy+PIUXBrFO5hFB1RO4Kxsv2t8zKWXVpOJNwa5HiLaDY3tRYO1717FxHIcZUaGUIEQ1LVoVQgsa1nST+dcD0yIVNRUg+Y7VjH1uz/xOcIYYTb9fF6PTBGlWgVrCtKyULpnRIUb6jjBcvZXykwAYgRCN7T00d/TS3NlLy+q1qOkAlYk8937wA+x67VV/je1iJqSFWBVbRUe0g5yZoyXSwobUBt677r3sHt3NM33PTEtV3tZ5G/f03ENrtHVBA1Jd06G0e5TK/gnC1zahRgz05vBFe7G5kNTG2kRTOndsaObxXX3cc0UrLfEgI/kyxw6PQyFPOJEkOzrCs1/+Ao2rurnu3gfqx3AsC9d1SLWvQaBVe+uB47jY5gwf+jSMkIbruIyezFPKW6TawmiLjIrUiI7Sk8CeqGCPFb0+fufYSs3oiGJ0rMUc8FKUhWcHKL40TPjaJkJXNqEsNtK8QAgh6i3I5oMEb9q5dJGOF/E5FcuLkIeKOCMV3PESFF2UTygz0m/ORAUR03HWeuuVUgOhC9A8448wlOp9FQxRn4a93CnThSKEQqqpjVRTG+uvuclzXmYnGO47zPDJI4z0HaH/iCfeD/zG7/Cj//FpsiPDjPXPb0rDVHxhW6EoQiERSJAIJLASFuPlcRpDjWxv3c5IaaSeqvzUK5/in17/J7a2bOXO7ju5uf1mkqHknKUDUkrMvhyFZ/tR4gaha5pRghrqAqIEn1MoimBtc5RXT2b42Td18/ShUf7l2aP8v3dtxDCL7Dl0hGvWdWKbJk8+/I8oqsqtH/zoqf6Mrkspn6d9/UZy45LvfPoVcmNlYg1B7v7EFWiGclZxU1SFcMKgXLAYODhJQ0eEcHxx/59CEegNQdSYjjVSxMlZKAH1nNddjbYoxv1RrMEChReGKDw3SPHlEcJXNxG6qhEleIlejmwXZ7SMNVTEHi5iDReniZcSMwi0e1PpRVCdsbakpgIoIY307Wu8tGk1Ypx6k66D43ipVLv6IUnWGy6LaaZKz0FZfaZe+1ebAOAV7tUfX0aBFEIQTaSJJtKs3rwVgHIxz3DfEaKphmkdeBbKJfqbdGmhqzotkRZaIl4z5pHiCG2RNt7e/XYOTx7muYHneHbgWZ7se5K2iLdOd9/q+9iY3kjMiM0oHXByJoVnB3AmTRLvXO29Rmv4ki5AXW7SEYN4SMd2JO++toMvvnCCezeN0lLsxzFCTJQsDnz3q2SGBnjLz32CSDJV37eYm6ShoxPXNfjOp3eSG/OMArmxMt/59Gs88CvXYDO3sNUIhDUc22X4WI54g0WiJbzosgDFUAl0xLDzJvZQEbdgLUl6Um+NkHznaqzhIoUXBik8P0hx1zChq5oIX910UTfblo7EHi9hD5ewhj0hs8dK1P77lLCG1hwmuC6F1hL2siRT368qaPjwJsY+v6e+xtbw4U3gctYZfTPOhVPrh1K6dVGszZ87k0C6roNruziuPUMgZ7zCEgtkMByla92VBAIR4k3NixY3f43tIsVxHSbNSQbyA4xXximbZXaP7ebp/qc5MHEAgWBLwxZu7riZu3vupj3aTlgPI22Xws4hMo8fJLguReSmdrSGEHrjyuiAcjEzWbTYeXycqKHzHz7/Ig2G5HduiCOMAP2vvkD/j7/OlW+5i6vvvKe+TzmfIxCN0tKzhtxEhc/9t2dnHPdDv38jZnnusTOnI6WkXLDRdIXGziiB8LkV/0tHYmfK2KMlzxCxhOJjjZYovjBI5dAkQlcIXdlI+NrmFS9wUkqcTMWLwoa8SMweKdX7RoqAitYcQm/2BExrCaPMMntuBrrimTSq63DSdr3uKBeI2QWSaWJ5ShhdHNueJpBu9fv5CGRt3dGIRRAJja//9Z/yB498lRPjGb9A+3KjbJfrzZhLTonx8jgvDL7A031PM14eJ6yF2dqylbd0vYU3KdcjHhvBLdik3rcONaRhdF+ew0OXg1dPZsiXbH708gH+4aVxfnlrmiv0DK998e9JdvZyzyf+Y71pr21WcGyb9g2b0XSdfKbCV//8VMQGEGsI8vaPb6k67hb+f2SbXllAsnlmWcBicE0Ha6SIu0TpyanYYyUKLw5ROZABTSF8ZYMncOcoykuBlBI3Z9WjMGuoiD1SRNZSxJqC3hRCmyJiamJ5G0VfbCxUIIWhoMQNbrvr7fbu119b0C+BL2yXEFJKclaO4eIwg4VBXNflSPYIP+3/KS8MvoDlWvzs5P18sP9uxm50aV/dQ3JNK3rEX1tbKnJli2d2HUTNDPAHOy0cs8wHB76C69h0P/jvuXp1GyFDw3UcSvkcHRs2EQh79WrlgsnEUInvfeb1+hrbW39uE8/+2yEURbDtHT2LirykKykXLIywfm5lAVPw0pMFpC2XzD1ZP/Z4mcLOISr7J0ARhK7wIjg1ev4Ezi1ap6KwalpRlqpRsyLQGoOegFWFTE0H/Q+Hy0TT2vYDk+Xc+oXs4wvbJYrlWmTKGfoKfeQrecpmicNv7OGtP93Cq6ED/N6qT7ElsZmbem7mru67aI+2E9H9biPnSqVY4Llnd1IiwJGcy+7HH2F16RhXvO/jKI3tRAyVNc1RipMZGru6iTc21/fNjpXY+9wAjR0x0m1hhBDYtsO+Z4d45YcnMIIa297RQ+vqxKLObSnLAmB505MAdqZC8cVByvuqAre5gfD1zahLPELJrdin1sSqYubmvU73CFBTwXoUpjeH0RqD/mzC84gvbD6zUrSKjPT1YX91EH0M9t0xzpPmDp7MPTMtVXn7qtu5ue1mGsINCyod8PFwbIu+fXuwHMne0QqFN3Zw4qnv8lzjzfz8+95OLKAyUTLpjkhaWptp6uqZJi6jJ/M886X9jA8Wue8/XjXt2JPDRZ57/DDZ0TLrt7dwxe0dqIuYuuA6LuW8TSQZOKeygGnHNJ26GChBdcmnQdiTFYo7hyjvHQcEoc1pwte1oMYXLnDScrFGawJWwh4q4kxOcSAmjGnpRK0x5JfAXGAWI2xn/YglhPgDKeV/n3JfBf5FSvmhRZyjzwUgKAMk9mjkh0G7sYGueIy702neHryHI5NHeKb/mRmuyju77mRTwyaSweRFMSD1QiOlZPT4cVzbIRqLETh0lL1PP0GoZzM7xZWk9uX4uauSBFyLgYJgY1vnjIjJLNnkxivEZzHyJJrDvO2jm3nlhyfY//wQw8dy3PjAamINC/sAspRlAfVjGipGRxQnb2EPF3ArzpKmJ7VEgPhbuohsbaH40jClN8YpvTFGcGOayPUtqI2hWc0W0nGxx8rTzB3OeLluVFAiOlpLmOCmVF3MLtmSg8uM+fwvdgkhfltK+adCiADwJeClZT4vnyVCSknlWJbCjgG05jDxja00JAP0NumMlccIaSF6E708tP4hXht7jSdPPsm/Hfw3Hjv4GJsbNnNT+028rfttdEQ7VuyA1JVAdmSIQmacSDJFKZdl3zf+FSOeZtPd7+aOfRV+cLTA27qDNGkOTkMPo0WbNuPUmpF0JVbZJjdepmV1fNbXUHWF6+7qpnV1ghe+eZTv/eMbXPO2VfRe3bjg/5elLAsAz9iixQzUsI49UcIeLSM0saRCocYDxN68ivD1NYEbw81ZJO7tZezRUy2o0g9tIPdcP+XdY9X5a55DUW8JE+hNoLd4a2Nq5MKbUnyWh7OmIoX3F/MwsBu4A/i2lPJ/LcfJ+KnIpcfOVph4dD+VQxlSD65DTQYI9JwaHiqlpGAVPMNJcRDHdchUMjw/+DxPnnzyVKqydSu3dtzKTe030RhqJKwvrKbmUqaUzzGwfy+heAKk5Aef/XtGTxxl68/+RyaNBEKo/OfvD7ExqfDf7t2EGolTMG1u6G2od/+3TYd9zw/x48/vZdt9PfRc1Tj3a+ZMnv/GEYaP5ujYkGLrPd0Yi1jjWuqygBr19GTO9KK3ZRhW6+Q9Z+Z4VdRqqKkAqXevpfjy8KlILH75OBRnXNPlbN/Pf5tZJeL048y6/7STOvPdqeN4ZqHtmp6DmVJ23ezPzs4Z/xKEENdNufs3wP8BngF+IoS4TkrpR20rHGm7FF8epnIwQ+i6ZpSogdYSmXaREUIQNaJEjShd8S4y5QwDhQESgQR3dt/J8exxnu57mmf7n+XJk6dSlW/pegub05tJBVPo6uX7ydc2TYaPHCIQ9pobv/zdbzB0+AA3ve+DdK3uZnffJCFd8I4enS8fNNmfhaviCq4rOTFeZE21+79tueTHPZt/vPHs6cVQzOC2D6xn/44hdv+kjyf689xw/2qauk5vsT43QghCUR3bdBg8PLlkZQHLnZ4EUKM6IqjO2oJKTQeJ3tyxZK91vpDyVLf96d1C5kacfk+c9oSYulHtee/rNL0XnBpfJLydlNn2rR9XTNn/TK8rpj0vTjsnIaYc8PRzAQpmsXDmdz47c33E+8vT7k8Am6uPS+AtC30xn/OLNVQg/3Q/asIgfGUjatxAi515wV1TNBrDjTSGGynZJcZKYwTVIN3xbt6/4f28OvrqjFTlm9rfxB2dd9AV7zrvA1IvNNJ1GTl+BADNMDi553Ve//H3WbvtTay+bjsAHckQR/rHuGd9kp8MZvjMM0f4Xw9dQzJscHy8SFsySNjQsC2H/IQnbLGG+RXLCyHYcGMrTd0xdjx+mB8/vI9NN7Wx+ZY2lAWmFTVDRdUUMsNFivlznBYw5fyWOz2JELO2oJptNM5KRTou0vRquYQQKBEDpVFHCZ72QWA2ATntuUsxKrVdZ2HdCfBdkZcsbslm/CsHKL82SuL+1eiN3vrCQgtqXemSM3MMFgYZKY2AhElzkucGnpuWqry+5Xpu7biVN7W9icZw42VROjDe30dmsJ9IMkV+fIxv/f/+gki6gbt/4VdQdS+KrVQqvHZshMY1m9hxPMtffm8/v/rWdbx1UwvZkkUirHFFR5KJgQI/+df9jA8UeMcvXXWWV56JbTq8/L0THH11lHR7hBvuX000tThTyFKXBdQ45Z40vYv2UqQndQVsd2YLKk25oN065mJaVCYBXaDGA6gRHXG6mPkghNgppdy6oH3mscYWAB4EepgS4Ukp/2AR5zgnvrAtDdKVXtusrx4guClNZHsrelsU7RybHFuOxUR5gr58HwWrgBSS45PHearvKZ4ffL4+VufGthu5vfN2NjduJh1ME1AvvQLwwmSGoUP7CceTuI7Dd//PJ8mPjXLP//PrxNLe+pjrupSyk2gt3RzKS9LhAL/xpVcYL5j8/YevJ6irjOQrXNeVpDxc5vuffZ1QVOfW9y/I2TyNE3vG2fntY0gpue6ubrqvaFjUcZajLACqbajyllfc7SxRcfcKa0E1G9KRyCmz3JSwhhozUMK6X05wFhYjbPPJCTwOTAI7gcXP6vY5b1hjZXI/PoES0ghf14ISNRZV83M6uqrTHGmmOdJMwSowWhpFFzpd8S5+ZuPPsGtk14xU5Y3tN3J7x+10J7qJG/FLonTAKpcZOXqYYDSOUBRe/NqXGe87we0/+/G6qAGUclnSHauINzUxcHQc03b5+Vt6+a2v7uaxXX18YFsXUUPjwFCOprz0HJE9cVxXUs6ZBMI66gIj7FWb0qTbIzz/tSM8//UjDB6e5Lq7uhecVlyOsgCYmp7UsCfKS5OerEY/Kw1pObimBCRC9aIyJap7tX5+gfeyMp/fpk4p5d3LfiY+S4JrOuSfPIEzViZ2ZzdCV9Cbw0ueFozoESJ6hM5oJ1kzy0B+gOtarmNr81Ymrcm62eQzuz/DF/Z8getbrufmjpu5ofUGmiPNxPTYRZmqdB2H4WOHUTUNVdM4/PKLHNjxU7bc/lZWbb6ivl0plyOSSJJoakZMGWuzpT3BTWsa+MpLJ3n75lbSEYORbAlnqIxrS+KNQRzLJRg1sCoOtukSiCzsoh9JBLj9QxvY+9MB3ni6n7E+z1jS0BFd8Ptd6rKAGkJV0BvDqLEA1nABZynTkxcI6VajsmoTZCWkoTcbKGENYagX5e/7xcp8/mJ+KoS4Ukq5e9nPxueckFJSPjhB8eVhjJ44ekdk2YeHqopKKpgiFUxRcSqMFcfoL/Tz9u63c3fv3RybPMaTJ5/kuYHneKrvKdoibdzQdgO3tN/CFY1XkA6lCWkXz2SB8f4+zFKJcDxBZnCAHf/2KM29a7j6znvr21iVCoqq0NjVjag2PK6NtSmaNh+9qYfnj4zz+R3H+OW3rCOqaRzvywMQbwzh2C6xdJBIIsD4YIFCpkIwoi3IEKIogs23tNPcE2fH1w7zo8/tZcut7Wx8U9uCU3+qphCK6eQmKpTy1pKWBSgBFaMzhlMbjVNxPCG4SERA2i5uxQG80S1qTEeJGhe9SF/szGeN7Q1gLXAELxUpACmlXPgK91nw19jODStbZvyf3sAeKZF871q0ZAhjVey8L0ZLKclb+VPNmKWXJnp5+GV+cvIn7J/Yj0B4qcq2G7m542bWJNbgSAfLtTBUg3QwveIclrmxUUaOHSGcSGKbFb79v/8Ks1zmvv/nNwjFvKJq13EoF3K0rz/V3LhGbaxNUzTIZ54+zOO7+vmbD1xDRyjA3ieOk3k9w7t+7RpsW9LcHSMUNZBSUsxUGBsoIIQgEF54ys4q2+z8znFO7BmncVWUG+5fTXiRqemlnhYwFem42ONl7LESQlNWZBcQKatRmVWNyoIqSiyAGqlGZb7xY8lZrjW2e86+ic+FRtouhWcGsPoLRG/tQAno6K2RC/KHJoQgZsSIGTG64l1MlifpK/RxbfO1bG3ZStbM8kz/M16q8rXP8OrIq/zStb/Ef33qv9Jf6Kc90s4n3/JJ1qXWrRhxM0tFRk8cqwvYc1/5V3KjI7zt479Uf0xKSTE3SXNX7wxRA0iEdRqjAfJlm/dv7eIHe4b57DNH+e1b1yILDmpIxVEF2LJu1hBCEEkFMSI64/15SjmTQERHWcD/qx7UuOGBXlrXxHn5ieM88ZnX2XpPD50bU2ff+TSWoyyghlAV9KYwanxlpSel7Xpi5nlTvIisSUcN6Us6tsdn6Zi33V8I0QzUK0ellMeX+mT8iG3xVI5kGP2nN1BTAeJv70FvCaPPsx7qfFG0ioyVx+jL9WG5FpqicTR7lDXJNfzu079Lf6G/vm17pJ2H73uYxtDcHTjOB45t079/D0II9ECQvT99khe//lWuvfsdbLn9bfXtitlJoukGGld1nzGVlitbvHBknMZogK+/2s8/PHWE37x5NcFnxxEBhQ33dtERDtK1KT3jQ4l0JbmJMhMDBTRDXZSg5MfLPPe1w0wMFOm9ppFr3roKbZGparNs49qS9BKXBUDVPZkzsYeLnnvyPKYnp9rxkSAMBTVuoEYMRMCPys43yxKxCSHuxyvKbgeGgW5gD7BlMSfps/S4JZvJ7x1HWi7RWzpQQxpaauV15w/rYcJ6mI5oBzkzx0BhAMd1aAo1TRM1gP5CP0WriGmYGOrSjilZCFJKxk4ew7YswrE4I8eP8tK3Hqdz0xVsvvVUjwKzXMIIhmjoWDXnBTgW1GlLhhjLm9xzRRvffHWAz+/q48NZlcYNScayFRpDxqwXT6EI4g0hghGd0ZN5ijmTUHQeE5mnEE0HecvPbuT1p/rZ++wgo8fz3PDAalKtC2+RZgQ1XMdl9GSeUt5a0rIAIQRatbbLHitjj5cQuoISWJ705NQiaYRAjegojSHUkIZYovfkc/6YTxz9h8CNwH4pZS/wVrzWWj4rAOlKCi8MYh6eJHydN6tKb7swKcj5ogiFRCDBxvRGtrdtJ6gGaY+0T9umPdLOSGmEHQM7GC2Ozux/d57Ijg6THx8nHItTLuR56pF/IhxP8qb3fbBuDHFsG9s0ae5ZjaKe/SLY0xDBclxURfDRN/WQz5pIWxJMBggpKgPFCq575vdrBDVaVydINIUo5Sxs01nQe1JUhSvf3MntH1yPbTn84J/3sP/5wUX9jE8vCyhml7YiSKieq9drLqDi5E2kc+7WfiklrungFCzcvIW0XdREgMCqOME1KYyOGFo84IvaRcp8hM2SUo4BihBCkVL+CLhmeU/LZ75Yw0VyT55ETQUIbk6jNQRX5KL7mTBUg5ZIC598yyfr4tYeaedPbv0T/urFv+Kvdv4VT/Y9yZ6xPZTt8nk9t3Ihz9iJ44RicVzX5Zkvfo5yIc+tH/oogZAX4UjXpZTP0tyzGj04vyg5ZKisSofJliy2daW4OlzdL6ahC0FZkQxn536viiJItURoXZ3w6t4K1oKFqbk7zp3/bgttaxO88oOTPPXFA5RrAzYXSCCsoRkKw8dyjPfncZZAfKaiBDSMVTH09ijScnEK9oLfr3Qkbsn2xKxoe6UwLWGM1QkCq5OegzisI9SV+6HQZ37MR9gyQogo8CTwsBDib4AF9+7yWXpc0yH7vWO4eYvYrR0oQQ0tvbLW1eaDIhTWpdbx8H0P890Hv8tn7/4sAsEtnbfQl+/jz57/M75//PvsHNrJUGHovERvtmUxdPgggXAYRVV57YdPMHBgH9ve+SANHavq2xVzWVKtbUSSCzNidKZCuFJiWy63Jj3zyQ9HJgFIRYPsGcwxlj979BOM6LStTRJJGJRyFo69MEEJhDVues8arr+7m9ETeZ74zOsMHMws6Bg1ppYFDB6apFJcnEieiVp6MtCbQEsFcAsWbmXuS5G0HJyC7UV6poMaD2B0xgiuTRJYFUdLBlH8GrNLjvl8tH8AKAO/BnwISAC/v5wn5XN2pJQUXxul/MYYwS0NqOkgRmvkov20qQhlmlEkokUwHZPOGzv5x9f+kX/Y/Q/sn9jP/WvupyPawerk6mWrf5Ouy+jxIyBBMwL079/Lqz/8Lquv3cbabTfWt6sUCgSjMZKt7XMcbXaCukpPQ4R9RyeImJJhDb55ZISbO1KsD8ZQgVdOZNjQGqcjNff7VFWFho4YoZjB2MmCV9S9gLIAIQSrr22icVWU5x4/zNNfOsja65u56i2dC57SvVzTAqa9RjU9qSYMrKGi554MaQhVmVkkHfSLpC9HzvpbK6UsSCkdKaUtpfxn4Angfyz/qfnMhT1ZJve9YyghjdC1zWipIMoSFc2uBJLBJNe2XEtPvIdfvvaXeWDNAzx58kn+8sW/5PWx13lp6CUGCgP1GrmlJDM0SDE7STAapZCZ4Jkvfo5kSyvb3/W++oXRNk0kkqbuHhRlcesw7akQwpaUs2a9o/9XDgwhVIGhKaQjAfYNZjk4nJ9zza1GOB6gbV2CYESnOGniLjAdGG8M8daf28S6bc0c3DnMD/5pD5MjpUW9N81QCUZ0MsNFBo9msSoLWwecD/X0ZFsUabo4eQtZcVCjOkZHlOCaJIHuuPe3Ebh4ir59zp0zCpsQ4iohxBNCiNeEEH8khGgRQnwF+D7wxvk7RZ/TkbZL7gcncCYqRG/tQA2qaCvM2r8UBNQAWxq30JPo4c7uO/kv2/8LJbvEnz7/pzzT/wz7x/fz2uhrFK3ikr2mVS4zMdhPOJbAsW2eeuSfcB2b2z74MTTDc2e6rkulWKC5dw26sfj+ibqq0Bw0qEyaxBuC3LOmiYoGhqYQDaqkIjqdqRAnxovsGchizUOoNF2lcVWUxs4oZsnBLC1s1UDVFK55Wxe3PLSOcsHi+//0Bgd3Di8q/SsUQShm4FgOAwcz5MfLS55GFkKgJbz0ZKAnTmBNEr01iho1Lnj9m8+FY658xT8Afwc8C9wNvAQ8AnxISnl+V/F9plE+NEHxpWGM1Qmva/9pw0MvJRSh0JPoIW7E0RWd//6m/87n3vgcn3vjc7w2+hof2fwRsmaW1fHVtEZbz7mg2yyXvPmHisLL33yM0RPHuPWDHyXe1FzfppSdJN2xilB0YUM9T0dKiVF2kLZETxj8zJZmZCrAx//lRU5OlOhMhfi7D19PZypEf6ZE6YTDFR0Jgmdx6gkhiKaDBCI6o315SjmLYGRhXfTb1iR4+8e38MI3jvDyE8cZOpJl673di2qltZxlATWEplyyfwM+C2eu34SAlPKfpJT7pJR/A7jAb/midmFxShaT3z6GUAWR7a2oibmHh14qpENprm25lsZQIz9/xc/zc5t/jt2ju/mDZ/+A49njHJw8yKsjr1KwFjxsdxrlfA5V0zm2exd7f/okG2++ne4rr6k/X8rliKTSJJpbzvEdgWtL8mOeQURGFJrbYvzml1/l5ISX/js5UeIXP7+znpasWC47j02QK8/PlKEHVFp74iRbQpTy1oLTgcGIzi0PreOat61i8PAkT3zmDYaOZhf2Jqssd1mAz8WPdBzcUgknm8UeHcU8cYLy3r2EhVhwkeVcEVtQCHEtp2a15oGrRDVRLaV8aeGn7nMuSFeS//FJ7MEC0ds6UCM6etPCC2svVkJaiCsbr+RY9hhSSNal1vGpVz7Fnz3/Z9y/9n7u672Pl4ZeoifRQ3ukHXWBa1/SdRGqSiSVIt3eycabb+fau99Zf96qlFF1jcZVXUuyXmPbLrlx73NiNB1CKtRFrcbJiRK15bVaE+Wdxya4siNBQ/TsaVChCBJNYYJRg7ETXsQUjMx/vUkIwbptLTR1xXju8cM8+YX9bLixlStua1/wlG5YvmkBPhcH0rKQpolbuxUKyGLR+2qaUzaUoGoITUMTYsH1S3PtMAD81ZT7g1PuS+AtM/bwWVbM/jz5ZwfQ2yMYvXG0lvBll35RFZXVydXEjTj7JvbxOzf8Dl/e/2UeP/g4r4++zn+69j9xbPIYI8UR1qXWETPmly6UrsvI8aM8/hd/RHZkmHhTM+/8td9GNwLYlonrOJjlEh0btqBqS2PScSyX/ESZQESjtz2Ga7l0pkLTxK0zFWJqBjFsaGiKMm/HZI1ASKN1TZzMcInsSHHBs96SLWHe9rFNvPKDk+x7bpDho1luuH81sYaFd7hZzmkBPhcWr0m0Wb+5pRJuoYBbLOIWi+A43jBYCQgQmo7QNEQgiBJZ+FilMzHvXpHnA79X5JlxTYfRf3wN81iO5LvXordFMDqil7XTq2gV2Tu+l5Jd4o2xN/i/u/8vEsm/v/Lfc03zNZSsEt2JbjqiHWcdcFrITPDI7/w62ZHh+mPxpmbe9zt/TLmQpziZoam7l1jD0vWuzAwX+danXkULqNz60DpOTJYINYf4xYdfqq+x/Y8Hr+L4WIErOpLT9nVcyXihQldDhNWNkQU1RS7lTMb68kgXjPDCLfB9+yd48ZtHcRzJtW/voufKhkX/Hi7ntACf5UE6znTxqkZcTqGIWy4Dsp7mQyigV8VL0xDz6MxzOq1XX3Vg0nEWNFb+4mlRcRkjpST//ADmkSzhba1ekWnL0g8PvdgI62GuarqKI5NHcFyHP77lj/nUK5/iky9/kjevejMf2fwRTuROMFIcYX16PXEjfsZjOZY1TdQAsiPDSOlSymWJNTYtqagBVIoWufEy3Vc04DqSpmiQEcvh4Y/fUE0/Sj71o0N8aedJfuH2Ndx3ZVt9X1URNEYDnBgvUrEcNrTG0OaZ0gvFDNrWJpkYKJCfqBCMLmzWW8f6FOm2CDu+foQXv3mUwcOTXH93N8YiOt4s57QAn8UhpQTLwjUtpGXiVipe1FUoIAtFpG3VNgQEqCpC1xG6jppMrojrki9sFwHWWJn8j06gpoMEN6bQW0J+D7sqmqKxLrWOuBHnQOYAv7n1N/nW0W/x2IHH2De+j1++7pdpi7Tx8vDLdEW7WBVfNWv0puo68abmGRGblNSbGy812dEStumSaPKGizY0h8mVK/RPlIlWReID27o4Pl7k739yiJLp8N7rO+v7C+GJ22i+QumkwxXtZ3dM1lA1hYbOqFfU3Z9HiIUVdYdiBrd/YD37dgzy2pP9jPcVuOH+XhpXLdwpWisLMMs2AwczyzItwGc60nVnSRkWcYte2hBXIgXeopMQXrSl64hqJ56VzrKnIoUQKvAi0CelfMdc2/qpyJlIx2Xskb2UXx8j8c7VGKtiGJ3nf3joxUDBKrBnbA8Vp8LJ3En+967/Tc7K8cGNH+SunrvIVXLoqs6G9AYSgcS0fV3HYeDAXr71t39ZX2O7/z//Vxzboqmrd959IOeL67i89N3j7PjaYd78oQ1E00Gau2PYmqiPtakXgzsuf/2DA/xk/wjv37qKD90w07ySLVkg4OpVSaIL7IBvmw5j/dWygOjCZr0BjPfnee7xIxQmK2y+uY1NN7cv+Bg1XMelnLeJJAPLUhZwOSFt+5RwVUxPtApF3JKXMpz2P6SoCE07lTZUVs7a/ZKmIoUQdwExKeWXT3v8Q8CwlPJ783yNX8Ebc3PmPJDPGSnuHqX8+hjBKxvQGkLoLSu7c/+FJKJHuLr5ag5nDuO4Dn9665/yD7v/gX9541/YPbqbX7z6F1EVlVeGX6Ej1kFXvAtd8UwLjmWx63vf5m0f/yWSLa0gBIXMBKm2jiUXNQDHlnVHZLwxiOt6xdWhgEp7KsRoziQR8s5NUxV+7W3rCWgKX3zxBCXL4eO39E4Tt5pj8sWj4/N2TNbQDJXm7ji58eqsN11BX0BaMd0e5c5/t5mXv3ecN54eYOiIZyyJJBdevH56WUBDR4RwfPFF8Jcy3tw465R4lcv1lGHdqFHdDqpGDV1HaDpa6tJ2U8/12/v7wDtnefwHwL8BZxU2IUQncB/wx8B/XswJXs44eZPsd46iRHXCVzehNYVR/PWHOdEVnfWp9cSMGIcyh/hP1/wnnux7ks+/8Xl+88nf5Jeu+SWuaLyCwcIgI6UR1ifXkw6lKRVyHNjxDNJx2Hb/gxQyEyRb24kkkstyno7lWf0DYQ0jpFEu2GhVl2J3OsJQtoJpuxhV16uqCP7THWsJ6ipfe6WfsuXwH9+8FnXKh5zFOibBS2tOnfVWypsEI/Of9aYHVLa/o5fW3jg7v3ucJz77Btff3U3X5vQCfiqn8MsCPKYZNSyr7i50CwWcYqm6zkW1KEvU17qUSHRRRo1LhbmELSylHDn9QSnloBBi5tz72flr4DeBMybehRCfAD4B0NXVNc/DXvpIVzL53aM4mQqxu7pRIwZayv/kOh+EELRH24kZMfaO7+XG1hvZmN7IJ1/6JH+y409455p38tCGh3Cly+6x3bSF27D29OFYFk09vZQLeULxOKnWtrO/2CKxTIf8RNmL1hyJPmUyc8hQubI9zqt9k8SFjl69oAsh+PgtvYQMlS++cIKy5fBrb1s/zTQytcdkyXIW7JiszXqbHC0yOVTCCKoLmrDdtaWBho4oO752mB2PH2bo8CTX3Nm1KEPI5VoWIE0Ta3wc68QJZHlKPwzJijRqLBciGEAxDHrWr+9d6L5zfQQKilkK44QQOnDWj4JCiHfgpSx3zrWdlPLTUsqtUsqtTU1NZz3hy4XK4QzFncME1iYw2iIrfnjoSiRmxLim6RpSwRRRLcof3PQH3NF1B1879DV+76e/x0R5goZgA6OlUXbt+gkADR3eh6umrt5lXWeoFC3y4xXiDSEcyyUQmv6nlo4GuLIjQaZkTusRKYTgwzd089GbenjywCh/9p29mKeNqlEVQUPVMblnIIu9wGbIiiJINXuz3qQL5fzCZr1FkgHe/OGNbL6ljaOvjfH9f3yD8f7FdYSpTQsQAgYPTzI5XETOoyH0xYiTL1A+fJjCjucxDxwAVUNNpU/d0mnURAIlHPbE7RIXNVmpcPxjH8M5cnTBJse5/nK/CvzD1Ois+v3fV587GzcD9wshjgL/CrxFCPH5hZ7g5Yhbscl8/TBCVwhf34LWGLqohoeuJHRVZ2N6I2uSayjbZT6y+SP86nW/ymBhkN966rd4pv8ZYlqEUt8weizCoDlCIBWvNzxeLrJjJayKQ7wphOvIWR2JDTVxK5ozGiA/eF0nv3D7GnYcGecPv/kGZWt6uyxlimNy18nMjOfnQzCi07o2QSQVWPCsN0URbLm1gzs+tAHXkfzwc3vZ++zAopsgn49pARcC6brYExMUX32V0s4XsQcGUeJx1FQaZZl/B1cabqVC5eBBct/7Hs7EBH2//MtYff2LOtYZXZHVaO2PgI8Dx/CyuKuAzwC/K6Wc9xRBIcSbgd/wXZHzI/Pdo+R/dILobR0EN6YJdCcu2jlrK4nJyiR7xvcgpaTiVPjbl/+WfRP7uKXlJjZ+cZxkbxeNN11DQ28PN6y+Zdk+EUspeem7x3juscPc/sH1xNIhWnrjBCOzp9mGJsu83j9JKmzMqFX7wZ4hPvnDA2xojfN779hMZBZHZLZkIRS4qnPhjskaxWyFsT4v6lpIWQCAWbbZ+Z1jnNwzQXN3jO3v7CV0Dv1NzbKNa8uLuizANU3skRGsEydwKxWUUBglfGkbOqbiZLOYR49iHjni3Y4exervB9f78NT1+c9x/MM/C8D7jh7ltXJpQf/JZ/wNlVLawG8JIX4fWFt9+KCUcnEDmnzmRaU/T+HpPvSOCEZP/KIeHrrSSAQSXNd8HQcmDjBWGuN3bvwd/u3gv/G9V77G2mIHRmcjm668AQIaw8VhmsJN5zwtYDacKT0i440hpATNOPPrtCQ8V+Zr/ZM0RALTDCNv3dRCUFf5iyf28d8e283v339F3U1Z41wckzXC8QBGSGNioEhhskIwMv+ibiOoceMDqzm6eoyXnzjOE//3dbbe10PH+oVNHZ96vOWeFrBcOPk81sAA9uAgAEo0hnaOUyJWMlJK7OHhaQJmHj2KMzZW30ZtaMDo7SV8440YPT0Yvb3ozc3oHe2Ljtjmsvu/5/RzBJJCiF1SytxCXkRK+WPgxws+u8sM6bhMfu0Q0pVEbmhHbwhdUsNDVwKGarCpYRMncyc5mj3Ku9a+i7bXLAq8SvPVW/jFp/4f+gv9tEfa+eRbPsm61LolFzfHcsmNlTGCKkZIxSw5Z51U3ZII4krJG4NZGsLTxe3mtY0ENIU//fZefvvfdvNHD1xBOjI9IjoXx2SN2qy3UExnrL+AqroYoflFb0IIeq9qpLFqLPnpVw6x+tomrn5r56JE6WIqC5COgz0xgXXiBE42i9ANlERyRdWKLQXSsjBPnDgViR09innsGLJYnZeoKOgdHQQ3b8bo7fVErKcHNT6zEsw1TTo++Un6fvmX4ejRBZ/LXKnIf5zl4TRwFfDzUsofLvjVzsLlnorM/bSfya8dIrK9ldDVjQR6EojL0OJ8vsiUM+wZ3cOez/wruSN9fOu+HP3FU58Q2yPtPHzfwzSGlraVVmGywtf+eheKJrj1oXUoujdeZj70TZTYO5idEbkB7D6Z4Q+/uYdkWOePHriC5vjM+rtz6TE5FaviMNaXp1ywCUUXNuvNdVxee7Kffc8NEmsIcuMDq0m2LD4N59gulaJNKGYQiuoYIQ3dUBfU5Hk5cMvlU+lG20EJhVBCl8ZAYKdQwDp6lMqUKMw6eRJsb7CtCAQwurtPCVhvL/qqVSiB+X/4qLkit911l/3Knj0L+oQ/VyryY7O+mBDdwKPADQt5IZ+5sTNlst87htoQJLAh5Q0P9UVtWUkGk1yR2sRLA2M0rO6lv/idac/3F/oxHfMMey+eSsnrEblqcxrHdgnF57/e1JEK4UrJgeEc6dMitys7k/zhA1fwe19/jf/yVS9yOz0ym+qYXGiPyanoAZWWnjjZsRITg0X0gDpvS7+iKlx1RyctPXGe/8YRfvDPe7jqjk7Wbm1e1HpZrSzANh0yQxaulAgBqq4QihgEIxp6UEMz1EUL+XyRUuLmclj9/VjDI14rxVgcTbs4zV9SSpzR0bp41SIxe/hU6zk1mcTo6SF87bV1IdNaWs65jk6WKzjlCkf37z+y0H0X/NOWUh6rWv59lgjpSjJfP4ws20Tv7EJLB9Gil5cj6kJhZfKUJzKsueVm2oPt9BemR2y1ziRLSX604jkiG4K4riQQWthrrEqHkVJycDhPQzSAMkUMNrTG+NN3X8nvPv46v/Vvr/KH919BT+P0slPlHHpMTqU26y0UNerrXQuZ9dbSG+ftP7+ZF791jF3fP8Hg4Um2vaP3jCaaOc9FiBni6joupZxJfqKCEN5aSiCkEowaBMKnorqlMJ9I28YeH8c6fhynWEToBmoqdVEZW6RtY/X3n0ojVr+6+by3gRBobW0Ya9cSvfPOeipRSy1urXQ5WbCwCSE2AP742yWk9LrXNit0VSN6U/iyGh56oTn++isAdHSs4X9t+Ut+7clfr6+x/eWb/xJVLL0pYXTAu1DEm0IgqXccWQhdDRGkhEOjeRoi08WttzHKn77nSn73sdf47X/bze/fv4X1LTMNCulIgGzJ4qXjE+fkmDRCGq2r40yOlJgcKWGE1HmvmwXCOjc9uIbDL4+w6wcneOIzr7P9vl5a1yTOvvNZUFQFI3TqZyul9Iw7YyUmRwAJiiYIhDVCUcMrRg+oC+pw4pZKWENDWCf7kK6DEomipRvO+dyXG7dUwjx2bLqInTgBlmd2F4aB3tXlGTpq6cSurosmlTqXeeTreB9yppIG2oCfXc6TupxwSxaT3ziCEtMJXdl4WQ4PvZD07duDUBQiqTRJN83n7v0cY6Ux+gv9DBWHMB2TVHDpPnm7jkt25FSPSClBncMRORddDWEcKTk6WqQhakwTt1WpMH/24FX87mOv8TuPvcZ/f8dmruiYKRZTHZNXdSRIL8IxCZ6IpFojhGIGoydylE2bwDxnvQkhWHNdM42rYux4/DBPPXqAddtauPLNHWc11SwEIQSaPl10XVdilWzKOavenUoPqoQiOoGIjh5Q0XRl2hqilBJ3chLzZB/2+BgoCmosviJbWEkpcTKZGa5Ee3Cw3o5LicUwenuJ33PPqfWw9vYV+X7my1wf0f7itPsSGAMOSCmXfuHhMiXz7SM4kxXib+9GTf//2TvrODnO+/6/n4FlOubTidkySDIzxzHEjpOGG3LTpoEmTRps4Je2aZI22LDjMBpjO+aYY5Bsy7KYT8e4d8swM8/vj9k73Umn0510rHm/Xvva3dmBZxbms9/v8wUP6gnk9ziMD9PI03VgHyU1dQgJbo+XYl8R7cl2/u2pf+Ps6rO5ecnNxHKxI7oBHC9GoWu27lFxeVSMvDzuGohCCOaX+rEkHOxNUuofntNVGfLw1RtX87l7tvCFe7fy2auXc/q8I91GAxGTm04gYnIAj1+3e721J0n0ZnH7tTGLU7jMy6V/v5zNf21m94YOOhtjnHX9AkKlk2clKIpA8WgMdX6ahkWiL0usNw0IhACPT8ftEYhUDDpbELkMwu1BLSqeMe5GaZrk29uPdCX29w+uo1VU4GpoIHDBBbYlNn8+avHMOYeJYrTgkSdHWi6EOFcI8VYp5Qcnb1gnB5n9faQ2duJeHEGvDeIqd5qHTiWZeIJoeytLzzoPhMDlsS+gC8ILWFe5jmdbnuWGhTfQlmibMGGzXWEZQqVeLJMjSmmNFyEEC8v8SClp7ktR4hsubiUBN1+98RT+/Z4t/L/7t/HJK5dy9sIjozwHakzuOM4ak0NRNYXS2qDd660liZEzx1zjUdUUTruinsoFIV687wCP3radUy+rY/6ppVP221A1ZZgYW6k06cZ2+tq7sQDh8aC5g3gsgVua6C4FTRMTFpgyEA0oLQuhKFi5HDIzfPbHymbJHzw4zArLNTYis4X1NA1XbS2+008ftMJc8+ah+Mda5nd2M6ZflRDiVOCtwJuA/YytpJbDKFh5k76799pls04vRy93modONS27tmEZBqX1DSiqilYIRQ64AlzZcCXPtDzDho4NuDQXDUYDHu3E29fksyaJ3gw1y4owDYtg8YnnXgkhWFQeQCJpiWYo8buGiUDYq/Mfb1jNl+7dylcf3MFHL1vCxUvLj9jPQFfug73JE4qYHMAfduP2avS0JkjFcuPq9Va1KMIV71vBhvsO8NKDjXaX7qsbxl315HiRUmIlEuQ7OrBicYSq4ikJDuaeWZYkm7FIJQHs8l66S+DxKLg8CromUDXGLcaDNRL/4R/It7Si11RT8+1vk2tuJv3SyyNW6RA+n22FXXYZ7sJ8mF5Tg9BP3hi/0ebYlgB/B7wF2wX5B+y8t4unaGxzmvjjTRgdKQIX1qCVelHDMzO5dC7TvH0rAEWVVfiC4WEXoXOrz2VheCGPNj7KudXn0pnqpD504t0nYj0ZchmTUIkXaTGuvmejIYRgUVkQS0JbX4bSw+bKAm6NL1+3iq/8ZRvffGQX2bzFVasqj9iPIgRlAQ/diSyZZpOVxxkxOcBgr7dolmhrYly93rwBF+e/eTG7N3Sw+fEWen+2lfXXzqd83uS1dpR5A6MvitHeAfkcuD2o4SOtdUURKC4x6MKUUmKakIibWP0GCHsdt1vg8SroLlvslGNUEVJcrkFRA8i3tNLykY9Q8alPE/3Vr1BLS+3Q+rPPHrTEtLIyx9NzGKN9w3YATwPXSin3AAgh/mVKRjXHyXckSTzdgl4bwDU/bDcPdb6YU4q0LNr37iJYUorq9uA5rPpBxBPhorqLuHXLreyP7UdTNKoD1WjKiQlRT7MdERkus62/iSwFpSiCJeVBLEvSEcseIW5el8oXXr+S/3pgO//3xB4yeZMbTqsZcV8TFTEJhV5vxR48Po2elgTpeKHX2xisNyEES9ZXUlYf4oV79vHkb3ex7OxKVp5fPeaSXmPBSqcxunswu7vt4/p8iHHUbhRCoGmgaUOCTCxJPg+ZtMmAVadpArdXweMRaJpA0wVCCMy+PpLPPUfwssuOKCOVb2lFnzePuttuQw3O3fJbE8lo39absC22x4UQD2JX6HeuvieIZVpE79pTKJtVhavCjzKOflcOE0Mum6Gn6SC1K1YhpMR92EVMV3Sunn81f9r1Jx5rfIyGUAN9mT5KfcdfhURakv4Ou7xQsMQDQk54dQxFESytDCFljK5ElhL/cHFzaQqfed1y/ueRXdz67H7SeZO/W1c34h+rgYjJlwo1Jo83YnLw2B6NivlhYt1p+jpS4+r1VlTp47J3L2fTY03seK6dzgNxzrx+PoGi43cPS8vCjMUwOjuRiQRoOiIQmLBSV0IR6Aro+qH31jQl6aRJMi6x0knE5o3IV5/D2rUVpMR35vojaiTqNdUoHvdJJWpSSjAMZKGSyXgZLXjkLuCuQquaG4B/ASqEED8A7pJSPnxcRzzJSb3YTu5ADN+ZlejlPscFOU10Nx4gl05RWteAUBR095EXyJpADedWn8uDBx4klU9xMH6QEm/JcVvXpmER682guVW7xqKcuICDoaiKYFlVCKutn95EluLDxE1XFT5xxVI8msJvXzxIKmfynnMbRjyvoRGTy6pCVEdOLEJRUQSRch/egD7upG7NpbL26gYq54fZ+MABHvnZNk6/Yh4LTi9F01SkBCHANC2M3NFb7Mh8HqM3itHZgTQMhNuDMkmd0g9HMbJYr72M3Pgc1rZNYBiIknLEpdehnHo2UVFG1be+Q9tHPzw4x1b97e+QjmUwUyaDtsXwu2GM+FYevr444qURdzDS+kc7jjjqkyPXlVIiTBNZEC9pGIe6gRc2UDweFK+XrJTjzps+pn9BSpkEfgP8RghRDNwMfApwhG2cmLEs/Q83opV68SwtQq/wOc1Dp4mDWzcDUFxTizccHvHCOhBE8uCBB3m65WmuariKeD5OyHV8czxG3iLRm7ErjhgST3DyJvdVRbC8MsTW1hi9ySPFTVUEH750MV5d5e5NLWQNkw9cuHBYLtwAAxGT29tipHMm808gYnIAt0+ncmGY/o4UsZ4Mbt/Y0wJqlxVRXO3nxXv30bi1h5qlRdz/89eI92QIlni46pZVaC5lmLhJCTKVxOjuxuztBSEQXh+Kb/KjBKVhYG57FWPj3zBeewmyWUS4CP2CK9DWno0yb+Gh718+SyzlofJHP0NBYkpBd3uKbCzB0LTigUcDn8KAJoz0PT58XQRHZiiPuP6RKx16bfTPX0oJpgmmibRMpHF4/zyBcOkobjfCHUDxeuzmqaqG0Au3wrkIbfyhw+PaQErZC/yocHMYJ9F79iIzBv7L6tHLfU7z0Gmkddd2XD4f3kAQX/DoofynlJ3CKWWn8PjBx7m64WpaE62Eio9X2EzivVlqlkSO2lx0ItFUhZXVtrhFUzmKfMNzJBUhuOWCBXhdKn96qZlM3uQjly45orgyHIqYbOxNkjVMllScWMQkgKoqFFcH8AR1epqTGDlrzO+JL+TiwrcsRVqSh3+2jXiPnfQe78nw4I+3cMV7V3JgczeqBqqZQ8T7UMwsukdDDwbQdYVJ6Eg0iLQszF3bbDF75UVIJ8EfQFt3Htrac1AXLTuqy9NIZogmM4PPBeDxzqyiDQPCJQfEawSXoXC7UdxeFK8HPB4UXUdoOsKejBy04kYsxC8P3QmhjHuuxrmyThGpoWWzqvxoxbOjNM1cxDTydB88QPm8BQhFweU9epBAkaeIi+su5tWuV3mt+zVWla1ivjkftzp+F3K8J0MubRAqtVvQ6FMwt6qpCiuqQ2xp6acvnSPiHS5uQgjeeXYDHl3lV883kslbfOLKpegjiNZAxGRnPEs6f+IRkwP4gm7ci3R625Mk+jJ4/fqYAkOEInD79EFRGyDek8E0LF56sHGErXKFm+0e0/RDQRyaJtAHnzNsuaYfuY465PHApdfav9sWs5efR8b6we1BW7MObe3ZqMtXI9RjX3L1oBd3ZRkoGlgG2fYu8vGpbYM5duFyH1O4jsWIbuhRXK1jwRG2KcDKmPT/eS9KyIX3lDJcVQHHBTmN9HV2kOjtYdG6sxFC4PIcPQBBUzQuqbuE3+34HQ83PszqstV0pjqpC9aN+7g9rXYH6lCJF4E4rhqRx4OuKqyqCbO5uY/+dI6w98jqNm9aW4dHV/nJ0/v4yv3b+PTVy48qWiUTGDE5gKorlNYG8AbG1+tNCDsQZ6i4BUs8eHWDSy9WMXUfpikwDGnf8oVb4XHeGP48l7NIJYesO8bYBYGFamTQDIFqrkU75Sz0kB89ErLLcuUF2vYcmpY/QiiH3vuKfSilFdz7w22HXKvvW4FOx4SK21QK13QwWh7bIqBCSvnsYcvPB1qllHsne3Bzhf4H9mH25wheMc92QZ5gtQmHE6N522sAlNTU4QkEjxkFV+4v54LaC/jTrj/Rne5GExrV/mrUcXpIogVhC5Z6EApT2i9MVxVW10TY3NxHLJMn5Dlyfu+6NdV4dYXv/nUPX7x3K//++hX4XCN/Vyc6YhLsf+6BIg9uv05Pc4JULH/MXm+maXHVLat48MdbDgnBe5eT7erAX3bi1WLswslgGJL8UEHs7iW7ZzfZxiaMVBpT82GVVmOV1WAGazEsBcOQZOISI5ofs0he/YE6HvvpYa7Vn27j4rcu4aVHdtgWoi5Gvdf1QnK4Zc1Z4ToWo11hvwV8ZoTl6cJr107CeOYc2QP9JF9sx704gnteEK3kxKtXOJwYLTu2oagagZJSfKFjX/z8up9L513KPXvv4bGDj/GmJW8imomOK/TfNCxiPWk0l4LLa1eQn+rcRZemsLo2zGtN/cTSeUIjtMu5fEUlHl3lfx7Zxefu3sKXrltJcAQRhOERk8urQlSdYMTkALrL7vUW703T2zZ6rzcjZ6G5FK79wHLbfJMW2fZOjERmxPXHi1042XZZutI9GC89h/HS37AO7rcj9xYuRT/nHLTTz0QERp97HSqSI1mLRl4SiIzsWtXcGtEew94uLxlpWupwNM3uHKG7lcJ7aPel0z0aulfDNbAspaJ7FHQ36G4L3W2iu0FzT37/usliNGFrkFJuPnyhlHKjEKJh8oY0d7AMi+idexBuFd8Z5eiVAad56DQjLYuOfbspqa1DUVXcY6ydtzSylHUVdv3INy5+I82J5vEJW94i0ZslVOLFMsAXmJ5yR25NZVVtmFeb+ohn8iOK1vmLy3BrCl99cAefues1vnz9qiMCTwYYjJgs1JhsKDnxiEmw589CpT48fhfdLQnSiUJS9wh/BvLxNOlt21GCwQn/syDjMYxXXiC/8W9Ye3YAoNQvwHXT29FOPwulaOwtaoaKJEf5D6Cr1oiuVbeW5+Lz7WuHlBLLAkNxYQoXlqKTFxqmqZA37CbW+bzEyJrkh9xyWZNkPDP43DKPrY6qrgz+sRj3zWPfH28iveZSUFWFhYvmzx/3tqO8Nppp4UQ+jIH4E00YnSkCF9biqgygHkcDRYeJJZ2IEW1rZfl5F9nNKUeZXxtKxBPh4vqLebb1WV5of4Gzq84mnosTdI0tadbIW8R7M1QtCmOZEtcU1TwcCY+usqYuwqamPhIZg8AI0bnr55fwhdev5P/dv41P3/ka/+/6VZQFR3Y3qoqgxO/mQE+STH5iIiYHcHk1KueP3uvN6OkBMf66jEdDplMYr27A2Pg3zB1bwLJQqmpwXXsz2hlno5RXTchxRiLT3MZV713Og7duHzbHZmRSuBctnFBXoWlYw4Rv2C1jYuSOXJbPmKT6c4PLTOPo+YIDKJoYoxhqg48DxW58ITf3/WQz/Z3ZCQ333yCEeL+U8idDFwoh3gu8NN4DnWzku9LEn2hGrwvgXhhGm8TWGw5jp2XHdqRlUVJbjycQQBnjPJmmaJxZeSYLIwt5+MDDnFdzHm2JNoLFYxO2RDRDNmXYLVgEY664MVl4dJVT6yK8cjBKImuMGACypi7C/7t+FV+6dyufunMz/+/6VUdN0J6siEkYqddbHrfPznOSeQOjsxPFHzihY8hcDnPLy+Q3/A1z6yYw8oiSMvTLr7XFrKZ+UlzHUkq7In8uhwSk4UONJLnuw6cAyqGEcyXARH9jBroYHE/H8gEsc0AcB+6No4vlgDhmTTKJ/OCykZLpr/7Aah76yZYj3LJjZTRh+yh25ZG3cUjI1gIu4A3HdbSTBCkl0Tt2AeBfX4le5Xeah84QBgJHIhWV+EYobjsa5b5yLqy5kJ9t/Rn7+vdhWibzwvPGFPrf01Loml3iAeSURUSOhi1uRbx8MEoya+AfQdyWV4X4yg2r+fc/b7EttxtWUV989PSIEr+b/gmOmBwc79Beb9Esbp+G1dcHkuMqgyUNA3P7Zjs8f/NLkM0gQhH08y9FW3sOSsOiyREzy0JmMnbVE0AJhVCrq1ADgcGK/LmMBRzbGppuFFXB7VNwj72s5hFISx4hgOEy73GLGoxeUqsDOEcIcTGwqrD4finlX4/7aCcJiULZLP+ZlejVAdSA0zx0ptC2Zyehsgp0jxe3d3z/8n26jwtqL+D23bfzaOOjvH/V++lKdVEbrD3mtr1tdkRkoMSDoioT2hn6RPC6VE6rj/DywSipnDFiFOSi8gBfvdHuxv2pOzfz5etWsaj86O9deBIiJgcY2uutuylG7mAb7sDYr6rSsjB3b7fFbNOLkEyAz4+29hw712zxigmrFQmFsHoJlmlipdJgmUihoEQiKOEIitsDqoohIZ+VkM0DEuRAIpdEIBCK3RlAUQRCnZxSbNOFUAQurzYsvUPTlSPmGsfDWP9OySE3h1Ew4jliD+xHK/PiWV6Cq9yp3D9TMHJZupsambf6VJDg8o7fPTwvNI9zqs/hoQMPkV6epjneTJW/atTQf8uS9HemUXUFt1fDNcMqzvhcGqcVLDeRM/GO4CatL/bx1ZtW87m7t/DZu1/jC9euZEXV0aMAJyticgB/2I2SkrSTJ5Pz4lbkUS/2UkqsA3sxXvobxkvPIfv7wO1GO2Ut6hlnoy47BTQNKe0WZ9IqRB1K+16CrTWDV7/hRaeG1PMffGa7FQ1kJoOCherS0UqK0SMR1IAPoRUiYxXb6hkqWEIIhGKXrbIsiZm33XX5vImZt11+liGHHQ9hj0lRhoufKOxvtjE0jeN4GC2PrQa7oWgG2xUpgDcJIf4beIOUsuW4jjjH6bt7DzJr4r+iCr3Sj5gBLicHm/Z9e8lnMpTU1uPy+VDU8c9aFHmKuLD2Qh468BBPND3BFQ1XHDP03zTswJFQiWdKSmkdD363xmn1RbzSGEUIRpwfqwp7bcvtni38+z1b+Nw1Kzi1LnLUfU5WxCQU5qZamyirC5Bz+dF9HlRhYUqFaHuKTCyFbG1Cbnoeuek56OkEVYMVp6KcejZixWlIlxsTsCQIAxQBigKKCgq2e1MoYnC5UCgID3atSfuuIEJ28IqVSUM2jUCi+Ly4quajFhWh+Cf2D660JJYpMU0Ly5RIS2IaFkbeLgBtFETQNMyCIA+XYFHoF6eoh6zBmSSAA2kc13/kVL5+t3vcJf5H+4V9D/iBlPLnQxcKId4JfB+4frwHm+ukt/fYZbPWlOKqC6KGHRfkTKJ5uz2/VlRVg/84q7mrisrqstWsKVvDX5v+yjULrqE5OXro/0Cof8X8ENJixllsAwTcGmvqI7zSGAVGFreyoJv/unE1/37PFr5071Y+ffUy1s8/esj7ZEVMWrEYZjKJXl2Nms3S8oH3HKqG//Wv0/X7X5B67BFQFNyrVuN94xvxrV9fEBiG3E7sYi4tC5lOY2UzCAR6OIQ2vw4tHEY5Do/AWBGKQFXEMZP87dQAWwTtmy2ERs62As28LYbZrIEcnNI7JIKDYj7UFTpFblAjZ2FgsXfP/v3j3Xa0X9gKKeURQSJSyl8KIT473gPNdcyMQfSuPYNls5zmoTOPlh3b8PgDeIIh3IHjj6Ir85Zxfs35bOraxKtdr7KyZCWJXIKAa+R9pvqzZJJ5QoXmolNZcWS8hDw6p9YX8crBo4tbkc/Ff75hNV+8dyv/+cAOPn75Es5fXHbUfU5GxGSuqdluNzNCx+nWT3yCyi99Cc/8efjPPnvEDtgngjRNrFQKmc8hhIJWXIRrwXzUcBjFNbP+zAohUFXBWJwTA6I3VAiNvB32b+RtEcwN5L/Jme0GHU3YRnwrhBDK0V47mYk9sB8rliN0ZQOu6oDTPHSGIS2LzgP7KGtYgHKM+pDHwqf7OLPqTP646488fOBh1pSvoT3VziLXohHX726OA3aNSISc9lD/YxH26pxWV8QrTbZb0q0dOd6gR+f/Xb+KL9+3ja8/tJN03uSKFZWj7neiIiatZBKjpwetpARpWSN3nK6tI3RV0XHtfyRkPm+LmWkgVBWtvByttBQ1GLTzyuYAiqowluyXw92gA4+NnIlZcIMaeQvLkMihLtCCS3TQ/TnEFTrRAjjaJ3KvEOInwEcLPdkoNB39JvCXCR3FLCfbGLPLZi0pwr0g7DQPnYFEO9pI9UVZetZ56F4fqnZiyfJ1oTrOrzmf23ffTleqC9M0qQ/W41KP/Mc+UPw4UORGd2mzIqIt7NNZUxtmU1M/wiNwjRDF6XNpfPHalfzXAzv47l/3kMmbXLemZvT9TkDEZK6tbVBMhKKM2HF6ItxlVjaLlU4hLAvhdqNXV6GVlKBMYJft2ciJuEEH5gFtASzMBWbM0d2gxzHG0YTtk8B/AY1CiMbCEecBv2DkGpInJdK0c9aER8O3tgK90mkeOhMZyF8rqqoZd/7aSIRdYc6vOZ97993Lo42P8qYlb6I71U11sHrYelJK+tpTqJrA7demteLIeCnyu1lTG+bV5n7CXn3EVjYeXeVz1yzn6w/t5CdP7yeTt3jT2tE7H5xIxKSVzWK0tQ12vE5v30HVV75C2+c+NzjHVvOd72DlcuM6VygEpGQyyHQaiUT1+XA3NKBGilD8PmdqYZwclxt0iAVoFoTPsPLj/jBHy2PLA/8qhPg8sAhbRvdIKVPjPchcJvb4QYzOtF02qzqAMoEJqQ4TR9O2Lai6TrCsDM8JVqkAO4hkUdEi1lXa9SP/btnf0ZRoojJQiTKkg6VlSOK9GYKlXjsicpZ1digOuFldE2JzSz8Rr2tEcdNVhX+7ahnffmwXv3q+kXTO5J1nzxtVCFyaQpHPNRgxOb90bHPSRmenHZFYsJh6b70VJRig7tafAbY1YeVyyEx2TOcnLQsrnUZmMwghUMJhXPX1qOEQygm4qx3Gx2hu0JyRHXcy2zHtaSllWkr5mpRys5QyJYS4XAjxyHgPNBexy2Y14aoL4llahFbs/BBmKh17d1NSW4+iqKM2Fh0PZb4yzqs5j5yV45mWZ8iZOfqyfcPWMQyLeG+2UHGEo1apn8mUBj2srgnTl8qRN0euhqEqgo9etoSrV1Vy+8vN/PipfVjHKEGvqQolfrsr9/a2GMZR9j2ANAxyBw+iBOwyZtldu8hs3ox7wUKsRAIrkcCMxY8patI0MeNxzN4erFg/WiiId+VK/Geeie+UU9Aryh1Rm+WMlsd2CfBDoBq4G/hP4JfYltt/TMXgZjKHymYJfGdW4ap2mofOVFLxGH3tbay44BJ0txtNn5hi1F7Ny5qyNSyKLOKRA49wUd1FNMebKfYUD66TjufIJPJ2jUg5dc1FJ5qyoIeV1bC1LUaRVx8xZF8Rgn+8cCEeXeWuV1pI500+dMli1FF+F4oQlPrHFjFp9PQgTXNwfq3vzjtRAgGCV1xxzPEfCv4wUbRC8EdJyZwK/nA4xGif6P8AtwDPAVcDzwOfl1J+eyoGNtNJbrDLZvnOrMRdH0SZoblJDtCyfStSWhRX1+I7zvy1o1ETqOH8mvO5bett7OnbQ22glmQ+iV+32+F0NxUiIqehuehEUxH2IKVka1uMEr97RMESQvDucxrw6iq/ffEgGcPi45cvGdGFOZSBiMlXChGTh9etlJZFrrFx0FrLHThAeuNGIm9+81HzxaxsFiuVREiJ8Hhw1dagFhfbuWwncfDHycBon66UUj4hpcxKKe8GuhxRszHiOfr/YpfN8q4qQStxKvfPZOzEbEGkohJvcPRmkOMl7A6zrnIdIVeIhw88jK7otCfbB18fiIj0R9y4POqsD0CojHhZXhWiJ5nFtEZ2NQoheMv6et577nye3dPNf/5lO1nDPOa+w14dRQg2HuilNzHcnWj29yPT6cE8sb4770T4fARf97rBdaSUWOk0ZrQXo7fXTlVYsADfGWfgW7cO17x5toXmiNqcZzQzIyKEuHHIczH0uZTyzskb1sym7+49yJyJ/5wq3DVBhDq7L1ZzndZdO4hUVKJ5vGPuvzZWFKHQEG44VD/SSNOebKc+WI+u6kTbkyiawOPTcY/QsXo2Uh3xYknJzvb4US03gBtOq8Gjq3z/iT18+d5tfO6aFSPWoRyKz6WhKoJNzX0srzwUMZlrbET4bCs419JC6rnnCN9wA2qhUayVSiEzadTiYjQn+OOkZ7S/Lk8C1w65DX3++skf2swkva1QNmtVGZ75ERTf3LhYzVXMfJ7ugwcomzcfzaWjuyY+x7DMaweRAPy1yW5+0ZXuwjItYt12jUgpmVWh/seitsjHkoogvamjW24AV62q5GOXL2FLaz+fv2cLicyxy/65NZUirx0xua8rgRGLYfb3D7oc+++6C6HrhF5/6DJkpVJ4Tz0V76pVTvCHw6jh/u+eyoHMBqycSfTuPahhF74zypzmobOA9n27MXI5iqtr8YYmtrTSAB7Nw+LIYtaUr+Hxg49z/YLraY43U6KVkejNUFYXnBHNRSeaumIfliXZ25WgJOBGOYqb9aKl5bg1ha89tJPP3v0aX75+FeFjWK9DIyYzHU3U6LYLMt/ZSfKppwhdffVgqSwzkbCrgIQm1s3sMHtxnM3joP8vdtks/znV6NVBp3noLKB5m932oqiqGt8Ez68NpTpQzfnV5xPLxXi582WyRpau3h7S8Tyh0pnTXHSimVfqZ36Zn55kdtTw/rMXlvL516+guS/Np+7cTE/i2HlmihCUqhBtbmNv0iJvmsTuvhsUhdB11wGFfmeZDK6GeRN1Sg5zgLn3S5skso0xki+04V5ShGdxEVpwZhU7dRiZ5h1b8YbC+MKR4+q/NlZC7hCry1ZT6avkocaH8Ogedu9tAiBQ4p1RzUUnmoYSPw0lfnoSOVtojsLp9UV8+bqV9CRy/Nudm2mPjSHvtquDkN9N1oCdO5qI//WvBC6+GK3E7ihgJRKoZaWoJ1DU2mHuMTd/aROMNC2it9tls/zrK9HLJibB12FykVLSsXc3ZfUNKJqGNgnzawMoQqE+WM/5teezO7qbjlQHPS0JAAIR16yrODIehBDML/VTX+KjO5kdVdxWVof5yg2rSGVNPnXHZpqioxQyyuehox38QQJuDdfjDyEtC+3qa4CCtZbL4prnWGsOwxmTsAkhzhFCvFUI8c6B22QPbCYRe7wJoyuN/8xKXHVBp3noLCHa3ko6HqOkth5fKDzpofYl3hLOrDoTl+LikQOPkOu2OyG7/fqMbC46kQghWFjmp7bIe0xxW1IR5D/fsBpTSj5952vs60qMvGJPt93SWlEgHkP/25OYa89it+WlJ5HBSiTQyssda83hCI55hRZC/Ar4BnAesK5wWzvJ45ox5LtSxB9vQq8L4l1VihpyXJCzhaathcLH1TWTFjgyFI/moT5Yz5lVZ/JMyzNkoxZ6GExpztjmohOJEILF5UGqI156kqO7JRtK/Xz1Daegqwqfufs1drTHhq9gmtDaBIWEbOWvD9sW3FWvJ+TR2deVoK27H61uSMFl0yj0CXM42RnLr20tdtPRk+4bI6UkevtuhCIInFOFXu5U+J5NtGzfguZyEy4tn9T5taFUB6o5r+Y8nm55mng0Q7jSSzwXQ9WP3mV6LiGEYEl5ECmhvT9D6ShtaWqKvPz3jav53D1b+Pw9W/j3a1awujZiv9jfB0YeNA1SScSTjyFPWwuV1ahAxMrT6gqQj5ks8VpoyQ7o2IIdfuoG1QWax36seUD3gKKBohbudftenft/OE5GxvKpbgEqgbZJHsuMI/liO7nGGP6zqnA3hJ3mobOMtj07KamrR9V1dPfU5DUFXUEWRRax2LcMJeXGV6zRmepilb5gSo4/E1AUwdKKoD3HGc9Q6j/6e18e8vDVG0/h8/ds4Yv3buPTVy9j7bwiaGkCn+1iFE8+hsikMa8s5K1JiTANihctoCueJZPNssrYgcsTsgVLmmCZkEtAth8sw35u761wX/ifLpSCCLpBdReE0Au6uyCAQ26qzpg6cTpMO2MRtlJgmxDiRWAwRldKed2kjWoGYA6UzSr34j2tzGkeOstIx2P0tbez8sJL8IRCU2ZpK0KhNlDLOa5LEAh6Pe0ElHIS+TgRNTIlY5gJKIpgaWUIKaErkaXEf/TfT7HfxX++YTVf+PMW/uMv2/nX8+s4N5WComLIZFD++jDWqjVQVwgSSSagtAy8PoqBTPcB9sZ6qK0L4NcFyoAQjQUphwhhHDJ9h54Pa3E5VAjdoA1YhANWobtgBapDRFBzhHCaGMun/8Xj3bkQ4gAQB0zAkFLOmrm56N17kHmLwDk1uCqdyv2zjebtWwBJUXUtvuDkz68NpcRXQn1mMV3ABvkUS/1voznRTMQTmdJxTDeqIlhWFcJq66fnGOIW9ur8xw2r+dJ92/jakwf58BIXlxaBePYJRDKBefW19opS2nNtVXanbmHmKMkcJOmOsLM9jioExQEXYY+O362hH6vcnRAgBoRwDH9eB4XQgGwM0lH7sbQYWQjVQxah7rFFceB+wBU6zDJ0hHAiOKawSSmfPMFjXCyl7D7BfUwp6a2FslmnlOJeHEGZhT20Tnaatm1BCIXi6lrcvqlNz3CrbtQ+P1Ik2Wg8zZv0G4hm8qTyKXz6yZUqoiqC5ZUhtrbG6E1mKR5F3PxujS9f1sB/3LOZb+3KkbNivP6RB7CWroD5i+yVEnEor4DCnKmWaAEp8XjceLCDKPuSObritnMp4NYo8bkIeDQ8usoJG+7DhHAMSMu2/qRhW4PWgDUoB7WvsGJh/5rtBh0QQ0WzrUShHrpXFFsAhQKIwvLCPRx6fsRr4rDHA6/NvT/tx/x0hBBnAd8FlgMuQAWSUso5Wb/GyhqDZbP86yvRipyac7OR1l3biVRWobvdUza/NhQjqqCFJJZi8WTX47yu6Go6U500hBumfCzTjaYqrKwOsaU1RjSVo8h39MhiT28nn1/h4mt7ofGRJxCxfqx3f8B+0bLAMKCyGgBhZHDHGjE8hyxyRQHfQMsbCTnT4mA0hQQ0RVDidxH2uvC5VbSp8MIIBVQFGGNN2QEhNHNgpAsWosQWwoF769C6g1biaOdyjLg/IYCCYA4IJ+KQeA6IqqIWLMqB19QxCu1IYjq5QjuWvx3fA/4O+BN2hOQ7gcVj3L8EHhZCSOBHUsofH76CEOIW7L5v1NfXj3G3k0f/X/ZjxXOErm7AVRt0XJCzENMw6G48wILT1+EJTH2bEsu0SEXz+EtcrPSv4Ym2x7lp+Y20JFqoCdagKydf4ewBcXutuY++dI6IdwRxy2aguwtXMMynVplkfvcE24rn8aJSx9ukRCQTUHHIWnPFDyKFgscXojQSQhMSQwq6+2Jk0kkQ4NIUXIWKL6Yp6U7k6IxnkUDIo1Pi1/G5NbxHaW465QwIoTqF35GRBFMWnlsGw0V1NKEdKqBDg3QOv4aOT2iDLsbt5hiTPS2l3COEUKWUJnCbEOJvY9z/uVLKViFEOfCIEGKHlPKpw/b9Y+DHAGvXrp3WlIJsYz/JF9pxL4ngXVHiNA+dpbTt2Ylp5CmumfjGomMhk8yT6s8xb1mYMwPnsaVjExs6NrCyeCU96R4q/ZVTPqaZgK4qrKqJ8FpLH/3pHOHDxa27276oKQr6C8/iSfTy+DVv4g/7M6QNyfsq84hKe25NySfR481oxfVU+QXa794AfQfRI/VUvfGXtOG3xW0IqioIqIesuYxhsr8nD4BbVSjyuwh79ULrnEl/O2YOQ92WM4HDBFNTxqZTQxnLmaSEEC5gkxDia0KIfwH8YxufbC3cdwJ3AevHO8CpQhoW0dt3o/g0/OdUoxU7lftnKwOJ2cU1dVM+vwbQ22ZfUEvKwiwJLaXCV8HDBx7G7/LTHG8eNXF5ruPSFFbXRPBoKrFM/tALhgFtLeAPgmWhPHQfsm4er796PdfVe/hzU5bvdXgxC2XR9NgBLM1FaVEY7fZ3Qt9Bez99B9FufycVXhNfx0vo8SaEMUJNSgEeXSXs1Ql7dTRVoSueZWdHnFeb+9jTmaA3mSNrWFPwrjgMY8A96QqCr4T6hcvmj3cXY1HCd2AL4D8D/wLUATcde2zCDyhSynjh8RXAl8c7wKki/oRdNitwUS2e+pDTPHQW07pzG75IEd5QaMIbi46F7ma7RFQo4sVVXMqFyoX8cecfaU20EnQFieVihN1TG6k5k3BpCqvrwmxu6iOWzhPy6hDtseeWVBWx8QVEZwfm+z+IUBTet9iL18zyh4MpMo/s4uMXVqKnOjE8xWhCHhK1AfoO4rKy1Dz3hcFFhiuM4Ssn7yvD8FWQ95YXntv3mu4ftOakhHTOZG86gUDg0RRKAm6CXg2fpuE04J4CNC8YGfjtzWi9u8ZtsY0lKrJRCOEFqqSUXxrHviuAuwr5QxrwWynlg+Md4FSQ70oRK5TN8p9W7jQPncVIKWnfs4vy+Qvx+AMo0xA+3duaRCjgC7ooLq5gnXsdd+++m4cbH+Zty99GS7zlpBY2sJuJrq6N8GpTH/FUlmBL8yFr7cF7kZXVyDVnACBSCd5+eg3eeRo//9sBjGQvn1/nQhMC08ihR+qHi1uknoy7hK7zv4ae6kBLddn36U7csYP42zeiWLlh4zF1P4bXFroBsTO85eT9FWRkKW19Fi19oApBxKcT8bnwu1VcJ5XPcpIw85BL2gn1uQRkE1C7Dv74jiP/tIyRsURFXotdK9IFzBdCnAp8+VgJ2lLKfcCa4xrVFCItSfRPuxCqIHheNVqJ44KczUTbWsgkExTX1uELT4949HWkCBR5UFSFokCIUquUs6vP5tmWZ3nrsrfSnek+KUP/D8ejq6ypi/Dqa/tIx1N4K/yIza8gWpsx33WLHTxgWXbdyMpKbprnwUeW7/+tjc8+r/LF8yXyb/+HvO57iD//s30RjNRjvPGXdMXSZEpWkClZceSBpUTN9qGlO9FTnWgp+96+tePr3oxipIdtYqkeW/S8ZWQ9ZSTdZfR6ylBCVfhLq/FHyvC5XHMxcn50pLSjN3NJW5CGCtTgsiGClR+6rPDYHKE339/ff9yiBmNP0F4PPGGfh9wkhGg47iPOMJIb2skdjNuV+xdGnOahs5ymLZsBKK2dh9s39VXfpSWJdaeJVPgAie5SqQnWcE71OTzZ/CRPNT/FuTXn0pXqYl7Yabfi1hQWG/3s9nlJZ/MEHrwXWVqGXHumvUIiZidjuz0gJTdW9xBYq/KNl0xuf/RZvpz5DdGiJXjecteRUZFHQwhMTxGmp4hs0dIjX5cSJZ8oiF4HeqqrIIK29eeN7qIkHx+2iSU0st5yZKAcNVRVuFVCsAKCleArGXvum+a1K5tIy55rMgqh/5OBZRbEaKggJYaL1DDBGrqscJPHmIdUdHAHwOUHV8C++cuOXOYKFJYFwFsMh1vi42As77Qhpeyfi8V/zdhA2SwfvjMr0QJO5f7ZTvOOrehuD8GSUlyeqbe+s2mDZH+O+hUlKJrdXLRUlFIfrGdRZBGPND7C5fMupyXZQnWw+qQM/R+KFY+jp1MsmVfK3idfQBzYh/nWvwdVta01S0KFHUWqprtRcwkuX1SKx5Xmgpd+SYtSSX/Z2QRkGAsNRRpI4qMf9FgIgeUKknUFyUYWjrxKPoU+xOIbuKnJDrSDz6Hm+oatL4WC8JfZIheoLNxXHBI+f7kd4j9kbmnAAuVNvyosH0HcjOwIgpMc2WI6YlkS8qP0wxtA9xaEpyBCvoLoDF3mPspjV8BONh+vfgjFPu8/vgPYNr5tGWMRZCHEWwFVCLEY+DAw1nD/GU307t3IvEXw/Gpc5WMK9HSY4bTt3kFJbR3uQABFnZ75NST4i9y4C+kiuqpT7ivnwtoLuXXLrezo3UFtoJbedC8V/oopH+NMIt/cjHC78WgakSceJBuOkD7jTDxQsNaqweUGy8TdtxvTbbexeX3+UcpFK/8d+ncu8C/kll9spjmaprbIyw/fuoYgB8mlj9LnbQKQuo+c3kAu1DDi68LIYsU7INGBK92JN9tFINeNO9OF2voyItnN8HwuYVt1N/4Y7vvosChP/vgOuP7/4NEvDRexXMKenxoNoQyxiPy26IRrhwiP/5CVNNRiGnzdN3ZLcyIx0qB5ke+8B+On1xjj3XwsI/4Q8FnsAsi/Ax4C/t94DzTTSG3pJrOtF++aMtxLix0X5BwgHY/R39lBzbKV+MNF0zKGnlb7YuoPu3ANaS5aGajklNJTCLqCPHzgYT50+odoijdR7is/aVshWakU+e5u1KJiMtu3k9++nfA730m/oiGyedySQWtNS3WiGFkMVwAlF6dkx29Ila3h+mvfzvt+Z4saQHM0zQd++yq/fddqSO+YtnOTmhtRVA9F9eSBjClpzZuYUiIEBHUoE/34cz240p0Qb4dEB7iDI0Z5orrs+Sl3EIJVx7CShizTvFNSMsuywEQipURKsKQs3ABpG96WtF83TYkhJaZlr2NaFoZlv26aEnPwtSiWhIONB9rHO56xREWmsIXts8dxvjMSK2PQd/ce1Igb/7nVqEHHBTkXaNq2BYCSunm4/dNjgfc0JxACfGHXsOaiQT1IyB3iwtoLuX/f/cRyMYQUJ3Xof669HaFqCCHov+MOlFCIoiuvxCtUdu1qgto63C43WHnc/XsxPHYVv+Idv0XJp+ha9T48btegqA3QHE3TkVb4n6dSzI8oNIRV5ocV6kLK1JTRGoGRksP3GmEgjDuwiOIKFyGPTsBfghghypNAOdzwgxMagxwiMFZBgKSUtiAVio2YUiKRWJYtLqYFpmVhSjDNwr0ceN2yXz8sL3OkmiODa0hQhBjUWnXgsbCXK9JCt/J4pIkqcwM7HPeHdlRhE0L8ebQNZ3Pbmv6/7MdK5glfswB3lf+k/cc812je9hpCUSiuqsY1DflrANGOFP6IG1VV0VyHvABCCGqDtZxZeSb377ufxxof45oF19CaaD0phc3K5TBaWlHCYbJ79pDetInI296G4nbjNU0WlgfYHilB5k2C6TaEZYCio8ebiOy/n1jDFeTC83FLg9oi7zBxqy3yYhh52hMmG9oMzMJVVVOgLqgME7v5EZVyn5jaa0AhOdxTKONlmJLOWJa2/gyBJCx+4y9RB5LOI/XIN/2KTCaNkTVOwPopWE+DQxCAHBQc+9nw14WwlyvC/v7azwVKQYRUVUXRxfgNQmkizDzCsm9YIAoyKIWKpfsxXUUYegCpeYjn1REy7EdnNIvtbKAJ2/34AqNX2Zw1ZA/0k3yxHffSIryrShAzpUacwwnTums7RZXVeIJhVG16gjJiXWlCZYXK8/pw93aJp4RSXymnlZ/GXw/+lTcsegPd6W7SRhqvdnKlmRgdnYBEKAr9d96J4vcTuvJKAKx4jMjC+ZxaXsGrBzoIRfdieCMAlG25FUv10LP87QDI/mZ++NY1fOC3rx6aY3vbaQTSrfz0SjdmOsHBhGR/TGFfDPYnYEuXxV8bD03b+HQGha4hrDI/ojA/rBJyT80lTxtizUkjw/5+L5VvvgNdgbwlaO6Kkkj2jt/6QaApApdqi9GUXsEtc1C4hJljqNklFQ1T92O6I1h6EKl5kKoLS3XZEZSH70oy7vIvowlbJXA58BbgrcD9wO+klFvHe5CZgl02axeKXyN4QY3TPHQOYeRydDXuZ8Hp66ctfy2XNkj2ZaldVoTmUlAOS97VVZ1KfyXn157Py50vs6FjA6tKVtGZ6mRe6OQJ/ZeGQb7pIEowRO7gQVIvvkj45ptRfD6kaXe61quqcLl0Tg3G2ddh4nYrhLpewt+xka6V78F0RwDIpRNEslv57d+fiqm6UQTkDIs4teCtBSmpMrNUmxnONzIo+QRKLk46GedAv8X+mGB/DPbFJU8dFNyfPxSMUeIVNIRtkRsQu/qQglubPIUQAmQ+TVvHIQtUxe5XN+OwTISVK4iX/UdhwPKyFB1L92G6iwri5S6Il3tKglGOeoRCweMHgQeFEG5sgXtCCPFlKeV3J31kk0DsrwcxujMEL63DVTt1XZUdJp/2vbuwTJOSunq8geC0jKG3LYmUEIi4cXtH/mlV+CpYHFlMpa+Shw88zFlVZ9lV/wM1aNMRfTYNGL29WIaBpmn033knwuMh9LrXAWDF+nE1NCBcLsinCaWbWVBTzc6OBAte+yk5fxX9C64d3JcwMmQtk1RWgjJC8JwQtkWgebCG/I8VUrLAyrHQzKIYWZRcHJGL0RtL0Nhnsi8O+2OCfTGLezpN8gWbQRFQHVAG3ZgNYdu1WeVXUOdiJxDLGGJ55YdNd1mqy3YbekqwNNttaKkupOqankjKIYx69IKgXYMtag3Ad4A7J39YE0++I0n8iWZc80L4zqh0mofOMZq22onZJbX16NOQvwbQ02JHRPrCbty+kX9aAT1A0BXk4vqL+d2O39GcaCaoB4lmopT5yqZyuNOClJJcYyOKP0C+rY3k3/5G6NprUYNB21oTAr2ikALRewAUlZDXw8r++3Anmmha+xnkkJYuai5BumzN+C+kQiBVN1J1Y7mAwnvvL4cVVp5VRhZhZlDyScjEaIsmaOzLsq8gdvuigmeajUG3oFuF+pAtdkNdmsWeKZ6/Ox4sA2EWLC/LGBQvCbaVpfsxPaVYuh+pTp14eXS75dCSRQsmrgiyEOIXwCrgAeBLUsotJzDGaUVakt7bdyM0heBFtWgRxwU512jZuR1/UTGBohI0fXrcNj0tCRDgj7jR3SP/tIQQ1AZqOaP8DO7YdQcPH3iYd658J03xJkq9pTP/IniCmH19yFQataSE3rvvRmgaode/HgCrvx/X/IK1lo1DrAl8pZCN49v8S/IVa2iPnEHAlGiqQMn2k/dXYHpLJnaQio7l0oEAprcUQlBaDqWWyTozgzCzCCNNLhWjuaufA9Ec++OwP2axodXk4f2HPsOQSwy6MQesu4awik8/+ufs8gYQ4VosUUg4728+8Zy8AYvLyqNY5uBiW7zcWHoAU/dj6T4szWOLvuIqNBadejy6QsawePfPN7CvJz2hRZDfASSBJcCHh/zgBCBnUwftxPNt5JviBM6pwj0/7DQPnWNIy6Jtz04qFyyelv5rA0TbU/jDblRdoLqOnhdZ5C3C7/JzTvU5PNPyDG9d/lYyRoZ4Pk7INWt+VsdFrqkJ4fVidHWRePJJgpdfjlZUhDQMuw/bgLXWsxdUtz3p9NIvIBtHP+9DLPQE2duZIOhS0CyTdHjkyiCTgqJiKX7Q7VQSJVBDfTnUS4sLCy5NYWSIx/o42NPPgd6sHbASM3lwn0HGPHTdqfQLGgbF7lA6gs8fJO6r5wO/eHV8CedSgjRQBqMNh4qXQGpuLM2P6Qpg6f6C1eXGEjoGCnnTwjClfZ+VhccZDEtimBZ50yI/+FhiWHL4NtbQx0NeG/LYsKzDtpFH2cbif9+0hi/du+2IVI6xMtoc25zIWDb6s8Qe3I9W6cN3dpXTPHQO0tvaQi6VoqS2Hm9weubXAPq70oRKPQgE2igJ/7qiU+Wv4tzac3mi+Qmean6K82vOpzXRSqh47gqbmUhgRaOoJaVEf/c7kJLwdXbWkBWP4Zo/37bW0n2Q6IRAmR3yvvUuWHYNlCyiGJBlAQ42NWJULENq05PWMQyhIDUvZiGy1RuoZGk1LJXStu7MLORTdPf1c6ArTmNvhv0xi30xixfbwJK24GkK/PSdS/hcIcITDiWcf+9NK7ln42YM08IwzcK9JGeBYUHegrxUyaOSkx7yUiFvCfIDr5sWhpXBMNPkzY5DQmLJo57W8aIpAk0V6IqCpgo0VUFXCveqQCss9+gqmlugD1muF9avCHmOW9RgjB20ZytSSvru2oM0JcEL69CLT+5q6nOVpm2Fwsf1DdNSHxIgnzdJRDNULQqje9RjegUqfBXU+Gvs+pEHHuHKeVfSle6iwWjAMxMu1pNAvrkFXG6MaJTEY48RuOgitLKy4daalNC9G1yFz/H5H4DmhrXvGdxPiduAilI2KaWUWHLmBm0MCVzBHaY4UEVxLZwO9pyWmcHIpmnp7edgZ5wDvWnCXn3EhPOs1LjtNTthWVMYFA1dVQpCodiPhwiER1cIHiYYuiKOWE9T7cT1geX6kOeHC85I6w0Ilq7aATTKBLjTQ17tiPzE8TCnhS29pZvMjl58p5fjXVrkNA+dozRv34rL5yNcUYnmmp4qMtHWJNKCQNHRA0eGEnAFCOgBLq67mJ+89hO29m6lLlBHZ6qT+lD9FIx4arHSafKdnahFRUR//WukYRC+4Qb7tXgM14IFCF2HZA+keiFYDs0b4eBzsP4f7MK7YAtfNkHJgnUsy3rZ0R6jxO+eueJ2FGQhAENxhagLVlA3D84FQh51xITz8oCLOz9wFpqqzvl5WLBTNn7w9jP4x1+/RNtxbD8n3I0jYaXy9N2zF7XYQ+D8Gqd56Bymddd2iqtqpq0+JByKiPRH3LiOEup/OLXBWlaXrB6sHxlwBWhJtGBY4675OuPJt7eDIrASCeIPPYT/nHPQq6qQ+TyoKnp5uV1wsHsneIJgGfDc/0GwGlbfdGhHmX4I1YCvmJoiL0srg/SmspiT4FKbDnKm5AdvP4PaIttirS3y8oO3n4EhQde0k0LUADJ5C4+m8Jv3ncmCEu+kFEGelfQ9YJfNilxWj17uuCDnKsm+KPHuLupWrMYXmr75qd5Wu/9XIOJBd40tkiziieDSXFxUdxH37b2PvmwfAkFfpo9SX+lkDndKkbkc+ZYW1GCIvttvR2YyhG+8EQArEce1cKFtrcU77JYqgTLYdg9E98PlX7YLAIMtdtKCkkWD+64t8mFZkj2dCYpnoeV2OJm8hUe3L+iWZDDhPJMfd/GNWU8mb5/3rj379o932zkpbNl9faQ2dOBZUYxnVSnCad8+Z2nebhfCKaufjz5N9SHBTs72hV2omkAbY5m2gSCSs6vO5r699/HYwce4fuH1HIwfpMRbMmf+nee7u0FKZDZL/C9/wbd+Pa76emQ+j9C0grVmQvcu8ITsUP+NP4OqNdBw/qEdpaJQvgL04Z9zfYkfKWFfd3KwvuGINRAlKIU5oIF6h2Kg7mFh+cDz6WTggu5w/Mw5YZN5i947dqMEdYIX1TvNQ+c4zdtfQ9E0Suvr0VzTl5/Y15EiVOJBURVUfex/pCr8FbQkWgbrR960+CZi2dicCf2Xpkn+4EGUQJD+++7DSiYJ32S7Fs14DPfixQhNg75mu8GmOwAbb4VMDM7+50MtV3Ip20UZqh7xOPNK/dQUeQcLBJuWXQBYFqrPm5ZdNDh3eOh5IWLQNCVZyyRvWljyUGX68YjkgDDOJJE8WZlzwhb760HMngyhKxtwVTvNQ+c6LTu2E6mowl9UPG0WjmmYxHuzVMwPHbWU1tHw636CriAX1V3Ey50v82Lbi5xSdsqcCf03olGsbA5FdxG79168p56Ke+FCZD6PorvQy8rsZpk9e8AThv5m2HInLL0aShfbO5HSbq5Ztx6Uo/9p0CbIMyOHCqPkmCI5mMd1FJEc4HhEcvxjH+U1jm8e8mj7HHVvxzGOo479OH7Yc0rYcm1J4k82414QxndGudM8dI6Tz2TobjrAwjPW4w1OX+uXaFsKaUkCRe5hzUXHSm2glv5sPxW+Ch5qfIizq8+mM9lJyBWiOjCyhTIbkFKSb2xE8ftJPPooViw23FpbssS21noP2PNnqm6H96s6rHvvoR2loxCpg0KF/8lGCDsPayIujicikoZpDetpZo/tsLGOUrJ/6LqHa4M47H7E/R9xrKMfd+j+jxzjscc32msynx13zP+cETZpSaJ37EK4FAKX1DnNQ08C2vbsQloWpfUNuL3TFyDUXQgc8Re5hzUXHSsRTwRd1bls3mX8ZvtvOBg/SG2glj19e3CrbkomumTUFGHFYpiJBGooTP899+BesQLP8uW2teYqWGtGFnr3gDcMLS9B47Ow/v3gK5yzmQehQPGC6T2Z42QiRfJkRZr53Hi3mTMmTeK5VvLNCfxnVuKZ51TuPxlo3m6XLy2ftwDNPX3za72tdqh/sMgzrLnoWNEUjWp/NWdUnMFpZadR7i0n7A4zLzSPxngjsVxsooc8JeSamhEeL4knnsDs7SUyxFpzzZ+PUFWINjI4m/Xc/0GwEla98dBO0n1QutRO0nZwGCNzQtiMaIbYgwfQa/z4z6p2moeeJLTs2EqwpJRwReW0/pHpbU3iC7lQNfWI5qJjpcxXRomnhI+c8RE++8xnef1dr+eWh2/BrbjZ37efVD41waOeXKxkEqOnB8Xjof/uu3EtWoTnlFOwcjkUlwuttNQOCIk22i7GnX+B3n1w5gcOiVguYb8WqprOU3GYhcx6YZNSEr17D1JC8JJ6p3L/SYJlmbTt2UWkqgZvaPrm18COiAwWe0ZsLjpW/LqfUm8pn3n6M7QmWwFoTbbysSc+RtgdZlvPNnLmuD0y00aurQ2haSSffRajo4PITTchhLDz1gattf2gapBPwYZbofIUmH+hvQNpQT4NZctGn4xxcBiBWS9s6c1dZHdG8Z9ejmdxkeOCPEnoaTpIPpOmpLYet3d66kMCWKZFvCdDoOTozUXHiiKUQVEboDXZihACwzLY3rN9VlQlsbJZjLY2hN9P/513otfX4z3jDNtac7ttay0TsyMgPWF4+dd2RZGzP3hIxNJ9EGmw89ocHMbJrBY2M5mn78970Uq9BC6sRRljxQeH2c9AYnbFgkXo7ulLzO7rTGGZkmCR57giIofiVt1U+4dHQVb7qxEIgu4giVyC3dHdWHJmJ+8anZ0gBOmNG8k3N9vWmqJgxYdYaz17QfdCrAW23A5LroKypYUdZEGoUDzu/pIODsAsF7b+v+zDShsEL65DK5qbFdEdRqZ5+xbc/gCltfWIUXKbJpue5kIprSIPrqM0Fx0rJd4SvnXxtwbFrdpfzVfO+wqJvB2cEvFG6Ep3caD/AHK0hKVpRBoGuYMHEYEg/XfcgVZdje+ss7CyWRSf17bW0lFIdoE7CC/8yO7EvP59h3aS6Yfy5XbYv4PDcTBro1Azu6OkXurEe0op3lUlTvPQk4zWXdspqqqe1saiAD2FiMhAsWvU5qJjQREK80Lz+OFlP0RRFFJGiq889xXcqpt/WfsvKEKh2FNMU7wJt+qmJlgzEacwoRg9PUjTJLt5M7n9+yn54AcRqorZF8WzYoU9VdC9G1w+aH0FDjwN6953KLw/Gwd/KQTKp/dEHGY1s9Jis3Im0Tv3oIRcBC+uQznBf8oOs4tYdxeJ3h57fs03vdVletuSeIM6uksbtbnoWPHpPuL5OF3pLixpcWr5qWzo2MDvd/wesPOiir3F7OnbQ3eq+4SPN5FIyyLX2IjwB+i//XbUsjIC559fsNZ8aCUlkOy25880Dzz3PQhUwOqb7R1YJuQzdni/M1fucALMSmGLPXYQM5oheEENeoVTNutko3XnNgDKGhage6Y3CjbaliJQ5BlTc9GxUu2vHgzvv3r+1VxWfxl/3vtnnmh6ArAtu4gnwrbebfRn+yfkmBOB2d+PTKfJ7d5NdtcuwjfcgNC0Q5GQUCh0HIRdD9jzbEPD+9N9ULLQrhfp4HACzDphy7UkSDzVjHtJEf4zKh0X5ElI8/atqLpO5YLFKMr0BQxZliTWkyZYPLbmomOlyFNEQA+QyCUQQvD3q/6eVaWr+Mnmn7C9ZztgJ3UHXUG2dG+ZMTluucZGhM9P/x13oBYVEbj44kPWWnExJDshn7Qtsw23QsUqWHCRvbGRtdvTROZN6zk4zA1mlbBJUxK9czfCoxG6rB7lBMOrHWYnLTu2UlRZTaC4eFrHEetOYxmSQLFnzM1Fx4KqqCwvWY4lLbJmFk3R+OjpH6XcV87/bvxf2pPtALhUFy7VxWvdr5E1sxN2/OPBjMcx+/rJNTWRee01Qtdei+JyIRNxuzu2tKBrF7hD8Mqv7QCSc4ZU78/ECgEjzm/a4cSZVcKWeLaFfEuC4LnVuGocd8XJSDaVoqf5IEXVtdM+v9bdHAcgWDL25qJjxat5WVGygkQugWEZBFwBPrnuk1hYfH3D1wetNJ/uQyLZ1rONvJWf0DGMh3xLC8Ltpv+OO1CCQYKXX46VyaD4/ba1Fm8FMwupHnjtdlhypZ18DXYUZKDcDhpxcJgAZo2wGb0ZYo804qoP4jurymkeepLStnsHUkrKGxbg8kxfYjZAb8tAqL97zM1Fx0PYHWZJ0RL6M/1IKakKVPGxMz5Ge7Kdb738LUzLBCDoCpLOp9nVu2tactysdJp8ZxdGVxfpl14idM01KF4vMplAnz8fYebt+TRvBF74ISgqrHt/YWPTLnRcusQJGHGYMGaFOkgpid61G4DQ5fOc5qEnMc07toIQVC5agqJOb0J+b1sST0DH7dXH1Vx0PFT6K6kL1hHNRAFYWbqS965+L5u7NvPLbb8cXC/sCRPNRNnfv3/Kc9zybW0IVaH/rrsQPh/Bq6/GSqdRAgHbWutvBmlC+xbY/xSc+tZD1lk6CiWL7fB/B4cJYlYIW+qVTrK7+/Cvr8Q9f3rrAjpMLy3btxIqKydSMf2FcaPtKQJFJ15K61jMC8+j2FM8GAF5Sf0lXLPgGh468BAPH3h4cL2IJ0JzvJmmeNOkjmcoMpcj39qK2ddP6vnnCV11Farfj5VK2nNrRhZ694IraFfv95fDKW+2NzYyoHntXmsODhPIjBc2M5Gj/759aBU+/BfWOM1DT2JMw6B9326Kq2vw+Kd3jlVakv6u9HE3Fx0PilBYUrQEl+IanFt72/K3cXr56fx86895tetV4FCO2/7YfjqSHZM6pgHyXV1gSfr//GeEy0Xommuw0mnUUAg1EoHoAbs81p5HoWc3nPkPdni/lJCOQcUK2zXp4DCBzHiV6L9/P1bGJHRZPVrQqdx/MtPVuB8jm6W0fj66Z3pLqMV7M5h5i2Cx57iai44XXdVZUbKCnJkjZ+ZQhMKHTv8QtYFavvXSt2iJtwCFHDd3hJ29O4mmo5M6Jmma5A8exEylSD79NMHLL0cNh7HSKVwNDYh8Cvqb7NJYG34CFSth4SX2xpl+ux2Nb3ojWx3mJjNa2DK7oqRe6cR7ahneZSVO5f6TnIHE7MoFi1G16Q0L72kpNBctOb7moseDT/exongFsVwMS1p4NS+fWPcJXIqL/97w34MNSTVFI+gOsrVnK4lcYtLGY/T2YuXzxO6/HxSF0HXX2dZaMGhbaz377PD9V39nz6WdXQjvtwx7zq108aSNzeHkZsYKm5Uzid61G7XITeiSesQkTc47zB6atm3BGwpR3rBguodCd3NB2Iq9x91c9Hgo8haxOLyYaDqKlJIyXxn/uu5fiWaifHPjNwfb2rhUF17dy5aeLWSMzISPQ0pJvrERK5sj8fjjBC+5BK24GCuVxD1/PiIbh3gbZFPw2h9h8RV2nhpAqs+OgtSnN6rVYe4yY9Ui9nAjZjRL8NJ6tBKncv/JjpSS1l3bKa6uwxOY/hzG3rYkbr+GJ6Add3PR46UqUEVVoIq+bB8Ai4sW8w9r/oHtvdv56Ws/HYyK9GgeBMLOcTMnNsfN6u/HTCZJPPggWBah66/HSqVQw2GUcNieT9M9sOHH9hzbukL1/nwK3H4IzbwCzg5zhxkpbLmmOIlnW/CsKMZ3SpnjgnSgv7ODVH8fJXX16NOcvwYQbUsSiLgntOLIWBFCsCC8gJArNOh+PK/mPG5cfCNPND3BffvuG1w34AqQMTPsjO4czHubCLJNzchcnvgjj+C/4AL0igqsdAp3QwMiHYVkD/Tuh31PwJq32AnYUkI2aVtuTsCIwyQy44RNmhbRO3ej+DRCV8xzmoc6AIfm18obFqLp09unS1qS/s40gSIPHv/0jEVVVJYWL0VBGXQ1vnHJGzmr6ix+u/23bGzfOLhu2B2mL9vH3v69E5LjZiaSWD09JP76V2Q+T+QNb8BKJlGLilBCIbstje61q/f7y2BNIbw/3QfhWvAWnfAYHBxGY8YJW/zpFvJtSYIX1qKXO5X7HWyad2xFc7upXrp8uodCPJrByFsES068ueiJ4FbdrCxZSdpIY1gGilD4p1P/iQXhBXz3le9yoP/A4LoRd4T2ZDuNscYTPm6+tcUOGnnwQXxnn41eU4OVTuOeNw+R6oFMn91nrXtXIbzfYweMCKB4+udHHeY+M0rYpGERe/QgrgVh/GdWOZX7HQZp2bGN4upa/NPcWBSgt9UupRUs8Zxwc9ETJeAKsKxoGX2ZPixp4VJd/Ou6f8Wv+/nahq8NViwRQlDkKeJg7CCtidbjPp6VyWC0d5B4+mlkOk3kxhtta624GDUYhK6ddnj/iz+B8hWw8FJ7w3TU7rOmO/PlDpPPpP4qhRARIcTtQogdQojtQoizR93AkpS8dRnhNyxwmoc6DJJJJOhtaaKkpm7a60PCYaH+M6BgQKmvlPnh+YMiVuQp4hPrPkEyn+QbG79BzswBh/q47e7bTW+697iOZXR0YOayxP/yF7xr1+JqaLCttYZ5EG+3g0O23AHpXjj7g3Z4fy4J7jAEp79ajMPJwWT/Kr8NPCilXAasAbaPtnK+K03fvXuRWQsjO32Vyh1mFq277K9NWcMCNNf01wntaUni9mn4w+4Z41WoC9ZR7iunL9MHwPzwfP75tH9mX98+fvDqDwbn1lRFJewKs613G/FcfFzHkPk8uaZmUs+/gJVIELnxRsxkErWkBNXvsyMhjQxs/gMsusxOyJbSFrbyZaBM/58Ah5ODSfumCSFCwAXArQBSypyUsu9Y25nRLNHf7MBK5iZraA6zjJad2xCKQs3SFdM9FACi7Un8kcmvETkehBAsiizCp/sGk7LXVa7jLcvewnOtz3H7rtsH19VVHa/mZWv31nE1Kc339GBl0sTvuw/PKafgXrIEOWCtxVrAyMHGnwEKrL/F3igdtZuHepwarw5Tx2T+hVoAdAG3CSFeEUL8VAhxRDSIEOIWIcRGIcRgGJcZzSINk+/+3bv45bs+wKOf+TxNjz+GaRiTOFyHmUrz9q1EKqqIVFRO91CQUtLXkbKFbZJrRI4XTdFYVrwMKeVg49FrF17LRXUXccfuO3i25dnBdT2aB0VR2NazbdBVORrSssg3NpJ+6WXMvj7CN96ImUiglZaget12oeNYM+x7HNb8nR3eb+bsHDYnYMRhipnMX6YGnA58SEr5ghDi28CngM8PXUlK+WPgxwCnVC2TAGqRG1NaWFh0ZVrp2tvMq3tfQfzoh3hEkGK/j/rF9ay86QbCi5ZM4ik4TDdGPk/Hvt00nHrGtNeHBEhEsxg5u0akNgNTUbyalxWlK9jUuQnVraIpGu9b/T7ak+388NUfUu4rZ3GRXcrKr/uJZ+Ps6N3BipIVaMrRLwdGNIqZTNJ/7724ly7FvWIFsrcX18oVdj1Iy4QXfmC3o1nzd/ZG6T6oPAW06XcfO5xcTKawNQPNUsoXCs9vxxa2UVGL3BS9bRlaxMtHfv8rOl96kR133ENbSyd9OZOUTNMS76Tl5QM89/JTqCKEX/VRHgkx/7SVLL35JtwzIHLOYWLo3L8HM5+nvL4BzTX9RbCjbXZEZKhs6mpEjpeQK8TSoqXs7N1JkbcITdH42NqP8blnPsc3Nn6D/zjvPyj12v3Qgu4gfZk+9vTtYUnREhRx5DlJKckfPEj6lU2Y3d2U3HILMplELStF9WjQvg+aN9gRkRd/xs5hy8bBWwzB6beyHU4+Jk3YpJTtQogmIcRSKeVO4FJg22jb6BU+St6/EsXvQnPbia/lZ6yn/Iz1g+uYmSwHHryPPY8+RWdfiphhEDf6iHW3s+eRXTzyyJ/RRZiQy0tVZSlLLjyb+iuvmvaiuQ7HR8tOO3CkYvHSGVGBprsQERkq9aLOgIjIo1HhryCVT9GcaKbYW0zIFeKT6z7J55/9PF978Wt86dwv4dXsCNOIJ0JnqhO34mZ+ZP4R+7LicYy+PmL33YdrwQI8p56KjPbimrcKeg/Yc2sbboWy5XbQiLQgn4Hq05yu2A7TwmRf7T8E/EYI4QL2Ae8ebWWhKbiKR0/KVj1uFt5wEwtvuGlwWaqri12//y2Nr26nJ5UnYeXpybbT09jCll++ivjVz3GLIEVeH3ULallxwzWUrDplAk7PYbJp2bEVf1Ex5fNmxjxNb0sSl0clWOKdEUI7GvPC80gZKfqz/YTdYWqDtXz09I/y1Re/yvde+R4fX/vxQQut2FNMU6IJt+qmOlg9bD/5piYyr27GaGuj7F//FZlMopWVoerYbshdD0CqGy7/EggFUj1QNB/cwWk4aweHSRY2KeUmYO1kHgPAV1bGqR/6CKcOWda7dTPb/vAn2ho7iBZcmG3JLtpea+TF155FEUF8ip+ycJCGU5aw/E034y0pneyhOowDKSUtO7ZRMX8hHt/MqELT257EX+TG7Z1582uHM9Cg9NWuV0nlU/h0H2vK1/D3q/6e27bcxm+3/5a3r3g7cCiBe3ffblyqi1Kf/VuwUilynV3E7r8fvbYW77p1WH1R9Pp5dsBIpt8O7194qR3eb2RB0aG4YRrP3OFkZ87654pXnsJ5Xz5klZmGQdOjD7Hrwcfo6o7TbxgkrX4Sve3sf2I3jz/xAJoIE9J9VJQVseS89cx/3TWonumf1zlZiba1kEnEKambh+ae/s9hICKyoiGE2zu99SrHykCD0lc6X0FTNFyqiysbrqQl3sJ9++6jOlDNJfV288+BBO7tvds5RT2FsDtMrr2d7Nat5JuaKP3wh5GpFHpFBapm2AnZr/7WPtCZhfD+TL/tglRnx/vjMDeZs8J2OKqm0XDVNTRcdc3gslRPN/vuupP9L26iJ5EjbuXpzXXQ29LC9j9sgT/8Go8SIuLxUTOvihXXXEX5unXTeBYnFy2FwscV8xfOCLdfqj9HPmPaFUdmUX/AgQalm3s2U+QuQlVU3rXyXbSn2rn1tVup8FWwsnQlYKcM+F1+tnZvZU1kBVZzC7EHHkCrqMB3zjlY/f3otbXQvQP6DsDev8Lp74RABWRidlSkv2x6T9jhpOekEbaR8JWUsup9t7DqfYeWRffsZsfvfk/LviaiGZOkzNKeaqJ9+0Fe2v4CigjgVfyUBgM0LF/AsptvIlBTO30nMYdp2bENl9c7Iwofw6EakeEy77TXiBwvAw1K9/TtodhbjKqofOT0j/D5Zz/PN1/6Jl857ytU+u0IRrfqxrRMtu98ltqtB8jt3UvJBz5gW2uVFahKDpLdsOFn4CuE91um7YasOcMJGHGYdk5qYRuJokWLOfvzh1LtTMOg/Zkn2XHvA3R29tOXN0hZCRr7Omh8bi9PPvcomggT0LxUlBax6MxTWXjDDehe3zSexdygZcc2Smrn4QvNjKoVPa12RGSkwos6xc1FJ4KqQBUpI0V7qp0iTxF+3W9HSj5jR0p++dwvE3DZTVx9wk2ytZ2e++9FLynGf8EFWIk4rpoa6H4NWjZA13a46FOg+2yhK1kE7ulvAuvg4AjbMVA1jZqLLqXmoksHl+XjCfbecwd7n91AdyxD3MzTl++hr62NnXdvg7v/hEsJEXH5qK4tZ9nVl1Jz3gXTeBazj1R/H33trdSuWI3unv7EbICe1iS6RyVQPDPGM16EEMwPzydlpIjn4gRdQSr9lXxs7cf4yvNf4dsvf5t/W/9vdqJ2Xz/+fe0oexrJv+U6rLQ9t6aQgkQnvPRzKF0Ci6+wLTXVDZH66T5FBwfAEbbjQg8GWPb2d7Hs7e8aXBZvPsiO3/yWph376c0YJGWWzkwLnXua2PTdlxDf+x5eEaA44Gfe0gZWvPEGQg0Lp/EsZjYthcLH5Q3zETOkeG601a4R6fXP3koaqqKyrHgZr3S+QsbI4NE8LC9ZzvtPeT8/fPWH/GLrL3jPyncjmtsRT7yADAXoX7+U9lgzC05ZDV2vwq6HbQvt0n+3w/szMag5HVTncuIwM3C+iRNEsLaedf/2KYaGlrS/8Bw777yHtrZe+vIGaZmiOdZJ84b9PLvhCVQRIqD6KC8OseCM1Sy96Y3oQceVA7YbUlFVapaunO6hAHZEZLQjRfm8ILpn5of6j4ZLdbGyZCWbujahKRqaonFR3UW0JFq4d++91KglXLVTQ2zbjXXT1YRw0xGw8MR2UdfXDK/9CRZcbJfLyvTbdSH9TqqMw8zBEbZJpPLMs6k881ALunw6xYH77mPPE8/Q1Zcibhr0G730d7ax+4GdPPTA3bgKVVOqq0tZeun51Fx86UlZNaV1p91YNFxePt1DASAVy5FLG4WIyNktbHCoQenWnq0Ue4tRhMJblr2FtkQbv9j3B87+ayURnxcuXI/I5QnXL2Rvy3OUbXsQD5Yd3m8ZYOZtl6SDwwzi5LtiTiO618fim9/E4pvfNLgs0dHOrt/9loOv7aI3bZCwsnRn2+je38zmn25C3PpTPCJAsc9P3aI6VrzhWoqWzYz2LZNFPpelY98eFp953owofAzQW6gRGS71zqpQ/9Eo9ZUy35jPgdgBSrwlKELhn5e8hx9s20PRlib6XncWIUtCZTmKmaAm1oHnwDNkTrkZT7CqEDCyBFxOoJTDzMIRtmkmUFHJ6R/9GKcPWdb56svs/NMdtDV1Ec0ZpGSGlkQXLZsO8Pymp1EKhZ/LwkEWnLqcZTffjLuoaNrOYaLp2LMbyzQpmzcPRZkZ1tFAqH9RlX/GNBedCOqCdaSMFD3pHiKeCJ6uOB/cUELG1c1/rtjNZ3JnESpbAL3bqNl2P3lPiE3z1rE6HcWveSHipLo4zDwcYZuBlK85nfI1h6TOzGRpfPhBdj/2BF09CWJGnoTRR7ynnX2P7ebRx+5HFyGCuo+qimKWXHgW866+Zta6MAcSs6uXzIz8NbCFTXerhEpnhgU5UQw0KE0baRKxHgLbduPetIvoZafTom/jf7L38blsDaXNL+Hr3UfL+veguYO81r2NU1fchGeG/PFwcBjK7LzynWSoHjcLrrueBdddP7gs1dXF7jv+SONLW+lJZkkMVE1pamHrr19D/PqXuJUQEY+Xuvk1LL/2aspOO32Uo8wcWnZuI1RaTkntzAkf722zIyI9/rlXKkpTNJYXL+fV/Q9iPf4sqqYSueoq/inbwLfjf+Yn237ON3duJF1UT9/8c/HmkiQCpWxLtbHaX4quzL33xGF24wjbLMVXVsaaD3yQNUOWRXfuYPvv/0DL/hb6skahakoX7VsPsmHrc4XCzz5KQ0EaVi1m+ZveiK+8YtrOYSSkZdG6czs1y1bg8c+MCFEpJdG2JGX1wRnZXHQicFsK9U0Zohu3Yl10JkJRObvhAlqjKf7U/Ci/dJlcetpbQUqQFoHSJfQbGXb27mR58XJUx3JzmEE4wjaHKFq6jHO+8IXB56Zh0Pz4o+z+yyN0dsfozxskrTiJaAcHnt7DE08/hCbCBDUvFWVFLD53HQtff+20Fn7uaT5INpWkdN58FHVmXCzT8TzZVCEicpaV0hor+e5ujMeeRAhB/KLTCJgmSkmAN6VrSCTTfLs4gqqZrM/EoXgeaG7CmpvedC/7+vexKLJoRtTzdHAAR9jmNKqmMe/yq5h3+VWDy/LxBLv/9Hv2vfgK3bEscStPNN9FtLWVHX/aCn/6re3CdHupqa9k+dWXU3n2OVM25sHGojMoeX2wa/YMby56vEjTJLN5M4lnniVw0UXooWo6whaRdCeVOx7iiz1R9pQ28H9776Rs6TuY7z9k5Rd5imhNtOJSXcwLzZvGs3BwOIQjbCcZejDAive8jxXvObSsf/8+dvzudzTvPkhvxiAls3Skm+nY2cTLOzcgvu3HK+zCz/OWLWDFzTcRqKublPG17NyG2x+gcvHMyY0aCPUvqvLNSavEiEbpv/8vYJqErrsO4XWTrVQwd91HpOlFupa/jo8vvZLPbv0xX9t3J/9Rs5ZiTzFgB58Ue4tp7G/ErboHCyk7OEwnjrA5EJ6/gDM/81nOLDw3DYOO559lxz3309ERpT9vkpYpDvZ3cvCFfTz9wmOohcLP5cURFq1fw+KbbpyQws8tO7dRNm8+/nDkhPc1UfS0JtFcCkWVcy9fS0pJZssWkk89hf/cc1H8Plw1NSwRrWR3PkjeHaJ7xeuJmAafXP5u/n3rj/jGhm/whXO+gFu1XdYDfdx2RXfhUlwUe4un+awcTnYcYXM4AlXTqD7vQqrPu3BwWT6dYu/dd7DvqRfpjmWImXn68z30d7Sx+97tPHDv7bhEmLDbS3VNGcuuuIiq8y4cV8pBoreHWGcHC89Yj6rNnEi73tYEgSIPHt/srRF5NKxYjP5770Vms4Suvx4hQY94UDY8jN57gN2nv5UMApeAeVXr+LDbzTc2fIPvb/o+Hzn9IyjCds2qikrIFWJb7zbWlK0h6ApO85k5nMwIKeV0j2GQtWvXyo0bN073MBzGSLy1hZ2/+w1NW/fRm86TkDkM2Q9YAAjhxSOCFPt91C+ex8qbrie8aGQXo5nL09PaQjYVxxcqIlJegeqaGeJ268efprQuwNX/sBqXd279F0y+8CJN//iPeFevpuT970OvrsYtG+HO94MnTPR13+C17s1Eylehhmw343177+PX23/NGxa/gTcvffOw/WWMDHkzz5ryNXg173ScksMcQwjxkpRy7Xi2mVu/UocpJVhdw9qPf5Kh37jOl15k+x130d7STV/OJCXTtMQ7aXn5AM+9/CRqoWpKeSTE/NNWsvTmm9C8frqbm/jz/36FWFcnobJyrvvY5yitrZt2cUvHc2SSeYIlnlnXXPRYWMkkfXfdhUylCN1wAwB6WIcnfw/JLrj4MxSpOosii9mtqpRIiRCCaxZcQ0uihbt230WNv4bzas8b3KdH82BK0+7AXbYGXZ0Zf04cTi4cYXOYUMrPWE/5GesHn5uZLPsf+DN7H3uGzr4kMcMkbvQR625nzyO7eOSRP3Pjp7/Ioz/9LrGuTgBiXZ38+X+/wpu/8DVCZdNbNT7aPqRr9ixsLjoamf37STz6KN7TTkMvLcFVW4PSthG23QUNF0DVGkh2U1V/DulsD23JNoo8RQgheO/q99KebOeHm39Ima+MpcVLB/fr1/3EsjF29O5geclyu7+bg8MU4nzjHCYV1eNm0RtuZtEbbh5clurqYtfvf0vjq9vpSeVx+7yDojZArKsTyzKnerhH0NuWAiBSMbcCR6xslv4//hErkbCtNUVB90t46hdgWXDWP0C6DyLzEN4IDe4gyXxysEGppmh8bO3H+Pwzn+d/Nv4PXznvK5T7DnViCLlD9GX62NO3hyVFSwbn4hwcpgLn2+Yw5fjKyjj1Qx/h+p/+kPf89lYCRaWEyoa3pwmVlc+IAsi9rQk0XaGkZmZUQZko8s3NxB9+BM/Klbiqq3DVVCH2/xX2PQ6r3wj+MhAqFC8ADjUoFQjSRhqAoCvIJ9Z/AlOafH3D10nlU8OOEfFE6Ex20tjfOOXn53By4wibw7TjD4e57mOfGxS3UFk5137ss/S8vHuaR2aH+geKPbh9c8e5IQ2D6O//gNnXd8ha8xiw4SfgLYLT3m5ba+XLQDsUCTrQoHQgQASgJlDDR0//KC2JFr77ynexpDXsWEXeIpriTbTGW6fwDB1Odhxhc5h2VJdOaW0db/7C13jvd27lzV/4Ggef6eQvd6V56P3/hZnNTtvYom1JgsXuOdFcdIB8RwexBx7AtWgRrnn1uKorEFt/D107YO17QVrgLYbAkXVEBxqU9mf7B0Vsddlq3r3q3bzS+Qq/3vbrYesP5Ljt7ttNd6p7Ss7PwcERNocZgerSCZWVEqmoIFRWyuk3raYh8yJ71DO58/0/Jt3cNOVjyiTzpON5AsUe1DnSXFRaFn1//CNmdzfhN7wBoajoWhxe/iUUL4QlV0I+Y1trR6myUuorZX54PtFMdHDZ5fMu56qGq/jL/r/waOOjw9ZXFZWIJ8L26HZiudiknp+DAzjC5jBD0fwBrvn5pzjN+xxd3mX88bNP0v3sM1M6hqFds5U50lzUiEbpv+fP6PPm4V68CFdNBWLjj+3w/nP+GbIxe17NPXqCdV2wjnJfOX3pvsFl71jxDtaUreG2LbexpXvLsPU1RcOv+9navfWIuTgHh4nGETaHGc053/wsF6/YS0aPcOfPetjzs18fe6MJIjqkRuRcoe9Pt2O0txO+/nqEqqLnG2HrndBwHpSvAEWHomMXMx5oUOpz+UjkEoBtmX3k9I9Q5a/imy99k9bE8Hk1t+pGUzS29WwjZ+Ym5fwcHMARNodZwPKPfIDrb3ahmykefr6cDZ/9+pQct7c1iTqHIiKNWIz+229Hq67Gs2I5rqpSxPPfBcuAMz8AmRiUL4cxJlUPNCiVUpI17XlQn+7jk+s/iSIUvrbha4OiN4BP92FIg2092zAsY8LP0cEBHGFzmCVUXnkFN39+HSXpPbzYcwYPve+rkx5U0tuWJFjswe2bG9UzYvfcQ765mfB116G43eh9L8G+J2DVTeAO2SH+/rJx7dOjeVhRuoJELjEoVOW+cv517b/Sne7mmy998wgBC7rsnLjd0d1HRFE6OEwEjrA5zBoCCxfxxh+9m/nZF9ijrefO9/9kUoNKegciIudAKS0zlSL6u9+hlpXhXXMKenkY8fx3wBOGU98KRg7Klhw1YGQ0Qq4QS4uW0p85FCm5tHgpt5xyC1t7tnLblts4vCZtxBOhO93N/v79R7zm4HCizP5frMNJherz8rrbPs3pvufp8i7lj599iu5nJj6oJJvKk+rP2TUi50Bz0fiDD5Hbt5/wtdfa1lrTfdC5Hda+B8w8lCwCl/+491/hr6AuWDcsUvKC2gu4ftH1PHbwMR7Y/8AR2xR5imiJt9CcaD7u4zo4jMTs/8U6nJSc/b+f4ZJV+8hoIe68rZfdP/3lhO5/oJRWqNQ765uLylyO3l/8ArWoCO8Zp6OXehAbfmJHPy66FDQPRE68cey88DxKPCX0Z/oHl7156ZtZX7meX237FS93vDxsfSEERd4i9vXtoyvVdcLHd3AYwBE2h1nLsg/9A9e/2YvLTPDIi1W88OmvTdi+ByMi50CNyPgTT5LduZPQNdeger3o238ByU44+4OQSxYCRk68sooiFJYULcGlugZD+hWh8E+n/hMN4Qa+8/J3OBg7eMQ2EU+E7b3b6c/2j7RbB4dx4wibw6ym8orLeNOXzqEkvYuN0bU8+L6vYqbSJ7zf3rYkqiYoqZ3dEZHSNOm59acogQDeM9ejBzKI1/4A886FkoV2dRH/xHVQ0FWdFSUryJm5wZB+j+bhE2s/gVfz8rUNXztCwDRFI+gKsqV7C8l8csLG4nDy4gibw6zHN6+BN/7oPSzIvsBebT13fOBnpJoOHnvDUehtTRAo9uAJzO6u2cnnnyfz6maCr3sdmt+H/ur3wcrD+veDaUDpyI1fTwSf7mNF8QriuThmoUNDsbeYT6z7BLFsjG9s/MYReWwu1YVbc7Ole8tg6oCDw/HiCJvDnED1ebn6tk9zRuB5uj1L+OPnnqb76eMPKultSxEq8czqiEgpJT0/+hHC58N/zjm41BbE3kdh5Y3gCtii5pocV2uRt4hF4UVEM9HBqMcFkQX806n/xO7obn68+cdHREN6NS8SybaebeSt/KSMy+HkYPb+ah0cRuCsb3yGS0/ZT04LcufPe9n9k1+Mex+5tEGyL2tHRM7i5qLpTa+S2rCR4BVXoAV9aC99yy6Vtfpm0H0Qrp3U41cFqqgN1g6LlDyr+izevPTNPNPyDHfvufuIbYKuIJl8hl29uwatPQeH8TJ7f7UODkdh6T/fwvVvDeAy4jyyoZoXPjW+oJKBGpGhUu9kDG/K6P7BDxC6jv/883GnX0J0brXD+8EOGJnkfndCCBpCDUTcEeLZ+ODyGxbdwHk15/GHnX/g+dbnj9gu5AkRzUTZ17/PyXFzOC4cYXOYk1Rceglv+sr5lKZ3sbFvLQ++d+xBJb1zICIyvWMnyWeeIXDppeghD+qm70NRAzScD6Fq8BVPyThURWVp8VKEONSgVAjBLafcwpKiJXx/0/fZ27f3iO0inghtiTaa4lPf1cFh9uMIm8OcxVdXz00/eg8Lc8+zVy8ElTQeOOZ20bYkiiYonsU1Int+8ANQFAIXXYSn+z5EogPO/Ee7skjJoikdi0t1sap01bAGpS7VxcfXfpywO8w3NnyDnnTPsG0Gctz2x/bTkeyY0vE6zH4cYXOY06g+L1f97DOsDb5gB5V84W90PfnkqNsM1Ij0BmdnRGS2sZH4o48SuPBC9BAoW38L9WfZCdmlS0D3TPmY/Lr/iAalYXeYT677JBkzw9c3fJ2MkRm2jSIUIu4IO6M7iaajI+3WwWFEHGFzOCk48+uf5rJTG8mpAe78ZYxdP7ztqOv2tiYJlXjR3bOza3bPD34IQOCSi/Ec/DXCzMEZ7wFPEEI10zaukRqU1oXq+PBpH6Yx1sj3N33/iKLIAzluW3u2HtEpwMHhaDjC5nDSsOSf3sf17wjhMWI8+nItz3/yv49YJ5cxSESzhEo9s7K5aL69ndhf/oL/nHPw+GMoBx6GlTeAtwjKloMyvT/5kRqUnlZxGu9Y8Q5ebH+RP+z8wxHbuFQXXt3Llp4tg/N0Dg6j4Qibw0lFxUUX8ab/uIDS9E5eiq3jL+/972FBJdF2uxRUcJZGRHb/+CfIfJ7AJZfg3vtThCsAy6+HcB14I9M9vMEGpX6Xf5gFdvX8q7m0/lLu2XMPTzYd6Sr2aB4Egu092wfn6RwcjoYjbA4nHd7aOm766ftZmH+e/fo6bv/AbaQO7ANsNyRApHz2CZsRjdJ/5514163D7zmA0rUFTn8XeEJQsmC6hzeIpmgsK16GRA7OqwkhePeqd7OyZCU/3vxjdvTuOGK7gCtAxsywM7rTyXFzGJVJEzYhxFIhxKYht5gQ4qOTdTwHh/Ggut1cdetnWBt6gR7PIv7wxRfo2byVonI3N3zsNCpqvVj52dXhueentyIzGUKXXYJrz88gUm/XhCxdCpp7uoc3DI/mYUXJClL51GAjUk3R+Jcz/oVyXzn/s/F/RoyGDLvD9GX72NO3x8lxczgqkyZsUsqdUspTpZSnAmcAKeCuyTqeg8PxcObXPs3lZzQTWTaPtKuKh2/bwd3/+wp3fus1eloTs0bcjESCvj/8Ac+ppxJQXkIkO2Dde+0Cx8HK6R7eiIRcIZYULRnWoDTgCvDJdZ/EkhZf3/D1wS4BQ4m4I3SkOmiMNU71kB1mCVPlirwU2CuldL6JDjOOxf/wHi5510r++qsdxHts11i8J8MDP95Guj9zjK1nBtFf/AIrkSB06bnoB/4IteugfAWULZv2gJHRqPBXUB+qHxYpWRWo4mNnfIy2ZBvffvnbR7gdhRAUeYo4GDtIa6J1qofsMAuYqm/83wG/G+kFIcQtQoiNQoiNXV1Os0GH6UGo6qCoDRDvyWBaR9lgBmFlMkR/9Wvcy5cT4gkws3DaOyDSYM+vzXDqQ/VHNChdWbqS96x6D692vcqvtv3qiG0G+rjt6dtzRHK3g8OkC5sQwgVcB/xppNellD+WUq6VUq4tKyub7OE4OIyIqkCwZHjicrDEgyJm/jxO9Le/w+zrI3zBGrSWRxHLXg+ReXYJrVmAIhSWFC/BrbmHuR4vnXcpr5v/Oh488CAPH3j4iO1URSXkCrGtdxuxXGwqh+www5kKi+1q4GUppVMXx2HG4g17uPqWFYPiFizxcPX7lpP4yTfIbPzrNI/u6Mhcjp7bbsO1YAFh8ZDdhmbFDXaRY232VE7RFZ3lxcvJm/lhvdrevuLtnFZ+Gj/f+nM2d20+cjtVx6f52Nq9lXgufkSCt8PJyVQI21s4ihvSwWGmoOgaJdUBbvqXU3jHl9dz07+cgj+6jdgffk/ju/+RxO+/M91DHJHonXdidnUROWc+Wt9WxJq3QHEDBMqne2jjxqf7WFEyvEGpIhQ+fPqHqQ3U8q2XvkVLouWI7TyaB03ReLXzVf7W8jde6XiFPdE9dCQ7iOVi5MycE0F5kiEm8wMXQviAJmCBlLL/WOuvXbtWbty4cdLG4+AwXrKbn6X5H95Hrk9S/qZzKf7CTxAzJBhDGgZ7r7gSVIX5lx5EcamIa/7HruDvnr0FnFsTreyO7qbEW4IQdvWXrlQXn3vmc3g0D1857ysEXcERt5VSkrdsqy9v5UGCRA6W5gq6ggT0AB7Ng1t1oynaVJ6aw3EghHhJSrl2PNtM6i9USpmSUpaMRdQcHGYi7lPOpeH+x/HPc9P5h7/R9s5LkJkjQ9Cng9hf/kK+tZWitUWomXbE6e+yixzPYlEDqPLbDUr7Mn2Dy8p8ZXx83cfpzfTyvxv/dzD37XCEELhUFwFXgCJPEUXeIoq9xQRcAbJmltZEK9t7tvNKxys83/Y8G9s3sqNnB23JNvqz/WSMjGPdzQEm1WIbL47F5jBTkfkcHR+4huizzfhqdWp/9WfUqobpG49lse+a12Olkiy8dBeiYjHi8q/AvLNB1adtXBOFaZls69lGMp8k6D5knT3T/Azf2/Q9Lq67mFtOuWXQojteDMsga2bteT0JCFBQ8Lv8hFwhgq4gHtWDW3OjK7P/fZ2NHI/F5tjhDg5jQOguKm99BPd//RMdv/or+6+7itrvfw/PusumZTzxxx4jt38/5VfWIGQGcdo77YCROSBqcKhB6abOTaSNNF7NLnF2Xu15tCRbuGv3XdQEanj9wtef0HE0RUNTNPy6f3CZJS3yVp6OVAct8RakkCDBrboHxc6n+3CrbjyaB0XMDNe0wyEcYXNwGAdFn/4++rIf0frF/6XxPR+k+tP/SPCtH53SMUgp6f6/76MVhykKb0QsvAxqzrCrjMwhXKqLlaUr2dS5CU1o6AXRvnnJzbQl2vjN9t9Q6a9kbeW4/swfE0UouFU3bnV4GTLDMojn4/Rke2x3ZcHZ5df9BF1BQq4QHt2DR/XgUmdPROpcxHFFOjgcB7ktz9F8y3vIRiXlbzyL4i/9bMqCShJPP0PT+99P+QU+iue1Iq77Hiy9Clz+Y288C+lJ97ClewvF3uJB6yhrZvnS375ES6KFb138LeqD9UgkAkHOypE1s1MytmHBKmYegUAKiSrUYcEqXs2LW3WjKrOzx9904rgiHRymCNeqs2m4/0ma33oFnX96gezei6n62QMIj2/Sj939f/+HGvJRXLEHseKttrU2R0UNoMRbwoLIAvb37afYW4wQArfq5hPrPsGvt/0al+Li/Q+/n9ZkK9X+ar558Tdxq+4pEbeBYJXDLTRLWmTNLPGEnbogkQgh8Kpegm5b8Hyab9AyPNG5wulASolEYkkLaYefYmEhpRzMJxx4TSLt9aUcXGdgmSUtLGlhShNTmiDBxMSy7GXCJcbd8t2x2BwcTgBpGHR84HVEn2nCW6NR+6s/o1XPn7TjpTZsoPEd76RsvUXJaom48cfQcB6oc/s/qpSSXdFddKe6iQzpK+dSXPzzY/9Ma/JQzchqfzU/ueInxPPxaRjp6OTNPDkrd1zBKlMlJFJKTGna62Gva1n24wGRHjoGhmqyxLZaB/y0AoQUg/OUCIbdDwi+QBxxD7Zb+KyFZ+02k+aS8bzPc/vX4OAwyQhNo/KnD+P+73+m4xePcuD6q6n93nfxnHn5pByv6/++j+p3UVx/AHHax6DqlDkvamBbRgsjC0kbaRK5BAGXndLgUl3DRA2gNdlKd7qbLz//ZYo9xYO3Ik/RsOcebdyGwAmjqzq6qh89WGUgAV0ymGM3qpBQEI6xComw3bWjCclI96qiogkNoQ7fbqYy938RDg5TQNG/fQ/X8p/S8u9fp/F9H6L6395H8O3/OqHHSG/eTOr55yk7NY2oXGGXzvIVT+gxZjIDDUo3dW0iY2QGu2pX+6uPsNhMaeLTfLQl2tjavZWUcWTuoVfzDhO7w4WvyFNExB2Z9KjHowWrDFRfmQ1CMtNwXJEODhNIbsvzNN/ybrJRSdlN6yn58s8nLKik6QP/SOr5p1l4TQva9V+D094O+uzr9H2ixHIxXu18lZA7hF/3kzNz/Mvj/zJsjs2luobNsWWMDL2ZXqKZKL2Z3mGPB+77sn22a24IAkHEHRkuft4jhXAgHcFh4jml/pRxuyIdYXNwmGCsaCfNb7uS5L4M4dNKqfrZgwjviQV3ZHbuYv/111O6Kk7J605HufF7dofsk5TOVCfbe7cPiopLcZ1wVKQlLWLZ2CHhy0ZHFMNkPnnEth7Vc4TYHf444o6MKSrSrbon5HzmCscjbI4r0sFhglGKyqn78wY6//Eaep8+SO6as6j95d1otQuPe5/dP/whii4oWmagnPV+CNVM4IhnH+W+clL5FAfjB1G8yoRc+Ad6vEU8ERaw4KjrZc0svemRha8308v23u1EM9GjWn+juT9rAjXkzBwffPSD0xLlOZMYEPilC5aOOxrLsdgcHCaRvq9/hPbbHkLzQ+33vo3nzCvHvY/s/v3se901lCyLUfq216Fc89/gLZqE0c4uLGmxvWc7/dl+wp7wdA9nGJa0iOfiR7g9Bx4PPE/kE8O2+9bF3+JrL37tiDnDL57zRb6/6fuoiooqVBShoAp18PngsqHPFWXw8cC6g9sNea4JbXDbwedj3HakMWiKdsJzgm7VPehifuoTT5Henx7XzhyLzcFhEol84tvoS39Gy7//N43v/QjVn3wPwXd+clz76PnRjxCKJLLajXLWLY6oFRhoULq5azPJfHJYpOF0owiFsDtM2B1mfvjoBkfOzA0TvsMDYcCO8vRqXrudTyEM35DGYHj+wO3w5wOh+dPFUKE7qigOEcahQvnR0z/KF//2xSPei7HiCJuDwyTjv+49NCxcRfMt76L5v35G2Y4tlHxlbEEl+ZYW+v/8Z4oWJVDPeR9UrJqCEc8eBhqUburcRFqk7ZB0IVCEMisiCV2qi0p/JZX+SsAuzzVSlGexp5j/OP8/xr3/YTlrljns8aAgSuuI54ZlHLHu0bYduu4R2x5+3FH2N/S5V/Met6iBI2wODlOCa+V6Gu5/mpa3XU7XnRvI7buQqtuOHVTS88PvAxaRM8pQ1/896FOfezXT8ek+Vpau5EDsAKZpYlK4mBaSi4HBPK0BpJSH8sEGjJohOV9CCBSUwVB7RRz2uCCaEy2gOSvHNy/+5hFRnjkrd+yNR0ARim01ocEsquYV1IMjWq9jxZljc3CYQqRh0PnB19P7ZCPeKpXaX92FVrt4xHXznZ3sveRiQvUJKj/3eZSz3gMzpMnpbGGw0saQCh0Dyc6Djw+r6GFatjAa2O4+wzKGuQBN61B1DsMyhgnosATqoYJ5+PNRBNSn+wjoAaS0q3JkjAw5KzcrLNCJwpljc3CYRQhNo+JHD+L+n4/SfuuDHLjhWmq/8794znndEev2fv9/kaZJ5LzFKKe90RG140AIYc/bTLK5MpKADithNYKwHk1ApZTEc3FbNC1r0AIdeO0IoYQRxVNKecjSPIoFOrBsprlus2YWt+rmJ1f8hKu+dNXIXWVHwRE2B4dpIPLxb6EvvY2Wz3+Vxls+RvUnXiP4rn8bfN2IRone8WdC83K4r/kwDKmP6DDzmHIBHVLH8XDxHLAuh1qVpnXIRWtapm15FgJO8lb+qMI5WPdxFKvzcJEcyXV7PMKZNbNkzSw79+3cP973yRE2B4dpwv/6d9OwaDXN730Hzf91G2XbN1Pyn79CKAq93/oSMi8JX7wWdeX4UwQc5iaTLaBjEc6BII8B4RwqlEcTzqHW6BHCKeRgMvqI7lrGb0Y6c2wODtOM1ddNy9uvILEnTcnfXUnR+z9Gvq0TaeRwzytDqxp5Ds7BYbYxVCSHznGO5K4dENDqSPVmK2+tGc9xHIvNwWGaUSKl1N79Ir3f/SKes66h8Z3vJt/Sil5TTe33voNaZiA056fqMPs5HotTGjI/3uM4s9EODjMAoWmE3/pB2j73OfItdohzvqWV5n/+MGZP1zSPzsFhduEIm4PDDEHm84OiNkC+pRWZH/cfVgeHkxpH2BwcZghC19Frqoct02uqEbp+lC0cHBxGwhE2B4cZglpSRu33vjMoboNzbCVl0zwyB4fZhTMj7eAwQxCahnvxUhp++2tkPo/QddSSMidwxMFhnDi/GAeHGYTQNLSKqukehoPDrMZxRTo4ODg4zCkcYXNwcHBwmFM4wubg4ODgMKdwhM3BwcHBYU7hCJuDg4ODw5zCETYHBwcHhzmFI2wODg4ODnMKR9gcHBwcHOYUjrA5ODg4OMwpHGFzcHBwcJhTOMLm4ODg4DCncITNwcHBwWFOIaSU0z2GQYQQcWDndI9jFlEKdE/3IOYgzvs6O3E+t7nJUillcDwbzLTq/jullGunexCzBSHERuf9mnic93V24nxucxMhxMbxbuO4Ih0cHBwc5hSOsDk4ODg4zClmmrD9eLoHMMtw3q/JwXlfZyfO5zY3GffnOqOCRxwcHBwcHE6UmWaxOTg4ODg4nBCOsDk4ODg4zCmmTdiEED8TQnQKIbZM1xhmG0KIA0KI14QQm44nBNbBRgjhEUK8KIR4VQixVQjxpcLyYiHEI0KI3YX7ouke68nOiXznhRB/m6xxOUwOI+mCEOIJIcS40jim02L7OXDVNB5/tnKxlPJUJ1/nhMgCl0gp1wCnAlcJIc4CPgU8JqVcDDxWeO4w/RzXd15Kec5kDchh0vg5E6AL0yZsUsqngN6hy4QQHxZCbBNCbBZC/H6ahjarcN6z8SNtEoWneuEmgeuBXxSW/wK4AUAIsbJg4W0qvM+Lp3rMDoco/IP/phDiKSHEdiHEOiHEnQVL+ytD1ksU7i8qbHO7EGKHEOI3QggxfWfgcDRG0oUCNxd+g7uEEOcfaz8zrfLIp4D5UsqsECIy3YOZgUjgYSGEBH4kpfwxznt2XAghVOAlYBHwf1LKF4QQFVLKNgApZZsQoryw+geAb0spfyOEcAHq9Iz6pGSk7zxATkp5gRDiI8A9wBnYF8S9QohvSil7DtvPacBKoBV4FjgXeGZKzsBhItCklOuFEK8DvgBcNurKUzOmMbMZ+I0Q4m7g7ukdyozkXClla+GC+4gQYgfOe3ZcSClN4NTCn4G7hBCrRln9OeCzQoha4E4p5e6pGKMDMPJ3HuDPhfvXgK0Df0iEEPuAOuBwYXtRStlcWGcT0IAjbLOJOwv3L2F/dqMy06IirwH+D/vf10tCiJkmvNOKlLK1cN8J3AWsx3nPTggpZR/wBLZfv0MIUQVQuO8srPNb4DogDTwkhLhkWgZ7EnKU7zzY86QA1pDHA89H+g0MXcc8yjoOM5eBz29Mn92METYhhALUSSkfBz4JRIDAtA5qBiGE8AshggOPgf/f3rmHWFVFcfj7lZampdgMYdKbSih6oFRS1tgfQ9Fj0gwDe4BFUaRYWAQVaUIvoehBb8JeSlEZpaU9bSpNrdEczaywoacoYZAaZrT6Y6/jnK73OqPdeXRcHxzuvvvuvc7a+xzOOvtx16oHviT6bKeRVJtN20rqTZrW+Io0Crjci11OmuJC0uHAGjN70Msc19k6745UuOdjF3XQJl321iJpJlAH1Ej6EZgKXCqpHyDgfn+bDhIHkKbMIF23GcA7wAfRZzvNQOAZX2fbA3jJzGZLWgi8JOkK4HvgIi8/BrhE0lZgLXBHVyi9G7LdPW9mcyXFbtWCUsYu3L5LcsKlVhAEQVAkus1UZBAEQRBUgzBsQRAEQaEIwxYEQRAUijBsQRAEQaEIwxYEQRAUijBsQYcjaX/3s7hM0lpJP3l6o6RHOuicEyVd1hGyOwNJ/SVduwv1Jkua1B10qSCrTtJ/ck6c8wFZK2luNfQKikUYtqDDMbNf3Tv7CcBjpP/bnWBmfc2sKg/MPO59ZRzpv37/RU5X+oTsD1S9b3aR/lRPlzqgKl73zWw98IukU6shLygOYdiCLsPf3md7erKkZyS97TG4Rkm612NxzZXU08sNkfShpM8lzctcYJVwJtBkZn95nXZHQHCdPpA0A2iWtKekaZKWeP2rvZwkPexy50h6U9Jo/61FUo2nh0qa7+k+SvGmlkhaKqnB88tFD7gbOMLzpnm5G3N6TMnpfIuk1ZLeBY5uR79Pdj3mS1ojaULutxskrfBjomdvp0uJvNf8eqyUdFUu/yxJTUpx796TdCjJofT1Lmu4pOlZv3mdbDTW1+s0+T3QUKE5rwFj22pzsJthZnHE0WkHMBmY5Ok6YHYu/2NSCJnjgc3A2f7bLFIImZ7AAqDW88cAT5c5xxRgfO77z8Denu7fhn51wCZSxASAq4BbPb038BlwGDCK5PllT+BA4DdgtJdrAWo8PRSY7+k7gUsyPYCvgT7AQ8BYz98L6E1y9Loip1c98ATJw8wewGzgdJKP0GZgH2A/4Nusf9u4Bgu8+iv5LwAAA4JJREFUPTUkh8E9c7L6kFyzrSR5xf+XLmXkDfDP3iSXV/sDtcAPuX4ckDv3pFzd6Vm/+feN/tkD2M/TNd4u5ct4ehDQ3NX3dRzd6whHoEF34i0z2yqpmWQwsvWTZtLD9WjgWJKXd7zML2XkDARW5b7vbASExWb2nafrgeNyo4p+wJEkozLTUpSAnyW93w659cD5uTWwXsDBlIkeoO3DhdX7sdS/93U99gVmmdlmAEmvl1aswBwz2wJskbSO5L7qNJe1yWW9Cgyn1ZN+JSZIGunpg1yvWqAx60czKxdja0cIuFPS6STHxoNcx7Ul5daRXiyCYBth2ILuxBYAM/tb0lYzy/y9ZR7bRQpRMqwNOX+QjEbGOSRDdD5wm6RjzKcpK7AplxZp9DcvX0ApLlQlf3R/0TrNn9dDwIVmtrqk/CpJi1zPeZKuBNaUlBFwl5k9XqLHxB3osSPKebvf6eCbkupITqSHmdlmn3bt5bLao9e2vlKy5nt5/liScRziLzst/LsvM3qRrncQbCPW2IL/E6uBWknDACT1lHRMmXKrSAFEK0aNkHSSpGfbcc55wDW5Nb6jlDzNNwIX+xrcQGBErk4LaVoP4MISWeP9AY6kE/2zXPSA30mjsXzdcZL6ep1BSjHKGoGRknorecI/L6sg6TpJ17WjjRmNwAWS9vE2jgQ+KqNLnn7ABjdqg4FTPH8hcIakw1yXAZ5fKquF1r5qIE2JZnLXuVEbARxS4fxHER7/gxLCsAX/G8zsT2A0cI+kL4BllN9h9xZphAZpuvJ5n95cSmsEhINp35v+U6TwQE2SVgCPk0Y3s4BvSNOkjwIf5upMAR6Q9BFpNJQxlfTgXu6ypnr+GGCFUgDMwcCzliJAf+KbOKaZ2dukXZ4LvS0vA/uaWRPwovfFKyRDlDGY7QNuVsRlTQcWA4uAp8xsaakuJdXmAj0kLff2fOqy1pPWJ1/1a/Wil3+DZIiXSRoOPEkygIuBk2kdLb8ADJX0GWn0lgUYLWUEMKe9bQx2D8K7f1BIJM0CbrIK0a79Af2cmS2v0vmmkzbCvFwNedVAacfpKH8hKCSSGoEGM9vQ1boE3YdYYwuKys2kTSRlDZuZ3di56nQ+ZnZuV+vQkUiqBe4LoxaUEiO2IAiCoFDEGlsQBEFQKMKwBUEQBIUiDFsQBEFQKMKwBUEQBIUiDFsQBEFQKP4BGFc+vKwkOV0AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "ax = make_tabular_results_plot('rank_roc', df_=df_[df_.time >= 1].copy(), exclude=['tabnet'], max_times=[1, 5, 30, 60*5, 60*60])\n", + "ax.invert_yaxis()\n", + "ax.set_xlim([np.log10(1.0), np.log10(3600)])\n", + "ax.legend([],[], frameon=False)\n", + "tikzplotlib.save(f'roc_raks_tabular.tex', axis_height='5cm', axis_width='6cm', strict=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 135, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAb0AAAGpCAYAAAAZaejJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAC590lEQVR4nOz9eXyk133f+X7Osz9VBaCwA7032Ww2yW6aIlviIooitVK0RC0U5UWOPUlsZ3IzM3EmuXN9PffeOJ7cjLNOPMnEsZ3Nuy2LpESttkVKokRRJJt7c+9mE0tjXwq1P+uZP56qagCNRgNoLAXgvF8vNIACquoAXahvne13hJQSRVEURdkNtK1ugKIoiqJsFhV6iqIoyq6hQk9RFEXZNVToKYqiKLuGCj1FURRl1zC2ugEr0dXVJQ8dOrTVzVAURVGayHPPPTclpexezXW2RegdOnSIU6dObXUzFEVRlCYihBhY7XXU8KaiKIqya6jQUxRFUXYNFXqKoijKrqFCT1EURdk1VOgpiqIou4YKPUVRFGXXUKGnKIqi7Boq9BRFUZRdQ4WeoiiKsmuo0FMURVF2DRV6iqIoyq6xYaEnhNgvhPiuEOJ1IcSrQoi/X7v814UQ54UQL9be7tuoNiiKoijKfBtZcDoE/qGU8nkhRAvwnBDir2tf+z+klP9qA+9bURRFUS6yYaEnpRwFRmsfF4QQrwN7N+r+FEVRFOVyNmVOTwhxCHgP8HTtov9BCPGyEOK/CCHaL3GdXxZCnBJCnJqcnNyMZiqKoig73IaHnhAiAzwE/IqUMg/8NnA1cBNJT/BfL3U9KeXvSilPSilPdnV1IaXc6KYqiqIoO9yGhp4QwiQJvD+WUj4MIKUcl1JGUsoY+D3gfZe7Hd/3GRkZwff9jWyuoiiKssNt5OpNAfxn4HUp5b+Zd3n/vG/7LHB6JbfneR4DAwPkcjniOF7fxiqKoii7wkau3nw/8DeAV4QQL9Yu+zXgZ4QQNwESeBf4Oyu5McdxiOOYyclJCoUCPT092La9/q1WFEVRdqyNXL35Q0As8aVvrvU2NU0jnU7j+z6Dg4N0dHTQ3t6Opqk99oqiKMrlbcu0sCwL13WZmZlhaGiIarW61U1SFEVRtoFtGXpwodcnpWRwcJDJyUmiKNrqZimKoihNbCPn9DaFZVmYpkkul6NYLNLb20sqldrqZimKoihNaNv29OYTQpBOp9F1neHhYSYmJgjDcKubpSiKojSZHRF6dYZhkE6nyefzDAwMUCwWt7pJiqIoShPZUaEHSa8vlUphmiYjIyOMjY0RBMFWN0tRFEVpAjsu9Orqvb5SqcTg4CCFQkGVMlMURdnldmzoQdLrc10Xy7IYHR1VpcwURVF2uR0denW6rpPJZPA8j8HBQVXKTFEUZZfaFaFX5zgOjuMwOTnJ+fPn8Txvq5ukKIqibKJdFXpwYVN7GIYMDAwwOzuren2Koii7xK4LvTrbtkmlUkxOTjI8PKxKmSmKouwCuzb0IOn1ZTIZ4jhmcHCQ6elpVcpMURRlB9vVoVdnWRbpdJqZmRkGBwepVCpb3SRFURRlA6jQq6mXMtM0jaGhISYmJlSvT1EUZYfZ9gWn15tpmhiGQT6fbxSwTqfTW90sRVEUZR2ont4S6qXMDMNgeHiYsbExVcBaURRlB1ChtwzDMMhkMhSLRQYGBlQpM0VRlG1Ohd5l1Ht99VJmo6OjqoC1oijKNrUtQk/XdRzHwTC2bgqyXsqsWq0yMDDA3Nyc6vUpiqJsM9si9CYmJviDP/gDwjDc0uCDpJSZbduMj4+rAtaKoijbzLYIPYBcLseXvvSlLQ89WFjAemBgQBWwVhRF2Sa2TegBTRcujuPgui4TExMMDw+rAtaKoihNbluFXjabZXp6muHh4a1uSsP8UmYDAwNMT083VTAriqIoF2yb0Mtms3z+85/n+eef57XXXtvq5lzEsixSqZQqZaYoitLExHZYgXjTTTfJxx9/nDAMqVarhGGI4zjMzc1RKBTYt2/fVjdxgSAI8DyP9vZ2Ojo60HV9q5ukKIqy4wghnpNSnlzNdbZ+VcgKRFHUOPrHMIzGYpYXXniBt956i2PHjnHbbbdhWdZWNrOhXspsbm6uUcoslUptdbMURVF2vW0Repdy55134jgOL7/8MsPDw9x1111N0+urb2oPw5Dh4WHa2tro7OxsitWniqIou9W2mdNbimEY3HbbbXz6059G13W++c1vcvbs2a1u1gKGYZBOpxulzIrF4lY3SVEUZdfa1qFX19vbywMPPMDJkyc5cOAAQFOVChNC4LoupmkyMjKiSpkpiqJskR0RepD0qG6++WZM0yQMQx555BF+8IMfNFXFlHoB63K5zODgIPl8XpUyUxRF2UQ7JvQWO3DgAK+//jpf/vKXOX/+/FY3ZwHXdbEsi7GxMVXKTFEUZRPtyNCrz/Xdf//96LrON77xDX7wgx801Zl4qpSZoijK5tuRoVfX19fHAw88wIkTJ5iZmUHTmu/HrZcym5yc5Pz586qUmaIoygba8evnDcPg9ttvJ45jNE2jWq3y4osvcvPNNzfNvj5N00in041eX1dXF9lstilDWlEUZTvbNc+q9QAZGhri5Zdf5qGHHmq6uT7btkmlUo36ovUN+YqiKMr62DWhV3fNNddw//33o2ka3/jGN/jhD3/YVNsH6r2+OI4ZGhpiamqKKIq2ulmKoig7wq4LPVg41/faa6/x9NNPb3WTLlIvYD07O8vg4CDlcnmrm6QoirLt7fg5vUupz/UdPnyY1tZWAEqlEpZlYZrmFrcuIYQgnU4TBAHDw8Nks1k6OjpUKTNFUZQ12vXPnn19fQBIKXn88ccpFot88IMfZM+ePVvcsgvqBazz+XyjgHU6nd7qZimKomw7u3J4cylCCE6ePIkQgq9//etNN9dXL2BtGAbnz59nbGysqfYdKoqibAcq9Obp7+/n85//PMePH+e1117jy1/+MrOzs1vdrAUWF7AuFAqqlJmiKMoK7frhzcUMw+COO+7g8OHDvPDCC2Qyma1u0kXqvb4oihgdHSWTydDd3d00c5GKoijNSvX0LqG/v5/77ruvUcD6W9/6FiMjI1vdrAXqpcyq1SoDAwPMzc2pXp+iKMoyVOitQLFYZG5ujq9//es8+eSTTTXXB0kpM8dxGB8fZ3h4WBWwVhRFuQQVeiuQzWZ54IEHOH78OK+++ipf/vKXm67Xp2kamUyGMAwZGBhgdnZWFbBWFEVZRIXeCpmmyR133MGnPvUpAJ599tmmHEq0bbtRwFqVMlMURVlo2yxkkVIihNjqZjRWeHqehxCCSqVCLpejv79/q5vWUO/1+b7P4OAgnZ2dZLNZdF3f6qYpiqJsqW3R09M0jWKx2DQ1KE3TbKzqfOGFF/ja177WlHN980uZDQ0NUalUtrpJiqIoW2pbhJ5pmuzZswff95tuuO69730vN9xwA6+++ioPPfQQo6OjW92kBTRNI5VKIYRgaGiIiYmJpnnxoCiKstm2RegBtLS0cODAARzHoVgsNs0iDdM0ef/7388nP/lJpJR87Wtf480339zqZl3ENE3S6TT5fJ6BgQFVwFpRlF1p24QeJE/c/f399Pb2UqlUmuqU8T179vD5z3+eG2+8kf379wM0TTDXzS9lNjw8zPj4uCplpijKrrKtQg+SJ+62tjYOHjyIYRhN1+u77bbbSKVSxHHM17/+dX70ox81XbDUS5kVCgUGBgYoFotNuRJVURRlvW270KuzLIu9e/fS3d1NpVJpug3ZcRzT0dHB6dOn+fKXv8zY2NhWN2mBeq/PNE3Onz/P6Oho0y3EURRFWW/bNvQgWaTR3t7OgQMH0DSNUqnUND0WwzC48847G3N9jz76aNP2+lpaWqhUKqqUmaIoO962Dr0627bZt28fHR0dlEqlpuqx1Of6rr/+egYHB5s2UFzXxbZtxsfHGRkZabqes6IoynrYNpvTL0fTNDo7O0mlUoyNjVEul3Fdtyk2tJumyZ133kkQBI0C1i+//DI33nhjU52CvriAdXd3N62trWjajnhtpCiKsjN6evO5rsuBAwdobW2lVCo11XBi/eifoaEhTp06xUMPPdR0c32QFLCulzI7f/58U62SVRRFuRI7LvQg6bH09PSwb98+wjCkXC431bDi4cOH+cmf/EniOObRRx/lqaeeaqpwhqTnnE6niaKIwcFBpqenm2aVrKIoylrtyNCrS6VSHDhwgEwmQ6lUaqpKJHv37uWBBx7g+uuv55VXXuGJJ57Y6iYtybIsXNdlZmaGoaGhpquIoyiKshobNqEkhNgP/AHQB8TA70opf0sI0QH8OXAIeBf4gpRydqPaYRgGfX19ZDIZxsfHgWQItBlYlsWdd97J4cOHSafTAHieh67rTTXXV+/11QtYt7e309HRoQpYK4qy7WxkTy8E/qGU8jrgNuDvCSGuB34VeExKeQ3wWO3zDZfJZDhw4ACu6zbVhnZIen3ZbBaAH/zgB00712dZFul0mlwux+DgoCplpijKtrNhoSelHJVSPl/7uAC8DuwFPg38fu3bfh/4zEa1YbH5Zcyq1WpTDtVdd911RFHUtHN9QgjS6TS6rjM8PMzExETTtVFRFOVSxGYs8BBCHAKeAI4Dg1LK7LyvzUop25e4zi8Dvwxw4MCBWwYGBta1Tb7vMz4+TrVaxXXdplqW7/s+Tz/9NK+//jptbW189KMfpaOjY6ubdREpJZVKBSEEvb29jeOWFEVRNoMQ4jkp5cnVXGfDn+mFEBngIeBXpJT5lV5PSvm7UsqTUsqT3d3d696u+WXMyuVyU23GtiyLD3zgA9x3332YponjOFvdpCXNL2U2MjLC2NhYUxUGUBRFWWxDV0sIIUySwPtjKeXDtYvHhRD9UspRIUQ/MLGRbViOpmlks1lc12V8fJxSqdQ4e64Z7Nu3j7179yKEII5jnnjiCa677jp6e3u3umkL1AtYl0olSqUSPT09ZDKZpvk9Koqi1G1YT08kz3j/GXhdSvlv5n3pUeAXah//AvDVjWrDSs0vY1Yul5uqt1IPjlKpxMjICI8++ig//vGPm24eTQiB67pYlsXo6KgqZaYoSlPasDk9IcSdwA+AV0i2LAD8GvA08CXgADAIPCilnFnutk6ePClPnTq1Ie1crFKpMDY2RhRFTVPGrM73fX784x/zxhtv0NbWxt133910vb66arVKHMd0dXWpUmaKomyItczpbcpCliu1maEHEEUR09PT5HI5HMdpqj1zAMPDw3z/+9/HMAwefPDBpg2UOI6pVCo4jkNPTw+2bW91kxRF2UFU6K2zcrnM2NgYUkocx2m6Xl+pVKK9vZ0wDJmZmaGnp2erm7Ukz/MIgoCuri6y2WzThrSiKNtLU67e3M7qZczqizSaqYyZZVm0tyc7PV588UW++tWvNuVcHyRzpqlUiqmpKYaHh5tyf6SiKLuDCr3LqJcx27NnD77vU6lUtrpJF7nxxhu59tprefnll3n44YeZmNiyBbGXpGkamUyGOI4bBayb6UWEoii7gwq9FVpcxqyZnrAty+Kuu+7ivvvuIwgCvvrVr/Laa69tdbOWVC9lNjMzw+DgYFO+iFAUZedSobcK88uYeZ7XdMN0+/bt48EHH+TYsWPs2bMHoKmOVKqrlzLTNI2hoSEmJiaa6kWEoig7V3MtS9wGhBC0tbXhui4TExOUSqWmKmNWr+ZS9/jjj5PJZLjllluabhWqaZoYhkE+n6dYLNLb29s4bUJRFGUjNMcz9TZkWRZ79uxpyjJmdXEcY5omL730UtPO9dVLmRmGwfDwMGNjY025GEdRlJ1Bhd4VqJcxO3jwIJqmUSqVmurIIk3TuOuuu/jEJz7RmOt75plnmjJUDMMgk8lQLBYZGBigUCg05dCsoijbmwq9ddDMZcwA9u/fz4MPPsjRo0d54403mq59dfVeX72U2ejoaNO2VVGU7UltTl9n1Wq1cdpAMxWvrqtUKriuSxzHvP766xw7dqxpT0CvlzLr7u6mtbW16X6XiqJsLbU5vQk4jsP+/fvJZrMUi8WmG0p0XReA8+fP8+STTzbtXB8kv0vbthkfH2dkZATP87a6SYqibHMq9DaArut0d3ezf/9+wjCkXC433fzU/v37uffee/F9vzHX14zbBnRdJ5PJ4Ps+AwMDTE5ONt0LCUVRtg8VehuoXsaspaWFUqnUdE/WBw4c4POf/zzXXHMNL774It/5zne2ukmXZNs26XSaubk5BgYGmJuba6pFQ4qibA/NtXFrBzIMo7H/bHx8nCAIGkOMzcC2be6++26uuuoqLMsCIAxDhBBNN9dXX+gSRRETExPkcjm6u7tJpVJb3TRFUbYJFXqbJJPJ4DgOExMTFItFXNdtqlA5cOBA4+Nnn32W4eFh7r77brq7u7ewVUvTdZ10Ok0QBAwPD9PS0kJnZ2cjtBVFUS5FDW9uIsMw6O/vp6+vrynLmNXt3bsXz/P4yle+0rRzfZBUdEmn01QqFQYGBlQRa0VRLkttWdgivu8zOTlJqVQilUo1TRmzOs/zeOqpp3jrrbdob2/nwx/+MB0dHVvdrEuqH1hbX0SUyWTUFgdF2eHUloVtpF7GrKenh0ql0nTL8etzfffeey9RFDVdKC+maRrpdBrDMBgdHVXn9imKsiQ1p7eFhBBks9lG8episdh0vb4DBw6wb9++RpueeeYZDh8+3JRzfXChnJnv+wwODtLW1kZHRwemaW510xRFaQLN8+y6i9m2zd69e+nq6mrK4tX1wCuXy7z11lt85Stf4dlnn23q+bP6uX31Wp6zs7Nqi4OiKCr0moWmaXR0dDRWUZZKpabb0J5KpXjwwQe55ppreOGFF3j44YeZmpra6mZdkhAC13VxHIepqSkGBgaa8veqKMrmUaHXZOaXMWvGDe31ub6Pf/zjeJ7Ht771raZr42L1+T5d1xkeHlYlzRRlF1Nzek2ovgIxnU4zNjaG7/u4rttUqxEPHjxIX18fuVwOwzCI45hcLtfUKzwNw6ClpQXP8xgYGKC9vZ329vamO1xXUZSNo3p6TSyVSnHw4EFaWlqasni1bdv09vYC8Nprr/HQQw9x6tSppp7rA1XSTFF2MxV6TU7XdXp7e9m7dy9BEFCpVLa6SUs6cuQIR44c4fnnn+eRRx5p6rk+WHh238TEBENDQ5TL5a1ulqIoG0yF3jaRyWQ4ePAgqVSKYrHYdL0px3G45557+NjHPkalUuGRRx7h1Vdf3epmXVa9pBnA8PAwo6OjTbd6VlGU9aMmM7YRwzDo6+sjk8kwPj6Opmk4jrPVzVrg0KFD9PX18dRTTzX1/N5ipmlimiaVSoV3332Xzs5OstlsU9VHVRTlyqnQ22aEELS0tCwoXt1sG9rrvb66H//4xxiGwXve856mDxHHcYjjmJmZGebm5lRJM0XZYZrnmVJZFdM0m7qMWZ2UkkqlwvPPP89XvvKVpp/rgwtbHEzTVCXNFGWHUaG3jdXLmB08eBDDMCgWi023ClEI0ZjrK5fLPPLII9tihSdcOLU9iiIGBwcZHx9vut+voiiro0JvB7Asq6nLmEEy1/fggw9y9dVX89JLL1EsFre6SStWL2mWy+WadvWsoigro+b0doh6GbNUKsX4+HjjyKJmmotyHIcPfehD5PN5WltbATh79iyHDx9uqjnJpQghME2TYrHYWO2pKMr209zPNMqq1cuYtbe3UyqVCIJgq5t0kXrgjY+P89hjj/HII48wPT29xa26PMuymnIIWVGUlVOhtwNpmkZXVxf79+8njmPK5XJTFlnu7e1tzPU9/PDDPPfcc00dKJqmEcdx0y4aUhTl8lTo7WCu63LgwAFaW1ubsowZXJjru+qqq3juuef49re/vdVNWpau69tqPlJRlIXUnN4Op+s6PT09TV282nEcPvzhD3PVVVc12lXv8TXbXJ9lWRQKBbq6uprqd6goyso01zOKsmHS6TQHDx4kk8k0ZRkzgMOHD3Po0CEATp8+zVe+8hXK5TKO42DbNo7jbPmJCJqmEUWRGuJUlG1K9fR2EcMw6O3tJZ1OMzExgRCi6cqY1bW2ttLa2oqmafzBH/wBuVyObDbLF77wBQzD2NKhWiEEpVKpaX93iqJcmurp7TL1MmYHDhzAtu2mXY146NAh7rvvPh599FFyuRwAuVyOL33pS1ve27Ntm3w+35SLgxRFWZ4KvV2qXsast7e3acuYCSEagVeXy+W2PGx0XScMw6YsAqAoyvJU6O1iQgja2tqatoxZvczafNlsllKpxIsvvril85JCCHX+nqJsQyr0lEYZs+7ubiqVStP0YMIw5Atf+EIj+Opzei+//DLPPPMMX/7ylxkaGtqStlmWRT6f35L7VhRl7cRWDxWtxMmTJ+WpU6e2uhm7QrVaZWJiAs/zmqKMmWEYGIaBlBIhBGEYEoYhg4ODPPXUU8zNzXHw4EHe//73k8lkNrVtpVKJgwcPYlnWpt6voigJIcRzUsqTq7mOWr2pLOA4Dvv27WN2dpbp6Wls28Y0zS1rTz3kFjtw4AB79+7llVde4eWXX96SYVkhBJVKRYWeomwjKvSUi2iaRmdnJ6lUirGxMcrlctNtaIdkQclNN93E8ePHG73BJ554gn379i3Y6L5RTNNkbm6Otra2Db0fRVHWj5rTUy5pfhmzUqnUlGXMgMYWBt/3mZqa4rHHHuPrX/86MzMzG3q/pmlSrVabsqi3oihLU6GnLKtexmzfvn0EQdC0xash2T/32c9+ljvvvJOZmRkeeughnnzyyQ1dmCOEUKeqK8o2okJPWZFUKtX0ZcwgGZq9/vrr+amf+imuu+46zp07t6H3Z5omhUJhQ+9DUZT1o0JPWTHDMOjr62Pv3r34vt/UPRzHcbjzzjv5qZ/6KSzLIo5jHn/8ccbHx9f1fkzTbOqhX0VRFlILWZRVy2Qy2LbN5OQkxWKRVCrVdKch1NVXnubzeUZGRjhz5gzXXHMNt956K6lU6opvXwiBlJJqtbrpWyYURVm9yz5TCSH+vhCiVST+sxDieSHExzajcUrzMk2T/v7+pi5jNl82m+WnfuqnuOmmmzh79ix//ud/zksvvbQuWx3UEKeibB8reXn+t6SUeeBjQDfwN4Hf3NBWKdtCs5cxW8w0Td73vvfx4IMP0t/fz9tvv70ut2tZFqVSqWnnORVFuWAlw5v1zU73Af9VSvmSaLYNW8qWqpcxm5ubY3JyEsuymnrDdltbG/feey+e56FpGp7n8dRTT3HzzTfT2tq66turD3HWq9goitK8VtLTe04I8VckofeXQogWoHlfzitbQtM02tvbOXjwIJqmUSqVmnZrQ51t2wBMTk7yzjvv8Bd/8RecOnVqTYtSdF1XQ5yKsg1ctvamEEIDbgLekVLmhBCdwF4p5cub0D5A1d7cbuI4bpoyZitVLBZ5+umnOXv2LJlMhttuu43Dhw+vuKpLHMd4nsfhw4ebdlGPouw0a6m9edm/TillDIwD1wsh7gJuALJraqGyK9TLmO3fv584jpt6Q3tdJpPhwx/+MJ/61KewLIu33nqrEXiGYeA4DrZt4zjOkofYaprWCD5FUZrXZef0hBD/HPgp4DWgPlMvgSc2sF3KDlAvYzY9PU0ul7tkYDST/v5+Pve5zzWquHieRxAEPPLII+RyucbxRoZhXDQMqmkaxWIR13W3oumKoqzASsZhPgNcK6W8T0r5qdrb/RvcLmWHmF/GLAzDbdHr0zQNx3GAZJFOPfAgObn9S1/60pLhbds2hUKh6X8+RdnNVhJ67wDNPymjNLVUKsWBAwfIZDLbanm/4ziNwKvL5XJLBpumaURRpIY4FaWJrWSsqQy8KIR4DGj8NUsp/6cNa5WyI9XLmGUymUY5sGYfChRCkM1mFwRfNpu95AIXIQSlUqnRU1QUpbmspKf3KPC/AT8Cnpv3pihrkslkOHDgAK7rNv2G9jAM+cIXvkA2mwVozOldaluDbdvk83k1xKkoTeqyPT0p5e+v5YaFEP8F+CQwIaU8Xrvs14FfAiZr3/ZrUspvruX2le2tXsYsn88zOTm5YB6tmYRhiGEY/PzP/zxSSoQQlzzNHZI5zGq1iu/7jX2AiqI0j0uGnhDiS1LKLwghXiFZrbmAlPLGy9z2fwP+PfAHiy7/P6SU/2q1DVV2nnoZM9d1mZiYoFQq4bpu0+1zWxxyMzMztLe3LzvEWS6XVegpShNarqf392vvP7mWG5ZSPiGEOLSW6yq7i2VZ7Nmzh3w+z8TERFOXMRseHuab3/wm9957LwcOHFjyeyzLIp/P097evsmtUxTlcpZ7Sf0fhRD/T2AfMCqlHJj/dgX3+T8IIV4WQvwXIcQlnxWEEL8shDglhDg1OTl5qW9TdghN08hmswvKmDXjXN+ePXtIp9O88sorl/wewzDwfX9DT2xXFGVtlgu93wPagf8/MCaE+JEQ4l8KIT4rhOhd4/39NnA1SVmzUeBfX+obpZS/K6U8KaU82d3dvca7U7Yb27bZt28fHR0dVCoVgiDY6iYtoGkaN9xwA+fPn2dmZmbZ761UKpvUKkVRVuqSoSel/LqU8teklHeTHCn0P5GUI/uXwMha7kxKOS6ljGqlzX4PeN9abkfZ2eaXMZNSNl3x6mPHjqHr+rK9PcuymJub28RWKYqyEsuu3hRCdAF31N5uAxzgO8BTa7kzIUS/lHK09ulngdNruR1ld3Ach/379zMzM8Ps7GzTFK92HIejR49y7ty5xurOxUzTpFgsEgRBU7RZUZTEcqs33wbmgIeAvwT+qZSyuNIbFkL8KXA30CWEGAb+MXC3EOImktWg7wJ/Z60NV3YHXdfp7u5ubGgvFoukUqktX+F58uRJbr311mVriQohqFarKvQUpYks19P7LyS9uweAE8BxIcRTwAtSysvWkJJS/swSF//nNbVS2fXqxavz+TxTU1MIIXAcZ8VH/2xEewCklEgplwxh0zQpFAq0tLRsdvMURbmES4aelPJ/r38shDhKMsT5S8AHhBCTUsoPbkL7FKWhvsIznU4zPT1NPp/f0iHPcrnMN7/5TU6cOMG111570ddN06RUKl1yCFRRlM132TEiIcRVJAtObiXp+XUD6ohoZcuYpklfX19joctWlTJzXRcpJadPn15yoY0QAikl1Wp109umKMrSLhl6QohHhBCjwDeBD5PU2/wZKWW3lHJNG9YVZT3Vhzx7enqoVqtUKpVNXeUphODEiRNMT08zOjq65PfUhzgVRWkOy/X0/itwQkp5TEr5N6WU/0lK+dpmNUxRVmL+pvZ0Ok2pVNrUvX1HjhzBcZxLbl+wLGtbHaWkKDvdcvv0HpVSTm1mYxRlreYPeQKbNuRpGAbXX389AwMDS+7Lqw9xqjP2FKU5qNl1ZUdxXZf9+/dv6irP66+/ntbWVjKZzJJf13WdQqFAKpXasDYoirIyzVXOXlHWwWYPeaZSKY4ePYqu60t+3bKspj83UFF2i+UWsnxcCPH5JS7/ohDioxvbLEW5cps55Cml5OWXX+aNN9646GuaphHHsRriVJQmsFxP758A31/i8seA39iY5ijK+qsPeW7kKk8hBAMDAzz33HNLBqumaRSLKy5opCjKBlku9FJSyovO9JFSjgHpjWuSoqy/+UOemUxmQ4Y8T5w4QalU4ty5cxd9zbZtCoVCUxXOVpTdaLnQc4QQFy10EUKYgLtxTVKUjWOaJr29vRsy5HngwAFaW1uX3L6gaRpRFKkhTkXZYsuF3sPA7wkhGr262sf/sfY1Rdm2Fg95lsvlK+6FaZrG8ePHmZiYYHx8/KKvCyEolUpXdB+KolyZ5ULv/0Nyft6AEOI5IcTzJCcjTNa+pijb2vwhz5aWFkql0hWfdn7ttdc2Tn9fzLZt8vm8GuJUlC0kLvcHKIRwgSO1T89IKTf9OOiTJ0/KU6dObfbdKrtMpVJhcnKSarW6YccXlUolDhw4gG3b637birLbCCGek1KeXM11ljtP73OLLpJAVgjxopRSFRNUdhzXddm3b19jY3v9srVsbC+VSszMzDTmDuuEEJTLZRV6irJFlqvI8qklLusAbhRC/G0p5eMb1CZF2TLzjy+amZlhbm4Oy7KwLGtVt/PUU08xPDzMF7/4xQVHH1mWRT6fp729fb2brijKCix3nt7fXOpyIcRB4EskRw0pyo5UX+XZ2trK5OTkqk9sP378OO+88w5vvfUWN9xwQ+NywzAac4erDVJFUa7cqictpJQDwNac2qkom6w+5Flf5bnSvX29vb10d3df8qy9SmXTp8YVRWENoSeEuBZQm42UXaM+5Ll37148z1vRvr76WXtzc3MMDg4u+JplWeqMPUXZIsstZPkayeKV+TqAfuBvbGSjFKUZua5LZ2cn09PTlzxRYb6rrrqKZ599lpmZGQ4ePNi43DRNisUiQRAsmO9TFGXjLbeQ5V8t+lwC08DbUsor28ykKNtUe3s75XIZz/MuuwJT0zQefPBBDOPiPzMhBNVqVYWeomyy5RayLFVsGiHE+4UQPyul/Hsb1yxFaU6aptHb28vg4CBRFF3yOKG6euCVy+UF5+mZpkmhUKClpWVD26soykIrmtMTQtwkhPgXQoh3gX8KXHx+iqLsEpZl0dvbu+LSZadPn+ZP//RPKZfLjctM06RcLhOG4UY2VVGURZY7T++oEOL/J4R4Hfj3wBBJBZd7pJT/btNaqChNqKWlhba2thWtwty3bx9RFPHaa681LhNCIKWkWq1uZDMVRVlkuZ7eG8CHgU9JKe+sBV20Oc1SlObX1dWFruuX3caQzWY5cOAAr7/++oKenWEYahWnomyy5ULvAWAM+K4Q4veEEB8GVl+PSVF2KMMw6OvrW9E2hhMnTlCpVDh79mzjMsuyKJVKRJF6Lakom+WSoSelfERK+VPAMeB7wD8AeoUQvy2E+NgmtU9Rmlp9G8P8+bql7Nmzh46ODl5//fXGZfUhTnXGnqJsnuW2LAAgpSwBfwz8sRCiA3gQ+FXgrza4bYqyLdS3MVSrVRzHWfJ7hBDcc889pNPpBZfrut4ocaYoysZbVUUWKeWMlPJ3pJQf2qgGKcp2U9/GEMfxskOVnZ2dF4VivTrLep3erijK8tb/wDBF2YVWuo1henqaRx55hFwuBySBGcexGuJUlE2iQk9R1slKtjG4rsv09DSnT59uXKZpGsVicTOaqCi73nL79I4IId6/xOUfEEJcvbHNUpTtqbu7e9ltDKlUiiNHjvDWW281ene2bVMoFFa00V1RlCuzXE/v3wJLbSKq1L6mKMoiuq5fdhvDiRMnCMOwsZJT0zSiKFJDnIqyCZYLvUNSypcXXyilPAUc2rAWKco2d7ltDJ2dnezZs4dXX321EYxCCEql0mY2U1F2peW2LCy99jrhrndDFGUnudw2hptvvpl8Pt8Y0rRtm3w+T0dHB0KoGhCKslGW6+k9K4T4pcUXCiH+NvDcxjVJUba/y21j2LNnD8eOHWuc0qDrOmEY4vvq1C5F2UjL9fR+BXhECPFFLoTcScACPrvB7VKUba++jWFkZIRMJnNRDy4MQ9544w16e3vp7u5GCEGlUrnsOX2KoqzdcmXIxqWUdwD/BHi39vZPpJS3SynHNqd5irK9LbeNQUrJqVOneOmll4AkJOfm5ja7iYqyq6x0n54E4tp7RVFW4VLbGEzT5NixY5w7d45isYhhGPi+r4Y4FWUDLbdPb68Q4mng14GrgCPArwshnhFC7N2k9inKtlffxuD7/kXbGI4fPw6wYLP6Ss7oUxRlbZbr6f174LellB+UUv7PUsp/IKX8YO3y/7A5zVOUncF1XTo6Oi7axpDJZDh8+DBvvPEGQRA0anEqirIxlgu966WU/23xhVLKPyA5bkhRlFVob2/Hdd2LTks/ceIEHR0dVCoVTNOkXC5f9mBaRVHWZrnQ05e6UAihXepriqJc2qW2MfT29nL//ffT2toKJBvVFwejoijrY7nQ+1rtxPTGAWC1j/8j8M0Nb5mi7EDLncZQLpeZnZ3FNE01xKkoG2S50PtfgDlgQAjxnBDiFMm2hTzwjzahbYqyIy21jUFKyaOPPsqTTz6JaZqUSqVlz+ZTFGVtltunF0gp/xGwH/jvgL8FHJRS/iMppVpTrShXYPE2BiEEx44dY2RkhOnpaQA1xKkoG+Cy+/SklBUp5StSypellGUhxEeFEH+9GY1TlJ1qqW0M1113HYZhcPr0aQzDIJ/Pb3ErFWXnWW6f3oeEEG8JIYpCiD8SQlxfG+L8TeC3N6+JirIzLd7GYNs2R48e5cyZM4RhqIY4FWUDLNfT+9fALwOdwJeBHwN/KKW8RUr58GY0TlF2usXbGE6cOIGUktHRUaSU6ow9RVlny4WelFJ+T0rpSSm/AkxKKX9rk9qlKLvC4m0MbW1tfPGLX+TIkSPouk6xWNzqJirKjrLcKQtZIcTn5n0u5n+uenuKsj7q2xhGR0dJp9OkUikgCcRCoUBXVxeattIyuYqiLGe50Ps+8KlLfC4BFXqKsk5aWlool8sUCgVSqRRPPfUU58+f595778XzPFxXndusKOvhkqEnpfybm9kQRdnturq6GiXIOjo6eOWVV5iYmGjM+ymKcuXUmImiNIn52xgOHz6M67q8+eabFAqFi6q3KIqyNir0FKWJuK5LZ2cnvu9z/fXXMzQ0xOzsrFrFqSjrRIWeojSZbDaL67pcddVV6LrOm2++SalU2upmKcqOsNxClgYhxB3AofnfXztiSFGUdVbfxuB5Hh/60Ifo7u4mn8/T0dGBEGKrm6co29plQ08I8YfA1cCLQL08hARU6CnKBqlvY4jjmHQ6Tblcxvd9bNve6qYpyra2kp7eSZIDZdVMuqJsovo2hjNnzvDGG2/wmc98RoWeolyhlczpnQb6NrohiqJcrKuriyiKGBoa4uWXX97q5ijKtreSnl4X8JoQ4hmgsYRMSnn/clcSQvwX4JPAhJTyeO2yDuDPSeYH3wW+IKWcXVPLFWUX0HWdkydPMjQ0xL59+xqHzKZSKVWlRVHWYCWh9+trvO3/Bvx7Fs79/SrwmJTyN4UQv1r7/P+1xttXlF3BcRzuuusuHnroIXK5HNlslp/+6Z+mp6dHBZ+irNJKztP7/lJvK7jeE8DMoos/Dfx+7ePfBz6z2gYrym5TLpcbgQeQy+X4sz/7s8aRRIqirNxlQ08IcZsQ4tnauXq+ECISQqz1dMteKeUoQO19zzL3+8tCiFNCiFOTk5NrvDtF2f7CMGwEXl0ulyMMw61pkKJsYysZG/n3wM8AbwMu8Iu1yzaUlPJ3pZQnpZQnu7u7N/ruFKVpGYZBNptdcFk2myWKInXIrKKs0oomBKSUZwBdShlJKf8rcPca729cCNEPUHs/scbbUZRdI5VK8dM//dON4Mtms3z605/m0Ucf5YknnmByclKVKVOUFVrJQpayEMICXhRC/AtgFEiv8f4eBX4B+M3a+6+u8XYUZdfQNI2enh5+8Rd/kTAMMQyDXC5HPp/n+9//PlNTU9x4442kUina29vVyk5FWYa43J5zIcRBYBywgH8AtAH/odb7W+56f0rSI+yqXf8fA18BvgQcAAaBB6WUixe7XOTkyZPy1KlTl/s2Rdk1pJSMjo7y+OOPc+bMGfr7+/nYxz5GGIZomkY2m6WlpQXLsra6qYqyYYQQz0kpT67mOpft6UkpB4QQLtAvpfwnK71hKeXPXOJLH17pbSiKsjQhBH19fXzkIx+hp6eHMAyxbRvbtonjmNnZWWZmZkin02SzWRzHUb0/RWFlqzc/RVJ389u1z28SQjy6we1SFOUy6sOeN9xwA0ePHgVgeHiYF154Add1SaVS+L7P+fPnGRgYUCs+FYWVLWT5deB9QA5ASvkiSUUVRVG2WP1EBsuyqFarDA4O8txzz/Gtb32LarWKZVmk02kMw2BycpJz584xPj5OpVJRB9Mqu9JKQi+UUs5teEsURVkTwzDo7+9HCMEtt9zCBz7wAUZHR3n44YcZGxsDknJm6XSaVCpFqVRieHiYwcFB8vm82vag7CorKjgthPhZQBdCXCOE+HfAjza4XYqirIJpmuzZs4c4jjly5Aif/vSn0XWdr33ta8wv7iCEwHEc0uk0QgjGx8c5d+6c2vag7BorCb3/EbiBpNj0nwJ54Fc2sE2KoqyBbdvs3bsX3/fJZrN87nOf49Zbb6WrqwvgouFMwzBIp9M4jkM+n2dgYIChoSGKxSJxHG/Fj6AoG+6yWxaagdqyoCgrVyqVGBkZwXXdxorNfD7Pd77zHe66665GCC7F932CIFDbHpRtYV23LFxuhebljhZSFGVrpNNpenp6GB8fb2xU9zyPSqXCV7/6VW6//Xauu+46hBAXXdeyLCzLUtselB1ruX16twNDJEOaTwMX/4UoitKU2traiKKIqakp0uk03d3dPPDAAzz++OP88Ic/ZGxsjA984AOYprnk9TVNI5VKIaXE8zyGh4cxTZP29nYymQyGsZJiTorSfJZ72dYH/BpwHPgt4KPA1EqPFlIUZWu1t7fT3t7eOILIcRw+8YlPcPLkSc6ePcsLL7xw2dsQQmDbdiPo5m97qFaratuDsu1c8uWalDIi2ZD+bSGETXLSwveEEL8hpfx3m9VARVHWRghBZ2cnURRRKBQaKzZvvvlm+vv7G3N7vu+vaN6uvu1BSkmpVCKfz2PbdqPep67rG/0jKcoVW3aMohZ2P0kSeIeA/xN4eOObpSjKeqhXbYmiiEqlguu6APT39wMQBAFf+cpX6Ovr44477ljRsGV92wMkZ/2NjY0hhKCtrY3W1lZs2964H0hRrtByC1l+n2Ro81vAP5FSnt60VimKsm40TaOvr4/z589TrVYbgQVJ7+3QoUO8+OKLTE5O8tGPfpTW1tYV37ZhGBiGQRzH5PN5crkcruvS3t6+YPWoojSLS25ZEELEQKn26fxvEoCUUq78L+MKqS0LinLlgiBgeHgYIcRFw5kDAwN873vfI45j7r77bg4fPrzm+6lve9B1vbHt4VILZhTlSqxly8IlX4ZJKTUpZUvtrXXeW8tmBp6iKOvDNE327t1LFEUEQbDgawcPHuRzn/sc2WyWF1988Yo2p9frfVqWxczMDOfOnWN0dJRyuawWvihbTq07VpRdxLIs9u7dy9DQEEKIBXN4LS0t3H///Xie19jbFwQBmUxmTfc1f9tDtVqlWCxiGIba9qBsKTXgrii7jOM47Nmzh2q1elGPTtd1UqkUAE8++SQPPfQQQ0NDV3R/9W0P6XQaXdeZnJzk3XffbWx7UJTNpEJPUXahdDpNX18f5XL5kkOZN998M+l0mm9961s8++yz61KPs17v03VdSqUSg4ODDA4OUigU1GkPyqZQoacou1Rrayvd3d2USqUl59qy2Syf+cxnOHr0KC+88ALf/OY3Gxvdr1R920Mmk0FKyejoKOfOnWNqagrf99flPhRlKWpQXVF2sfb2dsIwJJfLkUqlLqrHaRgGd999N/39/Tz//PMb0gbTNDFNkziOyeVyzM7Oqm0PyoZRoacou1xXV9eCqi1Lufbaazly5Ai6rhPHMWfPnuXIkSNLFq1eq/rCF0i2PYyMjKDremPhi9r2oKwH9RJKUXY5IQQ9PT2kUqllhy/rZcbeeecdvvvd7/KXf/mXG7YQpb7twTRNpqenG9seKpWK2vagXBEVeoqiNKq22LZ92SC7+uqrueOOOxgeHubhhx9mYmJiw9pVX02aTqepVqsMDw8zMDDA3NwcYRhu2P0qO5cKPUVpIjKSyCAm9kLickBU9InyHsF0hWC8jH++SFjwNqS3o+s6fX19CCHwPO+S3yeE4Pjx49x/f3Kk5qOPPspbb7217u1ZfJ/1bQ+apjExMcG5c+eYmJhQ2x6UVVFzeoqygWQkIZbIuPY+iiGGOIghjJFhjAwjZBhDBEmWLQ40gRASNAFCEBU9IsfE7HbRUus7z1Wv2jI0NEQQBMvOo/X09PC5z32OJ554gmw2u67tWE693qeUkmKxyNzc3LY67UFKSRzHSCkXfDz/fRzHRFHUeKt/Xn8vpUQIcdk3oLEQqP5+8ffM/3r9Oqt5v5bv3Uoq9BRlFS6E17wwiyRxOD/EkjdCeYke2bwQ0wRCEwhDBwu0lTwp2DqxH+ENFtAytfCz1+9PebmqLYs5jsPHPvaxxucvvfQS+/bto7Ozc93acynzT3sIgoCxsTE0TWvU+1zJcUmrdamQmv9+qbCa//lK9jsuDqX5Hy8O9fpjrB6i9bf5X5v/9aWut/iyehvq4brWkYXFt7E4AOcH81JBvTiUF19XCLHq0UoVeoqyArEXEYwWif3o4o4YIAQXQkwIhK6BucIQWwPN0sHSiash3rk8RtbG6HQQ5vr0chzHYe/evQwPD+O67op6T9VqlVdeeYVTp05x5513cu21165LW1Zi/raH2dlZZmZmSKVSZLPZxraHy4XV/ICK45gwDJfsYV3K/Cf2xb2n+ouH+V9bCxnFhNNVZCTRMyaaY6AZzd2zXWxx2C4ObaAxX7tcSPu+j6Zpq86wS56y0EzUKQvKVgrzHsFYCaFraHbzPcFIKZHVCBlLjE4HI+sgjPWZri8UCoyOjpJKpVb0ZF0ul3n88ccZGRnh6NGj3HnnnVtWY7N+2sP8IbXL9VwW96yW6m1tldiLCCdLEAO6QIYSJAhHR29JAlDou2eZRrlc5tixY6+EYXjjaq6nenqKcgkykoRTZcJZD83Vm/YJRQiBcA1kLAmnPaIZD6PbRW+1EfqVPUm3tLQQRRHj4+NkMpnLPumnUinuu+8+nn/+eZ5//nlyuRyf/vSntyQsLMvCsqxGwDXDfNJayFgSFQOimQrC0hBW8sJLmLVeTxgTTiWLeYSjN3qAzfp43Woq9BRlCbEfEZxPhjO1tLEtnjCFJtDTBjKSBBNlwpkKRncKPWMhtLW3P5vNEoYhMzMzpNPpy/4uNE3j5MmT9Pb24nnelv/utvr+r4SMYoLpKrISIhzjov9HIQSY+tIB6OroaRWAi6nQU5RFwrxHMFpGGAI9vf2qgAg9abcMY4KREpFdxehJoaXWHt6dnZ1EUUQ+n79k1ZbF9u/f3/j4zTffZHJykttvv73pV1c2i7gaEk4mxQI09/JP1RcFYBATTlZBgHANjIyJsJt3xGKzqNBTlBoZS8LJ5h/OXClhaOgZDRlE+EMFtJSB2ZNCc1b/Zy+EoLu7myiKKJfLjXJhKzU3N8drr73G5OQkH/nIR2hpaVl1G3YLGUuivE806yFsbU3zsxcHYEQwmSwO2e0BuPt+YkVZQuwnwRDmvGQ4cwc9GQgzmeeRQYT3bh5/pLYKdZU0TaO3txfbtqlUKqu67vve9z4++tGPksvlePjhhxkYGFj1/e8GMowJJstEOQ/h6uuyIEkIgTB1NNdAOHotACv4w0WCiTJxOUCuw7FR28XO+ctWlDUKCx7eu3lkEKGnzW09B7QczTbQ0gZxKcB7Z45gopzsJ1wFXdfp7+9H1/Vlq7Ys5fDhwzzwwANkMhn+8i//kpmZmVVdf6eLKyHBSBH8OBmKvoJ52EtZHIDSrwXg0O4JQDW8qexaMpaEUxXC6UryJLODeneX0ljpKSVhziPMebVtDvaKf37DMNizZ8+KqrYs1trayqc//WkGBwfp6OgAIIqiXT3PJ2NJNOcRzXkIa316dyshhABLR1Db9uJHhJMhUiRziHraXHLxzHa38//KFWUJjeHM2SpaxtwVgTefEAI9ZaA5OuFUBe/cHGHOSyrMrEC9aksQBKsu/GwYBldddRUA4+Pj/Nmf/Rnnz59f9c+wE8RBTDBRJprzEa6xaYG3mBAiCVzXQNg60gsJJyv4wwWCyTJxJVzxY6PZ7a6/dEUBwoKPvwuGM1ci2eZgIkyNcLyE9+4cUdFfUdkpx3HYs2cP1WqVKFr9HCEk4WmaJt/4xjd4/vnnd9WxQVE5IBwtQhBd0cra9ZYEoLEgAIOJMsFwgWBq+wegCj1l15CxJJgsE5wvICxtTasYdyqha2i1FwD+cAF/qEBcDi57vVQqRX9/P+VyeUU1JRdrb2/ns5/9LEeOHOHUqVN861vf2vGnJshYEs5UCSfKYGqIdaybut7qAai5Btg6srL9A1CFnrIryCDCHy4QTleTJ/ctGkZqdsLU0DMWMozxBvN45wvE3vK9uJaWFnp7eymXy2vqqZmmyT333MMHPvABRkdHefPNN9fa/KYXBzHBeJmoUBvO3EbD6kIIhL10AIbT2ycAm/clhqKsk6gUJKviBOiZ7bfZfCs0ClpXQrxzOYx2B6PDRZhLP0mvtmrLYkIIrrvuOvr7+2ltbQWSup8rKX22XUQln3CqitBXttm8mQkhwDaSRTCxJC6HRMUQIUBLm8kLS0tvykUw2/s3ryjLSIaRKoSTlWSJturdrZrmJCs9o7xfW+npYrQvvdKzs7OTOI6Zm5tbcdWWxern8lWrVb7yla/Q29vLBz/4QWzbvpIfY0vJWBLlqkT5INlsvsbeneXaWFkXKUBI8HMV/Mrqto1sBKEtDsDk8GMhBFrGREs1VwCq0FN2JBnE+GNF4nKYrM7cIb2FrbCwoHWFaLaK0eWit9kLnsiEEHR1dRGG4Zqqtsxn2zY/8RM/wdNPP83DDz/MRz/6Ubq6utbjx9lUsR8TTpWRQZxsNl/j49BybWRa4w//9I/I5XJks1m+8PkHsbCbIvjqLgrAYpAM5TZRAKqjhZQd58JwpkBzdu/+r40io5i4GiEMkRS0brEWPJlHUcTIyAi+7+O67hXd19jYGI899hjVapXbb7+d6667btu8gImKPuFMNTkk2Frb41BKycTMJP1H9vHQIw+Ty+UaX8tms/zcT3+RRx55BNdycGwbx3ZwLIfuji6yLW1JLz2OMPSt7d/IWIKfHH8lNIGWsZIVq1cQgOpoIWXXa6yKm6qgOZu3yXe3EbqGntaSklmjRaJpA7M3hZZK5kvrVVuGh4fxPO+Khib7+vr43Oc+x3e/+12Gh4e57rrr1uvH2DAyjolmPaK8j3BWXt8yljFzhTyTs1PomsbV+5O9jH/91Hf54tVfXBB4QPK5lgwF5/I5ql6VqLaC9r033Ey2pY18qcDD33kUwzBwLAfXdnAsm+uPHGNPdz9Vr8r5iREcy0kC07ZxLGfdiwUITYAzvwfo13qArEsAroYKPWVHSIYzS8SlYMNKOCkLCUNDNyxiP8IbLKBlTMwuF80xGlVbhoeH8X0fy7LWfD+u6/KJT3yCMAwRQpDP54miiPb29nX8adZH7EeEk2VkKBEr3Ht3+u3XGBo/z1RuurHRv6ejm6v3X4UQgo/e/iHaMq1ks9mLenqmYXL/PfcBSa8wDEMqfhXLSF6AWKbFLdffRMWrUvWqVD2PUrXcuJ/ZfI4nnvvRRW360K0f5GD/fiZnp3jpzdNJWNYC0bEd9vb049gOURwhWN1p8JcNwNq+0Y36G1ahp2x7cTnAHykCG7Q606xVupcShEjqVQY7uz7hajRWelZDvHN5jHYLvcNtVG0ZGhpC07QrOkFdCNEod/bkk08yOjrKBz7wAa655pr1+jGuiJTJ/FU4XUWY4qLVmX7gM5WbYWp2isnZaUqVEp/64CcQQjCbzxGGIUf2X0V3exfd7Z20Zlob1+3t7MYVNl/41AN86WsPXZjT+9QD2J6BeKuE7DQRWRPTNBeUhXNthxuPHr9ku3s6uvncR+6n6nlJKPpJOLa3ZgEIgoBiucjU7BRV32tsSfnkXffi2A7vDL3LD194Csu0cGwb13awLYdbT5wkk0ozm88xm8/h2Bd6mbZlN0JyRQFoaRe9eDAMg2w2y7Fjx46t9v9KhZ6ybclYEs5WCSdqw5mXWE5/RUwNwpip//Yq0ayH3m7T+XPXJZer4FtAcwyknZzyHc75GB0OVrvTCD7Xdddl2Oyuu+7iscce47vf/S6jo6PccccdVxSoV0pGMeFMlbgUIGwDKSTTuWk6WtvRNI0X33yFF15/qfH9rekWuto7G3Ntd958+/I9QimJCz7mk3l+/qNfQKY0RDkmenKW+GQK7fl88m0akDWRHSZ01t63GrDMbeu6TlumlbbM0l/f09PPZz70yVozJH7gU/GqZFLJ6tyOtnbec+zGpCfpJ8GZL+bRavc5MDq04Gev+5lPfB7Hdnh74CxD4+cbgZgMsToc6N0HRR9vtoxh6uitdrIIxtQwTZMwDPnSl77EzMzMql/lqoUsyrYkwxh/vERcDJLtCBs0FCJcg6n/9ArR7IUVcnq7TdcvnkBWVldzcjeRsUwWuwBGt0tFCxgdHyWVSq1qKOxS4jjm2Wef5aWXXqKzs5OPf/zjZDKXeObeQLEfURye5fzEKNOlWSZnp5jJzRLFEZ+6+xN0ZTsZnRxjfHqC7vYuuto7sa1VzHGWI7RnctimTesnDjH7F281Xny1/+y1zJbm8GcqMBMgpn3EdJB8HCbP69IQ0GEi6yHYaYGrLRuE68kPfMqVMlXfmzfEWuUnjp1AExqn336NtwbOUPU9PD/5G9OExs/f/zMIIfjh809xZvAdbMtKhlYtm49/8hN89dGvksvl+N3f/V1GRkZW9cOonp6y7TSGMyUbf7J5LBcEHpB8Hqle3nKElhS0lpEkmChj6oLOVJap4iyZlivfcK5pGrfeeit9fX289NJLl1ws41erVPJzmI6D6TgYpnVF912tVpmYmGBiYoJ9XXtoJ81MfoYfvPwUuq7Tle3g2FVH6W7voiWVhHB/dx/93X2rvi8xUEE8NwcRVG9yiPw8bb9wDE0IYimZnZvDr/qQ0iGlI/c5SEiG4fMhYiaA6QAxEyDeLKHVHrLS0Ro9QdlpQYcJ1sYs+rJMC8u89Hzu8Wuu5/g11wPJCxnP9/ACv/F/dHDPflJuiqpXTUKzWsUwjIsW9ayGCj1l25BSEs56hBNlNHuDhjPnCaaSTe16u31RTy+crJD/3hCpE91Yh1rVwplLEHpS0FqGMamSQZtnk/PnyHS0rkuP7+DBgxw4cAAhBEEQ8Oqrr3LjjTcS+j7F6SkqhTyabhDnZhCA0HSclgxuugXTdtCXORZJSokQgmq1ypNPPsnExASFQqHxdfMYdBy9jp6ePj59z0+SbWlbl58JL0Y8N4c2WEV2msS3ZaHFwK/6TI5NXP76QkCbiWwz4TBJEEYScsGFnuC0j3b+wmNatujIDuvCsGi7CfrmPqY1TcN1XFznwjaX/X372N+3b8H3pWz3okU9q6FCT9kWGsOZhY1fnSkjSfn5cUrPjmNd1UrHzxxj5k/fuDCn98XrqJzJEc16zH3zHFqLiXu8C/f6zm1fXmqjJCs9NbJmG/5kwNzANG17O5JFMFd627VewbvvvsszzzzDmbff4qaj15DJpHEWlTGLoxivVKIyNwdIdMPGbW3BclOUKlWmpqeZmJhgcnKSvr4+7rjjDizLYmpqiq6uLo4dPUan0UJnSztWJnlyNjWDjrZ1Wkk66qE9k4NqTHwig7wuA+vxWNcFdFpJz45aEPpxEoD1odFxDzFQSb4uSOYHO83G8Cgtxvq05Qr5uQpf+PyDfOnLf7Gm66s5PaXpRaWAYDQZztzoUAmnK+S/M0g4WcG+JkvLXfvQWq0lV2/KWOKfm6P8yhTBcBF0gXMki3tjN2bv2quR7HRSSkbHxigVi2Q6WtGzDtoV9toD36MwPc3Zt9/m5bfPoGsaH7n7Lg5dfQQhNKSMyU9PUy2XAKhUq1SqVdpaWgg9n+8+/QzFchkA0zDo6uriqquv5oYbbkjaHCcLdKKZKsISCGOdix6EMeLFAtqZMrLVSHp3HVtQJ7YcXZgfnKn1CoNF84P1EOywILV584PzWa6N3mLxoY99ODh9+vSq9sOo0FOalgxjgsnkgM2N3mwuY0n5hQlKT48hLI2Wu/fjHMmu+PrhTJXKK1NU35hBBjFGbwr3RBfOkazaJL+EOI4ZmRzFK3k4ZvIEptdfXKxC6PsUZqYoz+XQdAPLcSiWSrw7Msatt9/Oo48+2lji/9nPfpYXnjvFm2+9RaXqkXJd7vvQ3QC8MziIJjTaMhkc02g8j9upNE4qg1bR0AKxMSeJT/loT+egECGvTSNvbNn0ocVLkhIKEWLar4VhkAyTLjU/WBse3aj5wcXKc0Vu+djtb5S9yqoqFqjQU5qOlJKo4BOOJ6+8hbP2moUrEc5Wk97deBn76jZaPrivUV1ktWI/ovrGDJVXpohmPYSj497QiXu8C71l7Ru0d6Ioijg/MUIUx1jSSBYmZW30FhNxmbmxpcJu/mOke98B/vTP//yizdyf+MQn+OH3v0dHNktHto2ObPaSjy0ZS6KKRzjlAyBsDdtNYaVSGJaNfqXbJCKJeLWIeL0Irk58axv0boPC2vX5wcZCGR+Rv3D8VDI/mPQEZacJWROM9f/7XWvoqQkIpanIIMKfKBMX/GQrwgaeNyZjSeWlSYo/HkUYGq0fO4h9zaWfBFdCs3RSN3bjnugiGC5SfmWK8vMTlJ+fwDrcRupEF+a+nXNczpXQdZ3+7j6Gx0eIdImhG0Q5jzjvJUOeafOiXlXo+xRnpynlZtE0Aye99O9S07Qly3Z1ZLPc+p6bLts2KSWyHKMVNay0i9AFcRwTVKtUi0VAoBk6djqN5TiYloVYTX3LuQDtxznEbEh82EW+p3XTekhXbP784DVLzA/O+IgJHzGQHAaczA8ajZWiWz0/qEJPaQpSSqI5j2C8gtBAz2xsryjMeRQeGyQYLWEdbqXl7v3ruv1BCIG1vwVrfwtRwadyeorKq9Pk3plDb7eToc9j67OQYzszDZM93f2cnxhBCIHhGslm7+kqIu+htztorkEUBBRzM5RmZ9A0HSeVWXaYUcp4ybJdUl5+q4mMJDIXQiDBEo1Q1TQNzbIwaiXVojiiWixQyc+BBMOysdMpTNvBsKyle6tSIt4qIV4qgKkR3dkO+5zV/dKakaVBn43ss2mMHS6aHxQDFcSZZPRGGgLaa/sHN3l+UA1vKlsu9iKC8VJyDJBrIDZwPkNKSeXlKYpPjSB0jcwH9uJc274pPS8ZxlTP5Ki8MkU4XkaYGs6xDtwTXRgdO+CJ7wpU5hU+rldtkWFMWPHw/BKVuAimju26K5pTc1Jp3Gw7f/EXf9GY03vwwQep5GYbi1mWIj2JzAWAQFire0xEUUgYBNSf9U3HwU6lMW0L3bQQ5Qjt6TnEhI/cYxO/rw120ykgK5kf7Fi0f9BeuvdruTYtmQzv//gHg5dOv7yqV8iqp6dsGRlLwlyy704YYsNPNY/mPPKPDRKMlLAOttByz/4N71HOJwwN91gH7rEOgvEylVcmqbw2TeWVKcx9GVInurAOt+3KPX+u7dDf2cvo1HiyTyuOqRTnKOfyIAWmZiJcHSxghaOAVhzyqQ/djem6BJUKVhxSucT3ylgiyxEU4qR3t4b/A1030GtDnFJK4iikODMNMTjTGm3nksd3eEsGjuzCIW4hoNVAthoX7x+ct5FeG5m3fzCjN3qC9flBq8WhPdXK7B+9iZz2V/2koUJP2RJxNSQYLRH70cbvu5OSyulpSj8aAaDlQ/txruvY0icdszeF2XuQzPv3JsF3eoq5b72Llqnv+etY82Ka7SqdStPZ1s7Q0LvIahWhaViu2/h/koFETgdIV0Nk9GVHBNraWvnKb/46+ckLm7lbu3v4zK/++kU9vQXDmbZYl8eFEALdMDFig9QZiTUJfqtk7mqfyKyinc9hp1wsN4VhWmhbWDt0S11qfnA26QkuNT/Y9gvXM/snb15UKWmldulvWtkqMpKEsxXCqQrC0je8jFiU98k/PkgwXMTc30Lrh/Y31SpKzTVI39JL6j09+O/mKb8ySenHo5SeGcO+JkvqRBdGbyo5a2wHn/QQhSGl3CzlmWnMKKasCdL2whWZwhRICVQlshog01pyGvlS4SdZEHiQfB75PvnxUVp6+hBCEFdjmAtBCMQlhtLWypyUpN6QiBDKRwTefoEhXAxqJbdKZSqFIkiJYZlYqQyWs8x8YP1H8+qHsQK6Bvr6BHXTsDTotZG98+YHK1FjpajmGmsOPFChp2yiuBzgj5WQQZyszNvAP1QpJdXXZij+8DwALXfvw7mhs2mfHIQmsK9qw76qjXC2tufv9Rm8N2dxbuyk5f37mP6znXfSQxSGlOdyFGamkICdStGTSjNbzFOslHEX1dQUQoCVZD+lGFmKkS06wknOX4ujiKEXT+Hc8j5au3su6ukVpiZ56g//E6bj0rHnEB3dB+jcf5jUelVUAQglqbcl9iiEGSi9RxBnFj7uNE1Dm/ezRVFEpZCjnBSKwXRs7HQaw7KTEBQiWVFaiRCujpE2kX5MXA3Bi4jnLc3YkWHo6rAvqS8a2+Ki0oCroRayKBtOhjHhdIVwplo7AmhjJ++jok/h8SH8wQLm3gytH96P3roN9j8tEvsR1TdnsQ+3kfvqmR110kMchZRy88LOTaGJC70bKSVThVkqnoe7zKkEMpbJsKQG07MDvPnkdyjNTHHtB+7h5o9/km/81j8nPzlBa3cPn/wHv0puNsfgyy8xffYsMyPv4pWTcxjdliydew7RsfcQHf0HsZy1VdQxcpLUaxKtCtWDUD0sVr00vz4fGAYhkPTsLcfB0hyMrEvckmzl0DQNXdeT9yQn2cswRvoRcTWCMEbuwDC0XDuZ0/uTN/n4v/15Xh59Y1U/zJaEnhDiXaAAREAopTy53Pcf/4n3yD/6+uO1E3oFGsl/oFb/XCRzpBoaupa8GtQEaCL5ugCofS6S+6+9B4EAUf/4wnUForF6Vsz7fP71tV244GC1woJPOF5CRqClNnaTuZSS6huzFH8wjIwhc0c/7omubf0HDsnm/PF/9dxFl3f/3Z+g9MwY1r4MZl96wwtwr4c4CinNzZGfngQJtuuiaUu/CIrjmMn8LH7g4ywTfKW5Gd58+jGmhs7gtmS59oMfo+fao7jpDG1trY1e0txcnvJ0HvJR8kdtQHluhumRc8yMDDAzMkAYJC8sWjp7kxDcc4j2vv3oxmWG4WOJ+47EHoTYgdL1gii7Po+7MAjxymUCK0ZPGbS1tmI7LkEY4AcBYRASRBECiUQgkAhNS57/pIaGgFBCSDIyEMna5rnac79O/Ql0S0qKrUWqrYWe/r188JMfCV86/fKq5ki2cnjzHinl1Eq+MYgkQSgBiZQgG++BxZ9T21ha+1osZZJUSxC1N8mF69YvX/hZchtLvT6oh59AXAjceZcJjQsBvEQYi8Vf0wQaAl2/ENZLBq64OKznB3n9sauJi68rBBseBItLiGnOxj4hR6WAwneH8N/NY/anafnwAYzs9uvdLUlcPJyjt9vEXkj5hXHKz42DJpLFMfsyWHtrIdhE5c/iKKJcyJGfnEReJuzqNE2jqzXLxNwMfhBgLToRIfQ93nnxRwy8+gyabnDNe+/hwLFb0KSOnA2phMXGohUZS2QxglIEltZYOJXOdpLOdnLg+pPEcUx+apSZkXeZHnmXgVef5d1XnkZoOtnefXTuOUTn3kO0dvYtmHPTC5L0axK9BN6eZP7uSiuQSCkJopCwGiI0aDvYRbo1jWWayDgmjiNcLsxNx3FMHMdEcUwURfh+QBCG+EFAHMdJoJmAlZxXpyHQpYaISHrKoQRPsuDZT6992oRhWM7lefXd85w9e2Zgtdfdyp7eyZWG3rXHb5Jf/vb3N7ZRa1QP2PqvsRG4JIF74fIknC98TCOUlwprWfu81hFdJpQvfE2I+cmcXGvx/27Ss5X1UZNaKIoFwSzq4VsL0MW9a12rBakAXWi1nnb9+0GUQ8RkJVkc4OqN+6i3qhHm88O4HtbzfsjFYb9Ux1pKifdWjsITw8gwJnN7P+6N3Ttr2X/t9PbpP3p94ZyeoRGXAoLREsH5Iv5wkXCynDwgdIHZl04CcF8Gsze1odVtLiUJuznyU5PIWK4o7BYLo5Dx2WmEEJiGgZSSkbdf4e1T38OvlNhzzQmuOXk3durCIbIykBBLcDWEoyOLF282v+z9Bj658WGmR95l5vy7FGbGATAsh47+A3T0H+KQcS2tIymkCaVjgrDryh53YRQRRAFSQgqbdDaN05NCv4IpgbgWhFEUE0QhQRDiBz5BECJlXF8bBdRGy2INEXMhEKNFzz4C0Nnyv7H8zBwf/5ufP1uslI+s5npbFXrngFmSP8/fkVL+7hLf88vALwP07tl3y18/fXpzG7kLSCkbE+ByQfhe6D1DrbcMl+1dE8QYOQ+9HBLZ+tJFcxcn+BLqw1GLtbomXRmbjG1g6oK4HFD43jDeO3MYvSlaP3IAo32HbvI2tRWt3oy9iGCkiH++SHC+SDhZ25lmCMz+NNbeFqy9GYye1IYWAYjjmHI+d0VhN18QBozlpihNjfP2M4+RnxqjrXsPx27/KG3de5a8TvKYJPmd6QJxhb0vv1JmZvRdps+/S2Vshvek76LL2cf56lkG0+do27eXjj2HsN30qm43jmO8MNnUbpkmGdvFkRZ6u4loNTZ0VCaKokbvMAwjvMAnCAKCINmgDzJZKSoFOjoiBi2UiKg2ZDr/VaoGaJsXhtst9PZIKUeEED3AXwP/o5TyiUt9fzP39BSS0krFAG2mitAE0l7/hSpSghdGeGHyRN817eE+Pwl+TPq2PlI39Wz5K89mFFdDgpES/vkCwXCRcDrZ7yRMDbM/XRsObcHoXlmlk8veXxxTyefJT00QRxGW6zY2bF+JcmGO5574OoNvvoidynDNe++h/+obNn++VkqsEUidkUgkwy2DvDP7IjOj7xJ4ye820969YD7QWGI+UkqJHwbEsUTXNFrcNK7tYEgtCZNOAy29dbNPUkriOCas9xDDgMAP8IOQMEoW2MgYiCWaTHqGegRarG1aGK419LbktyqlHKm9nxBCPAK8D7hk6ClNzI+SsKtG4OjIDQoeIcAxddwY7OcnsYaK+G0W07f1kOlN0+mFZGwDY4cHXxxJhCZWPMWiOUZjKwRAXAkbvUD/fBH/R6OUGEVYGmZ/JlkUszeD0bW6EIxlTHUuz9w6h10UBrz+3BO8+szjxHHM0VvuovPICdKpza9oIjxJ+g2JOQ1BO5Su02hxDvMTHEZKSWF6vDYUeo6hN15g4NVnEUKjrWdPIwRTHT3EJHNnGSdF2nGxjGT7jvTiZNiwz0JscfFpIQS6rjdKwsGFERQpZW24NOkl+r6PH4ZJDzGKkhGiSEIMWizQItDrYQjJSI/Y/J5h3aaHnhAiDWhSykLt448Bv7HZ7VCuUCwRBR895yF1AamNeSi5KYPurIsuBJEXUfz2OYLzRarHO/CPtZMSgrIXkastPW9Pm3SmbdI7MAClhOnRZGFGW6eD6RqXWqN1SZpr4BzJNs4KjMtBEn7DSRAWB/IACFvH3JPG2pcMh+qdzpIhE8uYaj5PfmqSKIqwHGddwk5KyfCZ0zz/xNcpzs2w/8hx3nPXJ2nJdlKslJku5HAte9OCz5yQpN6UiAjK1wi8fSxY3CGEoLWrj9auPg7feBtRGJKbGE4WxZw/x9kXfsjZF36Iblp07z3MnoNHaT14FCvTmvy8lSiZf+wwN3TYeT0IITAMA6NWRSbluo2vJfOHMVEcEYZhspjGDwjCgCiqrRyNk5WkWgx6IEiW1cxbKr/BYbgVPb1e4JHag9UA/kRK+e0taIeyVl6EPl1BBHEylLlBD043ZdCXtsn9/muNBRztnz/K2EwZX7+wMMa1dFx0YgnFasRsqQgC2lNJAGZsA30HBGC1FBBUQ0xbZ+p8ESdtkOlwsa5gOFlLmTjXtONc0w5AVAwIzhcavcHiuVoIOnqyKGZvBmtfCyJr4hWLSZWTMMRyXSzHXe6uVmx2cpTnvvco40NnaOvs48Of/2X6DlzT+HrGTRHFEblSYcODTwQS9y2JPQ5hS7IVIU5f/v6EppHu6sft7OPwe+7EFoL8xAgTw2cZG3ib59/9GgBOqoW+vUfoO3qU/uPHSOvtG/azbAZN09A0DRMDFhUWiOKYOIoIa/OHQeDjB8mQqQyi5FVdmLzpXm3LBfWVtusXhttic7qa02sSsUSb89HmPKSpJasKN9CBvhYKf/jaRUv1W37hegZHCpdrKtUgwo8iNKElAZixSVv6tgxAKWFqqAiaRK9tRQi9iDCISbVaZNodjA34/4gKfhKAw0X88wXiQpC0xxLIDoHWbaP32pC58j2YXqXEyz/6K95++SlMy+HG93+ca268bckFMFJKcqU8hXIJ17704iUpJYHvIYSGpmto2srbacxI0q9LhA/VQ4LqQZZ9gSelJAhDwjhC13QyboqU7WAtscevmJ9l7NxbjA2+zdj5M3i1rRWt3T30H7mWviNH6b3qyIIXEoZpoZsmUsYIoREFAWHgr+hnaWb1+cMoigijmDBMgtD3fQIvgFgiw6QIgRYmWy50oVEqFPn4Lz64Peb0lO1HVEO06SoijJGuvqH7doQXYb05i9GeuqjUUDTroa/gvjUBKUsnVesBFqohMyUfIcS2DMBqKSDwQ5x5J1EYto5haXjlgErBJ511yLTZaOt4SrXeYuEe68C5tp1KsUBhcIxo3MeYE4jpCMYqRK9UwNEQXSai20R0mZDWVhwucRzx9ss/5uUf/RWBV+GaG2/nxjs+tuwqSCEE2XQrURxT9qpLVm0Jg4AoCHDbWpBxcgCt71UvLEEmqWqi6RpCN5K9rwCRxD0rcYYhSkHxFkHUeumfJYhCwjA5OTzlOHQ6KWzTWvbnT9ttXH3Dezly1x1gQG58jNEzbzJ25i3OPvc0bz71A4QQdO47QN+Roxy++b20dfXw8D//9UaFmfv/51/DMK1tH3zz5w+TnYeL5g8bWy4i/CAk8H28qo8UgiiOVl2SSIWesrwoRpvz0PIB0tKQ7sY9ZEQ1CTvr7ByEkuj9+5bclB2tcnSiHoDMC8Dpoo+uCzpSJu21IdBmzT8poTjrYS519poQmI4BUlLOe5TzHi3tDm6rtS4Vg6SUVEvJMGbo+5gZByebanyNUoycDJBTPnLSRw7X/q/ceSHYbSJSSw/Bjg2e4dR3v8rc9Bi9+49wy933097dv6K2CSHoyLQRxTFV32tUbYmlJKhUMCyLbP8ezHnDbBKQYUgUhURhSOj7hL5H6FeRMRhFQfZtA6OqUdkTU75aQzMv/j1GcUQQhkgpsU2LttYWHMtGX6ZQdON3Vo0hpSPaL5wd2d6/h/b+PVz/gXuIwpDpoQFGz7zF2Nm3ePX7j3HVze/j0X/zzxq1RPOTEzz6b/4Zn/1f/jGluRxOOoO+A09qEEJg6DpGbUFNat4Ieqk1TRiGq078nfdbUtaNKAdJ706yob07UQ2x3sglYRdJwgMZvOs6CKOQvp89Ru5P3mjM6WV/9hhjuUudinZ5jR6gpRPFkKsETBV9NG35AJRhTOxFCF0g7I0tp7aYXwmTXt5yJ1IIgeUayFiSn6lQynlkOh3cjLWm/7YFYed5mI6Lm2lZdJciGdbM6HDYSZ7QCxFyKkjexn3kUC0EU1qjFyi6TYpejhee+DpDZ06Taevgrk/9PPuOHF/173V+1RYv8NFlsuIz3d6B29qKEAtDSADCMNAMA9MG6p3JWCJO5xGvl5G2oPxeE68lJPJ9gnIEohamUQgCbMumNZ0hZbuYK1y4IyMJXgxZA9F66ceQbhj0HL6ansNX8xMf/QR+tYqTySx5akSlmOeR3/x1AEzbwclksNMZnEwGJ92CU/v4wmW1j9MZNH17H2CrrfFvUIWecrEwRst5aIUAaWvIDSpnJSphrWeXh1gS7M/gX99B3JoMclTKIWNA9y9cn6zelJKxXIVKeX2KLOsapK3kT6AegJNFH0MTtKctOlIWKV1D+BEYGmZfiqgYEBcChCHQnI3/85FAfqaKaa3sCUpoAjtlEoeSuYkKpVmf1i4HK7WylZ4S8GphF3hVTNvFbWld2X3XDgkVrQZc5V4IwckgeRvxkQNJCIbBDH3efg6+5zh733scI7P2knG6ptOZaWN4/Dym7dCxZ++Se+MuKR+i/TiHmAmID7rIW1qxLQ2bWsUf38PzqhDHdNoOJgIRRoRhQFitUn806rqOphvohn5R2Mp6zctuE+0Svd5LsRwH07KXPDXCcl1u/ewXqBaLVEtFqsUCXqlEcWaaqaFBvFIRGS99EoflprDTaZxMLRyXCEgn3YKdzmCnUts+JOvUQhblAikRpQBtxkuK19ob07sTlRDrjVmsd2phd6AF//p24iY55y6KoVr1iasRmqnRtq+F7r4MbW4yZBhXQ4LpyqaEn1cOmR4pLpjLW40ojAmqEbZr0NLpYF2irRfCbpKgWsF0HAxz/f4/pIw599oLvPPjp2mTnRzqvpGs1o2oJ0ZGvzAU2mWu6mw7v1ohiiLcjnZylQqGaWKuZKhPSsTbZcRLedAF8XvbYH8yfhaEIUFtriztpmnNZHAsB23e8KWUkigMiMJaj9Cr4lerhL5HPO+sHy0Q6LaB1mOjO2v7f6zP3dWHOFc6pyfjGL9aoVos4pUuBGPyft5ltc/9cmnJakgIge2mLoRi7c2u9yjn9yIzGWw3teyZgOuhUihwzc0nXwmi6MbVXE+FnpIIYrSZKlolTMJuA/YKifK8sJOS4GAL3nXtyCYJOwAiifAipKERZy1Cx6AURPhRjKkL9ra79LW6uJaehN9UhbjoI0wNzV7f8JPA1HByyKh+hSszQy8iCmKcFotMu93oOUrAL5eYq4edba9r2AFMjQ7y3Pe+ytToIJ29+7nlnvvp3nMoeXLNhbU5wQA5HV7YwNyqL1gYs9Rm7TiO8EolnEyGtp4+DMui6lU5PzGKbdnzNlYvoRShPZNDjPvIfpv4fW1ENvi+j5QSy7TItrTiOi7GKvcdSiCuzRdGeY/ADAhTEYFXIQqT8l5SyqRnaJjohrGi+bjNWL0ZxzF+uVwLwgJesXiJgEx6lN6iU+jrhBDY6XQtIFtwaj3KxT3LekhajruqkDTM5KDduz72seD0q6+t6gGrhjd3u8UlxDZgk7koB9iv5zDPzYGE4FBrEnZr7L1siHrY6RpRl4NMmaAJdKC1NrwbxZLhmQoD02U60xb7O1K07c0gvYhgqkK0zuHnV0LCaoSdufLbq6/09CshU0WfVKuNaYcUZ6cbYbd4zu5KVYp5XnzyW7zz6imcVIbbPv4Frrr+lsbQnxAC2k1EuwlHa2fjzQ/BgSrynaS0F20LQ9APq0gZ0963BzebbQzdOrZDX1cvo5NjuI67oGcGJI/3gSriueSxGJ5swdtvEMVVjNCgvTVL2k1fdKLDagiSws1GbOIcbsXouLCxPwpDosAnDAL8agWvXMIvl+eFR7KqVDdNDMNEMy7U3gwDf8NXamqalvTaMhmg77LfH0cRXrm0MBiLBaqlUhKapRLVYoGZkfN4pSJ+den5eKFpC3qQFw+z1j9voaWzs9HrnRsdWfV/lAq93cyPkoUq3saUEBOlAPuNWcxz+STsDrfiHdseYbcUXRNkUxZSSopexItDORxT52BHis6+NHbgEkyXa+Gno13BpnEJ5KerGKsY5rssITBtDb/qMXb2PH61TEtHC+n29GVXHa5GFIa88cIPOP3jx4ijkOtP3s3xWz+Mucx+OqhtOu4wER0mXFsLwdl5IXiuijxbTQ6RyBo4B9vQTQ2ZihDz5jzTborezm7GpidIzT+c1ovRTs0hhqpEnQaV9ziQSepetqQz2Ou00T32QmQE5r4MRmZhJ6Teq7NcSLW2XbhOHBH5Sc8t8Dy8Shm/VKZSyC+8vm6gmwa6YW748OFKaLqO29K64nnfKAyTkCxeCMR679GrBWe1WGBqaAavVCDwLj4d/f5/+L/yvT/4vYsW9qzUtgg9y9BodQ38MKa6RGV5ZZU2uISYKAXYr89ivpv8wTZ6dsutPrwEO2VgZh0iAbqEIFfFW4+FLJFE+BFSE5cNu8WEEGRsg4ydPCbfHC8gJgrszbr0dadIdUIwdWXh51dCgmq45rm8pQTVCsXcFEGljG5apLJteJUI3yuTbrWw0+aaV8RBMsd1/p3Xef77X6OQm2LvVddz8wc/SWt795puT2gCOk1Ep5kUaC6VYC7G8VIwGVB9ZYbqS9MgwOhJNY5RsvrTtKRbwNSwHBtTN4i9iMJfDuAPV6leb2PckKW3tQ3Xdi7uDV6BqBSiWRrW/gzaChcfAWiajubomI6DO6/DLaUkDPxGIHrlMn4l6VlJGQOiMSTdDCF4ObphkGptWxD4y4mC4EIo1oZaFy/oWa1tMadn918j3/srv8N/+OLN+EFM0Vuf1Xu7kh+jz9Y2mVt6UtpnnRilkOyZOTJDJRBQOJAhd3Ur0RpDNZ2xMNod/h9/+gLDsxX2tbv8x5+9GbcUrD345oVd3G6vKuyWvdlYUvRCgigmmzI50JGiTWhE0xXicoiwtBU/CUpgeriYzPusQ5WVsFqhmJvGr5STYbNFc3axlIRejK4LUlkb21n9cTZz0+M8971HGR14i9aOHm65+372HLr2itsOtd5BpUwq20ZbV29j/kuGMcFoqVExJpgoQQxoAvfGLjK39TPz528uKGFXLhZx2tKNupHrRcaSuBygt9qbcnZhHEcEVQ+vXKI8l6NSzCdHAAmBaTsY1vKb47czO5XmL/7p/0p+coJ/+9c/ZGgmt6ofdNuEXv8v/Fv2tbv8fz95PX/nD5/b6iYp8+xF8PPYfByTCPgaAX+Mx+TlDs67jN/5G7fwv339NYZnL8wD7Gt3+ZNfeC/FkeLqbmx+2GXtpNe5QbvRy35I2Y+wDI0DHSk6DR095604/PxKmNTWvMJe3oWwK6Gb1mUXqMSRJPRjDFsj3WZjrWBu0quWeeWpv+atF3+EYVncePvHOPoTd6zL8nYpJV61jECQ7duDm8ks+/2xHzUO1HVv6ib3yJmLCht0/eIJZGV9XzTLMCauRBi9KYz2zSuCPV8SglW8UonSXI5qqbBjQ3D+Stbf+JOHVx1622J4s254tsL+9hR/7+5VlVrb9UQYIYpJDbukXub6PPgz1YjrR6scnAmIBbzTbfF6r0Nkafz0Otz+/qy7IPAgeQxMeSGnhma5bW/28icpxLU5O00QtTvJfOIGl15JWQYpyyCIYs5OFjkD9Lfa9PW4OAWfqBhcMvwac3mrGBpbLKxWa2FXRDfNBSeKL0fTBZarEwUxcxMV7JRBqtXEMC9+mojjmDOvPM3LT34br1rhyI238hN3fBxnhfd12Z8hCAi8Cqm2dtq6u9FWsIJSs3Tsg63YB1sRjr5kCTvW+UV+7IUQgXWgBX0Nw/frRdN07FQaO5WmtbunEYLVUjHpCdbmBpMQtNEvUyatmYWBj2FaPPD//g1+67s/3tkVWfa1u6RtnXuPX35VkcK8EmI+skeHddpkruV9rNdnMQcLoAn8o23417bT5xorWO+1chldY1+7e1FPL1f2+WdPnaPTNbnv6i7uvaqL9sX7n7Yg7BYzdY3OtI2UkplSwOicR4uls7/dpqUcIpcIv6AS4q9xLi/0qpTmkgUAurHysFtMNzV0EwI/YnYsaYvbYmIYSTvHh87y3Pe+yuzkKD17r+KWe+6no2fvmu5rMSljquUyuq7Ttf8gdmp1p5A3CLFkCbv12ncqpSQuh2iWjrnK+bvNMD8E27p7iaOotlq0SDk3l4SgTOZNt2MIhoFPpVDgzTfefHO11902w5vv/ZXf4bd/7hYcQ1OLWVZgQQkxW1uXP3Yt72O/NoMxWARd4F/dhn8si9ygzdl2yqCSNvnv/+T5BXN6dsHnB2en+fqZSZ4fL2Bogjv3ZfnUkW6OtafQ/DgZxmyrDWM20flk1SCi5IfoAva5Fp2exIwkwtQQls70+SJxJDFWcYho6HmU5qbxykV03VhdNZLLkBLCIEbGEilKvPH8XzH09iukWrLc/MFPcuCaG9ftyTIMfIJqlUxnBy0d3Vc2RGpqEMZM/9HrjTm9zp+7Lnnhd4XPHzKWxKUAPetg9qSa/vy7pcwPwdJsDq+UzMMLodVC0Gz6ENzRm9NP3HSz/PpjP1CrN1cijNFmPbRSUkKMdZhQ1+Y87NdmMYZqYXekDf/ajQu7+eqrN2MB2hKrN4fzVb5+dpLvnJumHMYcaXO57/pe7jreh73Om8XXUxRLCtWAMIrpNnT6Y4FeDslNV3GyKwut0PMoz81QLeXRDAPD3Jj5pDDwOfvqDzn76pMAXHvTBzl++z1Y6xSu9d6dYZq09+3BctfnXD5MDWFoSXILgQzjKw+82vyd2ZdCz27N/N1GiKMIv1KhWipSmpvFL5VBSITQk9WhVhMVkKjZ0aGnKrKswOISYusQSFrOw35tBnO4hDRqYXe0HblUtf+tUhvGLEcxj00X+PpbkwzOlmmxDT56fS+fONFPX+vy+8O2kpSSkh9R8QLEWIWeANpsA93RkyfsJYS+Tzk/g1fMIzQdY4MOUpVSMvLuaV5/7q+olvPsOXSca2/+CKbRiqYL0mtc6TlfGHgEVZ+Wri4yHZ3run1gvcXVECRYezNoqSbaa7oBojBszAmWcrP4lRL1o5iMDajasxYq9HazIEabqaBVonUpIablPOxXZzDP18Lumiz+0Wxy280ilggvRgqI26xkzk7XkFJy+vwc33hllKfemUZKOHmonU+e2MNNB7JXtA9tIwWVkJnBAoEBRhjTEwmyhoaVMi+En5QUZ6co52fQNB3DcjaspzE3PcLpZ77F7OQgrR393PDeT9DZe7Dx9TiShEGMYWqksytb6TlfLGO8UgnTdsj29WM5zf3CJC6HaI6B2Z9uuvm7zRCFYTIcWixSyuXwqyWkTIpsb1UIqtDbjaREFAK02VoJsSsMJW221rM7X0KaGv41bXjXZGEbhN1Spooe3z49xl++OkauEtDf5vCTJ/r58HW9ZJpo6FNKSWGkRBzG6JZOHEPZD5CViGwQ02UZuBmLSnGGUm4ay01vWNh5lSJvvPAYQ2dewHJSHHvPh9l/9XsuufE5DmPCQGI6OpmsteRKz8UCv0ro+7R295Jub79QMaUJ1fffGVkHo3t7zt9thCgM8KtVqoUCpbkcQaUMQqBpemNOcKOp0NttFpUQu5KVidpMNQm7kfKFsDuahWZ6RRtLhB8jkcg2m3iZsFssiGKePDPFN14Z5Y2xArahcc+1PfzkiX4Oda1xdeA6Cioh+eES1hI1Nqt+SFgOMSZmSFVzZDpaV3zM0GrEUci5N57m7Ze/TxQGHL7uVq658YOY1srm18IgJg4lTsbAbbEaKz0X3Ecc4VXKWI5LtrfvsmXJtpoMYmIvxOxN76j5u40QhQF+pUKlWKA8lyOoVkFKNN3YsBBca+g1z8tdZWViiVbw0dahhJg2XQu70TLS0qje0IF/TVvzhJ2UycKDMNlkG7dZxC0rD7s6U9e4+9oe7r62hzMTRb75yiiPvzHBt18d44Y9rXzyxj3cdrgDY4OraFxKZcZDt5Z+QnUsA88vkGeSopNibtbDIimko+kCXRNoukAY9Y81dCEQIlmwqwmBJkRyeOolnrPHh9/itVPfppSfpnvPEW54771k2lZXOswwNaQBXiWiWirjtpi4LVajpmf9+J+2nl7Sbe1NHyD1+Tv7QOuOn79bD7ph1v7PW+no37swBHOzVOdKyeNRNzBtZ8WnvMsoQoYhMgyS91WP2POQlQrV3ByGWH2GqZ7eNiK8CG26ggjiZChzjb07fbqK/eoMxliZ2NLwj2bxr8nWNq5vMSmTkAtjJCBdA9liXdHPu5R8JeA7r4/zzdOjjOc9OtIW997Qx7039NGe3rz5iaASkj9fwkov/bfrlwoUx97FdNMIrXYcUCwhkshIIiKJCEGLgJBki4q48B4NIpL3mga6EOi6QBcCvzTDuy9/h9mxM6RaOjl688fp3nsNmgBNEwjEmn7lUiZHGSEETouOJnzcTEvj+J9mNn/+ztqTRphN8gJwmwuDAL9SplosUMrlCH0PiURDYOgGGiTB5gfElQrS85CVKjJaVD1HCDAMhG5Qnp3h5H/3t94oh+F1q2mL6ultB7FMNpnP+UhTQ7pr+2/TpypJ2I1XiC2N6okO/CPZ5gi7IEaEMUiIU0ZSF9PW12XLxVJaXZPP3byPT9+0l+cGZvnGKyP8yTODfOnUEHdc3clP3riH6/paNrxHUpnx0M2l7yOslimNv4vhpBqBB7VCzJpAzOuAxJCkjQQtJnkfSbQIrAj0CGQMEolfrjD0zpOMv/scmmbQe+09dBy+hVDTGclVG/V6JMnrDKGBIZKVe7oQ6BoYmkDXNIQGWi0chRBoGggEpqPjV8rkJ0NaOnpx23rQm2DF33Ia83fttfm7TS5msBPJOEYGAcL3scIIA42U7RJUfbx8nmp+jnK5RBgGIAWaoWPaLrptIxxn2Rqma/3/UaHX5EQlTObuIol013aSuT5ZC7uJCrGtUb2xE//qtq0PuzBG1PZNxbZO3OYkgb6Jw4y6Jnjf4Q7ed7iD87MVvnl6lMdeH+eJt6e4qivNfSf6+cTxPjKOQSwlmhDrtl80rIYElXDJXl7kVymMnUO3XDTdoDWb5sD+vRiGThhGDA6dJ59bdICnECAgrv/65oeplBDFzJ59mbGXv0folek8+BPsv/aDOEY6qUyXZCYIkFrSU5QiuSCWkiiKCZBJtspGxlL7lsb1ZRwi/QpmuoV0dg8hBrNncli2TluXg5s20WsBqtV6nZqoDc9uUc405u/60uhtav5upeqhJoMA6fvEnkdcLiMrFeJSCekHMK8Gr0QgzOScwFR7J+muHjqBMAwIPI9quUy5VMALfGQg0XQd07TWpZZrnQq9ZhXVNpkXfaSlJycirJI+UUkqqExUiG09CbsjbetWjmxNwhgRSogl0tKIOp1kT+FWtqlmb7vLL33gKn7u1oN8760JvvnKKE+emeKOI538nT96rlEV5rd/7hYc88orA5WnPTTj4ifXKPAojL6LpptohpkE3t79fOf3XqcwXaWl0+Ejf/s6Bhm6OPguoTQ5zMipv6I6M0aqex+HTv40bmc/EVCSEiFBxPU3iRYmvUM9JBkrBZACqYkkDDUuGm6WUhL7VRBg9+5Hs1NIIQhjSaxBuRwwfbaKZuuYrSaGqV9UklyIZA7W1DUMXSQfaxqWIdA0gaFpjeHXpPeZhOWVRFRcCZM2H2xDW+Moyk4lpWwEmgyCC6FWLhNXKsTV5JDf2mseQCAMA2GaCNtBS6+sFJ5hJIfmuukM7d09jRCslEtUSgW8agUQyRYJy0TT1h6Cak6vCSWbzNdYQkzKCz27ySqxo+Nf245/devWBUvthAMkSEsjzlhJj26re5qXIaXENDR+6Q9OXVT/819+/ka+8coYBzpSHOxI0dvqoK+imxJWQ+aGL57Li8OAwui55P/RSlY3Hj9xlO/83psUpquN72vpdLj754/y8gtvYqbEJXsmfinP2AuPM/fuqxipFvrf82HaDl2/8p6MlEkQ1kKxPmSqR8nnUO/dxURBBT2Vwc52IZZZrRf5ETKSmGkDu8VacHxSLJOjjmIpiWN54fPa0OxCovZsK9E1gakJjPlhqWuYenKZa+pYhrYgHBvzd66B1Z9BNPnjcSNcCLUAGfjEvp/00MoV4kqZuOrR6MLXuvTCMC4E2zof0XQpYRDg+x7VcpFKqUjV86nM5vjgf/933yyH4bHV3JZ6WdNMFpUQk6sZ5pMy6dm9OoMxlYRd9aYu/Ku2KOwiiQgiiEEaGlHWToKuWVaGroAQAsfUljzpwdA1/ujHA43LrFpx7AMdqeStM8XBjjQ9rfaSG+LLMxf38uIopDgxiIwjDDvVuNww9AWBB1CYrqILnYHveRiOwO3UcDs1Up0aZkojDgMmX/sxk68+BTKm58SddN9wO5qxynk1IZD6hQGqiMVDphJZrSAQpLr3YJsptFAiqlHjMA9JfVFN0lPULR2kTIZ3SyF2i4nVYqHpyfBm8vsSsIqHSlQLxTCWBFFMSUbJkGxcbwGYuiCbsmhzTRxdw/AijI6dPX8npYQgIJ4XarJcJi7N66nN7/iIeaFmmOhZd8uHeqM4xpMCTxrkRYaC4VAJPQJNEq/hP06FXjNYUEIM5Gq2IUiJPl4Lu+kqsatTeU8XwVWtmzo3BjT20iElUtOIW23iVK1Ht03nSDQhljzpoafF5s9/+TaGZysMTpcZmCkxOFPm9Mgc33trsvG9tqGxv/1CEB7oSLE3bWMWfZyWCwEk45jyxBCx72E4C/cOVkshLZ3ORT099IjuG0wqMzHlyYjC+QgpJUK8jV98gjjI07L3GHve+2GsTHbdfzdxFBL5Fey2DpyOXjTdIGj8QMnKUuL6CtMYLYjRItmYx9URSCEJcx7+nIeddbBaLNayVz15qItlj5qKYslsyWdqtgJBjNnn0qk7ZKsBGdvYsi0rV6IeajIIkkALgqSnVqkQlcoXQq0+/ihAGGYj2PRsdstDbT4pJV4YUQ1iil5AoZqcTVln6RqWoeG2pil6VaRk1XMManhzq621hJiUGGNlrNdmMKY9YtfAu66d4HDL5oZdXHsSiyRogrjFJE6ZYG3foJvPMTWqYczfXTynt8xpHyUvZGimzMBMmcF5bzOlC0d/2brG3laHfa0O+1ptuuISPaJMd+vCFaPlqYhwwuH2z17NY79/YU7vo794PQPDg405PSklhfNjjD3/13j5IYTeheHejW4ewEzN7wnqGM6V/b9IKYmqRYRmkOrZh+mu8gijWiCKOAlDwtqoQDGZW3NaLQzXQAiQmkj+JmorVq+U8CIkEPWk8HVB2Q+JZTJw2uaa9LTYtLomactA2+Len5QSwjDZnxYk+9TiIEDWF4pUKkTlCkLKpCddDzV93tCjcWW1UTeWTBaFhTGlashcNaDiJy/cJGBoScCZl3g+K05Oc/cXP/OGF/pqy8K2MK+EGJpYee9OSozRMvZrM+gzHnHKoHJzN8Hh1s07QkdK8OPklbwmiNMGMm0mod20f2BrUw1iHFPjj3/xVuLkx73s6s20bXCsv5Vj/a0LLi9WQ94ZK/D2O7OMegHD+QovjeV5YuDCXiRHz7EnpbHHNdivaex9G+x0RN6UHPu5a8iYOsUgouBcGHIMq2XGX/o+M2deQLcc9rzvXtqvvomgKChPx1SmI4qjEfmhCAgw04JULQTdTh3DXvn/WRz6hH4Fp60bJ7uyw10vIgTSEEn75w93d0IcRFTLEXoQk85aGJpABBItiJKTwGkcmpAUZ9BWGIgyGXKNbZ24ywVDwwKs2nBv0sOIOTtZIpYSXQg60hZdGYsW18Q19XUNj8am6yBEBj6EYbJIpLZHLa5UiatVBAuHHqUEoeuNObVm66ktJ4givCCmHETkqyGFatDYRqNrGpYuyNgbH9Iq9LZCrYSY5kXJiQUreUVZD7tXZ9BnPeK0QeWWboJDmxR29eookUSQBF1cD7odOh9SVw3WZ4tCxjE4bFvsv6oLo1bPtDI7wdTEKBMyxWglZqQcMVKOeH3K51jOoiwFhz5zmF/60osXDbH+u88d48XvP0b1jSeJQ4/Oo7fQc+NdGHZSOsxuA7tNo/0qI3lSn5NUpiPK0zH58xFzg0kIWhnR6AW6ndqS1WGklITVIpph0brn6ouGYNeLZupobcnp7bmcj5kySHU6GI6RjCZEtRdbYYzwI0RQ2/YS11daJCHRCERdJIelViLiNpM46yz5eE3mb3Wc2mb0WEoK1ZCpooeUYBoa3S02HWmLjG00vm+p3xPzemYyCJLeWaWS9M48D1n1ko3YNJqbTH/qOuh6I9S2U6AtFsVJD67qhxSqyZsXxklVFgSmIUibW9Ob3hahpwmYKXnMf1A3LHHRUgO2ovZvfU9RUpZJNL6m1fY4zS/XJBAXPhYXPl9w3UWfL6teQmzWS17prqR3JyXGSAn71Vn0XC3sTvYQHGrZ+LCpV0cJ4mT/V8pEZnZH0G2EsBoRlALMdLKy0ZubpjIzRra1jXYhuDabfF8cSc4/41MlJjquc2hfhu6Mxf9+7yE6XJ2ZSsQffedFnvo/f53KxAiDzj6e7ns/Uuui/e2Adisia2u0WxpZS6O99nGmTeBkTdqvTjZie3OS8nREZTomPxwxN5DMnVit9Z6gjtuhAT5x4OFke3CyXQs2ym+U5PR2jciPmBsqYrVYpDpsdOvCNocFf+dR7QXZ/ECsBhhzs+iVKWSbREYmctZEagYIA6mZSN0EoSOFlrzXNEBDaDoZQyNt6iA0whimcmVGJnKIMMQVki5Xo4UIV4Zovpf00Koeoj6HJkQyv11f8VgPtVRqXfedbbX583CFakChGlCZ9yKxMQ9nNUfcNEcrLiNlG9x2dee8DbGysTG2Pv4rJbWVtcnX4nmXy9qSWwnEsSSqLYeWJJPbsbxwe3EsiZHIGKLa5ZGUjW54FCUzp1JeWE5d/3gp9UAU1RBz1kMLYyJbh0hAJV74jXJegkuJO1ah9a0cVj4gSBnM3dRJZV8mqfQeRPMCXMy7r9rnS4R1/Xvq/yZVNJZodK06Sr0MWNzurMuRRbtdZbbaOCbIK85RmjqP6S6cw5NSMvFKQHUmpu8mk5a9Bq0m/IsP9fC9/+ufkp+coLW7h1/7O3+fZ7+5n7f3vg+z7TDHA8msJ5n1Y96YC5nz44tm+A1BIwSzVhKE7SmNbFYjaxq0VgVaXlKdjpkbiMidiwCJ3Spo2duBZqSxMqBvYmEV3dLRTI2wEpIbDHDaLNx2G23ximRdg9oqUxFW0eU4VmUQ4URELemkgk1URQsrtdI0EULWStfEEqIIGcWIMETGMQQh0vPBDyAIwA/JSFELSIMImIs1ZoRBbJo4bpq2lhTpjIvr1FbsaklgLnjTdBp/nNtSMgzsXWYerq2J9zs2b8vmEYC9RNX2ZiPlxYEbRxHhdDXZtLvHBlNbGNiNsL5wWXRujuiFSeSsB60W2gf2YB1upUNcqIQR18K8HtiNsG6E97wXBzIJd1nb/9RoW/3+BYhAIsJkL13s6IQpg9hJXpkiI6hG8165XvxnOz/zG9Eta4Ffu3Atvev6cv96xY7tKvSSXp6RMggqRcoTgxhO5qIje2bPhhTOR3QcNWjZm/x5WkGFv/y/fpP85AQA+ckJ/up3fotP/S+/QetLQ0veXywl+UCS82Jm/ZjZee9zfsy7xZAXvJhwiYGTNkvQ0Sc4GMPe2KGjqlF9o8DU6wUQ4HY6tOxJ0dKfIt2bWrDPbiMIITAcPelR5AO8OR+308FuTbY5ACAlmp/HLJ7HLI8jhUZktYKmI4MAvBCiKBl69APwPPBroeYH9Tu6sHxfiCRINRusFDj1UR6JIMaQkoyMawFaJfRL5Mcj5saS4ugpSyfjGNiGjr24AIHQ+b/be/M4W7Kqzve7doxnzJPDnfLONVNQjEUxCRQ8LRkUBFHsdqC7HyL9EKXBpvGpTdm0aONrUJR+Aj5EWxEnCmlmFKoBRaAKqurWQBU13PnmzXk6Q0x7vz92nMyT070375yV8c1PfCJODPvsiBMZv1h7r7U2yl0UxdzaxLEWKI5rty8XzIXJsYlU5cI/E7v9cM04Yy6y/XA6f8O/mP1w55MNIXobBcmz23dDYLNmgh6Zx0kN3mlSGxltiB6Zpvntk2STHZxGQOWH9hBc3X/BYohMqtGRFToCB9XwUSVvQZh1/gA4nXW9KKSrW9fWOl4WbLyKda11Ls7Y/YxmIUg50dbTK3DVhvsnA2hPWisvi9vMjxzECVY2cc0dT5l4MKU27DBwVc+/pskWBK/L7NjoUieHZSgRGr7Q8BX71tjHGMN82iOMsWaqkzHZjpnOhAczh290OnSUxq3DcKrYnSr2TGXsGG8zes8kGmhVFFm/h7c1pG97hYGqz0DJIzzPL6oigld2MNrQnuzQmY4o9XtU3Fn8+cM48Tza9UnDAdu3Nt/EjE3A7FxPk4axYuPkeV09D4IzTzu21hV3WAwrNAZaWcZMx+6tRKiFLrXQJXQdPIdcLPObPG3nb6H5OnIxtWe9dg26Aun4i5Prg/IWRVU5dr+uwJ6C1frhYhvoeMn74ZZTqpbZNTzAtVdfdeV6jy1E7wJgUk063iad7qACF1VZ+2Yz2hB9f5rmHSNkUxFOf0D9h/YSXN24IGJnMo2ObMe/Chy8rWVUxbvsR4M2xjDbSTk21WZ0ziZFroXemu7MlxNdK0/clPkTB3G8YIXXY2dKc/LuhLBfsfXJ3sJDuDl2lPkxh/qWrUuEr75lK2l2buFGIkLNE2qeYjeQRm1MpikN7iao9yP5Q7KVWCGcbCf5FHPPfIyeiinNpAy2UrYczVBHI9pMc4ejOeJqToaQ1F0aZY/+khXCwZLHQL48UPIon4VXpCjBDwzSHCd78CQtlWIGQ7zaAMQx5uQYjI1DmllB66ufvtDziAi5hWc/a41tCmwnYGyQfC30qAYugedz1u8GtikJdApZZJe71ueStOHdirngeuD4GPGIcImMy3wCc5GhndkmWSMKz/PxXeey6YfrpVQts70kjLzpjejHHlv3uE+X3xltYIwxZPMx6UgLY0BVvDX/oa3YTVnLbjrCGQip//BegivPv9iZTGNibV2+PYU7VMKpeKjLaUT00yAi9JU8+koeVyQVxuYiDk20SHVCyXMoX4b/nF3aUxHGpLRGDiHKXZEVJWlpjt9pM6vseIZtsjPGMP7Atxj57peJjz+Nl7757XzuD96z0Kf3sl9+B0eOTp+X+hmdkbabeOUqpcH9OH6wZHvZcyh7JXbV1x5QttVKGD06x+zxFsOjHXbNpUgE2SxMBCmH3YhvkXLc0eie2ztw1IIA9oph73ItcBebuZMWbmsMtzNuWwmqFdI0ZPpYhNs8TsVM4wVAqUQwtA11xZPQboBKI/Sj9xJPjZ+Xa7YelIKw56Uy04bpdsJkK8YAZdehWnIp+w6h67DGIPUrEbGW3ukwEGeGOEnpzMfMR22iOEFMBibDEygrh/pCFW0HhREH43gY5WMc3zr+qMAKo3IxojDK9nEiFyBcqd2CiXFkfAwmx5GJcZgYZ+ub3sjIr/4mybHjZ1Xs5fuk2GCYRBOPNtGzMarkruxo7+6nDZ2Hpmh9+yTZTC52L9lHcGXfeY4DMpgotXE9ruD0hzg1D/HPb7zRpSD0HHYPlBlulJhuxRyabDE+H+E5ilrorpr261KRRhmd6TbRzBHb0b9MULLEcPyOGJPB8LN83EDIojZHv/FpZo8+RH33NdSfcDNHR2Ne9rZbcR0hzQxHjk6fcbLpU9av08KgKW/diV89+8Fdy2WPfdcMwDUDttwooznSYu5Ei+qJFlsnI27EQRxBBnyiPo/piuKkkzEZpUy2E+4fm2Oqk6xwCnOVMBA6DHqaIS9lIFAMlEMGXc1AOsdAa4YGCcYJmHEGCDJNf3+D+f1P540fv3chqcAfvfbpVPnOJRG+Xhxl+/y6pJlmfD7Os+kIZc+hHriEgUPgqHVnqEkz2xXQTjJasZ26XRVKBM/xKJeDFS2n2fKCTIYYg+gYyR2ABG2D4VnZ8KrFhR6R1I4H4mMcd8FD1oqlY0Uy6lhRy8VsYT45DuNjSLu1tDpBAANDOH19Zy14sEEystx4443mjjvuuNTVWBVjDNlMTDLass5Z4ervESYzdB6apHXHSbKZGHcopPzM7QRXnD+xM9pgojyIVwlOX4BT95Fg4wvd6ZiPUk5Mtzkx00YbqAUe/mUwcsPciTmmDz4CJCti24y2gtca1+y8yac85NCaOM7hr91G0pxlx9NfzOB1N12Q305nKVnUwivXKQ3uuOBj3aWdjPmRFvMnrBB2piIAlKeobitR3WEdY4KBgNnYNqdONTtMzs0yPTfDVJQxHismYmGiY0iWPbYUhoZrGHQN/Y7hba+5ibd96oEVsY0f+6lr6Nx5+wU913PCQJRlJJnt8FZKqIYOtdD2kfrLnGJS3etsktKKMtL8rUEQXAc8tQ7r8ZzqnnvEGm2bWqMIpqdhahY9MwtTc5jpWZjO5+2l+WTxPOjvxwwOwuAQZmAIM7QFM7gFhrZBtQYi7N27ldE3vJ7k2HF+4uBB7u201/UPUlh654COM5KTTXQzQZXcVQc8NJmh8+AkzTtOomdj3C0l+l62D3//+RE7Y3KhyzSilBW6qoeE7uM2ie5qVAOXq7fV2DtYYXwu4vBki9lOQug5VC6RdZu0E6YeOwgmwivXVmwfeyChNabZeoNHaVAx8eAdnLjzH3DDMlfc8rNUtuw673UyxpBF9g26vHU3fuX8tjCshRs6NPbVaOyz1yFpp1YEj1sRnD1q85VaEQzoG8zYV5+hOqDR2yqYVGPm52FqGpNq5hyfSRUwmQoTiTCRqoXl44lCAp8tVY/f/5FhtpaF0Zbhv94+xrgO+fDDZXYHmp1hxu5AM+CayyyRkFp0iklgMtKMTdkUdo4SaoEDAu0kJTU24FvE4DmC7yrCNQYlPt+YJIGpGZiewUxN2+WpaUw+p7nUUsN1oNGA/j5k905o9CH9Ndvn2ldByiEiumckK+tNK8xAawbd8cDxmDJT7Pi93+PEW94CBw+uu96F6J0FRhuymchad47gVFe+JZtM0/nelBW7OSt2tZfvx99XP+eHjDEGE2eY1P6zOnUfpx5sOqFbDd9VDPeX2N4XMtNOODrVYnw+xlXWeeBihT0YY5h85DGyeI6wr7Fi+/TBlJmDGY39LtXtGUe+/ilmDj1AdfhKdj/3FbhheWWh54jOUtKohV9tUB7YjnLX7QNw3vBKLv376/Tvt04mSTOleWSS5rEpZkebHDmqOEIJxzPUqx1qYZN6NaLU56NKDn1AH5r9a5Q/7KX8ycsqNP7+J2D6MLsbe/iTV/4pjxrN7VM+bbN4H5TEMOxrhj3NTt/YuafZ6pnLLjQ11nCyqcGA6/gopRbCLGKBJrYPUSmDUgbHMfkyC+IoYrfLWnG6OSZNYXo2F7LppaI2PQ1zy5rXHQWNPuhvINdfA/0Nu9zfZ9dXq6d8PnXDtlbfaPLm1QxMxMjAMPPv/RDyr3/sTC/dAoXorRMdpSQjLXQ7RZVXiozJNJ0HJmneeRI9l+BuLVN74U78vecmdlboNCa12VGcqo/TF6BC1warFyxBKaG/4tNf8WnFKSdnOxydapNpQ8VfO43U+cAmfz5K8+QYQX/fiu3N0Yyx+xIq2xSV7RM8/LnbiOen2PbUF7Hlic8575bXgnUnQnXbXvzKxfVmPCU6w4mmCdsn6Gu00YMBWjtE47PMHouZnQuYa5aYmhoEwHU19XpCvR5T70sIw2zVB3fDzFD++9fB9GG7Yvowjb9/HU941cf5n1dNMWUCjkaKo7HiWCQcjRX3dRy+Nr/YWuNi2O5rdgWGnb5ml6/ZGWiGfU14SVvOe094aQqChQQeBtJUkaa5Q2fXOSVPgCEAOkPNzeDMTePOTeHMTqFmp5GZaWR6GubmlvbbKQWNOjQayLVXLVpt/XZOrXZBXrozA3OpYjpxmEkDnvDUG3njn9/N0ak2JyY7py9gGYXonSFGG9LJDul4C/EcnOrSt2STadr3T9K68yR6PsHdVqZ28278PbWzfojZAR6t5yVivUG9beU1m1ILVqfsu+wfqrK7v8xkM+bQRJPx+eiCxfy1J0aZPnwUr1JDLfNCiGY1I9+N8WsQ1O7n0S98EccP2f+DP011294VZRkDqdak2pB1Ey5jHRJcBZ7jnPJtXaeJHf6ntjj8z+WAZDFOZwJvfgRBkzkBaeZjxqag1cZXwtD2gC07DdAh6ihmZz1mZ31mZz0mJ61DkOdluQgm1GoRg/Iwjc4BQrMXve+FtJ/1G2QS4JiI0jffRclMceORXyRVFWKnQeI2iN1+ksDO56TBET3Ao+kADyd1jiYOhzqKb8056B4J2OJaAdzpa3Z1576mfgEvb2Wonx1X7MF1HNIs48Sjh2mOTy29rr3Wm06R2VlkZho1k4vZzDRqegqZnUbm5mzKtBwjCl2rk9Ub6N1X2nnfAKavj6zej6nWEKWQ3HJ0lEH1WpJ5y5MSkFNYktrAfCpMp8J0ophJhelEmEkV04kwnSpm8vl0IsylsuTa/9ULwoWm63/7mWDlF5yGy+M/4DJHt1PiE/OYRKPK3pK3GZNq2vdP0LpzFN1M8LZXqL14N/7ucxC7JEPHtglDVVwbYlD2FlJYFZwdrqPYWg/ZUguY7aScmGkzMnN+Y/7aUxPMHTuKobyQVLpL2rGOK0gC3M7xbx+gsn0fu5/3StxSlSRbRdyUUPKsOIe+g6tspv0ozWjGGe04I+txd3SV4DqCEkF3mjaH5I796x/+50JgQCXzuO0x3M4EBkVmPEyzBVMTNn+m5yHVlcmsg1CzJYzYstUmgI4ixeysz9w0zM14TEzUgBoVBcN+ROuxkPpT3sXn/3BxOKaX/vy7qAUTTDReiZdN46XT+Nk05egIrp5bCPa/If9OjUvsNkjK/UROg0np56Tp55Ae5JF0gPuTQW5vDdA04UI9645tKu1ahbtyURx0zTmlq60M9bN793ZG3vwmkmPH8XYOs/u97+Xo/Dyth76/VNRmppHpKWRudqmoAaZex/T1k+3Zj+lrYPoa6HxuanUbtN+D5FP3P6ObRcoYSFKFSfI4xAxmtTCnFbMZzGbCbCrMaWFWw5wWZjJZWK9X+H5aao6mzzM0XM2uMONJVUOfp2m4+dwzDPvJQtO1P/HAuq9l4b15CkyWB5lPRYivlgRwm1TTvm/CWnatFG9HhcpN2/F2Vc9K7BazoxhUycNpBFboLnCKp81OJ8kWYv6STFP2zz7mL5qdZuboY2SRTxrpJaKnM8Oxf4loT42j00+TzE/QuP559F//XEQUooTQVYSeTczrKuuU4Iic1smi68GXpIZ2kjLf6tButfCqg/iNLTiui+cInlI4l6IpXGuceBa3eRwnaZGJg0kEMzkD7bY1DYLAJmQ+HUZTjo/Q176XvvYBKtFjYAzj5moe0T/IseRJTDS38YOvfypf/5vvrxh494d+7mrufOcf4eoYR8c4xs49HeHqNl7WxNUt/KyJMjHKJIhJF+ayypilWlxSPBJswHfbeLSMT9t4RHjEeGgUgYJQGUIFgeRzZc4oE+fe//s/MXrrrUtc9b2dw2x7x69y9M1vtpcGMLVaj5j1L4paoytqp7+3jbFpgWezXKjSRcGaSZfOuyKWrnEWJWXocwz1fKopO++TjAEnYkBFNFREn4qo08FXEa4Nm8chxiHCNREOUf57RNRf/nbcT/8iTB/mxg/Nc8fxrPDePB9kzYRkZB6TGlRlsQnMJJr2feO0vjNqxW64Qv2WvXg71y92JtW5RWdQvs2O4lQ95AL2NxUs5VQxf9XAPWPHl6Q1z+yxgzh+ic5Mghso2yyZGZJMM30go3nyPtLOP6I8j6tv+dds23M1bj5IpqtOL25r4SpwlUPoanwdMzAQ0njCNYhfJko17SSlGdm0Umm81Cr0XYWr1AXxXpQswelM4jVPICYlyxySdgZTk6e06pbj6Bb19gP0tQ9Qb9+Ln80C0PT3cbzxI8yUbqAdDxAcOcTVhz7HtYcP0v+rn10ieABzEx20uBxqPO+M6q90nItjhKNj3CzC0RGu7uBmHSuSuo2btXGzFl7Wws2ahLpFVXdwsyaubuNo63mZZi6xsY/0BJcOLnPGBneLOCjl4Isdf9mXpZahU6ngDG2h8Z/fjfQPYqYmmPsf70N2bKf92p+1QlfvA3f1R3pHY8UrXlvIZruClgmJgZCYEhFlIkpi530qYlB1uMbp0O9ENLwOdYmo5VNZOpTy44JcsKxYxSgdobIIJ7UvE+vBIGjxycS3yR26fbVnQSF6y1hIITbVQYWLKcRMktE+MEHzu6OYdoq3s0r9h7fj71xfs9FCGjDTkx2levmnAXu84yhhsBowWA2Yj1JGZtocm25jThPzZ4yh1Wwy8ehDGM8nnUrIkgxHWU/Seslh/P42c499kSw+QP/wXp72ktcSVs+vM0kaR6RxRG1oG9X+rQt5PQNPUe/JeJ9khjjLiFPNfJQy38mYj9KFvKk2ePncrMLerClaa7JUYaaaNsOGUqe36owhTE7Q1z5AX/sA1c7DKDSpKjMbXs9M+QZmzH7MsQmcA4/hHP405QkbcG6CgGz3XhwyaoPhCkuvVheeedMYWSboTMgyIdNClqrF5ax3csiyCllWJcsUcbZ0uzFnco0MrkrwpIMvbXxp4ss8gbTwpYWn2vjSxpEOkcCkKObwaCmP2CmBF7K13Ef5P/93PvuxQ8xNHKY2GPLD//n/QdcNDwwM0cliosmTJFlMqmPSLMboGJPFiInwTExZIkpEDBCxSyJCIiq5WFWkQ0liSm5E6Fr7VJ0itytgfWh6DN9MPLQEdlI+Wny0BCRObWFZq8CKVz5f3Le73l9RRiYBRryFDsLrdB/Vxp6zFr6ieTPHGEM2F5OetLElEtrYLh1ntO8dp/XdMSt2u6tUnrkdf/jMxW4hO4oG8QSn7/GTHeXxTJxqJuYjDk226CQZQf6gjrNu7gpBJxHpiceolQPKYUhzpE25avsHReCxe47zva/9DSYb44pnPJ9rnvODqHMYj05rs+jXLYA2RO05vLBE//Y9+KX1hzoYY881TjWt3CpsRilxttQq9By1cF6rFeLEc7jNEzjJHFlq0K0MZmatVed7iL92ALzSEbXOg1boWvcSZBMAtLxdzJSfxLRcQ+ek4Bw+hHP4ICrPQ2p8n2zXHvTe/WR79qG37QClGLxyH9Xd1/K5D9/f06d3PfNHHmTikYPrvkZroTXLRFItFdRVJ0WW2SZvnUGaKbLMwbB2V8ZL33jDqs21t/ybqzn5oV+m7oxSd05Sc0bx1eoejSkuaVdEVGCzpahF4cly4ekKk1YBmfg967rru/v2ipt/2oTW54vKUD9XPWkQ75P/hhvfc3/RvHk2mCQjHm2h5+IFz0gdZ7TuGad11yimk+Hvrtk+ux1nNmL0iuwoAyFOdXNkR3m84LuKHY3FmL8TMx0cJdRDG/LgmoypgydgZwM/LDE91sKEzoJV+PAdd/HQNz6FiOLpP/IzbL/iurOui84MSZShHMFxFUZD1GmRxgm1wW1U6kPoTNGeT1gcL6pnmCfIl/I1Xdf17vBOAj5C4LkM+C7UhcwY4kwTZ3ZomVacMddZbJYSBF+lhMksQXsESTvoGJKZdo9VF67paRwko7ZvrnWAWudBFCmZBMyG13Hc+SHmJvrgkTHUoYOo0XsIAeO66F17iK+/wYrc9uEVzhfAgrD92Juvx4iDmIyJ759fwYPFmDjP617lFcm8zoiFIcDSrrWZi2eSIEmbvqFnrN5ci8vX516/ZL3rJIR+QhgkBIHGCw1+IAShIQiy1S7XZY/RBtIEkpT5+Sbfb7XZ+dK/IP3j16brLWtTi55NIRaRjLatMFV9K3bfHbGWXZTh76lReeaZid2i0PVkR6nlQrfJg8Y3MiJCo+zTKC9aKlmaMPLwYxhtCCpl0iSjPRvjl1yyNOXer3yOYw98E8ffwXNe86+oDw2c1XdrbUg6Vuz6hkqENR+Mpj0/y8DwAEO79+KXyj1edcaOTNMdCiq3DJdsX4jlygdTzhbndh1orfE0+FqjNfQFBqMNOtNEiSbutIjnRklmThK1O7RaGuabkBlU4OH4VRwlSAakWGE1KfX4Ifo7B+iPDlBKTwLQdrcz4j+fmamtxCMJzpHDqJNfwzMG4zjonbtJnv8iK3LDO88syTJW+M63yF0ouu79yjd45MMLLVAiKKlVm2vrAy5Pf8Y4UeQQRSqf22m+4zMx46xognVdTRBk+aQX56Fdd1FSlp0CY4wdISPJXUPBvl2UQmjUkSCk5Xt897sH+f4jhw6tt/xNK3qLKcRSVMnFpJrmt0do3ZWL3d46lZu24W07tdgtpgEzdjzIWpEd5fGOzjJGDz5KEkWUajatVnM6RpTQnpviO5/5OLNjx/DKT+c5r3k51cb6Y4lWEzsRiFstsixlcNce6oNbFgahlQUvT1kc2O18Ywy0p2D6KHrmJKkTEek2GS3SioPeGZIYh1aU0k4SotTgJpP0Ne+n0b6fvvb3cEyMFpdZ5xpGmk+hddLDHB1FnXgIMQ/iOQ7Zjl1Ez34h6e59JNt3YRwvH9+1K6IGbIaqxfViFj5bg1byXJD5JAKOg+RxjRutsWX68CFe9gvX89kPLjbXvuwXrmf68CE8z+B5KdVVelyMgTQROpFDvEwYWy2XqamVouh5i6LoBysF8nyLoklTSFMrdGAbI8LQBr6XQpuT0+sZsUanYDJUtj5nmC6bTvSMNqTTEemYTSEmntC8Y4T23WOYWOPvq1vLbtvafSPd7CikGiOLQldkR3n8Y7Rm7PBBomaTUs06o6RJRms2YurE97nnH/6OLDH41R/lxpc/nWpjfam+umInSqgPlShVPcDQac5htKbaP0D/9mG8MDxtWeeNLIXmGEw+ip6bJJmJSCabmCxDlULcoToLsq4zhrJDqNl7cCbuQTWPAhCrASZaNzAzWiU+2kSOjSD6XhvsvGsH8sLnoq7YC3t24freYnkYjLHej133AzuXxc/aICbDZAmSpfk+gsZBu2UyVcXEMTqJMJ0Omcmd08Bm/1d2FADlLE3VdTkJ48jd97H9KfDqt96A1tbwmT58iJG77zvlcSLg+QbPT6G2siXQGEgStcxKtMvz8x7xpFpFFHssxLDXWszwfX1KUTT56PU2VUy+0vdsirJyCL4ProOQITpDdIqYFKKWzcUJdogjJyAubyPT629P3lSipzspyUjTxsMJtO4eWxS7K/qoPHMb3pbVxW55dhSbBswvsqNsIowxjB87QnNqkkqjf2H93ESbh7/1JQ7d80/45e2o4GU88QU7aWw7c8FbTeyyNKEzP4tyXfq3D1Pp78fz1281njVJG2ZPYMYfJZudI55qk81HiOugKqVFD8x4FmfyAGriAM7UvUjaItMOzc5+WlPPpnU8pXPoJKQnQIRg7078Fz8XdeU+0l07aYtDlGo7pJAxSJTZMAxH2SbS/JlrrbjMxstlsc3oTx6f5oToUh3tVtBugHF8UD5KZKl7iLbHkiWYOEKSNkRNTJLYPrRUSLUiNW6PICq6/aGLgriYz/JiMXL3facVufUiAr6v8X1NbQ1RjGO1IIbxMlGcmAhYmhbN4PuaIND4fkrgJQROTOClBH6KVzKoUog0qojnIA4opxsCrzF0MJmLcUO0XyPzSvlwRYtT96JH0iEzqwROnoYNIXqJTjg0ewglCoVCiUJEcGTRKUQQ28TT/euuF0EyQU/HmMkYMk3yvRmiA1OQaLwr6pRu3IK7pYRCoTF5CRYdZ5jEXldVLrKjbEaMMSSdNnMT48yNjVLuSSA9Oz7B1//yo8yMHqF/59NpNX+A/U+usuOKMxOnRbGD2lBIqeySxB3a8y3Cao1tV1xFWKudk8fnujAGOjMwfQQ9dohkao5kuoPR9mHlDtRt4t+5QzgTd+NMHkDmDkJmaM3105zZT/OkQ3RkGpO0QI7g795B/eZnE157BeFVe1GlVaxUY8MpkkwTJRmtJKUTxURJhMpSwKCU4DguKqiiK4MYt4RxrAchZ3p9lGuFzAPCHicfo3F1jMoSJI2QtInEbUjmbF+9tg4mqXHI8Mm0S5a6C11OPaeRp+FaajFeTlbjehAht+RW15auKHY6iqglRG2xIhk7zM16TCQhS0RRDEEIflkIqg5+zcOvBfg1H79ewq0EyBkMJV8pu+zdNsS1V1915XrPaUOIXpRGHG8eBwPa9tLbjnoM0jW9F9ryrQAaY9v9nTa4EwbVhPIhqDwCKoX2Lpi7HtL+WWAWxhbLkDSfEEwg6JpAyUF5DtISVNuKbq/4qvx9UikrzN3tCpttQ6EWRFqJWiLMC5+Xi3XPulXnCx56K7cXnBvGGJKoQ2tmhrmJMdIoQpSiXF8ciuf4Qw/w9Y//OVmacMWNr+b4I/vYts9n/1PWHmG8y3Kx80NFGrWJ2hG1oS3UB4fOKvzgrNEZtCYwYw+TjY8Qj82SdTTiuqhqBdEdnKm7UY/dgzN5AKI52lM+0zPbaY5dR+doE5OkwBTeru3Unn8T4bX7Ca7eh1M+zfXQGrIYL0vwjKbsQL8L1Mpk7hYSt0IHj2bm0EwVzSizTZva/v+7yuC5GvdcOptEYZyQzAnBrwFDdr0xiEmQ1GZucZMWKm0hyRzKpNbM0EJmlI0pw0XjkqaCzr0wte7td7TzXqvxTEY8uKwwmfWkjCJUmhBqTagE+gNkRwilEqYcoks1MhUSRy6dJkRtiOc10XxKPJ8wM56QHOoAPV6pAn7Vs1PNI+hZ9mseXsmlWvXYWgn40gfvZXY0WvdQIRtC9ADq/jqDeTODmspQYxnuwwn+wzGkkO51ad8QoPsdSsv2l9ggBnQomJqy8/yS2uBdszjHkJrUesfBwjYAvdDsYv96Rbq7nO+w1Ke8Z3khA0x+zIIgIxgxSwS+W79uGcJSYXXEWViHsCjGPdt6Lefu9t59u2LeFXr7VSut69XEuns+3XXdsi9HsY47bdpzM8yOjZFEHZslo1TGDxfvFq019/zD57n39i9RbWzhmuf9JN/7F5/6kMMTnls55XksiJ1AdSDA9TUm7WC0z+CuPVQaDZyLOeRPGsHcCPr4AyTjEyQTLYy4qNDH88dsv9wjB5Cph4mmHWbHqzQnt9A+0Y+JUiDFGw6oPu8JhNfuJ7x6P051DbE2QJaATiCLWbypXQirUNkKfhkcH9wAxMHB+uWEQKOnoG5cYSexwfVzeWyh3WpwlYPvSB5XeA73lQhGfIzvW4ELe7xwdYrK4oWRxd2kiZe2kWwO8bt9hqCxzXIaD21sQHuWkYuiFcc0sZk/lzvj9DanXjSr0WjEZPZFyGS2TzfNFh1NEExYQQ9sw9QaENYwYQnjhRhxV1jdbg2qQ7BaZLPONEkzJZpLiOcS4vl8mkuYPdokbS1tchUFL33jk/nSh+9bEcJxpmwI0fMcj5pXI9YxURaddn9paZwjCd4DMd6jCWSQ7nOJbwjQjZ4fJDNIYhANxhP0gIMpKfC7D/Uew/zSP4/PmK4wr7WsjSY1iw+IrmCutryacItZFKiuRd0r3GJyYWZRlG1oWI+A5+uWW+i9oqlELbGW+4N++oI+Kl4F3zl/I30nnQ6tuWlmx8ZJ4g6CEJTK+H223871fBzPwxiN0ZpvfvJvuP+rX2bPk25k1xNv4d6vRfih8OSbazju6jeK1pB0UkSg0u/huBmYmLBcp2/rNsJKdcET86LQmcVMHiY79hDx6CS6Y0CBlz5m++Um7iE5OcfsaEBzokH75E7bFw6426pUn3UF4TX7Ca/Zj1Nf5XGmMytwWWKXBfu0dktQ6rfWlBvYad2/peC7Dr7rUA09hmrWQSKNY+I4IerEtFpNWm0bW6j1YtwixtiHsutilGNj/BznHJTEs5OqQDAEAbbfUScoHaPSCCdt4cRNVDoH+f+AAEq5iOuhlYsrzkKsntZWGI22y2ke7K61YHSeFnvBQ9X0jGhwhhajMYterUZbb8ju/6EBMo02CiMeWvkYvw/dqKOrfeiwgglKK8NGDBB3F9YZOucpGAjwBwKW3wk606TNlHQ+IWmmpM2EsOqdteDBBhG9h6cf5ue/+PO870XvI3CCtYUvNTjHEoLvRLiPJaAh3ecR3+Cj+3Kx0waJrEVnXEE3HEx5UegeD/RaWBtJrGGpSC9YzPm6E60THJ07CgIlp8RQaWhBBD1nfdaRFboZ5sbHiDsdRHKhqzeW7Od6PmkS84n/diuzY6PUt2zlljf+Mtuvvo4s28J9X22RpYan/nANv7RStBbEDijVFI6f4ToZ9a3bqPUPXlwvTK2hPWWtuuOHScamIJ7Di76PP3Mf6aFHaZ50aY6VaI2W0ZG1bt0tA5SfuX9B5NxGT6uLwTZ1ZTGY3CPPGHA8CHqst664yfr6Jo0xkGWYNLWu7VlmP8OSe1t5Hm4Y4tdr1EslJAwRzwPXJUEw6WIZpt1GRxG02+iog+lES9/FjLFepY4Vx4X5Ob+UGEhje62yKO8zbELcRPJclIa8Q9Cxo4Qvv17G2KF5tO5aipBmQpLawPYkla5yAilkGiUZSjQqdwoSBTgBxg0xWmF0fl7iguuj+vpQ1SpSKqOCwF7Hy4iw6q2IWVwPGyINWWl/yVx161UMV4a59bm38oG7PoCrXBxxcPO3pL52hWc/+gSefPIKlBYe3n6Cu/cdpFWNcXFwM8fOHRdVcnFCH8d3bTmqW467pFxHObiy+nZHOSvGSiu4eMRZTCftkOoUEaHslhksDdLnry2CSdShPTvL7MQYSbsNIvilEq630tJI44iZ0ZMM7NzN/3rfbzObp70CqG/Zyqv+07v40kceYXIk5SkvqjK4c2kZXbEDCEoaL4SwWqGxbQflet9CbsyLQhpjZk6QHb6X+MhBZPz7qNYj6KMP0DnSpDka0BorkeXPEHewYZ1OuiI30MhPqmu9xXncgFm03oI6+NV1WW82CDldKmhG92wHRFBhgAQBqlRChSX72fPA9RDfQ85RkIz127f1yOc6itCdDqbdwXTa6E6HXq+VhaQ3udUouTjKGgmfT0sWL05xKxfDFnRf8A02TsHxQOX3tk4Xp9xUs7kJFFpKZCokkxKp+KSZSxQZ0hjQthVHEKRcwqvXcGoVvHKAhCHqMo8vdn1FGms+/6F7+fUP/VsOjz24rgpvCEuvy/Hmccpumfl4nsxkpDql1inx0uPP4YWTz8Axitsbd/LXQ1/kmHeSbCaDmQtXHyVqUQhPIZC9wrrwuSvY+b6n2r7a8nJhPuX2nvq4yt1QYh04Ab7ybbMqstDE7Tv+kibOOIs5Pn+cI/oIABWvwkBpgColpJPSmpwmbrUQR+GH4YIHZprETBw7wszJEWZGR5g+OcLMyRHmpyfBGH7ynUsFD2B2bJQk1kyeSLnmpvISwdMa0k6K1ileYCjVXGqDQ9S3bCUon7q/77wTN9EnHib53lfRj/4L5sRDRAdP0BpxaI0FpG0PaOA0qoRPucr2yV1zBd5gf4+4pdCetuWp3HqrbgOvlItbwGo5F43WC0LWFTVh6Qu2QVBhaF3YSyW7nFsW4rpW2HqDki8QYv32T5kbFFi0FLvimCTWaux00O02JuqQzcV02x8X6i3KBsbn4ojjrBRpx198USgt7TdcEMOkDXHTCiKAV7YvHH7J/jaOhyjPerimGSqKcOOIbluCDJZR/Q1UtW7737yALE9vl0QZaaxJWmmPf4AVUqUUjisoRxB16fve01jj+oofffNTeM8ng3i9x28o0RuuDNMf9vNbz/8tZCYj/Kc23gP2Jkuu9Gg/0eem4P/gWdmLQUFWgawMiZ+RmozUpGTaimVXNLvzVKen377s8/J9Mm2/Y63t7aS95Du6+652/IVEkDO2aFfbvmhhry74qwr7GQj28jL6/D4w8Kbb38Tx5nGGK8NrNnH3imAWxzSnpnng4YdIWh1EIPTLhLEinZynPT7J7OhJpk+OMD81sRD1rByH+tBWBnfv4Ypn3ERj23ZKtT7qW7ausPRaMxm7rgvYda19pJjcskuTCL8E9UaJgV3DVPoGcE/zMD2vGIOZPo6+6zaib36GzoOP0D4e0RoNSNsOUMOphoRPvIq+a68kvHov7kAN0cmi9RbNgluG8qAVOTe04pZbzyYXMuIMk87bgGOWjQunFCoMkXIZKYXWSvN9K2i9oraBWBCtU7Ag9l1RXGY16nYb05xfuOdgQSMXrMVeyxHl2skrQ9hY/TuzzAbfz3cQ07YvE4GP29eH6u/HKZftb3EG19sYg04NWabJUk2WaJJORhJbYcySvGl5wdQ1OI5COZJPF+eFOo01nWbCg9/73oPrPfaSNG+KyEuA38c6Zv2xMeZ3TrV/aX/JvOB3X8D7XvQ+/HmFfG0G//5c7K7wiJ/gQ2A7qk1FYWoKEwjnNFTxJcQY62yyXJB7hXG5OJ9u+3rEvVd8VxX5Zd/Ru682644VXZPfe9Hv8Z5vvceGq+QMV4Z553PeyZ/c9yfU/Tr1oE7dr1NVFUqZh9/UOGMtmGkTz87TmZymNT5JZ2pm8UGjhLC/QX3rVvq3DTO0YxcD23dSG9yyotmx26f3qfe+e6FP74f//Tt44BtNdl8LIMSdmLjdJqw5DGwfYGDnTsJq9eLF1gFMHyX68kdofeWztB85SWvEJWnZB7Qqe5Su3ktw7TWEV+3EG6oj3YBr5UFQs5NXwoiLMQ5k9uFtsm7T2aKDhLiubQYrlZAgQMplK2i5kJ2JOGx2eptSTZKgkwTTamGiCN3uYDodTJr0xBHmv0LX8SbLMFlmrWfHwelr4DT6cKrVhd/jgtRbW0HsCmOaaPuyF2mS2FqLvY5qIvQI4vkVxU4z4cobdhxIsvjJ6znuot+ZIuIAHwB+CDgKfFtEPmWMuX+tY67vewKfeuHfMfedEdpfOWH7g/e5xNf6mKrClHOhCzeu0PXSDR9wcM6rl+LFQBt9SlFcSzRXW95Z3clrrno1PzH8Krq5l/7m+G1UvApHpg+jJ5v40wn9cx6NeY/GnE+95aLy2E0thnbF0OlzyHYEyEAFf7BOqb+B61doig+qTOLGRO407Q5UvSqhG+Iq+6+RJjHVgSFe+873oLMUnSke+OYkO66ATrNDEnUoNwL2Xb2PxrYtFyy2LksT4okxSFLwXPyBIfR9t9P8zMdo3XEnrSMdkvlc5MIy4f6t1K67lvDq/XjbB20Tmwpt5hKnBHQzjnj2IZUCqUZ5BgldVM02N0qptGiddQVtg6Tp10Yz2ZkkzmJ8x2cgHLhsmvbP2GpMEhv/mCZorZlWHWKT4otDQ6o4vm+ddi5Sk6MowVWOdVpdtc49opjmohhZUYyjlKyd9Xhw2y6LBUF01Rn3Jzq+4FQTrr7+qnUPXXIpXsduAh42xjwKICIfB14JrCl6yckWkx+5n/4fv4b2+BQPH/gS6fEIRrEi9zgQus2AsODgfUbETzvGj1z5Iv7mv7xjwcp66Zvexrc/8gluucvHmMWSPD/ADX1ki4fxXZJQEbsgkiI6JU0SOiPzxCMzwJEV36WAQFxCPAJxKCmfihPwzKe9jCuv/wG+8P89tJDo9yWvv56ZQ1/j4L2fpdpXJ5oPOHLMWaXU80P/juvZNnQtJ97yH0iOHcfbOcyOd7+b0fd+hM7dd2N8SHbVSZ47THTFbrJtA+CFdnICIAXlIYFGvAQJFMo3iC+Im4LjIl4uZiJAkk+zNtl/lE8biLJrXz7eevtbF5rG33vzewFopa1LWbWzYuF8vrzsfBJozVzm5yPYQMsQMFhRzOyoHlYYM5KOIUsydKbByBJRFLXYn6iUUCtVqZYC3vaVt/HY/GPrbiO/6M2bIvIa4CXGmNfnn38WeJYx5hfXOubJO64zn33dh3H6A2o/dyUffMvrLlZ1Cy4hr3jbr3H7n314RX/aLW/4ZW77759EqUHEGUKcfkQuTP/QWoN3vuoXruLoLS+8IN+5nF1/8Aec/J3fJjm22Mzr7Rxm8H3v4XWffR0Ht4EpXvyWsFbT+Ntvejtv+cpbLl3FzpLH2/mcC73X4uFbH6b9WPuy995crYIrlFdE3gC8AeCG7dcCkE1F+F7AdUN7L2gFCy4P+nfsWtVzsm/rNm7aGgHH8+nC0dhy46qDd5qgxJGfOftBYdfD3iv2LhE8gOTYcYKBIX7mFb/GRc98vAHYU9uzRCDAen/vqe3hN579G5eoVmfP4+18zhZjzKrXYj1cCtE7Cuzu+byLVZ5cxpgPAR8Ca+kBOP0BKOHlH/jAxahnwSVmdmx8Vc9J5bg893fffVHqMD8+v+rgncoRbvn12y5KHdonbZPmcktPPI+fvP5fXZQ6bDTG2+MMV4ZXWEb9YT8/ee1PXsKanR2Pt/M5F1a7FuvhUvTqfhu4WkT2i4gP/BTwqdMd5PQH9P/0dajKxnLsKDh7Kn19vOKtv059y1bACt4r3vrrVPr6Llodwr6Al77hemqDNjShNhjy0jdcT9h38Yb48Qe3sOMP34+3cxiwgrfjD9+PP7jlotVhozEQDvD+F7+f4Yq9ZsOVYd7/4vcz0Js7cwPxeDufc2H5tVgvlypk4WXA72FDFj5ijPmtU+3/jKc+3Xzjy19DVXzcYGPF9hScG1mc0JyZQesMpRwqfX04/sW9B9IkoTMTLQzeGfYFuBc5xmyF9+bgloubmHoDcjl7b54Nj7fzORe61+IFz3xBfP+996/rDXRDpCG78cYbzR133HGpq1FQUFBQcBkhIncaY25czzGb8zWhoKCgoGBTUoheQUFBQcGmoRC9goKCgoJNQyF6BQUFBQWbhkL0CgoKCgo2DYXoFRQUFBRsGgrRKygoKCjYNBSiV1BQUFCwaShEr6CgoKBg01CIXkFBQUHBpqEQvYKCgoKCTUMhegUFBQUFm4YNkXBaROaABy91PTYYQ8D4pa7E44zimm5Mit/t8cu1xpjaeg64FIPIng0PrjeT9mZHRO4ortn5pbimG5Pid3v8IiLrHn6naN4sKCgoKNg0FKJXUFBQULBp2Cii96FLXYENSHHNzj/FNd2YFL/b45d1/7YbwpGloKCgoKDgfLBRLL2CgoKCgoJzphC9goKCgoJNw2UneiLyEREZFZF7L3VdNhIiclBEDojIXWfjxlsAIhKKyLdE5G4RuU9EfjNfPyAiXxKR7+fz/ktd183OudzvIvLPF6peBReG1XRBRG4XkXWHolx2ogd8FHjJpa7EBuVFxpinFjFJZ00EvNgY8xTgqcBLROTZwDuAfzTGXA38Y/654NJzVve7Mea5F6pCBReMj3KedOGyEz1jzFeByd51IvJLInK/iNwjIh+/RFXbcBTXbX0Yy3z+0csnA7wS+NN8/Z8CPwYgIk/MLcO78mt89cWuc8Ei+Zv/+0TkqyLygIg8U0Q+kVvo/7Vnv/l8fnN+zN+KyPdE5C9ERC7dGRSsxWq6kPMT+f/gQyLy/DMpa6NkZHkHsN8YE4lI41JX5jLFAF8UEQN80BjzIYrrtm5ExAHuBK4CPmCM+aaIbDPGnAAwxpwQka357m8Eft8Y8xci4gPOpan1pmS1+x0gNsa8QER+Gfh74BnYh+UjIvI+Y8zEsnKeBjwROA78E/A84OsX5QwKzgeuMeYmEXkZ8E7gB097wIWv03nhHuAvROSTwCcvbVUuW55njDmeP5C/JCLfo7hu68YYkwFPzV8SbhORJ51i928AvyYiu4BPGGO+fzHqWACsfr8DfCqfHwDu676siMijwG5gueh9yxhzNN/nLmAfhehtJD6Rz+/E/nan5bJr3lyDlwMfwL613SkiG0WsLxrGmOP5fBS4DbiJ4rqdNcaYaeB2bD/CSRHZAZDPR/N9Pga8AmgDXxCRF1+Sym5C1rjfwfbLAuie5e7n1e7/3n2yNfYpuHzp/n5n/Ntd9qInIgrYbYz5CvB2oAFUL2mlLjNEpCIite4ycAtwP8V1WxcisqXbDCwiJWxTyfew1sPr8t1eh202Q0SuAB41xrw/3+fJF7vOm5E17vfC27vgjLjs3mpE5C+Bm4EhETkKvAv4WRHpAwR4X/4WXrDINmxTHNjf9GPAl4CvFNdtXewA/jTv11PAXxtjPi0i3wD+WkT+T+Aw8BP5/q8FfkZEEmAE+C+XotKbkBX3uzHm8yJSeNU+TllFF9551mUVacgKCgoKCjYLl33zZkFBQUFBwfmiEL2CgoKCgk1DIXoFBQUFBZuGQvQKCgoKCjYNhegVFBQUFGwaCtEruGSIyGCet/IuERkRkWP58ryI/I8L9J1vEZGfuxBlXwxEpCEi/9dZHHeriPzK5VCXNcq6WUTOKRF0T07NLSLy+fNRr4LHH4XoFVwyjDETeZb8pwJ/hI0lfKoxpmqMOS8P017yjDT/DhvHeC7lXMocmw3gvF+bs6TB+avLzcB5Gf3AGDMGnBCR552P8goeXxSiV3DZkb/1fzpfvlVE/lREvpiPofZqEXlPPpba50XEy/d7hoj8bxG5U0S+0E0btowXA98xxqT5MWc8CkVep6+IyMeAAyLiiMjvisi38+N/Id9PROQP83I/IyKfFZHX5NsOishQvnyjiNyeL1fEjhf2bRH5roi8Ml+/2igOvwNcma/73Xy//9hTj9/sqfOviciDIvIPwLVncN1vzetxu4g8KiK/1LPtrSJybz69JV+9oi7Lyvtk/nvcJyJv6Fn/EhH5jthxC/9RRPZhk3f/h7ys54vIR7vXLT+ma8VV82O+k98Dr1zjdD4J/PTpzrlgE2KMKaZiuuQTcCvwK/nyzcCne9Z/HTvMz1OAFvDSfNtt2GF+POCfgS35+tcCH1nlO34TeHPP5+NAkC83TlO/m4EmdtQKgDcAv54vB8AdwH7g1dhsOA4wDEwDr8n3OwgM5cs3Arfny+8GfqZbD+AhoAL8AfDT+XofKGGT6t7bU69bgA9hs+4o4NPAC7D5Vg8AZaAOPNy9vqf5Df45P58hbHJmr6esCjaV3X3Y0QmW1GWV8gbyeQmbJmwQ2AIc6bmOAz3f/Ss9x360e93yz/P53AXq+fJQfl7Su0++vBM4cKnv62K6/KbLLg1ZQcEqfM4Yk4jIAayYdPtrDmAfvNcCT8Jm2yff58Qq5ewAHuj5vN5RKL5ljHksX74FeHKPNdIHXI0VnL80drSG4yLy5TMo9xbgFT19biGwh1VGcZCVw73dkk/fzT9X83rUgNuMMS0AEfnU8gPX4DPGmAiIRGQUm/LrB/KymnlZnwCez+KIBmvxSyLyqnx5d16vLcBXu9fRGLPaGGmnQoB3i8gLsEmkd+Z1HFm23yj2paOgYAmF6BVsBCIAY4wWkcQY082d182cL9hhZJ5zmnLaWEHp8nKsSL0C+A0ReaLJmz7XoNmzLFir8Qu9O4gd12ut3H4pi10KvfUQ4MeNMQ8u2/8BEflmXs8viMjrgUeX7SPAbxtjPrisHm85RT1OxWqjDqx7YFURuRmbsPs5xphW3pQb5mWdSb0WrpVYpffz9T+NFc5n5C9CB1l6LbuE2N+7oGAJRZ9eweOBB4EtIvIcABHxROSJq+z3AHZw2DVH7xCRm0Tkz87gO78A/PuePsVrxGb8/yrwU3mf3w7gRT3HHMQ2FQL8+LKy3pw/3BGRp+Xz1UZxmMNacb3H/jsRqebH7BQ7xtxXgVeJSEnsiAQ/2j1ARH5RRH7xDM6xy1eBHxORcn6OrwK+tkpdeukDpnLBuw54dr7+G8ALRWR/XpeBfP3ysg6yeK1eiW1m7ZY7mgvei4C9a3z/NRQjLxSsQiF6BRseY0wMvAb4byJyN3AXq3sCfg5r2YFtAv3zvMn0uyyOQrGHM7MQ/hg7fNN3RORe4INYq+g24PvYptf/F/jfPcf8JvD7IvI1rBXV5V3Yh/o9eVnvyte/FrhX7OCm1wF/ZuzI3/+UO5T8rjHmi1hv1G/k5/K3QM0Y8x3gr/Jr8XdYkepyHSsHU12TvKyPAt8Cvgn8sTHmu8vrsuywzwOuiNyTn8+/5GWNYftDP5H/Vn+V7/+/sCJ9l4g8H/gwVhy/BTyLRSv7L4AbReQOrNXXHTx2OS8CPnOm51iweShGWSjYVIjIbcDbzRqjnOcP7/9pjLnnPH3fR7FOOX97Pso7H4j1jH11/rLwuEREvgq80hgzdanrUnB5UfTpFWw23oF1aFlV9Iwx//HiVufiY4z5kUtdhwuJiGwB3lsIXsFqFJZeQUFBQcGmoejTKygoKCjYNBSiV1BQUFCwaShEr6CgoKBg01CIXkFBQUHBpqEQvYKCgoKCTcP/D+rlxukxG+8hAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "ax = make_tabular_results_plot('wins_roc', df_=df_[df_.time >= 1].copy(), exclude=exclude, max_times=[1, 5, 30, 60*5, 60*60])\n", + "ax.set_xlim([np.log10(1.0), np.log10(3600)])\n", + "ax.legend([],[], frameon=False)\n", + "tikzplotlib.save(f'roc_wins_tabular.tex', axis_height='5cm', axis_width='6cm', strict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Big Table metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [], + "source": [ + "max_time = '3600'" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [], + "source": [ + "global_results_filtered = {**global_results}\n", + "global_results_filtered = {k: global_results_filtered[k] for k in global_results_filtered.keys() if '_time_'+str(max_time)+tabular_baselines.get_scoring_string(metric_used, usage='')+'_' in k or 'transformer' in k}\n" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [], + "source": [ + "roc_matrix, roc_matrix_stds = make_metric_matrix(global_results_filtered, methods, pos, 'roc', test_datasets_multiclass_filtered)\n", + "acc_matrix, acc_matrix_stds = make_metric_matrix(global_results_filtered, methods, pos, 'acc', test_datasets_multiclass_filtered)\n", + "cross_entropy_matrix, cross_entropy_matrix_stds = make_metric_matrix(global_results_filtered, methods, pos, 'cross_entropy', test_datasets_multiclass_filtered)\n", + "time_matrix, time_matrix_stds = make_metric_matrix(global_results_filtered, methods, pos, 'time', test_datasets_multiclass_filtered)\n", + "\n", + "roc_rank, rocs_wins = make_ranks_and_wins_table(roc_matrix.copy())\n", + "acc_rank, acc_wins = make_ranks_and_wins_table(acc_matrix.copy())\n", + "cross_entropy_rank, cross_entropy_wins = make_ranks_and_wins_table(-cross_entropy_matrix.copy())" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [], + "source": [ + "def wins_vs_idx(matrix, idx):\n", + " wins_auc = np.array([[(matrix.values[:, j] < matrix.values[:, i]).sum() if i != j else 0 for i,method in enumerate(methods)] for j in [idx]])\n", + " ties_auc = np.array([[(matrix.values[:, j] == matrix.values[:, i]).sum() if i != j else 0 for i,method in enumerate(methods)] for j in [idx]])\n", + " losses_auc = np.array([[(matrix.values[:, j] > matrix.values[:, i]).sum() if i != j else 0 for i,method in enumerate(methods)] for j in [idx]])\n", + " \n", + " return wins_auc, ties_auc, losses_auc\n", + "\n", + "transformer_idx = np.where(roc_matrix.columns == 'transformer')[0][0]\n", + "\n", + "wins_roc_vs_us, ties_roc_vs_us, losses_roc_vs_us = wins_vs_idx(roc_matrix, transformer_idx)\n", + "wins_acc_vs_us, ties_acc_vs_us, losses_acc_vs_us = wins_vs_idx(acc_matrix, transformer_idx)\n", + "wins_ce_vs_us, ties_ce_vs_us, losses_ce_vs_us = wins_vs_idx(-cross_entropy_matrix, transformer_idx)" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "metadata": {}, + "outputs": [], + "source": [ + "def rename(table):\n", + " return table.rename(columns=relabeler).T.rename(columns={'blood-transfusion-service-center': 'blood-transfus..'\n", + " , 'jungle_chess_2pcs_raw_endgame_complete': 'jungle\\_chess..', 'bank-marketing': 'bank-market..'}).T\n", + "\n", + "def get_suffix(i, k):\n", + " suffix = ''\n", + " suffix = suffix+'s' if test_datasets[i][5]['samples_capped'] == True else suffix\n", + " suffix = suffix+'f' if test_datasets[i][5]['feats_capped'] == True else suffix\n", + " suffix = suffix+'c' if test_datasets[i][5]['classes_capped'] == True else suffix\n", + " suffix = '' if len(suffix) == 0 else f' [{suffix}]'\n", + " \n", + " return k + suffix" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "metadata": {}, + "outputs": [], + "source": [ + "relabeler = {'transformer': 'Tabular PFN'\n", + " , 'autogluon': 'Autogluon'\n", + " , 'autosklearn2': 'Autosklearn2'\n", + " , 'gp': 'GP (RBF)'\n", + " , 'logistic': 'Log. Regr.'\n", + " , 'knn': 'KNN'\n", + " , 'catboost': 'Catboost'\n", + " , 'xgb': 'XGB'}" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 KNNLog. Regr.GP (RBF)CatboostXGBAutosklearn2AutogluonTabular PFN
balance-scale0.8800.9620.9830.9240.9940.9970.9920.996
mfeat-fourier0.9700.9760.9230.9820.9800.9830.9840.982
breast-w0.9840.9940.9920.9930.9900.9940.9930.993
mfeat-karhunen0.9940.9950.9150.9990.9980.9980.9990.998
mfeat-morphological0.9510.9640.9590.9630.9610.9670.9700.968
mfeat-zernike0.9760.9790.9810.9760.9730.9810.7940.983
cmc0.6340.6750.6730.7260.7300.7380.7330.726
credit-approval0.9140.9120.9210.9390.9420.9410.9410.932
credit-g0.7260.7600.7780.7820.7850.7930.7940.790
diabetes0.8060.8370.8470.8380.8380.8340.8390.842
tic-tac-toe0.9860.9940.9980.9991.0000.9941.0000.966
vehicle0.8810.9340.8980.9300.9280.9500.9420.958
eucalyptus0.7980.8850.8500.8980.9000.9130.9200.927
analcatdata_authorship1.0001.0000.9831.0001.0001.0000.9991.000
analcatdata_dmft0.5430.5660.5720.5590.5740.5750.5660.580
pc40.8220.8910.8990.9410.9290.9330.9430.939
pc30.7530.7720.7950.8250.8290.8270.8280.834
kc20.7900.8260.8280.8320.8230.8310.8240.836
pc10.7850.8210.8240.8600.8490.8530.8580.877
banknote-authentication0.9991.0001.0001.0001.0001.0001.0001.000
blood-transfus..0.7080.7420.7570.7430.7310.7500.7370.759
ilpd0.6460.7090.6630.7280.7170.7210.7230.736
qsar-biodeg0.8920.9200.9220.9220.9190.9250.9280.933
wdbc0.9910.9950.9950.9930.9900.9950.9960.996
cylinder-bands0.7810.8230.8390.8760.8780.8720.8880.846
dresses-sales0.5570.6090.6240.5700.5820.5710.5510.534
MiceProtein0.9970.9980.9881.0001.0001.0000.8011.000
car0.9240.9780.8950.9960.9950.9980.9970.997
steel-plates-fault0.9200.9350.9390.9660.9660.9690.9670.966
climate-model-simulation-crashes0.8490.9320.9410.9340.9250.9290.9390.939
Wins AUC OVO0.0000.0003.0000.0001.0004.0005.00010.000
Wins Acc.0.0002.0001.0001.0003.0004.0005.00011.000
Wins CE0.0000.0002.0001.0002.0003.0009.00012.000
Win/T/L AUC vs Us2/0/283/0/274/1/257/0/235/0/2511/1/1814/1/150/0/0
Win/T/L Acc vs Us2/0/285/0/256/1/238/0/2210/0/2011/1/1811/0/190/0/0
Win/T/L CE vs Us0/0/301/0/294/0/265/0/257/0/238/0/2214/0/160/0/0
Mean AUC OVO0.8490.8790.8730.8900.8910.8940.8820.894
Mean AUC OVO Stds0.0150.0110.0270.0110.0110.0100.0390.010
Mean Acc.0.7850.8080.7860.8180.8210.8210.8180.826
Mean Acc. Stds0.0160.0130.0580.0110.0130.0160.0400.011
Mean CE1.7910.7700.8490.7670.7580.8150.7300.731
Mean CE Stds1.1430.0240.0520.0620.0470.0600.0370.015
M. rank AUC OVO7.4335.4005.3004.0674.5333.1333.3172.817
Mean rank Acc.6.8675.0505.3004.1004.4333.9833.4502.817
Mean rank CE7.1505.1335.6174.4674.2504.6172.4332.333
Mean time (s)0.500602836.7183717.0763289.9893601.1003130.2620.688
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 88, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table = roc_matrix.copy()\n", + "#table = roc_ovr_matrix.copy()\n", + "#table = acc_matrix.copy()\n", + "#table = cross_entropy_matrix.copy()\n", + "\n", + "#table = table_acc\n", + "table.index = [get_suffix(i, k) for i, k in enumerate(table.index[0:table.shape[0]])]\n", + "\n", + "table.loc['Wins AUC OVO'] = rocs_wins.values\n", + "#table.loc['Mean AUC OVR'] = roc_ovr_matrix.mean(skipna=True)\n", + "table.loc['Wins Acc.'] = acc_wins.values\n", + "#table.loc['Mean Bal. Acc.'] = balanced_acc_matrix.mean()\n", + "table.loc['Wins CE'] = cross_entropy_wins.values\n", + "\n", + "table.loc['Win/T/L AUC vs Us'] = [\"{:d}/{:d}/{:d}\".format(w, t, l) for w,t,l in zip(wins_roc_vs_us[-1, :], ties_roc_vs_us[-1, :], losses_roc_vs_us[-1, :])]\n", + "table.loc['Win/T/L Acc vs Us'] = [\"{:d}/{:d}/{:d}\".format(w, t, l) for w,t,l in zip(wins_acc_vs_us[-1, :], ties_acc_vs_us[-1, :], losses_acc_vs_us[-1, :])]\n", + "table.loc['Win/T/L CE vs Us'] = [\"{:d}/{:d}/{:d}\".format(w, t, l) for w,t,l in zip(wins_ce_vs_us[-1, :], ties_ce_vs_us[-1, :], losses_ce_vs_us[-1, :])]\n", + "\n", + "table.loc['Mean AUC OVO'] = roc_matrix.mean(skipna=True)\n", + "table.loc['Mean AUC OVO Stds'] = roc_matrix_stds.mean(skipna=True)\n", + "\n", + "#table.loc['Mean AUC OVR'] = roc_ovr_matrix.mean(skipna=True)\n", + "table.loc['Mean Acc.'] = acc_matrix.mean()\n", + "table.loc['Mean Acc. Stds'] = acc_matrix_stds.mean(skipna=True)\n", + "\n", + "#table.loc['Mean Bal. Acc.'] = balanced_acc_matrix.mean()\n", + "table.loc['Mean CE'] = cross_entropy_matrix.mean()\n", + "table.loc['Mean CE Stds'] = cross_entropy_matrix_stds.mean()\n", + "\n", + "table.loc['M. rank AUC OVO'] = roc_rank.values\n", + "#table.loc['Mean rank AUC OVR'] = roc_ovr_rank.values\n", + "table.loc['Mean rank Acc.'] = acc_rank.values\n", + "#table.loc['Mean rank Bal. Acc.'] = balanced_acc_rank.values\n", + "table.loc['Mean rank CE'] = cross_entropy_rank.values\n", + "\n", + "table.loc['Mean time (s)'] = time_matrix.mean()\n", + "table.loc['Mean time (s)', 'knn'] = 0.5\n", + "table.loc['Mean time (s)', 'logistic'] = 60\n", + "\n", + "table = table[['knn', 'logistic', 'gp', 'catboost', 'xgb', 'autosklearn2', 'autogluon', 'transformer']]\n", + "rename(table).round(decimals=3).style.highlight_max(axis = 1, props= 'font-weight: bold;').format(precision=3)" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "metadata": {}, + "outputs": [], + "source": [ + "def bold_extreme_values(data, format_string=\"%.3g\", max_=True):\n", + " data = data.astype(float).round(3)\n", + " if max_:\n", + " extrema = data != data.max()\n", + " else:\n", + " extrema = data != data.min()\n", + " bolded = data.apply(lambda x : \"\\\\textbf{%s}\" % format_string % x)\n", + " formatted = data.apply(lambda x : format_string % x)\n", + " return formatted.where(extrema, bolded) \n", + "\n", + "def to_str(data, format_string=\"%.3g\"):\n", + " formatted = data.apply(lambda x : format_string % x)\n", + " return formatted" + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
KNNLog. Regr.GP (RBF)CatboostXGBAutosklearn2AutogluonTabular PFN
balance-scale0.880.9620.9830.9240.994\\textbf{0.997}0.9920.996
mfeat-fourier0.970.9760.9230.9820.980.983\\textbf{0.984}0.982
breast-w0.984\\textbf{0.994}0.9920.9930.99\\textbf{0.994}0.9930.993
mfeat-karhunen0.9940.9950.915\\textbf{0.999}0.9980.998\\textbf{0.999}0.998
mfeat-morphological0.9510.9640.9590.9630.9610.967\\textbf{0.97}0.968
mfeat-zernike0.9760.9790.9810.9760.9730.9810.794\\textbf{0.983}
cmc0.6340.6750.6730.7260.73\\textbf{0.738}0.7330.726
credit-approval0.9140.9120.9210.939\\textbf{0.942}0.9410.9410.932
credit-g0.7260.760.7780.7820.7850.793\\textbf{0.794}0.79
diabetes0.8060.837\\textbf{0.847}0.8380.8380.8340.8390.842
tic-tac-toe0.9860.9940.9980.999\\textbf{1}0.994\\textbf{1}0.966
vehicle0.8810.9340.8980.930.9280.950.942\\textbf{0.958}
eucalyptus0.7980.8850.850.8980.90.9130.92\\textbf{0.927}
analcatdata_authorship\\textbf{1}\\textbf{1}0.983\\textbf{1}\\textbf{1}\\textbf{1}0.999\\textbf{1}
analcatdata_dmft0.5430.5660.5720.5590.5740.5750.566\\textbf{0.58}
pc40.8220.8910.8990.9410.9290.933\\textbf{0.943}0.939
pc30.7530.7720.7950.8250.8290.8270.828\\textbf{0.834}
kc20.790.8260.8280.8320.8230.8310.824\\textbf{0.836}
pc10.7850.8210.8240.860.8490.8530.858\\textbf{0.877}
banknote-authentication0.999\\textbf{1}\\textbf{1}\\textbf{1}\\textbf{1}\\textbf{1}\\textbf{1}\\textbf{1}
blood-transfus..0.7080.7420.7570.7430.7310.750.737\\textbf{0.759}
ilpd0.6460.7090.6630.7280.7170.7210.723\\textbf{0.736}
qsar-biodeg0.8920.920.9220.9220.9190.9250.928\\textbf{0.933}
wdbc0.9910.9950.9950.9930.990.995\\textbf{0.996}\\textbf{0.996}
cylinder-bands0.7810.8230.8390.8760.8780.872\\textbf{0.888}0.846
dresses-sales0.5570.609\\textbf{0.624}0.570.5820.5710.5510.534
MiceProtein0.9970.9980.988\\textbf{1}\\textbf{1}\\textbf{1}0.801\\textbf{1}
car0.9240.9780.8950.9960.995\\textbf{0.998}0.9970.997
steel-plates-fault0.920.9350.9390.9660.966\\textbf{0.969}0.9670.966
climate-model-simulation-crashes0.8490.932\\textbf{0.941}0.9340.9250.9290.9390.939
Wins AUC OVO0030145\\textbf{10}
Wins Acc.0211345\\textbf{11}
Wins CE0021239\\textbf{12}
Win/T/L AUC vs Us2/0/283/0/274/1/257/0/235/0/2511/1/1814/1/150/0/0
Win/T/L Acc vs Us2/0/285/0/256/1/238/0/2210/0/2011/1/1811/0/190/0/0
Win/T/L CE vs Us0/0/301/0/294/0/265/0/257/0/238/0/2214/0/160/0/0
Mean AUC OVO0.849$\\pm$0.0150.879$\\pm$0.0110.873$\\pm$0.0270.89$\\pm$0.0110.891$\\pm$0.011\\textbf{0.894}$\\pm$0.010.882$\\pm$0.039\\textbf{0.894}$\\pm$0.0097
Mean Acc.0.785$\\pm$0.0160.808$\\pm$0.0130.786$\\pm$0.0580.818$\\pm$0.0110.821$\\pm$0.0130.821$\\pm$0.0160.818$\\pm$0.04\\textbf{0.826}$\\pm$0.011
Mean CE1.79$\\pm$1.10.77$\\pm$0.0240.849$\\pm$0.0520.767$\\pm$0.0620.758$\\pm$0.0470.815$\\pm$0.06\\textbf{0.73}$\\pm$0.0370.731$\\pm$0.015
M. rank AUC OVO7.435.45.34.074.533.133.32\\textbf{2.82}
Mean rank Acc.6.875.055.34.14.433.983.45\\textbf{2.82}
Mean rank CE7.155.135.624.474.254.622.43\\textbf{2.33}
Mean time (s)\\textbf{0.5}60283737173290360131300.688
\n", + "
" + ], + "text/plain": [ + " KNN Log. Regr. \\\n", + "balance-scale 0.88 0.962 \n", + "mfeat-fourier 0.97 0.976 \n", + "breast-w 0.984 \\textbf{0.994} \n", + "mfeat-karhunen 0.994 0.995 \n", + "mfeat-morphological 0.951 0.964 \n", + "mfeat-zernike 0.976 0.979 \n", + "cmc 0.634 0.675 \n", + "credit-approval 0.914 0.912 \n", + "credit-g 0.726 0.76 \n", + "diabetes 0.806 0.837 \n", + "tic-tac-toe 0.986 0.994 \n", + "vehicle 0.881 0.934 \n", + "eucalyptus 0.798 0.885 \n", + "analcatdata_authorship \\textbf{1} \\textbf{1} \n", + "analcatdata_dmft 0.543 0.566 \n", + "pc4 0.822 0.891 \n", + "pc3 0.753 0.772 \n", + "kc2 0.79 0.826 \n", + "pc1 0.785 0.821 \n", + "banknote-authentication 0.999 \\textbf{1} \n", + "blood-transfus.. 0.708 0.742 \n", + "ilpd 0.646 0.709 \n", + "qsar-biodeg 0.892 0.92 \n", + "wdbc 0.991 0.995 \n", + "cylinder-bands 0.781 0.823 \n", + "dresses-sales 0.557 0.609 \n", + "MiceProtein 0.997 0.998 \n", + "car 0.924 0.978 \n", + "steel-plates-fault 0.92 0.935 \n", + "climate-model-simulation-crashes 0.849 0.932 \n", + "Wins AUC OVO 0 0 \n", + "Wins Acc. 0 2 \n", + "Wins CE 0 0 \n", + "Win/T/L AUC vs Us 2/0/28 3/0/27 \n", + "Win/T/L Acc vs Us 2/0/28 5/0/25 \n", + "Win/T/L CE vs Us 0/0/30 1/0/29 \n", + "Mean AUC OVO 0.849$\\pm$0.015 0.879$\\pm$0.011 \n", + "Mean Acc. 0.785$\\pm$0.016 0.808$\\pm$0.013 \n", + "Mean CE 1.79$\\pm$1.1 0.77$\\pm$0.024 \n", + "M. rank AUC OVO 7.43 5.4 \n", + "Mean rank Acc. 6.87 5.05 \n", + "Mean rank CE 7.15 5.13 \n", + "Mean time (s) \\textbf{0.5} 60 \n", + "\n", + " GP (RBF) Catboost \\\n", + "balance-scale 0.983 0.924 \n", + "mfeat-fourier 0.923 0.982 \n", + "breast-w 0.992 0.993 \n", + "mfeat-karhunen 0.915 \\textbf{0.999} \n", + "mfeat-morphological 0.959 0.963 \n", + "mfeat-zernike 0.981 0.976 \n", + "cmc 0.673 0.726 \n", + "credit-approval 0.921 0.939 \n", + "credit-g 0.778 0.782 \n", + "diabetes \\textbf{0.847} 0.838 \n", + "tic-tac-toe 0.998 0.999 \n", + "vehicle 0.898 0.93 \n", + "eucalyptus 0.85 0.898 \n", + "analcatdata_authorship 0.983 \\textbf{1} \n", + "analcatdata_dmft 0.572 0.559 \n", + "pc4 0.899 0.941 \n", + "pc3 0.795 0.825 \n", + "kc2 0.828 0.832 \n", + "pc1 0.824 0.86 \n", + "banknote-authentication \\textbf{1} \\textbf{1} \n", + "blood-transfus.. 0.757 0.743 \n", + "ilpd 0.663 0.728 \n", + "qsar-biodeg 0.922 0.922 \n", + "wdbc 0.995 0.993 \n", + "cylinder-bands 0.839 0.876 \n", + "dresses-sales \\textbf{0.624} 0.57 \n", + "MiceProtein 0.988 \\textbf{1} \n", + "car 0.895 0.996 \n", + "steel-plates-fault 0.939 0.966 \n", + "climate-model-simulation-crashes \\textbf{0.941} 0.934 \n", + "Wins AUC OVO 3 0 \n", + "Wins Acc. 1 1 \n", + "Wins CE 2 1 \n", + "Win/T/L AUC vs Us 4/1/25 7/0/23 \n", + "Win/T/L Acc vs Us 6/1/23 8/0/22 \n", + "Win/T/L CE vs Us 4/0/26 5/0/25 \n", + "Mean AUC OVO 0.873$\\pm$0.027 0.89$\\pm$0.011 \n", + "Mean Acc. 0.786$\\pm$0.058 0.818$\\pm$0.011 \n", + "Mean CE 0.849$\\pm$0.052 0.767$\\pm$0.062 \n", + "M. rank AUC OVO 5.3 4.07 \n", + "Mean rank Acc. 5.3 4.1 \n", + "Mean rank CE 5.62 4.47 \n", + "Mean time (s) 2837 3717 \n", + "\n", + " XGB Autosklearn2 \\\n", + "balance-scale 0.994 \\textbf{0.997} \n", + "mfeat-fourier 0.98 0.983 \n", + "breast-w 0.99 \\textbf{0.994} \n", + "mfeat-karhunen 0.998 0.998 \n", + "mfeat-morphological 0.961 0.967 \n", + "mfeat-zernike 0.973 0.981 \n", + "cmc 0.73 \\textbf{0.738} \n", + "credit-approval \\textbf{0.942} 0.941 \n", + "credit-g 0.785 0.793 \n", + "diabetes 0.838 0.834 \n", + "tic-tac-toe \\textbf{1} 0.994 \n", + "vehicle 0.928 0.95 \n", + "eucalyptus 0.9 0.913 \n", + "analcatdata_authorship \\textbf{1} \\textbf{1} \n", + "analcatdata_dmft 0.574 0.575 \n", + "pc4 0.929 0.933 \n", + "pc3 0.829 0.827 \n", + "kc2 0.823 0.831 \n", + "pc1 0.849 0.853 \n", + "banknote-authentication \\textbf{1} \\textbf{1} \n", + "blood-transfus.. 0.731 0.75 \n", + "ilpd 0.717 0.721 \n", + "qsar-biodeg 0.919 0.925 \n", + "wdbc 0.99 0.995 \n", + "cylinder-bands 0.878 0.872 \n", + "dresses-sales 0.582 0.571 \n", + "MiceProtein \\textbf{1} \\textbf{1} \n", + "car 0.995 \\textbf{0.998} \n", + "steel-plates-fault 0.966 \\textbf{0.969} \n", + "climate-model-simulation-crashes 0.925 0.929 \n", + "Wins AUC OVO 1 4 \n", + "Wins Acc. 3 4 \n", + "Wins CE 2 3 \n", + "Win/T/L AUC vs Us 5/0/25 11/1/18 \n", + "Win/T/L Acc vs Us 10/0/20 11/1/18 \n", + "Win/T/L CE vs Us 7/0/23 8/0/22 \n", + "Mean AUC OVO 0.891$\\pm$0.011 \\textbf{0.894}$\\pm$0.01 \n", + "Mean Acc. 0.821$\\pm$0.013 0.821$\\pm$0.016 \n", + "Mean CE 0.758$\\pm$0.047 0.815$\\pm$0.06 \n", + "M. rank AUC OVO 4.53 3.13 \n", + "Mean rank Acc. 4.43 3.98 \n", + "Mean rank CE 4.25 4.62 \n", + "Mean time (s) 3290 3601 \n", + "\n", + " Autogluon \\\n", + "balance-scale 0.992 \n", + "mfeat-fourier \\textbf{0.984} \n", + "breast-w 0.993 \n", + "mfeat-karhunen \\textbf{0.999} \n", + "mfeat-morphological \\textbf{0.97} \n", + "mfeat-zernike 0.794 \n", + "cmc 0.733 \n", + "credit-approval 0.941 \n", + "credit-g \\textbf{0.794} \n", + "diabetes 0.839 \n", + "tic-tac-toe \\textbf{1} \n", + "vehicle 0.942 \n", + "eucalyptus 0.92 \n", + "analcatdata_authorship 0.999 \n", + "analcatdata_dmft 0.566 \n", + "pc4 \\textbf{0.943} \n", + "pc3 0.828 \n", + "kc2 0.824 \n", + "pc1 0.858 \n", + "banknote-authentication \\textbf{1} \n", + "blood-transfus.. 0.737 \n", + "ilpd 0.723 \n", + "qsar-biodeg 0.928 \n", + "wdbc \\textbf{0.996} \n", + "cylinder-bands \\textbf{0.888} \n", + "dresses-sales 0.551 \n", + "MiceProtein 0.801 \n", + "car 0.997 \n", + "steel-plates-fault 0.967 \n", + "climate-model-simulation-crashes 0.939 \n", + "Wins AUC OVO 5 \n", + "Wins Acc. 5 \n", + "Wins CE 9 \n", + "Win/T/L AUC vs Us 14/1/15 \n", + "Win/T/L Acc vs Us 11/0/19 \n", + "Win/T/L CE vs Us 14/0/16 \n", + "Mean AUC OVO 0.882$\\pm$0.039 \n", + "Mean Acc. 0.818$\\pm$0.04 \n", + "Mean CE \\textbf{0.73}$\\pm$0.037 \n", + "M. rank AUC OVO 3.32 \n", + "Mean rank Acc. 3.45 \n", + "Mean rank CE 2.43 \n", + "Mean time (s) 3130 \n", + "\n", + " Tabular PFN \n", + "balance-scale 0.996 \n", + "mfeat-fourier 0.982 \n", + "breast-w 0.993 \n", + "mfeat-karhunen 0.998 \n", + "mfeat-morphological 0.968 \n", + "mfeat-zernike \\textbf{0.983} \n", + "cmc 0.726 \n", + "credit-approval 0.932 \n", + "credit-g 0.79 \n", + "diabetes 0.842 \n", + "tic-tac-toe 0.966 \n", + "vehicle \\textbf{0.958} \n", + "eucalyptus \\textbf{0.927} \n", + "analcatdata_authorship \\textbf{1} \n", + "analcatdata_dmft \\textbf{0.58} \n", + "pc4 0.939 \n", + "pc3 \\textbf{0.834} \n", + "kc2 \\textbf{0.836} \n", + "pc1 \\textbf{0.877} \n", + "banknote-authentication \\textbf{1} \n", + "blood-transfus.. \\textbf{0.759} \n", + "ilpd \\textbf{0.736} \n", + "qsar-biodeg \\textbf{0.933} \n", + "wdbc \\textbf{0.996} \n", + "cylinder-bands 0.846 \n", + "dresses-sales 0.534 \n", + "MiceProtein \\textbf{1} \n", + "car 0.997 \n", + "steel-plates-fault 0.966 \n", + "climate-model-simulation-crashes 0.939 \n", + "Wins AUC OVO \\textbf{10} \n", + "Wins Acc. \\textbf{11} \n", + "Wins CE \\textbf{12} \n", + "Win/T/L AUC vs Us 0/0/0 \n", + "Win/T/L Acc vs Us 0/0/0 \n", + "Win/T/L CE vs Us 0/0/0 \n", + "Mean AUC OVO \\textbf{0.894}$\\pm$0.0097 \n", + "Mean Acc. \\textbf{0.826}$\\pm$0.011 \n", + "Mean CE 0.731$\\pm$0.015 \n", + "M. rank AUC OVO \\textbf{2.82} \n", + "Mean rank Acc. \\textbf{2.82} \n", + "Mean rank CE \\textbf{2.33} \n", + "Mean time (s) 0.688 " + ] + }, + "execution_count": 90, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "keys_max = [\"Mean rank CE\", \"Mean rank Acc.\", \"Mean rank AUC OVO\", \"Mean rank AUC OVR\", \"Mean rank Bal. Acc.\", \"Mean AUC OVO\", \"Mean Acc.\"]\n", + "keys_max = [\"Mean AUC OVO\", \"Mean Acc.\", \"Wins AUC OVO\", \"Wins Acc.\", \"Wins CE\"]\n", + "\n", + "keys_min = [\"Mean rank CE\", \"Mean rank Acc.\", \"M. rank AUC OVO\", \"Mean CE\"]\n", + "\n", + "table_latex = rename(table).copy()\n", + "\n", + "table_latex.iloc[0:30] = table_latex.iloc[0:30].apply(lambda data : bold_extreme_values(data),axis=1)\n", + "table_latex.loc[[\"Mean time (s)\"]] = table_latex.loc[[\"Mean time (s)\"]].apply(lambda data : bold_extreme_values(data, format_string=\"%.4g\", max_=False), axis=1)\n", + "table_latex.loc[keys_max] = table_latex.loc[keys_max].apply(lambda data : bold_extreme_values(data),axis=1)\n", + "table_latex.loc[keys_min] = table_latex.loc[keys_min].apply(lambda data : bold_extreme_values(data, max_=False),axis=1)\n", + "\n", + "table_latex.loc[['Mean CE Stds']] = table_latex.loc[['Mean CE Stds']].apply(lambda data : to_str(data, format_string=\"%.2g\"),axis=1)\n", + "table_latex.loc['Mean CE'] = table_latex.loc['Mean CE'] + '$\\pm$' + table_latex.loc['Mean CE Stds']\n", + "table_latex = table_latex.drop(['Mean CE Stds'])\n", + "\n", + "table_latex.loc[['Mean Acc. Stds']] = table_latex.loc[['Mean Acc. Stds']].apply(lambda data : to_str(data, format_string=\"%.2g\"),axis=1)\n", + "table_latex.loc['Mean Acc.'] = table_latex.loc['Mean Acc.'] + '$\\pm$' + table_latex.loc['Mean Acc. Stds']\n", + "table_latex = table_latex.drop(['Mean Acc. Stds'])\n", + "\n", + "table_latex.loc[['Mean AUC OVO Stds']] = table_latex.loc[['Mean AUC OVO Stds']].apply(lambda data : to_str(data, format_string=\"%.2g\"),axis=1)\n", + "table_latex.loc['Mean AUC OVO'] = table_latex.loc['Mean AUC OVO'] + '$\\pm$' + table_latex.loc['Mean AUC OVO Stds']\n", + "table_latex = table_latex.drop(['Mean AUC OVO Stds'])\n", + "\n", + "table_latex\n", + "#print(table_latex.to_latex(escape=False))" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\\begin{tabular}{lllllllll}\n", + "\\toprule\n", + "{} & KNN & Log. Regr. & GP (RBF) & Catboost & XGB & Autosklearn2 & Autogluon & Tabular PFN \\\\\n", + "\\midrule\n", + "balance-scale & 0.88 & 0.962 & 0.983 & 0.924 & 0.994 & \\textbf{0.997} & 0.992 & 0.996 \\\\\n", + "mfeat-fourier & 0.97 & 0.976 & 0.923 & 0.982 & 0.98 & 0.983 & \\textbf{0.984} & 0.982 \\\\\n", + "breast-w & 0.984 & \\textbf{0.994} & 0.992 & 0.993 & 0.99 & \\textbf{0.994} & 0.993 & 0.993 \\\\\n", + "mfeat-karhunen & 0.994 & 0.995 & 0.915 & \\textbf{0.999} & 0.998 & 0.998 & \\textbf{0.999} & 0.998 \\\\\n", + "mfeat-morphological & 0.951 & 0.964 & 0.959 & 0.963 & 0.961 & 0.967 & \\textbf{0.97} & 0.968 \\\\\n", + "mfeat-zernike & 0.976 & 0.979 & 0.981 & 0.976 & 0.973 & 0.981 & 0.794 & \\textbf{0.983} \\\\\n", + "cmc & 0.634 & 0.675 & 0.673 & 0.726 & 0.73 & \\textbf{0.738} & 0.733 & 0.726 \\\\\n", + "credit-approval & 0.914 & 0.912 & 0.921 & 0.939 & \\textbf{0.942} & 0.941 & 0.941 & 0.932 \\\\\n", + "credit-g & 0.726 & 0.76 & 0.778 & 0.782 & 0.785 & 0.793 & \\textbf{0.794} & 0.79 \\\\\n", + "diabetes & 0.806 & 0.837 & \\textbf{0.847} & 0.838 & 0.838 & 0.834 & 0.839 & 0.842 \\\\\n", + "tic-tac-toe & 0.986 & 0.994 & 0.998 & 0.999 & \\textbf{1} & 0.994 & \\textbf{1} & 0.966 \\\\\n", + "vehicle & 0.881 & 0.934 & 0.898 & 0.93 & 0.928 & 0.95 & 0.942 & \\textbf{0.958} \\\\\n", + "eucalyptus & 0.798 & 0.885 & 0.85 & 0.898 & 0.9 & 0.913 & 0.92 & \\textbf{0.927} \\\\\n", + "analcatdata_authorship & \\textbf{1} & \\textbf{1} & 0.983 & \\textbf{1} & \\textbf{1} & \\textbf{1} & 0.999 & \\textbf{1} \\\\\n", + "analcatdata_dmft & 0.543 & 0.566 & 0.572 & 0.559 & 0.574 & 0.575 & 0.566 & \\textbf{0.58} \\\\\n", + "pc4 & 0.822 & 0.891 & 0.899 & 0.941 & 0.929 & 0.933 & \\textbf{0.943} & 0.939 \\\\\n", + "pc3 & 0.753 & 0.772 & 0.795 & 0.825 & 0.829 & 0.827 & 0.828 & \\textbf{0.834} \\\\\n", + "kc2 & 0.79 & 0.826 & 0.828 & 0.832 & 0.823 & 0.831 & 0.824 & \\textbf{0.836} \\\\\n", + "pc1 & 0.785 & 0.821 & 0.824 & 0.86 & 0.849 & 0.853 & 0.858 & \\textbf{0.877} \\\\\n", + "banknote-authentication & 0.999 & \\textbf{1} & \\textbf{1} & \\textbf{1} & \\textbf{1} & \\textbf{1} & \\textbf{1} & \\textbf{1} \\\\\n", + "blood-transfus.. & 0.708 & 0.742 & 0.757 & 0.743 & 0.731 & 0.75 & 0.737 & \\textbf{0.759} \\\\\n", + "ilpd & 0.646 & 0.709 & 0.663 & 0.728 & 0.717 & 0.721 & 0.723 & \\textbf{0.736} \\\\\n", + "qsar-biodeg & 0.892 & 0.92 & 0.922 & 0.922 & 0.919 & 0.925 & 0.928 & \\textbf{0.933} \\\\\n", + "wdbc & 0.991 & 0.995 & 0.995 & 0.993 & 0.99 & 0.995 & \\textbf{0.996} & \\textbf{0.996} \\\\\n", + "cylinder-bands & 0.781 & 0.823 & 0.839 & 0.876 & 0.878 & 0.872 & \\textbf{0.888} & 0.846 \\\\\n", + "dresses-sales & 0.557 & 0.609 & \\textbf{0.624} & 0.57 & 0.582 & 0.571 & 0.551 & 0.534 \\\\\n", + "MiceProtein & 0.997 & 0.998 & 0.988 & \\textbf{1} & \\textbf{1} & \\textbf{1} & 0.801 & \\textbf{1} \\\\\n", + "car & 0.924 & 0.978 & 0.895 & 0.996 & 0.995 & \\textbf{0.998} & 0.997 & 0.997 \\\\\n", + "steel-plates-fault & 0.92 & 0.935 & 0.939 & 0.966 & 0.966 & \\textbf{0.969} & 0.967 & 0.966 \\\\\n", + "climate-model-simulation-crashes & 0.849 & 0.932 & \\textbf{0.941} & 0.934 & 0.925 & 0.929 & 0.939 & 0.939 \\\\\n", + "Wins AUC OVO & 0 & 0 & 3 & 0 & 1 & 4 & 5 & \\textbf{10} \\\\\n", + "Wins Acc. & 0 & 2 & 1 & 1 & 3 & 4 & 5 & \\textbf{11} \\\\\n", + "Wins CE & 0 & 0 & 2 & 1 & 2 & 3 & 9 & \\textbf{12} \\\\\n", + "Win/T/L AUC vs Us & 2/0/28 & 3/0/27 & 4/1/25 & 7/0/23 & 5/0/25 & 11/1/18 & 14/1/15 & 0/0/0 \\\\\n", + "Win/T/L Acc vs Us & 2/0/28 & 5/0/25 & 6/1/23 & 8/0/22 & 10/0/20 & 11/1/18 & 11/0/19 & 0/0/0 \\\\\n", + "Win/T/L CE vs Us & 0/0/30 & 1/0/29 & 4/0/26 & 5/0/25 & 7/0/23 & 8/0/22 & 14/0/16 & 0/0/0 \\\\\n", + "Mean AUC OVO & 0.849$\\pm$0.015 & 0.879$\\pm$0.011 & 0.873$\\pm$0.027 & 0.89$\\pm$0.011 & 0.891$\\pm$0.011 & \\textbf{0.894}$\\pm$0.01 & 0.882$\\pm$0.039 & \\textbf{0.894}$\\pm$0.0097 \\\\\n", + "Mean Acc. & 0.785$\\pm$0.016 & 0.808$\\pm$0.013 & 0.786$\\pm$0.058 & 0.818$\\pm$0.011 & 0.821$\\pm$0.013 & 0.821$\\pm$0.016 & 0.818$\\pm$0.04 & \\textbf{0.826}$\\pm$0.011 \\\\\n", + "Mean CE & 1.79$\\pm$1.1 & 0.77$\\pm$0.024 & 0.849$\\pm$0.052 & 0.767$\\pm$0.062 & 0.758$\\pm$0.047 & 0.815$\\pm$0.06 & \\textbf{0.73}$\\pm$0.037 & 0.731$\\pm$0.015 \\\\\n", + "M. rank AUC OVO & 7.43 & 5.4 & 5.3 & 4.07 & 4.53 & 3.13 & 3.32 & \\textbf{2.82} \\\\\n", + "Mean rank Acc. & 6.87 & 5.05 & 5.3 & 4.1 & 4.43 & 3.98 & 3.45 & \\textbf{2.82} \\\\\n", + "Mean rank CE & 7.15 & 5.13 & 5.62 & 4.47 & 4.25 & 4.62 & 2.43 & \\textbf{2.33} \\\\\n", + "Mean time (s) & \\textbf{0.5} & 60 & 2837 & 3717 & 3290 & 3601 & 3130 & 0.688 \\\\\n", + "\\bottomrule\n", + "\\end{tabular}\n", + "\n" + ] + } + ], + "source": [ + "print(table_latex.to_latex(escape=False))" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
KNNLog. Regr.GP (RBF)CatboostXGBAutosklearn2AutogluonTabular PFN
Wins AUC OVO0030145\\textbf{10}
Wins Acc.0211345\\textbf{11}
Wins CE0021239\\textbf{12}
Win/T/L AUC vs Us2/0/283/0/274/1/257/0/235/0/2511/1/1814/1/150/0/0
Win/T/L Acc vs Us2/0/285/0/256/1/238/0/2210/0/2011/1/1811/0/190/0/0
Win/T/L CE vs Us0/0/301/0/294/0/265/0/257/0/238/0/2214/0/160/0/0
Mean AUC OVO0.849$\\pm$0.0150.879$\\pm$0.0110.873$\\pm$0.0270.89$\\pm$0.0110.891$\\pm$0.011\\textbf{0.894}$\\pm$0.010.882$\\pm$0.039\\textbf{0.894}$\\pm$0.0097
Mean Acc.0.785$\\pm$0.0160.808$\\pm$0.0130.786$\\pm$0.0580.818$\\pm$0.0110.821$\\pm$0.0130.821$\\pm$0.0160.818$\\pm$0.04\\textbf{0.826}$\\pm$0.011
Mean CE1.79$\\pm$1.10.77$\\pm$0.0240.849$\\pm$0.0520.767$\\pm$0.0620.758$\\pm$0.0470.815$\\pm$0.06\\textbf{0.73}$\\pm$0.0370.731$\\pm$0.015
M. rank AUC OVO7.435.45.34.074.533.133.32\\textbf{2.82}
Mean rank Acc.6.875.055.34.14.433.983.45\\textbf{2.82}
Mean rank CE7.155.135.624.474.254.622.43\\textbf{2.33}
Mean time (s)\\textbf{0.5}60283737173290360131300.688
\n", + "
" + ], + "text/plain": [ + " KNN Log. Regr. GP (RBF) \\\n", + "Wins AUC OVO 0 0 3 \n", + "Wins Acc. 0 2 1 \n", + "Wins CE 0 0 2 \n", + "Win/T/L AUC vs Us 2/0/28 3/0/27 4/1/25 \n", + "Win/T/L Acc vs Us 2/0/28 5/0/25 6/1/23 \n", + "Win/T/L CE vs Us 0/0/30 1/0/29 4/0/26 \n", + "Mean AUC OVO 0.849$\\pm$0.015 0.879$\\pm$0.011 0.873$\\pm$0.027 \n", + "Mean Acc. 0.785$\\pm$0.016 0.808$\\pm$0.013 0.786$\\pm$0.058 \n", + "Mean CE 1.79$\\pm$1.1 0.77$\\pm$0.024 0.849$\\pm$0.052 \n", + "M. rank AUC OVO 7.43 5.4 5.3 \n", + "Mean rank Acc. 6.87 5.05 5.3 \n", + "Mean rank CE 7.15 5.13 5.62 \n", + "Mean time (s) \\textbf{0.5} 60 2837 \n", + "\n", + " Catboost XGB Autosklearn2 \\\n", + "Wins AUC OVO 0 1 4 \n", + "Wins Acc. 1 3 4 \n", + "Wins CE 1 2 3 \n", + "Win/T/L AUC vs Us 7/0/23 5/0/25 11/1/18 \n", + "Win/T/L Acc vs Us 8/0/22 10/0/20 11/1/18 \n", + "Win/T/L CE vs Us 5/0/25 7/0/23 8/0/22 \n", + "Mean AUC OVO 0.89$\\pm$0.011 0.891$\\pm$0.011 \\textbf{0.894}$\\pm$0.01 \n", + "Mean Acc. 0.818$\\pm$0.011 0.821$\\pm$0.013 0.821$\\pm$0.016 \n", + "Mean CE 0.767$\\pm$0.062 0.758$\\pm$0.047 0.815$\\pm$0.06 \n", + "M. rank AUC OVO 4.07 4.53 3.13 \n", + "Mean rank Acc. 4.1 4.43 3.98 \n", + "Mean rank CE 4.47 4.25 4.62 \n", + "Mean time (s) 3717 3290 3601 \n", + "\n", + " Autogluon Tabular PFN \n", + "Wins AUC OVO 5 \\textbf{10} \n", + "Wins Acc. 5 \\textbf{11} \n", + "Wins CE 9 \\textbf{12} \n", + "Win/T/L AUC vs Us 14/1/15 0/0/0 \n", + "Win/T/L Acc vs Us 11/0/19 0/0/0 \n", + "Win/T/L CE vs Us 14/0/16 0/0/0 \n", + "Mean AUC OVO 0.882$\\pm$0.039 \\textbf{0.894}$\\pm$0.0097 \n", + "Mean Acc. 0.818$\\pm$0.04 \\textbf{0.826}$\\pm$0.011 \n", + "Mean CE \\textbf{0.73}$\\pm$0.037 0.731$\\pm$0.015 \n", + "M. rank AUC OVO 3.32 \\textbf{2.82} \n", + "Mean rank Acc. 3.45 \\textbf{2.82} \n", + "Mean rank CE 2.43 \\textbf{2.33} \n", + "Mean time (s) 3130 0.688 " + ] + }, + "execution_count": 92, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table_latex_small = table_latex.iloc[-len(keys_min+keys_max)-1-3:]\n", + "table_latex_small" + ] + }, + { + "cell_type": "code", + "execution_count": 438, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\\begin{tabular}{lllllllll}\n", + "\\toprule\n", + "{} & KNN & Log. Regr. & GP (RBF) & Catboost & XGB & Autosklearn2 & Autogluon & Tabular PFN \\\\\n", + "\\midrule\n", + "Wins AUC OVO & 0 & 0 & 2 & 0 & 1 & 5 & 5 & \\textbf{8} \\\\\n", + "Wins Acc. & 0 & 3 & 1 & 2 & 1 & 4 & 6 & \\textbf{10} \\\\\n", + "Wins CE & 0 & 0 & 1 & 0 & 2 & 1 & \\textbf{13} & 11 \\\\\n", + "Win/T/L AUC vs Us & 2/0/28 & 3/0/27 & 3/1/26 & 7/1/22 & 5/0/25 & 11/1/18 & 13/1/16 & 0/0/0 \\\\\n", + "Win/T/L Acc vs Us & 2/0/28 & 6/0/24 & 4/1/25 & 10/0/20 & 7/0/23 & 11/0/19 & 11/0/19 & 0/0/0 \\\\\n", + "Win/T/L CE vs Us & 0/0/30 & 1/0/29 & 3/0/27 & 4/0/26 & 4/0/26 & 6/0/24 & 15/0/15 & 0/0/0 \\\\\n", + "Mean AUC OVO & 0.849$\\pm$0.015 & 0.88$\\pm$0.012 & 0.873$\\pm$0.024 & 0.888$\\pm$0.012 & 0.89$\\pm$0.012 & \\textbf{0.895}$\\pm$0.0097 & 0.889$\\pm$0.022 & 0.894$\\pm$0.0097 \\\\\n", + "Mean Acc. & 0.785$\\pm$0.016 & 0.809$\\pm$0.014 & 0.774$\\pm$0.075 & 0.818$\\pm$0.013 & 0.82$\\pm$0.014 & 0.818$\\pm$0.025 & 0.825$\\pm$0.021 & \\textbf{0.826}$\\pm$0.011 \\\\\n", + "Mean CE & 1.79$\\pm$1.1 & 0.769$\\pm$0.027 & 0.859$\\pm$0.068 & 0.774$\\pm$0.073 & 0.763$\\pm$0.059 & 0.839$\\pm$0.065 & \\textbf{0.72}$\\pm$0.019 & 0.731$\\pm$0.015 \\\\\n", + "M. rank AUC OVO & 7.53 & 5.28 & 5.55 & 4.22 & 4.72 & \\textbf{2.75} & 3.13 & 2.82 \\\\\n", + "Mean rank Acc. & 6.78 & 4.98 & 5.67 & 4.22 & 4.53 & 4.08 & 2.97 & \\textbf{2.77} \\\\\n", + "Mean rank CE & 7.17 & 5.17 & 5.77 & 4.52 & 4.7 & 4.77 & \\textbf{1.78} & 2.13 \\\\\n", + "Mean time & \\textbf{0.5} & 60 & 941.1 & 1121 & 921.9 & 901.5 & 809.1 & 0.688 \\\\\n", + "\\bottomrule\n", + "\\end{tabular}\n", + "\n" + ] + } + ], + "source": [ + "print(table_latex_small.to_latex(escape=False))" + ] + }, + { + "cell_type": "code", + "execution_count": 1130, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
logisticgpknncatboostxgbautosklearn2autogluontabnettransformer
balance-scale0.9620.9830.8800.9260.979\\textbf{0.997}0.9930.9180.996
mfeat-fourier0.9760.9230.9700.9810.9800.983\\textbf{0.985}0.8990.980
breast-w\\textbf{0.994}0.9920.9840.9930.992\\textbf{0.994}0.9930.9830.992
mfeat-karhunen0.9950.9150.994\\textbf{0.999}0.9980.998\\textbf{0.999}0.9870.998
mfeat-morphological0.9640.9590.9510.9630.9610.967\\textbf{0.970}0.8860.967
mfeat-zernike0.9790.9810.9760.9760.9740.981\\textbf{0.991}0.9520.983
cmc0.6750.6730.6340.7260.724\\textbf{0.738}0.7340.6590.727
credit-approval0.9120.9210.9140.9390.9400.941\\textbf{0.943}0.8520.932
credit-g0.7600.7780.7260.7820.780\\textbf{0.793}0.7920.7100.787
diabetes0.837\\textbf{0.847}0.8060.8380.8310.8340.8370.7480.843
tic-tac-toe0.9940.9980.9860.9990.9960.994\\textbf{1.000}0.8680.958
vehicle0.9340.8980.8810.9270.9270.9500.9410.635\\textbf{0.958}
eucalyptus0.8850.8500.7980.8960.9000.9130.9200.675\\textbf{0.925}
analcatdata_authorship\\textbf{1.000}0.983\\textbf{1.000}\\textbf{1.000}\\textbf{1.000}\\textbf{1.000}\\textbf{1.000}0.986\\textbf{1.000}
analcatdata_dmft0.5660.5720.5430.5630.5760.5750.5710.550\\textbf{0.584}
pc40.8910.8990.8220.9370.9310.933\\textbf{0.941}0.8480.935
pc30.7720.7950.7530.8240.8230.8270.8300.802\\textbf{0.834}
kc20.8260.8280.7900.8260.8280.8310.8290.754\\textbf{0.832}
pc10.8210.8240.7850.8400.8480.8530.8650.759\\textbf{0.867}
banknote-authentication\\textbf{1.000}\\textbf{1.000}0.999\\textbf{1.000}0.963\\textbf{1.000}\\textbf{1.000}\\textbf{1.000}\\textbf{1.000}
blood-transfusion-service-center0.7420.7570.7080.7400.7330.7500.7420.669\\textbf{0.762}
ilpd0.7090.6630.6460.7210.7170.7210.7260.683\\textbf{0.734}
qsar-biodeg0.9200.9220.8920.9210.9210.9250.9260.804\\textbf{0.928}
wdbc0.9950.9950.9910.9940.9930.995\\textbf{0.996}0.962\\textbf{0.996}
cylinder-bands0.8230.8390.7810.8680.8760.872\\textbf{0.879}0.5560.849
dresses-sales0.609\\textbf{0.624}0.5570.5890.5760.5710.5560.5040.536
MiceProtein0.9980.9880.997\\textbf{1.000}\\textbf{1.000}\\textbf{1.000}\\textbf{1.000}0.889\\textbf{1.000}
car0.9780.8950.9240.9960.995\\textbf{0.998}\\textbf{0.998}0.9810.996
steel-plates-fault0.9350.9390.9200.9650.964\\textbf{0.969}0.9670.8350.965
climate-model-simulation-crashes0.932\\textbf{0.941}0.8490.9330.9180.9290.9310.8370.939
Mean AUC OVO0.8794450.8727120.8485960.8887290.8881260.8943360.8950640.8063750.893478
Mean CE0.7695420.848521.7905640.770230.8954840.8149980.717031.1973070.73392
Mean rank AUC5.555.4666677.7833334.255.0333333.0333332.5666678.3666672.95
Mean rank CE4.9333334.2666672.3166675.354.95.258.4833342.2833337.216667
Mean time1210.628412836.718452695.035413546.8573351525.185583601.1001623144.0609553674.0929020.678991
\n", + "
" + ], + "text/plain": [ + " logistic gp \\\n", + "balance-scale 0.962 0.983 \n", + "mfeat-fourier 0.976 0.923 \n", + "breast-w \\textbf{0.994} 0.992 \n", + "mfeat-karhunen 0.995 0.915 \n", + "mfeat-morphological 0.964 0.959 \n", + "mfeat-zernike 0.979 0.981 \n", + "cmc 0.675 0.673 \n", + "credit-approval 0.912 0.921 \n", + "credit-g 0.760 0.778 \n", + "diabetes 0.837 \\textbf{0.847} \n", + "tic-tac-toe 0.994 0.998 \n", + "vehicle 0.934 0.898 \n", + "eucalyptus 0.885 0.850 \n", + "analcatdata_authorship \\textbf{1.000} 0.983 \n", + "analcatdata_dmft 0.566 0.572 \n", + "pc4 0.891 0.899 \n", + "pc3 0.772 0.795 \n", + "kc2 0.826 0.828 \n", + "pc1 0.821 0.824 \n", + "banknote-authentication \\textbf{1.000} \\textbf{1.000} \n", + "blood-transfusion-service-center 0.742 0.757 \n", + "ilpd 0.709 0.663 \n", + "qsar-biodeg 0.920 0.922 \n", + "wdbc 0.995 0.995 \n", + "cylinder-bands 0.823 0.839 \n", + "dresses-sales 0.609 \\textbf{0.624} \n", + "MiceProtein 0.998 0.988 \n", + "car 0.978 0.895 \n", + "steel-plates-fault 0.935 0.939 \n", + "climate-model-simulation-crashes 0.932 \\textbf{0.941} \n", + "Mean AUC OVO 0.879445 0.872712 \n", + "Mean CE 0.769542 0.84852 \n", + "Mean rank AUC 5.55 5.466667 \n", + "Mean rank CE 4.933333 4.266667 \n", + "Mean time 1210.62841 2836.718452 \n", + "\n", + " knn catboost \\\n", + "balance-scale 0.880 0.926 \n", + "mfeat-fourier 0.970 0.981 \n", + "breast-w 0.984 0.993 \n", + "mfeat-karhunen 0.994 \\textbf{0.999} \n", + "mfeat-morphological 0.951 0.963 \n", + "mfeat-zernike 0.976 0.976 \n", + "cmc 0.634 0.726 \n", + "credit-approval 0.914 0.939 \n", + "credit-g 0.726 0.782 \n", + "diabetes 0.806 0.838 \n", + "tic-tac-toe 0.986 0.999 \n", + "vehicle 0.881 0.927 \n", + "eucalyptus 0.798 0.896 \n", + "analcatdata_authorship \\textbf{1.000} \\textbf{1.000} \n", + "analcatdata_dmft 0.543 0.563 \n", + "pc4 0.822 0.937 \n", + "pc3 0.753 0.824 \n", + "kc2 0.790 0.826 \n", + "pc1 0.785 0.840 \n", + "banknote-authentication 0.999 \\textbf{1.000} \n", + "blood-transfusion-service-center 0.708 0.740 \n", + "ilpd 0.646 0.721 \n", + "qsar-biodeg 0.892 0.921 \n", + "wdbc 0.991 0.994 \n", + "cylinder-bands 0.781 0.868 \n", + "dresses-sales 0.557 0.589 \n", + "MiceProtein 0.997 \\textbf{1.000} \n", + "car 0.924 0.996 \n", + "steel-plates-fault 0.920 0.965 \n", + "climate-model-simulation-crashes 0.849 0.933 \n", + "Mean AUC OVO 0.848596 0.888729 \n", + "Mean CE 1.790564 0.77023 \n", + "Mean rank AUC 7.783333 4.25 \n", + "Mean rank CE 2.316667 5.35 \n", + "Mean time 695.03541 3546.857335 \n", + "\n", + " xgb autosklearn2 \\\n", + "balance-scale 0.979 \\textbf{0.997} \n", + "mfeat-fourier 0.980 0.983 \n", + "breast-w 0.992 \\textbf{0.994} \n", + "mfeat-karhunen 0.998 0.998 \n", + "mfeat-morphological 0.961 0.967 \n", + "mfeat-zernike 0.974 0.981 \n", + "cmc 0.724 \\textbf{0.738} \n", + "credit-approval 0.940 0.941 \n", + "credit-g 0.780 \\textbf{0.793} \n", + "diabetes 0.831 0.834 \n", + "tic-tac-toe 0.996 0.994 \n", + "vehicle 0.927 0.950 \n", + "eucalyptus 0.900 0.913 \n", + "analcatdata_authorship \\textbf{1.000} \\textbf{1.000} \n", + "analcatdata_dmft 0.576 0.575 \n", + "pc4 0.931 0.933 \n", + "pc3 0.823 0.827 \n", + "kc2 0.828 0.831 \n", + "pc1 0.848 0.853 \n", + "banknote-authentication 0.963 \\textbf{1.000} \n", + "blood-transfusion-service-center 0.733 0.750 \n", + "ilpd 0.717 0.721 \n", + "qsar-biodeg 0.921 0.925 \n", + "wdbc 0.993 0.995 \n", + "cylinder-bands 0.876 0.872 \n", + "dresses-sales 0.576 0.571 \n", + "MiceProtein \\textbf{1.000} \\textbf{1.000} \n", + "car 0.995 \\textbf{0.998} \n", + "steel-plates-fault 0.964 \\textbf{0.969} \n", + "climate-model-simulation-crashes 0.918 0.929 \n", + "Mean AUC OVO 0.888126 0.894336 \n", + "Mean CE 0.895484 0.814998 \n", + "Mean rank AUC 5.033333 3.033333 \n", + "Mean rank CE 4.9 5.25 \n", + "Mean time 1525.18558 3601.100162 \n", + "\n", + " autogluon tabnet \\\n", + "balance-scale 0.993 0.918 \n", + "mfeat-fourier \\textbf{0.985} 0.899 \n", + "breast-w 0.993 0.983 \n", + "mfeat-karhunen \\textbf{0.999} 0.987 \n", + "mfeat-morphological \\textbf{0.970} 0.886 \n", + "mfeat-zernike \\textbf{0.991} 0.952 \n", + "cmc 0.734 0.659 \n", + "credit-approval \\textbf{0.943} 0.852 \n", + "credit-g 0.792 0.710 \n", + "diabetes 0.837 0.748 \n", + "tic-tac-toe \\textbf{1.000} 0.868 \n", + "vehicle 0.941 0.635 \n", + "eucalyptus 0.920 0.675 \n", + "analcatdata_authorship \\textbf{1.000} 0.986 \n", + "analcatdata_dmft 0.571 0.550 \n", + "pc4 \\textbf{0.941} 0.848 \n", + "pc3 0.830 0.802 \n", + "kc2 0.829 0.754 \n", + "pc1 0.865 0.759 \n", + "banknote-authentication \\textbf{1.000} \\textbf{1.000} \n", + "blood-transfusion-service-center 0.742 0.669 \n", + "ilpd 0.726 0.683 \n", + "qsar-biodeg 0.926 0.804 \n", + "wdbc \\textbf{0.996} 0.962 \n", + "cylinder-bands \\textbf{0.879} 0.556 \n", + "dresses-sales 0.556 0.504 \n", + "MiceProtein \\textbf{1.000} 0.889 \n", + "car \\textbf{0.998} 0.981 \n", + "steel-plates-fault 0.967 0.835 \n", + "climate-model-simulation-crashes 0.931 0.837 \n", + "Mean AUC OVO 0.895064 0.806375 \n", + "Mean CE 0.71703 1.197307 \n", + "Mean rank AUC 2.566667 8.366667 \n", + "Mean rank CE 8.483334 2.283333 \n", + "Mean time 3144.060955 3674.092902 \n", + "\n", + " transformer \n", + "balance-scale 0.996 \n", + "mfeat-fourier 0.980 \n", + "breast-w 0.992 \n", + "mfeat-karhunen 0.998 \n", + "mfeat-morphological 0.967 \n", + "mfeat-zernike 0.983 \n", + "cmc 0.727 \n", + "credit-approval 0.932 \n", + "credit-g 0.787 \n", + "diabetes 0.843 \n", + "tic-tac-toe 0.958 \n", + "vehicle \\textbf{0.958} \n", + "eucalyptus \\textbf{0.925} \n", + "analcatdata_authorship \\textbf{1.000} \n", + "analcatdata_dmft \\textbf{0.584} \n", + "pc4 0.935 \n", + "pc3 \\textbf{0.834} \n", + "kc2 \\textbf{0.832} \n", + "pc1 \\textbf{0.867} \n", + "banknote-authentication \\textbf{1.000} \n", + "blood-transfusion-service-center \\textbf{0.762} \n", + "ilpd \\textbf{0.734} \n", + "qsar-biodeg \\textbf{0.928} \n", + "wdbc \\textbf{0.996} \n", + "cylinder-bands 0.849 \n", + "dresses-sales 0.536 \n", + "MiceProtein \\textbf{1.000} \n", + "car 0.996 \n", + "steel-plates-fault 0.965 \n", + "climate-model-simulation-crashes 0.939 \n", + "Mean AUC OVO 0.893478 \n", + "Mean CE 0.73392 \n", + "Mean rank AUC 2.95 \n", + "Mean rank CE 7.216667 \n", + "Mean time 0.678991 " + ] + }, + "execution_count": 1130, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table_latex = table.copy()\n", + "\n", + "table_latex.iloc[:-5] = table_latex.iloc[:-5].apply(lambda data : bold_extreme_values(data),axis=1)\n", + "table_latex.iloc[-5:-5] = table_latex.iloc[-5:-5].apply(lambda data : bold_extreme_values(data, max_=False),axis=1)\n", + "\n", + "table_latex\n", + "#print(table_latex.to_latex(escape=False))" + ] + }, + { + "cell_type": "code", + "execution_count": 230, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 Log. Regr.GPKNNCatboostXGBautosklearnautosklearn_fasth2oh2o_fasttransformerdiff
Mean Bal. Acc.0.4610.5930.6280.6840.6450.6830.6710.6930.7010.6690.015
Mean CE0.9890.7871.2230.8080.8441.0110.9100.9350.8330.820-0.011
Mean rank AUC7.8896.7088.3754.2084.6394.1394.7084.2785.0145.042-0.833
Mean rank AUC OVR7.8896.7088.4314.1534.5694.2504.7644.2504.9445.042-0.889
Mean rank Acc.8.0697.3197.5834.1114.5563.7644.4034.4585.1395.597-1.486
Mean rank Bal. Acc.8.3067.2927.3193.8194.9173.8474.5564.3615.0695.514-1.694
Mean rank CE5.5005.5005.5005.5005.5005.5005.5005.5005.5005.5000.000
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 230, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rename(table[-7:]).round(decimals=3).style.highlight_min(axis = 1, props= 'font-weight: bold;').format(precision=3)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/TabPFN/TrainingTuningAndPrediction.ipynb b/TabPFN/TrainingTuningAndPrediction.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..791a17a9d1dca3ca44b09ebaa48188508544e318 --- /dev/null +++ b/TabPFN/TrainingTuningAndPrediction.ipynb @@ -0,0 +1,1925 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import time\n", + "import warnings\n", + "from datetime import datetime\n", + "\n", + "import torch\n", + "\n", + "import numpy as np\n", + "\n", + "import matplotlib.pyplot as plt\n", + "from scripts.differentiable_pfn_evaluation import eval_model_range\n", + "from model_builder import get_model, get_default_spec, save_model, load_model\n", + "from scripts.transformer_prediction_interface import transformer_predict, get_params_from_config, load_model_workflow\n", + "\n", + "from scripts.model_configs import *\n", + "\n", + "from datasets import load_openml_list, open_cc_dids, open_cc_valid_dids\n", + "from priors.utils import plot_prior, plot_features\n", + "from priors.utils import uniform_int_sampler_f\n", + "\n", + "from scripts.tabular_metrics import calculate_score_per_method, calculate_score\n", + "from scripts.tabular_evaluation import evaluate\n", + "\n", + "from priors.differentiable_prior import DifferentiableHyperparameterList, draw_random_style, merge_style_with_info\n", + "from scripts import tabular_metrics\n", + "from notebook_utils import *" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "large_datasets = True\n", + "max_samples = 10000 if large_datasets else 5000\n", + "bptt = 10000 if large_datasets else 3000\n", + "suite='cc'" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "device = 'cpu'\n", + "base_path = '.'\n", + "max_features = 100" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def print_models(model_string):\n", + " print(model_string)\n", + "\n", + " for i in range(80):\n", + " for e in range(50):\n", + " exists = Path(os.path.join(base_path, f'models_diff/prior_diff_real_checkpoint{model_string}_n_{i}_epoch_{e}.cpkt')).is_file()\n", + " if exists:\n", + " print(os.path.join(base_path, f'models_diff/prior_diff_real_checkpoint{model_string}_n_{i}_epoch_{e}.cpkt'))\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "def train_function(config_sample, i, add_name=''):\n", + " start_time = time.time()\n", + " N_epochs_to_save = 50\n", + " \n", + " def save_callback(model, epoch):\n", + " if not hasattr(model, 'last_saved_epoch'):\n", + " model.last_saved_epoch = 0\n", + " if ((time.time() - start_time) / (maximum_runtime * 60 / N_epochs_to_save)) > model.last_saved_epoch:\n", + " print('Saving model..')\n", + " config_sample['epoch_in_training'] = epoch\n", + " save_model(model, base_path, f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{model.last_saved_epoch}.cpkt',\n", + " config_sample)\n", + " model.last_saved_epoch = model.last_saved_epoch + 1 # TODO: Rename to checkpoint\n", + " \n", + " model = get_model(config_sample\n", + " , device\n", + " , should_train=True\n", + " , verbose=1\n", + " , epoch_callback = save_callback)\n", + " \n", + " return" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Datasets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "cc_test_datasets_multiclass, cc_test_datasets_multiclass_df = load_openml_list(open_cc_dids, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = max_samples, num_feats=100, return_capped=True)\n", + "cc_valid_datasets_multiclass, cc_valid_datasets_multiclass_df = load_openml_list(open_cc_valid_dids, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = max_samples, num_feats=100, return_capped=True)\n", + "\n", + "# Loading longer OpenML Datasets for generalization experiments (optional)\n", + "# test_datasets_multiclass, test_datasets_multiclass_df = load_openml_list(test_dids_classification, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 10000, num_feats=100, return_capped=True)\n", + "\n", + "random.seed(0)\n", + "random.shuffle(cc_valid_datasets_multiclass)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def get_datasets(selector, task_type, suite='cc'):\n", + " if task_type == 'binary':\n", + " ds = valid_datasets_binary if selector == 'valid' else test_datasets_binary\n", + " else:\n", + " if suite == 'openml':\n", + " ds = valid_datasets_multiclass if selector == 'valid' else test_datasets_multiclass\n", + " elif suite == 'cc':\n", + " ds = cc_valid_datasets_multiclass if selector == 'valid' else cc_test_datasets_multiclass\n", + " else:\n", + " raise Exception(\"Unknown suite\")\n", + " return ds" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Fitting a PFN for our prior" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Define prior settings" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "def get_prior_config(config_type):\n", + " if config_type == 'causal':\n", + " return get_prior_config_causal()\n", + " elif config_type == 'gp':\n", + " return get_prior_config_gp()\n", + " elif config_type == 'bnn':\n", + " return get_prior_config_bnn()\n", + " elif config_type == 'bag_gp_bnn':\n", + " return get_prior_config_bag_gp_bnn()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def get_prior_config_gp():\n", + " config_general = get_general_config(max_features, 50, eval_positions=[30])\n", + " config_general_real_world = {**config_general}\n", + "\n", + " config_flexible_categorical = get_flexible_categorical_config(max_features)\n", + " config_flexible_categorical_real_world = {**config_flexible_categorical}\n", + "\n", + " config_gp = {}\n", + "\n", + " config_diff = get_diff_config()\n", + "\n", + " config = {**config_general_real_world, **config_flexible_categorical_real_world, **config_diff, **config_gp}\n", + " \n", + " config_sample['differentiable_hyperparameters']['prior_bag_exp_weights_1'] = {'distribution': 'uniform', 'min': 0.0, 'max': .01} # Never select MLP\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def get_prior_config_bnn():\n", + " config_general = get_general_config(max_features, 50, eval_positions=[30])\n", + " config_general_real_world = {**config_general}\n", + "\n", + " config_flexible_categorical = get_flexible_categorical_config(max_features)\n", + " config_flexible_categorical_real_world = {**config_flexible_categorical}\n", + "\n", + " config_gp = {}\n", + " config_mlp = {}\n", + "\n", + " config_diff = get_diff_config()\n", + "\n", + " config = {**config_general_real_world, **config_flexible_categorical_real_world, **config_diff, **config_gp, **config_mlp}\n", + " \n", + " config_sample['differentiable_hyperparameters']['prior_bag_exp_weights_1'] = {'distribution': 'uniform', 'min': 1000.0, 'max': 1001.0} # Always select MLP\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def get_prior_config_causal():\n", + " config_general = get_general_config(max_features, 50, eval_positions=[30])\n", + " config_general_real_world = {**config_general}\n", + "\n", + " config_flexible_categorical = get_flexible_categorical_config(max_features)\n", + " config_flexible_categorical_real_world = {**config_flexible_categorical}\n", + " config_flexible_categorical_real_world['num_categorical_features_sampler_a'] = -1.0 # Categorical features disabled by default\n", + "\n", + " config_gp = {}\n", + " config_mlp = {}\n", + "\n", + " config_diff = get_diff_config()\n", + "\n", + " config = {**config_general_real_world, **config_flexible_categorical_real_world, **config_diff, **config_gp,\n", + " **config_mlp}\n", + " \n", + " return config" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "def reload_config(config_type='causal', task_type='multiclass', longer=0):\n", + " config = get_prior_config(config_type=config_type)\n", + " \n", + " config['prior_type'], config['differentiable'], config['flexible'] = 'prior_bag', True, True\n", + " \n", + " model_string = ''\n", + " \n", + " config['epochs'] = 12000\n", + " config['recompute_attn'] = True\n", + "\n", + " config['max_num_classes'] = 10\n", + " config['num_classes'] = uniform_int_sampler_f(2, config['max_num_classes'])\n", + " config['balanced'] = False\n", + " model_string = model_string + '_multiclass'\n", + " \n", + " model_string = model_string + '_'+datetime.now().strftime(\"%m_%d_%Y_%H_%M_%S\")\n", + " \n", + " return config, model_string" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Visualize Prior samples" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "config, model_string = reload_config(longer=1)\n", + "config_sample = evaluate_hypers(config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config_sample['batch_size'] = 4\n", + "model = get_model(config_sample, device, should_train=False, verbose=2) # , state_dict=model[2].state_dict()\n", + "(hp_embedding, data, targets_), targets = next(iter(model[3]))\n", + "\n", + "from utils import normalize_data\n", + "fig = plt.figure(figsize=(8, 8))\n", + "N = 100\n", + "plot_features(data[0:N, 0, 0:4], targets[0:N, 0], fig=fig)\n", + "\n", + "d = np.concatenate([data[:, 0, :].T, np.expand_dims(targets[:, 0], -1).T])\n", + "d[np.isnan(d)] = 0\n", + "c = np.corrcoef(d)\n", + "plt.matshow(np.abs(c), vmin=0, vmax=1)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Training" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using style prior: True\n", + "Using cpu:0 device\n", + "Not using distributed\n", + "DataLoader.__dict__ {'num_steps': 100, 'fuse_x_y': False, 'get_batch_kwargs': {'batch_size': 4, 'seq_len': 50, 'seq_len_maximum': 50, 'device': 'cpu:0', 'num_features': 100, 'hyperparameters': {'lr': 0.0001733787235341751, 'dropout': 0.0, 'emsize': 256, 'batch_size': 4, 'nlayers': 12, 'num_features': 100, 'nhead': 4, 'nhid_factor': 2, 'bptt': 50, 'eval_positions': [47], 'seq_len_used': 50, 'sampling': 'normal', 'epochs': 12000, 'num_steps': 100, 'verbose': True, 'pre_sample_causes': True, 'mix_activations': False, 'nan_prob_unknown_reason_reason_prior': 1.0, 'categorical_feature_p': 0.0, 'nan_prob_no_reason': 0.2, 'nan_prob_unknown_reason': 0.0, 'nan_prob_a_reason': 0.0, 'max_num_classes': 10, 'num_classes': .. at 0x7f3dd119d560>, 'noise_type': 'Gaussian', 'balanced': False, 'normalize_to_ranking': False, 'set_value_to_nan': 0.5, 'normalize_by_used_features': True, 'num_features_used': .. at 0x7f3dd119db90>, 'num_categorical_features_sampler_a': -1.0, 'differentiable_hyperparameters': {'distribution': 'uniform', 'min': 100000.0, 'max': 100001.0}, 'prior_type': 'prior_bag', 'differentiable': True, 'flexible': True, 'recompute_attn': True, 'aggregate_k_gradients': 1, 'multiclass_type': 'rank', 'bptt_extra_samples': None, 'prior_bag_get_batch': (.make_get_batch.. at 0x7f3de63f0ef0>, .make_get_batch.. at 0x7f3de63f0e60>), 'prior_bag_exp_weights_1': 2.0}, 'num_outputs': 1, 'dynamic_batch_size': 2, 'get_batch': .make_get_batch.. at 0x7f3de63f00e0>, 'differentiable_hyperparameters': {'prior_bag_exp_weights_1': {'distribution': 'uniform', 'min': 100000.0, 'max': 100001.0}, 'num_layers': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 6, 'min_mean': 1, 'round': True, 'lower_bound': 2}, 'prior_mlp_hidden_dim': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 130, 'min_mean': 5, 'round': True, 'lower_bound': 4}, 'prior_mlp_dropout_prob': {'distribution': 'meta_beta', 'scale': 0.9, 'min': 0.1, 'max': 5.0}, 'noise_std': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 0.3, 'min_mean': 0.0001, 'round': False, 'lower_bound': 0.0}, 'init_std': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 0.01, 'round': False, 'lower_bound': 0.0}, 'num_causes': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 12, 'min_mean': 1, 'round': True, 'lower_bound': 1}, 'is_causal': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'pre_sample_weights': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'y_is_effect': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'prior_mlp_activations': {'distribution': 'meta_choice_mixed', 'choice_values': [, , , . at 0x7f3dd119d3b0>, ]}, 'block_wise_dropout': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'sort_features': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'in_clique': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'outputscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 1e-05, 'round': False, 'lower_bound': 0}, 'lengthscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 1e-05, 'round': False, 'lower_bound': 0}, 'noise': {'distribution': 'meta_choice', 'choice_values': [1e-05, 0.0001, 0.01]}, 'output_multiclass_ordered_p': {'distribution': 'uniform', 'min': 0.0, 'max': 0.5}, 'multiclass_type': {'distribution': 'meta_choice', 'choice_values': ['value', 'rank']}}}, 'num_features': 100, 'num_outputs': 1}\n", + "PRIOR_BAG: tensor([1.0000e+00, 1.0000e+05]) [1]\n", + "{'is_causal': True, 'num_causes': 4, 'prior_mlp_hidden_dim': 25, 'num_layers': 6, 'noise_std': 0.039215336075737864, 'y_is_effect': False, 'pre_sample_weights': True, 'prior_mlp_dropout_prob': 0.6926878062137257, 'pre_sample_causes': True}\n", + "Hparams dict_keys(['prior_bag_exp_weights_1', 'num_layers_log_mean', 'num_layers_log_std', 'prior_mlp_hidden_dim_log_mean', 'prior_mlp_hidden_dim_log_std', 'prior_mlp_dropout_prob_b', 'prior_mlp_dropout_prob_k', 'noise_std_log_mean', 'noise_std_log_std', 'init_std_log_mean', 'init_std_log_std', 'num_causes_log_mean', 'num_causes_log_std', 'is_causal_choice_1_weight', 'pre_sample_weights_choice_1_weight', 'y_is_effect_choice_1_weight', 'prior_mlp_activations_choice_1_weight', 'prior_mlp_activations_choice_2_weight', 'prior_mlp_activations_choice_3_weight', 'prior_mlp_activations_choice_4_weight', 'block_wise_dropout_choice_1_weight', 'sort_features_choice_1_weight', 'in_clique_choice_1_weight', 'outputscale_log_mean', 'outputscale_log_std', 'lengthscale_log_mean', 'lengthscale_log_std', 'noise_choice_1_weight', 'noise_choice_2_weight', 'output_multiclass_ordered_p', 'multiclass_type_choice_1_weight'])\n", + "Using a Transformer with 6.52 M parameters\n", + "PRIOR_BAG: tensor([1.0000e+00, 1.0000e+05]) [1]\n", + "{'is_causal': True, 'num_causes': 7, 'prior_mlp_hidden_dim': 18, 'num_layers': 7, 'noise_std': 0.18324918445872412, 'y_is_effect': False, 'pre_sample_weights': True, 'prior_mlp_dropout_prob': 0.24372190159948676, 'pre_sample_causes': True}\n", + "Hparams dict_keys(['prior_bag_exp_weights_1', 'num_layers_log_mean', 'num_layers_log_std', 'prior_mlp_hidden_dim_log_mean', 'prior_mlp_hidden_dim_log_std', 'prior_mlp_dropout_prob_b', 'prior_mlp_dropout_prob_k', 'noise_std_log_mean', 'noise_std_log_std', 'init_std_log_mean', 'init_std_log_std', 'num_causes_log_mean', 'num_causes_log_std', 'is_causal_choice_1_weight', 'pre_sample_weights_choice_1_weight', 'y_is_effect_choice_1_weight', 'prior_mlp_activations_choice_1_weight', 'prior_mlp_activations_choice_2_weight', 'prior_mlp_activations_choice_3_weight', 'prior_mlp_activations_choice_4_weight', 'block_wise_dropout_choice_1_weight', 'sort_features_choice_1_weight', 'in_clique_choice_1_weight', 'outputscale_log_mean', 'outputscale_log_std', 'lengthscale_log_mean', 'lengthscale_log_std', 'noise_choice_1_weight', 'noise_choice_2_weight', 'output_multiclass_ordered_p', 'multiclass_type_choice_1_weight'])\n" + ] + } + ], + "source": [ + "model = get_model(config_sample, device, should_train=True, verbose=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Prior tuning and inference of a fitted PFN (pretrained model provided)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Settings" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "model_string, longer, task_type = '', 1, 'multiclass'\n", + "eval_positions = [1000]\n", + "bptt = 2000\n", + " \n", + "test_datasets, valid_datasets = get_datasets('test', task_type, suite=suite), get_datasets('valid', task_type, suite=suite)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "model_string = ''\n", + "i, e = '8x_lr0.0003', -1\n", + "\n", + "# File which contains result of hyperparameter tuning run: style (i.e. hyperparameters) and a dataframe with results.\n", + "style_file = 'prior_tuning_result.pkl'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Setup helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "def load_result(path, i, e, ensemble=0, split_setter=None):\n", + " print(f'loading {path}')\n", + " with open(path, 'rb') as output:\n", + " c, metrics, metrics_valid, style, temperature, optimization_route = CustomUnpickler(output).load()\n", + "\n", + " metrics = metrics[ensemble]\n", + "\n", + " with warnings.catch_warnings():\n", + " warnings.simplefilter(\"ignore\")\n", + " for i in range(len(metrics)):\n", + " calculate_score_per_method(tabular_metrics.auc_metric, 'roc', metrics[i], test_datasets, eval_positions)\n", + " calculate_score_per_method(tabular_metrics.cross_entropy, 'cross_entropy', metrics[i], test_datasets, eval_positions)\n", + " calculate_score_per_method(tabular_metrics.time_metric, 'time', metrics[i], test_datasets, eval_positions)\n", + " calculate_score_per_method(tabular_metrics.auc_metric, 'roc', metrics_valid, valid_datasets, eval_positions)\n", + " calculate_score_per_method(tabular_metrics.cross_entropy, 'cross_entropy', metrics_valid, valid_datasets, eval_positions)\n", + "\n", + " df = {'checkpoint_path': path\n", + " , 'epoch_evaluated': e\n", + " , 'model_id': i}\n", + "\n", + " hparams = ['dropout', 'multiclass_loss_type', 'aggregate_k_gradients'\n", + " , 'num_classes_in_training', 'nlayers', 'nhead', 'bptt_in_training', 'lr', 'bptt'\n", + " , 'batch_size_in_training', 'emsize', 'nan_prob_unknown_reason', 'num_classes_in_training', 'epoch_in_training'\n", + " , 'normalize_to_ranking', 'categorical_feature_p', 'noise_type', 'set_value_to_nan', 'sampling'\n", + " , 'mix_activations', 'multiclass_type', 'output_multiclass_ordered_p', 'nan_prob_unknown_reason_reason_prior', 'num_steps']\n", + "\n", + " df.update({k: c[k] if k in c else None for k in hparams})\n", + " roc, ce, time = [], [], []\n", + " for split in range(0,5):\n", + " time += [metrics[split]['mean_time']]\n", + "\n", + " if split_setter is None:\n", + " for split in range(0,5):\n", + " roc += [metrics[split]['mean_roc']]\n", + " ce += [metrics[split]['mean_cross_entropy']]\n", + " else:\n", + " roc += [metrics[split_setter]['mean_roc']]\n", + " ce += [metrics[split_setter]['mean_cross_entropy']]\n", + " df['split_setter'] = split_setter\n", + " df['ensemble'] = ensemble\n", + " df.update({'mean_time_test': np.mean(time), 'mean_auc_test': np.mean(roc), 'mean_auc_valid': metrics_valid['mean_roc_at_1000'],\n", + " 'mean_cross_entropy_test': np.mean(ce), 'mean_cross_entropy_valid': metrics_valid['mean_cross_entropy_at_1000']})\n", + "\n", + " diff_list = DifferentiableHyperparameterList(c['differentiable_hyperparameters'], c['nhid_factor'] * c['emsize'], device)\n", + " diff_hparams_keys, diff_hparams_f = diff_list.get_hyperparameter_info()\n", + "\n", + " hyper = merge_style_with_info(diff_hparams_keys, diff_hparams_f, style, transform=False)\n", + "\n", + " return hyper, df, optimization_route, metrics, style, temperature" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Predict using a Fitted and Tuned Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Loading the model" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading models_diff/prior_diff_real_checkpoint_n_8x_lr0.0003_epoch_49.cpkt\n", + "Using style prior: True\n", + "Using cpu:0 device\n", + "Not using distributed\n", + "DataLoader.__dict__ {'num_steps': 768, 'fuse_x_y': False, 'get_batch_kwargs': {'batch_size': 1, 'seq_len': 10, 'seq_len_maximum': 10, 'device': 'cpu:0', 'num_features': 100, 'hyperparameters': {'lr': 0.0003, 'dropout': 0.0, 'emsize': 512, 'batch_size': 1, 'nlayers': 12, 'num_features': 100, 'nhead': 4, 'nhid_factor': 2, 'bptt': 10, 'eval_positions': [972], 'seq_len_used': 50, 'sampling': 'normal', 'epochs': 1800, 'num_steps': 768, 'verbose': False, 'pre_sample_causes': True, 'mix_activations': False, 'nan_prob_unknown_reason_reason_prior': 1.0, 'output_multiclass_ordered_p': 0.0, 'categorical_feature_p': 0.1, 'nan_prob_no_reason': 0.0, 'nan_prob_unknown_reason': 0.1, 'nan_prob_a_reason': 0.0, 'max_num_classes': 10, 'num_classes': 2, 'noise_type': 'Gaussian', 'balanced': False, 'multiclass_type': 'rank', 'normalize_to_ranking': False, 'set_value_to_nan': 0.1, 'normalize_by_used_features': True, 'num_features_used': . at 0x7f3dd11fdcb0>, 'num_categorical_features_sampler_a': -1.0, 'differentiable_hyperparameters': {'distribution': 'uniform', 'min': 0.5, 'max': 8.0}, 'prior_type': 'prior_bag', 'differentiable': True, 'flexible': True, 'aggregate_k_gradients': 8, 'recompute_attn': True, 'bptt_extra_samples': None, 'dynamic_batch_size': False, 'multiclass_loss_type': 'nono', 'total_available_time_in_s': None, 'done_part_in_training': 0.8805555555555555, 'categorical_features_sampler': . at 0x7f3dd1202290>, 'num_features_used_in_training': '.. at 0x7fe1a28895e0>', 'num_classes_in_training': '.. at 0x7fe1a2889550>', 'batch_size_in_training': 8, 'bptt_in_training': 1024, 'bptt_extra_samples_in_training': None, 'prior_bag_get_batch': (.make_get_batch.. at 0x7f3dca889ef0>, .make_get_batch.. at 0x7f3dca889dd0>), 'prior_bag_exp_weights_1': 2.0}, 'num_outputs': 1, 'dynamic_batch_size': 2, 'get_batch': .make_get_batch.. at 0x7f3dca889200>, 'differentiable_hyperparameters': {'prior_bag_exp_weights_1': {'distribution': 'uniform', 'min': 0.5, 'max': 8.0}, 'num_layers': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 6, 'min_mean': 1, 'round': True, 'lower_bound': 2}, 'prior_mlp_hidden_dim': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 130, 'min_mean': 5, 'round': True, 'lower_bound': 4}, 'prior_mlp_dropout_prob': {'distribution': 'meta_beta', 'scale': 0.9, 'min': 0.1, 'max': 5.0}, 'noise_std': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 0.3, 'min_mean': 0.0001, 'round': False, 'lower_bound': 0.0}, 'init_std': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 0.01, 'round': False, 'lower_bound': 0.0}, 'num_causes': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 12, 'min_mean': 1, 'round': True, 'lower_bound': 1}, 'is_causal': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'pre_sample_weights': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'y_is_effect': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'prior_mlp_activations': {'distribution': 'meta_choice_mixed', 'choice_values': [, , , , ], 'choice_values_used': [\"\", \"\", \"\", '. at 0x7fe1a2889670>', \"\"]}, 'block_wise_dropout': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'sort_features': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'in_clique': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'outputscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 1e-05, 'round': False, 'lower_bound': 0}, 'lengthscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 1e-05, 'round': False, 'lower_bound': 0}, 'noise': {'distribution': 'meta_choice', 'choice_values': [1e-05, 0.0001, 0.01]}}}, 'num_features': 100, 'num_outputs': 1}\n", + "Using a Transformer with 25.89 M parameters\n" + ] + } + ], + "source": [ + "model, c, results_file = load_model_workflow(i, e, add_name=model_string, base_path=base_path, device='cpu', eval_addition='')" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading prior_tuning_result.pkl\n" + ] + } + ], + "source": [ + "hyper_, df_, optimization_route, metric, style, temperature = load_result(style_file, i, e)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "#### Quick demo: Predict for a given dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[(0, 'balance-scale'),\n", + " (1, 'mfeat-fourier'),\n", + " (2, 'breast-w'),\n", + " (3, 'mfeat-karhunen'),\n", + " (4, 'mfeat-morphological'),\n", + " (5, 'mfeat-zernike'),\n", + " (6, 'cmc'),\n", + " (7, 'credit-approval'),\n", + " (8, 'credit-g'),\n", + " (9, 'diabetes'),\n", + " (10, 'tic-tac-toe'),\n", + " (11, 'vehicle'),\n", + " (12, 'eucalyptus'),\n", + " (13, 'analcatdata_authorship'),\n", + " (14, 'analcatdata_dmft'),\n", + " (15, 'pc4'),\n", + " (16, 'pc3'),\n", + " (17, 'kc2'),\n", + " (18, 'pc1'),\n", + " (19, 'banknote-authentication'),\n", + " (20, 'blood-transfusion-service-center'),\n", + " (21, 'ilpd'),\n", + " (22, 'qsar-biodeg'),\n", + " (23, 'wdbc'),\n", + " (24, 'cylinder-bands'),\n", + " (25, 'dresses-sales'),\n", + " (26, 'MiceProtein'),\n", + " (27, 'car'),\n", + " (28, 'steel-plates-fault'),\n", + " (29, 'climate-model-simulation-crashes')]" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[(i, test_datasets[i][0]) for i in range(len(test_datasets))]" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluation dataset name: balance-scale shape torch.Size([625, 4])\n" + ] + } + ], + "source": [ + "evaluation_dataset_index = 0 # Index of the dataset to predict\n", + "ds = test_datasets[evaluation_dataset_index]\n", + "print(f'Evaluation dataset name: {ds[0]} shape {ds[1].shape}')" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "# This parameter defines the number of inferences to average, the runtime scales almost linearly with N_ensemble_configurations.\n", + "# Higher values mostly affect cross entropy. \n", + "N_ensemble_configurations = 10" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "eval_xs, eval_ys = ds[1].clone().unsqueeze(1), ds[2].clone().unsqueeze(1)\n", + "eval_position = eval_xs.shape[0] // 2\n", + "\n", + "start = time.time()\n", + "prediction = transformer_predict(model[2], eval_xs, eval_ys, eval_position,\n", + " device='cpu',\n", + " style=style,\n", + " inference_mode=True,\n", + " N_ensemble_configurations=N_ensemble_configurations,\n", + " softmax_temperature=temperature, **get_params_from_config(c))\n", + "prediction_, y_ = prediction.squeeze(0), eval_ys.squeeze(1).long()[eval_position:]\n", + "time_taken = time.time() - start" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor(0.9991, dtype=torch.float64), tensor(0.5725), 17.07512354850769)" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "roc, ce = tabular_metrics.auc_metric(y_, prediction_), tabular_metrics.cross_entropy(y_, prediction_)\n", + "roc, ce, time_taken" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Testing speed\n", + "Define as 'ex' your scheduler / execution environment to perform a speed test on (e.g. one cpu kernel, ..) - alternatively remove scheduling.\n", + "The code also programatically sets torch and libraries to use one kernel. If you wish to use more remove the appropriate code in 'submit_speed_test'." + ] + }, + { + "cell_type": "code", + "execution_count": 197, + "metadata": {}, + "outputs": [], + "source": [ + "def submit_speed_test(device, N_ensemble_configurations):\n", + " import os\n", + " os.environ[\"OMP_NUM_THREADS\"] = \"1\" # export OMP_NUM_THREADS=4\n", + " os.environ[\"OPENBLAS_NUM_THREADS\"] = \"1\" # export OPENBLAS_NUM_THREADS=4 \n", + " os.environ[\"MKL_NUM_THREADS\"] = \"1\" # export MKL_NUM_THREADS=6\n", + " os.environ[\"VECLIB_MAXIMUM_THREADS\"] = \"1\" # export VECLIB_MAXIMUM_THREADS=4\n", + " os.environ[\"NUMEXPR_NUM_THREADS\"] = \"1\" # export NUMEXPR_NUM_THREADS=6\n", + " torch.set_num_threads(1)\n", + " \n", + " result = evaluate(datasets=test_datasets, model=model[2],\n", + " method='transformer'\n", + " , device=device\n", + " , overwrite=True, style=style\n", + " , save=False\n", + " , path_interfix=''\n", + " , metric_used=tabular_metrics.cross_entropy\n", + " , return_tensor=False\n", + " , verbose=False\n", + " , eval_positions=[1000]\n", + " , bptt=2000\n", + " , inference_mode=True\n", + " , softmax_temperature=torch.tensor([0.0]).repeat(3)\n", + " , base_path=None\n", + " , N_ensemble_configurations = N_ensemble_configurations\n", + " )\n", + " with open(f'speed_results/speed_result_{device}_{N_ensemble_configurations}.pkl', 'wb') as f:\n", + " pickle.dump(result, f)\n", + " \n", + " return result" + ] + }, + { + "cell_type": "code", + "execution_count": 198, + "metadata": {}, + "outputs": [], + "source": [ + "speed_job_list = []\n", + "for device in ['cpu']:\n", + " for ens in [1,5,10,20,50,100]:\n", + " raise Exception(\"Define ex as your scheduler\")\n", + " speed_job_list += [ex.submit(submit_speed_test, device, ens)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for device in ['cpu']:\n", + " for ens in [1,5,10,20,50,100]:\n", + " for prep in ['mix', 'robust_all', 'power_all']: \n", + " speed_result = pickle.load(open(f'speed_results/speed_result_{device}_{ens}_{prep}.pkl',\"rb\"))\n", + " calculate_score_per_method(tabular_metrics.time_metric, 'time', speed_result, cc_test_datasets_multiclass, [1000])\n", + " print(f'{device}_{ens}_{prep}', speed_result['mean_time'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Printing Prior Hyperparameter tables (optional for reference)" + ] + }, + { + "cell_type": "code", + "execution_count": 277, + "metadata": {}, + "outputs": [], + "source": [ + "hps_df = pd.DataFrame(hps).T\n", + "hps_df.loc[hps_df['choice_values_used'].isna(), 'choice_values_used'] = hps_df[hps_df['choice_values_used'].isna()]['choice_values']\n", + "hps_df = hps_df.drop(columns=['choice_values'])\n", + "hps_df.loc[hps_df['distribution'] == 'meta_choice_mixed', 'distribution'] = 'meta_choice'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_table = (hps_df[hps_df['distribution'] == 'uniform'][['min', 'max']].rename(index={'prior_bag_exp_weights_1': 'GP sampling likelihood'})\n", + ".rename(columns={'distribution': 'Sampling distribution $p(\\psi)$', 'min': 'Minimum', 'max': 'Maximum'}))\n", + "print(print_table.to_latex(index=True, escape=False))\n", + "print_table = (hps_df[hps_df['distribution'] == 'meta_beta'][['min', 'max', 'scale']].rename(index={'prior_mlp_dropout_prob': 'MLP weight dropout'})\n", + ".rename(columns={'distribution': 'Sampling distribution $p(\\psi)$', 'min': 'Min $\\alpha$ and $\\beta$', 'max': 'Max $\\alpha$ and $\\beta$', 'scale': 'Output scaling'}))\n", + "print(print_table.to_latex(index=True, escape=False))\n", + "print_table = (hps_df[hps_df['distribution'] == 'meta_choice'][['choice_values_used']].rename(index={'is_causal': 'Sample SCM', 'pre_sample_weights': 'Share Gaussian Noise mean for all nodes', 'y_is_effect': 'Sample y Node in last MLP layer'\n", + " , 'block_wise_dropout': 'Blockwise Dropout', 'sort_features': 'Keep SCM feature order', 'prior_mlp_activations': 'MLP Activation Functions', 'in_clique': 'Sample feature nodes in blocks', 'noise': 'GP noise'})\n", + ".rename(columns={'distribution': 'Sampling distribution $p(\\psi)$', 'choice_values_used': 'Choices'}))\n", + "print(print_table.to_latex(index=True, escape=False))\n", + "print_table = (hps_df[hps_df['distribution'] == 'meta_trunc_norm_log_scaled'][['max_mean', 'min_mean', 'round', 'lower_bound']]\n", + " .rename(index={'num_layers': 'MLP \\#layers', 'prior_mlp_hidden_dim': 'MLP \\#hidden nodes per layer', 'noise_std': 'Gaussian Noise Std.'\n", + " , 'init_std': 'MLP Weights Std.', 'num_causes': 'SCM \\#nodes at layer 1', 'outputscale': 'GP outputscale', 'lengthscale': 'GP lengthscale'})\n", + ".rename(columns={'distribution': 'Sampling distribution $p(\\psi)$', 'max_mean': 'Max Mean', 'min_mean': 'Min Mean', 'round': 'Round value'\n", + " , 'lower_bound': 'Lower bound'}))\n", + "print(print_table.to_latex(index=True, escape=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Prior tuning and full datasets evaluation\n", + "This section runs a differentiable hyperparameter tuning run and saves the results to a results file, which can be inserted in TabularEval.ipynb to compare to other baselines." + ] + }, + { + "cell_type": "code", + "execution_count": 298, + "metadata": {}, + "outputs": [], + "source": [ + "# Enabling prior tuning will run differentiable hyperparameter optimization, which is not feasable on CPU.\n", + "# If you would like to run on a random configuration to get quick results, disable prior tuning.\n", + "enable_prior_tuning = True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "eval_positions=[1000]\n", + "bptt=2000\n", + "\n", + "N_models = 3\n", + "models_per_block = 1\n", + "\n", + "eval_addition = 'user_run'\n", + "\n", + "eval_model(i=i, e=e\n", + " , valid_datasets=cc_valid_datasets_multiclass[90:] if enable_prior_tuning else cc_valid_datasets_multiclass[:1]#valid_datasets[40:] #valid_datasets[40:]\n", + " , test_datasets=cc_test_datasets_multiclass\n", + " , train_datasets=cc_valid_datasets_multiclass[:90] if enable_prior_tuning else cc_valid_datasets_multiclass[:1]\n", + " , eval_positions_valid=[bptt//2]\n", + " , eval_positions_test=[bptt//2]\n", + " , bptt_valid=bptt\n", + " , bptt_test=bptt\n", + " , add_name=model_string\n", + " , base_path=base_path\n", + " , N_draws=5 if enable_prior_tuning else 0\n", + " , N_grad_steps=20 if enable_prior_tuning else 0\n", + " , selection_metric='ce'\n", + " , eval_addition=eval_addition\n", + " , n_parallel_configurations = 3\n", + " , N_ensemble_configurations = 10\n", + " , device=device)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "#### Plot optimization of a differentiable HP tuning run\n", + "Change 'style_file' to your evaluation results path." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hyper_, df_, optimization_route, metric, style, temperature = load_result(style_file, i, e)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "from matplotlib.ticker import FormatStrFormatter\n", + "renamer = {\"loss\": \"Validation CE Loss\", \"test_loss\": \"Training CE Loss\"}\n", + "for i, optimization_route in enumerate(optimization_routes):\n", + " f, ax = plt.subplots(figsize=(7, 7))\n", + " x, y = 'loss', 'test_loss'\n", + " #x, y = 'select', 'test_select'\n", + " for route in optimization_route:\n", + " route[y], route[x] = np.array(route[y]), np.array(route[x])\n", + " plt.plot(route[x][~np.isnan(route[y])], np.array(route[y][~np.isnan(route[y])]), '-o', markersize=3)\n", + " ax.set_xlabel(renamer[x])\n", + " ax.set_ylabel(renamer[y])\n", + " ax.yaxis.set_major_formatter(FormatStrFormatter('%.3f'))\n", + " ax.xaxis.set_major_formatter(FormatStrFormatter('%.3f'))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### Model Sequence Length Generalization Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 300, + "metadata": {}, + "outputs": [], + "source": [ + "test_datasets_longer_generalization = [ds for ds in test_datasets_multiclass if ds[1].shape[0] >= 10000]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "mlp_jobs_eval_longer_list = []\n", + "\n", + "for bptt_ in [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000, 5500, 6000, 6500, 7000, 7500, 8000, 8500, 9000, 9500, 10000]:\n", + " eval_addition = f'model_generalization_to_longer_newest_{bptt_}'\n", + " eval_model(i=i, e=e\n", + " , valid_datasets=test_datasets_longer_generalization[:1]#valid_datasets[40:] #valid_datasets[40:]\n", + " , test_datasets=test_datasets_longer_generalization\n", + " , train_datasets=test_datasets_longer_generalization[:1]\n", + " , eval_positions_valid=[bptt_//2]\n", + " , eval_positions_test=[bptt_//2]\n", + " , bptt_valid=bptt_\n", + " , bptt_test=bptt_\n", + " , add_name=model_string\n", + " , base_path=base_path\n", + " , N_draws=10#30\n", + " , N_grad_steps=20#30\n", + " , selection_metric='ce'\n", + " , eval_addition=eval_addition\n", + " , n_parallel_configurations = 3\n", + " , N_ensemble_configurations = 10\n", + " , device=device)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "#### Generalization Visualization Code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Warning this breaks things\n", + "eval_addition = 'model_generalization_to_longer_newest_'\n", + "test_datasets = test_datasets_longer_generalization\n", + "eval_positions = np.array([500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000, 5500, 6000, 6500, 7000, 7500, 8000, 8500, 9000, 9500, 10000]) // 2\n", + "ensembles = [0]\n", + "\n", + "match_string = f'{eval_addition}'\n", + "files = [os.path.join(base_path, 'models_diff/', file) for file in files if match_string in file]" + ] + }, + { + "cell_type": "code", + "execution_count": 1293, + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
checkpoint_pathepoch_evaluatedmodel_iddropoutmulticlass_loss_typeaggregate_k_gradientsnum_classes_in_trainingnlayersnheadbptt_in_training...nan_prob_unknown_reason_reason_priornum_stepsensemblemean_time_testmean_auc_testmean_auc_validmean_cross_entropy_testmean_cross_entropy_validbatch_size_effectivesplit_setter
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl00/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.076806.0022240.778491NaN0.790930NaN64NaN
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl01/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.076806.0022240.781297NaN0.792994NaN641.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl02/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.076806.0022240.779897NaN0.790729NaN642.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl03/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.076806.0022240.777390NaN0.789066NaN643.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl04/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.076806.0022240.775381NaN0.790932NaN644.0
..................................................................
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl00/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.0768028.6982310.790622NaN0.778693NaN64NaN
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl01/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.0768028.6982310.791781NaN0.778138NaN641.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl02/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.0768028.6982310.791817NaN0.778924NaN642.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl03/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.0768028.6982310.787148NaN0.779340NaN643.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl04/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.0768028.6982310.791743NaN0.778370NaN644.0
\n", + "

100 rows × 34 columns

\n", + "
" + ], + "text/plain": [ + " checkpoint_path \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "\n", + " epoch_evaluated model_id \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "... ... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "\n", + " dropout \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "\n", + " multiclass_loss_type \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "\n", + " aggregate_k_gradients \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "\n", + " num_classes_in_training \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "\n", + " nlayers nhead \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "... ... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "\n", + " bptt_in_training ... \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "... ... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "\n", + " nan_prob_unknown_reason_reason_prior \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "\n", + " num_steps ensemble \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "... ... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "\n", + " mean_time_test \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 6.002224 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 6.002224 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 6.002224 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 6.002224 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 6.002224 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 28.698231 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 28.698231 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 28.698231 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 28.698231 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 28.698231 \n", + "\n", + " mean_auc_test \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.778491 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.781297 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.779897 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.777390 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.775381 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.790622 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.791781 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.791817 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.787148 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.791743 \n", + "\n", + " mean_auc_valid \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "\n", + " mean_cross_entropy_test \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.790930 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.792994 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.790729 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.789066 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.790932 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.778693 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.778138 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.778924 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.779340 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.778370 \n", + "\n", + " mean_cross_entropy_valid \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "\n", + " batch_size_effective \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "\n", + " split_setter \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 2.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 3.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 4.0 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 2.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 3.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 4.0 \n", + "\n", + "[100 rows x 34 columns]" + ] + }, + "execution_count": 1293, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = {}\n", + "optimization_routes = []\n", + "df_hyper = {}\n", + "metrics = []\n", + "for i, file in enumerate(files):\n", + " for ensemble in ensembles:\n", + " for split_setter in range(0, 5):\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + " df_hyper[file+str(ensemble)+str(split_setter)], df[file+str(ensemble)+str(split_setter)] = hyper_, df_\n", + " optimization_routes += [optimization_route]\n", + " metrics += [metric]\n", + " \n", + "df = pd.DataFrame.from_dict(df, orient='index')\n", + "df_hyper = pd.DataFrame.from_dict(df_hyper, orient='index')\n", + "#df['num_classes_in_training'] = df['num_classes_in_training'].astype(str).str.slice(0, 5)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 1298, + "metadata": {}, + "outputs": [], + "source": [ + "df = df.sort_values('mean_time_test')\n", + "df['train_pos'] = np.array([[i,i,i,i,i] for i in eval_positions]).flatten()" + ] + }, + { + "cell_type": "code", + "execution_count": 1305, + "metadata": {}, + "outputs": [], + "source": [ + "#filehandler = open(f'model_generalization_result.pkl',\"wb\")\n", + "#pickle.dump(df, filehandler)" + ] + }, + { + "cell_type": "code", + "execution_count": 1304, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABB2UlEQVR4nO3deZxldXng/89z97q39q6lt+qNXqABaaBFRSKIaMCIuEXBmMT8MoMkQ0acaEYzxmhiFmOc6G9GQ4gxJGpUgjgiQkBNwBEXuhsa6IWGpumlequta7v7OeeZP86p6ltV91ZXF3W7tuf9el3PvWe731O057nnuzxfUVWMMcaY8UKzXQBjjDFzkwUIY4wxZVmAMMYYU5YFCGOMMWVZgDDGGFOWBQhjjDFlVTVAiMj1IrJPRPaLyEfLbP+IiOwMXrtExBWR5mDbB4N1u0XkjmqW0xhjzERSrXEQIhIGngfeCHQC24BbVHVPhf1vBD6kqteKyEXAN4ErgALwb8DvqOoLVSmsMcaYCar5BHEFsF9VD6hqAf+Gf9Mk+98CfCN4fwHwc1XNqKoDPAa8vYplNcYYM06kiudeARwp+dwJvKrcjiKSBK4Hbg9W7QL+TESWAFngzcD2CsfeCtwKkEqlLj///PNnpPDzzr59/nLTptkthzELmCp4qjie4noeruevF0AkeC+CTOPcniq5okfOcckVXfJFDzeo4REgHg2TiIZIRMIkomEiodPf4nhKbTwyWoazsWPHjh5VbS23rZoBolxRK9Vn3Qg8rqp9AKq6V0Q+A/wAGAaeBpxyB6rqXcBdAFu3btXt28vGkYXvmmv85aOPzmYpjFlQCo5HOu/Qny3QO1xgKO8gQCQUoiYaJhY5u0qYXNGlayjPycEcXYM5Tgzm6RrKcag3w9H+LAAJYH1zkk3ttWxsr2NTex2rl6QIhyrf/fvSBV61rplENHzW1ygihyptq2aA6AQ6Sj6vBI5V2PdmTlcvAaCq/wD8A4CI/HlwPmOMqZrSgNAzlGe44I4JCC2p+BmP7xrKcTK48Z8cDIJB8H4gWxyzfywcoq0+zorGGl5/fhub2uvY0FZLKl7NW/PUVbMU24ANIrIWOIofBN47ficRaQCuBt43bn2bqnaJyCrgHcBrqljW+e/jH5/tEhgzp2lQNeS4iuN5ZPIuB3rSPH9ykOMDOYZzDtmii6eKpzBSUeR6Hq6C6/pVPo6neN5INZP/Gs47dA3m6csUxnxnJCS01sVpr0/w6rUp2usTtNUnaA/WNSSjhKZTL3SOVC1AqKojIrcDDwNh4CuqultEbgu23xns+nbgEVVNjzvFt4M2iCLwX1T1VLXKuiBcd91sl8CYqlBVVP36aW/0vb/0t5e2CyiO6/l1+I5H3vEYzjkc6ktzqDdN56kcx/qzHO3PcnIwh1eh0luAcEiIhIWwCOHQyCtEJHT6cyQkhEJCMhbm8tVNtNX7N/62ujhL6xM0pWJVCQDF4BoLrocAiWh40iqo6apaN9fZsKjbIHbu9JdbtsxmKYyZVNH1KLoeBcej4Pq/4jNFh2zew/E8VBVX/aDglQSG8WT0f09vdVyPE4N5jvZnORa8jvbnODGYww0iQUhgaX2CVUuSrGpKsrShhra6OE2pKLFwiPqaKIloeE79qldV8o4fEDxVFD8gLEnFaErFSMbC1ETDyDTLLCI7VHVruW1zo6LLvHx33OEvrZHazCLX09GbfyG4qaXzDum8S7bo4Hg62ntF8ev2o2EJliFQpegqGccjW3DIFj0yBf8c2YJLuuCQKbhkgmW64JLJO/QM5zk2cDoQCLC0IcGq5iRXnreEVc1JOpqTLEnFRp9EBKExGaGlNk4qHqE/U+TYQJZTmQKRUIjaeKQqv8qn8jfMBz2ZRsJffU2UVXVJ6muiJONh4pGzb4yeDgsQxphpKTgemYLDUM6hL51nOO9SDKo8wA8ARccbvaEP512Gcw4DuSL9mSKD2SL9WX85lC8GTxPu6E1+MoloiGQ0QjIeJhkLs7yxhlevOx0IVjbVEA2HyBVP32hDItTXRGmpjVFXEyUVGxsAGpMxVi9JMphz6BnKc3wgS9FVYuEQqSoGi4LjkXf86iIUwmGhKRmjoylJbU1kQjnPJQsQxpgzUlWyRZd03qU/U6A3XWAwW6TzVIbDfVl6h/0AMZgrMpA9/So4XtnzxcIhGpJRGmqiNCSjrGyqIRmPkIqFScYiJGPh4BUhFT/9fmRZ7obpjyPwA8Jw3iEkQmMyelY3WhHxy1QTZW1LisFcMeiFlMdxlUQkTDI+/SqoMcEAf+xEMhqmtS5OYzJGKv7yqotmmgUIY8wEjuuRKboMZx36MgWOD2Q52JPmYG+GI30ZDvX5y5Ef+9Gw0FATo7EmSn1NlI6mGhpqYjTUREfXNSb9ZUMiSiIamvZNUFXHtGN4QUNFKCQ0pfxf3nVBQAi9jF/eoZDQmIzRmIxxXqvHYM7hxECO7uEcqvjBIlb5Zj5ZMGhIRqmJ+sEgEp67OVMtQBizyLmef8PNFFwGsgU6+7LsOT7Iwd40h3ozHO7LcKw/OxoMGmuinNdWy6vWLuG81hTrW2tprYvP+K9eLwgERUcpeqcDgYhQEw1Rn4xQF4+SjIWJR8Mko+GXFRAmEwmHaE7FaE7FKLq1DGSLHOvP0psuIEBNNOy3v5QEg1QsMq+CQTkWIBaKP//z2S6BmSNUg+6ewY3fDwB+989s0SXvuEGvGI9T6Twv9WQ41JvxA0JfhhMDudFzNadinNea4pfWt3BeWy3rW2tpTsXwlNFf8I7ncaqk///4FgQJRhSInE5DMf494JfRO92G4d9koyypi1IXj5CI+ikmYpHQrNXJA0TDIVpq47TUxsk7LgOZIicGc9REw/M6GJRjAWKhuPLK2S6BmQXZgstQrkhvuuDf/It+Q/HIQC9FRzuDDmaKo1VDB4OA0DWUHz1Xa12c81pTXHd+G+e11rKuJUVtIkrB9Sg6Hl5w6+/LFIiG/V4+LXUxahMR4mG/V81IN8zRLqrBZycok+v53VddL1jveagHHpCKhf1zRcLEIyHikelXQ50r8UiYtvowbfWJ2S5KVViAWCh++lN/aYFiQSs4HsN5v9dQ91CefNAIHI/4ydsSEb+qpWuowIvdQ7zYPcyL3WkO9AzTnzmd5mF5Q4KN7XXccNEy1rWmWNFYQ000THH0F7yAKIrSlIxSm4hQE/WrcuKRoEuqWfAsQCwUf/iH/tLGQSwonqcMFxwGMkW6hvIM5fybfCQUIhkLUxuP0jucZ9vBPl7sGuZAT5oD3cOkCy7gjwbuaKrhslVNnNday3mtKda2pEjGImQL/iA1Vb+ffUNNhPpE1P8FHw0RC4eqVqdv5gcLEMbMIRqkfB7KFekeytObLuCpEhYJRs+eThY3nHP46s8P8cAzx/z++pEQa5ekeN3G1tEqotVLUmMyjo4EhUzBpTEZ5YIldTQkY9PKAmoWPgsQxsyi06NmvQnVRolImIaaicncCo7HA88c454dR8jkXV6/qY23XbqcVc3lU0KXPilYUDBnwwKEMeeA6+loArlswR9NPJx1yBTd0UbkaChETVBtVOkcjz3fxVd/fpie4TyXrWri/VeuZm1L7YR9LSiYmWABwpgZ5LjeaGK1TMEN0kj4eYRC4geCEEIsEiIWCdE8hVGzqsqTh/u5+6cvcbA3w/rWWu64bgOXrGwcs58FBTPTLEAsFJ///GyXYNEayhU50pdlIFsYrR4CP/dPPOI39qZqp/d/tf1dw/zjT1/imc4B2uvjfORNm7hqQ8uYaqdc0WUoX6QpGbOgYGaUBYiFwtJ8n3PZgsvB3jTHB3LUBPMEV6oeOlsnBnN89WeH+PEL3dQnIvznX1rHDRctndC9NFNwKLgel69upqFmZr7bmBEWIBaKH/7QX9rEQVVXcDw6T/mjj6PhEC2p2IwN6BrIFrln+xEefPY4oZDwq5ev5J2XrSw7BeVgtkg4BJetapozU1SahcX+VS0Un/60v7QAUTWupxwfyPJSdxpPlaZkbMZSPuSKLvc/fYxvP9lJruhy3QXtvPeKVSypLT8Hcn+mQE0szEUrGqw6yVSNBQhjzsDzlJ7hPPu7hyk4Hg2J6MvKs6Oq9Af5e04M5jjen+XhPSfpSxd41dpmfuM1a1jVnKx4bG8mT3Myzubl9Tai2VSVBQhjJtGfKbD/5DBDeYf6RJS6KbYxZAsuJ4MAMLI8MZDj5FCek4O5CfMkXLCsnj/45U1cuLyh4jk9VXrTeZY11LCxvW5WE9aZxcEChDFlDOcdDnQP0zNcoDbmT0tZjqfKjkOn2Ht8kJODOU4O5jkxmGMgWxyzX000zNKGBCsba7h8VSNL6xO01ydob0jQXpcYM9q5HNdT+tJ5VrekWNeSmvNJ7MzCUNUAISLXA18AwsCXVfUvx23/CPBrJWW5AGhV1T4R+RDwn/C7jj8L/Jaq5jCminLFoGdSf5ZEJEJrhcBQdD0e29fNfU91cuRUlpBAW12CpQ0JXr22mfaGxGgQWFqfoC4RmfZNvej66bQ3La1jZVP5qidjqkFUzzz/67ROLBIGngfeCHQC24BbVHVPhf1vBD6kqteKyArgJ8BmVc2KyD3Ag6p692TfuXXrVt2+fftMXsb8sW+fv9y0aXbLMU8VHI+j/X4a7GhIqE9Ey97QMwWHf9t1gu8+fYy+dIG1LSnecekKrlrfUpX8/3nHZSjncOHy+gWbUtrMLhHZoapby22r5hPEFcB+VT0QFOKbwE1A2QAB3AJ8Y1zZakSkCCSBY1Us6/xngWFaHNfjxGCOl3rSuJ7SVFO+Z9KpdIH7nz7GQ7uOky64vGJlAx+8dgOXrmqsWnVPpuCQK7pcuqqRxmSsKt9hzGSqGSBWAEdKPncCryq3o4gkgeuB2wFU9aiI/DVwGMgCj6jqIxWOvRW4FWDVqlUzVvh553vf85c33ji75ZgnXE/pGszxYs8wjqsVeyYdPZXlO0918qPnuvBUec15Lbzj0hVsbK+ravmGcw4eyuVrmqm1MQ5mllTzX165n1WV6rNuBB5X1T4AEWnCf9pYC/QD/yoi71PVr004oepdwF3gVzHNQLnnp899zl9agJjU+C6r9Yko0cTEwPD8ySHu3dHJzw/0EgkLb9zcztu2rGB5Y03VyziQLRCLhNiyoomamI1xMLOnmgGiE+go+bySytVENzO2euk64CVV7QYQkfuAK4EJAcKYqfA8v4voi91pskWH+nhsQpdVVWXH4VPc9+RRnj06QCoe5le3dvCWVyyj6RxV8fRl8tQnoly4vOGMPZuMqbZqBohtwAYRWQscxQ8C7x2/k4g0AFcD7ytZfRh4dVD1lAXeACzS1mfzcqgqpzJFXuwaYrjg+l1WU2Mbex3X4//u7+G+Jzs52JuhpTbGb1+1ljdtbicZOzfVO6pKX6ZAa22cTUvrFsSE92b+q9q/flV1ROR24GH8bq5fUdXdInJbsP3OYNe347cxpEuO/YWI3As8CTjAUwTVSMZMhaoykC2yv3uYoWyR2niUltTYLqtF1+OHe09y745OuobyrGpO8qHrNvBLG1qnNUJZVfHUXyqg6o+TUAWlZJv6da2l21xP6WiuYX1rnU3zaeaMqnVznQ2LupvrNdf4S5uTmoFskZe6h+nLFEnFwhOeAvKOy8O7T3Lfk530pgtsaq/j3Vs72LqmacLsbZNxXI90wcXx/FHRIfFTfIdDQliEcChEKETJ5zKvYH0kHKIpWb5rrTHVNFvdXM259NWvznYJZt1QrshLPWl6h/PURCcOcssWXB7adZzv7DxKf6bIhcvrueO6jVyysmHKN+a845ItuLieEo2EWNoQp6U2Tl0iaqkvzIJjAWKh6Og48z4LVKbgcLAnzcnBPIlImJbaxITtDzxznP+z8yhDOYctHY2855c7uGhF5bxHI1SVXNEjG8zUloqHWdOSojEZpTY+/dHRxswHFiAWim99y1++5z2zW45zqOj68zIc7PHnZVgybl6G4ZzD/U8f5f5njpHOu2xd3cR7tnZw/rL6Sc/rqZIpuOQdF4CmZJRVS+poqIlZt1OzqFiAWCj+9m/95SIIECNjGZ7vGvJHP4+bl2EgW+S7O4/ywDPHyRZdXr2umfdsXcX6ttqK5yxtTwiHhCWpGG31tdQnotbd1CxaFiDMvDKQKfJC9xBDWYeGmuiY3kan0gXue+ooD+06TsHxeO36Ft69tYO1LamK5yu6HoO5IpGQsLQhYe0JxpSwAGHmhVzR5aWeNMcHsqTGpd/uGc7z7Sc7eWT3SRzP43UbW3n35R10VJh0B/zAMJAtEouE2NReR1t9woKCMeNYgDBzmuN6HOvPcqAnHcz/HB9tZzjcl+G+Jzt57PluFLh2UxvvunzlpOkwRp8YwsL5S+torYvboDRjKrAAYeYkVaV7KM8LXcMUXY/GIMuqqrL72ADffrKTbQdPEYuEuP6ipbxtywraJ0mH7bgegzmHcBg2ttXRVm+BwZgzsQCxUNx772yXYMYM5ors7xpmIFOkPhGlPhHFU+VnB3q578lOnjsxRF0iwnuvWMWbL15GQ03laUAd12MgaGM4ry3F0vqEBQZjpsgCxELR0jLbJXjZTs/mlqMmGqalNk7R9Xh49wm+89RRjvZnaa+Pc9vr1vGGC9pJRCt3OXU9ZSBbIBQSzmutZWlDYlrpM4xZzCxALBR33+0v3//+2SxFWap+riEvyE3kacn7YH06mAM6FHQxzRRc7t3Ryf1PH+VUpsi61hR/8MubuPK8lkkbk11P6c8WCIuwtiXFssYaCwzGTJMFiIViFgNEruhyuC9DwfFwPA/XBcfzcDzF9Ty8IN2XCKCCn57u9IQhI+8bamL0Zwrc/dODPLTrBNmiy5aORv7bG1eeMR3GyBODCKxrSbG0ocbGLxjzMlmAMC9Lf6bArqMDqEIsEiIkQkiEaDhEPCKEhCmlozjcl+GrP3+BR/d146ly1foW3nHZSs5rrTy4zfWUbMEl77qEBFYvSbG80QKDMTPFAoSZFlXlaH+W508MUZeITtoeUE6u6LLv5BB7jg2y6+gAzxwd8HskXbiUmy5dwdIKPZKKrkdmZMSzCC11cdrqaqkfN2jOGPPyWYAwZ63oeuzvGubEQI7mVHxKA8z6MwX2Hh9kz/FBdh8b5MXuYTz1q5bWtqQm7ZGUd1wyBRdPlXgkxLJgxHNtImKD24ypIgsQ5qxkCg57jg6SLjgTkuONUFWOD+TYEwSEPccGOdqfBSAaFja21/GuyzvYvKye85fWkYpHJhyfLbpki36yvNp4hLUtKZpSMVKxsGVQNeYcsQCxUDz4YNW/onc4z+5jA0TDYZpLZmdzPeWlnjR7jg+w55gfFE5lioB/c9+8rJ43bm7nwmX1nNdWW7YqqLQ9AaA5FWPNkiQNydhZV18ZY2aGBYiFIlk579DLpaoc7s2wv3uYxprYmEbgEwM5Pvm93aNPCG11cS5Z2cjm5fVsXlZPR3Ny0lnaSscrtNXFaam19gRj5goLEAvFl77kL3/3d2f0tAXH44WTQ3QN51gyrr3hpZ5h/vj+3RRd5UPXbeDiFY201sUnOdtpqspArojnKWuWpFjeZOMVjJlrLEAsFPfc4y9nMEAM5x12Hx2g4Hi0pMb2Knr26ACf/v4ekrEwn37bxayaJHPqhPPmHHKOw/LGGlYvSVkVkjFzVFV/sonI9SKyT0T2i8hHy2z/iIjsDF67RMQVkWYR2VSyfqeIDIrIHdUsqxmreyjHjkOnAGhMxsZs+9mBXv74/l00p2L81TsvmXJwyBZcuofzpBJhXrl2CZuW1ltwMGYOq9oThIiEgS8CbwQ6gW0icr+q7hnZR1U/C3w22P9G4EOq2gf0AVtKznMU+E61ympO8zzlYG+al3rSNCVjE6p9Ht59gi89up8NbXV84i2bqZ8kUd6IouvRny1QF49waUcjTanYGY8xxsy+alYxXQHsV9UDACLyTeAmYE+F/W8BvlFm/RuAF1X1UFVKaUblHZfnjg9xKlOgpTY+pnFZVbl3Ryf//PNDXLaqkY/dcMEZf/2P5EWKhkNctLzBP6eNWzBm3qhmgFgBHCn53Am8qtyOIpIErgduL7P5ZsoHjpFjbwVuBVi1atV0y7roDeaKoykzlqTGNjR7qvzDT17i/qePcfXGVj74hg2TNih7qgwGDdDrWvz0F5Zi25j5p5oBotxPRa2w743A40H10ukTiMSAtwIfq/QlqnoXcBfA1q1bK51/4Xv00WkdpqqcGMjx3IkhUrEINfGxTwVF1+MLP3qBx57v5q2XLOe3r1o7abfVoVyRvOOxsqmGjuaktTEYM49VM0B0Ah0ln1cCxyrsW+kp4QbgSVU9OcNlM/iT6ezvHuZ4f5bGmtiEX/m5ostfPLSXJw/38xuvXs27Ll9ZcRRzpuCQLji01cVZ21I7YXS0MWb+qeb/i7cBG0RkLX4j883Ae8fvJCINwNXA+8qco1K7hBnvr//aX374w1PaPZ132HtskHTRYUnJPM8jBrNF/uSBPbzQNcTtr1/PL1+4tOK5TmUK1ERDXLaqaUKPJ2PM/FW1AKGqjojcDjwMhIGvqOpuEbkt2H5nsOvbgUdUNV16fNAu8UbgA9Uq44LywAP+cgoBomswx97jg8QjYZqTEwe2dQ3l+OP7d3NyMMdHb7iA16xbUvY8qkpfpkBTMsbm5fU20M2YBaaq9QCq+iDw4Lh1d477fDdwd5ljM0D5O5OZlpGcSYd6y3dhBX9ehj++fxeZgsun3noRF69oKHsuVaU3k2dpfQ0b2+ssq6oxC5BVFC8SuaLLnmMDDOYcWmsnVikBPHd8kD95YA+RsPCX77iYtS3lJ+txPaUvk2dVU5J1rbXWddWYBcoCxCLQly6w+9gAYZEJXVhHbD/Ux1889BxLUjH+5K0XsbSh/IQ9juvRlymwoa2Wjuakpd42ZgGzALFQ1NRMWOV5yuG+NC92pydkYS316L4uPv+jF1i9JMknb7yQpgoNzUXXoz9T4MJl9SxtnPh9xpiFxQLEQvHQQ2M+joyK7ktPHBVd6vH9PfzND5/nwuUNfPxXLiAZK/9PIld0Gc47vGJlAy115Z8ujDELiwWIBWggU2TXsQEAWmorp9/eeaSfv35kH5va/bxKlQa1ZQoOecflstVNZacENcYsTBYgFoo//VNUlaO/92GePzlEXTw66Sjm508O8WcP7mFlUw2feMuFFfcdzjl4KJetbqbWBr8Zs6hYx/UFwvvhj8j82yM8f3KY5mR80uBwpC/DJ+/fTUNNlE+99SJqE+Vv/IO5IuEwXL66yYKDMYuQ/b9+ASi6Htl8EU+hdZIqJfAHyf3Rd3cRDgt/etNFNFdIvd2XzlOXiHDhigbiEcunZMxiZAFiAegZylPjQSQ8eZfT/kyBT9y/28+x9I5XsKxhYk8kVaU3XaClLsYFS+stC6sxi5j9v3+eU1WOnMqccSRzpuDwye/tpns4zx+9ZTNrW1IT9vFU6UnnWdaY4MJlDRYcjFnk7AlinhvKO6TzLl5zc8V9Co7Hp7+/l4O9GT7+5gu4cPnE9Bmup/Sl86xpSbG2JWUD4IwxFiDmu+P9WWLhEIfv+mrZ7a6n/NXDz/Hs0QF+/40b2bpmYiBxPT+v0sb2OjqmOL+0MWbhszqEeazgeBwfyFXshaSq/K9/f4FfvNTHrb+0jms2tZXd71Q2z8Y2Cw7GmLEsQMxjvcN5AEIitP/lp2j/y0+NblNVvvL4QX70XBe3vLKDGy9ZXvYcQ7kiTck4Kyx1hjFmHKtimqdUlc5TmdHxCckdT4zZfu+TnfyfnUd5y8XLuOWK8nN1F10Px/PY1F5nGVmNMRPYE8Q8NZR3GM67ZccoPLz7BP/8s0NcvbGV//y6dWUbnFWVgWyB85fWUxOzcQ7GmIksQMxTx/uzZSf8eXx/D196dD+Xr27ijjdsqJikbyBbpL0+QWvd5APrjDGLlwWIeajgeJwYzFE3rnF6OO/4yfeW1vPR68+vOI4hV3QJhYT1bXXWndUYU5EFiHnoVDqPKmOeDvqa2niimPST7/1K5cysniqDuSIXLq+vOD+EMcaANVLPO6rK4b7MmOR5h/syfPSVHyB1VYTPTJJ8D+BUpsDqJSkaK0wKZIwxI6r6E1JErheRfSKyX0Q+Wmb7R0RkZ/DaJSKuiDQH2xpF5F4ReU5E9orIa6pZ1vlifON00fX402Ae6cmS74GfbiMVC7NmiY13MMacWdUChIiEgS8CNwCbgVtEZHPpPqr6WVXdoqpbgI8Bj6lqX7D5C8C/qer5wCXA3mqVdT4Z3zj9H/u6ODGY42u7vsGlf/PJise5npIpuFyw3HIsGWOmpppVTFcA+1X1AICIfBO4CdhTYf9bgG8E+9YDrwPeD6CqBaBQxbLOCyON0401/lOC6yn/ur2T9a21rH7shUmPPZXNs6G9zuZ1MMZMWTV/Sq4AjpR87gzWTSAiSeB64NvBqnVAN/CPIvKUiHxZRCamH/WPvVVEtovI9u7u7pkr/Rw0vnH6xy90c2Iwx7tf2THpcYPZIs3JOMvLpPc2xphKKgYIEWkdXyUUrL9QRFqncO5y/Se1wr43Ao+XVC9FgMuAv1XVS4E0MKENA0BV71LVraq6tbV1KsWan0Yap1Mx/wnA9ZR7th9hzZIkr1pbOZNr0fVw1WPTUhstbYw5O5M9QfwvoNwddyV++8CZdAKlP21XAscq7HszQfVSybGdqvqL4PO9+AFj0RppnB7pvvqzA710nsry7q0dFQfDqSr9GX+09GRTkBpjTDmTBYiLVfWx8StV9WHgFVM49zZgg4isFZEYfhC4f/xOItIAXA18t+Q7TgBHRGRTsOoNVG67WBRODORGG6c9Vb617TArGmu48rwWAArr1lNYt37MMQPZIssaa2irT5zz8hpj5r/JWiyj09wGgKo6InI78DAQBr6iqrtF5LZg+53Brm8HHlHV9LhT/B7w9SC4HAB+60zfuVAVXY/jA9nRxultB/s42JvhQ9dtGJ1J7uhnxj7UjYyWPq+19pyX1xizMEwWIF4QkTer6oOlK0XkBvwb9hkFxz44bt2d4z7fDdxd5tidwNapfM9C1zd8unFaVfnmtiO018e5emP5+R08VYbyDpetarTR0saYaZssQHwIeEBE3g3sCNZtBV4DvKXaBTO+8Y3TTx3uZ3/XMLe/fv2YeahX/PcPAv6TxKlMgTVLkjZa2hjzslQMEKr6vIhcDLwXuChY/RjwAVXNnYvCmdNzTi+pjftPD9uP0FIb49rzxz49xA7sB/zR0rWxCKuXlO0VbIwxUzbpqClVzQP/eI7KYso4MZAbHfm86+gAe48P8oHXrSub6hsgU3B55drmMU8XxhgzHZONgxgSkcGS14CIvBgMWltyLgu5WI00To+k9f7m9iM0JaO8cXN7xf03tNfaaGljzIyoGCBUtU5V60teDfhtELuBOysdZ2ZOaeP03uODPNM5wNsvXVF2FjnXU6LhkM0tbYyZMWf1U1NVTwF/IyK/XqXymMD4xulvbT9CfSLCDRctm7Bv0fUYOv9CljbU2ARAxpgZc9Z1ESISnc5x5uz4I6cdWmoT7O8aZsehU/zGq1eXHRE9kC2y/AtfIGJPD8aYGVTxRi8i7yizugl4D37qC1NF/shpPxh8a/thUvEwv/KKiU8P6bxDXU3ERksbY2bcZE8CN477rEAv8AVV/X71imRKR04f7Enz8wN93PLKDpKxsf+5PFUyBYetK5oJ/UZQ6/e1r81CiY0xC9Fk4yAqprYQkVeq6rbqFMmUNk7fs+MINdEwN16yfMJ+A9kiHc1J6hNR6OychZIaYxayKbclBKm/b8af2GcAS4NRFaWN052nMvzkhR7eedlK6hJj018VXQ8RbECcMaZqJg0QIrIaPyDcAjjAamCrqh6sftEWp9LG6X/d3kk0EuKmLeWfHi5YWme5lowxVTPZQLmf4ifaiwLvUtXLgSELDtU10jh9YiDHo893ccOFSyfkVErnHeprIrQ3WMO0MaZ6JnuC6Maf5Kcdf+KgF6g8I5yZASON0w2JGF/92UHCIeHtl46dpXWkYfqVK5rHjnl4zWvOcWmNMQvdZI3UNwWT+bwT+JSIrAcaReQKVX3inJVwERlpnO5LF/jRc1286cKlLKmNj9lnIFtkVXNyQpsEf/EX57CkxpjFYNIKbFUdUNWvqOobgVcBnwA+LyJHzknpFpHSxun7nuxEgXeOe3oouh4hgVXWMG2MOQem3MKpql2q+r9U9UrgqiqWaVEaDhqnswWXh/ec4Nrz2yYMfuvPFtnYXqFh+p3v9F/GGDNDppUyQ1UPzXRBFruRxul7d3Tiesq7Lls5Zns679CYjNJaFy9/gt7ec1BKY8xiYn0k54CC43FsIIvrKQ/tOs7rNrayvCSvkqdKtuiwoa3WkvEZY84ZS7o3Bxzrz6IK33/mGAXH492Xd4zZ3p8p0NGcmtgwbYwxVTTZOIi/EpHbyqz/kIh8ZionF5HrRWSfiOwXkY+W2f4REdkZvHaJiCsizcG2gyLybLBt+9lc1HySK7oc7E0TkRAPPHOcK9e30NGcHN1edD3CIWFVyTpjjDkXJnuCeAun56Iu9QXgGeC/T3ZiEQkDXwTeCHQC20TkflXdM7KPqn4W+Gyw/43Ah1S1r+Q0r1fVnqlcyHx1uC9NOCQ8+OxxskWX92wd2/bQny1y0fL6M4+YfsMbqlhKY8xiNFmAUFX1yqz0ZGoV4VcA+1X1AICIfBO4CdhTYf9bgG9M4bwLRjrvcPRUjppoiPufPsar1jaztqV2dPtw3qFpsobpUn/0R1UsqTFmMZrsZ2lGRDaMXxmsy07h3CuA0vESncG6CUQkCVwPfLtktQKPiMgOEbm10peIyK0isl1Etnd3d0+hWHPHSz3DxCMhHtp1kuG8w7u3nm57GGmYXm8N08aYWTJZgPgE8JCIvF9ELg5evwV8P9h2JuXuapVSddwIPD6ueum1qnoZcAPwX0TkdeUOVNW7VHWrqm5tbW2dQrHmhoFMka6hPIlomO8+fZRLOxrZ2F43ur0/U2DV2TRM33CD/zLGmBkyWaqNh0TkbcBHgN8LVu8G3qmqz07h3J1AaXeclcCxCvvezLjqJVU9Fiy7ROQ7+FVWP57C9855qsr+7iFSsQhPvNRHf6bIW689nbG16HqEw2fZMJ2dykOdMcZM3aTdXFV1F/CbIlLrf9T0WZx7G7BBRNYCR/GDwHvH7xTke7oaeF/JuhQQUtWh4P2bgD85i++e03qH8wxmHVpq4zyy5yQttTEuXdU0un0gV+TCZVNomDbGmCqa9A4kIr8rIoeBQ8BhETkkIr87lROrqgPcDjwM7AXuUdXdInLbuO6zbwceGRd82oGfiMjTwBPA91X136Z+WXOX5yn7u9PUxiN0DeZ46vAprrugnXDIr5Ebzjs01kyxYdoYY6qo4hOEiHwcuBK4pqQn0jrgCyLSrKqfPtPJVfVB/DklStfdOe7z3cDd49YdAC6Z2iXML12DOXJFlyWpOPc96U8Tet0F7YDfMJ0ruly8ssEapo0xs26yKqZfBy5R1dzIClU9ICLvBp4GzhggzFhF12N/zzD1iSiup/xgbxdbOhppD5Ly+Q3TSWrj0xjg/pa3zHBpjTGL3ZnaIHJl1mVFZML4CHNmx/uzOK4STYTYfqiPnuE8/+mqtYCfjykcFlYtmeaI6Q9/eAZLaowxk7dBdIrIhOG5InItcLx6RVqY8o6fUqOxxp8+9JHdJ6lPRLhibTMAg7kiG9vqiIatYdoYMzdM9gTxX4HvishPgB34YxheCbwWf0S0OQtH+rIIQjgknMoUeOJgHze+YjnRcIhswaWuJvLyGqavucZfPvroTBTXGGMqP0Go6m78XEw/BtYA64L3FwXbzBRlCg5H+jLU1/iD3v79uS5cT3nThX7jdKbo0NFYYw3Txpg5ZSptEF8pXSciYRH5NVX9elVLtoAc7EkTDYcIiaCqPLL7BJuX1dPRdLq9oSEZm8USGmPMRJOl+64XkY+JyP8WkTeK73bgAPDuc1fE+W0wV+TEYJ76hB+Ldx8b5NhAjjdt9p8eckWXukSURDQ8m8U0xpgJJnuC+CpwCvgZ8J+BPwBiwE2qurP6RZv/VJUDXcMko+HR6qOH95wgGQvz2vUtAKQLDptKcjAZY8xcMVmAWKeqFwOIyJeBHmCVqg6dk5ItAKcyRU5lCrTU+uMchnMOP93fyxsuaBt9YlCFptQMVC+92x7qjDEza7IAURx5o6quiLxkwWHqPE95sWuI2vjpbKyPPd9FwfV40+alwEj1UmRmqpd+d0oZUIwxZsomCxCXiMhg8F6AmuCz4Cfuq6966eaxnuE8Q3mH1uDpQVV5eM9J1rWmWN/mTwqULjhsbJuh6qVMxl8mbWpSY8zMmCzdt7WaTpPjeuzvHqYhcbrqaH/XMC/1pPmdq88bXaeqM1O9BPDmN/tLGwdhjJkhNmy3Ck4M5ig43ph03Y/sOUksEuJ1G/1JjfKOS208Qk3M4rAxZm6yADHDCo7HS91pGkpmgssWXB57vpurzmsZTcSXzjssa6yZrWIaY8wZWYCYYZ2nMniqREpyKj2+v4ds0R0dOQ3gKTTPVPWSMcZUgQWIGZQtuBzuy9BQM/bG/8ieE6xorGHzMr9dv+B4JGNhkrFppPU2xphzxO5QM+hQn59SY2R2OIBDvWn2nhjit65cMzpYLl1wWNuSmtkvf//7Z/Z8xphFzwLEDBnKFTnen2VJamxG1h/sOUkkJFx7ftvoOs/Tma9esgBhjJlhVsU0Q17qSZOIRMZkZC26Hv++r4tXrW2mMUjGV3Q9ErEwyZnuvdTT47+MMWaG2BPEDEjnHfqGCyypHfv08PMDvQzlnNGR0yP7rl6SnPnU3u96l7+0cRDGmBlS1ScIEbleRPaJyH4R+WiZ7R8RkZ3Ba5eIuCLSXLI9LCJPicgD1SznyzWcK/rjy8d5ZM9J2uribFnVOLrOVaW59mVMDGSMMedI1QKEiISBLwI3AJuBW0Rkc+k+qvpZVd2iqluAjwGPqWpfyS4fBPZWq4wzpTddIBEZW2V0YjDHziP9XHdBO6HgaaHoesQjIVI2OM4YMw9U8wniCmC/qh5Q1QLwTSafqvQW4BsjH0RkJfArwJerWMaXTVX9ADEu4d4P95wkJHDdBafHPqQLDssbbOY4Y8z8UM0AsQI4UvK5M1g3gYgkgeuBb5es/jz+HBTeZF8iIreKyHYR2d7d3f2yCjwdmYKL5+mYrq2up/xg70kuW9U0Zp5p11Waa21wnDFmfqhmgCj3M1kr7Hsj8PhI9ZKIvAXoUtUdZ/oSVb1LVbeq6tbW1tbpl3aa0nlnwkXtOHSKvnRhdNY48KuXYtHQaKqNGfc7v+O/jDFmhlSzF1Mn0FHyeSVwrMK+N1NSvQS8FniriLwZSAD1IvI1VX1fVUr6MvSmC8THtT88sucEjTVRXrlmtL2ddN5hZXMVq5fe857qnNcYs2hV8wliG7BBRNaKSAw/CNw/ficRaQCuBr47sk5VP6aqK1V1TXDcv8/F4KCq9A4XqClpf+hLF9h2sI83XNA2Jh+TqzqhG+yMOnLEfxljzAyp2hOEqjoicjvwMBAGvqKqu0XktmD7ncGubwceUdV0tcpSLdmii+t5Y9offrT3JJ4yZuyD43pEwyHqqlW9BPDrv+4vbRyEMWaGVHWgnKo+CDw4bt2d4z7fDdw9yTkeBR6d8cLNgHTexStpgfDUb5y+aHk9y0tSeacLLssaEtZ7yRgzr1iqjZehL50nET4dY589OsDxgRxvunDpmP0cz6OlzgbHGWPmFwsQ06Sq9AwXSERLZo3bfZJUPMyV5y0ZXed6SiRU5eolY4ypAgsQ05Qrejju6YmBBrNFfvpiD6/f2DamV9Nw3mFpQ5xQyKqXjDHzi/2snaZ0wcHT0+0Pjz7fhePpmFnjwK9eaq1NVL9Av//71f8OY8yiYgFimvrSBeIR/+lBVXlk90k2tNWytqV2dB+/ekmoS5yDP/ONN1b/O4wxi4pVMU1T73B+NP/S8yeHOdSXGdO1FfzBce31iXNTvbRvn/8yxpgZYk8Q05AruhQcj9p4FICH95wgHgnxuo0tY/Yret6YXExV9YEP+EsbB2GMmSH2BDENpfmXHNfjJy/0cNX6FpKx0/H2dPVSdHYKaYwxL5MFiGk4lSkQDfl/un0nh8gWXa5Y2zxmn3TeobUuPmaUtTHGzCcWIKahZ7hATTDpz1NH+gkJvGJl45h9ip5HW9056L1kjDFVYgHiLOWKLrmiSzQY//D0kX42tNWNSePtBvND1NdY9ZIxZv6yRuqzlCm4o+/TeYfnTw7xq5d3jNvHoe1cVy99/OPn7ruMMYuCBYizdCpdIBK0PzxzdABPYUtH45h9Cq5H67muXrruunP7fcaYBc+qmM5S73CeZND+sPNIP4loiE1L60a3e6qERKg/F4PjSu3c6b+MMWaG2BPEWcg7Lpmiy5KgO+vOw6e4aHnDaHsEQCbv0loXHzNZ0Dlxxx3+0sZBGGNmiD1BnIVM/nT7Q9dgjmMDuQnVS3nXpc1SextjFgALEGehP3u6/WFnZz8wtv3BU0UE671kjFkQLECchdL5p3ce6ac5FWNVc3J0e6bg0lobH1PlZIwx85Xdyaao4HgM5x1ikRCeKk8f6WfLysYx04jmHZe2ehscZ4xZGKyReooyBWf0/YHuNIM5h0tKqpc0mBuiYbaql/78z2fne40xC1ZVnyBE5HoR2Sci+0Xko2W2f0REdgavXSLiikiziCRE5AkReVpEdovIp6pZzqkYyBQJB08LO4/0A2PbHzIFl5bZrF668kr/ZYwxM6RqdzMRCQNfBG4ANgO3iMjm0n1U9bOqukVVtwAfAx5T1T4gD1yrqpcAW4DrReTV1SrrVJTmX9p55BSrm5M0p2Kj23OOS/tsVi/99Kf+yxhjZkg1q5iuAPar6gEAEfkmcBOwp8L+twDfAFC/vmY4WB8NXlrhuKoruh5D+SJLUnHyjsue44O8+aJlo9tnvXoJ4A//0F/aOAhjzAypZn3ICuBIyefOYN0EIpIErge+XbIuLCI7gS7gB6r6iwrH3ioi20Vke3d390yVfYzS8Q97jg1SdJUtqxpH12WLLk3JKLGItfkbYxaOat7RymWqq/QUcCPweFC95O+o6gZVTyuBK0TkonIHqupdqrpVVbe2tra+3DKXNZAtECppf4iEhIuWN4xuzxZnuXrJGGOqoJoBohMoTXO6EjhWYd+bCaqXxlPVfuBR/CeMWdGTHjv+4fyldaPzUY+wmeOMMQtNNQPENmCDiKwVkRh+ELh//E4i0gBcDXy3ZF2riDQG72uA64DnqljWihzXYyhXJB4J0Z8pcKAnzZZVTaPb/alFQ6MJ/IwxZqGoWiO1qjoicjvwMBAGvqKqu0XktmD7ncGubwceUdV0yeHLgH8KekKFgHtU9YFqlXUy6YKLKogIT3cOAHDpmO6tDm318TED5mbF5z8/u99vjFlwqjpQTlUfBB4ct+7OcZ/vBu4et+4Z4NJqlm2qBrNFQoy0P5yiNh7hvNba0e0F12NJSXfXWbNly2yXwBizwFi3mzPoTeepiYVRVXYeGeAVKxvGzBSnCrXneu6Hcn74Q/9ljDEzZA7c2eYu11MGMg5NyShH+7P0DOd599aVo9vzjkttPEw8MgfaHz79aX9pM8sZY2aIPUFMIl1wUBQRqZhew7q3GmMWKgsQkxjKFseMf2ivj7OsoWZ0u6dKQ3IOtD8YY0wVWICYRM9wgUQkjON6PNM5wJaO091bPVUEoTZutXTGmIXJAkQFrqcMZIvEoyGe7xomW3THdG/NFV2aUtExDdbGGLOQ2M/fCjIFB0+VkAhPH+lHgFesHJteY/WSZOUTnGt/93ezXQJjzAJjAaKCoawzmkzqqSP9rG+rHZtOQ+dYeo1Nm2a7BMaYBcaqmCrozRRIRMNkCg77TgyO6b3kuB7RyBxLr/G97/kvY4yZIfYEUYbnKf3pAvU1UbYd7MPTsd1bs0WX1ro5kF6j1Oc+5y9vvHF2y2GMWTDsCaKMTNHF9fz2h52H+4lHQlywrH50+5xJr2GMMVVkAaKM4VxxdDaLp470c+HyhglzTc+J9BrGGFNFFiDK6B3253/oHspztD87pnurn14jMjfSaxhjTBVZgBjH85S+TIF4JMzTFdJrtNXFZ6dwxhhzDlk9yTjZoovnKeGQ8NSRfhqT0THjHeZseo2vfnW2S2CMWWAsQIwznCui+IHg6c5+Lu1oHO2tNDJwbk6m1+joOPM+xhhzFqyKaZzejJ9/6WBPmoFscUz1Uq7o0pyKzc30Gt/6lv8yxpgZMgd/Cs8eVaVvuEhtPFI2vXeu6LJmLqXXKPW3f+sv3/Oe2S2HMWbBsCeIEtmii+t5o+0PHc1JltSebpBWhbqaOZRewxhjqsgCRInhvIMCBcdjz7HBMd1bR9Jr1ESte6sxZnGoaoAQketFZJ+I7BeRj5bZ/hER2Rm8domIKyLNItIhIv8hIntFZLeIfLCa5RzRN+x3b917fJCC63HJysbRbSPdW+dUeg1jjKmiqgUIEQkDXwRuADYDt4jI5tJ9VPWzqrpFVbcAHwMeU9U+wAF+X1UvAF4N/Jfxx840VaUnGCD31JF+wiHhohWn02sUPW9MdZMxxix01WykvgLYr6oHAETkm8BNwJ4K+98CfANAVY8Dx4P3QyKyF1gxybEvW67ojbY/7DxyivOX1pGMnf7zKMzN7q0j7r13tktgjFlgqlnFtAI4UvK5M1g3gYgkgeuBb5fZtga4FPhFhWNvFZHtIrK9u7t72oUdaX8YyBY50J2e0HupPh4lFpnDTTYtLf7LGGNmSDXveOUq67XCvjcCjwfVS6dPIFKLHzTuUNXBcgeq6l2qulVVt7a2tk67sKcyBeLhMM909qNM7N7aVj/Hq5fuvtt/GWPMDKlmgOgESof3rgSOVdj3ZoLqpREiEsUPDl9X1fuqUsIS3UN5EtEQO4/0k4qF2dBWN7rNVaUhOce7t1qAMMbMsGoGiG3ABhFZKyIx/CBw//idRKQBuBr4bsk6Af4B2Kuq/7OKZQT8JwTHHWl/6OfilQ2jo6VH02vE5nD7gzHGVEHVAoSqOsDtwMPAXuAeVd0tIreJyG0lu74deERV0yXrXgv8OnBtSTfYN1errJ76NV/HB3J0DeXZ0tE0ui1bcFmSihGai+k1jDGmiqr6s1hVHwQeHLfuznGf7wbuHrfuJ5Rvw6iqp4L0GqUD5HKOy7q61LkuijHGzLo53C3n3Nt55BRtdXGWNSTGrK9LzPH2B2OMqQKrWA+4nvJM5wBXrW8ZHS1ddD3ikRCJ6DyIow8+eOZ9jDHmLFiACLzUkyZTcCfMHre0YZ6k10jO0Syzxph5ax78ND43dh8bRIBXlORfcjyP5tQcH/8w4ktf8l/GGDNDLEAEdh0bYF1rioYgnbcGPZvmdHqNUvfc47+MMWaGWIAA0nmHF7vSY7q35h2PusQcT69hjDFVZHc/YNvBPlzVsd1bi356b2OMWawsQAA/fbGXaFi4YNnp9N7zIr2GMcZUkQUI4Gcv9rKpvW60Osn1lLCl1zDGLHIy0hi7EIhIN3BoGgeGQrFkrXquA0AoFMLzXC3msjNcxGpqAXpmuxCzyK7frt+uf3pWq2rZVNgLKkAsZiKyXVW3znY5Zotdv12/Xf/MX79VMRljjCnLAoQxxpiyLEAsHHfNdgFmmV3/4mbXXwXWBmGMMaYse4IwxhhTlgUIY4wxZVmAmKNE5Csi0iUiu0rWNYvID0TkhWDZVLLtYyKyX0T2icgvl6y/XESeDbb9/zIvcpeDiHSIyH+IyF4R2S0iHwzWL4q/gYgkROQJEXk6uP5PBesXxfWPEJGwiDwlIg8EnxfN9YvIwaDcO0Vke7Du3F6/qtprDr6A1wGXAbtK1v0V8NHg/UeBzwTvNwNPA3FgLfAiEA62PQG8Bn8K14eAG2b72qZ4/cuAy4L3dcDzwXUuir9BUNba4H0U+AXw6sVy/SV/h/8G/AvwQPB50Vw/cBBoGbfunF6/PUHMUar6Y6Bv3OqbgH8K3v8T8LaS9d9U1byqvgTsB64QkWVAvar+TP1/Kf9ccsycpqrHVfXJ4P0QsBdYwSL5G6hvOPgYDV7KIrl+ABFZCfwK8OWS1Yvm+is4p9dvAWJ+aVfV4+DfQIG2YP0K4EjJfp3BuhXB+/Hr5xURWQNciv8retH8DYLqlZ1AF/ADVV1U1w98HvgDwCtZt5iuX4FHRGSHiNwarDun12/Z6BaGcnWKOsn6eUNEaoFvA3eo6uAk1acL7m+gqi6wRUQage+IyEWT7L6grl9E3gJ0qeoOEblmKoeUWTdvrz/wWlU9JiJtwA9E5LlJ9q3K9dsTxPxyMnhkJFh2Bes7gY6S/VYCx4L1K8usnxdEJIofHL6uqvcFqxfV3wBAVfuBR4HrWTzX/1rgrSJyEPgmcK2IfI3Fc/2o6rFg2QV8B7iCc3z9FiDml/uB3wze/ybw3ZL1N4tIXETWAhuAJ4JH0CEReXXQc+E3So6Z04Ly/gOwV1X/Z8mmRfE3EJHW4MkBEakBrgOeY5Fcv6p+TFVXquoa4Gbg31X1fSyS6xeRlIjUjbwH3gTs4lxf/2y31NurYg+GbwDHgSL+r4DfBpYAPwJeCJbNJfv/D/yeC/so6aUAbA3+Yb0I/G+C0fNz/QVchf8o/AywM3i9ebH8DYBXAE8F178L+ESwflFc/7i/xTWc7sW0KK4fWIffK+lpYDfwP2bj+i3VhjHGmLKsiskYY0xZFiCMMcaUZQHCGGNMWRYgjDHGlGUBwhhjTFkWIMyMExEVkc+VfP6wiHxyhs59t4i8aybOdYbv+VXxM8n+x7j1a0TkvdM850+nsM+XRWTzdM4/m0TkURHZOtvlMDPLAoSphjzwDhFpme2ClBKR8Fns/tvA76rq68etXwOUDRAiMmnqGlW98kxfqqr/SVX3TLWQxlSTBQhTDQ7+HLkfGr9h/BOAiAwHy2tE5DERuUdEnheRvxSRXxN/ToRnReS8ktNcJyL/N9jvLcHxYRH5rIhsE5FnROQDJef9DxH5F+DZMuW5JTj/LhH5TLDuE/gD9e4Ukc+OO+QvgV8SP0f/h0Tk/SLyryLyPfzEarUi8iMReTI4700VrvVREblXRJ4Tka8Ho1zH/BIXkWER+TPx54T4uYi0B+vPCz5vE5E/GTnvuOtKicj3g2N3ich7Rq4tOG6XiNw17nv/RkR+HDw5vVJE7hN/3oFPB/usCcr7T8Hf+F4RSZb57jeJyM+Cv8G/ip9Pi+C/6Z7g2L8ef5yZg2Z7xKC9Ft4LGAbq8fPZNwAfBj4ZbLsbeFfpvsHyGqAffx6IOHAU+FSw7YPA50uO/zf8Hzcb8EeZJ4BbgY8H+8SB7fh58a8B0sDaMuVcDhwGWvETV/478LZg26PA1jLHXEMwqjf4/P6gDM3B5wh+emWAFvy0y1LmWgfw8+KEgJ8BV43/XvyR5DcG7/+q5PoeAG4J3t82ct5x5Xwn8PclnxuCZenI26+WnP9RTs8t8EH8fD0j/y068UfwrgnK9Npgv68AHy4td3DNPwZSwfr/DnwCaMYf4Tvyt2ic7X+n9jrzy54gTFWo6iB+7vn/ehaHbVN/Hog8flqAR4L1z+LfnEbco6qeqr4AHADOx89V8xvip8f+Bf4NbUOw/xPq58gf75XAo6raraoO8HX8iZrO1g9UdWTuDgH+XESeAX6In1q5vcwxT6hqp6p6+GlE1pTZp4AfDAB2lOzzGuBfg/f/UqFMz+I/aX1GRH5JVQeC9a8XkV+IyLPAtcCFJcfcX3Ls7pL/Fgc4nQjuiKo+Hrz/Gv6TVqlX409e83jw3+I3gdXAIJADviwi7wAyFcpt5hBL922q6fPAk8A/lqxzCKo2g+qNWMm2fMl7r+Szx9h/q+Pzw4ykNf49VX24dIP4qaLTFco3U1NPlp7/1/CfSC5X1aL42UgTZY4pvVaX8v9fLGrwc3uSfcpS1edF5HL8/FV/ISKP4D+FfAn/CeWI+B0HSstW+vce/99i5LvL/e1LCX7AvGV8mUTkCuAN+Mn3bscPUGYOsycIUzXBr+p78Bt8RxwELg/e34Q/U9rZ+lURCQXtEuvwqy4eBn5H/BThiMhG8bNgTuYXwNUi0hI0YN8CPHaGY4bwp0CtpAF/HoOiiLwe/9fzTPs5fhUS+DfbCURkOZBR1a8Bf40/fe1IMOgJ2gWm0xtslYi8Jnh/C/CTMmV7rYisD8qRDP5b1OJXcz0I3AFsmcZ3m3PMniBMtX0O/9fiiL8HvisiT+Bno6z0634y+/Bv5O3AbaqaE5Ev41fBPBk8mXRzhqkVVfW4iHwM+A/8X74PquqZUiE/Azgi8jR+e8ipcdu/DnxP/Enmd+Kn6J5pdwBfE5HfB76P354x3sXAZ0XEw88I/Duq2i8if49fhXQQ2DaN794L/KaI/B1+RtG/Ld2oqt0i8n7gGyISD1Z/HD+wfldEEvh/6wkdGMzcY9lcjZlngp5DWVVVEbkZv8H6pjMdNwPfuwa/gX6yme3MAmJPEMbMP5cD/zt4UuoH/r/ZLY5ZqOwJwhhjTFnWSG2MMaYsCxDGGGPKsgBhjDGmLAsQxhhjyrIAYYwxpqz/B1vFcb7qCMBqAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "sns.lineplot(data=df, x='train_pos', y='mean_auc_test')\n", + "plt.axvline(x=1024, linestyle='dashed', color='red')\n", + "plt.xlabel('Number of training samples')\n", + "plt.ylabel('ROC AUC')\n", + "plt.ylim((0.715,0.79))\n", + "tikzplotlib.save(f'model_generalization.tex', axis_height='6cm', axis_width='6cm')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/TabPFN/__pycache__/encoders.cpython-37.pyc b/TabPFN/__pycache__/encoders.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61aec819068402a3ffa6fc192078cc4d28c4588c Binary files /dev/null and b/TabPFN/__pycache__/encoders.cpython-37.pyc differ diff --git a/TabPFN/__pycache__/layer.cpython-37.pyc b/TabPFN/__pycache__/layer.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c76ab52eafe19cb5861816445db8010d9a2b202 Binary files /dev/null and b/TabPFN/__pycache__/layer.cpython-37.pyc differ diff --git a/TabPFN/__pycache__/model_builder.cpython-37.pyc b/TabPFN/__pycache__/model_builder.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b585d814cea29561d185190898a4802b5d7fe073 Binary files /dev/null and b/TabPFN/__pycache__/model_builder.cpython-37.pyc differ diff --git a/TabPFN/__pycache__/notebook_utils.cpython-37.pyc b/TabPFN/__pycache__/notebook_utils.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1884110312b29fbff32b52d358226af6f7917ede Binary files /dev/null and b/TabPFN/__pycache__/notebook_utils.cpython-37.pyc differ diff --git a/TabPFN/__pycache__/positional_encodings.cpython-37.pyc b/TabPFN/__pycache__/positional_encodings.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b59a49111cfb9b285c14288aaf5f50cc902b819b Binary files /dev/null and b/TabPFN/__pycache__/positional_encodings.cpython-37.pyc differ diff --git a/TabPFN/__pycache__/train.cpython-37.pyc b/TabPFN/__pycache__/train.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..348eeae76beecafc9f0c6f11f2a11fba25d5ef45 Binary files /dev/null and b/TabPFN/__pycache__/train.cpython-37.pyc differ diff --git a/TabPFN/__pycache__/transformer.cpython-37.pyc b/TabPFN/__pycache__/transformer.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..271a056101d8a8d54effe21378560d72cbdead83 Binary files /dev/null and b/TabPFN/__pycache__/transformer.cpython-37.pyc differ diff --git a/TabPFN/__pycache__/utils.cpython-37.pyc b/TabPFN/__pycache__/utils.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b54a19a60e2134402ef5e9827908b9d8a94729f4 Binary files /dev/null and b/TabPFN/__pycache__/utils.cpython-37.pyc differ diff --git a/TabPFN/__pycache__/utils.cpython-38.pyc b/TabPFN/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..911ad74b46f353cf4415642a1faa03b8ddf0aaee Binary files /dev/null and b/TabPFN/__pycache__/utils.cpython-38.pyc differ diff --git a/TabPFN/datasets/__init__.py b/TabPFN/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..38a42a279fe237176fba8989ee757ef8e37c0ac7 --- /dev/null +++ b/TabPFN/datasets/__init__.py @@ -0,0 +1,149 @@ +import pandas as pd +import torch +import numpy as np +import openml + + +def get_openml_classification(did, max_samples, multiclass=True, shuffled=True): + dataset = openml.datasets.get_dataset(did) + X, y, categorical_indicator, attribute_names = dataset.get_data( + dataset_format="array", target=dataset.default_target_attribute + ) + + if not multiclass: + X = X[y < 2] + y = y[y < 2] + + if multiclass and not shuffled: + raise NotImplementedError("This combination of multiclass and shuffling isn't implemented") + + if not isinstance(X, np.ndarray) or not isinstance(y, np.ndarray): + print('Not a NP Array, skipping') + return None, None, None, None + + if not shuffled: + sort = np.argsort(y) if y.mean() < 0.5 else np.argsort(-y) + pos = int(y.sum()) if y.mean() < 0.5 else int((1 - y).sum()) + X, y = X[sort][-pos * 2:], y[sort][-pos * 2:] + y = torch.tensor(y).reshape(2, -1).transpose(0, 1).reshape(-1).flip([0]).float() + X = torch.tensor(X).reshape(2, -1, X.shape[1]).transpose(0, 1).reshape(-1, X.shape[1]).flip([0]).float() + else: + order = np.arange(y.shape[0]) + np.random.seed(13) + np.random.shuffle(order) + X, y = torch.tensor(X[order]), torch.tensor(y[order]) + if max_samples: + X, y = X[:max_samples], y[:max_samples] + + return X, y, list(np.where(categorical_indicator)[0]), attribute_names + +def load_openml_list(dids, filter_for_nan=False + , num_feats=100 + , min_samples = 100 + , max_samples=400 + , multiclass=True + , max_num_classes=10 + , shuffled=True + , return_capped = False): + datasets = [] + openml_list = openml.datasets.list_datasets(dids) + print(f'Number of datasets: {len(openml_list)}') + + datalist = pd.DataFrame.from_dict(openml_list, orient="index") + if filter_for_nan: + datalist = datalist[datalist['NumberOfInstancesWithMissingValues'] == 0] + print(f'Number of datasets after Nan and feature number filtering: {len(datalist)}') + + for ds in datalist.index: + modifications = {'samples_capped': False, 'classes_capped': False, 'feats_capped': False} + entry = datalist.loc[ds] + + print('Loading', entry['name'], entry.did, '..') + + if entry['NumberOfClasses'] == 0.0: + raise Exception("Regression not supported") + #X, y, categorical_feats, attribute_names = get_openml_regression(int(entry.did), max_samples) + else: + X, y, categorical_feats, attribute_names = get_openml_classification(int(entry.did), max_samples + , multiclass=multiclass, shuffled=shuffled) + if X is None: + continue + + if X.shape[1] > num_feats: + if return_capped: + X = X[:, 0:num_feats] + categorical_feats = [c for c in categorical_feats if c < num_feats] + modifications['feats_capped'] = True + else: + print('Too many features') + continue + if X.shape[0] == max_samples: + modifications['samples_capped'] = True + + if X.shape[0] < min_samples: + print(f'Too few samples left') + continue + + if len(np.unique(y)) > max_num_classes: + if return_capped: + X = X[y < np.unique(y)[10]] + y = y[y < np.unique(y)[10]] + modifications['classes_capped'] = True + else: + print(f'Too many classes') + continue + + datasets += [[entry['name'], X, y, categorical_feats, attribute_names, modifications]] + + return datasets, datalist + + +# Classification +valid_dids_classification = [13, 59, 4, 15, 40710, 43, 1498] +test_dids_classification = [973, 1596, 40981, 1468, 40984, 40975, 41163, 41147, 1111, 41164, 1169, 1486, 41143, 1461, 41167, 40668, 41146, 41169, 41027, 23517, 41165, 41161, 41159, 41138, 1590, 41166, 1464, 41168, 41150, 1489, 41142, 3, 12, 31, 54, 1067] +valid_large_classification = [ 943, 23512, 49, 838, 1131, 767, 1142, 748, 1112, + 1541, 384, 912, 1503, 796, 20, 30, 903, 4541, + 961, 805, 1000, 4135, 1442, 816, 1130, 906, 1511, + 184, 181, 137, 1452, 1481, 949, 449, 50, 913, + 1071, 831, 843, 9, 896, 1532, 311, 39, 451, + 463, 382, 778, 474, 737, 1162, 1538, 820, 188, + 452, 1156, 37, 957, 911, 1508, 1054, 745, 1220, + 763, 900, 25, 387, 38, 757, 1507, 396, 4153, + 806, 779, 746, 1037, 871, 717, 1480, 1010, 1016, + 981, 1547, 1002, 1126, 1459, 846, 837, 1042, 273, + 1524, 375, 1018, 1531, 1458, 6332, 1546, 1129, 679, + 389] + +open_cc_dids = [11, + 14, + 15, + 16, + 18, + 22, + 23, + 29, + 31, + 37, + 50, + 54, + 188, + 458, + 469, + 1049, + 1050, + 1063, + 1068, + 1510, + 1494, + 1480, + 1462, + 1464, + 6332, + 23381, + 40966, + 40982, + 40994, + 40975] +# Filtered by N_samples < 2000, N feats < 100, N classes < 10 + +open_cc_valid_dids = [13,25,35,40,41,43,48,49,51,53,55,56,59,61,187,285,329,333,334,335,336,337,338,377,446,450,451,452,460,463,464,466,470,475,481,679,694,717,721,724,733,738,745,747,748,750,753,756,757,764,765,767,774,778,786,788,795,796,798,801,802,810,811,814,820,825,826,827,831,839,840,841,844,852,853,854,860,880,886,895,900,906,907,908,909,915,925,930,931,934,939,940,941,949,966,968,984,987,996,1048,1054,1071,1073,1100,1115,1412,1442,1443,1444,1446,1447,1448,1451,1453,1488,1490,1495,1498,1499,1506,1508,1511,1512,1520,1523,4153,23499,40496,40646,40663,40669,40680,40682,40686,40690,40693,40705,40706,40710,40711,40981,41430,41538,41919,41976,42172,42261,42544,42585,42638] diff --git a/TabPFN/datasets/utils.py b/TabPFN/datasets/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d193dbe021c4daa3808fa0f8823a6decfe3f634e --- /dev/null +++ b/TabPFN/datasets/utils.py @@ -0,0 +1,8 @@ +def normalize_data(eval_xs): + mean = eval_xs.mean(0) + std = eval_xs.std(0) + .000001 + eval_xs = (eval_xs - mean) / std + + return eval_xs + + diff --git a/TabPFN/decoders.py b/TabPFN/decoders.py new file mode 100644 index 0000000000000000000000000000000000000000..efdd8fcb624c1b0a6d127fcf0aa8548c4ca1524d --- /dev/null +++ b/TabPFN/decoders.py @@ -0,0 +1,30 @@ +import torch +from torch import nn +import random + + +class ScaledDecoder(nn.Module): + def __init__(self, ninp, nhid, nout): + super().__init__() + self.linear = nn.Linear(ninp, nhid) + self.linear1 = nn.Linear(nhid, nout) + self.linear2 = nn.Linear(nhid, 10) + + def forward(self, x): + #return torch.cat([self.linear1(x), self.linear2(x)], -1) + x = self.linear(x) + x = nn.GELU()(x) + temps = self.linear2(x).softmax(-1) @ torch.tensor([1.,1.4,1.7,2.,5.,10.,20.,40.,80.,160.], device=x.device) + if random.random() > .99: + print(temps.shape,temps[:,:2]) + return self.linear1(x) / temps.unsqueeze(-1) + +class FixedScaledDecoder(nn.Module): + def __init__(self, ninp, nhid, nout): + super().__init__() + self.mapper = nn.Sequential(nn.Linear(ninp, nhid), nn.GELU(), nn.Linear(nhid, nout)) + self.T = nn.Parameter(torch.ones(10000)/10000) + + def forward(self, x): + return self.mapper(x)/self.T.sum() + diff --git a/TabPFN/differentiable_pfn_evaluation.py b/TabPFN/differentiable_pfn_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..a4ff651b036318fcab7f989bb8d1ce89310c7cf8 --- /dev/null +++ b/TabPFN/differentiable_pfn_evaluation.py @@ -0,0 +1,345 @@ +import os +import torch +import numpy as np +import time +import pickle +from scripts import tabular_metrics +from scripts.tabular_metrics import calculate_score_per_method +from scripts.tabular_evaluation import evaluate +from priors.differentiable_prior import draw_random_style +from tqdm import tqdm +import random +from scripts.transformer_prediction_interface import get_params_from_config, load_model_workflow + +""" +=============================== +PUBLIC FUNCTIONS FOR EVALUATION +=============================== +""" + + +def eval_model_range(i_range, *args, **kwargs): + for i in i_range: + eval_model(i, *args, **kwargs) + + + +def eval_model(i, e, valid_datasets, test_datasets, train_datasets, eval_positions_valid, eval_positions_test, + bptt_valid, + bptt_test, add_name, base_path, device='cpu', eval_addition='', **extra_tuning_args): + """ + Differentiable model evaliation workflow. Evaluates and saves results to disk. + + :param i: + :param e: + :param valid_datasets: + :param test_datasets: + :param train_datasets: + :param eval_positions_valid: + :param eval_positions_test: + :param bptt_valid: + :param bptt_test: + :param add_name: + :param base_path: + :param device: + :param eval_addition: + :param extra_tuning_args: + :return: + """ + model, c, results_file = load_model_workflow(i, e, add_name, base_path, device, eval_addition) + params = {'bptt': bptt_valid + , 'bptt_final': bptt_test + , 'eval_positions': eval_positions_valid + , 'eval_positions_test': eval_positions_test + , 'valid_datasets': valid_datasets + , 'test_datasets': test_datasets + , 'train_datasets': train_datasets + , 'verbose': True + , 'device': device + } + + params.update(get_params_from_config(c)) + + start = time.time() + metrics, metrics_valid, style, temperature, optimization_route = evaluate_differentiable_model(model, **params, + **extra_tuning_args) + print('Evaluation time: ', time.time() - start) + + print(results_file) + r = [c.copy(), metrics, metrics_valid, style.to('cpu'), temperature.to('cpu'), optimization_route] + with open(results_file, 'wb') as output: + del r[0]['num_features_used'] + del r[0]['categorical_features_sampler'] + pickle.dump(r, output) + + _, _, _, style, temperature, _ = r + + return r, model + +""" +=============================== +INTERNAL HELPER FUNCTIONS +=============================== +""" + +def evaluate_differentiable_model(model + , valid_datasets + , test_datasets + , train_datasets + , N_draws=100 + , N_grad_steps=10 + , eval_positions=None + , eval_positions_test=None + , bptt=100 + , bptt_final=200 + , style=None + , n_parallel_configurations=1 + , device='cpu' + , selection_metric='auc' + , final_splits=[1, 2, 3, 4, 5] + , N_ensemble_configurations_list=[1, 5, 10, 20, 50, 100] + , **kwargs): + """ + Evaluation function for diffable model evaluation. Returns a list of results. + + :param model: + :param valid_datasets: + :param test_datasets: + :param train_datasets: + :param N_draws: + :param N_grad_steps: + :param eval_positions: + :param eval_positions_test: + :param bptt: + :param bptt_final: + :param style: + :param n_parallel_configurations: + :param device: + :param selection_metric: + :param final_splits: + :param N_ensemble_configurations_list: + :param kwargs: + :return: + """ + torch.manual_seed(0) + np.random.seed(0) + random.seed(0) + + diffable_metric = tabular_metrics.cross_entropy + evaluation_metric = tabular_metrics.auc_metric + if selection_metric in ('auc', 'roc'): + selection_metric_min_max = 'max' + selection_metric = tabular_metrics.auc_metric + evaluation_metric = selection_metric + elif selection_metric in ('ce', 'selection_metric'): + selection_metric_min_max = 'min' + selection_metric = tabular_metrics.cross_entropy + evaluation_metric = selection_metric + + print('Diffable metric', diffable_metric, ' Selection metric', selection_metric, ' Evaluation metric', + evaluation_metric) + print('N PARALLEL CONFIGURATIONS', n_parallel_configurations) + print('eval_positions', eval_positions) + + def evaluate_valid(style, softmax_temperature, results, results_tracked): + result_valid = eval_step(valid_datasets, style, softmax_temperature=softmax_temperature, + return_tensor=False, inference_mode=True, selection_metric=selection_metric, + evaluation_metric=evaluation_metric, eval_positions=eval_positions, bptt=bptt, model=model[2]) + result_valid = [float(result_valid[f'mean_select_at_{pos}']) for pos in eval_positions] + results += [result_valid] + results_tracked += [np.nanmean(result_valid)] + + model[2].to(device) + model[2].eval() + + results_on_valid, results_on_valid_tracked = [], [] + best_style, best_softmax_temperature = style, torch.cat( + [torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)], 0) + optimization_routes = [] + + best_style = torch.cat([draw_random_style(model[3], device).detach() for n in range(0, n_parallel_configurations)], + 0) + best_softmax_temperature = torch.cat([torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)], + 0) + + + for _ in tqdm(range(0, N_draws), desc='Iterate over Optimization initializations'): # Evaluates N hparam draws + style = torch.cat([draw_random_style(model[3], device).detach() for n in range(0, n_parallel_configurations)], + 0) + softmax_temperature = torch.cat([torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)], + 0) + + evaluate_valid(style, softmax_temperature, results_on_valid, results_on_valid_tracked) + + print(f'Draw --> Valid Selection metric: {results_on_valid[-1]}') + + if N_grad_steps > 0: + gradient_optimize_result = gradient_optimize_style(model, style, N_grad_steps + , softmax_temperature=softmax_temperature + , model=model[2] + , train_datasets=train_datasets + , valid_datasets=valid_datasets + , selection_metric_min_max=selection_metric_min_max + , **kwargs) + optimization_routes += [gradient_optimize_result['optimization_route']] + + evaluate_valid(gradient_optimize_result['best_style'] + , gradient_optimize_result['best_temperature'] + , results_on_valid, results_on_valid_tracked) + + print(f'After diff --> Valid Selection metric: {results_on_valid[-1]}') + + if selection_metric_min_max == 'min': + is_best = (results_on_valid_tracked[-1] <= min(results_on_valid_tracked)) + else: + is_best = (results_on_valid_tracked[-1] >= max(results_on_valid_tracked)) + + if is_best or best_style is None: + best_style = gradient_optimize_result['best_style'].clone() + best_softmax_temperature = gradient_optimize_result['best_temperature'].clone() + torch.cuda.empty_cache() + + def final_evaluation(): + print('Running eval dataset with final params (no gradients)..') + print(best_style, best_softmax_temperature) + result_test = [] + for N_ensemble_configurations in N_ensemble_configurations_list: + print(f'Running with {N_ensemble_configurations} ensemble_configurations') + kwargs['N_ensemble_configurations'] = N_ensemble_configurations + splits = [] + for split in final_splits: + splits += [eval_step(test_datasets, best_style, softmax_temperature=best_softmax_temperature + , return_tensor=False, eval_positions=eval_positions_test, + bptt=bptt_final, inference_mode=True, split_number=split, model=model[2] + , selection_metric=selection_metric, evaluation_metric=evaluation_metric)] + result_test += [splits] + + print('Running valid dataset with final params (no gradients)..') + result_valid = eval_step(valid_datasets, best_style, softmax_temperature=best_softmax_temperature + , return_tensor=False, eval_positions=eval_positions_test, + bptt=bptt_final, inference_mode=True, model=model[2] + , selection_metric=selection_metric, evaluation_metric=evaluation_metric) + + return result_test, result_valid + + result_test, result_valid = final_evaluation() + + return result_test, result_valid, best_style, best_softmax_temperature, optimization_routes + + +def eval_step(ds, used_style, selection_metric, evaluation_metric, eval_positions, return_tensor=True, **kwargs): + def step(): + return evaluate(datasets=ds, + method='transformer' + , overwrite=True + , style=used_style + , eval_positions=eval_positions + , metric_used=selection_metric + , save=False + , path_interfix=None + , base_path=None + , verbose=True + , **kwargs) + + if return_tensor: + r = step() + else: + with torch.no_grad(): + r = step() + + calculate_score_per_method(selection_metric, 'select', r, ds, eval_positions, aggregator='mean') + calculate_score_per_method(evaluation_metric, 'eval', r, ds, eval_positions, aggregator='mean') + + return r + + +def gradient_optimize_style(model, init_style, steps, softmax_temperature, train_datasets, valid_datasets, learning_rate=0.03, optimize_all=False, + limit_style=True, N_datasets_sampled=90, optimize_softmax_temperature=True, selection_metric_min_max='max', **kwargs): + """ + Uses gradient based methods to optimize 'style' on the 'train_datasets' and uses stopping with 'valid_datasets'. + + :param model: + :param init_style: + :param steps: + :param learning_rate: + :param softmax_temperature: + :param train_datasets: + :param valid_datasets: + :param optimize_all: + :param limit_style: + :param N_datasets_sampled: + :param optimize_softmax_temperature: + :param selection_metric_min_max: + :param kwargs: + :return: + """ + grad_style = torch.nn.Parameter(init_style.detach(), requires_grad=True) + + best_style, best_temperature, best_selection_metric, best_diffable_metric = grad_style.detach(), softmax_temperature.detach(), None, None + softmax_temperature = torch.nn.Parameter(softmax_temperature.detach(), requires_grad=optimize_softmax_temperature) + variables_to_optimize = model[2].parameters() if optimize_all else [grad_style, softmax_temperature] + optimizer = torch.optim.Adam(variables_to_optimize, lr=learning_rate) + + optimization_route_selection, optimization_route_diffable = [], [] + optimization_route_selection_valid, optimization_route_diffable_valid = [], [] + + def eval_opt(ds, return_tensor=True, inference_mode=False): + result = eval_step(ds, grad_style, softmax_temperature=softmax_temperature, return_tensor=return_tensor + , inference_mode=inference_mode, model=model[2], **kwargs) + + diffable_metric = result['mean_metric'] + selection_metric = result['mean_select'] + + return diffable_metric, selection_metric + + def eval_all_datasets(datasets, propagate=True): + selection_metrics_this_step, diffable_metrics_this_step = [], [] + for ds in datasets: + diffable_metric_train, selection_metric_train = eval_opt([ds], inference_mode=(not propagate)) + if not torch.isnan(diffable_metric_train).any(): + if propagate and diffable_metric_train.requires_grad == True: + diffable_metric_train.backward() + selection_metrics_this_step += [selection_metric_train] + diffable_metrics_this_step += [float(diffable_metric_train.detach().cpu().numpy())] + diffable_metric_train = np.nanmean(diffable_metrics_this_step) + selection_metric_train = np.nanmean(selection_metrics_this_step) + + return diffable_metric_train, selection_metric_train + + for t in tqdm(range(steps), desc='Iterate over Optimization steps'): + optimizer.zero_grad() + + # Select subset of datasets + random.seed(t) + train_datasets_ = random.sample(train_datasets, N_datasets_sampled) + + # Get score on train + diffable_metric_train, selection_metric_train = eval_all_datasets(train_datasets_, propagate=True) + optimization_route_selection += [float(selection_metric_train)] + optimization_route_diffable += [float(diffable_metric_train)] + + # Get score on valid + diffable_metric_valid, selection_metric_valid = eval_all_datasets(valid_datasets, propagate=False) + optimization_route_selection_valid += [float(selection_metric_valid)] + optimization_route_diffable_valid += [float(diffable_metric_valid)] + + is_best = (selection_metric_min_max == 'min' and best_selection_metric > selection_metric_valid) + is_best = is_best or (selection_metric_min_max == 'max' and best_selection_metric < selection_metric_valid) + if (best_selection_metric is None) or (not np.isnan(selection_metric_valid) and is_best): + print('New best', best_selection_metric, selection_metric_valid) + best_style = grad_style.detach().clone() + best_temperature = softmax_temperature.detach().clone() + best_selection_metric, best_diffable_metric = selection_metric_valid, diffable_metric_valid + + optimizer.step() + + if limit_style: + grad_style = grad_style.detach().clamp(-1.74, 1.74) + + print(f'Valid: Diffable metric={diffable_metric_valid} Selection metric={selection_metric_valid};' + + f'Train: Diffable metric={diffable_metric_train} Selection metric={selection_metric_train}') + + print(f'Return best:{best_style} {best_selection_metric}') + return {'best_style': best_style, 'best_temperature': best_temperature + , 'optimization_route': {'select': optimization_route_selection, 'loss': optimization_route_diffable, + 'test_select': optimization_route_selection_valid, 'test_loss': optimization_route_diffable_valid}} \ No newline at end of file diff --git a/TabPFN/encoders.py b/TabPFN/encoders.py new file mode 100644 index 0000000000000000000000000000000000000000..72885327200d5e3c026d0715568cd0f1c1f35ac4 --- /dev/null +++ b/TabPFN/encoders.py @@ -0,0 +1,225 @@ +import math + +import torch +import torch.nn as nn +from utils import normalize_data +import torch.nn.functional as F +from torch.nn import TransformerEncoder, TransformerEncoderLayer + + +class StyleEncoder(nn.Module): + def __init__(self, em_size, hyperparameter_definitions): + super().__init__() + # self.embeddings = {} + self.em_size = em_size + # self.hyperparameter_definitions = {} + # for hp in hyperparameter_definitions: + # self.embeddings[hp] = nn.Linear(1, self.em_size) + # self.embeddings = nn.ModuleDict(self.embeddings) + self.embedding = nn.Linear(hyperparameter_definitions.shape[0], self.em_size) + + def forward(self, hyperparameters): # T x B x num_features + # Make faster by using matrices + # sampled_embeddings = [torch.stack([ + # self.embeddings[hp](torch.tensor([batch[hp]], device=self.embeddings[hp].weight.device, dtype=torch.float)) + # for hp in batch + # ], -1).sum(-1) for batch in hyperparameters] + # return torch.stack(sampled_embeddings, 0) + return self.embedding(hyperparameters) + + +class _PositionalEncoding(nn.Module): + def __init__(self, d_model, dropout=0.): + super().__init__() + self.dropout = nn.Dropout(p=dropout) + self.d_model = d_model + self.device_test_tensor = nn.Parameter(torch.tensor(1.)) + + def forward(self, x):# T x B x num_features + assert self.d_model % x.shape[-1]*2 == 0 + d_per_feature = self.d_model // x.shape[-1] + pe = torch.zeros(*x.shape, d_per_feature, device=self.device_test_tensor.device) + #position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + interval_size = 10 + div_term = (1./interval_size) * 2*math.pi*torch.exp(torch.arange(0, d_per_feature, 2, device=self.device_test_tensor.device).float()*math.log(math.sqrt(2))) + #print(div_term/2/math.pi) + pe[..., 0::2] = torch.sin(x.unsqueeze(-1) * div_term) + pe[..., 1::2] = torch.cos(x.unsqueeze(-1) * div_term) + return self.dropout(pe).view(x.shape[0],x.shape[1],self.d_model) + + +Positional = lambda _, emsize: _PositionalEncoding(d_model=emsize) + +class EmbeddingEncoder(nn.Module): + def __init__(self, num_features, em_size, num_embs=100): + super().__init__() + self.num_embs = num_embs + self.embeddings = nn.Embedding(num_embs * num_features, em_size, max_norm=True) + self.init_weights(.1) + self.min_max = (-2,+2) + + @property + def width(self): + return self.min_max[1] - self.min_max[0] + + def init_weights(self, initrange): + self.embeddings.weight.data.uniform_(-initrange, initrange) + + def discretize(self, x): + split_size = self.width / self.num_embs + return (x - self.min_max[0] // split_size).int().clamp(0, self.num_embs - 1) + + def forward(self, x): # T x B x num_features + x_idxs = self.discretize(x) + x_idxs += torch.arange(x.shape[-1], device=x.device).view(1, 1, -1) * self.num_embs + # print(x_idxs,self.embeddings.weight.shape) + return self.embeddings(x_idxs).mean(-2) + + +class Normalize(nn.Module): + def __init__(self, mean, std): + super().__init__() + self.mean = mean + self.std = std + + def forward(self, x): + return (x-self.mean)/self.std + + +def get_normalized_uniform_encoder(encoder_creator): + """ + This can be used to wrap an encoder that is fed uniform samples in [0,1] and normalizes these to 0 mean and 1 std. + For example, it can be used as `encoder_creator = get_normalized_uniform_encoder(encoders.Linear)`, now this can + be initialized with `encoder_creator(feature_dim, in_dim)`. + :param encoder: + :return: + """ + return lambda in_dim, out_dim: nn.Sequential(Normalize(.5, math.sqrt(1/12)), encoder_creator(in_dim, out_dim)) + + +Linear = nn.Linear +MLP = lambda num_features, emsize: nn.Sequential(nn.Linear(num_features+1,emsize*2), + nn.ReLU(), + nn.Linear(emsize*2,emsize)) + +class NanHandlingEncoder(nn.Module): + def __init__(self, num_features, emsize, keep_nans=True): + super().__init__() + self.num_features = 2 * num_features if keep_nans else num_features + self.emsize = emsize + self.keep_nans = keep_nans + self.layer = nn.Linear(self.num_features, self.emsize) + + def forward(self, x): + if self.keep_nans: + x = torch.cat([torch.nan_to_num(x, nan=0.0), normalize_data(torch.isnan(x) * -1 + + torch.logical_and(torch.isinf(x), torch.sign(x) == 1) * 1 + + torch.logical_and(torch.isinf(x), torch.sign(x) == -1) * 2 + )], -1) + else: + x = torch.nan_to_num(x, nan=0.0) + return self.layer(x) + +class Linear(nn.Linear): + def __init__(self, num_features, emsize): + super().__init__(num_features, emsize) + self.num_features = num_features + self.emsize = emsize + + def forward(self, x): + x = torch.nan_to_num(x, nan=0.0) + return super().forward(x) + +class SequenceSpanningEncoder(nn.Module): + # Regular Encoder transforms Seq_len, B, S -> Seq_len, B, E attending only to last dimension + # This Encoder accesses the Seq_Len dimension additionally + + # Why would we want this? We can learn normalization and embedding of features + # , this might be more important for e.g. categorical, ordinal feats, nan detection + # However maybe this can be easily learned through transformer as well? + # A problem is to make this work across any sequence length and be independent of ordering + + # We could use average and maximum pooling and use those with a linear layer + + + # Another idea !! Similar to this we would like to encode features so that their number is variable + # We would like to embed features, also using knowledge of the features in the entire sequence + + # We could use convolution or another transformer + # Convolution: + + # Transformer/Conv across sequence dimension that encodes and normalizes features + # -> Transformer across feature dimension that encodes features to a constant size + + # Conv with flexible features but no sequence info: S,B,F -(reshape)-> S*B,1,F + # -(Conv1d)-> S*B,N,F -(AvgPool,MaxPool)-> S*B,N,1 -> S,B,N + # This probably won't work since it's missing a way to recognize which feature is encoded + + # Transformer with flexible features: S,B,F -> F,B*S,1 -> F2,B*S,1 -> S,B,F2 + + def __init__(self, num_features, em_size): + super().__init__() + + raise NotImplementedError() + # Seq_len, B, S -> Seq_len, B, E + # + self.convs = torch.nn.ModuleList([nn.Conv1d(64 if i else 1, 64, 3) for i in range(5)]) + # self.linear = nn.Linear(64, emsize) + +class TransformerBasedFeatureEncoder(nn.Module): + def __init__(self, num_features, emsize): + super().__init__() + + hidden_emsize = emsize + encoder = Linear(1, hidden_emsize) + n_out = emsize + nhid = 2*emsize + dropout =0.0 + nhead=4 + nlayers=4 + model = nn.Transformer(nhead=nhead, num_encoder_layers=4, num_decoder_layers=4, d_model=1) + + def forward(self, *input): + # S,B,F -> F,S*B,1 -> F2,S*B,1 -> S,B,F2 + input = input.transpose() + self.model(input) + +class Conv(nn.Module): + def __init__(self, input_size, emsize): + super().__init__() + self.convs = torch.nn.ModuleList([nn.Conv2d(64 if i else 1, 64, 3) for i in range(5)]) + self.linear = nn.Linear(64,emsize) + + + def forward(self, x): + size = math.isqrt(x.shape[-1]) + assert size*size == x.shape[-1] + x = x.reshape(*x.shape[:-1], 1, size, size) + for conv in self.convs: + if x.shape[-1] < 4: + break + x = conv(x) + x.relu_() + x = nn.AdaptiveAvgPool2d((1,1))(x).squeeze(-1).squeeze(-1) + return self.linear(x) + + + + +class CanEmb(nn.Embedding): + def __init__(self, num_features, num_embeddings: int, embedding_dim: int, *args, **kwargs): + assert embedding_dim % num_features == 0 + embedding_dim = embedding_dim // num_features + super().__init__(num_embeddings, embedding_dim, *args, **kwargs) + + def forward(self, x): + lx = x.long() + assert (lx == x).all(), "CanEmb only works with tensors of whole numbers" + x = super().forward(lx) + return x.view(*x.shape[:-2], -1) + +def get_Canonical(num_classes): + return lambda num_features, emsize: CanEmb(num_features, num_classes, emsize) + +def get_Embedding(num_embs_per_feature=100): + return lambda num_features, emsize: EmbeddingEncoder(num_features, emsize, num_embs=num_embs_per_feature) diff --git a/TabPFN/initializers.py b/TabPFN/initializers.py new file mode 100644 index 0000000000000000000000000000000000000000..4a2de2711a62676223950c35e5ce88cabcb086a0 --- /dev/null +++ b/TabPFN/initializers.py @@ -0,0 +1,9 @@ +import torch +from torch import nn + +def get_NormalInitializer(std): + def initializer(m): + if isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, std) + nn.init.normal_(m.bias, 0, std) + return initializer \ No newline at end of file diff --git a/TabPFN/layer.py b/TabPFN/layer.py new file mode 100644 index 0000000000000000000000000000000000000000..3354d3b137263542d2fc8ace85da2d2a740b10e4 --- /dev/null +++ b/TabPFN/layer.py @@ -0,0 +1,125 @@ +from functools import partial + +from torch import nn +from torch.nn.modules.transformer import * +from torch.nn.modules.transformer import _get_activation_fn + +from torch.utils.checkpoint import checkpoint + + +class TransformerEncoderLayer(Module): + r"""TransformerEncoderLayer is made up of self-attn and feedforward network. + This standard encoder layer is based on the paper "Attention Is All You Need". + Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, + Lukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. In Advances in + Neural Information Processing Systems, pages 6000-6010. Users may modify or implement + in a different way during application. + + Args: + d_model: the number of expected features in the input (required). + nhead: the number of heads in the multiheadattention models (required). + dim_feedforward: the dimension of the feedforward network model (default=2048). + dropout: the dropout value (default=0.1). + activation: the activation function of intermediate layer, relu or gelu (default=relu). + layer_norm_eps: the eps value in layer normalization components (default=1e-5). + batch_first: If ``True``, then the input and output tensors are provided + as (batch, seq, feature). Default: ``False``. + + Examples:: + >>> encoder_layer = nn.TransformerEncoderLayer(d_model=512, nhead=8) + >>> src = torch.rand(10, 32, 512) + >>> out = encoder_layer(src) + + Alternatively, when ``batch_first`` is ``True``: + >>> encoder_layer = nn.TransformerEncoderLayer(d_model=512, nhead=8, batch_first=True) + >>> src = torch.rand(32, 10, 512) + >>> out = encoder_layer(src) + """ + __constants__ = ['batch_first'] + + def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu", + layer_norm_eps=1e-5, batch_first=False, pre_norm=False, + device=None, dtype=None, recompute_attn=False) -> None: + factory_kwargs = {'device': device, 'dtype': dtype} + super().__init__() + self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout, batch_first=batch_first, + **factory_kwargs) + # Implementation of Feedforward model + self.linear1 = Linear(d_model, dim_feedforward, **factory_kwargs) + self.dropout = Dropout(dropout) + self.linear2 = Linear(dim_feedforward, d_model, **factory_kwargs) + + self.norm1 = LayerNorm(d_model, eps=layer_norm_eps, **factory_kwargs) + self.norm2 = LayerNorm(d_model, eps=layer_norm_eps, **factory_kwargs) + self.dropout1 = Dropout(dropout) + self.dropout2 = Dropout(dropout) + self.pre_norm = pre_norm + self.recompute_attn = recompute_attn + + self.activation = _get_activation_fn(activation) + + def __setstate__(self, state): + if 'activation' not in state: + state['activation'] = F.relu + super().__setstate__(state) + + def forward(self, src: Tensor, src_mask: Optional[Tensor] = None, src_key_padding_mask: Optional[Tensor] = None) -> Tensor: + r"""Pass the input through the encoder layer. + + Args: + src: the sequence to the encoder layer (required). + src_mask: the mask for the src sequence (optional). + src_key_padding_mask: the mask for the src keys per batch (optional). + + Shape: + see the docs in Transformer class. + """ + if self.pre_norm: + src_ = self.norm1(src) + else: + src_ = src + if isinstance(src_mask, tuple): + # global attention setup + assert not self.self_attn.batch_first + assert src_key_padding_mask is None + + global_src_mask, trainset_src_mask, valset_src_mask = src_mask + + num_global_tokens = global_src_mask.shape[0] + num_train_tokens = trainset_src_mask.shape[0] + + global_tokens_src = src_[:num_global_tokens] + train_tokens_src = src_[num_global_tokens:num_global_tokens+num_train_tokens] + global_and_train_tokens_src = src_[:num_global_tokens+num_train_tokens] + eval_tokens_src = src_[num_global_tokens+num_train_tokens:] + + + attn = partial(checkpoint, self.self_attn) if self.recompute_attn else self.self_attn + + global_tokens_src2 = attn(global_tokens_src, global_and_train_tokens_src, global_and_train_tokens_src, None, True, global_src_mask)[0] + train_tokens_src2 = attn(train_tokens_src, global_tokens_src, global_tokens_src, None, True, trainset_src_mask)[0] + eval_tokens_src2 = attn(eval_tokens_src, src_, src_, + None, True, valset_src_mask)[0] + + src2 = torch.cat([global_tokens_src2, train_tokens_src2, eval_tokens_src2], dim=0) + + else: + if self.recompute_attn: + src2 = checkpoint(self.self_attn, src_, src_, src_, src_key_padding_mask, True, src_mask)[0] + else: + src2 = self.self_attn(src_, src_, src_, attn_mask=src_mask, + key_padding_mask=src_key_padding_mask)[0] + src = src + self.dropout1(src2) + if not self.pre_norm: + src = self.norm1(src) + + if self.pre_norm: + src_ = self.norm2(src) + else: + src_ = src + src2 = self.linear2(self.dropout(self.activation(self.linear1(src_)))) + src = src + self.dropout2(src2) + + if not self.pre_norm: + src = self.norm2(src) + return src \ No newline at end of file diff --git a/TabPFN/losses.py b/TabPFN/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..734d8f7ab3b6613eb2fba9feffdfaa46215106af --- /dev/null +++ b/TabPFN/losses.py @@ -0,0 +1,41 @@ +import torch +from torch import nn + +class CrossEntropyForMulticlassLoss(torch.nn.CrossEntropyLoss): + # This loss applies cross entropy after reducing the number of prediction + # dimensions to the number of classes in the target + + # TODO: loss.item() doesn't work so the displayed losses are Nans + def __init__(self, num_classes, weight=None, size_average=None, ignore_index: int = -100, + reduce=None, reduction: str = 'mean', label_smoothing: float = 0.0) -> None: + super().__init__(size_average=size_average, reduce=reduce, reduction=reduction, ignore_index=ignore_index) + self.num_classes = num_classes + + def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + loss = torch.zeros_like(input[:, :, 0]) + for b in range(target.shape[1]): + l = super().forward(input[:, b, 0:len(torch.unique(target[:, b]))], target[:, b]) + loss[:, b] += l + return loss.flatten() + +def JointBCELossWithLogits(output, target): + # output shape: (S, B, NS) with NS = Number of sequences + # target shape: (S, B, SL) + # Loss = -log(mean_NS(prod_SL(p(target_SL, output_NS)))) + # Here at the moment NS = SL + output = output.unsqueeze(-1).repeat(1, 1, 1, target.shape[-1]) # (S, B, NS, SL) + output = output.permute(2, 0, 1, 3) # (NS, S, B, SL) + print(target.shape, output.shape) + loss = (target * torch.sigmoid(output)) + ((1-target) * (1-torch.sigmoid(output))) + loss = loss.prod(-1) + loss = loss.mean(0) + loss = -torch.log(loss) + loss = loss.mean() + return loss + +class ScaledSoftmaxCE(nn.Module): + def forward(self, x, label): + logits = x[..., :-10] + temp_scales = x[..., -10:] + + logprobs = logits.softmax(-1) diff --git a/TabPFN/model_builder.py b/TabPFN/model_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..df134764925c430b9177d063da9d96f0ddef28fa --- /dev/null +++ b/TabPFN/model_builder.py @@ -0,0 +1,273 @@ +from train import train, Losses +import priors +import encoders + +from collections import defaultdict + +from priors.utils import trunc_norm_sampler_f, gamma_sampler_f +from utils import get_uniform_single_eval_pos_sampler +import torch +import math + +def save_model(model, path, filename, config_sample): + config_sample = {**config_sample} + + def make_serializable(config_sample): + if isinstance(config_sample, dict): + config_sample = {k: make_serializable(config_sample[k]) for k in config_sample} + if isinstance(config_sample, list): + config_sample = [make_serializable(v) for v in config_sample] + if callable(config_sample): + config_sample = str(config_sample) + return config_sample + + #if 'num_features_used' in config_sample: + # del config_sample['num_features_used'] + + #config_sample['num_classes_as_str'] = str(config_sample['num_classes']) + #del config_sample['num_classes'] + + config_sample = make_serializable(config_sample) + + torch.save((model.state_dict(), None, config_sample), os.path.join(path, filename)) + + +import subprocess as sp +import os + +def get_gpu_memory(): + command = "nvidia-smi" + memory_free_info = sp.check_output(command.split()).decode('ascii') + return memory_free_info + + +def load_model(path, filename, device, eval_positions, verbose): + # TODO: This function only restores evaluation functionality but training canät be continued. It is also not flexible. + + model_state, optimizer_state, config_sample = torch.load( + os.path.join(path, filename), map_location='cpu') + if ('differentiable_hyperparameters' in config_sample + and 'prior_mlp_activations' in config_sample['differentiable_hyperparameters']): + config_sample['differentiable_hyperparameters']['prior_mlp_activations']['choice_values_used'] = config_sample[ + 'differentiable_hyperparameters'][ + 'prior_mlp_activations'][ + 'choice_values'] + config_sample['differentiable_hyperparameters']['prior_mlp_activations']['choice_values'] = [ + torch.nn.Tanh for k in config_sample['differentiable_hyperparameters']['prior_mlp_activations']['choice_values']] + + config_sample['categorical_features_sampler'] = lambda: lambda x: ([], [], []) + config_sample['num_features_used_in_training'] = config_sample['num_features_used'] + config_sample['num_features_used'] = lambda: config_sample['num_features'] + config_sample['num_classes_in_training'] = config_sample['num_classes'] + config_sample['num_classes'] = 2 + config_sample['batch_size_in_training'] = config_sample['batch_size'] + config_sample['batch_size'] = 1 + config_sample['bptt_in_training'] = config_sample['bptt'] + config_sample['bptt'] = 10 + config_sample['bptt_extra_samples_in_training'] = config_sample['bptt_extra_samples'] + config_sample['bptt_extra_samples'] = None + + #print('Memory', str(get_gpu_memory())) + + model = get_model(config_sample, device=device, should_train=False, verbose=verbose) + module_prefix = 'module.' + model_state = {k.replace(module_prefix, ''): v for k, v in model_state.items()} + model[2].load_state_dict(model_state) + model[2].to(device) + + return model, config_sample + +def fix_loaded_config_sample(loaded_config_sample, config): + def copy_to_sample(*k): + t,s = loaded_config_sample, config + for k_ in k[:-1]: + t = t[k_] + s = s[k_] + t[k[-1]] = s[k[-1]] + copy_to_sample('num_features_used') + copy_to_sample('num_classes') + copy_to_sample('differentiable_hyperparameters','prior_mlp_activations','choice_values') + +def load_config_sample(path, template_config): + model_state, optimizer_state, loaded_config_sample = torch.load(path, map_location='cpu') + fix_loaded_config_sample(loaded_config_sample, template_config) + return loaded_config_sample + +def get_default_spec(test_datasets, valid_datasets): + bptt = 10000 + eval_positions = [1000, 2000, 3000, 4000, 5000] # list(2 ** np.array([4, 5, 6, 7, 8, 9, 10, 11, 12])) + max_features = max([X.shape[1] for (_, X, _, _, _, _) in test_datasets] + [X.shape[1] for (_, X, _, _, _, _) in valid_datasets]) + max_splits = 5 + + return bptt, eval_positions, max_features, max_splits + +def get_mlp_prior_hyperparameters(config): + config = {hp: (list(config[hp].values())[0]) if type(config[hp]) is dict else config[hp] for hp in config} + + if "prior_sigma_gamma_k" in config: + sigma_sampler = gamma_sampler_f(config["prior_sigma_gamma_k"], config["prior_sigma_gamma_theta"]) + config['init_std'] = sigma_sampler + if "prior_noise_std_gamma_k" in config: + noise_std_sampler = gamma_sampler_f(config["prior_noise_std_gamma_k"], config["prior_noise_std_gamma_theta"]) + config['noise_std'] = noise_std_sampler + + return config + + +def get_gp_mix_prior_hyperparameters(config): + return {'lengthscale_concentration': config["prior_lengthscale_concentration"], + 'nu': config["prior_nu"], + 'outputscale_concentration': config["prior_outputscale_concentration"], + 'categorical_data': config["prior_y_minmax_norm"], + 'y_minmax_norm': config["prior_lengthscale_concentration"], + 'noise_concentration': config["prior_noise_concentration"], + 'noise_rate': config["prior_noise_rate"]} + +def get_gp_prior_hyperparameters(config): + return {hp: (list(config[hp].values())[0]) if type(config[hp]) is dict else config[hp] for hp in config} + + +def get_meta_gp_prior_hyperparameters(config): + config = {hp: (list(config[hp].values())[0]) if type(config[hp]) is dict else config[hp] for hp in config} + + if "outputscale_mean" in config: + outputscale_sampler = trunc_norm_sampler_f(config["outputscale_mean"] + , config["outputscale_mean"] * config["outputscale_std_f"]) + config['outputscale'] = outputscale_sampler + if "lengthscale_mean" in config: + lengthscale_sampler = trunc_norm_sampler_f(config["lengthscale_mean"], + config["lengthscale_mean"] * config["lengthscale_std_f"]) + config['lengthscale'] = lengthscale_sampler + + return config + + +def get_model(config, device, should_train=True, verbose=False, state_dict=None, epoch_callback=None): + extra_kwargs = {} + verbose_train, verbose_prior = verbose >= 1, verbose >= 2 + config['verbose'] = verbose_prior + + if 'aggregate_k_gradients' not in config or config['aggregate_k_gradients'] is None: + config['aggregate_k_gradients'] = math.ceil(config['batch_size'] * ((config['nlayers'] * config['emsize'] * config['bptt'] * config['bptt']) / 10824640000)) + + config['num_steps'] = math.ceil(config['num_steps'] * config['aggregate_k_gradients']) + config['batch_size'] = math.ceil(config['batch_size'] / config['aggregate_k_gradients']) + config['recompute_attn'] = config['recompute_attn'] if 'recompute_attn' in config else False + + def make_get_batch(model_proto, **extra_kwargs): + extra_kwargs = defaultdict(lambda: None, **extra_kwargs) + return (lambda batch_size, seq_len, num_features, hyperparameters + , device, model_proto=model_proto, get_batch=extra_kwargs['get_batch'] + , prior_bag_priors=extra_kwargs['prior_bag_priors']: model_proto.get_batch( + batch_size=batch_size + , seq_len=seq_len + , device=device + , get_batch=get_batch + , hyperparameters=hyperparameters + , num_features=num_features)) + + if config['prior_type'] == 'prior_bag': + # Prior bag combines priors + get_batch_gp = make_get_batch(priors.fast_gp) + get_batch_mlp = make_get_batch(priors.mlp) + if 'flexible' in config and config['flexible']: + get_batch_gp = make_get_batch(priors.flexible_categorical, **{'get_batch': get_batch_gp}) + get_batch_mlp = make_get_batch(priors.flexible_categorical, **{'get_batch': get_batch_mlp}) + prior_bag_hyperparameters = {'prior_bag_get_batch': (get_batch_gp, get_batch_mlp) + , 'prior_bag_exp_weights_1': 2.0} + prior_hyperparameters = {**get_mlp_prior_hyperparameters(config), **get_gp_prior_hyperparameters(config) + , **prior_bag_hyperparameters} + model_proto = priors.prior_bag + else: + if config['prior_type'] == 'mlp': + prior_hyperparameters = get_mlp_prior_hyperparameters(config) + model_proto = priors.mlp + elif config['prior_type'] == 'gp': + prior_hyperparameters = get_gp_prior_hyperparameters(config) + model_proto = priors.fast_gp + elif config['prior_type'] == 'gp_mix': + prior_hyperparameters = get_gp_mix_prior_hyperparameters(config) + model_proto = priors.fast_gp_mix + else: + raise Exception() + + if 'flexible' in config and config['flexible']: + get_batch_base = make_get_batch(model_proto) + extra_kwargs['get_batch'] = get_batch_base + model_proto = priors.flexible_categorical + + use_style = False + + if 'differentiable' in config and config['differentiable']: + get_batch_base = make_get_batch(model_proto, **extra_kwargs) + extra_kwargs = {'get_batch': get_batch_base, 'differentiable_hyperparameters': config['differentiable_hyperparameters']} + model_proto = priors.differentiable_prior + use_style = True + print(f"Using style prior: {use_style}") + + if (('nan_prob_no_reason' in config and config['nan_prob_no_reason'] > 0.0) or + ('nan_prob_a_reason' in config and config['nan_prob_a_reason'] > 0.0) or + ('nan_prob_unknown_reason' in config and config['nan_prob_unknown_reason'] > 0.0)): + encoder = encoders.NanHandlingEncoder + else: + encoder = encoders.Linear + + num_outputs = config['num_outputs'] if 'num_outputs' in config else 1 + if config['max_num_classes'] == 2: + if 'joint_loss' in config and config['joint_loss']: + loss = JointBCELossWithLogits + else: + loss = Losses.bce + elif config['max_num_classes'] > 2: + loss = Losses.ce(torch.ones((config['max_num_classes']))) + else: + loss = BarDistribution(borders=get_bucket_limits(500, full_range=(-10, 10))) + + aggregate_k_gradients = 1 if 'aggregate_k_gradients' not in config else config['aggregate_k_gradients'] + check_is_compatible = False if 'multiclass_loss_type' not in config else (config['multiclass_loss_type'] == 'compatible') + config['multiclass_type'] = config['multiclass_type'] if 'multiclass_type' in config else 'rank' + config['mix_activations'] = config['mix_activations'] if 'mix_activations' in config else False + + config['bptt_extra_samples'] = config['bptt_extra_samples'] if 'bptt_extra_samples' in config else None + config['eval_positions'] = [int(config['bptt'] * 0.95)] if config['bptt_extra_samples'] is None else [int(config['bptt'])] + + epochs = 0 if not should_train else config['epochs'] + model = train(model_proto.DataLoader + , loss + , encoder + , style_encoder_generator = encoders.StyleEncoder if use_style else None + , emsize=config['emsize'] + , nhead=config['nhead'] + , y_encoder_generator= encoders.get_Canonical(config['max_num_classes']) if config.get('canonical_y_encoder', False) else encoders.Linear + , pos_encoder_generator=None + , batch_size=config['batch_size'] + , nlayers=config['nlayers'] + , nhid=config['emsize'] * config['nhid_factor'] + , epochs=epochs + , total_available_time_in_s=config.get('total_available_time_in_s', None) + , warmup_epochs=20 + , bptt=config['bptt'] + , gpu_device=device + , dropout=config['dropout'] + , steps_per_epoch=config['num_steps'] + , single_eval_pos_gen=get_uniform_single_eval_pos_sampler(config['bptt']) + , load_weights_from_this_state_dict=state_dict + , aggregate_k_gradients=aggregate_k_gradients + , check_is_compatible=check_is_compatible + , recompute_attn=config['recompute_attn'] + , epoch_callback=epoch_callback + , bptt_extra_samples = config['bptt_extra_samples'] + , extra_prior_kwargs_dict={ + 'num_features': config['num_features'] + , 'fuse_x_y': False + , 'hyperparameters': prior_hyperparameters + , 'num_outputs':num_outputs + , 'dynamic_batch_size': 1 if ('num_global_att_tokens' in config and config['num_global_att_tokens']) else 2 + , **extra_kwargs + } + , lr=config['lr'] + , verbose=verbose_train, + weight_decay=config.get('weight_decay', 0.0), + normalize_labels=True) + + return model \ No newline at end of file diff --git a/TabPFN/models_diff/gp_ablation_model.cpkt b/TabPFN/models_diff/gp_ablation_model.cpkt new file mode 100644 index 0000000000000000000000000000000000000000..196f8b2d9405851cfc61680c229eb43c6499876f --- /dev/null +++ b/TabPFN/models_diff/gp_ablation_model.cpkt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7b0c8febc553cca3fdee265b5a1cd7567dbf83da855969940be4707a9218ffb +size 69460013 diff --git a/TabPFN/models_diff/prior_diff_real_checkpoint_n_8x_lr0.0003_epoch_49.cpkt b/TabPFN/models_diff/prior_diff_real_checkpoint_n_8x_lr0.0003_epoch_49.cpkt new file mode 100644 index 0000000000000000000000000000000000000000..23987f7b71cc06e37f69ac3be1a6c63951bc48cb --- /dev/null +++ b/TabPFN/models_diff/prior_diff_real_checkpoint_n_8x_lr0.0003_epoch_49.cpkt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dae97f45bd53d719fc2b23fac4ec55eab16d63892196d939b1bb1c3b408be242 +size 103616779 diff --git a/TabPFN/notebook_utils.py b/TabPFN/notebook_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c085c98f9865f993163bc1cfd29fd895f6a27b27 --- /dev/null +++ b/TabPFN/notebook_utils.py @@ -0,0 +1,32 @@ +import os +from pathlib import Path + +import io +import torch +import pickle + +def print_models(base_path, model_string): + print(model_string) + + for i in range(80): + for e in range(50): + exists = Path(os.path.join(base_path, f'models_diff/prior_diff_real_checkpoint{model_string}_n_{i}_epoch_{e}.cpkt')).is_file() + if exists: + print(os.path.join(base_path, f'models_diff/prior_diff_real_checkpoint{model_string}_n_{i}_epoch_{e}.cpkt')) + print() + +class CustomUnpickler(pickle.Unpickler): + def find_class(self, module, name): + if name == 'Manager': + from settings import Manager + return Manager + try: + return self.find_class_cpu(module, name) + except: + return None + + def find_class_cpu(self, module, name): + if module == 'torch.storage' and name == '_load_from_bytes': + return lambda b: torch.load(io.BytesIO(b), map_location='cpu') + else: + return super().find_class(module, name) \ No newline at end of file diff --git a/TabPFN/positional_encodings.py b/TabPFN/positional_encodings.py new file mode 100644 index 0000000000000000000000000000000000000000..05580e052d6bb1fe782441e7e65088f7989e8e0b --- /dev/null +++ b/TabPFN/positional_encodings.py @@ -0,0 +1,70 @@ +import math + +import torch +from torch import nn + + +# Protocol for positonal encodings. +# __init__(d_model, max_len=..[, more optionals]) +# forward(x: (seq_len, bs, d_model)) -> Tensor of shape (*x.shape[:2],d_model) containing pos. embeddings + + +class NoPositionalEncoding(nn.Module): + def __init__(self, d_model, max_len=None): + super(NoPositionalEncoding, self).__init__() + pass + + def forward(self, x): + return x #* math.sqrt(x.shape[-1]) + + +class PositionalEncoding(nn.Module): + def __init__(self, d_model, max_len=5000): + super(PositionalEncoding, self).__init__() + pe = torch.zeros(max_len, d_model) + position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0).transpose(0, 1) + self.register_buffer('pe', pe) + + def forward(self, x): + x = self.pe[:x.size(0), :] + x # * math.sqrt(x.shape[-1]) + return x + + +class LearnedPositionalEncoding(nn.Module): + def __init__(self, d_model, max_len=5000): + super(LearnedPositionalEncoding, self).__init__() + self.max_seq_len = max_len + #self.positional_embeddings = nn.Embedding(max_len, d_model) + self.positional_embeddings = nn.Parameter(torch.empty(max_len, d_model)) + nn.init.normal_(self.positional_embeddings, mean=0, std=d_model ** -0.5) + + def forward(self, x): + seq_len, bs, d_model = x.shape + assert seq_len <= len(self.positional_embeddings), 'seq_len can be at most max_len.' + pos_emb = self.positional_embeddings[:seq_len] + return pos_emb.unsqueeze(1).expand(seq_len, bs, d_model) + x #* math.sqrt(x.shape[-1]) + + +class PairedScrambledPositionalEncodings(LearnedPositionalEncoding): + # TODO check whether it is a problem to use the same perm. for full batch + def forward(self, x): + seq_len, bs, d_model = x.shape + assert seq_len <= len(self.positional_embeddings), 'seq_len can be at most max_len.' + assert len(self.positional_embeddings) % 2 == 0, 'Please specify an even max_len.' + + paired_embs = self.positional_embeddings.view(len(self.positional_embeddings), -1, 2) + pos_emb = paired_embs[torch.randperm(len(paired_embs))].view(*self.positional_embeddings.shape)[:seq_len] + + return pos_emb.unsqueeze(1).expand(seq_len, bs, d_model) + x #* math.sqrt(x.shape[-1]) + + + + + + + + diff --git a/TabPFN/prior_tuning_result.pkl b/TabPFN/prior_tuning_result.pkl new file mode 100644 index 0000000000000000000000000000000000000000..3c8ccbb3e791393060cd87ca014df27fb2f38947 --- /dev/null +++ b/TabPFN/prior_tuning_result.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24d2189bbc836aeea888cf6c540f2c1b45b5351822931189e8bf10a0bc80a0b6 +size 18668851 diff --git a/TabPFN/priors/__init__.py b/TabPFN/priors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3407398b08379f975aa59cb35e731b82d2a50360 --- /dev/null +++ b/TabPFN/priors/__init__.py @@ -0,0 +1,4 @@ +from . import fast_gp, mlp, flexible_categorical, differentiable_prior, prior_bag + + + diff --git a/TabPFN/priors/__pycache__/__init__.cpython-37.pyc b/TabPFN/priors/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5614b244decae44f20078e0b4d938f585e1886b9 Binary files /dev/null and b/TabPFN/priors/__pycache__/__init__.cpython-37.pyc differ diff --git a/TabPFN/priors/__pycache__/__init__.cpython-38.pyc b/TabPFN/priors/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2cfceae4c7220b46522e15f01431d6c0070430c2 Binary files /dev/null and b/TabPFN/priors/__pycache__/__init__.cpython-38.pyc differ diff --git a/TabPFN/priors/__pycache__/differentiable_prior.cpython-37.pyc b/TabPFN/priors/__pycache__/differentiable_prior.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe09b4d8901b3345447456907f421f3220c77d8f Binary files /dev/null and b/TabPFN/priors/__pycache__/differentiable_prior.cpython-37.pyc differ diff --git a/TabPFN/priors/__pycache__/fast_gp.cpython-37.pyc b/TabPFN/priors/__pycache__/fast_gp.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d7103ee496f54ded51c1a4188e86655a2845698f Binary files /dev/null and b/TabPFN/priors/__pycache__/fast_gp.cpython-37.pyc differ diff --git a/TabPFN/priors/__pycache__/fast_gp.cpython-38.pyc b/TabPFN/priors/__pycache__/fast_gp.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b736dd8c5ac7506644b53a26cce52c2b7302f8c7 Binary files /dev/null and b/TabPFN/priors/__pycache__/fast_gp.cpython-38.pyc differ diff --git a/TabPFN/priors/__pycache__/flexible_categorical.cpython-37.pyc b/TabPFN/priors/__pycache__/flexible_categorical.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d734c07230fca0cef320eb6f54db980d7d6f0b02 Binary files /dev/null and b/TabPFN/priors/__pycache__/flexible_categorical.cpython-37.pyc differ diff --git a/TabPFN/priors/__pycache__/mlp.cpython-37.pyc b/TabPFN/priors/__pycache__/mlp.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba6fc92b418307f3a4b3d8d9c76b49047953cdc7 Binary files /dev/null and b/TabPFN/priors/__pycache__/mlp.cpython-37.pyc differ diff --git a/TabPFN/priors/__pycache__/prior.cpython-37.pyc b/TabPFN/priors/__pycache__/prior.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5823fdc3c01ca0b849ccf9019fcb3d279dcb7680 Binary files /dev/null and b/TabPFN/priors/__pycache__/prior.cpython-37.pyc differ diff --git a/TabPFN/priors/__pycache__/prior_bag.cpython-37.pyc b/TabPFN/priors/__pycache__/prior_bag.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca9f4453a0551afe639e4235d447d0a903f23929 Binary files /dev/null and b/TabPFN/priors/__pycache__/prior_bag.cpython-37.pyc differ diff --git a/TabPFN/priors/__pycache__/utils.cpython-37.pyc b/TabPFN/priors/__pycache__/utils.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d2e452dfaa762966c4e48f1ab43a42c7d8db6d76 Binary files /dev/null and b/TabPFN/priors/__pycache__/utils.cpython-37.pyc differ diff --git a/TabPFN/priors/differentiable_prior.py b/TabPFN/priors/differentiable_prior.py new file mode 100644 index 0000000000000000000000000000000000000000..631d684880012d5af5327f097c70fcc4b5bf4b6f --- /dev/null +++ b/TabPFN/priors/differentiable_prior.py @@ -0,0 +1,293 @@ +import torch +from torch import nn +import math + +from .utils import get_batch_to_dataloader +from utils import default_device +from .utils import order_by_y, normalize_by_used_features_f + +from .utils import trunc_norm_sampler_f, beta_sampler_f, gamma_sampler_f, uniform_sampler_f, zipf_sampler_f, scaled_beta_sampler_f, uniform_int_sampler_f + + +def unpack_dict_of_tuples(d): + # Returns list of dicts where each dict i contains values of tuple position i + # {'a': (1,2), 'b': (3,4)} -> [{'a': 1, 'b': 3}, {'a': 2, 'b': 4}] + return [dict(zip(d.keys(), v)) for v in list(zip(*list(d.values())))] + +class DifferentiableHyperparameter(nn.Module): + ## We can sample this and get a hyperparameter value and a normalized hyperparameter indicator + def __init__(self, distribution, embedding_dim, device, **args): + super(DifferentiableHyperparameter, self).__init__() + + self.distribution = distribution + self.embedding_dim = embedding_dim + self.device=device + for key in args: + setattr(self, key, args[key]) + + def get_sampler(): + #if self.distribution == "beta": + # return beta_sampler_f(self.a, self.b), 0, 1 + #elif self.distribution == "gamma": + # return gamma_sampler_f(self.a, self.b), 0, 1 + #elif self.distribution == "beta_int": + # return scaled_beta_sampler_f(self.a, self.b, self.scale, self.min), self.scale + self.min, self.min, self.a / (self.a + self.b) + if self.distribution == "uniform": + if not hasattr(self, 'sample'): + return uniform_sampler_f(self.min, self.max), self.min, self.max, (self.max+self.min) / 2, math.sqrt(1/12*(self.max-self.min)*(self.max-self.min)) + else: + return lambda: self.sample, self.min, self.max, None, None + elif self.distribution == "uniform_int": + return uniform_int_sampler_f(self.min, self.max), self.min, self.max, (self.max+self.min) / 2, math.sqrt(1/12*(self.max-self.min)*(self.max-self.min)) + + if self.distribution.startswith("meta"): + self.hparams = {} + def sample_meta(f): + indicators, passed = unpack_dict_of_tuples({hp: self.hparams[hp]() for hp in self.hparams}) + # sampled_embeddings = list(itertools.chain.from_iterable([sampled_embeddings[k] for k in sampled_embeddings])) + meta_passed = f(**passed) + return indicators, meta_passed + + args_passed = {'device': device, 'embedding_dim': embedding_dim} + if self.distribution == "meta_beta": + ## Truncated normal where std and mean are drawn randomly logarithmically scaled + if hasattr(self, 'b') and hasattr(self, 'k'): + self.hparams = {'b': lambda: (None, self.b), 'k': lambda: (None, self.k)} + else: + self.hparams = {"b": DifferentiableHyperparameter(distribution="uniform", min=self.min + , max=self.max, **args_passed) + , "k": DifferentiableHyperparameter(distribution="uniform", min=self.min + , max=self.max, **args_passed)} + def make_beta(b, k): + return lambda b=b, k=k: self.scale * beta_sampler_f(b, k)() + self.sampler = lambda make_beta=make_beta : sample_meta(make_beta) + elif self.distribution == "meta_trunc_norm_log_scaled": + # these choices are copied down below, don't change these without changing `replace_differentiable_distributions` + self.min_std = self.min_std if hasattr(self, 'min_std') else 0.001 + self.max_std = self.max_std if hasattr(self, 'max_std') else self.max_mean + ## Truncated normal where std and mean are drawn randomly logarithmically scaled + if not hasattr(self, 'log_mean'): + self.hparams = {"log_mean": DifferentiableHyperparameter(distribution="uniform", min=math.log(self.min_mean) + , max=math.log(self.max_mean), **args_passed) + , "log_std": DifferentiableHyperparameter(distribution="uniform", min=math.log(self.min_std) + , max=math.log(self.max_std), **args_passed)} + else: + self.hparams = {'log_mean': lambda: (None, self.log_mean), 'log_std': lambda: (None, self.log_std)} + def make_trunc_norm(log_mean, log_std): + return ((lambda : self.lower_bound + round(trunc_norm_sampler_f(math.exp(log_mean), math.exp(log_std))())) if self.round + else (lambda: self.lower_bound + trunc_norm_sampler_f(math.exp(log_mean), math.exp(log_std))())) + + self.sampler = lambda make_trunc_norm=make_trunc_norm: sample_meta(make_trunc_norm) + elif self.distribution == "meta_trunc_norm": + self.min_std = self.min_std if hasattr(self, 'min_std') else 0 + self.max_std = self.max_std if hasattr(self, 'max_std') else self.max_mean + self.hparams = {"mean": DifferentiableHyperparameter(distribution="uniform", min=self.min_mean + , max=self.max_mean, **args_passed) + , "std": DifferentiableHyperparameter(distribution="uniform", min=self.min_std + , max=self.max_std, **args_passed)} + def make_trunc_norm(mean, std): + return ((lambda: self.lower_bound + round( + trunc_norm_sampler_f(math.exp(mean), math.exp(std))())) if self.round + else ( + lambda make_trunc_norm=make_trunc_norm: self.lower_bound + trunc_norm_sampler_f(math.exp(mean), math.exp(std))())) + self.sampler = lambda : sample_meta(make_trunc_norm) + elif self.distribution == "meta_choice": + if hasattr(self, 'choice_1_weight'): + self.hparams = {f'choice_{i}_weight': lambda: (None, getattr(self, f'choice_{i}_weight')) for i in range(1, len(self.choice_values))} + else: + self.hparams = {f"choice_{i}_weight": DifferentiableHyperparameter(distribution="uniform", min=-5.0 + , max=6.0, **args_passed) for i in range(1, len(self.choice_values))} + def make_choice(**choices): + weights = torch.softmax(torch.tensor([1.0] + [choices[i] for i in choices], dtype=torch.float), 0) # create a tensor of weights + sample = torch.multinomial(weights, 1, replacement=True).numpy()[0] + return self.choice_values[sample] + + self.sampler = lambda make_choice=make_choice: sample_meta(make_choice) + elif self.distribution == "meta_choice_mixed": + if hasattr(self, 'choice_1_weight'): + self.hparams = {f'choice_{i}_weight': lambda: (None, getattr(self, f'choice_{i}_weight')) for i in range(1, len(self.choice_values))} + else: + self.hparams = {f"choice_{i}_weight": DifferentiableHyperparameter(distribution="uniform", min=-5.0 + , max=6.0, **args_passed) for i in range(1, len(self.choice_values))} + def make_choice(**choices): + weights = torch.softmax(torch.tensor([1.0] + [choices[i] for i in choices], dtype=torch.float), 0) # create a tensor of weights + def sample(): + s = torch.multinomial(weights, 1, replacement=True).numpy()[0] + return self.choice_values[s]() + return lambda: sample + + self.sampler = lambda make_choice=make_choice: sample_meta(make_choice) + else: + def return_two(x, min, max, mean, std): + # Returns (a hyperparameter value, and an indicator value passed to the model) + if mean is not None: + ind = (x-mean)/std#(2 * (x-min) / (max-min) - 1) + else: + ind = None + return ind, x # normalize indicator to [-1, 1] + # def sample_standard(sampler_f, embedding): + # s = torch.tensor([sampler_f()], device = self.device) + # return s, embedding(s) + self.sampler_f, self.sampler_min, self.sampler_max, self.sampler_mean, self.sampler_std = get_sampler() + self.sampler = lambda : return_two(self.sampler_f(), min=self.sampler_min, max=self.sampler_max + , mean=self.sampler_mean, std=self.sampler_std) + # self.embedding_layer = nn.Linear(1, self.embedding_dim, device=self.device) + # self.embed = lambda x : self.embedding_layer( + # (x - self.sampler_min) / (self.sampler_max - self.sampler_min)) + #self.sampler = lambda : sample_standard(self.sampler_f, self.embedding) + + + def forward(self): + s, s_passed = self.sampler() + return s, s_passed + + + +class DifferentiableHyperparameterList(nn.Module): + def __init__(self, hyperparameters, embedding_dim, device): + super().__init__() + + self.device = device + hyperparameters = {k: v for (k, v) in hyperparameters.items() if v} + self.hyperparameters = nn.ModuleDict({hp: DifferentiableHyperparameter(embedding_dim = embedding_dim + , name = hp + , device = device, **hyperparameters[hp]) for hp in hyperparameters}) + def get_hyperparameter_info(self): + sampled_hyperparameters_f, sampled_hyperparameters_keys = [], [] + def append_hp(hp_key, hp_val): + sampled_hyperparameters_keys.append(hp_key) + # Function remaps hyperparameters from [-1, 1] range to true value + s_min, s_max, s_mean, s_std = hp_val.sampler_min, hp_val.sampler_max, hp_val.sampler_mean, hp_val.sampler_std + sampled_hyperparameters_f.append((lambda x: (x-s_mean)/s_std, lambda y : (y * s_std)+s_mean)) + #sampled_hyperparameters_f.append(((lambda x: ((x - s_min) / (s_max - s_min) * (2) - 1) + # , (lambda y: ((y + 1) * (1 / 2) * (s_max - s_min) + s_min)))) + for hp in self.hyperparameters: + hp_val = self.hyperparameters[hp] + if hasattr(hp_val, 'hparams'): + for hp_ in hp_val.hparams: + append_hp(f'{hp}_{hp_}', hp_val.hparams[hp_]) + else: + append_hp(hp, hp_val) + + + return sampled_hyperparameters_keys, sampled_hyperparameters_f + + def sample_parameter_object(self): + sampled_hyperparameters, s_passed = {}, {} + for hp in self.hyperparameters: + sampled_hyperparameters_, s_passed_ = self.hyperparameters[hp]() + s_passed[hp] = s_passed_ + if isinstance(sampled_hyperparameters_, dict): + sampled_hyperparameters_ = {hp + '_' + str(key): val for key, val in sampled_hyperparameters_.items()} + sampled_hyperparameters.update(sampled_hyperparameters_) + else: + sampled_hyperparameters[hp] = sampled_hyperparameters_ + + # s_passed contains the values passed to the get_batch function + # sampled_hyperparameters contains the indicator of the sampled value, i.e. only number that describe the sampled object + return s_passed, sampled_hyperparameters#self.pack_parameter_object(sampled_embeddings) + +class DifferentiablePrior(torch.nn.Module): + def __init__(self, get_batch, hyperparameters, differentiable_hyperparameters, args): + super(DifferentiablePrior, self).__init__() + + self.h = hyperparameters + self.args = args + self.get_batch = get_batch + self.differentiable_hyperparameters = DifferentiableHyperparameterList(differentiable_hyperparameters + , embedding_dim=self.h['emsize'] + , device=self.args['device']) + + def forward(self): + # Sample hyperparameters + sampled_hyperparameters_passed, sampled_hyperparameters_indicators = self.differentiable_hyperparameters.sample_parameter_object() + + hyperparameters = {**self.h, **sampled_hyperparameters_passed} + x, y, y_ = self.get_batch(hyperparameters=hyperparameters, **self.args) + + return x, y, y_, sampled_hyperparameters_indicators + + +# TODO: Make this a class that keeps objects +@torch.no_grad() +def get_batch(batch_size, seq_len, num_features, get_batch + , device=default_device, differentiable_hyperparameters={} + , hyperparameters=None, batch_size_per_gp_sample=None, **kwargs): + batch_size_per_gp_sample = batch_size_per_gp_sample or (min(64, batch_size)) + num_models = batch_size // batch_size_per_gp_sample + assert num_models * batch_size_per_gp_sample == batch_size, f'Batch size ({batch_size}) not divisible by batch_size_per_gp_sample ({batch_size_per_gp_sample})' + + args = {'device': device, 'seq_len': seq_len, 'num_features': num_features, 'batch_size': batch_size_per_gp_sample} + + models = [DifferentiablePrior(get_batch, hyperparameters, differentiable_hyperparameters, args) for _ in range(num_models)] + sample = sum([[model()] for model in models], []) + + x, y, y_, hyperparameter_dict = zip(*sample) + + if 'verbose' in hyperparameters and hyperparameters['verbose']: + print('Hparams', hyperparameter_dict[0].keys()) + + hyperparameter_matrix = [] + for batch in hyperparameter_dict: + hyperparameter_matrix.append([batch[hp] for hp in batch]) + + transposed_hyperparameter_matrix = list(zip(*hyperparameter_matrix)) + assert all([all([hp is None for hp in hp_]) or all([hp is not None for hp in hp_]) for hp_ in transposed_hyperparameter_matrix]), 'it should always be the case that when a hyper-parameter is None, once it is always None' + # we remove columns that are only None (i.e. not sampled) + hyperparameter_matrix = [[hp for hp in hp_ if hp is not None] for hp_ in hyperparameter_matrix] + if len(hyperparameter_matrix[0]) > 0: + packed_hyperparameters = torch.tensor(hyperparameter_matrix) + packed_hyperparameters = torch.repeat_interleave(packed_hyperparameters, repeats=batch_size_per_gp_sample, dim=0).detach() + else: + packed_hyperparameters = None + + x, y, y_, packed_hyperparameters = (torch.cat(x, 1).detach() + , torch.cat(y, 1).detach() + , torch.cat(y_, 1).detach() + , packed_hyperparameters)#list(itertools.chain.from_iterable(itertools.repeat(x, batch_size_per_gp_sample) for x in packed_hyperparameters)))#torch.repeat_interleave(torch.stack(packed_hyperparameters, 0).detach(), repeats=batch_size_per_gp_sample, dim=0)) + + return x, y, y_, packed_hyperparameters + +DataLoader = get_batch_to_dataloader(get_batch) +DataLoader.num_outputs = 1 +#DataLoader.validate = lambda : 0 + +def draw_random_style(dl, device): + (hp_embedding, data, targets_), targets = next(iter(dl)) + return hp_embedding.to(device)[0:1, :] + +def merge_style_with_info(diff_hparams_keys, diff_hparams_f, style, transform=True): + params = dict(zip(diff_hparams_keys, zip(diff_hparams_f, style.detach().cpu().numpy().tolist()[0]))) + def t(v): + if transform: + return v[0][1](v[1]) + else: + return v[1] + return {k : t(v) for k, v in params.items()} + + +import ConfigSpace.hyperparameters as CSH + +def replace_differentiable_distributions(config): + diff_config = config['differentiable_hyperparameters'] + for name, diff_hp_dict in diff_config.items(): + distribution = diff_hp_dict['distribution'] + if distribution == 'uniform': + diff_hp_dict['sample'] = CSH.UniformFloatHyperparameter(name, diff_hp_dict['min'], diff_hp_dict['max']) + elif distribution == 'meta_beta': + diff_hp_dict['k'] = CSH.UniformFloatHyperparameter(name+'_k', diff_hp_dict['min'], diff_hp_dict['max']) + diff_hp_dict['b'] = CSH.UniformFloatHyperparameter(name+'_b', diff_hp_dict['min'], diff_hp_dict['max']) + elif distribution == 'meta_choice': + for i in range(1, len(diff_hp_dict['choice_values'])): + diff_hp_dict[f'choice_{i}_weight'] = CSH.UniformFloatHyperparameter(name+f'choice_{i}_weight', -5.0, 6.0) + elif distribution == 'meta_choice_mixed': + for i in range(1, len(diff_hp_dict['choice_values'])): + diff_hp_dict[f'choice_{i}_weight'] = CSH.UniformFloatHyperparameter(name+f'choice_{i}_weight', -5.0, 6.0) + elif distribution == 'meta_trunc_norm_log_scaled': + diff_hp_dict['log_mean'] = CSH.UniformFloatHyperparameter(name+'_log_mean', math.log(diff_hp_dict['min_mean']), math.log(diff_hp_dict['max_mean'])) + min_std = diff_hp_dict['min_std'] if 'min_std' in diff_hp_dict else 0.001 + max_std = diff_hp_dict['max_std'] if 'max_std' in diff_hp_dict else diff_hp_dict['max_mean'] + diff_hp_dict['log_std'] = CSH.UniformFloatHyperparameter(name+'_log_std', math.log(min_std), math.log(max_std)) + else: + raise ValueError(f'Unknown distribution {distribution}') + diff --git a/TabPFN/priors/fast_gp.py b/TabPFN/priors/fast_gp.py new file mode 100644 index 0000000000000000000000000000000000000000..1c4b061c547d9ec5c17135b9f58562bbb4a54da1 --- /dev/null +++ b/TabPFN/priors/fast_gp.py @@ -0,0 +1,144 @@ +import time + +import torch +from torch import nn +import gpytorch + +from .utils import get_batch_to_dataloader +from utils import default_device + + +# We will use the simplest form of GP model, exact inference +class ExactGPModel(gpytorch.models.ExactGP): + def __init__(self, train_x, train_y, likelihood): + super(ExactGPModel, self).__init__(train_x, train_y, likelihood) + self.mean_module = gpytorch.means.ConstantMean() + self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel()) + + def forward(self, x): + mean_x = self.mean_module(x) + covar_x = self.covar_module(x) + return gpytorch.distributions.MultivariateNormal(mean_x, covar_x) + + +def get_model(x, y, hyperparameters): + likelihood = gpytorch.likelihoods.GaussianLikelihood(noise_constraint=gpytorch.constraints.GreaterThan(1.e-9)) + model = ExactGPModel(x, y, likelihood) + model.likelihood.noise = torch.ones_like(model.likelihood.noise) * hyperparameters["noise"] + model.covar_module.outputscale = torch.ones_like(model.covar_module.outputscale) * hyperparameters["outputscale"] + model.covar_module.base_kernel.lengthscale = torch.ones_like(model.covar_module.base_kernel.lengthscale) * \ + hyperparameters["lengthscale"] + return model, likelihood + + +@torch.no_grad() +def get_batch(batch_size, seq_len, num_features, device=default_device, hyperparameters=None, + equidistant_x=False, fix_x=None, **kwargs): + if isinstance(hyperparameters, (tuple, list)): + hyperparameters = {"noise": hyperparameters[0] + , "outputscale": hyperparameters[1] + , "lengthscale": hyperparameters[2] + , "is_binary_classification": hyperparameters[3] + # , "num_features_used": hyperparameters[4] + , "normalize_by_used_features": hyperparameters[5] + , "order_y": hyperparameters[6] + , "sampling": hyperparameters[7] + } + elif hyperparameters is None: + hyperparameters = {"noise": .1, "outputscale": .1, "lengthscale": .1} + + if 'verbose' in hyperparameters and hyperparameters['verbose']: + print({"noise": hyperparameters['noise'], "outputscale": hyperparameters['outputscale'] + , "lengthscale": hyperparameters['lengthscale'], 'batch_size': batch_size, 'sampling': hyperparameters['sampling']}) + + # hyperparameters = {k: hyperparameters[k]() if callable(hyperparameters[k]) else hyperparameters[k] for k in + # hyperparameters.keys()} + assert not (equidistant_x and (fix_x is not None)) + + with gpytorch.settings.fast_computations(*hyperparameters.get('fast_computations', (True, True, True))): + if equidistant_x: + assert num_features == 1 + x = torch.linspace(0, 1., seq_len).unsqueeze(0).repeat(batch_size, 1).unsqueeze(-1) + elif fix_x is not None: + assert fix_x.shape == (seq_len, num_features) + x = fix_x.unsqueeze(0).repeat(batch_size, 1, 1).to(device) + else: + if hyperparameters.get('sampling','uniform') == 'uniform': + x = torch.rand(batch_size, seq_len, num_features, device=device) + else: + x = torch.randn(batch_size, seq_len, num_features, device=device) + model, likelihood = get_model(x, torch.Tensor(), hyperparameters) + model.to(device) + # trained_model = ExactGPModel(train_x, train_y, likelihood).cuda() + # trained_model.eval() + is_fitted = False + while not is_fitted: + try: + with gpytorch.settings.prior_mode(True): + model, likelihood = get_model(x, torch.Tensor(), hyperparameters) + model.to(device) + + d = model(x) + d = likelihood(d) + sample = d.sample().transpose(0, 1) + is_fitted = True + except RuntimeError: # This can happen when torch.linalg.eigh fails. Restart with new init resolves this. + print('GP Fitting unsuccessful, retrying.. ') + print(x) + print(hyperparameters) + + if bool(torch.any(torch.isnan(x)).detach().cpu().numpy()): + print({"noise": hyperparameters['noise'], "outputscale": hyperparameters['outputscale'] + , "lengthscale": hyperparameters['lengthscale'], 'batch_size': batch_size}) + + # TODO: Multi output + return x.transpose(0, 1), sample, sample # x.shape = (T,B,H) + +DataLoader = get_batch_to_dataloader(get_batch) +DataLoader.num_outputs = 1 + +def get_model_on_device(x,y,hyperparameters,device): + model, likelihood = get_model(x, y, hyperparameters) + model.to(device) + return model, likelihood + + +@torch.no_grad() +def evaluate(x, y, y_non_noisy, use_mse=False, hyperparameters={}, get_model_on_device=get_model_on_device, device=default_device, step_size=1, start_pos=0): + start_time = time.time() + losses_after_t = [.0] if start_pos == 0 else [] + all_losses_after_t = [] + + with gpytorch.settings.fast_computations(*hyperparameters.get('fast_computations',(True,True,True))), gpytorch.settings.fast_pred_var(False): + for t in range(max(start_pos, 1), len(x), step_size): + loss_sum = 0. + model, likelihood = get_model_on_device(x[:t].transpose(0, 1), y[:t].transpose(0, 1), hyperparameters, device) + + + model.eval() + # print([t.shape for t in model.train_inputs]) + # print(x[:t].transpose(0,1).shape, x[t].unsqueeze(1).shape, y[:t].transpose(0,1).shape) + f = model(x[t].unsqueeze(1)) + l = likelihood(f) + means = l.mean.squeeze() + varis = l.covariance_matrix.squeeze() + # print(l.variance.squeeze(), l.mean.squeeze(), y[t]) + + assert len(means.shape) == len(varis.shape) == 1 + assert len(means) == len(varis) == x.shape[1] + + if use_mse: + c = nn.MSELoss(reduction='none') + ls = c(means, y[t]) + else: + ls = -l.log_prob(y[t].unsqueeze(1)) + + losses_after_t.append(ls.mean()) + all_losses_after_t.append(ls.flatten()) + return torch.stack(all_losses_after_t).to('cpu'), torch.tensor(losses_after_t).to('cpu'), time.time() - start_time + +if __name__ == '__main__': + hps = (.1,.1,.1) + for redo_idx in range(1): + print( + evaluate(*get_batch(1000, 10, hyperparameters=hps, num_features=10), use_mse=False, hyperparameters=hps)) diff --git a/TabPFN/priors/flexible_categorical.py b/TabPFN/priors/flexible_categorical.py new file mode 100644 index 0000000000000000000000000000000000000000..b24a83d49018fc7ebd62f803ceec643de9bc206e --- /dev/null +++ b/TabPFN/priors/flexible_categorical.py @@ -0,0 +1,240 @@ +import time +import random + +import torch +from torch import nn + +from .utils import get_batch_to_dataloader +from utils import normalize_data, nan_handling_missing_for_unknown_reason_value, nan_handling_missing_for_no_reason_value, nan_handling_missing_for_a_reason_value, to_ranking_low_mem, remove_outliers +from .utils import normalize_by_used_features_f, randomize_classes, CategoricalActivation +from .utils import uniform_int_sampler_f + +time_it = False + +class BalancedBinarize(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x): + return (x > torch.median(x)).float() + +def class_sampler_f(min_, max_): + def s(): + if random.random() > 0.5: + return uniform_int_sampler_f(min_, max_)() + return 2 + return s + +class MulticlassRank(nn.Module): + def __init__(self, num_classes, ordered_p=0.5): + super().__init__() + self.num_classes = class_sampler_f(2, num_classes)() + self.ordered_p = ordered_p + + def forward(self, x): + # x has shape (T,B,H) + + # CAUTION: This samples the same idx in sequence for each class boundary in a batch + class_boundaries = torch.randint(0, x.shape[0], (self.num_classes - 1,)) + class_boundaries = x[class_boundaries].unsqueeze(1) + + d = (x > class_boundaries).sum(axis=0) + + randomized_classes = torch.rand((d.shape[1], )) > self.ordered_p + d[:, randomized_classes] = randomize_classes(d[:, randomized_classes], self.num_classes) + reverse_classes = torch.rand((d.shape[1],)) > 0.5 + d[:, reverse_classes] = self.num_classes - 1 - d[:, reverse_classes] + return d + +class MulticlassValue(nn.Module): + def __init__(self, num_classes, ordered_p=0.5): + super().__init__() + self.num_classes = class_sampler_f(2, num_classes)() + self.classes = nn.Parameter(torch.randn(num_classes-1), requires_grad=False) + self.ordered_p = ordered_p + + def forward(self, x): + # x has shape (T,B,H) + d = (x > (self.classes.unsqueeze(-1).unsqueeze(-1))).sum(axis=0) + + randomized_classes = torch.rand((d.shape[1],)) > self.ordered_p + d[:, randomized_classes] = randomize_classes(d[:, randomized_classes], self.num_classes) + reverse_classes = torch.rand((d.shape[1],)) > 0.5 + d[:, reverse_classes] = self.num_classes - 1 - d[:, reverse_classes] + return d + +class MulticlassMultiNode(nn.Module): + def __init__(self, num_classes, ordered_p=0.5): + super().__init__() + self.num_classes = class_sampler_f(2, num_classes)() + self.classes = nn.Parameter(torch.randn(num_classes-1), requires_grad=False) + self.alt_multi_class = MulticlassValue(num_classes, ordered_p) + + def forward(self, x): + # x has shape T, B, H + if len(x.shape) == 2: + return self.alt_multi_class(x) + T = 3 + x[torch.isnan(x)] = 0.00001 + d = torch.multinomial(torch.pow(0.00001+torch.sigmoid(x[:, :, 0:self.num_classes]).reshape(-1, self.num_classes), T), 1, replacement=True).reshape(x.shape[0], x.shape[1]).float() + return d + + +class FlexibleCategorical(torch.nn.Module): + def __init__(self, get_batch, hyperparameters, args): + super(FlexibleCategorical, self).__init__() + + self.h = {k: hyperparameters[k]() if callable(hyperparameters[k]) else hyperparameters[k] for k in + hyperparameters.keys()} + self.args = args + self.args_passed = {**self.args} + self.args_passed.update({'num_features': self.h['num_features_used']}) + self.get_batch = get_batch + + if self.h['num_classes'] > 1 and not self.h['balanced']: + if self.h['multiclass_type'] == 'rank': + self.class_assigner = MulticlassRank(self.h['num_classes'] + , ordered_p=self.h['output_multiclass_ordered_p'] + ) + elif self.h['multiclass_type'] == 'value': + self.class_assigner = MulticlassValue(self.h['num_classes'] + , ordered_p=self.h['output_multiclass_ordered_p'] + ) + elif self.h['multiclass_type'] == 'multi_node': + self.class_assigner = MulticlassMultiNode(self.h['num_classes']) + else: + raise ValueError("Unknow Multiclass type") + elif self.h['num_classes'] == 2 and self.h['balanced']: + self.class_assigner = BalancedBinarize() + elif self.h['num_classes'] > 2 and self.h['balanced']: + raise NotImplementedError("Balanced multiclass training is not possible") + else: + self.class_assigner = lambda x:x # Regression + + def drop_for_reason(self, x, v): + nan_prob_sampler = CategoricalActivation(ordered_p=0.0 + , categorical_p=1.0 + , keep_activation_size=False, + num_classes_sampler=lambda: 20) + d = nan_prob_sampler(x) + # TODO: Make a different ordering for each activation + x[d < torch.rand((1,), device=x.device) * 20 * self.h['nan_prob_no_reason'] * random.random()] = v + return x + + def drop_for_no_reason(self, x, v): + x[torch.rand(x.shape, device=self.args['device']) < self.h['nan_prob_no_reason']] = v + return x + + def forward(self, batch_size): + start = time.time() + x, y, y_ = self.get_batch(hyperparameters=self.h, **self.args_passed) + if time_it: + print('Flex Forward Block 1', round(time.time() - start, 3)) + + start = time.time() + + if self.h['nan_prob_no_reason']+self.h['nan_prob_a_reason']+self.h['nan_prob_unknown_reason'] > 0 and random.random() > 0.5: # Only one out of two datasets should have nans + if self.h['nan_prob_no_reason'] > 0 and random.random() > 0.5: # Missing for no reason + x = self.drop_for_no_reason(x, nan_handling_missing_for_no_reason_value(self.h['set_value_to_nan'])) + + if self.h['nan_prob_a_reason'] > 0 and random.random() > 0.5: # Missing for a reason + x = self.drop_for_reason(x, nan_handling_missing_for_a_reason_value(self.h['set_value_to_nan'])) + + if self.h['nan_prob_unknown_reason'] > 0: # Missing for unknown reason and random.random() > 0.5 + if random.random() < self.h['nan_prob_unknown_reason_reason_prior']: + x = self.drop_for_no_reason(x, nan_handling_missing_for_unknown_reason_value(self.h['set_value_to_nan'])) + else: + x = self.drop_for_reason(x, nan_handling_missing_for_unknown_reason_value(self.h['set_value_to_nan'])) + + # Categorical features + if 'categorical_feature_p' in self.h and random.random() > 1 - self.h['categorical_feature_p']: + p = random.random() + for col in range(x.shape[2]): + m = MulticlassRank(10, ordered_p=0.3) + if random.random() > p: + x[:, :, col] = m(x[:, :, col]) + + if time_it: + print('Flex Forward Block 2', round(time.time() - start, 3)) + start = time.time() + + if self.h['normalize_to_ranking']: + x = to_ranking_low_mem(x) + else: + x = remove_outliers(x) + x, y = normalize_data(x), normalize_data(y) + + if time_it: + print('Flex Forward Block 3', round(time.time() - start, 3)) + start = time.time() + + # Cast to classification if enabled + y = self.class_assigner(y).float() + + if time_it: + print('Flex Forward Block 4', round(time.time() - start, 3)) + start = time.time() + if self.h['normalize_by_used_features']: + x = normalize_by_used_features_f(x, self.h['num_features_used'], self.args['num_features'], normalize_with_sqrt=self.h.get('normalize_with_sqrt',False)) + if time_it: + print('Flex Forward Block 5', round(time.time() - start, 3)) + + start = time.time() + # Append empty features if enabled + x = torch.cat( + [x, torch.zeros((x.shape[0], x.shape[1], self.args['num_features'] - self.h['num_features_used']), + device=self.args['device'])], -1) + if time_it: + print('Flex Forward Block 6', round(time.time() - start, 3)) + + return x, y, y # x.shape = (T,B,H) + +import torch.cuda as cutorch + +@torch.no_grad() +def get_batch(batch_size, seq_len, num_features, get_batch, device, hyperparameters=None, batch_size_per_gp_sample=None, **kwargs): + batch_size_per_gp_sample = batch_size_per_gp_sample or (min(32, batch_size)) + num_models = batch_size // batch_size_per_gp_sample + assert num_models > 0, f'Batch size ({batch_size}) is too small for batch_size_per_gp_sample ({batch_size_per_gp_sample})' + assert num_models * batch_size_per_gp_sample == batch_size, f'Batch size ({batch_size}) not divisible by batch_size_per_gp_sample ({batch_size_per_gp_sample})' + + # Sample one seq_len for entire batch + seq_len = hyperparameters['seq_len_used']() if callable(hyperparameters['seq_len_used']) else seq_len + + args = {'device': device, 'seq_len': seq_len, 'num_features': num_features, 'batch_size': batch_size_per_gp_sample} + + models = [FlexibleCategorical(get_batch, hyperparameters, args).to(device) for _ in range(num_models)] + + start = time.time() + sample = sum([[model(batch_size=batch_size_per_gp_sample)] for model in models], []) + #print('sample', time.time() - start) + + x, y, y_ = zip(*sample) + x, y, y_ = torch.cat(x, 1).detach(), torch.cat(y, 1).detach(), torch.cat(y_, 1).detach() + + # # TODO: Reintegrate this code (Doesn't work on batch dim), could be applied to each batch sample individually + # if hyperparameters['is_binary_classification'] and hyperparameters['order_y']: + # x, y = order_by_y(x, y) + + return x, y, y_ + +# num_features_used = num_features_used_sampler() +# prior_outputscale = prior_outputscale_sampler() +# prior_lengthscale = prior_lengthscale_sampler() +# +# x, sample = normalize_data(x), normalize_data(sample) +# +# if is_binary_classification: +# sample = (sample > torch.median(sample, dim=0)[0]).float() +# +# if normalize_by_used_features: +# x = normalize_by_used_features_f(x, num_features_used, num_features) +# +# # # if is_binary_classification and order_y: +# # # x, sample = order_by_y(x, sample) +# # +# # Append empty features if enabled +# x = torch.cat([x, torch.zeros((x.shape[0], x.shape[1], num_features - num_features_used), device=device)], -1) + +DataLoader = get_batch_to_dataloader(get_batch) +DataLoader.num_outputs = 1 \ No newline at end of file diff --git a/TabPFN/priors/mlp.py b/TabPFN/priors/mlp.py new file mode 100644 index 0000000000000000000000000000000000000000..e489556e52196ca16463c7c8f0e25a69aaa3c630 --- /dev/null +++ b/TabPFN/priors/mlp.py @@ -0,0 +1,173 @@ +import random +import math + +import torch +from torch import nn +import numpy as np + +from utils import default_device +from .utils import get_batch_to_dataloader + +class GaussianNoise(nn.Module): + def __init__(self, std, device): + super().__init__() + self.std = std + self.device=device + + def forward(self, x): + return x + torch.normal(torch.zeros_like(x), self.std) + + +def causes_sampler_f(num_causes): + means = np.random.normal(0, 1, (num_causes)) + std = np.abs(np.random.normal(0, 1, (num_causes)) * means) + return means, std + +def get_batch(batch_size, seq_len, num_features, hyperparameters, device=default_device, num_outputs=1, sampling='normal', **kwargs): + if ('mix_activations' in hyperparameters) and hyperparameters['mix_activations']: + s = hyperparameters['prior_mlp_activations']() + hyperparameters['prior_mlp_activations'] = lambda : s + + class MLP(torch.nn.Module): + def __init__(self, hyperparameters): + super(MLP, self).__init__() + + with torch.no_grad(): + + for key in hyperparameters: + setattr(self, key, hyperparameters[key]) + + assert (self.num_layers >= 2) + + if 'verbose' in hyperparameters and self.verbose: + print({k : hyperparameters[k] for k in ['is_causal', 'num_causes', 'prior_mlp_hidden_dim' + , 'num_layers', 'noise_std', 'y_is_effect', 'pre_sample_weights', 'prior_mlp_dropout_prob' + , 'pre_sample_causes']}) + + if self.is_causal: + self.prior_mlp_hidden_dim = max(self.prior_mlp_hidden_dim, num_outputs + 2 * num_features) + else: + self.num_causes = num_features + + # This means that the mean and standard deviation of each cause is determined in advance + if self.pre_sample_causes: + self.causes_mean, self.causes_std = causes_sampler_f(self.num_causes) + self.causes_mean = torch.tensor(self.causes_mean, device=device).unsqueeze(0).unsqueeze(0).tile( + (seq_len, 1, 1)) + self.causes_std = torch.tensor(self.causes_std, device=device).unsqueeze(0).unsqueeze(0).tile( + (seq_len, 1, 1)) + + def generate_module(layer_idx, out_dim): + # Determine std of each noise term in initialization, so that is shared in runs + # torch.abs(torch.normal(torch.zeros((out_dim)), self.noise_std)) - Change std for each dimension? + noise = (GaussianNoise(torch.abs(torch.normal(torch.zeros(size=(1, out_dim), device=device), float(self.noise_std))), device=device) + if self.pre_sample_weights else GaussianNoise(float(self.noise_std), device=device)) + return [ + nn.Sequential(*[self.prior_mlp_activations() + , nn.Linear(self.prior_mlp_hidden_dim, out_dim) + , noise]) + ] + + self.layers = [nn.Linear(self.num_causes, self.prior_mlp_hidden_dim, device=device)] + self.layers += [module for layer_idx in range(self.num_layers-1) for module in generate_module(layer_idx, self.prior_mlp_hidden_dim)] + if not self.is_causal: + self.layers += generate_module(-1, num_outputs) + self.layers = nn.Sequential(*self.layers) + + # Initialize Model parameters + for i, (n, p) in enumerate(self.layers.named_parameters()): + if self.block_wise_dropout: + if len(p.shape) == 2: # Only apply to weight matrices and not bias + nn.init.zeros_(p) + # TODO: N blocks should be a setting + n_blocks = random.randint(1, math.ceil(math.sqrt(min(p.shape[0], p.shape[1])))) + w, h = p.shape[0] // n_blocks, p.shape[1] // n_blocks + keep_prob = (n_blocks*w*h) / p.numel() + for block in range(0, n_blocks): + nn.init.normal_(p[w * block: w * (block+1), h * block: h * (block+1)], std=self.init_std / keep_prob**(1/2)) + else: + if len(p.shape) == 2: # Only apply to weight matrices and not bias + dropout_prob = self.prior_mlp_dropout_prob if i > 0 else 0.0 # Don't apply dropout in first layer + dropout_prob = min(dropout_prob, 0.99) + nn.init.normal_(p, std=self.init_std / (1. - dropout_prob)**(1/2)) + p *= torch.bernoulli(torch.zeros_like(p) + 1. - dropout_prob) + + def forward(self): + def sample_normal(): + if self.pre_sample_causes: + causes = torch.normal(self.causes_mean, self.causes_std.abs()).float() + else: + causes = torch.normal(0., 1., (seq_len, 1, self.num_causes), device=device).float() + return causes + + if self.sampling == 'normal': + causes = sample_normal() + elif self.sampling == 'mixed': + zipf_p, multi_p, normal_p = random.random() * 0.66, random.random() * 0.66, random.random() * 0.66 + def sample_cause(n): + if random.random() > normal_p: + if self.pre_sample_causes: + return torch.normal(self.causes_mean[:, :, n], self.causes_std[:, :, n].abs()).float() + else: + return torch.normal(0., 1., (seq_len, 1), device=device).float() + elif random.random() > multi_p: + x = torch.multinomial(torch.rand((random.randint(2, 10))), seq_len, replacement=True).to(device).unsqueeze(-1).float() + x = (x - torch.mean(x)) / torch.std(x) + return x + else: + x = torch.minimum(torch.tensor(np.random.zipf(2.0 + random.random() * 2, size=(seq_len)), + device=device).unsqueeze(-1).float(), torch.tensor(10.0, device=device)) + return x - torch.mean(x) + causes = torch.cat([sample_cause(n).unsqueeze(-1) for n in range(self.num_causes)], -1) + elif self.sampling == 'uniform': + causes = torch.rand((seq_len, 1, self.num_causes), device=device) + else: + raise ValueError(f'Sampling is set to invalid setting: {sampling}.') + + outputs = [causes] + for layer in self.layers: + outputs.append(layer(outputs[-1])) + outputs = outputs[2:] + + if self.is_causal: + ## Sample nodes from graph if model is causal + outputs_flat = torch.cat(outputs, -1) + + if self.in_clique: + random_perm = random.randint(0, outputs_flat.shape[-1] - num_outputs - num_features) + torch.randperm(num_outputs + num_features, device=device) + else: + random_perm = torch.randperm(outputs_flat.shape[-1]-1, device=device) + + random_idx_y = list(range(-num_outputs, -0)) if self.y_is_effect else random_perm[0:num_outputs] + random_idx = random_perm[num_outputs:num_outputs + num_features] + + if self.sort_features: + random_idx, _ = torch.sort(random_idx) + y = outputs_flat[:, :, random_idx_y] + + x = outputs_flat[:, :, random_idx] + else: + y = outputs[-1][:, :, :] + x = causes + + if bool(torch.any(torch.isnan(x)).detach().cpu().numpy()) or bool(torch.any(torch.isnan(y)).detach().cpu().numpy()): + x[:] = 0.0 + y[:] = 1.0 + + return x, y + + model = MLP(hyperparameters).to(device) + + sample = sum([[model()] for _ in range(0, batch_size)], []) + + x, y = zip(*sample) + y = torch.cat(y, 1).detach().squeeze(2) + x = torch.cat(x, 1).detach() + x = x[..., torch.randperm(x.shape[-1])] + + return x, y, y + + +DataLoader = get_batch_to_dataloader(get_batch) +DataLoader.num_outputs = 1 + diff --git a/TabPFN/priors/prior.py b/TabPFN/priors/prior.py new file mode 100644 index 0000000000000000000000000000000000000000..64ef7ea7eeb8bf251a56e9dd5fac752ab46241b3 --- /dev/null +++ b/TabPFN/priors/prior.py @@ -0,0 +1,12 @@ +from torch.utils.data import DataLoader + + +class PriorDataLoader(DataLoader): + pass + # init accepts num_steps as first argument + + # has two attributes set on class or object level: + # num_features: int and + # num_outputs: int + # fuse_x_y: bool + # Optional: validate function that accepts a transformer model diff --git a/TabPFN/priors/prior_bag.py b/TabPFN/priors/prior_bag.py new file mode 100644 index 0000000000000000000000000000000000000000..c664fb2f3e18c5f2c8a36d5d9a29b60d0cbd2fd3 --- /dev/null +++ b/TabPFN/priors/prior_bag.py @@ -0,0 +1,32 @@ +import torch + +from .utils import get_batch_to_dataloader +from utils import default_device + +def get_batch(batch_size, seq_len, num_features, device=default_device + , hyperparameters=None, batch_size_per_gp_sample=None, **kwargs): + batch_size_per_gp_sample = batch_size_per_gp_sample or (min(64, batch_size)) + num_models = batch_size // batch_size_per_gp_sample + assert num_models * batch_size_per_gp_sample == batch_size, f'Batch size ({batch_size}) not divisible by batch_size_per_gp_sample ({batch_size_per_gp_sample})' + + args = {'device': device, 'seq_len': seq_len, 'num_features': num_features, 'batch_size': batch_size_per_gp_sample} + + prior_bag_priors_get_batch = hyperparameters['prior_bag_get_batch'] + prior_bag_priors_p = [1.0] + [hyperparameters[f'prior_bag_exp_weights_{i}'] for i in range(1, len(prior_bag_priors_get_batch))] + + weights = torch.tensor(prior_bag_priors_p, dtype=torch.float) # create a tensor of weights + batch_assignments = torch.multinomial(torch.softmax(weights, 0), num_models, replacement=True).numpy() + + if 'verbose' in hyperparameters and hyperparameters['verbose']: + print('PRIOR_BAG:', weights, batch_assignments) + + sample = sum([[prior_bag_priors_get_batch[int(prior_idx)](hyperparameters=hyperparameters, **args)] for prior_idx in batch_assignments], []) + + x, y, y_ = zip(*sample) + x, y, y_ = (torch.cat(x, 1).detach() + , torch.cat(y, 1).detach() + , torch.cat(y_, 1).detach()) + return x, y, y_ + +DataLoader = get_batch_to_dataloader(get_batch) +DataLoader.num_outputs = 1 \ No newline at end of file diff --git a/TabPFN/priors/utils.py b/TabPFN/priors/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..eaf236d1b58969d8086cc4f8ddba79334663f8fd --- /dev/null +++ b/TabPFN/priors/utils.py @@ -0,0 +1,163 @@ +import random + +import torch + +from utils import set_locals_in_self +from .prior import PriorDataLoader +from torch import nn +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.gridspec as gridspec +import scipy.stats as stats +import math + +def get_batch_to_dataloader(get_batch_method_): + class DL(PriorDataLoader): + get_batch_method = get_batch_method_ + + # Caution, you might need to set self.num_features manually if it is not part of the args. + def __init__(self, num_steps, fuse_x_y=False, **get_batch_kwargs): + set_locals_in_self(locals()) + # The stuff outside the or is set as class attribute before instantiation. + self.num_features = get_batch_kwargs.get('num_features') or self.num_features + self.num_outputs = get_batch_kwargs.get('num_outputs') or self.num_outputs + print('DataLoader.__dict__', self.__dict__) + + @staticmethod + def gbm(*args, fuse_x_y=True, **kwargs): + dynamic_seq_len = callable(kwargs['seq_len']) + kwargs['seq_len'] = kwargs['seq_len']() if dynamic_seq_len else kwargs['seq_len'] + # Scales the batch size dynamically with the power of 'dynamic_batch_size'. + # A transformer with quadratic memory usage in the seq len would need a power of 2 to keep memory constant. + if dynamic_seq_len and 'dynamic_batch_size' in kwargs and kwargs['dynamic_batch_size'] > 0: + kwargs['batch_size'] = kwargs['batch_size'] * math.floor(math.pow(kwargs['seq_len_maximum'], kwargs['dynamic_batch_size']) / math.pow(kwargs['seq_len'], kwargs['dynamic_batch_size'])) + batch = get_batch_method_(*args, **kwargs) + x, y, target_y, style = batch if len(batch) == 4 else (batch[0], batch[1], batch[2], None) + if fuse_x_y: + return torch.cat([x, torch.cat([torch.zeros_like(y[:1]), y[:-1]], 0).unsqueeze(-1).float()], + -1), target_y + else: + return (style, x, y), target_y + + def __len__(self): + return self.num_steps + + def __iter__(self): + return iter(self.gbm(**self.get_batch_kwargs, fuse_x_y=self.fuse_x_y) for _ in range(self.num_steps)) + + + return DL + +import seaborn as sns +def plot_features(data, targets, fig=None): + if torch.is_tensor(data): + data = data.detach().cpu().numpy() + targets = targets.detach().cpu().numpy() + #data = np.concatenate([data, data[:, -1:]], -1) + #df = pd.DataFrame(data, columns=list(range(0, data.shape[1]))) + #g = sns.pairplot(df, hue=data.shape[1]-1, palette="Set2", diag_kind="kde", height=2.5) + #plt.legend([], [], frameon=False) + #g._legend.remove() + #g = sns.PairGrid(df, hue=data.shape[1]-1) + #g.map_diag(sns.histplot) + #g.map_offdiag(sns.scatterplot) + #g._legend.remove() + + fig2 = fig if fig else plt.figure(figsize=(8, 8)) + spec2 = gridspec.GridSpec(ncols=data.shape[1], nrows=data.shape[1], figure=fig2) + for d in range(0, data.shape[1]): + for d2 in range(0, data.shape[1]): + sub_ax = fig2.add_subplot(spec2[d, d2]) + if d == d2: + sns.kdeplot(data[:, d],hue=targets[:],ax=sub_ax,legend=False, palette="deep") + sub_ax.set(ylabel=None) + else: + sns.scatterplot(x=data[:, d], y=data[:, d2], + hue=targets[:],legend=False, palette="deep") + #plt.scatter(data[:, d], data[:, d2], + # c=targets[:]) + sub_ax.get_xaxis().set_ticks([]) + sub_ax.get_yaxis().set_ticks([]) + plt.subplots_adjust(wspace=0.05, hspace=0.05) + fig2.show() + + +def plot_prior(prior): + s = np.array([prior() for _ in range(0, 1000)]) + count, bins, ignored = plt.hist(s, 50, density=True) + print(s.min()) + plt.show() + +trunc_norm_sampler_f = lambda mu, sigma : lambda: stats.truncnorm((0 - mu) / sigma, (1000000 - mu) / sigma, loc=mu, scale=sigma).rvs(1)[0] +beta_sampler_f = lambda a, b : lambda : np.random.beta(a, b) +gamma_sampler_f = lambda a, b : lambda : np.random.gamma(a, b) +uniform_sampler_f = lambda a, b : lambda : np.random.uniform(a, b) +uniform_int_sampler_f = lambda a, b : lambda : round(np.random.uniform(a, b)) +def zipf_sampler_f(a, b, c): + x = np.arange(b, c) + weights = x ** (-a) + weights /= weights.sum() + return lambda : stats.rv_discrete(name='bounded_zipf', values=(x, weights)).rvs(1) +scaled_beta_sampler_f = lambda a, b, scale, minimum : lambda : minimum + round(beta_sampler_f(a, b)() * (scale - minimum)) + + +def normalize_by_used_features_f(x, num_features_used, num_features, normalize_with_sqrt=False): + if normalize_with_sqrt: + return x / (num_features_used / num_features)**(1 / 2) + return x / (num_features_used / num_features) + + +def order_by_y(x, y): + order = torch.argsort(y if random.randint(0, 1) else -y, dim=0)[:, 0, 0] + order = order.reshape(2, -1).transpose(0, 1).reshape(-1)#.reshape(seq_len) + x = x[order] # .reshape(2, -1).transpose(0, 1).reshape(-1).flip([0]).reshape(seq_len, 1, -1) + y = y[order] # .reshape(2, -1).transpose(0, 1).reshape(-1).reshape(seq_len, 1, -1) + + return x, y + +def randomize_classes(x, num_classes): + classes = torch.arange(0, num_classes, device=x.device) + random_classes = torch.randperm(num_classes, device=x.device).type(x.type()) + x = ((x.unsqueeze(-1) == classes) * random_classes).sum(-1) + return x + + +class CategoricalActivation(nn.Module): + def __init__(self, categorical_p=0.1, ordered_p=0.7 + , keep_activation_size=False + , num_classes_sampler=zipf_sampler_f(0.8, 1, 10)): + self.categorical_p = categorical_p + self.ordered_p = ordered_p + self.keep_activation_size = keep_activation_size + self.num_classes_sampler = num_classes_sampler + + super().__init__() + + def forward(self, x): + # x shape: T, B, H + + x = nn.Softsign()(x) + + num_classes = self.num_classes_sampler() + hid_strength = torch.abs(x).mean(0).unsqueeze(0) if self.keep_activation_size else None + + categorical_classes = torch.rand((x.shape[1], x.shape[2])) < self.categorical_p + class_boundaries = torch.zeros((num_classes - 1, x.shape[1], x.shape[2]), device=x.device, dtype=x.dtype) + # Sample a different index for each hidden dimension, but shared for all batches + for b in range(x.shape[1]): + for h in range(x.shape[2]): + ind = torch.randint(0, x.shape[0], (num_classes - 1,)) + class_boundaries[:, b, h] = x[ind, b, h] + + for b in range(x.shape[1]): + x_rel = x[:, b, categorical_classes[b]] + boundaries_rel = class_boundaries[:, b, categorical_classes[b]].unsqueeze(1) + x[:, b, categorical_classes[b]] = (x_rel > boundaries_rel).sum(dim=0).float() - num_classes / 2 + + ordered_classes = torch.rand((x.shape[1],x.shape[2])) < self.ordered_p + ordered_classes = torch.logical_and(ordered_classes, categorical_classes) + x[:, ordered_classes] = randomize_classes(x[:, ordered_classes], num_classes) + + x = x * hid_strength if self.keep_activation_size else x + + return x diff --git a/TabPFN/requirements.txt b/TabPFN/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..39c7d0769d74021e6594663cb78aa8d712c7d5d4 --- /dev/null +++ b/TabPFN/requirements.txt @@ -0,0 +1,15 @@ +# Please use python V 3.7 to be compatible with all packages +gpytorch==1.5.0 +torch==1.9.0 +scikit-learn==0.24.2 +pyyaml==5.4.1 +seaborn==0.11.2 +xgboost==1.4.0 +tqdm==4.62.1 +numpy==1.21.2 +openml==0.12.2 +catboost==0.26.1 +auto-sklearn==0.14.5 +hyperopt==0.2.5 +configspace==0.4.21 +# autogluon==0.4.0 \ No newline at end of file diff --git a/TabPFN/scripts/__pycache__/transformer_prediction_interface.cpython-37.pyc b/TabPFN/scripts/__pycache__/transformer_prediction_interface.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c226ddf110bc32e7f92be1a3fa7d2a567410a296 Binary files /dev/null and b/TabPFN/scripts/__pycache__/transformer_prediction_interface.cpython-37.pyc differ diff --git a/TabPFN/scripts/__pycache__/transformer_prediction_interface.cpython-38.pyc b/TabPFN/scripts/__pycache__/transformer_prediction_interface.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ccdf40badb8c45e0b00fd3256209c3b964aca15c Binary files /dev/null and b/TabPFN/scripts/__pycache__/transformer_prediction_interface.cpython-38.pyc differ diff --git a/TabPFN/scripts/baseline_prediction_interface.py b/TabPFN/scripts/baseline_prediction_interface.py new file mode 100644 index 0000000000000000000000000000000000000000..298a046c4c3c39cbddbcdc5ee47c68606c706b2c --- /dev/null +++ b/TabPFN/scripts/baseline_prediction_interface.py @@ -0,0 +1,38 @@ +import tqdm +import numpy as np + +def baseline_predict(metric_function, eval_xs, eval_ys, categorical_feats, metric_used=None, eval_pos=2, max_time=300, **kwargs): + """ + Baseline prediction interface. + :param metric_function: + :param eval_xs: + :param eval_ys: + :param categorical_feats: + :param metric_used: + :param eval_pos: + :param max_time: Scheduled maximum time + :param kwargs: + :return: list [np.array(metrics), np.array(outputs), best_configs] or [None, None, None] if failed + """ + + metrics = [] + outputs = [] + best_configs = [] + eval_splits = list(zip(eval_xs.transpose(0, 1), eval_ys.transpose(0, 1))) + for eval_x, eval_y in tqdm.tqdm(eval_splits, desc='Calculating splits'+str(metric_function)+' '+str(eval_pos)): + try: + metric, output, best_config = metric_function(eval_x[:eval_pos], + eval_y[:eval_pos], + eval_x[eval_pos:], + eval_y[eval_pos:], + categorical_feats, + metric_used=metric_used + , max_time=max_time) + metrics += [metric] + outputs += [output] + best_configs += [best_config] + return np.array(metrics), np.array(outputs), best_configs + except Exception as e: + print(f'There was an exception in {metric_function}') + print(e) + return None, None, None \ No newline at end of file diff --git a/TabPFN/scripts/differentiable_pfn_evaluation.py b/TabPFN/scripts/differentiable_pfn_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..91c01377a11b42f3a813459eb27f4009d2cd77e8 --- /dev/null +++ b/TabPFN/scripts/differentiable_pfn_evaluation.py @@ -0,0 +1,391 @@ +import os +import torch +import numpy as np +import time +import pickle +from scripts import tabular_metrics +from scripts.tabular_metrics import calculate_score_per_method +from scripts.tabular_evaluation import evaluate +from priors.differentiable_prior import draw_random_style +from tqdm import tqdm +from pathlib import Path +import random +from model_builder import load_model +from scripts.transformer_prediction_interface import get_params_from_config + +""" +=============================== +PUBLIC FUNCTIONS FOR EVALUATION +=============================== +""" + + +def eval_model_range(i_range, *args, **kwargs): + for i in i_range: + eval_model(i, *args, **kwargs) + + +def load_model_workflow(i, e, add_name, base_path, device='cpu', eval_addition=''): + """ + Workflow for loading a model and setting appropriate parameters for diffable hparam tuning. + + :param i: + :param e: + :param eval_positions_valid: + :param add_name: + :param base_path: + :param device: + :param eval_addition: + :return: + """ + def check_file(e): + model_file = f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{e}.cpkt' + model_path = os.path.join(base_path, model_file) + # print('Evaluate ', model_path) + results_file = os.path.join(base_path, + f'models_diff/prior_diff_real_results{add_name}_n_{i}_epoch_{e}_{eval_addition}.pkl') + if not Path(model_path).is_file(): # or Path(results_file).is_file(): + return None, None, None + return model_file, model_path, results_file + + model_file = None + if e == -1: + for e_ in range(100, -1, -1): + model_file_, model_path_, results_file_ = check_file(e_) + if model_file_ is not None: + e = e_ + model_file, model_path, results_file = model_file_, model_path_, results_file_ + break + else: + model_file, model_path, results_file = check_file(e) + + if model_file is None: + print('No checkpoint found') + return None + + print(f'Loading {model_file}') + + model, c = load_model(base_path, model_file, device, eval_positions=[], verbose=False) + + return model, c, results_file + + +def eval_model(i, e, valid_datasets, test_datasets, train_datasets, eval_positions_valid, eval_positions_test, + bptt_valid, + bptt_test, add_name, base_path, device='cpu', eval_addition='', **extra_tuning_args): + """ + Differentiable model evaliation workflow. Evaluates and saves results to disk. + + :param i: + :param e: + :param valid_datasets: + :param test_datasets: + :param train_datasets: + :param eval_positions_valid: + :param eval_positions_test: + :param bptt_valid: + :param bptt_test: + :param add_name: + :param base_path: + :param device: + :param eval_addition: + :param extra_tuning_args: + :return: + """ + model, c, results_file = load_model_workflow(i, e, add_name, base_path, device, eval_addition) + params = {'bptt': bptt_valid + , 'bptt_final': bptt_test + , 'eval_positions': eval_positions_valid + , 'eval_positions_test': eval_positions_test + , 'valid_datasets': valid_datasets + , 'test_datasets': test_datasets + , 'train_datasets': train_datasets + , 'verbose': True + , 'device': device + } + + params.update(get_params_from_config(c)) + + start = time.time() + metrics, metrics_valid, style, temperature, optimization_route = evaluate_differentiable_model(model, **params, + **extra_tuning_args) + print('Evaluation time: ', time.time() - start) + + print(results_file) + r = [c.copy(), metrics, metrics_valid, style.to('cpu'), temperature.to('cpu'), optimization_route] + with open(results_file, 'wb') as output: + del r[0]['num_features_used'] + del r[0]['categorical_features_sampler'] + pickle.dump(r, output) + + _, _, _, style, temperature, _ = r + + return r, model + +""" +=============================== +INTERNAL HELPER FUNCTIONS +=============================== +""" + +def evaluate_differentiable_model(model + , valid_datasets + , test_datasets + , train_datasets + , N_draws=100 + , N_grad_steps=10 + , eval_positions=None + , eval_positions_test=None + , bptt=100 + , bptt_final=200 + , style=None + , n_parallel_configurations=1 + , device='cpu' + , selection_metric='auc' + , final_splits=[1, 2, 3, 4, 5] + , N_ensemble_configurations_list=[1, 5, 10, 20, 50, 100] + , **kwargs): + """ + Evaluation function for diffable model evaluation. Returns a list of results. + + :param model: + :param valid_datasets: + :param test_datasets: + :param train_datasets: + :param N_draws: + :param N_grad_steps: + :param eval_positions: + :param eval_positions_test: + :param bptt: + :param bptt_final: + :param style: + :param n_parallel_configurations: + :param device: + :param selection_metric: + :param final_splits: + :param N_ensemble_configurations_list: + :param kwargs: + :return: + """ + torch.manual_seed(0) + np.random.seed(0) + random.seed(0) + + diffable_metric = tabular_metrics.cross_entropy + evaluation_metric = tabular_metrics.auc_metric + if selection_metric in ('auc', 'roc'): + selection_metric_min_max = 'max' + selection_metric = tabular_metrics.auc_metric + evaluation_metric = selection_metric + elif selection_metric in ('ce', 'selection_metric'): + selection_metric_min_max = 'min' + selection_metric = tabular_metrics.cross_entropy + evaluation_metric = selection_metric + + print('Diffable metric', diffable_metric, ' Selection metric', selection_metric, ' Evaluation metric', + evaluation_metric) + print('N PARALLEL CONFIGURATIONS', n_parallel_configurations) + print('eval_positions', eval_positions) + + def evaluate_valid(style, softmax_temperature, results, results_tracked): + result_valid = eval_step(valid_datasets, style, softmax_temperature=softmax_temperature, + return_tensor=False, inference_mode=True, selection_metric=selection_metric, + evaluation_metric=evaluation_metric, eval_positions=eval_positions, bptt=bptt, model=model[2]) + result_valid = [float(result_valid[f'mean_select_at_{pos}']) for pos in eval_positions] + results += [result_valid] + results_tracked += [np.nanmean(result_valid)] + + model[2].to(device) + model[2].eval() + + results_on_valid, results_on_valid_tracked = [], [] + best_style, best_softmax_temperature = style, torch.cat( + [torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)], 0) + optimization_routes = [] + + best_style = torch.cat([draw_random_style(model[3], device).detach() for n in range(0, n_parallel_configurations)], + 0) + best_softmax_temperature = torch.cat([torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)], + 0) + + + for _ in tqdm(range(0, N_draws), desc='Iterate over Optimization initializations'): # Evaluates N hparam draws + style = torch.cat([draw_random_style(model[3], device).detach() for n in range(0, n_parallel_configurations)], + 0) + softmax_temperature = torch.cat([torch.tensor([0.0]).to(device) for n in range(0, n_parallel_configurations)], + 0) + + evaluate_valid(style, softmax_temperature, results_on_valid, results_on_valid_tracked) + + print(f'Draw --> Valid Selection metric: {results_on_valid[-1]}') + + if N_grad_steps > 0: + gradient_optimize_result = gradient_optimize_style(model, style, N_grad_steps + , softmax_temperature=softmax_temperature + , model=model[2] + , train_datasets=train_datasets + , valid_datasets=valid_datasets + , selection_metric_min_max=selection_metric_min_max + , **kwargs) + optimization_routes += [gradient_optimize_result['optimization_route']] + + evaluate_valid(gradient_optimize_result['best_style'] + , gradient_optimize_result['best_temperature'] + , results_on_valid, results_on_valid_tracked) + + print(f'After diff --> Valid Selection metric: {results_on_valid[-1]}') + + if selection_metric_min_max == 'min': + is_best = (results_on_valid_tracked[-1] <= min(results_on_valid_tracked)) + else: + is_best = (results_on_valid_tracked[-1] >= max(results_on_valid_tracked)) + + if is_best or best_style is None: + best_style = gradient_optimize_result['best_style'].clone() + best_softmax_temperature = gradient_optimize_result['best_temperature'].clone() + torch.cuda.empty_cache() + + def final_evaluation(): + print('Running eval dataset with final params (no gradients)..') + print(best_style, best_softmax_temperature) + result_test = [] + for N_ensemble_configurations in N_ensemble_configurations_list: + print(f'Running with {N_ensemble_configurations} ensemble_configurations') + kwargs['N_ensemble_configurations'] = N_ensemble_configurations + splits = [] + for split in final_splits: + splits += [eval_step(test_datasets, best_style, softmax_temperature=best_softmax_temperature + , return_tensor=False, eval_positions=eval_positions_test, + bptt=bptt_final, inference_mode=True, split_number=split, model=model[2] + , selection_metric=selection_metric, evaluation_metric=evaluation_metric)] + result_test += [splits] + + print('Running valid dataset with final params (no gradients)..') + result_valid = eval_step(valid_datasets, best_style, softmax_temperature=best_softmax_temperature + , return_tensor=False, eval_positions=eval_positions_test, + bptt=bptt_final, inference_mode=True, model=model[2] + , selection_metric=selection_metric, evaluation_metric=evaluation_metric) + + return result_test, result_valid + + result_test, result_valid = final_evaluation() + + return result_test, result_valid, best_style, best_softmax_temperature, optimization_routes + + +def eval_step(ds, used_style, selection_metric, evaluation_metric, eval_positions, return_tensor=True, **kwargs): + def step(): + return evaluate(datasets=ds, + method='transformer' + , overwrite=True + , style=used_style + , eval_positions=eval_positions + , metric_used=selection_metric + , save=False + , path_interfix=None + , base_path=None + , verbose=True + , **kwargs) + + if return_tensor: + r = step() + else: + with torch.no_grad(): + r = step() + + calculate_score_per_method(selection_metric, 'select', r, ds, eval_positions, aggregator='mean') + calculate_score_per_method(evaluation_metric, 'eval', r, ds, eval_positions, aggregator='mean') + + return r + + +def gradient_optimize_style(model, init_style, steps, softmax_temperature, train_datasets, valid_datasets, learning_rate=0.03, optimize_all=False, + limit_style=True, N_datasets_sampled=90, optimize_softmax_temperature=True, selection_metric_min_max='max', **kwargs): + """ + Uses gradient based methods to optimize 'style' on the 'train_datasets' and uses stopping with 'valid_datasets'. + + :param model: + :param init_style: + :param steps: + :param learning_rate: + :param softmax_temperature: + :param train_datasets: + :param valid_datasets: + :param optimize_all: + :param limit_style: + :param N_datasets_sampled: + :param optimize_softmax_temperature: + :param selection_metric_min_max: + :param kwargs: + :return: + """ + grad_style = torch.nn.Parameter(init_style.detach(), requires_grad=True) + + best_style, best_temperature, best_selection_metric, best_diffable_metric = grad_style.detach(), softmax_temperature.detach(), None, None + softmax_temperature = torch.nn.Parameter(softmax_temperature.detach(), requires_grad=optimize_softmax_temperature) + variables_to_optimize = model[2].parameters() if optimize_all else [grad_style, softmax_temperature] + optimizer = torch.optim.Adam(variables_to_optimize, lr=learning_rate) + + optimization_route_selection, optimization_route_diffable = [], [] + optimization_route_selection_valid, optimization_route_diffable_valid = [], [] + + def eval_opt(ds, return_tensor=True, inference_mode=False): + result = eval_step(ds, grad_style, softmax_temperature=softmax_temperature, return_tensor=return_tensor + , inference_mode=inference_mode, model=model[2], **kwargs) + + diffable_metric = result['mean_metric'] + selection_metric = result['mean_select'] + + return diffable_metric, selection_metric + + def eval_all_datasets(datasets, propagate=True): + selection_metrics_this_step, diffable_metrics_this_step = [], [] + for ds in datasets: + diffable_metric_train, selection_metric_train = eval_opt([ds], inference_mode=(not propagate)) + if not torch.isnan(diffable_metric_train).any(): + if propagate and diffable_metric_train.requires_grad == True: + diffable_metric_train.backward() + selection_metrics_this_step += [selection_metric_train] + diffable_metrics_this_step += [float(diffable_metric_train.detach().cpu().numpy())] + diffable_metric_train = np.nanmean(diffable_metrics_this_step) + selection_metric_train = np.nanmean(selection_metrics_this_step) + + return diffable_metric_train, selection_metric_train + + for t in tqdm(range(steps), desc='Iterate over Optimization steps'): + optimizer.zero_grad() + + # Select subset of datasets + random.seed(t) + train_datasets_ = random.sample(train_datasets, N_datasets_sampled) + + # Get score on train + diffable_metric_train, selection_metric_train = eval_all_datasets(train_datasets_, propagate=True) + optimization_route_selection += [float(selection_metric_train)] + optimization_route_diffable += [float(diffable_metric_train)] + + # Get score on valid + diffable_metric_valid, selection_metric_valid = eval_all_datasets(valid_datasets, propagate=False) + optimization_route_selection_valid += [float(selection_metric_valid)] + optimization_route_diffable_valid += [float(diffable_metric_valid)] + + is_best = (selection_metric_min_max == 'min' and best_selection_metric > selection_metric_valid) + is_best = is_best or (selection_metric_min_max == 'max' and best_selection_metric < selection_metric_valid) + if (best_selection_metric is None) or (not np.isnan(selection_metric_valid) and is_best): + print('New best', best_selection_metric, selection_metric_valid) + best_style = grad_style.detach().clone() + best_temperature = softmax_temperature.detach().clone() + best_selection_metric, best_diffable_metric = selection_metric_valid, diffable_metric_valid + + optimizer.step() + + if limit_style: + grad_style = grad_style.detach().clamp(-1.74, 1.74) + + print(f'Valid: Diffable metric={diffable_metric_valid} Selection metric={selection_metric_valid};' + + f'Train: Diffable metric={diffable_metric_train} Selection metric={selection_metric_train}') + + print(f'Return best:{best_style} {best_selection_metric}') + return {'best_style': best_style, 'best_temperature': best_temperature + , 'optimization_route': {'select': optimization_route_selection, 'loss': optimization_route_diffable, + 'test_select': optimization_route_selection_valid, 'test_loss': optimization_route_diffable_valid}} \ No newline at end of file diff --git a/TabPFN/scripts/model_configs.py b/TabPFN/scripts/model_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..89e2decf949b6b102f373c817c56999346a3844d --- /dev/null +++ b/TabPFN/scripts/model_configs.py @@ -0,0 +1,210 @@ +from copy import deepcopy +from priors.utils import uniform_int_sampler_f +from priors.differentiable_prior import DifferentiableHyperparameter +from ConfigSpace import hyperparameters as CSH +import torch +from priors.differentiable_prior import replace_differentiable_distributions + +import ConfigSpace as CS + +def get_general_config(max_features, bptt, eval_positions=None): + """" + Returns the general PFN training hyperparameters. + """ + config_general = { + "lr": CSH.UniformFloatHyperparameter('lr', lower=0.00002, upper=0.0002, log=True), + "dropout": CSH.CategoricalHyperparameter('dropout', [0.0]), + "emsize": CSH.CategoricalHyperparameter('emsize', [2 ** i for i in range(8, 9)]), ## upper bound is -1 + "batch_size": CSH.CategoricalHyperparameter('batch_size', [2 ** i for i in range(8, 9)]), + "nlayers": CSH.CategoricalHyperparameter('nlayers', [12]), + "num_features": max_features, + "nhead": CSH.CategoricalHyperparameter('nhead', [4]), + "nhid_factor": 2, + "bptt": bptt, + "eval_positions": None, + "seq_len_used": bptt, + "sampling": 'normal',#hp.choice('sampling', ['mixed', 'normal']), # uniform + "epochs": 80, + "num_steps": 100, + "verbose": False, + "pre_sample_causes": True, # This is MLP + "mix_activations": False,#hp.choice('mix_activations', [True, False]), + } + + return config_general + +def get_flexible_categorical_config(max_features): + """" + Returns the configuration parameters for the tabular multiclass wrapper. + """ + config_flexible_categorical = { + "nan_prob_unknown_reason_reason_prior": CSH.CategoricalHyperparameter('nan_prob_unknown_reason_reason_prior', [1.0]), + "categorical_feature_p": CSH.CategoricalHyperparameter('categorical_feature_p', [0.0]), + "nan_prob_no_reason": CSH.CategoricalHyperparameter('nan_prob_no_reason', [0.0, 0.1, 0.2]), + "nan_prob_unknown_reason": CSH.CategoricalHyperparameter('nan_prob_unknown_reason', [0.0]), + "nan_prob_a_reason": CSH.CategoricalHyperparameter('nan_prob_a_reason', [0.0]), + # "num_classes": lambda : random.randint(2, 10), "balanced": False, + "max_num_classes": 2, + "num_classes": 2, + "noise_type": CSH.CategoricalHyperparameter('noise_type', ["Gaussian"]), # NN + "balanced": True, + "normalize_to_ranking": CSH.CategoricalHyperparameter('normalize_to_ranking', [False]), + "set_value_to_nan": CSH.CategoricalHyperparameter('set_value_to_nan', [0.5, 0.2, 0.0]), + "normalize_by_used_features": True, + "num_features_used": + {'uniform_int_sampler_f(3,max_features)': uniform_int_sampler_f(1, max_features)} + # hp.choice('conv_activation', [{'distribution': 'uniform', 'min': 2.0, 'max': 8.0}, None]), + } + return config_flexible_categorical + +def get_diff_flex(): + """" + Returns the configuration parameters for a differentiable wrapper around the tabular multiclass wrapper. + """ + diff_flex = { + # "ordinal_pct": {'distribution': 'uniform', 'min': 0.0, 'max': 0.5}, + # "num_categorical_features_sampler_a": hp.choice('num_categorical_features_sampler_a', + # [{'distribution': 'uniform', 'min': 0.3, 'max': 0.9}, None]), + # "num_categorical_features_sampler_b": {'distribution': 'uniform', 'min': 0.3, 'max': 0.9}, + "output_multiclass_ordered_p": {'distribution': 'uniform', 'min': 0.0, 'max': 0.5}, #CSH.CategoricalHyperparameter('output_multiclass_ordered_p', [0.0, 0.1, 0.2]), + "multiclass_type": {'distribution': 'meta_choice', 'choice_values': ['value', 'rank']}, + } + + return diff_flex + +def get_diff_gp(): + """" + Returns the configuration parameters for a differentiable wrapper around GP. + """ + diff_gp = { + 'outputscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10., 'min_mean': 0.00001, 'round': False, + 'lower_bound': 0}, + 'lengthscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10., 'min_mean': 0.00001, 'round': False, + 'lower_bound': 0}, + 'noise': {'distribution': 'meta_choice', 'choice_values': [0.00001, 0.0001, 0.01]} + } + + return diff_gp + +def get_diff_causal(): + """" + Returns the configuration parameters for a differentiable wrapper around MLP / Causal mixture. + """ + diff_causal = { + "num_layers": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 6, 'min_mean': 1, 'round': True, + 'lower_bound': 2}, + # Better beta? + "prior_mlp_hidden_dim": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 130, 'min_mean': 5, + 'round': True, 'lower_bound': 4}, + + "prior_mlp_dropout_prob": {'distribution': 'meta_beta', 'scale': 0.9, 'min': 0.1, 'max': 5.0}, + # This mustn't be too high since activations get too large otherwise + + "noise_std": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': .3, 'min_mean': 0.0001, 'round': False, + 'lower_bound': 0.0}, + "init_std": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 0.01, 'round': False, + 'lower_bound': 0.0}, + "num_causes": {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 12, 'min_mean': 1, 'round': True, + 'lower_bound': 1}, + "is_causal": {'distribution': 'meta_choice', 'choice_values': [True, False]}, + "pre_sample_weights": {'distribution': 'meta_choice', 'choice_values': [True, False]}, + "y_is_effect": {'distribution': 'meta_choice', 'choice_values': [True, False]}, + "prior_mlp_activations": {'distribution': 'meta_choice_mixed', 'choice_values': [ + torch.nn.Tanh + , torch.nn.ReLU + , torch.nn.Identity + , lambda : torch.nn.LeakyReLU(negative_slope=0.1) + , torch.nn.ELU + ]}, + "block_wise_dropout": {'distribution': 'meta_choice', 'choice_values': [True, False]}, + "sort_features": {'distribution': 'meta_choice', 'choice_values': [True, False]}, + "in_clique": {'distribution': 'meta_choice', 'choice_values': [True, False]}, + } + + return diff_causal + +def get_diff_prior_bag(): + """" + Returns the configuration parameters for a GP and MLP / Causal mixture. + """ + diff_prior_bag = { + 'prior_bag_exp_weights_1': {'distribution': 'uniform', 'min': 100000., 'max': 100001.}, + # MLP Weight (Biased, since MLP works better, 1.0 is weight for prior number 0) + } + + return diff_prior_bag + +def get_diff_config(): + """" + Returns the configuration parameters for a differentiable wrapper around GP and MLP / Causal mixture priors. + """ + diff_prior_bag = get_diff_prior_bag() + diff_causal = get_diff_causal() + diff_gp = get_diff_gp() + diff_flex = get_diff_flex() + + config_diff = {'differentiable_hyperparameters': {**diff_prior_bag, **diff_causal, **diff_gp, **diff_flex}} + + return config_diff + + +def sample_differentiable(config): + """" + Returns sampled hyperparameters from a differentiable wrapper, that is it makes a non-differentiable out of + differentiable. + """ + # config is a dict of dicts, dicts that have a 'distribution' key are treated as distributions to be sampled + result = deepcopy(config) + del result['differentiable_hyperparameters'] + + for k, v in config['differentiable_hyperparameters'].items(): + s_indicator, s_hp = DifferentiableHyperparameter(**v, embedding_dim=None, + device=None)() # both of these are actually not used to the best of my knowledge + result[k] = s_hp + + return result + +def list_all_hps_in_nested(config): + """" + Returns a list of hyperparameters from a neszed dict of hyperparameters. + """ + + if isinstance(config, CSH.Hyperparameter): + return [config] + elif isinstance(config, dict): + result = [] + for k, v in config.items(): + result += list_all_hps_in_nested(v) + return result + else: + return [] + +def create_configspace_from_hierarchical(config): + cs = CS.ConfigurationSpace() + for hp in list_all_hps_in_nested(config): + cs.add_hyperparameter(hp) + return cs + +def fill_in_configsample(config, configsample): + # config is our dict that defines config distribution + # configsample is a CS.Configuration + hierarchical_configsample = deepcopy(config) + for k, v in config.items(): + if isinstance(v, CSH.Hyperparameter): + hierarchical_configsample[k] = configsample[v.name] + elif isinstance(v, dict): + hierarchical_configsample[k] = fill_in_configsample(v, configsample) + return hierarchical_configsample + + +def evaluate_hypers(config, sample_diff_hps=False): + """" + Samples a hyperparameter configuration from a sampleable configuration (can be used in HP search). + """ + if sample_diff_hps: + # I do a deepcopy here, such that the config stays the same and can still be used with diff. hps + config = deepcopy(config) + replace_differentiable_distributions(config) + cs = create_configspace_from_hierarchical(config) + cs_sample = cs.sample_configuration() + return fill_in_configsample(config, cs_sample) diff --git a/TabPFN/scripts/tabular_baselines.py b/TabPFN/scripts/tabular_baselines.py new file mode 100644 index 0000000000000000000000000000000000000000..a8a7911d96d72200372a53e2f90dc4ea206e3918 --- /dev/null +++ b/TabPFN/scripts/tabular_baselines.py @@ -0,0 +1,421 @@ +from catboost import CatBoostClassifier, Pool + +import math + +from sklearn.impute import SimpleImputer + +import xgboost as xgb +from sklearn import neighbors +from sklearn.gaussian_process import GaussianProcessClassifier +from sklearn.gaussian_process.kernels import RBF +import numpy as np + +from scripts import tabular_metrics +import pandas as pd + +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import cross_val_score +import time + +from hyperopt import fmin, tpe, hp, STATUS_OK, Trials , space_eval, rand +from sklearn.compose import ColumnTransformer +from sklearn.preprocessing import OneHotEncoder +from sklearn.preprocessing import MinMaxScaler + +import autosklearn.classification + +CV = 5 +MULTITHREAD = 1 # Number of threads baselines are able to use at most +param_grid, param_grid_hyperopt = {}, {} + +def get_scoring_direction(metric_used): + # Not needed + if metric_used == tabular_metrics.auc_metric: + return -1 + elif metric_used == tabular_metrics.cross_entropy: + return 1 + else: + raise Exception('No scoring string found for metric') + +def get_scoring_string(metric_used, multiclass=True, usage="sklearn_cv"): + if metric_used == tabular_metrics.auc_metric: + if usage == 'sklearn_cv': + return 'roc_auc_ovo' + elif usage == 'autogluon': + return 'log_loss' # Autogluon crashes when using 'roc_auc' with some datasets usning logloss gives better scores; + # We might be able to fix this, but doesn't work out of box. + # File bug report? Error happens with dataset robert and fabert + if multiclass: + return 'roc_auc_ovo_macro' + else: + return 'roc_auc' + elif usage == 'autosklearn': + if multiclass: + return autosklearn.metrics.log_loss # roc_auc only works for binary, use logloss instead + else: + return autosklearn.metrics.roc_auc + elif usage == 'catboost': + return 'MultiClass' # Effectively LogLoss, ROC not available + elif usage == 'xgb': + return 'logloss' + return 'roc_auc' + elif metric_used == tabular_metrics.cross_entropy: + if usage == 'sklearn_cv': + return 'neg_log_loss' + elif usage == 'autogluon': + return 'log_loss' + elif usage == 'autosklearn': + return autosklearn.metrics.log_loss + elif usage == 'catboost': + return 'MultiClass' # Effectively LogLoss + return 'logloss' + else: + raise Exception('No scoring string found for metric') + +def eval_f(params, clf_, x, y, metric_used, start_time, max_time): + if time.time() - start_time > max_time: + return np.nan + scores = cross_val_score(clf_(**params), x, y, cv=CV, scoring=get_scoring_string(metric_used)) + + return -np.nanmean(scores) + +def preprocess_impute(x, y, test_x, test_y, impute, one_hot, standardize, cat_features=[]): + import warnings + def warn(*args, **kwargs): + pass + + warnings.warn = warn + + x, y, test_x, test_y = x.cpu().numpy(), y.cpu().long().numpy(), test_x.cpu().numpy(), test_y.cpu().long().numpy() + + if impute: + imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean') + imp_mean.fit(x) + x, test_x = imp_mean.transform(x), imp_mean.transform(test_x) + + if one_hot: + def make_pd_from_np(x): + data = pd.DataFrame(x) + for c in cat_features: + data.iloc[:, c] = data.iloc[:, c].astype('int') + return data + x, test_x = make_pd_from_np(x), make_pd_from_np(test_x) + transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(handle_unknown='ignore', sparse=False), cat_features)], remainder="passthrough") + transformer.fit(x) + x, test_x = transformer.transform(x), transformer.transform(test_x) + + if standardize: + scaler = MinMaxScaler() + scaler.fit(x) + x, test_x = scaler.transform(x), scaler.transform(test_x) + + return x, y, test_x, test_y + +## Auto Gluon +def autogluon_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300): + from autogluon.tabular import TabularPredictor # Inside function so package can be sued without installation + x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y + , one_hot=False + , cat_features=cat_features + , impute=False + , standardize=False) + train_data = pd.DataFrame(np.concatenate([x, y[:, np.newaxis]], 1)) + test_data = pd.DataFrame(np.concatenate([test_x, test_y[:, np.newaxis]], 1)) + + # AutoGluon automatically infers datatypes, we don't specify the categorical labels + predictor = TabularPredictor( + label=train_data.columns[-1], + eval_metric=get_scoring_string(metric_used, usage='autogluon', multiclass=(len(np.unique(y)) > 2)), + problem_type='multiclass' if len(np.unique(y)) > 2 else 'binary' + ## seed=int(y[:].sum()) doesn't accept seed + ).fit( + train_data=train_data, + time_limit=max_time, + presets=['best_quality'] + # The seed is deterministic but varies for each dataset and each split of it + ) + + pred = predictor.predict_proba(test_data, as_multiclass=True).values + + metric = metric_used(test_y, pred) + + return metric, pred, predictor.fit_summary() + +## AUTO Sklearn +def autosklearn_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300): + return autosklearn2_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=max_time, version=1) + +from autosklearn.experimental.askl2 import AutoSklearn2Classifier +from autosklearn.classification import AutoSklearnClassifier +def autosklearn2_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300, version=2): + x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y + , one_hot=False + , cat_features=cat_features + , impute=False + , standardize=False) + + def make_pd_from_np(x): + data = pd.DataFrame(x) + for c in cat_features: + data.iloc[:, c] = data.iloc[:, c].astype('category') + return data + + x = make_pd_from_np(x) + test_x = make_pd_from_np(test_x) + + clf_ = AutoSklearn2Classifier if version == 2 else AutoSklearnClassifier + clf = clf_(time_left_for_this_task=max_time, + memory_limit=4000, + n_jobs=MULTITHREAD, + seed=int(y[:].sum()), + # The seed is deterministic but varies for each dataset and each split of it + metric=get_scoring_string(metric_used, usage='autosklearn', multiclass=len(np.unique(y)) > 2)) + + # fit model to data + clf.fit(x, y) + + pred = clf.predict_proba(test_x) + metric = metric_used(test_y, pred) + + return metric, pred, None + +param_grid_hyperopt['logistic'] = { + 'penalty': hp.choice('penalty', ['l1', 'l2', 'none']) + , 'max_iter': hp.randint('max_iter', [50, 500]) + , 'fit_intercept': hp.choice('fit_intercept', [True, False]) + , 'C': hp.loguniform('C', -5, math.log(5.0))} # 'normalize': [False], + +def logistic_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300): + x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y + , one_hot=True, impute=True, standardize=True + , cat_features=cat_features) + + def clf_(**params): + return LogisticRegression(solver='saga', tol=1e-4, n_jobs=1, **params) + + start_time = time.time() + + def stop(trial): + return time.time() - start_time > max_time, [] + + best = fmin( + fn=lambda params: eval_f(params, clf_, x, y, metric_used, start_time, max_time), + space=param_grid_hyperopt['logistic'], + algo=rand.suggest, + rstate=np.random.RandomState(int(y[:].sum())), + early_stop_fn=stop, + # The seed is deterministic but varies for each dataset and each split of it + max_evals=10000) + best = space_eval(param_grid_hyperopt['logistic'], best) + + clf = clf_(**best) + clf.fit(x, y) + + pred = clf.predict_proba(test_x) + metric = metric_used(test_y, pred) + + return metric, pred, best + +## KNN +param_grid_hyperopt['knn'] = {'n_neighbors': hp.randint('n_neighbors', 1,16) + } +def knn_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300): + x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y, + one_hot=True, impute=True, standardize=True, + cat_features=cat_features) + + def clf_(**params): + return neighbors.KNeighborsClassifier(n_jobs=1, **params) + + start_time = time.time() + + def stop(trial): + return time.time() - start_time > max_time, [] + + best = fmin( + fn=lambda params: eval_f(params, clf_, x, y, metric_used, start_time, max_time), + space=param_grid_hyperopt['knn'], + algo=rand.suggest, + rstate=np.random.RandomState(int(y[:].sum())), + early_stop_fn=stop, + # The seed is deterministic but varies for each dataset and each split of it + max_evals=10000) + best = space_eval(param_grid_hyperopt['knn'], best) + + clf = clf_(**best) + clf.fit(x, y) + + pred = clf.predict_proba(test_x) + metric = metric_used(test_y, pred) + + return metric, pred, best + +## GP +param_grid_hyperopt['gp'] = { + 'params_y_scale': hp.loguniform('params_y_scale', math.log(0.05), math.log(5.0)), + 'params_length_scale': hp.loguniform('params_length_scale', math.log(0.1), math.log(1.0)), + 'n_jobs': hp.choice('njobs', [1]) +} +def gp_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300): + x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y, + one_hot=True, impute=True, standardize=True, + cat_features=cat_features) + + def clf_(params_y_scale,params_length_scale, **params): + return GaussianProcessClassifier(kernel= params_y_scale * RBF(params_length_scale), **params) + + start_time = time.time() + def stop(trial): + return time.time() - start_time > max_time, [] + + + best = fmin( + fn=lambda params: eval_f(params, clf_, x, y, metric_used, start_time, max_time), + space=param_grid_hyperopt['gp'], + algo=rand.suggest, + rstate=np.random.RandomState(int(y[:].sum())), + early_stop_fn=stop, + # The seed is deterministic but varies for each dataset and each split of it + max_evals=1000) + best = space_eval(param_grid_hyperopt['gp'], best) + + clf = clf_(**best) + clf.fit(x, y) + + pred = clf.predict_proba(test_x) + metric = metric_used(test_y, pred) + + return metric, pred, best + + +# Catboost +# Hyperparameter space: https://arxiv.org/pdf/2106.03253.pdf + +param_grid_hyperopt['catboost'] = { + 'learning_rate': hp.loguniform('learning_rate', math.log(math.pow(math.e, -5)), math.log(1)), + 'random_strength': hp.randint('random_strength', 1, 20), + 'l2_leaf_reg': hp.loguniform('l2_leaf_reg', math.log(1), math.log(10)), + 'bagging_temperature': hp.uniform('bagging_temperature', 0., 1), + 'leaf_estimation_iterations': hp.randint('leaf_estimation_iterations', 1, 20), + 'iterations': hp.randint('iterations', 100, 4000), # This is smaller than in paper, 4000 leads to ram overusage +} + +def catboost_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300): + print(x) + + x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y + , one_hot=False + , cat_features=cat_features + , impute=False + , standardize=False) + + # Nans in categorical features must be encoded as separate class + x[:, cat_features], test_x[:, cat_features] = np.nan_to_num(x[:, cat_features], -1), np.nan_to_num( + test_x[:, cat_features], -1) + + def make_pd_from_np(x): + data = pd.DataFrame(x) + for c in cat_features: + data.iloc[:, c] = data.iloc[:, c].astype('int') + return data + + x = make_pd_from_np(x) + test_x = make_pd_from_np(test_x) + + def clf_(**params): + return CatBoostClassifier( + loss_function=get_scoring_string(metric_used, usage='catboost'), + thread_count = MULTITHREAD, + used_ram_limit='4gb', + random_seed=int(y[:].sum()), + logging_level='Silent', + cat_features=cat_features, + **params) + + start_time = time.time() + def stop(trial): + return time.time() - start_time > max_time, [] + + best = fmin( + fn=lambda params: eval_f(params, clf_, x, y, metric_used, start_time, max_time), + space=param_grid_hyperopt['catboost'], + algo=rand.suggest, + rstate=np.random.RandomState(int(y[:].sum())), + early_stop_fn=stop, + # The seed is deterministic but varies for each dataset and each split of it + max_evals=1000) + best = space_eval(param_grid_hyperopt['catboost'], best) + + clf = clf_(**best) + clf.fit(x, y) + + pred = clf.predict_proba(test_x) + metric = metric_used(test_y, pred) + + return metric, pred, best + + +# XGBoost +# Hyperparameter space: https://arxiv.org/pdf/2106.03253.pdf +param_grid_hyperopt['xgb'] = { + 'learning_rate': hp.loguniform('learning_rate', -7, math.log(1)), + 'max_depth': hp.randint('max_depth', 1, 10), + 'subsample': hp.uniform('subsample', 0.2, 1), + 'colsample_bytree': hp.uniform('colsample_bytree', 0.2, 1), + 'colsample_bylevel': hp.uniform('colsample_bylevel', 0.2, 1), + 'min_child_weight': hp.loguniform('min_child_weight', -16, 5), + 'alpha': hp.loguniform('alpha', -16, 2), + 'lambda': hp.loguniform('lambda', -16, 2), + 'gamma': hp.loguniform('gamma', -16, 2), + 'n_estimators': hp.randint('n_estimators', 100, 4000), # This is smaller than in paper +} + +def xgb_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300): + # XGB Documentation: + # XGB handles categorical data appropriately without using One Hot Encoding, categorical features are experimetal + # XGB handles missing values appropriately without imputation + + x, y, test_x, test_y = preprocess_impute(x, y, test_x, test_y + , one_hot=False + , cat_features=cat_features + , impute=False + , standardize=False) + + def clf_(**params): + return xgb.XGBClassifier(use_label_encoder=False + , nthread=1 + , **params + , eval_metric=get_scoring_string(metric_used, usage='xgb') # AUC not implemented + ) + + start_time = time.time() + def stop(trial): + return time.time() - start_time > max_time, [] + + best = fmin( + fn=lambda params: eval_f(params, clf_, x, y, metric_used, start_time, max_time), + space=param_grid_hyperopt['xgb'], + algo=rand.suggest, + rstate=np.random.RandomState(int(y[:].sum())), + early_stop_fn=stop, + # The seed is deterministic but varies for each dataset and each split of it + max_evals=1000) + best = space_eval(param_grid_hyperopt['xgb'], best) + + clf = clf_(**best) + clf.fit(x, y) + + pred = clf.predict_proba(test_x) + metric = metric_used(test_y, pred) + + return metric, pred, best + + +clf_dict = {'gp': gp_metric + , 'knn': knn_metric + , 'catboost': catboost_metric + , 'xgb': xgb_metric + , 'logistic': logistic_metric + , 'autosklearn': autosklearn_metric + , 'autosklearn2': autosklearn2_metric + , 'autogluon': autogluon_metric} \ No newline at end of file diff --git a/TabPFN/scripts/tabular_evaluation.py b/TabPFN/scripts/tabular_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..cd7f36e32948f80d0e266b47828df5f51fe3f78e --- /dev/null +++ b/TabPFN/scripts/tabular_evaluation.py @@ -0,0 +1,284 @@ +import time +import os +from pathlib import Path + +from tqdm import tqdm +import random +import numpy as np + +from torch import nn + +from utils import torch_nanmean +from datasets import * +from model_builder import load_model +from scripts.tabular_baselines import get_scoring_string +from scripts import tabular_metrics +from scripts.transformer_prediction_interface import * +from scripts.baseline_prediction_interface import * +""" +=============================== +PUBLIC FUNCTIONS FOR EVALUATION +=============================== +""" + + +def eval_model(i, e, valid_datasets, test_datasets, eval_positions, bptt, add_name, base_path, device='cpu', eval_addition='', **kwargs): + metrics_test, config_sample, model_path = eval_model_on_ds(i, e, test_datasets, eval_positions, bptt, add_name, base_path, device=device, eval_addition=eval_addition, **kwargs) + metrics_valid, _, _ = eval_model_on_ds(i, e, valid_datasets, eval_positions, bptt, add_name, base_path, device=device, eval_addition=eval_addition, **kwargs) + return {'mean_auc_test': metrics_test['mean_roc_at_1000'], 'mean_auc_valid': metrics_valid['mean_roc_at_1000'], 'mean_ce_test': metrics_test['mean_ce_at_1000'], 'mean_ce_valid': metrics_valid['mean_ce_at_1000'], 'config_sample': config_sample, 'model_path': model_path} + +def eval_model_on_ds(i, e, valid_datasets, eval_positions, bptt, add_name, base_path, device='cpu', eval_addition='', **kwargs): + + # How to use: evaluate_without_fitting(i,0,valid_datasets, [1024], 100000, add_name=model_string, base_path=base_path,) + def check_file(e): + model_file = f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{e}.cpkt' + model_path = os.path.join(base_path, model_file) + # print('Evaluate ', model_path) + results_file = os.path.join(base_path, + f'models_diff/prior_diff_real_results{add_name}_n_{i}_epoch_{e}_{eval_addition}.pkl') + if not Path(model_path).is_file(): # or Path(results_file).is_file(): + # print('checkpoint exists: ', Path(model_file).is_file(), ', results are written:', Path(results_file).is_file()) + return None, None, None + return model_file, model_path, results_file + + if e == -1: # use last checkpoint, if e == -1 + for e_ in range(100, -1, -1): + model_file_, model_path_, results_file_ = check_file(e_) + if model_file_ is not None: + e = e_ + model_file, model_path, results_file = model_file_, model_path_, results_file_ + break + else: + model_file, model_path, results_file = check_file(e) + + model, config_sample = load_model(base_path, model_file, device, None, verbose=False) + print(model[2].style_encoder) + + params = {'max_features': config_sample['num_features'] + , 'rescale_features': config_sample["normalize_by_used_features"] + , 'normalize_to_ranking': config_sample["normalize_to_ranking"] + , 'normalize_with_sqrt': config_sample.get("normalize_with_sqrt", False) + } + metrics_valid = evaluate(datasets=valid_datasets, model=model[2], method='transformer', device=device, overwrite=True, + extend_features=True + # just removed the style keyword but transformer is trained with style, just empty + , save=False + , metric_used=tabular_metrics.cross_entropy + , return_tensor=True + , verbose=False + , eval_positions=eval_positions + , bptt=bptt + , base_path=None + , inference_mode=True + , **params + , **kwargs) + + tabular_metrics.calculate_score_per_method(tabular_metrics.auc_metric, 'roc', metrics_valid, valid_datasets, eval_positions) + tabular_metrics.calculate_score_per_method(tabular_metrics.cross_entropy, 'ce', metrics_valid, valid_datasets, eval_positions) + + return metrics_valid, config_sample, model_path + + +def evaluate(datasets, bptt, eval_positions, metric_used, model + , verbose=False + , return_tensor=False + , **kwargs): + """ + Evaluates a list of datasets for a model function. + + :param datasets: List of datasets + :param bptt: maximum sequence length + :param eval_positions: List of positions where to evaluate models + :param verbose: If True, is verbose. + :param metric_used: Which metric is optimized for. + :param return_tensor: Wheater to return results as a pytorch.tensor or numpy, this is only relevant for transformer. + :param kwargs: + :return: + """ + overall_result = {'metric_used': get_scoring_string(metric_used) + , 'bptt': bptt + , 'eval_positions': eval_positions} + + aggregated_metric_datasets, num_datasets = torch.tensor(0.0), 0 + + # For each dataset + for [ds_name, X, y, categorical_feats, _, _] in tqdm.tqdm(datasets, desc='Iterate over datasets') if verbose else datasets: + dataset_bptt = min(len(X), bptt) + # if verbose and dataset_bptt < bptt: + # print(f'Dataset too small for given sequence length, reducing to {len(X)} ({bptt})') + + aggregated_metric, num = torch.tensor(0.0), 0 + ds_result = {} + + for eval_position in (eval_positions if verbose else eval_positions): + eval_position_real = int(dataset_bptt * 0.5) if 2 * eval_position > dataset_bptt else eval_position + eval_position_bptt = int(eval_position_real * 2.0) + + r = evaluate_position(X, y, model=model + , num_classes=len(torch.unique(y)) + , categorical_feats = categorical_feats + , bptt = eval_position_bptt + , ds_name=ds_name + , eval_position = eval_position_real + , metric_used = metric_used + ,**kwargs) + + if r is None: + continue + + _, outputs, ys, best_configs, time_used = r + + if torch.is_tensor(outputs): + outputs = outputs.to(outputs.device) + ys = ys.to(outputs.device) + + ys = ys.T + ds_result[f'{ds_name}_best_configs_at_{eval_position}'] = best_configs + ds_result[f'{ds_name}_outputs_at_{eval_position}'] = outputs + ds_result[f'{ds_name}_ys_at_{eval_position}'] = ys + ds_result[f'{ds_name}_time_at_{eval_position}'] = time_used + + new_metric = torch_nanmean(torch.stack([metric_used(ys[i], outputs[i]) for i in range(ys.shape[0])])) + + if not return_tensor: + make_scalar = lambda x: float(x.detach().cpu().numpy()) if (torch.is_tensor(x) and (len(x.shape) == 0)) else x + new_metric = make_scalar(new_metric) + ds_result = {k: make_scalar(ds_result[k]) for k in ds_result.keys()} + + lib = torch if return_tensor else np + if not lib.isnan(new_metric).any(): + aggregated_metric, num = aggregated_metric + new_metric, num + 1 + + overall_result.update(ds_result) + if num > 0: + aggregated_metric_datasets, num_datasets = (aggregated_metric_datasets + (aggregated_metric / num)), num_datasets + 1 + + overall_result['mean_metric'] = aggregated_metric_datasets / num_datasets + + return overall_result + +""" +=============================== +INTERNAL HELPER FUNCTIONS +=============================== +""" + +def check_file_exists(path): + """Checks if a pickle file exists. Returns None if not, else returns the unpickled file.""" + if (os.path.isfile(path)): + print(f'loading results from {path}') + with open(path, 'rb') as f: + return np.load(f, allow_pickle=True).tolist() + return None + +def generate_valid_split(X, y, bptt, eval_position, split_number=1): + """Generates a deteministic train-(test/valid) split. Both splits must contain the same classes and all classes in + the entire datasets. If no such split can be sampled in 7 passes, returns None. + + :param X: torch tensor, feature values + :param y: torch tensor, class values + :param bptt: Number of samples in train + test + :param eval_position: Number of samples in train, i.e. from which index values are in test + :param split_number: The split id + :return: + """ + done, seed = False, 13 + + torch.manual_seed(split_number) + perm = torch.randperm(X.shape[0]) if split_number > 1 else torch.arange(0, X.shape[0]) + X, y = X[perm], y[perm] + + while not done: + if seed > 20: + return None, None # No split could be generated in 7 passes, return None + random.seed(seed) + i = random.randint(0, len(X) - bptt) if len(X) - bptt > 0 else 0 + y_ = y[i:i + bptt] + + # Checks if all classes from dataset are contained and classes in train and test are equal (contain same + # classes) and + done = len(torch.unique(y_)) == len(torch.unique(y)) + done = done and torch.all(torch.unique(y_) == torch.unique(y)) + done = done and len(torch.unique(y_[:eval_position])) == len(torch.unique(y_[eval_position:])) + done = done and torch.all(torch.unique(y_[:eval_position]) == torch.unique(y_[eval_position:])) + seed = seed + 1 + + eval_xs = torch.stack([X[i:i + bptt].clone()], 1) + eval_ys = torch.stack([y[i:i + bptt].clone()], 1) + + return eval_xs, eval_ys + + +def evaluate_position(X, y, categorical_feats, model, bptt + , eval_position, overwrite, save, base_path, path_interfix, method, ds_name, fetch_only=False + , max_time=300, split_number=1 + , per_step_normalization=False, **kwargs): + """ + Evaluates a dataset with a 'bptt' number of training samples. + + :param X: Dataset X + :param y: Dataset labels + :param categorical_feats: Indices of categorical features. + :param model: Model function + :param bptt: Sequence length. + :param eval_position: Number of training samples. + :param overwrite: Wheater to ove + :param overwrite: If True, results on disk are overwritten. + :param save: + :param path_interfix: Used for constructing path to write on disk. + :param method: Model name. + :param ds_name: Datset name. + :param fetch_only: Wheater to calculate or only fetch results. + :param per_step_normalization: + :param kwargs: + :return: + """ + + if save: + path = os.path.join(base_path, f'results/tabular/{path_interfix}/results_{method}_{ds_name}_{eval_position}_{bptt}_{split_number}.npy') + #log_path = + + ## Load results if on disk + if not overwrite: + result = check_file_exists(path) + if result is not None: + if not fetch_only: + print(f'Loaded saved result for {path}') + return result + elif fetch_only: + print(f'Could not load saved result for {path}') + return None + + ## Generate data splits + eval_xs, eval_ys = generate_valid_split(X, y, bptt, eval_position, split_number=split_number) + if eval_xs is None: + return None + print(f"No dataset could be generated {ds_name} {bptt}") + + eval_ys = (eval_ys > torch.unique(eval_ys).unsqueeze(0)).sum(axis=1).unsqueeze(-1) + + start_time = time.time() + + if isinstance(model, nn.Module): # Two separate predict interfaces for transformer and baselines + outputs, best_configs = transformer_predict(model, eval_xs, eval_ys, eval_position, categorical_feats=categorical_feats, **kwargs), None + else: + _, outputs, best_configs = baseline_predict(model, eval_xs, eval_ys, categorical_feats + , eval_pos=eval_position + , max_time=max_time, **kwargs) + + eval_ys = eval_ys[eval_position:] + if outputs is None: + return None + + if torch.is_tensor(outputs): # Transfers data to cpu for saving + outputs = outputs.cpu() + eval_ys = eval_ys.cpu() + + ds_result = None, outputs, eval_ys, best_configs, time.time() - start_time + + if save: + with open(path, 'wb') as f: + np.save(f, ds_result) + print(f'saved results to {path}') + + return ds_result \ No newline at end of file diff --git a/TabPFN/scripts/tabular_metrics.py b/TabPFN/scripts/tabular_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..f5e9cd64356045b30da040a344408c55fac6e420 --- /dev/null +++ b/TabPFN/scripts/tabular_metrics.py @@ -0,0 +1,181 @@ +""" +=============================== +Metrics calculation +=============================== +Includes a few metric as well as functions composing metrics on results files. + +""" + + + +import numpy as np +import torch +from sklearn.metrics import roc_auc_score, accuracy_score, balanced_accuracy_score, average_precision_score +from scipy.stats import rankdata +import pandas as pd + +""" +=============================== +Metrics calculation +=============================== +""" +def auc_metric(target, pred, multi_class='ovo', numpy=False): + lib = np if numpy else torch + try: + if not numpy: + target = torch.tensor(target) if not torch.is_tensor(target) else target + pred = torch.tensor(pred) if not torch.is_tensor(pred) else pred + if len(lib.unique(target)) > 2: + if not numpy: + return torch.tensor(roc_auc_score(target, pred, multi_class=multi_class)) + return roc_auc_score(target, pred, multi_class=multi_class) + else: + if len(pred.shape) == 2: + pred = pred[:, 1] + if not numpy: + return torch.tensor(roc_auc_score(target, pred)) + return roc_auc_score(target, pred) + except ValueError as e: + print(e) + return np.nan + +def accuracy_metric(target, pred): + target = torch.tensor(target) if not torch.is_tensor(target) else target + pred = torch.tensor(pred) if not torch.is_tensor(pred) else pred + if len(torch.unique(target)) > 2: + return torch.tensor(accuracy_score(target, torch.argmax(pred, -1))) + else: + return torch.tensor(accuracy_score(target, pred[:, 1] > 0.5)) + +def average_precision_metric(target, pred): + target = torch.tensor(target) if not torch.is_tensor(target) else target + pred = torch.tensor(pred) if not torch.is_tensor(pred) else pred + if len(torch.unique(target)) > 2: + return torch.tensor(average_precision_score(target, torch.argmax(pred, -1))) + else: + return torch.tensor(average_precision_score(target, pred[:, 1] > 0.5)) + +def balanced_accuracy_metric(target, pred): + target = torch.tensor(target) if not torch.is_tensor(target) else target + pred = torch.tensor(pred) if not torch.is_tensor(pred) else pred + if len(torch.unique(target)) > 2: + return torch.tensor(balanced_accuracy_score(target, torch.argmax(pred, -1))) + else: + return torch.tensor(balanced_accuracy_score(target, pred[:, 1] > 0.5)) + +def cross_entropy(target, pred): + target = torch.tensor(target) if not torch.is_tensor(target) else target + pred = torch.tensor(pred) if not torch.is_tensor(pred) else pred + if len(torch.unique(target)) > 2: + ce = torch.nn.CrossEntropyLoss() + return ce(pred.float(), target.long()) + else: + bce = torch.nn.BCELoss() + return bce(pred[:, 1].float(), target.float()) + +def time_metric(): + """ + Dummy function, will just be used as a handler. + """ + pass + +def count_metric(x, y): + """ + Dummy function, returns one count per dataset. + """ + return 1 + +""" +=============================== +Metrics composition +=============================== +""" +def calculate_score_per_method(metric, name:str, global_results:dict, ds:list, eval_positions:list, aggregator:str='mean'): + """ + Calculates the metric given by 'metric' and saves it under 'name' in the 'global_results' + + :param metric: Metric function + :param name: Name of metric in 'global_results' + :param global_results: Dicrtonary containing the results for current method for a collection of datasets + :param ds: Dataset to calculate metrics on, a list of dataset properties + :param eval_positions: List of positions to calculate metrics on + :param aggregator: Specifies way to aggregate results across evaluation positions + :return: + """ + aggregator_f = np.nanmean if aggregator == 'mean' else np.nansum + for pos in eval_positions: + valid_positions = 0 + for d in ds: + if f'{d[0]}_outputs_at_{pos}' in global_results: + preds = global_results[f'{d[0]}_outputs_at_{pos}'] + y = global_results[f'{d[0]}_ys_at_{pos}'] + + preds, y = preds.detach().cpu().numpy() if torch.is_tensor( + preds) else preds, y.detach().cpu().numpy() if torch.is_tensor(y) else y + + try: + if metric == time_metric: + global_results[f'{d[0]}_{name}_at_{pos}'] = global_results[f'{d[0]}_time_at_{pos}'] + valid_positions = valid_positions + 1 + else: + global_results[f'{d[0]}_{name}_at_{pos}'] = aggregator_f( + [metric(y[split], preds[split]) for split in range(y.shape[0])]) + valid_positions = valid_positions + 1 + except Exception as err: + print(f'Error calculating metric with {err}, {type(err)} at {d[0]} {pos} {name}') + global_results[f'{d[0]}_{name}_at_{pos}'] = np.nan + else: + global_results[f'{d[0]}_{name}_at_{pos}'] = np.nan + + if valid_positions > 0: + global_results[f'{aggregator}_{name}_at_{pos}'] = aggregator_f([global_results[f'{d[0]}_{name}_at_{pos}'] for d in ds]) + else: + global_results[f'{aggregator}_{name}_at_{pos}'] = np.nan + + for d in ds: + metrics = [global_results[f'{d[0]}_{name}_at_{pos}'] for pos in eval_positions] + metrics = [m for m in metrics if not np.isnan(m)] + global_results[f'{d[0]}_{aggregator}_{name}'] = aggregator_f(metrics) if len(metrics) > 0 else np.nan + + metrics = [global_results[f'{aggregator}_{name}_at_{pos}'] for pos in eval_positions] + metrics = [m for m in metrics if not np.isnan(m)] + global_results[f'{aggregator}_{name}'] = aggregator_f(metrics) if len(metrics) > 0 else np.nan + + +def calculate_score(metric, name, global_results, ds, eval_positions, aggregator='mean', limit_to=''): + """ + Calls calculate_metrics_by_method with a range of methods. See arguments of that method. + :param limit_to: This method will not get metric calculations. + """ + for m in global_results: + if limit_to not in m: + continue + calculate_score_per_method(metric, name, global_results[m], ds, eval_positions, aggregator=aggregator) + + +def make_metric_matrix(global_results, methods, pos, name, ds): + result = [] + for m in global_results: + result += [[global_results[m][d[0] + '_' + name + '_at_' + str(pos)] for d in ds]] + result = np.array(result) + result = pd.DataFrame(result.T, index=[d[0] for d in ds], columns=[k[:-8] for k in list(global_results.keys())]) + + matrix_means, matrix_stds = [], [] + + for method in methods: + matrix_means += [result.iloc[:, [(method) in c for c in result.columns]].mean(axis=1)] + matrix_stds += [result.iloc[:, [(method) in c for c in result.columns]].std(axis=1)] + + matrix_means = pd.DataFrame(matrix_means, index=methods).T + matrix_stds = pd.DataFrame(matrix_stds, index=methods).T + + return matrix_means, matrix_stds + + +def make_ranks_and_wins_table(matrix): + for dss in matrix.T: + matrix.loc[dss] = rankdata(-matrix.round(3).loc[dss]) + ranks_acc = matrix.mean() + wins_acc = (matrix == 1).sum() + + return ranks_acc, wins_acc \ No newline at end of file diff --git a/TabPFN/scripts/transformer_prediction_interface.py b/TabPFN/scripts/transformer_prediction_interface.py new file mode 100644 index 0000000000000000000000000000000000000000..54f29c4f0fe537c74fa12650593aaed2c5468ab7 --- /dev/null +++ b/TabPFN/scripts/transformer_prediction_interface.py @@ -0,0 +1,357 @@ +import torch +import random + +from torch.utils.checkpoint import checkpoint + +from utils import normalize_data, to_ranking_low_mem, remove_outliers +from priors.utils import normalize_by_used_features_f +from utils import NOP + +from sklearn.preprocessing import PowerTransformer, QuantileTransformer, RobustScaler + +from notebook_utils import CustomUnpickler + +import numpy as np +from sklearn.base import BaseEstimator, ClassifierMixin +from sklearn.utils.validation import check_X_y, check_array, check_is_fitted +from sklearn.utils.multiclass import check_classification_targets +from sklearn.utils import column_or_1d +from pathlib import Path +from model_builder import load_model +import os + +def load_model_workflow(i, e, add_name, base_path, device='cpu', eval_addition=''): + """ + Workflow for loading a model and setting appropriate parameters for diffable hparam tuning. + + :param i: + :param e: + :param eval_positions_valid: + :param add_name: + :param base_path: + :param device: + :param eval_addition: + :return: + """ + def check_file(e): + model_file = f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{e}.cpkt' + model_path = os.path.join(base_path, model_file) + # print('Evaluate ', model_path) + results_file = os.path.join(base_path, + f'models_diff/prior_diff_real_results{add_name}_n_{i}_epoch_{e}_{eval_addition}.pkl') + if not Path(model_path).is_file(): # or Path(results_file).is_file(): + return None, None, None + return model_file, model_path, results_file + + model_file = None + if e == -1: + for e_ in range(100, -1, -1): + model_file_, model_path_, results_file_ = check_file(e_) + if model_file_ is not None: + e = e_ + model_file, model_path, results_file = model_file_, model_path_, results_file_ + break + else: + model_file, model_path, results_file = check_file(e) + + if model_file is None: + print('No checkpoint found') + return None + + print(f'Loading {model_file}') + + model, c = load_model(base_path, model_file, device, eval_positions=[], verbose=False) + + return model, c, results_file + + +class TabPFNClassifier(BaseEstimator, ClassifierMixin): + + def __init__(self, device='cpu', base_path='.'): + # Model file specification (Model name, Epoch) + model_string = '' + i, e = '8x_lr0.0003', -1 + + # File which contains result of hyperparameter tuning run: style (i.e. hyperparameters) and a dataframe with results. + style_file = 'prior_tuning_result.pkl' + + model, c, results_file = load_model_workflow(i, e, add_name=model_string, base_path=base_path, device=device, + eval_addition='') + style, temperature = self.load_result_minimal(style_file, i, e, base_path=base_path) + + self.device = device + self.base_path = base_path + self.model = model + self.c = c + self.style = style + self.temperature = temperature + + self.max_num_features = self.c['num_features'] + self.max_num_classes = self.c['max_num_classes'] + + def load_result_minimal(self, path, i, e, base_path='.'): + with open(os.path.join(base_path,path), 'rb') as output: + _, _, _, style, temperature, optimization_route = CustomUnpickler(output).load() + + return style, temperature + + def fit(self, X, y): + # Check that X and y have correct shape + X, y = check_X_y(X, y) + y = self._validate_targets(y) + + self.X_ = X + self.y_ = y + + if X.shape[1] > self.max_num_features: + raise ValueError("The number of features for this classifier is restricted to ", self.max_num_features) + if len(np.unique(y)) > self.max_num_classes: + raise ValueError("The number of classes for this classifier is restricted to ", self.max_num_classes) + + # Return the classifier + return self + + def _validate_targets(self, y): + y_ = column_or_1d(y, warn=True) + check_classification_targets(y) + cls, y = np.unique(y_, return_inverse=True) + if len(cls) < 2: + raise ValueError( + "The number of classes has to be greater than one; got %d class" + % len(cls) + ) + + self.classes_ = cls + + return np.asarray(y, dtype=np.float64, order="C") + + def predict_proba(self, X): + # Check is fit had been called + check_is_fitted(self) + + # Input validation + X = check_array(X) + + X_full = np.concatenate([self.X_, X], axis=0) + X_full = torch.tensor(X_full, device=self.device).float().unsqueeze(1) + y_full = np.concatenate([self.y_, self.y_[0] + np.zeros_like(X[:, 0])], axis=0) + y_full = torch.tensor(y_full, device=self.device).float().unsqueeze(1) + + eval_pos = self.X_.shape[0] + + prediction = transformer_predict(self.model[2], X_full, y_full, eval_pos, + device=self.device, + style=self.style, + inference_mode=True, + N_ensemble_configurations=10, + softmax_temperature=self.temperature + , **get_params_from_config(self.c)) + prediction_ = prediction.squeeze(0) + + return prediction_.detach().cpu().numpy() + + def predict(self, X, return_winning_probability=False): + p = self.predict_proba(X) + y = np.argmax(self.predict_proba(X), axis=-1) + y = self.classes_.take(np.asarray(y, dtype=np.intp)) + if return_winning_probability: + return y, p.max(axis=-1) + return y + +def transformer_predict(model, eval_xs, eval_ys, eval_position, + device='cpu', + max_features=100, + style=None, + inference_mode=False, + num_classes=2, + extend_features=True, + normalize_to_ranking=False, + softmax_temperature=0.0, + multiclass_decoder='permutation', + preprocess_transform='mix', + categorical_feats=[], + feature_shift_decoder=True, + N_ensemble_configurations=10, + average_logits=True, + normalize_with_sqrt=False, **kwargs): + """ + + :param model: + :param eval_xs: + :param eval_ys: should be classes that are 0-indexed and every class until num_classes-1 is present + :param eval_position: + :param rescale_features: + :param device: + :param max_features: + :param style: + :param inference_mode: + :param num_classes: + :param extend_features: + :param normalize_to_ranking: + :param softmax_temperature: + :param multiclass_decoder: + :param preprocess_transform: + :param categorical_feats: + :param feature_shift_decoder: + :param N_ensemble_configurations: + :param average_logits: + :param normalize_with_sqrt: + :param metric_used: + :return: + """ + num_classes = len(torch.unique(eval_ys)) + + def predict(eval_xs, eval_ys, used_style, softmax_temperature, return_logits): + # Initialize results array size S, B, Classes + + inference_mode_call = torch.inference_mode() if inference_mode else NOP() + with inference_mode_call: + output = model( + (used_style.repeat(eval_xs.shape[1], 1) if used_style is not None else None, eval_xs, eval_ys.float()), + single_eval_pos=eval_position)[:, :, 0:num_classes] + + output = output[:, :, 0:num_classes] / torch.exp(softmax_temperature) + if not return_logits: + output = torch.nn.functional.softmax(output, dim=-1) + #else: + # output[:, :, 1] = model((style.repeat(eval_xs.shape[1], 1) if style is not None else None, eval_xs, eval_ys.float()), + # single_eval_pos=eval_position) + + # output[:, :, 1] = torch.sigmoid(output[:, :, 1]).squeeze(-1) + # output[:, :, 0] = 1 - output[:, :, 1] + + #print('RESULTS', eval_ys.shape, torch.unique(eval_ys, return_counts=True), output.mean(axis=0)) + + return output + + def preprocess_input(eval_xs, preprocess_transform): + import warnings + + if eval_xs.shape[1] > 1: + raise Exception("Transforms only allow one batch dim - TODO") + if preprocess_transform != 'none': + if preprocess_transform == 'power' or preprocess_transform == 'power_all': + pt = PowerTransformer(standardize=True) + elif preprocess_transform == 'quantile' or preprocess_transform == 'quantile_all': + pt = QuantileTransformer(output_distribution='normal') + elif preprocess_transform == 'robust' or preprocess_transform == 'robust_all': + pt = RobustScaler(unit_variance=True) + + # eval_xs, eval_ys = normalize_data(eval_xs), normalize_data(eval_ys) + eval_xs = normalize_data(eval_xs) + + # Removing empty features + eval_xs = eval_xs[:, 0, :].cpu().numpy() + sel = [len(np.unique(eval_xs[0:eval_ys.shape[0], col])) > 1 for col in range(eval_xs.shape[1])] + eval_xs = np.array(eval_xs[:, sel]) + + warnings.simplefilter('error') + if preprocess_transform != 'none': + feats = set(range(eval_xs.shape[1])) if 'all' in preprocess_transform else set( + range(eval_xs.shape[1])) - set(categorical_feats) + for col in feats: + try: + pt.fit(eval_xs[0:eval_ys.shape[0], col:col + 1]) + trans = pt.transform(eval_xs[:, col:col + 1]) + # print(scipy.stats.spearmanr(trans[~np.isnan(eval_xs[:, col:col+1])], eval_xs[:, col:col+1][~np.isnan(eval_xs[:, col:col+1])])) + eval_xs[:, col:col + 1] = trans + except: + pass + warnings.simplefilter('default') + + eval_xs = torch.tensor(eval_xs).float().unsqueeze(1).to(device) + + # eval_xs = normalize_data(eval_xs) + + # TODO: Cautian there is information leakage when to_ranking is used, we should not use it + eval_xs = remove_outliers(eval_xs) if not normalize_to_ranking else normalize_data(to_ranking_low_mem(eval_xs)) + + # Rescale X + eval_xs = normalize_by_used_features_f(eval_xs, eval_xs.shape[-1], max_features, + normalize_with_sqrt=normalize_with_sqrt) + return eval_xs.detach() + + eval_xs, eval_ys = eval_xs.to(device), eval_ys.to(device) + eval_ys = eval_ys[:eval_position] + + model.to(device) + style = style.to(device) + + model.eval() + + import itertools + style = style.unsqueeze(0) if len(style.shape) == 1 else style + num_styles = style.shape[0] + styles_configurations = range(0, num_styles) + preprocess_transform_configurations = [preprocess_transform if i % 2 == 0 else 'none' for i in range(0, num_styles)] + if preprocess_transform == 'mix': + def get_preprocess(i): + if i == 0: + return 'power_all' + if i == 1: + return 'robust_all' + if i == 2: + return 'none' + preprocess_transform_configurations = [get_preprocess(i) for i in range(0, num_styles)] + styles_configurations = zip(styles_configurations, preprocess_transform_configurations) + + feature_shift_configurations = range(0, eval_xs.shape[2]) if feature_shift_decoder else [0] + class_shift_configurations = range(0, len(torch.unique(eval_ys))) if multiclass_decoder == 'permutation' else [0] + + ensemble_configurations = list(itertools.product(styles_configurations, feature_shift_configurations, class_shift_configurations)) + random.shuffle(ensemble_configurations) + ensemble_configurations = ensemble_configurations[0:N_ensemble_configurations] + + output = None + + eval_xs_transformed = {} + for ensemble_configuration in ensemble_configurations: + (styles_configuration, preprocess_transform_configuration), feature_shift_configuration, class_shift_configuration = ensemble_configuration + + style_ = style[styles_configuration:styles_configuration+1, :] + softmax_temperature_ = softmax_temperature[styles_configuration] + + eval_xs_, eval_ys_ = eval_xs.clone(), eval_ys.clone() + + if preprocess_transform_configuration in eval_xs_transformed: + eval_xs_ = eval_xs_transformed['preprocess_transform_configuration'].clone() + else: + eval_xs_ = preprocess_input(eval_xs_, preprocess_transform=preprocess_transform_configuration) + eval_xs_transformed['preprocess_transform_configuration'] = eval_xs_ + + eval_ys_ = ((eval_ys_ + class_shift_configuration) % num_classes).float() + + eval_xs_ = torch.cat([eval_xs_[..., feature_shift_configuration:],eval_xs_[..., :feature_shift_configuration]],dim=-1) + + # Extend X + if extend_features: + eval_xs_ = torch.cat( + [eval_xs_, + torch.zeros((eval_xs_.shape[0], eval_xs_.shape[1], max_features - eval_xs_.shape[2])).to(device)], -1) + + #preprocess_transform_ = preprocess_transform if styles_configuration % 2 == 0 else 'none' + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="None of the inputs have requires_grad=True. Gradients will be None") + output_ = checkpoint(predict, eval_xs_, eval_ys_, style_, softmax_temperature_, True) + output_ = torch.cat([output_[..., class_shift_configuration:],output_[..., :class_shift_configuration]],dim=-1) + + #output_ = predict(eval_xs, eval_ys, style_, preprocess_transform_) + if not average_logits: + output_ = torch.nn.functional.softmax(output_, dim=-1) + output = output_ if output is None else output + output_ + + output = output / len(ensemble_configurations) + if average_logits: + output = torch.nn.functional.softmax(output, dim=-1) + + output = torch.transpose(output, 0, 1) + + return output + +def get_params_from_config(c): + return {'max_features': c['num_features'] + , 'rescale_features': c["normalize_by_used_features"] + , 'normalize_to_ranking': c["normalize_to_ranking"] + , 'normalize_with_sqrt': c.get("normalize_with_sqrt", False) + } \ No newline at end of file diff --git a/TabPFN/tabular_evaluation.py b/TabPFN/tabular_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..168eb7ffaa6999785513c0e9f6af26d6548fcb7a --- /dev/null +++ b/TabPFN/tabular_evaluation.py @@ -0,0 +1,283 @@ +import time +import os +from pathlib import Path + +from tqdm import tqdm +import random +import numpy as np + +from torch import nn + +from utils import torch_nanmean +from datasets import * +from model_builder import load_model +from scripts.tabular_baselines import get_scoring_string +from scripts import tabular_metrics +from scripts.transformer_prediction_interface import * +from scripts.baseline_prediction_interface import * +""" +=============================== +PUBLIC FUNCTIONS FOR EVALUATION +=============================== +""" + + +def eval_model(i, e, valid_datasets, test_datasets, eval_positions, bptt, add_name, base_path, device='cpu', eval_addition='', **kwargs): + metrics_test, config_sample, model_path = eval_model_on_ds(i, e, test_datasets, eval_positions, bptt, add_name, base_path, device=device, eval_addition=eval_addition, **kwargs) + metrics_valid, _, _ = eval_model_on_ds(i, e, valid_datasets, eval_positions, bptt, add_name, base_path, device=device, eval_addition=eval_addition, **kwargs) + return {'mean_auc_test': metrics_test['mean_roc_at_1000'], 'mean_auc_valid': metrics_valid['mean_roc_at_1000'], 'mean_ce_test': metrics_test['mean_ce_at_1000'], 'mean_ce_valid': metrics_valid['mean_ce_at_1000'], 'config_sample': config_sample, 'model_path': model_path} + +def eval_model_on_ds(i, e, valid_datasets, eval_positions, bptt, add_name, base_path, device='cpu', eval_addition='', **kwargs): + + # How to use: evaluate_without_fitting(i,0,valid_datasets, [1024], 100000, add_name=model_string, base_path=base_path,) + def check_file(e): + model_file = f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{e}.cpkt' + model_path = os.path.join(base_path, model_file) + # print('Evaluate ', model_path) + results_file = os.path.join(base_path, + f'models_diff/prior_diff_real_results{add_name}_n_{i}_epoch_{e}_{eval_addition}.pkl') + if not Path(model_path).is_file(): # or Path(results_file).is_file(): + # print('checkpoint exists: ', Path(model_file).is_file(), ', results are written:', Path(results_file).is_file()) + return None, None, None + return model_file, model_path, results_file + + if e == -1: # use last checkpoint, if e == -1 + for e_ in range(100, -1, -1): + model_file_, model_path_, results_file_ = check_file(e_) + if model_file_ is not None: + e = e_ + model_file, model_path, results_file = model_file_, model_path_, results_file_ + break + else: + model_file, model_path, results_file = check_file(e) + + model, config_sample = load_model(base_path, model_file, device, None, verbose=False) + + params = {'max_features': config_sample['num_features'] + , 'rescale_features': config_sample["normalize_by_used_features"] + , 'normalize_to_ranking': config_sample["normalize_to_ranking"] + , 'normalize_with_sqrt': config_sample.get("normalize_with_sqrt", False) + } + metrics_valid = evaluate(datasets=valid_datasets, model=model[2], method='transformer', device=device, overwrite=True, + extend_features=True + # just removed the style keyword but transformer is trained with style, just empty + , save=False + , metric_used=tabular_metrics.cross_entropy + , return_tensor=True + , verbose=False + , eval_positions=eval_positions + , bptt=bptt + , base_path=None + , inference_mode=True + , **params + , **kwargs) + + tabular_metrics.calculate_score_per_method(tabular_metrics.auc_metric, 'roc', metrics_valid, valid_datasets, eval_positions) + tabular_metrics.calculate_score_per_method(tabular_metrics.cross_entropy, 'ce', metrics_valid, valid_datasets, eval_positions) + + return metrics_valid, config_sample, model_path + + +def evaluate(datasets, bptt, eval_positions, metric_used, model + , verbose=False + , return_tensor=False + , **kwargs): + """ + Evaluates a list of datasets for a model function. + + :param datasets: List of datasets + :param bptt: maximum sequence length + :param eval_positions: List of positions where to evaluate models + :param verbose: If True, is verbose. + :param metric_used: Which metric is optimized for. + :param return_tensor: Wheater to return results as a pytorch.tensor or numpy, this is only relevant for transformer. + :param kwargs: + :return: + """ + overall_result = {'metric_used': get_scoring_string(metric_used) + , 'bptt': bptt + , 'eval_positions': eval_positions} + + aggregated_metric_datasets, num_datasets = torch.tensor(0.0), 0 + + # For each dataset + for [ds_name, X, y, categorical_feats, _, _] in tqdm.tqdm(datasets, desc='Iterate over datasets') if verbose else datasets: + dataset_bptt = min(len(X), bptt) + # if verbose and dataset_bptt < bptt: + # print(f'Dataset too small for given sequence length, reducing to {len(X)} ({bptt})') + + aggregated_metric, num = torch.tensor(0.0), 0 + ds_result = {} + + for eval_position in (eval_positions if verbose else eval_positions): + eval_position_real = int(dataset_bptt * 0.5) if 2 * eval_position > dataset_bptt else eval_position + eval_position_bptt = int(eval_position_real * 2.0) + + r = evaluate_position(X, y, model=model + , num_classes=len(torch.unique(y)) + , categorical_feats = categorical_feats + , bptt = eval_position_bptt + , ds_name=ds_name + , eval_position = eval_position_real + , metric_used = metric_used + ,**kwargs) + + if r is None: + continue + + _, outputs, ys, best_configs, time_used = r + + if torch.is_tensor(outputs): + outputs = outputs.to(outputs.device) + ys = ys.to(outputs.device) + + ys = ys.T + ds_result[f'{ds_name}_best_configs_at_{eval_position}'] = best_configs + ds_result[f'{ds_name}_outputs_at_{eval_position}'] = outputs + ds_result[f'{ds_name}_ys_at_{eval_position}'] = ys + ds_result[f'{ds_name}_time_at_{eval_position}'] = time_used + + new_metric = torch_nanmean(torch.stack([metric_used(ys[i], outputs[i]) for i in range(ys.shape[0])])) + + if not return_tensor: + make_scalar = lambda x: float(x.detach().cpu().numpy()) if (torch.is_tensor(x) and (len(x.shape) == 0)) else x + new_metric = make_scalar(new_metric) + ds_result = {k: make_scalar(ds_result[k]) for k in ds_result.keys()} + + lib = torch if return_tensor else np + if not lib.isnan(new_metric).any(): + aggregated_metric, num = aggregated_metric + new_metric, num + 1 + + overall_result.update(ds_result) + if num > 0: + aggregated_metric_datasets, num_datasets = (aggregated_metric_datasets + (aggregated_metric / num)), num_datasets + 1 + + overall_result['mean_metric'] = aggregated_metric_datasets / num_datasets + + return overall_result + +""" +=============================== +INTERNAL HELPER FUNCTIONS +=============================== +""" + +def check_file_exists(path): + """Checks if a pickle file exists. Returns None if not, else returns the unpickled file.""" + if (os.path.isfile(path)): + print(f'loading results from {path}') + with open(path, 'rb') as f: + return np.load(f, allow_pickle=True).tolist() + return None + +def generate_valid_split(X, y, bptt, eval_position, split_number=1): + """Generates a deteministic train-(test/valid) split. Both splits must contain the same classes and all classes in + the entire datasets. If no such split can be sampled in 7 passes, returns None. + + :param X: torch tensor, feature values + :param y: torch tensor, class values + :param bptt: Number of samples in train + test + :param eval_position: Number of samples in train, i.e. from which index values are in test + :param split_number: The split id + :return: + """ + done, seed = False, 13 + + torch.manual_seed(split_number) + perm = torch.randperm(X.shape[0]) if split_number > 1 else torch.arange(0, X.shape[0]) + X, y = X[perm], y[perm] + + while not done: + if seed > 20: + return None, None # No split could be generated in 7 passes, return None + random.seed(seed) + i = random.randint(0, len(X) - bptt) if len(X) - bptt > 0 else 0 + y_ = y[i:i + bptt] + + # Checks if all classes from dataset are contained and classes in train and test are equal (contain same + # classes) and + done = len(torch.unique(y_)) == len(torch.unique(y)) + done = done and torch.all(torch.unique(y_) == torch.unique(y)) + done = done and len(torch.unique(y_[:eval_position])) == len(torch.unique(y_[eval_position:])) + done = done and torch.all(torch.unique(y_[:eval_position]) == torch.unique(y_[eval_position:])) + seed = seed + 1 + + eval_xs = torch.stack([X[i:i + bptt].clone()], 1) + eval_ys = torch.stack([y[i:i + bptt].clone()], 1) + + return eval_xs, eval_ys + + +def evaluate_position(X, y, categorical_feats, model, bptt + , eval_position, overwrite, save, base_path, path_interfix, method, ds_name, fetch_only=False + , max_time=300, split_number=1 + , per_step_normalization=False, **kwargs): + """ + Evaluates a dataset with a 'bptt' number of training samples. + + :param X: Dataset X + :param y: Dataset labels + :param categorical_feats: Indices of categorical features. + :param model: Model function + :param bptt: Sequence length. + :param eval_position: Number of training samples. + :param overwrite: Wheater to ove + :param overwrite: If True, results on disk are overwritten. + :param save: + :param path_interfix: Used for constructing path to write on disk. + :param method: Model name. + :param ds_name: Datset name. + :param fetch_only: Wheater to calculate or only fetch results. + :param per_step_normalization: + :param kwargs: + :return: + """ + + if save: + path = os.path.join(base_path, f'results/tabular/{path_interfix}/results_{method}_{ds_name}_{eval_position}_{bptt}_{split_number}.npy') + #log_path = + + ## Load results if on disk + if not overwrite: + result = check_file_exists(path) + if result is not None: + if not fetch_only: + print(f'Loaded saved result for {path}') + return result + elif fetch_only: + print(f'Could not load saved result for {path}') + return None + + ## Generate data splits + eval_xs, eval_ys = generate_valid_split(X, y, bptt, eval_position, split_number=split_number) + if eval_xs is None: + return None + print(f"No dataset could be generated {ds_name} {bptt}") + + eval_ys = (eval_ys > torch.unique(eval_ys).unsqueeze(0)).sum(axis=1).unsqueeze(-1) + + start_time = time.time() + + if isinstance(model, nn.Module): # Two separate predict interfaces for transformer and baselines + outputs, best_configs = transformer_predict(model, eval_xs, eval_ys, eval_position, categorical_feats=categorical_feats, **kwargs), None + else: + _, outputs, best_configs = baseline_predict(model, eval_xs, eval_ys, categorical_feats + , eval_pos=eval_position + , max_time=max_time, **kwargs) + + eval_ys = eval_ys[eval_position:] + if outputs is None: + return None + + if torch.is_tensor(outputs): # Transfers data to cpu for saving + outputs = outputs.cpu() + eval_ys = eval_ys.cpu() + + ds_result = None, outputs, eval_ys, best_configs, time.time() - start_time + + if save: + with open(path, 'wb') as f: + np.save(f, ds_result) + print(f'saved results to {path}') + + return ds_result \ No newline at end of file diff --git a/TabPFN/train.py b/TabPFN/train.py new file mode 100644 index 0000000000000000000000000000000000000000..edfe5cd0a5d9811cff22f362936cb15d5ed504a4 --- /dev/null +++ b/TabPFN/train.py @@ -0,0 +1,386 @@ +import os +import itertools +import argparse +import time +import datetime +import yaml +from contextlib import nullcontext + + +import torch +from torch import nn + +import utils +from transformer import TransformerModel +from utils import get_cosine_schedule_with_warmup, get_openai_lr, StoreDictKeyPair, get_weighted_single_eval_pos_sampler, get_uniform_single_eval_pos_sampler +import priors +import encoders +import positional_encodings +from utils import init_dist +from torch.cuda.amp import autocast + +class Losses(): + gaussian = nn.GaussianNLLLoss(full=True, reduction='none') + mse = nn.MSELoss(reduction='none') + ce = lambda weight : nn.CrossEntropyLoss(reduction='none', weight=weight) + bce = nn.BCEWithLogitsLoss(reduction='none') + + +def train(priordataloader_class, criterion, encoder_generator, emsize=200, nhid=200, nlayers=6, nhead=2, dropout=0.2, + epochs=10, steps_per_epoch=100, batch_size=200, bptt=10, lr=None, weight_decay=0.0, warmup_epochs=10, input_normalization=False, + y_encoder_generator=None, pos_encoder_generator=None, decoder=None, extra_prior_kwargs_dict={}, scheduler=get_cosine_schedule_with_warmup, + load_weights_from_this_state_dict=None, validation_period=10, single_eval_pos_gen=None, bptt_extra_samples=None, gpu_device='cuda:0', + aggregate_k_gradients=1, verbose=True, style_encoder_generator=None, check_is_compatible=True, epoch_callback=None, + initializer=None, initialize_with_model=None, train_mixed_precision=False, total_available_time_in_s=None, normalize_labels=True, **model_extra_args + ): + assert (epochs is None) != (total_available_time_in_s is None) + start_of_training = time.time() + device = gpu_device if torch.cuda.is_available() else 'cpu:0' + print(f'Using {device} device') + using_dist, rank, device = init_dist(device) + bptt_sampler = (lambda : single_eval_pos_gen() + bptt_extra_samples if callable(single_eval_pos_gen) else single_eval_pos_gen + bptt_extra_samples) if bptt_extra_samples is not None else bptt + dl = priordataloader_class(num_steps=steps_per_epoch, batch_size=batch_size, seq_len=bptt_sampler, seq_len_maximum=bptt+(bptt_extra_samples if bptt_extra_samples else 0), device=device, **extra_prior_kwargs_dict) + if dl.fuse_x_y: + raise Exception("Illegal parameter") + + encoder = encoder_generator(dl.num_features+1 if dl.fuse_x_y else dl.num_features,emsize) + style_def = next(iter(dl))[0][0] # This is (style, x, y), target with x and y with batch size + + style_encoder = style_encoder_generator(hyperparameter_definitions=style_def[0], em_size=emsize) if (style_def is not None) else None + n_out = dl.num_outputs + if isinstance(criterion, nn.GaussianNLLLoss): + n_out *= 2 + elif isinstance(criterion, nn.CrossEntropyLoss): + n_out *= criterion.weight.shape[0] + model = TransformerModel(encoder, n_out, emsize, nhead, nhid, nlayers, dropout, style_encoder=style_encoder, + y_encoder=y_encoder_generator(dl.num_outputs, emsize), input_normalization=input_normalization, + pos_encoder=(pos_encoder_generator or positional_encodings.NoPositionalEncoding)(emsize, bptt*2), + decoder=decoder, init_method=initializer, **model_extra_args + ) + model.criterion = criterion + if load_weights_from_this_state_dict is not None: + model.load_state_dict(load_weights_from_this_state_dict) + if initialize_with_model is not None: + model.init_from_small_model(initialize_with_model) + + print(f"Using a Transformer with {sum(p.numel() for p in model.parameters())/1000/1000:.{2}f} M parameters") + + try: + for (k, v), (k2, v2) in zip(model.state_dict().items(), initialize_with_model.state_dict().items()): + print(k, ((v - v2) / v).abs().mean(), v.shape) + except Exception: + pass + + model.to(device) + if using_dist: + print("Distributed training") + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[rank], output_device=rank, broadcast_buffers=False) + + + # learning rate + if lr is None: + lr = get_openai_lr(model) + print(f"Using OpenAI max lr of {lr}.") + optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay) + scheduler = scheduler(optimizer, warmup_epochs, epochs if epochs is not None else 100) # when training for fixed time lr schedule takes 100 steps + + def train_step(): + model.train() # Turn on the train mode + total_loss = 0. + total_positional_losses = 0. + total_positional_losses_recorded = 0 + before_get_batch = time.time() + assert len(dl) % aggregate_k_gradients == 0, 'Please set the number of steps per epoch s.t. `aggregate_k_gradients` divides it.' + valid_batch_steps = 0.0 + for batch, (data, targets) in enumerate(dl): + if using_dist and not (batch % aggregate_k_gradients == aggregate_k_gradients - 1): + cm = model.no_sync() + #print(f'p={rank}, no_sync', force=True) + else: + cm = nullcontext() + #print(f'p={rank}, sync', force=True) + with cm: + time_to_get_batch = time.time() - before_get_batch + before_forward = time.time() + if bptt_extra_samples is None: + single_eval_pos = single_eval_pos_gen() if callable(single_eval_pos_gen) else single_eval_pos_gen + else: + single_eval_pos = targets.shape[0] - bptt_extra_samples + + is_compatible = torch.ones((targets.shape[1])).bool() + if check_is_compatible or normalize_labels: + for b in range(targets.shape[1]): + targets_in_train = torch.unique(targets[:single_eval_pos, b], sorted=True) + targets_in_eval = torch.unique(targets[single_eval_pos:, b], sorted=True) + + if check_is_compatible: + is_compatible[b] = len(targets_in_train) == len(targets_in_eval) and (targets_in_train == targets_in_eval).all() + is_compatible[b] = is_compatible[b] and len(targets_in_train) > 1 + + # Set targets to range starting from 0 (e.g. targets 0, 2, 5, 2 will be converted to 0, 1, 2, 1) + if normalize_labels: + targets[:, b] = (targets[:, b] > torch.unique(targets[:, b]).unsqueeze(1)).sum(axis=0).unsqueeze(0) + valid_batch_steps += is_compatible.float().mean() + is_compatible = is_compatible.to(device) + #if using_dist and check_is_compatible: + # print('step share before reduce',curr_step_share, force=True) + # curr_step_share = curr_step_share.to(device) + # torch.distributed.all_reduce_multigpu([curr_step_share], op=torch.distributed.ReduceOp.SUM) + # curr_step_share = curr_step_share.cpu() / torch.distributed.get_world_size() + # print('step share after reduce',curr_step_share, torch.distributed.get_world_size(), force=True) + + # If style is set to None, it should not be transferred to device + output = model(tuple(e.to(device) if torch.is_tensor(e) else e for e in data) if isinstance(data, tuple) else data.to(device) + , single_eval_pos=single_eval_pos) + + forward_time = time.time() - before_forward + + #output, targets = output[:, is_compatible], targets[:, is_compatible] + + if single_eval_pos is not None: + targets = targets[single_eval_pos:] + if isinstance(criterion, nn.GaussianNLLLoss): + assert output.shape[-1] == 2, \ + 'need to write a little bit of code to handle multiple regression targets at once' + + mean_pred = output[..., 0] + var_pred = output[..., 1].abs() + losses = criterion(mean_pred.flatten(), targets.to(device).flatten(), var=var_pred.flatten()) + elif isinstance(criterion, (nn.MSELoss, nn.BCEWithLogitsLoss)): + losses = criterion(output.flatten(), targets.to(device).flatten()) + elif isinstance(criterion, (nn.CrossEntropyLoss)): + #print(n_out, targets.min(), targets.max(), force=True) + losses = criterion(output.reshape(-1, n_out), targets.to(device).long().flatten()) + else: + losses = criterion(output.reshape(-1, n_out), targets.to(device).flatten()) + losses = losses.view(*output.shape[0:2]) + loss = losses.mean(0) @ is_compatible.float() / losses.shape[1] + #loss = torch_nanmean(losses, axis=[0, 1]) * is_compatible.float().mean() + # not sure whether we can go without the nan checks. + + loss.backward() + + if ((batch % aggregate_k_gradients == aggregate_k_gradients - 1) and (not check_is_compatible or using_dist))\ + or (valid_batch_steps >= aggregate_k_gradients and (check_is_compatible and not using_dist)): + with torch.no_grad(): + for p in model.parameters(): + if p.grad is not None: + p.grad.div_(valid_batch_steps) + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.) + try: + optimizer.step() + except: + print("Invalid optimization step encountered") + optimizer.zero_grad() + valid_batch_steps = 0.0 + + step_time = time.time() - before_forward + + if not torch.isnan(loss): + total_loss += loss.item() + total_positional_losses += losses.mean(1).cpu().detach() if single_eval_pos is None else \ + nn.functional.one_hot(torch.tensor(single_eval_pos), bptt)*loss.cpu().detach() + + total_positional_losses_recorded += torch.ones(bptt) if single_eval_pos is None else \ + nn.functional.one_hot(torch.tensor(single_eval_pos), bptt) + + before_get_batch = time.time() + return total_loss / steps_per_epoch, ( + total_positional_losses / total_positional_losses_recorded).tolist(), time_to_get_batch, forward_time, step_time + + best_val_loss = float("inf") + best_model = None + total_loss = float('inf') + total_positional_losses = float('inf') + try: + for epoch in (range(1, epochs + 1) if epochs is not None else itertools.count(1)): + + epoch_start_time = time.time() + if train_mixed_precision: + with autocast(): + total_loss, total_positional_losses, time_to_get_batch, forward_time, step_time = train_step() + else: + total_loss, total_positional_losses, time_to_get_batch, forward_time, step_time = train_step() + if hasattr(dl, 'validate') and epoch % validation_period == 0: + with torch.no_grad(): + val_score = dl.validate(model) + else: + val_score = None + + if verbose: + print('-' * 89) + print( + f'| end of epoch {epoch:3d} | time: {(time.time() - epoch_start_time):5.2f}s | mean loss {total_loss:5.2f} | ' + f"pos losses {','.join([f'{l:5.2f}' for l in total_positional_losses])}, lr {scheduler.get_last_lr()[0]}" + f' data time {time_to_get_batch:5.2f} step time {step_time:5.2f}' + f' forward time {forward_time:5.2f}' + (f'val score {val_score}' if val_score is not None else '')) + print('-' * 89) + + # stepping with wallclock time based scheduler + current_time = time.time() + if epoch_callback is not None and rank == 0: + epoch_callback(model, epoch / epochs if total_available_time_in_s is None else # noqa + (current_time - start_of_training) / total_available_time_in_s # noqa + ) + if epochs is None and (current_time - start_of_training) > total_available_time_in_s: # noqa + break + if epochs is None: + scheduler.step((current_time - epoch_start_time) / total_available_time_in_s * 100) + else: + scheduler.step() + except KeyboardInterrupt: + pass + + return total_loss, total_positional_losses, model.to('cpu'), dl + +def _parse_args(config_parser, parser): + # Do we have a config file to parse? + args_config, remaining = config_parser.parse_known_args() + if args_config.config: + with open(args_config.config, 'r') as f: + cfg = yaml.safe_load(f) + parser.set_defaults(**cfg) + + # The main arg parser parses the rest of the args, the usual + # defaults will have been overridden if config file specified. + args = parser.parse_args(remaining) + + # Cache the args as a text string to save them in the output dir later + args_text = yaml.safe_dump(args.__dict__, default_flow_style=False) + return args, args_text + + +if __name__ == '__main__': + config_parser = argparse.ArgumentParser(description='Only used as a first parser for the config file path.') + config_parser.add_argument('--config') + parser = argparse.ArgumentParser() + parser.add_argument('prior') + parser.add_argument('--loss_function', default='barnll') + # Optional Arg's for `--loss_function barnll` + parser.add_argument('--min_y', type=float, help='barnll can only model y in strict ranges, this is the minimum y can take.') + parser.add_argument('--max_y', type=float, help='barnll can only model y in strict ranges, this is the maximum y can take.') + parser.add_argument('--num_buckets', default=100, type=int) + #parser.add_argument('--num_features', default=None, type=int, help='Specify depending on the prior.') + parser.add_argument("--extra_prior_kwargs_dict", default={'fuse_x_y': False}, dest="extra_prior_kwargs_dict", action=StoreDictKeyPair, nargs="+", metavar="KEY=VAL", help='Specify depending on the prior.') + parser.add_argument('--encoder', default='linear', type=str, help='Specify depending on the prior.') + parser.add_argument('--y_encoder', default='linear', type=str, help='Specify depending on the prior. You should specify this if you do not fuse x and y.') + parser.add_argument('--pos_encoder', default='sinus', type=str, help='Specify depending on the prior.') + parser.add_argument('--bptt', default=10, type=int) + parser.add_argument('--epochs', default=200, type=int) + parser.add_argument('--warmup_epochs', default=50, type=int) + parser.add_argument('--validation_period', default=10, type=int) + parser.add_argument('--permutation_invariant_max_eval_pos', default=None, type=int, help='Set this to an int to ') + parser.add_argument('--permutation_invariant_sampling', default='weighted', help="Only relevant if --permutation_invariant_max_eval_pos is set.") + + # these can likely be mostly left at defaults + parser.add_argument('--emsize', default=512, type=int) # sometimes even larger is better e.g. 1024 + parser.add_argument('--nlayers', default=6, type=int) + parser.add_argument('--nhid', default=None, type=int) # 2*emsize is the default + parser.add_argument('--nhead', default=4, type=int) # nhead = emsize / 64 in the original paper + parser.add_argument('--dropout', default=.0, type=float) + parser.add_argument('--steps_per_epoch', default=10, type=int) + parser.add_argument('--batch_size', default=1000, type=int) + parser.add_argument('--lr', '--learning_rate', default=.001, type=float) # try also .0003, .0001, go lower with lower batch size + + args, _ = _parse_args(config_parser, parser) + + if args.nhid is None: + args.nhid = 2*args.emsize + + prior = args.__dict__.pop('prior') + + if prior == 'gp': + prior = priors.fast_gp.DataLoader + elif prior == 'ridge': + prior = priors.ridge.DataLoader + elif prior == 'stroke': + prior = priors.stroke.DataLoader + elif prior == 'mix_gp': + prior = priors.fast_gp_mix.DataLoader + else: + raise NotImplementedError(f'Prior == {prior}.') + + loss_function = args.__dict__.pop('loss_function') + + criterion = nn.GaussianNLLLoss(reduction='none', full=True) + classificiation_criterion = nn.CrossEntropyLoss(reduction='none') + num_buckets = args.__dict__.pop('num_buckets') + max_y = args.__dict__.pop('max_y') + min_y = args.__dict__.pop('min_y') + # criterion = nn.MSELoss(reduction='none') + + def get_y_sample(): + dl = prior(num_steps=1, batch_size=args.batch_size * args.steps_per_epoch, seq_len=args.bptt, device=device, + **args.extra_prior_kwargs_dict) + y_sample = next(iter(dl))[-1] + print(f'Creating Bar distribution with borders from y sample of size {y_sample.numel()}') + return y_sample + + if loss_function == 'ce': + criterion = nn.CrossEntropyLoss(reduction='none') + elif loss_function == 'gaussnll': + criterion = nn.GaussianNLLLoss(reduction='none', full=True) + elif loss_function == 'mse': + criterion = nn.MSELoss(reduction='none') + elif loss_function == 'barnll': + criterion = BarDistribution(borders=get_bucket_limits(num_buckets, full_range=(min_y,max_y))) + elif loss_function == 'adaptivebarnll': + borders = get_bucket_limits(num_buckets, ys=get_y_sample(), full_range=(min_y,max_y)) + criterion = BarDistribution(borders=borders) + elif loss_function == 'adaptivefullsupportbarnll': + assert min_y is None and max_y is None, "Please do not specify `min_y` and `max_y` with `unboundedadaptivebarnll`." + borders = get_bucket_limits(num_buckets, ys=get_y_sample()) + criterion = FullSupportBarDistribution(borders=borders) + else: + raise NotImplementedError(f'loss_function == {loss_function}.') + + + + encoder = args.__dict__.pop('encoder') + y_encoder = args.__dict__.pop('y_encoder') + + def get_encoder_generator(encoder): + if encoder == 'linear': + encoder_generator = encoders.Linear + elif encoder == 'mlp': + encoder_generator = encoders.MLP + elif encoder == 'positional': + encoder_generator = encoders.Positional + else: + raise NotImplementedError(f'A {encoder} encoder is not valid.') + return encoder_generator + + encoder_generator = get_encoder_generator(encoder) + y_encoder_generator = get_encoder_generator(y_encoder) + + pos_encoder = args.__dict__.pop('pos_encoder') + + if pos_encoder == 'none': + pos_encoder_generator = None + elif pos_encoder == 'sinus': + pos_encoder_generator = positional_encodings.PositionalEncoding + elif pos_encoder == 'learned': + pos_encoder_generator = positional_encodings.LearnedPositionalEncoding + elif pos_encoder == 'paired_scrambled_learned': + pos_encoder_generator = positional_encodings.PairedScrambledPositionalEncodings + else: + raise NotImplementedError(f'pos_encoer == {pos_encoder} is not valid.') + + permutation_invariant_max_eval_pos = args.__dict__.pop('permutation_invariant_max_eval_pos') + permutation_invariant_sampling = args.__dict__.pop('permutation_invariant_sampling') + if permutation_invariant_max_eval_pos is not None: + if permutation_invariant_sampling == 'weighted': + get_sampler = get_weighted_single_eval_pos_sampler + elif permutation_invariant_sampling == 'uniform': + get_sampler = get_uniform_single_eval_pos_sampler + else: + raise ValueError() + args.__dict__['single_eval_pos_gen'] = get_sampler(permutation_invariant_max_eval_pos) + + + print("ARGS for `train`:", args.__dict__) + + train(prior, criterion, encoder_generator, + y_encoder_generator=y_encoder_generator, pos_encoder_generator=pos_encoder_generator, + **args.__dict__) + diff --git a/TabPFN/transformer.py b/TabPFN/transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..85903897bad661b8617696d534e06cc205cb448b --- /dev/null +++ b/TabPFN/transformer.py @@ -0,0 +1,226 @@ +import math +from typing import Optional + +import torch +import torch.nn as nn +from torch import Tensor +from torch.nn import Module, TransformerEncoder + +from layer import TransformerEncoderLayer, _get_activation_fn +from utils import SeqBN, bool_mask_to_att_mask + + + +class TransformerModel(nn.Module): + def __init__(self, encoder, n_out, ninp, nhead, nhid, nlayers, dropout=0.0, style_encoder=None, y_encoder=None, + pos_encoder=None, decoder=None, input_normalization=False, init_method=None, pre_norm=False, + activation='gelu', recompute_attn=False, num_global_att_tokens=0, full_attention=False, + all_layers_same_init=True): + super().__init__() + self.model_type = 'Transformer' + encoder_layer_creator = lambda: TransformerEncoderLayer(ninp, nhead, nhid, dropout, activation=activation, + pre_norm=pre_norm, recompute_attn=recompute_attn) + self.transformer_encoder = TransformerEncoder(encoder_layer_creator(), nlayers)\ + if all_layers_same_init else TransformerEncoderDiffInit(encoder_layer_creator, nlayers) + self.ninp = ninp + self.encoder = encoder + self.y_encoder = y_encoder + self.pos_encoder = pos_encoder + self.decoder = decoder(ninp, nhid, n_out) if decoder is not None else nn.Sequential(nn.Linear(ninp, nhid), nn.GELU(), nn.Linear(nhid, n_out)) + self.input_ln = SeqBN(ninp) if input_normalization else None + self.style_encoder = style_encoder + self.init_method = init_method + if num_global_att_tokens is not None: + assert not full_attention + self.global_att_embeddings = nn.Embedding(num_global_att_tokens, ninp) if num_global_att_tokens else None + self.full_attention = full_attention + + self.n_out = n_out + self.nhid = nhid + + self.init_weights() + + @staticmethod + def generate_square_subsequent_mask(sz): + mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1) + return bool_mask_to_att_mask(mask) + + @staticmethod + def generate_D_q_matrix(sz, query_size): + train_size = sz-query_size + mask = torch.zeros(sz,sz) == 0 + mask[:,train_size:].zero_() + mask |= torch.eye(sz) == 1 + return bool_mask_to_att_mask(mask) + + @staticmethod + def generate_global_att_query_matrix(num_global_att_tokens, seq_len, num_query_tokens): + train_size = seq_len + num_global_att_tokens - num_query_tokens + sz = seq_len + num_global_att_tokens + mask = torch.zeros(num_query_tokens, sz) == 0 + mask[:,train_size:].zero_() + mask[:,train_size:] |= torch.eye(num_query_tokens) == 1 + return bool_mask_to_att_mask(mask) + + @staticmethod + def generate_global_att_trainset_matrix(num_global_att_tokens, seq_len, num_query_tokens): + train_size = seq_len + num_global_att_tokens - num_query_tokens + trainset_size = seq_len - num_query_tokens + mask = torch.zeros(trainset_size, num_global_att_tokens) == 0 + #mask[:,num_global_att_tokens:].zero_() + #mask[:,num_global_att_tokens:] |= torch.eye(trainset_size) == 1 + return bool_mask_to_att_mask(mask) + + @staticmethod + def generate_global_att_globaltokens_matrix(num_global_att_tokens, seq_len, num_query_tokens): + mask = torch.zeros(num_global_att_tokens, num_global_att_tokens+seq_len-num_query_tokens) == 0 + return bool_mask_to_att_mask(mask) + + def init_weights(self): + initrange = 1. + # if isinstance(self.encoder,EmbeddingEncoder): + # self.encoder.weight.data.uniform_(-initrange, initrange) + # self.decoder.bias.data.zero_() + # self.decoder.weight.data.uniform_(-initrange, initrange) + if self.init_method is not None: + self.apply(self.init_method) + for layer in self.transformer_encoder.layers: + nn.init.zeros_(layer.linear2.weight) + nn.init.zeros_(layer.linear2.bias) + attns = layer.self_attn if isinstance(layer.self_attn, nn.ModuleList) else [layer.self_attn] + for attn in attns: + nn.init.zeros_(attn.out_proj.weight) + nn.init.zeros_(attn.out_proj.bias) + + def forward(self, src, src_mask=None, single_eval_pos=None): + assert isinstance(src, tuple), 'fuse_x_y is forbidden, that is inputs have to be given as (x,y) or (style,x,y)' + + if len(src) == 2: + src = (None,) + src + + style_src, style_src_size = (src[0], (0 if (src[0] is None) else 1)) + if src_mask is not None: assert self.global_att_embeddings is None or isinstance(src_mask, tuple) + if src_mask is None: + x_src = src[1] + if self.global_att_embeddings is None: + full_len = len(x_src) + style_src_size + if self.full_attention: + src_mask = bool_mask_to_att_mask(torch.ones((full_len, full_len), dtype=torch.bool)).to(x_src.device) + else: + src_mask = self.generate_D_q_matrix(len(x_src) + style_src_size, len(x_src) + style_src_size -single_eval_pos).to(x_src.device) + else: + src_mask_args = (self.global_att_embeddings.num_embeddings, + len(x_src) + style_src_size, + len(x_src) + style_src_size - single_eval_pos) + src_mask = (self.generate_global_att_globaltokens_matrix(*src_mask_args).to(x_src.device), + self.generate_global_att_trainset_matrix(*src_mask_args).to(x_src.device), + self.generate_global_att_query_matrix(*src_mask_args).to(x_src.device)) + + style_src, x_src, y_src = src + x_src = self.encoder(x_src) + y_src = self.y_encoder(y_src.unsqueeze(-1) if len(y_src.shape) < len(x_src.shape) else y_src) + style_src = self.style_encoder(style_src).unsqueeze(0) if self.style_encoder else torch.tensor([], device=x_src.device) + global_src = torch.tensor([], device=x_src.device) if self.global_att_embeddings is None else \ + self.global_att_embeddings.weight.unsqueeze(1).repeat(1, x_src.shape[1], 1) + train_x = x_src[:single_eval_pos] + y_src[:single_eval_pos] + src = torch.cat([global_src, style_src, train_x, x_src[single_eval_pos:]], 0) + + if self.input_ln is not None: + src = self.input_ln(src) + + if self.pos_encoder is not None: + src = self.pos_encoder(src) + + # If we have style input, drop its output + output = self.transformer_encoder(src, src_mask)[style_src_size:] + output = self.decoder(output) + return output[single_eval_pos+(self.global_att_embeddings.num_embeddings if self.global_att_embeddings else 0):] + + @torch.no_grad() + def init_from_small_model(self, small_model): + assert isinstance(self.decoder, nn.Linear) and isinstance(self.encoder, (nn.Linear, nn.Sequential)) \ + and isinstance(self.y_encoder, (nn.Linear, nn.Sequential)) + + def set_encoder_weights(my_encoder, small_model_encoder): + my_encoder_linear, small_encoder_linear = (my_encoder, small_model_encoder) \ + if isinstance(my_encoder, nn.Linear) else (my_encoder[-1], small_model_encoder[-1]) + small_in_dim = small_encoder_linear.out_features + my_encoder_linear.weight.zero_() + my_encoder_linear.bias.zero_() + my_encoder_linear.weight[:small_in_dim] = small_encoder_linear.weight + my_encoder_linear.bias[:small_in_dim] = small_encoder_linear.bias + + set_encoder_weights(self.encoder, small_model.encoder) + set_encoder_weights(self.y_encoder, small_model.y_encoder) + + small_in_dim = small_model.decoder.in_features + + self.decoder.weight[:, :small_in_dim] = small_model.decoder.weight + self.decoder.bias = small_model.decoder.bias + + for my_layer, small_layer in zip(self.transformer_encoder.layers, small_model.transformer_encoder.layers): + small_hid_dim = small_layer.linear1.out_features + my_in_dim = my_layer.linear1.in_features + + # packed along q,k,v order in first dim + my_in_proj_w = my_layer.self_attn.in_proj_weight + small_in_proj_w = small_layer.self_attn.in_proj_weight + + my_in_proj_w.view(3, my_in_dim, my_in_dim)[:, :small_in_dim, :small_in_dim] = small_in_proj_w.view(3, + small_in_dim, + small_in_dim) + my_layer.self_attn.in_proj_bias.view(3, my_in_dim)[:, + :small_in_dim] = small_layer.self_attn.in_proj_bias.view(3, small_in_dim) + + my_layer.self_attn.out_proj.weight[:small_in_dim, :small_in_dim] = small_layer.self_attn.out_proj.weight + my_layer.self_attn.out_proj.bias[:small_in_dim] = small_layer.self_attn.out_proj.bias + + my_layer.linear1.weight[:small_hid_dim, :small_in_dim] = small_layer.linear1.weight + my_layer.linear1.bias[:small_hid_dim] = small_layer.linear1.bias + + my_layer.linear2.weight[:small_in_dim, :small_hid_dim] = small_layer.linear2.weight + my_layer.linear2.bias[:small_in_dim] = small_layer.linear2.bias + + my_layer.norm1.weight[:small_in_dim] = math.sqrt(small_in_dim / my_in_dim) * small_layer.norm1.weight + my_layer.norm2.weight[:small_in_dim] = math.sqrt(small_in_dim / my_in_dim) * small_layer.norm2.weight + + my_layer.norm1.bias[:small_in_dim] = small_layer.norm1.bias + my_layer.norm2.bias[:small_in_dim] = small_layer.norm2.bias + + +class TransformerEncoderDiffInit(Module): + r"""TransformerEncoder is a stack of N encoder layers + + Args: + encoder_layer_creator: a function generating objects of TransformerEncoderLayer class without args (required). + num_layers: the number of sub-encoder-layers in the encoder (required). + norm: the layer normalization component (optional). + """ + __constants__ = ['norm'] + + def __init__(self, encoder_layer_creator, num_layers, norm=None): + super().__init__() + self.layers = nn.ModuleList([encoder_layer_creator() for _ in range(num_layers)]) + self.num_layers = num_layers + self.norm = norm + + def forward(self, src: Tensor, mask: Optional[Tensor] = None, src_key_padding_mask: Optional[Tensor] = None) -> Tensor: + r"""Pass the input through the encoder layers in turn. + + Args: + src: the sequence to the encoder (required). + mask: the mask for the src sequence (optional). + src_key_padding_mask: the mask for the src keys per batch (optional). + + Shape: + see the docs in Transformer class. + """ + output = src + + for mod in self.layers: + output = mod(output, src_mask=mask, src_key_padding_mask=src_key_padding_mask) + + if self.norm is not None: + output = self.norm(output) + + return output diff --git a/TabPFN/utils.py b/TabPFN/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b600696b0baf302858ceabb99a1e4e24986c0624 --- /dev/null +++ b/TabPFN/utils.py @@ -0,0 +1,236 @@ +import os +import math +import argparse +import random +import datetime + +import torch +from torch import nn +from torch.optim.lr_scheduler import LambdaLR +import numpy as np + +# copied from huggingface +def get_cosine_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, num_cycles=0.5, last_epoch=-1): + """ Create a schedule with a learning rate that decreases following the + values of the cosine function between 0 and `pi * cycles` after a warmup + period during which it increases linearly between 0 and 1. + """ + + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps)) + return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) + + return LambdaLR(optimizer, lr_lambda, last_epoch) + +# copied from huggingface +def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1): + """ + Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after + a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer. + + Args: + optimizer (:class:`~torch.optim.Optimizer`): + The optimizer for which to schedule the learning rate. + num_warmup_steps (:obj:`int`): + The number of steps for the warmup phase. + num_training_steps (:obj:`int`): + The total number of training steps. + last_epoch (:obj:`int`, `optional`, defaults to -1): + The index of the last epoch when resuming training. + + Return: + :obj:`torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. + """ + + def lr_lambda(current_step: int): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + return max( + 0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps)) + ) + + return LambdaLR(optimizer, lr_lambda, last_epoch) + + +def get_openai_lr(transformer_model): + num_params = sum(p.numel() for p in transformer_model.parameters()) + return 0.003239 - 0.0001395 * math.log(num_params) + + +def get_weighted_single_eval_pos_sampler(max_len): + """ + This gives a sampler that can be used for `single_eval_pos` which yields good performance for all positions p, + where p <= `max_len`. At most `max_len` - 1 examples are shown to the Transformer. + :return: Sampler that can be fed to `train()` as `single_eval_pos_gen`. + """ + return lambda: random.choices(range(max_len), [1 / (max_len - i) for i in range(max_len)])[0] + + +def get_uniform_single_eval_pos_sampler(max_len, min_len=0): + """ + Just sample any evaluation position with the same weight + :return: Sampler that can be fed to `train()` as `single_eval_pos_gen`. + """ + return lambda: random.choices(range(min_len, max_len))[0] + + +class SeqBN(nn.Module): + def __init__(self, d_model): + super().__init__() + self.bn = nn.BatchNorm1d(d_model) + self.d_model = d_model + + def forward(self, x): + assert self.d_model == x.shape[-1] + flat_x = x.view(-1, self.d_model) + flat_x = self.bn(flat_x) + return flat_x.view(*x.shape) + + +def set_locals_in_self(locals): + self = locals['self'] + for var_name, val in locals.items(): + if var_name != 'self': setattr(self, var_name, val) + + +default_device = 'cuda:0' if torch.cuda.is_available() else 'cpu:0' + + +# Copied from StackOverflow, but we do an eval on the values additionally +class StoreDictKeyPair(argparse.Action): + def __init__(self, option_strings, dest, nargs=None, **kwargs): + self._nargs = nargs + super(StoreDictKeyPair, self).__init__(option_strings, dest, nargs=nargs, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + my_dict = {} + for kv in values: + k, v = kv.split("=") + try: + my_dict[k] = eval(v) + except NameError: + my_dict[k] = v + setattr(namespace, self.dest, my_dict) + print("dict values: {}".format(my_dict)) + +def get_nan_value(v, set_value_to_nan=0.0): + if random.random() < set_value_to_nan: + return v + else: + return random.choice([-999, 0, 1, 999]) + +def to_ranking(data): + x = (data >= data.unsqueeze(-3)) + x = x.sum(0) + return x +# TODO: Is there a better way to do this? +# 1. Cmparing to unique elements: When all values are different we still get quadratic blowup +# 2. Argsort(Argsort()) returns ranking, but with duplicate values there is an ordering which is problematic +# 3. Argsort(Argsort(Unique))->Scatter seems a bit complicated, doesn't have quadratic blowup, but how fast? +def to_ranking_low_mem(data): + x = torch.zeros_like(data) + for col in range(data.shape[-1]): + x_ = (data[:, :, col] >= data[:, :, col].unsqueeze(-2)) + x_ = x_.sum(0) + x[:, :, col] = x_ + return x + +def nan_handling_missing_for_unknown_reason_value(set_value_to_nan=0.0): + return get_nan_value(float('nan'), set_value_to_nan) + +def nan_handling_missing_for_no_reason_value(set_value_to_nan=0.0): + return get_nan_value(float('-inf'), set_value_to_nan) + +def nan_handling_missing_for_a_reason_value(set_value_to_nan=0.0): + return get_nan_value(float('inf'), set_value_to_nan) + +def torch_nanmean(x, axis=0): + num = torch.where(torch.isnan(x), torch.full_like(x, 0), torch.full_like(x, 1)).sum(axis=axis) + value = torch.where(torch.isnan(x), torch.full_like(x, 0), x).sum(axis=axis) + return value / num + +def torch_nanstd(x, axis=0): + num = torch.where(torch.isnan(x), torch.full_like(x, 0), torch.full_like(x, 1)).sum(axis=axis) + value = torch.where(torch.isnan(x), torch.full_like(x, 0), x).sum(axis=axis) + mean = value / num + mean_broadcast = torch.repeat_interleave(mean.unsqueeze(axis), x.shape[axis], dim=axis) + return torch.sqrt(torch.nansum(torch.square(mean_broadcast - x), axis=axis) / (num - 1)) + +def normalize_data(data, normalize_positions=-1): + if normalize_positions > 0: + mean = torch_nanmean(data[:normalize_positions], axis=0) + std = torch_nanstd(data[:normalize_positions], axis=0) + .000001 + else: + mean = torch_nanmean(data, axis=0) + std = torch_nanstd(data, axis=0) + .000001 + data = (data - mean) / std + data = torch.clip(data, min=-100, max=100) + + return data + +def remove_outliers(X, n_sigma=4): + # Expects T, B, H + assert len(X.shape) == 3, "X must be T,B,H" + #for b in range(X.shape[1]): + #for col in range(X.shape[2]): + data = X + data_mean, data_std = torch_nanmean(data, axis=0), torch_nanstd(data, axis=0) + cut_off = data_std * n_sigma + lower, upper = data_mean - cut_off, data_mean + cut_off + + data_clean = X[:].clone() + data_clean[torch.logical_or(data > upper, data < lower)] = np.nan + data_mean, data_std = torch_nanmean(data_clean, axis=0), torch_nanstd(data_clean, axis=0) + cut_off = data_std * n_sigma + lower, upper = data_mean - cut_off, data_mean + cut_off + + X = torch.maximum(-torch.log(1+torch.abs(X)) + lower, X) + X = torch.minimum(torch.log(1+torch.abs(X)) + upper, X) + # print(ds[1][data < lower, col], ds[1][data > upper, col], ds[1][~np.isnan(data), col].shape, data_mean, data_std) + return X + +def bool_mask_to_att_mask(mask): + return mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0)) + +def print_on_master_only(is_master): + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + +def init_dist(device): + if 'SLURM_PROCID' in os.environ and torch.cuda.device_count() > 1: + assert device != 'cpu:0' + rank = int(os.environ['SLURM_PROCID']) + os.environ['MASTER_ADDR'] = 'localhost' + os.environ['MASTER_PORT'] = '12355' + torch.cuda.set_device(rank) + os.environ['CUDA_VISIBLE_DEVICES'] = str(rank) + torch.distributed.init_process_group(backend="nccl", init_method="env://", timeout=datetime.timedelta(seconds=20), + world_size=torch.cuda.device_count(), rank=rank) + torch.distributed.barrier() + print_on_master_only(rank == 0) + print(f"Distributed training on {torch.cuda.device_count()} GPUs, this is rank {rank}, " + "only I can print, but when using print(..., force=True) it will print on all ranks.") + + return True, rank, f'cuda:{rank}' + else: + print('Not using distributed') + # will not change any of the behavior of print, but allows putting the force=True in the print calls + print_on_master_only(True) + return False, 0, device + +# NOP function for python with statements (x = NOP(); with x:) +class NOP(): + def __enter__(self): + pass + def __exit__(self, type, value, traceback): + pass \ No newline at end of file diff --git a/Untitled.ipynb b/Untitled.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..14b5c62deef912c7c1edbcb86773ba53b235b86b --- /dev/null +++ b/Untitled.ipynb @@ -0,0 +1,260 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 4, + "id": "1e0cd6a7", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.insert(0,'..')" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ba81c2ba", + "metadata": {}, + "outputs": [], + "source": [ + "from scripts.transformer_prediction_interface import TabPFNClassifier" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "0fe8a920", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/Users/samuelmueller/TabPFN/TabPFN\r\n" + ] + } + ], + "source": [ + "!pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "fd08a53d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching examples at: '/Users/samuelmueller/TabPFN/TabPFN/gradio_cached_examples/670/log.csv'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/samuelmueller/opt/anaconda3/envs/TabPFN/lib/python3.7/site-packages/gradio/networking.py:59: ResourceWarning: unclosed \n", + " s = socket.socket() # create a socket object\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/samuelmueller/opt/anaconda3/envs/TabPFN/lib/python3.7/site-packages/gradio/networking.py:59: ResourceWarning: unclosed \n", + " s = socket.socket() # create a socket object\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running on local URL: http://127.0.0.1:7898/\n", + "\n", + "To create a public link, set `share=True` in `launch()`.\n" + ] + }, + { + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(, 'http://127.0.0.1:7898/', None)" + ] + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import torch\n", + "import gradio as gr\n", + "import openml\n", + "\n", + "\n", + "def compute(table: np.array):\n", + " vfunc = np.vectorize(lambda s: len(s))\n", + " non_empty_row_mask = (vfunc(table).sum(1) != 0)\n", + " print(table)\n", + " table = table[non_empty_row_mask]\n", + " empty_mask = table == ''\n", + " empty_inds = np.where(empty_mask)\n", + " assert np.all(empty_inds[1][0] == empty_inds[1])\n", + " y_column = empty_inds[1][0]\n", + " eval_lines = empty_inds[0]\n", + "\n", + " train_table = np.delete(table, eval_lines, axis=0)\n", + " eval_table = table[eval_lines]\n", + "\n", + " try:\n", + " x_train = torch.tensor(np.delete(train_table, y_column, axis=1).astype(np.float32))\n", + " x_eval = torch.tensor(np.delete(eval_table, y_column, axis=1).astype(np.float32))\n", + "\n", + " y_train = train_table[:, y_column]\n", + " except ValueError:\n", + " return \"Please only add numbers (to the inputs) or leave fields empty.\", None\n", + "\n", + " classifier = TabPFNClassifier(base_path='..', device='cpu')\n", + " classifier.fit(x_train, y_train)\n", + " y_eval, p_eval = classifier.predict(x_eval, return_winning_probability=True)\n", + " print(x_train, y_train, x_eval, y_eval)\n", + "\n", + " # print(file, type(file))\n", + " out_table = table.copy().astype(str)\n", + " out_table[eval_lines, y_column] = [f\"{y_e} (p={p_e:.2f})\" for y_e, p_e in zip(y_eval, p_eval)]\n", + " return None, out_table\n", + "\n", + "\n", + "def upload_file(file):\n", + " if file.name.endswith('.arff'):\n", + " dataset = openml.datasets.OpenMLDataset('t', 'test', data_file=file.name)\n", + " X_, _, categorical_indicator_, attribute_names_ = dataset.get_data(\n", + " dataset_format=\"array\"\n", + " )\n", + " return X_\n", + " elif file.name.endswith('.csv') or file.name.endswith('.data'):\n", + " df = pd.read_csv(file.name)\n", + " return df.to_numpy()\n", + "\n", + "\n", + "example = \\\n", + " [\n", + " [1, 2, 1],\n", + " [2, 1, 1],\n", + " [1, 1, 1],\n", + " [2, 2, 2],\n", + " [3, 4, 2],\n", + " [3, 2, 2],\n", + " [2, 3, '']\n", + " ]\n", + "\n", + "with gr.Blocks() as demo:\n", + " gr.Markdown(\"\"\"This demo allows you to play with the **TabPFN**.\n", + " You can either change the table manually (we have filled it with a toy benchmark, sum up to 3 has label 1 and over that label 2).\n", + " The network predicts fields you leave empty. Only one column can have empty entries that are predicted.\n", + " Please, provide everything but the label column as numeric values. It is ok to encode classes as integers.\n", + " \"\"\")\n", + " inp_table = gr.DataFrame(type='numpy', value=example, headers=[''] * 3)\n", + " inp_file = gr.File(\n", + " label='Drop either a .csv (without header, only numeric values for all but the labels) or a .arff file.')\n", + " btn = gr.Button(\"Predict Empty Table Cells\")\n", + "\n", + " inp_file.change(fn=upload_file, inputs=inp_file, outputs=inp_table)\n", + "\n", + " out_text = gr.Textbox()\n", + " out_table = gr.DataFrame()\n", + "\n", + " btn.click(fn=compute, inputs=inp_table, outputs=[out_text, out_table])\n", + " examples = gr.Examples(examples=['./iris.csv'],\n", + " inputs=[inp_file],\n", + " outputs=[inp_table],\n", + " fn=upload_file,\n", + " cache_examples=True)\n", + "\n", + "demo.launch()" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "c4510232", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame({'hi':[1,2,'j']})" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "2403f193", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[[1], [2], ['j']]" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "sys:1: ResourceWarning: unclosed socket \n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + } + ], + "source": [ + "df.to_numpy().tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "adf1a91c", + "metadata": {}, + "outputs": [], + "source": [ + "k" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e2b3282a6cb4048d6991aa86798e9b13687f27c5 --- /dev/null +++ b/app.py @@ -0,0 +1,96 @@ +import sys +tabpfn_path = 'TabPFN' +sys.path.insert(0, tabpfn_path) # our submodule of the TabPFN repo (at 045c8400203ebd062346970b4f2c0ccda5a40618) +from TabPFN.scripts.transformer_prediction_interface import TabPFNClassifier + +import numpy as np +import pandas as pd +import torch +import gradio as gr +import openml + + +def compute(table: np.array): + vfunc = np.vectorize(lambda s: len(s)) + non_empty_row_mask = (vfunc(table).sum(1) != 0) + table = table[non_empty_row_mask] + empty_mask = table == '' + empty_inds = np.where(empty_mask) + if not len(empty_inds[0]): + return "**Please leave at least one field blank for prediction.**", None + if not np.all(empty_inds[1][0] == empty_inds[1]): + return "**Please only leave fields of one column blank for prediction.**", None + y_column = empty_inds[1][0] + eval_lines = empty_inds[0] + + train_table = np.delete(table, eval_lines, axis=0) + eval_table = table[eval_lines] + + try: + x_train = torch.tensor(np.delete(train_table, y_column, axis=1).astype(np.float32)) + x_eval = torch.tensor(np.delete(eval_table, y_column, axis=1).astype(np.float32)) + + y_train = train_table[:, y_column] + except ValueError: + return "**Please only add numbers (to the inputs) or leave fields empty.**", None + + classifier = TabPFNClassifier(base_path=tabpfn_path, device='cpu') + classifier.fit(x_train, y_train) + y_eval, p_eval = classifier.predict(x_eval, return_winning_probability=True) + + # print(file, type(file)) + out_table = table.copy().astype(str) + out_table[eval_lines, y_column] = [f"{y_e} (p={p_e:.2f})" for y_e, p_e in zip(y_eval, p_eval)] + return None, out_table + + +def upload_file(file): + if file.name.endswith('.arff'): + dataset = openml.datasets.OpenMLDataset('t', 'test', data_file=file.name) + X_, _, categorical_indicator_, attribute_names_ = dataset.get_data( + dataset_format="array" + ) + df = pd.DataFrame(X_, columns=attribute_names_) + return df + elif file.name.endswith('.csv') or file.name.endswith('.data'): + df = pd.read_csv(file.name, header=None) + df.columns = np.arange(len(df.columns)) + print(df) + return df + + +example = \ + [ + [1, 2, 1], + [2, 1, 1], + [1, 1, 1], + [2, 2, 2], + [3, 4, 2], + [3, 2, 2], + [2, 3, ''] + ] + +with gr.Blocks() as demo: + gr.Markdown("""This demo allows you to play with the **TabPFN**. + You can either change the table manually (we have filled it with a toy benchmark, sum up to 3 has label 1 and over that label 2). + The network predicts fields you leave empty. Only one column can have empty entries that are predicted. + Please, provide everything but the label column as numeric values. It is ok to encode classes as integers. + """) + inp_table = gr.DataFrame(type='numpy', value=example, headers=[''] * 3) + inp_file = gr.File( + label='Drop either a .csv (without header, only numeric values for all but the labels) or a .arff file.') + examples = gr.Examples(examples=['iris.csv', 'balance-scale.arff'], + inputs=[inp_file], + outputs=[inp_table], + fn=upload_file, + cache_examples=True) + btn = gr.Button("Predict Empty Table Cells") + + inp_file.change(fn=upload_file, inputs=inp_file, outputs=inp_table) + + out_text = gr.Markdown() + out_table = gr.DataFrame() + + btn.click(fn=compute, inputs=inp_table, outputs=[out_text, out_table]) + +demo.launch() \ No newline at end of file diff --git a/balance-scale.arff b/balance-scale.arff new file mode 100644 index 0000000000000000000000000000000000000000..b0c5d965d115e034a698e477b447029b7eb8645f --- /dev/null +++ b/balance-scale.arff @@ -0,0 +1,694 @@ +%1. Title: Balance Scale Weight & Distance Database +% +%2. Source Information: +% (a) Source: Generated to model psychological experiments reported +% by Siegler, R. S. (1976). Three Aspects of Cognitive +% Development. Cognitive Psychology, 8, 481-520. +% (b) Donor: Tim Hume (hume@ics.uci.edu) +% (c) Date: 22 April 1994 +% +%3. Past Usage: (possibly different formats of this data) +% - Publications +% 1. Klahr, D., & Siegler, R.S. (1978). The Representation of +% Children's Knowledge. In H. W. Reese & L. P. Lipsitt (Eds.), +% Advances in Child Development and Behavior, pp. 61-116. New +% York: Academic Press +% 2. Langley,P. (1987). A General Theory of Discrimination +% Learning. In D. Klahr, P. Langley, & R. Neches (Eds.), +% Production System Models of Learning and Development, pp. +% 99-161. Cambridge, MA: MIT Press +% 3. Newell, A. (1990). Unified Theories of Cognition. +% Cambridge, MA: Harvard University Press +% 4. McClelland, J.L. (1988). Parallel Distibuted Processing: +% Implications for Cognition and Development. Technical +% Report AIP-47, Department of Psychology, Carnegie-Mellon +% University +% 5. Shultz, T., Mareschal, D., & Schmidt, W. (1994). Modeling +% Cognitive Development on Balance Scale Phenomena. Machine +% Learning, Vol. 16, pp. 59-88. +% +%4. Relevant Information: +% This data set was generated to model psychological +% experimental results. Each example is classified as having the +% balance scale tip to the right, tip to the left, or be +% balanced. The attributes are the left weight, the left +% distance, the right weight, and the right distance. The +% correct way to find the class is the greater of +% (left-distance * left-weight) and (right-distance * +% right-weight). If they are equal, it is balanced. +% +%5. Number of Instances: 625 (49 balanced, 288 left, 288 right) +% +%6. Number of Attributes: 4 (numeric) + class name = 5 +% +%7. Attribute Information: +% 1. Class Name: 3 (L, B, R) +% 2. Left-Weight: 5 (1, 2, 3, 4, 5) +% 3. Left-Distance: 5 (1, 2, 3, 4, 5) +% 4. Right-Weight: 5 (1, 2, 3, 4, 5) +% 5. Right-Distance: 5 (1, 2, 3, 4, 5) +% +%8. Missing Attribute Values: +% none +% +%9. Class Distribution: +% 1. 46.08 percent are L +% 2. 07.84 percent are B +% 3. 46.08 percent are R +% + +@relation balance-scale +@attribute 'left-weight' real +@attribute 'left-distance' real +@attribute 'right-weight' real +@attribute 'right-distance' real +@attribute 'class' { L, B, R} +@data +1,1,1,1,B +1,1,1,2,R +1,1,1,3,R +1,1,1,4,R +1,1,1,5,R +1,1,2,1,R +1,1,2,2,R +1,1,2,3,R +1,1,2,4,R +1,1,2,5,R +1,1,3,1,R +1,1,3,2,R +1,1,3,3,R +1,1,3,4,R +1,1,3,5,R +1,1,4,1,R +1,1,4,2,R +1,1,4,3,R +1,1,4,4,R +1,1,4,5,R +1,1,5,1,R +1,1,5,2,R +1,1,5,3,R +1,1,5,4,R +1,1,5,5,R +1,2,1,1,L +1,2,1,2,B +1,2,1,3,R +1,2,1,4,R +1,2,1,5,R +1,2,2,1,B +1,2,2,2,R +1,2,2,3,R +1,2,2,4,R +1,2,2,5,R +1,2,3,1,R +1,2,3,2,R +1,2,3,3,R +1,2,3,4,R +1,2,3,5,R +1,2,4,1,R +1,2,4,2,R +1,2,4,3,R +1,2,4,4,R +1,2,4,5,R +1,2,5,1,R +1,2,5,2,R +1,2,5,3,R +1,2,5,4,R +1,2,5,5,R +1,3,1,1,L +1,3,1,2,L +1,3,1,3,B +1,3,1,4,R +1,3,1,5,R +1,3,2,1,L +1,3,2,2,R +1,3,2,3,R +1,3,2,4,R +1,3,2,5,R +1,3,3,1,B +1,3,3,2,R +1,3,3,3,R +1,3,3,4,R +1,3,3,5,R +1,3,4,1,R +1,3,4,2,R +1,3,4,3,R +1,3,4,4,R +1,3,4,5,R +1,3,5,1,R +1,3,5,2,R +1,3,5,3,R +1,3,5,4,R +1,3,5,5,R +1,4,1,1,L +1,4,1,2,L +1,4,1,3,L +1,4,1,4,B +1,4,1,5,R +1,4,2,1,L +1,4,2,2,B +1,4,2,3,R +1,4,2,4,R +1,4,2,5,R +1,4,3,1,L +1,4,3,2,R +1,4,3,3,R +1,4,3,4,R +1,4,3,5,R +1,4,4,1,B +1,4,4,2,R +1,4,4,3,R +1,4,4,4,R +1,4,4,5,R +1,4,5,1,R +1,4,5,2,R +1,4,5,3,R +1,4,5,4,R +1,4,5,5,R +1,5,1,1,L +1,5,1,2,L +1,5,1,3,L +1,5,1,4,L +1,5,1,5,B +1,5,2,1,L +1,5,2,2,L +1,5,2,3,R +1,5,2,4,R +1,5,2,5,R +1,5,3,1,L +1,5,3,2,R +1,5,3,3,R +1,5,3,4,R +1,5,3,5,R +1,5,4,1,L +1,5,4,2,R +1,5,4,3,R +1,5,4,4,R +1,5,4,5,R +1,5,5,1,B +1,5,5,2,R +1,5,5,3,R +1,5,5,4,R +1,5,5,5,R +2,1,1,1,L +2,1,1,2,B +2,1,1,3,R +2,1,1,4,R +2,1,1,5,R +2,1,2,1,B +2,1,2,2,R +2,1,2,3,R +2,1,2,4,R +2,1,2,5,R +2,1,3,1,R +2,1,3,2,R +2,1,3,3,R +2,1,3,4,R +2,1,3,5,R +2,1,4,1,R +2,1,4,2,R +2,1,4,3,R +2,1,4,4,R +2,1,4,5,R +2,1,5,1,R +2,1,5,2,R +2,1,5,3,R +2,1,5,4,R +2,1,5,5,R +2,2,1,1,L +2,2,1,2,L +2,2,1,3,L +2,2,1,4,B +2,2,1,5,R +2,2,2,1,L +2,2,2,2,B +2,2,2,3,R +2,2,2,4,R +2,2,2,5,R +2,2,3,1,L +2,2,3,2,R +2,2,3,3,R +2,2,3,4,R +2,2,3,5,R +2,2,4,1,B +2,2,4,2,R +2,2,4,3,R +2,2,4,4,R +2,2,4,5,R +2,2,5,1,R +2,2,5,2,R +2,2,5,3,R +2,2,5,4,R +2,2,5,5,R +2,3,1,1,L +2,3,1,2,L +2,3,1,3,L +2,3,1,4,L +2,3,1,5,L +2,3,2,1,L +2,3,2,2,L +2,3,2,3,B +2,3,2,4,R +2,3,2,5,R +2,3,3,1,L +2,3,3,2,B +2,3,3,3,R +2,3,3,4,R +2,3,3,5,R +2,3,4,1,L +2,3,4,2,R +2,3,4,3,R +2,3,4,4,R +2,3,4,5,R +2,3,5,1,L +2,3,5,2,R +2,3,5,3,R +2,3,5,4,R +2,3,5,5,R +2,4,1,1,L +2,4,1,2,L +2,4,1,3,L +2,4,1,4,L +2,4,1,5,L +2,4,2,1,L +2,4,2,2,L +2,4,2,3,L +2,4,2,4,B +2,4,2,5,R +2,4,3,1,L +2,4,3,2,L +2,4,3,3,R +2,4,3,4,R +2,4,3,5,R +2,4,4,1,L +2,4,4,2,B +2,4,4,3,R +2,4,4,4,R +2,4,4,5,R +2,4,5,1,L +2,4,5,2,R +2,4,5,3,R +2,4,5,4,R +2,4,5,5,R +2,5,1,1,L +2,5,1,2,L +2,5,1,3,L +2,5,1,4,L +2,5,1,5,L +2,5,2,1,L +2,5,2,2,L +2,5,2,3,L +2,5,2,4,L +2,5,2,5,B +2,5,3,1,L +2,5,3,2,L +2,5,3,3,L +2,5,3,4,R +2,5,3,5,R +2,5,4,1,L +2,5,4,2,L +2,5,4,3,R +2,5,4,4,R +2,5,4,5,R +2,5,5,1,L +2,5,5,2,B +2,5,5,3,R +2,5,5,4,R +2,5,5,5,R +3,1,1,1,L +3,1,1,2,L +3,1,1,3,B +3,1,1,4,R +3,1,1,5,R +3,1,2,1,L +3,1,2,2,R +3,1,2,3,R +3,1,2,4,R +3,1,2,5,R +3,1,3,1,B +3,1,3,2,R +3,1,3,3,R +3,1,3,4,R +3,1,3,5,R +3,1,4,1,R +3,1,4,2,R +3,1,4,3,R +3,1,4,4,R +3,1,4,5,R +3,1,5,1,R +3,1,5,2,R +3,1,5,3,R +3,1,5,4,R +3,1,5,5,R +3,2,1,1,L +3,2,1,2,L +3,2,1,3,L +3,2,1,4,L +3,2,1,5,L +3,2,2,1,L +3,2,2,2,L +3,2,2,3,B +3,2,2,4,R +3,2,2,5,R +3,2,3,1,L +3,2,3,2,B +3,2,3,3,R +3,2,3,4,R +3,2,3,5,R +3,2,4,1,L +3,2,4,2,R +3,2,4,3,R +3,2,4,4,R +3,2,4,5,R +3,2,5,1,L +3,2,5,2,R +3,2,5,3,R +3,2,5,4,R +3,2,5,5,R +3,3,1,1,L +3,3,1,2,L +3,3,1,3,L +3,3,1,4,L +3,3,1,5,L +3,3,2,1,L +3,3,2,2,L +3,3,2,3,L +3,3,2,4,L +3,3,2,5,R +3,3,3,1,L +3,3,3,2,L +3,3,3,3,B +3,3,3,4,R +3,3,3,5,R +3,3,4,1,L +3,3,4,2,L +3,3,4,3,R +3,3,4,4,R +3,3,4,5,R +3,3,5,1,L +3,3,5,2,R +3,3,5,3,R +3,3,5,4,R +3,3,5,5,R +3,4,1,1,L +3,4,1,2,L +3,4,1,3,L +3,4,1,4,L +3,4,1,5,L +3,4,2,1,L +3,4,2,2,L +3,4,2,3,L +3,4,2,4,L +3,4,2,5,L +3,4,3,1,L +3,4,3,2,L +3,4,3,3,L +3,4,3,4,B +3,4,3,5,R +3,4,4,1,L +3,4,4,2,L +3,4,4,3,B +3,4,4,4,R +3,4,4,5,R +3,4,5,1,L +3,4,5,2,L +3,4,5,3,R +3,4,5,4,R +3,4,5,5,R +3,5,1,1,L +3,5,1,2,L +3,5,1,3,L +3,5,1,4,L +3,5,1,5,L +3,5,2,1,L +3,5,2,2,L +3,5,2,3,L +3,5,2,4,L +3,5,2,5,L +3,5,3,1,L +3,5,3,2,L +3,5,3,3,L +3,5,3,4,L +3,5,3,5,B +3,5,4,1,L +3,5,4,2,L +3,5,4,3,L +3,5,4,4,R +3,5,4,5,R +3,5,5,1,L +3,5,5,2,L +3,5,5,3,B +3,5,5,4,R +3,5,5,5,R +4,1,1,1,L +4,1,1,2,L +4,1,1,3,L +4,1,1,4,B +4,1,1,5,R +4,1,2,1,L +4,1,2,2,B +4,1,2,3,R +4,1,2,4,R +4,1,2,5,R +4,1,3,1,L +4,1,3,2,R +4,1,3,3,R +4,1,3,4,R +4,1,3,5,R +4,1,4,1,B +4,1,4,2,R +4,1,4,3,R +4,1,4,4,R +4,1,4,5,R +4,1,5,1,R +4,1,5,2,R +4,1,5,3,R +4,1,5,4,R +4,1,5,5,R +4,2,1,1,L +4,2,1,2,L +4,2,1,3,L +4,2,1,4,L +4,2,1,5,L +4,2,2,1,L +4,2,2,2,L +4,2,2,3,L +4,2,2,4,B +4,2,2,5,R +4,2,3,1,L +4,2,3,2,L +4,2,3,3,R +4,2,3,4,R +4,2,3,5,R +4,2,4,1,L +4,2,4,2,B +4,2,4,3,R +4,2,4,4,R +4,2,4,5,R +4,2,5,1,L +4,2,5,2,R +4,2,5,3,R +4,2,5,4,R +4,2,5,5,R +4,3,1,1,L +4,3,1,2,L +4,3,1,3,L +4,3,1,4,L +4,3,1,5,L +4,3,2,1,L +4,3,2,2,L +4,3,2,3,L +4,3,2,4,L +4,3,2,5,L +4,3,3,1,L +4,3,3,2,L +4,3,3,3,L +4,3,3,4,B +4,3,3,5,R +4,3,4,1,L +4,3,4,2,L +4,3,4,3,B +4,3,4,4,R +4,3,4,5,R +4,3,5,1,L +4,3,5,2,L +4,3,5,3,R +4,3,5,4,R +4,3,5,5,R +4,4,1,1,L +4,4,1,2,L +4,4,1,3,L +4,4,1,4,L +4,4,1,5,L +4,4,2,1,L +4,4,2,2,L +4,4,2,3,L +4,4,2,4,L +4,4,2,5,L +4,4,3,1,L +4,4,3,2,L +4,4,3,3,L +4,4,3,4,L +4,4,3,5,L +4,4,4,1,L +4,4,4,2,L +4,4,4,3,L +4,4,4,4,B +4,4,4,5,R +4,4,5,1,L +4,4,5,2,L +4,4,5,3,L +4,4,5,4,R +4,4,5,5,R +4,5,1,1,L +4,5,1,2,L +4,5,1,3,L +4,5,1,4,L +4,5,1,5,L +4,5,2,1,L +4,5,2,2,L +4,5,2,3,L +4,5,2,4,L +4,5,2,5,L +4,5,3,1,L +4,5,3,2,L +4,5,3,3,L +4,5,3,4,L +4,5,3,5,L +4,5,4,1,L +4,5,4,2,L +4,5,4,3,L +4,5,4,4,L +4,5,4,5,B +4,5,5,1,L +4,5,5,2,L +4,5,5,3,L +4,5,5,4,B +4,5,5,5,R +5,1,1,1,L +5,1,1,2,L +5,1,1,3,L +5,1,1,4,L +5,1,1,5,B +5,1,2,1,L +5,1,2,2,L +5,1,2,3,R +5,1,2,4,R +5,1,2,5,R +5,1,3,1,L +5,1,3,2,R +5,1,3,3,R +5,1,3,4,R +5,1,3,5,R +5,1,4,1,L +5,1,4,2,R +5,1,4,3,R +5,1,4,4,R +5,1,4,5,R +5,1,5,1,B +5,1,5,2,R +5,1,5,3,R +5,1,5,4,R +5,1,5,5,R +5,2,1,1,L +5,2,1,2,L +5,2,1,3,L +5,2,1,4,L +5,2,1,5,L +5,2,2,1,L +5,2,2,2,L +5,2,2,3,L +5,2,2,4,L +5,2,2,5,B +5,2,3,1,L +5,2,3,2,L +5,2,3,3,L +5,2,3,4,R +5,2,3,5,R +5,2,4,1,L +5,2,4,2,L +5,2,4,3,R +5,2,4,4,R +5,2,4,5,R +5,2,5,1,L +5,2,5,2,B +5,2,5,3,R +5,2,5,4,R +5,2,5,5,R +5,3,1,1,L +5,3,1,2,L +5,3,1,3,L +5,3,1,4,L +5,3,1,5,L +5,3,2,1,L +5,3,2,2,L +5,3,2,3,L +5,3,2,4,L +5,3,2,5,L +5,3,3,1,L +5,3,3,2,L +5,3,3,3,L +5,3,3,4,L +5,3,3,5,B +5,3,4,1,L +5,3,4,2,L +5,3,4,3,L +5,3,4,4,R +5,3,4,5,R +5,3,5,1,L +5,3,5,2,L +5,3,5,3,B +5,3,5,4,R +5,3,5,5,R +5,4,1,1,L +5,4,1,2,L +5,4,1,3,L +5,4,1,4,L +5,4,1,5,L +5,4,2,1,L +5,4,2,2,L +5,4,2,3,L +5,4,2,4,L +5,4,2,5,L +5,4,3,1,L +5,4,3,2,L +5,4,3,3,L +5,4,3,4,L +5,4,3,5,L +5,4,4,1,L +5,4,4,2,L +5,4,4,3,L +5,4,4,4,L +5,4,4,5,B +5,4,5,1,L +5,4,5,2,L +5,4,5,3,L +5,4,5,4,B +5,4,5,5,R +5,5,1,1,L +5,5,1,2,L +5,5,1,3,L +5,5,1,4,L +5,5,1,5,L +5,5,2,1,L +5,5,2,2,L +5,5,2,3,L +5,5,2,4,L +5,5,2,5,L +5,5,3,1,L +5,5,3,2,L +5,5,3,3,L +5,5,3,4,L +5,5,3,5,L +5,5,4,1,L +5,5,4,2,L +5,5,4,3,L +5,5,4,4,L +5,5,4,5,L +5,5,5,1,L +5,5,5,2,L +5,5,5,3,L +5,5,5,4,L +5,5,5,5,B +% +% +% diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..d8cd5a944c3f6a9912725bbe72647ba2cacecb64 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,16 @@ +# Please use python V 3.7 to be compatible with all packages +gpytorch==1.5.0 +torch==1.9.0 +scikit-learn==0.24.2 +pyyaml==5.4.1 +seaborn==0.11.2 +xgboost==1.4.0 +tqdm==4.62.1 +numpy==1.21.2 +openml==0.12.2 +catboost==0.26.1 +auto-sklearn==0.14.5 +hyperopt==0.2.5 +configspace==0.4.21 +# autogluon==0.4.0 +gradio==3.1.1 \ No newline at end of file