diff --git "a/TabPFN/TrainingTuningAndPrediction.ipynb" "b/TabPFN/TrainingTuningAndPrediction.ipynb" new file mode 100644--- /dev/null +++ "b/TabPFN/TrainingTuningAndPrediction.ipynb" @@ -0,0 +1,1925 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import time\n", + "import warnings\n", + "from datetime import datetime\n", + "\n", + "import torch\n", + "\n", + "import numpy as np\n", + "\n", + "import matplotlib.pyplot as plt\n", + "from scripts.differentiable_pfn_evaluation import eval_model_range\n", + "from model_builder import get_model, get_default_spec, save_model, load_model\n", + "from scripts.transformer_prediction_interface import transformer_predict, get_params_from_config, load_model_workflow\n", + "\n", + "from scripts.model_configs import *\n", + "\n", + "from datasets import load_openml_list, open_cc_dids, open_cc_valid_dids\n", + "from priors.utils import plot_prior, plot_features\n", + "from priors.utils import uniform_int_sampler_f\n", + "\n", + "from scripts.tabular_metrics import calculate_score_per_method, calculate_score\n", + "from scripts.tabular_evaluation import evaluate\n", + "\n", + "from priors.differentiable_prior import DifferentiableHyperparameterList, draw_random_style, merge_style_with_info\n", + "from scripts import tabular_metrics\n", + "from notebook_utils import *" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "large_datasets = True\n", + "max_samples = 10000 if large_datasets else 5000\n", + "bptt = 10000 if large_datasets else 3000\n", + "suite='cc'" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "device = 'cpu'\n", + "base_path = '.'\n", + "max_features = 100" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def print_models(model_string):\n", + " print(model_string)\n", + "\n", + " for i in range(80):\n", + " for e in range(50):\n", + " exists = Path(os.path.join(base_path, f'models_diff/prior_diff_real_checkpoint{model_string}_n_{i}_epoch_{e}.cpkt')).is_file()\n", + " if exists:\n", + " print(os.path.join(base_path, f'models_diff/prior_diff_real_checkpoint{model_string}_n_{i}_epoch_{e}.cpkt'))\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "def train_function(config_sample, i, add_name=''):\n", + " start_time = time.time()\n", + " N_epochs_to_save = 50\n", + " \n", + " def save_callback(model, epoch):\n", + " if not hasattr(model, 'last_saved_epoch'):\n", + " model.last_saved_epoch = 0\n", + " if ((time.time() - start_time) / (maximum_runtime * 60 / N_epochs_to_save)) > model.last_saved_epoch:\n", + " print('Saving model..')\n", + " config_sample['epoch_in_training'] = epoch\n", + " save_model(model, base_path, f'models_diff/prior_diff_real_checkpoint{add_name}_n_{i}_epoch_{model.last_saved_epoch}.cpkt',\n", + " config_sample)\n", + " model.last_saved_epoch = model.last_saved_epoch + 1 # TODO: Rename to checkpoint\n", + " \n", + " model = get_model(config_sample\n", + " , device\n", + " , should_train=True\n", + " , verbose=1\n", + " , epoch_callback = save_callback)\n", + " \n", + " return" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Datasets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "cc_test_datasets_multiclass, cc_test_datasets_multiclass_df = load_openml_list(open_cc_dids, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = max_samples, num_feats=100, return_capped=True)\n", + "cc_valid_datasets_multiclass, cc_valid_datasets_multiclass_df = load_openml_list(open_cc_valid_dids, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = max_samples, num_feats=100, return_capped=True)\n", + "\n", + "# Loading longer OpenML Datasets for generalization experiments (optional)\n", + "# test_datasets_multiclass, test_datasets_multiclass_df = load_openml_list(test_dids_classification, multiclass=True, shuffled=True, filter_for_nan=False, max_samples = 10000, num_feats=100, return_capped=True)\n", + "\n", + "random.seed(0)\n", + "random.shuffle(cc_valid_datasets_multiclass)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def get_datasets(selector, task_type, suite='cc'):\n", + " if task_type == 'binary':\n", + " ds = valid_datasets_binary if selector == 'valid' else test_datasets_binary\n", + " else:\n", + " if suite == 'openml':\n", + " ds = valid_datasets_multiclass if selector == 'valid' else test_datasets_multiclass\n", + " elif suite == 'cc':\n", + " ds = cc_valid_datasets_multiclass if selector == 'valid' else cc_test_datasets_multiclass\n", + " else:\n", + " raise Exception(\"Unknown suite\")\n", + " return ds" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Fitting a PFN for our prior" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Define prior settings" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "def get_prior_config(config_type):\n", + " if config_type == 'causal':\n", + " return get_prior_config_causal()\n", + " elif config_type == 'gp':\n", + " return get_prior_config_gp()\n", + " elif config_type == 'bnn':\n", + " return get_prior_config_bnn()\n", + " elif config_type == 'bag_gp_bnn':\n", + " return get_prior_config_bag_gp_bnn()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def get_prior_config_gp():\n", + " config_general = get_general_config(max_features, 50, eval_positions=[30])\n", + " config_general_real_world = {**config_general}\n", + "\n", + " config_flexible_categorical = get_flexible_categorical_config(max_features)\n", + " config_flexible_categorical_real_world = {**config_flexible_categorical}\n", + "\n", + " config_gp = {}\n", + "\n", + " config_diff = get_diff_config()\n", + "\n", + " config = {**config_general_real_world, **config_flexible_categorical_real_world, **config_diff, **config_gp}\n", + " \n", + " config_sample['differentiable_hyperparameters']['prior_bag_exp_weights_1'] = {'distribution': 'uniform', 'min': 0.0, 'max': .01} # Never select MLP\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def get_prior_config_bnn():\n", + " config_general = get_general_config(max_features, 50, eval_positions=[30])\n", + " config_general_real_world = {**config_general}\n", + "\n", + " config_flexible_categorical = get_flexible_categorical_config(max_features)\n", + " config_flexible_categorical_real_world = {**config_flexible_categorical}\n", + "\n", + " config_gp = {}\n", + " config_mlp = {}\n", + "\n", + " config_diff = get_diff_config()\n", + "\n", + " config = {**config_general_real_world, **config_flexible_categorical_real_world, **config_diff, **config_gp, **config_mlp}\n", + " \n", + " config_sample['differentiable_hyperparameters']['prior_bag_exp_weights_1'] = {'distribution': 'uniform', 'min': 1000.0, 'max': 1001.0} # Always select MLP\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def get_prior_config_causal():\n", + " config_general = get_general_config(max_features, 50, eval_positions=[30])\n", + " config_general_real_world = {**config_general}\n", + "\n", + " config_flexible_categorical = get_flexible_categorical_config(max_features)\n", + " config_flexible_categorical_real_world = {**config_flexible_categorical}\n", + " config_flexible_categorical_real_world['num_categorical_features_sampler_a'] = -1.0 # Categorical features disabled by default\n", + "\n", + " config_gp = {}\n", + " config_mlp = {}\n", + "\n", + " config_diff = get_diff_config()\n", + "\n", + " config = {**config_general_real_world, **config_flexible_categorical_real_world, **config_diff, **config_gp,\n", + " **config_mlp}\n", + " \n", + " return config" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "def reload_config(config_type='causal', task_type='multiclass', longer=0):\n", + " config = get_prior_config(config_type=config_type)\n", + " \n", + " config['prior_type'], config['differentiable'], config['flexible'] = 'prior_bag', True, True\n", + " \n", + " model_string = ''\n", + " \n", + " config['epochs'] = 12000\n", + " config['recompute_attn'] = True\n", + "\n", + " config['max_num_classes'] = 10\n", + " config['num_classes'] = uniform_int_sampler_f(2, config['max_num_classes'])\n", + " config['balanced'] = False\n", + " model_string = model_string + '_multiclass'\n", + " \n", + " model_string = model_string + '_'+datetime.now().strftime(\"%m_%d_%Y_%H_%M_%S\")\n", + " \n", + " return config, model_string" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Visualize Prior samples" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "config, model_string = reload_config(longer=1)\n", + "config_sample = evaluate_hypers(config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config_sample['batch_size'] = 4\n", + "model = get_model(config_sample, device, should_train=False, verbose=2) # , state_dict=model[2].state_dict()\n", + "(hp_embedding, data, targets_), targets = next(iter(model[3]))\n", + "\n", + "from utils import normalize_data\n", + "fig = plt.figure(figsize=(8, 8))\n", + "N = 100\n", + "plot_features(data[0:N, 0, 0:4], targets[0:N, 0], fig=fig)\n", + "\n", + "d = np.concatenate([data[:, 0, :].T, np.expand_dims(targets[:, 0], -1).T])\n", + "d[np.isnan(d)] = 0\n", + "c = np.corrcoef(d)\n", + "plt.matshow(np.abs(c), vmin=0, vmax=1)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Training" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using style prior: True\n", + "Using cpu:0 device\n", + "Not using distributed\n", + "DataLoader.__dict__ {'num_steps': 100, 'fuse_x_y': False, 'get_batch_kwargs': {'batch_size': 4, 'seq_len': 50, 'seq_len_maximum': 50, 'device': 'cpu:0', 'num_features': 100, 'hyperparameters': {'lr': 0.0001733787235341751, 'dropout': 0.0, 'emsize': 256, 'batch_size': 4, 'nlayers': 12, 'num_features': 100, 'nhead': 4, 'nhid_factor': 2, 'bptt': 50, 'eval_positions': [47], 'seq_len_used': 50, 'sampling': 'normal', 'epochs': 12000, 'num_steps': 100, 'verbose': True, 'pre_sample_causes': True, 'mix_activations': False, 'nan_prob_unknown_reason_reason_prior': 1.0, 'categorical_feature_p': 0.0, 'nan_prob_no_reason': 0.2, 'nan_prob_unknown_reason': 0.0, 'nan_prob_a_reason': 0.0, 'max_num_classes': 10, 'num_classes': .. at 0x7f3dd119d560>, 'noise_type': 'Gaussian', 'balanced': False, 'normalize_to_ranking': False, 'set_value_to_nan': 0.5, 'normalize_by_used_features': True, 'num_features_used': .. at 0x7f3dd119db90>, 'num_categorical_features_sampler_a': -1.0, 'differentiable_hyperparameters': {'distribution': 'uniform', 'min': 100000.0, 'max': 100001.0}, 'prior_type': 'prior_bag', 'differentiable': True, 'flexible': True, 'recompute_attn': True, 'aggregate_k_gradients': 1, 'multiclass_type': 'rank', 'bptt_extra_samples': None, 'prior_bag_get_batch': (.make_get_batch.. at 0x7f3de63f0ef0>, .make_get_batch.. at 0x7f3de63f0e60>), 'prior_bag_exp_weights_1': 2.0}, 'num_outputs': 1, 'dynamic_batch_size': 2, 'get_batch': .make_get_batch.. at 0x7f3de63f00e0>, 'differentiable_hyperparameters': {'prior_bag_exp_weights_1': {'distribution': 'uniform', 'min': 100000.0, 'max': 100001.0}, 'num_layers': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 6, 'min_mean': 1, 'round': True, 'lower_bound': 2}, 'prior_mlp_hidden_dim': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 130, 'min_mean': 5, 'round': True, 'lower_bound': 4}, 'prior_mlp_dropout_prob': {'distribution': 'meta_beta', 'scale': 0.9, 'min': 0.1, 'max': 5.0}, 'noise_std': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 0.3, 'min_mean': 0.0001, 'round': False, 'lower_bound': 0.0}, 'init_std': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 0.01, 'round': False, 'lower_bound': 0.0}, 'num_causes': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 12, 'min_mean': 1, 'round': True, 'lower_bound': 1}, 'is_causal': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'pre_sample_weights': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'y_is_effect': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'prior_mlp_activations': {'distribution': 'meta_choice_mixed', 'choice_values': [, , , . at 0x7f3dd119d3b0>, ]}, 'block_wise_dropout': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'sort_features': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'in_clique': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'outputscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 1e-05, 'round': False, 'lower_bound': 0}, 'lengthscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 1e-05, 'round': False, 'lower_bound': 0}, 'noise': {'distribution': 'meta_choice', 'choice_values': [1e-05, 0.0001, 0.01]}, 'output_multiclass_ordered_p': {'distribution': 'uniform', 'min': 0.0, 'max': 0.5}, 'multiclass_type': {'distribution': 'meta_choice', 'choice_values': ['value', 'rank']}}}, 'num_features': 100, 'num_outputs': 1}\n", + "PRIOR_BAG: tensor([1.0000e+00, 1.0000e+05]) [1]\n", + "{'is_causal': True, 'num_causes': 4, 'prior_mlp_hidden_dim': 25, 'num_layers': 6, 'noise_std': 0.039215336075737864, 'y_is_effect': False, 'pre_sample_weights': True, 'prior_mlp_dropout_prob': 0.6926878062137257, 'pre_sample_causes': True}\n", + "Hparams dict_keys(['prior_bag_exp_weights_1', 'num_layers_log_mean', 'num_layers_log_std', 'prior_mlp_hidden_dim_log_mean', 'prior_mlp_hidden_dim_log_std', 'prior_mlp_dropout_prob_b', 'prior_mlp_dropout_prob_k', 'noise_std_log_mean', 'noise_std_log_std', 'init_std_log_mean', 'init_std_log_std', 'num_causes_log_mean', 'num_causes_log_std', 'is_causal_choice_1_weight', 'pre_sample_weights_choice_1_weight', 'y_is_effect_choice_1_weight', 'prior_mlp_activations_choice_1_weight', 'prior_mlp_activations_choice_2_weight', 'prior_mlp_activations_choice_3_weight', 'prior_mlp_activations_choice_4_weight', 'block_wise_dropout_choice_1_weight', 'sort_features_choice_1_weight', 'in_clique_choice_1_weight', 'outputscale_log_mean', 'outputscale_log_std', 'lengthscale_log_mean', 'lengthscale_log_std', 'noise_choice_1_weight', 'noise_choice_2_weight', 'output_multiclass_ordered_p', 'multiclass_type_choice_1_weight'])\n", + "Using a Transformer with 6.52 M parameters\n", + "PRIOR_BAG: tensor([1.0000e+00, 1.0000e+05]) [1]\n", + "{'is_causal': True, 'num_causes': 7, 'prior_mlp_hidden_dim': 18, 'num_layers': 7, 'noise_std': 0.18324918445872412, 'y_is_effect': False, 'pre_sample_weights': True, 'prior_mlp_dropout_prob': 0.24372190159948676, 'pre_sample_causes': True}\n", + "Hparams dict_keys(['prior_bag_exp_weights_1', 'num_layers_log_mean', 'num_layers_log_std', 'prior_mlp_hidden_dim_log_mean', 'prior_mlp_hidden_dim_log_std', 'prior_mlp_dropout_prob_b', 'prior_mlp_dropout_prob_k', 'noise_std_log_mean', 'noise_std_log_std', 'init_std_log_mean', 'init_std_log_std', 'num_causes_log_mean', 'num_causes_log_std', 'is_causal_choice_1_weight', 'pre_sample_weights_choice_1_weight', 'y_is_effect_choice_1_weight', 'prior_mlp_activations_choice_1_weight', 'prior_mlp_activations_choice_2_weight', 'prior_mlp_activations_choice_3_weight', 'prior_mlp_activations_choice_4_weight', 'block_wise_dropout_choice_1_weight', 'sort_features_choice_1_weight', 'in_clique_choice_1_weight', 'outputscale_log_mean', 'outputscale_log_std', 'lengthscale_log_mean', 'lengthscale_log_std', 'noise_choice_1_weight', 'noise_choice_2_weight', 'output_multiclass_ordered_p', 'multiclass_type_choice_1_weight'])\n" + ] + } + ], + "source": [ + "model = get_model(config_sample, device, should_train=True, verbose=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Prior tuning and inference of a fitted PFN (pretrained model provided)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Settings" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "model_string, longer, task_type = '', 1, 'multiclass'\n", + "eval_positions = [1000]\n", + "bptt = 2000\n", + " \n", + "test_datasets, valid_datasets = get_datasets('test', task_type, suite=suite), get_datasets('valid', task_type, suite=suite)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "model_string = ''\n", + "i, e = '8x_lr0.0003', -1\n", + "\n", + "# File which contains result of hyperparameter tuning run: style (i.e. hyperparameters) and a dataframe with results.\n", + "style_file = 'prior_tuning_result.pkl'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Setup helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "def load_result(path, i, e, ensemble=0, split_setter=None):\n", + " print(f'loading {path}')\n", + " with open(path, 'rb') as output:\n", + " c, metrics, metrics_valid, style, temperature, optimization_route = CustomUnpickler(output).load()\n", + "\n", + " metrics = metrics[ensemble]\n", + "\n", + " with warnings.catch_warnings():\n", + " warnings.simplefilter(\"ignore\")\n", + " for i in range(len(metrics)):\n", + " calculate_score_per_method(tabular_metrics.auc_metric, 'roc', metrics[i], test_datasets, eval_positions)\n", + " calculate_score_per_method(tabular_metrics.cross_entropy, 'cross_entropy', metrics[i], test_datasets, eval_positions)\n", + " calculate_score_per_method(tabular_metrics.time_metric, 'time', metrics[i], test_datasets, eval_positions)\n", + " calculate_score_per_method(tabular_metrics.auc_metric, 'roc', metrics_valid, valid_datasets, eval_positions)\n", + " calculate_score_per_method(tabular_metrics.cross_entropy, 'cross_entropy', metrics_valid, valid_datasets, eval_positions)\n", + "\n", + " df = {'checkpoint_path': path\n", + " , 'epoch_evaluated': e\n", + " , 'model_id': i}\n", + "\n", + " hparams = ['dropout', 'multiclass_loss_type', 'aggregate_k_gradients'\n", + " , 'num_classes_in_training', 'nlayers', 'nhead', 'bptt_in_training', 'lr', 'bptt'\n", + " , 'batch_size_in_training', 'emsize', 'nan_prob_unknown_reason', 'num_classes_in_training', 'epoch_in_training'\n", + " , 'normalize_to_ranking', 'categorical_feature_p', 'noise_type', 'set_value_to_nan', 'sampling'\n", + " , 'mix_activations', 'multiclass_type', 'output_multiclass_ordered_p', 'nan_prob_unknown_reason_reason_prior', 'num_steps']\n", + "\n", + " df.update({k: c[k] if k in c else None for k in hparams})\n", + " roc, ce, time = [], [], []\n", + " for split in range(0,5):\n", + " time += [metrics[split]['mean_time']]\n", + "\n", + " if split_setter is None:\n", + " for split in range(0,5):\n", + " roc += [metrics[split]['mean_roc']]\n", + " ce += [metrics[split]['mean_cross_entropy']]\n", + " else:\n", + " roc += [metrics[split_setter]['mean_roc']]\n", + " ce += [metrics[split_setter]['mean_cross_entropy']]\n", + " df['split_setter'] = split_setter\n", + " df['ensemble'] = ensemble\n", + " df.update({'mean_time_test': np.mean(time), 'mean_auc_test': np.mean(roc), 'mean_auc_valid': metrics_valid['mean_roc_at_1000'],\n", + " 'mean_cross_entropy_test': np.mean(ce), 'mean_cross_entropy_valid': metrics_valid['mean_cross_entropy_at_1000']})\n", + "\n", + " diff_list = DifferentiableHyperparameterList(c['differentiable_hyperparameters'], c['nhid_factor'] * c['emsize'], device)\n", + " diff_hparams_keys, diff_hparams_f = diff_list.get_hyperparameter_info()\n", + "\n", + " hyper = merge_style_with_info(diff_hparams_keys, diff_hparams_f, style, transform=False)\n", + "\n", + " return hyper, df, optimization_route, metrics, style, temperature" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Predict using a Fitted and Tuned Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Loading the model" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading models_diff/prior_diff_real_checkpoint_n_8x_lr0.0003_epoch_49.cpkt\n", + "Using style prior: True\n", + "Using cpu:0 device\n", + "Not using distributed\n", + "DataLoader.__dict__ {'num_steps': 768, 'fuse_x_y': False, 'get_batch_kwargs': {'batch_size': 1, 'seq_len': 10, 'seq_len_maximum': 10, 'device': 'cpu:0', 'num_features': 100, 'hyperparameters': {'lr': 0.0003, 'dropout': 0.0, 'emsize': 512, 'batch_size': 1, 'nlayers': 12, 'num_features': 100, 'nhead': 4, 'nhid_factor': 2, 'bptt': 10, 'eval_positions': [972], 'seq_len_used': 50, 'sampling': 'normal', 'epochs': 1800, 'num_steps': 768, 'verbose': False, 'pre_sample_causes': True, 'mix_activations': False, 'nan_prob_unknown_reason_reason_prior': 1.0, 'output_multiclass_ordered_p': 0.0, 'categorical_feature_p': 0.1, 'nan_prob_no_reason': 0.0, 'nan_prob_unknown_reason': 0.1, 'nan_prob_a_reason': 0.0, 'max_num_classes': 10, 'num_classes': 2, 'noise_type': 'Gaussian', 'balanced': False, 'multiclass_type': 'rank', 'normalize_to_ranking': False, 'set_value_to_nan': 0.1, 'normalize_by_used_features': True, 'num_features_used': . at 0x7f3dd11fdcb0>, 'num_categorical_features_sampler_a': -1.0, 'differentiable_hyperparameters': {'distribution': 'uniform', 'min': 0.5, 'max': 8.0}, 'prior_type': 'prior_bag', 'differentiable': True, 'flexible': True, 'aggregate_k_gradients': 8, 'recompute_attn': True, 'bptt_extra_samples': None, 'dynamic_batch_size': False, 'multiclass_loss_type': 'nono', 'total_available_time_in_s': None, 'done_part_in_training': 0.8805555555555555, 'categorical_features_sampler': . at 0x7f3dd1202290>, 'num_features_used_in_training': '.. at 0x7fe1a28895e0>', 'num_classes_in_training': '.. at 0x7fe1a2889550>', 'batch_size_in_training': 8, 'bptt_in_training': 1024, 'bptt_extra_samples_in_training': None, 'prior_bag_get_batch': (.make_get_batch.. at 0x7f3dca889ef0>, .make_get_batch.. at 0x7f3dca889dd0>), 'prior_bag_exp_weights_1': 2.0}, 'num_outputs': 1, 'dynamic_batch_size': 2, 'get_batch': .make_get_batch.. at 0x7f3dca889200>, 'differentiable_hyperparameters': {'prior_bag_exp_weights_1': {'distribution': 'uniform', 'min': 0.5, 'max': 8.0}, 'num_layers': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 6, 'min_mean': 1, 'round': True, 'lower_bound': 2}, 'prior_mlp_hidden_dim': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 130, 'min_mean': 5, 'round': True, 'lower_bound': 4}, 'prior_mlp_dropout_prob': {'distribution': 'meta_beta', 'scale': 0.9, 'min': 0.1, 'max': 5.0}, 'noise_std': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 0.3, 'min_mean': 0.0001, 'round': False, 'lower_bound': 0.0}, 'init_std': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 0.01, 'round': False, 'lower_bound': 0.0}, 'num_causes': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 12, 'min_mean': 1, 'round': True, 'lower_bound': 1}, 'is_causal': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'pre_sample_weights': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'y_is_effect': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'prior_mlp_activations': {'distribution': 'meta_choice_mixed', 'choice_values': [, , , , ], 'choice_values_used': [\"\", \"\", \"\", '. at 0x7fe1a2889670>', \"\"]}, 'block_wise_dropout': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'sort_features': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'in_clique': {'distribution': 'meta_choice', 'choice_values': [True, False]}, 'outputscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 1e-05, 'round': False, 'lower_bound': 0}, 'lengthscale': {'distribution': 'meta_trunc_norm_log_scaled', 'max_mean': 10.0, 'min_mean': 1e-05, 'round': False, 'lower_bound': 0}, 'noise': {'distribution': 'meta_choice', 'choice_values': [1e-05, 0.0001, 0.01]}}}, 'num_features': 100, 'num_outputs': 1}\n", + "Using a Transformer with 25.89 M parameters\n" + ] + } + ], + "source": [ + "model, c, results_file = load_model_workflow(i, e, add_name=model_string, base_path=base_path, device='cpu', eval_addition='')" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading prior_tuning_result.pkl\n" + ] + } + ], + "source": [ + "hyper_, df_, optimization_route, metric, style, temperature = load_result(style_file, i, e)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "#### Quick demo: Predict for a given dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[(0, 'balance-scale'),\n", + " (1, 'mfeat-fourier'),\n", + " (2, 'breast-w'),\n", + " (3, 'mfeat-karhunen'),\n", + " (4, 'mfeat-morphological'),\n", + " (5, 'mfeat-zernike'),\n", + " (6, 'cmc'),\n", + " (7, 'credit-approval'),\n", + " (8, 'credit-g'),\n", + " (9, 'diabetes'),\n", + " (10, 'tic-tac-toe'),\n", + " (11, 'vehicle'),\n", + " (12, 'eucalyptus'),\n", + " (13, 'analcatdata_authorship'),\n", + " (14, 'analcatdata_dmft'),\n", + " (15, 'pc4'),\n", + " (16, 'pc3'),\n", + " (17, 'kc2'),\n", + " (18, 'pc1'),\n", + " (19, 'banknote-authentication'),\n", + " (20, 'blood-transfusion-service-center'),\n", + " (21, 'ilpd'),\n", + " (22, 'qsar-biodeg'),\n", + " (23, 'wdbc'),\n", + " (24, 'cylinder-bands'),\n", + " (25, 'dresses-sales'),\n", + " (26, 'MiceProtein'),\n", + " (27, 'car'),\n", + " (28, 'steel-plates-fault'),\n", + " (29, 'climate-model-simulation-crashes')]" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[(i, test_datasets[i][0]) for i in range(len(test_datasets))]" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluation dataset name: balance-scale shape torch.Size([625, 4])\n" + ] + } + ], + "source": [ + "evaluation_dataset_index = 0 # Index of the dataset to predict\n", + "ds = test_datasets[evaluation_dataset_index]\n", + "print(f'Evaluation dataset name: {ds[0]} shape {ds[1].shape}')" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "# This parameter defines the number of inferences to average, the runtime scales almost linearly with N_ensemble_configurations.\n", + "# Higher values mostly affect cross entropy. \n", + "N_ensemble_configurations = 10" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "eval_xs, eval_ys = ds[1].clone().unsqueeze(1), ds[2].clone().unsqueeze(1)\n", + "eval_position = eval_xs.shape[0] // 2\n", + "\n", + "start = time.time()\n", + "prediction = transformer_predict(model[2], eval_xs, eval_ys, eval_position,\n", + " device='cpu',\n", + " style=style,\n", + " inference_mode=True,\n", + " N_ensemble_configurations=N_ensemble_configurations,\n", + " softmax_temperature=temperature, **get_params_from_config(c))\n", + "prediction_, y_ = prediction.squeeze(0), eval_ys.squeeze(1).long()[eval_position:]\n", + "time_taken = time.time() - start" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor(0.9991, dtype=torch.float64), tensor(0.5725), 17.07512354850769)" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "roc, ce = tabular_metrics.auc_metric(y_, prediction_), tabular_metrics.cross_entropy(y_, prediction_)\n", + "roc, ce, time_taken" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Testing speed\n", + "Define as 'ex' your scheduler / execution environment to perform a speed test on (e.g. one cpu kernel, ..) - alternatively remove scheduling.\n", + "The code also programatically sets torch and libraries to use one kernel. If you wish to use more remove the appropriate code in 'submit_speed_test'." + ] + }, + { + "cell_type": "code", + "execution_count": 197, + "metadata": {}, + "outputs": [], + "source": [ + "def submit_speed_test(device, N_ensemble_configurations):\n", + " import os\n", + " os.environ[\"OMP_NUM_THREADS\"] = \"1\" # export OMP_NUM_THREADS=4\n", + " os.environ[\"OPENBLAS_NUM_THREADS\"] = \"1\" # export OPENBLAS_NUM_THREADS=4 \n", + " os.environ[\"MKL_NUM_THREADS\"] = \"1\" # export MKL_NUM_THREADS=6\n", + " os.environ[\"VECLIB_MAXIMUM_THREADS\"] = \"1\" # export VECLIB_MAXIMUM_THREADS=4\n", + " os.environ[\"NUMEXPR_NUM_THREADS\"] = \"1\" # export NUMEXPR_NUM_THREADS=6\n", + " torch.set_num_threads(1)\n", + " \n", + " result = evaluate(datasets=test_datasets, model=model[2],\n", + " method='transformer'\n", + " , device=device\n", + " , overwrite=True, style=style\n", + " , save=False\n", + " , path_interfix=''\n", + " , metric_used=tabular_metrics.cross_entropy\n", + " , return_tensor=False\n", + " , verbose=False\n", + " , eval_positions=[1000]\n", + " , bptt=2000\n", + " , inference_mode=True\n", + " , softmax_temperature=torch.tensor([0.0]).repeat(3)\n", + " , base_path=None\n", + " , N_ensemble_configurations = N_ensemble_configurations\n", + " )\n", + " with open(f'speed_results/speed_result_{device}_{N_ensemble_configurations}.pkl', 'wb') as f:\n", + " pickle.dump(result, f)\n", + " \n", + " return result" + ] + }, + { + "cell_type": "code", + "execution_count": 198, + "metadata": {}, + "outputs": [], + "source": [ + "speed_job_list = []\n", + "for device in ['cpu']:\n", + " for ens in [1,5,10,20,50,100]:\n", + " raise Exception(\"Define ex as your scheduler\")\n", + " speed_job_list += [ex.submit(submit_speed_test, device, ens)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for device in ['cpu']:\n", + " for ens in [1,5,10,20,50,100]:\n", + " for prep in ['mix', 'robust_all', 'power_all']: \n", + " speed_result = pickle.load(open(f'speed_results/speed_result_{device}_{ens}_{prep}.pkl',\"rb\"))\n", + " calculate_score_per_method(tabular_metrics.time_metric, 'time', speed_result, cc_test_datasets_multiclass, [1000])\n", + " print(f'{device}_{ens}_{prep}', speed_result['mean_time'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Printing Prior Hyperparameter tables (optional for reference)" + ] + }, + { + "cell_type": "code", + "execution_count": 277, + "metadata": {}, + "outputs": [], + "source": [ + "hps_df = pd.DataFrame(hps).T\n", + "hps_df.loc[hps_df['choice_values_used'].isna(), 'choice_values_used'] = hps_df[hps_df['choice_values_used'].isna()]['choice_values']\n", + "hps_df = hps_df.drop(columns=['choice_values'])\n", + "hps_df.loc[hps_df['distribution'] == 'meta_choice_mixed', 'distribution'] = 'meta_choice'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_table = (hps_df[hps_df['distribution'] == 'uniform'][['min', 'max']].rename(index={'prior_bag_exp_weights_1': 'GP sampling likelihood'})\n", + ".rename(columns={'distribution': 'Sampling distribution $p(\\psi)$', 'min': 'Minimum', 'max': 'Maximum'}))\n", + "print(print_table.to_latex(index=True, escape=False))\n", + "print_table = (hps_df[hps_df['distribution'] == 'meta_beta'][['min', 'max', 'scale']].rename(index={'prior_mlp_dropout_prob': 'MLP weight dropout'})\n", + ".rename(columns={'distribution': 'Sampling distribution $p(\\psi)$', 'min': 'Min $\\alpha$ and $\\beta$', 'max': 'Max $\\alpha$ and $\\beta$', 'scale': 'Output scaling'}))\n", + "print(print_table.to_latex(index=True, escape=False))\n", + "print_table = (hps_df[hps_df['distribution'] == 'meta_choice'][['choice_values_used']].rename(index={'is_causal': 'Sample SCM', 'pre_sample_weights': 'Share Gaussian Noise mean for all nodes', 'y_is_effect': 'Sample y Node in last MLP layer'\n", + " , 'block_wise_dropout': 'Blockwise Dropout', 'sort_features': 'Keep SCM feature order', 'prior_mlp_activations': 'MLP Activation Functions', 'in_clique': 'Sample feature nodes in blocks', 'noise': 'GP noise'})\n", + ".rename(columns={'distribution': 'Sampling distribution $p(\\psi)$', 'choice_values_used': 'Choices'}))\n", + "print(print_table.to_latex(index=True, escape=False))\n", + "print_table = (hps_df[hps_df['distribution'] == 'meta_trunc_norm_log_scaled'][['max_mean', 'min_mean', 'round', 'lower_bound']]\n", + " .rename(index={'num_layers': 'MLP \\#layers', 'prior_mlp_hidden_dim': 'MLP \\#hidden nodes per layer', 'noise_std': 'Gaussian Noise Std.'\n", + " , 'init_std': 'MLP Weights Std.', 'num_causes': 'SCM \\#nodes at layer 1', 'outputscale': 'GP outputscale', 'lengthscale': 'GP lengthscale'})\n", + ".rename(columns={'distribution': 'Sampling distribution $p(\\psi)$', 'max_mean': 'Max Mean', 'min_mean': 'Min Mean', 'round': 'Round value'\n", + " , 'lower_bound': 'Lower bound'}))\n", + "print(print_table.to_latex(index=True, escape=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "#### Prior tuning and full datasets evaluation\n", + "This section runs a differentiable hyperparameter tuning run and saves the results to a results file, which can be inserted in TabularEval.ipynb to compare to other baselines." + ] + }, + { + "cell_type": "code", + "execution_count": 298, + "metadata": {}, + "outputs": [], + "source": [ + "# Enabling prior tuning will run differentiable hyperparameter optimization, which is not feasable on CPU.\n", + "# If you would like to run on a random configuration to get quick results, disable prior tuning.\n", + "enable_prior_tuning = True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "eval_positions=[1000]\n", + "bptt=2000\n", + "\n", + "N_models = 3\n", + "models_per_block = 1\n", + "\n", + "eval_addition = 'user_run'\n", + "\n", + "eval_model(i=i, e=e\n", + " , valid_datasets=cc_valid_datasets_multiclass[90:] if enable_prior_tuning else cc_valid_datasets_multiclass[:1]#valid_datasets[40:] #valid_datasets[40:]\n", + " , test_datasets=cc_test_datasets_multiclass\n", + " , train_datasets=cc_valid_datasets_multiclass[:90] if enable_prior_tuning else cc_valid_datasets_multiclass[:1]\n", + " , eval_positions_valid=[bptt//2]\n", + " , eval_positions_test=[bptt//2]\n", + " , bptt_valid=bptt\n", + " , bptt_test=bptt\n", + " , add_name=model_string\n", + " , base_path=base_path\n", + " , N_draws=5 if enable_prior_tuning else 0\n", + " , N_grad_steps=20 if enable_prior_tuning else 0\n", + " , selection_metric='ce'\n", + " , eval_addition=eval_addition\n", + " , n_parallel_configurations = 3\n", + " , N_ensemble_configurations = 10\n", + " , device=device)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "#### Plot optimization of a differentiable HP tuning run\n", + "Change 'style_file' to your evaluation results path." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hyper_, df_, optimization_route, metric, style, temperature = load_result(style_file, i, e)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "from matplotlib.ticker import FormatStrFormatter\n", + "renamer = {\"loss\": \"Validation CE Loss\", \"test_loss\": \"Training CE Loss\"}\n", + "for i, optimization_route in enumerate(optimization_routes):\n", + " f, ax = plt.subplots(figsize=(7, 7))\n", + " x, y = 'loss', 'test_loss'\n", + " #x, y = 'select', 'test_select'\n", + " for route in optimization_route:\n", + " route[y], route[x] = np.array(route[y]), np.array(route[x])\n", + " plt.plot(route[x][~np.isnan(route[y])], np.array(route[y][~np.isnan(route[y])]), '-o', markersize=3)\n", + " ax.set_xlabel(renamer[x])\n", + " ax.set_ylabel(renamer[y])\n", + " ax.yaxis.set_major_formatter(FormatStrFormatter('%.3f'))\n", + " ax.xaxis.set_major_formatter(FormatStrFormatter('%.3f'))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### Model Sequence Length Generalization Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 300, + "metadata": {}, + "outputs": [], + "source": [ + "test_datasets_longer_generalization = [ds for ds in test_datasets_multiclass if ds[1].shape[0] >= 10000]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "mlp_jobs_eval_longer_list = []\n", + "\n", + "for bptt_ in [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000, 5500, 6000, 6500, 7000, 7500, 8000, 8500, 9000, 9500, 10000]:\n", + " eval_addition = f'model_generalization_to_longer_newest_{bptt_}'\n", + " eval_model(i=i, e=e\n", + " , valid_datasets=test_datasets_longer_generalization[:1]#valid_datasets[40:] #valid_datasets[40:]\n", + " , test_datasets=test_datasets_longer_generalization\n", + " , train_datasets=test_datasets_longer_generalization[:1]\n", + " , eval_positions_valid=[bptt_//2]\n", + " , eval_positions_test=[bptt_//2]\n", + " , bptt_valid=bptt_\n", + " , bptt_test=bptt_\n", + " , add_name=model_string\n", + " , base_path=base_path\n", + " , N_draws=10#30\n", + " , N_grad_steps=20#30\n", + " , selection_metric='ce'\n", + " , eval_addition=eval_addition\n", + " , n_parallel_configurations = 3\n", + " , N_ensemble_configurations = 10\n", + " , device=device)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "#### Generalization Visualization Code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Warning this breaks things\n", + "eval_addition = 'model_generalization_to_longer_newest_'\n", + "test_datasets = test_datasets_longer_generalization\n", + "eval_positions = np.array([500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000, 5500, 6000, 6500, 7000, 7500, 8000, 8500, 9000, 9500, 10000]) // 2\n", + "ensembles = [0]\n", + "\n", + "match_string = f'{eval_addition}'\n", + "files = [os.path.join(base_path, 'models_diff/', file) for file in files if match_string in file]" + ] + }, + { + "cell_type": "code", + "execution_count": 1293, + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_5500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_6500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_1500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_2500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_3000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_7500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_8500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_9500.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/tmp/ipykernel_443022/3307425477.py:8: ResourceWarning: unclosed file <_io.BufferedReader name='/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl'>\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
checkpoint_pathepoch_evaluatedmodel_iddropoutmulticlass_loss_typeaggregate_k_gradientsnum_classes_in_trainingnlayersnheadbptt_in_training...nan_prob_unknown_reason_reason_priornum_stepsensemblemean_time_testmean_auc_testmean_auc_validmean_cross_entropy_testmean_cross_entropy_validbatch_size_effectivesplit_setter
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl00/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.076806.0022240.778491NaN0.790930NaN64NaN
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl01/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.076806.0022240.781297NaN0.792994NaN641.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl02/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.076806.0022240.779897NaN0.790729NaN642.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl03/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.076806.0022240.777390NaN0.789066NaN643.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_4500.pkl04/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.076806.0022240.775381NaN0.790932NaN644.0
..................................................................
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl00/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.0768028.6982310.790622NaN0.778693NaN64NaN
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl01/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.0768028.6982310.791781NaN0.778138NaN641.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl02/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.0768028.6982310.791817NaN0.778924NaN642.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl03/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.0768028.6982310.787148NaN0.779340NaN643.0
/work/dlclarge1/hollmann-PFN_Tabular/models_diff/prior_diff_real_results_longer_multiclass_causal_05_02_2022_12_49_44_sams_n_8x_lr0.0003_epoch_44_model_generalization_to_longer_newest_10000.pkl04/work/dlclarge1/hollmann-PFN_Tabular/models_di...-140.0nono8<function <lambda>.<locals>.<lambda> at 0x7fe1...1241024...1.0768028.6982310.791743NaN0.778370NaN644.0
\n", + "

100 rows × 34 columns

\n", + "
" + ], + "text/plain": [ + " checkpoint_path \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... /work/dlclarge1/hollmann-PFN_Tabular/models_di... \n", + "\n", + " epoch_evaluated model_id \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "... ... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... -1 4 \n", + "\n", + " dropout \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.0 \n", + "\n", + " multiclass_loss_type \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... nono \n", + "\n", + " aggregate_k_gradients \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 8 \n", + "\n", + " num_classes_in_training \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... .. at 0x7fe1... \n", + "\n", + " nlayers nhead \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "... ... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 12 4 \n", + "\n", + " bptt_in_training ... \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "... ... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1024 ... \n", + "\n", + " nan_prob_unknown_reason_reason_prior \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "\n", + " num_steps ensemble \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "... ... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 768 0 \n", + "\n", + " mean_time_test \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 6.002224 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 6.002224 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 6.002224 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 6.002224 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 6.002224 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 28.698231 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 28.698231 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 28.698231 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 28.698231 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 28.698231 \n", + "\n", + " mean_auc_test \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.778491 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.781297 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.779897 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.777390 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.775381 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.790622 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.791781 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.791817 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.787148 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.791743 \n", + "\n", + " mean_auc_valid \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "\n", + " mean_cross_entropy_test \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.790930 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.792994 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.790729 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.789066 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.790932 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.778693 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.778138 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.778924 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.779340 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 0.778370 \n", + "\n", + " mean_cross_entropy_valid \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "\n", + " batch_size_effective \\\n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 64 \n", + "\n", + " split_setter \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 2.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 3.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 4.0 \n", + "... ... \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... NaN \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 1.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 2.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 3.0 \n", + "/work/dlclarge1/hollmann-PFN_Tabular/models_dif... 4.0 \n", + "\n", + "[100 rows x 34 columns]" + ] + }, + "execution_count": 1293, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = {}\n", + "optimization_routes = []\n", + "df_hyper = {}\n", + "metrics = []\n", + "for i, file in enumerate(files):\n", + " for ensemble in ensembles:\n", + " for split_setter in range(0, 5):\n", + " hyper_, df_, optimization_route, metric = load_result(file, i, e, ensemble=ensemble, split_setter=split_setter)\n", + " df_hyper[file+str(ensemble)+str(split_setter)], df[file+str(ensemble)+str(split_setter)] = hyper_, df_\n", + " optimization_routes += [optimization_route]\n", + " metrics += [metric]\n", + " \n", + "df = pd.DataFrame.from_dict(df, orient='index')\n", + "df_hyper = pd.DataFrame.from_dict(df_hyper, orient='index')\n", + "#df['num_classes_in_training'] = df['num_classes_in_training'].astype(str).str.slice(0, 5)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 1298, + "metadata": {}, + "outputs": [], + "source": [ + "df = df.sort_values('mean_time_test')\n", + "df['train_pos'] = np.array([[i,i,i,i,i] for i in eval_positions]).flatten()" + ] + }, + { + "cell_type": "code", + "execution_count": 1305, + "metadata": {}, + "outputs": [], + "source": [ + "#filehandler = open(f'model_generalization_result.pkl',\"wb\")\n", + "#pickle.dump(df, filehandler)" + ] + }, + { + "cell_type": "code", + "execution_count": 1304, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABB2UlEQVR4nO3deZxldXng/89z97q39q6lt+qNXqABaaBFRSKIaMCIuEXBmMT8MoMkQ0acaEYzxmhiFmOc6G9GQ4gxJGpUgjgiQkBNwBEXuhsa6IWGpumlequta7v7OeeZP86p6ltV91ZXF3W7tuf9el3PvWe731O057nnuzxfUVWMMcaY8UKzXQBjjDFzkwUIY4wxZVmAMMYYU5YFCGOMMWVZgDDGGFOWBQhjjDFlVTVAiMj1IrJPRPaLyEfLbP+IiOwMXrtExBWR5mDbB4N1u0XkjmqW0xhjzERSrXEQIhIGngfeCHQC24BbVHVPhf1vBD6kqteKyEXAN4ErgALwb8DvqOoLVSmsMcaYCar5BHEFsF9VD6hqAf+Gf9Mk+98CfCN4fwHwc1XNqKoDPAa8vYplNcYYM06kiudeARwp+dwJvKrcjiKSBK4Hbg9W7QL+TESWAFngzcD2CsfeCtwKkEqlLj///PNnpPDzzr59/nLTptkthzELmCp4qjie4noeruevF0AkeC+CTOPcniq5okfOcckVXfJFDzeo4REgHg2TiIZIRMIkomEiodPf4nhKbTwyWoazsWPHjh5VbS23rZoBolxRK9Vn3Qg8rqp9AKq6V0Q+A/wAGAaeBpxyB6rqXcBdAFu3btXt28vGkYXvmmv85aOPzmYpjFlQCo5HOu/Qny3QO1xgKO8gQCQUoiYaJhY5u0qYXNGlayjPycEcXYM5Tgzm6RrKcag3w9H+LAAJYH1zkk3ttWxsr2NTex2rl6QIhyrf/fvSBV61rplENHzW1ygihyptq2aA6AQ6Sj6vBI5V2PdmTlcvAaCq/wD8A4CI/HlwPmOMqZrSgNAzlGe44I4JCC2p+BmP7xrKcTK48Z8cDIJB8H4gWxyzfywcoq0+zorGGl5/fhub2uvY0FZLKl7NW/PUVbMU24ANIrIWOIofBN47ficRaQCuBt43bn2bqnaJyCrgHcBrqljW+e/jH5/tEhgzp2lQNeS4iuN5ZPIuB3rSPH9ykOMDOYZzDtmii6eKpzBSUeR6Hq6C6/pVPo6neN5INZP/Gs47dA3m6csUxnxnJCS01sVpr0/w6rUp2usTtNUnaA/WNSSjhKZTL3SOVC1AqKojIrcDDwNh4CuqultEbgu23xns+nbgEVVNjzvFt4M2iCLwX1T1VLXKuiBcd91sl8CYqlBVVP36aW/0vb/0t5e2CyiO6/l1+I5H3vEYzjkc6ktzqDdN56kcx/qzHO3PcnIwh1eh0luAcEiIhIWwCOHQyCtEJHT6cyQkhEJCMhbm8tVNtNX7N/62ujhL6xM0pWJVCQDF4BoLrocAiWh40iqo6apaN9fZsKjbIHbu9JdbtsxmKYyZVNH1KLoeBcej4Pq/4jNFh2zew/E8VBVX/aDglQSG8WT0f09vdVyPE4N5jvZnORa8jvbnODGYww0iQUhgaX2CVUuSrGpKsrShhra6OE2pKLFwiPqaKIloeE79qldV8o4fEDxVFD8gLEnFaErFSMbC1ETDyDTLLCI7VHVruW1zo6LLvHx33OEvrZHazCLX09GbfyG4qaXzDum8S7bo4Hg62ntF8ev2o2EJliFQpegqGccjW3DIFj0yBf8c2YJLuuCQKbhkgmW64JLJO/QM5zk2cDoQCLC0IcGq5iRXnreEVc1JOpqTLEnFRp9EBKExGaGlNk4qHqE/U+TYQJZTmQKRUIjaeKQqv8qn8jfMBz2ZRsJffU2UVXVJ6muiJONh4pGzb4yeDgsQxphpKTgemYLDUM6hL51nOO9SDKo8wA8ARccbvaEP512Gcw4DuSL9mSKD2SL9WX85lC8GTxPu6E1+MoloiGQ0QjIeJhkLs7yxhlevOx0IVjbVEA2HyBVP32hDItTXRGmpjVFXEyUVGxsAGpMxVi9JMphz6BnKc3wgS9FVYuEQqSoGi4LjkXf86iIUwmGhKRmjoylJbU1kQjnPJQsQxpgzUlWyRZd03qU/U6A3XWAwW6TzVIbDfVl6h/0AMZgrMpA9/So4XtnzxcIhGpJRGmqiNCSjrGyqIRmPkIqFScYiJGPh4BUhFT/9fmRZ7obpjyPwA8Jw3iEkQmMyelY3WhHxy1QTZW1LisFcMeiFlMdxlUQkTDI+/SqoMcEAf+xEMhqmtS5OYzJGKv7yqotmmgUIY8wEjuuRKboMZx36MgWOD2Q52JPmYG+GI30ZDvX5y5Ef+9Gw0FATo7EmSn1NlI6mGhpqYjTUREfXNSb9ZUMiSiIamvZNUFXHtGN4QUNFKCQ0pfxf3nVBQAi9jF/eoZDQmIzRmIxxXqvHYM7hxECO7uEcqvjBIlb5Zj5ZMGhIRqmJ+sEgEp67OVMtQBizyLmef8PNFFwGsgU6+7LsOT7Iwd40h3ozHO7LcKw/OxoMGmuinNdWy6vWLuG81hTrW2tprYvP+K9eLwgERUcpeqcDgYhQEw1Rn4xQF4+SjIWJR8Mko+GXFRAmEwmHaE7FaE7FKLq1DGSLHOvP0psuIEBNNOy3v5QEg1QsMq+CQTkWIBaKP//z2S6BmSNUg+6ewY3fDwB+989s0SXvuEGvGI9T6Twv9WQ41JvxA0JfhhMDudFzNadinNea4pfWt3BeWy3rW2tpTsXwlNFf8I7ncaqk///4FgQJRhSInE5DMf494JfRO92G4d9koyypi1IXj5CI+ikmYpHQrNXJA0TDIVpq47TUxsk7LgOZIicGc9REw/M6GJRjAWKhuPLK2S6BmQXZgstQrkhvuuDf/It+Q/HIQC9FRzuDDmaKo1VDB4OA0DWUHz1Xa12c81pTXHd+G+e11rKuJUVtIkrB9Sg6Hl5w6+/LFIiG/V4+LXUxahMR4mG/V81IN8zRLqrBZycok+v53VddL1jveagHHpCKhf1zRcLEIyHikelXQ50r8UiYtvowbfWJ2S5KVViAWCh++lN/aYFiQSs4HsN5v9dQ91CefNAIHI/4ydsSEb+qpWuowIvdQ7zYPcyL3WkO9AzTnzmd5mF5Q4KN7XXccNEy1rWmWNFYQ000THH0F7yAKIrSlIxSm4hQE/WrcuKRoEuqWfAsQCwUf/iH/tLGQSwonqcMFxwGMkW6hvIM5fybfCQUIhkLUxuP0jucZ9vBPl7sGuZAT5oD3cOkCy7gjwbuaKrhslVNnNday3mtKda2pEjGImQL/iA1Vb+ffUNNhPpE1P8FHw0RC4eqVqdv5gcLEMbMIRqkfB7KFekeytObLuCpEhYJRs+eThY3nHP46s8P8cAzx/z++pEQa5ekeN3G1tEqotVLUmMyjo4EhUzBpTEZ5YIldTQkY9PKAmoWPgsQxsyi06NmvQnVRolImIaaicncCo7HA88c454dR8jkXV6/qY23XbqcVc3lU0KXPilYUDBnwwKEMeeA6+loArlswR9NPJx1yBTd0UbkaChETVBtVOkcjz3fxVd/fpie4TyXrWri/VeuZm1L7YR9LSiYmWABwpgZ5LjeaGK1TMEN0kj4eYRC4geCEEIsEiIWCdE8hVGzqsqTh/u5+6cvcbA3w/rWWu64bgOXrGwcs58FBTPTLEAsFJ///GyXYNEayhU50pdlIFsYrR4CP/dPPOI39qZqp/d/tf1dw/zjT1/imc4B2uvjfORNm7hqQ8uYaqdc0WUoX6QpGbOgYGaUBYiFwtJ8n3PZgsvB3jTHB3LUBPMEV6oeOlsnBnN89WeH+PEL3dQnIvznX1rHDRctndC9NFNwKLgel69upqFmZr7bmBEWIBaKH/7QX9rEQVVXcDw6T/mjj6PhEC2p2IwN6BrIFrln+xEefPY4oZDwq5ev5J2XrSw7BeVgtkg4BJetapozU1SahcX+VS0Un/60v7QAUTWupxwfyPJSdxpPlaZkbMZSPuSKLvc/fYxvP9lJruhy3QXtvPeKVSypLT8Hcn+mQE0szEUrGqw6yVSNBQhjzsDzlJ7hPPu7hyk4Hg2J6MvKs6Oq9Af5e04M5jjen+XhPSfpSxd41dpmfuM1a1jVnKx4bG8mT3Myzubl9Tai2VSVBQhjJtGfKbD/5DBDeYf6RJS6KbYxZAsuJ4MAMLI8MZDj5FCek4O5CfMkXLCsnj/45U1cuLyh4jk9VXrTeZY11LCxvW5WE9aZxcEChDFlDOcdDnQP0zNcoDbmT0tZjqfKjkOn2Ht8kJODOU4O5jkxmGMgWxyzX000zNKGBCsba7h8VSNL6xO01ydob0jQXpcYM9q5HNdT+tJ5VrekWNeSmvNJ7MzCUNUAISLXA18AwsCXVfUvx23/CPBrJWW5AGhV1T4R+RDwn/C7jj8L/Jaq5jCminLFoGdSf5ZEJEJrhcBQdD0e29fNfU91cuRUlpBAW12CpQ0JXr22mfaGxGgQWFqfoC4RmfZNvej66bQ3La1jZVP5qidjqkFUzzz/67ROLBIGngfeCHQC24BbVHVPhf1vBD6kqteKyArgJ8BmVc2KyD3Ag6p692TfuXXrVt2+fftMXsb8sW+fv9y0aXbLMU8VHI+j/X4a7GhIqE9Ey97QMwWHf9t1gu8+fYy+dIG1LSnecekKrlrfUpX8/3nHZSjncOHy+gWbUtrMLhHZoapby22r5hPEFcB+VT0QFOKbwE1A2QAB3AJ8Y1zZakSkCCSBY1Us6/xngWFaHNfjxGCOl3rSuJ7SVFO+Z9KpdIH7nz7GQ7uOky64vGJlAx+8dgOXrmqsWnVPpuCQK7pcuqqRxmSsKt9hzGSqGSBWAEdKPncCryq3o4gkgeuB2wFU9aiI/DVwGMgCj6jqIxWOvRW4FWDVqlUzVvh553vf85c33ji75ZgnXE/pGszxYs8wjqsVeyYdPZXlO0918qPnuvBUec15Lbzj0hVsbK+ravmGcw4eyuVrmqm1MQ5mllTzX165n1WV6rNuBB5X1T4AEWnCf9pYC/QD/yoi71PVr004oepdwF3gVzHNQLnnp899zl9agJjU+C6r9Yko0cTEwPD8ySHu3dHJzw/0EgkLb9zcztu2rGB5Y03VyziQLRCLhNiyoomamI1xMLOnmgGiE+go+bySytVENzO2euk64CVV7QYQkfuAK4EJAcKYqfA8v4voi91pskWH+nhsQpdVVWXH4VPc9+RRnj06QCoe5le3dvCWVyyj6RxV8fRl8tQnoly4vOGMPZuMqbZqBohtwAYRWQscxQ8C7x2/k4g0AFcD7ytZfRh4dVD1lAXeACzS1mfzcqgqpzJFXuwaYrjg+l1WU2Mbex3X4//u7+G+Jzs52JuhpTbGb1+1ljdtbicZOzfVO6pKX6ZAa22cTUvrFsSE92b+q9q/flV1ROR24GH8bq5fUdXdInJbsP3OYNe347cxpEuO/YWI3As8CTjAUwTVSMZMhaoykC2yv3uYoWyR2niUltTYLqtF1+OHe09y745OuobyrGpO8qHrNvBLG1qnNUJZVfHUXyqg6o+TUAWlZJv6da2l21xP6WiuYX1rnU3zaeaMqnVznQ2LupvrNdf4S5uTmoFskZe6h+nLFEnFwhOeAvKOy8O7T3Lfk530pgtsaq/j3Vs72LqmacLsbZNxXI90wcXx/FHRIfFTfIdDQliEcChEKETJ5zKvYH0kHKIpWb5rrTHVNFvdXM259NWvznYJZt1QrshLPWl6h/PURCcOcssWXB7adZzv7DxKf6bIhcvrueO6jVyysmHKN+a845ItuLieEo2EWNoQp6U2Tl0iaqkvzIJjAWKh6Og48z4LVKbgcLAnzcnBPIlImJbaxITtDzxznP+z8yhDOYctHY2855c7uGhF5bxHI1SVXNEjG8zUloqHWdOSojEZpTY+/dHRxswHFiAWim99y1++5z2zW45zqOj68zIc7PHnZVgybl6G4ZzD/U8f5f5njpHOu2xd3cR7tnZw/rL6Sc/rqZIpuOQdF4CmZJRVS+poqIlZt1OzqFiAWCj+9m/95SIIECNjGZ7vGvJHP4+bl2EgW+S7O4/ywDPHyRZdXr2umfdsXcX6ttqK5yxtTwiHhCWpGG31tdQnotbd1CxaFiDMvDKQKfJC9xBDWYeGmuiY3kan0gXue+ooD+06TsHxeO36Ft69tYO1LamK5yu6HoO5IpGQsLQhYe0JxpSwAGHmhVzR5aWeNMcHsqTGpd/uGc7z7Sc7eWT3SRzP43UbW3n35R10VJh0B/zAMJAtEouE2NReR1t9woKCMeNYgDBzmuN6HOvPcqAnHcz/HB9tZzjcl+G+Jzt57PluFLh2UxvvunzlpOkwRp8YwsL5S+torYvboDRjKrAAYeYkVaV7KM8LXcMUXY/GIMuqqrL72ADffrKTbQdPEYuEuP6ipbxtywraJ0mH7bgegzmHcBg2ttXRVm+BwZgzsQCxUNx772yXYMYM5ors7xpmIFOkPhGlPhHFU+VnB3q578lOnjsxRF0iwnuvWMWbL15GQ03laUAd12MgaGM4ry3F0vqEBQZjpsgCxELR0jLbJXjZTs/mlqMmGqalNk7R9Xh49wm+89RRjvZnaa+Pc9vr1vGGC9pJRCt3OXU9ZSBbIBQSzmutZWlDYlrpM4xZzCxALBR33+0v3//+2SxFWap+riEvyE3kacn7YH06mAM6FHQxzRRc7t3Ryf1PH+VUpsi61hR/8MubuPK8lkkbk11P6c8WCIuwtiXFssYaCwzGTJMFiIViFgNEruhyuC9DwfFwPA/XBcfzcDzF9Ty8IN2XCKCCn57u9IQhI+8bamL0Zwrc/dODPLTrBNmiy5aORv7bG1eeMR3GyBODCKxrSbG0ocbGLxjzMlmAMC9Lf6bArqMDqEIsEiIkQkiEaDhEPCKEhCmlozjcl+GrP3+BR/d146ly1foW3nHZSs5rrTy4zfWUbMEl77qEBFYvSbG80QKDMTPFAoSZFlXlaH+W508MUZeITtoeUE6u6LLv5BB7jg2y6+gAzxwd8HskXbiUmy5dwdIKPZKKrkdmZMSzCC11cdrqaqkfN2jOGPPyWYAwZ63oeuzvGubEQI7mVHxKA8z6MwX2Hh9kz/FBdh8b5MXuYTz1q5bWtqQm7ZGUd1wyBRdPlXgkxLJgxHNtImKD24ypIgsQ5qxkCg57jg6SLjgTkuONUFWOD+TYEwSEPccGOdqfBSAaFja21/GuyzvYvKye85fWkYpHJhyfLbpki36yvNp4hLUtKZpSMVKxsGVQNeYcsQCxUDz4YNW/onc4z+5jA0TDYZpLZmdzPeWlnjR7jg+w55gfFE5lioB/c9+8rJ43bm7nwmX1nNdWW7YqqLQ9AaA5FWPNkiQNydhZV18ZY2aGBYiFIlk579DLpaoc7s2wv3uYxprYmEbgEwM5Pvm93aNPCG11cS5Z2cjm5fVsXlZPR3Ny0lnaSscrtNXFaam19gRj5goLEAvFl77kL3/3d2f0tAXH44WTQ3QN51gyrr3hpZ5h/vj+3RRd5UPXbeDiFY201sUnOdtpqspArojnKWuWpFjeZOMVjJlrLEAsFPfc4y9nMEAM5x12Hx2g4Hi0pMb2Knr26ACf/v4ekrEwn37bxayaJHPqhPPmHHKOw/LGGlYvSVkVkjFzVFV/sonI9SKyT0T2i8hHy2z/iIjsDF67RMQVkWYR2VSyfqeIDIrIHdUsqxmreyjHjkOnAGhMxsZs+9mBXv74/l00p2L81TsvmXJwyBZcuofzpBJhXrl2CZuW1ltwMGYOq9oThIiEgS8CbwQ6gW0icr+q7hnZR1U/C3w22P9G4EOq2gf0AVtKznMU+E61ympO8zzlYG+al3rSNCVjE6p9Ht59gi89up8NbXV84i2bqZ8kUd6IouvRny1QF49waUcjTanYGY8xxsy+alYxXQHsV9UDACLyTeAmYE+F/W8BvlFm/RuAF1X1UFVKaUblHZfnjg9xKlOgpTY+pnFZVbl3Ryf//PNDXLaqkY/dcMEZf/2P5EWKhkNctLzBP6eNWzBm3qhmgFgBHCn53Am8qtyOIpIErgduL7P5ZsoHjpFjbwVuBVi1atV0y7roDeaKoykzlqTGNjR7qvzDT17i/qePcfXGVj74hg2TNih7qgwGDdDrWvz0F5Zi25j5p5oBotxPRa2w743A40H10ukTiMSAtwIfq/QlqnoXcBfA1q1bK51/4Xv00WkdpqqcGMjx3IkhUrEINfGxTwVF1+MLP3qBx57v5q2XLOe3r1o7abfVoVyRvOOxsqmGjuaktTEYM49VM0B0Ah0ln1cCxyrsW+kp4QbgSVU9OcNlM/iT6ezvHuZ4f5bGmtiEX/m5ostfPLSXJw/38xuvXs27Ll9ZcRRzpuCQLji01cVZ21I7YXS0MWb+qeb/i7cBG0RkLX4j883Ae8fvJCINwNXA+8qco1K7hBnvr//aX374w1PaPZ132HtskHTRYUnJPM8jBrNF/uSBPbzQNcTtr1/PL1+4tOK5TmUK1ERDXLaqaUKPJ2PM/FW1AKGqjojcDjwMhIGvqOpuEbkt2H5nsOvbgUdUNV16fNAu8UbgA9Uq44LywAP+cgoBomswx97jg8QjYZqTEwe2dQ3l+OP7d3NyMMdHb7iA16xbUvY8qkpfpkBTMsbm5fU20M2YBaaq9QCq+iDw4Lh1d477fDdwd5ljM0D5O5OZlpGcSYd6y3dhBX9ehj++fxeZgsun3noRF69oKHsuVaU3k2dpfQ0b2+ssq6oxC5BVFC8SuaLLnmMDDOYcWmsnVikBPHd8kD95YA+RsPCX77iYtS3lJ+txPaUvk2dVU5J1rbXWddWYBcoCxCLQly6w+9gAYZEJXVhHbD/Ux1889BxLUjH+5K0XsbSh/IQ9juvRlymwoa2Wjuakpd42ZgGzALFQ1NRMWOV5yuG+NC92pydkYS316L4uPv+jF1i9JMknb7yQpgoNzUXXoz9T4MJl9SxtnPh9xpiFxQLEQvHQQ2M+joyK7ktPHBVd6vH9PfzND5/nwuUNfPxXLiAZK/9PIld0Gc47vGJlAy115Z8ujDELiwWIBWggU2TXsQEAWmorp9/eeaSfv35kH5va/bxKlQa1ZQoOecflstVNZacENcYsTBYgFoo//VNUlaO/92GePzlEXTw66Sjm508O8WcP7mFlUw2feMuFFfcdzjl4KJetbqbWBr8Zs6hYx/UFwvvhj8j82yM8f3KY5mR80uBwpC/DJ+/fTUNNlE+99SJqE+Vv/IO5IuEwXL66yYKDMYuQ/b9+ASi6Htl8EU+hdZIqJfAHyf3Rd3cRDgt/etNFNFdIvd2XzlOXiHDhigbiEcunZMxiZAFiAegZylPjQSQ8eZfT/kyBT9y/28+x9I5XsKxhYk8kVaU3XaClLsYFS+stC6sxi5j9v3+eU1WOnMqccSRzpuDwye/tpns4zx+9ZTNrW1IT9vFU6UnnWdaY4MJlDRYcjFnk7AlinhvKO6TzLl5zc8V9Co7Hp7+/l4O9GT7+5gu4cPnE9Bmup/Sl86xpSbG2JWUD4IwxFiDmu+P9WWLhEIfv+mrZ7a6n/NXDz/Hs0QF+/40b2bpmYiBxPT+v0sb2OjqmOL+0MWbhszqEeazgeBwfyFXshaSq/K9/f4FfvNTHrb+0jms2tZXd71Q2z8Y2Cw7GmLEsQMxjvcN5AEIitP/lp2j/y0+NblNVvvL4QX70XBe3vLKDGy9ZXvYcQ7kiTck4Kyx1hjFmHKtimqdUlc5TmdHxCckdT4zZfu+TnfyfnUd5y8XLuOWK8nN1F10Px/PY1F5nGVmNMRPYE8Q8NZR3GM67ZccoPLz7BP/8s0NcvbGV//y6dWUbnFWVgWyB85fWUxOzcQ7GmIksQMxTx/uzZSf8eXx/D196dD+Xr27ijjdsqJikbyBbpL0+QWvd5APrjDGLlwWIeajgeJwYzFE3rnF6OO/4yfeW1vPR68+vOI4hV3QJhYT1bXXWndUYU5EFiHnoVDqPKmOeDvqa2niimPST7/1K5cysniqDuSIXLq+vOD+EMcaANVLPO6rK4b7MmOR5h/syfPSVHyB1VYTPTJJ8D+BUpsDqJSkaK0wKZIwxI6r6E1JErheRfSKyX0Q+Wmb7R0RkZ/DaJSKuiDQH2xpF5F4ReU5E9orIa6pZ1vlifON00fX402Ae6cmS74GfbiMVC7NmiY13MMacWdUChIiEgS8CNwCbgVtEZHPpPqr6WVXdoqpbgI8Bj6lqX7D5C8C/qer5wCXA3mqVdT4Z3zj9H/u6ODGY42u7vsGlf/PJise5npIpuFyw3HIsGWOmpppVTFcA+1X1AICIfBO4CdhTYf9bgG8E+9YDrwPeD6CqBaBQxbLOCyON0401/lOC6yn/ur2T9a21rH7shUmPPZXNs6G9zuZ1MMZMWTV/Sq4AjpR87gzWTSAiSeB64NvBqnVAN/CPIvKUiHxZRCamH/WPvVVEtovI9u7u7pkr/Rw0vnH6xy90c2Iwx7tf2THpcYPZIs3JOMvLpPc2xphKKgYIEWkdXyUUrL9QRFqncO5y/Se1wr43Ao+XVC9FgMuAv1XVS4E0MKENA0BV71LVraq6tbV1KsWan0Yap1Mx/wnA9ZR7th9hzZIkr1pbOZNr0fVw1WPTUhstbYw5O5M9QfwvoNwddyV++8CZdAKlP21XAscq7HszQfVSybGdqvqL4PO9+AFj0RppnB7pvvqzA710nsry7q0dFQfDqSr9GX+09GRTkBpjTDmTBYiLVfWx8StV9WHgFVM49zZgg4isFZEYfhC4f/xOItIAXA18t+Q7TgBHRGRTsOoNVG67WBRODORGG6c9Vb617TArGmu48rwWAArr1lNYt37MMQPZIssaa2irT5zz8hpj5r/JWiyj09wGgKo6InI78DAQBr6iqrtF5LZg+53Brm8HHlHV9LhT/B7w9SC4HAB+60zfuVAVXY/jA9nRxultB/s42JvhQ9dtGJ1J7uhnxj7UjYyWPq+19pyX1xizMEwWIF4QkTer6oOlK0XkBvwb9hkFxz44bt2d4z7fDdxd5tidwNapfM9C1zd8unFaVfnmtiO018e5emP5+R08VYbyDpetarTR0saYaZssQHwIeEBE3g3sCNZtBV4DvKXaBTO+8Y3TTx3uZ3/XMLe/fv2YeahX/PcPAv6TxKlMgTVLkjZa2hjzslQMEKr6vIhcDLwXuChY/RjwAVXNnYvCmdNzTi+pjftPD9uP0FIb49rzxz49xA7sB/zR0rWxCKuXlO0VbIwxUzbpqClVzQP/eI7KYso4MZAbHfm86+gAe48P8oHXrSub6hsgU3B55drmMU8XxhgzHZONgxgSkcGS14CIvBgMWltyLgu5WI00To+k9f7m9iM0JaO8cXN7xf03tNfaaGljzIyoGCBUtU5V60teDfhtELuBOysdZ2ZOaeP03uODPNM5wNsvXVF2FjnXU6LhkM0tbYyZMWf1U1NVTwF/IyK/XqXymMD4xulvbT9CfSLCDRctm7Bv0fUYOv9CljbU2ARAxpgZc9Z1ESISnc5x5uz4I6cdWmoT7O8aZsehU/zGq1eXHRE9kC2y/AtfIGJPD8aYGVTxRi8i7yizugl4D37qC1NF/shpPxh8a/thUvEwv/KKiU8P6bxDXU3ERksbY2bcZE8CN477rEAv8AVV/X71imRKR04f7Enz8wN93PLKDpKxsf+5PFUyBYetK5oJ/UZQ6/e1r81CiY0xC9Fk4yAqprYQkVeq6rbqFMmUNk7fs+MINdEwN16yfMJ+A9kiHc1J6hNR6OychZIaYxayKbclBKm/b8af2GcAS4NRFaWN052nMvzkhR7eedlK6hJj018VXQ8RbECcMaZqJg0QIrIaPyDcAjjAamCrqh6sftEWp9LG6X/d3kk0EuKmLeWfHi5YWme5lowxVTPZQLmf4ifaiwLvUtXLgSELDtU10jh9YiDHo893ccOFSyfkVErnHeprIrQ3WMO0MaZ6JnuC6Maf5Kcdf+KgF6g8I5yZASON0w2JGF/92UHCIeHtl46dpXWkYfqVK5rHjnl4zWvOcWmNMQvdZI3UNwWT+bwT+JSIrAcaReQKVX3inJVwERlpnO5LF/jRc1286cKlLKmNj9lnIFtkVXNyQpsEf/EX57CkxpjFYNIKbFUdUNWvqOobgVcBnwA+LyJHzknpFpHSxun7nuxEgXeOe3oouh4hgVXWMG2MOQem3MKpql2q+r9U9UrgqiqWaVEaDhqnswWXh/ec4Nrz2yYMfuvPFtnYXqFh+p3v9F/GGDNDppUyQ1UPzXRBFruRxul7d3Tiesq7Lls5Zns679CYjNJaFy9/gt7ec1BKY8xiYn0k54CC43FsIIvrKQ/tOs7rNrayvCSvkqdKtuiwoa3WkvEZY84ZS7o3Bxzrz6IK33/mGAXH492Xd4zZ3p8p0NGcmtgwbYwxVTTZOIi/EpHbyqz/kIh8ZionF5HrRWSfiOwXkY+W2f4REdkZvHaJiCsizcG2gyLybLBt+9lc1HySK7oc7E0TkRAPPHOcK9e30NGcHN1edD3CIWFVyTpjjDkXJnuCeAun56Iu9QXgGeC/T3ZiEQkDXwTeCHQC20TkflXdM7KPqn4W+Gyw/43Ah1S1r+Q0r1fVnqlcyHx1uC9NOCQ8+OxxskWX92wd2/bQny1y0fL6M4+YfsMbqlhKY8xiNFmAUFX1yqz0ZGoV4VcA+1X1AICIfBO4CdhTYf9bgG9M4bwLRjrvcPRUjppoiPufPsar1jaztqV2dPtw3qFpsobpUn/0R1UsqTFmMZrsZ2lGRDaMXxmsy07h3CuA0vESncG6CUQkCVwPfLtktQKPiMgOEbm10peIyK0isl1Etnd3d0+hWHPHSz3DxCMhHtp1kuG8w7u3nm57GGmYXm8N08aYWTJZgPgE8JCIvF9ELg5evwV8P9h2JuXuapVSddwIPD6ueum1qnoZcAPwX0TkdeUOVNW7VHWrqm5tbW2dQrHmhoFMka6hPIlomO8+fZRLOxrZ2F43ur0/U2DV2TRM33CD/zLGmBkyWaqNh0TkbcBHgN8LVu8G3qmqz07h3J1AaXeclcCxCvvezLjqJVU9Fiy7ROQ7+FVWP57C9855qsr+7iFSsQhPvNRHf6bIW689nbG16HqEw2fZMJ2dykOdMcZM3aTdXFV1F/CbIlLrf9T0WZx7G7BBRNYCR/GDwHvH7xTke7oaeF/JuhQQUtWh4P2bgD85i++e03qH8wxmHVpq4zyy5yQttTEuXdU0un0gV+TCZVNomDbGmCqa9A4kIr8rIoeBQ8BhETkkIr87lROrqgPcDjwM7AXuUdXdInLbuO6zbwceGRd82oGfiMjTwBPA91X136Z+WXOX5yn7u9PUxiN0DeZ46vAprrugnXDIr5Ebzjs01kyxYdoYY6qo4hOEiHwcuBK4pqQn0jrgCyLSrKqfPtPJVfVB/DklStfdOe7z3cDd49YdAC6Z2iXML12DOXJFlyWpOPc96U8Tet0F7YDfMJ0ruly8ssEapo0xs26yKqZfBy5R1dzIClU9ICLvBp4GzhggzFhF12N/zzD1iSiup/xgbxdbOhppD5Ly+Q3TSWrj0xjg/pa3zHBpjTGL3ZnaIHJl1mVFZML4CHNmx/uzOK4STYTYfqiPnuE8/+mqtYCfjykcFlYtmeaI6Q9/eAZLaowxk7dBdIrIhOG5InItcLx6RVqY8o6fUqOxxp8+9JHdJ6lPRLhibTMAg7kiG9vqiIatYdoYMzdM9gTxX4HvishPgB34YxheCbwWf0S0OQtH+rIIQjgknMoUeOJgHze+YjnRcIhswaWuJvLyGqavucZfPvroTBTXGGMqP0Go6m78XEw/BtYA64L3FwXbzBRlCg5H+jLU1/iD3v79uS5cT3nThX7jdKbo0NFYYw3Txpg5ZSptEF8pXSciYRH5NVX9elVLtoAc7EkTDYcIiaCqPLL7BJuX1dPRdLq9oSEZm8USGmPMRJOl+64XkY+JyP8WkTeK73bgAPDuc1fE+W0wV+TEYJ76hB+Ldx8b5NhAjjdt9p8eckWXukSURDQ8m8U0xpgJJnuC+CpwCvgZ8J+BPwBiwE2qurP6RZv/VJUDXcMko+HR6qOH95wgGQvz2vUtAKQLDptKcjAZY8xcMVmAWKeqFwOIyJeBHmCVqg6dk5ItAKcyRU5lCrTU+uMchnMOP93fyxsuaBt9YlCFptQMVC+92x7qjDEza7IAURx5o6quiLxkwWHqPE95sWuI2vjpbKyPPd9FwfV40+alwEj1UmRmqpd+d0oZUIwxZsomCxCXiMhg8F6AmuCz4Cfuq6966eaxnuE8Q3mH1uDpQVV5eM9J1rWmWN/mTwqULjhsbJuh6qVMxl8mbWpSY8zMmCzdt7WaTpPjeuzvHqYhcbrqaH/XMC/1pPmdq88bXaeqM1O9BPDmN/tLGwdhjJkhNmy3Ck4M5ig43ph03Y/sOUksEuJ1G/1JjfKOS208Qk3M4rAxZm6yADHDCo7HS91pGkpmgssWXB57vpurzmsZTcSXzjssa6yZrWIaY8wZWYCYYZ2nMniqREpyKj2+v4ds0R0dOQ3gKTTPVPWSMcZUgQWIGZQtuBzuy9BQM/bG/8ieE6xorGHzMr9dv+B4JGNhkrFppPU2xphzxO5QM+hQn59SY2R2OIBDvWn2nhjit65cMzpYLl1wWNuSmtkvf//7Z/Z8xphFzwLEDBnKFTnen2VJamxG1h/sOUkkJFx7ftvoOs/Tma9esgBhjJlhVsU0Q17qSZOIRMZkZC26Hv++r4tXrW2mMUjGV3Q9ErEwyZnuvdTT47+MMWaG2BPEDEjnHfqGCyypHfv08PMDvQzlnNGR0yP7rl6SnPnU3u96l7+0cRDGmBlS1ScIEbleRPaJyH4R+WiZ7R8RkZ3Ba5eIuCLSXLI9LCJPicgD1SznyzWcK/rjy8d5ZM9J2uribFnVOLrOVaW59mVMDGSMMedI1QKEiISBLwI3AJuBW0Rkc+k+qvpZVd2iqluAjwGPqWpfyS4fBPZWq4wzpTddIBEZW2V0YjDHziP9XHdBO6HgaaHoesQjIVI2OM4YMw9U8wniCmC/qh5Q1QLwTSafqvQW4BsjH0RkJfArwJerWMaXTVX9ADEu4d4P95wkJHDdBafHPqQLDssbbOY4Y8z8UM0AsQI4UvK5M1g3gYgkgeuBb5es/jz+HBTeZF8iIreKyHYR2d7d3f2yCjwdmYKL5+mYrq2up/xg70kuW9U0Zp5p11Waa21wnDFmfqhmgCj3M1kr7Hsj8PhI9ZKIvAXoUtUdZ/oSVb1LVbeq6tbW1tbpl3aa0nlnwkXtOHSKvnRhdNY48KuXYtHQaKqNGfc7v+O/jDFmhlSzF1Mn0FHyeSVwrMK+N1NSvQS8FniriLwZSAD1IvI1VX1fVUr6MvSmC8THtT88sucEjTVRXrlmtL2ddN5hZXMVq5fe857qnNcYs2hV8wliG7BBRNaKSAw/CNw/ficRaQCuBr47sk5VP6aqK1V1TXDcv8/F4KCq9A4XqClpf+hLF9h2sI83XNA2Jh+TqzqhG+yMOnLEfxljzAyp2hOEqjoicjvwMBAGvqKqu0XktmD7ncGubwceUdV0tcpSLdmii+t5Y9offrT3JJ4yZuyD43pEwyHqqlW9BPDrv+4vbRyEMWaGVHWgnKo+CDw4bt2d4z7fDdw9yTkeBR6d8cLNgHTexStpgfDUb5y+aHk9y0tSeacLLssaEtZ7yRgzr1iqjZehL50nET4dY589OsDxgRxvunDpmP0cz6OlzgbHGWPmFwsQ06Sq9AwXSERLZo3bfZJUPMyV5y0ZXed6SiRU5eolY4ypAgsQ05Qrejju6YmBBrNFfvpiD6/f2DamV9Nw3mFpQ5xQyKqXjDHzi/2snaZ0wcHT0+0Pjz7fhePpmFnjwK9eaq1NVL9Av//71f8OY8yiYgFimvrSBeIR/+lBVXlk90k2tNWytqV2dB+/ekmoS5yDP/ONN1b/O4wxi4pVMU1T73B+NP/S8yeHOdSXGdO1FfzBce31iXNTvbRvn/8yxpgZYk8Q05AruhQcj9p4FICH95wgHgnxuo0tY/Yret6YXExV9YEP+EsbB2GMmSH2BDENpfmXHNfjJy/0cNX6FpKx0/H2dPVSdHYKaYwxL5MFiGk4lSkQDfl/un0nh8gWXa5Y2zxmn3TeobUuPmaUtTHGzCcWIKahZ7hATTDpz1NH+gkJvGJl45h9ip5HW9056L1kjDFVYgHiLOWKLrmiSzQY//D0kX42tNWNSePtBvND1NdY9ZIxZv6yRuqzlCm4o+/TeYfnTw7xq5d3jNvHoe1cVy99/OPn7ruMMYuCBYizdCpdIBK0PzxzdABPYUtH45h9Cq5H67muXrruunP7fcaYBc+qmM5S73CeZND+sPNIP4loiE1L60a3e6qERKg/F4PjSu3c6b+MMWaG2BPEWcg7Lpmiy5KgO+vOw6e4aHnDaHsEQCbv0loXHzNZ0Dlxxx3+0sZBGGNmiD1BnIVM/nT7Q9dgjmMDuQnVS3nXpc1SextjFgALEGehP3u6/WFnZz8wtv3BU0UE671kjFkQLECchdL5p3ce6ac5FWNVc3J0e6bg0lobH1PlZIwx85Xdyaao4HgM5x1ikRCeKk8f6WfLysYx04jmHZe2ehscZ4xZGKyReooyBWf0/YHuNIM5h0tKqpc0mBuiYbaql/78z2fne40xC1ZVnyBE5HoR2Sci+0Xko2W2f0REdgavXSLiikiziCRE5AkReVpEdovIp6pZzqkYyBQJB08LO4/0A2PbHzIFl5bZrF668kr/ZYwxM6RqdzMRCQNfBG4ANgO3iMjm0n1U9bOqukVVtwAfAx5T1T4gD1yrqpcAW4DrReTV1SrrVJTmX9p55BSrm5M0p2Kj23OOS/tsVi/99Kf+yxhjZkg1q5iuAPar6gEAEfkmcBOwp8L+twDfAFC/vmY4WB8NXlrhuKoruh5D+SJLUnHyjsue44O8+aJlo9tnvXoJ4A//0F/aOAhjzAypZn3ICuBIyefOYN0EIpIErge+XbIuLCI7gS7gB6r6iwrH3ioi20Vke3d390yVfYzS8Q97jg1SdJUtqxpH12WLLk3JKLGItfkbYxaOat7RymWqq/QUcCPweFC95O+o6gZVTyuBK0TkonIHqupdqrpVVbe2tra+3DKXNZAtECppf4iEhIuWN4xuzxZnuXrJGGOqoJoBohMoTXO6EjhWYd+bCaqXxlPVfuBR/CeMWdGTHjv+4fyldaPzUY+wmeOMMQtNNQPENmCDiKwVkRh+ELh//E4i0gBcDXy3ZF2riDQG72uA64DnqljWihzXYyhXJB4J0Z8pcKAnzZZVTaPb/alFQ6MJ/IwxZqGoWiO1qjoicjvwMBAGvqKqu0XktmD7ncGubwceUdV0yeHLgH8KekKFgHtU9YFqlXUy6YKLKogIT3cOAHDpmO6tDm318TED5mbF5z8/u99vjFlwqjpQTlUfBB4ct+7OcZ/vBu4et+4Z4NJqlm2qBrNFQoy0P5yiNh7hvNba0e0F12NJSXfXWbNly2yXwBizwFi3mzPoTeepiYVRVXYeGeAVKxvGzBSnCrXneu6Hcn74Q/9ljDEzZA7c2eYu11MGMg5NyShH+7P0DOd599aVo9vzjkttPEw8MgfaHz79aX9pM8sZY2aIPUFMIl1wUBQRqZhew7q3GmMWKgsQkxjKFseMf2ivj7OsoWZ0u6dKQ3IOtD8YY0wVWICYRM9wgUQkjON6PNM5wJaO091bPVUEoTZutXTGmIXJAkQFrqcMZIvEoyGe7xomW3THdG/NFV2aUtExDdbGGLOQ2M/fCjIFB0+VkAhPH+lHgFesHJteY/WSZOUTnGt/93ezXQJjzAJjAaKCoawzmkzqqSP9rG+rHZtOQ+dYeo1Nm2a7BMaYBcaqmCrozRRIRMNkCg77TgyO6b3kuB7RyBxLr/G97/kvY4yZIfYEUYbnKf3pAvU1UbYd7MPTsd1bs0WX1ro5kF6j1Oc+5y9vvHF2y2GMWTDsCaKMTNHF9fz2h52H+4lHQlywrH50+5xJr2GMMVVkAaKM4VxxdDaLp470c+HyhglzTc+J9BrGGFNFFiDK6B3253/oHspztD87pnurn14jMjfSaxhjTBVZgBjH85S+TIF4JMzTFdJrtNXFZ6dwxhhzDlk9yTjZoovnKeGQ8NSRfhqT0THjHeZseo2vfnW2S2CMWWAsQIwznCui+IHg6c5+Lu1oHO2tNDJwbk6m1+joOPM+xhhzFqyKaZzejJ9/6WBPmoFscUz1Uq7o0pyKzc30Gt/6lv8yxpgZMgd/Cs8eVaVvuEhtPFI2vXeu6LJmLqXXKPW3f+sv3/Oe2S2HMWbBsCeIEtmii+t5o+0PHc1JltSebpBWhbqaOZRewxhjqsgCRInhvIMCBcdjz7HBMd1bR9Jr1ESte6sxZnGoaoAQketFZJ+I7BeRj5bZ/hER2Rm8domIKyLNItIhIv8hIntFZLeIfLCa5RzRN+x3b917fJCC63HJysbRbSPdW+dUeg1jjKmiqgUIEQkDXwRuADYDt4jI5tJ9VPWzqrpFVbcAHwMeU9U+wAF+X1UvAF4N/Jfxx840VaUnGCD31JF+wiHhohWn02sUPW9MdZMxxix01WykvgLYr6oHAETkm8BNwJ4K+98CfANAVY8Dx4P3QyKyF1gxybEvW67ojbY/7DxyivOX1pGMnf7zKMzN7q0j7r13tktgjFlgqlnFtAI4UvK5M1g3gYgkgeuBb5fZtga4FPhFhWNvFZHtIrK9u7t72oUdaX8YyBY50J2e0HupPh4lFpnDTTYtLf7LGGNmSDXveOUq67XCvjcCjwfVS6dPIFKLHzTuUNXBcgeq6l2qulVVt7a2tk67sKcyBeLhMM909qNM7N7aVj/Hq5fuvtt/GWPMDKlmgOgESof3rgSOVdj3ZoLqpREiEsUPDl9X1fuqUsIS3UN5EtEQO4/0k4qF2dBWN7rNVaUhOce7t1qAMMbMsGoGiG3ABhFZKyIx/CBw//idRKQBuBr4bsk6Af4B2Kuq/7OKZQT8JwTHHWl/6OfilQ2jo6VH02vE5nD7gzHGVEHVAoSqOsDtwMPAXuAeVd0tIreJyG0lu74deERV0yXrXgv8OnBtSTfYN1errJ76NV/HB3J0DeXZ0tE0ui1bcFmSihGai+k1jDGmiqr6s1hVHwQeHLfuznGf7wbuHrfuJ5Rvw6iqp4L0GqUD5HKOy7q61LkuijHGzLo53C3n3Nt55BRtdXGWNSTGrK9LzPH2B2OMqQKrWA+4nvJM5wBXrW8ZHS1ddD3ikRCJ6DyIow8+eOZ9jDHmLFiACLzUkyZTcCfMHre0YZ6k10jO0Syzxph5ax78ND43dh8bRIBXlORfcjyP5tQcH/8w4ktf8l/GGDNDLEAEdh0bYF1rioYgnbcGPZvmdHqNUvfc47+MMWaGWIAA0nmHF7vSY7q35h2PusQcT69hjDFVZHc/YNvBPlzVsd1bi356b2OMWawsQAA/fbGXaFi4YNnp9N7zIr2GMcZUkQUI4Gcv9rKpvW60Osn1lLCl1zDGLHIy0hi7EIhIN3BoGgeGQrFkrXquA0AoFMLzXC3msjNcxGpqAXpmuxCzyK7frt+uf3pWq2rZVNgLKkAsZiKyXVW3znY5Zotdv12/Xf/MX79VMRljjCnLAoQxxpiyLEAsHHfNdgFmmV3/4mbXXwXWBmGMMaYse4IwxhhTlgUIY4wxZVmAmKNE5Csi0iUiu0rWNYvID0TkhWDZVLLtYyKyX0T2icgvl6y/XESeDbb9/zIvcpeDiHSIyH+IyF4R2S0iHwzWL4q/gYgkROQJEXk6uP5PBesXxfWPEJGwiDwlIg8EnxfN9YvIwaDcO0Vke7Du3F6/qtprDr6A1wGXAbtK1v0V8NHg/UeBzwTvNwNPA3FgLfAiEA62PQG8Bn8K14eAG2b72qZ4/cuAy4L3dcDzwXUuir9BUNba4H0U+AXw6sVy/SV/h/8G/AvwQPB50Vw/cBBoGbfunF6/PUHMUar6Y6Bv3OqbgH8K3v8T8LaS9d9U1byqvgTsB64QkWVAvar+TP1/Kf9ccsycpqrHVfXJ4P0QsBdYwSL5G6hvOPgYDV7KIrl+ABFZCfwK8OWS1Yvm+is4p9dvAWJ+aVfV4+DfQIG2YP0K4EjJfp3BuhXB+/Hr5xURWQNciv8retH8DYLqlZ1AF/ADVV1U1w98HvgDwCtZt5iuX4FHRGSHiNwarDun12/Z6BaGcnWKOsn6eUNEaoFvA3eo6uAk1acL7m+gqi6wRUQage+IyEWT7L6grl9E3gJ0qeoOEblmKoeUWTdvrz/wWlU9JiJtwA9E5LlJ9q3K9dsTxPxyMnhkJFh2Bes7gY6S/VYCx4L1K8usnxdEJIofHL6uqvcFqxfV3wBAVfuBR4HrWTzX/1rgrSJyEPgmcK2IfI3Fc/2o6rFg2QV8B7iCc3z9FiDml/uB3wze/ybw3ZL1N4tIXETWAhuAJ4JH0CEReXXQc+E3So6Z04Ly/gOwV1X/Z8mmRfE3EJHW4MkBEakBrgOeY5Fcv6p+TFVXquoa4Gbg31X1fSyS6xeRlIjUjbwH3gTs4lxf/2y31NurYg+GbwDHgSL+r4DfBpYAPwJeCJbNJfv/D/yeC/so6aUAbA3+Yb0I/G+C0fNz/QVchf8o/AywM3i9ebH8DYBXAE8F178L+ESwflFc/7i/xTWc7sW0KK4fWIffK+lpYDfwP2bj+i3VhjHGmLKsiskYY0xZFiCMMcaUZQHCGGNMWRYgjDHGlGUBwhhjTFkWIMyMExEVkc+VfP6wiHxyhs59t4i8aybOdYbv+VXxM8n+x7j1a0TkvdM850+nsM+XRWTzdM4/m0TkURHZOtvlMDPLAoSphjzwDhFpme2ClBKR8Fns/tvA76rq68etXwOUDRAiMmnqGlW98kxfqqr/SVX3TLWQxlSTBQhTDQ7+HLkfGr9h/BOAiAwHy2tE5DERuUdEnheRvxSRXxN/ToRnReS8ktNcJyL/N9jvLcHxYRH5rIhsE5FnROQDJef9DxH5F+DZMuW5JTj/LhH5TLDuE/gD9e4Ukc+OO+QvgV8SP0f/h0Tk/SLyryLyPfzEarUi8iMReTI4700VrvVREblXRJ4Tka8Ho1zH/BIXkWER+TPx54T4uYi0B+vPCz5vE5E/GTnvuOtKicj3g2N3ich7Rq4tOG6XiNw17nv/RkR+HDw5vVJE7hN/3oFPB/usCcr7T8Hf+F4RSZb57jeJyM+Cv8G/ip9Pi+C/6Z7g2L8ef5yZg2Z7xKC9Ft4LGAbq8fPZNwAfBj4ZbLsbeFfpvsHyGqAffx6IOHAU+FSw7YPA50uO/zf8Hzcb8EeZJ4BbgY8H+8SB7fh58a8B0sDaMuVcDhwGWvETV/478LZg26PA1jLHXEMwqjf4/P6gDM3B5wh+emWAFvy0y1LmWgfw8+KEgJ8BV43/XvyR5DcG7/+q5PoeAG4J3t82ct5x5Xwn8PclnxuCZenI26+WnP9RTs8t8EH8fD0j/y068UfwrgnK9Npgv68AHy4td3DNPwZSwfr/DnwCaMYf4Tvyt2ic7X+n9jrzy54gTFWo6iB+7vn/ehaHbVN/Hog8flqAR4L1z+LfnEbco6qeqr4AHADOx89V8xvip8f+Bf4NbUOw/xPq58gf75XAo6raraoO8HX8iZrO1g9UdWTuDgH+XESeAX6In1q5vcwxT6hqp6p6+GlE1pTZp4AfDAB2lOzzGuBfg/f/UqFMz+I/aX1GRH5JVQeC9a8XkV+IyLPAtcCFJcfcX3Ls7pL/Fgc4nQjuiKo+Hrz/Gv6TVqlX409e83jw3+I3gdXAIJADviwi7wAyFcpt5hBL922q6fPAk8A/lqxzCKo2g+qNWMm2fMl7r+Szx9h/q+Pzw4ykNf49VX24dIP4qaLTFco3U1NPlp7/1/CfSC5X1aL42UgTZY4pvVaX8v9fLGrwc3uSfcpS1edF5HL8/FV/ISKP4D+FfAn/CeWI+B0HSstW+vce/99i5LvL/e1LCX7AvGV8mUTkCuAN+Mn3bscPUGYOsycIUzXBr+p78Bt8RxwELg/e34Q/U9rZ+lURCQXtEuvwqy4eBn5H/BThiMhG8bNgTuYXwNUi0hI0YN8CPHaGY4bwp0CtpAF/HoOiiLwe/9fzTPs5fhUS+DfbCURkOZBR1a8Bf40/fe1IMOgJ2gWm0xtslYi8Jnh/C/CTMmV7rYisD8qRDP5b1OJXcz0I3AFsmcZ3m3PMniBMtX0O/9fiiL8HvisiT+Bno6z0634y+/Bv5O3AbaqaE5Ev41fBPBk8mXRzhqkVVfW4iHwM+A/8X74PquqZUiE/Azgi8jR+e8ipcdu/DnxP/Enmd+Kn6J5pdwBfE5HfB76P354x3sXAZ0XEw88I/Duq2i8if49fhXQQ2DaN794L/KaI/B1+RtG/Ld2oqt0i8n7gGyISD1Z/HD+wfldEEvh/6wkdGMzcY9lcjZlngp5DWVVVEbkZv8H6pjMdNwPfuwa/gX6yme3MAmJPEMbMP5cD/zt4UuoH/r/ZLY5ZqOwJwhhjTFnWSG2MMaYsCxDGGGPKsgBhjDGmLAsQxhhjyrIAYYwxpqz/B1vFcb7qCMBqAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "sns.lineplot(data=df, x='train_pos', y='mean_auc_test')\n", + "plt.axvline(x=1024, linestyle='dashed', color='red')\n", + "plt.xlabel('Number of training samples')\n", + "plt.ylabel('ROC AUC')\n", + "plt.ylim((0.715,0.79))\n", + "tikzplotlib.save(f'model_generalization.tex', axis_height='6cm', axis_width='6cm')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}