Training in progress, step 1000

cmd.txt

@@ -0,0 +1 @@
+/var/spool/slurmd/job117568/slurm_script 05-06_03-05

commit.txt

@@ -0,0 +1,5 @@
+commit 001432feaa4bcad3e709036376d5dd95b14abfc0
+Author: Shteyman <dshteyma@isl-iam1.rr.intel.com>
+Date:   Wed Jun 5 02:59:17 2024 -0700
+
+     unnecessary file delete

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "JackFram/llama-68m",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 12,
+  "pad_token_id": 1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.41.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

events.out.tfevents.1717581940.isl-gpu3.8841.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa74e7b5cfe6aa38454b4aec704fd8c38a85af75b76f4b1763b5255bc48cb0ee
+size 5694

experiment_code/config/config1.yaml

@@ -0,0 +1,28 @@
+config_name: "JackFram/llama-68m"
+tokenizer_name: "JackFram/llama-68m"
+validation_split_percentage: 2
+train_file: "/home/dshteyma/shareGPT_data/ShareGPT_V3_unfiltered_cleaned_split.json"
+dataset_name_hub: "anon8231489123/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json"
+dataset_name_local: "ShareGPT"
+# max_train_samples: 1000
+# max_eval_samples: 10
+do_train: True
+do_eval: True
+output_dir: "/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs"
+overwrite_output_dir: True
+per_device_train_batch_size: 4
+gradient_accumulation_steps: 1
+report_to: "tensorboard"
+logging_dir: "/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs"
+logging_steps: 500
+save_steps: 1000
+eval_strategy: "steps"
+eval_steps: 1000
+learning_rate: 0.0001
+gradient_accumulation_steps: 1
+weight_decay: 0.01
+warmup_ratio: 0.05
+push_to_hub: True
+hub_model_id: "DorinSht/ShareGPT_llama2_68M"
+hub_strategy: "all_checkpoints"
+

experiment_code/config/config_redpajama.yaml

@@ -0,0 +1,27 @@
+config_name: "JackFram/llama-68m"
+tokenizer_name: "JackFram/llama-68m"
+validation_split_percentage: 2
+train_file: "/home/dshteyma/target_draft_coupling_code/dataset_dict.json"
+dataset_name_local: "RedPajama"
+dataset_name: "togethercomputer/RedPajama-Data-1T-Sample"
+dataset_name_hub: "togethercomputer/RedPajama-Data-1T-Sample"
+# max_train_samples: 1000
+# max_eval_samples: 10
+do_train: True
+do_eval: True
+output_dir: "/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs"
+overwrite_output_dir: True
+per_device_train_batch_size: 4
+gradient_accumulation_steps: 3
+report_to: "tensorboard"
+logging_dir: "/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs"
+logging_steps: 10000
+save_steps: 10000
+eval_strategy: "steps"
+eval_steps: 10000
+learning_rate: 0.0001
+weight_decay: 0.01
+warmup_ratio: 0.05
+push_to_hub: False
+hub_model_id: "DorinSht/llama_68M_redpajama"
+hub_strategy: "all_checkpoints"

experiment_code/prepare_sharegpt.py

@@ -0,0 +1,44 @@
+"""
+This script is largely copied from the Vicuna repo: https://github.com/lm-sys/FastChat/blob/main/fastchat/data/split_long_conversation.py
+We fixed a bug in `split_one_sample`, which previously includes long conversations in the processed data. Now we skip these long conversations.
+"""
+import argparse
+from concurrent.futures import ProcessPoolExecutor
+import json
+import transformers
+from tqdm import tqdm
+
+def shareGPT_pipeline(tokenizer, raw_datasets, overwrite_cache):
+    
+    def preprocess_conversation(convo):
+        key_mapping = {"role" : "from", "content" : "value"}
+        value_mapping = {"user" : "user", "human" : "user", "gpt" : "assistant", 'system': 'assitant', 'bing': 'assitant', 'chatgpt': 'assitant', 'bard': 'assitant'}
+        # mapping = {"human" : "user", "gpt" : "assitant"}
+        if value_mapping[convo[0][key_mapping['role']]] != 'user': 
+            convo = convo[1:]
+        preproc_convos_user = [{"role": 'user', "content": convo_elem[key_mapping['content']]} for i, convo_elem in enumerate(convo) if (i % 2 == 0 and value_mapping[convo_elem[key_mapping['role']]] == 'user')]
+        preproc_convos_assistant = [{"role": 'assistant', "content": convo_elem[key_mapping['content']]} for i, convo_elem in enumerate(convo) if (i % 2 == 1 and value_mapping[convo_elem[key_mapping['role']]] == 'assistant')]
+        if len(preproc_convos_user) != len(preproc_convos_assistant):
+            return []
+        preproc_convos = [conv_elem for pair in zip(preproc_convos_user, preproc_convos_assistant) for conv_elem in pair]
+        return preproc_convos
+
+    def filter_incorrect_conversations(examples):
+        convos = examples["conversations"]
+        ids_to_remove = [True if preprocess_conversation(convo) == [] else False for convo in convos]
+        return { "ids_to_remove" : ids_to_remove, }
+
+    def formatting_prompts_func(examples):
+        convos = examples["conversations"]
+        # preproc_convos = [convo for convo in convos if (convo[0]['from'] == 'human' or convo[0]['from'] == 'user')]
+        preproc_convos = [preprocess_conversation(convo) for convo in convos]
+        # preproc_convos2 = [preproc_convo for preproc_convo in preproc_convos if preproc_convo[0]['role'] == 'user']
+        texts = [tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = False) for i, convo in enumerate(preproc_convos)]
+        return { "text" : texts,}
+
+    filtered_datasets = raw_datasets.filter(lambda example: example['conversations'] != [], load_from_cache_file=not overwrite_cache,)
+    dataset = filtered_datasets.map(filter_incorrect_conversations, batched = True, load_from_cache_file=not overwrite_cache,)
+    filtered_datasets2 = dataset.filter(lambda example: example['ids_to_remove'] == False, load_from_cache_file=not overwrite_cache,)
+    raw_datasets_preprocessed = filtered_datasets2.map(formatting_prompts_func, batched = True, load_from_cache_file=not overwrite_cache,)    
+    
+    return raw_datasets_preprocessed

experiment_code/requirements.txt

@@ -0,0 +1,2 @@
+huggingface-hub==0.22.2
+-e git+https://github.com/huggingface/transformers.git@bbaa8ceff696c479aecdb4575b2deb1349efd3aa#egg=transformers

experiment_code/run_clm.py

@@ -0,0 +1,754 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset.
+
+Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
+https://huggingface.co/models?filter=text-generation
+"""
+# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.
+import random
+import logging
+import math
+import os
+from datetime import datetime
+import sys
+import warnings
+from dataclasses import dataclass, field
+from itertools import chain
+from typing import Optional
+import datasets
+import evaluate
+import torch
+from datasets import load_dataset
+import argparse
+import transformers
+from prepare_sharegpt import shareGPT_pipeline
+from transformers import (
+    CONFIG_MAPPING,
+    MODEL_FOR_CAUSAL_LM_MAPPING,
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    HfArgumentParser,
+    Trainer,
+    TrainingArguments,
+    default_data_collator,
+    set_seed,
+)
+from transformers.testing_utils import CaptureLogger
+from transformers.trainer_utils import get_last_checkpoint
+from transformers.utils import check_min_version, send_example_telemetry
+from transformers.utils.versions import require_version
+from functools import partial
+
+from omegaconf import DictConfig, OmegaConf
+import hydra 
+
+# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
+check_min_version("4.41.0.dev0")
+
+require_version("datasets>=2.14.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt")
+
+logger = logging.getLogger(__name__)
+
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+
+random.seed(42)
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
+    """
+
+    model_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The model checkpoint for weights initialization. Don't set if you want to train a model from scratch."
+            )
+        },
+    )
+    model_type: Optional[str] = field(
+        default=None,
+        metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
+    )
+    padding_side: str = field(
+        default="right", metadata={"help": "The padding side in tokenizer"}
+    ) 
+    config_overrides: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override some existing default config settings when a model is trained from scratch. Example: "
+                "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
+            )
+        },
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    use_fast_tokenizer: bool = field(
+        default=True,
+        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
+    )
+    model_revision: str = field(
+        default="main",
+        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
+    )
+    token: str = field(
+        default=None,
+        metadata={
+            "help": (
+                "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token "
+                "generated when running `huggingface-cli login` (stored in `~/.huggingface`)."
+            )
+        },
+    )
+    use_auth_token: bool = field(
+        default=None,
+        metadata={
+            "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead."
+        },
+    )
+    trust_remote_code: bool = field(
+        default=True,
+        metadata={
+            "help": (
+                "Whether or not to allow for custom models defined on the Hub in their own modeling files. This option "
+                "should only be set to `True` for repositories you trust and in which you have read the code, as it will "
+                "execute code present on the Hub on your local machine."
+            )
+        },
+    )
+    torch_dtype: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the "
+                "dtype will be automatically derived from the model's weights."
+            ),
+            "choices": ["auto", "bfloat16", "float16", "float32"],
+        },
+    )
+    low_cpu_mem_usage: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded. "
+                "set True will benefit LLM loading time and RAM consumption."
+            )
+        },
+    )
+
+    def __post_init__(self):
+        if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
+            raise ValueError(
+                "--config_overrides can't be used in combination with --config_name or --model_name_or_path"
+            )
+            
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    """
+    dataset_name: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+    dataset_name_hub: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+    dataset_name_local: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset for identification."}
+    )
+    dataset_config_name: Optional[str] = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
+    validation_file: Optional[str] = field(
+        default=None,
+        metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
+    )
+    max_train_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of training examples to this "
+                "value if set."
+            )
+        },
+    )
+    max_eval_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
+                "value if set."
+            )
+        },
+    )
+    streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"})
+    block_size: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Optional input sequence length after tokenization. "
+                "The training dataset will be truncated in block of this size for training. "
+                "Default to the model max input length for single sentence inputs (take into account special tokens)."
+            )
+        },
+    )
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
+    )
+    validation_split_percentage: Optional[int] = field(
+        default=5,
+        metadata={
+            "help": "The percentage of the train set used as validation set in case there's no validation split"
+        },
+    )
+    preprocessing_num_workers: Optional[int] = field(
+        default=None,
+        metadata={"help": "The number of processes to use for the preprocessing."},
+    )
+    keep_linebreaks: bool = field(
+        default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
+    )
+    lazy_preprocess: bool = False
+
+    def __post_init__(self):
+        if self.streaming:
+            require_version("datasets>=2.0.0", "The streaming feature requires `datasets>=2.0.0`")
+
+        if self.dataset_name is None and self.train_file is None and self.validation_file is None:
+            raise ValueError("Need either a dataset name or a training/validation file.")
+        else:
+            if self.train_file is not None:
+                extension = self.train_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
+            if self.validation_file is not None:
+                extension = self.validation_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."   
+
+# @dataclass
+# class TrainingArguments(transformers.TrainingArguments):
+#     cache_dir: Optional[str] = field(default=None)
+#     optim: str = field(default="adamw_torch")
+#     model_max_length: int = field(
+#         default=2048,
+#         metadata={
+#             "help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)."
+#         },
+#     )
+
+def create_output_directory(dir_root_path):
+    # Get the current date and time
+    current_time = datetime.now()
+    # Format the date and time as a string
+    # Example format: YYYYMMDD_HHMMSS
+    formatted_time = current_time.strftime("%Y%m%d_%H%M%S")
+    # Define the directory name with the formatted time
+    directory_full_path = os.path.join(dir_root_path, f"training_outputs_{formatted_time}")
+    # Create the directory
+    os.makedirs(directory_full_path)
+    print(f"Directory '{directory_full_path}' created successfully.")
+    return directory_full_path
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+    parser = argparse.ArgumentParser(description="parser for arguments from .py script call")
+    parser.add_argument('--output_dir', type=str, help='Path for training_args.output_dir')
+    parser.add_argument('--logging_dir', type=str, help='Path for training_args.logging_dir')
+    parser.add_argument('--config_file', type=str, help='An additional required option.')
+    args = parser.parse_args()
+    
+    parser_hf = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+    if args.config_file is not None and args.output_dir is not None and args.output_dir is not None:
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser_hf.parse_yaml_file(args.config_file)
+        training_args.output_dir = args.output_dir
+        training_args.logging_dir = args.logging_dir
+    else:
+        # use the preset config file defined in the slurm .sh script
+        # model_args, data_args, training_args = parser_hf.parse_yaml_file(os.getenv("DEFAULT_CONFIG_FILE"))
+        model_args, data_args, training_args = parser_hf.parse_yaml_file('./config/config1.yaml')
+
+
+    if model_args.use_auth_token is not None:
+        warnings.warn(
+            "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead.",
+            FutureWarning,
+        )
+        if model_args.token is not None:
+            raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.")
+        model_args.token = model_args.use_auth_token
+
+    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
+    # information sent is the one passed as arguments along with your Python/PyTorch versions.
+    send_example_telemetry("run_clm", model_args, data_args)
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+
+    if training_args.should_log:
+        # The default of training_args.log_level is passive, so we set log level at info here to have that default.
+        transformers.utils.logging.set_verbosity_info()
+
+    log_level = training_args.get_process_log_level()
+    logger.setLevel(log_level)
+    datasets.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.enable_default_handler()
+    transformers.utils.logging.enable_explicit_format()
+
+    # Log on each process the small summary:
+    logger.warning(
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, "
+        + f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}"
+    )
+    logger.info(f"Training/evaluation parameters {training_args}")
+
+    # Detecting last checkpoint.
+    last_checkpoint = None
+    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
+        last_checkpoint = get_last_checkpoint(training_args.output_dir)
+        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
+            raise ValueError(
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Use --overwrite_output_dir to overcome."
+            )
+        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
+            logger.info(
+                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
+                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
+            )
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
+    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
+    # (the dataset will be downloaded automatically from the datasets Hub).
+    #
+    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
+    # 'text' is found. You can easily tweak this behavior (see below).
+    #
+    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
+    # download the dataset.
+    if data_args.dataset_name is not None:
+        # Downloading and loading a dataset from the hub.
+        raw_datasets = load_dataset(
+            data_args.dataset_name,
+            data_args.dataset_config_name,
+            cache_dir=model_args.cache_dir,
+            token=model_args.token,
+            streaming=data_args.streaming,
+        )
+        if "validation" not in raw_datasets.keys():
+            raw_datasets["validation"] = load_dataset(
+                data_args.dataset_name,
+                data_args.dataset_config_name,
+                split=f"train[:{data_args.validation_split_percentage}%]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                streaming=data_args.streaming,
+            )
+            raw_datasets["train"] = load_dataset(
+                data_args.dataset_name,
+                data_args.dataset_config_name,
+                split=f"train[{data_args.validation_split_percentage}%:]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                streaming=data_args.streaming,
+            )
+    else:
+        data_files = {}
+        dataset_args = {}
+        if data_args.train_file is not None:
+            data_files["train"] = data_args.train_file
+        if data_args.validation_file is not None:
+            data_files["validation"] = data_args.validation_file
+        extension = (
+            data_args.train_file.split(".")[-1]
+            if data_args.train_file is not None
+            else data_args.validation_file.split(".")[-1]
+        )
+        if extension == "txt":
+            extension = "text"
+            dataset_args["keep_linebreaks"] = data_args.keep_linebreaks
+        raw_datasets = load_dataset(
+            extension,
+            data_files=data_files,
+            cache_dir=model_args.cache_dir,
+            token=model_args.token,
+            **dataset_args,
+        )
+        # If no validation data is there, validation_split_percentage will be used to divide the dataset.
+        if "validation" not in raw_datasets.keys():
+            raw_datasets["validation"] = load_dataset(
+                extension,
+                data_files=data_files,
+                split=f"train[:{data_args.validation_split_percentage}%]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                **dataset_args,
+            )
+            raw_datasets["train"] = load_dataset(
+                extension,
+                data_files=data_files,
+                split=f"train[{data_args.validation_split_percentage}%:]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                **dataset_args,
+            )
+    
+    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
+    # https://huggingface.co/docs/datasets/loading_datasets.
+
+    # Load pretrained model and tokenizer
+    #
+    # Distributed training:
+    # The .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+
+    config_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "revision": model_args.model_revision,
+        "token": model_args.token,
+        "trust_remote_code": model_args.trust_remote_code,
+    }
+    if model_args.config_name:
+        config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)
+    elif model_args.model_name_or_path:
+        config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
+    else:
+        config = CONFIG_MAPPING[model_args.model_type]()
+        logger.warning("You are instantiating a new config instance from scratch.")
+        if model_args.config_overrides is not None:
+            logger.info(f"Overriding config: {model_args.config_overrides}")
+            config.update_from_string(model_args.config_overrides)
+            logger.info(f"New config: {config}")
+
+    tokenizer_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "use_fast": model_args.use_fast_tokenizer,
+        "revision": model_args.model_revision,
+        "token": model_args.token,
+        "padding": 'max_length',
+        "trust_remote_code": model_args.trust_remote_code,
+        "model_max_length": config.max_position_embeddings,
+        "return_tensors":'pt'
+    }
+    if model_args.tokenizer_name:
+        tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)
+    elif model_args.model_name_or_path:
+        tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs)
+    else:
+        raise ValueError(
+            "You are instantiating a new tokenizer from scratch. This is not supported by this script. "
+            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
+        )
+    if tokenizer.pad_token != tokenizer.unk_token:
+        tokenizer.pad_token = tokenizer.unk_token
+
+    if model_args.model_name_or_path:
+        torch_dtype = (
+            model_args.torch_dtype
+            if model_args.torch_dtype in ["auto", None]
+            else getattr(torch, model_args.torch_dtype)
+        )
+        model = AutoModelForCausalLM.from_pretrained(
+            model_args.model_name_or_path,
+            from_tf=bool(".ckpt" in model_args.model_name_or_path),
+            config=config,
+            cache_dir=model_args.cache_dir,
+            revision=model_args.model_revision,
+            token=model_args.token,
+            trust_remote_code=model_args.trust_remote_code,
+            torch_dtype=torch_dtype,
+            low_cpu_mem_usage=model_args.low_cpu_mem_usage,
+        )
+    else:
+        model = AutoModelForCausalLM.from_config(config, trust_remote_code=model_args.trust_remote_code)
+        n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
+        logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
+
+    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch
+    # on a small vocab and want a smaller embedding size, remove this test.
+    embedding_size = model.get_input_embeddings().weight.shape[0]
+    if len(tokenizer) > embedding_size:
+        model.resize_token_embeddings(len(tokenizer))    
+    
+    if "ShareGPT" == data_args.dataset_name_local:
+        raw_datasets_preprocessed = shareGPT_pipeline(tokenizer=tokenizer, raw_datasets=raw_datasets, overwrite_cache=data_args.overwrite_cache) 
+    if "RedPajama" == data_args.dataset_name_local:
+        raw_datasets_preprocessed = raw_datasets
+    
+### HEREE
+    # Preprocessing the datasets.
+    # First we tokenize all the texts.
+    if training_args.do_train:
+        column_names = list(raw_datasets_preprocessed["train"].features)
+    else:
+        column_names = list(raw_datasets_preprocessed["validation"].features)
+    text_column_name = "text"
+
+
+    # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function
+    tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base")
+    
+    def tokenize_function(examples):
+        with CaptureLogger(tok_logger) as cl:
+            # print(tokenizer(examples[text_column_name]))
+            # output = tokenizer(examples[text_column_name])
+            output = tokenizer(
+                            examples[text_column_name],
+                            return_tensors="pt",
+                            padding="max_length",
+                            max_length=tokenizer.model_max_length,
+                            truncation=True,
+                        )
+            # output = input_ids.clone() 
+        # clm input could be much much longer than block_size
+        if "Token indices sequence length is longer than the" in cl.out:
+            tok_logger.warning(
+                "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits"
+                " before being passed to the model."
+            )
+        return output
+    
+    with training_args.main_process_first(desc="dataset map tokenization"):
+        if not data_args.streaming:
+            tokenized_datasets = raw_datasets_preprocessed.map(
+                tokenize_function,
+                batched=True,
+                num_proc=data_args.preprocessing_num_workers,
+                remove_columns=column_names,
+                load_from_cache_file=not data_args.overwrite_cache,
+                desc="Running tokenizer on dataset",
+            )
+        else:
+            tokenized_datasets = raw_datasets_preprocessed.map(
+                tokenize_function,
+                batched=True,
+                remove_columns=column_names,
+                load_from_cache_file=not data_args.overwrite_cache,
+            )
+    if hasattr(config, "max_position_embeddings"):
+        max_pos_embeddings = config.max_position_embeddings
+    else:
+        # Define a default value if the attribute is missing in the config.
+        max_pos_embeddings = 1024
+
+    if data_args.block_size is None:
+        block_size = tokenizer.model_max_length
+        if block_size > max_pos_embeddings:
+            logger.warning(
+                f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). "
+                f"Using block_size={min(1024, max_pos_embeddings)} instead. You can change that default value by passing --block_size xxx."
+            )
+            if max_pos_embeddings > 0:
+                block_size = min(1024, max_pos_embeddings)
+            else:
+                block_size = 1024
+    else:
+        if data_args.block_size > tokenizer.model_max_length:
+            logger.warning(
+                f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model "
+                f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}."
+            )
+        block_size = min(data_args.block_size, tokenizer.model_max_length)
+
+    # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.
+    def group_texts(examples):
+        # Concatenate all texts.
+        concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}
+        total_length = len(concatenated_examples[list(examples.keys())[0]])
+        # We drop the small remainder, and if the total_length < block_size  we exclude this batch and return an empty dict.
+        # We could add padding if the model supported it instead of this drop, you can customize this part to your needs.
+        total_length = (total_length // block_size) * block_size
+        # Split by chunks of max_len.
+        result = {
+            k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
+            for k, t in concatenated_examples.items()
+        }
+        result["labels"] = result["input_ids"].copy()
+        return result
+
+    # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder
+    # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower
+    # to preprocess.
+    #
+    # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
+    # https://huggingface.co/docs/datasets/process#map
+
+    with training_args.main_process_first(desc="grouping texts together"):
+        if not data_args.streaming:
+            lm_datasets = tokenized_datasets.map(
+                group_texts,
+                batched=True,
+                num_proc=data_args.preprocessing_num_workers,
+                load_from_cache_file=not data_args.overwrite_cache,
+                desc=f"Grouping texts in chunks of {block_size}",
+            )
+        else:
+            lm_datasets = tokenized_datasets.map(
+                group_texts,
+                batched=True,
+                load_from_cache_file=not data_args.overwrite_cache,
+            )
+
+    if training_args.do_train:
+        if "train" not in tokenized_datasets:
+            raise ValueError("--do_train requires a train dataset")
+        train_dataset = lm_datasets["train"]
+        if data_args.max_train_samples is not None:
+            max_train_samples = min(len(train_dataset), data_args.max_train_samples)
+            train_dataset = train_dataset.select(range(max_train_samples))
+
+    if training_args.do_eval:
+        if "validation" not in tokenized_datasets:
+            raise ValueError("--do_eval requires a validation dataset")
+        eval_dataset = lm_datasets["validation"]
+        if data_args.max_eval_samples is not None:
+            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
+            eval_dataset = eval_dataset.select(range(max_eval_samples))
+
+        def preprocess_logits_for_metrics(logits, labels):
+            if isinstance(logits, tuple):
+                # Depending on the model and config, logits may contain extra tensors,
+                # like past_key_values, but logits always come first
+                logits = logits[0]
+            return logits.argmax(dim=-1)
+
+        
+        def compute_metrics(eval_preds):
+            accuracy = evaluate.load("accuracy", cache_dir=model_args.cache_dir)
+            perplexity = evaluate.load("perplexity", module_type="metric")
+            preds, labels = eval_preds
+            # preds have the same shape as the labels, after the argmax(-1) has been calculated
+            # by preprocess_logits_for_metrics but we need to shift the labels
+            labels = labels[:, 1:].reshape(-1)
+            preds = preds[:, :-1].reshape(-1)
+            accuracy = accuracy.compute(predictions=preds, references=labels)
+            # perplexity = perplexity.compute(predictions=preds, model_id='llama')
+            return accuracy
+
+    # Initialize the optimizer
+    optimizer = torch.optim.AdamW(model.parameters(), lr=training_args.learning_rate, weight_decay=training_args.weight_decay)
+    # Calculate the number of training steps
+    train_steps = (len(train_dataset) // (training_args.per_device_train_batch_size * training_args._n_gpu)) * training_args.num_train_epochs
+
+    # Initialize the scheduler
+    linear_scheduler = transformers.get_linear_schedule_with_warmup(
+        optimizer,
+        num_warmup_steps=train_steps*training_args.warmup_ratio,
+        num_training_steps=train_steps
+    )
+        
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset if training_args.do_train else None,
+        eval_dataset=eval_dataset if training_args.do_eval else None,
+        tokenizer=tokenizer,
+        optimizers=(optimizer, linear_scheduler),
+        # Data collator will default to DataCollatorWithPadding, so we change it.
+        data_collator=default_data_collator,
+        compute_metrics=compute_metrics if training_args.do_eval else None,
+        preprocess_logits_for_metrics=preprocess_logits_for_metrics
+        if training_args.do_eval else None,
+    )
+
+    # Training
+    if training_args.do_train:
+        checkpoint = None
+        if training_args.resume_from_checkpoint is not None:
+            checkpoint = training_args.resume_from_checkpoint
+        elif last_checkpoint is not None:
+            checkpoint = last_checkpoint
+        train_result = trainer.train(resume_from_checkpoint=checkpoint)
+        trainer.save_model()  # Saves the tokenizer too for easy upload
+
+        metrics = train_result.metrics
+
+        max_train_samples = (
+            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
+        )
+        metrics["train_samples"] = min(max_train_samples, len(train_dataset))
+
+        trainer.log_metrics("train", metrics)
+        trainer.save_metrics("train", metrics)
+        trainer.save_state()
+        try:
+            torch.save([vars(a) for a in [training_args, data_args, model_args]], os.path.join(training_args.output_dir, "args.bin"))
+        except:
+            logger.info("Failed to save arguments")
+
+    # Evaluation
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+
+        metrics = trainer.evaluate()
+
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
+        try:
+            perplexity = math.exp(metrics["eval_loss"])
+        except OverflowError:
+            perplexity = float("inf")
+        metrics["perplexity"] = perplexity
+
+        trainer.log_metrics("eval", metrics)
+        trainer.save_metrics("eval", metrics)
+
+    kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-generation"}
+    if data_args.dataset_name is not None:
+        kwargs["dataset_tags"] = data_args.dataset_name
+        if data_args.dataset_config_name is not None:
+            kwargs["dataset_args"] = data_args.dataset_config_name
+            kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}"
+        else:
+            kwargs["dataset"] = data_args.dataset_name
+    elif data_args.dataset_name_hub is not None:
+        kwargs["dataset"] = data_args.dataset_name_hub
+
+    if training_args.push_to_hub:
+        trainer.push_to_hub(**kwargs)
+    else:
+        trainer.create_model_card(**kwargs)
+
+if __name__ == "__main__":
+    main()

experiment_code/submit_job.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+#SBATCH -p g24
+#SBATCH --job-name=myjob_shareGPT
+#SBATCH --qos=high
+#SBATCH --nodes=1                 # Number of nodes
+#SBATCH --ntasks=1         # Number of tasks (one for each script)
+#SBATCH --cpus-per-task=60
+#SBATCH --gres=gpu:6
+#SBATCH --array=1-1                      # Array range
+# #SBATCH --output=./slurm_outputs/run_clm_job_%A_task_%a.out  # Standard output
+#SBATCH --output=/dev/null     # Discard standard output  # Because we write to the log.txt file
+
+# # Get the current date and time
+current_time=$(date +"%d-%m_%H-%M")
+OUTPUT_DIR="./training_outputs_job_${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}_${current_time}"
+export DEFAULT_CONFIG_FILE="./config/config1.yaml"
+
+while test $# -gt 0; do
+    echo $1
+    case "$1" in 
+        --output_dir)
+            shift
+            OUTPUT_DIR=$1
+            shift
+            ;;
+    esac
+done
+
+mkdir_is_exists() {
+    if [ -d "$1" ]; then
+        echo "Directory '$1' already exists."
+    else
+        mkdir -p "$1"
+        echo "Directory '$1' created."
+    fi
+}
+
+
+mkdir_is_exists $OUTPUT_DIR
+mkdir_is_exists $OUTPUT_DIR/experiment_code
+git log -n 1 > $OUTPUT_DIR/commit.txt
+pip freeze > $OUTPUT_DIR/pip_freeze.txt
+echo $0 $ARGS $current_time > $OUTPUT_DIR/cmd.txt
+cp -r ./run_clm.py $OUTPUT_DIR/experiment_code
+cp -r ./prepare_sharegpt.py $OUTPUT_DIR/experiment_code
+cp -r config $OUTPUT_DIR/experiment_code
+cp -r ./submit_job.sh $OUTPUT_DIR/experiment_code
+cp -r ./requirements.txt $OUTPUT_DIR/experiment_code
+
+# Define the Python scripts and their corresponding input files
+declare -A scripts_and_inputs=(
+        ["1"]="./config/config1.yaml"
+    #  ["2"]="./config/config_redpajama.yaml"
+    # ["3"]="./config/config1.yaml"
+    # ["4"]="./config/config1.yaml"
+    # ["5"]="./config/config1.yaml"
+)
+
+# Launch each script with its corresponding input file as a separate task
+echo "Starting job array task: $SLURM_ARRAY_TASK_ID"
+PARAMS="--output_dir $OUTPUT_DIR --logging_dir $OUTPUT_DIR --config_file ${scripts_and_inputs[$SLURM_ARRAY_TASK_ID]}"
+
+srun --exclusive python run_clm.py $PARAMS 2>&1 | tee $OUTPUT_DIR/log.txt
+
+
+# Wait for all background jobs to complete
+wait
+
+# Print a message indicating completion
+echo "All Python scripts have been executed."

log.txt

@@ -0,0 +1,306 @@
+2024-06-05 03:05:23.315324: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
+2024-06-05 03:05:23.315399: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
+2024-06-05 03:05:23.317038: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
+2024-06-05 03:05:23.326431: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
+To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2024-06-05 03:05:26.483871: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
+06/05/2024 03:05:35 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 6, distributed training: False, 16-bits training: False
+06/05/2024 03:05:35 - INFO - __main__ - Training/evaluation parameters TrainingArguments(
+_n_gpu=6,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=True,
+do_predict=False,
+do_train=True,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=1000,
+eval_strategy=steps,
+evaluation_strategy=None,
+fp16=False,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=1,
+gradient_checkpointing=False,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=DorinSht/ShareGPT_llama2_68M,
+hub_private_repo=False,
+hub_strategy=all_checkpoints,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0001,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=./training_outputs_job_117568_1_05-06_03-05,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=500,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=linear,
+max_grad_norm=1.0,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=3.0,
+optim=adamw_torch,
+optim_args=None,
+optim_target_modules=None,
+output_dir=./training_outputs_job_117568_1_05-06_03-05,
+overwrite_output_dir=True,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=4,
+prediction_loss_only=False,
+push_to_hub=True,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=True,
+report_to=['tensorboard'],
+restore_callback_states_from_checkpoint=False,
+resume_from_checkpoint=None,
+run_name=/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=1000,
+save_strategy=steps,
+save_total_limit=None,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.05,
+warmup_steps=0,
+weight_decay=0.01,
+)
+Using custom data configuration default-afe4b27d28cbdcb1
+06/05/2024 03:05:35 - INFO - datasets.builder - Using custom data configuration default-afe4b27d28cbdcb1
+Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+06/05/2024 03:05:35 - INFO - datasets.info - Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+Overwrite dataset info from restored data version if exists.
+06/05/2024 03:05:36 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists.
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/05/2024 03:05:36 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/05/2024 03:05:36 - INFO - datasets.builder - Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/05/2024 03:05:36 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+Using custom data configuration default-afe4b27d28cbdcb1
+06/05/2024 03:05:36 - INFO - datasets.builder - Using custom data configuration default-afe4b27d28cbdcb1
+Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+06/05/2024 03:05:36 - INFO - datasets.info - Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+06/05/2024 03:05:36 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists.
+Overwrite dataset info from restored data version if exists.
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/05/2024 03:05:36 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+06/05/2024 03:05:36 - INFO - datasets.builder - Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/05/2024 03:05:36 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+Using custom data configuration default-afe4b27d28cbdcb1
+06/05/2024 03:05:37 - INFO - datasets.builder - Using custom data configuration default-afe4b27d28cbdcb1
+Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+06/05/2024 03:05:37 - INFO - datasets.info - Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+Overwrite dataset info from restored data version if exists.
+06/05/2024 03:05:37 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists.
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/05/2024 03:05:37 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/05/2024 03:05:37 - INFO - datasets.builder - Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/05/2024 03:05:37 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+[INFO|configuration_utils.py:726] 2024-06-05 03:05:37,626 >> loading configuration file config.json from cache at /home/dshteyma/.cache/huggingface/hub/models--JackFram--llama-68m/snapshots/964a5d77df908b69f8d6476fb70e940425b04cb5/config.json
+[INFO|configuration_utils.py:789] 2024-06-05 03:05:37,628 >> Model config LlamaConfig {
+  "_name_or_path": "JackFram/llama-68m",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 12,
+  "pad_token_id": 1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.41.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+[INFO|tokenization_utils_base.py:2102] 2024-06-05 03:05:37,771 >> loading file tokenizer.model from cache at /home/dshteyma/.cache/huggingface/hub/models--JackFram--llama-68m/snapshots/964a5d77df908b69f8d6476fb70e940425b04cb5/tokenizer.model
+[INFO|tokenization_utils_base.py:2102] 2024-06-05 03:05:37,771 >> loading file tokenizer.json from cache at None
+[INFO|tokenization_utils_base.py:2102] 2024-06-05 03:05:37,771 >> loading file added_tokens.json from cache at None
+[INFO|tokenization_utils_base.py:2102] 2024-06-05 03:05:37,771 >> loading file special_tokens_map.json from cache at /home/dshteyma/.cache/huggingface/hub/models--JackFram--llama-68m/snapshots/964a5d77df908b69f8d6476fb70e940425b04cb5/special_tokens_map.json
+[INFO|tokenization_utils_base.py:2102] 2024-06-05 03:05:37,771 >> loading file tokenizer_config.json from cache at /home/dshteyma/.cache/huggingface/hub/models--JackFram--llama-68m/snapshots/964a5d77df908b69f8d6476fb70e940425b04cb5/tokenizer_config.json
+[WARNING|logging.py:329] 2024-06-05 03:05:37,772 >> You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+[WARNING|logging.py:329] 2024-06-05 03:05:37,868 >> You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+[INFO|configuration_utils.py:936] 2024-06-05 03:05:38,449 >> Generate config GenerationConfig {
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "pad_token_id": 1
+}
+
+06/05/2024 03:05:39 - INFO - __main__ - Training new model from scratch - Total size=64.88M params
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-988d048fea8d2473.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-988d048fea8d2473.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-4e281c930893bca9.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-4e281c930893bca9.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-3fe350bccdda6078.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-3fe350bccdda6078.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-35d09b588a0c62b9.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-35d09b588a0c62b9.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-4e5279ee31a5d8d3.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-4e5279ee31a5d8d3.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-63d56456928edd43.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-63d56456928edd43.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-6a784a78d9818240.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-6a784a78d9818240.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-46540f58a00a92bf.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-46540f58a00a92bf.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-73605724efaea9d2.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-73605724efaea9d2.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-83d3df87e1b82021.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-83d3df87e1b82021.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-efdbb02491aa6344.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-efdbb02491aa6344.arrow
+06/05/2024 03:05:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-0cf2ae38fef927f3.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-0cf2ae38fef927f3.arrow
+06/05/2024 03:05:39 - WARNING - accelerate.utils.other - Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
+[INFO|trainer.py:2068] 2024-06-05 03:05:40,316 >> ***** Running training *****
+[INFO|trainer.py:2069] 2024-06-05 03:05:40,316 >>   Num examples = 90,745
+[INFO|trainer.py:2070] 2024-06-05 03:05:40,316 >>   Num Epochs = 3
+[INFO|trainer.py:2071] 2024-06-05 03:05:40,316 >>   Instantaneous batch size per device = 4
+[INFO|trainer.py:2073] 2024-06-05 03:05:40,316 >>   Training with DataParallel so batch size has been adjusted to: 24
+[INFO|trainer.py:2074] 2024-06-05 03:05:40,316 >>   Total train batch size (w. parallel, distributed & accumulation) = 24
+[INFO|trainer.py:2075] 2024-06-05 03:05:40,316 >>   Gradient Accumulation steps = 1
+[INFO|trainer.py:2076] 2024-06-05 03:05:40,316 >>   Total optimization steps = 11,346
+[INFO|trainer.py:2077] 2024-06-05 03:05:40,317 >>   Number of trainable parameters = 68,030,208
+
  0%|          | 0/11346 [00:00<?, ?it/s]/home/dshteyma/miniconda3/lib/python3.9/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
+  warnings.warn('Was asked to gather along dimension 0, but all '
+
  0%|          | 1/11346 [00:04<15:15:06,  4.84s/it]
  0%|          | 2/11346 [00:05<8:04:02,  2.56s/it] 
  0%|          | 3/11346 [00:06<5:46:22,  1.83s/it]
  0%|          | 4/11346 [00:07<4:41:45,  1.49s/it]
  0%|          | 5/11346 [00:08<4:06:03,  1.30s/it]
  0%|          | 6/11346 [00:09<3:44:22,  1.19s/it]
  0%|          | 7/11346 [00:10<3:30:41,  1.11s/it]
  0%|          | 8/11346 [00:11<3:21:54,  1.07s/it]
  0%|          | 9/11346 [00:12<3:15:57,  1.04s/it]
  0%|          | 10/11346 [00:13<3:11:49,  1.02s/it]
  0%|          | 11/11346 [00:14<3:09:05,  1.00s/it]
  0%|          | 12/11346 [00:15<3:07:02,  1.01it/s]
  0%|          | 13/11346 [00:16<3:05:37,  1.02it/s]
  0%|          | 14/11346 [00:17<3:04:44,  1.02it/s]
  0%|          | 15/11346 [00:18<3:04:01,  1.03it/s]
  0%|          | 16/11346 [00:19<3:03:31,  1.03it/s]
  0%|          | 17/11346 [00:20<3:04:21,  1.02it/s]
  0%|          | 18/11346 [00:21<3:06:59,  1.01it/s]
  0%|          | 19/11346 [00:22<3:05:57,  1.02it/s]
  0%|          | 20/11346 [00:23<3:05:22,  1.02it/s]
  0%|          | 21/11346 [00:24<3:04:28,  1.02it/s]
  0%|          | 22/11346 [00:25<3:03:54,  1.03it/s]
  0%|          | 23/11346 [00:26<3:03:38,  1.03it/s]
  0%|          | 24/11346 [00:27<3:03:17,  1.03it/s]
  0%|          | 25/11346 [00:28<3:03:02,  1.03it/s]
  0%|          | 26/11346 [00:29<3:02:51,  1.03it/s]
  0%|          | 27/11346 [00:30<3:02:38,  1.03it/s]
  0%|          | 28/11346 [00:31<3:02:32,  1.03it/s]
  0%|          | 29/11346 [00:32<3:02:30,  1.03it/s]
  0%|          | 30/11346 [00:32<3:02:27,  1.03it/s]
  0%|          | 31/11346 [00:33<3:02:27,  1.03it/s]
  0%|          | 32/11346 [00:34<3:02:28,  1.03it/s]
  0%|          | 33/11346 [00:35<3:02:33,  1.03it/s]
  0%|          | 34/11346 [00:36<3:02:39,  1.03it/s]
  0%|          | 35/11346 [00:37<3:02:48,  1.03it/s]
  0%|          | 36/11346 [00:38<3:02:46,  1.03it/s]
  0%|          | 37/11346 [00:39<3:02:52,  1.03it/s]
  0%|          | 38/11346 [00:40<3:02:45,  1.03it/s]
  0%|          | 39/11346 [00:41<3:02:40,  1.03it/s]
  0%|          | 40/11346 [00:42<3:02:37,  1.03it/s]
  0%|          | 41/11346 [00:43<3:02:27,  1.03it/s]
  0%|          | 42/11346 [00:44<3:02:25,  1.03it/s]
  0%|          | 43/11346 [00:45<3:02:24,  1.03it/s]
  0%|          | 44/11346 [00:46<3:02:22,  1.03it/s]
  0%|          | 45/11346 [00:47<3:02:21,  1.03it/s]
  0%|          | 46/11346 [00:48<3:02:27,  1.03it/s]
  0%|          | 47/11346 [00:49<3:02:26,  1.03it/s]
  0%|          | 48/11346 [00:50<3:02:23,  1.03it/s]
  0%|          | 49/11346 [00:51<3:02:30,  1.03it/s]
  0%|          | 50/11346 [00:52<3:02:36,  1.03it/s]
  0%|          | 51/11346 [00:53<3:02:38,  1.03it/s]
  0%|          | 52/11346 [00:54<3:02:47,  1.03it/s]
  0%|          | 53/11346 [00:55<3:02:52,  1.03it/s]
  0%|          | 54/11346 [00:56<3:02:47,  1.03it/s]
  0%|          | 55/11346 [00:57<3:02:48,  1.03it/s]
  0%|          | 56/11346 [00:58<3:02:50,  1.03it/s]
  1%|          | 57/11346 [00:59<3:02:51,  1.03it/s]
  1%|          | 58/11346 [01:00<3:02:49,  1.03it/s]
  1%|          | 59/11346 [01:01<3:02:35,  1.03it/s]
  1%|          | 60/11346 [01:02<3:02:36,  1.03it/s]
  1%|          | 61/11346 [01:03<3:02:29,  1.03it/s]
  1%|          | 62/11346 [01:04<3:02:34,  1.03it/s]
  1%|          | 63/11346 [01:04<3:02:34,  1.03it/s]
  1%|          | 64/11346 [01:05<3:02:36,  1.03it/s]
  1%|          | 65/11346 [01:06<3:02:35,  1.03it/s]
  1%|          | 66/11346 [01:07<3:02:31,  1.03it/s]
  1%|          | 67/11346 [01:08<3:02:33,  1.03it/s]
  1%|          | 68/11346 [01:09<3:02:32,  1.03it/s]
  1%|          | 69/11346 [01:10<3:02:34,  1.03it/s]
  1%|          | 70/11346 [01:11<3:02:37,  1.03it/s]
  1%|          | 71/11346 [01:12<3:02:31,  1.03it/s]
  1%|          | 72/11346 [01:13<3:02:35,  1.03it/s]
  1%|          | 73/11346 [01:14<3:02:39,  1.03it/s]
  1%|          | 74/11346 [01:15<3:02:39,  1.03it/s]
  1%|          | 75/11346 [01:16<3:02:28,  1.03it/s]
  1%|          | 76/11346 [01:17<3:02:26,  1.03it/s]
  1%|          | 77/11346 [01:18<3:02:26,  1.03it/s]
  1%|          | 78/11346 [01:19<3:02:28,  1.03it/s]
  1%|          | 79/11346 [01:20<3:02:26,  1.03it/s]
  1%|          | 80/11346 [01:21<3:02:30,  1.03it/s]
  1%|          | 81/11346 [01:22<3:02:28,  1.03it/s]
  1%|          | 82/11346 [01:23<3:02:30,  1.03it/s]
  1%|          | 83/11346 [01:24<3:02:24,  1.03it/s]
  1%|          | 84/11346 [01:25<3:02:28,  1.03it/s]
  1%|          | 85/11346 [01:26<3:02:25,  1.03it/s]
  1%|          | 86/11346 [01:27<3:02:27,  1.03it/s]
  1%|          | 87/11346 [01:28<3:02:31,  1.03it/s]
  1%|          | 88/11346 [01:29<3:02:34,  1.03it/s]
  1%|          | 89/11346 [01:30<3:02:35,  1.03it/s]
  1%|          | 90/11346 [01:31<3:02:33,  1.03it/s]
  1%|          | 91/11346 [01:32<3:02:30,  1.03it/s]
  1%|          | 92/11346 [01:33<3:02:28,  1.03it/s]
  1%|          | 93/11346 [01:34<3:02:18,  1.03it/s]
  1%|          | 94/11346 [01:35<3:02:18,  1.03it/s]
  1%|          | 95/11346 [01:36<3:02:19,  1.03it/s]
  1%|          | 96/11346 [01:37<3:02:13,  1.03it/s]
  1%|          | 97/11346 [01:38<3:02:16,  1.03it/s]
  1%|          | 98/11346 [01:38<3:02:10,  1.03it/s]
  1%|          | 99/11346 [01:39<3:02:15,  1.03it/s]
  1%|          | 100/11346 [01:40<3:02:17,  1.03it/s]
  1%|          | 101/11346 [01:41<3:02:09,  1.03it/s]
  1%|          | 102/11346 [01:42<3:02:11,  1.03it/s]
  1%|          | 103/11346 [01:43<3:02:13,  1.03it/s]
  1%|          | 104/11346 [01:44<3:02:14,  1.03it/s]
  1%|          | 105/11346 [01:45<3:02:17,  1.03it/s]
  1%|          | 106/11346 [01:46<3:02:24,  1.03it/s]
  1%|          | 107/11346 [01:47<3:02:23,  1.03it/s]
  1%|          | 108/11346 [01:48<3:02:19,  1.03it/s]
  1%|          | 109/11346 [01:49<3:02:12,  1.03it/s]
  1%|          | 110/11346 [01:50<3:02:17,  1.03it/s]
  1%|          | 111/11346 [01:51<3:02:21,  1.03it/s]
  1%|          | 112/11346 [01:52<3:02:20,  1.03it/s]
  1%|          | 113/11346 [01:53<3:02:08,  1.03it/s]
  1%|          | 114/11346 [01:54<3:02:11,  1.03it/s]
  1%|          | 115/11346 [01:55<3:02:06,  1.03it/s]
  1%|          | 116/11346 [01:56<3:02:02,  1.03it/s]
  1%|          | 117/11346 [01:57<3:02:01,  1.03it/s]
  1%|          | 118/11346 [01:58<3:02:04,  1.03it/s]
  1%|          | 119/11346 [01:59<3:02:06,  1.03it/s]
  1%|          | 120/11346 [02:00<3:02:03,  1.03it/s]
  1%|          | 121/11346 [02:01<3:02:03,  1.03it/s]
  1%|          | 122/11346 [02:02<3:02:00,  1.03it/s]
  1%|          | 123/11346 [02:03<3:02:07,  1.03it/s]
  1%|          | 124/11346 [02:04<3:02:06,  1.03it/s]
  1%|          | 125/11346 [02:05<3:02:04,  1.03it/s]
  1%|          | 126/11346 [02:06<3:02:01,  1.03it/s]
  1%|          | 127/11346 [02:07<3:02:09,  1.03it/s]
  1%|          | 128/11346 [02:08<3:02:00,  1.03it/s]
  1%|          | 129/11346 [02:09<3:02:01,  1.03it/s]
  1%|          | 130/11346 [02:10<3:02:00,  1.03it/s]
  1%|          | 131/11346 [02:11<3:02:05,  1.03it/s]
  1%|          | 132/11346 [02:12<3:02:04,  1.03it/s]
  1%|          | 133/11346 [02:13<3:01:55,  1.03it/s]
  1%|          | 134/11346 [02:14<3:01:57,  1.03it/s]
  1%|          | 135/11346 [02:15<3:01:53,  1.03it/s]
  1%|          | 136/11346 [02:15<3:01:44,  1.03it/s]
  1%|          | 137/11346 [02:16<3:01:44,  1.03it/s]
  1%|          | 138/11346 [02:17<3:01:41,  1.03it/s]
  1%|          | 139/11346 [02:18<3:01:44,  1.03it/s]
  1%|          | 140/11346 [02:19<3:01:46,  1.03it/s]
  1%|          | 141/11346 [02:20<3:01:46,  1.03it/s]
  1%|▏         | 142/11346 [02:21<3:01:48,  1.03it/s]
  1%|▏         | 143/11346 [02:22<3:01:39,  1.03it/s]
  1%|▏         | 144/11346 [02:23<3:01:43,  1.03it/s]
  1%|▏         | 145/11346 [02:24<3:01:43,  1.03it/s]
  1%|▏         | 146/11346 [02:25<3:01:37,  1.03it/s]
  1%|▏         | 147/11346 [02:26<3:01:42,  1.03it/s]
  1%|▏         | 148/11346 [02:27<3:01:44,  1.03it/s]
  1%|▏         | 149/11346 [02:28<3:01:44,  1.03it/s]
  1%|▏         | 150/11346 [02:29<3:01:47,  1.03it/s]
  1%|▏         | 151/11346 [02:30<3:01:43,  1.03it/s]
  1%|▏         | 152/11346 [02:31<3:01:43,  1.03it/s]
  1%|▏         | 153/11346 [02:32<3:01:37,  1.03it/s]
  1%|▏         | 154/11346 [02:33<3:01:38,  1.03it/s]
  1%|▏         | 155/11346 [02:34<3:01:32,  1.03it/s]
  1%|▏         | 156/11346 [02:35<3:01:33,  1.03it/s]
  1%|▏         | 157/11346 [02:36<3:01:29,  1.03it/s]
  1%|▏         | 158/11346 [02:37<3:01:24,  1.03it/s]
  1%|▏         | 159/11346 [02:38<3:01:23,  1.03it/s]
  1%|▏         | 160/11346 [02:39<3:01:30,  1.03it/s]
  1%|▏         | 161/11346 [02:40<3:01:33,  1.03it/s]
  1%|▏         | 162/11346 [02:41<3:01:28,  1.03it/s]
  1%|▏         | 163/11346 [02:42<3:01:27,  1.03it/s]
  1%|▏         | 164/11346 [02:43<3:01:27,  1.03it/s]
  1%|▏         | 165/11346 [02:44<3:01:27,  1.03it/s]
  1%|▏         | 166/11346 [02:45<3:01:20,  1.03it/s]
  1%|▏         | 167/11346 [02:46<3:01:20,  1.03it/s]
  1%|▏         | 168/11346 [02:47<3:01:15,  1.03it/s]
  1%|▏         | 169/11346 [02:48<3:01:17,  1.03it/s]
  1%|▏         | 170/11346 [02:49<3:01:17,  1.03it/s]
  2%|▏         | 171/11346 [02:50<3:01:21,  1.03it/s]
  2%|▏         | 172/11346 [02:51<3:01:16,  1.03it/s]
  2%|▏         | 173/11346 [02:52<3:01:20,  1.03it/s]
  2%|▏         | 174/11346 [02:52<3:01:22,  1.03it/s]
  2%|▏         | 175/11346 [02:53<3:01:23,  1.03it/s]
  2%|▏         | 176/11346 [02:54<3:01:27,  1.03it/s]
  2%|▏         | 177/11346 [02:55<3:01:28,  1.03it/s]
  2%|▏         | 178/11346 [02:56<3:01:20,  1.03it/s]
  2%|▏         | 179/11346 [02:57<3:01:14,  1.03it/s]
  2%|▏         | 180/11346 [02:58<3:01:12,  1.03it/s]
  2%|▏         | 181/11346 [02:59<3:01:27,  1.03it/s]
  2%|▏         | 182/11346 [03:00<3:01:28,  1.03it/s]
  2%|▏         | 183/11346 [03:01<3:01:20,  1.03it/s]
  2%|▏         | 184/11346 [03:02<3:01:20,  1.03it/s]
  2%|▏         | 185/11346 [03:03<3:01:08,  1.03it/s]
  2%|▏         | 186/11346 [03:04<3:01:13,  1.03it/s]
  2%|▏         | 187/11346 [03:05<3:01:19,  1.03it/s]
  2%|▏         | 188/11346 [03:06<3:01:16,  1.03it/s]
  2%|▏         | 189/11346 [03:07<3:01:13,  1.03it/s]
  2%|▏         | 190/11346 [03:08<3:01:09,  1.03it/s]
  2%|▏         | 191/11346 [03:09<3:01:10,  1.03it/s]
  2%|▏         | 192/11346 [03:10<3:01:10,  1.03it/s]
  2%|▏         | 193/11346 [03:11<3:01:02,  1.03it/s]
  2%|▏         | 194/11346 [03:12<3:01:06,  1.03it/s]
  2%|▏         | 195/11346 [03:13<3:01:03,  1.03it/s]
  2%|▏         | 196/11346 [03:14<3:01:09,  1.03it/s]
  2%|▏         | 197/11346 [03:15<3:01:08,  1.03it/s]
  2%|▏         | 198/11346 [03:16<3:01:05,  1.03it/s]
  2%|▏         | 199/11346 [03:17<3:01:02,  1.03it/s]
  2%|▏         | 200/11346 [03:18<3:00:55,  1.03it/s]
  2%|▏         | 201/11346 [03:19<3:00:59,  1.03it/s]
  2%|▏         | 202/11346 [03:20<3:01:04,  1.03it/s]
  2%|▏         | 203/11346 [03:21<3:01:01,  1.03it/s]
  2%|▏         | 204/11346 [03:22<3:01:02,  1.03it/s]
  2%|▏         | 205/11346 [03:23<3:00:57,  1.03it/s]
  2%|▏         | 206/11346 [03:24<3:00:55,  1.03it/s]
  2%|▏         | 207/11346 [03:25<3:00:55,  1.03it/s]
  2%|▏         | 208/11346 [03:26<3:00:50,  1.03it/s]
  2%|▏         | 209/11346 [03:27<3:00:50,  1.03it/s]
  2%|▏         | 210/11346 [03:28<3:00:49,  1.03it/s]
  2%|▏         | 211/11346 [03:29<3:00:48,  1.03it/s]
  2%|▏         | 212/11346 [03:30<3:00:46,  1.03it/s]
  2%|▏         | 213/11346 [03:30<3:00:49,  1.03it/s]
  2%|▏         | 214/11346 [03:31<3:00:51,  1.03it/s]
  2%|▏         | 215/11346 [03:32<3:00:50,  1.03it/s]
  2%|▏         | 216/11346 [03:33<3:00:49,  1.03it/s]
  2%|▏         | 217/11346 [03:34<3:00:41,  1.03it/s]
  2%|▏         | 218/11346 [03:35<3:00:43,  1.03it/s]
  2%|▏         | 219/11346 [03:36<3:00:43,  1.03it/s]
  2%|▏         | 220/11346 [03:37<3:00:45,  1.03it/s]
  2%|▏         | 221/11346 [03:38<3:00:43,  1.03it/s]
  2%|▏         | 222/11346 [03:39<3:00:34,  1.03it/s]
  2%|▏         | 223/11346 [03:40<3:00:36,  1.03it/s]
  2%|▏         | 224/11346 [03:41<3:00:39,  1.03it/s]
  2%|▏         | 225/11346 [03:42<3:00:41,  1.03it/s]
  2%|▏         | 226/11346 [03:43<3:00:35,  1.03it/s]
  2%|▏         | 227/11346 [03:44<3:00:33,  1.03it/s]
  2%|▏         | 228/11346 [03:45<3:00:32,  1.03it/s]
  2%|▏         | 229/11346 [03:46<3:00:33,  1.03it/s]
  2%|▏         | 230/11346 [03:47<3:00:32,  1.03it/s]
  2%|▏         | 231/11346 [03:48<3:00:35,  1.03it/s]
  2%|▏         | 232/11346 [03:49<3:00:36,  1.03it/s]
  2%|▏         | 233/11346 [03:50<3:00:32,  1.03it/s]
  2%|▏         | 234/11346 [03:51<3:00:35,  1.03it/s]
  2%|▏         | 235/11346 [03:52<3:00:35,  1.03it/s]
  2%|▏         | 236/11346 [03:53<3:00:35,  1.03it/s]
  2%|▏         | 237/11346 [03:54<3:00:34,  1.03it/s]
  2%|▏         | 238/11346 [03:55<3:00:38,  1.02it/s]
  2%|▏         | 239/11346 [03:56<3:00:34,  1.03it/s]
  2%|▏         | 240/11346 [03:57<3:00:31,  1.03it/s]
  2%|▏         | 241/11346 [03:58<3:00:30,  1.03it/s]
  2%|▏         | 242/11346 [03:59<3:00:23,  1.03it/s]
  2%|▏         | 243/11346 [04:00<3:00:22,  1.03it/s]
  2%|▏         | 244/11346 [04:01<3:00:22,  1.03it/s]
  2%|▏         | 245/11346 [04:02<3:00:21,  1.03it/s]
  2%|▏         | 246/11346 [04:03<3:00:11,  1.03it/s]
  2%|▏         | 247/11346 [04:04<3:00:14,  1.03it/s]
  2%|▏         | 248/11346 [04:05<3:00:16,  1.03it/s]
  2%|▏         | 249/11346 [04:06<3:00:15,  1.03it/s]
  2%|▏         | 250/11346 [04:07<3:00:15,  1.03it/s]
  2%|▏         | 251/11346 [04:08<3:00:12,  1.03it/s]
  2%|▏         | 252/11346 [04:08<3:00:11,  1.03it/s]
  2%|▏         | 253/11346 [04:09<3:00:11,  1.03it/s]
  2%|▏         | 254/11346 [04:10<3:00:13,  1.03it/s]
  2%|▏         | 255/11346 [04:11<3:00:12,  1.03it/s]
  2%|▏         | 256/11346 [04:12<3:00:13,  1.03it/s]
  2%|▏         | 257/11346 [04:13<3:00:06,  1.03it/s]
  2%|▏         | 258/11346 [04:14<3:00:05,  1.03it/s]
  2%|▏         | 259/11346 [04:15<3:00:08,  1.03it/s]
  2%|▏         | 260/11346 [04:16<3:00:08,  1.03it/s]
  2%|▏         | 261/11346 [04:17<3:00:05,  1.03it/s]
  2%|▏         | 262/11346 [04:18<3:00:01,  1.03it/s]
  2%|▏         | 263/11346 [04:19<3:00:04,  1.03it/s]
  2%|▏         | 264/11346 [04:20<3:00:04,  1.03it/s]
  2%|▏         | 265/11346 [04:21<3:00:02,  1.03it/s]
  2%|▏         | 266/11346 [04:22<3:00:05,  1.03it/s]
  2%|▏         | 267/11346 [04:23<3:00:03,  1.03it/s]
  2%|▏         | 268/11346 [04:24<2:59:55,  1.03it/s]
  2%|▏         | 269/11346 [04:25<2:59:51,  1.03it/s]
  2%|▏         | 270/11346 [04:26<2:59:51,  1.03it/s]
  2%|▏         | 271/11346 [04:27<2:59:44,  1.03it/s]
  2%|▏         | 272/11346 [04:28<2:59:48,  1.03it/s]
  2%|▏         | 273/11346 [04:29<2:59:59,  1.03it/s]
  2%|▏         | 274/11346 [04:30<3:00:03,  1.02it/s]
  2%|▏         | 275/11346 [04:31<2:59:58,  1.03it/s]
  2%|▏         | 276/11346 [04:32<3:00:01,  1.02it/s]
  2%|▏         | 277/11346 [04:33<3:00:00,  1.02it/s]
  2%|▏         | 278/11346 [04:34<2:59:52,  1.03it/s]
  2%|▏         | 279/11346 [04:35<2:59:53,  1.03it/s]
  2%|▏         | 280/11346 [04:36<2:59:54,  1.03it/s]
  2%|▏         | 281/11346 [04:37<2:59:49,  1.03it/s]
  2%|▏         | 282/11346 [04:38<2:59:50,  1.03it/s]
  2%|▏         | 283/11346 [04:39<2:59:49,  1.03it/s]
  3%|▎         | 284/11346 [04:40<2:59:42,  1.03it/s]
  3%|▎         | 285/11346 [04:41<2:59:45,  1.03it/s]
  3%|▎         | 286/11346 [04:42<2:59:45,  1.03it/s]
  3%|▎         | 287/11346 [04:43<2:59:46,  1.03it/s]
  3%|▎         | 288/11346 [04:44<2:59:38,  1.03it/s]
  3%|▎         | 289/11346 [04:45<2:59:34,  1.03it/s]
  3%|▎         | 290/11346 [04:46<2:59:25,  1.03it/s]
  3%|▎         | 291/11346 [04:47<2:59:28,  1.03it/s]
  3%|▎         | 292/11346 [04:47<2:59:31,  1.03it/s]
  3%|▎         | 293/11346 [04:48<2:59:32,  1.03it/s]
  3%|▎         | 294/11346 [04:49<2:59:30,  1.03it/s]
  3%|▎         | 295/11346 [04:50<2:59:31,  1.03it/s]
  3%|▎         | 296/11346 [04:51<2:59:32,  1.03it/s]
  3%|▎         | 297/11346 [04:52<2:59:27,  1.03it/s]
  3%|▎         | 298/11346 [04:53<2:59:26,  1.03it/s]
  3%|▎         | 299/11346 [04:54<2:59:23,  1.03it/s]
  3%|▎         | 300/11346 [04:55<2:59:19,  1.03it/s]
  3%|▎         | 301/11346 [04:56<2:59:21,  1.03it/s]
  3%|▎         | 302/11346 [04:57<2:59:26,  1.03it/s]
  3%|▎         | 303/11346 [04:58<2:59:25,  1.03it/s]
  3%|▎         | 304/11346 [04:59<2:59:23,  1.03it/s]
  3%|▎         | 305/11346 [05:00<2:59:26,  1.03it/s]
  3%|▎         | 306/11346 [05:01<2:59:22,  1.03it/s]
  3%|▎         | 307/11346 [05:02<2:59:30,  1.02it/s]
  3%|▎         | 308/11346 [05:03<2:59:20,  1.03it/s]
  3%|▎         | 309/11346 [05:04<2:59:21,  1.03it/s]
  3%|▎         | 310/11346 [05:05<2:59:16,  1.03it/s]
  3%|▎         | 311/11346 [05:06<2:59:15,  1.03it/s]
  3%|▎         | 312/11346 [05:07<2:59:11,  1.03it/s]
  3%|▎         | 313/11346 [05:08<2:59:02,  1.03it/s]
  3%|▎         | 314/11346 [05:09<2:59:00,  1.03it/s]
  3%|▎         | 315/11346 [05:10<2:59:03,  1.03it/s]
  3%|▎         | 316/11346 [05:11<2:59:04,  1.03it/s]
  3%|▎         | 317/11346 [05:12<2:59:08,  1.03it/s]
  3%|▎         | 318/11346 [05:13<2:59:07,  1.03it/s]
  3%|▎         | 319/11346 [05:14<2:59:05,  1.03it/s]
  3%|▎         | 320/11346 [05:15<2:59:01,  1.03it/s]
  3%|▎         | 321/11346 [05:16<2:59:02,  1.03it/s]
  3%|▎         | 322/11346 [05:17<2:59:08,  1.03it/s]
  3%|▎         | 323/11346 [05:18<2:59:03,  1.03it/s]
  3%|▎         | 324/11346 [05:19<2:58:55,  1.03it/s]
  3%|▎         | 325/11346 [05:20<2:59:00,  1.03it/s]
  3%|▎         | 326/11346 [05:21<2:58:53,  1.03it/s]
  3%|▎         | 327/11346 [05:22<2:58:46,  1.03it/s]
  3%|▎         | 328/11346 [05:23<2:58:50,  1.03it/s]
  3%|▎         | 329/11346 [05:24<2:58:42,  1.03it/s]
  3%|▎         | 330/11346 [05:25<2:58:39,  1.03it/s]
  3%|▎         | 331/11346 [05:25<2:58:42,  1.03it/s]
  3%|▎         | 332/11346 [05:26<2:58:47,  1.03it/s]
  3%|▎         | 333/11346 [05:27<2:58:45,  1.03it/s]
  3%|▎         | 334/11346 [05:28<2:58:46,  1.03it/s]
  3%|▎         | 335/11346 [05:29<2:58:40,  1.03it/s]
  3%|▎         | 336/11346 [05:30<2:58:34,  1.03it/s]
  3%|▎         | 337/11346 [05:31<2:58:29,  1.03it/s]
  3%|▎         | 338/11346 [05:32<2:58:31,  1.03it/s]
  3%|▎         | 339/11346 [05:33<2:58:38,  1.03it/s]
  3%|▎         | 340/11346 [05:34<2:58:38,  1.03it/s]
  3%|▎         | 341/11346 [05:35<2:58:41,  1.03it/s]
  3%|▎         | 342/11346 [05:36<2:58:42,  1.03it/s]
  3%|▎         | 343/11346 [05:37<2:58:40,  1.03it/s]
  3%|▎         | 344/11346 [05:38<2:58:33,  1.03it/s]
  3%|▎         | 345/11346 [05:39<2:58:34,  1.03it/s]
  3%|▎         | 346/11346 [05:40<2:58:37,  1.03it/s]
  3%|▎         | 347/11346 [05:41<2:58:31,  1.03it/s]
  3%|▎         | 348/11346 [05:42<2:58:31,  1.03it/s]
  3%|▎         | 349/11346 [05:43<2:58:30,  1.03it/s]
  3%|▎         | 350/11346 [05:44<2:58:27,  1.03it/s]
  3%|▎         | 351/11346 [05:45<2:58:22,  1.03it/s]
  3%|▎         | 352/11346 [05:46<2:58:23,  1.03it/s]
  3%|▎         | 353/11346 [05:47<2:58:23,  1.03it/s]
  3%|▎         | 354/11346 [05:48<2:58:23,  1.03it/s]
  3%|▎         | 355/11346 [05:49<2:58:22,  1.03it/s]
  3%|▎         | 356/11346 [05:50<2:58:22,  1.03it/s]
  3%|▎         | 357/11346 [05:51<2:58:23,  1.03it/s]
  3%|▎         | 358/11346 [05:52<2:58:22,  1.03it/s]
  3%|▎         | 359/11346 [05:53<2:58:24,  1.03it/s]
  3%|▎         | 360/11346 [05:54<2:58:23,  1.03it/s]
  3%|▎         | 361/11346 [05:55<2:58:18,  1.03it/s]
  3%|▎         | 362/11346 [05:56<2:58:22,  1.03it/s]
  3%|▎         | 363/11346 [05:57<2:58:12,  1.03it/s]
  3%|▎         | 364/11346 [05:58<2:58:13,  1.03it/s]
  3%|▎         | 365/11346 [05:59<2:58:18,  1.03it/s]
  3%|▎         | 366/11346 [06:00<2:58:18,  1.03it/s]
  3%|▎         | 367/11346 [06:01<2:58:12,  1.03it/s]
  3%|▎         | 368/11346 [06:02<2:58:16,  1.03it/s]
  3%|▎         | 369/11346 [06:03<2:58:15,  1.03it/s]
  3%|▎         | 370/11346 [06:03<2:58:11,  1.03it/s]
  3%|▎         | 371/11346 [06:04<2:58:12,  1.03it/s]
  3%|▎         | 372/11346 [06:05<2:58:20,  1.03it/s]
  3%|▎         | 373/11346 [06:06<2:58:15,  1.03it/s]
  3%|▎         | 374/11346 [06:07<2:58:06,  1.03it/s]
  3%|▎         | 375/11346 [06:08<2:58:09,  1.03it/s]
  3%|▎         | 376/11346 [06:09<2:58:10,  1.03it/s]
  3%|▎         | 377/11346 [06:10<2:58:08,  1.03it/s]
  3%|▎         | 378/11346 [06:11<2:57:58,  1.03it/s]
  3%|▎         | 379/11346 [06:12<2:58:02,  1.03it/s]
  3%|▎         | 380/11346 [06:13<2:58:02,  1.03it/s]
  3%|▎         | 381/11346 [06:14<2:57:59,  1.03it/s]
  3%|▎         | 382/11346 [06:15<2:57:58,  1.03it/s]
  3%|▎         | 383/11346 [06:16<2:57:57,  1.03it/s]
  3%|▎         | 384/11346 [06:17<2:57:59,  1.03it/s]
  3%|▎         | 385/11346 [06:18<2:57:55,  1.03it/s]
  3%|▎         | 386/11346 [06:19<2:57:57,  1.03it/s]
  3%|▎         | 387/11346 [06:20<2:57:56,  1.03it/s]
  3%|▎         | 388/11346 [06:21<2:57:53,  1.03it/s]
  3%|▎         | 389/11346 [06:22<2:57:54,  1.03it/s]
  3%|▎         | 390/11346 [06:23<2:57:56,  1.03it/s]
  3%|▎         | 391/11346 [06:24<2:57:50,  1.03it/s]
  3%|▎         | 392/11346 [06:25<2:57:51,  1.03it/s]
  3%|▎         | 393/11346 [06:26<2:57:45,  1.03it/s]
  3%|▎         | 394/11346 [06:27<2:57:41,  1.03it/s]
  3%|▎         | 395/11346 [06:28<2:57:39,  1.03it/s]
  3%|▎         | 396/11346 [06:29<2:57:34,  1.03it/s]
  3%|▎         | 397/11346 [06:30<2:57:32,  1.03it/s]
  4%|▎         | 398/11346 [06:31<2:57:25,  1.03it/s]
  4%|▎         | 399/11346 [06:32<2:57:23,  1.03it/s]
  4%|▎         | 400/11346 [06:33<2:57:23,  1.03it/s]
  4%|▎         | 401/11346 [06:34<2:57:26,  1.03it/s]
  4%|▎         | 402/11346 [06:35<2:57:28,  1.03it/s]
  4%|▎         | 403/11346 [06:36<2:57:24,  1.03it/s]
  4%|▎         | 404/11346 [06:37<2:57:26,  1.03it/s]
  4%|▎         | 405/11346 [06:38<2:57:22,  1.03it/s]
  4%|▎         | 406/11346 [06:39<2:57:16,  1.03it/s]
  4%|▎         | 407/11346 [06:39<2:57:09,  1.03it/s]
  4%|▎         | 408/11346 [06:40<2:57:09,  1.03it/s]
  4%|▎         | 409/11346 [06:41<2:57:08,  1.03it/s]
  4%|▎         | 410/11346 [06:42<2:57:07,  1.03it/s]
  4%|▎         | 411/11346 [06:43<2:57:03,  1.03it/s]
  4%|▎         | 412/11346 [06:44<2:57:04,  1.03it/s]
  4%|▎         | 413/11346 [06:45<2:57:00,  1.03it/s]
  4%|▎         | 414/11346 [06:46<2:56:59,  1.03it/s]
  4%|▎         | 415/11346 [06:47<2:57:02,  1.03it/s]
  4%|▎         | 416/11346 [06:48<2:57:00,  1.03it/s]
  4%|▎         | 417/11346 [06:49<2:57:00,  1.03it/s]
  4%|▎         | 418/11346 [06:50<2:57:03,  1.03it/s]
  4%|▎         | 419/11346 [06:51<2:57:04,  1.03it/s]
  4%|▎         | 420/11346 [06:52<2:57:04,  1.03it/s]
  4%|▎         | 421/11346 [06:53<2:57:10,  1.03it/s]
  4%|▎         | 422/11346 [06:54<2:57:09,  1.03it/s]
  4%|▎         | 423/11346 [06:55<2:57:04,  1.03it/s]
  4%|▎         | 424/11346 [06:56<2:57:00,  1.03it/s]
  4%|▎         | 425/11346 [06:57<2:56:57,  1.03it/s]
  4%|▍         | 426/11346 [06:58<2:56:58,  1.03it/s]
  4%|▍         | 427/11346 [06:59<2:56:54,  1.03it/s]
  4%|▍         | 428/11346 [07:00<2:56:55,  1.03it/s]
  4%|▍         | 429/11346 [07:01<2:56:55,  1.03it/s]
  4%|▍         | 430/11346 [07:02<2:56:53,  1.03it/s]
  4%|▍         | 431/11346 [07:03<2:56:50,  1.03it/s]
  4%|▍         | 432/11346 [07:04<2:56:46,  1.03it/s]
  4%|▍         | 433/11346 [07:05<2:56:43,  1.03it/s]
  4%|▍         | 434/11346 [07:06<2:56:44,  1.03it/s]
  4%|▍         | 435/11346 [07:07<2:56:51,  1.03it/s]
  4%|▍         | 436/11346 [07:08<2:56:47,  1.03it/s]
  4%|▍         | 437/11346 [07:09<2:56:43,  1.03it/s]
  4%|▍         | 438/11346 [07:10<2:56:53,  1.03it/s]
  4%|▍         | 439/11346 [07:11<2:56:45,  1.03it/s]
  4%|▍         | 440/11346 [07:12<2:56:42,  1.03it/s]
  4%|▍         | 441/11346 [07:13<2:56:45,  1.03it/s]
  4%|▍         | 442/11346 [07:14<2:56:42,  1.03it/s]
  4%|▍         | 443/11346 [07:14<2:56:39,  1.03it/s]
  4%|▍         | 444/11346 [07:15<2:56:37,  1.03it/s]
  4%|▍         | 445/11346 [07:16<2:56:38,  1.03it/s]
  4%|▍         | 446/11346 [07:17<2:56:35,  1.03it/s]
  4%|▍         | 447/11346 [07:18<2:56:38,  1.03it/s]
  4%|▍         | 448/11346 [07:19<2:56:38,  1.03it/s]
  4%|▍         | 449/11346 [07:20<2:56:34,  1.03it/s]
  4%|▍         | 450/11346 [07:21<2:56:31,  1.03it/s]
  4%|▍         | 451/11346 [07:22<2:56:32,  1.03it/s]
  4%|▍         | 452/11346 [07:23<2:56:31,  1.03it/s]
  4%|▍         | 453/11346 [07:24<2:56:29,  1.03it/s]
  4%|▍         | 454/11346 [07:25<2:56:29,  1.03it/s]
  4%|▍         | 455/11346 [07:26<2:56:25,  1.03it/s]
  4%|▍         | 456/11346 [07:27<2:56:25,  1.03it/s]
  4%|▍         | 457/11346 [07:28<2:56:24,  1.03it/s]
  4%|▍         | 458/11346 [07:29<2:56:23,  1.03it/s]
  4%|▍         | 459/11346 [07:30<2:56:22,  1.03it/s]
  4%|▍         | 460/11346 [07:31<2:56:22,  1.03it/s]
  4%|▍         | 461/11346 [07:32<2:56:19,  1.03it/s]
  4%|▍         | 462/11346 [07:33<2:56:19,  1.03it/s]
  4%|▍         | 463/11346 [07:34<2:56:20,  1.03it/s]
  4%|▍         | 464/11346 [07:35<2:56:20,  1.03it/s]
  4%|▍         | 465/11346 [07:36<2:56:21,  1.03it/s]
  4%|▍         | 466/11346 [07:37<2:56:16,  1.03it/s]
  4%|▍         | 467/11346 [07:38<2:56:10,  1.03it/s]
  4%|▍         | 468/11346 [07:39<2:56:11,  1.03it/s]
  4%|▍         | 469/11346 [07:40<2:56:07,  1.03it/s]
  4%|▍         | 470/11346 [07:41<2:56:09,  1.03it/s]
  4%|▍         | 471/11346 [07:42<2:56:10,  1.03it/s]
  4%|▍         | 472/11346 [07:43<2:56:12,  1.03it/s]
  4%|▍         | 473/11346 [07:44<2:56:18,  1.03it/s]
  4%|▍         | 474/11346 [07:45<2:56:16,  1.03it/s]
  4%|▍         | 475/11346 [07:46<2:56:08,  1.03it/s]
  4%|▍         | 476/11346 [07:47<2:56:06,  1.03it/s]
  4%|▍         | 477/11346 [07:48<2:56:05,  1.03it/s]
  4%|▍         | 478/11346 [07:49<2:56:03,  1.03it/s]
  4%|▍         | 479/11346 [07:49<2:55:59,  1.03it/s]
  4%|▍         | 480/11346 [07:50<2:56:00,  1.03it/s]
  4%|▍         | 481/11346 [07:51<2:55:59,  1.03it/s]
  4%|▍         | 482/11346 [07:52<2:55:57,  1.03it/s]
  4%|▍         | 483/11346 [07:53<2:55:58,  1.03it/s]
  4%|▍         | 484/11346 [07:54<2:55:55,  1.03it/s]
  4%|▍         | 485/11346 [07:55<2:55:58,  1.03it/s]
  4%|▍         | 486/11346 [07:56<2:55:56,  1.03it/s]
  4%|▍         | 487/11346 [07:57<2:55:50,  1.03it/s]
  4%|▍         | 488/11346 [07:58<2:55:48,  1.03it/s]
  4%|▍         | 489/11346 [07:59<2:55:47,  1.03it/s]
  4%|▍         | 490/11346 [08:00<2:55:48,  1.03it/s]
  4%|▍         | 491/11346 [08:01<2:55:51,  1.03it/s]
  4%|▍         | 492/11346 [08:02<2:55:51,  1.03it/s]
  4%|▍         | 493/11346 [08:03<2:55:48,  1.03it/s]
  4%|▍         | 494/11346 [08:04<2:55:50,  1.03it/s]
  4%|▍         | 495/11346 [08:05<2:55:51,  1.03it/s]
  4%|▍         | 496/11346 [08:06<2:55:49,  1.03it/s]
  4%|▍         | 497/11346 [08:07<2:55:49,  1.03it/s]
  4%|▍         | 498/11346 [08:08<2:55:44,  1.03it/s]
  4%|▍         | 499/11346 [08:09<2:55:43,  1.03it/s]
  4%|▍         | 500/11346 [08:10<2:55:42,  1.03it/s]
                                                     
  4%|▍         | 500/11346 [08:10<2:55:42,  1.03it/s]
  4%|▍         | 501/11346 [08:11<2:55:54,  1.03it/s]
  4%|▍         | 502/11346 [08:12<2:55:53,  1.03it/s]
  4%|▍         | 503/11346 [08:13<2:55:51,  1.03it/s]
  4%|▍         | 504/11346 [08:14<2:55:43,  1.03it/s]
  4%|▍         | 505/11346 [08:15<2:55:43,  1.03it/s]
  4%|▍         | 506/11346 [08:16<2:55:40,  1.03it/s]
  4%|▍         | 507/11346 [08:17<2:55:35,  1.03it/s]
  4%|▍         | 508/11346 [08:18<2:55:34,  1.03it/s]
  4%|▍         | 509/11346 [08:19<2:55:35,  1.03it/s]
  4%|▍         | 510/11346 [08:20<2:55:32,  1.03it/s]
  5%|▍         | 511/11346 [08:21<2:55:29,  1.03it/s]
  5%|▍         | 512/11346 [08:22<2:55:32,  1.03it/s]
  5%|▍         | 513/11346 [08:23<2:55:28,  1.03it/s]
  5%|▍         | 514/11346 [08:24<2:55:27,  1.03it/s]
  5%|▍         | 515/11346 [08:24<2:55:28,  1.03it/s]
  5%|▍         | 516/11346 [08:25<2:55:36,  1.03it/s]
  5%|▍         | 517/11346 [08:26<2:55:34,  1.03it/s]
  5%|▍         | 518/11346 [08:27<2:55:29,  1.03it/s]
  5%|▍         | 519/11346 [08:28<2:55:30,  1.03it/s]
  5%|▍         | 520/11346 [08:29<2:55:30,  1.03it/s]
  5%|▍         | 521/11346 [08:30<2:55:31,  1.03it/s]
  5%|▍         | 522/11346 [08:31<2:55:30,  1.03it/s]
  5%|▍         | 523/11346 [08:32<2:55:32,  1.03it/s]
  5%|▍         | 524/11346 [08:33<2:55:27,  1.03it/s]
  5%|▍         | 525/11346 [08:34<2:55:25,  1.03it/s]
  5%|▍         | 526/11346 [08:35<2:55:29,  1.03it/s]
  5%|▍         | 527/11346 [08:36<2:55:25,  1.03it/s]
  5%|▍         | 528/11346 [08:37<2:55:19,  1.03it/s]
  5%|▍         | 529/11346 [08:38<2:55:22,  1.03it/s]
  5%|▍         | 530/11346 [08:39<2:55:23,  1.03it/s]
  5%|▍         | 531/11346 [08:40<2:55:20,  1.03it/s]
  5%|▍         | 532/11346 [08:41<2:55:13,  1.03it/s]
  5%|▍         | 533/11346 [08:42<2:55:13,  1.03it/s]
  5%|▍         | 534/11346 [08:43<2:55:10,  1.03it/s]
  5%|▍         | 535/11346 [08:44<2:55:10,  1.03it/s]
  5%|▍         | 536/11346 [08:45<2:55:09,  1.03it/s]
  5%|▍         | 537/11346 [08:46<2:55:11,  1.03it/s]
  5%|▍         | 538/11346 [08:47<2:55:07,  1.03it/s]
  5%|▍         | 539/11346 [08:48<2:55:08,  1.03it/s]
  5%|▍         | 540/11346 [08:49<2:55:08,  1.03it/s]
  5%|▍         | 541/11346 [08:50<2:55:11,  1.03it/s]
  5%|▍         | 542/11346 [08:51<2:55:07,  1.03it/s]
  5%|▍         | 543/11346 [08:52<2:55:04,  1.03it/s]
  5%|▍         | 544/11346 [08:53<2:55:06,  1.03it/s]
  5%|▍         | 545/11346 [08:54<2:55:03,  1.03it/s]
  5%|▍         | 546/11346 [08:55<2:55:00,  1.03it/s]
  5%|▍         | 547/11346 [08:56<2:54:58,  1.03it/s]
  5%|▍         | 548/11346 [08:57<2:54:57,  1.03it/s]
  5%|▍         | 549/11346 [08:58<2:54:57,  1.03it/s]
  5%|▍         | 550/11346 [08:59<2:54:58,  1.03it/s]
  5%|▍         | 551/11346 [08:59<2:54:53,  1.03it/s]
  5%|▍         | 552/11346 [09:00<2:54:50,  1.03it/s]
  5%|▍         | 553/11346 [09:01<2:54:48,  1.03it/s]
  5%|▍         | 554/11346 [09:02<2:54:51,  1.03it/s]
  5%|▍         | 555/11346 [09:03<2:54:53,  1.03it/s]
  5%|▍         | 556/11346 [09:04<2:54:48,  1.03it/s]
  5%|▍         | 557/11346 [09:05<2:54:44,  1.03it/s]
  5%|▍         | 558/11346 [09:06<2:54:55,  1.03it/s]
  5%|▍         | 559/11346 [09:07<2:54:51,  1.03it/s]
  5%|▍         | 560/11346 [09:08<2:54:48,  1.03it/s]
  5%|▍         | 561/11346 [09:09<2:54:47,  1.03it/s]
  5%|▍         | 562/11346 [09:10<2:54:45,  1.03it/s]
  5%|▍         | 563/11346 [09:11<2:54:42,  1.03it/s]
  5%|▍         | 564/11346 [09:12<2:54:41,  1.03it/s]
  5%|▍         | 565/11346 [09:13<2:54:45,  1.03it/s]
  5%|▍         | 566/11346 [09:14<2:54:41,  1.03it/s]
  5%|▍         | 567/11346 [09:15<2:54:43,  1.03it/s]
  5%|▌         | 568/11346 [09:16<2:54:40,  1.03it/s]
  5%|▌         | 569/11346 [09:17<2:54:35,  1.03it/s]
  5%|▌         | 570/11346 [09:18<2:54:32,  1.03it/s]
  5%|▌         | 571/11346 [09:19<2:54:34,  1.03it/s]
  5%|▌         | 572/11346 [09:20<2:54:32,  1.03it/s]
  5%|▌         | 573/11346 [09:21<2:54:33,  1.03it/s]
  5%|▌         | 574/11346 [09:22<2:54:33,  1.03it/s]
  5%|▌         | 575/11346 [09:23<2:54:29,  1.03it/s]
  5%|▌         | 576/11346 [09:24<2:54:28,  1.03it/s]
  5%|▌         | 577/11346 [09:25<2:54:25,  1.03it/s]
  5%|▌         | 578/11346 [09:26<2:54:24,  1.03it/s]
  5%|▌         | 579/11346 [09:27<2:54:27,  1.03it/s]
  5%|▌         | 580/11346 [09:28<2:54:25,  1.03it/s]
  5%|▌         | 581/11346 [09:29<2:54:21,  1.03it/s]
  5%|▌         | 582/11346 [09:30<2:54:26,  1.03it/s]
  5%|▌         | 583/11346 [09:31<2:54:24,  1.03it/s]
  5%|▌         | 584/11346 [09:32<2:54:22,  1.03it/s]
  5%|▌         | 585/11346 [09:33<2:54:21,  1.03it/s]
  5%|▌         | 586/11346 [09:34<2:54:17,  1.03it/s]
  5%|▌         | 587/11346 [09:34<2:54:17,  1.03it/s]
  5%|▌         | 588/11346 [09:35<2:54:13,  1.03it/s]
  5%|▌         | 589/11346 [09:36<2:54:18,  1.03it/s]
  5%|▌         | 590/11346 [09:37<2:54:15,  1.03it/s]
  5%|▌         | 591/11346 [09:38<2:54:09,  1.03it/s]
  5%|▌         | 592/11346 [09:39<2:54:09,  1.03it/s]
  5%|▌         | 593/11346 [09:40<2:54:13,  1.03it/s]
  5%|▌         | 594/11346 [09:41<2:54:12,  1.03it/s]
  5%|▌         | 595/11346 [09:42<2:54:13,  1.03it/s]
  5%|▌         | 596/11346 [09:43<2:54:10,  1.03it/s]
  5%|▌         | 597/11346 [09:44<2:54:09,  1.03it/s]
  5%|▌         | 598/11346 [09:45<2:54:06,  1.03it/s]
  5%|▌         | 599/11346 [09:46<2:54:12,  1.03it/s]
  5%|▌         | 600/11346 [09:47<2:54:08,  1.03it/s]
  5%|▌         | 601/11346 [09:48<2:54:08,  1.03it/s]
  5%|▌         | 602/11346 [09:49<2:54:07,  1.03it/s]
  5%|▌         | 603/11346 [09:50<2:54:12,  1.03it/s]
  5%|▌         | 604/11346 [09:51<2:54:10,  1.03it/s]
  5%|▌         | 605/11346 [09:52<2:54:10,  1.03it/s]
  5%|▌         | 606/11346 [09:53<2:54:09,  1.03it/s]
  5%|▌         | 607/11346 [09:54<2:54:06,  1.03it/s]
  5%|▌         | 608/11346 [09:55<2:54:06,  1.03it/s]
  5%|▌         | 609/11346 [09:56<2:54:06,  1.03it/s]
  5%|▌         | 610/11346 [09:57<2:54:03,  1.03it/s]
  5%|▌         | 611/11346 [09:58<2:54:02,  1.03it/s]
  5%|▌         | 612/11346 [09:59<2:54:01,  1.03it/s]
  5%|▌         | 613/11346 [10:00<2:53:56,  1.03it/s]
  5%|▌         | 614/11346 [10:01<2:53:56,  1.03it/s]
  5%|▌         | 615/11346 [10:02<2:53:53,  1.03it/s]
  5%|▌         | 616/11346 [10:03<2:53:48,  1.03it/s]
  5%|▌         | 617/11346 [10:04<2:53:48,  1.03it/s]
  5%|▌         | 618/11346 [10:05<2:53:46,  1.03it/s]
  5%|▌         | 619/11346 [10:06<2:53:50,  1.03it/s]
  5%|▌         | 620/11346 [10:07<2:53:49,  1.03it/s]
  5%|▌         | 621/11346 [10:08<2:53:50,  1.03it/s]
  5%|▌         | 622/11346 [10:09<2:53:46,  1.03it/s]
  5%|▌         | 623/11346 [10:09<2:53:45,  1.03it/s]
  5%|▌         | 624/11346 [10:10<2:53:45,  1.03it/s]
  6%|▌         | 625/11346 [10:11<2:53:48,  1.03it/s]
  6%|▌         | 626/11346 [10:12<2:53:47,  1.03it/s]
  6%|▌         | 627/11346 [10:13<2:53:42,  1.03it/s]
  6%|▌         | 628/11346 [10:14<2:53:45,  1.03it/s]
  6%|▌         | 629/11346 [10:15<2:53:41,  1.03it/s]
  6%|▌         | 630/11346 [10:16<2:53:39,  1.03it/s]
  6%|▌         | 631/11346 [10:17<2:53:36,  1.03it/s]
  6%|▌         | 632/11346 [10:18<2:53:35,  1.03it/s]
  6%|▌         | 633/11346 [10:19<2:53:36,  1.03it/s]
  6%|▌         | 634/11346 [10:20<2:53:35,  1.03it/s]
  6%|▌         | 635/11346 [10:21<2:53:34,  1.03it/s]
  6%|▌         | 636/11346 [10:22<2:53:36,  1.03it/s]
  6%|▌         | 637/11346 [10:23<2:53:29,  1.03it/s]
  6%|▌         | 638/11346 [10:24<2:53:28,  1.03it/s]
  6%|▌         | 639/11346 [10:25<2:53:30,  1.03it/s]
  6%|▌         | 640/11346 [10:26<2:53:28,  1.03it/s]
  6%|▌         | 641/11346 [10:27<2:53:28,  1.03it/s]
  6%|▌         | 642/11346 [10:28<2:53:24,  1.03it/s]
  6%|▌         | 643/11346 [10:29<2:53:26,  1.03it/s]
  6%|▌         | 644/11346 [10:30<2:53:27,  1.03it/s]
  6%|▌         | 645/11346 [10:31<2:53:27,  1.03it/s]
  6%|▌         | 646/11346 [10:32<2:53:26,  1.03it/s]
  6%|▌         | 647/11346 [10:33<2:53:26,  1.03it/s]
  6%|▌         | 648/11346 [10:34<2:53:25,  1.03it/s]
  6%|▌         | 649/11346 [10:35<2:53:23,  1.03it/s]
  6%|▌         | 650/11346 [10:36<2:53:19,  1.03it/s]
  6%|▌         | 651/11346 [10:37<2:53:19,  1.03it/s]
  6%|▌         | 652/11346 [10:38<2:53:27,  1.03it/s]
  6%|▌         | 653/11346 [10:39<2:53:24,  1.03it/s]
  6%|▌         | 654/11346 [10:40<2:53:21,  1.03it/s]
  6%|▌         | 655/11346 [10:41<2:53:29,  1.03it/s]
  6%|▌         | 656/11346 [10:42<2:53:21,  1.03it/s]
  6%|▌         | 657/11346 [10:43<2:53:16,  1.03it/s]
  6%|▌         | 658/11346 [10:44<2:53:16,  1.03it/s]
  6%|▌         | 659/11346 [10:45<2:53:17,  1.03it/s]
  6%|▌         | 660/11346 [10:45<2:53:15,  1.03it/s]
  6%|▌         | 661/11346 [10:46<2:53:14,  1.03it/s]
  6%|▌         | 662/11346 [10:47<2:53:15,  1.03it/s]
  6%|▌         | 663/11346 [10:48<2:53:12,  1.03it/s]
  6%|▌         | 664/11346 [10:49<2:53:10,  1.03it/s]
  6%|▌         | 665/11346 [10:50<2:53:03,  1.03it/s]
  6%|▌         | 666/11346 [10:51<2:53:05,  1.03it/s]
  6%|▌         | 667/11346 [10:52<2:53:08,  1.03it/s]
  6%|▌         | 668/11346 [10:53<2:53:06,  1.03it/s]
  6%|▌         | 669/11346 [10:54<2:53:03,  1.03it/s]
  6%|▌         | 670/11346 [10:55<2:52:58,  1.03it/s]
  6%|▌         | 671/11346 [10:56<2:53:01,  1.03it/s]
  6%|▌         | 672/11346 [10:57<2:53:05,  1.03it/s]
  6%|▌         | 673/11346 [10:58<2:53:06,  1.03it/s]
  6%|▌         | 674/11346 [10:59<2:53:06,  1.03it/s]
  6%|▌         | 675/11346 [11:00<2:52:57,  1.03it/s]
  6%|▌         | 676/11346 [11:01<2:53:00,  1.03it/s]
  6%|▌         | 677/11346 [11:02<2:52:58,  1.03it/s]
  6%|▌         | 678/11346 [11:03<2:52:58,  1.03it/s]
  6%|▌         | 679/11346 [11:04<2:52:51,  1.03it/s]
  6%|▌         | 680/11346 [11:05<2:52:48,  1.03it/s]
  6%|▌         | 681/11346 [11:06<2:52:44,  1.03it/s]
  6%|▌         | 682/11346 [11:07<2:52:45,  1.03it/s]
  6%|▌         | 683/11346 [11:08<2:52:45,  1.03it/s]
  6%|▌         | 684/11346 [11:09<2:52:44,  1.03it/s]
  6%|▌         | 685/11346 [11:10<2:52:48,  1.03it/s]
  6%|▌         | 686/11346 [11:11<2:52:43,  1.03it/s]
  6%|▌         | 687/11346 [11:12<2:52:38,  1.03it/s]
  6%|▌         | 688/11346 [11:13<2:52:38,  1.03it/s]
  6%|▌         | 689/11346 [11:14<2:52:39,  1.03it/s]
  6%|▌         | 690/11346 [11:15<2:52:38,  1.03it/s]
  6%|▌         | 691/11346 [11:16<2:52:46,  1.03it/s]
  6%|▌         | 692/11346 [11:17<2:52:42,  1.03it/s]
  6%|▌         | 693/11346 [11:18<2:52:41,  1.03it/s]
  6%|▌         | 694/11346 [11:19<2:52:39,  1.03it/s]
  6%|▌         | 695/11346 [11:20<2:52:39,  1.03it/s]
  6%|▌         | 696/11346 [11:20<2:52:35,  1.03it/s]
  6%|▌         | 697/11346 [11:21<2:52:37,  1.03it/s]
  6%|▌         | 698/11346 [11:22<2:52:38,  1.03it/s]
  6%|▌         | 699/11346 [11:23<2:52:33,  1.03it/s]
  6%|▌         | 700/11346 [11:24<2:52:31,  1.03it/s]
  6%|▌         | 701/11346 [11:25<2:52:29,  1.03it/s]
  6%|▌         | 702/11346 [11:26<2:52:29,  1.03it/s]
  6%|▌         | 703/11346 [11:27<2:52:29,  1.03it/s]
  6%|▌         | 704/11346 [11:28<2:52:26,  1.03it/s]
  6%|▌         | 705/11346 [11:29<2:52:21,  1.03it/s]
  6%|▌         | 706/11346 [11:30<2:52:24,  1.03it/s]
  6%|▌         | 707/11346 [11:31<2:52:20,  1.03it/s]
  6%|▌         | 708/11346 [11:32<2:52:20,  1.03it/s]
  6%|▌         | 709/11346 [11:33<2:52:20,  1.03it/s]
  6%|▋         | 710/11346 [11:34<2:52:20,  1.03it/s]
  6%|▋         | 711/11346 [11:35<2:52:18,  1.03it/s]
  6%|▋         | 712/11346 [11:36<2:52:15,  1.03it/s]
  6%|▋         | 713/11346 [11:37<2:52:13,  1.03it/s]
  6%|▋         | 714/11346 [11:38<2:52:13,  1.03it/s]
  6%|▋         | 715/11346 [11:39<2:52:13,  1.03it/s]
  6%|▋         | 716/11346 [11:40<2:52:14,  1.03it/s]
  6%|▋         | 717/11346 [11:41<2:52:14,  1.03it/s]
  6%|▋         | 718/11346 [11:42<2:52:13,  1.03it/s]
  6%|▋         | 719/11346 [11:43<2:52:08,  1.03it/s]
  6%|▋         | 720/11346 [11:44<2:52:05,  1.03it/s]
  6%|▋         | 721/11346 [11:45<2:52:00,  1.03it/s]
  6%|▋         | 722/11346 [11:46<2:52:01,  1.03it/s]
  6%|▋         | 723/11346 [11:47<2:52:05,  1.03it/s]
  6%|▋         | 724/11346 [11:48<2:52:05,  1.03it/s]
  6%|▋         | 725/11346 [11:49<2:52:02,  1.03it/s]
  6%|▋         | 726/11346 [11:50<2:52:06,  1.03it/s]
  6%|▋         | 727/11346 [11:51<2:52:06,  1.03it/s]
  6%|▋         | 728/11346 [11:52<2:52:02,  1.03it/s]
  6%|▋         | 729/11346 [11:53<2:52:02,  1.03it/s]
  6%|▋         | 730/11346 [11:54<2:52:03,  1.03it/s]
  6%|▋         | 731/11346 [11:55<2:52:02,  1.03it/s]
  6%|▋         | 732/11346 [11:55<2:52:00,  1.03it/s]
  6%|▋         | 733/11346 [11:56<2:52:01,  1.03it/s]
  6%|▋         | 734/11346 [11:57<2:51:58,  1.03it/s]
  6%|▋         | 735/11346 [11:58<2:52:00,  1.03it/s]
  6%|▋         | 736/11346 [11:59<2:51:54,  1.03it/s]
  6%|▋         | 737/11346 [12:00<2:51:58,  1.03it/s]
  7%|▋         | 738/11346 [12:01<2:51:58,  1.03it/s]
  7%|▋         | 739/11346 [12:02<2:51:56,  1.03it/s]
  7%|▋         | 740/11346 [12:03<2:51:52,  1.03it/s]
  7%|▋         | 741/11346 [12:04<2:52:01,  1.03it/s]
  7%|▋         | 742/11346 [12:05<2:51:56,  1.03it/s]
  7%|▋         | 743/11346 [12:06<2:51:52,  1.03it/s]
  7%|▋         | 744/11346 [12:07<2:51:49,  1.03it/s]
  7%|▋         | 745/11346 [12:08<2:51:51,  1.03it/s]
  7%|▋         | 746/11346 [12:09<2:51:55,  1.03it/s]
  7%|▋         | 747/11346 [12:10<2:51:54,  1.03it/s]
  7%|▋         | 748/11346 [12:11<3:04:33,  1.04s/it]
  7%|▋         | 749/11346 [12:12<3:00:39,  1.02s/it]
  7%|▋         | 750/11346 [12:13<2:57:52,  1.01s/it]
  7%|▋         | 751/11346 [12:14<2:56:02,  1.00it/s]
  7%|▋         | 752/11346 [12:15<2:54:40,  1.01it/s]
  7%|▋         | 753/11346 [12:16<2:53:46,  1.02it/s]
  7%|▋         | 754/11346 [12:17<2:53:09,  1.02it/s]
  7%|▋         | 755/11346 [12:18<2:52:43,  1.02it/s]
  7%|▋         | 756/11346 [12:19<2:52:19,  1.02it/s]
  7%|▋         | 757/11346 [12:20<2:52:06,  1.03it/s]
  7%|▋         | 758/11346 [12:21<2:51:53,  1.03it/s]
  7%|▋         | 759/11346 [12:22<2:51:44,  1.03it/s]
  7%|▋         | 760/11346 [12:23<2:51:40,  1.03it/s]
  7%|▋         | 761/11346 [12:24<2:51:34,  1.03it/s]
  7%|▋         | 762/11346 [12:25<2:51:31,  1.03it/s]
  7%|▋         | 763/11346 [12:26<2:51:28,  1.03it/s]
  7%|▋         | 764/11346 [12:27<2:51:28,  1.03it/s]
  7%|▋         | 765/11346 [12:28<2:51:27,  1.03it/s]
  7%|▋         | 766/11346 [12:29<2:51:20,  1.03it/s]
  7%|▋         | 767/11346 [12:30<2:51:18,  1.03it/s]
  7%|▋         | 768/11346 [12:31<2:51:19,  1.03it/s]
  7%|▋         | 769/11346 [12:32<2:51:21,  1.03it/s]
  7%|▋         | 770/11346 [12:33<2:51:22,  1.03it/s]
  7%|▋         | 771/11346 [12:34<2:51:17,  1.03it/s]
  7%|▋         | 772/11346 [12:35<2:51:17,  1.03it/s]
  7%|▋         | 773/11346 [12:36<2:51:16,  1.03it/s]
  7%|▋         | 774/11346 [12:37<2:51:12,  1.03it/s]
  7%|▋         | 775/11346 [12:38<2:51:12,  1.03it/s]
  7%|▋         | 776/11346 [12:39<2:51:08,  1.03it/s]
  7%|▋         | 777/11346 [12:39<2:51:11,  1.03it/s]
  7%|▋         | 778/11346 [12:40<2:51:10,  1.03it/s]
  7%|▋         | 779/11346 [12:41<2:51:10,  1.03it/s]
  7%|▋         | 780/11346 [12:42<2:51:08,  1.03it/s]
  7%|▋         | 781/11346 [12:43<2:51:05,  1.03it/s]
  7%|▋         | 782/11346 [12:44<2:51:02,  1.03it/s]
  7%|▋         | 783/11346 [12:45<2:51:01,  1.03it/s]
  7%|▋         | 784/11346 [12:46<2:51:02,  1.03it/s]
  7%|▋         | 785/11346 [12:47<2:51:05,  1.03it/s]
  7%|▋         | 786/11346 [12:48<2:51:04,  1.03it/s]
  7%|▋         | 787/11346 [12:49<2:51:02,  1.03it/s]
  7%|▋         | 788/11346 [12:50<2:51:02,  1.03it/s]
  7%|▋         | 789/11346 [12:51<2:51:01,  1.03it/s]
  7%|▋         | 790/11346 [12:52<2:50:58,  1.03it/s]
  7%|▋         | 791/11346 [12:53<2:51:00,  1.03it/s]
  7%|▋         | 792/11346 [12:54<2:50:56,  1.03it/s]
  7%|▋         | 793/11346 [12:55<2:50:56,  1.03it/s]
  7%|▋         | 794/11346 [12:56<2:50:57,  1.03it/s]
  7%|▋         | 795/11346 [12:57<2:50:53,  1.03it/s]
  7%|▋         | 796/11346 [12:58<2:50:54,  1.03it/s]
  7%|▋         | 797/11346 [12:59<2:50:55,  1.03it/s]
  7%|▋         | 798/11346 [13:00<2:50:59,  1.03it/s]
  7%|▋         | 799/11346 [13:01<2:50:54,  1.03it/s]
  7%|▋         | 800/11346 [13:02<2:50:47,  1.03it/s]
  7%|▋         | 801/11346 [13:03<2:50:48,  1.03it/s]
  7%|▋         | 802/11346 [13:04<2:50:51,  1.03it/s]
  7%|▋         | 803/11346 [13:05<2:50:51,  1.03it/s]
  7%|▋         | 804/11346 [13:06<2:50:50,  1.03it/s]
  7%|▋         | 805/11346 [13:07<2:50:49,  1.03it/s]
  7%|▋         | 806/11346 [13:08<2:50:47,  1.03it/s]
  7%|▋         | 807/11346 [13:09<2:50:44,  1.03it/s]
  7%|▋         | 808/11346 [13:10<2:50:40,  1.03it/s]
  7%|▋         | 809/11346 [13:11<2:50:35,  1.03it/s]
  7%|▋         | 810/11346 [13:12<2:50:41,  1.03it/s]
  7%|▋         | 811/11346 [13:13<2:50:39,  1.03it/s]
  7%|▋         | 812/11346 [13:13<2:50:37,  1.03it/s]
  7%|▋         | 813/11346 [13:14<2:50:35,  1.03it/s]
  7%|▋         | 814/11346 [13:15<2:50:35,  1.03it/s]
  7%|▋         | 815/11346 [13:16<2:50:42,  1.03it/s]
  7%|▋         | 816/11346 [13:17<2:50:37,  1.03it/s]
  7%|▋         | 817/11346 [13:18<2:50:36,  1.03it/s]
  7%|▋         | 818/11346 [13:19<2:50:35,  1.03it/s]
  7%|▋         | 819/11346 [13:20<2:50:33,  1.03it/s]
  7%|▋         | 820/11346 [13:21<2:50:35,  1.03it/s]
  7%|▋         | 821/11346 [13:22<2:50:38,  1.03it/s]
  7%|▋         | 822/11346 [13:23<2:50:32,  1.03it/s]
  7%|▋         | 823/11346 [13:24<2:50:27,  1.03it/s]
  7%|▋         | 824/11346 [13:25<2:50:24,  1.03it/s]
  7%|▋         | 825/11346 [13:26<2:50:27,  1.03it/s]
  7%|▋         | 826/11346 [13:27<2:50:27,  1.03it/s]
  7%|▋         | 827/11346 [13:28<2:50:26,  1.03it/s]
  7%|▋         | 828/11346 [13:29<2:50:33,  1.03it/s]
  7%|▋         | 829/11346 [13:30<2:50:29,  1.03it/s]
  7%|▋         | 830/11346 [13:31<2:50:24,  1.03it/s]
  7%|▋         | 831/11346 [13:32<2:50:23,  1.03it/s]
  7%|▋         | 832/11346 [13:33<2:50:24,  1.03it/s]
  7%|▋         | 833/11346 [13:34<2:50:20,  1.03it/s]
  7%|▋         | 834/11346 [13:35<2:50:16,  1.03it/s]
  7%|▋         | 835/11346 [13:36<2:50:12,  1.03it/s]
  7%|▋         | 836/11346 [13:37<2:50:16,  1.03it/s]
  7%|▋         | 837/11346 [13:38<2:50:17,  1.03it/s]
  7%|▋         | 838/11346 [13:39<2:50:15,  1.03it/s]
  7%|▋         | 839/11346 [13:40<2:50:13,  1.03it/s]
  7%|▋         | 840/11346 [13:41<2:50:10,  1.03it/s]
  7%|▋         | 841/11346 [13:42<2:50:10,  1.03it/s]
  7%|▋         | 842/11346 [13:43<2:50:10,  1.03it/s]
  7%|▋         | 843/11346 [13:44<2:50:07,  1.03it/s]
  7%|▋         | 844/11346 [13:45<2:50:02,  1.03it/s]
  7%|▋         | 845/11346 [13:46<2:49:58,  1.03it/s]
  7%|▋         | 846/11346 [13:47<2:50:00,  1.03it/s]
  7%|▋         | 847/11346 [13:48<2:50:03,  1.03it/s]
  7%|▋         | 848/11346 [13:48<2:50:03,  1.03it/s]
  7%|▋         | 849/11346 [13:49<2:50:05,  1.03it/s]
  7%|▋         | 850/11346 [13:50<2:50:01,  1.03it/s]
  8%|▊         | 851/11346 [13:51<2:50:09,  1.03it/s]
  8%|▊         | 852/11346 [13:52<2:50:04,  1.03it/s]
  8%|▊         | 853/11346 [13:53<2:50:03,  1.03it/s]
  8%|▊         | 854/11346 [13:54<2:50:04,  1.03it/s]
  8%|▊         | 855/11346 [13:55<2:50:01,  1.03it/s]
  8%|▊         | 856/11346 [13:56<2:49:57,  1.03it/s]
  8%|▊         | 857/11346 [13:57<2:49:57,  1.03it/s]
  8%|▊         | 858/11346 [13:58<2:49:55,  1.03it/s]
  8%|▊         | 859/11346 [13:59<2:49:52,  1.03it/s]
  8%|▊         | 860/11346 [14:00<2:49:51,  1.03it/s]
  8%|▊         | 861/11346 [14:01<2:49:48,  1.03it/s]
  8%|▊         | 862/11346 [14:02<2:49:46,  1.03it/s]
  8%|▊         | 863/11346 [14:03<2:49:45,  1.03it/s]
  8%|▊         | 864/11346 [14:04<2:49:47,  1.03it/s]
  8%|▊         | 865/11346 [14:05<2:49:48,  1.03it/s]
  8%|▊         | 866/11346 [14:06<2:49:48,  1.03it/s]
  8%|▊         | 867/11346 [14:07<2:49:48,  1.03it/s]
  8%|▊         | 868/11346 [14:08<2:49:51,  1.03it/s]
  8%|▊         | 869/11346 [14:09<2:49:48,  1.03it/s]
  8%|▊         | 870/11346 [14:10<2:49:48,  1.03it/s]
  8%|▊         | 871/11346 [14:11<2:49:43,  1.03it/s]
  8%|▊         | 872/11346 [14:12<2:49:38,  1.03it/s]
  8%|▊         | 873/11346 [14:13<2:49:38,  1.03it/s]
  8%|▊         | 874/11346 [14:14<2:49:41,  1.03it/s]
  8%|▊         | 875/11346 [14:15<2:49:38,  1.03it/s]
  8%|▊         | 876/11346 [14:16<2:49:35,  1.03it/s]
  8%|▊         | 877/11346 [14:17<2:49:34,  1.03it/s]
  8%|▊         | 878/11346 [14:18<2:49:31,  1.03it/s]
  8%|▊         | 879/11346 [14:19<2:49:34,  1.03it/s]
  8%|▊         | 880/11346 [14:20<2:49:32,  1.03it/s]
  8%|▊         | 881/11346 [14:21<2:49:32,  1.03it/s]
  8%|▊         | 882/11346 [14:22<2:49:39,  1.03it/s]
  8%|▊         | 883/11346 [14:23<2:49:32,  1.03it/s]
  8%|▊         | 884/11346 [14:23<2:49:25,  1.03it/s]
  8%|▊         | 885/11346 [14:24<2:49:24,  1.03it/s]
  8%|▊         | 886/11346 [14:25<2:49:23,  1.03it/s]
  8%|▊         | 887/11346 [14:26<2:49:24,  1.03it/s]
  8%|▊         | 888/11346 [14:27<2:49:25,  1.03it/s]
  8%|▊         | 889/11346 [14:28<2:49:21,  1.03it/s]
  8%|▊         | 890/11346 [14:29<2:49:17,  1.03it/s]
  8%|▊         | 891/11346 [14:30<2:49:20,  1.03it/s]
  8%|▊         | 892/11346 [14:31<2:49:22,  1.03it/s]
  8%|▊         | 893/11346 [14:32<2:49:17,  1.03it/s]
  8%|▊         | 894/11346 [14:33<2:49:19,  1.03it/s]
  8%|▊         | 895/11346 [14:34<2:49:22,  1.03it/s]
  8%|▊         | 896/11346 [14:35<2:49:19,  1.03it/s]
  8%|▊         | 897/11346 [14:36<2:49:17,  1.03it/s]
  8%|▊         | 898/11346 [14:37<2:49:24,  1.03it/s]
  8%|▊         | 899/11346 [14:38<2:49:20,  1.03it/s]
  8%|▊         | 900/11346 [14:39<2:49:14,  1.03it/s]
  8%|▊         | 901/11346 [14:40<2:49:16,  1.03it/s]
  8%|▊         | 902/11346 [14:41<2:49:12,  1.03it/s]
  8%|▊         | 903/11346 [14:42<2:49:08,  1.03it/s]
  8%|▊         | 904/11346 [14:43<2:49:13,  1.03it/s]
  8%|▊         | 905/11346 [14:44<2:49:17,  1.03it/s]
  8%|▊         | 906/11346 [14:45<2:49:15,  1.03it/s]
  8%|▊         | 907/11346 [14:46<2:49:08,  1.03it/s]
  8%|▊         | 908/11346 [14:47<2:49:09,  1.03it/s]
  8%|▊         | 909/11346 [14:48<2:49:06,  1.03it/s]
  8%|▊         | 910/11346 [14:49<2:49:07,  1.03it/s]
  8%|▊         | 911/11346 [14:50<2:49:04,  1.03it/s]
  8%|▊         | 912/11346 [14:51<2:49:13,  1.03it/s]
  8%|▊         | 913/11346 [14:52<2:49:08,  1.03it/s]
  8%|▊         | 914/11346 [14:53<2:49:06,  1.03it/s]
  8%|▊         | 915/11346 [14:54<2:49:03,  1.03it/s]
  8%|▊         | 916/11346 [14:55<2:49:04,  1.03it/s]
  8%|▊         | 917/11346 [14:56<2:49:02,  1.03it/s]
  8%|▊         | 918/11346 [14:57<2:49:00,  1.03it/s]
  8%|▊         | 919/11346 [14:58<2:48:58,  1.03it/s]
  8%|▊         | 920/11346 [14:58<2:49:00,  1.03it/s]
  8%|▊         | 921/11346 [14:59<2:48:58,  1.03it/s]
  8%|▊         | 922/11346 [15:00<2:49:00,  1.03it/s]
  8%|▊         | 923/11346 [15:01<2:49:00,  1.03it/s]
  8%|▊         | 924/11346 [15:02<2:48:57,  1.03it/s]
  8%|▊         | 925/11346 [15:03<2:48:52,  1.03it/s]
  8%|▊         | 926/11346 [15:04<2:48:47,  1.03it/s]
  8%|▊         | 927/11346 [15:05<2:48:45,  1.03it/s]
  8%|▊         | 928/11346 [15:06<2:48:44,  1.03it/s]
  8%|▊         | 929/11346 [15:07<2:48:45,  1.03it/s]
  8%|▊         | 930/11346 [15:08<2:48:47,  1.03it/s]
  8%|▊         | 931/11346 [15:09<2:48:46,  1.03it/s]
  8%|▊         | 932/11346 [15:10<2:48:43,  1.03it/s]
  8%|▊         | 933/11346 [15:11<2:48:43,  1.03it/s]
  8%|▊         | 934/11346 [15:12<2:48:42,  1.03it/s]
  8%|▊         | 935/11346 [15:13<2:48:41,  1.03it/s]
  8%|▊         | 936/11346 [15:14<2:48:37,  1.03it/s]
  8%|▊         | 937/11346 [15:15<2:48:32,  1.03it/s]
  8%|▊         | 938/11346 [15:16<2:48:42,  1.03it/s]
  8%|▊         | 939/11346 [15:17<2:48:41,  1.03it/s]
  8%|▊         | 940/11346 [15:18<2:48:38,  1.03it/s]
  8%|▊         | 941/11346 [15:19<2:48:35,  1.03it/s]
  8%|▊         | 942/11346 [15:20<2:48:35,  1.03it/s]
  8%|▊         | 943/11346 [15:21<2:48:30,  1.03it/s]
  8%|▊         | 944/11346 [15:22<2:48:26,  1.03it/s]
  8%|▊         | 945/11346 [15:23<2:48:24,  1.03it/s]
  8%|▊         | 946/11346 [15:24<2:48:26,  1.03it/s]
  8%|▊         | 947/11346 [15:25<2:48:34,  1.03it/s]
  8%|▊         | 948/11346 [15:26<2:48:28,  1.03it/s]
  8%|▊         | 949/11346 [15:27<2:48:28,  1.03it/s]
  8%|▊         | 950/11346 [15:28<2:48:30,  1.03it/s]
  8%|▊         | 951/11346 [15:29<2:48:30,  1.03it/s]
  8%|▊         | 952/11346 [15:30<2:48:31,  1.03it/s]
  8%|▊         | 953/11346 [15:31<2:48:28,  1.03it/s]
  8%|▊         | 954/11346 [15:32<2:48:31,  1.03it/s]
  8%|▊         | 955/11346 [15:33<2:48:25,  1.03it/s]
  8%|▊         | 956/11346 [15:33<2:48:19,  1.03it/s]
  8%|▊         | 957/11346 [15:34<2:48:20,  1.03it/s]
  8%|▊         | 958/11346 [15:35<2:48:17,  1.03it/s]
  8%|▊         | 959/11346 [15:36<2:48:16,  1.03it/s]
  8%|▊         | 960/11346 [15:37<2:48:15,  1.03it/s]
  8%|▊         | 961/11346 [15:38<2:48:15,  1.03it/s]
  8%|▊         | 962/11346 [15:39<2:48:13,  1.03it/s]
  8%|▊         | 963/11346 [15:40<2:48:11,  1.03it/s]
  8%|▊         | 964/11346 [15:41<2:48:13,  1.03it/s]
  9%|▊         | 965/11346 [15:42<2:48:12,  1.03it/s]
  9%|▊         | 966/11346 [15:43<2:48:09,  1.03it/s]
  9%|▊         | 967/11346 [15:44<2:48:09,  1.03it/s]
  9%|▊         | 968/11346 [15:45<2:48:11,  1.03it/s]
  9%|▊         | 969/11346 [15:46<2:48:09,  1.03it/s]
  9%|▊         | 970/11346 [15:47<2:48:04,  1.03it/s]
  9%|▊         | 971/11346 [15:48<2:48:03,  1.03it/s]
  9%|▊         | 972/11346 [15:49<2:48:06,  1.03it/s]
  9%|▊         | 973/11346 [15:50<2:48:06,  1.03it/s]
  9%|▊         | 974/11346 [15:51<2:48:00,  1.03it/s]
  9%|▊         | 975/11346 [15:52<2:48:02,  1.03it/s]
  9%|▊         | 976/11346 [15:53<2:48:03,  1.03it/s]
  9%|▊         | 977/11346 [15:54<2:48:03,  1.03it/s]
  9%|▊         | 978/11346 [15:55<2:47:59,  1.03it/s]
  9%|▊         | 979/11346 [15:56<2:47:59,  1.03it/s]
  9%|▊         | 980/11346 [15:57<2:47:57,  1.03it/s]
  9%|▊         | 981/11346 [15:58<2:47:51,  1.03it/s]
  9%|▊         | 982/11346 [15:59<2:47:47,  1.03it/s]
  9%|▊         | 983/11346 [16:00<2:47:45,  1.03it/s]
  9%|▊         | 984/11346 [16:01<2:47:51,  1.03it/s]
  9%|▊         | 985/11346 [16:02<2:47:50,  1.03it/s]
  9%|▊         | 986/11346 [16:03<2:47:49,  1.03it/s]
  9%|▊         | 987/11346 [16:04<2:47:50,  1.03it/s]
  9%|▊         | 988/11346 [16:05<2:47:47,  1.03it/s]
  9%|▊         | 989/11346 [16:06<2:47:46,  1.03it/s]
  9%|▊         | 990/11346 [16:07<2:47:42,  1.03it/s]
  9%|▊         | 991/11346 [16:08<2:47:49,  1.03it/s]
  9%|▊         | 992/11346 [16:08<2:47:49,  1.03it/s]
  9%|▉         | 993/11346 [16:09<2:47:48,  1.03it/s]
  9%|▉         | 994/11346 [16:10<2:47:55,  1.03it/s]
  9%|▉         | 995/11346 [16:11<2:47:53,  1.03it/s]
  9%|▉         | 996/11346 [16:12<2:47:48,  1.03it/s]
  9%|▉         | 997/11346 [16:13<2:47:48,  1.03it/s]
  9%|▉         | 998/11346 [16:14<2:47:47,  1.03it/s]
  9%|▉         | 999/11346 [16:15<2:47:44,  1.03it/s]
  9%|▉         | 1000/11346 [16:16<2:47:39,  1.03it/s]
                                                      
  9%|▉         | 1000/11346 [16:16<2:47:39,  1.03it/s][INFO|trainer.py:3662] 2024-06-05 03:21:57,117 >> ***** Running Evaluation *****
+[INFO|trainer.py:3664] 2024-06-05 03:21:57,117 >>   Num examples = 1840
+[INFO|trainer.py:3667] 2024-06-05 03:21:57,117 >>   Batch size = 48
+{'loss': 5.1118, 'grad_norm': 0.8546391725540161, 'learning_rate': 8.816009873931059e-05, 'epoch': 0.13}
+{'loss': 3.406, 'grad_norm': 0.8593688607215881, 'learning_rate': 9.59831475011252e-05, 'epoch': 0.26}
+
+
  0%|          | 0/39 [00:00<?, ?it/s]
+
  5%|▌         | 2/39 [00:01<00:27,  1.34it/s]
+
  8%|▊         | 3/39 [00:02<00:37,  1.05s/it]
+
 10%|█         | 4/39 [00:04<00:42,  1.22s/it]
+
 13%|█▎        | 5/39 [00:05<00:44,  1.31s/it]
+
 15%|█▌        | 6/39 [00:07<00:45,  1.37s/it]
+
 18%|█▊        | 7/39 [00:08<00:44,  1.41s/it]
+
 21%|██        | 8/39 [00:10<00:44,  1.43s/it]
+
 23%|██▎       | 9/39 [00:11<00:43,  1.45s/it]
+
 26%|██▌       | 10/39 [00:13<00:42,  1.46s/it]
+
 28%|██▊       | 11/39 [00:14<00:41,  1.47s/it]
+
 31%|███       | 12/39 [00:16<00:39,  1.47s/it]
+
 33%|███▎      | 13/39 [00:17<00:38,  1.48s/it]
+
 36%|███▌      | 14/39 [00:19<00:36,  1.48s/it]
+
 38%|███▊      | 15/39 [00:20<00:35,  1.48s/it]
+
 41%|████      | 16/39 [00:22<00:34,  1.48s/it]
+
 44%|████▎     | 17/39 [00:23<00:32,  1.48s/it]
+
 46%|████▌     | 18/39 [00:25<00:31,  1.48s/it]
+
 49%|████▊     | 19/39 [00:26<00:29,  1.48s/it]
+
 51%|█████▏    | 20/39 [00:28<00:28,  1.48s/it]
+
 54%|█████▍    | 21/39 [00:29<00:26,  1.48s/it]
+
 56%|█████▋    | 22/39 [00:31<00:25,  1.48s/it]
+
 59%|█████▉    | 23/39 [00:32<00:23,  1.48s/it]
+
 62%|██████▏   | 24/39 [00:34<00:22,  1.48s/it]
+
 64%|██████▍   | 25/39 [00:35<00:20,  1.48s/it]
+
 67%|██████▋   | 26/39 [00:37<00:19,  1.48s/it]
+
 69%|██████▉   | 27/39 [00:38<00:17,  1.48s/it]
+
 72%|███████▏  | 28/39 [00:40<00:16,  1.48s/it]
+
 74%|███████▍  | 29/39 [00:41<00:14,  1.48s/it]
+
 77%|███████▋  | 30/39 [00:43<00:13,  1.48s/it]
+
 79%|███████▉  | 31/39 [00:44<00:11,  1.48s/it]
+
 82%|████████▏ | 32/39 [00:46<00:10,  1.48s/it]
+
 85%|████████▍ | 33/39 [00:47<00:08,  1.48s/it]
+
 87%|████████▋ | 34/39 [00:48<00:07,  1.48s/it]
+
 90%|████████▉ | 35/39 [00:50<00:05,  1.48s/it]
+
 92%|█████████▏| 36/39 [00:51<00:04,  1.48s/it]
+
 95%|█████████▍| 37/39 [00:53<00:02,  1.48s/it]
+
 97%|█████████▋| 38/39 [00:54<00:01,  1.46s/it]
+
                                                      
+
                                               
  9%|▉         | 1000/11346 [17:30<2:47:39,  1.03it/s]
+
+
                                               [INFO|trainer.py:3353] 2024-06-05 03:23:11,169 >> Saving model checkpoint to ./training_outputs_job_117568_1_05-06_03-05/checkpoint-1000
+[INFO|configuration_utils.py:471] 2024-06-05 03:23:11,183 >> Configuration saved in ./training_outputs_job_117568_1_05-06_03-05/checkpoint-1000/config.json
+[INFO|configuration_utils.py:705] 2024-06-05 03:23:11,189 >> Configuration saved in ./training_outputs_job_117568_1_05-06_03-05/checkpoint-1000/generation_config.json
+[INFO|modeling_utils.py:2592] 2024-06-05 03:23:12,091 >> Model weights saved in ./training_outputs_job_117568_1_05-06_03-05/checkpoint-1000/model.safetensors
+[INFO|tokenization_utils_base.py:2503] 2024-06-05 03:23:12,105 >> tokenizer config file saved in ./training_outputs_job_117568_1_05-06_03-05/checkpoint-1000/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2512] 2024-06-05 03:23:12,110 >> Special tokens file saved in ./training_outputs_job_117568_1_05-06_03-05/checkpoint-1000/special_tokens_map.json
+[INFO|tokenization_utils_base.py:2503] 2024-06-05 03:23:14,277 >> tokenizer config file saved in ./training_outputs_job_117568_1_05-06_03-05/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2512] 2024-06-05 03:23:14,282 >> Special tokens file saved in ./training_outputs_job_117568_1_05-06_03-05/special_tokens_map.json
+/home/dshteyma/miniconda3/lib/python3.9/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
+  warnings.warn('Was asked to gather along dimension 0, but all '

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c47fea650ec741b07e6a187541e4217459975b8d59c02f835544382c220ca692
+size 272123144

pip_freeze.txt

@@ -0,0 +1,330 @@
+absl-py==2.1.0
+accelerate==0.26.1
+aiofiles==23.2.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+altair==5.3.0
+annotated-types==0.6.0
+antlr4-python3-runtime==4.9.3
+anyio==4.0.0
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==2.4.0
+astunparse==1.6.3
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.1.0
+auto-gptq==0.6.0
+Babel==2.13.0
+backcall @ file:///home/ktietz/src/ci/backcall_1611930011877/work
+beartype==0.17.2
+beautifulsoup4==4.12.2
+bitsandbytes==0.43.1
+bleach==6.1.0
+blis==0.7.11
+brotlipy==0.7.0
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2023.7.22
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.0
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+colorama @ file:///tmp/build/80754af9/colorama_1607707115595/work
+coloredlogs==15.0.1
+comm==0.1.4
+conda==4.12.0
+conda-content-trust @ file:///tmp/build/80754af9/conda-content-trust_1617045594566/work
+conda-package-handling @ file:///tmp/build/80754af9/conda-package-handling_1649105784853/work
+confection==0.1.4
+contextlib2==21.6.0
+contexttimer==0.3.3
+contourpy==1.1.1
+cryptography @ file:///tmp/build/80754af9/cryptography_1639414572950/work
+cycler==0.12.1
+cymem==2.0.8
+dataclasses-json==0.6.4
+DataProperty==1.0.1
+datasets==2.19.1
+debugpy==1.8.0
+decorator @ file:///opt/conda/conda-bld/decorator_1643638310831/work
+defusedxml==0.7.1
+dill==0.3.7
+dnspython==2.6.1
+docstring_parser==0.16
+dos2unix==1
+einops==0.8.0
+eval_type_backport==0.2.0
+evaluate==0.4.1
+exceptiongroup==1.1.3
+executing==2.0.0
+fastapi==0.111.0
+fastapi-cli==0.0.2
+fastchat==0.1.0
+fastjsonschema==2.18.1
+ffmpy==0.3.2
+filelock==3.12.4
+fire==0.5.0
+flash-attn==2.5.8
+flatbuffers==23.5.26
+fonttools==4.43.1
+fqdn==1.5.1
+frozenlist==1.4.0
+fschat==0.2.36
+fsspec==2023.6.0
+gast==0.5.4
+gekko==1.0.6
+globals==0.3.36
+google-auth==2.27.0
+google-auth-oauthlib==1.2.0
+google-pasta==0.2.0
+gradio==4.29.0
+gradio_client==0.16.1
+greenlet==3.0.3
+grpcio==1.60.1
+h11==0.14.0
+h5py==3.10.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.22.2
+humanfriendly==10.0
+hydra-core==1.3.2
+hydra-joblib-launcher==1.2.0
+hydra-submitit-launcher==1.2.0
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.1.0
+ipykernel==6.25.2
+ipython==8.18.1
+isoduration==20.11.0
+jedi==0.19.1
+Jinja2==3.1.2
+joblib==1.3.2
+json5==0.9.14
+jsonlines==4.0.0
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.19.1
+jsonschema-specifications==2023.7.1
+jupyter-events==0.7.0
+jupyter-lsp==2.2.0
+jupyter_client==8.3.1
+jupyter_core==5.3.2
+jupyter_server==2.7.3
+jupyter_server_terminals==0.4.4
+jupyterlab==4.0.6
+jupyterlab-pygments==0.2.2
+jupyterlab_server==2.25.0
+keras==2.15.0
+kiwisolver==1.4.5
+langchain==0.1.8
+langchain-community==0.0.21
+langchain-core==0.1.25
+langcodes==3.3.0
+langdetect==1.0.9
+langsmith==0.1.5
+libclang==16.0.6
+lxml==5.1.0
+Markdown==3.5.2
+markdown-it-py==3.0.0
+markdown2==2.4.13
+MarkupSafe==2.1.5
+marshmallow==3.20.2
+matplotlib==3.8.0
+matplotlib-inline @ file:///opt/conda/conda-bld/matplotlib-inline_1662014470464/work
+mbstrdecoder==1.1.3
+mdurl==0.1.2
+mistune==3.0.2
+ml-collections==0.1.1
+ml-dtypes==0.2.0
+more-itertools==10.2.0
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.15
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.8.0
+nbconvert==7.9.2
+nbformat==5.9.2
+nest-asyncio==1.5.8
+networkx==3.1
+nh3==0.2.17
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==7.0.4
+notebook_shim==0.2.3
+numexpr==2.9.0
+numpy==1.26.0
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-ml-py3==7.352.0
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.2.140
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+opt-einsum==3.3.0
+optimum==1.16.2
+orjson==3.10.3
+overrides==7.4.0
+packaging==23.2
+pandas==2.1.1
+pandocfilters==1.5.0
+parso @ file:///opt/conda/conda-bld/parso_1641458642106/work
+pathvalidate==3.2.0
+patsy==0.5.3
+peft==0.8.2
+pexpect @ file:///tmp/build/80754af9/pexpect_1605563209008/work
+pickleshare @ file:///tmp/build/80754af9/pickleshare_1606932040724/work
+Pillow==10.0.1
+platformdirs==3.11.0
+plotly==5.17.0
+plotly-express==0.4.1
+portalocker==2.8.2
+preshed==3.0.9
+prometheus-client==0.17.1
+prompt-toolkit==3.0.43
+protobuf==3.20.3
+psutil==5.9.5
+ptyprocess @ file:///tmp/build/80754af9/ptyprocess_1609355006118/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl
+pure-eval @ file:///opt/conda/conda-bld/pure_eval_1646925070566/work
+pyarrow==13.0.0
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pybind11==2.11.1
+pycosat==0.6.3
+pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
+pydantic==2.6.1
+pydantic_core==2.16.2
+pydub==0.25.1
+Pygments==2.16.1
+pyOpenSSL @ file:///opt/conda/conda-bld/pyopenssl_1643788558760/work
+pyparsing==3.1.1
+PySocks @ file:///tmp/build/80754af9/pysocks_1605305812635/work
+pytablewriter==1.2.0
+python-dateutil==2.8.2
+python-dotenv==1.0.1
+python-helper==0.3.74
+python-json-logger==2.0.7
+python-multipart==0.0.9
+pytz==2023.3.post1
+PyYAML==6.0.1
+pyzmq==25.1.1
+referencing==0.30.2
+regex==2023.10.3
+requests==2.31.0
+requests-oauthlib==1.3.1
+responses==0.18.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.1
+rotary-embedding-torch==0.5.3
+rouge==1.0.1
+rouge-score==0.1.2
+rpds-py==0.10.4
+rsa==4.9
+ruamel-yaml-conda @ file:///tmp/build/80754af9/ruamel_yaml_1616016711199/work
+ruff==0.4.3
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.4.1.post1
+scipy==1.11.3
+seaborn==0.13.0
+semantic-version==2.10.0
+Send2Trash==1.8.2
+sentencepiece==0.2.0
+shellingham==1.5.4
+shortuuid==1.0.13
+shtab==1.7.1
+six @ file:///tmp/build/80754af9/six_1644875935023/work
+smart-open==6.4.0
+sniffio==1.3.0
+soupsieve==2.5
+spacy==3.7.4
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+speculative-decoding==0.1.2
+SQLAlchemy==2.0.27
+sqlitedict==2.1.0
+srsly==2.4.8
+stack-data==0.6.3
+starlette==0.37.2
+statsmodels==0.14.0
+submitit==1.5.1
+svgwrite==1.4.3
+sympy==1.12
+tabledata==1.3.3
+tabulate==0.9.0
+tcolorpy==0.1.4
+tenacity==8.2.3
+tensorboard==2.15.1
+tensorboard-data-server==0.7.2
+tensorflow==2.15.0.post1
+tensorflow-estimator==2.15.0
+tensorflow-io-gcs-filesystem==0.35.0
+tensorrt==8.6.1.post1
+tensorrt-bindings==8.6.1
+tensorrt-libs==8.6.1
+termcolor==2.4.0
+terminado==0.17.1
+thinc==8.2.3
+threadpoolctl==3.3.0
+tiktoken==0.6.0
+tinycss2==1.2.1
+tk==0.1.0
+tokenizers==0.19.1
+tomli==2.0.1
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.1.0
+torchaudio==2.1.0
+torchvision==0.16.0
+tornado==6.3.3
+tqdm==4.66.1
+tqdm-multiprocess==0.0.11
+traitlets==5.11.2
+-e git+https://github.com/huggingface/transformers.git@bbaa8ceff696c479aecdb4575b2deb1349efd3aa#egg=transformers
+triton==2.1.0
+trl==0.8.6
+typepy==1.3.2
+typer==0.12.3
+types-python-dateutil==2.8.19.14
+typing-inspect==0.9.0
+typing_extensions==4.8.0
+tyro==0.8.3
+tzdata==2023.3
+ujson==5.9.0
+unsloth @ git+https://github.com/unslothai/unsloth.git@4211cc01409e3ced4f7abebaf68e244193b46e2c
+uri-template==1.3.0
+urllib3==2.0.6
+uvicorn==0.29.0
+uvloop==0.19.0
+wasabi==1.1.2
+watchfiles==0.21.0
+wavedrom==2.0.3.post3
+wcwidth==0.2.8
+weasel==0.3.4
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.6.4
+websockets==11.0.3
+Werkzeug==3.0.1
+word2number==1.1
+wrapt==1.14.1
+xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp39-cp39-manylinux2014_x86_64.whl
+xxhash==3.4.1
+yarl==1.9.2
+zipp==3.17.0
+zstandard==0.22.0

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "<unk>",
+  "padding": "max_length",
+  "return_tensors": "pt",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84a8c3b8deb94d253a20c7d637102edeaa129f75276f850abb181ddfd4b1ddcf
+size 5176