| 2024-08-30 20:25:26.549777: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. | |
| 2024-08-30 20:25:26.568217: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered | |
| 2024-08-30 20:25:26.590253: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered | |
| 2024-08-30 20:25:26.597224: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered | |
| 2024-08-30 20:25:26.612962: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. | |
| To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2024-08-30 20:25:27.916342: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT | |
| /usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1494: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead | |
| warnings.warn( | |
| 08/30/2024 20:25:29 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: False | |
| 08/30/2024 20:25:29 - INFO - __main__ - Training/evaluation parameters TrainingArguments( | |
| _n_gpu=1, | |
| accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None, 'use_configured_state': False}, | |
| adafactor=False, | |
| adam_beta1=0.9, | |
| adam_beta2=0.999, | |
| adam_epsilon=1e-08, | |
| auto_find_batch_size=False, | |
| batch_eval_metrics=False, | |
| bf16=False, | |
| bf16_full_eval=False, | |
| data_seed=None, | |
| dataloader_drop_last=False, | |
| dataloader_num_workers=0, | |
| dataloader_persistent_workers=False, | |
| dataloader_pin_memory=True, | |
| dataloader_prefetch_factor=None, | |
| ddp_backend=None, | |
| ddp_broadcast_buffers=None, | |
| ddp_bucket_cap_mb=None, | |
| ddp_find_unused_parameters=None, | |
| ddp_timeout=1800, | |
| debug=[], | |
| deepspeed=None, | |
| disable_tqdm=False, | |
| dispatch_batches=None, | |
| do_eval=True, | |
| do_predict=True, | |
| do_train=True, | |
| eval_accumulation_steps=None, | |
| eval_delay=0, | |
| eval_do_concat_batches=True, | |
| eval_on_start=False, | |
| eval_steps=None, | |
| eval_strategy=epoch, | |
| evaluation_strategy=epoch, | |
| fp16=False, | |
| fp16_backend=auto, | |
| fp16_full_eval=False, | |
| fp16_opt_level=O1, | |
| fsdp=[], | |
| fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, | |
| fsdp_min_num_params=0, | |
| fsdp_transformer_layer_cls_to_wrap=None, | |
| full_determinism=False, | |
| gradient_accumulation_steps=2, | |
| gradient_checkpointing=False, | |
| gradient_checkpointing_kwargs=None, | |
| greater_is_better=True, | |
| group_by_length=False, | |
| half_precision_backend=auto, | |
| hub_always_push=False, | |
| hub_model_id=None, | |
| hub_private_repo=False, | |
| hub_strategy=every_save, | |
| hub_token=<HUB_TOKEN>, | |
| ignore_data_skip=False, | |
| include_inputs_for_metrics=False, | |
| include_num_input_tokens_seen=False, | |
| include_tokens_per_second=False, | |
| jit_mode_eval=False, | |
| label_names=None, | |
| label_smoothing_factor=0.0, | |
| learning_rate=5e-05, | |
| length_column_name=length, | |
| load_best_model_at_end=True, | |
| local_rank=0, | |
| log_level=passive, | |
| log_level_replica=warning, | |
| log_on_each_node=True, | |
| logging_dir=/content/dissertation/scripts/ner/output/tb, | |
| logging_first_step=False, | |
| logging_nan_inf_filter=True, | |
| logging_steps=500, | |
| logging_strategy=steps, | |
| lr_scheduler_kwargs={}, | |
| lr_scheduler_type=linear, | |
| max_grad_norm=1.0, | |
| max_steps=-1, | |
| metric_for_best_model=f1, | |
| mp_parameters=, | |
| neftune_noise_alpha=None, | |
| no_cuda=False, | |
| num_train_epochs=10.0, | |
| optim=adamw_torch, | |
| optim_args=None, | |
| optim_target_modules=None, | |
| output_dir=/content/dissertation/scripts/ner/output, | |
| overwrite_output_dir=True, | |
| past_index=-1, | |
| per_device_eval_batch_size=8, | |
| per_device_train_batch_size=32, | |
| prediction_loss_only=False, | |
| push_to_hub=True, | |
| push_to_hub_model_id=None, | |
| push_to_hub_organization=None, | |
| push_to_hub_token=<PUSH_TO_HUB_TOKEN>, | |
| ray_scope=last, | |
| remove_unused_columns=True, | |
| report_to=['tensorboard'], | |
| restore_callback_states_from_checkpoint=False, | |
| resume_from_checkpoint=None, | |
| run_name=/content/dissertation/scripts/ner/output, | |
| save_on_each_node=False, | |
| save_only_model=False, | |
| save_safetensors=True, | |
| save_steps=500, | |
| save_strategy=epoch, | |
| save_total_limit=None, | |
| seed=42, | |
| skip_memory_metrics=True, | |
| split_batches=None, | |
| tf32=None, | |
| torch_compile=False, | |
| torch_compile_backend=None, | |
| torch_compile_mode=None, | |
| torchdynamo=None, | |
| tpu_metrics_debug=False, | |
| tpu_num_cores=None, | |
| use_cpu=False, | |
| use_ipex=False, | |
| use_legacy_prediction_loop=False, | |
| use_mps_device=False, | |
| warmup_ratio=0.0, | |
| warmup_steps=0, | |
| weight_decay=0.0, | |
| ) | |
| Downloading builder script: 0%| | 0.00/3.54k [00:00<?, ?B/s] Downloading builder script: 100%|██████████| 3.54k/3.54k [00:00<00:00, 38.9kB/s] | |
| Downloading data: 0%| | 0.00/27.6M [00:00<?, ?B/s] Downloading data: 38%|███▊ | 10.5M/27.6M [00:00<00:00, 17.2MB/s] Downloading data: 76%|███████▌ | 21.0M/27.6M [00:00<00:00, 28.2MB/s] Downloading data: 100%|██████████| 27.6M/27.6M [00:00<00:00, 30.4MB/s] | |
| Downloading data: 0%| | 0.00/6.96M [00:00<?, ?B/s] Downloading data: 100%|██████████| 6.96M/6.96M [00:00<00:00, 9.97MB/s] Downloading data: 100%|██████████| 6.96M/6.96M [00:00<00:00, 9.90MB/s] | |
| Downloading data: 0%| | 0.00/12.3M [00:00<?, ?B/s] Downloading data: 85%|████████▌ | 10.5M/12.3M [00:00<00:00, 14.4MB/s] Downloading data: 100%|██████████| 12.3M/12.3M [00:00<00:00, 15.7MB/s] | |
| Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 680 examples [00:00, 6770.03 examples/s] Generating train split: 1700 examples [00:00, 6444.09 examples/s] Generating train split: 2366 examples [00:00, 6429.84 examples/s] Generating train split: 3354 examples [00:00, 6497.31 examples/s] Generating train split: 4366 examples [00:00, 6591.28 examples/s] Generating train split: 5026 examples [00:00, 6590.04 examples/s] Generating train split: 5752 examples [00:00, 6774.12 examples/s] Generating train split: 6734 examples [00:01, 6683.22 examples/s] Generating train split: 7743 examples [00:01, 6692.84 examples/s] Generating train split: 8733 examples [00:01, 6654.49 examples/s] Generating train split: 9716 examples [00:01, 6618.43 examples/s] Generating train split: 10731 examples [00:01, 6660.28 examples/s] Generating train split: 11751 examples [00:01, 6698.38 examples/s] Generating train split: 12779 examples [00:01, 6743.63 examples/s] Generating train split: 13741 examples [00:02, 6639.47 examples/s] Generating train split: 14750 examples [00:02, 6661.83 examples/s] Generating train split: 15753 examples [00:02, 6667.19 examples/s] Generating train split: 16770 examples [00:02, 6696.94 examples/s] Generating train split: 17747 examples [00:02, 6632.86 examples/s] Generating train split: 18718 examples [00:02, 6581.28 examples/s] Generating train split: 19734 examples [00:02, 6632.61 examples/s] Generating train split: 20717 examples [00:03, 6602.84 examples/s] Generating train split: 21721 examples [00:03, 6627.21 examples/s] Generating train split: 22396 examples [00:03, 6650.95 examples/s] Generating train split: 23379 examples [00:03, 6615.02 examples/s] Generating train split: 24364 examples [00:03, 6593.55 examples/s] Generating train split: 25347 examples [00:03, 6543.21 examples/s] Generating train split: 26334 examples [00:03, 6535.03 examples/s] Generating train split: 27000 examples [00:04, 6443.51 examples/s] Generating train split: 27229 examples [00:04, 6605.43 examples/s] | |
| Generating validation split: 0 examples [00:00, ? examples/s] Generating validation split: 707 examples [00:00, 7037.34 examples/s] Generating validation split: 1683 examples [00:00, 6583.30 examples/s] Generating validation split: 2366 examples [00:00, 6499.90 examples/s] Generating validation split: 3358 examples [00:00, 6523.55 examples/s] Generating validation split: 4347 examples [00:00, 6486.94 examples/s] Generating validation split: 5000 examples [00:00, 6445.93 examples/s] Generating validation split: 5723 examples [00:00, 6658.31 examples/s] Generating validation split: 6737 examples [00:01, 6690.91 examples/s] Generating validation split: 6810 examples [00:01, 6559.78 examples/s] | |
| Generating test split: 0 examples [00:00, ? examples/s] Generating test split: 781 examples [00:00, 7779.50 examples/s] Generating test split: 1895 examples [00:00, 7535.97 examples/s] Generating test split: 3000 examples [00:00, 7402.47 examples/s] Generating test split: 3773 examples [00:00, 7504.86 examples/s] Generating test split: 4877 examples [00:00, 7443.50 examples/s] Generating test split: 5656 examples [00:00, 7538.62 examples/s] Generating test split: 6467 examples [00:00, 7699.53 examples/s] Generating test split: 7244 examples [00:00, 7715.50 examples/s] Generating test split: 8436 examples [00:01, 7798.01 examples/s] Generating test split: 9636 examples [00:01, 7864.66 examples/s] Generating test split: 10476 examples [00:01, 7824.56 examples/s] Generating test split: 11677 examples [00:01, 7882.93 examples/s] Generating test split: 12646 examples [00:01, 5028.97 examples/s] Generating test split: 13401 examples [00:01, 5421.91 examples/s] Generating test split: 14168 examples [00:02, 5869.47 examples/s] Generating test split: 14614 examples [00:02, 6796.13 examples/s] | |
| [INFO|configuration_utils.py:733] 2024-08-30 20:25:41,919 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--PlanTL-GOB-ES--bsc-bio-ehr-es/snapshots/1e543adb2d21f19d85a89305eebdbd64ab656b99/config.json | |
| [INFO|configuration_utils.py:800] 2024-08-30 20:25:41,923 >> Model config RobertaConfig { | |
| "_name_or_path": "PlanTL-GOB-ES/bsc-bio-ehr-es", | |
| "architectures": [ | |
| "RobertaForMaskedLM" | |
| ], | |
| "attention_probs_dropout_prob": 0.1, | |
| "bos_token_id": 0, | |
| "classifier_dropout": null, | |
| "eos_token_id": 2, | |
| "finetuning_task": "ner", | |
| "gradient_checkpointing": false, | |
| "hidden_act": "gelu", | |
| "hidden_dropout_prob": 0.1, | |
| "hidden_size": 768, | |
| "id2label": { | |
| "0": "O", | |
| "1": "B-ENFERMEDAD", | |
| "2": "I-ENFERMEDAD" | |
| }, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 3072, | |
| "label2id": { | |
| "B-ENFERMEDAD": 1, | |
| "I-ENFERMEDAD": 2, | |
| "O": 0 | |
| }, | |
| "layer_norm_eps": 1e-05, | |
| "max_position_embeddings": 514, | |
| "model_type": "roberta", | |
| "num_attention_heads": 12, | |
| "num_hidden_layers": 12, | |
| "pad_token_id": 1, | |
| "position_embedding_type": "absolute", | |
| "transformers_version": "4.42.4", | |
| "type_vocab_size": 1, | |
| "use_cache": true, | |
| "vocab_size": 50262 | |
| } | |
| [INFO|configuration_utils.py:733] 2024-08-30 20:25:42,016 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--PlanTL-GOB-ES--bsc-bio-ehr-es/snapshots/1e543adb2d21f19d85a89305eebdbd64ab656b99/config.json | |
| [INFO|configuration_utils.py:800] 2024-08-30 20:25:42,017 >> Model config RobertaConfig { | |
| "_name_or_path": "PlanTL-GOB-ES/bsc-bio-ehr-es", | |
| "architectures": [ | |
| "RobertaForMaskedLM" | |
| ], | |
| "attention_probs_dropout_prob": 0.1, | |
| "bos_token_id": 0, | |
| "classifier_dropout": null, | |
| "eos_token_id": 2, | |
| "gradient_checkpointing": false, | |
| "hidden_act": "gelu", | |
| "hidden_dropout_prob": 0.1, | |
| "hidden_size": 768, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 3072, | |
| "layer_norm_eps": 1e-05, | |
| "max_position_embeddings": 514, | |
| "model_type": "roberta", | |
| "num_attention_heads": 12, | |
| "num_hidden_layers": 12, | |
| "pad_token_id": 1, | |
| "position_embedding_type": "absolute", | |
| "transformers_version": "4.42.4", | |
| "type_vocab_size": 1, | |
| "use_cache": true, | |
| "vocab_size": 50262 | |
| } | |
| [INFO|tokenization_utils_base.py:2161] 2024-08-30 20:25:42,027 >> loading file vocab.json from cache at /root/.cache/huggingface/hub/models--PlanTL-GOB-ES--bsc-bio-ehr-es/snapshots/1e543adb2d21f19d85a89305eebdbd64ab656b99/vocab.json | |
| [INFO|tokenization_utils_base.py:2161] 2024-08-30 20:25:42,028 >> loading file merges.txt from cache at /root/.cache/huggingface/hub/models--PlanTL-GOB-ES--bsc-bio-ehr-es/snapshots/1e543adb2d21f19d85a89305eebdbd64ab656b99/merges.txt | |
| [INFO|tokenization_utils_base.py:2161] 2024-08-30 20:25:42,028 >> loading file tokenizer.json from cache at None | |
| [INFO|tokenization_utils_base.py:2161] 2024-08-30 20:25:42,028 >> loading file added_tokens.json from cache at None | |
| [INFO|tokenization_utils_base.py:2161] 2024-08-30 20:25:42,028 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--PlanTL-GOB-ES--bsc-bio-ehr-es/snapshots/1e543adb2d21f19d85a89305eebdbd64ab656b99/special_tokens_map.json | |
| [INFO|tokenization_utils_base.py:2161] 2024-08-30 20:25:42,028 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--PlanTL-GOB-ES--bsc-bio-ehr-es/snapshots/1e543adb2d21f19d85a89305eebdbd64ab656b99/tokenizer_config.json | |
| [INFO|configuration_utils.py:733] 2024-08-30 20:25:42,028 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--PlanTL-GOB-ES--bsc-bio-ehr-es/snapshots/1e543adb2d21f19d85a89305eebdbd64ab656b99/config.json | |
| [INFO|configuration_utils.py:800] 2024-08-30 20:25:42,029 >> Model config RobertaConfig { | |
| "_name_or_path": "PlanTL-GOB-ES/bsc-bio-ehr-es", | |
| "architectures": [ | |
| "RobertaForMaskedLM" | |
| ], | |
| "attention_probs_dropout_prob": 0.1, | |
| "bos_token_id": 0, | |
| "classifier_dropout": null, | |
| "eos_token_id": 2, | |
| "gradient_checkpointing": false, | |
| "hidden_act": "gelu", | |
| "hidden_dropout_prob": 0.1, | |
| "hidden_size": 768, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 3072, | |
| "layer_norm_eps": 1e-05, | |
| "max_position_embeddings": 514, | |
| "model_type": "roberta", | |
| "num_attention_heads": 12, | |
| "num_hidden_layers": 12, | |
| "pad_token_id": 1, | |
| "position_embedding_type": "absolute", | |
| "transformers_version": "4.42.4", | |
| "type_vocab_size": 1, | |
| "use_cache": true, | |
| "vocab_size": 50262 | |
| } | |
| [INFO|configuration_utils.py:733] 2024-08-30 20:25:42,112 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--PlanTL-GOB-ES--bsc-bio-ehr-es/snapshots/1e543adb2d21f19d85a89305eebdbd64ab656b99/config.json | |
| [INFO|configuration_utils.py:800] 2024-08-30 20:25:42,113 >> Model config RobertaConfig { | |
| "_name_or_path": "PlanTL-GOB-ES/bsc-bio-ehr-es", | |
| "architectures": [ | |
| "RobertaForMaskedLM" | |
| ], | |
| "attention_probs_dropout_prob": 0.1, | |
| "bos_token_id": 0, | |
| "classifier_dropout": null, | |
| "eos_token_id": 2, | |
| "gradient_checkpointing": false, | |
| "hidden_act": "gelu", | |
| "hidden_dropout_prob": 0.1, | |
| "hidden_size": 768, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 3072, | |
| "layer_norm_eps": 1e-05, | |
| "max_position_embeddings": 514, | |
| "model_type": "roberta", | |
| "num_attention_heads": 12, | |
| "num_hidden_layers": 12, | |
| "pad_token_id": 1, | |
| "position_embedding_type": "absolute", | |
| "transformers_version": "4.42.4", | |
| "type_vocab_size": 1, | |
| "use_cache": true, | |
| "vocab_size": 50262 | |
| } | |
| [INFO|modeling_utils.py:3556] 2024-08-30 20:25:42,300 >> loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--PlanTL-GOB-ES--bsc-bio-ehr-es/snapshots/1e543adb2d21f19d85a89305eebdbd64ab656b99/pytorch_model.bin | |
| [INFO|modeling_utils.py:4354] 2024-08-30 20:25:42,438 >> Some weights of the model checkpoint at PlanTL-GOB-ES/bsc-bio-ehr-es were not used when initializing RobertaForTokenClassification: ['lm_head.bias', 'lm_head.decoder.bias', 'lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight'] | |
| - This IS expected if you are initializing RobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model). | |
| - This IS NOT expected if you are initializing RobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). | |
| [WARNING|modeling_utils.py:4366] 2024-08-30 20:25:42,438 >> Some weights of RobertaForTokenClassification were not initialized from the model checkpoint at PlanTL-GOB-ES/bsc-bio-ehr-es and are newly initialized: ['classifier.bias', 'classifier.weight'] | |
| You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. | |
| Map: 0%| | 0/27229 [00:00<?, ? examples/s] Map: 7%|▋ | 2000/27229 [00:00<00:02, 10564.72 examples/s] Map: 15%|█▍ | 4000/27229 [00:00<00:02, 10953.22 examples/s] Map: 22%|██▏ | 6000/27229 [00:00<00:01, 11153.09 examples/s] Map: 29%|██▉ | 8000/27229 [00:00<00:01, 11313.41 examples/s] Map: 37%|███▋ | 10000/27229 [00:00<00:01, 11222.22 examples/s] Map: 44%|████▍ | 12000/27229 [00:01<00:01, 11320.82 examples/s] Map: 51%|█████▏ | 14000/27229 [00:01<00:01, 11244.86 examples/s] Map: 59%|█████▉ | 16000/27229 [00:01<00:00, 11231.80 examples/s] Map: 66%|██████▌ | 18000/27229 [00:01<00:00, 11231.70 examples/s] Map: 73%|███████▎ | 20000/27229 [00:01<00:00, 11355.32 examples/s] Map: 81%|████████ | 22000/27229 [00:01<00:00, 11344.14 examples/s] Map: 88%|████████▊ | 24000/27229 [00:02<00:00, 11393.55 examples/s] Map: 95%|█████████▌| 26000/27229 [00:02<00:00, 11349.27 examples/s] Map: 100%|██████████| 27229/27229 [00:02<00:00, 10948.22 examples/s] Map: 100%|██████████| 27229/27229 [00:02<00:00, 11170.55 examples/s] | |
| Map: 0%| | 0/6810 [00:00<?, ? examples/s] Map: 29%|██▉ | 2000/6810 [00:00<00:00, 11001.37 examples/s] Map: 59%|█████▊ | 4000/6810 [00:00<00:00, 11030.44 examples/s] Map: 88%|████████▊ | 6000/6810 [00:00<00:00, 11133.09 examples/s] Map: 100%|██████████| 6810/6810 [00:00<00:00, 11043.95 examples/s] | |
| Map: 0%| | 0/14614 [00:00<?, ? examples/s] Map: 14%|█▎ | 2000/14614 [00:00<00:00, 12902.53 examples/s] Map: 27%|██▋ | 4000/14614 [00:00<00:01, 6743.67 examples/s] Map: 41%|████ | 6000/14614 [00:00<00:00, 8771.21 examples/s] Map: 55%|█████▍ | 8000/14614 [00:00<00:00, 8837.23 examples/s] Map: 68%|██████▊ | 10000/14614 [00:01<00:00, 10043.88 examples/s] Map: 82%|████████▏ | 12000/14614 [00:01<00:00, 10956.27 examples/s] Map: 96%|█████████▌| 14000/14614 [00:01<00:00, 11520.66 examples/s] Map: 100%|██████████| 14614/14614 [00:01<00:00, 10168.24 examples/s] | |
| /content/dissertation/scripts/ner/run_ner_train.py:397: FutureWarning: load_metric is deprecated and will be removed in the next major version of datasets. Use 'evaluate.load' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate | |
| metric = load_metric("seqeval", trust_remote_code=True) | |
| [INFO|trainer.py:805] 2024-08-30 20:25:48,288 >> The following columns in the training set don't have a corresponding argument in `RobertaForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `RobertaForTokenClassification.forward`, you can safely ignore this message. | |
| [INFO|trainer.py:2128] 2024-08-30 20:25:48,850 >> ***** Running training ***** | |
| [INFO|trainer.py:2129] 2024-08-30 20:25:48,850 >> Num examples = 27,229 | |
| [INFO|trainer.py:2130] 2024-08-30 20:25:48,850 >> Num Epochs = 10 | |
| [INFO|trainer.py:2131] 2024-08-30 20:25:48,850 >> Instantaneous batch size per device = 32 | |
| [INFO|trainer.py:2134] 2024-08-30 20:25:48,850 >> Total train batch size (w. parallel, distributed & accumulation) = 64 | |
| [INFO|trainer.py:2135] 2024-08-30 20:25:48,851 >> Gradient Accumulation steps = 2 | |
| [INFO|trainer.py:2136] 2024-08-30 20:25:48,851 >> Total optimization steps = 4,250 | |
| [INFO|trainer.py:2137] 2024-08-30 20:25:48,851 >> Number of trainable parameters = 124,055,043 | |
| 0%| | 0/4250 [00:00<?, ?it/s] 0%| | 1/4250 [00:01<1:18:15, 1.11s/it] 0%| | 2/4250 [00:01<40:12, 1.76it/s] 0%| | 3/4250 [00:01<28:24, 2.49it/s] 0%| | 4/4250 [00:01<22:38, 3.13it/s] 0%| | 5/4250 [00:01<20:43, 3.41it/s] 0%| | 6/4250 [00:02<18:37, 3.80it/s] 0%| | 7/4250 [00:02<17:55, 3.94it/s] 0%| | 8/4250 [00:02<16:59, 4.16it/s] 0%| | 9/4250 [00:02<17:55, 3.94it/s] 0%| | 10/4250 [00:03<17:19, 4.08it/s] 0%| | 11/4250 [00:03<16:45, 4.21it/s] 0%| | 12/4250 [00:03<15:21, 4.60it/s] 0%| | 13/4250 [00:03<17:23, 4.06it/s] 0%| | 14/4250 [00:04<16:58, 4.16it/s] 0%| | 15/4250 [00:04<16:38, 4.24it/s] 0%| | 16/4250 [00:04<15:46, 4.47it/s] 0%| | 17/4250 [00:04<14:44, 4.78it/s] 0%| | 18/4250 [00:04<14:27, 4.88it/s] 0%| | 19/4250 [00:05<15:16, 4.62it/s] 0%| | 20/4250 [00:05<14:22, 4.90it/s] 0%| | 21/4250 [00:05<16:37, 4.24it/s] 1%| | 22/4250 [00:05<15:57, 4.42it/s] 1%| | 23/4250 [00:05<15:28, 4.55it/s] 1%| | 24/4250 [00:06<15:18, 4.60it/s] 1%| | 25/4250 [00:06<16:46, 4.20it/s] 1%| | 26/4250 [00:06<15:14, 4.62it/s] 1%| | 27/4250 [00:06<15:15, 4.61it/s] 1%| | 28/4250 [00:07<14:02, 5.01it/s] 1%| | 29/4250 [00:07<13:43, 5.12it/s] 1%| | 30/4250 [00:07<16:48, 4.18it/s] 1%| | 31/4250 [00:07<15:45, 4.46it/s] 1%| | 32/4250 [00:07<16:53, 4.16it/s] 1%| | 33/4250 [00:08<16:20, 4.30it/s] 1%| | 34/4250 [00:08<16:34, 4.24it/s] 1%| | 35/4250 [00:08<15:53, 4.42it/s] 1%| | 36/4250 [00:08<16:15, 4.32it/s] 1%| | 37/4250 [00:09<15:17, 4.59it/s] 1%| | 38/4250 [00:09<15:07, 4.64it/s] 1%| | 39/4250 [00:09<16:28, 4.26it/s] 1%| | 40/4250 [00:09<15:19, 4.58it/s] 1%| | 41/4250 [00:09<14:59, 4.68it/s] 1%| | 42/4250 [00:10<15:06, 4.64it/s] 1%| | 43/4250 [00:10<17:34, 3.99it/s] 1%| | 44/4250 [00:11<27:51, 2.52it/s] 1%| | 45/4250 [00:11<23:41, 2.96it/s] 1%| | 46/4250 [00:11<22:36, 3.10it/s] 1%| | 47/4250 [00:11<19:58, 3.51it/s] 1%| | 48/4250 [00:12<18:45, 3.73it/s] 1%| | 49/4250 [00:12<16:12, 4.32it/s] 1%| | 50/4250 [00:12<16:45, 4.18it/s] 1%| | 51/4250 [00:12<16:59, 4.12it/s] 1%| | 52/4250 [00:13<16:34, 4.22it/s] 1%| | 53/4250 [00:13<17:18, 4.04it/s] 1%|▏ | 54/4250 [00:13<20:58, 3.33it/s] 1%|▏ | 55/4250 [00:13<19:06, 3.66it/s] 1%|▏ | 56/4250 [00:14<18:27, 3.79it/s] 1%|▏ | 57/4250 [00:14<17:06, 4.08it/s] 1%|▏ | 58/4250 [00:14<15:39, 4.46it/s] 1%|▏ | 59/4250 [00:14<15:05, 4.63it/s] 1%|▏ | 60/4250 [00:14<14:49, 4.71it/s] 1%|▏ | 61/4250 [00:15<15:52, 4.40it/s] 1%|▏ | 62/4250 [00:15<15:58, 4.37it/s] 1%|▏ | 63/4250 [00:15<15:02, 4.64it/s] 2%|▏ | 64/4250 [00:15<14:48, 4.71it/s] 2%|▏ | 65/4250 [00:16<15:34, 4.48it/s] 2%|▏ | 66/4250 [00:16<16:39, 4.19it/s] 2%|▏ | 67/4250 [00:16<16:33, 4.21it/s] 2%|▏ | 68/4250 [00:17<20:48, 3.35it/s] 2%|▏ | 69/4250 [00:17<18:35, 3.75it/s] 2%|▏ | 70/4250 [00:17<19:57, 3.49it/s] 2%|▏ | 71/4250 [00:17<18:59, 3.67it/s] 2%|▏ | 72/4250 [00:18<17:43, 3.93it/s] 2%|▏ | 73/4250 [00:18<17:16, 4.03it/s] 2%|▏ | 74/4250 [00:18<17:31, 3.97it/s] 2%|▏ | 75/4250 [00:18<15:35, 4.46it/s] 2%|▏ | 76/4250 [00:18<16:09, 4.30it/s] 2%|▏ | 77/4250 [00:19<15:16, 4.55it/s] 2%|▏ | 78/4250 [00:19<14:19, 4.85it/s] 2%|▏ | 79/4250 [00:19<14:56, 4.65it/s] 2%|▏ | 80/4250 [00:19<15:15, 4.56it/s] 2%|▏ | 81/4250 [00:19<15:31, 4.47it/s] 2%|▏ | 82/4250 [00:20<14:46, 4.70it/s] 2%|▏ | 83/4250 [00:20<16:51, 4.12it/s] 2%|▏ | 84/4250 [00:20<15:25, 4.50it/s] 2%|▏ | 85/4250 [00:20<15:09, 4.58it/s] 2%|▏ | 86/4250 [00:21<15:01, 4.62it/s] 2%|▏ | 87/4250 [00:21<15:03, 4.61it/s] 2%|▏ | 88/4250 [00:21<13:40, 5.07it/s] 2%|▏ | 89/4250 [00:21<13:52, 5.00it/s] 2%|▏ | 90/4250 [00:21<16:14, 4.27it/s] 2%|▏ | 91/4250 [00:22<14:35, 4.75it/s] 2%|▏ | 92/4250 [00:22<14:04, 4.93it/s] 2%|▏ | 93/4250 [00:22<13:20, 5.19it/s] 2%|▏ | 94/4250 [00:22<13:59, 4.95it/s] 2%|▏ | 95/4250 [00:22<15:09, 4.57it/s] 2%|▏ | 96/4250 [00:23<16:58, 4.08it/s] 2%|▏ | 97/4250 [00:23<15:55, 4.34it/s] 2%|▏ | 98/4250 [00:23<14:47, 4.68it/s] 2%|▏ | 99/4250 [00:23<14:10, 4.88it/s] 2%|▏ | 100/4250 [00:24<14:30, 4.77it/s] 2%|▏ | 101/4250 [00:24<18:02, 3.83it/s] 2%|▏ | 102/4250 [00:24<20:27, 3.38it/s] 2%|▏ | 103/4250 [00:25<18:31, 3.73it/s] 2%|▏ | 104/4250 [00:25<17:01, 4.06it/s] 2%|▏ | 105/4250 [00:25<16:41, 4.14it/s] 2%|▏ | 106/4250 [00:25<16:12, 4.26it/s] 3%|▎ | 107/4250 [00:25<15:58, 4.32it/s] 3%|▎ | 108/4250 [00:26<19:11, 3.60it/s] 3%|▎ | 109/4250 [00:26<18:38, 3.70it/s] 3%|▎ | 110/4250 [00:26<17:15, 4.00it/s] 3%|▎ | 111/4250 [00:26<15:36, 4.42it/s] 3%|▎ | 112/4250 [00:27<16:11, 4.26it/s] 3%|▎ | 113/4250 [00:27<15:49, 4.36it/s] 3%|▎ | 114/4250 [00:27<14:38, 4.71it/s] 3%|▎ | 115/4250 [00:27<14:08, 4.87it/s] 3%|▎ | 116/4250 [00:28<15:38, 4.40it/s] 3%|▎ | 117/4250 [00:28<15:11, 4.54it/s] 3%|▎ | 118/4250 [00:28<14:46, 4.66it/s] 3%|▎ | 119/4250 [00:28<15:08, 4.55it/s] 3%|▎ | 120/4250 [00:28<14:14, 4.83it/s] 3%|▎ | 121/4250 [00:29<14:19, 4.80it/s] 3%|▎ | 122/4250 [00:29<20:15, 3.40it/s] 3%|▎ | 123/4250 [00:29<18:33, 3.71it/s] 3%|▎ | 124/4250 [00:29<16:55, 4.06it/s] 3%|▎ | 125/4250 [00:30<17:07, 4.01it/s] 3%|▎ | 126/4250 [00:30<15:33, 4.42it/s] 3%|▎ | 127/4250 [00:30<14:50, 4.63it/s] 3%|▎ | 128/4250 [00:30<14:17, 4.81it/s] 3%|▎ | 129/4250 [00:31<17:35, 3.90it/s] 3%|▎ | 130/4250 [00:31<17:02, 4.03it/s] 3%|▎ | 131/4250 [00:31<15:32, 4.42it/s] 3%|▎ | 132/4250 [00:31<14:40, 4.68it/s] 3%|▎ | 133/4250 [00:31<15:39, 4.38it/s] 3%|▎ | 134/4250 [00:32<15:20, 4.47it/s] 3%|▎ | 135/4250 [00:32<15:08, 4.53it/s] 3%|▎ | 136/4250 [00:32<15:13, 4.50it/s] 3%|▎ | 137/4250 [00:32<15:00, 4.57it/s] 3%|▎ | 138/4250 [00:33<14:29, 4.73it/s] 3%|▎ | 139/4250 [00:33<14:58, 4.58it/s] 3%|▎ | 140/4250 [00:33<15:55, 4.30it/s] 3%|▎ | 141/4250 [00:33<15:59, 4.28it/s] 3%|▎ | 142/4250 [00:33<15:00, 4.56it/s] 3%|▎ | 143/4250 [00:34<14:38, 4.67it/s] 3%|▎ | 144/4250 [00:34<14:27, 4.73it/s] 3%|▎ | 145/4250 [00:34<16:59, 4.03it/s] 3%|▎ | 146/4250 [00:35<22:15, 3.07it/s] 3%|▎ | 147/4250 [00:35<19:50, 3.45it/s] 3%|▎ | 148/4250 [00:35<19:43, 3.47it/s] 4%|▎ | 149/4250 [00:35<18:58, 3.60it/s] 4%|▎ | 150/4250 [00:36<17:38, 3.87it/s] 4%|▎ | 151/4250 [00:36<16:10, 4.23it/s] 4%|▎ | 152/4250 [00:36<17:03, 4.00it/s] 4%|▎ | 153/4250 [00:36<16:24, 4.16it/s] 4%|▎ | 154/4250 [00:37<15:22, 4.44it/s] 4%|▎ | 155/4250 [00:37<14:32, 4.70it/s] 4%|▎ | 156/4250 [00:37<14:50, 4.60it/s] 4%|▎ | 157/4250 [00:37<14:30, 4.70it/s] 4%|▎ | 158/4250 [00:37<17:37, 3.87it/s] 4%|▎ | 159/4250 [00:38<18:35, 3.67it/s] 4%|▍ | 160/4250 [00:38<17:00, 4.01it/s] 4%|▍ | 161/4250 [00:38<16:02, 4.25it/s] 4%|▍ | 162/4250 [00:38<14:55, 4.56it/s] 4%|▍ | 163/4250 [00:39<14:47, 4.60it/s] 4%|▍ | 164/4250 [00:39<18:40, 3.65it/s] 4%|▍ | 165/4250 [00:39<18:13, 3.74it/s] 4%|▍ | 166/4250 [00:40<18:24, 3.70it/s] 4%|▍ | 167/4250 [00:40<17:29, 3.89it/s] 4%|▍ | 168/4250 [00:40<15:56, 4.27it/s] 4%|▍ | 169/4250 [00:40<15:32, 4.38it/s] 4%|▍ | 170/4250 [00:40<15:28, 4.39it/s] 4%|▍ | 171/4250 [00:41<16:01, 4.24it/s] 4%|▍ | 172/4250 [00:41<15:13, 4.46it/s] 4%|▍ | 173/4250 [00:41<18:22, 3.70it/s] 4%|▍ | 174/4250 [00:41<16:38, 4.08it/s] 4%|▍ | 175/4250 [00:42<19:24, 3.50it/s] 4%|▍ | 176/4250 [00:42<19:20, 3.51it/s] 4%|▍ | 177/4250 [00:42<17:04, 3.98it/s] 4%|▍ | 178/4250 [00:42<16:07, 4.21it/s] 4%|▍ | 179/4250 [00:43<15:41, 4.32it/s] 4%|▍ | 180/4250 [00:43<14:28, 4.69it/s] 4%|▍ | 181/4250 [00:43<14:38, 4.63it/s] 4%|▍ | 182/4250 [00:43<17:11, 3.94it/s] 4%|▍ | 183/4250 [00:44<16:36, 4.08it/s] 4%|▍ | 184/4250 [00:44<16:28, 4.11it/s] 4%|▍ | 185/4250 [00:44<15:07, 4.48it/s] 4%|▍ | 186/4250 [00:44<14:39, 4.62it/s] 4%|▍ | 187/4250 [00:44<13:47, 4.91it/s] 4%|▍ | 188/4250 [00:45<14:41, 4.61it/s] 4%|▍ | 189/4250 [00:45<14:11, 4.77it/s] 4%|▍ | 190/4250 [00:45<13:45, 4.92it/s] 4%|▍ | 191/4250 [00:45<14:23, 4.70it/s] 5%|▍ | 192/4250 [00:45<14:11, 4.76it/s] 5%|▍ | 193/4250 [00:46<13:55, 4.86it/s] 5%|▍ | 194/4250 [00:46<16:42, 4.05it/s] 5%|▍ | 195/4250 [00:46<15:38, 4.32it/s] 5%|▍ | 196/4250 [00:46<14:26, 4.68it/s] 5%|▍ | 197/4250 [00:47<14:55, 4.53it/s] 5%|▍ | 198/4250 [00:47<15:19, 4.41it/s] 5%|▍ | 199/4250 [00:47<15:25, 4.38it/s] 5%|▍ | 200/4250 [00:47<15:35, 4.33it/s] 5%|▍ | 201/4250 [00:48<15:14, 4.43it/s] 5%|▍ | 202/4250 [00:48<15:31, 4.35it/s] 5%|▍ | 203/4250 [00:48<14:26, 4.67it/s] 5%|▍ | 204/4250 [00:48<13:47, 4.89it/s] 5%|▍ | 205/4250 [00:48<14:44, 4.57it/s] 5%|▍ | 206/4250 [00:49<14:29, 4.65it/s] 5%|▍ | 207/4250 [00:49<16:45, 4.02it/s] 5%|▍ | 208/4250 [00:49<15:19, 4.40it/s] 5%|▍ | 209/4250 [00:49<15:12, 4.43it/s] 5%|▍ | 210/4250 [00:49<14:01, 4.80it/s] 5%|▍ | 211/4250 [00:50<15:09, 4.44it/s] 5%|▍ | 212/4250 [00:50<14:26, 4.66it/s] 5%|▌ | 213/4250 [00:50<17:31, 3.84it/s] 5%|▌ | 214/4250 [00:50<15:24, 4.37it/s] 5%|▌ | 215/4250 [00:51<15:17, 4.40it/s] 5%|▌ | 216/4250 [00:51<19:04, 3.52it/s] 5%|▌ | 217/4250 [00:51<17:03, 3.94it/s] 5%|▌ | 218/4250 [00:51<15:56, 4.21it/s] 5%|▌ | 219/4250 [00:52<16:35, 4.05it/s] 5%|▌ | 220/4250 [00:52<16:40, 4.03it/s] 5%|▌ | 221/4250 [00:52<18:41, 3.59it/s] 5%|▌ | 222/4250 [00:53<20:01, 3.35it/s] 5%|▌ | 223/4250 [00:53<18:26, 3.64it/s] 5%|▌ | 224/4250 [00:53<16:31, 4.06it/s] 5%|▌ | 225/4250 [00:53<15:07, 4.43it/s] 5%|▌ | 226/4250 [00:53<14:23, 4.66it/s] 5%|▌ | 227/4250 [00:54<14:01, 4.78it/s] 5%|▌ | 228/4250 [00:54<13:47, 4.86it/s] 5%|▌ | 229/4250 [00:54<12:58, 5.17it/s] 5%|▌ | 230/4250 [00:54<14:21, 4.67it/s] 5%|▌ | 231/4250 [00:54<13:45, 4.87it/s] 5%|▌ | 232/4250 [00:55<13:35, 4.93it/s] 5%|▌ | 233/4250 [00:55<13:23, 5.00it/s] 6%|▌ | 234/4250 [00:55<15:41, 4.26it/s] 6%|▌ | 235/4250 [00:56<20:37, 3.24it/s] 6%|▌ | 236/4250 [00:56<21:51, 3.06it/s] 6%|▌ | 237/4250 [00:56<19:42, 3.39it/s] 6%|▌ | 238/4250 [00:56<17:25, 3.84it/s] 6%|▌ | 239/4250 [00:57<17:28, 3.83it/s] 6%|▌ | 240/4250 [00:57<19:56, 3.35it/s] 6%|▌ | 241/4250 [00:57<17:57, 3.72it/s] 6%|▌ | 242/4250 [00:57<16:26, 4.06it/s] 6%|▌ | 243/4250 [00:58<15:57, 4.19it/s] 6%|▌ | 244/4250 [00:58<15:58, 4.18it/s] 6%|▌ | 245/4250 [00:58<15:56, 4.19it/s] 6%|▌ | 246/4250 [00:59<24:58, 2.67it/s] 6%|▌ | 247/4250 [00:59<22:20, 2.99it/s] 6%|▌ | 248/4250 [00:59<19:38, 3.40it/s] 6%|▌ | 249/4250 [01:00<23:51, 2.80it/s] 6%|▌ | 250/4250 [01:00<25:17, 2.64it/s] 6%|▌ | 251/4250 [01:01<22:48, 2.92it/s] 6%|▌ | 252/4250 [01:01<19:31, 3.41it/s] 6%|▌ | 253/4250 [01:01<17:13, 3.87it/s] 6%|▌ | 254/4250 [01:01<17:53, 3.72it/s] 6%|▌ | 255/4250 [01:01<18:10, 3.66it/s] 6%|▌ | 256/4250 [01:02<17:55, 3.71it/s] 6%|▌ | 257/4250 [01:02<16:43, 3.98it/s] 6%|▌ | 258/4250 [01:02<15:35, 4.27it/s] 6%|▌ | 259/4250 [01:02<14:28, 4.59it/s] 6%|▌ | 260/4250 [01:03<15:44, 4.22it/s] 6%|▌ | 261/4250 [01:03<15:00, 4.43it/s] 6%|▌ | 262/4250 [01:03<13:52, 4.79it/s] 6%|▌ | 263/4250 [01:03<12:53, 5.15it/s] 6%|▌ | 264/4250 [01:03<14:53, 4.46it/s] 6%|▌ | 265/4250 [01:04<15:50, 4.19it/s] 6%|▋ | 266/4250 [01:04<15:46, 4.21it/s] 6%|▋ | 267/4250 [01:04<15:43, 4.22it/s] 6%|▋ | 268/4250 [01:04<14:47, 4.49it/s] 6%|▋ | 269/4250 [01:05<14:19, 4.63it/s] 6%|▋ | 270/4250 [01:05<16:43, 3.97it/s] 6%|▋ | 271/4250 [01:05<15:07, 4.38it/s] 6%|▋ | 272/4250 [01:05<14:35, 4.54it/s] 6%|▋ | 273/4250 [01:05<14:08, 4.69it/s] 6%|▋ | 274/4250 [01:06<13:43, 4.83it/s] 6%|▋ | 275/4250 [01:06<13:31, 4.90it/s] 6%|▋ | 276/4250 [01:06<12:58, 5.11it/s] 7%|▋ | 277/4250 [01:06<13:41, 4.84it/s] 7%|▋ | 278/4250 [01:06<13:31, 4.89it/s] 7%|▋ | 279/4250 [01:07<14:09, 4.67it/s] 7%|▋ | 280/4250 [01:07<17:33, 3.77it/s] 7%|▋ | 281/4250 [01:07<16:06, 4.10it/s] 7%|▋ | 282/4250 [01:07<15:58, 4.14it/s] 7%|▋ | 283/4250 [01:08<15:01, 4.40it/s] 7%|▋ | 284/4250 [01:08<14:33, 4.54it/s] 7%|▋ | 285/4250 [01:08<14:43, 4.49it/s] 7%|▋ | 286/4250 [01:08<14:18, 4.62it/s] 7%|▋ | 287/4250 [01:09<15:53, 4.16it/s] 7%|▋ | 288/4250 [01:09<14:44, 4.48it/s] 7%|▋ | 289/4250 [01:09<14:55, 4.42it/s] 7%|▋ | 290/4250 [01:09<14:05, 4.69it/s] 7%|▋ | 291/4250 [01:09<14:55, 4.42it/s] 7%|▋ | 292/4250 [01:10<14:01, 4.70it/s] 7%|▋ | 293/4250 [01:10<13:19, 4.95it/s] 7%|▋ | 294/4250 [01:10<12:53, 5.11it/s] 7%|▋ | 295/4250 [01:10<14:15, 4.62it/s] 7%|▋ | 296/4250 [01:10<13:25, 4.91it/s] 7%|▋ | 297/4250 [01:11<13:07, 5.02it/s] 7%|▋ | 298/4250 [01:11<13:03, 5.04it/s] 7%|▋ | 299/4250 [01:11<13:43, 4.80it/s] 7%|▋ | 300/4250 [01:11<15:32, 4.23it/s] 7%|▋ | 301/4250 [01:12<14:53, 4.42it/s] 7%|▋ | 302/4250 [01:12<16:06, 4.09it/s] 7%|▋ | 303/4250 [01:12<17:14, 3.82it/s] 7%|▋ | 304/4250 [01:12<16:34, 3.97it/s] 7%|▋ | 305/4250 [01:13<16:20, 4.03it/s] 7%|▋ | 306/4250 [01:13<18:10, 3.62it/s] 7%|▋ | 307/4250 [01:13<16:03, 4.09it/s] 7%|▋ | 308/4250 [01:13<15:15, 4.31it/s] 7%|▋ | 309/4250 [01:14<15:01, 4.37it/s] 7%|▋ | 310/4250 [01:14<14:24, 4.56it/s] 7%|▋ | 311/4250 [01:14<13:58, 4.70it/s] 7%|▋ | 312/4250 [01:14<16:55, 3.88it/s] 7%|▋ | 313/4250 [01:15<16:12, 4.05it/s] 7%|▋ | 314/4250 [01:15<16:17, 4.03it/s] 7%|▋ | 315/4250 [01:15<15:12, 4.31it/s] 7%|▋ | 316/4250 [01:15<14:32, 4.51it/s] 7%|▋ | 317/4250 [01:15<15:41, 4.18it/s] 7%|▋ | 318/4250 [01:16<15:25, 4.25it/s] 8%|▊ | 319/4250 [01:16<15:46, 4.15it/s] 8%|▊ | 320/4250 [01:16<16:25, 3.99it/s] 8%|▊ | 321/4250 [01:16<15:03, 4.35it/s] 8%|▊ | 322/4250 [01:17<14:35, 4.49it/s] 8%|▊ | 323/4250 [01:17<13:27, 4.86it/s] 8%|▊ | 324/4250 [01:17<13:13, 4.95it/s] 8%|▊ | 325/4250 [01:17<13:22, 4.89it/s] 8%|▊ | 326/4250 [01:17<12:56, 5.05it/s] 8%|▊ | 327/4250 [01:18<12:34, 5.20it/s] 8%|▊ | 328/4250 [01:18<16:35, 3.94it/s] 8%|▊ | 329/4250 [01:18<17:45, 3.68it/s] 8%|▊ | 330/4250 [01:18<16:40, 3.92it/s] 8%|▊ | 331/4250 [01:19<16:15, 4.02it/s] 8%|▊ | 332/4250 [01:19<15:10, 4.31it/s] 8%|▊ | 333/4250 [01:19<14:15, 4.58it/s] 8%|▊ | 334/4250 [01:19<14:33, 4.48it/s] 8%|▊ | 335/4250 [01:20<15:52, 4.11it/s] 8%|▊ | 336/4250 [01:20<15:40, 4.16it/s] 8%|▊ | 337/4250 [01:20<14:30, 4.50it/s] 8%|▊ | 338/4250 [01:20<13:51, 4.70it/s] 8%|▊ | 339/4250 [01:21<17:39, 3.69it/s] 8%|▊ | 340/4250 [01:21<16:24, 3.97it/s] 8%|▊ | 341/4250 [01:21<17:23, 3.75it/s] 8%|▊ | 342/4250 [01:21<15:45, 4.13it/s] 8%|▊ | 343/4250 [01:22<16:07, 4.04it/s] 8%|▊ | 344/4250 [01:22<15:55, 4.09it/s] 8%|▊ | 345/4250 [01:22<17:25, 3.74it/s] 8%|▊ | 346/4250 [01:22<18:30, 3.52it/s] 8%|▊ | 347/4250 [01:23<17:09, 3.79it/s] 8%|▊ | 348/4250 [01:23<18:01, 3.61it/s] 8%|▊ | 349/4250 [01:23<16:19, 3.98it/s] 8%|▊ | 350/4250 [01:23<16:02, 4.05it/s] 8%|▊ | 351/4250 [01:24<15:28, 4.20it/s] 8%|▊ | 352/4250 [01:24<15:45, 4.12it/s] 8%|▊ | 353/4250 [01:24<14:05, 4.61it/s] 8%|▊ | 354/4250 [01:24<14:12, 4.57it/s] 8%|▊ | 355/4250 [01:25<15:18, 4.24it/s] 8%|▊ | 356/4250 [01:25<14:48, 4.38it/s] 8%|▊ | 357/4250 [01:25<14:00, 4.63it/s] 8%|▊ | 358/4250 [01:25<13:29, 4.81it/s] 8%|▊ | 359/4250 [01:25<12:50, 5.05it/s] 8%|▊ | 360/4250 [01:26<14:46, 4.39it/s] 8%|▊ | 361/4250 [01:26<14:14, 4.55it/s] 9%|▊ | 362/4250 [01:26<13:28, 4.81it/s] 9%|▊ | 363/4250 [01:26<14:21, 4.51it/s] 9%|▊ | 364/4250 [01:27<20:35, 3.15it/s] 9%|▊ | 365/4250 [01:27<19:18, 3.35it/s] 9%|▊ | 366/4250 [01:27<18:56, 3.42it/s] 9%|▊ | 367/4250 [01:27<17:11, 3.77it/s] 9%|▊ | 368/4250 [01:28<15:13, 4.25it/s] 9%|▊ | 369/4250 [01:28<16:11, 4.00it/s] 9%|▊ | 370/4250 [01:28<15:10, 4.26it/s] 9%|▊ | 371/4250 [01:28<14:18, 4.52it/s] 9%|▉ | 372/4250 [01:29<13:52, 4.66it/s] 9%|▉ | 373/4250 [01:29<15:47, 4.09it/s] 9%|▉ | 374/4250 [01:29<15:20, 4.21it/s] 9%|▉ | 375/4250 [01:30<25:51, 2.50it/s] 9%|▉ | 376/4250 [01:30<22:05, 2.92it/s] 9%|▉ | 377/4250 [01:30<19:34, 3.30it/s] 9%|▉ | 378/4250 [01:30<17:28, 3.69it/s] 9%|▉ | 379/4250 [01:31<15:52, 4.06it/s] 9%|▉ | 380/4250 [01:31<15:12, 4.24it/s] 9%|▉ | 381/4250 [01:31<14:26, 4.46it/s] 9%|▉ | 382/4250 [01:31<13:57, 4.62it/s] 9%|▉ | 383/4250 [01:32<15:16, 4.22it/s] 9%|▉ | 384/4250 [01:32<14:11, 4.54it/s] 9%|▉ | 385/4250 [01:32<16:37, 3.87it/s] 9%|▉ | 386/4250 [01:32<18:50, 3.42it/s] 9%|▉ | 387/4250 [01:33<16:38, 3.87it/s] 9%|▉ | 388/4250 [01:33<15:54, 4.05it/s] 9%|▉ | 389/4250 [01:33<15:11, 4.24it/s] 9%|▉ | 390/4250 [01:33<15:52, 4.05it/s] 9%|▉ | 391/4250 [01:33<14:31, 4.43it/s] 9%|▉ | 392/4250 [01:34<14:36, 4.40it/s] 9%|▉ | 393/4250 [01:34<13:42, 4.69it/s] 9%|▉ | 394/4250 [01:34<13:42, 4.69it/s] 9%|▉ | 395/4250 [01:34<13:04, 4.91it/s] 9%|▉ | 396/4250 [01:34<12:50, 5.00it/s] 9%|▉ | 397/4250 [01:35<13:30, 4.75it/s] 9%|▉ | 398/4250 [01:35<14:10, 4.53it/s] 9%|▉ | 399/4250 [01:35<13:17, 4.83it/s] 9%|▉ | 400/4250 [01:35<14:40, 4.37it/s] 9%|▉ | 401/4250 [01:36<14:06, 4.54it/s] 9%|▉ | 402/4250 [01:36<15:53, 4.04it/s] 9%|▉ | 403/4250 [01:36<15:28, 4.14it/s] 10%|▉ | 404/4250 [01:36<14:41, 4.36it/s] 10%|▉ | 405/4250 [01:37<15:17, 4.19it/s] 10%|▉ | 406/4250 [01:37<15:58, 4.01it/s] 10%|▉ | 407/4250 [01:38<23:04, 2.78it/s] 10%|▉ | 408/4250 [01:38<21:23, 2.99it/s] 10%|▉ | 409/4250 [01:38<18:34, 3.45it/s] 10%|▉ | 410/4250 [01:38<17:37, 3.63it/s] 10%|▉ | 411/4250 [01:38<17:21, 3.68it/s] 10%|▉ | 412/4250 [01:39<18:26, 3.47it/s] 10%|▉ | 413/4250 [01:39<17:12, 3.72it/s] 10%|▉ | 414/4250 [01:39<16:36, 3.85it/s] 10%|▉ | 415/4250 [01:39<15:21, 4.16it/s] 10%|▉ | 416/4250 [01:40<14:56, 4.28it/s] 10%|▉ | 417/4250 [01:40<14:55, 4.28it/s] 10%|▉ | 418/4250 [01:40<13:45, 4.64it/s] 10%|▉ | 419/4250 [01:40<14:56, 4.27it/s] 10%|▉ | 420/4250 [01:41<15:00, 4.25it/s] 10%|▉ | 421/4250 [01:41<15:28, 4.12it/s] 10%|▉ | 422/4250 [01:41<14:59, 4.26it/s] 10%|▉ | 423/4250 [01:41<13:50, 4.61it/s] 10%|▉ | 424/4250 [01:42<14:34, 4.37it/s] 10%|█ | 425/4250 [01:42<14:28, 4.40it/s][INFO|trainer.py:805] 2024-08-30 20:27:31,190 >> The following columns in the evaluation set don't have a corresponding argument in `RobertaForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `RobertaForTokenClassification.forward`, you can safely ignore this message. | |
| [INFO|trainer.py:3788] 2024-08-30 20:27:31,192 >> | |
| ***** Running Evaluation ***** | |
| [INFO|trainer.py:3790] 2024-08-30 20:27:31,192 >> Num examples = 6810 | |
| [INFO|trainer.py:3793] 2024-08-30 20:27:31,192 >> Batch size = 8 | |
| 0%| | 0/852 [00:00<?, ?it/s][A | |
| 1%| | 9/852 [00:00<00:09, 88.66it/s][A | |
| 2%|▏ | 18/852 [00:00<00:10, 79.48it/s][A | |
| 3%|▎ | 27/852 [00:00<00:10, 78.72it/s][A | |
| 4%|▍ | 35/852 [00:00<00:10, 77.17it/s][A | |
| 5%|▌ | 43/852 [00:00<00:10, 77.47it/s][A | |
| 6%|▌ | 51/852 [00:00<00:10, 78.01it/s][A | |
| 7%|▋ | 60/852 [00:00<00:10, 78.82it/s][A | |
| 8%|▊ | 68/852 [00:00<00:10, 76.55it/s][A | |
| 9%|▉ | 76/852 [00:00<00:10, 76.57it/s][A | |
| 10%|▉ | 84/852 [00:01<00:10, 74.53it/s][A | |
| 11%|█ | 92/852 [00:01<00:10, 74.88it/s][A | |
| 12%|█▏ | 100/852 [00:01<00:10, 74.67it/s][A | |
| 13%|█▎ | 108/852 [00:01<00:09, 74.65it/s][A | |
| 14%|█▎ | 116/852 [00:01<00:09, 75.39it/s][A | |
| 15%|█▍ | 125/852 [00:01<00:09, 76.70it/s][A | |
| 16%|█▌ | 133/852 [00:01<00:09, 72.98it/s][A | |
| 17%|█▋ | 141/852 [00:01<00:09, 73.08it/s][A | |
| 17%|█▋ | 149/852 [00:01<00:09, 72.57it/s][A | |
| 18%|█▊ | 157/852 [00:02<00:09, 74.49it/s][A | |
| 19%|█▉ | 165/852 [00:02<00:09, 75.40it/s][A | |
| 20%|██ | 173/852 [00:02<00:08, 75.96it/s][A | |
| 21%|██ | 181/852 [00:02<00:08, 76.27it/s][A | |
| 22%|██▏ | 190/852 [00:02<00:08, 77.54it/s][A | |
| 23%|██▎ | 198/852 [00:02<00:08, 77.17it/s][A | |
| 24%|██▍ | 206/852 [00:02<00:08, 76.97it/s][A | |
| 25%|██▌ | 214/852 [00:02<00:08, 75.85it/s][A | |
| 26%|██▌ | 222/852 [00:02<00:08, 77.02it/s][A | |
| 27%|██▋ | 230/852 [00:03<00:07, 77.88it/s][A | |
| 28%|██▊ | 238/852 [00:03<00:07, 77.06it/s][A | |
| 29%|██▉ | 246/852 [00:03<00:08, 75.49it/s][A | |
| 30%|██▉ | 255/852 [00:03<00:07, 77.83it/s][A | |
| 31%|███ | 263/852 [00:03<00:07, 78.44it/s][A | |
| 32%|███▏ | 271/852 [00:03<00:07, 77.33it/s][A | |
| 33%|███▎ | 280/852 [00:03<00:07, 78.78it/s][A | |
| 34%|███▍ | 288/852 [00:03<00:07, 78.13it/s][A | |
| 35%|███▍ | 296/852 [00:03<00:07, 78.46it/s][A | |
| 36%|███▌ | 305/852 [00:03<00:06, 79.51it/s][A | |
| 37%|███▋ | 313/852 [00:04<00:06, 77.59it/s][A | |
| 38%|███▊ | 322/852 [00:04<00:06, 79.45it/s][A | |
| 39%|███▊ | 330/852 [00:04<00:06, 78.56it/s][A | |
| 40%|███▉ | 338/852 [00:04<00:06, 75.62it/s][A | |
| 41%|████ | 346/852 [00:04<00:06, 76.01it/s][A | |
| 42%|████▏ | 354/852 [00:04<00:06, 75.26it/s][A | |
| 42%|████▏ | 362/852 [00:04<00:06, 75.94it/s][A | |
| 43%|████▎ | 370/852 [00:04<00:06, 76.55it/s][A | |
| 44%|████▍ | 378/852 [00:04<00:06, 77.17it/s][A | |
| 45%|████▌ | 386/852 [00:05<00:06, 76.59it/s][A | |
| 46%|████▋ | 395/852 [00:05<00:05, 77.79it/s][A | |
| 47%|████▋ | 403/852 [00:05<00:05, 77.58it/s][A | |
| 48%|████▊ | 411/852 [00:05<00:05, 75.31it/s][A | |
| 49%|████▉ | 419/852 [00:05<00:05, 76.29it/s][A | |
| 50%|█████ | 427/852 [00:05<00:05, 74.65it/s][A | |
| 51%|█████ | 435/852 [00:05<00:05, 75.66it/s][A | |
| 52%|█████▏ | 443/852 [00:05<00:05, 76.76it/s][A | |
| 53%|█████▎ | 451/852 [00:05<00:05, 77.61it/s][A | |
| 54%|█████▍ | 460/852 [00:05<00:05, 77.74it/s][A | |
| 55%|█████▍ | 468/852 [00:06<00:05, 75.24it/s][A | |
| 56%|█████▌ | 476/852 [00:06<00:05, 72.34it/s][A | |
| 57%|█████▋ | 484/852 [00:06<00:05, 72.40it/s][A | |
| 58%|█████▊ | 492/852 [00:06<00:04, 74.29it/s][A | |
| 59%|█████▉ | 501/852 [00:06<00:04, 76.52it/s][A | |
| 60%|█████▉ | 509/852 [00:06<00:04, 75.58it/s][A | |
| 61%|██████ | 517/852 [00:06<00:04, 76.67it/s][A | |
| 62%|██████▏ | 525/852 [00:06<00:04, 75.52it/s][A | |
| 63%|██████▎ | 533/852 [00:06<00:04, 75.91it/s][A | |
| 63%|██████▎ | 541/852 [00:07<00:04, 77.03it/s][A | |
| 64%|██████▍ | 549/852 [00:07<00:04, 75.19it/s][A | |
| 65%|██████▌ | 557/852 [00:07<00:03, 75.67it/s][A | |
| 66%|██████▋ | 566/852 [00:07<00:03, 77.52it/s][A | |
| 67%|██████▋ | 574/852 [00:07<00:03, 77.51it/s][A | |
| 68%|██████▊ | 582/852 [00:07<00:03, 76.66it/s][A | |
| 69%|██████▉ | 590/852 [00:07<00:03, 75.98it/s][A | |
| 70%|███████ | 598/852 [00:07<00:03, 76.35it/s][A | |
| 71%|███████ | 606/852 [00:07<00:03, 76.06it/s][A | |
| 72%|███████▏ | 614/852 [00:08<00:03, 75.05it/s][A | |
| 73%|███████▎ | 622/852 [00:08<00:03, 75.43it/s][A | |
| 74%|███████▍ | 630/852 [00:08<00:02, 74.34it/s][A | |
| 75%|███████▍ | 638/852 [00:08<00:02, 75.78it/s][A | |
| 76%|███████▌ | 646/852 [00:08<00:02, 73.52it/s][A | |
| 77%|███████▋ | 654/852 [00:08<00:02, 75.08it/s][A | |
| 78%|███████▊ | 662/852 [00:08<00:02, 75.52it/s][A | |
| 79%|███████▊ | 670/852 [00:08<00:02, 76.49it/s][A | |
| 80%|███████▉ | 678/852 [00:08<00:02, 77.21it/s][A | |
| 81%|████████ | 686/852 [00:08<00:02, 77.55it/s][A | |
| 82%|████████▏ | 695/852 [00:09<00:01, 78.56it/s][A | |
| 83%|████████▎ | 704/852 [00:09<00:01, 79.29it/s][A | |
| 84%|████████▎ | 713/852 [00:09<00:01, 79.99it/s][A | |
| 85%|████████▍ | 721/852 [00:09<00:01, 78.74it/s][A | |
| 86%|████████▌ | 730/852 [00:09<00:01, 79.95it/s][A | |
| 87%|████████▋ | 738/852 [00:09<00:01, 79.68it/s][A | |
| 88%|████████▊ | 747/852 [00:09<00:01, 80.02it/s][A | |
| 89%|████████▊ | 756/852 [00:09<00:01, 80.47it/s][A | |
| 90%|████████▉ | 765/852 [00:09<00:01, 80.21it/s][A | |
| 91%|█████████ | 774/852 [00:10<00:00, 79.37it/s][A | |
| 92%|█████████▏| 782/852 [00:10<00:00, 78.22it/s][A | |
| 93%|█████████▎| 790/852 [00:10<00:00, 77.38it/s][A | |
| 94%|█████████▎| 798/852 [00:10<00:00, 76.61it/s][A | |
| 95%|█████████▍| 807/852 [00:10<00:00, 78.06it/s][A | |
| 96%|█████████▌| 815/852 [00:10<00:00, 76.73it/s][A | |
| 97%|█████████▋| 824/852 [00:10<00:00, 77.89it/s][A | |
| 98%|█████████▊| 833/852 [00:10<00:00, 78.72it/s][A | |
| 99%|█████████▊| 841/852 [00:10<00:00, 77.89it/s][A | |
| 100%|█████████▉| 849/852 [00:11<00:00, 75.32it/s][A | |
| [A 10%|█ | 425/4250 [01:57<14:28, 4.40it/s] | |
| 100%|██████████| 852/852 [00:14<00:00, 75.32it/s][A | |
| [A[INFO|trainer.py:3478] 2024-08-30 20:27:45,881 >> Saving model checkpoint to /content/dissertation/scripts/ner/output/checkpoint-425 | |
| [INFO|configuration_utils.py:472] 2024-08-30 20:27:45,882 >> Configuration saved in /content/dissertation/scripts/ner/output/checkpoint-425/config.json | |
| [INFO|modeling_utils.py:2690] 2024-08-30 20:27:47,247 >> Model weights saved in /content/dissertation/scripts/ner/output/checkpoint-425/model.safetensors | |
| [INFO|tokenization_utils_base.py:2574] 2024-08-30 20:27:47,248 >> tokenizer config file saved in /content/dissertation/scripts/ner/output/checkpoint-425/tokenizer_config.json | |
| [INFO|tokenization_utils_base.py:2583] 2024-08-30 20:27:47,248 >> Special tokens file saved in /content/dissertation/scripts/ner/output/checkpoint-425/special_tokens_map.json | |
| [INFO|tokenization_utils_base.py:2574] 2024-08-30 20:27:50,017 >> tokenizer config file saved in /content/dissertation/scripts/ner/output/tokenizer_config.json | |
| [INFO|tokenization_utils_base.py:2583] 2024-08-30 20:27:50,017 >> Special tokens file saved in /content/dissertation/scripts/ner/output/special_tokens_map.json | |
| 10%|█ | 426/4250 [02:01<6:17:25, 5.92s/it] 10%|█ | 427/4250 [02:01<4:27:24, 4.20s/it] 10%|█ | 428/4250 [02:01<3:11:11, 3.00s/it] 10%|█ | 429/4250 [02:02<2:20:01, 2.20s/it] 10%|█ | 430/4250 [02:02<1:41:14, 1.59s/it] 10%|█ | 431/4250 [02:02<1:15:55, 1.19s/it] 10%|█ | 432/4250 [02:02<56:49, 1.12it/s] 10%|█ | 433/4250 [02:02<43:08, 1.47it/s] 10%|█ | 434/4250 [02:03<35:31, 1.79it/s] 10%|█ | 435/4250 [02:03<28:25, 2.24it/s] 10%|█ | 436/4250 [02:03<24:21, 2.61it/s] 10%|█ | 437/4250 [02:03<20:52, 3.04it/s] 10%|█ | 438/4250 [02:04<18:32, 3.43it/s] 10%|█ | 439/4250 [02:04<17:05, 3.72it/s] 10%|█ | 440/4250 [02:04<15:35, 4.07it/s] 10%|█ | 441/4250 [02:04<14:45, 4.30it/s] 10%|█ | 442/4250 [02:04<14:42, 4.32it/s] 10%|█ | 443/4250 [02:05<16:22, 3.88it/s] 10%|█ | 444/4250 [02:05<15:35, 4.07it/s] 10%|█ | 445/4250 [02:05<14:09, 4.48it/s] 10%|█ | 446/4250 [02:05<15:10, 4.18it/s] 11%|█ | 447/4250 [02:06<14:17, 4.43it/s] 11%|█ | 448/4250 [02:06<13:13, 4.79it/s] 11%|█ | 449/4250 [02:06<13:06, 4.83it/s] 11%|█ | 450/4250 [02:06<15:05, 4.20it/s] 11%|█ | 451/4250 [02:07<15:49, 4.00it/s] 11%|█ | 452/4250 [02:07<14:54, 4.25it/s] 11%|█ | 453/4250 [02:07<15:41, 4.03it/s] 11%|█ | 454/4250 [02:07<15:39, 4.04it/s] 11%|█ | 455/4250 [02:08<15:58, 3.96it/s] 11%|█ | 456/4250 [02:08<14:30, 4.36it/s] 11%|█ | 457/4250 [02:08<13:21, 4.73it/s] 11%|█ | 458/4250 [02:08<14:27, 4.37it/s] 11%|█ | 459/4250 [02:09<24:59, 2.53it/s] 11%|█ | 460/4250 [02:09<25:16, 2.50it/s] 11%|█ | 461/4250 [02:10<23:13, 2.72it/s] 11%|█ | 462/4250 [02:10<20:18, 3.11it/s] 11%|█ | 463/4250 [02:10<18:00, 3.51it/s] 11%|█ | 464/4250 [02:10<16:52, 3.74it/s] 11%|█ | 465/4250 [02:10<15:29, 4.07it/s] 11%|█ | 466/4250 [02:11<14:52, 4.24it/s] 11%|█ | 467/4250 [02:11<15:16, 4.13it/s] 11%|█ | 468/4250 [02:11<15:22, 4.10it/s] 11%|█ | 469/4250 [02:11<14:56, 4.22it/s] 11%|█ | 470/4250 [02:12<14:16, 4.41it/s] 11%|█ | 471/4250 [02:12<13:02, 4.83it/s] 11%|█ | 472/4250 [02:12<14:34, 4.32it/s] 11%|█ | 473/4250 [02:12<14:11, 4.43it/s] 11%|█ | 474/4250 [02:13<15:34, 4.04it/s] 11%|█ | 475/4250 [02:13<19:44, 3.19it/s] 11%|█ | 476/4250 [02:13<19:06, 3.29it/s] 11%|█ | 477/4250 [02:14<18:12, 3.45it/s] 11%|█ | 478/4250 [02:14<18:34, 3.39it/s] 11%|█▏ | 479/4250 [02:14<17:12, 3.65it/s] 11%|█▏ | 480/4250 [02:14<15:17, 4.11it/s] 11%|█▏ | 481/4250 [02:15<16:05, 3.90it/s] 11%|█▏ | 482/4250 [02:15<17:51, 3.52it/s] 11%|█▏ | 483/4250 [02:15<16:15, 3.86it/s] 11%|█▏ | 484/4250 [02:15<14:39, 4.28it/s] 11%|█▏ | 485/4250 [02:16<14:48, 4.24it/s] 11%|█▏ | 486/4250 [02:16<15:40, 4.00it/s] 11%|█▏ | 487/4250 [02:16<14:28, 4.33it/s] 11%|█▏ | 488/4250 [02:16<14:12, 4.41it/s] 12%|█▏ | 489/4250 [02:17<18:04, 3.47it/s] 12%|█▏ | 490/4250 [02:17<16:37, 3.77it/s] 12%|█▏ | 491/4250 [02:17<16:44, 3.74it/s] 12%|█▏ | 492/4250 [02:17<15:59, 3.92it/s] 12%|█▏ | 493/4250 [02:18<14:35, 4.29it/s] 12%|█▏ | 494/4250 [02:18<15:03, 4.16it/s] 12%|█▏ | 495/4250 [02:18<17:00, 3.68it/s] 12%|█▏ | 496/4250 [02:18<16:06, 3.89it/s] 12%|█▏ | 497/4250 [02:19<14:40, 4.26it/s] |