Duplicate from tmnam/vihealthbert-w_mlm-ViMedNLI

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,89 @@
+---
+base_model: demdecuong/vihealthbert-base-word
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: vihealthbert-w_mlm-ViMedNLI
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# vihealthbert-w_mlm-ViMedNLI
+
+This model is a fine-tuned version of [demdecuong/vihealthbert-base-word](https://huggingface.co/demdecuong/vihealthbert-base-word) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.1156
+- Accuracy: 0.8341
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-05
+- train_batch_size: 32
+- eval_batch_size: 32
+- seed: 19161
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- training_steps: 30000
+
+### Training results
+
+| Training Loss | Epoch    | Step  | Validation Loss | Accuracy |
+|:-------------:|:--------:|:-----:|:---------------:|:--------:|
+| 5.5327        | 10.5263  | 1000  | 2.7528          | 0.5890   |
+| 1.9051        | 21.0526  | 2000  | 1.4678          | 0.7783   |
+| 1.1194        | 31.5789  | 3000  | 1.1543          | 0.8020   |
+| 0.831         | 42.1053  | 4000  | 1.0972          | 0.8147   |
+| 0.6805        | 52.6316  | 5000  | 0.9968          | 0.8256   |
+| 0.5937        | 63.1579  | 6000  | 1.0310          | 0.8243   |
+| 0.5258        | 73.6842  | 7000  | 1.1045          | 0.8151   |
+| 0.4569        | 84.2105  | 8000  | 1.0393          | 0.8254   |
+| 0.4007        | 94.7368  | 9000  | 1.0684          | 0.8217   |
+| 0.3632        | 105.2632 | 10000 | 1.1223          | 0.8182   |
+| 0.3343        | 115.7895 | 11000 | 1.1048          | 0.8230   |
+| 0.2998        | 126.3158 | 12000 | 1.0996          | 0.8218   |
+| 0.2817        | 136.8421 | 13000 | 1.0880          | 0.8320   |
+| 0.2568        | 147.3684 | 14000 | 1.1189          | 0.8216   |
+| 0.2396        | 157.8947 | 15000 | 1.1026          | 0.8267   |
+| 0.219         | 168.4211 | 16000 | 1.1284          | 0.8241   |
+| 0.2028        | 178.9474 | 17000 | 1.1205          | 0.8243   |
+| 0.1927        | 189.4737 | 18000 | 1.1104          | 0.8313   |
+| 0.1841        | 200.0    | 19000 | 1.0284          | 0.8348   |
+| 0.1687        | 210.5263 | 20000 | 1.1662          | 0.8266   |
+| 0.1627        | 221.0526 | 21000 | 1.1330          | 0.8278   |
+| 0.1564        | 231.5789 | 22000 | 1.1413          | 0.8265   |
+| 0.1483        | 242.1053 | 23000 | 1.1836          | 0.8246   |
+| 0.1439        | 252.6316 | 24000 | 1.2169          | 0.8179   |
+| 0.1396        | 263.1579 | 25000 | 1.1871          | 0.8266   |
+| 0.1364        | 273.6842 | 26000 | 1.1696          | 0.8301   |
+| 0.1314        | 284.2105 | 27000 | 1.1557          | 0.8324   |
+| 0.1295        | 294.7368 | 28000 | 1.1712          | 0.8298   |
+| 0.1296        | 305.2632 | 29000 | 1.1821          | 0.8273   |
+| 0.1251        | 315.7895 | 30000 | 1.1567          | 0.8262   |
+
+
+### Framework versions
+
+- Transformers 4.40.2
+- Pytorch 2.2.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 64000
+}

all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 315.7894736842105,
+    "eval_accuracy": 0.8340939202119829,
+    "eval_loss": 1.1156361103057861,
+    "eval_runtime": 0.9791,
+    "eval_samples": 399,
+    "eval_samples_per_second": 407.517,
+    "eval_steps_per_second": 13.277,
+    "perplexity": 3.051508658043527,
+    "total_flos": 6.261324507818496e+16,
+    "train_loss": 0.5399098532358806,
+    "train_runtime": 6604.7234,
+    "train_samples": 3012,
+    "train_samples_per_second": 145.351,
+    "train_steps_per_second": 4.542
+}

checkpoint-1000/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 64000
+}

checkpoint-1000/config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "demdecuong/vihealthbert-base-word",
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "finetuning_task": "word-level",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 258,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tokenizer_class": "PhobertTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.40.2",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 64001
+}

checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fde2113281f99b697a1c9b4fa1cc9b1c412ffc94e8cfc01909c918f354376ccd
+size 540279452

checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f359bbd67da057ea08ad75f5155d032cafc0e71594dafecd311efddbddebbd87
+size 1080631290

checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32cb63b93fa8fc30aa1a3595f75ce57e9b0eadf08b988b70abfb88a4fbce5349
+size 14308

checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a63abf127bfb7a79cfff9a24142b4b60edd1ec07fff069d4830232793c6f6bf
+size 1064

checkpoint-1000/special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,54 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64000": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "PhobertTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-1000/trainer_state.json

@@ -0,0 +1,44 @@
+{
+  "best_metric": 0.588971337116252,
+  "best_model_checkpoint": "/home/tmnam/Desktop/crosslingual-mining-for-domain-nli/output/pretraining/vihealthbert-w_mlm-ViMedNLI/lr3e-5_wr0.1_wd0.0/checkpoint-1000",
+  "epoch": 10.526315789473685,
+  "eval_steps": 1000,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010526315789473684,
+      "grad_norm": 44.032386779785156,
+      "learning_rate": 1e-08,
+      "loss": 11.5293,
+      "step": 1
+    },
+    {
+      "epoch": 10.526315789473685,
+      "grad_norm": 5.0217604637146,
+      "learning_rate": 9.999999999999999e-06,
+      "loss": 5.5327,
+      "step": 1000
+    },
+    {
+      "epoch": 10.526315789473685,
+      "eval_accuracy": 0.588971337116252,
+      "eval_loss": 2.7528159618377686,
+      "eval_runtime": 0.9763,
+      "eval_samples_per_second": 408.668,
+      "eval_steps_per_second": 13.315,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 30000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 316,
+  "save_steps": 1000,
+  "total_flos": 2088029746083840.0,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60e8bf70fd8d429fbee6487cc8f4c553c61462925a541b83c0ef6e04da9e123c
+size 5432

checkpoint-10000/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 64000
+}

checkpoint-10000/config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "demdecuong/vihealthbert-base-word",
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "finetuning_task": "word-level",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 258,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tokenizer_class": "PhobertTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.40.2",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 64001
+}

checkpoint-10000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5eeb8905245ac97ed1e919b8c820da04604f5addc6788d6bc7e0a494c68922b
+size 540279452

checkpoint-10000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e237e8afbb2e15d5e3ee41184a877fbbf7391c3415cb659515eab479003bd89
+size 1080631290

checkpoint-10000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa784583a63c1ef06bf24646e41befddb28ec405a513ae587fee4d0005a2c3e0
+size 14308

checkpoint-10000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16708bba77628982fefaa5f607419f239634259413dfae59721b020fe02e11e7
+size 1064

checkpoint-10000/special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-10000/tokenizer_config.json

@@ -0,0 +1,54 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64000": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "PhobertTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-10000/trainer_state.json

@@ -0,0 +1,188 @@
+{
+  "best_metric": 0.8255746290369508,
+  "best_model_checkpoint": "/home/tmnam/Desktop/crosslingual-mining-for-domain-nli/output/pretraining/vihealthbert-w_mlm-ViMedNLI/lr3e-5_wr0.1_wd0.0/checkpoint-5000",
+  "epoch": 105.26315789473684,
+  "eval_steps": 1000,
+  "global_step": 10000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010526315789473684,
+      "grad_norm": 44.032386779785156,
+      "learning_rate": 1e-08,
+      "loss": 11.5293,
+      "step": 1
+    },
+    {
+      "epoch": 10.526315789473685,
+      "grad_norm": 5.0217604637146,
+      "learning_rate": 9.999999999999999e-06,
+      "loss": 5.5327,
+      "step": 1000
+    },
+    {
+      "epoch": 10.526315789473685,
+      "eval_accuracy": 0.588971337116252,
+      "eval_loss": 2.7528159618377686,
+      "eval_runtime": 0.9763,
+      "eval_samples_per_second": 408.668,
+      "eval_steps_per_second": 13.315,
+      "step": 1000
+    },
+    {
+      "epoch": 21.05263157894737,
+      "grad_norm": 3.756152629852295,
+      "learning_rate": 1.9999999999999998e-05,
+      "loss": 1.9051,
+      "step": 2000
+    },
+    {
+      "epoch": 21.05263157894737,
+      "eval_accuracy": 0.7783207463349623,
+      "eval_loss": 1.467841625213623,
+      "eval_runtime": 0.9735,
+      "eval_samples_per_second": 409.849,
+      "eval_steps_per_second": 13.353,
+      "step": 2000
+    },
+    {
+      "epoch": 31.57894736842105,
+      "grad_norm": 2.861485481262207,
+      "learning_rate": 3e-05,
+      "loss": 1.1194,
+      "step": 3000
+    },
+    {
+      "epoch": 31.57894736842105,
+      "eval_accuracy": 0.8019903409922435,
+      "eval_loss": 1.1543285846710205,
+      "eval_runtime": 0.9729,
+      "eval_samples_per_second": 410.113,
+      "eval_steps_per_second": 13.362,
+      "step": 3000
+    },
+    {
+      "epoch": 42.10526315789474,
+      "grad_norm": 3.267728805541992,
+      "learning_rate": 2.8888888888888888e-05,
+      "loss": 0.831,
+      "step": 4000
+    },
+    {
+      "epoch": 42.10526315789474,
+      "eval_accuracy": 0.8146666666666667,
+      "eval_loss": 1.097205638885498,
+      "eval_runtime": 0.973,
+      "eval_samples_per_second": 410.092,
+      "eval_steps_per_second": 13.361,
+      "step": 4000
+    },
+    {
+      "epoch": 52.63157894736842,
+      "grad_norm": 2.2974698543548584,
+      "learning_rate": 2.777777777777778e-05,
+      "loss": 0.6805,
+      "step": 5000
+    },
+    {
+      "epoch": 52.63157894736842,
+      "eval_accuracy": 0.8255746290369508,
+      "eval_loss": 0.9968159794807434,
+      "eval_runtime": 0.9748,
+      "eval_samples_per_second": 409.295,
+      "eval_steps_per_second": 13.335,
+      "step": 5000
+    },
+    {
+      "epoch": 63.1578947368421,
+      "grad_norm": 2.481663703918457,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 0.5937,
+      "step": 6000
+    },
+    {
+      "epoch": 63.1578947368421,
+      "eval_accuracy": 0.8242695979172693,
+      "eval_loss": 1.0310105085372925,
+      "eval_runtime": 0.9734,
+      "eval_samples_per_second": 409.886,
+      "eval_steps_per_second": 13.355,
+      "step": 6000
+    },
+    {
+      "epoch": 73.6842105263158,
+      "grad_norm": 2.574080467224121,
+      "learning_rate": 2.5555555555555557e-05,
+      "loss": 0.5258,
+      "step": 7000
+    },
+    {
+      "epoch": 73.6842105263158,
+      "eval_accuracy": 0.8151164452064581,
+      "eval_loss": 1.1044955253601074,
+      "eval_runtime": 0.9728,
+      "eval_samples_per_second": 410.166,
+      "eval_steps_per_second": 13.364,
+      "step": 7000
+    },
+    {
+      "epoch": 84.21052631578948,
+      "grad_norm": 2.3108158111572266,
+      "learning_rate": 2.4444444444444445e-05,
+      "loss": 0.4569,
+      "step": 8000
+    },
+    {
+      "epoch": 84.21052631578948,
+      "eval_accuracy": 0.8254382152687237,
+      "eval_loss": 1.0392757654190063,
+      "eval_runtime": 0.9756,
+      "eval_samples_per_second": 408.981,
+      "eval_steps_per_second": 13.325,
+      "step": 8000
+    },
+    {
+      "epoch": 94.73684210526316,
+      "grad_norm": 2.3435771465301514,
+      "learning_rate": 2.3333333333333336e-05,
+      "loss": 0.4007,
+      "step": 9000
+    },
+    {
+      "epoch": 94.73684210526316,
+      "eval_accuracy": 0.8216863990759458,
+      "eval_loss": 1.0684024095535278,
+      "eval_runtime": 0.9715,
+      "eval_samples_per_second": 410.719,
+      "eval_steps_per_second": 13.382,
+      "step": 9000
+    },
+    {
+      "epoch": 105.26315789473684,
+      "grad_norm": 4.333491325378418,
+      "learning_rate": 2.222222222222222e-05,
+      "loss": 0.3632,
+      "step": 10000
+    },
+    {
+      "epoch": 105.26315789473684,
+      "eval_accuracy": 0.8181686046511628,
+      "eval_loss": 1.122326374053955,
+      "eval_runtime": 0.9742,
+      "eval_samples_per_second": 409.576,
+      "eval_steps_per_second": 13.345,
+      "step": 10000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 30000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 316,
+  "save_steps": 1000,
+  "total_flos": 2.087108169272832e+16,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-10000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60e8bf70fd8d429fbee6487cc8f4c553c61462925a541b83c0ef6e04da9e123c
+size 5432

checkpoint-11000/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 64000
+}

checkpoint-11000/config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "demdecuong/vihealthbert-base-word",
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "finetuning_task": "word-level",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 258,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tokenizer_class": "PhobertTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.40.2",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 64001
+}

checkpoint-11000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d39d82b8c5ab4bc2a111eb62f5afc990665a77d040d4dc7cc722c59383a06719
+size 540279452

checkpoint-11000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b558c060eb4395a9f6c05c72e06bd41b76ede63a13df200565f9ae6b8ee32c02
+size 1080631290

checkpoint-11000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f94b819fcea5812fb5cab2f349a90f8420c83038260c610b716d6bfdd5c3c08
+size 14308

checkpoint-11000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3f3cf82b6a539bfb71271613d780d1122a5eb60bc12ea263ed134affaffcf3c
+size 1064

checkpoint-11000/special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-11000/tokenizer_config.json

@@ -0,0 +1,54 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64000": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "PhobertTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-11000/trainer_state.json

@@ -0,0 +1,204 @@
+{
+  "best_metric": 0.8255746290369508,
+  "best_model_checkpoint": "/home/tmnam/Desktop/crosslingual-mining-for-domain-nli/output/pretraining/vihealthbert-w_mlm-ViMedNLI/lr3e-5_wr0.1_wd0.0/checkpoint-5000",
+  "epoch": 115.78947368421052,
+  "eval_steps": 1000,
+  "global_step": 11000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010526315789473684,
+      "grad_norm": 44.032386779785156,
+      "learning_rate": 1e-08,
+      "loss": 11.5293,
+      "step": 1
+    },
+    {
+      "epoch": 10.526315789473685,
+      "grad_norm": 5.0217604637146,
+      "learning_rate": 9.999999999999999e-06,
+      "loss": 5.5327,
+      "step": 1000
+    },
+    {
+      "epoch": 10.526315789473685,
+      "eval_accuracy": 0.588971337116252,
+      "eval_loss": 2.7528159618377686,
+      "eval_runtime": 0.9763,
+      "eval_samples_per_second": 408.668,
+      "eval_steps_per_second": 13.315,
+      "step": 1000
+    },
+    {
+      "epoch": 21.05263157894737,
+      "grad_norm": 3.756152629852295,
+      "learning_rate": 1.9999999999999998e-05,
+      "loss": 1.9051,
+      "step": 2000
+    },
+    {
+      "epoch": 21.05263157894737,
+      "eval_accuracy": 0.7783207463349623,
+      "eval_loss": 1.467841625213623,
+      "eval_runtime": 0.9735,
+      "eval_samples_per_second": 409.849,
+      "eval_steps_per_second": 13.353,
+      "step": 2000
+    },
+    {
+      "epoch": 31.57894736842105,
+      "grad_norm": 2.861485481262207,
+      "learning_rate": 3e-05,
+      "loss": 1.1194,
+      "step": 3000
+    },
+    {
+      "epoch": 31.57894736842105,
+      "eval_accuracy": 0.8019903409922435,
+      "eval_loss": 1.1543285846710205,
+      "eval_runtime": 0.9729,
+      "eval_samples_per_second": 410.113,
+      "eval_steps_per_second": 13.362,
+      "step": 3000
+    },
+    {
+      "epoch": 42.10526315789474,
+      "grad_norm": 3.267728805541992,
+      "learning_rate": 2.8888888888888888e-05,
+      "loss": 0.831,
+      "step": 4000
+    },
+    {
+      "epoch": 42.10526315789474,
+      "eval_accuracy": 0.8146666666666667,
+      "eval_loss": 1.097205638885498,
+      "eval_runtime": 0.973,
+      "eval_samples_per_second": 410.092,
+      "eval_steps_per_second": 13.361,
+      "step": 4000
+    },
+    {
+      "epoch": 52.63157894736842,
+      "grad_norm": 2.2974698543548584,
+      "learning_rate": 2.777777777777778e-05,
+      "loss": 0.6805,
+      "step": 5000
+    },
+    {
+      "epoch": 52.63157894736842,
+      "eval_accuracy": 0.8255746290369508,
+      "eval_loss": 0.9968159794807434,
+      "eval_runtime": 0.9748,
+      "eval_samples_per_second": 409.295,
+      "eval_steps_per_second": 13.335,
+      "step": 5000
+    },
+    {
+      "epoch": 63.1578947368421,
+      "grad_norm": 2.481663703918457,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 0.5937,
+      "step": 6000
+    },
+    {
+      "epoch": 63.1578947368421,
+      "eval_accuracy": 0.8242695979172693,
+      "eval_loss": 1.0310105085372925,
+      "eval_runtime": 0.9734,
+      "eval_samples_per_second": 409.886,
+      "eval_steps_per_second": 13.355,
+      "step": 6000
+    },
+    {
+      "epoch": 73.6842105263158,
+      "grad_norm": 2.574080467224121,
+      "learning_rate": 2.5555555555555557e-05,
+      "loss": 0.5258,
+      "step": 7000
+    },
+    {
+      "epoch": 73.6842105263158,
+      "eval_accuracy": 0.8151164452064581,
+      "eval_loss": 1.1044955253601074,
+      "eval_runtime": 0.9728,
+      "eval_samples_per_second": 410.166,
+      "eval_steps_per_second": 13.364,
+      "step": 7000
+    },
+    {
+      "epoch": 84.21052631578948,
+      "grad_norm": 2.3108158111572266,
+      "learning_rate": 2.4444444444444445e-05,
+      "loss": 0.4569,
+      "step": 8000
+    },
+    {
+      "epoch": 84.21052631578948,
+      "eval_accuracy": 0.8254382152687237,
+      "eval_loss": 1.0392757654190063,
+      "eval_runtime": 0.9756,
+      "eval_samples_per_second": 408.981,
+      "eval_steps_per_second": 13.325,
+      "step": 8000
+    },
+    {
+      "epoch": 94.73684210526316,
+      "grad_norm": 2.3435771465301514,
+      "learning_rate": 2.3333333333333336e-05,
+      "loss": 0.4007,
+      "step": 9000
+    },
+    {
+      "epoch": 94.73684210526316,
+      "eval_accuracy": 0.8216863990759458,
+      "eval_loss": 1.0684024095535278,
+      "eval_runtime": 0.9715,
+      "eval_samples_per_second": 410.719,
+      "eval_steps_per_second": 13.382,
+      "step": 9000
+    },
+    {
+      "epoch": 105.26315789473684,
+      "grad_norm": 4.333491325378418,
+      "learning_rate": 2.222222222222222e-05,
+      "loss": 0.3632,
+      "step": 10000
+    },
+    {
+      "epoch": 105.26315789473684,
+      "eval_accuracy": 0.8181686046511628,
+      "eval_loss": 1.122326374053955,
+      "eval_runtime": 0.9742,
+      "eval_samples_per_second": 409.576,
+      "eval_steps_per_second": 13.345,
+      "step": 10000
+    },
+    {
+      "epoch": 115.78947368421052,
+      "grad_norm": 2.059063196182251,
+      "learning_rate": 2.111111111111111e-05,
+      "loss": 0.3343,
+      "step": 11000
+    },
+    {
+      "epoch": 115.78947368421052,
+      "eval_accuracy": 0.8230374174614821,
+      "eval_loss": 1.1047978401184082,
+      "eval_runtime": 0.9722,
+      "eval_samples_per_second": 410.402,
+      "eval_steps_per_second": 13.371,
+      "step": 11000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 30000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 316,
+  "save_steps": 1000,
+  "total_flos": 2.295911143881216e+16,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-11000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60e8bf70fd8d429fbee6487cc8f4c553c61462925a541b83c0ef6e04da9e123c
+size 5432

checkpoint-12000/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 64000
+}

checkpoint-12000/config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "demdecuong/vihealthbert-base-word",
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "finetuning_task": "word-level",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 258,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tokenizer_class": "PhobertTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.40.2",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 64001
+}

checkpoint-12000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a49de740a03b7fd359a7014f1848dc4c35e92de380396a6417a5b922083245cf
+size 540279452

checkpoint-12000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fbe23771933f88548e6203c5cbfc08c180daa9513a29aa097818c33d525913c
+size 1080631290

checkpoint-12000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b10a5f96c9da18eef368331f46689d8b06a095af3651b031974c83a29a3eea53
+size 14308

checkpoint-12000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f00858c4bde6fa49429f0c635a130d8a10928b5995514184697fc31d699936bf
+size 1064

checkpoint-12000/special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-12000/tokenizer_config.json

@@ -0,0 +1,54 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64000": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "PhobertTokenizer",
+  "unk_token": "<unk>"
+}