upload

README.md

@@ -1,9 +1,14 @@
 ---
 license: apache-2.0
+base_model: meta-llama/Llama-2-7b-hf
 language:
 - en
-library_name: peft
 pipeline_tag: text2text-generation
+tags:
+- punica
+- llama-factory
+- lora
+- generated_from_trainer
 ---
 
 * Base Model: [Llama-2-7b-hf](https://huggingface.co/meta-llama/Llama-2-7b-hf)
@@ -11,4 +16,22 @@ pipeline_tag: text2text-generation
 * LoRA rank: 16
 * Training epochs: 4
 
-See <https://github.com/punica-ai/punica/tree/master/examples/finetune>
+See <https://github.com/punica-ai/punica/tree/master/examples/finetune>
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 32
+- eval_batch_size: 8
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 4.0
+
+### Framework versions
+
+- Transformers 4.34.1
+- Pytorch 2.2.0.dev20230911+cu121
+- Datasets 2.14.4
+- Tokenizers 0.14.1

adapter_config.json

@@ -0,0 +1,28 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16.0,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "k_proj",
+    "q_proj",
+    "up_proj",
+    "v_proj",
+    "gate_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f9f8258c444218b6c652eb2c84cc29b36ca1c0223bfbd31c34460defeb01d8d
+size 160069834

all_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 4.0,
+    "train_loss": 0.40969020433914966,
+    "train_runtime": 4321.1174,
+    "train_samples_per_second": 6.918,
+    "train_steps_per_second": 0.217
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,40 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}

train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 4.0,
+    "train_loss": 0.40969020433914966,
+    "train_runtime": 4321.1174,
+    "train_samples_per_second": 6.918,
+    "train_steps_per_second": 0.217
+}

trainer_log.jsonl

@@ -0,0 +1,94 @@
+{"current_steps": 10, "total_steps": 936, "loss": 1.0499, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.9985919525527434e-05, "epoch": 0.04, "percentage": 1.07, "elapsed_time": "0:00:46", "remaining_time": "1:12:01"}
+{"current_steps": 20, "total_steps": 936, "loss": 0.7822, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.994369396289063e-05, "epoch": 0.09, "percentage": 2.14, "elapsed_time": "0:01:33", "remaining_time": "1:11:07"}
+{"current_steps": 30, "total_steps": 936, "loss": 0.575, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.987337087656614e-05, "epoch": 0.13, "percentage": 3.21, "elapsed_time": "0:02:19", "remaining_time": "1:10:17"}
+{"current_steps": 40, "total_steps": 936, "loss": 0.5199, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.977502948114772e-05, "epoch": 0.17, "percentage": 4.27, "elapsed_time": "0:03:04", "remaining_time": "1:08:50"}
+{"current_steps": 50, "total_steps": 936, "loss": 0.5358, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.964878055211597e-05, "epoch": 0.21, "percentage": 5.34, "elapsed_time": "0:03:50", "remaining_time": "1:08:04"}
+{"current_steps": 60, "total_steps": 936, "loss": 0.508, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.949476630105669e-05, "epoch": 0.26, "percentage": 6.41, "elapsed_time": "0:04:37", "remaining_time": "1:07:26"}
+{"current_steps": 70, "total_steps": 936, "loss": 0.4915, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.9313160215468334e-05, "epoch": 0.3, "percentage": 7.48, "elapsed_time": "0:05:25", "remaining_time": "1:07:04"}
+{"current_steps": 80, "total_steps": 936, "loss": 0.4856, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.910416686333906e-05, "epoch": 0.34, "percentage": 8.55, "elapsed_time": "0:06:10", "remaining_time": "1:06:05"}
+{"current_steps": 90, "total_steps": 936, "loss": 0.479, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.886802166271364e-05, "epoch": 0.38, "percentage": 9.62, "elapsed_time": "0:06:56", "remaining_time": "1:05:18"}
+{"current_steps": 100, "total_steps": 936, "loss": 0.4707, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.8604990616509616e-05, "epoch": 0.43, "percentage": 10.68, "elapsed_time": "0:07:42", "remaining_time": "1:04:23"}
+{"current_steps": 110, "total_steps": 936, "loss": 0.4687, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.8315370012881514e-05, "epoch": 0.47, "percentage": 11.75, "elapsed_time": "0:08:31", "remaining_time": "1:03:58"}
+{"current_steps": 120, "total_steps": 936, "loss": 0.4834, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.799948609147061e-05, "epoch": 0.51, "percentage": 12.82, "elapsed_time": "0:09:16", "remaining_time": "1:03:06"}
+{"current_steps": 130, "total_steps": 936, "loss": 0.4541, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.765769467591625e-05, "epoch": 0.56, "percentage": 13.89, "elapsed_time": "0:10:04", "remaining_time": "1:02:26"}
+{"current_steps": 140, "total_steps": 936, "loss": 0.4442, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.7290380773042575e-05, "epoch": 0.6, "percentage": 14.96, "elapsed_time": "0:10:46", "remaining_time": "1:01:16"}
+{"current_steps": 150, "total_steps": 936, "loss": 0.4528, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.68979581391722e-05, "epoch": 0.64, "percentage": 16.03, "elapsed_time": "0:11:33", "remaining_time": "1:00:36"}
+{"current_steps": 160, "total_steps": 936, "loss": 0.436, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.6480868814055424e-05, "epoch": 0.68, "percentage": 17.09, "elapsed_time": "0:12:19", "remaining_time": "0:59:47"}
+{"current_steps": 170, "total_steps": 936, "loss": 0.4504, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.6039582622939854e-05, "epoch": 0.73, "percentage": 18.16, "elapsed_time": "0:13:04", "remaining_time": "0:58:56"}
+{"current_steps": 180, "total_steps": 936, "loss": 0.4348, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.557459664734141e-05, "epoch": 0.77, "percentage": 19.23, "elapsed_time": "0:13:50", "remaining_time": "0:58:09"}
+{"current_steps": 190, "total_steps": 936, "loss": 0.4283, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.5086434665112864e-05, "epoch": 0.81, "percentage": 20.3, "elapsed_time": "0:14:35", "remaining_time": "0:57:17"}
+{"current_steps": 200, "total_steps": 936, "loss": 0.4527, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.457564656044056e-05, "epoch": 0.85, "percentage": 21.37, "elapsed_time": "0:15:23", "remaining_time": "0:56:39"}
+{"current_steps": 210, "total_steps": 936, "loss": 0.4525, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.404280770443398e-05, "epoch": 0.9, "percentage": 22.44, "elapsed_time": "0:16:09", "remaining_time": "0:55:52"}
+{"current_steps": 220, "total_steps": 936, "loss": 0.4315, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.348851830700593e-05, "epoch": 0.94, "percentage": 23.5, "elapsed_time": "0:16:55", "remaining_time": "0:55:05"}
+{"current_steps": 230, "total_steps": 936, "loss": 0.4309, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.2913402740773294e-05, "epoch": 0.98, "percentage": 24.57, "elapsed_time": "0:17:42", "remaining_time": "0:54:22"}
+{"current_steps": 240, "total_steps": 936, "loss": 0.4207, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.231810883773999e-05, "epoch": 1.03, "percentage": 25.64, "elapsed_time": "0:18:26", "remaining_time": "0:53:28"}
+{"current_steps": 250, "total_steps": 936, "loss": 0.4189, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.170330715955444e-05, "epoch": 1.07, "percentage": 26.71, "elapsed_time": "0:19:12", "remaining_time": "0:52:43"}
+{"current_steps": 260, "total_steps": 936, "loss": 0.4353, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.1069690242163484e-05, "epoch": 1.11, "percentage": 27.78, "elapsed_time": "0:19:58", "remaining_time": "0:51:56"}
+{"current_steps": 270, "total_steps": 936, "loss": 0.4054, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.0417971815713584e-05, "epoch": 1.15, "percentage": 28.85, "elapsed_time": "0:20:47", "remaining_time": "0:51:16"}
+{"current_steps": 280, "total_steps": 936, "loss": 0.4152, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.974888600057808e-05, "epoch": 1.2, "percentage": 29.91, "elapsed_time": "0:21:33", "remaining_time": "0:50:29"}
+{"current_steps": 290, "total_steps": 936, "loss": 0.3879, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.906318648041617e-05, "epoch": 1.24, "percentage": 30.98, "elapsed_time": "0:22:18", "remaining_time": "0:49:42"}
+{"current_steps": 300, "total_steps": 936, "loss": 0.4164, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.8361645653195026e-05, "epoch": 1.28, "percentage": 32.05, "elapsed_time": "0:23:05", "remaining_time": "0:48:57"}
+{"current_steps": 310, "total_steps": 936, "loss": 0.406, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.764505376113138e-05, "epoch": 1.32, "percentage": 33.12, "elapsed_time": "0:23:51", "remaining_time": "0:48:10"}
+{"current_steps": 320, "total_steps": 936, "loss": 0.4076, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.69142180005327e-05, "epoch": 1.37, "percentage": 34.19, "elapsed_time": "0:24:38", "remaining_time": "0:47:25"}
+{"current_steps": 330, "total_steps": 936, "loss": 0.3953, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.6169961612540645e-05, "epoch": 1.41, "percentage": 35.26, "elapsed_time": "0:25:21", "remaining_time": "0:46:33"}
+{"current_steps": 340, "total_steps": 936, "loss": 0.4105, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.5413122955801005e-05, "epoch": 1.45, "percentage": 36.32, "elapsed_time": "0:26:04", "remaining_time": "0:45:43"}
+{"current_steps": 350, "total_steps": 936, "loss": 0.4014, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.4644554562104634e-05, "epoch": 1.5, "percentage": 37.39, "elapsed_time": "0:26:51", "remaining_time": "0:44:57"}
+{"current_steps": 360, "total_steps": 936, "loss": 0.4234, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.386512217606339e-05, "epoch": 1.54, "percentage": 38.46, "elapsed_time": "0:27:37", "remaining_time": "0:44:11"}
+{"current_steps": 370, "total_steps": 936, "loss": 0.4012, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.307570377990245e-05, "epoch": 1.58, "percentage": 39.53, "elapsed_time": "0:28:23", "remaining_time": "0:43:26"}
+{"current_steps": 380, "total_steps": 936, "loss": 0.3979, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.227718860446782e-05, "epoch": 1.62, "percentage": 40.6, "elapsed_time": "0:29:10", "remaining_time": "0:42:41"}
+{"current_steps": 390, "total_steps": 936, "loss": 0.3877, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.147047612756302e-05, "epoch": 1.67, "percentage": 41.67, "elapsed_time": "0:29:53", "remaining_time": "0:41:50"}
+{"current_steps": 400, "total_steps": 936, "loss": 0.4024, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.065647506074306e-05, "epoch": 1.71, "percentage": 42.74, "elapsed_time": "0:30:46", "remaining_time": "0:41:13"}
+{"current_steps": 410, "total_steps": 936, "loss": 0.3994, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.983610232570728e-05, "epoch": 1.75, "percentage": 43.8, "elapsed_time": "0:31:33", "remaining_time": "0:40:29"}
+{"current_steps": 420, "total_steps": 936, "loss": 0.4052, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.9010282021444008e-05, "epoch": 1.79, "percentage": 44.87, "elapsed_time": "0:32:17", "remaining_time": "0:39:40"}
+{"current_steps": 430, "total_steps": 936, "loss": 0.4038, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.8179944383290274e-05, "epoch": 1.84, "percentage": 45.94, "elapsed_time": "0:33:00", "remaining_time": "0:38:50"}
+{"current_steps": 440, "total_steps": 936, "loss": 0.3999, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.7346024735079486e-05, "epoch": 1.88, "percentage": 47.01, "elapsed_time": "0:33:48", "remaining_time": "0:38:06"}
+{"current_steps": 450, "total_steps": 936, "loss": 0.3848, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.6509462435557152e-05, "epoch": 1.92, "percentage": 48.08, "elapsed_time": "0:34:33", "remaining_time": "0:37:19"}
+{"current_steps": 460, "total_steps": 936, "loss": 0.4126, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.5671199820251534e-05, "epoch": 1.97, "percentage": 49.15, "elapsed_time": "0:35:19", "remaining_time": "0:36:33"}
+{"current_steps": 470, "total_steps": 936, "loss": 0.3996, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.48321811399911e-05, "epoch": 2.01, "percentage": 50.21, "elapsed_time": "0:36:07", "remaining_time": "0:35:48"}
+{"current_steps": 480, "total_steps": 936, "loss": 0.3672, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.399335149726463e-05, "epoch": 2.05, "percentage": 51.28, "elapsed_time": "0:36:52", "remaining_time": "0:35:02"}
+{"current_steps": 490, "total_steps": 936, "loss": 0.3728, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.3155655781621793e-05, "epoch": 2.09, "percentage": 52.35, "elapsed_time": "0:37:40", "remaining_time": "0:34:17"}
+{"current_steps": 500, "total_steps": 936, "loss": 0.3901, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.2320037605313808e-05, "epoch": 2.14, "percentage": 53.42, "elapsed_time": "0:38:29", "remaining_time": "0:33:33"}
+{"current_steps": 510, "total_steps": 936, "loss": 0.3848, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.148743824037269e-05, "epoch": 2.18, "percentage": 54.49, "elapsed_time": "0:39:15", "remaining_time": "0:32:47"}
+{"current_steps": 520, "total_steps": 936, "loss": 0.3729, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.0658795558326743e-05, "epoch": 2.22, "percentage": 55.56, "elapsed_time": "0:40:03", "remaining_time": "0:32:03"}
+{"current_steps": 530, "total_steps": 936, "loss": 0.3709, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.98350429737465e-05, "epoch": 2.26, "percentage": 56.62, "elapsed_time": "0:40:51", "remaining_time": "0:31:17"}
+{"current_steps": 540, "total_steps": 936, "loss": 0.3811, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.9017108392811065e-05, "epoch": 2.31, "percentage": 57.69, "elapsed_time": "0:41:36", "remaining_time": "0:30:30"}
+{"current_steps": 550, "total_steps": 936, "loss": 0.3639, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.820591316807939e-05, "epoch": 2.35, "percentage": 58.76, "elapsed_time": "0:42:19", "remaining_time": "0:29:42"}
+{"current_steps": 560, "total_steps": 936, "loss": 0.3693, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.740237106064383e-05, "epoch": 2.39, "percentage": 59.83, "elapsed_time": "0:43:04", "remaining_time": "0:28:55"}
+{"current_steps": 570, "total_steps": 936, "loss": 0.3769, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.6607387210834887e-05, "epoch": 2.44, "percentage": 60.9, "elapsed_time": "0:43:51", "remaining_time": "0:28:09"}
+{"current_steps": 580, "total_steps": 936, "loss": 0.3621, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.582185711863681e-05, "epoch": 2.48, "percentage": 61.97, "elapsed_time": "0:44:35", "remaining_time": "0:27:22"}
+{"current_steps": 590, "total_steps": 936, "loss": 0.3585, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.5046665634962476e-05, "epoch": 2.52, "percentage": 63.03, "elapsed_time": "0:45:25", "remaining_time": "0:26:38"}
+{"current_steps": 600, "total_steps": 936, "loss": 0.3705, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.4282685964923642e-05, "epoch": 2.56, "percentage": 64.1, "elapsed_time": "0:46:08", "remaining_time": "0:25:50"}
+{"current_steps": 610, "total_steps": 936, "loss": 0.3662, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.3530778684219648e-05, "epoch": 2.61, "percentage": 65.17, "elapsed_time": "0:46:52", "remaining_time": "0:25:03"}
+{"current_steps": 620, "total_steps": 936, "loss": 0.3831, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.2791790769752232e-05, "epoch": 2.65, "percentage": 66.24, "elapsed_time": "0:47:37", "remaining_time": "0:24:16"}
+{"current_steps": 630, "total_steps": 936, "loss": 0.3874, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.2066554645558578e-05, "epoch": 2.69, "percentage": 67.31, "elapsed_time": "0:48:24", "remaining_time": "0:23:30"}
+{"current_steps": 640, "total_steps": 936, "loss": 0.3563, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.1355887245137383e-05, "epoch": 2.74, "percentage": 68.38, "elapsed_time": "0:49:13", "remaining_time": "0:22:45"}
+{"current_steps": 650, "total_steps": 936, "loss": 0.3612, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.0660589091223855e-05, "epoch": 2.78, "percentage": 69.44, "elapsed_time": "0:49:59", "remaining_time": "0:21:59"}
+{"current_steps": 660, "total_steps": 936, "loss": 0.3721, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.981443394050525e-06, "epoch": 2.82, "percentage": 70.51, "elapsed_time": "0:50:44", "remaining_time": "0:21:13"}
+{"current_steps": 670, "total_steps": 936, "loss": 0.3744, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.3192151691096e-06, "epoch": 2.86, "percentage": 71.58, "elapsed_time": "0:51:30", "remaining_time": "0:20:26"}
+{"current_steps": 680, "total_steps": 936, "loss": 0.3553, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.67465037541038e-06, "epoch": 2.91, "percentage": 72.65, "elapsed_time": "0:52:18", "remaining_time": "0:19:41"}
+{"current_steps": 690, "total_steps": 936, "loss": 0.3685, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.048475075202727e-06, "epoch": 2.95, "percentage": 73.72, "elapsed_time": "0:53:02", "remaining_time": "0:18:54"}
+{"current_steps": 700, "total_steps": 936, "loss": 0.3731, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.441394616113062e-06, "epoch": 2.99, "percentage": 74.79, "elapsed_time": "0:53:45", "remaining_time": "0:18:07"}
+{"current_steps": 710, "total_steps": 936, "loss": 0.3529, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.854092836613948e-06, "epoch": 3.03, "percentage": 75.85, "elapsed_time": "0:54:30", "remaining_time": "0:17:20"}
+{"current_steps": 720, "total_steps": 936, "loss": 0.3542, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.28723129572247e-06, "epoch": 3.08, "percentage": 76.92, "elapsed_time": "0:55:18", "remaining_time": "0:16:35"}
+{"current_steps": 730, "total_steps": 936, "loss": 0.3587, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.741448527795137e-06, "epoch": 3.12, "percentage": 77.99, "elapsed_time": "0:56:07", "remaining_time": "0:15:50"}
+{"current_steps": 740, "total_steps": 936, "loss": 0.3583, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.217359323258459e-06, "epoch": 3.16, "percentage": 79.06, "elapsed_time": "0:56:53", "remaining_time": "0:15:04"}
+{"current_steps": 750, "total_steps": 936, "loss": 0.3589, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.715554036085673e-06, "epoch": 3.21, "percentage": 80.13, "elapsed_time": "0:57:41", "remaining_time": "0:14:18"}
+{"current_steps": 760, "total_steps": 936, "loss": 0.3424, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.236597918799709e-06, "epoch": 3.25, "percentage": 81.2, "elapsed_time": "0:58:28", "remaining_time": "0:13:32"}
+{"current_steps": 770, "total_steps": 936, "loss": 0.3455, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.7810304857511914e-06, "epoch": 3.29, "percentage": 82.26, "elapsed_time": "0:59:13", "remaining_time": "0:12:45"}
+{"current_steps": 780, "total_steps": 936, "loss": 0.3667, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.3493649053890326e-06, "epoch": 3.33, "percentage": 83.33, "elapsed_time": "1:00:00", "remaining_time": "0:12:00"}
+{"current_steps": 790, "total_steps": 936, "loss": 0.3579, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.942087422208051e-06, "epoch": 3.38, "percentage": 84.4, "elapsed_time": "1:00:45", "remaining_time": "0:11:13"}
+{"current_steps": 800, "total_steps": 936, "loss": 0.342, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.5596568090246548e-06, "epoch": 3.42, "percentage": 85.47, "elapsed_time": "1:01:32", "remaining_time": "0:10:27"}
+{"current_steps": 810, "total_steps": 936, "loss": 0.3542, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.2025038501977486e-06, "epoch": 3.46, "percentage": 86.54, "elapsed_time": "1:02:19", "remaining_time": "0:09:41"}
+{"current_steps": 820, "total_steps": 936, "loss": 0.3385, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.8710308563769124e-06, "epoch": 3.5, "percentage": 87.61, "elapsed_time": "1:03:03", "remaining_time": "0:08:55"}
+{"current_steps": 830, "total_steps": 936, "loss": 0.3564, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.5656112113243721e-06, "epoch": 3.55, "percentage": 88.68, "elapsed_time": "1:03:46", "remaining_time": "0:08:08"}
+{"current_steps": 840, "total_steps": 936, "loss": 0.357, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.286588951321363e-06, "epoch": 3.59, "percentage": 89.74, "elapsed_time": "1:04:32", "remaining_time": "0:07:22"}
+{"current_steps": 850, "total_steps": 936, "loss": 0.3492, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.034278377632636e-06, "epoch": 3.63, "percentage": 90.81, "elapsed_time": "1:05:20", "remaining_time": "0:06:36"}
+{"current_steps": 860, "total_steps": 936, "loss": 0.3547, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.089637024655483e-07, "epoch": 3.68, "percentage": 91.88, "elapsed_time": "1:06:04", "remaining_time": "0:05:50"}
+{"current_steps": 870, "total_steps": 936, "loss": 0.3401, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.108987288226536e-07, "epoch": 3.72, "percentage": 92.95, "elapsed_time": "1:06:55", "remaining_time": "0:05:04"}
+{"current_steps": 880, "total_steps": 936, "loss": 0.3417, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.403065646083809e-07, "epoch": 3.76, "percentage": 94.02, "elapsed_time": "1:07:41", "remaining_time": "0:04:18"}
+{"current_steps": 890, "total_steps": 936, "loss": 0.3709, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.973793713118039e-07, "epoch": 3.8, "percentage": 95.09, "elapsed_time": "1:08:27", "remaining_time": "0:03:32"}
+{"current_steps": 900, "total_steps": 936, "loss": 0.3702, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.8227814754865068e-07, "epoch": 3.85, "percentage": 96.15, "elapsed_time": "1:09:13", "remaining_time": "0:02:46"}
+{"current_steps": 910, "total_steps": 936, "loss": 0.3401, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.513254770636137e-08, "epoch": 3.89, "percentage": 97.22, "elapsed_time": "1:09:58", "remaining_time": "0:01:59"}
+{"current_steps": 920, "total_steps": 936, "loss": 0.3566, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.604073589645596e-08, "epoch": 3.93, "percentage": 98.29, "elapsed_time": "1:10:44", "remaining_time": "0:01:13"}
+{"current_steps": 930, "total_steps": 936, "loss": 0.3785, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.069275378746796e-09, "epoch": 3.97, "percentage": 99.36, "elapsed_time": "1:11:29", "remaining_time": "0:00:27"}
+{"current_steps": 936, "total_steps": 936, "loss": null, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": null, "epoch": 4.0, "percentage": 100.0, "elapsed_time": "1:11:57", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,586 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 936,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.04,
+      "learning_rate": 4.9985919525527434e-05,
+      "loss": 1.0499,
+      "step": 10
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.994369396289063e-05,
+      "loss": 0.7822,
+      "step": 20
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.987337087656614e-05,
+      "loss": 0.575,
+      "step": 30
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.977502948114772e-05,
+      "loss": 0.5199,
+      "step": 40
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.964878055211597e-05,
+      "loss": 0.5358,
+      "step": 50
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.949476630105669e-05,
+      "loss": 0.508,
+      "step": 60
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.9313160215468334e-05,
+      "loss": 0.4915,
+      "step": 70
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.910416686333906e-05,
+      "loss": 0.4856,
+      "step": 80
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 4.886802166271364e-05,
+      "loss": 0.479,
+      "step": 90
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 4.8604990616509616e-05,
+      "loss": 0.4707,
+      "step": 100
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 4.8315370012881514e-05,
+      "loss": 0.4687,
+      "step": 110
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.799948609147061e-05,
+      "loss": 0.4834,
+      "step": 120
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.765769467591625e-05,
+      "loss": 0.4541,
+      "step": 130
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.7290380773042575e-05,
+      "loss": 0.4442,
+      "step": 140
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 4.68979581391722e-05,
+      "loss": 0.4528,
+      "step": 150
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.6480868814055424e-05,
+      "loss": 0.436,
+      "step": 160
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 4.6039582622939854e-05,
+      "loss": 0.4504,
+      "step": 170
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 4.557459664734141e-05,
+      "loss": 0.4348,
+      "step": 180
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 4.5086434665112864e-05,
+      "loss": 0.4283,
+      "step": 190
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 4.457564656044056e-05,
+      "loss": 0.4527,
+      "step": 200
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.404280770443398e-05,
+      "loss": 0.4525,
+      "step": 210
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.348851830700593e-05,
+      "loss": 0.4315,
+      "step": 220
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.2913402740773294e-05,
+      "loss": 0.4309,
+      "step": 230
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 4.231810883773999e-05,
+      "loss": 0.4207,
+      "step": 240
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 4.170330715955444e-05,
+      "loss": 0.4189,
+      "step": 250
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 4.1069690242163484e-05,
+      "loss": 0.4353,
+      "step": 260
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 4.0417971815713584e-05,
+      "loss": 0.4054,
+      "step": 270
+    },
+    {
+      "epoch": 1.2,
+      "learning_rate": 3.974888600057808e-05,
+      "loss": 0.4152,
+      "step": 280
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 3.906318648041617e-05,
+      "loss": 0.3879,
+      "step": 290
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 3.8361645653195026e-05,
+      "loss": 0.4164,
+      "step": 300
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 3.764505376113138e-05,
+      "loss": 0.406,
+      "step": 310
+    },
+    {
+      "epoch": 1.37,
+      "learning_rate": 3.69142180005327e-05,
+      "loss": 0.4076,
+      "step": 320
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 3.6169961612540645e-05,
+      "loss": 0.3953,
+      "step": 330
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 3.5413122955801005e-05,
+      "loss": 0.4105,
+      "step": 340
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 3.4644554562104634e-05,
+      "loss": 0.4014,
+      "step": 350
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 3.386512217606339e-05,
+      "loss": 0.4234,
+      "step": 360
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 3.307570377990245e-05,
+      "loss": 0.4012,
+      "step": 370
+    },
+    {
+      "epoch": 1.62,
+      "learning_rate": 3.227718860446782e-05,
+      "loss": 0.3979,
+      "step": 380
+    },
+    {
+      "epoch": 1.67,
+      "learning_rate": 3.147047612756302e-05,
+      "loss": 0.3877,
+      "step": 390
+    },
+    {
+      "epoch": 1.71,
+      "learning_rate": 3.065647506074306e-05,
+      "loss": 0.4024,
+      "step": 400
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 2.983610232570728e-05,
+      "loss": 0.3994,
+      "step": 410
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 2.9010282021444008e-05,
+      "loss": 0.4052,
+      "step": 420
+    },
+    {
+      "epoch": 1.84,
+      "learning_rate": 2.8179944383290274e-05,
+      "loss": 0.4038,
+      "step": 430
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 2.7346024735079486e-05,
+      "loss": 0.3999,
+      "step": 440
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 2.6509462435557152e-05,
+      "loss": 0.3848,
+      "step": 450
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 2.5671199820251534e-05,
+      "loss": 0.4126,
+      "step": 460
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 2.48321811399911e-05,
+      "loss": 0.3996,
+      "step": 470
+    },
+    {
+      "epoch": 2.05,
+      "learning_rate": 2.399335149726463e-05,
+      "loss": 0.3672,
+      "step": 480
+    },
+    {
+      "epoch": 2.09,
+      "learning_rate": 2.3155655781621793e-05,
+      "loss": 0.3728,
+      "step": 490
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 2.2320037605313808e-05,
+      "loss": 0.3901,
+      "step": 500
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 2.148743824037269e-05,
+      "loss": 0.3848,
+      "step": 510
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 2.0658795558326743e-05,
+      "loss": 0.3729,
+      "step": 520
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 1.98350429737465e-05,
+      "loss": 0.3709,
+      "step": 530
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 1.9017108392811065e-05,
+      "loss": 0.3811,
+      "step": 540
+    },
+    {
+      "epoch": 2.35,
+      "learning_rate": 1.820591316807939e-05,
+      "loss": 0.3639,
+      "step": 550
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 1.740237106064383e-05,
+      "loss": 0.3693,
+      "step": 560
+    },
+    {
+      "epoch": 2.44,
+      "learning_rate": 1.6607387210834887e-05,
+      "loss": 0.3769,
+      "step": 570
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 1.582185711863681e-05,
+      "loss": 0.3621,
+      "step": 580
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 1.5046665634962476e-05,
+      "loss": 0.3585,
+      "step": 590
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 1.4282685964923642e-05,
+      "loss": 0.3705,
+      "step": 600
+    },
+    {
+      "epoch": 2.61,
+      "learning_rate": 1.3530778684219648e-05,
+      "loss": 0.3662,
+      "step": 610
+    },
+    {
+      "epoch": 2.65,
+      "learning_rate": 1.2791790769752232e-05,
+      "loss": 0.3831,
+      "step": 620
+    },
+    {
+      "epoch": 2.69,
+      "learning_rate": 1.2066554645558578e-05,
+      "loss": 0.3874,
+      "step": 630
+    },
+    {
+      "epoch": 2.74,
+      "learning_rate": 1.1355887245137383e-05,
+      "loss": 0.3563,
+      "step": 640
+    },
+    {
+      "epoch": 2.78,
+      "learning_rate": 1.0660589091223855e-05,
+      "loss": 0.3612,
+      "step": 650
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 9.981443394050525e-06,
+      "loss": 0.3721,
+      "step": 660
+    },
+    {
+      "epoch": 2.86,
+      "learning_rate": 9.3192151691096e-06,
+      "loss": 0.3744,
+      "step": 670
+    },
+    {
+      "epoch": 2.91,
+      "learning_rate": 8.67465037541038e-06,
+      "loss": 0.3553,
+      "step": 680
+    },
+    {
+      "epoch": 2.95,
+      "learning_rate": 8.048475075202727e-06,
+      "loss": 0.3685,
+      "step": 690
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 7.441394616113062e-06,
+      "loss": 0.3731,
+      "step": 700
+    },
+    {
+      "epoch": 3.03,
+      "learning_rate": 6.854092836613948e-06,
+      "loss": 0.3529,
+      "step": 710
+    },
+    {
+      "epoch": 3.08,
+      "learning_rate": 6.28723129572247e-06,
+      "loss": 0.3542,
+      "step": 720
+    },
+    {
+      "epoch": 3.12,
+      "learning_rate": 5.741448527795137e-06,
+      "loss": 0.3587,
+      "step": 730
+    },
+    {
+      "epoch": 3.16,
+      "learning_rate": 5.217359323258459e-06,
+      "loss": 0.3583,
+      "step": 740
+    },
+    {
+      "epoch": 3.21,
+      "learning_rate": 4.715554036085673e-06,
+      "loss": 0.3589,
+      "step": 750
+    },
+    {
+      "epoch": 3.25,
+      "learning_rate": 4.236597918799709e-06,
+      "loss": 0.3424,
+      "step": 760
+    },
+    {
+      "epoch": 3.29,
+      "learning_rate": 3.7810304857511914e-06,
+      "loss": 0.3455,
+      "step": 770
+    },
+    {
+      "epoch": 3.33,
+      "learning_rate": 3.3493649053890326e-06,
+      "loss": 0.3667,
+      "step": 780
+    },
+    {
+      "epoch": 3.38,
+      "learning_rate": 2.942087422208051e-06,
+      "loss": 0.3579,
+      "step": 790
+    },
+    {
+      "epoch": 3.42,
+      "learning_rate": 2.5596568090246548e-06,
+      "loss": 0.342,
+      "step": 800
+    },
+    {
+      "epoch": 3.46,
+      "learning_rate": 2.2025038501977486e-06,
+      "loss": 0.3542,
+      "step": 810
+    },
+    {
+      "epoch": 3.5,
+      "learning_rate": 1.8710308563769124e-06,
+      "loss": 0.3385,
+      "step": 820
+    },
+    {
+      "epoch": 3.55,
+      "learning_rate": 1.5656112113243721e-06,
+      "loss": 0.3564,
+      "step": 830
+    },
+    {
+      "epoch": 3.59,
+      "learning_rate": 1.286588951321363e-06,
+      "loss": 0.357,
+      "step": 840
+    },
+    {
+      "epoch": 3.63,
+      "learning_rate": 1.034278377632636e-06,
+      "loss": 0.3492,
+      "step": 850
+    },
+    {
+      "epoch": 3.68,
+      "learning_rate": 8.089637024655483e-07,
+      "loss": 0.3547,
+      "step": 860
+    },
+    {
+      "epoch": 3.72,
+      "learning_rate": 6.108987288226536e-07,
+      "loss": 0.3401,
+      "step": 870
+    },
+    {
+      "epoch": 3.76,
+      "learning_rate": 4.403065646083809e-07,
+      "loss": 0.3417,
+      "step": 880
+    },
+    {
+      "epoch": 3.8,
+      "learning_rate": 2.973793713118039e-07,
+      "loss": 0.3709,
+      "step": 890
+    },
+    {
+      "epoch": 3.85,
+      "learning_rate": 1.8227814754865068e-07,
+      "loss": 0.3702,
+      "step": 900
+    },
+    {
+      "epoch": 3.89,
+      "learning_rate": 9.513254770636137e-08,
+      "loss": 0.3401,
+      "step": 910
+    },
+    {
+      "epoch": 3.93,
+      "learning_rate": 3.604073589645596e-08,
+      "loss": 0.3566,
+      "step": 920
+    },
+    {
+      "epoch": 3.97,
+      "learning_rate": 5.069275378746796e-09,
+      "loss": 0.3785,
+      "step": 930
+    },
+    {
+      "epoch": 4.0,
+      "step": 936,
+      "total_flos": 4.98594351337046e+17,
+      "train_loss": 0.40969020433914966,
+      "train_runtime": 4321.1174,
+      "train_samples_per_second": 6.918,
+      "train_steps_per_second": 0.217
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 936,
+  "num_train_epochs": 4,
+  "save_steps": 200,
+  "total_flos": 4.98594351337046e+17,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbbed73135e44f6ca90ec72ca7dffc3ba0a828c08a457d2a39a4dcf2d902806e
+size 4664