Model save

adapter_config.json

@@ -16,10 +16,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
+    "k_proj",
     "v_proj",
-    "o_proj",
-    "k_proj"
+    "q_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:38c4cc3db797cde9c3e4214f38f663a87bbca856e02bb4d2a95bb006aca3741f
+oid sha256:b55781a0314f695d8e8230b4bb6fa7c6f34fea0b06e4d3c140c535cd51b3cbba
 size 109086672

all_results.json

@@ -1,13 +1,13 @@
 {
     "epoch": 0.67,
-    "eval_loss": 1.0666559934616089,
-    "eval_runtime": 2404.6287,
+    "eval_loss": 1.0666674375534058,
+    "eval_runtime": 2391.2283,
     "eval_samples": 23110,
-    "eval_samples_per_second": 9.611,
+    "eval_samples_per_second": 9.664,
     "eval_steps_per_second": 0.151,
-    "train_loss": 1.1599171240540112,
-    "train_runtime": 73159.9281,
+    "train_loss": 1.1599184581462074,
+    "train_runtime": 72731.1653,
     "train_samples": 207865,
-    "train_samples_per_second": 2.841,
+    "train_samples_per_second": 2.858,
     "train_steps_per_second": 0.006
 }

config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.35.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.67,
-    "eval_loss": 1.0666559934616089,
-    "eval_runtime": 2404.6287,
+    "eval_loss": 1.0666674375534058,
+    "eval_runtime": 2391.2283,
     "eval_samples": 23110,
-    "eval_samples_per_second": 9.611,
+    "eval_samples_per_second": 9.664,
     "eval_steps_per_second": 0.151
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.67,
-    "train_loss": 1.1599171240540112,
-    "train_runtime": 73159.9281,
+    "train_loss": 1.1599184581462074,
+    "train_runtime": 72731.1653,
     "train_samples": 207865,
-    "train_samples_per_second": 2.841,
+    "train_samples_per_second": 2.858,
     "train_steps_per_second": 0.006
 }

trainer_state.json

@@ -65,19 +65,19 @@
     {
       "epoch": 0.11,
       "learning_rate": 1.939986331449053e-05,
-      "loss": 1.2736,
+      "loss": 1.2735,
       "step": 45
     },
     {
       "epoch": 0.12,
       "learning_rate": 1.926084840336821e-05,
-      "loss": 1.2586,
+      "loss": 1.2587,
       "step": 50
     },
     {
       "epoch": 0.14,
       "learning_rate": 1.910797282022027e-05,
-      "loss": 1.2334,
+      "loss": 1.2333,
       "step": 55
     },
     {
@@ -251,7 +251,7 @@
     {
       "epoch": 0.49,
       "learning_rate": 1.0232116539815558e-05,
-      "loss": 1.0671,
+      "loss": 1.0672,
       "step": 200
     },
     {
@@ -269,13 +269,13 @@
     {
       "epoch": 0.53,
       "learning_rate": 9.072784204417995e-06,
-      "loss": 1.0795,
+      "loss": 1.0796,
       "step": 215
     },
     {
       "epoch": 0.54,
       "learning_rate": 8.688345254588579e-06,
-      "loss": 1.0686,
+      "loss": 1.0685,
       "step": 220
     },
     {
@@ -340,19 +340,19 @@
     },
     {
       "epoch": 0.67,
-      "eval_loss": 1.066710114479065,
-      "eval_runtime": 2407.1881,
-      "eval_samples_per_second": 9.6,
-      "eval_steps_per_second": 0.15,
+      "eval_loss": 1.0667219161987305,
+      "eval_runtime": 2389.1593,
+      "eval_samples_per_second": 9.673,
+      "eval_steps_per_second": 0.152,
       "step": 272
     },
     {
       "epoch": 0.67,
       "step": 272,
       "total_flos": 4.932776606721638e+16,
-      "train_loss": 1.1599171240540112,
-      "train_runtime": 73159.9281,
-      "train_samples_per_second": 2.841,
+      "train_loss": 1.1599184581462074,
+      "train_runtime": 72731.1653,
+      "train_samples_per_second": 2.858,
       "train_steps_per_second": 0.006
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:894ab50c9898c2dee04f953d44234befab17dc2499229944290efadb69ed352d
+oid sha256:5a7bafc436c3a41a9a2a47904c2a1fb60e8d46cd8f97932f14a89887397ae60d
 size 5624