use bin instead of safetensors with max shard of 2GB

.gitattributes

@@ -33,8 +33,26 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-model-00001-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00002-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00003-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00004-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
 tokenizer.model filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00013-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00017-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00001-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00002-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00006-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00008-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model.bin.index.json filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00010-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00016-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00018-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00020-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00003-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00007-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00011-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00021-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00014-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00015-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00019-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00004-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00005-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00009-of-00021.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00012-of-00021.bin filter=lfs diff=lfs merge=lfs -text

config.json

@@ -27,8 +27,8 @@
   },
   "rope_theta": 1000000,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.36.2",
+  "torch_dtype": "float16",
+  "transformers_version": "4.33.2",
   "use_cache": true,
   "vocab_size": 92544
 }

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 1,
   "eos_token_id": 2,
   "pad_token_id": 2,
-  "transformers_version": "4.36.2"
+  "transformers_version": "4.33.2"
 }

model.safetensors.index.json

@@ -1,346 +0,0 @@
-{
-  "metadata": {
-    "total_size": 39722299392
-  },
-  "weight_map": {
-    "model.layers.0.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.10.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.10.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.10.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.10.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.10.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.10.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.10.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.11.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.11.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.11.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.19.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.19.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.19.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.19.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.19.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.19.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.19.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.2.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.20.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.20.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.20.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.20.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.20.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.20.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.20.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.21.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.21.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.21.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.21.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.21.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.21.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.21.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.22.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.22.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.22.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.22.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.22.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.22.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.22.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.23.attention.wo.weight": "model-00002-of-00004.safetensors",
-    "model.layers.23.attention.wqkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.23.attention_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.23.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.23.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
-    "model.layers.23.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
-    "model.layers.23.ffn_norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.24.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.28.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.28.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.28.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.28.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.28.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.28.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.28.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.29.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.29.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.29.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.29.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.29.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.29.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.29.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.3.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.30.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.30.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.30.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.30.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.30.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.30.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.30.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.31.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.31.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.31.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.31.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.31.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.31.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.31.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.32.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.32.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.32.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.32.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.32.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.32.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.32.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.33.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.33.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.33.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.33.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.33.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.33.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.33.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.34.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.34.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.34.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.34.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.34.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.34.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.34.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.35.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.35.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.35.attention_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.35.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.35.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
-    "model.layers.35.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.35.ffn_norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.36.attention.wo.weight": "model-00003-of-00004.safetensors",
-    "model.layers.36.attention.wqkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.36.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.36.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.36.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.36.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
-    "model.layers.36.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.37.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.37.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.37.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.37.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.37.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.37.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.37.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.38.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.38.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.38.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.38.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.38.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.38.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.38.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.39.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.39.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.39.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.39.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.39.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.39.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.39.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.4.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.40.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.40.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.40.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.40.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.40.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.40.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.40.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.41.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.41.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.41.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.41.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.41.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.41.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.41.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.42.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.42.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.42.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.42.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.42.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.42.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.42.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.43.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.43.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.43.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.43.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.43.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.43.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.43.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.44.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.44.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.44.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.44.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.44.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.44.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.44.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.45.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.45.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.45.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.45.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.45.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.45.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.45.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.46.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.46.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.46.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.46.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.46.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.46.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.46.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.47.attention.wo.weight": "model-00004-of-00004.safetensors",
-    "model.layers.47.attention.wqkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.47.attention_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.47.feed_forward.w1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.47.feed_forward.w2.weight": "model-00004-of-00004.safetensors",
-    "model.layers.47.feed_forward.w3.weight": "model-00004-of-00004.safetensors",
-    "model.layers.47.ffn_norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.5.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.9.attention.wo.weight": "model-00001-of-00004.safetensors",
-    "model.layers.9.attention.wqkv.weight": "model-00001-of-00004.safetensors",
-    "model.layers.9.attention_norm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.9.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
-    "model.layers.9.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
-    "model.layers.9.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
-    "model.layers.9.ffn_norm.weight": "model-00001-of-00004.safetensors",
-    "model.norm.weight": "model-00004-of-00004.safetensors",
-    "model.tok_embeddings.weight": "model-00001-of-00004.safetensors",
-    "output.weight": "model-00004-of-00004.safetensors"
-  }
-}

modeling_internlm2.py

@@ -133,7 +133,7 @@ class InternLM2RotaryEmbedding(nn.Module):
     def forward(self, x, seq_len=None):
         # x: [bs, num_attention_heads, seq_len, head_size]
         if seq_len > self.max_seq_len_cached:
-            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=torch.float32)
 
         return (
             self.cos_cached[:seq_len].to(dtype=x.dtype),
@@ -196,20 +196,10 @@ def rotate_half(x):
 
 
 def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
-    # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
-    cos = cos.squeeze(1).squeeze(0)  # [seq_len, dim]
-    sin = sin.squeeze(1).squeeze(0)  # [seq_len, dim]
-    cos = cos.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)
-    sin = sin.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)
-    if q.size(2) == 1:
-        q_embed = (q * cos[:, :, -1, :]) + (rotate_half(q) * sin[:, :, -1, :])
-    else:
-        q_embed = (q * cos) + (rotate_half(q) * sin)
-
-    if k.size(2) == 1:
-        k_embed = (k * cos[:, :, -1, :]) + (rotate_half(k) * sin[:, :, -1, :])
-    else:
-        k_embed = (k * cos) + (rotate_half(k) * sin)
+    cos = cos[position_ids].unsqueeze(1)
+    sin = sin[position_ids].unsqueeze(1)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
 
     return q_embed, k_embed
 
@@ -289,8 +279,15 @@ class InternLM2Attention(nn.Module):
                     base=self.config.rope_theta,
                     scaling_factor=scaling_factor
                 )
+            elif scaling_type == "linear":
+                self.rotary_emb = InternLM2LinearScalingRotaryEmbedding(
+                    self.head_dim,
+                    max_position_embeddings=self.max_position_embeddings,
+                    base=self.config.rope_theta,
+                    scaling_factor=scaling_factor
+                )
             else:
-                raise ValueError("Currently we only support rotary embedding's type being 'dynamic'.")
+                raise ValueError("Currently we only support rotary embedding's type being 'dynamic' or 'linear'.")
         return self.rotary_emb
 
     def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
@@ -1032,7 +1029,6 @@ class InternLM2ForCausalLM(InternLM2PreTrainedModel):
         for record in history:
             prompt += f"""[UNUSED_TOKEN_146]user\n{record[0]}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n{record[1]}[UNUSED_TOKEN_145]\n"""
         prompt += f"""[UNUSED_TOKEN_146]user\n{query}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n"""
-        print(prompt)
         return tokenizer([prompt], return_tensors="pt")
 
     @torch.no_grad()
@@ -1268,5 +1264,5 @@ class InternLM2ForSequenceClassification(InternLM2PreTrainedModel):
             logits=pooled_logits,
             past_key_values=transformer_outputs.past_key_values,
             hidden_states=transformer_outputs.hidden_states,
-            attentions=transformer_outputs,
+            attentions=transformer_outputs.attentions,
         )

model-00001-of-00004.safetensors → pytorch_model-00001-of-00021.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b59946828e4d922485feaed00b1a933c1676a3167fb1ce6a2405493c54b77ada
-size 9895166840
+oid sha256:722819f7b217898ee32c8c65421006303081106b7af8048378774423de824f26
+size 1917348897

model-00002-of-00004.safetensors → pytorch_model-00002-of-00021.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce679ab57cb0bd5aedc3ae7986f3d45d16c2927421ab8c1b1039e79f64dddc39
-size 9965995968
+oid sha256:8d4ee6af265497cbd169c7163f0e9ac97c56e389c7a89f825713894ff3120fee
+size 1937823489

model-00003-of-00004.safetensors → pytorch_model-00003-of-00021.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d0e57dd25dca35feb3ced58513f2893564f2b012cf9ef6d9146f8ccd5f0893a
-size 9940805472
+oid sha256:fc0d11d9813cea7bba73cfbcc0e212fff344e7282adb370fdfb20adfda9f164a
+size 1963014163

model-00004-of-00004.safetensors → pytorch_model-00004-of-00021.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a592a88592e01386864f0f3feaaa5ca1a2f181e8615580ec16c17b07a3e9fee
-size 9920369776
+oid sha256:aa74f5be136e888c2a7c43032bd97dca226d8644ec06ec6cef986a3f5be375be
+size 1937823489

pytorch_model-00005-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3b295c170c33adf209f507609fbec201a3534edd0ca2530d6524cc3dc63b989
+size 1963014163

pytorch_model-00006-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb3d4c2c9b32ac6e07eb649294497621e5511c244ba780b8476e3fa22b71ea6a
+size 1937823489

pytorch_model-00007-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02c8539dc2ee3003f392001d61cd67a924fd6a4abdb6fddd70b366b703a8141c
+size 1963014227

pytorch_model-00008-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7fe91599763599e612ff79bca50d1be9dd7cb9030d1e6dbacdb51dc45e22afa
+size 1937823489

pytorch_model-00009-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df8dae357eb00345ef42506045de1723869dabe8cf89fa192e06137cf3cd395d
+size 1963014227

pytorch_model-00010-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8add13dc25f54db6df0a8b5e8135a63f670ac7bcbcb5706b541daa96bdfdd6c
+size 1937823489

pytorch_model-00011-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32af852ccb9936da7d332e22beda929de7fcb1f96c39262b836bca387fe0161d
+size 1963014227

pytorch_model-00012-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff97793ae1fe4d27af4f9209fbf2e532cd4fb1ccc39e399aa2f58bddb765f186
+size 1937823489

pytorch_model-00013-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d0c08ce8a307fa5fa3caebff931e1bf540e861e2fa38b1bfa4bcf91f7277e10
+size 1963014227

pytorch_model-00014-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15ebf9dcba9bd6b42c165736792b2ebdc33840bffc65c4314a0c704e34917d9e
+size 1937823489

pytorch_model-00015-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e353fef3f3a242e8e5af05afca8e5113582d7ea3225fbe1ce111e32816e48a2c
+size 1963014227

pytorch_model-00016-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ade1ad7844c0523e10e50cf541e20becd8d0a011305a0b5d49becc94cdf6692a
+size 1937823489

pytorch_model-00017-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d969e5162667a98b6ed514bbf8a92f611994673ff47155a526b0092863ebc6e9
+size 1963014227

pytorch_model-00018-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a8721d0d742ce15ae0f3081249b7cd7bb907701ee543101979d297cd9d6120f
+size 1937823489

pytorch_model-00019-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15b81635b5847683c9c943f8d819c29934a01e5a9e00a99464d8c5c3564d2938
+size 1963014227

pytorch_model-00020-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d0dc96e4be3de25d042446799c96aa3bfb5204132cfe1d48db6f456aaf79bb1
+size 1560347805

pytorch_model-00021-of-00021.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1783c28825d2881dd23254e19422472464fdcd28416c9d9ae79b4fb46911ca1b
+size 1137181610

pytorch_model.bin.index.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e110c305db80b79307dbccd08c7e923304c1deff95dc6460ea62b7fa74716f46
+size 27171

tokenizer_config.json

@@ -1,30 +1,4 @@
 {
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
   "auto_map": {
     "AutoTokenizer": [
       "tokenization_internlm.InternLMTokenizer",