Set use_memory_efficient_attention to False and modify config.json of 2_Dense_8192

2_Dense/config.json

@@ -1,6 +1,6 @@
 {
  "in_features": 1024,
- "out_features": 8192,
+ "out_features": 1024,
  "bias": true,
  "activation_function": "torch.nn.modules.linear.Identity"
 }

2_Dense/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6b6a267eb9d083406ccf911efc73d9bd26ba371fba499e50c833cc9d166af04
-size 33587400
+oid sha256:caed226f6fc2317d1dde81540cbc2cc40055ab27dbf8e4377facfb281ac16fdd
+size 4198592

2_Dense/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0da3d28f4e0afdc78b9246c0fd4199843a41e8e096fe18a3d46b5831a6a72dd
-size 33588732
+oid sha256:13e6f799c3a50eef2673ebd4b363e96787520bb6fe077104c4bc5aec5e0209b2
+size 4199932

2_Dense_8192/config.json

@@ -1,6 +1,6 @@
 {
  "in_features": 1024,
- "out_features": 8192,
+ "out_features": 1024,
  "bias": true,
  "activation_function": "torch.nn.modules.linear.Identity"
 }

2_Dense_8192/config_original.json

@@ -0,0 +1,6 @@
+{
+ "in_features": 1024,
+ "out_features": 8192,
+ "bias": true,
+ "activation_function": "torch.nn.modules.linear.Identity"
+}

config.json

@@ -1,38 +1 @@
-{
- "architectures": [
-  "NewModel"
- ],
- "attention_probs_dropout_prob": 0.0,
- "auto_map": {
-  "AutoConfig": "configuration.NewConfig",
-  "AutoModel": "modeling.NewModel"
- },
- "classifier_dropout": null,
- "hidden_act": "gelu",
- "hidden_dropout_prob": 0.1,
- "hidden_size": 1024,
- "initializer_range": 0.02,
- "intermediate_size": 4096,
- "layer_norm_eps": 1e-12,
- "layer_norm_type": "layer_norm",
- "logn_attention_clip1": false,
- "logn_attention_scale": false,
- "max_position_embeddings": 8192,
- "model_type": "new",
- "num_attention_heads": 16,
- "num_hidden_layers": 24,
- "pack_qkv": true,
- "pad_token_id": 0,
- "position_embedding_type": "rope",
- "rope_scaling": {
-  "factor": 2.0,
-  "type": "ntk"
- },
- "rope_theta": 160000,
- "torch_dtype": "float32",
- "transformers_version": "4.41.2",
- "type_vocab_size": 2,
- "unpad_inputs": true,
- "use_memory_efficient_attention": true,
- "vocab_size": 30528
-}
+{"architectures": ["NewModel"], "attention_probs_dropout_prob": 0.0, "auto_map": {"AutoConfig": "configuration.NewConfig", "AutoModel": "modeling.NewModel"}, "classifier_dropout": null, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 1024, "initializer_range": 0.02, "intermediate_size": 4096, "layer_norm_eps": 1e-12, "layer_norm_type": "layer_norm", "logn_attention_clip1": false, "logn_attention_scale": false, "max_position_embeddings": 8192, "model_type": "new", "num_attention_heads": 16, "num_hidden_layers": 24, "pack_qkv": true, "pad_token_id": 0, "position_embedding_type": "rope", "rope_scaling": {"factor": 2.0, "type": "ntk"}, "rope_theta": 160000, "torch_dtype": "float32", "transformers_version": "4.41.2", "type_vocab_size": 2, "unpad_inputs": false, "use_memory_efficient_attention": false, "vocab_size": 30528}