Upload 9 files

MODEL_LICENSE.md

@@ -0,0 +1,22 @@
+Microsoft.
+Copyright (c) Microsoft Corporation.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,93 @@
----
-license: mit
----
+---
+frameworks:
+- Pytorch
+license: other
+tasks:
+- text-embedding
+---
+
+## CodeFuse-CGE-Small
+<p align="center">
+    <img src="https://modelscope.cn/api/v1/models/codefuse-ai/CodeFuse-QWen-14B/repo?Revision=master&FilePath=LOGO.jpg&View=true" width="800"/>
+<p>
+
+## Model Description
+CodeFuse-CGE-Small is the Small version of the CodeFuse-CGE family which is fine-tuned based on Phi-3.5-mini-instruct. CodeFuse-CGE-Small is distinguish on text2code task for it's powerful ability of capturing the semantic relationship between code and text.  
+  
+This model has the following notable features:  
+● Instruction-tuning is enabled for both query and code snippet sides.  
+● The model obtains sentence-level and code-level representations through a layer of cross-attention computation module.  
+● The model has a smaller dimensional size without significant degradation in performance.
+
+Model Configuration  
+Model Size: 3.8B  
+Embedding Dimension: 1024  
+Hidden Layers: 32  
+Max Input Tokens: 1024  
+
+
+Requirements  
+```
+flash_attn==2.4.2
+torch==2.1.0
+accelerate==0.28.0
+transformers==4.43.0
+```
+
+
+## How to Use
+### transformers
+```
+from transformers import AutoTokenizer, AutoModel
+
+model_name_or_path = "CodeFuse-CGE-Small"
+model = AutoModel.from_pretrained(model_name_or_path, trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True, truncation_side='right', padding_side='right')
+model = model.to(torch.bfloat16)
+
+if torch.cuda.is_available():
+    device = 'cuda'
+else:
+    device = 'cpu'
+model.to(device)
+
+prefix_dict =  {'python':{'query':'Retrieve the Python code that solves the following query:', 'passage':'Python code:'},
+                'java':{'query':'Retrieve the Java code that solves the following query:', 'passage':'Java code:'},
+                'go':{'query':'Retrieve the Go code that solves the following query:', 'passage':'Go code:'},
+                'c++':{'query':'Retrieve the C++ code that solves the following query:', 'passage':'C++ code:'},
+                'javascript':{'query':'Retrieve the Javascript code that solves the following query:', 'passage':'Javascript code:'},
+                'php':{'query':'Retrieve the PHP code that solves the following query:', 'passage':'PHP code:'},
+                'ruby':{'query':'Retrieve the Ruby code that solves the following query:', 'passage':'Ruby code:'},
+                'default':{'query':'Retrieve the code that solves the following query:', 'passage':'Code:'}
+                }
+
+text = ["Writes a Boolean to the stream.",
+        "def writeBoolean(self, n): t = TYPE_BOOL_TRUE if n is False: t = TYPE_BOOL_FALSE self.stream.write(t)"]
+text[0] += prefix_dict['python']['query']
+text[0] += prefix_dict['python']['passage']
+embed = model.encode(tokenizer, text)
+score = embed[0] @ embed[1].T
+print("score", score)
+
+```
+
+## Benchmark the Performance
+We use MRR metric to evaluate the ability on text2code retrieval tasks: AdvTest, CosQA, CSN
+
+
+![result](./result.png)
+
+## Acknowledgement
+Thanks to the authors of open-sourced datasets, including CSN, Adv, CoSQA.
+
+## License
+Since CodeFuse-CGE-Small is fine-tuned based on Phi3 model, our usage license follows the same terms as that of Phi3 model.
+
+## 加入我们
+我们是平台技术事业群AI Native团队，负责蚂蚁蚂蚁集团平台工程的智能化，团队成立3年多以来，支持了蚂蚁集团云计算基础设施智能化运维的升级改造。团队的Mission是，通过世界级的技术创新和影响，构建有广泛用户的算法服务和平台，支撑内外部产品和业务落地。团队秉承创新基因，在支撑业务落地的同时，推动技术影响。3年以来在ICLR、NeurIPS、KDD、ACL等顶会发表论文20余篇，创新业务结果获得两次蚂蚁技术最高奖T-Star，1次蚂蚁集团最高奖SuperMA。开源项目CodeFuse获得4K点赞(2024年2月)，Huggingface和modelscope上模型累积下载量超过150万次。
+
+我们正在寻找行业中的佼佼者加入我们的团队！如果您希望在一个充满活力、创新和卓越文化的环境中发展您的职业生涯，欢迎您查看我们的社招&校招机会，加入我们，一起创造下一个行业里程碑。
+
+校招：https://hrrecommend.antgroup.com/guide.html?code=8uoP5mlus5DqQYbE_EnqcE2FD5JZH21MwvMUIb9mb6X3osXPuBraG54SyM8GLn_7
+
+社招：https://talent.antgroup.com/off-campus-position?positionId=1933830

config.json

@@ -0,0 +1,149 @@
+{
+  "_name_or_path": "CodeFuse_CGE_Small",
+  "architectures": [
+    "CodeFuse_CGE_Small"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_codefuse_cge_small.CodeFuseCGESmallConfig",
+    "AutoModel": "modeling_codefuse_cge_small.CodeFuse_CGE_Small"
+  },
+  "bos_token_id": 1,
+  "compress_dim": 1024,
+  "embd_pdrop": 0.0,
+  "embedding_dim": 4096,
+  "embedding_method": "pma",
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "inf_seq_length": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "keep_max_layer": 24,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": 32000,
+  "padding_side": "right",
+  "pma_ln": true,
+  "pma_norm": false,
+  "pma_norm_mode": "post_normal",
+  "pma_num_heads": 32,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "long_factor": [
+      1.0800000429153442,
+      1.1100000143051147,
+      1.1399999856948853,
+      1.340000033378601,
+      1.5899999141693115,
+      1.600000023841858,
+      1.6200000047683716,
+      2.620000123977661,
+      3.2300000190734863,
+      3.2300000190734863,
+      4.789999961853027,
+      7.400000095367432,
+      7.700000286102295,
+      9.09000015258789,
+      12.199999809265137,
+      17.670000076293945,
+      24.46000099182129,
+      28.57000160217285,
+      30.420001983642578,
+      30.840002059936523,
+      32.590003967285156,
+      32.93000411987305,
+      42.320003509521484,
+      44.96000289916992,
+      50.340003967285156,
+      50.45000457763672,
+      57.55000305175781,
+      57.93000411987305,
+      58.21000289916992,
+      60.1400032043457,
+      62.61000442504883,
+      62.62000274658203,
+      62.71000289916992,
+      63.1400032043457,
+      63.1400032043457,
+      63.77000427246094,
+      63.93000411987305,
+      63.96000289916992,
+      63.970001220703125,
+      64.02999877929688,
+      64.06999969482422,
+      64.08000183105469,
+      64.12000274658203,
+      64.41000366210938,
+      64.4800033569336,
+      64.51000213623047,
+      64.52999877929688,
+      64.83999633789062
+    ],
+    "short_factor": [
+      1.0,
+      1.0199999809265137,
+      1.0299999713897705,
+      1.0299999713897705,
+      1.0499999523162842,
+      1.0499999523162842,
+      1.0499999523162842,
+      1.0499999523162842,
+      1.0499999523162842,
+      1.0699999332427979,
+      1.0999999046325684,
+      1.1099998950958252,
+      1.1599998474121094,
+      1.1599998474121094,
+      1.1699998378753662,
+      1.2899998426437378,
+      1.339999794960022,
+      1.679999828338623,
+      1.7899998426437378,
+      1.8199998140335083,
+      1.8499997854232788,
+      1.8799997568130493,
+      1.9099997282028198,
+      1.9399996995925903,
+      1.9899996519088745,
+      2.0199997425079346,
+      2.0199997425079346,
+      2.0199997425079346,
+      2.0199997425079346,
+      2.0199997425079346,
+      2.0199997425079346,
+      2.0299997329711914,
+      2.0299997329711914,
+      2.0299997329711914,
+      2.0299997329711914,
+      2.0299997329711914,
+      2.0299997329711914,
+      2.0299997329711914,
+      2.0299997329711914,
+      2.0299997329711914,
+      2.0799996852874756,
+      2.0899996757507324,
+      2.189999580383301,
+      2.2199995517730713,
+      2.5899994373321533,
+      2.729999542236328,
+      2.749999523162842,
+      2.8399994373321533
+    ],
+    "type": "longrope"
+  },
+  "rope_theta": 10000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 32064
+}

configuration_codefuse_cge_small.py

@@ -0,0 +1,142 @@
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+
+class CodeFuseCGESmallConfig(PretrainedConfig):
+    model_type = "phi3"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=32064,
+        hidden_size=3072,
+        intermediate_size=8192,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        resid_pdrop=0.0,
+        embd_pdrop=0.0,
+        attention_dropout=0.0,
+        hidden_act="silu",
+        max_position_embeddings=4096,
+        original_max_position_embeddings=4096,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        bos_token_id=1,
+        eos_token_id=32000,
+        pad_token_id=32000,
+        sliding_window=None,
+        embedding_method="pma",
+        inf_seq_length=1024,
+        padding_side="right",
+        compress_dim=1024,
+        keep_max_layer=32,
+        pma_num_heads=32,
+        pma_ln=True,
+        pma_norm=False,
+        pma_norm_mode="post_normal",
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.resid_pdrop = resid_pdrop
+        self.embd_pdrop = embd_pdrop
+        self.attention_dropout = attention_dropout
+        self.hidden_act = hidden_act
+        self.max_position_embeddings = max_position_embeddings
+        self.original_max_position_embeddings = original_max_position_embeddings
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self._rope_scaling_adjustment()
+        self._rope_scaling_validation()
+        self.sliding_window = sliding_window
+
+        self.embedding_method = embedding_method
+        self.inf_seq_length = inf_seq_length
+        self.padding_side = padding_side
+        self.compress_dim = compress_dim
+        self.keep_max_layer = keep_max_layer
+        self.pma_num_heads = pma_num_heads
+        self.pma_ln = pma_ln
+        self.pma_norm = pma_norm
+        self.pma_norm_mode = pma_norm_mode
+
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+    def _rope_scaling_adjustment(self):
+        """
+        Adjust the `type` of the `rope_scaling` configuration for backward compatibility.
+        """
+        if self.rope_scaling is None:
+            return
+
+        rope_scaling_type = self.rope_scaling.get("type", None)
+
+        # For backward compatibility if previous version used "su" or "yarn"
+        if rope_scaling_type is not None and rope_scaling_type in ["su", "yarn"]:
+            self.rope_scaling["type"] = "longrope"
+
+    def _rope_scaling_validation(self):
+        """
+        Validate the `rope_scaling` configuration.
+        """
+        if self.rope_scaling is None:
+            return
+
+        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3:
+            raise ValueError(
+                "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, "
+                f"got {self.rope_scaling}"
+            )
+        rope_scaling_type = self.rope_scaling.get("type", None)
+        rope_scaling_short_factor = self.rope_scaling.get("short_factor", None)
+        rope_scaling_long_factor = self.rope_scaling.get("long_factor", None)
+        if rope_scaling_type is None or rope_scaling_type not in ["longrope"]:
+            raise ValueError(f"`rope_scaling`'s type field must be one of ['longrope'], got {rope_scaling_type}")
+        if not (
+            isinstance(rope_scaling_short_factor, list)
+            and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor)
+        ):
+            raise ValueError(
+                f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}"
+            )
+        if not len(rope_scaling_short_factor) == self.hidden_size // self.num_attention_heads // 2:
+            raise ValueError(
+                f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}"
+            )
+        if not (
+            isinstance(rope_scaling_long_factor, list)
+            and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor)
+        ):
+            raise ValueError(
+                f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}"
+            )
+        if not len(rope_scaling_long_factor) == self.hidden_size // self.num_attention_heads // 2:
+            raise ValueError(
+                f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}"
+            )

model.safetensors.index.json

@@ -0,0 +1,215 @@
+{
+  "metadata": {
+    "total_size": 7658954752
+  },
+  "weight_map": {
+    "mha_pma.S": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.fc_k.bias": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.fc_k.weight": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.fc_o.bias": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.fc_o.weight": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.fc_q.bias": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.fc_q.weight": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.fc_v.bias": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.fc_v.weight": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.ln0.bias": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.ln0.weight": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.ln1.bias": "model-00002-of-00002.safetensors",
+    "mha_pma.mab.ln1.weight": "model-00002-of-00002.safetensors",
+    "plm_model.lm_head.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "plm_model.model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "plm_model.model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

modeling_codefuse_cge_small.py

@@ -0,0 +1,243 @@
+import inspect
+import math
+import os
+import warnings
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+from transformers import PretrainedConfig
+
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask, _prepare_4d_causal_attention_mask_for_sdpa, _prepare_4d_attention_mask, _prepare_4d_attention_mask_for_sdpa
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    is_flash_attn_2_available,
+    is_flash_attn_greater_or_equal_2_10,
+    logging,
+    replace_return_docstrings,
+)
+import numpy as np
+from transformers import Phi3ForCausalLM
+import inspect
+import math
+import os
+import warnings
+from typing import List, Optional, Tuple, Union
+from tqdm import tqdm, trange
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask, _prepare_4d_causal_attention_mask_for_sdpa, _prepare_4d_attention_mask, _prepare_4d_attention_mask_for_sdpa
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    is_flash_attn_2_available,
+    is_flash_attn_greater_or_equal_2_10,
+    logging,
+    replace_return_docstrings,
+)
+import numpy as np
+import torch
+import os
+import argparse
+import json
+from tqdm import tqdm
+from typing import cast, List, Union, Tuple
+from transformers import AutoTokenizer, AutoModel  # pylint: disable=C0413
+from peft import LoraConfig, get_peft_model, TaskType
+import time
+import torch.nn.functional as F
+import sys
+import time
+import torch 
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+from tqdm import tqdm, trange
+from collections import defaultdict
+from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM, AutoConfig
+import torch.distributed as dist
+from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+import sys
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+import re
+
+
+class MAB_POST(nn.Module):
+    def __init__(self, dim_Q, dim_K, dim_V, num_heads, ln=False):
+        super(MAB_POST, self).__init__()
+        self.dim_V = dim_V
+        self.num_heads = num_heads
+        self.fc_q = nn.Linear(dim_Q, dim_V)
+        self.fc_k = nn.Linear(dim_K, dim_V)
+        self.fc_v = nn.Linear(dim_K, dim_V)
+        if ln:
+            self.ln0 = nn.LayerNorm(dim_V)
+            self.ln1 = nn.LayerNorm(dim_V)
+        self.fc_o = nn.Linear(dim_V, dim_V)
+        nn.init.xavier_uniform_(self.fc_q.weight)
+        nn.init.xavier_uniform_(self.fc_k.weight)
+        nn.init.xavier_uniform_(self.fc_v.weight)
+        nn.init.xavier_uniform_(self.fc_o.weight)
+
+    def forward(self, Q, K, pad_mask=None):
+
+        Q_ = self.fc_q(Q)
+        K_, V_ = self.fc_k(K), self.fc_v(K)
+
+        dim_split = self.dim_V // self.num_heads
+        Q_ = torch.cat(Q_.split(dim_split, 2), 0)
+        K_ = torch.cat(K_.split(dim_split, 2), 0)
+        V_ = torch.cat(V_.split(dim_split, 2), 0)
+
+        pad_mask = pad_mask.unsqueeze(1).repeat(self.num_heads, Q.size(1), 1)
+        score = Q_.bmm(K_.transpose(1,2))/math.sqrt(self.dim_V)
+        score = score.masked_fill(pad_mask == 0, -1e12)
+        A = torch.softmax(score, 2)
+        A = A * pad_mask
+        O = torch.cat(A.bmm(V_).split(Q.size(0), 0), 2) 
+        O = Q + O
+        O = O if getattr(self, 'ln0', None) is None else self.ln0(O)
+        O = O + F.relu(self.fc_o(O))
+        O = O if getattr(self, 'ln1', None) is None else self.ln1(O)
+        return O
+
+
+class PMA(nn.Module):
+    def __init__(self, dim, compress_dim, num_heads, num_seeds, ln=False, pma_mode=None):
+        super(PMA, self).__init__()
+        self.S = nn.Parameter(torch.Tensor(1, num_seeds, compress_dim))
+        nn.init.xavier_uniform_(self.S)
+        if pma_mode == 'post_normal':
+            self.mab = MAB_POST(compress_dim, dim, compress_dim, num_heads, ln=ln)
+        elif pma_mode == 'pre_normal':
+            self.mab = MAB_PRE_NORMAL(compress_dim, dim, compress_dim, num_heads, ln=ln)
+        elif pma_mode == 'pre_gptj':
+            self.mab = MAB_PRE_GPTJ(compress_dim, dim, compress_dim, num_heads, ln=ln)
+        else:
+            raise ValueError(f"Error, the pma_mode {pma_mode} is not implemented !")
+
+    def forward(self, X, pad_mask):
+        if self.S.dtype != torch.bfloat16:
+            X = X.float()
+        return self.mab(self.S.repeat(X.size(0), 1, 1), X, pad_mask)
+
+
+class CodeFuse_CGE_Small(PreTrainedModel):
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.plm_model = Phi3ForCausalLM(config)
+        self.embedding_method = config.embedding_method
+        self.inf_seq_length = config.inf_seq_length
+        self.padding_side = config.padding_side
+
+        self.keep_max_layer = config.keep_max_layer
+        self.emb_dim = self.plm_model.model.embed_tokens.weight.size(1)
+        self.num_heads = config.pma_num_heads
+        self.ln = config.pma_ln
+        self.norm = config.pma_norm
+        self.compress_dim = config.compress_dim
+        self.pma_mode = config.pma_norm_mode
+        self.mha_pma = PMA(self.emb_dim, self.compress_dim, self.num_heads, 1, ln=self.ln, pma_mode=self.pma_mode)
+
+    def last_embedding(self, A, index):
+        bs, seq, emb = A.size()
+        res = A[torch.arange(bs), index, :]
+        return res
+
+    def mean_embedding(self, A, mask):
+        bs, seq, emb = A.size()
+        res = (A * (mask.unsqueeze(-1))).sum(1) / (mask.sum(1).unsqueeze(-1))
+        return res
+
+    def weighted_embedding(self, A, mask):
+        weights = (torch.arange(start=1, end=A.size(1) + 1).unsqueeze(0).unsqueeze(-1).expand(A.size()).float()).to(A.device)
+        input_mask_expanded = (mask.squeeze(1).unsqueeze(-1).expand(A.size()).float()).to(A.device)
+        sum_embedding = torch.sum(A * input_mask_expanded * weights, dim=1)
+        sum_mask = torch.sum(input_mask_expanded * weights, dim=1)
+        weighted_embedding = sum_embedding / sum_mask
+        return weighted_embedding
+
+    def pma_embedding(self, A, mask):
+        res = self.mha_pma(A, mask).squeeze(1)
+        return res
+
+    def get_sentence_embedding(self, embedding_method, **inputs):
+        outputs = self.plm_model(inputs['input_ids'], inputs['attention_mask'], output_hidden_states=True)
+        if embedding_method == 'last':
+            embedding = outputs.hidden_states[self.keep_max_layer]
+            index = inputs['attention_mask'].sum(-1).long() - 1
+            res_embedding = self.last_embedding(embedding, index)
+        elif embedding_method == 'mean':
+            embedding = outputs.hidden_states[self.keep_max_layer]
+            res_embedding = self.mean_embedding(embedding, inputs['attention_mask'])
+        elif embedding_method == 'weighted':
+            embedding = outputs.hidden_states[self.keep_max_layer]
+            res_embedding = self.weighted_embedding(embedding, inputs['attention_mask'])
+        elif embedding_method == 'pma':
+            embedding = outputs.hidden_states[self.keep_max_layer]
+            attention_mask = inputs['attention_mask']
+            res_embedding = self.pma_embedding(embedding, attention_mask)
+        else:
+            logger.debug('Error, no {} way to obtain embbedings'.format(embedding_method))
+
+        if not self.norm:
+            res_embedding = torch.nn.functional.normalize(res_embedding, p=2.0, dim=-1, eps=1e-12, out=None)
+        return res_embedding
+
+
+    def encode(self, tokenizer, sentences, batch_size=32, convert_to_numpy=True,
+            convert_to_tensor=False, show_progress_bar=True, max_seq_length=None, **kwargs):
+
+        if max_seq_length is None:
+            max_seq_length = self.inf_seq_length
+
+        input_is_string = False        
+        if isinstance(sentences, str) or not hasattr(sentences, "__len__"):
+            sentences = [sentences]
+            input_is_string = True
+
+        all_embeddings = []
+        length_sorted_idx = np.argsort([-len(s) for s in sentences])
+        sentences_sorted = [sentences[idx] for idx in length_sorted_idx]
+        with torch.no_grad():
+            for start_index in trange(0, len(sentences), batch_size, desc="Batches", disable=not show_progress_bar):
+                sentences_batch = sentences_sorted[start_index: start_index + batch_size]
+                with torch.no_grad():
+                    inputs = tokenizer(sentences_batch, padding=True, truncation=True, max_length=max_seq_length, add_special_tokens=False, return_tensors='pt').to(self.plm_model.device)
+                    embeddings = self.get_sentence_embedding(self.embedding_method, **inputs)
+                embeddings = embeddings.detach()
+                if convert_to_numpy:
+                    if embeddings.dtype == torch.bfloat16:
+                        embeddings = embeddings.cpu().to(torch.float32)
+                    else:
+                        embeddings = embeddings.cpu()
+                all_embeddings.extend(embeddings)
+        all_embeddings = [all_embeddings[idx] for idx in np.argsort(length_sorted_idx)]
+        if convert_to_tensor:
+            all_embeddings = torch.stack(all_embeddings)
+        elif convert_to_numpy:
+            all_embeddings = np.asarray([emb.numpy() for emb in all_embeddings])
+
+        if input_is_string:
+            all_embeddings = all_embeddings[0]
+        return all_embeddings

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,130 @@
+{
+    "add_bos_token": false,
+    "add_eos_token": false,
+    "added_tokens_decoder": {
+        "0": {
+            "content": "<unk>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "1": {
+            "content": "<s>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "2": {
+            "content": "</s>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": false
+        },
+        "32000": {
+            "content": "<|endoftext|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "32001": {
+            "content": "<|assistant|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": true
+        },
+        "32002": {
+            "content": "<|placeholder1|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": true
+        },
+        "32003": {
+            "content": "<|placeholder2|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": true
+        },
+        "32004": {
+            "content": "<|placeholder3|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": true
+        },
+        "32005": {
+            "content": "<|placeholder4|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": true
+        },
+        "32006": {
+            "content": "<|system|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": true
+        },
+        "32007": {
+            "content": "<|end|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": true
+        },
+        "32008": {
+            "content": "<|placeholder5|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": true
+        },
+        "32009": {
+            "content": "<|placeholder6|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": true
+        },
+        "32010": {
+            "content": "<|user|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": true,
+            "single_word": false,
+            "special": true
+        }
+    },
+    "bos_token": "<s>",
+    "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+    "clean_up_tokenization_spaces": false,
+    "eos_token": "<|endoftext|>",
+    "legacy": false,
+    "model_max_length": 131072,
+    "pad_token": "<|endoftext|>",
+    "padding_side": "left",
+    "sp_model_kwargs": {},
+    "tokenizer_class": "LlamaTokenizer",
+    "unk_token": "<unk>",
+    "use_default_system_prompt": false
+}