Upload folder using huggingface_hub

README.md

@@ -0,0 +1,182 @@
+---
+base_model:
+- DewEfresh/neo_7b
+- m-a-p/neo_7b
+tags:
+- merge
+- mergekit
+- lazymergekit
+- DewEfresh/neo_7b
+- m-a-p/neo_7b
+---
+
+# Neo_7b-merge15
+
+Neo_7b-merge15 is a merge of the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
+* [DewEfresh/neo_7b](https://huggingface.co/DewEfresh/neo_7b)
+* [m-a-p/neo_7b](https://huggingface.co/m-a-p/neo_7b)
+
+## 🧩 Configuration
+
+```yaml
+# Define the slices for the model merging process
+slices:
+  - sources:
+      # First part: merge layer 0 with layer 3
+      - model: DewEfresh/neo_7b
+        layer_range: [0, 0]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  - sources:
+      # Second part: merge layer 1 with layer 3
+      - model: DewEfresh/neo_7b
+        layer_range: [1, 1]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  - sources:
+      # Third part: merge layer 2 with layer 3
+      - model: DewEfresh/neo_7b
+        layer_range: [2, 2]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  - sources:
+      # Fourth part: merge layer 4 with layer 7
+      - model: DewEfresh/neo_7b
+        layer_range: [4, 4]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  - sources:
+      # Fifth part: merge layer 5 with layer 7
+      - model: DewEfresh/neo_7b
+        layer_range: [5, 5]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  - sources:
+      # Sixth part: merge layer 6 with layer 7
+      - model: DewEfresh/neo_7b
+        layer_range: [6, 6]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  - sources:
+      # Seventh part: merge layer 8 with layer 11
+      - model: DewEfresh/neo_7b
+        layer_range: [8, 8]
+      - model: m-a-p/neo_7b
+        layer_range: [11, 11]
+  - sources:
+      # Eighth part: merge layer 9 with layer 11
+      - model: DewEfresh/neo_7b
+        layer_range: [9, 9]
+      - model: m-a-p/neo_7b
+        layer_range: [11, 11]
+  - sources:
+      # Ninth part: merge layer 10 with layer 11
+      - model: DewEfresh/neo_7b
+        layer_range: [10, 10]
+      - model: m-a-p/neo_7b
+        layer_range: [11, 11]
+  - sources:
+      # Tenth part: merge layer 12 with layer 15
+      - model: DewEfresh/neo_7b
+        layer_range: [12, 12]
+      - model: m-a-p/neo_7b
+        layer_range: [15, 15]
+  - sources:
+      # Eleventh part: merge layer 13 with layer 15
+      - model: DewEfresh/neo_7b
+        layer_range: [13, 13]
+      - model: m-a-p/neo_7b
+        layer_range: [15, 15]
+  - sources:
+      # Twelfth part: merge layer 14 with layer 15
+      - model: DewEfresh/neo_7b
+        layer_range: [14, 14]
+      - model: m-a-p/neo_7b
+        layer_range: [15, 15]
+  - sources:
+      # Thirteenth part: merge layer 16 with layer 19
+      - model: DewEfresh/neo_7b
+        layer_range: [16, 16]
+      - model: m-a-p/neo_7b
+        layer_range: [19, 19]
+  - sources:
+      # Fourteenth part: merge layer 17 with layer 19
+      - model: DewEfresh/neo_7b
+        layer_range: [17, 17]
+      - model: m-a-p/neo_7b
+        layer_range: [19, 19]
+  - sources:
+      # Fifteenth part: merge layer 18 with layer 19
+      - model: DewEfresh/neo_7b
+        layer_range: [18, 18]
+      - model: m-a-p/neo_7b
+        layer_range: [19, 19]
+  - sources:
+      # Sixteenth part: merge layer 20 with layer 23
+      - model: DewEfresh/neo_7b
+        layer_range: [20, 20]
+      - model: m-a-p/neo_7b
+        layer_range: [23, 23]
+  - sources:
+      # Seventeenth part: merge layer 21 with layer 23
+      - model: DewEfresh/neo_7b
+        layer_range: [21, 21]
+      - model: m-a-p/neo_7b
+        layer_range: [23, 23]
+  - sources:
+      # Eighteenth part: merge layer 22 with layer 23
+      - model: DewEfresh/neo_7b
+        layer_range: [22, 22]
+      - model: m-a-p/neo_7b
+        layer_range: [23, 23]
+  - sources:
+      # Nineteenth part: merge layer 24 with layer 27
+      - model: DewEfresh/neo_7b
+        layer_range: [24, 24]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+  - sources:
+      # Twentieth part: merge layer 25 with layer 27
+      - model: DewEfresh/neo_7b
+        layer_range: [25, 25]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+  - sources:
+      # Twenty-first part: merge layer 26 with layer 27
+      - model: DewEfresh/neo_7b
+        layer_range: [26, 26]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+# Specify the merging method for the slices
+merge_method: slerp
+base_model: DewEfresh/neo_7b
+parameters:
+  t: 0.3333  # Set global interpolation value to 33.33%
+dtype: bfloat16
+
+```
+
+## 💻 Usage
+
+```python
+!pip install -qU transformers accelerate
+
+from transformers import AutoTokenizer
+import transformers
+import torch
+
+model = "DewEfresh/Neo_7b-merge15"
+messages = [{"role": "user", "content": "What is a large language model?"}]
+
+tokenizer = AutoTokenizer.from_pretrained(model)
+prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+pipeline = transformers.pipeline(
+    "text-generation",
+    model=model,
+    torch_dtype=torch.float16,
+    device_map="auto",
+)
+
+outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+print(outputs[0]["generated_text"])
+```

added_tokens.json

@@ -0,0 +1,7 @@
+{
+  "<|CLS|>": 64000,
+  "<|EOD|>": 64002,
+  "<|MASK|>": 64003,
+  "<|PAD|>": 64004,
+  "<|SEP|>": 64001
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "DewEfresh/neo_7b",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 24576,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 0,
+  "num_key_value_heads": 16,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": true,
+  "vocab_size": 64256
+}

mergekit_config.yml

@@ -0,0 +1,136 @@
+
+# Define the slices for the model merging process
+slices:
+  - sources:
+      # First part: merge layer 0 with layer 3
+      - model: DewEfresh/neo_7b
+        layer_range: [0, 0]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  - sources:
+      # Second part: merge layer 1 with layer 3
+      - model: DewEfresh/neo_7b
+        layer_range: [1, 1]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  - sources:
+      # Third part: merge layer 2 with layer 3
+      - model: DewEfresh/neo_7b
+        layer_range: [2, 2]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  - sources:
+      # Fourth part: merge layer 4 with layer 7
+      - model: DewEfresh/neo_7b
+        layer_range: [4, 4]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  - sources:
+      # Fifth part: merge layer 5 with layer 7
+      - model: DewEfresh/neo_7b
+        layer_range: [5, 5]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  - sources:
+      # Sixth part: merge layer 6 with layer 7
+      - model: DewEfresh/neo_7b
+        layer_range: [6, 6]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  - sources:
+      # Seventh part: merge layer 8 with layer 11
+      - model: DewEfresh/neo_7b
+        layer_range: [8, 8]
+      - model: m-a-p/neo_7b
+        layer_range: [11, 11]
+  - sources:
+      # Eighth part: merge layer 9 with layer 11
+      - model: DewEfresh/neo_7b
+        layer_range: [9, 9]
+      - model: m-a-p/neo_7b
+        layer_range: [11, 11]
+  - sources:
+      # Ninth part: merge layer 10 with layer 11
+      - model: DewEfresh/neo_7b
+        layer_range: [10, 10]
+      - model: m-a-p/neo_7b
+        layer_range: [11, 11]
+  - sources:
+      # Tenth part: merge layer 12 with layer 15
+      - model: DewEfresh/neo_7b
+        layer_range: [12, 12]
+      - model: m-a-p/neo_7b
+        layer_range: [15, 15]
+  - sources:
+      # Eleventh part: merge layer 13 with layer 15
+      - model: DewEfresh/neo_7b
+        layer_range: [13, 13]
+      - model: m-a-p/neo_7b
+        layer_range: [15, 15]
+  - sources:
+      # Twelfth part: merge layer 14 with layer 15
+      - model: DewEfresh/neo_7b
+        layer_range: [14, 14]
+      - model: m-a-p/neo_7b
+        layer_range: [15, 15]
+  - sources:
+      # Thirteenth part: merge layer 16 with layer 19
+      - model: DewEfresh/neo_7b
+        layer_range: [16, 16]
+      - model: m-a-p/neo_7b
+        layer_range: [19, 19]
+  - sources:
+      # Fourteenth part: merge layer 17 with layer 19
+      - model: DewEfresh/neo_7b
+        layer_range: [17, 17]
+      - model: m-a-p/neo_7b
+        layer_range: [19, 19]
+  - sources:
+      # Fifteenth part: merge layer 18 with layer 19
+      - model: DewEfresh/neo_7b
+        layer_range: [18, 18]
+      - model: m-a-p/neo_7b
+        layer_range: [19, 19]
+  - sources:
+      # Sixteenth part: merge layer 20 with layer 23
+      - model: DewEfresh/neo_7b
+        layer_range: [20, 20]
+      - model: m-a-p/neo_7b
+        layer_range: [23, 23]
+  - sources:
+      # Seventeenth part: merge layer 21 with layer 23
+      - model: DewEfresh/neo_7b
+        layer_range: [21, 21]
+      - model: m-a-p/neo_7b
+        layer_range: [23, 23]
+  - sources:
+      # Eighteenth part: merge layer 22 with layer 23
+      - model: DewEfresh/neo_7b
+        layer_range: [22, 22]
+      - model: m-a-p/neo_7b
+        layer_range: [23, 23]
+  - sources:
+      # Nineteenth part: merge layer 24 with layer 27
+      - model: DewEfresh/neo_7b
+        layer_range: [24, 24]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+  - sources:
+      # Twentieth part: merge layer 25 with layer 27
+      - model: DewEfresh/neo_7b
+        layer_range: [25, 25]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+  - sources:
+      # Twenty-first part: merge layer 26 with layer 27
+      - model: DewEfresh/neo_7b
+        layer_range: [26, 26]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+# Specify the merging method for the slices
+merge_method: slerp
+base_model: DewEfresh/neo_7b
+parameters:
+  t: 0.3333  # Set global interpolation value to 33.33%
+dtype: bfloat16
+

model-00001-of-00001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41209159383a6438fa1e73146df470aae582c40bf510ec2495a3f8780477ce87
+size 789584192

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4235f15b69871ff7026193156d555c33c61671569b44c1eaf0bc53445bd2014
+size 4998668576

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cb0f120f24ce52d9f1f4b81b42b45adcf23386e2776a092c26fcfa71612c608
+size 4926336584

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44e521e63a165fd616257f81ca7f33e2bba57601fccd6fe71d4da4a989d5ab8b
+size 3019978728

model.safetensors.index.json

@@ -0,0 +1 @@
+{"metadata": {"mergekit_version": "0.0.4.4", "total_size": 789583872}, "weight_map": {"lm_head.weight": "model-00001-of-00001.safetensors", "model.embed_tokens.weight": "model-00001-of-00001.safetensors", "model.norm.weight": "model-00001-of-00001.safetensors"}}

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "additional_special_tokens": [
+    "<|CLS|>",
+    "<|SEP|>",
+    "<|EOD|>",
+    "<|MASK|>",
+    "<|PAD|>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6a2447b0e5664cabb2481587597102d82f42f0ccb7ef22e1c2d95494a8b03c5
+size 1002561

tokenizer_config.json

@@ -0,0 +1,95 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": true,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": true,
+      "special": true
+    },
+    "64000": {
+      "content": "<|CLS|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64001": {
+      "content": "<|SEP|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64002": {
+      "content": "<|EOD|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64003": {
+      "content": "<|MASK|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64004": {
+      "content": "<|PAD|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|CLS|>",
+    "<|SEP|>",
+    "<|EOD|>",
+    "<|MASK|>",
+    "<|PAD|>"
+  ],
+  "auto_map": {
+    "AutoTokenizer": [
+      "DewEfresh/neo_7b--tokenization_neo.NEOTokenizer",
+      null
+    ]
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% set system_message = 'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\\n\\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don\\'t know the answer to a question, please don\\'t share false information.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if loop.index0 == 0 and system_message is defined %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '<s>' + '[INST] ' + content + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 4096,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "NEOTokenizer",
+  "unk_token": "<unk>",
+  "use_fast": false
+}