Upload folder using huggingface_hub

README.md

@@ -1,8 +1,8 @@
 ---
 base_model:
-- kromcomp/L3.1-Fatboy-10B
-- kromcomp/L3.1-Npos-10B
-- kromcomp/L3.1-Adlib-10B
+- kromcomp/L3.1-Rays-10B
+- kromcomp/L3.1-Bloat-10B
+- kromcomp/L3.1-Extend-10B
 library_name: transformers
 tags:
 - mergekit
@@ -16,20 +16,21 @@ This is a merge of pre-trained language models created using [mergekit](https://
 ## Merge Details
 ### Merge Method
 
-This model was merged using the della merge method using [kromcomp/L3.1-Fatboy-10B](https://huggingface.co/kromcomp/L3.1-Fatboy-10B) as a base.
+This model was merged using the [DELLA](https://arxiv.org/abs/2406.11617) merge method using [kromcomp/L3.1-Extend-10B](https://huggingface.co/kromcomp/L3.1-Extend-10B) as a base.
 
 ### Models Merged
 
 The following models were included in the merge:
-* [kromcomp/L3.1-Npos-10B](https://huggingface.co/kromcomp/L3.1-Npos-10B)
-* [kromcomp/L3.1-Adlib-10B](https://huggingface.co/kromcomp/L3.1-Adlib-10B)
+* [kromcomp/L3.1-Rays-10B](https://huggingface.co/kromcomp/L3.1-Rays-10B)
+* [kromcomp/L3.1-Bloat-10B](https://huggingface.co/kromcomp/L3.1-Bloat-10B)
 
 ### Configuration
 
 The following YAML configuration was used to produce this model:
 
 ```yaml
-base_model: kromcomp/L3.1-Fatboy-10B
+base_model: kromcomp/L3.1-Extend-10B
+chat_template: llama3
 dtype: float32
 merge_method: della
 parameters:
@@ -38,22 +39,22 @@ parameters:
 slices:
 - sources:
   - layer_range: [0, 42]
-    model: kromcomp/L3.1-Npos-10B
+    model: kromcomp/L3.1-Rays-10B
     parameters:
       density: 0.7
       epsilon: 0.05
-      weight: 0.4
+      weight: [0.0, 0.2, 0.6]
   - layer_range: [0, 42]
-    model: kromcomp/L3.1-Adlib-10B
+    model: kromcomp/L3.1-Bloat-10B
     parameters:
-      density: 0.5
+      density: 0.7
       epsilon: 0.05
-      weight: 0.3
+      weight: [0.9, 0.3, 0.3]
   - layer_range: [0, 42]
-    model: kromcomp/L3.1-Fatboy-10B
+    model: kromcomp/L3.1-Extend-10B
     parameters:
-      density: 0.7
-      epsilon: 0.1
-      weight: 0.4
-tokenizer_source: union
+      density: 0.65
+      epsilon: 0.05
+      weight: [0.1, 0.5, 0.1]
+tokenizer: {}
 ```

config.json

@@ -1,16 +1,12 @@
 {
-  "_name_or_path": "kromcomp/L3.1-Fatboy-10B",
+  "_name_or_path": "kromcomp/L3.1-Extend-10B",
   "architectures": [
     "LlamaForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
   "bos_token_id": 128000,
-  "eos_token_id": [
-    128001,
-    128008,
-    128009
-  ],
+  "eos_token_id": 128009,
   "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 4096,
@@ -34,7 +30,7 @@
   "rope_theta": 500000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
-  "transformers_version": "4.46.3",
-  "use_cache": true,
+  "transformers_version": "4.47.0",
+  "use_cache": false,
   "vocab_size": 128256
 }

mergekit_config.yml

@@ -1,4 +1,5 @@
-base_model: kromcomp/L3.1-Fatboy-10B
+base_model: kromcomp/L3.1-Extend-10B
+chat_template: llama3
 dtype: float32
 merge_method: della
 parameters:
@@ -7,21 +8,21 @@ parameters:
 slices:
 - sources:
   - layer_range: [0, 42]
-    model: kromcomp/L3.1-Npos-10B
+    model: kromcomp/L3.1-Rays-10B
     parameters:
       density: 0.7
       epsilon: 0.05
-      weight: 0.4
+      weight: [0.0, 0.2, 0.6]
   - layer_range: [0, 42]
-    model: kromcomp/L3.1-Adlib-10B
+    model: kromcomp/L3.1-Bloat-10B
     parameters:
-      density: 0.5
+      density: 0.7
       epsilon: 0.05
-      weight: 0.3
+      weight: [0.9, 0.3, 0.3]
   - layer_range: [0, 42]
-    model: kromcomp/L3.1-Fatboy-10B
+    model: kromcomp/L3.1-Extend-10B
     parameters:
-      density: 0.7
-      epsilon: 0.1
-      weight: 0.4
-tokenizer_source: union
+      density: 0.65
+      epsilon: 0.05
+      weight: [0.1, 0.5, 0.1]
+tokenizer: {}

model-00001-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c4040d796795b41536d5f4c5de8979e985265189d110974a5e0c6136c06fa9f
+oid sha256:e8d29c8f2aeb3140e9e890ef57e8168adc7780372e895ef5aed1c909b4a24ddf
 size 2101346432

model-00002-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:687695ecca93afa394410508a8f95fef23046dcd88c00f40a9eedff91f2de093
+oid sha256:b3814600395597f0cfc96a402d522f4fb7ebfd4d18e4f872fc35724ef537e74c
 size 2973811968

model-00003-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d80a0404bfbc0be583120ac7d9ed598650cb466eb3607ea73088a96bd0e13ba
+oid sha256:83cf478bfc3539a6a38099f9f9164727b44ce643dc5396ebb0cec3d7a65f906a
 size 2852228272

model-00004-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2587e9daeeecc073c47ae942a3ddedbe1598159df12c82b0812555784d82afc
+oid sha256:aff07065f3f7ce95548dce8e57fd6876ffc169195142f92130352c778262d3d5
 size 2852228280

model-00005-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e35e5a8a94f1b4496088471c2b73d50280e168c1c70a5ee79a97fed8b0f6577f
+oid sha256:f9dca2b2bd10bb442136f9d87ec681debd1e192a1e70db28d94495f6a9ed2eef
 size 2936131096

model-00006-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:021380e7effe9854c2ea4c1fc2b55de556d6b5ab7722d879fac3a62c80e0bb65
+oid sha256:8415562f84c1e1db4a2395e1e7bc2520b4f003b916232b3395c43d9166a27afe
 size 2936131072

model-00007-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c440072ebb0b814d966fa40bf117adc19e8ab13872fa8810020c68aad2979a02
+oid sha256:cd76bf0ffdb79aea6ab1bc418d8373154e44fa925bf5b5208fe0d982a155dbf3
 size 2852228280

model-00008-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0c3372a4af7b335e36f30d253187507d2eea2c145d0ef054452ea8cb9453389
+oid sha256:3fad8211d6a842d267c5b68d087f04e03155c5b5310608a60f80972f25a2149b
 size 2936131096

model-00009-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17231f2cdf1895e97051df766f35d651d6f4e8b44c569cede0d174d6d6bd13b5
+oid sha256:c1604aef44348df28904b31e1601c365e7a2034fc548699bbe0a63c8769860c3
 size 2936131072

model-00010-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2ae43537b651da70c02cffa28699782076cc85cbc9934e7b3c8eab2b618d4f5
+oid sha256:cc6d61c73a872eec6922d3dca2ea674a419ecefec9f60913213eb8072b39e36d
 size 2852228280

model-00011-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:779c8dc7fb771324b89534ed26e156e6acd3d6faa33129a3f57de3dedc1370c9
+oid sha256:a5289042677fdf966875ee07c5526052e1d63fac827ed527ec10d902a0f5d591
 size 2936131096

model-00012-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a876d02d2409da48a86979ee9be308ecd7958a69a9be78575b1a3752dc7347e
+oid sha256:8539922f0371ae204b1ff38898e68c173d12ff00ba5f71a6d591bd9eee427082
 size 2936131080

model-00013-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0edc1efe2cbdd45952f7132f7d6f435474c6c4178d8fa75d29e2223983dc952e
+oid sha256:d9fb388bc32e97a288ec9f686a670ec018d3e83127d8b6194cbd6601db5d0dc6
 size 2852228272

model-00014-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da704bd0800ed01bda6423e9da386839a20ffaee797337eac6054a508ed83b4b
+oid sha256:e8961b528499497cd875269edd244d42976a3e10d7d7fce58112ee51bf5b2af6
 size 2936131064

model-00015-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:192b811fc7fb1eadc4d87d9c3f808249de05d1bee43217ad50dbdd19d388fbd4
+oid sha256:0e6d3ec5d559d9f4a6d38b8f1c913c5b95c70f4a369f367b24093a51ad059a86
 size 956351824

special_tokens_map.json

@@ -12,5 +12,12 @@
     "normalized": false,
     "rstrip": false,
     "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
   }
 }

tokenizer_config.json

@@ -2050,13 +2050,15 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
-  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "chat_template": "{% set loop_messages = messages %}\n{% for message in loop_messages %}\n{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' %}\n{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}\n{{ content }}\n{% endfor %}\n{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}{% endif %}\n",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }