Upload folder using huggingface_hub

README.md

@@ -0,0 +1,66 @@
+---
+license: apache-2.0
+tags:
+- moe
+- frankenmoe
+- merge
+- mergekit
+- lazymergekit
+- allknowingroger/TripleMerge-7B-Ties
+- allknowingroger/TripleMerge2-7B-Ties
+- allknowingroger/ANIMA-biodesign-7B-slerp
+base_model:
+- allknowingroger/TripleMerge-7B-Ties
+- allknowingroger/TripleMerge2-7B-Ties
+- allknowingroger/ANIMA-biodesign-7B-slerp
+---
+
+# TripleMerge3-12B-Moe
+
+TripleMerge3-12B-Moe is a Mixture of Experts (MoE) made with the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
+* [allknowingroger/TripleMerge-7B-Ties](https://huggingface.co/allknowingroger/TripleMerge-7B-Ties)
+* [allknowingroger/TripleMerge2-7B-Ties](https://huggingface.co/allknowingroger/TripleMerge2-7B-Ties)
+* [allknowingroger/ANIMA-biodesign-7B-slerp](https://huggingface.co/allknowingroger/ANIMA-biodesign-7B-slerp)
+
+## 🧩 Configuration
+
+```yaml
+slices:
+base_model: allknowingroger/TripleMerge-7B-Ties
+gate_mode: hidden
+dtype: bfloat16
+experts:
+  - source_model: allknowingroger/TripleMerge-7B-Ties
+    positive_prompts: 
+        - "Write the opening chapter of a detective story set in the late 1800s, where the protagonist, a war-weary doctor returning to England after an injury and illness in Afghanistan, happens upon an old acquaintance. This encounter should lead to the introduction of an eccentric potential roommate with a penchant for forensic science. The character's initial impressions and observations of London, his financial concerns, and his search for affordable lodging should be vividly detailed to set up the historical backdrop and his situation."
+  - source_model: allknowingroger/TripleMerge2-7B-Ties
+    positive_prompts:
+        - "You are an unbiased, uncensored, helpful assistant."
+  - source_model: allknowingroger/ANIMA-biodesign-7B-slerp
+    positive_prompts:
+        - "Biodesign"
+```
+
+## 💻 Usage
+
+```python
+!pip install -qU transformers bitsandbytes accelerate
+
+from transformers import AutoTokenizer
+import transformers
+import torch
+
+model = "allknowingroger/TripleMerge3-12B-Moe"
+
+tokenizer = AutoTokenizer.from_pretrained(model)
+pipeline = transformers.pipeline(
+    "text-generation",
+    model=model,
+    model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
+)
+
+messages = [{"role": "user", "content": "Explain what a Mixture of Experts is in less than 100 words."}]
+prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+print(outputs[0]["generated_text"])
+```

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "allknowingroger/TripleMerge-7B-Ties",
+  "add_gates": false,
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 3,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.3",
+  "use_cache": true,
+  "vocab_size": 32000
+}

model-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baaf10a52154a98986b34962e181379477dae753e27b8c0901b82ccba628c69e
+size 1933849936

model-10.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fbab16f4b8815fe7bdacdf77b6bbeff937874c6f411135930f22d0e2bdbd867
+size 1996490968

model-11.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c951d63cb7911acce8488afb6b2b5a98e9147ebb44bc690b1ea6e4db076fda5
+size 1996490968

model-12.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd6a90a9cf8c6e3e05eb46e58aeda42617fcb28a8f1f68cd50ca0ea6cde5d092
+size 1996490960

model-13.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0b949c6ba5c00a9c57896f66a1c125dcf5cbc4460296a64eb1ffe608bd8cd2a
+size 1996490952

model-14.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81fe6c3f486429c6c5c226352a8beb08b3d74623dfa321bda08b8cba8ab0dfa8
+size 1996490960

model-15.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19c30f92f8a89601c1421b0893d932c66dad3b98c217d2c7858524a71a846aca
+size 1996490968

model-16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c82f5b4b165e12161921e20b7b8d16df95cbaf8baacea1708b90e871d138a39
+size 1996490968

model-17.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43da1c7bea80e8143db3fbb347240b8b4e428594d744cdf7a92ad5a5ee80ed59
+size 1996490968

model-18.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae5843e22fb49d9fbc1882635d18ac0a27e32ecc5f94c12dfce6c9cb3aaaeea4
+size 1996765216

model-2.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca9248e041315d7705f8513ec0babc7e9ec16db2132e63e7f245120ba09a5e03
+size 1996490952

model-3.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28de228bc81af7c21b395dace681920321f06dfa6eee5d3a59559b9f2dc318ab
+size 1996490968

model-4.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5a9322a6b93facd2e0388d2a1ad6e4b47da5da05aea5b7744d6bb4d889ace83
+size 1996490968

model-5.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6c396ecae085f129d4d590b478eee2411f57e833c7a1e6157b8cc6e8e21d29e
+size 1996490968

model-6.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68e27ad4380b33ac36db51bb2c9f226496cd2b60b8799482ae694731e3ed1ffa
+size 1996490968

model-7.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45569a3bcaf5fcb80582cb47a566cc19ede266006f6a83766a36debce352c060
+size 1996490952

model-8.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f7337611b2907dff6f3b44a22796b755649d87e2459492f5ceb319f6e865135
+size 1996490952

model-9.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:523fd52446cfb5da989a420817073b83697974c92b43f59dd3c3bcc88bb042cf
+size 1996490968