blueprintninja commited on
Commit
d73dabe
1 Parent(s): 7712b7a

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. README.md +66 -0
  2. config.json +31 -0
README.md ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ tags:
4
+ - moe
5
+ - frankenmoe
6
+ - merge
7
+ - mergekit
8
+ - lazymergekit
9
+ - cognitivecomputations/WizardLM-7B-Uncensored
10
+ - dreamgen/WizardLM-2-7B
11
+ base_model:
12
+ - cognitivecomputations/WizardLM-7B-Uncensored
13
+ - dreamgen/WizardLM-2-7B
14
+ ---
15
+
16
+ # UndeadWizard
17
+
18
+ UndeadWizard is a Mixture of Experts (MoE) made with the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
19
+ * [cognitivecomputations/WizardLM-7B-Uncensored](https://huggingface.co/cognitivecomputations/WizardLM-7B-Uncensored)
20
+ * [dreamgen/WizardLM-2-7B](https://huggingface.co/dreamgen/WizardLM-2-7B)
21
+
22
+ ## 🧩 Configuration
23
+
24
+ ```yaml
25
+ base_model: dreamgen/WizardLM-2-7B
26
+ experts:
27
+ - source_model: cognitivecomputations/WizardLM-7B-Uncensored
28
+ positive_prompts:
29
+ - "chat"
30
+ - "write"
31
+ - "scene"
32
+ - "story"
33
+ - "explain"
34
+ - source_model: dreamgen/WizardLM-2-7B
35
+ positive_prompts:
36
+ - "reason"
37
+ - "math"
38
+ - "code"
39
+ - "solve"
40
+ - "python"
41
+
42
+ ```
43
+
44
+ ## 💻 Usage
45
+
46
+ ```python
47
+ !pip install -qU transformers bitsandbytes accelerate
48
+
49
+ from transformers import AutoTokenizer
50
+ import transformers
51
+ import torch
52
+
53
+ model = "blueprintninja/UndeadWizard"
54
+
55
+ tokenizer = AutoTokenizer.from_pretrained(model)
56
+ pipeline = transformers.pipeline(
57
+ "text-generation",
58
+ model=model,
59
+ model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
60
+ )
61
+
62
+ messages = [{"role": "user", "content": "Explain what a Mixture of Experts is in less than 100 words."}]
63
+ prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
64
+ outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
65
+ print(outputs[0]["generated_text"])
66
+ ```
config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "dreamgen/WizardLM-2-7B",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 4096,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 14336,
13
+ "max_position_embeddings": 32768,
14
+ "model_type": "mixtral",
15
+ "num_attention_heads": 32,
16
+ "num_experts_per_tok": 2,
17
+ "num_hidden_layers": 32,
18
+ "num_key_value_heads": 8,
19
+ "num_local_experts": 2,
20
+ "output_router_logits": false,
21
+ "rms_norm_eps": 1e-05,
22
+ "rope_theta": 10000.0,
23
+ "router_aux_loss_coef": 0.001,
24
+ "router_jitter_noise": 0.0,
25
+ "sliding_window": null,
26
+ "tie_word_embeddings": false,
27
+ "torch_dtype": "bfloat16",
28
+ "transformers_version": "4.40.0",
29
+ "use_cache": false,
30
+ "vocab_size": 32000
31
+ }