Upload folder using huggingface_hub

xDAN APUS4.0: xDAN-L2-moe-Random-v4.3-0402

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "xDAN2099/xDAN-L2-RL-v7.3-Agent-Dlora-0318-APUS-xDAN4.0-e3",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 6,
+  "eos_token_id": 7,
+  "hidden_act": "silu",
+  "hidden_size": 7168,
+  "initializer_range": 0.02,
+  "intermediate_size": 20480,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 56,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 60,
+  "num_key_value_heads": 8,
+  "num_local_experts": 4,
+  "output_router_logits": false,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.2",
+  "use_cache": true,
+  "vocab_size": 64000
+}

mergekit_moe_config.yml

@@ -0,0 +1,68 @@
+base_model: xDAN2099/xDAN-L2-RL-v7.3-Agent-Dlora-0318-APUS-xDAN4.0-e3
+gate_mode: random # Use the "hidden" mode for gate operation, implying internal decision-making without exposing the process.
+tokenizer_source: base # Use the base tokenizer for processing inputs.
+dtype: bfloat16 # Use bfloat16 data type for model output, balancing performance and precision.
+experts_per_token: 2 # Assign two experts per token for enhanced decision-making.
+experts:
+  - source_model: xDAN2099/xDAN-L2-RL-Mix378-BagelMath-0310-e2-Chat-v7.2-DPO-QDora-0317-epoch05
+    positive_prompts:
+      - "Explain quantum mechanics concepts in simple terms"
+      - "Detailed walkthrough of solving linear algebra problems"
+      - "Interpretation of complex statistical data for research papers"
+      - "Advanced calculus applications in engineering"
+      - "Mathematical modeling for economic forecasts"
+    negative_prompts:
+      - "General knowledge trivia questions"
+      - "Creating a screenplay for a movie"
+      - "Advice on personal relationships"
+      - "Cooking recipes for beginners"
+      - "Trends in digital marketing strategies"
+
+  - source_model: NousResearch/Nous-Hermes-2-Yi-34B
+    positive_prompts:
+      - "Generate creative writing prompts for a novel"
+      - "Dialogue script for a video game scenario"
+      - "Constructing an engaging blog post on technology trends"
+      - "Python coding tips for beginners"
+      - "Developing characters for a fantasy story"
+    negative_prompts:
+      - "In-depth analysis of a medical research paper"
+      - "Theoretical physics problem sets"
+      - "Investment portfolio optimization"
+      - "Architectural design principles"
+      - "Advanced machine learning algorithm explanations"
+
+  - source_model: xDAN2099/xDAN-L2-RL-v7.3-Agent-Dlora-0318-e1
+    positive_prompts:
+      - "Strategies for effective online teaching"
+      - "Guide to writing a technical paper in computer science"
+      - "Explaining software development life cycle (SDLC)"
+      - "Tips for engaging online content creation"
+      - "Overview of the latest web development frameworks"
+    negative_prompts:
+      - "Performing a detailed company financial analysis"
+      - "Creating a workout plan for athletes"
+      - "Restoration tips for classic cars"
+      - "Psychological assessment techniques"
+      - "Surgical techniques in modern medicine"
+
+  - source_model: xDAN2099/xDAN-APUS4-Preference-DPO-0331-v2-e1
+    positive_prompts:
+      - "Mathematics"
+      - "Physics"
+      - "Chemistry"
+      - "Biology"
+      - "Medicine"
+      - "Engineering"
+      - "Computer Science"
+    negative_prompts:
+      - "History"
+      - "Philosophy"
+      - "Linguistics"
+      - "Literature"
+      - "Art and Art History"
+      - "Music Theory and Composition"
+      - "Performing Arts (Theater, Dance)"
+
+
+ #CUDA_VISIBLE_DEVICES='' mergekit-moe xDAN-L2-moe-Random-v4.2-0327.yaml xDAN-L2-moe-Random-v4.2-0327

model-00001-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42feec9537e5ba7f180ccead79c4d195fbccf8d1d5789213ebbfa3872bf04671
+size 9879789120

model-00002-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69a5aedaebb481cca6926009ede7c4ada05a72f98e9a48c2b96bad641983abf4
+size 9865065968

model-00003-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4e4faf30493b57a8b4ecc94a7cca7d6b7ea802fa73bb165251119142634c4e8
+size 9806374952

model-00004-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61d2714e9054ac3f654d00fc21f00c283fbc1c906fe615e36cd0649208cc1e31
+size 9997201336

model-00005-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:065ac921e4c5140e42fb2b66aba64f0c068d20c371f94c1ea6b6a5cfcc362adb
+size 9967841040

model-00006-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23d6da37e684c01bca3e50c59e9682b66958a4ba9921086e4951ee1dade64b18
+size 9806375000

model-00007-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e412eb6ba16f046381022b05980f7fda5ca4028d52e08b6d3c010c31ee758cd9
+size 9865066016

model-00008-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dea951f08c00278ae9a85907d45a2cf7bb0c677c1e71fe5dd053d45adf2f97a
+size 9806375000

model-00009-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f50762b9205d49aa6c7f090b386aa1d24095890260500fd1cd3c868a42a059e3
+size 9806375000

model-00010-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2194c80f26b6d57e5f911fb8088bdee57f8e46231a252d14662debea3ca620ba
+size 9865066016

model-00011-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02a447ba0ef5a3d024382cdab886b75ec40d2b9d47601df5e74a65b0f0925dbe
+size 9806375000

model-00012-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:390128feadc7bb33d2a079cfc1ce077e6cf09db1b2eec26a3275e2688717d178
+size 9997201392

model-00013-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd40e8e51380a32c75842d5b71c2e6433fb6fb7e0e48b169eb9ee22a01e3edac
+size 9967841040

model-00014-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:154126d4b03402225cf6fdb2aaaa78a550f0604f690eb9a2c3670fd073e64781
+size 9806375000

model-00015-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a94c5bbb78a29c0cf869b7b7062a9d5308f64acfae581b131e612d064cbccef4
+size 9865066016

model-00016-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:849bc25ac4db5043e710c14825af3958859e9870d13faf013c5410a658eb8539
+size 9806375000

model-00017-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:443e35413e625b8acba43c512871ba9076ac4d3ec3b95988fc41c6c8342e4dd7
+size 9806375000

model-00018-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fdd7944a06b5fdd5d44f5c06bd32bbe0a289a1aa45eb231f69c8ed520219e44
+size 9865066016

model-00019-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96d755b268c4fbde1b52e3acacfac90aad79e3799c18795420950ac9fe1baa3a
+size 9806375000

model-00020-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ddf0bd1f47615b68624b382ddc7163f15a703746ec293a133e46d2274e000cb
+size 9997201392

model-00021-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68d3dff150421361e0700ead6f3e4c2ff9ef38409d5925abd1c114086ad37f6c
+size 9967841040

model-00022-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbb7e99090ee9b29f44d6da557c02dd4238a51c6f216908b1e44f079632bcb1f
+size 9806375000

model-00023-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6979da55a1439bbff437207a75c3dbf07e54aa66cb14e2730c93cc61eeee8dc0
+size 9865066016

model-00024-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cf90071779251dd237a6798e35227b2a7203be2a99ffbe7e16e1f4083008d73
+size 297048896

special_tokens_map.json

@@ -0,0 +1,27 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_start|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386c49cf943d71aa110361135338c50e38beeff0a66593480421f37b319e1a39
+size 1033105

tokenizer_config.json

@@ -0,0 +1,65 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% set system_message = 'You are a helpful assistant named APUS-xDAN-4.0 MoE.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\\n' + system_message + '<|im_end|>\\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\\n' + content + '<|im_end|>\\n<|im_start|>assistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": true,
+  "model_max_length": 200000,
+  "pad_token": "<|im_start|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}