File size: 3,074 Bytes
6e30527 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 |
# Merging MISCHIEVOUS-12B-Mix Models with Sliced SLERP
base_model: bamec66557/mergekit-slerp-tsgkafq
dtype: bfloat16
merge_method: slerp
tokenizer_source: union
# Slices Configuration (Layer-Specific Merging)
slices:
- name: initial_layers
sources:
- model: bamec66557/mergekit-slerp-uhnpbqg
layer_range: [0, 10]
- model: bamec66557/mergekit-slerp-tsgkafq
layer_range: [0, 10]
parameters:
t:
- name: self_attn
value: [0.8, 0.85, 0.9, 0.95, 1.0]
- name: mlp
value: [0.9, 0.95, 1.0, 1.05, 1.1]
- name: layer_norm
value: [0.6, 0.65, 0.7, 0.75, 0.8]
- name: embed_tokens
value: [1.0]
- name: middle_layers
sources:
- model: bamec66557/mergekit-slerp-uhnpbqg
layer_range: [10, 20]
- model: bamec66557/mergekit-slerp-tsgkafq
layer_range: [10, 20]
parameters:
t:
- name: self_attn
value: [0.7, 0.75, 0.8, 0.85, 0.9]
- name: mlp
value: [1.0, 0.95, 0.9, 0.85, 0.8]
- name: layer_norm
value: [0.5, 0.55, 0.6, 0.65, 0.7]
- name: embed_tokens
value: [1.0]
- name: upper_middle_layers
sources:
- model: bamec66557/mergekit-slerp-uhnpbqg
layer_range: [20, 30]
- model: bamec66557/mergekit-slerp-tsgkafq
layer_range: [20, 30]
parameters:
t:
- name: self_attn
value: [0.6, 0.65, 0.7, 0.75, 0.8]
- name: mlp
value: [0.8, 0.75, 0.7, 0.65, 0.6]
- name: layer_norm
value: [0.4, 0.45, 0.5, 0.55, 0.6]
- name: embed_tokens
value: [1.0]
- name: final_layers
sources:
- model: bamec66557/mergekit-slerp-uhnpbqg
layer_range: [30, 40]
- model: bamec66557/mergekit-slerp-tsgkafq
layer_range: [30, 40]
parameters:
t:
- name: self_attn
value: [0.9, 1.0, 1.1, 1.2, 1.3]
- name: mlp
value: [0.7, 0.65, 0.6, 0.55, 0.5]
- name: layer_norm
value: [0.7, 0.75, 0.8, 0.85, 0.9]
- name: embed_tokens
value: [1.0]
# Regularization (Prevent Overfitting During Merging)
regularization:
methods:
- name: weight_clipping
clip_range: [-0.2, 0.2]
- name: random_noise
scale: 0.015
- name: l2_norm
scale: 0.01
# Postprocessing (Enhance Merged Model Quality)
postprocessing:
operations:
- name: random_noise
scale: 0.0025
- name: non_linear_scaling
parameters:
function: tanh
- name: sharpening
intensity: 0.3
- name: gaussian_smoothing
sigma: 1.5
- name: smoothing
parameters:
adaptive: true
range: [0.8, 1.2]
kernel_size: 5
- name: normalize
- name: dynamic_scaling
scale_range: [0.75, 1.25]
# Optional: Ties Merging (Advanced Technique)
ties:
enabled: false # Set to true if ties merging is required
method: greedy # Options: greedy, optimal, random
layers: [0, 10, 20, 30] # Example layers for ties merging
|