File size: 579 Bytes
32b542e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 |
_BASE_: "7tasks_berttiny_training.yaml"
MOE:
MOE: True
MOE_TYPE: 'attribute'
TAG_Transform: True
ATTRIBUTE_LENGTH: 8
EP_WORLD_SIZE: 1 # tag moe only
NUM_EXPERTS: 8
TOP_K: 2
CAPACITY_FACTOR: 3.0
EVAL_MIN_CAPACITY: 4.0
MIN_CAPACITY: 4
NOISY_GATE_POLICY: 'vmoe'
MOE_PARAM_GROUP: True
MOE_EXPERT_TYPE: 'FFN,SA'
SA_LINEAR_OUT_MOE: True
MOE_EXPERT_LOCATION: 'all' # 'odd'
# MOE_LAYER_START_IDX: 3
# MOE_LAYER_END_IDX: 21
# MOE_LAYER_START_IDX: 18
# MOE_LAYER_END_IDX: 12
BATCH_PRIO: True
USE_TUTEL: True
FFN_SHARE_GATE_DECISION: True |