| _BASE_: "7tasks_berttiny_training.yaml" | |
| MOE: | |
| MOE: True | |
| MOE_TYPE: 'attribute' | |
| TAG_Transform: True | |
| ATTRIBUTE_LENGTH: 8 | |
| EP_WORLD_SIZE: 1 # tag moe only | |
| NUM_EXPERTS: 8 | |
| TOP_K: 2 | |
| CAPACITY_FACTOR: 3.0 | |
| EVAL_MIN_CAPACITY: 4.0 | |
| MIN_CAPACITY: 4 | |
| NOISY_GATE_POLICY: 'vmoe' | |
| MOE_PARAM_GROUP: True | |
| MOE_EXPERT_TYPE: 'FFN,SA' | |
| SA_LINEAR_OUT_MOE: True | |
| MOE_EXPERT_LOCATION: 'all' # 'odd' | |
| # MOE_LAYER_START_IDX: 3 | |
| # MOE_LAYER_END_IDX: 21 | |
| # MOE_LAYER_START_IDX: 18 | |
| # MOE_LAYER_END_IDX: 12 | |
| BATCH_PRIO: True | |
| USE_TUTEL: True | |
| FFN_SHARE_GATE_DECISION: True |