File size: 222 Bytes
cbdc7d9
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
{
  "embed_dim": 768,
  "num_heads": 4,
  "dropout": 0.025,
  "bias": true,
  "gate_min": 0.05,
  "gate_max": 0.95,
  "gate_dropout": 0.01,
  "dropout_gate_open": 0.075,
  "dropout_gate_close": 0.05,
  "CLS_self_attn": 0
}