JonasGeiping commited on
Commit
9e699be
1 Parent(s): 77c5c17

Upload train_budget_hours_24.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. train_budget_hours_24.json +41 -0
train_budget_hours_24.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "optim": {
3
+ "type": "AdamW",
4
+ "lr": 0.001,
5
+ "betas": [
6
+ 0.9,
7
+ 0.98
8
+ ],
9
+ "eps": 1e-12,
10
+ "weight_decay": 0.01,
11
+ "amsgrad": false,
12
+ "fused": null
13
+ },
14
+ "optim_mod": {
15
+ "name": "none"
16
+ },
17
+ "name": "bert-o4",
18
+ "limited_decay_keys": [
19
+ "bias",
20
+ "LayerNorm.bias",
21
+ "LayerNorm.weight",
22
+ "norm"
23
+ ],
24
+ "warmup_steps": 0,
25
+ "cooldown_steps": 0,
26
+ "steps": 900000,
27
+ "scheduler": "budget-triangle2",
28
+ "batch_size": 8192,
29
+ "batch_size_ramp": 0.6,
30
+ "gradient_clipping": 0.5,
31
+ "pretrain_in_train_mode": false,
32
+ "objective": {
33
+ "name": "masked-lm",
34
+ "mlm_probability": 0.25,
35
+ "use_80_20_rule": true,
36
+ "disable_mlm": false,
37
+ "token_drop": 0.0
38
+ },
39
+ "reverse_dataset_order": false,
40
+ "budget": 24
41
+ }