NDugar commited on
Commit
5d8a119
1 Parent(s): 5a85ced

Upload ds_config.json

Browse files
Files changed (1) hide show
  1. ds_config.json +23 -0
ds_config.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+
4
+ "enabled": true,
5
+
6
+ "initial_scale_power": 12
7
+
8
+ },
9
+ "zero_optimization": {
10
+
11
+ "stage": 2,
12
+
13
+ "reduce_bucket_size": 5e7,
14
+
15
+ "allgather_bucket_size": 1.25e9,
16
+
17
+ "overlap_comm": true,
18
+
19
+ "contiguous_gradients": true
20
+
21
+ },
22
+ "zero_allow_untested_optimizer": true
23
+ }