pretrain model

scripts/pretrain-model.yaml

@@ -86,7 +86,7 @@ train:
   max_steps:
 
   # Limits the length of samples. Off by default (type: Optional[int], default: null)
-  # max_seq_length: 8193
+  max_seq_length: 8193
 
   # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
   tie_embeddings: true
@@ -121,15 +121,15 @@ optimizer:
 
   init_args:
     #   (type: float, default: 0.001)
-    lr: 1e-4
+    lr: 4e-04
 
     #   (type: float, default: 0.01)
-    weight_decay: 1e-2
+    weight_decay: 0.1
 
     #   (type: tuple, default: (0.9,0.999))
     betas:
       - 0.9
-      - 0.999
+      - 0.95
 
 # How many devices/GPUs to use. Uses all GPUs by default. (type: Union[int, str], default: auto)
 devices: auto