| aws_output_bucket: s3://sagemaker-us-east-1-107457652907/experiments/llama2.13b.wudao.sft.combine.v1.0.seq2k.w16.adamw.NA100.0803.ds | |
| data_dir: null | |
| dist_load_data_barrier: false | |
| train_file: /tmp/data-train/panda-sft-combine/ | |
| dev_file: null | |
| test_file: null | |
| model: | |
| _target_: models.llama.LlamaForConditionalGeneration.from_pretrained | |
| use_peft: false | |
| gradient_checkpointing: true | |
| enable_flash_attention: true | |
| flash_attention_vanilla_torch: true | |
| read_tensor: | |
| _target_: data.collators.zh_instruct.TextDatasetCombineV2 | |
| extra_data: | |
| _target_: data.collators.zh_instruct.WuDaoCorpusDataset | |
| tokenizer: null | |
| file_path: /tmp/data-train/WuDaoCorpus2.0_base_200G/ | |
| file_num: 50 | |
| extended_vocab: null | |
| collator: | |
| _target_: data.collators.flan.CombineCollator | |
| max_seq_length: 2048 | |
| tokenizer: ${model_name_or_path} | |
| decoder_only: true | |
| padding: longest | |
| padding_side: right | |
| num_workers: 4 | |
| prefetch_factor: 2 | |
| do_preprocess: false | |
| model_name_or_path: /tmp/llama2.13b.vocab_ext_80k.v3.0.seq2k.w16.adamw.NA100.0728/checkpoint-2000 | |
| pretrain: null | |
| exp_name: llama2.13b.wudao.sft.combine.v1.0.seq2k.w16.adamw.NA100.0803.ds | |
| exp_notes: null | |
| output_dir: /tmp/${exp_name} | |
| resume: /tmp/checkpoint-500 | |
| do_train: true | |
| evaluate_during_training: false | |
| do_eval: false | |
| eval_sub_path: checkpoint-* | |
| per_gpu_train_batch_size: 2 | |
| per_gpu_eval_batch_size: 1 | |
| learning_rate: 1.0e-05 | |
| gradient_accumulation_steps: 64 | |
| weight_decay: 0.01 | |
| adam_epsilon: 1.0e-06 | |
| adam_betas: (0.9, 0.99) | |
| max_grad_norm: 1.0 | |
| num_train_epochs: 1 | |
| total_dataset_len: 10000000 | |
| max_steps: 0 | |
| warmup_proportion: 0 | |
| warmup_steps: 0 | |
| optimizer: null | |
| use_nvlamb: null | |
| bit_training: null | |
| logging_steps: 1 | |
| save_best: false | |
| save_steps: 250 | |
| eval_steps: 250 | |
| ddp_eval: true | |
| no_cuda: false | |
| seed: 42 | |
| local_rank: 0 | |
| fp16: true | |
| fp16_opt_level: O1 | |
| fp16_bfloat16: true | |
| prediction_cfg: | |
| metric: acc | |
| measure: 1 | |
| best_checkpoint: null | |
| best_result: null | |
| eval_forward_fn: | |
| _target_: general_util.evaluator.DiscriminatorForwardFn | |
| post_process: null | |
| fairscale_config: | |
| _target_: general_util.fsdp_utils.default_initialize | |
| fp16: ${fp16} | |
| move_grads_to_cpu: false | |
| move_params_to_cpu: false | |
| flatten_parameters: false | |
| with_lightseq: false | |
| load_lr_scheduler_states: false | |
| ds_cfg: | |
| train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size} | |
| gradient_accumulation_steps: ${gradient_accumulation_steps} | |
| optimizer: | |
| type: AdamW | |
| params: | |
| lr: ${learning_rate} | |
| betas: | |
| - 0.9 | |
| - 0.96 | |
| eps: ${adam_epsilon} | |
| weight_decay: ${weight_decay} | |
| scheduler: | |
| type: WarmupDecayLR | |
| params: | |
| total_num_steps: 4882 | |
| warmup_max_lr: ${learning_rate} | |
| warmup_num_steps: 0 | |
| warmup_type: linear | |
| gradient_clipping: ${max_grad_norm} | |
| bf16: | |
| enabled: ${fp16} | |
| zero_optimization: | |
| stage: 1 | |
| contiguous_gradients: true | |
| overlap_comm: true | |
| reduce_scatter: true | |
| reduce_bucket_size: 500000000.0 | |
| allgather_bucket_size: 500000000.0 | |
| offload_optimizer: | |
| device: cpu | |
| pin_memory: true | |
| steps_per_print: 1 | |
| summary_helper: | |
| _target_: general_util.tensorboard_helper.WandbWriter | |
| batch_index_or_keys: null | |
| outputs_index_or_keys: null | |
| n_gpu: 1 | |
| device: cuda:0 | |
| train_batch_size: 2 | |
| eval_batch_size: null | |
| world_size: 16 | |
| world_rank: null | |