| env_defaults: | |
| # NOTE: this kind of string leaded by > will append a new line to the end of the string | |
| SHARED_CMD_ARGS: >- | |
| -m src.train | |
| +model=base_sca_multitask_v2 | |
| training.do_train=True | |
| training.do_eval=True | |
| training.do_inference=True | |
| +data.streaming=False | |
| training.max_eval_samples=800 | |
| training.max_steps=200000 | |
| training.fp16=True | |
| training.output_dir=$AMLT_OUTPUT_DIR | |
| training.output_log_dir=$AMLT_LOGS_DIR | |
| model.cache_dir=/mnt/blob/weights/.model.cache/ | |
| training.save_strategy=steps | |
| training.save_steps=5000 | |
| training.save_total_limit=3 | |
| training.optim=adamw_torch | |
| training.evaluate_before_train=True | |
| training.per_device_train_batch_size=1 | |
| training.evaluation_strategy=steps | |
| training.eval_steps=5000 | |
| training.logging_steps=1000 | |
| training.logging_first_step=True | |
| training.dataloader_num_workers=4 | |
| training.num_masks_per_sample=16 | |
| wandb.project=$AMLT_EXPERIMENT_NAME | |
| wandb.name=$AMLT_JOB_NAME | |
| model.num_caption_tokens=8 | |
| model.additional_num_hidden_layers=12 | |
| model.num_task_tokens=6 | |
| training.lr_scheduler_type=cosine | |
| model.lm_head_model_name_or_path=gpt2-large | |
| training.learning_rate=1e-4 | |
| training.weight_decay=1e-4 | |
| training.warmup_steps=200 | |
| training.warmup_ratio=0.33333333 | |
| training.compute_metrics=True | |
| environment: | |
| image: nvidia/pytorch:23.07-py3 # NCCL on PHLRR4076 cannot initialized successfully | |
| # image: nvidia/pytorch:23.06-py3 # NCCL on PHLRR4076 cannot initialized successfully | |
| # image: nvidia/pytorch:22.12-py3 # Pydantic has bug: https://github.com/pydantic/pydantic/issues/545#issuecomment-1573776471 pip install pydantic==1.10.8; not support adamw_torch_fused, as it requires PyTorch 2.0 or higher | |
| registry: nvcr.io | |
| code: | |
| local_dir: $CONFIG_DIR/../ | |
| jobs: | |
| - name: gpt2-large | |
| preemptible: True | |
| sku: ${NUM_NODES}xG${NUM_GPUS} | |
| process_count_per_node: 1 # Each node should run 1 process | |
| command: | |
| - . amlt_configs/setup.sh | |
| - source ~/.bashrc | |
| - . amlt_configs/setup_accelerate_on_azure.sh | |
| - >- | |
| accelerate launch --config_file amlt_configs/accelerate_deepspeed_config.yaml | |
| $SHARED_CMD_ARGS | |
| train_data='[vg-densecap-local]' | |
| eval_data='[vg-densecap-local]' | |
| model.lm_head_model_name_or_path=gpt2-large | |
| $EXTRA_ARGS | |
| submit_args: | |
| env: | |
| SHARED_MEMORY_PERCENT: 0.5 | |
| HYDRA_FULL_ERROR: 1 | |
| container_args: | |
| shm_size: 256g | |
| - name: open_llama_3b_v2 | |
| preemptible: True | |
| sku: ${NUM_NODES}xG${NUM_GPUS} | |
| process_count_per_node: 1 # Each node should run 1 process | |
| command: | |
| - . amlt_configs/setup.sh | |
| - source ~/.bashrc | |
| - . amlt_configs/setup_accelerate_on_azure.sh | |
| - >- | |
| accelerate launch --config_file amlt_configs/accelerate_deepspeed_config.yaml | |
| $SHARED_CMD_ARGS | |
| train_data='[vg-densecap-local]' | |
| eval_data='[vg-densecap-local]' | |
| model.lm_head_model_name_or_path=openlm-research/open_llama_3b_v2 | |
| training.gradient_checkpointing=true | |
| $EXTRA_ARGS | |
| submit_args: | |
| env: | |
| SHARED_MEMORY_PERCENT: 0.5 | |
| HYDRA_FULL_ERROR: 1 | |
| container_args: | |
| shm_size: 256g | |
| # sing resrch 1x8 no-pre lsj | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0" -t msrresrchvc -w msrresrchws --sku=G8-V100 --no-pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :1=`date +"%m%d%y"`.resrch-1x8-v100-16g-no_pre.ollm3bv2-large-lsj train-sca-ablat-lsj-scale_lr-110423 | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0" -t msrresrchvc -w msrresrchws --sku=G8-V100 --no-pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :0=`date +"%m%d%y"`.resrch-1x8-v100-16g-no_pre.gpt2-large-lsj train-sca-ablat-lsj-scale_lr-110423 | |
| # sing octo 4x8 no-pre lsj | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0 training.learning_rate=4e-4" -t msroctovc -w msroctows --sku=4xG8-V100 --no-pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :1=`date +"%m%d%y"`.octo-4x8-v100-16g-no_pre.ollm3bv2-large-lsj-1xlr train-sca-ablat-lsj-scale_lr-110423 | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0 training.learning_rate=4e-4" -t msroctovc -w msroctows --sku=4xG8-V100 --no-pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :0=`date +"%m%d%y"`.octo-4x8-v100-16g-no_pre.gpt2-large-lsj-1xlr train-sca-ablat-lsj-scale_lr-110423 | |
| # The maximum scale lr with BS 64: 8e-4 (too big to achieve better) | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0 training.learning_rate=8e-4" -t msrresrchvc -w msrresrchws --sku=16xG4-V100-IB --pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :1=`date +"%m%d%y"`.resrch-16x4-v100-16g-pre.ollm3bv2-large-lsj-1xlr train-sca-ablat-lsj-scale_lr-110423 | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0 training.learning_rate=8e-4" -t msrresrchvc -w msrresrchws --sku=16xG4-V100-IB --pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :0=`date +"%m%d%y"`.resrch-16x4-v100-16g-no_pre.gpt2-large-lsj-1xlr train-sca-ablat-lsj-scale_lr-110423 | |
| # The maximum scale lr with BS 64: 4e-4 (try to achieve better with that from BS 32) | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0 training.learning_rate=4e-4" -t msrresrchvc -w msrresrchws --sku=16xG4-V100-IB --pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :1=`date +"%m%d%y"`.resrch-16x4-v100-16g-pre.ollm3bv2-large-lsj-1xlr-4e_4 train-sca-ablat-lsj-scale_lr-110423 | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0 training.learning_rate=4e-4" -t msrresrchvc -w msrresrchws --sku=16xG4-V100-IB --pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :0=`date +"%m%d%y"`.resrch-16x4-v100-16g-no_pre.gpt2-large-lsj-1xlr-4e_4 train-sca-ablat-lsj-scale_lr-110423 | |
| # 1x8, 4e-4 | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0 training.learning_rate=4e-4" -t itplabrr1cl1 -w resrchvc --sku=G8-V100 --pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :1=`date +"%m%d%y"`.rr1-1x8-v100-16g-pre.ollm3bv2-large-lsj-4e_4 train-sca-ablat-lsj-scale_lr-110423 | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0 training.learning_rate=4e-4" -t itplabrr1cl1 -w resrchvc --sku=G8-V100 --pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :0=`date +"%m%d%y"`.rr1-1x8-v100-16g-pre.gpt2-large-lsj-4e_4 train-sca-ablat-lsj-scale_lr-110423 | |
| # The maximum scale lr with BS 64: 4e-4 (try to achieve better with that from BS 32) | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0 training.learning_rate=4e-4" -t msrresrchvc -w msrresrchws --sku=16xG4-V100-IB --pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :1=`date +"%m%d%y"`.resrch-16x4-v100-16g-pre.ollm3bv2-large-lsj-1xlr-4e_4 train-sca-ablat-lsj-scale_lr-110423 | |
| # amlt run -d "" --extra-args "+data_transforms=lsj-0_1-2_0 training.learning_rate=4e-4" -t msrresrchvc -w msrresrchws --sku=16xG4-V100-IB --pre amlt_configs/train-sca-ablat-lsj-scale_lr-110423.yaml :0=`date +"%m%d%y"`.resrch-16x4-v100-16g-no_pre.gpt2-large-lsj-1xlr-4e_4 train-sca-ablat-lsj-scale_lr-110423 |