File size: 1,328 Bytes
d727a17 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
#!/bin/bash
#SBATCH --job-name=alpaca-7 # Specify job name
#SBATCH --partition=pgpu # Specify partition name
#SBATCH --mem=0 # Use entire memory of node
#SBATCH --gres=gpu:8 # Generic resources; 8 GPU
#SBATCH --exclusive # Do not share node
#SBATCH --time=48:00:00 # Set a limit on the total run time
#SBATCH --output=logs_alp-7.o%j # File name for standard output
#SBATCH --error=errors_alp-7.e%j # File name for standard error output
cd /path/to/gitrepo
# activate conda environment
source /home/user/miniconda3/etc/profile.d/conda.sh
conda activate medalpaca
# recommended to manually set the hf cache dir, as the files are huge
export HF_HOME="/path/to/your/hfcache"
# feel free to adapt the below command, to run the training
# in 8bit with LoRA, fp16 with LoRA or bf16 and fsdp
torchrun --nproc_per_node=8 --master_port=9876 medalpaca/train.py \
--model 'decapoda-research/llama-7b-hf' \
--data_path 'medical_meadow_small.json' \
--output_dir './lora-alpaca-7b' \
--train_in_8bit False \
--use_lora False \
--bf16 True \
--tf32 True \
--fp16 False \
--gradient_checkpointing True \
--global_batch_size 256 \
--per_device_batch_size 4 \
--wandb_project 'medalpaca' \
--use_wandb False
|