Upload llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16

llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/bench.slurm

@@ -1,16 +1,16 @@
 #!/bin/bash
 
 #SBATCH --job-name=bench_cluster
-#SBATCH --time=01:30:00
+#SBATCH --time=02:00:00
 #SBATCH --partition=hopper-prod
 #SBATCH --nodes=8
 #SBATCH --gres=gpu:8
-#SBATCH --qos=high
+#SBATCH --qos=prod
 #SBATCH --ntasks-per-node=1
 #SBATCH --cpus-per-task=96
 #SBATCH --exclusive
-#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/log.out
-#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/log.out
+#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/log.out
+#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/log.out
 
 # Function to update status based on squeue output
 update_status() {
@@ -53,7 +53,7 @@ huggingface-cli login --token $HUGGINGFACE_TOKEN
 
 
 NANOTRON_REPO="/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron"
-CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/config.yaml"
+CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/config.yaml"
 
 LAUNCHER="torchrun \
    --nproc_per_node 8 \
@@ -72,7 +72,7 @@ cd ..
 job_id=${SLURM_JOB_ID}
 
 # Update status to "pending" or "running" in the background
-update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt &
+update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt &
 
 # Run the main command
 srun -u $LAUNCHER $CMD
@@ -80,28 +80,28 @@ exit_status=$?
 
 # Update status based on the exit status of `srun`
 if [ $exit_status -eq 0 ]; then
-    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt
+    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt
 else
-    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt
-    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt
-    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/log.out; then
-        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt
+    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/log.out; then
+        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt
+    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/log.out; then
+        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt
+    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/log.out; then
+        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt
     else
-        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt
+        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt
     fi 
 fi
 
 # Run the report script if the job completed successfully
 if [ $exit_status -eq 0 ]; then
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16 --is_logs
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16 --is_profiler
+    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16 --is_logs
+    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16 --is_profiler
 fi
 
 
 # Push to hub the folder using huggingface_cli
-huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16 llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16 --commit-message "Upload llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16"
+huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/tmp/bench_cluster/llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16 llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16 --commit-message "Upload llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16"
 
 # Verify the upload
 if [ $? -eq 0 ]; then

llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/config.yaml

@@ -48,7 +48,7 @@ parallelism:
   dp: 1
   expert_parallel_size: 1
   pp: 64
-  pp_engine: 1f1b
+  pp_engine: afab
   tp: 1
   tp_linear_async_communication: false
   tp_mode: REDUCE_SCATTER

llama-1B/64_GPUS/dp-1_tp-1_pp-64_mbz-16/status.txt

@@ -1 +1 @@
-timeout
+fail