Upload llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8

.gitattributes

@@ -77,3 +77,4 @@ llama-1B/16_GPUS/dp-2_tp-4_pp-2_mbz-4/profiler/ip-26-0-173-202_732190.1719951899
 llama-1B/16_GPUS/dp-4_tp-4_pp-1_mbz-2/profiler/ip-26-0-165-24_808737.1719952159405509926.pt.trace.json.tmp filter=lfs diff=lfs merge=lfs -text
 llama-1B/64_GPUS/dp-4_tp-16_pp-1_mbz-32/profiler/ip-26-0-160-192_1104448.1719964747268413011.pt.trace.json filter=lfs diff=lfs merge=lfs -text
 llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-2/profiler/ip-26-0-160-225_1766358.1719966036233626173.pt.trace.json filter=lfs diff=lfs merge=lfs -text
+llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/profiler/ip-26-0-160-225_1769215.1719966492698530506.pt.trace.json filter=lfs diff=lfs merge=lfs -text

llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/bench.slurm

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+#SBATCH --job-name=bench_cluster
+#SBATCH --time=01:30:00
+#SBATCH --partition=hopper-prod
+#SBATCH --nodes=8
+#SBATCH --gres=gpu:8
+#SBATCH --qos=high
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=96
+#SBATCH --exclusive
+#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/log.out
+#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/log.out
+
+# Function to update status based on squeue output
+update_status() {
+    job_id=$1
+    status_file=$2
+    # For unknown reasons, it doenst update status for pending. It only works for running 
+    while true; do
+        job_status=$(squeue --job $job_id --noheader --format=%T)
+        echo "Job status: $job_status"
+        if [ -z "$job_status" ]; then
+            # Job has finished or is not found
+            break
+        elif [ "$job_status" = "RUNNING" ]; then
+            printf "running" > $status_file
+            break
+        fi
+        sleep 10
+    done
+}
+
+# Misc initializations.
+echo "========================"
+echo "START TIME: $(date)"
+source /fsx/ferdinandmom/miniforge3/etc/profile.d/conda.sh
+conda activate /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster
+echo python3 version = $(python3 --version)
+echo "========================"
+
+# Slurm stuff
+export HOSTNAMES=$(scontrol show hostnames "$SLURM_JOB_NODELIST")
+export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
+export MASTER_PORT=$((1024 + RANDOM % 64511))
+
+export TMPDIR=/scratch
+export HF_DATASETS_CACHE="/admin/home/ferdinand_mom/.cache"
+export CUBLAS_WORKSPACE_CONFIG=":4096:8"
+export CUDA_DEVICE_MAX_CONNECTIONS="1"
+
+huggingface-cli login --token $HUGGINGFACE_TOKEN
+
+
+NANOTRON_REPO="/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron"
+CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/config.yaml"
+
+LAUNCHER="torchrun \
+   --nproc_per_node 8 \
+   --nnodes 8 \
+   --rdzv_endpoint ${MASTER_ADDR}:${MASTER_PORT} \
+   --rdzv_backend c10d \
+   --max_restarts 0 \
+   --tee 3 \
+   --node_rank ${SLURM_PROCID}"
+
+# Checkout the bench_cluster branch
+cd $NANOTRON_REPO
+git checkout bench_cluster
+cd ..
+# Get the current job ID
+job_id=${SLURM_JOB_ID}
+
+# Update status to "pending" or "running" in the background
+update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/status.txt &
+
+# Run the main command
+srun -u $LAUNCHER $CMD
+exit_status=$?
+
+# Update status based on the exit status of `srun`
+if [ $exit_status -eq 0 ]; then
+    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/status.txt
+else
+    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/log.out; then
+        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/status.txt
+    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/log.out; then
+        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/status.txt
+    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/log.out; then
+        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/status.txt
+    else
+        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/status.txt
+    fi 
+fi
+
+# Run the report script if the job completed successfully
+if [ $exit_status -eq 0 ]; then
+    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8 --is_logs
+    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8 --is_profiler
+fi
+
+
+# Push to hub the folder using huggingface_cli
+huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8 llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8 --commit-message "Upload llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8"
+
+# Verify the upload
+if [ $? -eq 0 ]; then
+    echo "Uploading to Huggingface Hub successful"
+else
+    echo "Failed to upload to Huggingface Hub"
+fi

llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/config.yaml

@@ -0,0 +1,90 @@
+general:
+  project: bench_cluster
+  seed: 42
+model:
+  ddp_bucket_cap_mb: 25
+  dtype: bfloat16
+  init_method:
+    std: 0.025
+  make_vocab_size_divisible_by: 1
+  model_config:
+    bos_token_id: 1
+    eos_token_id: 2
+    hidden_act: silu
+    hidden_size: 2048
+    initializer_range: 0.02
+    intermediate_size: 4096
+    is_llama_config: true
+    max_position_embeddings: 4096
+    num_attention_heads: 32
+    num_hidden_layers: 24
+    num_key_value_heads: 32
+    pad_token_id: null
+    pretraining_tp: 1
+    rms_norm_eps: 1.0e-05
+    rope_scaling: null
+    rope_theta: 10000.0
+    tie_word_embeddings: true
+    use_cache: true
+    vocab_size: 50257
+optimizer:
+  accumulate_grad_in_fp32: true
+  clip_grad: 1.0
+  learning_rate_scheduler:
+    learning_rate: 0.0001
+    lr_decay_style: linear
+    lr_warmup_style: linear
+    lr_warmup_steps: 1
+    min_decay_lr: 1.0e-05
+  optimizer_factory:
+    adam_beta1: 0.9
+    adam_beta2: 0.95
+    adam_eps: 1.0e-08
+    name: adamW
+    torch_adam_is_fused: true
+  weight_decay: 0.01
+  zero_stage: 1
+parallelism:
+  dp: 32
+  expert_parallel_size: 1
+  pp: 1
+  pp_engine: 1f1b
+  tp: 2
+  tp_linear_async_communication: false
+  tp_mode: REDUCE_SCATTER
+profiler:
+  profiler_export_path: /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8
+tokenizer:
+  tokenizer_max_length: null
+  tokenizer_name_or_path: openai-community/gpt2
+  tokenizer_revision: null
+data_stages:
+- name: Training Stage
+  start_training_step: 1
+  data:
+    dataset:
+      dataset_overwrite_cache: false
+      dataset_processing_num_proc_per_process: 64
+      hf_dataset_config_name: null
+      hf_dataset_or_datasets: roneneldan/TinyStories
+      hf_dataset_splits: train
+      text_column_name: text
+    num_loading_workers: 0
+    seed: 42
+lighteval: null
+tokens:
+  train_steps: 20
+  val_check_interval: -1
+  batch_accumulation_per_replica: 4
+  limit_test_batches: 0
+  limit_val_batches: 0
+  micro_batch_size: 8
+  sequence_length: 4096
+logging:
+  iteration_step_info_interval: 1
+  log_level: info
+  log_level_replica: info
+checkpoints:
+  checkpoint_interval: 100000
+  checkpoints_path: /dev/null
+  resume_checkpoint_path: null

llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/log_metrics.csv

@@ -0,0 +1,21 @@
+iteration,consumed_tokens,elapsed_time_per_iteration_ms,tokens_per_sec,tokens_per_sec_per_gpu,global_batch_size,lm_loss,lr,model_tflops_per_gpu,hardware_tflops_per_gpu,grad_norm,memory_usage_MiB,peak_allocated_MiB,peak_reserved_MiB
+1,4190000.0000000005,18200.0,230000.0,3600.0,1020.0,11.5,0.0001,32.7,32.7,26.4,3474.18,42258.96,44002.0
+2,8390000.0,8700.0,482000.0,7540.0,1020.0,11.5,9.53e-05,68.4,68.4,26.6,3474.18,42258.96,44002.0
+3,12600000.0,8650.0,485000.0,7570.0,1020.0,11.5,9.05e-05,68.7,68.7,262.0,3474.18,42258.96,44002.0
+4,16800000.0,8650.0,485000.0,7580.0,1020.0,14.6,8.58e-05,68.8,68.8,29.1,3474.18,5623.96,44002.0
+5,21000000.0,8670.0,484000.0,7560.0,1020.0,10.8,8.11e-05,68.6,68.6,30.9,3474.18,42258.96,44002.0
+6,25200000.0,8730.0,481000.0,7510.0,1020.0,10.6,7.63e-05,68.1,68.1,27.5,3474.18,42258.96,44002.0
+7,29400000.0,2130.0,1970000.0,30700.0,1020.0,10.2,7.16e-05,279.0,279.0,9.42,3474.18,42258.96,44002.0
+8,33600000.0,8650.0,485000.0,7580.0,1020.0,12.8,6.68e-05,68.8,68.8,77.8,3474.18,42258.96,44002.0
+9,37700000.0,8690.0,482000.0,7540.0,1020.0,9.42,6.21e-05,68.4,68.4,11.5,3474.18,42258.96,44002.0
+10,41900000.0,8650.0,485000.0,7580.0,1020.0,9.2,5.74e-05,68.8,68.8,6.89,3474.18,42258.96,44002.0
+11,46100000.0,8670.0,484000.0,7560.0,1020.0,8.92,5.26e-05,68.6,68.6,5.87,3474.18,42258.96,44002.0
+12,50300000.0,8730.0,481000.0,7510.0,1020.0,8.55,4.79e-05,68.2,68.2,6.31,3474.18,42258.96,44002.0
+13,54500000.0,8700.0,482000.0,7530.0,1020.0,8.09,4.32e-05,68.3,68.3,5.4,3474.18,42258.96,44002.0
+14,58700000.0,8690.0,483000.0,7540.0,1020.0,7.69,3.84e-05,68.4,68.4,4.65,3474.18,42258.96,44002.0
+15,62900000.0,8680.0,483000.0,7550.0,1020.0,7.51,3.37e-05,68.5,68.5,7.06,3474.18,42258.96,44002.0
+16,67099999.99999999,8640.0,486000.0,7590.0,1020.0,7.4,2.89e-05,68.9,68.9,7.08,3474.18,42258.96,44002.0
+17,71300000.0,8700.0,482000.0,7530.0,1020.0,7.35,2.42e-05,68.3,68.3,6.09,3474.18,42258.96,44002.0
+18,75500000.0,8690.0,482000.0,7540.0,1020.0,7.32,1.95e-05,68.4,68.4,6.83,3474.18,42258.96,44002.0
+19,79700000.0,8690.0,483000.0,7540.0,1020.0,7.21,1.47e-05,68.4,68.4,5.56,3474.18,42258.96,44002.0
+20,83900000.0,8700.0,482000.0,7530.0,1020.0,7.11,1e-05,68.3,68.3,4.28,,,

llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/profiler.csv

@@ -0,0 +1,2 @@
+forward,backward
+1ms 91μs,0ms 406μs

llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/profiler/ip-26-0-160-225_1769215.1719966492698530506.pt.trace.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88902d02a23b63f84c2179728c3aa07a1a08929c528e4886cbc8bf752841af51
+size 635836966

llama-1B/64_GPUS/dp-32_tp-2_pp-1_mbz-8/status.txt

@@ -0,0 +1 @@
+completed