Upload llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2

llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/bench.slurm

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+#SBATCH --job-name=bench_cluster
+#SBATCH --time=00:59:00
+#SBATCH --partition=hopper-prod
+#SBATCH --nodes=2
+#SBATCH --gres=gpu:8
+#SBATCH --qos=high
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=96
+#SBATCH --exclusive
+#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/log.out
+#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/log.out
+
+# Function to update status based on squeue output
+update_status() {
+    job_id=$1
+    status_file=$2
+    # For unknown reasons, it doenst update status for pending. It only works for running 
+    while true; do
+        job_status=$(squeue --job $job_id --noheader --format=%T)
+        echo "Job status: $job_status"
+        if [ -z "$job_status" ]; then
+            # Job has finished or is not found
+            break
+        elif [ "$job_status" = "RUNNING" ]; then
+            printf "running" > $status_file
+            break
+        fi
+        sleep 10
+    done
+}
+
+# Misc initializations.
+echo "========================"
+echo "START TIME: $(date)"
+source /fsx/ferdinandmom/miniforge3/etc/profile.d/conda.sh
+conda activate /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster
+echo python3 version = $(python3 --version)
+echo "========================"
+
+# Slurm stuff
+export HOSTNAMES=$(scontrol show hostnames "$SLURM_JOB_NODELIST")
+export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
+export MASTER_PORT=$((1024 + RANDOM % 64511))
+
+export TMPDIR=/scratch
+export HF_DATASETS_CACHE="/admin/home/ferdinand_mom/.cache"
+export CUBLAS_WORKSPACE_CONFIG=":4096:8"
+export CUDA_DEVICE_MAX_CONNECTIONS="1"
+
+huggingface-cli login --token $HUGGINGFACE_TOKEN
+
+
+NANOTRON_REPO="/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron"
+CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/config.yaml"
+
+LAUNCHER="torchrun \
+   --nproc_per_node 8 \
+   --nnodes 2 \
+   --rdzv_endpoint ${MASTER_ADDR}:${MASTER_PORT} \
+   --rdzv_backend c10d \
+   --max_restarts 0 \
+   --tee 3 \
+   --node_rank ${SLURM_PROCID}"
+
+# Checkout the bench_cluster branch
+cd $NANOTRON_REPO
+git checkout bench_cluster
+cd ..
+# Get the current job ID
+job_id=${SLURM_JOB_ID}
+
+# Update status to "pending" or "running" in the background
+update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/status.txt &
+
+# Run the main command
+srun -u $LAUNCHER $CMD
+exit_status=$?
+
+# Update status based on the exit status of `srun`
+if [ $exit_status -eq 0 ]; then
+    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/status.txt
+else
+    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/log.out; then
+        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/status.txt
+    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/log.out; then
+        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/status.txt
+    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/log.out; then
+        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/status.txt
+    else
+        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/status.txt
+    fi 
+fi
+
+# Run the report script if the job completed successfully
+if [ $exit_status -eq 0 ]; then
+    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2 --is_logs
+    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2 --is_profiler
+fi
+
+
+# Push to hub the folder using huggingface_cli
+huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2 llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2 --commit-message "Upload llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2"
+
+# Verify the upload
+if [ $? -eq 0 ]; then
+    echo "Uploading to Huggingface Hub successful"
+else
+    echo "Failed to upload to Huggingface Hub"
+fi

llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/config.yaml

@@ -0,0 +1,90 @@
+general:
+  project: bench_cluster
+  seed: 42
+model:
+  ddp_bucket_cap_mb: 25
+  dtype: bfloat16
+  init_method:
+    std: 0.025
+  make_vocab_size_divisible_by: 1
+  model_config:
+    bos_token_id: 1
+    eos_token_id: 2
+    hidden_act: silu
+    hidden_size: 2048
+    initializer_range: 0.02
+    intermediate_size: 4096
+    is_llama_config: true
+    max_position_embeddings: 4096
+    num_attention_heads: 32
+    num_hidden_layers: 24
+    num_key_value_heads: 32
+    pad_token_id: null
+    pretraining_tp: 1
+    rms_norm_eps: 1.0e-05
+    rope_scaling: null
+    rope_theta: 10000.0
+    tie_word_embeddings: true
+    use_cache: true
+    vocab_size: 50257
+optimizer:
+  accumulate_grad_in_fp32: true
+  clip_grad: 1.0
+  learning_rate_scheduler:
+    learning_rate: 0.0001
+    lr_decay_style: linear
+    lr_warmup_style: linear
+    lr_warmup_steps: 1
+    min_decay_lr: 1.0e-05
+  optimizer_factory:
+    adam_beta1: 0.9
+    adam_beta2: 0.95
+    adam_eps: 1.0e-08
+    name: adamW
+    torch_adam_is_fused: true
+  weight_decay: 0.01
+  zero_stage: 1
+parallelism:
+  dp: 16
+  expert_parallel_size: 1
+  pp: 1
+  pp_engine: 1f1b
+  tp: 1
+  tp_linear_async_communication: false
+  tp_mode: REDUCE_SCATTER
+profiler:
+  profiler_export_path: /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2
+tokenizer:
+  tokenizer_max_length: null
+  tokenizer_name_or_path: openai-community/gpt2
+  tokenizer_revision: null
+data_stages:
+- name: Training Stage
+  start_training_step: 1
+  data:
+    dataset:
+      dataset_overwrite_cache: false
+      dataset_processing_num_proc_per_process: 64
+      hf_dataset_config_name: null
+      hf_dataset_or_datasets: roneneldan/TinyStories
+      hf_dataset_splits: train
+      text_column_name: text
+    num_loading_workers: 32
+    seed: 42
+lighteval: null
+tokens:
+  train_steps: 20
+  val_check_interval: -1
+  batch_accumulation_per_replica: 32
+  limit_test_batches: 0
+  limit_val_batches: 0
+  micro_batch_size: 2
+  sequence_length: 4096
+logging:
+  iteration_step_info_interval: 1
+  log_level: info
+  log_level_replica: info
+checkpoints:
+  checkpoint_interval: 100000
+  checkpoints_path: /dev/null
+  resume_checkpoint_path: null

llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/log.out

@@ -0,0 +1,133 @@
+========================
+START TIME: Tue Jul  2 16:30:13 UTC 2024
+python3 version = Python 3.10.14
+========================
+The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.
+Token is valid (permission: write).
+Your token has been saved to /admin/home/ferdinand_mom/.cache/huggingface/token
+Login successful
+Already on 'bench_cluster'
+M	examples/config_tiny_llama.py
+M	examples/config_tiny_llama.yaml
+M	examples/train_tiny_llama.sh
+M	src/nanotron/models/llama.py
+M	src/nanotron/trainer.py
+Your branch is up to date with 'origin/bench_cluster'.
+Job status: RUNNING
+W0702 16:30:16.125000 140327896655680 torch/distributed/run.py:757] 
+W0702 16:30:16.125000 140327896655680 torch/distributed/run.py:757] *****************************************
+W0702 16:30:16.125000 140327896655680 torch/distributed/run.py:757] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0702 16:30:16.125000 140327896655680 torch/distributed/run.py:757] *****************************************
+W0702 16:30:16.135000 139775981082432 torch/distributed/run.py:757] 
+W0702 16:30:16.135000 139775981082432 torch/distributed/run.py:757] *****************************************
+W0702 16:30:16.135000 139775981082432 torch/distributed/run.py:757] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0702 16:30:16.135000 139775981082432 torch/distributed/run.py:757] *****************************************
+[default0]:[W socket.cpp:464] [c10d] The server socket has failed to bind to [::]:36391 (errno: 98 - Address already in use).
+[default0]:[W socket.cpp:464] [c10d] The server socket has failed to bind to ?UNKNOWN? (errno: 98 - Address already in use).
+[default0]:[E socket.cpp:500] [c10d] The server socket has failed to listen on any local network address.
+[default0]:Traceback (most recent call last):
+[default0]:  File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/run_train.py", line 233, in <module>
+[default0]:    trainer = DistributedTrainer(config_file)
+[default0]:  File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/trainer.py", line 145, in __init__
+[default0]:    self.parallel_context = ParallelContext(
+[default0]:  File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/parallel/context.py", line 53, in __init__
+[default0]:    dist.initialize_torch_distributed()
+[default0]:  File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/distributed.py", line 278, in initialize_torch_distributed
+[default0]:    dist.init_process_group(
+[default0]:  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/c10d_logger.py", line 75, in wrapper
+[default0]:    return func(*args, **kwargs)
+[default0]:  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/c10d_logger.py", line 89, in wrapper
+[default0]:    func_return = func(*args, **kwargs)
+[default0]:  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 1305, in init_process_group
+[default0]:    store, rank, world_size = next(rendezvous_iterator)
+[default0]:  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/rendezvous.py", line 246, in _env_rendezvous_handler
+[default0]:    store = _create_c10d_store(master_addr, master_port, rank, world_size, timeout, use_libuv)
+[default0]:  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/rendezvous.py", line 174, in _create_c10d_store
+[default0]:    return TCPStore(
+[default0]:torch.distributed.DistNetworkError: The server socket has failed to listen on any local network address. The server socket has failed to bind to [::]:36391 (errno: 98 - Address already in use). The server socket has failed to bind to ?UNKNOWN? (errno: 98 - Address already in use).
+W0702 16:30:27.302000 140327896655680 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 853935 closing signal SIGTERM
+W0702 16:30:27.303000 140327896655680 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 853936 closing signal SIGTERM
+W0702 16:30:27.303000 140327896655680 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 853937 closing signal SIGTERM
+W0702 16:30:27.303000 140327896655680 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 853938 closing signal SIGTERM
+W0702 16:30:27.303000 140327896655680 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 853939 closing signal SIGTERM
+W0702 16:30:27.303000 140327896655680 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 853940 closing signal SIGTERM
+W0702 16:30:27.304000 140327896655680 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 853941 closing signal SIGTERM
+E0702 16:30:27.706000 140327896655680 torch/distributed/elastic/multiprocessing/api.py:826] failed (exitcode: 1) local_rank: 0 (pid: 853934) of binary: /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/bin/python3.10
+Traceback (most recent call last):
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 347, in wrapper
+    return f(*args, **kwargs)
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/run.py", line 879, in main
+    run(args)
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/run.py", line 870, in run
+    elastic_launch(
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 263, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/run_train.py FAILED
+------------------------------------------------------------
+Failures:
+  <NO_OTHER_FAILURES>
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2024-07-02_16:30:27
+  host      : ip-26-0-163-43.ec2.internal
+  rank      : 0 (local_rank: 0)
+  exitcode  : 1 (pid: 853934)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
+srun: error: ip-26-0-163-43: task 0: Exited with exit code 1
+W0702 16:30:31.395000 139770314262272 torch/distributed/elastic/rendezvous/dynamic_rendezvous.py:1252] The node 'ip-26-0-169-207.ec2.internal_2422832_0' has failed to send a keep-alive heartbeat to the rendezvous 'none' due to an error of type RendezvousConnectionError.
+W0702 16:30:32.306000 139775981082432 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2422901 closing signal SIGTERM
+W0702 16:30:32.307000 139775981082432 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2422902 closing signal SIGTERM
+W0702 16:30:32.307000 139775981082432 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2422903 closing signal SIGTERM
+W0702 16:30:32.307000 139775981082432 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2422904 closing signal SIGTERM
+W0702 16:30:32.307000 139775981082432 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2422905 closing signal SIGTERM
+W0702 16:30:32.307000 139775981082432 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2422906 closing signal SIGTERM
+W0702 16:30:32.307000 139775981082432 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2422907 closing signal SIGTERM
+W0702 16:30:32.307000 139775981082432 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2422908 closing signal SIGTERM
+W0702 16:30:32.813000 139775981082432 torch/distributed/elastic/rendezvous/dynamic_rendezvous.py:1203] The node 'ip-26-0-169-207.ec2.internal_2422832_0' has failed to shutdown the rendezvous 'none' due to an error of type RendezvousConnectionError.
+W0702 16:30:32.820000 139775981082432 torch/distributed/elastic/rendezvous/dynamic_rendezvous.py:1203] The node 'ip-26-0-169-207.ec2.internal_2422832_0' has failed to shutdown the rendezvous 'none' due to an error of type RendezvousConnectionError.
+Traceback (most recent call last):
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/rendezvous/c10d_rendezvous_backend.py", line 113, in _call_store
+    return getattr(self._store, store_op)(*args, **kwargs)
+torch.distributed.DistNetworkError: Broken pipe
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 347, in wrapper
+    return f(*args, **kwargs)
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/run.py", line 879, in main
+    run(args)
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/run.py", line 870, in run
+    elastic_launch(
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 254, in launch_agent
+    result = agent.run()
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 123, in wrapper
+    result = f(*args, **kwargs)
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 733, in run
+    result = self._invoke_run(role)
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    num_nodes_waiting = rdzv_handler.num_nodes_waiting()
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/rendezvous/dynamic_rendezvous.py", line 1174, in num_nodes_waiting
+    self._state_holder.sync()
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/rendezvous/dynamic_rendezvous.py", line 419, in sync
+    get_response = self._backend.get_state()
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/rendezvous/c10d_rendezvous_backend.py", line 73, in get_state
+    base64_state: bytes = self._call_store("get", self._key)
+  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/rendezvous/c10d_rendezvous_backend.py", line 115, in _call_store
+    raise RendezvousConnectionError(
+torch.distributed.elastic.rendezvous.api.RendezvousConnectionError: The connection to the C10d store has failed. See inner exception for details.
+srun: error: ip-26-0-169-207: task 1: Exited with exit code 1
+Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See https://huggingface.co/docs/huggingface_hub/hf_transfer for more details.

llama-1B/16_GPUS/dp-16_tp-1_pp-1_mbz-2/status.txt

@@ -0,0 +1 @@
+fail