7b

c4_original-open_lm_7b-1.0/checkpoints/epoch_17.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b04707ca718e40d012b9bb43b38c01b2f5684884d81144a744a4eaf7df43e138
+size 27560991570

c4_original-open_lm_7b-1.0/params.txt

@@ -0,0 +1,123 @@
+accum_freq: 1
+attn_activation: None
+attn_name: auto
+attn_seq_scalar: None
+attn_seq_scalar_alpha: None
+average: None
+average_coefficients: None
+beta1: 0.9
+beta2: 0.95
+checkpoint_path: /admin/home-sy/dcnlp_logs/c4_original-open_lm_7b-1.0/checkpoints
+copy_codebase: False
+data_key: txt
+dataset_manifest: None
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+delete_previous_checkpoint: True
+device: cuda:0
+disable_buffer: False
+dist_backend: nccl
+dist_url: env://
+distill_model: None
+distill_pretrained: None
+distributed: False
+epochs: 5
+epochs_cooldown: None
+eps: 1e-08
+experimental_meta_device: False
+ffn_type: swiglu
+force_distributed: False
+force_min_lr: 0.0
+fsdp: False
+fsdp_amp: False
+fsdp_backward_prefetch: False
+fsdp_checkpoint: False
+fsdp_cpu_offload: False
+fsdp_hybrid: False
+fsdp_hybrid_o2: False
+fsdp_limit_all_gathers: False
+fsdp_pure_bf16: False
+fsdp_use_orig_params: False
+global_batch_size: 1
+global_val_batch_size: 1
+grad_checkpointing: False
+grad_clip_norm: 1.0
+hf_fsdp_block: None
+hf_model: None
+hf_seq_len: None
+ignore_parse_errors: False
+load_pretrained_state: False
+local_rank: 0
+log_every_n_steps: 20
+log_level: 20
+log_local: False
+log_logit_mean: False
+log_path: /admin/home-sy/dcnlp_logs/c4_original-open_lm_7b-1.0/out.log
+logs: /admin/home-sy/dcnlp_logs
+lr: 0.0003
+lr_cooldown_end: 3e-05
+lr_cooldown_power: 1.0
+lr_scheduler: cosine
+model: open_lm_7b
+model_norm: lp_layer_norm
+moe_capacity_factor: 1.25
+moe_expert_model_parallelism: False
+moe_freq: 0
+moe_loss_weight: 0.1
+moe_num_experts: None
+moe_top_k: 2
+moe_weight_parallelism: False
+multiple_data_passes: False
+name: c4_original-open_lm_7b-1.0
+no_set_device_rank: False
+optimizer: adamw
+per_gpu_batch_size: 1
+per_gpu_val_batch_size: 1
+positional_embedding_type: rotary
+precision: amp_bfloat16
+pretrained: None
+qk_norm: True
+rank: 0
+remote_sync: s3://dcnlp-west/dcnlp_experiments_v3
+remote_sync_frequency: 300
+remote_sync_protocol: s3
+report_to: 
+resume: s3://dcnlp-west/dcnlp_experiments_v3/c4_original-open_lm_7b-1.0/checkpoints/epoch_17.pt
+save_frequency: 1
+save_most_recent: False
+seed: 124
+seq_len: 2048
+skip_scheduler: False
+squash_mask_left: True
+target_mask_individual: 50400
+target_mask_left: 50300
+tensorboard: False
+tensorboard_path: 
+torchcompile: False
+torchscript: False
+trace: False
+train_data: None
+train_data_mix_weights: None
+train_data_upsampling_factors: None
+train_num_samples: None
+use_bn_sync: False
+use_bnb_linear: None
+val_data: ['training/eval_data/val_tok_mult/de-en/val_de-en_000.tar', 'training/eval_data/val_tok_mult/de-en/val_de-en_010.tar', 'training/eval_data/val_tok_mult/de-en/val_de-en_020.tar', 'training/eval_data/val_tok_mult/de-en/val_de-en_030.tar', 'training/eval_data/val_tok_mult/de-en/val_de-en_040.tar', 'training/eval_data/val_tok_mult/de-en/val_de-en_050.tar', 'training/eval_data/val_tok_mult/de-en/val_de-en_060.tar', 'training/eval_data/val_tok_mult/de-en/val_de-en_070.tar', 'training/eval_data/val_tok_mult/de-en/val_de-en_080.tar', 'training/eval_data/val_tok_mult/de-en/val_de-en_090.tar', 'training/eval_data/val_tok_mult/de-en/val_de-en_100.tar']
+val_data_key: ['json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz']
+val_frequency: 5
+val_iter_ci: 10000
+val_max_pop_ci: 300000
+val_num_samples: None
+val_seq_ci: True
+val_tok_ci: True
+vocab_size: 50432
+wandb: False
+wandb_notes: 
+wandb_project_name: open-lm
+warmup: 5000
+wd: 0.33
+workers: 2
+world_size: 1
+z_loss_coefficient: 0.0001

rpj-open_lm_7b-1.0/checkpoints/epoch_39.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1c1a2008f6f48a3a406b047a4f3ba689fe4c00c50362477c416c2807cdca19f
+size 27560991506

rpj-open_lm_7b-1.0/params.txt

@@ -0,0 +1,100 @@
+accum_freq: 4
+average: None
+average_coefficients: None
+batch_size: 16
+beta1: 0.9
+beta2: 0.95
+checkpoint_path: ./logs/vaishaal_open_lm_7b_cc1_without_replacement_137B_tokens_h100_rpj_only/checkpoints
+copy_codebase: False
+data_key: json
+dataset_manifest: ['s3://permanent-813987666268/users/vaishaal/mlr/open_lm/rpj_tokenized_upsampled_eleutherai/manifest.jsonl']
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+delete_previous_checkpoint: False
+device: cuda:0
+disable_buffer: False
+dist_backend: nccl
+dist_url: env://
+distill_model: None
+distill_pretrained: None
+distributed: True
+epochs: 64
+epochs_cooldown: None
+eps: 1e-08
+ffn_type: swiglu
+force_min_lr: 0.0
+fsdp: True
+fsdp_amp: False
+fsdp_backward_prefetch: False
+fsdp_checkpoint: False
+fsdp_cpu_offload: False
+fsdp_hybrid: False
+fsdp_hybrid_o2: False
+fsdp_limit_all_gathers: True
+fsdp_pure_bf16: True
+fsdp_use_orig_params: False
+grad_checkpointing: False
+grad_clip_norm: 1.0
+hf_fsdp_block: None
+hf_model: None
+hf_seq_len: None
+ignore_parse_errors: True
+load_pretrained_state: False
+local_rank: 0
+log_every_n_steps: 20
+log_level: 20
+log_local: False
+log_logit_mean: False
+log_path: ./logs/vaishaal_open_lm_7b_cc1_without_replacement_137B_tokens_h100_rpj_only/out.log
+logs: ./logs/
+lr: 0.0003
+lr_cooldown_end: 3e-05
+lr_cooldown_power: 1.0
+lr_scheduler: cosine
+model: open_lm_7b
+model_norm: lp_layer_norm
+name: vaishaal_open_lm_7b_cc1_without_replacement_137B_tokens_h100_rpj_only
+no_set_device_rank: False
+optimizer: adamw
+positional_embedding_type: rotary
+precision: amp_bfloat16
+pretrained: None
+qk_norm: True
+rank: 0
+remote_sync: s3://permanent-813987666268/users/vaishaal/mlr/open_lm/checkpoints
+remote_sync_frequency: 300
+remote_sync_protocol: s3
+report_to: wandb
+resume: s3://permanent-813987666268/users/vaishaal/mlr/open_lm/checkpoints/vaishaal_open_lm_7b_cc1_without_replacement_137B_tokens_h100_rpj_only/checkpoints/epoch_23.pt
+save_frequency: 1
+save_most_recent: False
+seed: 124
+seq_len: 2048
+skip_scheduler: False
+target_mask_individual: None
+target_mask_left: None
+tensorboard: False
+tensorboard_path: 
+torchcompile: False
+torchscript: False
+trace: False
+train_data: None
+train_data_mix_weights: None
+train_data_upsampling_factors: None
+train_num_samples: 1052856
+use_bn_sync: False
+use_bnb_linear: None
+val_data: None
+val_frequency: 1
+val_num_samples: None
+vocab_size: 50432
+wandb: True
+wandb_notes: 
+wandb_project_name: open_lm
+warmup: 5000
+wd: 0.1
+workers: 4
+world_size: 64
+z_loss_coefficient: 0.0001

rw_original-open_lm_7b-1.0/checkpoints/epoch_47.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169754a84143f9b9956684e0596a67a8379576beee50e186220abe277d0dd422
+size 27560991506

rw_original-open_lm_7b-1.0/params.txt

@@ -0,0 +1,102 @@
+accum_freq: 4
+average: None
+average_coefficients: None
+batch_size: 16
+beta1: 0.9
+beta2: 0.95
+checkpoint_path: ./logs/refined_web_7b_CC1_experiment_16_nodes/checkpoints
+copy_codebase: False
+data_key: json.gz
+dataset_manifest: ['s3://permanent-813987666268/users/vaishaal/mlr/refined_web_tokenized/manifest.jsonl']
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+delete_previous_checkpoint: False
+device: cuda:0
+disable_buffer: False
+dist_backend: nccl
+dist_url: env://
+distill_model: None
+distill_pretrained: None
+distributed: True
+epochs: 64
+epochs_cooldown: None
+eps: 1e-08
+ffn_type: swiglu
+force_min_lr: 0.0
+fsdp: True
+fsdp_amp: False
+fsdp_backward_prefetch: False
+fsdp_checkpoint: False
+fsdp_cpu_offload: False
+fsdp_hybrid: False
+fsdp_hybrid_o2: False
+fsdp_limit_all_gathers: True
+fsdp_pure_bf16: True
+fsdp_use_orig_params: False
+grad_checkpointing: False
+grad_clip_norm: 1.0
+hf_fsdp_block: None
+hf_model: None
+hf_seq_len: None
+ignore_parse_errors: True
+load_pretrained_state: False
+local_rank: 0
+log_every_n_steps: 20
+log_level: 20
+log_local: False
+log_logit_mean: False
+log_path: ./logs/refined_web_7b_CC1_experiment_16_nodes/out.log
+logs: ./logs/
+lr: 0.0003
+lr_cooldown_end: 3e-05
+lr_cooldown_power: 1.0
+lr_scheduler: cosine
+model: open_lm_7b
+model_norm: lp_layer_norm
+name: refined_web_7b_CC1_experiment_16_nodes
+no_set_device_rank: False
+optimizer: adamw
+positional_embedding_type: rotary
+precision: amp_bfloat16
+pretrained: None
+qk_norm: True
+rank: 0
+remote_sync: s3://permanent-813987666268/users/vaishaal/mlr/open_lm/checkpoints
+remote_sync_frequency: 300
+remote_sync_protocol: s3
+report_to: wandb
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 124
+seq_len: 2048
+skip_scheduler: False
+target_mask_individual: None
+target_mask_left: None
+tensorboard: False
+tensorboard_path: 
+torchcompile: False
+torchscript: False
+trace: False
+train_data: None
+train_data_mix_weights: None
+train_data_upsampling_factors: None
+train_num_samples: 1052856
+use_bn_sync: False
+use_bnb_linear: None
+val_batch_size: None
+val_data: None
+val_data_key: txt
+val_frequency: 1
+val_num_samples: None
+vocab_size: 50432
+wandb: True
+wandb_notes: 
+wandb_project_name: open_lm
+warmup: 5000
+wd: 0.1
+workers: 4
+world_size: 128
+z_loss_coefficient: 0.0001