| { | |
| "version": ".unknown.", | |
| "runner": { | |
| "runner_type": "pdsh", | |
| "hostsfile": null, | |
| "hosts": null, | |
| "master_port": 29500, | |
| "master_addr": null, | |
| "script": "src/scaling/transformer/train.py", | |
| "default_gpu_count": 8, | |
| "docker_config": { | |
| "docker_container": null, | |
| "docker_sudo": false, | |
| "docker_mounts": [ | |
| [ | |
| "/mnt/", | |
| "/mnt/" | |
| ] | |
| ] | |
| }, | |
| "use_determined": true | |
| }, | |
| "logger": { | |
| "log_level": "info", | |
| "log_dir": null, | |
| "metrics_ranks": null, | |
| "use_wandb": true, | |
| "wandb_ranks": null, | |
| "wandb_host": "https://api.wandb.ai", | |
| "wandb_team": "aleph-alpha", | |
| "wandb_project": "ng-semantic-embedding", | |
| "wandb_group": "pharia_7B_embedding_all_source_256-2024-10-01-19-28-52", | |
| "wandb_api_key": "d80060c84c496915fcad15283809dbde65087d73", | |
| "use_tensorboard": false, | |
| "tensorboard_ranks": null, | |
| "determined_metrics_ranks": null | |
| }, | |
| "topology": { | |
| "global_rank": 0, | |
| "world_size": 256, | |
| "local_slot": 0, | |
| "model_parallel_size": 1, | |
| "pipe_parallel_size": 1, | |
| "data_parallel_size": 256, | |
| "global_batch_size": 2048, | |
| "micro_batch_size": 8, | |
| "gradient_accumulation_steps": 1, | |
| "pipe_partition_method": "balanced", | |
| "pipe_partition_overwrite": null, | |
| "activation_checkpointing_type": "every_layer", | |
| "sequence_parallel": false | |
| }, | |
| "optimizer": { | |
| "method": "adamw", | |
| "beta1": 0.9, | |
| "beta2": 0.95, | |
| "eps": 1e-15, | |
| "gradient_clipping": 0.0, | |
| "allreduce_bucket_size": 500000000, | |
| "loss_scaler": { | |
| "enable": false, | |
| "initial_scale": 4294967296.0, | |
| "window": 1000, | |
| "hysteresis": 2.0, | |
| "consecutive_hysteresis": false, | |
| "min_scale": 1.0, | |
| "factor": 2.0 | |
| }, | |
| "zero": true, | |
| "zero_save_static": false, | |
| "debug_log": false | |
| }, | |
| "learning_rate_scheduler": { | |
| "learning_rate": 2e-05, | |
| "learning_rate_minimum": 0.0, | |
| "learning_rate_decay_style": "linear", | |
| "learning_rate_decay_iters": 938, | |
| "learning_rate_warmup_steps": 50 | |
| }, | |
| "embedding_learning_rate_scheduler": { | |
| "learning_rate": 0.0, | |
| "learning_rate_minimum": 0.0, | |
| "learning_rate_decay_style": "cosine", | |
| "learning_rate_decay_iters": 0, | |
| "learning_rate_warmup_steps": 0 | |
| }, | |
| "training": { | |
| "weight_decay": 0.0001, | |
| "finetune": true, | |
| "finetunable_parameters": [ | |
| "embed" | |
| ], | |
| "parameters_exclude": [], | |
| "use_separate_lr_on_embeddings": false, | |
| "use_deterministic_torch_algorithms": false, | |
| "loss_function_config": { | |
| "loss_type": "contrastive_loss", | |
| "number_of_hard_negatives": 1, | |
| "use_instructions": true, | |
| "query_side_only": false, | |
| "scale": 50, | |
| "log_verbose_metrics": true | |
| } | |
| }, | |
| "trainer": { | |
| "save_dir": "checkpoints", | |
| "save_interval": 938, | |
| "load_dir": "/scratch/samuel/Pharia-1-LLM-7B-control", | |
| "train_iterations": 938, | |
| "assert_checkpoint_loaded": true, | |
| "load_optimizer_states": false, | |
| "delete_past_optimizer_states": true, | |
| "load_context": false, | |
| "allowed_missing_keys_in_checkpoint": [ | |
| "embed" | |
| ], | |
| "allowed_unexpected_keys_in_checkpoint": null, | |
| "ignore_keys_in_checkpoint": null, | |
| "merge_lora_after_loading_checkpoint": false, | |
| "seed": 42, | |
| "dataloader_num_workers": 0, | |
| "dataloader_pin_memory": true, | |
| "dataloader_prefetch_factor": null, | |
| "eval_iterations": 1, | |
| "eval_interval": null, | |
| "separate_file_for_parameters": [ | |
| "adapter_embed" | |
| ] | |
| }, | |
| "profiler": { | |
| "profile_steps": 0, | |
| "profile_start_at_step": 0, | |
| "profiler_output": null | |
| }, | |
| "transformer_architecture": { | |
| "vocab_size": 128000, | |
| "vocab_file": "/scratch/samuel/Pharia-1-LLM-7B-control/vocab.json", | |
| "hidden_size": 4608, | |
| "num_layers": 27, | |
| "num_attention_heads": 36, | |
| "num_local_attention_heads": 0, | |
| "local_attention_window_size": null, | |
| "rotary_embedding_base": 1000000, | |
| "rotary_percentage": 1.0, | |
| "sequence_length": 2048, | |
| "norm_type": "layernorm", | |
| "relative_position_embedding_type": "rotary_complex", | |
| "mlp_type": "default", | |
| "mlp_factor": 4.0, | |
| "attention_bias": true, | |
| "attention_qkv_in_one": false, | |
| "attention_num_kv_heads": 4, | |
| "attention_use_matmul": false, | |
| "mlp_bias": true, | |
| "key_query_norm": false, | |
| "weight_tying": false, | |
| "masked_softmax": { | |
| "kernel": "torch", | |
| "softmax_in_fp32": true, | |
| "scale": 1.0, | |
| "deterministic_flash_attn_bwd": false | |
| }, | |
| "layernorm": { | |
| "optimization_type": "torch", | |
| "layernorm_epsilon": 1e-05 | |
| }, | |
| "precision": "bfloat16", | |
| "dropout_embedding": 0.0, | |
| "dropout_attention_probs": 0.0, | |
| "dropout_after_attention": 0.0, | |
| "dropout_after_mlp": 0.0, | |
| "bitfit_bias_config": null, | |
| "finetunable_token_ids": [], | |
| "image_encoder": false, | |
| "dropout_image_encoder": 0.0, | |
| "softprompt_config": null, | |
| "adapter_config": { | |
| "name": "embed", | |
| "attention_downsampling_factor": 0.25, | |
| "mlp_downsampling_factor": 0.25, | |
| "init_std": 1e-05, | |
| "version": ".unknown." | |
| }, | |
| "lora_config": null, | |
| "embedding_head_config": { | |
| "name": "pooling_only", | |
| "proj_layers": [ | |
| 256 | |
| ], | |
| "pooling": "weighted_mean" | |
| }, | |
| "lm_head": false, | |
| "causal": true | |
| }, | |
| "data": { | |
| "legacy_dataset": false, | |
| "load_mmap_index_to_memory": false, | |
| "use_mmap": false, | |
| "load_data_item_mmap_index_to_memory": false, | |
| "finetuning_dataset": false, | |
| "finetuning_chat_dataset": false, | |
| "finetuning_dataset_memory_map": false, | |
| "embedding_dataset": true, | |
| "embedding_dataset_memory_map": false, | |
| "data_prefixes": [ | |
| "/scratch/samuel/all_source_data_templated/allnli_eng.jsonl", | |
| "/scratch/samuel/all_source_data_templated/hotpot_with_bm25_negatives.jsonl", | |
| "/scratch/samuel/all_source_data_templated/medi_1.jsonl", | |
| "/scratch/samuel/all_source_data_templated/msmarco.jsonl" | |
| ], | |
| "validation_data_prefixes": null, | |
| "blended_dataset": { | |
| "weight_by_num_documents": true, | |
| "weighted_sampler_alpha": 1.0, | |
| "weights": null, | |
| "weight_examples_proportional": false, | |
| "ep_maximum": null, | |
| "ep_temperature": 1.0, | |
| "minimum_dataset_size": 0, | |
| "cache_directory": "/scratch/samuel/blended_dataset_cache", | |
| "shuffle_dataset_indices": false, | |
| "load_dataset_indices_to_memory": false | |
| }, | |
| "only_full_sequences": false, | |
| "allow_incomplete_sequences_every_n": 0 | |
| }, | |
| "determined_experiment_id": 2965, | |
| "determined_trial_id": 86040 | |
| } |