_wandb:
    value:
        cli_version: 0.18.1
        m: []
        python_version: 3.11.10
        t:
            "1":
                - 1
                - 11
                - 41
                - 49
                - 50
                - 51
                - 55
                - 71
                - 100
            "2":
                - 1
                - 11
                - 41
                - 49
                - 50
                - 51
                - 55
                - 71
                - 100
            "3":
                - 2
                - 15
                - 16
                - 23
                - 55
                - 61
            "4": 3.11.10
            "5": 0.18.1
            "6": 4.44.2
            "8":
                - 5
            "12": 0.18.1
            "13": linux-x86_64
checkpoint:
    value:
        every_steps: 2500
data:
    value:
        before_mask_input_length: 1137
        input_length: 1024
        mean_noise_span_length: 3
        mlm_probability: 0.15
        num_workers: 16
        target_length: 229
device:
    value: gpu
eval:
    value:
        corrected_steps: 500
        every_steps: 1000000000
        steps: 500
eval_only:
    value: false
logging:
    value:
        every_steps: 25
        grad_l2: true
        use_wandb: true
        wandb_config:
            entity: pszemraj
            mode: online
            project: nanoT5
            tags:
                - 24x24
                - "1024"
        weights_l2: true
mode:
    value: pt
model:
    value:
        checkpoint_path: ""
        compile: true
        klass: hf_t5
        name: pszemraj/tFINE-850m-24x24-512ctx
        overwrite:
            dropout_rate: 0
        random_init: false
n_all_param:
    value: 853929472
optim:
    value:
        base_lr: 0.01
        batch_size: 128
        epochs: -1
        final_cosine: 2e-05
        grad_acc: 8
        grad_clip: 1
        lr_scheduler: cosine
        name: adamwscale
        total_steps: 20000
        warmup_steps: 5000
        weight_decay: 0
precision:
    value: bf16
predict_only:
    value: false
seed:
    value: 34534
slurm_id:
    value: none
tokenizer:
    value:
        name: BEE-spoke-data/slimpajama_tok-48128-BPE-forT5
working_dir:
    value: /workspace/nanoT5/outputs/2024-09-26/05-19-51