PIPELINE: 'satvisiontoapretrain'

MODEL:
  TYPE: swinv2
  NAME: mim_satvision_pretrain-giant
  DROP_PATH_RATE: 0.1
  SWINV2:
    IN_CHANS: 14
    EMBED_DIM: 512
    DEPTHS: [ 2, 2, 42, 2 ]
    NUM_HEADS: [ 16, 32, 64, 128 ]
    WINDOW_SIZE: 8
    NORM_PERIOD: 6

DATA:
  DATAMODULE: False
  BATCH_SIZE: 64 
  LENGTH: 1_920_000
  PIN_MEMORY: True
  NUM_WORKERS: 4
  DATA_PATHS: [/explore/nobackup/projects/ilab/projects/3DClouds/data/mosaic-v3/webdatasets/shards]
  IMG_SIZE: 128
  MASK_PATCH_SIZE: 8
  MASK_RATIO: 0.6

TRAIN:
  ACCELERATOR: 'gpu'
  STRATEGY: 'deepspeed'
  USE_CHECKPOINT: True
  EPOCHS: 50 
  WARMUP_EPOCHS: 10
  BASE_LR: 3e-4
  MIN_LR: 2e-4
  WARMUP_LR: 1e-4
  WEIGHT_DECAY: 0.05
  LR_SCHEDULER:
    NAME: 'multistep'
    GAMMA: 0.1
    MULTISTEPS: [700,]

DEEPSPEED:
  STAGE: 2

PRECISION: 'bf16'

PRINT_FREQ: 10 
SAVE_FREQ: 50
TAG: mim_pretrain_giant_satvision_128_scaled_bt_minmax_50ep