config.yaml

callbacks:
  rollout_lh:
    tasks:
      _target_: calvin_env.envs.tasks.Tasks
      tasks:
        rotate_red_block_right:
        - rotate_object
        - block_red
        - -60
        rotate_red_block_left:
        - rotate_object
        - block_red
        - 60
        rotate_blue_block_right:
        - rotate_object
        - block_blue
        - -60
        rotate_blue_block_left:
        - rotate_object
        - block_blue
        - 60
        rotate_pink_block_right:
        - rotate_object
        - block_pink
        - -60
        rotate_pink_block_left:
        - rotate_object
        - block_pink
        - 60
        push_red_block_right:
        - push_object
        - block_red
        - 0.1
        - 0
        push_red_block_left:
        - push_object
        - block_red
        - -0.1
        - 0
        push_blue_block_right:
        - push_object
        - block_blue
        - 0.1
        - 0
        push_blue_block_left:
        - push_object
        - block_blue
        - -0.1
        - 0
        push_pink_block_right:
        - push_object
        - block_pink
        - 0.1
        - 0
        push_pink_block_left:
        - push_object
        - block_pink
        - -0.1
        - 0
        move_slider_left:
        - move_door_rel
        - base__slide
        - 0.15
        move_slider_right:
        - move_door_rel
        - base__slide
        - -0.15
        open_drawer:
        - move_door_rel
        - base__drawer
        - 0.12
        close_drawer:
        - move_door_rel
        - base__drawer
        - -0.12
        lift_red_block_table:
        - lift_object
        - block_red
        - 0.05
        - table
        - base_link
        lift_red_block_slider:
        - lift_object
        - block_red
        - 0.03
        - table
        - plank_link
        lift_red_block_drawer:
        - lift_object
        - block_red
        - 0.05
        - table
        - drawer_link
        lift_blue_block_table:
        - lift_object
        - block_blue
        - 0.05
        - table
        - base_link
        lift_blue_block_slider:
        - lift_object
        - block_blue
        - 0.03
        - table
        - plank_link
        lift_blue_block_drawer:
        - lift_object
        - block_blue
        - 0.05
        - table
        - drawer_link
        lift_pink_block_table:
        - lift_object
        - block_pink
        - 0.05
        - table
        - base_link
        lift_pink_block_slider:
        - lift_object
        - block_pink
        - 0.03
        - table
        - plank_link
        lift_pink_block_drawer:
        - lift_object
        - block_pink
        - 0.05
        - table
        - drawer_link
        place_in_slider:
        - place_object
        - table
        - plank_link
        place_in_drawer:
        - place_object
        - table
        - drawer_link
        stack_block:
        - stack_objects
        unstack_block:
        - unstack_objects
        turn_on_lightbulb:
        - toggle_light
        - lightbulb
        - 0
        - 1
        turn_off_lightbulb:
        - toggle_light
        - lightbulb
        - 1
        - 0
        turn_on_led:
        - toggle_light
        - led
        - 0
        - 1
        turn_off_led:
        - toggle_light
        - led
        - 1
        - 0
        push_into_drawer:
        - push_object_into
        - - block_red
          - block_blue
          - block_pink
        - table
        - base_link
        - table
        - drawer_link
    val_annotations:
      rotate_red_block_right:
      - take the red block and rotate it to the right
      rotate_red_block_left:
      - take the red block and rotate it to the left
      rotate_blue_block_right:
      - take the blue block and rotate it to the right
      rotate_blue_block_left:
      - take the blue block and rotate it to the left
      rotate_pink_block_right:
      - take the pink block and rotate it to the right
      rotate_pink_block_left:
      - take the pink block and rotate it to the left
      push_red_block_right:
      - go push the red block right
      push_red_block_left:
      - go push the red block left
      push_blue_block_right:
      - go push the blue block right
      push_blue_block_left:
      - go push the blue block left
      push_pink_block_right:
      - go push the pink block right
      push_pink_block_left:
      - go push the pink block left
      move_slider_left:
      - push the sliding door to the left side
      move_slider_right:
      - push the sliding door to the right side
      open_drawer:
      - pull the handle to open the drawer
      close_drawer:
      - push the handle to close the drawer
      lift_red_block_table:
      - grasp and lift the red block
      lift_blue_block_table:
      - grasp and lift the blue block
      lift_pink_block_table:
      - grasp and lift the pink block
      lift_red_block_slider:
      - lift the red block from the sliding cabinet
      lift_blue_block_slider:
      - lift the blue block from the sliding cabinet
      lift_pink_block_slider:
      - lift the pink block from the sliding cabinet
      lift_red_block_drawer:
      - Take the red block from the drawer
      lift_blue_block_drawer:
      - Take the blue block from the drawer
      lift_pink_block_drawer:
      - Take the pink block from the drawer
      place_in_slider:
      - store the grasped block in the sliding cabinet
      place_in_drawer:
      - store the grasped block in the drawer
      push_into_drawer:
      - slide the block that it falls into the drawer
      stack_block:
      - stack the grasped block
      unstack_block:
      - remove the stacked block
      turn_on_lightbulb:
      - use the switch to turn on the light bulb
      turn_off_lightbulb:
      - use the switch to turn off the light bulb
      turn_on_led:
      - press the button to turn on the led light
      turn_off_led:
      - press the button to turn off the led light
    _target_: mode.rollout.rollout_long_horizon.RolloutLongHorizon
    _recursive_: false
    env_cfg:
      _target_: mode.wrappers.hulc_wrapper.HulcWrapper
    skip_epochs: ${rollout_lh_skip_epochs}
    rollout_freq: 5
    num_videos: 0
    num_sequences: 1000
    replan_freq: 30
    ep_len: 360
    empty_cache: false
    log_video_to_file: false
    save_dir: ./videos
    lang_folder: ${lang_folder}
    debug: false
  ema:
    _target_: mode.callbacks.ema.EMA
    decay: 0.999
    start_step: 0
    save_ema_weights_in_callback_state: true
    evaluate_ema_weights_instead: true
    power: 0.6666666666666666
    inv_gamma: 1.0
    min_value: 0.0
    max_value: 0.9999
  checkpoint:
    _target_: pytorch_lightning.callbacks.ModelCheckpoint
    save_top_k: 1
    verbose: true
    monitor: eval_lh/avg_seq_len
    mode: max
    dirpath: saved_models
    filename: '{epoch:02d}_{eval_lh/avg_seq_len:.2f}'
    every_n_epochs: ${callbacks.rollout_lh.rollout_freq}
datamodule:
  transforms:
    train:
      rgb_static:
      - _target_: torchvision.transforms.Resize
        size: 224
        antialias: true
      - _target_: mode.utils.transforms.RandomShiftsAug
        pad: 10
      - _target_: mode.utils.transforms.ScaleImageTensor
      - _target_: torchvision.transforms.Normalize
        mean:
        - 0.48145466
        - 0.4578275
        - 0.40821073
        std:
        - 0.26862954
        - 0.26130258
        - 0.27577711
      rgb_gripper:
      - _target_: torchvision.transforms.Resize
        size: 112
        antialias: true
      - _target_: mode.utils.transforms.RandomShiftsAug
        pad: 4
      - _target_: mode.utils.transforms.ScaleImageTensor
      - _target_: torchvision.transforms.Normalize
        mean:
        - 0.48145466
        - 0.4578275
        - 0.40821073
        std:
        - 0.26862954
        - 0.26130258
        - 0.27577711
      robot_obs:
      - _target_: mode.utils.transforms.NormalizeVector
      scene_obs:
      - _target_: mode.utils.transforms.NormalizeVector
    val:
      rgb_static:
      - _target_: torchvision.transforms.Resize
        size: 224
        antialias: true
      - _target_: mode.utils.transforms.ScaleImageTensor
      - _target_: torchvision.transforms.Normalize
        mean:
        - 0.48145466
        - 0.4578275
        - 0.40821073
        std:
        - 0.26862954
        - 0.26130258
        - 0.27577711
      rgb_gripper:
      - _target_: torchvision.transforms.Resize
        size: 112
        antialias: true
      - _target_: mode.utils.transforms.ScaleImageTensor
      - _target_: torchvision.transforms.Normalize
        mean:
        - 0.48145466
        - 0.4578275
        - 0.40821073
        std:
        - 0.26862954
        - 0.26130258
        - 0.27577711
      robot_obs:
      - _target_: mode.utils.transforms.NormalizeVector
      scene_obs:
      - _target_: mode.utils.transforms.NormalizeVector
  _target_: mode.datasets.hulc_data_module.HulcDataModule
  _recursive_: false
  root_data_dir: ${root_data_dir}
  action_space: 7
  action_max:
  - 1.0
  - 1.0
  - 1.0
  - 1.0
  - 1.0
  - 1.0
  - 1.0
  action_min:
  - -1.0
  - -1.0
  - -1.0
  - -1.0
  - -1.0
  - -1.0
  - -1
  shuffle_val: false
  observation_space:
    rgb_obs:
    - rgb_static
    - rgb_gripper
    depth_obs: []
    state_obs:
    - robot_obs
    actions:
    - rel_actions
    language:
    - language
  proprioception_dims:
    n_state_obs: 8
    keep_indices:
    - - 0
      - 7
    - - 14
      - 15
    robot_orientation_idx:
    - 3
    - 6
    normalize: true
    normalize_robot_orientation: true
  datasets:
    lang_dataset:
      _target_: mode.datasets.disk_dataset.ExtendedDiskDataset
      key: lang
      save_format: npz
      batch_size: ${batch_size}
      min_window_size: ${act_seq_len}
      max_window_size: ${act_seq_len}
      proprio_state: ${datamodule.proprioception_dims}
      obs_space: ${datamodule.observation_space}
      skip_frames: 1
      pad: false
      lang_folder: ${lang_folder}
      aux_lang_loss_window: 8
      num_workers: ${num_workers}
      action_seq_len: ${act_seq_len}
      obs_seq_len: ${obs_seq_len}
      future_range: 1
      use_extracted_rel_actions: ${use_extracted_rel_actions}
model:
  _target_: mode.models.mode_agent.MoDEAgent
  _recursive_: false
  multistep: ${multistep}
  use_lr_scheduler: true
  entropy_gamma: 0.0
  router_z_delta: 0.0
  use_proprio: false
  seed: ${seed}
  sampler_type: ddim
  num_sampling_steps: 10
  sigma_data: 0.5
  sigma_min: 0.001
  sigma_max: 80
  noise_scheduler: exponential
  sigma_sample_density_type: loglogistic
  ckpt_path: /home/hk-project-sustainebot/ft4740/code/MoDE_Diffusion_Policy/pretrained_models/300k
  start_from_pretrained: true
  act_window_size: ${act_seq_len}
  latent_dim: 1024
  obs_enc_dim: ${obs_dim}
  cond_dim: 512
  resnet_type: '50'
  optimizer:
    _target_: torch.optim.AdamW
    transformer_weight_decay: 0.05
    obs_encoder_weight_decay: 0.05
    learning_rate: 0.0001
    betas:
    - 0.9
    - 0.95
  lr_scheduler:
    lr_scheduler:
      init_lr: 0.0001
      init_lr_scale: 0.1
      final_lr_scale: 1.0e-06
      total_steps: 45000
      phase_ratio: (0.02, 0.08, 0.9)
      lr: 0.0001
  model:
    _target_: mode.models.edm_diffusion.score_wrappers.GCDenoiser
    _recursive_: false
    sigma_data: ${model.sigma_data}
    inner_model:
      _target_: mode.models.networks.modedit.MoDeDiT
      action_dim: ${datamodule.action_space}
      goal_dim: ${model.cond_dim}
      obs_dim: 2048
      goal_conditioned: true
      causal: true
      use_custom_attn_mask: false
      use_proprio: ${model.use_proprio}
      state_dim: ${proprio_dims}
      embed_dim: ${model.latent_dim}
      n_layers: 12
      goal_seq_len: 1
      obs_seq_len: ${obs_seq_len}
      action_seq_len: ${act_seq_len}
      embed_pdrob: 0
      goal_drop: 0.1
      attn_pdrop: 0.3
      mlp_pdrop: 0.1
      n_heads: 8
      device: ${device}
      linear_output: true
      cond_router: true
      num_experts: 4
      top_k: 2
      router_normalize: true
      use_goal_in_routing: false
      use_argmax: false
      use_shared_expert: false
      use_noise_token_as_input: true
      init_style: olmoe
  language_goal:
    _target_: mode.models.networks.clip_lang_encoder.LangClip
    _recursive_: false
    model_name: ${clip_lang_model_name}
root_data_dir: /hkfs/work/workspace/scratch/ft4740-play3/data/task_ABC_D
lang_folder: lang_clip_resnet50
vis_clip_model_name: ViT-B/16
clip_lang_model_name: ViT-B/32
log_dir: ./logs
slurm: false
seed: 242
device: cuda
batch_size: 64
devices: 4
act_dim: 7
proprio_dims: 7
obs_dim: 512
goal_dim: 512
obs_seq_len: 1
act_seq_len: 10
multistep: 10
p_last_state: 0
gen_img_res: 112
max_epochs: 15
rollout_lh_skip_epochs: 4
num_workers: 12
benchmark_name: calvin_abc
use_extracted_rel_actions: true
trainer:
  devices: ${devices}
  precision: 16
  max_epochs: ${max_epochs}
  sync_batchnorm: true
  accelerator: gpu
  strategy: ddp
  limit_train_batches: 1000
  limit_val_batches: 4
logger:
  _target_: pytorch_lightning.loggers.WandbLogger
  save_dir: .
  name: logger
  group: mode
  log_model: false
  project: ${benchmark_name}
  entity: bennoq
  id: ???