Upload folder using huggingface_hub

README.md

@@ -1,3 +1,10 @@
----
-license: apache-2.0
----
+# In-Context Imitation Learning via Next-Token Prediction
+by <a href="https://max-fu.github.io">Max (Letian) Fu*</a>, <a href="https://qingh097.github.io/">Huang Huang*</a>, <a href="https://www.linkedin.com/in/gaurav-datta/">Gaurav Datta*</a>, <a href="https://yunliangchen.github.io/">Lawrence Yunliang Chen</a>, <a href="https://autolab.berkeley.edu/people">William Chung-Ho Panitch</a>, <a href="https://fangchenliu.github.io/">Fangchen Liu</a>, <a href="https://www.research.autodesk.com/people/hui-li/">Hui Li</a>, and <a href="https://goldberg.berkeley.edu">Ken Goldberg</a> at UC Berkeley and Autodesk (*equal contribution).
+
+[[Paper](https://openreview.net/forum?id=tFEOOH9eH0)] | [[Project Page](https://github.com/Max-Fu/icrt)] | [[Checkpoints](https://huggingface.co/mlfu7/Touch-Vision-Language-Models)] | [[Dataset](https://huggingface.co/datasets/mlfu7/Touch-Vision-Language-Dataset)] | [[Citation](#citation)]
+
+This repo contains the checkpoints for *In-Context Imitation Learning via Next-Token Prediction*. We investigate how to bring few-shot, in-context learning capability that exists in next-token prediction models (i.e. GPT) into real-robot imitation learning policies. 
+
+In particular, we store the pre-trained vision encoder and ICRT model separately. Please find them in [encoder](crossmae_rtx/cross-mae-rtx-vitb.pth) and [ICRT](icrt_vitb_droid_pretrained/icrt_vitb_droid_pretrained.pth) separately.
+
+Please refer to the [project page](https://github.com/Max-Fu/icrt) on installing the repo, training and inferencing the model. 

crossmae_rtx/cross-mae-rtx-vitb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2743c5a1ba4cbe296870a2f12a64d2653e7c257c7e8d25c3e2b197ace461f8fe
+size 509274961

icrt_vitb_droid_pretrained/icrt_vitb_droid_pretrained.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edaf5b17fa3c61e5ff0cf3dc254c8f8573197c607a63a39c7073b6066a69a545
+size 366419235

icrt_vitb_droid_pretrained/run.yaml

@@ -0,0 +1,81 @@
+!!python/object:icrt.util.args.ExperimentConfig
+dataset_cfg: !!python/object:icrt.util.args.DatasetConfig
+  action_noise: 0.0
+  dataset_json: config/real_expanded_4_icrt_dataset_config.json
+  goal_conditioned: false
+  non_overlapping: 32
+  num_repeat_traj: 2
+  num_weighted_steps: 30
+  proprio_noise: 0.005
+  rebalance_tasks: true
+  shuffle_repeat_traj: true
+  skip_step: false
+  sort_by_lang: true
+  task_barrier: true
+  task_names: null
+  vision_aug: true
+device: cuda
+dist_on_itp: false
+dist_url: env://
+load_config: null
+local_rank: -1
+logging_cfg: !!python/object:icrt.util.args.LoggingConfig
+  log_dir: /shared/projects/icrt/output/240604_2024
+  log_name: '240604_2024'
+  output_dir: /shared/projects/icrt/output/240604_2024
+model_cfg: !!python/object:icrt.util.args.ModelConfig
+  policy_cfg: !!python/object:icrt.util.args.PolicyConfig
+    adapter_mlp_ratio: 4.0
+    adapter_num_heads: 8
+    camera_pos_emb: false
+    decoder_pred_head: mlp
+    kl_div_loss: false
+    llama_ckpt_dir: /home/mfu/checkpoints/llama-2/llama-2-7b
+    load_llama: true
+    lora_layer_idxs: null
+    lora_rank: 4
+    loss_w_action: 1.0
+    modality_pos_emb: false
+    multikv_attn_pool: false
+    no_prompt_loss: true
+    phase: pretrain
+    pred_action_only: true
+    pretrained_path: /shared/projects/icrt/output/240604_1852/checkpoint-3.pth
+    remove_proprio: false
+    scale_loss: 1.0
+    scratch_llama_config: ../config/model_config/custom_transformer.json
+    separate_camera_adapter: true
+    step_weight: 1.0
+  vision_encoder_cfg: !!python/object:icrt.util.args.VisionEncoderConfig
+    vision_encoder: /home/mfu/Documents/icrt/crossmae_ckpt/cross-mae-rtx.pth
+    vision_lora: false
+    vision_lora_rank: 8
+    vision_nonpretrained: false
+    vision_unfreeze_all: false
+    vision_unfreeze_last_n: 0
+optimizer_cfg: !!python/object:icrt.util.args.OptimizerConfig
+  blr: 0.001
+  lr: 0.0005
+  min_lr: 0.0
+  warmup_epochs: 1.25
+  weight_decay: 0.01
+shared_cfg: !!python/object:icrt.util.args.SharedConfig
+  batch_size: 1
+  num_cameras: 2
+  num_pred_steps: 16
+  num_stages: 1
+  resume: null
+  rot_6d: true
+  save_every: 5
+  seed: 0
+  seq_length: 512
+  split_epoch: 1
+  start_epoch: 0
+  use_delta_action: true
+train: true
+trainer_cfg: !!python/object:icrt.util.args.TrainerConfig
+  accum_iter: 8
+  epochs: 125
+  num_workers: 20
+  pin_memory: true
+world_size: 1