Upload 14 files

talc_finetuned_modelscope_t2v/config.yaml

@@ -0,0 +1,57 @@
+pretrained_model_path: damo-vilab/text-to-video-ms-1.7b
+output_dir: /local2/hbansal/mst2vg/msrvtt_vatex_modelscope_multi_lr_1e-5_constant_warmup_max_n_scenes_4_e20_wo_rescale_unfiltered_frames_8_grad_4_olddata_time_aligned
+extra_train_data: []
+dataset_types: json
+shuffle: true
+validation_steps: 100
+trainable_modules: all
+trainable_text_modules:
+- all
+extra_unet_params: null
+extra_text_encoder_params: null
+train_batch_size: 1
+num_train_epochs: 20
+learning_rate: 1.0e-05
+scale_lr: false
+lr_scheduler: constant_with_warmup
+lr_warmup_steps: 0
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_weight_decay: 0
+adam_epsilon: 1.0e-08
+max_grad_norm: 1.0
+gradient_accumulation_steps: 4
+gradient_checkpointing: true
+text_encoder_gradient_checkpointing: false
+checkpointing_steps: 1000
+resume_from_checkpoint: null
+resume_step: null
+mixed_precision: fp16
+use_8bit_adam: true
+enable_xformers_memory_efficient_attention: false
+enable_torch_2_attn: true
+seed: 64
+extend_dataset: false
+cache_latents: false
+cached_latent_dir: null
+save_pretrained_model: true
+logger_type: wandb
+image: false
+prediction_type: epsilon
+conditioning_dropout_prob: 0.0
+kwargs:
+  train_path: /local2/hbansal/msr-vtt/old_train_msr_vatex_16.jsonl
+  val_path: /local2/hbansal/msr-vtt/old_val_msr_vatex_16.jsonl
+  root: /local2/hbansal/mst2vg/
+  time_aligned_caption: true
+  max_scenes: 4
+offset_noise_strength: 0.1
+rescale_schedule: false
+train_data:
+  width: 256
+  height: 256
+  use_bucketing: false
+  sample_start_idx: 1
+  n_sample_frames: 8
+train_text_encoder: false
+use_offset_noise: false

talc_finetuned_modelscope_t2v/model_index.json

@@ -0,0 +1,25 @@
+{
+  "_class_name": "TextToVideoSDPipeline",
+  "_diffusers_version": "0.25.0.dev0",
+  "_name_or_path": "damo-vilab/text-to-video-ms-1.7b",
+  "scheduler": [
+    "diffusers",
+    "DDIMScheduler"
+  ],
+  "text_encoder": [
+    "transformers",
+    "CLIPTextModel"
+  ],
+  "tokenizer": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "unet": [
+    "diffusers",
+    "UNet3DConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ]
+}

talc_finetuned_modelscope_t2v/scheduler/scheduler_config.json

@@ -0,0 +1,20 @@
+{
+  "_class_name": "DDIMScheduler",
+  "_diffusers_version": "0.25.0.dev0",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "clip_sample_range": 1.0,
+  "dynamic_thresholding_ratio": 0.995,
+  "num_train_timesteps": 1000,
+  "prediction_type": "epsilon",
+  "rescale_betas_zero_snr": false,
+  "sample_max_value": 1.0,
+  "set_alpha_to_one": false,
+  "skip_prk_steps": true,
+  "steps_offset": 1,
+  "thresholding": false,
+  "timestep_spacing": "leading",
+  "trained_betas": null
+}

talc_finetuned_modelscope_t2v/text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "damo-vilab/text-to-video-ms-1.7b",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_size": 1024,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 23,
+  "pad_token_id": 1,
+  "projection_dim": 512,
+  "torch_dtype": "float32",
+  "transformers_version": "4.28.1",
+  "vocab_size": 49408
+}

talc_finetuned_modelscope_t2v/text_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ec178e5f07ed24deb82bf41fdfde9392c0471755d5bf8d0576b21d7c6dafd46
+size 1361597016

talc_finetuned_modelscope_t2v/tokenizer/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "!",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

talc_finetuned_modelscope_t2v/tokenizer/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

talc_finetuned_modelscope_t2v/unet/config.json

@@ -0,0 +1,35 @@
+{
+  "_class_name": "UNet3DConditionModel",
+  "_diffusers_version": "0.25.0.dev0",
+  "_name_or_path": "damo-vilab/text-to-video-ms-1.7b",
+  "act_fn": "silu",
+  "apply_history": false,
+  "attention_head_dim": 64,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "cross_attention_dim": 1024,
+  "down_block_types": [
+    "CrossAttnDownBlock3D",
+    "CrossAttnDownBlock3D",
+    "CrossAttnDownBlock3D",
+    "DownBlock3D"
+  ],
+  "downsample_padding": 1,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "out_channels": 4,
+  "sample_size": 32,
+  "up_block_types": [
+    "UpBlock3D",
+    "CrossAttnUpBlock3D",
+    "CrossAttnUpBlock3D",
+    "CrossAttnUpBlock3D"
+  ]
+}

talc_finetuned_modelscope_t2v/unet/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d100db2564ae4569a0eeab7de0896ee9deb919bb7ac58202ea0275ee469baa1
+size 5645118400

talc_finetuned_modelscope_t2v/vae/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.25.0.dev0",
+  "_name_or_path": "damo-vilab/text-to-video-ms-1.7b",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 512,
+  "scaling_factor": 0.18215,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ]
+}

talc_finetuned_modelscope_t2v/vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc436097e5cbb105c44df4403ae84acf7e81de1f64acfedc334888e9a8eea223
+size 334643268