config_hydra.yaml

training_type: pretrain
wandb_project: speechlmm-v1
wandb_watch: 'false'
num_gpus: 4
num_nodes: 1
accelerate_config: /net/scratch/hscra/plgrid/plgstefanop/llava/conf/accelerate/deepspeed.yaml
deepspeed_config: /net/scratch/hscra/plgrid/plgstefanop/llava/conf/deepspeed/zero3.json
adjustments: []
data:
  data_config_path: /net/scratch/hscra/plgrid/plgstefanop/llava/conf/datasets/speechlmm_v1/speechlmm_v1.yml
  num_proc_for_preprocessing: 64
  dataloader_debug: false
  filter_broken_samples: false
  organize_eval_dataset_per_task: true
  group_dataset_by_task:
    train: true
    eval: false
    test: false
  task_weights:
    ASR: 0.3
    ST: 0.2
    SSUM: 0.1
    SQA: 0.1
    SLU_INTENT_ONLY: 0.1
    VSR: 0.2
  multi_task_sampler: alternating
  replacement: true
  rebuild_dataset_cache: false
  cache_final_datasets: true
  audio_input_sampling_rate: 16000
  codec_sampling_rate: null
  codec_frame_rate: null
  image_folder: null
  image_aspect_ratio: square
  is_multimodal: false
  lazy_preprocess: true
  align_text_to_audio: false
  use_text_tokens: true
  align_with_whisper: false
  restore_punctuation_and_spaces: true
  max_condition_audio_duration: 10
  variable_batch_size: false
  max_length_per_batch: null
model:
  add_lm_head: true
  vision_select_layer: -1
  vision_use_patch_token: true
  vision_patch_merge_type: flat
  vision_select_feature: patch
  mm_use_im_start_end: false
  mm_use_audio_start_end: false
  use_audio_encoder_as_codec_encoder: false
  add_all_multimodal_tokens: true
  perturb_codes: true
  perturb_prob: 0.2
  pad_audio_weight: 0.5
  epad_audio_weight: 1
  pad_epad_audio_weight_decay: 0.5
  perturb_prob_decay: 0.5
  conversation_version: null
  codebook_weights:
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  - 1
  chunk_size_in_seconds: 15
  chunk_overlap_in_seconds: 1
  chunk_encoding_strategy: loop
  audio_loss_decay: 1
  audio_loss_weight: 1
  tokenizer_padding_side: right
  audio_encoder:
    _name_or_path: meetween/seamless-m4t-v2-large-speech-encoder
    adaptor_dropout: 0.1
    adaptor_kernel_size: 8
    adaptor_stride: 8
    add_adapter: true
    architectures:
    - SeamlessM4Tv2SpeechEncoder
    conv_depthwise_kernel_size: 31
    feature_projection_input_dim: 160
    hidden_size: 1024
    initializer_range: 0.02
    layer_norm_eps: 1.0e-05
    left_max_position_embeddings: 64
    model_type: seamless_m4t_v2
    num_adapter_layers: 1
    position_embeddings_type: relative_key
    right_max_position_embeddings: 8
    speech_encoder_attention_heads: 16
    speech_encoder_chunk_size: 20000
    speech_encoder_dropout: 0.0
    speech_encoder_hidden_act: swish
    speech_encoder_intermediate_size: 4096
    speech_encoder_layerdrop: 0.1
    speech_encoder_layers: 24
    speech_encoder_left_chunk_num: 128
    torch_dtype: float32
    transformers_version: 4.37.2
    use_cache: true
    sampling_rate: 16000
  audio_adapter:
    model_type: qformer
    input_dim: 1024
    output_dim: 2048
    hidden_size: 768
    num_hidden_layers: 4
    num_attention_heads: 12
    intermediate_size: 3072
    hidden_act: gelu
    hidden_dropout_prob: 0.1
    attention_probs_dropout_prob: 0.1
    initializer_range: 0.02
    layer_norm_eps: 1.0e-12
    add_cross_attention: true
    num_queries: 1
    cross_attention_every_n_layers: 1
    compress_factor: 2
    triplet_loss: false
  text_decoder:
    _name_or_path: meta-llama/Llama-3.2-1B-Instruct
    architectures:
    - LlamaForCausalLM
    attention_bias: false
    attention_dropout: 0.0
    bos_token_id: 128000
    eos_token_id:
    - 128001
    - 128008
    - 128009
    head_dim: 64
    hidden_act: silu
    hidden_size: 2048
    initializer_range: 0.02
    intermediate_size: 8192
    max_position_embeddings: 131072
    mlp_bias: false
    model_type: llama
    num_attention_heads: 32
    num_hidden_layers: 16
    num_key_value_heads: 8
    pretraining_tp: 1
    rms_norm_eps: 1.0e-05
    rope_scaling:
      factor: 32.0
      high_freq_factor: 4.0
      low_freq_factor: 1.0
      original_max_position_embeddings: 8192
      rope_type: llama3
    rope_theta: 500000.0
    tie_word_embeddings: true
    torch_dtype: bfloat16
    transformers_version: 4.45.0.dev0
    use_cache: true
    vocab_size: 128256
    conversation_version: llama_3_1
  video_encoder:
    hidden_size: 768
    _name_or_path: /net/scratch/hscra/plgrid/plgstefanop/auto_avsr_ckpt/vsr_trlrs3vox2_base.pth
    model_type: auto_avsr
    adim: 768
    aheads: 12
    eunits: 3072
    elayers: 12
    transformer_input_layer: conv3d
    dropout_rate: 0.1
    transformer_attn_dropout_rate: 0.1
    transformer_encoder_attn_layer_type: rel_mha
    macaron_style: true
    use_cnn_module: true
    cnn_module_kernel: 31
    zero_triu: false
    a_upsample_ratio: 1
    relu_type: swish
    ddim: 768
    dheads: 12
    dunits: 3072
    dlayers: 6
    lsm_weight: 0.1
    transformer_length_normalized_loss: false
    mtlalpha: 0.1
    ctc_type: builtin
    rel_pos_type: latest
  video_adapter:
    model_type: mlp
    input_dim: 768
    output_dim: 2048
    hidden_layers: 4
    hidden_size: 4096
    residual_type: interpolation
    force_input_projection: true
    force_output_projection: true
training:
  modality: audio
  group_by_modality_length: false
  load_in_4bit: false
  load_in_8bit: false
  bnb_4bit_quant_type: nf4
  bnb_4bit_use_double_quant: true
  lora_adapters: []
  num_steps_between_each_restart: null
  lr_min: 1.0e-06
  eval_temperature: 0
  eval_max_new_tokens: 200
  eval_num_batched_generations: 4
  cache_dir: null
  resume_from_checkpoint: null
  model_max_length: 2048
  freeze_modules:
  - audio_encoder
  - video_encoder
  - text_decoder
  attn_implementation: flash_attention_2
  mpt_attn_impl: triton
  mm_projector_lr: null
  output_dir: /net/scratch/hscra/plgrid/plgstefanop/checkpoints/speechlmm-v1/llava-huggingface-s
  report_to: wandb
  run_name: pretrain-audio-seamless-qformer-auto_avsr-mlp-llama_3_1b-speechlmm/v1/s
  num_train_epochs: 0
  max_steps: 0
  per_device_train_batch_size: 4
  per_device_eval_batch_size: 8
  gradient_accumulation_steps: 12
  gradient_checkpointing: true
  optim: adamw_torch
  learning_rate: 5.0e-05
  weight_decay: 0.0
  lr_scheduler_type: cosine
  warmup_ratio: 0.03
  logging_steps: 1
  save_strategy: steps
  save_steps: 200
  save_total_limit: 10
  eval_strategy: 'no'
  eval_steps: 0
  push_to_hub: true
  hub_model_id: meetween/Llama-speechlmm-1.0-s
  hub_strategy: end
  hub_token: null
  hub_private_repo: false
  fp16: false
  bf16: true
  tf32: true
  dataloader_num_workers: 4
  remove_unused_columns: false
  seed: 42
  pretrained_checkpoint: /net/scratch/hscra/plgrid/plgstefanop/checkpoints/speechlmm-v1/llava-pretrain-audio-seamless-qformer-auto_avsr-mlp-llama_3_1b-speechlmm/v1/s/checkpoint-800
run_name: speechlmm-s