config.json

{
  "model_type": "csumlm",
  "text_encoder": {
    "model_name": "bert-base-uncased",
    "pretrained_weights": "path/to/bert-base-uncased"
  },
  "image_encoder": {
    "model_name": "resnet50",
    "pretrained_weights": "path/to/resnet50"
  },
  "audio_encoder": {
    "model_name": "wav2vec2-base",
    "pretrained_weights": "path/to/wav2vec2-base"
  },
  "fusion_layer": {
    "type": "concatenate",
    "output_dim": 1024
  },
  "output_layer": {
    "type": "dense",
    "units": 1,
    "activation": "sigmoid"
  },
  "attention_mechanism": {
    "traditional_attention": {
      "type": "bahdanau"
    },
    "self_attention": {
      "type": "scaled_dot_product"
    },
    "linear_attention": {
      "type": "linear"
    }
  },
  "belief_desire_intent_tree": {
    "type": "recursive_neural_network"
  },
  "iragel": {
    "retrieval_model": "path/to/retrieval_model",
    "generation_model": "path/to/generation_model"
  },
  "explainer": {
    "type": "attention_visualization"
  },
  "realtime_learning": {
    "type": "online_learning",
    "learning_rate": 0.001
  },
  "knowledge_base": {
    "type": "knowledge_graph_embeddings"
  }
}