camembertav2_base_p2_17k_last_layer.yaml

# Layer dimensions
mlp_input: 1024
mlp_tag_hidden: 16
mlp_arc_hidden: 512
mlp_lab_hidden: 128
# Lexers
lexers:
  - name: word_embeddings
    type: words
    embedding_size: 256
    word_dropout: 0.5
  - name: char_level_embeddings
    type: chars_rnn
    embedding_size: 64
    lstm_output_size: 128
  - name: fasttext
    type: fasttext
  - name: camembertav2_base_p2_17k_last_layer
    type: bert
    model: /scratch/camembertv2/runs/models/camembertav2-base-bf16/post/ckpt-p2-17000/pt/discriminator/
    layers: [11]
    subwords_reduction: "mean"
# Training hyperparameters
encoder_dropout: 0.5
mlp_dropout: 0.5
batch_size: 8
epochs: 64
lr:
  base: 0.00003
  schedule:
    shape: linear
    warmup_steps: 100