refactor: disable lora by default

Signed-off-by: jupyterjazz <saba.sturua@jina.ai>

configuration_xlm_roberta.py

@@ -26,6 +26,7 @@ class XLMRobertaFlashConfig(PretrainedConfig):
             lora_rank=4,
             lora_dropout_p=0.0,
             lora_alpha=1,
+            lora_main_params_trainable=False,
             load_trained_adapters=False,
             use_flash_attn=True,
             torch_dtype=None,
@@ -55,6 +56,7 @@ class XLMRobertaFlashConfig(PretrainedConfig):
         self.lora_rank = lora_rank
         self.lora_dropout_p = lora_dropout_p
         self.lora_alpha = lora_alpha
+        self.lora_main_params_trainable = lora_main_params_trainable
         self.use_flash_attn = use_flash_attn
         self.emb_pooler = emb_pooler
         if torch_dtype and hasattr(torch, torch_dtype) and type(getattr(torch, torch_dtype)) is torch.dtype:

modeling_lora.py

@@ -226,10 +226,10 @@ class XLMRobertaLoRA(XLMRobertaModel):
             dropout_p=self._dropout_p,
             alpha=self._alpha,
         )
-        self.main_params_trainable = False
+        self.main_params_trainable = config.lora_main_params_trainable
         self._task_idx = None
-        # By default, we select the first LoRA
-        self.current_task = 0
+        # By default, disable LoRA until it's specified which adapter/task to use
+        self.current_task = None
 
 
     @property