Fix initialize_lisa_module during validation in train_ds.py

Former-commit-id: 7232bb8fa764f6d3a9b7928c975e26a67fde545c

train_ds.py

@@ -161,7 +161,8 @@ def main(args):
     model.get_model().initialize_vision_modules(model.get_model().config)
     vision_tower = model.get_model().get_vision_tower()
     vision_tower.to(dtype=torch_dtype, device=args.local_rank)
-    model.get_model().initialize_lisa_modules(model.get_model().config)
+    if not args.eval_only:
+        model.get_model().initialize_lisa_modules(model.get_model().config)
 
     for p in vision_tower.parameters():
         p.requires_grad = False