Upload model

modeling_hf_nomic_bert.py

@@ -2002,7 +2002,30 @@ class NomicMultiHeadAttentionPooling(nn.Module):
 
         return hidden_states
 
-class NomicVisionModel(PreTrainedModel):
+class NomicVisionPreTrainedModel(PreTrainedModel):
+    """An abstract class to handle weights initialization and
+    a simple interface for dowloading and loading pretrained models.
+    """
+
+    config_class = NomicBertConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Block"]
+    _skip_keys_device_placement = "past_key_values"
+
+    def __init__(self, config, *inputs, **kwargs):
+        super().__init__(config)
+        if not isinstance(config, GPT2Config):
+            raise ValueError(
+                "Parameter config in `{}(config)` should be an instance of class `GPT2Config`. "
+                "To create a model from a Google pretrained model use "
+                "`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`".format(
+                    self.__class__.__name__, self.__class__.__name__
+                )
+            )
+        self.config = config
+
+class NomicVisionModel(NomicVisionPreTrainedModel):
     def __init__(self, config):
         super().__init__(config)