Upload Moondream

config.json

@@ -11,5 +11,5 @@
     "model_type": "phi"
   },
   "torch_dtype": "float16",
-  "transformers_version": "4.38.2"
+  "transformers_version": "4.36.2"
 }

configuration_moondream.py

@@ -25,7 +25,6 @@ class PhiConfig(PretrainedConfig):
         rope_theta=10000.0,
         rope_scaling=None,
         partial_rotary_factor=0.5,
-        qk_layernorm=False,
         bos_token_id=1,
         eos_token_id=2,
         **kwargs,
@@ -51,7 +50,6 @@ class PhiConfig(PretrainedConfig):
         self.rope_theta = rope_theta
         self.rope_scaling = rope_scaling
         self.partial_rotary_factor = partial_rotary_factor
-        self.qk_layernorm = qk_layernorm
         self._rope_scaling_validation()
 
         super().__init__(

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.38.2"
+  "transformers_version": "4.36.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7840817a7015edf729fa3d60099c35f08fc30511a1dc8ea231acd0e9a6555bb8
+oid sha256:927694193ed81f83b9b269c0d1ffa8dc823dec90bce4703a54b22ebd6c9632b6
 size 3733912224

modeling_phi.py

@@ -16,14 +16,13 @@
 """ PyTorch Phi model."""
 
 
-import math
 from typing import List, Optional, Tuple, Union
 
 import torch
 import torch.nn.functional as F
 import torch.utils.checkpoint
 from torch import nn
-from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+from torch.nn import CrossEntropyLoss
 
 from transformers.activations import ACT2FN
 from transformers.cache_utils import Cache, DynamicCache
@@ -31,7 +30,6 @@ from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_m
 from transformers.modeling_outputs import (
     BaseModelOutputWithPast,
     CausalLMOutputWithPast,
-    SequenceClassifierOutputWithPast,
 )
 from transformers.modeling_utils import PreTrainedModel
 from transformers.utils import (
@@ -287,19 +285,6 @@ class PhiAttention(nn.Module):
             self.num_heads * self.head_dim, self.hidden_size, bias=True
         )
 
-        self.qk_layernorm = config.qk_layernorm
-        if self.qk_layernorm:
-            self.q_layernorm = nn.LayerNorm(
-                config.hidden_size // self.num_heads,
-                eps=config.layer_norm_eps,
-                elementwise_affine=True,
-            )
-            self.k_layernorm = nn.LayerNorm(
-                config.hidden_size // self.num_heads,
-                eps=config.layer_norm_eps,
-                elementwise_affine=True,
-            )
-
         self._init_rope()
 
     def _init_rope(self):
@@ -344,10 +329,6 @@ class PhiAttention(nn.Module):
             3, dim=-1
         )
 
-        if self.qk_layernorm:
-            query_states = self.q_layernorm(query_states)
-            key_states = self.k_layernorm(key_states)
-
         query_states = query_states.view(
             bsz, q_len, self.num_heads, self.head_dim
         ).transpose(1, 2)
@@ -451,10 +432,6 @@ class PhiFlashAttention2(PhiAttention):
             3, dim=-1
         )
 
-        if self.qk_layernorm:
-            query_states = self.q_layernorm(query_states)
-            key_states = self.k_layernorm(key_states)
-
         # Flash attention requires the input to have the shape
         # batch_size x seq_length x head_dim x hidden_dim
         # therefore we just need to keep the original shape

moondream.py

@@ -59,6 +59,9 @@ class Moondream(PreTrainedModel):
 
         return torch.cat(embeds, dim=1)
 
+    def get_input_embeddings(self):
+        return self.text_model.get_input_embeddings()
+
     def generate(
         self,
         image_embeds,