Crystalcareai
/

GemMoE-Medium-v0.4

Text Generation

Model card Files Files and versions Community

Crystalcareai commited on Mar 21

Commit

ac546a8

•

1 Parent(s): 75df32a

Update modeling_gemmoe.py

Files changed (1) hide show

modeling_gemmoe.py +2 -2

modeling_gemmoe.py CHANGED Viewed

@@ -746,7 +746,7 @@ class GemmoeDecoderLayer(nn.Module):
         # Check if the tensor sizes match before adding residual
         if hidden_states.size() != residual.size():
-            hidden_states = hidden_states.view(residual.size())
         hidden_states = residual + hidden_states
@@ -757,7 +757,7 @@ class GemmoeDecoderLayer(nn.Module):
         # Check if the tensor sizes match before adding residual
         if hidden_states.size() != residual.size():
-            hidden_states = hidden_states.view(residual.size())
         hidden_states = residual + hidden_states

         # Check if the tensor sizes match before adding residual
         if hidden_states.size() != residual.size():
+            hidden_states = hidden_states[:, -residual.size(1):, :]
         hidden_states = residual + hidden_states
         # Check if the tensor sizes match before adding residual
         if hidden_states.size() != residual.size():
+            hidden_states = hidden_states[:, -residual.size(1):, :]
         hidden_states = residual + hidden_states