Update configuration_gemmoe.py

configuration_gemmoe.py

@@ -126,6 +126,8 @@ class GemmoeConfig(PretrainedConfig):
         pad_token_id=0,
         eos_token_id=1,
         bos_token_id=2,
+        hidden_act="gelu_pytorch_tanh",
+        hidden_activation=None,
         tie_word_embeddings=True,
         rope_theta=10000.0,
         attention_bias=False,
@@ -143,6 +145,8 @@ class GemmoeConfig(PretrainedConfig):
         self.num_hidden_layers = num_hidden_layers
         self.num_attention_heads = num_attention_heads
         self.head_dim = head_dim
+        self.hidden_act = hidden_act
+        self.hidden_activation = hidden_activation
         self.num_key_value_heads = num_key_value_heads
         self.hidden_act = hidden_act
         self.initializer_range = initializer_range