Update configuration_telechat.py

#2
Files changed (1) hide show
  1. configuration_telechat.py +2 -0
configuration_telechat.py CHANGED
@@ -119,6 +119,7 @@ class TELECHATConfig(PretrainedConfig):
119
  n_embd=768,
120
  n_layer=12,
121
  n_head=12,
 
122
  n_inner=None,
123
  activation_function="gelu_new",
124
  resid_pdrop=0.1,
@@ -159,6 +160,7 @@ class TELECHATConfig(PretrainedConfig):
159
  self.n_embd = n_embd
160
  self.n_layer = n_layer
161
  self.n_head = n_head
 
162
  self.n_inner = n_inner
163
  self.activation_function = activation_function
164
  self.resid_pdrop = resid_pdrop
 
119
  n_embd=768,
120
  n_layer=12,
121
  n_head=12,
122
+ num_key_value_heads=64,
123
  n_inner=None,
124
  activation_function="gelu_new",
125
  resid_pdrop=0.1,
 
160
  self.n_embd = n_embd
161
  self.n_layer = n_layer
162
  self.n_head = n_head
163
+ self.num_key_value_heads = num_key_value_heads
164
  self.n_inner = n_inner
165
  self.activation_function = activation_function
166
  self.resid_pdrop = resid_pdrop