Xidong commited on
Commit
69c118d
1 Parent(s): 9304507

Upload Apollo-34B/configuration_yi.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. Apollo-34B/configuration_yi.py +121 -0
Apollo-34B/configuration_yi.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ Yi model configuration"""
2
+ from transformers.configuration_utils import PretrainedConfig
3
+ from transformers.utils import logging
4
+
5
+ logger = logging.get_logger(__name__)
6
+
7
+ Yi_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
8
+
9
+
10
+ class YiConfig(PretrainedConfig):
11
+ r"""
12
+ This is the configuration class to store the configuration of a [`YiModel`]. It is used to instantiate an Yi
13
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
14
+ defaults will yield a similar configuration to that of the Yi model.
15
+
16
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
17
+ documentation from [`PretrainedConfig`] for more information.
18
+
19
+
20
+ Args:
21
+ vocab_size (`int`, *optional*, defaults to 64000):
22
+ Vocabulary size of the Yi model. Defines the number of different tokens that can be represented by the
23
+ `inputs_ids` passed when calling [`YiModel`]
24
+ hidden_size (`int`, *optional*, defaults to 4096):
25
+ Dimension of the hidden representations.
26
+ intermediate_size (`int`, *optional*, defaults to 11008):
27
+ Dimension of the MLP representations.
28
+ num_hidden_layers (`int`, *optional*, defaults to 32):
29
+ Number of hidden layers in the Transformer encoder.
30
+ num_attention_heads (`int`, *optional*, defaults to 32):
31
+ Number of attention heads for each attention layer in the Transformer encoder.
32
+ num_key_value_heads (`int`, *optional*):
33
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
34
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
35
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
36
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
37
+ by meanpooling all the original heads within that group. For more details checkout [this
38
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
39
+ `num_attention_heads`.
40
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
41
+ The non-linear activation function (function or string) in the decoder.
42
+ max_position_embeddings (`int`, *optional*, defaults to 4096):
43
+ The maximum sequence length that this model might ever be used with. Typically set this to something large
44
+ just in case (e.g., 512 or 1024 or 2048 or 4096).
45
+ initializer_range (`float`, *optional*, defaults to 0.02):
46
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
47
+ rms_norm_eps (`float`, *optional*, defaults to 1e-5):
48
+ The epsilon used by the rms normalization layers.
49
+ use_cache (`bool`, *optional*, defaults to `True`):
50
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
51
+ relevant if `config.is_decoder=True`.
52
+ tie_word_embeddings(`bool`, *optional*, defaults to `False`):
53
+ Whether to tie weight embeddings
54
+ output_attentions (`bool`, *optional*, defaults to `False`):
55
+ Whether or not to output attentions.
56
+ rope_theta (`float`, *optional*, defaults to 5000000.0):
57
+ The base period of the RoPE embeddings.
58
+ Example:
59
+
60
+ ```python
61
+ >>> from transformers import YiModel, YiConfig
62
+
63
+ >>> # Initializing a Yi style configuration
64
+ >>> configuration = YiConfig()
65
+
66
+ >>> # Initializing a model from the Yi style configuration
67
+ >>> model = YiModel(configuration)
68
+
69
+ >>> # Accessing the model configuration
70
+ >>> configuration = model.config
71
+ ```"""
72
+ model_type = "Yi"
73
+ keys_to_ignore_at_inference = ["past_key_values"]
74
+
75
+ def __init__(
76
+ self,
77
+ vocab_size=64000,
78
+ hidden_size=4096,
79
+ intermediate_size=11008,
80
+ num_hidden_layers=32,
81
+ num_attention_heads=32,
82
+ num_key_value_heads=4,
83
+ hidden_act="silu",
84
+ max_position_embeddings=4096,
85
+ initializer_range=0.02,
86
+ rms_norm_eps=1e-5,
87
+ use_cache=True,
88
+ pad_token_id=0,
89
+ bos_token_id=1,
90
+ eos_token_id=2,
91
+ tie_word_embeddings=False,
92
+ output_attentions=False,
93
+ rope_theta=5000000.0,
94
+ **kwargs,
95
+ ):
96
+ self.vocab_size = vocab_size
97
+ self.max_position_embeddings = max_position_embeddings
98
+ self.hidden_size = hidden_size
99
+ self.intermediate_size = intermediate_size
100
+ self.num_hidden_layers = num_hidden_layers
101
+ self.num_attention_heads = num_attention_heads
102
+
103
+ # for backward compatibility
104
+ if num_key_value_heads is None:
105
+ num_key_value_heads = num_attention_heads
106
+
107
+ self.num_key_value_heads = num_key_value_heads
108
+ self.hidden_act = hidden_act
109
+ self.initializer_range = initializer_range
110
+ self.rms_norm_eps = rms_norm_eps
111
+ self.use_cache = use_cache
112
+ self.output_attentions = output_attentions
113
+ self.rope_theta = rope_theta
114
+
115
+ super().__init__(
116
+ pad_token_id=pad_token_id,
117
+ bos_token_id=bos_token_id,
118
+ eos_token_id=eos_token_id,
119
+ tie_word_embeddings=tie_word_embeddings,
120
+ **kwargs,
121
+ )