txsun commited on
Commit
cb817a2
1 Parent(s): 9ed415a

Upload configuration_moss.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. configuration_moss.py +123 -0
configuration_moss.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ Moss model configuration"""
2
+
3
+ from transformers.utils import logging
4
+ from transformers.configuration_utils import PretrainedConfig
5
+
6
+
7
+ logger = logging.get_logger(__name__)
8
+
9
+
10
+ class MossConfig(PretrainedConfig):
11
+ r"""
12
+ This is the configuration class to store the configuration of a [`MossModel`]. It is used to instantiate a
13
+ Moss model according to the specified arguments, defining the model architecture. Instantiating a configuration
14
+ with the defaults will yield a similar configuration to that of the Moss
15
+ [fnlp/moss-moon-003-base](https://huggingface.co/fnlp/moss-moon-003-base) architecture. Configuration objects
16
+ inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from
17
+ [`PretrainedConfig`] for more information.
18
+
19
+ Args:
20
+ vocab_size (`int`, *optional*, defaults to 107008):
21
+ Vocabulary size of the Moss model. Defines the number of different tokens that can be represented by the
22
+ `inputs_ids` passed when calling [`MossModel`].
23
+ n_positions (`int`, *optional*, defaults to 2048):
24
+ The maximum sequence length that this model might ever be used with. Typically set this to something large
25
+ just in case (e.g., 512 or 1024 or 2048).
26
+ n_embd (`int`, *optional*, defaults to 4096):
27
+ Dimensionality of the embeddings and hidden states.
28
+ n_layer (`int`, *optional*, defaults to 28):
29
+ Number of hidden layers in the Transformer encoder.
30
+ n_head (`int`, *optional*, defaults to 16):
31
+ Number of attention heads for each attention layer in the Transformer encoder.
32
+ rotary_dim (`int`, *optional*, defaults to 64):
33
+ Number of dimensions in the embedding that Rotary Position Embedding is applied to.
34
+ n_inner (`int`, *optional*, defaults to None):
35
+ Dimensionality of the inner feed-forward layers. `None` will set it to 4 times n_embd
36
+ activation_function (`str`, *optional*, defaults to `"gelu_new"`):
37
+ Activation function, to be selected in the list `["relu", "silu", "gelu", "tanh", "gelu_new"]`.
38
+ resid_pdrop (`float`, *optional*, defaults to 0.1):
39
+ The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
40
+ embd_pdrop (`int`, *optional*, defaults to 0.1):
41
+ The dropout ratio for the embeddings.
42
+ attn_pdrop (`float`, *optional*, defaults to 0.1):
43
+ The dropout ratio for the attention.
44
+ layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
45
+ The epsilon to use in the layer normalization layers.
46
+ initializer_range (`float`, *optional*, defaults to 0.02):
47
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
48
+ use_cache (`bool`, *optional*, defaults to `True`):
49
+ Whether or not the model should return the last key/values attentions (not used by all models).
50
+
51
+ Example:
52
+
53
+ ```python
54
+ >>> from modeling_moss import MossModel
55
+ >>> from configuration_moss import MossConfig
56
+
57
+ >>> # Initializing a moss-moon-003-base configuration
58
+ >>> configuration = MossConfig()
59
+
60
+ >>> # Initializing a model (with random weights) from the configuration
61
+ >>> model = MossModel(configuration)
62
+
63
+ >>> # Accessing the model configuration
64
+ >>> configuration = model.config
65
+ ```"""
66
+
67
+ model_type = "moss"
68
+ attribute_map = {
69
+ "max_position_embeddings": "n_positions",
70
+ "hidden_size": "n_embd",
71
+ "num_attention_heads": "n_head",
72
+ "num_hidden_layers": "n_layer",
73
+ }
74
+
75
+ def __init__(
76
+ self,
77
+ vocab_size=107008,
78
+ n_positions=2048,
79
+ n_ctx=2048,
80
+ n_embd=4096,
81
+ n_layer=28,
82
+ n_head=16,
83
+ rotary_dim=64,
84
+ n_inner=None,
85
+ activation_function="gelu_new",
86
+ resid_pdrop=0.0,
87
+ embd_pdrop=0.0,
88
+ attn_pdrop=0.0,
89
+ layer_norm_epsilon=1e-5,
90
+ initializer_range=0.02,
91
+ use_cache=True,
92
+ bos_token_id=106028,
93
+ eos_token_id=106068,
94
+ tie_word_embeddings=False,
95
+ wbits=32,
96
+ groupsize=128,
97
+ **kwargs,
98
+ ):
99
+ self.vocab_size = vocab_size
100
+ self.n_ctx = n_ctx
101
+ self.n_positions = n_positions
102
+ self.n_embd = n_embd
103
+ self.n_layer = n_layer
104
+ self.n_head = n_head
105
+ self.n_inner = n_inner
106
+ self.rotary_dim = rotary_dim
107
+ self.activation_function = activation_function
108
+ self.resid_pdrop = resid_pdrop
109
+ self.embd_pdrop = embd_pdrop
110
+ self.attn_pdrop = attn_pdrop
111
+ self.layer_norm_epsilon = layer_norm_epsilon
112
+ self.initializer_range = initializer_range
113
+ self.use_cache = use_cache
114
+ self.wbits = wbits
115
+ self.groupsize = groupsize
116
+
117
+ self.bos_token_id = bos_token_id
118
+ self.eos_token_id = eos_token_id
119
+
120
+ super().__init__(
121
+ bos_token_id=bos_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs
122
+ )
123
+