Hugging Face's logo

theblackcat102
/
pythia-3b-deduped-sft-r1

Text Generation
Transformers
PyTorch
English
gpt_neox
sft
text-generation-inference
Inference Endpoints
Model card Files Community
theblackcat102 commited on
Commit
ee5b07d
•
1 Parent(s): c476bc0

Upload 5 files

Browse files
Files changed (5) hide show
  1. config.json +25 -0
  2. special_tokens_map.json +13 -0
  3. tokenizer.json +0 -0
  4. tokenizer_config.json +10 -0
  5. trainer_state.json +223 -0
config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "EleutherAI/pythia-2.8b-deduped",
3
+ "architectures": [
4
+ "GPTNeoXForCausalLM"
5
+ ],
6
+ "bos_token_id": 0,
7
+ "eos_token_id": 0,
8
+ "hidden_act": "gelu",
9
+ "hidden_size": 2560,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 10240,
12
+ "layer_norm_eps": 1e-05,
13
+ "max_position_embeddings": 2048,
14
+ "model_type": "gpt_neox",
15
+ "num_attention_heads": 32,
16
+ "num_hidden_layers": 32,
17
+ "rotary_emb_base": 10000,
18
+ "rotary_pct": 0.25,
19
+ "tie_word_embeddings": false,
20
+ "torch_dtype": "float16",
21
+ "transformers_version": "4.25.1",
22
+ "use_cache": true,
23
+ "use_parallel_residual": true,
24
+ "vocab_size": 50281
25
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<human>",
4
+ "<bot>",
5
+ "<prefix>",
6
+ "</prefix>"
7
+ ],
8
+ "bos_token": "<|endoftext|>",
9
+ "eos_token": "<|endoftext|>",
10
+ "pad_token": "<|padding|>",
11
+ "sep_token": "<|endoftext|>",
12
+ "unk_token": "<|endoftext|>"
13
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": "<|endoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "model_max_length": 1000000000000000019884624838656,
6
+ "name_or_path": "EleutherAI/pythia-2.8b-deduped",
7
+ "special_tokens_map_file": "/fsx/home-hailey/.cache/huggingface/hub/models--EleutherAI--gpt-neox-20b/snapshots/3523781c8df75f7741687a4284f6f70e1afa12f4/special_tokens_map.json",
8
+ "tokenizer_class": "GPTNeoXTokenizer",
9
+ "unk_token": "<|endoftext|>"
10
+ }
trainer_state.json ADDED
@@ -0,0 +1,223 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 9.090225563909774,
5
+ "global_step": 300,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.3,
12
+ "learning_rate": 4.999999999999999e-07,
13
+ "loss": 1.8578,
14
+ "step": 10
15
+ },
16
+ {
17
+ "epoch": 0.6,
18
+ "learning_rate": 6.505149978319905e-07,
19
+ "loss": 1.719,
20
+ "step": 20
21
+ },
22
+ {
23
+ "epoch": 0.9,
24
+ "learning_rate": 7.38560627359831e-07,
25
+ "loss": 1.7023,
26
+ "step": 30
27
+ },
28
+ {
29
+ "epoch": 1.21,
30
+ "learning_rate": 8.01029995663981e-07,
31
+ "loss": 1.7111,
32
+ "step": 40
33
+ },
34
+ {
35
+ "epoch": 1.51,
36
+ "learning_rate": 8.494850021680092e-07,
37
+ "loss": 1.6419,
38
+ "step": 50
39
+ },
40
+ {
41
+ "epoch": 1.81,
42
+ "learning_rate": 8.890756251918216e-07,
43
+ "loss": 1.5888,
44
+ "step": 60
45
+ },
46
+ {
47
+ "epoch": 2.12,
48
+ "learning_rate": 9.225490200071283e-07,
49
+ "loss": 1.5672,
50
+ "step": 70
51
+ },
52
+ {
53
+ "epoch": 2.42,
54
+ "learning_rate": 9.515449934959715e-07,
55
+ "loss": 1.525,
56
+ "step": 80
57
+ },
58
+ {
59
+ "epoch": 2.72,
60
+ "learning_rate": 9.771212547196622e-07,
61
+ "loss": 1.5381,
62
+ "step": 90
63
+ },
64
+ {
65
+ "epoch": 3.03,
66
+ "learning_rate": 9.999999999999997e-07,
67
+ "loss": 1.6184,
68
+ "step": 100
69
+ },
70
+ {
71
+ "epoch": 3.03,
72
+ "eval_oa_private_accuracy": 0.6486486486486487,
73
+ "eval_oa_private_loss": 1.4580078125,
74
+ "eval_oa_private_runtime": 2.0191,
75
+ "eval_oa_private_samples_per_second": 8.42,
76
+ "eval_oa_private_steps_per_second": 0.495,
77
+ "step": 100
78
+ },
79
+ {
80
+ "epoch": 3.33,
81
+ "learning_rate": 9.941935483870966e-07,
82
+ "loss": 1.4745,
83
+ "step": 110
84
+ },
85
+ {
86
+ "epoch": 3.63,
87
+ "learning_rate": 9.87741935483871e-07,
88
+ "loss": 1.5171,
89
+ "step": 120
90
+ },
91
+ {
92
+ "epoch": 3.93,
93
+ "learning_rate": 9.812903225806452e-07,
94
+ "loss": 1.5434,
95
+ "step": 130
96
+ },
97
+ {
98
+ "epoch": 4.24,
99
+ "learning_rate": 9.741935483870968e-07,
100
+ "loss": 1.5055,
101
+ "step": 140
102
+ },
103
+ {
104
+ "epoch": 4.54,
105
+ "learning_rate": 9.67741935483871e-07,
106
+ "loss": 1.4663,
107
+ "step": 150
108
+ },
109
+ {
110
+ "epoch": 4.84,
111
+ "learning_rate": 9.612903225806452e-07,
112
+ "loss": 1.4662,
113
+ "step": 160
114
+ },
115
+ {
116
+ "epoch": 5.15,
117
+ "learning_rate": 9.548387096774193e-07,
118
+ "loss": 1.4806,
119
+ "step": 170
120
+ },
121
+ {
122
+ "epoch": 5.45,
123
+ "learning_rate": 9.483870967741935e-07,
124
+ "loss": 1.4755,
125
+ "step": 180
126
+ },
127
+ {
128
+ "epoch": 5.75,
129
+ "learning_rate": 9.419354838709677e-07,
130
+ "loss": 1.4745,
131
+ "step": 190
132
+ },
133
+ {
134
+ "epoch": 6.06,
135
+ "learning_rate": 9.354838709677418e-07,
136
+ "loss": 1.4963,
137
+ "step": 200
138
+ },
139
+ {
140
+ "epoch": 6.06,
141
+ "eval_oa_private_accuracy": 0.6705276705276705,
142
+ "eval_oa_private_loss": 1.490234375,
143
+ "eval_oa_private_runtime": 0.8407,
144
+ "eval_oa_private_samples_per_second": 20.222,
145
+ "eval_oa_private_steps_per_second": 1.19,
146
+ "step": 200
147
+ },
148
+ {
149
+ "epoch": 6.36,
150
+ "learning_rate": 9.290322580645161e-07,
151
+ "loss": 1.4592,
152
+ "step": 210
153
+ },
154
+ {
155
+ "epoch": 6.66,
156
+ "learning_rate": 9.225806451612903e-07,
157
+ "loss": 1.447,
158
+ "step": 220
159
+ },
160
+ {
161
+ "epoch": 6.96,
162
+ "learning_rate": 9.161290322580644e-07,
163
+ "loss": 1.4527,
164
+ "step": 230
165
+ },
166
+ {
167
+ "epoch": 7.27,
168
+ "learning_rate": 9.096774193548387e-07,
169
+ "loss": 1.4361,
170
+ "step": 240
171
+ },
172
+ {
173
+ "epoch": 7.57,
174
+ "learning_rate": 9.032258064516129e-07,
175
+ "loss": 1.4065,
176
+ "step": 250
177
+ },
178
+ {
179
+ "epoch": 7.87,
180
+ "learning_rate": 8.96774193548387e-07,
181
+ "loss": 1.4299,
182
+ "step": 260
183
+ },
184
+ {
185
+ "epoch": 8.18,
186
+ "learning_rate": 8.896774193548387e-07,
187
+ "loss": 1.4363,
188
+ "step": 270
189
+ },
190
+ {
191
+ "epoch": 8.48,
192
+ "learning_rate": 8.832258064516129e-07,
193
+ "loss": 1.4161,
194
+ "step": 280
195
+ },
196
+ {
197
+ "epoch": 8.78,
198
+ "learning_rate": 8.767741935483871e-07,
199
+ "loss": 1.4356,
200
+ "step": 290
201
+ },
202
+ {
203
+ "epoch": 9.09,
204
+ "learning_rate": 8.703225806451613e-07,
205
+ "loss": 1.4495,
206
+ "step": 300
207
+ },
208
+ {
209
+ "epoch": 9.09,
210
+ "eval_oa_private_accuracy": 0.6664378860672615,
211
+ "eval_oa_private_loss": 1.3466796875,
212
+ "eval_oa_private_runtime": 0.9592,
213
+ "eval_oa_private_samples_per_second": 17.722,
214
+ "eval_oa_private_steps_per_second": 1.042,
215
+ "step": 300
216
+ }
217
+ ],
218
+ "max_steps": 1650,
219
+ "num_train_epochs": 50,
220
+ "total_flos": 8.233564448028099e+17,
221
+ "trial_name": null,
222
+ "trial_params": null
223
+ }