Commit
•
72da9b3
1
Parent(s):
190d22e
Update README.md
Browse files
README.md
CHANGED
|
@@ -8,6 +8,15 @@ Based on GPT-Neo architecture.
|
|
| 8 |
|
| 9 |
License: mit
|
| 10 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
------ EXAMPLE USAGE ---
|
| 12 |
|
| 13 |
from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
|
|
|
|
| 8 |
|
| 9 |
License: mit
|
| 10 |
|
| 11 |
+
---- hyperparams used to train this model ----
|
| 12 |
+
lr = 5e-4
|
| 13 |
+
lr_schedule = constant
|
| 14 |
+
wd=0.1
|
| 15 |
+
adam_beta1=0.9, adam_beta2 = 0.95
|
| 16 |
+
context length=512
|
| 17 |
+
batch size=80
|
| 18 |
+
gradient accumulation steps=16
|
| 19 |
+
|
| 20 |
------ EXAMPLE USAGE ---
|
| 21 |
|
| 22 |
from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
|