Commit
•
8744194
1
Parent(s):
73cb078
Upload 9 files
Browse files- config.json +51 -0
- configuration_olmo.py +1 -0
- modeling_olmo.py +1 -0
- requirements.txt +1 -0
- revisions.txt +558 -0
- special_tokens_map.json +4 -0
- tokenization_olmo_fast.py +1 -0
- tokenizer.json +0 -0
- tokenizer_config.json +242 -0
config.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_type": "swiglu",
|
| 3 |
+
"alibi": false,
|
| 4 |
+
"alibi_bias_max": 8.0,
|
| 5 |
+
"architectures": [
|
| 6 |
+
"OlmoModelForCausalLM"
|
| 7 |
+
],
|
| 8 |
+
"attention_dropout": 0.0,
|
| 9 |
+
"attention_layer_norm": false,
|
| 10 |
+
"attention_layer_norm_with_affine": false,
|
| 11 |
+
"bias_for_layer_norm": false,
|
| 12 |
+
"block_group_size": 1,
|
| 13 |
+
"block_type": "sequential",
|
| 14 |
+
"d_model": 4096,
|
| 15 |
+
"embedding_dropout": 0.0,
|
| 16 |
+
"embedding_size": 50304,
|
| 17 |
+
"eos_token_id": 50279,
|
| 18 |
+
"flash_attention": true,
|
| 19 |
+
"include_bias": false,
|
| 20 |
+
"init_cutoff_factor": null,
|
| 21 |
+
"init_device": "meta",
|
| 22 |
+
"init_fn": "mitchell",
|
| 23 |
+
"init_std": 0.02,
|
| 24 |
+
"layer_norm_type": "default",
|
| 25 |
+
"layer_norm_with_affine": false,
|
| 26 |
+
"max_sequence_length": 2048,
|
| 27 |
+
"mlp_hidden_size": 22016,
|
| 28 |
+
"mlp_ratio": 4,
|
| 29 |
+
"model_type": "olmo",
|
| 30 |
+
"multi_query_attention": false,
|
| 31 |
+
"n_heads": 32,
|
| 32 |
+
"n_layers": 32,
|
| 33 |
+
"pad_token_id": 1,
|
| 34 |
+
"precision": "amp_bf16",
|
| 35 |
+
"residual_dropout": 0.0,
|
| 36 |
+
"rope": true,
|
| 37 |
+
"rope_full_precision": true,
|
| 38 |
+
"scale_logits": false,
|
| 39 |
+
"transformers_version": "4.36.2",
|
| 40 |
+
"use_cache": true,
|
| 41 |
+
"vocab_size": 50280,
|
| 42 |
+
"weight_tying": false,
|
| 43 |
+
"auto_map": {
|
| 44 |
+
"AutoConfig": "configuration_olmo.OLMoConfig",
|
| 45 |
+
"AutoModelForCausalLM": "modeling_olmo.OLMoForCausalLM",
|
| 46 |
+
"AutoTokenizer": [
|
| 47 |
+
"tokenization_olmo_fast.OLMoTokenizerFast",
|
| 48 |
+
"tokenization_olmo_fast.OLMoTokenizerFast"
|
| 49 |
+
]
|
| 50 |
+
}
|
| 51 |
+
}
|
configuration_olmo.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
from hf_olmo import OLMoConfig
|
modeling_olmo.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
from hf_olmo import OLMoForCausalLM
|
requirements.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
ai2-olmo>=0.2.2
|
revisions.txt
ADDED
|
@@ -0,0 +1,558 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
step0-tokens0B
|
| 2 |
+
step1000-tokens4B
|
| 3 |
+
step2000-tokens9B
|
| 4 |
+
step3000-tokens13B
|
| 5 |
+
step4000-tokens18B
|
| 6 |
+
step5000-tokens22B
|
| 7 |
+
step6000-tokens27B
|
| 8 |
+
step7000-tokens31B
|
| 9 |
+
step8000-tokens35B
|
| 10 |
+
step9000-tokens40B
|
| 11 |
+
step10000-tokens44B
|
| 12 |
+
step11000-tokens49B
|
| 13 |
+
step12000-tokens53B
|
| 14 |
+
step13000-tokens58B
|
| 15 |
+
step14000-tokens62B
|
| 16 |
+
step15000-tokens66B
|
| 17 |
+
step16000-tokens71B
|
| 18 |
+
step17000-tokens75B
|
| 19 |
+
step18000-tokens80B
|
| 20 |
+
step19000-tokens84B
|
| 21 |
+
step20000-tokens88B
|
| 22 |
+
step21000-tokens93B
|
| 23 |
+
step22000-tokens97B
|
| 24 |
+
step23000-tokens102B
|
| 25 |
+
step24000-tokens106B
|
| 26 |
+
step25000-tokens111B
|
| 27 |
+
step26000-tokens115B
|
| 28 |
+
step27000-tokens119B
|
| 29 |
+
step28000-tokens124B
|
| 30 |
+
step29000-tokens128B
|
| 31 |
+
step30000-tokens133B
|
| 32 |
+
step31000-tokens137B
|
| 33 |
+
step32000-tokens142B
|
| 34 |
+
step33000-tokens146B
|
| 35 |
+
step34000-tokens150B
|
| 36 |
+
step35000-tokens155B
|
| 37 |
+
step36000-tokens159B
|
| 38 |
+
step37000-tokens164B
|
| 39 |
+
step38000-tokens168B
|
| 40 |
+
step39000-tokens173B
|
| 41 |
+
step40000-tokens177B
|
| 42 |
+
step41000-tokens181B
|
| 43 |
+
step42000-tokens186B
|
| 44 |
+
step43000-tokens190B
|
| 45 |
+
step44000-tokens195B
|
| 46 |
+
step45000-tokens199B
|
| 47 |
+
step46000-tokens203B
|
| 48 |
+
step47000-tokens208B
|
| 49 |
+
step48000-tokens212B
|
| 50 |
+
step49000-tokens217B
|
| 51 |
+
step50000-tokens221B
|
| 52 |
+
step51000-tokens226B
|
| 53 |
+
step52000-tokens230B
|
| 54 |
+
step53000-tokens234B
|
| 55 |
+
step54000-tokens239B
|
| 56 |
+
step55000-tokens243B
|
| 57 |
+
step56000-tokens248B
|
| 58 |
+
step57000-tokens252B
|
| 59 |
+
step58000-tokens257B
|
| 60 |
+
step59000-tokens261B
|
| 61 |
+
step60000-tokens265B
|
| 62 |
+
step61000-tokens270B
|
| 63 |
+
step62000-tokens274B
|
| 64 |
+
step63000-tokens279B
|
| 65 |
+
step64000-tokens283B
|
| 66 |
+
step65000-tokens288B
|
| 67 |
+
step66000-tokens292B
|
| 68 |
+
step67000-tokens296B
|
| 69 |
+
step68000-tokens301B
|
| 70 |
+
step69000-tokens305B
|
| 71 |
+
step70000-tokens310B
|
| 72 |
+
step71000-tokens314B
|
| 73 |
+
step72000-tokens319B
|
| 74 |
+
step73000-tokens323B
|
| 75 |
+
step74000-tokens327B
|
| 76 |
+
step75000-tokens332B
|
| 77 |
+
step76000-tokens336B
|
| 78 |
+
step77000-tokens341B
|
| 79 |
+
step78000-tokens345B
|
| 80 |
+
step79000-tokens349B
|
| 81 |
+
step80000-tokens354B
|
| 82 |
+
step81000-tokens358B
|
| 83 |
+
step82000-tokens363B
|
| 84 |
+
step83000-tokens367B
|
| 85 |
+
step84000-tokens372B
|
| 86 |
+
step85000-tokens376B
|
| 87 |
+
step86000-tokens380B
|
| 88 |
+
step87000-tokens385B
|
| 89 |
+
step88000-tokens389B
|
| 90 |
+
step89000-tokens394B
|
| 91 |
+
step90000-tokens398B
|
| 92 |
+
step91000-tokens403B
|
| 93 |
+
step92000-tokens407B
|
| 94 |
+
step93000-tokens411B
|
| 95 |
+
step94000-tokens416B
|
| 96 |
+
step95000-tokens420B
|
| 97 |
+
step96000-tokens425B
|
| 98 |
+
step97000-tokens429B
|
| 99 |
+
step98000-tokens434B
|
| 100 |
+
step99000-tokens438B
|
| 101 |
+
step100000-tokens442B
|
| 102 |
+
step101000-tokens447B
|
| 103 |
+
step102000-tokens451B
|
| 104 |
+
step103000-tokens456B
|
| 105 |
+
step104000-tokens460B
|
| 106 |
+
step105000-tokens464B
|
| 107 |
+
step106000-tokens469B
|
| 108 |
+
step107000-tokens473B
|
| 109 |
+
step108000-tokens478B
|
| 110 |
+
step109000-tokens482B
|
| 111 |
+
step110000-tokens487B
|
| 112 |
+
step111000-tokens491B
|
| 113 |
+
step112000-tokens495B
|
| 114 |
+
step113000-tokens500B
|
| 115 |
+
step114000-tokens504B
|
| 116 |
+
step115000-tokens509B
|
| 117 |
+
step116000-tokens513B
|
| 118 |
+
step117000-tokens518B
|
| 119 |
+
step118000-tokens522B
|
| 120 |
+
step119000-tokens526B
|
| 121 |
+
step120000-tokens531B
|
| 122 |
+
step121000-tokens535B
|
| 123 |
+
step122000-tokens540B
|
| 124 |
+
step123000-tokens544B
|
| 125 |
+
step124000-tokens549B
|
| 126 |
+
step125000-tokens553B
|
| 127 |
+
step126000-tokens557B
|
| 128 |
+
step127000-tokens562B
|
| 129 |
+
step128000-tokens566B
|
| 130 |
+
step129000-tokens571B
|
| 131 |
+
step130000-tokens575B
|
| 132 |
+
step131000-tokens580B
|
| 133 |
+
step132000-tokens584B
|
| 134 |
+
step133000-tokens588B
|
| 135 |
+
step134000-tokens593B
|
| 136 |
+
step135000-tokens597B
|
| 137 |
+
step136000-tokens602B
|
| 138 |
+
step137000-tokens606B
|
| 139 |
+
step138000-tokens610B
|
| 140 |
+
step139000-tokens615B
|
| 141 |
+
step140000-tokens619B
|
| 142 |
+
step141000-tokens624B
|
| 143 |
+
step142000-tokens628B
|
| 144 |
+
step143000-tokens633B
|
| 145 |
+
step144000-tokens637B
|
| 146 |
+
step145000-tokens641B
|
| 147 |
+
step146000-tokens646B
|
| 148 |
+
step147000-tokens650B
|
| 149 |
+
step148000-tokens655B
|
| 150 |
+
step149000-tokens659B
|
| 151 |
+
step150000-tokens664B
|
| 152 |
+
step151000-tokens668B
|
| 153 |
+
step152000-tokens672B
|
| 154 |
+
step153000-tokens677B
|
| 155 |
+
step154000-tokens681B
|
| 156 |
+
step155000-tokens686B
|
| 157 |
+
step156000-tokens690B
|
| 158 |
+
step157000-tokens695B
|
| 159 |
+
step158000-tokens699B
|
| 160 |
+
step159000-tokens703B
|
| 161 |
+
step160000-tokens708B
|
| 162 |
+
step161000-tokens712B
|
| 163 |
+
step162000-tokens717B
|
| 164 |
+
step163000-tokens721B
|
| 165 |
+
step164000-tokens725B
|
| 166 |
+
step165000-tokens730B
|
| 167 |
+
step166000-tokens734B
|
| 168 |
+
step167000-tokens739B
|
| 169 |
+
step168000-tokens743B
|
| 170 |
+
step169000-tokens748B
|
| 171 |
+
step170000-tokens752B
|
| 172 |
+
step171000-tokens756B
|
| 173 |
+
step172000-tokens761B
|
| 174 |
+
step173000-tokens765B
|
| 175 |
+
step174000-tokens770B
|
| 176 |
+
step175000-tokens774B
|
| 177 |
+
step176000-tokens779B
|
| 178 |
+
step177000-tokens783B
|
| 179 |
+
step178000-tokens787B
|
| 180 |
+
step179000-tokens792B
|
| 181 |
+
step180000-tokens796B
|
| 182 |
+
step181000-tokens801B
|
| 183 |
+
step182000-tokens805B
|
| 184 |
+
step183000-tokens810B
|
| 185 |
+
step184000-tokens814B
|
| 186 |
+
step185000-tokens818B
|
| 187 |
+
step186000-tokens823B
|
| 188 |
+
step187000-tokens827B
|
| 189 |
+
step188000-tokens832B
|
| 190 |
+
step189000-tokens836B
|
| 191 |
+
step190000-tokens840B
|
| 192 |
+
step191000-tokens845B
|
| 193 |
+
step192000-tokens849B
|
| 194 |
+
step193000-tokens854B
|
| 195 |
+
step194000-tokens858B
|
| 196 |
+
step195000-tokens863B
|
| 197 |
+
step196000-tokens867B
|
| 198 |
+
step197000-tokens871B
|
| 199 |
+
step198000-tokens876B
|
| 200 |
+
step199000-tokens880B
|
| 201 |
+
step200000-tokens885B
|
| 202 |
+
step201000-tokens889B
|
| 203 |
+
step202000-tokens894B
|
| 204 |
+
step203000-tokens898B
|
| 205 |
+
step204000-tokens902B
|
| 206 |
+
step205000-tokens907B
|
| 207 |
+
step206000-tokens911B
|
| 208 |
+
step207000-tokens916B
|
| 209 |
+
step208000-tokens920B
|
| 210 |
+
step209000-tokens925B
|
| 211 |
+
step210000-tokens929B
|
| 212 |
+
step211000-tokens933B
|
| 213 |
+
step212000-tokens938B
|
| 214 |
+
step213000-tokens942B
|
| 215 |
+
step214000-tokens947B
|
| 216 |
+
step215000-tokens951B
|
| 217 |
+
step216000-tokens956B
|
| 218 |
+
step217000-tokens960B
|
| 219 |
+
step218000-tokens964B
|
| 220 |
+
step219000-tokens969B
|
| 221 |
+
step220000-tokens973B
|
| 222 |
+
step221000-tokens978B
|
| 223 |
+
step222000-tokens982B
|
| 224 |
+
step223000-tokens986B
|
| 225 |
+
step224000-tokens991B
|
| 226 |
+
step225000-tokens995B
|
| 227 |
+
step226000-tokens1000B
|
| 228 |
+
step227000-tokens1004B
|
| 229 |
+
step228000-tokens1009B
|
| 230 |
+
step229000-tokens1013B
|
| 231 |
+
step230000-tokens1017B
|
| 232 |
+
step231000-tokens1022B
|
| 233 |
+
step232000-tokens1026B
|
| 234 |
+
step233000-tokens1031B
|
| 235 |
+
step234000-tokens1035B
|
| 236 |
+
step235000-tokens1040B
|
| 237 |
+
step236000-tokens1044B
|
| 238 |
+
step237000-tokens1048B
|
| 239 |
+
step238000-tokens1053B
|
| 240 |
+
step239000-tokens1057B
|
| 241 |
+
step240000-tokens1062B
|
| 242 |
+
step241000-tokens1066B
|
| 243 |
+
step242000-tokens1071B
|
| 244 |
+
step243000-tokens1075B
|
| 245 |
+
step244000-tokens1079B
|
| 246 |
+
step245000-tokens1084B
|
| 247 |
+
step246000-tokens1088B
|
| 248 |
+
step247000-tokens1093B
|
| 249 |
+
step248000-tokens1097B
|
| 250 |
+
step249000-tokens1101B
|
| 251 |
+
step250000-tokens1106B
|
| 252 |
+
step251000-tokens1110B
|
| 253 |
+
step252000-tokens1115B
|
| 254 |
+
step253000-tokens1119B
|
| 255 |
+
step254000-tokens1124B
|
| 256 |
+
step255000-tokens1128B
|
| 257 |
+
step256000-tokens1132B
|
| 258 |
+
step257000-tokens1137B
|
| 259 |
+
step258000-tokens1141B
|
| 260 |
+
step259000-tokens1146B
|
| 261 |
+
step260000-tokens1150B
|
| 262 |
+
step261000-tokens1155B
|
| 263 |
+
step262000-tokens1159B
|
| 264 |
+
step263000-tokens1163B
|
| 265 |
+
step264000-tokens1168B
|
| 266 |
+
step265000-tokens1172B
|
| 267 |
+
step266000-tokens1177B
|
| 268 |
+
step267000-tokens1181B
|
| 269 |
+
step268000-tokens1186B
|
| 270 |
+
step269000-tokens1190B
|
| 271 |
+
step270000-tokens1194B
|
| 272 |
+
step271000-tokens1199B
|
| 273 |
+
step272000-tokens1203B
|
| 274 |
+
step273000-tokens1208B
|
| 275 |
+
step274000-tokens1212B
|
| 276 |
+
step275000-tokens1217B
|
| 277 |
+
step276000-tokens1221B
|
| 278 |
+
step277000-tokens1225B
|
| 279 |
+
step278000-tokens1230B
|
| 280 |
+
step279000-tokens1234B
|
| 281 |
+
step280000-tokens1239B
|
| 282 |
+
step281000-tokens1243B
|
| 283 |
+
step282000-tokens1247B
|
| 284 |
+
step283000-tokens1252B
|
| 285 |
+
step284000-tokens1256B
|
| 286 |
+
step285000-tokens1261B
|
| 287 |
+
step286000-tokens1265B
|
| 288 |
+
step287000-tokens1270B
|
| 289 |
+
step288000-tokens1274B
|
| 290 |
+
step289000-tokens1278B
|
| 291 |
+
step290000-tokens1283B
|
| 292 |
+
step291000-tokens1287B
|
| 293 |
+
step292000-tokens1292B
|
| 294 |
+
step293000-tokens1296B
|
| 295 |
+
step294000-tokens1301B
|
| 296 |
+
step295000-tokens1305B
|
| 297 |
+
step296000-tokens1309B
|
| 298 |
+
step297000-tokens1314B
|
| 299 |
+
step298000-tokens1318B
|
| 300 |
+
step299000-tokens1323B
|
| 301 |
+
step300000-tokens1327B
|
| 302 |
+
step301000-tokens1332B
|
| 303 |
+
step302000-tokens1336B
|
| 304 |
+
step303000-tokens1340B
|
| 305 |
+
step304000-tokens1345B
|
| 306 |
+
step305000-tokens1349B
|
| 307 |
+
step306000-tokens1354B
|
| 308 |
+
step307000-tokens1358B
|
| 309 |
+
step308000-tokens1362B
|
| 310 |
+
step309000-tokens1367B
|
| 311 |
+
step310000-tokens1371B
|
| 312 |
+
step311000-tokens1376B
|
| 313 |
+
step312000-tokens1380B
|
| 314 |
+
step313000-tokens1385B
|
| 315 |
+
step314000-tokens1389B
|
| 316 |
+
step315000-tokens1393B
|
| 317 |
+
step316000-tokens1398B
|
| 318 |
+
step317000-tokens1402B
|
| 319 |
+
step318000-tokens1407B
|
| 320 |
+
step319000-tokens1411B
|
| 321 |
+
step320000-tokens1416B
|
| 322 |
+
step321000-tokens1420B
|
| 323 |
+
step322000-tokens1424B
|
| 324 |
+
step323000-tokens1429B
|
| 325 |
+
step324000-tokens1433B
|
| 326 |
+
step325000-tokens1438B
|
| 327 |
+
step326000-tokens1442B
|
| 328 |
+
step327000-tokens1447B
|
| 329 |
+
step328000-tokens1451B
|
| 330 |
+
step329000-tokens1455B
|
| 331 |
+
step330000-tokens1460B
|
| 332 |
+
step331000-tokens1464B
|
| 333 |
+
step332000-tokens1469B
|
| 334 |
+
step333000-tokens1473B
|
| 335 |
+
step334000-tokens1478B
|
| 336 |
+
step335000-tokens1482B
|
| 337 |
+
step336000-tokens1486B
|
| 338 |
+
step337000-tokens1491B
|
| 339 |
+
step338000-tokens1495B
|
| 340 |
+
step339000-tokens1500B
|
| 341 |
+
step340000-tokens1504B
|
| 342 |
+
step341000-tokens1508B
|
| 343 |
+
step342000-tokens1513B
|
| 344 |
+
step343000-tokens1517B
|
| 345 |
+
step344000-tokens1522B
|
| 346 |
+
step345000-tokens1526B
|
| 347 |
+
step346000-tokens1531B
|
| 348 |
+
step347000-tokens1535B
|
| 349 |
+
step348000-tokens1539B
|
| 350 |
+
step349000-tokens1544B
|
| 351 |
+
step350000-tokens1548B
|
| 352 |
+
step351000-tokens1553B
|
| 353 |
+
step352000-tokens1557B
|
| 354 |
+
step353000-tokens1562B
|
| 355 |
+
step354000-tokens1566B
|
| 356 |
+
step355000-tokens1570B
|
| 357 |
+
step356000-tokens1575B
|
| 358 |
+
step357000-tokens1579B
|
| 359 |
+
step358000-tokens1584B
|
| 360 |
+
step359000-tokens1588B
|
| 361 |
+
step360000-tokens1593B
|
| 362 |
+
step361000-tokens1597B
|
| 363 |
+
step362000-tokens1601B
|
| 364 |
+
step363000-tokens1606B
|
| 365 |
+
step364000-tokens1610B
|
| 366 |
+
step365000-tokens1615B
|
| 367 |
+
step366000-tokens1619B
|
| 368 |
+
step367000-tokens1623B
|
| 369 |
+
step368000-tokens1628B
|
| 370 |
+
step369000-tokens1632B
|
| 371 |
+
step370000-tokens1637B
|
| 372 |
+
step371000-tokens1641B
|
| 373 |
+
step372000-tokens1646B
|
| 374 |
+
step373000-tokens1650B
|
| 375 |
+
step374000-tokens1654B
|
| 376 |
+
step375000-tokens1659B
|
| 377 |
+
step376000-tokens1663B
|
| 378 |
+
step377000-tokens1668B
|
| 379 |
+
step378000-tokens1672B
|
| 380 |
+
step379000-tokens1677B
|
| 381 |
+
step380000-tokens1681B
|
| 382 |
+
step381000-tokens1685B
|
| 383 |
+
step382000-tokens1690B
|
| 384 |
+
step383000-tokens1694B
|
| 385 |
+
step384000-tokens1699B
|
| 386 |
+
step385000-tokens1703B
|
| 387 |
+
step386000-tokens1708B
|
| 388 |
+
step387000-tokens1712B
|
| 389 |
+
step388000-tokens1716B
|
| 390 |
+
step389000-tokens1721B
|
| 391 |
+
step390000-tokens1725B
|
| 392 |
+
step391000-tokens1730B
|
| 393 |
+
step392000-tokens1734B
|
| 394 |
+
step393000-tokens1739B
|
| 395 |
+
step394000-tokens1743B
|
| 396 |
+
step395000-tokens1747B
|
| 397 |
+
step396000-tokens1752B
|
| 398 |
+
step397000-tokens1756B
|
| 399 |
+
step398000-tokens1761B
|
| 400 |
+
step399000-tokens1765B
|
| 401 |
+
step400000-tokens1769B
|
| 402 |
+
step401000-tokens1774B
|
| 403 |
+
step402000-tokens1778B
|
| 404 |
+
step403000-tokens1783B
|
| 405 |
+
step404000-tokens1787B
|
| 406 |
+
step405000-tokens1792B
|
| 407 |
+
step406000-tokens1796B
|
| 408 |
+
step407000-tokens1800B
|
| 409 |
+
step408000-tokens1805B
|
| 410 |
+
step409000-tokens1809B
|
| 411 |
+
step410000-tokens1814B
|
| 412 |
+
step411000-tokens1818B
|
| 413 |
+
step412000-tokens1823B
|
| 414 |
+
step413000-tokens1827B
|
| 415 |
+
step414000-tokens1831B
|
| 416 |
+
step415000-tokens1836B
|
| 417 |
+
step416000-tokens1840B
|
| 418 |
+
step417000-tokens1845B
|
| 419 |
+
step418000-tokens1849B
|
| 420 |
+
step419000-tokens1854B
|
| 421 |
+
step420000-tokens1858B
|
| 422 |
+
step421000-tokens1862B
|
| 423 |
+
step422000-tokens1867B
|
| 424 |
+
step423000-tokens1871B
|
| 425 |
+
step424000-tokens1876B
|
| 426 |
+
step425000-tokens1880B
|
| 427 |
+
step426000-tokens1884B
|
| 428 |
+
step427000-tokens1889B
|
| 429 |
+
step428000-tokens1893B
|
| 430 |
+
step429000-tokens1898B
|
| 431 |
+
step430000-tokens1902B
|
| 432 |
+
step431000-tokens1907B
|
| 433 |
+
step432000-tokens1911B
|
| 434 |
+
step433000-tokens1915B
|
| 435 |
+
step434000-tokens1920B
|
| 436 |
+
step435000-tokens1924B
|
| 437 |
+
step436000-tokens1929B
|
| 438 |
+
step437000-tokens1933B
|
| 439 |
+
step438000-tokens1938B
|
| 440 |
+
step439000-tokens1942B
|
| 441 |
+
step440000-tokens1946B
|
| 442 |
+
step441000-tokens1951B
|
| 443 |
+
step442000-tokens1955B
|
| 444 |
+
step443000-tokens1960B
|
| 445 |
+
step444000-tokens1964B
|
| 446 |
+
step445000-tokens1969B
|
| 447 |
+
step446000-tokens1973B
|
| 448 |
+
step447000-tokens1977B
|
| 449 |
+
step448000-tokens1982B
|
| 450 |
+
step449000-tokens1986B
|
| 451 |
+
step450000-tokens1991B
|
| 452 |
+
step451000-tokens1995B
|
| 453 |
+
step452000-tokens2000B
|
| 454 |
+
step453000-tokens2004B
|
| 455 |
+
step454000-tokens2008B
|
| 456 |
+
step455000-tokens2013B
|
| 457 |
+
step456000-tokens2017B
|
| 458 |
+
step457000-tokens2022B
|
| 459 |
+
step458000-tokens2026B
|
| 460 |
+
step459000-tokens2030B
|
| 461 |
+
step460000-tokens2035B
|
| 462 |
+
step461000-tokens2039B
|
| 463 |
+
step462000-tokens2044B
|
| 464 |
+
step463000-tokens2048B
|
| 465 |
+
step464000-tokens2053B
|
| 466 |
+
step465000-tokens2057B
|
| 467 |
+
step466000-tokens2061B
|
| 468 |
+
step467000-tokens2066B
|
| 469 |
+
step468000-tokens2070B
|
| 470 |
+
step469000-tokens2075B
|
| 471 |
+
step470000-tokens2079B
|
| 472 |
+
step471000-tokens2084B
|
| 473 |
+
step472000-tokens2088B
|
| 474 |
+
step473000-tokens2092B
|
| 475 |
+
step474000-tokens2097B
|
| 476 |
+
step475000-tokens2101B
|
| 477 |
+
step476000-tokens2106B
|
| 478 |
+
step477000-tokens2110B
|
| 479 |
+
step478000-tokens2115B
|
| 480 |
+
step479000-tokens2119B
|
| 481 |
+
step480000-tokens2123B
|
| 482 |
+
step481000-tokens2128B
|
| 483 |
+
step482000-tokens2132B
|
| 484 |
+
step483000-tokens2137B
|
| 485 |
+
step484000-tokens2141B
|
| 486 |
+
step485000-tokens2145B
|
| 487 |
+
step486000-tokens2150B
|
| 488 |
+
step487000-tokens2154B
|
| 489 |
+
step488000-tokens2159B
|
| 490 |
+
step489000-tokens2163B
|
| 491 |
+
step490000-tokens2168B
|
| 492 |
+
step491000-tokens2172B
|
| 493 |
+
step492000-tokens2176B
|
| 494 |
+
step493000-tokens2181B
|
| 495 |
+
step494000-tokens2185B
|
| 496 |
+
step495000-tokens2190B
|
| 497 |
+
step496000-tokens2194B
|
| 498 |
+
step497000-tokens2199B
|
| 499 |
+
step498000-tokens2203B
|
| 500 |
+
step499000-tokens2207B
|
| 501 |
+
step500000-tokens2212B
|
| 502 |
+
step501000-tokens2216B
|
| 503 |
+
step502000-tokens2221B
|
| 504 |
+
step503000-tokens2225B
|
| 505 |
+
step504000-tokens2230B
|
| 506 |
+
step505000-tokens2234B
|
| 507 |
+
step506000-tokens2238B
|
| 508 |
+
step507000-tokens2243B
|
| 509 |
+
step508000-tokens2247B
|
| 510 |
+
step509000-tokens2252B
|
| 511 |
+
step510000-tokens2256B
|
| 512 |
+
step511000-tokens2261B
|
| 513 |
+
step512000-tokens2265B
|
| 514 |
+
step513000-tokens2269B
|
| 515 |
+
step514000-tokens2274B
|
| 516 |
+
step515000-tokens2278B
|
| 517 |
+
step516000-tokens2283B
|
| 518 |
+
step517000-tokens2287B
|
| 519 |
+
step518000-tokens2291B
|
| 520 |
+
step519000-tokens2296B
|
| 521 |
+
step520000-tokens2300B
|
| 522 |
+
step521000-tokens2305B
|
| 523 |
+
step522000-tokens2309B
|
| 524 |
+
step523000-tokens2314B
|
| 525 |
+
step524000-tokens2318B
|
| 526 |
+
step525000-tokens2322B
|
| 527 |
+
step526000-tokens2327B
|
| 528 |
+
step527000-tokens2331B
|
| 529 |
+
step528000-tokens2336B
|
| 530 |
+
step529000-tokens2340B
|
| 531 |
+
step530000-tokens2345B
|
| 532 |
+
step531000-tokens2349B
|
| 533 |
+
step532000-tokens2353B
|
| 534 |
+
step533000-tokens2358B
|
| 535 |
+
step534000-tokens2362B
|
| 536 |
+
step535000-tokens2367B
|
| 537 |
+
step536000-tokens2371B
|
| 538 |
+
step537000-tokens2376B
|
| 539 |
+
step538000-tokens2380B
|
| 540 |
+
step539000-tokens2384B
|
| 541 |
+
step540000-tokens2389B
|
| 542 |
+
step541000-tokens2393B
|
| 543 |
+
step542000-tokens2398B
|
| 544 |
+
step543000-tokens2402B
|
| 545 |
+
step544000-tokens2406B
|
| 546 |
+
step545000-tokens2411B
|
| 547 |
+
step546000-tokens2415B
|
| 548 |
+
step547000-tokens2420B
|
| 549 |
+
step548000-tokens2424B
|
| 550 |
+
step549000-tokens2429B
|
| 551 |
+
step550000-tokens2433B
|
| 552 |
+
step551000-tokens2437B
|
| 553 |
+
step552000-tokens2442B
|
| 554 |
+
step553000-tokens2446B
|
| 555 |
+
step554000-tokens2451B
|
| 556 |
+
step555000-tokens2455B
|
| 557 |
+
step556000-tokens2460B
|
| 558 |
+
step557000-tokens2464B
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"eos_token": "<|endoftext|>",
|
| 3 |
+
"pad_token": "<|padding|>"
|
| 4 |
+
}
|
tokenization_olmo_fast.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
from hf_olmo.tokenization_olmo_fast import OLMoTokenizerFast
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"added_tokens_decoder": {
|
| 3 |
+
"0": {
|
| 4 |
+
"content": "|||IP_ADDRESS|||",
|
| 5 |
+
"lstrip": false,
|
| 6 |
+
"normalized": true,
|
| 7 |
+
"rstrip": false,
|
| 8 |
+
"single_word": false,
|
| 9 |
+
"special": false
|
| 10 |
+
},
|
| 11 |
+
"1": {
|
| 12 |
+
"content": "<|padding|>",
|
| 13 |
+
"lstrip": false,
|
| 14 |
+
"normalized": false,
|
| 15 |
+
"rstrip": false,
|
| 16 |
+
"single_word": false,
|
| 17 |
+
"special": true
|
| 18 |
+
},
|
| 19 |
+
"50254": {
|
| 20 |
+
"content": " ",
|
| 21 |
+
"lstrip": false,
|
| 22 |
+
"normalized": true,
|
| 23 |
+
"rstrip": false,
|
| 24 |
+
"single_word": false,
|
| 25 |
+
"special": false
|
| 26 |
+
},
|
| 27 |
+
"50255": {
|
| 28 |
+
"content": " ",
|
| 29 |
+
"lstrip": false,
|
| 30 |
+
"normalized": true,
|
| 31 |
+
"rstrip": false,
|
| 32 |
+
"single_word": false,
|
| 33 |
+
"special": false
|
| 34 |
+
},
|
| 35 |
+
"50256": {
|
| 36 |
+
"content": " ",
|
| 37 |
+
"lstrip": false,
|
| 38 |
+
"normalized": true,
|
| 39 |
+
"rstrip": false,
|
| 40 |
+
"single_word": false,
|
| 41 |
+
"special": false
|
| 42 |
+
},
|
| 43 |
+
"50257": {
|
| 44 |
+
"content": " ",
|
| 45 |
+
"lstrip": false,
|
| 46 |
+
"normalized": true,
|
| 47 |
+
"rstrip": false,
|
| 48 |
+
"single_word": false,
|
| 49 |
+
"special": false
|
| 50 |
+
},
|
| 51 |
+
"50258": {
|
| 52 |
+
"content": " ",
|
| 53 |
+
"lstrip": false,
|
| 54 |
+
"normalized": true,
|
| 55 |
+
"rstrip": false,
|
| 56 |
+
"single_word": false,
|
| 57 |
+
"special": false
|
| 58 |
+
},
|
| 59 |
+
"50259": {
|
| 60 |
+
"content": " ",
|
| 61 |
+
"lstrip": false,
|
| 62 |
+
"normalized": true,
|
| 63 |
+
"rstrip": false,
|
| 64 |
+
"single_word": false,
|
| 65 |
+
"special": false
|
| 66 |
+
},
|
| 67 |
+
"50260": {
|
| 68 |
+
"content": " ",
|
| 69 |
+
"lstrip": false,
|
| 70 |
+
"normalized": true,
|
| 71 |
+
"rstrip": false,
|
| 72 |
+
"single_word": false,
|
| 73 |
+
"special": false
|
| 74 |
+
},
|
| 75 |
+
"50261": {
|
| 76 |
+
"content": " ",
|
| 77 |
+
"lstrip": false,
|
| 78 |
+
"normalized": true,
|
| 79 |
+
"rstrip": false,
|
| 80 |
+
"single_word": false,
|
| 81 |
+
"special": false
|
| 82 |
+
},
|
| 83 |
+
"50262": {
|
| 84 |
+
"content": " ",
|
| 85 |
+
"lstrip": false,
|
| 86 |
+
"normalized": true,
|
| 87 |
+
"rstrip": false,
|
| 88 |
+
"single_word": false,
|
| 89 |
+
"special": false
|
| 90 |
+
},
|
| 91 |
+
"50263": {
|
| 92 |
+
"content": " ",
|
| 93 |
+
"lstrip": false,
|
| 94 |
+
"normalized": true,
|
| 95 |
+
"rstrip": false,
|
| 96 |
+
"single_word": false,
|
| 97 |
+
"special": false
|
| 98 |
+
},
|
| 99 |
+
"50264": {
|
| 100 |
+
"content": " ",
|
| 101 |
+
"lstrip": false,
|
| 102 |
+
"normalized": true,
|
| 103 |
+
"rstrip": false,
|
| 104 |
+
"single_word": false,
|
| 105 |
+
"special": false
|
| 106 |
+
},
|
| 107 |
+
"50265": {
|
| 108 |
+
"content": " ",
|
| 109 |
+
"lstrip": false,
|
| 110 |
+
"normalized": true,
|
| 111 |
+
"rstrip": false,
|
| 112 |
+
"single_word": false,
|
| 113 |
+
"special": false
|
| 114 |
+
},
|
| 115 |
+
"50266": {
|
| 116 |
+
"content": " ",
|
| 117 |
+
"lstrip": false,
|
| 118 |
+
"normalized": true,
|
| 119 |
+
"rstrip": false,
|
| 120 |
+
"single_word": false,
|
| 121 |
+
"special": false
|
| 122 |
+
},
|
| 123 |
+
"50267": {
|
| 124 |
+
"content": " ",
|
| 125 |
+
"lstrip": false,
|
| 126 |
+
"normalized": true,
|
| 127 |
+
"rstrip": false,
|
| 128 |
+
"single_word": false,
|
| 129 |
+
"special": false
|
| 130 |
+
},
|
| 131 |
+
"50268": {
|
| 132 |
+
"content": " ",
|
| 133 |
+
"lstrip": false,
|
| 134 |
+
"normalized": true,
|
| 135 |
+
"rstrip": false,
|
| 136 |
+
"single_word": false,
|
| 137 |
+
"special": false
|
| 138 |
+
},
|
| 139 |
+
"50269": {
|
| 140 |
+
"content": " ",
|
| 141 |
+
"lstrip": false,
|
| 142 |
+
"normalized": true,
|
| 143 |
+
"rstrip": false,
|
| 144 |
+
"single_word": false,
|
| 145 |
+
"special": false
|
| 146 |
+
},
|
| 147 |
+
"50270": {
|
| 148 |
+
"content": " ",
|
| 149 |
+
"lstrip": false,
|
| 150 |
+
"normalized": true,
|
| 151 |
+
"rstrip": false,
|
| 152 |
+
"single_word": false,
|
| 153 |
+
"special": false
|
| 154 |
+
},
|
| 155 |
+
"50271": {
|
| 156 |
+
"content": " ",
|
| 157 |
+
"lstrip": false,
|
| 158 |
+
"normalized": true,
|
| 159 |
+
"rstrip": false,
|
| 160 |
+
"single_word": false,
|
| 161 |
+
"special": false
|
| 162 |
+
},
|
| 163 |
+
"50272": {
|
| 164 |
+
"content": " ",
|
| 165 |
+
"lstrip": false,
|
| 166 |
+
"normalized": true,
|
| 167 |
+
"rstrip": false,
|
| 168 |
+
"single_word": false,
|
| 169 |
+
"special": false
|
| 170 |
+
},
|
| 171 |
+
"50273": {
|
| 172 |
+
"content": " ",
|
| 173 |
+
"lstrip": false,
|
| 174 |
+
"normalized": true,
|
| 175 |
+
"rstrip": false,
|
| 176 |
+
"single_word": false,
|
| 177 |
+
"special": false
|
| 178 |
+
},
|
| 179 |
+
"50274": {
|
| 180 |
+
"content": " ",
|
| 181 |
+
"lstrip": false,
|
| 182 |
+
"normalized": true,
|
| 183 |
+
"rstrip": false,
|
| 184 |
+
"single_word": false,
|
| 185 |
+
"special": false
|
| 186 |
+
},
|
| 187 |
+
"50275": {
|
| 188 |
+
"content": " ",
|
| 189 |
+
"lstrip": false,
|
| 190 |
+
"normalized": true,
|
| 191 |
+
"rstrip": false,
|
| 192 |
+
"single_word": false,
|
| 193 |
+
"special": false
|
| 194 |
+
},
|
| 195 |
+
"50276": {
|
| 196 |
+
"content": " ",
|
| 197 |
+
"lstrip": false,
|
| 198 |
+
"normalized": true,
|
| 199 |
+
"rstrip": false,
|
| 200 |
+
"single_word": false,
|
| 201 |
+
"special": false
|
| 202 |
+
},
|
| 203 |
+
"50277": {
|
| 204 |
+
"content": "|||EMAIL_ADDRESS|||",
|
| 205 |
+
"lstrip": false,
|
| 206 |
+
"normalized": true,
|
| 207 |
+
"rstrip": false,
|
| 208 |
+
"single_word": false,
|
| 209 |
+
"special": false
|
| 210 |
+
},
|
| 211 |
+
"50278": {
|
| 212 |
+
"content": "|||PHONE_NUMBER|||",
|
| 213 |
+
"lstrip": false,
|
| 214 |
+
"normalized": true,
|
| 215 |
+
"rstrip": false,
|
| 216 |
+
"single_word": false,
|
| 217 |
+
"special": false
|
| 218 |
+
},
|
| 219 |
+
"50279": {
|
| 220 |
+
"content": "<|endoftext|>",
|
| 221 |
+
"lstrip": false,
|
| 222 |
+
"normalized": false,
|
| 223 |
+
"rstrip": false,
|
| 224 |
+
"single_word": false,
|
| 225 |
+
"special": true
|
| 226 |
+
}
|
| 227 |
+
},
|
| 228 |
+
"clean_up_tokenization_spaces": true,
|
| 229 |
+
"eos_token": "<|endoftext|>",
|
| 230 |
+
"max_length": null,
|
| 231 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 232 |
+
"pad_token": "<|padding|>",
|
| 233 |
+
"tokenizer_class": "OLMoTokenizer",
|
| 234 |
+
"truncation": "right",
|
| 235 |
+
"auto_map": {
|
| 236 |
+
"AutoConfig": "configuration_olmo.OLMoConfig",
|
| 237 |
+
"AutoTokenizer": [
|
| 238 |
+
"tokenization_olmo_fast.OLMoTokenizerFast",
|
| 239 |
+
"tokenization_olmo_fast.OLMoTokenizerFast"
|
| 240 |
+
]
|
| 241 |
+
}
|
| 242 |
+
}
|