Hugging Face's logo

tangledgroup
/
tangled-llama-n-128k-v0.1

Model card Files Community
mtasic85 commited on
Commit
f8d385f
·
1 Parent(s): c698c63

tokenizer

Browse files
Files changed (17) hide show
  1. TRAIN.md +113 -0
  2. merges.txt +0 -0
  3. misc/logo.png +3 -0
  4. scripts/contrain-model-0.yaml +156 -0
  5. scripts/contrain_datasets.py +204 -0
  6. scripts/prepare_contrain_datasets.py +43 -0
  7. scripts/prepare_pretrain_datasets.py +46 -0
  8. scripts/pretrain-model-0.yaml +150 -0
  9. scripts/pretrain_datasets.py +78 -0
  10. scripts/requirements.in +22 -0
  11. scripts/tokenizer_datasets.py +53 -0
  12. scripts/train_tokenizer.py +91 -0
  13. scripts/utils.py +96 -0
  14. special_tokens_map.json +5 -0
  15. tokenizer.json +3 -0
  16. tokenizer_config.json +4108 -0
  17. vocab.json +0 -0
TRAIN.md ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Train
2
+
3
+ ## Environment
4
+
5
+ ```bash
6
+ cd scripts
7
+ python -m venv venv
8
+ source venv/bin/activate
9
+ pip install -U -r requirements.in
10
+ ```
11
+
12
+ ## Train Tokenizer
13
+
14
+ ```bash
15
+ time python -B train_tokenizer.py
16
+ ```
17
+
18
+ ## Pretrain
19
+
20
+ ```bash
21
+ python -B prepare_pretrain_datasets.py
22
+ ```
23
+
24
+ ```bash
25
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt pretrain --config pretrain-model-0.yaml
26
+ litgpt convert_pretrained_checkpoint ../out/pretrain-0/final/ ../out/pretrain-0-final-checkpoint
27
+
28
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt pretrain --config pretrain-model-1.yaml
29
+ litgpt convert_pretrained_checkpoint ../out/pretrain-1/final/ ../out/pretrain-1-final-checkpoint
30
+
31
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt pretrain --config pretrain-model-2.yaml
32
+ litgpt convert_pretrained_checkpoint ../out/pretrain-2/final/ ../out/pretrain-2-final-checkpoint
33
+
34
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt pretrain --config pretrain-model-3.yaml
35
+ litgpt convert_pretrained_checkpoint ../out/pretrain-3/final/ ../out/pretrain-3-final-checkpoint
36
+
37
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt pretrain --config pretrain-model-4.yaml
38
+ litgpt convert_pretrained_checkpoint ../out/pretrain-4/final/ ../out/pretrain-4-final-checkpoint
39
+
40
+ # NOTE: unused
41
+ # CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt pretrain --config pretrain-model-5.yaml
42
+ # litgpt convert_pretrained_checkpoint ../out/pretrain-5/final/ ../out/pretrain-5-final-checkpoint
43
+ ```
44
+
45
+ ### Continued Pretraining
46
+
47
+ ```bash
48
+ python -B prepare_contrain_datasets.py
49
+ ```
50
+
51
+ ```bash
52
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt pretrain --config contrain-model-0.yaml
53
+ litgpt convert_pretrained_checkpoint ../out/contrain-0/final/ ../out/contrain-0-final-checkpoint
54
+
55
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt pretrain --config contrain-model-1.yaml
56
+ litgpt convert_pretrained_checkpoint ../out/contrain-1/final/ ../out/contrain-1-final-checkpoint
57
+ ```
58
+
59
+ ## Chat with Pretrained model
60
+
61
+ ```bash
62
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt chat out/pretrain-0/final/
63
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt chat out/pretrain-1/final/
64
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt chat out/pretrain-2/final/
65
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt chat out/pretrain-3/final/
66
+ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt chat out/pretrain-4/final/
67
+ # CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt chat out/pretrain-5/final/
68
+ ```
69
+
70
+ <!-- OLD -->
71
+
72
+ ## Model
73
+
74
+ ### Pretraining
75
+
76
+ ```bash
77
+ litgpt pretrain --config ./pretrain-model.yaml
78
+ litgpt convert_from_litgpt out/pretrain/final/ out/converted_pretrain
79
+ cp config.json out/pretrain/final/
80
+ cp config.json out/converted_pretrain/
81
+ ```
82
+
83
+ ```python
84
+ import torch
85
+ from safetensors.torch import save_file
86
+
87
+ state_dict = torch.load('out/converted_pretrain/model.pth', map_location='cpu')
88
+ save_file(state_dict, 'out/converted_pretrain/model.safetensors')
89
+ ```
90
+
91
+ ### Continued Pretraining
92
+
93
+ ```bash
94
+ litgpt convert_pretrained_checkpoint out/pretrain/final/ out/pretrain_checkpoint/final/
95
+ cp config.json out/pretrain_checkpoint/final/
96
+
97
+ litgpt pretrain --config ./contrain-model.yaml
98
+ litgpt convert_from_litgpt out/contrain/final/ out/converted_contrain
99
+ cp config.json out/converted_contrain/
100
+ ```
101
+
102
+ ```python
103
+ import torch
104
+ from safetensors.torch import save_file
105
+
106
+ state_dict = torch.load('out/converted_contrain/model.pth', map_location='cpu')
107
+ save_file(state_dict, 'out/converted_contrain/model.safetensors')
108
+ ```
109
+
110
+ ```bash
111
+ cp out/converted_contrain/model.pth ./
112
+ cp out/converted_contrain/model.safetensors ./
113
+ ```
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
misc/logo.png ADDED

Git LFS Details

  • SHA256: 591bee6fa56315a84eeec47c5e04ff6331f842c773ce50b7a59e508b4d2904cf
  • Pointer size: 131 Bytes
  • Size of remote file: 684 kB
scripts/contrain-model-0.yaml ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/blob/main/config.json
2
+
3
+ # The name of the model to pretrain. Choose from names in ``litgpt.config``. Mutually exclusive with
4
+ # ``model_config``. (type: Optional[str], default: null)
5
+ model_name: "Llama-3.2-1B"
6
+
7
+ # A ``litgpt.Config`` object to define the model architecture. Mutually exclusive with
8
+ # ``model_config``. (type: Optional[Config], default: null)
9
+ model_config:
10
+ padded_vocab_size: 65536
11
+ vocab_size: 65536
12
+ block_size: 131072
13
+ n_layer: 32
14
+ n_head: 16
15
+ head_size: 64
16
+ n_embd: 768
17
+ n_query_groups: 4
18
+ rotary_percentage: 1.0
19
+ parallel_residual: false
20
+ shared_attention_norm: false
21
+ bias: false
22
+ # attn_bias: true # qwen 2.5
23
+ norm_class_name: "RMSNorm"
24
+ mlp_class_name: "LLaMAMLP"
25
+ intermediate_size: 2048
26
+ # rope_base: 500000 # llama 3.2
27
+ rope_base: 1000000 # qwen 2.5
28
+ rope_adjustments: # llama 3.2
29
+ factor: 32.0
30
+ low_freq_factor: 1.0
31
+ high_freq_factor: 4.0
32
+ original_max_seq_len: 8192
33
+
34
+ # Directory in which to save checkpoints and logs. If running in a Lightning Studio Job, look for it in
35
+ # /teamspace/jobs/<job-name>/share. (type: <class 'Path'>, default: out/pretrain)
36
+ out_dir: "../out/contrain-0/"
37
+
38
+ # The precision to use for pretraining. Possible choices: "bf16-true", "bf16-mixed", "32-true". (type: Optional[str], default: null)
39
+ # precision: bf16-mixed
40
+ precision: bf16-true
41
+
42
+ # Optional path to a checkpoint directory to initialize the model from.
43
+ # Useful for continued pretraining. Mutually exclusive with ``resume``. (type: Optional[Path], default: null)
44
+ initial_checkpoint_dir: "../out/pretrain-4-final-checkpoint/"
45
+
46
+ # Path to a checkpoint directory to resume from in case training was interrupted, or ``True`` to resume
47
+ # from the latest checkpoint in ``out_dir``. An error will be raised if no checkpoint is found. Passing
48
+ # ``'auto'`` will resume from the latest checkpoint but not error if no checkpoint exists.
49
+ # (type: Union[bool, Literal["auto"], Path], default: False)
50
+ # resume:
51
+
52
+ # Data-related arguments. If not provided, the default is ``litgpt.data.TinyLlama``.
53
+ data:
54
+ class_path: LitData
55
+
56
+ init_args:
57
+ data_path: "../contrain-data-0-4097-16388000/"
58
+ num_workers: 32
59
+
60
+ # Training-related arguments. See ``litgpt.args.TrainArgs`` for details
61
+ train:
62
+ # Number of optimizer steps between saving checkpoints (type: Optional[int], default: 1000)
63
+ save_interval: 100
64
+
65
+ # Number of iterations between logging calls (type: int, default: 1)
66
+ log_interval: 1
67
+
68
+ # Number of samples between optimizer steps across data-parallel ranks (type: int, default: 512)
69
+ global_batch_size: 512
70
+
71
+ # Number of samples per data-parallel rank (type: int, default: 4)
72
+ micro_batch_size: 3
73
+
74
+ # Number of iterations with learning rate warmup active (type: int, default: 2000)
75
+ lr_warmup_steps: 0
76
+
77
+ # Number of epochs to train on (type: Optional[int], default: null)
78
+ epochs:
79
+
80
+ # Total number of tokens to train on (type: Optional[int], default: 3000000000000)
81
+ max_tokens: 1527816367 # 4_097 * 372_911
82
+
83
+ # Limits the number of optimizer steps to run. (type: Optional[int], default: null)
84
+ max_steps:
85
+
86
+ # Limits the length of samples. Off by default (type: Optional[int], default: null)
87
+ max_seq_length: 4097
88
+
89
+ # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
90
+ tie_embeddings: true
91
+
92
+ # (type: Optional[float], default: 1.0)
93
+ max_norm: 1.0
94
+
95
+ # (type: float, default: 4e-05)
96
+ min_lr: 1e-06
97
+
98
+ # Evaluation-related arguments. See ``litgpt.args.EvalArgs`` for details
99
+ eval:
100
+ # Number of optimizer steps between evaluation calls (type: int, default: 1000)
101
+ interval: 25
102
+
103
+ # Number of tokens to generate (type: Optional[int], default: null)
104
+ max_new_tokens:
105
+
106
+ # Number of iterations (type: int, default: 100)
107
+ max_iters: 100
108
+
109
+ # Whether to evaluate on the validation set at the beginning of the training
110
+ initial_validation: false
111
+
112
+ # Whether to evaluate on the validation set at the end the training
113
+ final_validation: true
114
+
115
+ # Optimizer-related arguments
116
+ optimizer:
117
+ class_path: grokadamw.GrokAdamW
118
+
119
+ init_args:
120
+ # (type: float, default: 0.001)
121
+ lr: 1e-05
122
+
123
+ # (type: float, default: 0.01)
124
+ weight_decay: 1e-2
125
+
126
+ # (type: tuple, default: (0.9,0.999))
127
+ betas:
128
+ - 0.9
129
+ - 0.999
130
+
131
+ # optimizer:
132
+ # class_path: sophia_opt.SophiaG
133
+ #
134
+ # init_args:
135
+ # lr: 4e-4
136
+ # betas:
137
+ # - 0.965
138
+ # - 0.99
139
+ # rho: 0.01
140
+ # weight_decay: 1e-1
141
+
142
+ # How many devices/GPUs to use. Uses all GPUs by default. (type: Union[int, str], default: auto)
143
+ devices: auto
144
+
145
+ # How many nodes to use. (type: int, default: 1)
146
+ num_nodes: 1
147
+
148
+ # Optional path to the tokenizer dir that was used for preprocessing the dataset. Only some data
149
+ # module require this. (type: Optional[Path], default: null)
150
+ tokenizer_dir: "../"
151
+
152
+ # The name of the logger to send metrics to. (type: Literal['wandb', 'tensorboard', 'csv'], default: tensorboard)
153
+ logger_name: "wandb"
154
+
155
+ # The random seed to use for reproducibility. (type: int, default: 42)
156
+ seed: 23
scripts/contrain_datasets.py ADDED
@@ -0,0 +1,204 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ roles_map = {
2
+ 'system': 'system',
3
+ 'user': 'user',
4
+ 'human': 'user',
5
+ 'assistant': 'assistant',
6
+ 'gpt': 'assistant',
7
+ 'AI': 'assistant',
8
+ }
9
+
10
+
11
+ contrain_datasets = [
12
+ #
13
+ # general instructs
14
+ #
15
+ # mlabonne/open-perfectblend - 1.48 GB, 1,420,909
16
+ # meta-math/MetaMathQA 395,000
17
+ # openbmb/UltraInteract_sft 288,579
18
+ # HuggingFaceH4/ultrachat_200k 207,865
19
+ # microsoft/orca-math-word-problems-200k 200,035
20
+ # HuggingFaceH4/ultrafeedback_binarized 187,405
21
+ # theblackcat102/evol-codealpaca-v1 111,272
22
+ # Post-training-Data-Flywheel/AutoIF-instruct-61k 61,492
23
+ # mlabonne/lmsys-arena-human-preference-55k-sharegpt 57,362
24
+ *[
25
+ {'path': 'mlabonne/open-perfectblend', 'split': f'train[{i}%:{i + 20}%]', 'field': 'conversations', 'transform': lambda msgs: [
26
+ {'role': roles_map[m['from']], 'content': m['value']}
27
+ for m in msgs
28
+ ]}
29
+ for i in range(0, 100, 20)
30
+ ],
31
+ # arcee-ai/The-Tome - 4.58 GB, 1,752,473
32
+ # - arcee-ai/infini-instruct-top-500k (BAAI/Infinity-Instruct)
33
+ # - TIGER-Lab/WebInstructSub (top-500k) - IGNORE
34
+ # - jondurbin/airoboros-3.2
35
+ # - gardner/glaive-function-calling-v2-sharegpt
36
+ # - arcee-ai/reasoning-sharegpt (SkunkworksAI/reasoning-0.01)
37
+ # - arcee-ai/self-instruct-sharegpt (bigcode/self-oss-instruct-sc2-exec-filter-50k)
38
+ # - cognitivecomputations/ultrainteract_trajectories_sharegpt
39
+ # - cognitivecomputations/SystemChat-2.0
40
+ # - arcee-ai/qwen2-72b-magpie-en
41
+ *[
42
+ {'path': 'arcee-ai/The-Tome', 'split': f'train[{i}%:{i + 20}%]', 'field': 'conversations', 'transform': lambda msgs: [
43
+ {'role': roles_map[m['from']], 'content': m['value']}
44
+ for m in msgs
45
+ ]}
46
+ for i in range(0, 100, 20)
47
+ ],
48
+ # rombodawg/Everything_Instruct_Multilingual - 2.48 GB, 5,808,694
49
+ # Science:
50
+ # antiven0m/physical-reasoning-dpoScience
51
+ # LawalAfeez/science-dataset
52
+ # Social media:
53
+ # Kyle1668/AG-Tweets
54
+ # euclaise/reddit-instruct-curated
55
+ # General Knowledge:
56
+ # NousResearch/CharacterCodex_Characters
57
+ # jstet/quotes-500k_Famous_Quotes
58
+ # FronkonGames/steam-games-dataset_Video_Games
59
+ # totuta_youtube_subs_howto100M_HowTo
60
+ # Multi-lingual:
61
+ # Amani27/massive_translation_dataset
62
+ # udmurtNLP/udmurt-russian-english-labse
63
+ # grosenthal/latin_english
64
+ # msarmi9/korean-english-multitarget-ted-talks-task
65
+ # HaiderSultanArc/MT-Urdu-English_Translate
66
+ # Garsa3112/ChineseEnglishTranslationDataset
67
+ # Cooking:
68
+ # andrewsiah/se_cooking_preference_sft
69
+ # Hieu-Phamkaggle/food_recipes
70
+ # Writing:
71
+ # shahules786/PoetryFoundationData
72
+ # euclaise/writingprompts
73
+ # qwedsacf/ivypanda-essaysEssay
74
+ # Medicine:
75
+ # keivalya/MedQuad-MedicalQnADataset
76
+ # nuvocare/MSD
77
+ # History:
78
+ # ambrosfitz10k/history_data_v4
79
+ # Law:
80
+ # dzunggg/legal-qa-v1
81
+ # Role-Play:
82
+ # roleplay4/fun_CoupleRP
83
+ # Undi95andrijdavid/roleplay-conversation-sharegpt
84
+ # News:
85
+ # RealTimeData/bbc_news_alltime
86
+ # Coding: (rombodawg/code_bagel)
87
+ # layoric/tiny-codes-alpaca
88
+ # glaiveai/glaive-code-assistant-v3
89
+ # ajibawa-2023/Code-290k-ShareGPT
90
+ # chargoddard/commitpack-ft-instruct-rated
91
+ # iamtarun/code_instructions_120k_alpaca
92
+ # ise-uiuc/Magicoder-Evol-Instruct-110K
93
+ # cognitivecomputations/dolphin-coder
94
+ # nickrosh/Evol-Instruct-Code-80k-v1
95
+ # coseal/CodeUltraFeedback_binarized
96
+ # CyberNative/Code_Vulnerability_Security_DPO
97
+ # Math: (rombodawg/code_bagel)
98
+ # TIGER-Lab/MathInstruct
99
+ # Function calling: (rombodawg/code_bagel)
100
+ # glaiveai/glaive-function-calling-v2
101
+ # General Instruct: (rombodawg/OpenHermes-2.5-Uncensored)
102
+ # teknium/OpenHermes-2.5
103
+ *[
104
+ {'path': 'rombodawg/Everything_Instruct_Multilingual', 'split': f'train[{i}%:{i + 20}%]', 'transform': lambda r: [
105
+ {'role': 'system', 'content': r['instruction']},
106
+ {'role': 'user', 'content': r['input']},
107
+ {'role': 'assistant', 'content': r['output']},
108
+ ]}
109
+ for i in range(0, 100, 20)
110
+ ],
111
+
112
+ #
113
+ # tool/function calling
114
+ #
115
+ # 65.7 MB, 11,578
116
+ {'path': 'NousResearch/hermes-function-calling-v1', 'field': 'conversations', 'transform': lambda msgs: [
117
+ {'role': roles_map[m['from']], 'content': m['value']}
118
+ for m in msgs
119
+ ]},
120
+
121
+ #
122
+ # agent
123
+ #
124
+ # 1.51 GB, 485,874
125
+ *[
126
+ {'path': 'arcee-ai/agent-data', 'split': f'train[{i}%:{i + 20}%]', 'field': 'conversations', 'transform': lambda msgs: [
127
+ {'role': roles_map[m['from']], 'content': m['value']}
128
+ for m in msgs
129
+ ]}
130
+ for i in range(0, 100, 20)
131
+ ],
132
+
133
+ #
134
+ # general reasoning
135
+ #
136
+ *[
137
+ # 10.8 MB, 15,770
138
+ {'path': 'AtlasUnified/Atlas-Reasoning', 'data_files': 'reasoning.csv', 'transform': lambda r: [
139
+ {'role': 'user', 'content': r['Prompt']},
140
+ {'role': 'assistant', 'content': r['Step-by-step reasoning'] + '\n' + r['Solution']},
141
+ ]},
142
+ ],
143
+
144
+ #
145
+ # math reasoning
146
+ #
147
+ # 8.99 MB, 6,914
148
+ {'path': 'thesven/gsm8k-reasoning', 'transform': lambda r: [
149
+ {'role': 'user', 'content': r['question']},
150
+ {'role': 'assistant', 'content': (r['generation'] or '') + '\n' + r['answer'] + '\n' + r['short_answer']},
151
+ ]},
152
+
153
+ # 1.79 MB, 3,963
154
+ {'path': 'AlgorithmicResearchGroup/math_reasoning_autoformalization_track', 'transform': lambda r: [
155
+ {'role': 'user', 'content': r['informal_statement']},
156
+ {'role': 'assistant', 'content': r['informal_proof'] + '\n' + r['formal_proof']},
157
+ ]},
158
+
159
+ # 307 MB, 19,944
160
+ {'path': 'KingNish/reasoning-base-20k', 'transform': lambda r: [
161
+ {'role': 'user', 'content': r['user']},
162
+ {'role': 'assistant', 'content': r['reasoning'] + '\n' + r['assistant']},
163
+ ]},
164
+
165
+ # 9.45 MB, 10,000
166
+ {'path': 'Aarushhh/math-reasoning-10k', 'transform': lambda r: [
167
+ {'role': 'user', 'content': r['problem']},
168
+ {'role': 'assistant', 'content': r['plan'] + '\n' + r['solution']},
169
+ ]},
170
+
171
+ #
172
+ # reflection
173
+ #
174
+ # 4.17 MB, 1,000
175
+ {'path': 'dvilasuero/reflection-v1-gpt-4o-judge', 'transform': lambda r: [
176
+ {'role': 'system', 'content': r['system']},
177
+ {'role': 'user', 'content': r['prompt']},
178
+ {'role': 'assistant', 'content': r['response']},
179
+ ]},
180
+ # 12.4 MB, 3,000
181
+ {'path': 'dvilasuero/reflection-v1-openai-o-mini-judge', 'transform': lambda r: [
182
+ {'role': 'system', 'content': r['system']},
183
+ {'role': 'user', 'content': r['prompt']},
184
+ {'role': 'assistant', 'content': r['response']},
185
+ ]},
186
+ # 70.8 MB, 36,549
187
+ {'path': 'dvilasuero/reflection-v1-final-dedup', 'transform': lambda r: [
188
+ {'role': 'system', 'content': r['system']},
189
+ {'role': 'user', 'content': r['prompt']},
190
+ {'role': 'assistant', 'content': r['response']},
191
+ ]},
192
+ # 30.6 MB, 25,391
193
+ {'path': 'flozi00/reflection-qwen2.5-72b-260924', 'transform': lambda r: [
194
+ r['system'][0],
195
+ {'role': 'user', 'content': r['input']},
196
+ {'role': 'assistant', 'content': r['reflection'] + '\n' + r['output']},
197
+ ]},
198
+
199
+ #
200
+ # general instructs
201
+ #
202
+ # 971 MB, 484,570
203
+ {'path': 'HuggingFaceTB/smol-smoltalk', 'field': 'messages'},
204
+ ]
scripts/prepare_contrain_datasets.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from functools import partial
2
+
3
+ from litgpt.tokenizer import Tokenizer
4
+ from litdata import optimize, TokensLoader, StreamingDataset
5
+ from transformers import AutoTokenizer
6
+
7
+ from utils import tokenize_chat_fn
8
+ from contrain_datasets import contrain_datasets
9
+
10
+
11
+ #
12
+ # optimize datasets
13
+ #
14
+ for i, (block_size, subchunk_size) in enumerate([(4097, 4000)]):
15
+ chunk_size = block_size * subchunk_size
16
+ output_dir = f'../contrain-data-{i}-{block_size}-{chunk_size}'
17
+
18
+ outputs = optimize(
19
+ fn=partial(
20
+ tokenize_chat_fn,
21
+ hf_tokenizer=AutoTokenizer.from_pretrained('..', trust_remote_code=True, use_fast=True),
22
+ tokenizer=Tokenizer('..'),
23
+ ),
24
+ inputs=contrain_datasets,
25
+ output_dir=output_dir,
26
+ chunk_size=chunk_size, # Number of tokens to store by chunks. This is roughly 64MB of tokens per chunk.
27
+ num_workers=32,
28
+ reorder_files=False,
29
+ )
30
+
31
+ #
32
+ # total number of chunks in datasets
33
+ #
34
+ for i, (block_size, subchunk_size) in enumerate([(4097, 4000)]):
35
+ chunk_size = block_size * subchunk_size
36
+ input_dir = f'../contrain-data-{i}-{block_size}-{chunk_size}'
37
+
38
+ dataset = StreamingDataset(
39
+ input_dir=input_dir,
40
+ item_loader=TokensLoader(block_size=block_size),
41
+ )
42
+
43
+ print(f'{i=}, {block_size=}, {chunk_size=}, {len(dataset)=}, {len(dataset) * block_size=}')
scripts/prepare_pretrain_datasets.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from functools import partial
2
+
3
+ from litgpt.tokenizer import Tokenizer
4
+ from litdata import optimize, TokensLoader, StreamingDataset
5
+ from transformers import AutoTokenizer
6
+
7
+ from utils import tokenize_text_fn
8
+ from pretrain_datasets import pretrain_datasets
9
+
10
+
11
+ #
12
+ # optimize datasets
13
+ #
14
+ for i, (block_size, subchunk_size) in enumerate([(4097, 4000)]):
15
+ chunk_size = block_size * subchunk_size
16
+ output_dir = f'../pretrain-data-{i}-{block_size}-{chunk_size}'
17
+
18
+ outputs = optimize(
19
+ fn=partial(
20
+ tokenize_text_fn,
21
+ hf_tokenizer=AutoTokenizer.from_pretrained('..', trust_remote_code=True, use_fast=True),
22
+ tokenizer=Tokenizer('..'),
23
+ ),
24
+ inputs=pretrain_datasets,
25
+ output_dir=output_dir,
26
+ chunk_size=chunk_size, # Number of tokens to store by chunks. This is roughly 64MB of tokens per chunk.
27
+ num_workers=32,
28
+ reorder_files=False,
29
+ ## This is important to inform LitData that we are encoding contiguous 1D array (tokens).
30
+ ## LitData skips storing metadata for each sample e.g all the tokens are concatenated to form one large tensor.
31
+ # item_loader=TokensLoader(block_size=block_size),
32
+ )
33
+
34
+ #
35
+ # total number of chunks in datasets
36
+ #
37
+ for i, (block_size, subchunk_size) in enumerate([(4097, 4000)]):
38
+ chunk_size = block_size * subchunk_size
39
+ input_dir = f'../pretrain-data-{i}-{block_size}-{chunk_size}'
40
+
41
+ dataset = StreamingDataset(
42
+ input_dir=input_dir,
43
+ item_loader=TokensLoader(block_size=block_size),
44
+ )
45
+
46
+ print(f'{i=}, {block_size=}, {chunk_size=}, {len(dataset)=}, {len(dataset) * block_size=}')
scripts/pretrain-model-0.yaml ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/blob/main/config.json
2
+
3
+ # The name of the model to pretrain. Choose from names in ``litgpt.config``. Mutually exclusive with
4
+ # ``model_config``. (type: Optional[str], default: null)
5
+ model_name: "Qwen2.5-7B"
6
+
7
+ # A ``litgpt.Config`` object to define the model architecture. Mutually exclusive with
8
+ # ``model_config``. (type: Optional[Config], default: null)
9
+ model_config:
10
+ block_size: 131072
11
+ vocab_size: 131072
12
+ padded_vocab_size: 131072
13
+ n_layer: 28
14
+ n_head: 28
15
+ head_size: 128
16
+ n_embd: 896
17
+ n_query_groups: 4
18
+ rotary_percentage: 1.0
19
+ parallel_residual: False
20
+ bias: False
21
+ attn_bias: True
22
+ norm_class_name: "RMSNorm"
23
+ mlp_class_name: "LLaMAMLP"
24
+ intermediate_size: 4864
25
+ norm_eps: 1e-6
26
+ rope_base: 1000000
27
+
28
+ # Directory in which to save checkpoints and logs. If running in a Lightning Studio Job, look for it in
29
+ # /teamspace/jobs/<job-name>/share. (type: <class 'Path'>, default: out/pretrain)
30
+ out_dir: "../out/pretrain-0/"
31
+
32
+ # The precision to use for pretraining. Possible choices: "bf16-true", "bf16-mixed", "32-true". (type: Optional[str], default: null)
33
+ # precision: bf16-mixed
34
+ precision: bf16-true
35
+
36
+ # Optional path to a checkpoint directory to initialize the model from.
37
+ # Useful for continued pretraining. Mutually exclusive with ``resume``. (type: Optional[Path], default: null)
38
+ initial_checkpoint_dir:
39
+
40
+ # Path to a checkpoint directory to resume from in case training was interrupted, or ``True`` to resume
41
+ # from the latest checkpoint in ``out_dir``. An error will be raised if no checkpoint is found. Passing
42
+ # ``'auto'`` will resume from the latest checkpoint but not error if no checkpoint exists.
43
+ # (type: Union[bool, Literal["auto"], Path], default: False)
44
+ resume: "auto"
45
+
46
+ # Data-related arguments. If not provided, the default is ``litgpt.data.TinyLlama``.
47
+ data:
48
+ class_path: LitData
49
+
50
+ init_args:
51
+ data_path: "../pretrain-data-0-0-513-32704-16777152/"
52
+ num_workers: 32
53
+
54
+ # Training-related arguments. See ``litgpt.args.TrainArgs`` for details
55
+ train:
56
+ # Number of optimizer steps between saving checkpoints (type: Optional[int], default: 1000)
57
+ save_interval: 100
58
+
59
+ # Number of iterations between logging calls (type: int, default: 1)
60
+ log_interval: 1
61
+
62
+ # Number of samples between optimizer steps across data-parallel ranks (type: int, default: 512)
63
+ global_batch_size: 512
64
+
65
+ # Number of samples per data-parallel rank (type: int, default: 4)
66
+ micro_batch_size: 24
67
+
68
+ # Number of iterations with learning rate warmup active (type: int, default: 2000)
69
+ lr_warmup_steps: 0
70
+
71
+ # Number of epochs to train on (type: Optional[int], default: null)
72
+ epochs:
73
+
74
+ # Total number of tokens to train on (type: Optional[int], default: 3000000000000)
75
+ max_tokens: 1945266624 # 32_704 * 59_481
76
+
77
+ # Limits the number of optimizer steps to run. (type: Optional[int], default: null)
78
+ max_steps:
79
+
80
+ # Limits the length of samples. Off by default (type: Optional[int], default: null)
81
+ max_seq_length: 513
82
+
83
+ # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
84
+ tie_embeddings: true
85
+
86
+ # (type: Optional[float], default: 1.0)
87
+ max_norm: 1.0
88
+
89
+ # (type: float, default: 4e-05)
90
+ min_lr: 1e-05
91
+
92
+ # Evaluation-related arguments. See ``litgpt.args.EvalArgs`` for details
93
+ eval:
94
+ # Number of optimizer steps between evaluation calls (type: int, default: 1000)
95
+ interval: 100
96
+
97
+ # Number of tokens to generate (type: Optional[int], default: null)
98
+ max_new_tokens:
99
+
100
+ # Number of iterations (type: int, default: 100)
101
+ max_iters: 100
102
+
103
+ # Whether to evaluate on the validation set at the beginning of the training
104
+ initial_validation: false
105
+
106
+ # Whether to evaluate on the validation set at the end the training
107
+ final_validation: true
108
+
109
+ # Optimizer-related arguments
110
+ optimizer:
111
+ class_path: grokadamw.GrokAdamW
112
+
113
+ init_args:
114
+ # (type: float, default: 0.001)
115
+ lr: 1e-04
116
+
117
+ # (type: float, default: 0.01)
118
+ weight_decay: 1e-2
119
+
120
+ # (type: tuple, default: (0.9,0.999))
121
+ betas:
122
+ - 0.9
123
+ - 0.999
124
+
125
+ # optimizer:
126
+ # class_path: sophia_opt.SophiaG
127
+ #
128
+ # init_args:
129
+ # lr: 4e-4
130
+ # betas:
131
+ # - 0.965
132
+ # - 0.99
133
+ # rho: 0.01
134
+ # weight_decay: 1e-1
135
+
136
+ # How many devices/GPUs to use. Uses all GPUs by default. (type: Union[int, str], default: auto)
137
+ devices: auto
138
+
139
+ # How many nodes to use. (type: int, default: 1)
140
+ num_nodes: 1
141
+
142
+ # Optional path to the tokenizer dir that was used for preprocessing the dataset. Only some data
143
+ # module require this. (type: Optional[Path], default: null)
144
+ tokenizer_dir: "../"
145
+
146
+ # The name of the logger to send metrics to. (type: Literal['wandb', 'tensorboard', 'csv'], default: tensorboard)
147
+ logger_name: "wandb"
148
+
149
+ # The random seed to use for reproducibility. (type: int, default: 42)
150
+ seed: 23
scripts/pretrain_datasets.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pretrain_datasets = [
2
+ #
3
+ # multilingual
4
+ #
5
+ # 3.17 GB, 2,226,907
6
+ *[
7
+ {'path': 'ontocord/fineweb-permissive-multilingual-2m', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['text']}
8
+ for i in range(0, 100, 10)
9
+ ],
10
+ # 1.64 GB, 1,001,000
11
+ *[
12
+ {'path': 'distily/c4_multilingual_1M', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['text']}
13
+ for i in range(0, 100, 10)
14
+ ],
15
+
16
+ #
17
+ # general knowledge
18
+ #
19
+ # 65.1 MB, 7,819
20
+ {'path': 'Sketched33/Cities_Wikipedia_Information', 'format': lambda n: n['wikipedia_content']},
21
+ # 135 MB, 1,795
22
+ {'path': 'open-phi/textbooks', 'format': lambda n: n['markdown']},
23
+ # 631 MB, 111,048
24
+ {'path': 'open-phi/programming_books_llama', 'format': lambda n: n['markdown']},
25
+
26
+ #
27
+ # misc
28
+ #
29
+ # 472 KB, 5,034
30
+ {'path': 'badrex/llm-emoji-dataset', 'format': '{short description}. {LLM description}. {character}'},
31
+
32
+ #
33
+ # math
34
+ #
35
+ # # 12.6 GB, 21,972,791 - we use 1M subset - 639 MB, 1,000,000
36
+ # *[
37
+ # {'path': 'nvidia/OpenMathInstruct-2', 'split': f'train_1M[{i}%:{i + 10}%]', 'format': '{problem} {generated_solution} {expected_answer}'}
38
+ # for i in range(0, 100, 10)
39
+ # ],
40
+ # 12.6 GB, 14M rows
41
+ *[
42
+ {'path': 'nvidia/OpenMathInstruct-2', 'split': f'train[{i}%:{i + 5}%]', 'format': '{problem} {generated_solution} {expected_answer}'}
43
+ for i in range(0, 100, 5)
44
+ ],
45
+
46
+ #
47
+ # stem
48
+ #
49
+ # 1.44 GB, 63,357
50
+ *[
51
+ {'path': 'neuralwork/arxiver', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['abstract']}
52
+ for i in range(0, 100, 10)
53
+ ],
54
+ *[
55
+ {'path': 'neuralwork/arxiver', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['markdown']}
56
+ for i in range(0, 100, 10)
57
+ ],
58
+
59
+ #
60
+ # code
61
+ #
62
+ # 7.81 GB, ~2,804,025
63
+ *[
64
+ {'path': 'rombodawg/code_bagel_hermes-2.5', 'split': f'train[{i}%:{i + 10}%]', 'format': '{input} {output}'}
65
+ for i in range(0, 100, 10)
66
+ ],
67
+
68
+ #
69
+ # general knowledge
70
+ #
71
+ # 3.18 GB, 1,010,500 - paper says that extracted is 6GB
72
+ *[
73
+ {'path': 'JeanKaddour/minipile', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['text']}
74
+ for i in range(0, 100, 10)
75
+ ],
76
+ {'path': 'JeanKaddour/minipile', 'split': 'validation', 'format': lambda n: n['text']},
77
+ {'path': 'JeanKaddour/minipile', 'split': 'test', 'format': lambda n: n['text']},
78
+ ]
scripts/requirements.in ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
2
+ torch>=2.5.0,<2.6.0
3
+ numpy<2.0
4
+
5
+ tqdm
6
+ datasets
7
+ jinja2
8
+ transformers
9
+ wandb
10
+ # litgpt[all]
11
+ litgpt[all] @ git+https://github.com/Lightning-AI/litgpt.git
12
+ # litgpt @ git+https://github.com/Lightning-AI/litgpt.git
13
+ # litdata
14
+ # litdata @ git+https://github.com/Lightning-AI/litdata.git
15
+ lm_eval[ifeval,math]
16
+ grokadamw
17
+ # bitsandbytes
18
+ # pyzstd
19
+ # zstd
20
+ Pillow
21
+
22
+ sophia-opt
scripts/tokenizer_datasets.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tokenizer_datasets = [
2
+ #
3
+ # multilingual
4
+ #
5
+ # 3.8 GB, 19,454,996
6
+ *[
7
+ {'path': 'sentence-transformers/parallel-sentences-wikimatrix', 'data_dir': 'all', 'split': f'train[{i}%:{i + 5}%]', 'format': lambda n: n['non_english']}
8
+ for i in range(0, 100, 5)
9
+ ],
10
+ # 3.17 GB, 2,226,907
11
+ *[
12
+ {'path': 'ontocord/fineweb-permissive-multilingual-2m', 'split': f'train[{i}%:{i + 5}%]', 'format': lambda n: n['text']}
13
+ for i in range(0, 100, 5)
14
+ ],
15
+ # 1.64 GB, 1,001,000
16
+ *[
17
+ {'path': 'distily/c4_multilingual_1M', 'split': f'train[{i}%:{i + 5}%]', 'format': lambda n: n['text']}
18
+ for i in range(0, 100, 5)
19
+ ],
20
+
21
+ #
22
+ # stem
23
+ #
24
+ # 1.44 GB, 63,357
25
+ *[
26
+ {'path': 'neuralwork/arxiver', 'split': f'train[{i}%:{i + 5}%]', 'format': lambda n: n['abstract']}
27
+ for i in range(0, 100, 5)
28
+ ],
29
+ *[
30
+ {'path': 'neuralwork/arxiver', 'split': f'train[{i}%:{i + 5}%]', 'format': lambda n: n['markdown']}
31
+ for i in range(0, 100, 5)
32
+ ],
33
+
34
+ #
35
+ # code
36
+ #
37
+ # 7.81 GB, ~2,804,025
38
+ *[
39
+ {'path': 'rombodawg/code_bagel_hermes-2.5', 'split': f'train[{i}%:{i + 5}%]', 'format': '{input} {output}'}
40
+ for i in range(0, 100, 5)
41
+ ],
42
+
43
+ #
44
+ # general knowledge
45
+ #
46
+ # 3.18 GB, 1,010,500 - paper says that extracted is 6GB
47
+ *[
48
+ {'path': 'JeanKaddour/minipile', 'split': f'train[{i}%:{i + 5}%]', 'format': lambda n: n['text']}
49
+ for i in range(0, 100, 5)
50
+ ],
51
+ {'path': 'JeanKaddour/minipile', 'split': 'validation', 'format': lambda n: n['text']},
52
+ {'path': 'JeanKaddour/minipile', 'split': 'test', 'format': lambda n: n['text']},
53
+ ]
scripts/train_tokenizer.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import PreTrainedTokenizerFast
2
+ from tokenizers import Tokenizer, normalizers, pre_tokenizers, processors, decoders
3
+ from tokenizers.models import BPE
4
+ from tokenizers.trainers import BpeTrainer
5
+
6
+ from utils import batch_text_iterator
7
+ from tokenizer_datasets import tokenizer_datasets
8
+
9
+ #
10
+ # special_tokens
11
+ #
12
+ bos_token = '<|endoftext|>'
13
+ eos_token = '<|endoftext|>'
14
+ pad_token = '<|endoftext|>'
15
+
16
+ special_tokens = [
17
+ eos_token,
18
+ '<|im_start|>',
19
+ '<|im_end|>',
20
+ 'system',
21
+ 'user',
22
+ 'assistant',
23
+ ]
24
+
25
+ for i in range(256 - len(special_tokens)):
26
+ special_tokens.append(f'<|reserved_{i}|>')
27
+
28
+ for i in range(256):
29
+ special_tokens.append(f'<0x{i:02X}>')
30
+
31
+ #
32
+ # BPE Tokenizer
33
+ #
34
+ bpe = BPE(unk_token=None, byte_fallback=True)
35
+ tokenizer = Tokenizer(bpe)
36
+
37
+ # normalizer
38
+ tokenizer.normalizer = None
39
+
40
+ # pre-tokenizer
41
+ tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False, trim_offsets=True, use_regex=True)
42
+
43
+ # post-processor
44
+ tokenizer.post_processor = processors.ByteLevel(add_prefix_space=True, trim_offsets=False, use_regex=True)
45
+
46
+ # decoder
47
+ tokenizer.decoder = decoders.ByteLevel(add_prefix_space=True, trim_offsets=True, use_regex=True)
48
+
49
+ #
50
+ # BPE Trainer
51
+ #
52
+ trainer = BpeTrainer(
53
+ vocab_size=131072, # 128 * 1024
54
+ min_frequency=3,
55
+ special_tokens=special_tokens,
56
+ max_token_length=16,
57
+ )
58
+
59
+ tokenizer.train_from_iterator(
60
+ (batch_text_iterator(**n) for n in tokenizer_datasets),
61
+ trainer,
62
+ )
63
+
64
+ tokenizer.save('../tokenizer.json')
65
+ tokenizer.model.save('../')
66
+
67
+ #
68
+ # PreTrainedTokenizerFast
69
+ #
70
+ CHAT_TEMPLATE = (
71
+ "{% for message in messages %}"
72
+ "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
73
+ "{% endfor %}"
74
+
75
+ "{% if add_generation_prompt %}"
76
+ "{{ '<|im_start|>assistant\n' }}"
77
+ "{% else %}"
78
+ "{{ eos_token }}"
79
+ "{% endif %}"
80
+ )
81
+
82
+ fast_tokenizer = PreTrainedTokenizerFast(
83
+ tokenizer_object=tokenizer,
84
+ chat_template=CHAT_TEMPLATE,
85
+ bos_token=bos_token,
86
+ eos_token=eos_token,
87
+ pad_token=pad_token,
88
+ clean_up_tokenization_spaces=False,
89
+ )
90
+
91
+ fast_tokenizer.save_pretrained('../')
scripts/utils.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gc
2
+ from typing import Union, Optional, Iterator, Callable
3
+
4
+ import torch
5
+ from datasets import load_dataset
6
+ from litgpt.tokenizer import Tokenizer
7
+ from transformers import AutoTokenizer
8
+
9
+
10
+ def batch_text_iterator(path: str,
11
+ name: Optional[str]=None,
12
+ data_dir: Optional[str]=None,
13
+ data_files: Optional[str]=None,
14
+ keep_in_memory: bool=False,
15
+ revision: Optional[str]=None,
16
+ split: str='train',
17
+ num_proc: Optional[int]=None,
18
+ format: Optional[Callable|str]=None) -> Iterator[str]:
19
+ assert isinstance(format, str) or callable(format), f'{path=} {format=}'
20
+
21
+ dataset = load_dataset(path=path,
22
+ name=name,
23
+ data_dir=data_dir,
24
+ data_files=data_files,
25
+ keep_in_memory=keep_in_memory,
26
+ revision=revision,
27
+ split=split,
28
+ trust_remote_code=True,
29
+ num_proc=num_proc)
30
+
31
+ if callable(format):
32
+ for row in dataset:
33
+ text = format(row)
34
+ yield text
35
+ else:
36
+ for row in dataset:
37
+ text = format.format(**row)
38
+ yield text
39
+
40
+ del dataset
41
+ gc.collect()
42
+
43
+
44
+ def batch_chat_iterator(path: str,
45
+ name: Optional[str]=None,
46
+ data_dir: Optional[str]=None,
47
+ data_files: Optional[str]=None,
48
+ keep_in_memory: bool=False,
49
+ revision: Optional[str]=None,
50
+ split: str='train',
51
+ num_proc: Optional[int]=None,
52
+ field: Optional[str]=None,
53
+ transform: Optional[Callable]=None) -> Iterator[list[dict[str, str]]]:
54
+
55
+ dataset = load_dataset(path=path,
56
+ name=name,
57
+ data_dir=data_dir,
58
+ data_files=data_files,
59
+ keep_in_memory=keep_in_memory,
60
+ revision=revision,
61
+ split=split,
62
+ trust_remote_code=True,
63
+ num_proc=num_proc)
64
+
65
+ if callable(transform):
66
+ for row in dataset:
67
+ if field:
68
+ messages = transform(row[field])
69
+ else:
70
+ messages = transform(row)
71
+
72
+ yield messages
73
+ else:
74
+ for row in dataset:
75
+ if field:
76
+ messages = row[field]
77
+ else:
78
+ raise ValueError(field)
79
+
80
+ yield messages
81
+
82
+ del dataset
83
+ gc.collect()
84
+
85
+
86
+ def tokenize_text_fn(dataset_config: dict, hf_tokenizer: AutoTokenizer, tokenizer: Tokenizer) -> Iterator[torch.Tensor]:
87
+ for text in batch_text_iterator(**dataset_config):
88
+ text_ids: torch.Tensor = tokenizer.encode(text, bos=False, eos=True)
89
+ yield text_ids
90
+
91
+
92
+ def tokenize_chat_fn(dataset_config: dict, hf_tokenizer: AutoTokenizer, tokenizer: Tokenizer) -> Iterator[torch.Tensor]:
93
+ for messages in batch_chat_iterator(**dataset_config):
94
+ text: str = hf_tokenizer.apply_chat_template(messages, tokenize=False)
95
+ text_ids: torch.Tensor = tokenizer.encode(text, bos=False, eos=False)
96
+ yield text_ids
special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>"
5
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4bf3b7e915e35508fc236a489c2d9f10870d5c3d6564bbe982d3e225ecf1484f
3
+ size 9650692
tokenizer_config.json ADDED
@@ -0,0 +1,4108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<|im_start|>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<|im_end|>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "system",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "user",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "5": {
44
+ "content": "assistant",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "6": {
52
+ "content": "<|reserved_0|>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "7": {
60
+ "content": "<|reserved_1|>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "8": {
68
+ "content": "<|reserved_2|>",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "9": {
76
+ "content": "<|reserved_3|>",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "10": {
84
+ "content": "<|reserved_4|>",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ },
91
+ "11": {
92
+ "content": "<|reserved_5|>",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "12": {
100
+ "content": "<|reserved_6|>",
101
+ "lstrip": false,
102
+ "normalized": false,
103
+ "rstrip": false,
104
+ "single_word": false,
105
+ "special": true
106
+ },
107
+ "13": {
108
+ "content": "<|reserved_7|>",
109
+ "lstrip": false,
110
+ "normalized": false,
111
+ "rstrip": false,
112
+ "single_word": false,
113
+ "special": true
114
+ },
115
+ "14": {
116
+ "content": "<|reserved_8|>",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false,
121
+ "special": true
122
+ },
123
+ "15": {
124
+ "content": "<|reserved_9|>",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "16": {
132
+ "content": "<|reserved_10|>",
133
+ "lstrip": false,
134
+ "normalized": false,
135
+ "rstrip": false,
136
+ "single_word": false,
137
+ "special": true
138
+ },
139
+ "17": {
140
+ "content": "<|reserved_11|>",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false,
145
+ "special": true
146
+ },
147
+ "18": {
148
+ "content": "<|reserved_12|>",
149
+ "lstrip": false,
150
+ "normalized": false,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": true
154
+ },
155
+ "19": {
156
+ "content": "<|reserved_13|>",
157
+ "lstrip": false,
158
+ "normalized": false,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": true
162
+ },
163
+ "20": {
164
+ "content": "<|reserved_14|>",
165
+ "lstrip": false,
166
+ "normalized": false,
167
+ "rstrip": false,
168
+ "single_word": false,
169
+ "special": true
170
+ },
171
+ "21": {
172
+ "content": "<|reserved_15|>",
173
+ "lstrip": false,
174
+ "normalized": false,
175
+ "rstrip": false,
176
+ "single_word": false,
177
+ "special": true
178
+ },
179
+ "22": {
180
+ "content": "<|reserved_16|>",
181
+ "lstrip": false,
182
+ "normalized": false,
183
+ "rstrip": false,
184
+ "single_word": false,
185
+ "special": true
186
+ },
187
+ "23": {
188
+ "content": "<|reserved_17|>",
189
+ "lstrip": false,
190
+ "normalized": false,
191
+ "rstrip": false,
192
+ "single_word": false,
193
+ "special": true
194
+ },
195
+ "24": {
196
+ "content": "<|reserved_18|>",
197
+ "lstrip": false,
198
+ "normalized": false,
199
+ "rstrip": false,
200
+ "single_word": false,
201
+ "special": true
202
+ },
203
+ "25": {
204
+ "content": "<|reserved_19|>",
205
+ "lstrip": false,
206
+ "normalized": false,
207
+ "rstrip": false,
208
+ "single_word": false,
209
+ "special": true
210
+ },
211
+ "26": {
212
+ "content": "<|reserved_20|>",
213
+ "lstrip": false,
214
+ "normalized": false,
215
+ "rstrip": false,
216
+ "single_word": false,
217
+ "special": true
218
+ },
219
+ "27": {
220
+ "content": "<|reserved_21|>",
221
+ "lstrip": false,
222
+ "normalized": false,
223
+ "rstrip": false,
224
+ "single_word": false,
225
+ "special": true
226
+ },
227
+ "28": {
228
+ "content": "<|reserved_22|>",
229
+ "lstrip": false,
230
+ "normalized": false,
231
+ "rstrip": false,
232
+ "single_word": false,
233
+ "special": true
234
+ },
235
+ "29": {
236
+ "content": "<|reserved_23|>",
237
+ "lstrip": false,
238
+ "normalized": false,
239
+ "rstrip": false,
240
+ "single_word": false,
241
+ "special": true
242
+ },
243
+ "30": {
244
+ "content": "<|reserved_24|>",
245
+ "lstrip": false,
246
+ "normalized": false,
247
+ "rstrip": false,
248
+ "single_word": false,
249
+ "special": true
250
+ },
251
+ "31": {
252
+ "content": "<|reserved_25|>",
253
+ "lstrip": false,
254
+ "normalized": false,
255
+ "rstrip": false,
256
+ "single_word": false,
257
+ "special": true
258
+ },
259
+ "32": {
260
+ "content": "<|reserved_26|>",
261
+ "lstrip": false,
262
+ "normalized": false,
263
+ "rstrip": false,
264
+ "single_word": false,
265
+ "special": true
266
+ },
267
+ "33": {
268
+ "content": "<|reserved_27|>",
269
+ "lstrip": false,
270
+ "normalized": false,
271
+ "rstrip": false,
272
+ "single_word": false,
273
+ "special": true
274
+ },
275
+ "34": {
276
+ "content": "<|reserved_28|>",
277
+ "lstrip": false,
278
+ "normalized": false,
279
+ "rstrip": false,
280
+ "single_word": false,
281
+ "special": true
282
+ },
283
+ "35": {
284
+ "content": "<|reserved_29|>",
285
+ "lstrip": false,
286
+ "normalized": false,
287
+ "rstrip": false,
288
+ "single_word": false,
289
+ "special": true
290
+ },
291
+ "36": {
292
+ "content": "<|reserved_30|>",
293
+ "lstrip": false,
294
+ "normalized": false,
295
+ "rstrip": false,
296
+ "single_word": false,
297
+ "special": true
298
+ },
299
+ "37": {
300
+ "content": "<|reserved_31|>",
301
+ "lstrip": false,
302
+ "normalized": false,
303
+ "rstrip": false,
304
+ "single_word": false,
305
+ "special": true
306
+ },
307
+ "38": {
308
+ "content": "<|reserved_32|>",
309
+ "lstrip": false,
310
+ "normalized": false,
311
+ "rstrip": false,
312
+ "single_word": false,
313
+ "special": true
314
+ },
315
+ "39": {
316
+ "content": "<|reserved_33|>",
317
+ "lstrip": false,
318
+ "normalized": false,
319
+ "rstrip": false,
320
+ "single_word": false,
321
+ "special": true
322
+ },
323
+ "40": {
324
+ "content": "<|reserved_34|>",
325
+ "lstrip": false,
326
+ "normalized": false,
327
+ "rstrip": false,
328
+ "single_word": false,
329
+ "special": true
330
+ },
331
+ "41": {
332
+ "content": "<|reserved_35|>",
333
+ "lstrip": false,
334
+ "normalized": false,
335
+ "rstrip": false,
336
+ "single_word": false,
337
+ "special": true
338
+ },
339
+ "42": {
340
+ "content": "<|reserved_36|>",
341
+ "lstrip": false,
342
+ "normalized": false,
343
+ "rstrip": false,
344
+ "single_word": false,
345
+ "special": true
346
+ },
347
+ "43": {
348
+ "content": "<|reserved_37|>",
349
+ "lstrip": false,
350
+ "normalized": false,
351
+ "rstrip": false,
352
+ "single_word": false,
353
+ "special": true
354
+ },
355
+ "44": {
356
+ "content": "<|reserved_38|>",
357
+ "lstrip": false,
358
+ "normalized": false,
359
+ "rstrip": false,
360
+ "single_word": false,
361
+ "special": true
362
+ },
363
+ "45": {
364
+ "content": "<|reserved_39|>",
365
+ "lstrip": false,
366
+ "normalized": false,
367
+ "rstrip": false,
368
+ "single_word": false,
369
+ "special": true
370
+ },
371
+ "46": {
372
+ "content": "<|reserved_40|>",
373
+ "lstrip": false,
374
+ "normalized": false,
375
+ "rstrip": false,
376
+ "single_word": false,
377
+ "special": true
378
+ },
379
+ "47": {
380
+ "content": "<|reserved_41|>",
381
+ "lstrip": false,
382
+ "normalized": false,
383
+ "rstrip": false,
384
+ "single_word": false,
385
+ "special": true
386
+ },
387
+ "48": {
388
+ "content": "<|reserved_42|>",
389
+ "lstrip": false,
390
+ "normalized": false,
391
+ "rstrip": false,
392
+ "single_word": false,
393
+ "special": true
394
+ },
395
+ "49": {
396
+ "content": "<|reserved_43|>",
397
+ "lstrip": false,
398
+ "normalized": false,
399
+ "rstrip": false,
400
+ "single_word": false,
401
+ "special": true
402
+ },
403
+ "50": {
404
+ "content": "<|reserved_44|>",
405
+ "lstrip": false,
406
+ "normalized": false,
407
+ "rstrip": false,
408
+ "single_word": false,
409
+ "special": true
410
+ },
411
+ "51": {
412
+ "content": "<|reserved_45|>",
413
+ "lstrip": false,
414
+ "normalized": false,
415
+ "rstrip": false,
416
+ "single_word": false,
417
+ "special": true
418
+ },
419
+ "52": {
420
+ "content": "<|reserved_46|>",
421
+ "lstrip": false,
422
+ "normalized": false,
423
+ "rstrip": false,
424
+ "single_word": false,
425
+ "special": true
426
+ },
427
+ "53": {
428
+ "content": "<|reserved_47|>",
429
+ "lstrip": false,
430
+ "normalized": false,
431
+ "rstrip": false,
432
+ "single_word": false,
433
+ "special": true
434
+ },
435
+ "54": {
436
+ "content": "<|reserved_48|>",
437
+ "lstrip": false,
438
+ "normalized": false,
439
+ "rstrip": false,
440
+ "single_word": false,
441
+ "special": true
442
+ },
443
+ "55": {
444
+ "content": "<|reserved_49|>",
445
+ "lstrip": false,
446
+ "normalized": false,
447
+ "rstrip": false,
448
+ "single_word": false,
449
+ "special": true
450
+ },
451
+ "56": {
452
+ "content": "<|reserved_50|>",
453
+ "lstrip": false,
454
+ "normalized": false,
455
+ "rstrip": false,
456
+ "single_word": false,
457
+ "special": true
458
+ },
459
+ "57": {
460
+ "content": "<|reserved_51|>",
461
+ "lstrip": false,
462
+ "normalized": false,
463
+ "rstrip": false,
464
+ "single_word": false,
465
+ "special": true
466
+ },
467
+ "58": {
468
+ "content": "<|reserved_52|>",
469
+ "lstrip": false,
470
+ "normalized": false,
471
+ "rstrip": false,
472
+ "single_word": false,
473
+ "special": true
474
+ },
475
+ "59": {
476
+ "content": "<|reserved_53|>",
477
+ "lstrip": false,
478
+ "normalized": false,
479
+ "rstrip": false,
480
+ "single_word": false,
481
+ "special": true
482
+ },
483
+ "60": {
484
+ "content": "<|reserved_54|>",
485
+ "lstrip": false,
486
+ "normalized": false,
487
+ "rstrip": false,
488
+ "single_word": false,
489
+ "special": true
490
+ },
491
+ "61": {
492
+ "content": "<|reserved_55|>",
493
+ "lstrip": false,
494
+ "normalized": false,
495
+ "rstrip": false,
496
+ "single_word": false,
497
+ "special": true
498
+ },
499
+ "62": {
500
+ "content": "<|reserved_56|>",
501
+ "lstrip": false,
502
+ "normalized": false,
503
+ "rstrip": false,
504
+ "single_word": false,
505
+ "special": true
506
+ },
507
+ "63": {
508
+ "content": "<|reserved_57|>",
509
+ "lstrip": false,
510
+ "normalized": false,
511
+ "rstrip": false,
512
+ "single_word": false,
513
+ "special": true
514
+ },
515
+ "64": {
516
+ "content": "<|reserved_58|>",
517
+ "lstrip": false,
518
+ "normalized": false,
519
+ "rstrip": false,
520
+ "single_word": false,
521
+ "special": true
522
+ },
523
+ "65": {
524
+ "content": "<|reserved_59|>",
525
+ "lstrip": false,
526
+ "normalized": false,
527
+ "rstrip": false,
528
+ "single_word": false,
529
+ "special": true
530
+ },
531
+ "66": {
532
+ "content": "<|reserved_60|>",
533
+ "lstrip": false,
534
+ "normalized": false,
535
+ "rstrip": false,
536
+ "single_word": false,
537
+ "special": true
538
+ },
539
+ "67": {
540
+ "content": "<|reserved_61|>",
541
+ "lstrip": false,
542
+ "normalized": false,
543
+ "rstrip": false,
544
+ "single_word": false,
545
+ "special": true
546
+ },
547
+ "68": {
548
+ "content": "<|reserved_62|>",
549
+ "lstrip": false,
550
+ "normalized": false,
551
+ "rstrip": false,
552
+ "single_word": false,
553
+ "special": true
554
+ },
555
+ "69": {
556
+ "content": "<|reserved_63|>",
557
+ "lstrip": false,
558
+ "normalized": false,
559
+ "rstrip": false,
560
+ "single_word": false,
561
+ "special": true
562
+ },
563
+ "70": {
564
+ "content": "<|reserved_64|>",
565
+ "lstrip": false,
566
+ "normalized": false,
567
+ "rstrip": false,
568
+ "single_word": false,
569
+ "special": true
570
+ },
571
+ "71": {
572
+ "content": "<|reserved_65|>",
573
+ "lstrip": false,
574
+ "normalized": false,
575
+ "rstrip": false,
576
+ "single_word": false,
577
+ "special": true
578
+ },
579
+ "72": {
580
+ "content": "<|reserved_66|>",
581
+ "lstrip": false,
582
+ "normalized": false,
583
+ "rstrip": false,
584
+ "single_word": false,
585
+ "special": true
586
+ },
587
+ "73": {
588
+ "content": "<|reserved_67|>",
589
+ "lstrip": false,
590
+ "normalized": false,
591
+ "rstrip": false,
592
+ "single_word": false,
593
+ "special": true
594
+ },
595
+ "74": {
596
+ "content": "<|reserved_68|>",
597
+ "lstrip": false,
598
+ "normalized": false,
599
+ "rstrip": false,
600
+ "single_word": false,
601
+ "special": true
602
+ },
603
+ "75": {
604
+ "content": "<|reserved_69|>",
605
+ "lstrip": false,
606
+ "normalized": false,
607
+ "rstrip": false,
608
+ "single_word": false,
609
+ "special": true
610
+ },
611
+ "76": {
612
+ "content": "<|reserved_70|>",
613
+ "lstrip": false,
614
+ "normalized": false,
615
+ "rstrip": false,
616
+ "single_word": false,
617
+ "special": true
618
+ },
619
+ "77": {
620
+ "content": "<|reserved_71|>",
621
+ "lstrip": false,
622
+ "normalized": false,
623
+ "rstrip": false,
624
+ "single_word": false,
625
+ "special": true
626
+ },
627
+ "78": {
628
+ "content": "<|reserved_72|>",
629
+ "lstrip": false,
630
+ "normalized": false,
631
+ "rstrip": false,
632
+ "single_word": false,
633
+ "special": true
634
+ },
635
+ "79": {
636
+ "content": "<|reserved_73|>",
637
+ "lstrip": false,
638
+ "normalized": false,
639
+ "rstrip": false,
640
+ "single_word": false,
641
+ "special": true
642
+ },
643
+ "80": {
644
+ "content": "<|reserved_74|>",
645
+ "lstrip": false,
646
+ "normalized": false,
647
+ "rstrip": false,
648
+ "single_word": false,
649
+ "special": true
650
+ },
651
+ "81": {
652
+ "content": "<|reserved_75|>",
653
+ "lstrip": false,
654
+ "normalized": false,
655
+ "rstrip": false,
656
+ "single_word": false,
657
+ "special": true
658
+ },
659
+ "82": {
660
+ "content": "<|reserved_76|>",
661
+ "lstrip": false,
662
+ "normalized": false,
663
+ "rstrip": false,
664
+ "single_word": false,
665
+ "special": true
666
+ },
667
+ "83": {
668
+ "content": "<|reserved_77|>",
669
+ "lstrip": false,
670
+ "normalized": false,
671
+ "rstrip": false,
672
+ "single_word": false,
673
+ "special": true
674
+ },
675
+ "84": {
676
+ "content": "<|reserved_78|>",
677
+ "lstrip": false,
678
+ "normalized": false,
679
+ "rstrip": false,
680
+ "single_word": false,
681
+ "special": true
682
+ },
683
+ "85": {
684
+ "content": "<|reserved_79|>",
685
+ "lstrip": false,
686
+ "normalized": false,
687
+ "rstrip": false,
688
+ "single_word": false,
689
+ "special": true
690
+ },
691
+ "86": {
692
+ "content": "<|reserved_80|>",
693
+ "lstrip": false,
694
+ "normalized": false,
695
+ "rstrip": false,
696
+ "single_word": false,
697
+ "special": true
698
+ },
699
+ "87": {
700
+ "content": "<|reserved_81|>",
701
+ "lstrip": false,
702
+ "normalized": false,
703
+ "rstrip": false,
704
+ "single_word": false,
705
+ "special": true
706
+ },
707
+ "88": {
708
+ "content": "<|reserved_82|>",
709
+ "lstrip": false,
710
+ "normalized": false,
711
+ "rstrip": false,
712
+ "single_word": false,
713
+ "special": true
714
+ },
715
+ "89": {
716
+ "content": "<|reserved_83|>",
717
+ "lstrip": false,
718
+ "normalized": false,
719
+ "rstrip": false,
720
+ "single_word": false,
721
+ "special": true
722
+ },
723
+ "90": {
724
+ "content": "<|reserved_84|>",
725
+ "lstrip": false,
726
+ "normalized": false,
727
+ "rstrip": false,
728
+ "single_word": false,
729
+ "special": true
730
+ },
731
+ "91": {
732
+ "content": "<|reserved_85|>",
733
+ "lstrip": false,
734
+ "normalized": false,
735
+ "rstrip": false,
736
+ "single_word": false,
737
+ "special": true
738
+ },
739
+ "92": {
740
+ "content": "<|reserved_86|>",
741
+ "lstrip": false,
742
+ "normalized": false,
743
+ "rstrip": false,
744
+ "single_word": false,
745
+ "special": true
746
+ },
747
+ "93": {
748
+ "content": "<|reserved_87|>",
749
+ "lstrip": false,
750
+ "normalized": false,
751
+ "rstrip": false,
752
+ "single_word": false,
753
+ "special": true
754
+ },
755
+ "94": {
756
+ "content": "<|reserved_88|>",
757
+ "lstrip": false,
758
+ "normalized": false,
759
+ "rstrip": false,
760
+ "single_word": false,
761
+ "special": true
762
+ },
763
+ "95": {
764
+ "content": "<|reserved_89|>",
765
+ "lstrip": false,
766
+ "normalized": false,
767
+ "rstrip": false,
768
+ "single_word": false,
769
+ "special": true
770
+ },
771
+ "96": {
772
+ "content": "<|reserved_90|>",
773
+ "lstrip": false,
774
+ "normalized": false,
775
+ "rstrip": false,
776
+ "single_word": false,
777
+ "special": true
778
+ },
779
+ "97": {
780
+ "content": "<|reserved_91|>",
781
+ "lstrip": false,
782
+ "normalized": false,
783
+ "rstrip": false,
784
+ "single_word": false,
785
+ "special": true
786
+ },
787
+ "98": {
788
+ "content": "<|reserved_92|>",
789
+ "lstrip": false,
790
+ "normalized": false,
791
+ "rstrip": false,
792
+ "single_word": false,
793
+ "special": true
794
+ },
795
+ "99": {
796
+ "content": "<|reserved_93|>",
797
+ "lstrip": false,
798
+ "normalized": false,
799
+ "rstrip": false,
800
+ "single_word": false,
801
+ "special": true
802
+ },
803
+ "100": {
804
+ "content": "<|reserved_94|>",
805
+ "lstrip": false,
806
+ "normalized": false,
807
+ "rstrip": false,
808
+ "single_word": false,
809
+ "special": true
810
+ },
811
+ "101": {
812
+ "content": "<|reserved_95|>",
813
+ "lstrip": false,
814
+ "normalized": false,
815
+ "rstrip": false,
816
+ "single_word": false,
817
+ "special": true
818
+ },
819
+ "102": {
820
+ "content": "<|reserved_96|>",
821
+ "lstrip": false,
822
+ "normalized": false,
823
+ "rstrip": false,
824
+ "single_word": false,
825
+ "special": true
826
+ },
827
+ "103": {
828
+ "content": "<|reserved_97|>",
829
+ "lstrip": false,
830
+ "normalized": false,
831
+ "rstrip": false,
832
+ "single_word": false,
833
+ "special": true
834
+ },
835
+ "104": {
836
+ "content": "<|reserved_98|>",
837
+ "lstrip": false,
838
+ "normalized": false,
839
+ "rstrip": false,
840
+ "single_word": false,
841
+ "special": true
842
+ },
843
+ "105": {
844
+ "content": "<|reserved_99|>",
845
+ "lstrip": false,
846
+ "normalized": false,
847
+ "rstrip": false,
848
+ "single_word": false,
849
+ "special": true
850
+ },
851
+ "106": {
852
+ "content": "<|reserved_100|>",
853
+ "lstrip": false,
854
+ "normalized": false,
855
+ "rstrip": false,
856
+ "single_word": false,
857
+ "special": true
858
+ },
859
+ "107": {
860
+ "content": "<|reserved_101|>",
861
+ "lstrip": false,
862
+ "normalized": false,
863
+ "rstrip": false,
864
+ "single_word": false,
865
+ "special": true
866
+ },
867
+ "108": {
868
+ "content": "<|reserved_102|>",
869
+ "lstrip": false,
870
+ "normalized": false,
871
+ "rstrip": false,
872
+ "single_word": false,
873
+ "special": true
874
+ },
875
+ "109": {
876
+ "content": "<|reserved_103|>",
877
+ "lstrip": false,
878
+ "normalized": false,
879
+ "rstrip": false,
880
+ "single_word": false,
881
+ "special": true
882
+ },
883
+ "110": {
884
+ "content": "<|reserved_104|>",
885
+ "lstrip": false,
886
+ "normalized": false,
887
+ "rstrip": false,
888
+ "single_word": false,
889
+ "special": true
890
+ },
891
+ "111": {
892
+ "content": "<|reserved_105|>",
893
+ "lstrip": false,
894
+ "normalized": false,
895
+ "rstrip": false,
896
+ "single_word": false,
897
+ "special": true
898
+ },
899
+ "112": {
900
+ "content": "<|reserved_106|>",
901
+ "lstrip": false,
902
+ "normalized": false,
903
+ "rstrip": false,
904
+ "single_word": false,
905
+ "special": true
906
+ },
907
+ "113": {
908
+ "content": "<|reserved_107|>",
909
+ "lstrip": false,
910
+ "normalized": false,
911
+ "rstrip": false,
912
+ "single_word": false,
913
+ "special": true
914
+ },
915
+ "114": {
916
+ "content": "<|reserved_108|>",
917
+ "lstrip": false,
918
+ "normalized": false,
919
+ "rstrip": false,
920
+ "single_word": false,
921
+ "special": true
922
+ },
923
+ "115": {
924
+ "content": "<|reserved_109|>",
925
+ "lstrip": false,
926
+ "normalized": false,
927
+ "rstrip": false,
928
+ "single_word": false,
929
+ "special": true
930
+ },
931
+ "116": {
932
+ "content": "<|reserved_110|>",
933
+ "lstrip": false,
934
+ "normalized": false,
935
+ "rstrip": false,
936
+ "single_word": false,
937
+ "special": true
938
+ },
939
+ "117": {
940
+ "content": "<|reserved_111|>",
941
+ "lstrip": false,
942
+ "normalized": false,
943
+ "rstrip": false,
944
+ "single_word": false,
945
+ "special": true
946
+ },
947
+ "118": {
948
+ "content": "<|reserved_112|>",
949
+ "lstrip": false,
950
+ "normalized": false,
951
+ "rstrip": false,
952
+ "single_word": false,
953
+ "special": true
954
+ },
955
+ "119": {
956
+ "content": "<|reserved_113|>",
957
+ "lstrip": false,
958
+ "normalized": false,
959
+ "rstrip": false,
960
+ "single_word": false,
961
+ "special": true
962
+ },
963
+ "120": {
964
+ "content": "<|reserved_114|>",
965
+ "lstrip": false,
966
+ "normalized": false,
967
+ "rstrip": false,
968
+ "single_word": false,
969
+ "special": true
970
+ },
971
+ "121": {
972
+ "content": "<|reserved_115|>",
973
+ "lstrip": false,
974
+ "normalized": false,
975
+ "rstrip": false,
976
+ "single_word": false,
977
+ "special": true
978
+ },
979
+ "122": {
980
+ "content": "<|reserved_116|>",
981
+ "lstrip": false,
982
+ "normalized": false,
983
+ "rstrip": false,
984
+ "single_word": false,
985
+ "special": true
986
+ },
987
+ "123": {
988
+ "content": "<|reserved_117|>",
989
+ "lstrip": false,
990
+ "normalized": false,
991
+ "rstrip": false,
992
+ "single_word": false,
993
+ "special": true
994
+ },
995
+ "124": {
996
+ "content": "<|reserved_118|>",
997
+ "lstrip": false,
998
+ "normalized": false,
999
+ "rstrip": false,
1000
+ "single_word": false,
1001
+ "special": true
1002
+ },
1003
+ "125": {
1004
+ "content": "<|reserved_119|>",
1005
+ "lstrip": false,
1006
+ "normalized": false,
1007
+ "rstrip": false,
1008
+ "single_word": false,
1009
+ "special": true
1010
+ },
1011
+ "126": {
1012
+ "content": "<|reserved_120|>",
1013
+ "lstrip": false,
1014
+ "normalized": false,
1015
+ "rstrip": false,
1016
+ "single_word": false,
1017
+ "special": true
1018
+ },
1019
+ "127": {
1020
+ "content": "<|reserved_121|>",
1021
+ "lstrip": false,
1022
+ "normalized": false,
1023
+ "rstrip": false,
1024
+ "single_word": false,
1025
+ "special": true
1026
+ },
1027
+ "128": {
1028
+ "content": "<|reserved_122|>",
1029
+ "lstrip": false,
1030
+ "normalized": false,
1031
+ "rstrip": false,
1032
+ "single_word": false,
1033
+ "special": true
1034
+ },
1035
+ "129": {
1036
+ "content": "<|reserved_123|>",
1037
+ "lstrip": false,
1038
+ "normalized": false,
1039
+ "rstrip": false,
1040
+ "single_word": false,
1041
+ "special": true
1042
+ },
1043
+ "130": {
1044
+ "content": "<|reserved_124|>",
1045
+ "lstrip": false,
1046
+ "normalized": false,
1047
+ "rstrip": false,
1048
+ "single_word": false,
1049
+ "special": true
1050
+ },
1051
+ "131": {
1052
+ "content": "<|reserved_125|>",
1053
+ "lstrip": false,
1054
+ "normalized": false,
1055
+ "rstrip": false,
1056
+ "single_word": false,
1057
+ "special": true
1058
+ },
1059
+ "132": {
1060
+ "content": "<|reserved_126|>",
1061
+ "lstrip": false,
1062
+ "normalized": false,
1063
+ "rstrip": false,
1064
+ "single_word": false,
1065
+ "special": true
1066
+ },
1067
+ "133": {
1068
+ "content": "<|reserved_127|>",
1069
+ "lstrip": false,
1070
+ "normalized": false,
1071
+ "rstrip": false,
1072
+ "single_word": false,
1073
+ "special": true
1074
+ },
1075
+ "134": {
1076
+ "content": "<|reserved_128|>",
1077
+ "lstrip": false,
1078
+ "normalized": false,
1079
+ "rstrip": false,
1080
+ "single_word": false,
1081
+ "special": true
1082
+ },
1083
+ "135": {
1084
+ "content": "<|reserved_129|>",
1085
+ "lstrip": false,
1086
+ "normalized": false,
1087
+ "rstrip": false,
1088
+ "single_word": false,
1089
+ "special": true
1090
+ },
1091
+ "136": {
1092
+ "content": "<|reserved_130|>",
1093
+ "lstrip": false,
1094
+ "normalized": false,
1095
+ "rstrip": false,
1096
+ "single_word": false,
1097
+ "special": true
1098
+ },
1099
+ "137": {
1100
+ "content": "<|reserved_131|>",
1101
+ "lstrip": false,
1102
+ "normalized": false,
1103
+ "rstrip": false,
1104
+ "single_word": false,
1105
+ "special": true
1106
+ },
1107
+ "138": {
1108
+ "content": "<|reserved_132|>",
1109
+ "lstrip": false,
1110
+ "normalized": false,
1111
+ "rstrip": false,
1112
+ "single_word": false,
1113
+ "special": true
1114
+ },
1115
+ "139": {
1116
+ "content": "<|reserved_133|>",
1117
+ "lstrip": false,
1118
+ "normalized": false,
1119
+ "rstrip": false,
1120
+ "single_word": false,
1121
+ "special": true
1122
+ },
1123
+ "140": {
1124
+ "content": "<|reserved_134|>",
1125
+ "lstrip": false,
1126
+ "normalized": false,
1127
+ "rstrip": false,
1128
+ "single_word": false,
1129
+ "special": true
1130
+ },
1131
+ "141": {
1132
+ "content": "<|reserved_135|>",
1133
+ "lstrip": false,
1134
+ "normalized": false,
1135
+ "rstrip": false,
1136
+ "single_word": false,
1137
+ "special": true
1138
+ },
1139
+ "142": {
1140
+ "content": "<|reserved_136|>",
1141
+ "lstrip": false,
1142
+ "normalized": false,
1143
+ "rstrip": false,
1144
+ "single_word": false,
1145
+ "special": true
1146
+ },
1147
+ "143": {
1148
+ "content": "<|reserved_137|>",
1149
+ "lstrip": false,
1150
+ "normalized": false,
1151
+ "rstrip": false,
1152
+ "single_word": false,
1153
+ "special": true
1154
+ },
1155
+ "144": {
1156
+ "content": "<|reserved_138|>",
1157
+ "lstrip": false,
1158
+ "normalized": false,
1159
+ "rstrip": false,
1160
+ "single_word": false,
1161
+ "special": true
1162
+ },
1163
+ "145": {
1164
+ "content": "<|reserved_139|>",
1165
+ "lstrip": false,
1166
+ "normalized": false,
1167
+ "rstrip": false,
1168
+ "single_word": false,
1169
+ "special": true
1170
+ },
1171
+ "146": {
1172
+ "content": "<|reserved_140|>",
1173
+ "lstrip": false,
1174
+ "normalized": false,
1175
+ "rstrip": false,
1176
+ "single_word": false,
1177
+ "special": true
1178
+ },
1179
+ "147": {
1180
+ "content": "<|reserved_141|>",
1181
+ "lstrip": false,
1182
+ "normalized": false,
1183
+ "rstrip": false,
1184
+ "single_word": false,
1185
+ "special": true
1186
+ },
1187
+ "148": {
1188
+ "content": "<|reserved_142|>",
1189
+ "lstrip": false,
1190
+ "normalized": false,
1191
+ "rstrip": false,
1192
+ "single_word": false,
1193
+ "special": true
1194
+ },
1195
+ "149": {
1196
+ "content": "<|reserved_143|>",
1197
+ "lstrip": false,
1198
+ "normalized": false,
1199
+ "rstrip": false,
1200
+ "single_word": false,
1201
+ "special": true
1202
+ },
1203
+ "150": {
1204
+ "content": "<|reserved_144|>",
1205
+ "lstrip": false,
1206
+ "normalized": false,
1207
+ "rstrip": false,
1208
+ "single_word": false,
1209
+ "special": true
1210
+ },
1211
+ "151": {
1212
+ "content": "<|reserved_145|>",
1213
+ "lstrip": false,
1214
+ "normalized": false,
1215
+ "rstrip": false,
1216
+ "single_word": false,
1217
+ "special": true
1218
+ },
1219
+ "152": {
1220
+ "content": "<|reserved_146|>",
1221
+ "lstrip": false,
1222
+ "normalized": false,
1223
+ "rstrip": false,
1224
+ "single_word": false,
1225
+ "special": true
1226
+ },
1227
+ "153": {
1228
+ "content": "<|reserved_147|>",
1229
+ "lstrip": false,
1230
+ "normalized": false,
1231
+ "rstrip": false,
1232
+ "single_word": false,
1233
+ "special": true
1234
+ },
1235
+ "154": {
1236
+ "content": "<|reserved_148|>",
1237
+ "lstrip": false,
1238
+ "normalized": false,
1239
+ "rstrip": false,
1240
+ "single_word": false,
1241
+ "special": true
1242
+ },
1243
+ "155": {
1244
+ "content": "<|reserved_149|>",
1245
+ "lstrip": false,
1246
+ "normalized": false,
1247
+ "rstrip": false,
1248
+ "single_word": false,
1249
+ "special": true
1250
+ },
1251
+ "156": {
1252
+ "content": "<|reserved_150|>",
1253
+ "lstrip": false,
1254
+ "normalized": false,
1255
+ "rstrip": false,
1256
+ "single_word": false,
1257
+ "special": true
1258
+ },
1259
+ "157": {
1260
+ "content": "<|reserved_151|>",
1261
+ "lstrip": false,
1262
+ "normalized": false,
1263
+ "rstrip": false,
1264
+ "single_word": false,
1265
+ "special": true
1266
+ },
1267
+ "158": {
1268
+ "content": "<|reserved_152|>",
1269
+ "lstrip": false,
1270
+ "normalized": false,
1271
+ "rstrip": false,
1272
+ "single_word": false,
1273
+ "special": true
1274
+ },
1275
+ "159": {
1276
+ "content": "<|reserved_153|>",
1277
+ "lstrip": false,
1278
+ "normalized": false,
1279
+ "rstrip": false,
1280
+ "single_word": false,
1281
+ "special": true
1282
+ },
1283
+ "160": {
1284
+ "content": "<|reserved_154|>",
1285
+ "lstrip": false,
1286
+ "normalized": false,
1287
+ "rstrip": false,
1288
+ "single_word": false,
1289
+ "special": true
1290
+ },
1291
+ "161": {
1292
+ "content": "<|reserved_155|>",
1293
+ "lstrip": false,
1294
+ "normalized": false,
1295
+ "rstrip": false,
1296
+ "single_word": false,
1297
+ "special": true
1298
+ },
1299
+ "162": {
1300
+ "content": "<|reserved_156|>",
1301
+ "lstrip": false,
1302
+ "normalized": false,
1303
+ "rstrip": false,
1304
+ "single_word": false,
1305
+ "special": true
1306
+ },
1307
+ "163": {
1308
+ "content": "<|reserved_157|>",
1309
+ "lstrip": false,
1310
+ "normalized": false,
1311
+ "rstrip": false,
1312
+ "single_word": false,
1313
+ "special": true
1314
+ },
1315
+ "164": {
1316
+ "content": "<|reserved_158|>",
1317
+ "lstrip": false,
1318
+ "normalized": false,
1319
+ "rstrip": false,
1320
+ "single_word": false,
1321
+ "special": true
1322
+ },
1323
+ "165": {
1324
+ "content": "<|reserved_159|>",
1325
+ "lstrip": false,
1326
+ "normalized": false,
1327
+ "rstrip": false,
1328
+ "single_word": false,
1329
+ "special": true
1330
+ },
1331
+ "166": {
1332
+ "content": "<|reserved_160|>",
1333
+ "lstrip": false,
1334
+ "normalized": false,
1335
+ "rstrip": false,
1336
+ "single_word": false,
1337
+ "special": true
1338
+ },
1339
+ "167": {
1340
+ "content": "<|reserved_161|>",
1341
+ "lstrip": false,
1342
+ "normalized": false,
1343
+ "rstrip": false,
1344
+ "single_word": false,
1345
+ "special": true
1346
+ },
1347
+ "168": {
1348
+ "content": "<|reserved_162|>",
1349
+ "lstrip": false,
1350
+ "normalized": false,
1351
+ "rstrip": false,
1352
+ "single_word": false,
1353
+ "special": true
1354
+ },
1355
+ "169": {
1356
+ "content": "<|reserved_163|>",
1357
+ "lstrip": false,
1358
+ "normalized": false,
1359
+ "rstrip": false,
1360
+ "single_word": false,
1361
+ "special": true
1362
+ },
1363
+ "170": {
1364
+ "content": "<|reserved_164|>",
1365
+ "lstrip": false,
1366
+ "normalized": false,
1367
+ "rstrip": false,
1368
+ "single_word": false,
1369
+ "special": true
1370
+ },
1371
+ "171": {
1372
+ "content": "<|reserved_165|>",
1373
+ "lstrip": false,
1374
+ "normalized": false,
1375
+ "rstrip": false,
1376
+ "single_word": false,
1377
+ "special": true
1378
+ },
1379
+ "172": {
1380
+ "content": "<|reserved_166|>",
1381
+ "lstrip": false,
1382
+ "normalized": false,
1383
+ "rstrip": false,
1384
+ "single_word": false,
1385
+ "special": true
1386
+ },
1387
+ "173": {
1388
+ "content": "<|reserved_167|>",
1389
+ "lstrip": false,
1390
+ "normalized": false,
1391
+ "rstrip": false,
1392
+ "single_word": false,
1393
+ "special": true
1394
+ },
1395
+ "174": {
1396
+ "content": "<|reserved_168|>",
1397
+ "lstrip": false,
1398
+ "normalized": false,
1399
+ "rstrip": false,
1400
+ "single_word": false,
1401
+ "special": true
1402
+ },
1403
+ "175": {
1404
+ "content": "<|reserved_169|>",
1405
+ "lstrip": false,
1406
+ "normalized": false,
1407
+ "rstrip": false,
1408
+ "single_word": false,
1409
+ "special": true
1410
+ },
1411
+ "176": {
1412
+ "content": "<|reserved_170|>",
1413
+ "lstrip": false,
1414
+ "normalized": false,
1415
+ "rstrip": false,
1416
+ "single_word": false,
1417
+ "special": true
1418
+ },
1419
+ "177": {
1420
+ "content": "<|reserved_171|>",
1421
+ "lstrip": false,
1422
+ "normalized": false,
1423
+ "rstrip": false,
1424
+ "single_word": false,
1425
+ "special": true
1426
+ },
1427
+ "178": {
1428
+ "content": "<|reserved_172|>",
1429
+ "lstrip": false,
1430
+ "normalized": false,
1431
+ "rstrip": false,
1432
+ "single_word": false,
1433
+ "special": true
1434
+ },
1435
+ "179": {
1436
+ "content": "<|reserved_173|>",
1437
+ "lstrip": false,
1438
+ "normalized": false,
1439
+ "rstrip": false,
1440
+ "single_word": false,
1441
+ "special": true
1442
+ },
1443
+ "180": {
1444
+ "content": "<|reserved_174|>",
1445
+ "lstrip": false,
1446
+ "normalized": false,
1447
+ "rstrip": false,
1448
+ "single_word": false,
1449
+ "special": true
1450
+ },
1451
+ "181": {
1452
+ "content": "<|reserved_175|>",
1453
+ "lstrip": false,
1454
+ "normalized": false,
1455
+ "rstrip": false,
1456
+ "single_word": false,
1457
+ "special": true
1458
+ },
1459
+ "182": {
1460
+ "content": "<|reserved_176|>",
1461
+ "lstrip": false,
1462
+ "normalized": false,
1463
+ "rstrip": false,
1464
+ "single_word": false,
1465
+ "special": true
1466
+ },
1467
+ "183": {
1468
+ "content": "<|reserved_177|>",
1469
+ "lstrip": false,
1470
+ "normalized": false,
1471
+ "rstrip": false,
1472
+ "single_word": false,
1473
+ "special": true
1474
+ },
1475
+ "184": {
1476
+ "content": "<|reserved_178|>",
1477
+ "lstrip": false,
1478
+ "normalized": false,
1479
+ "rstrip": false,
1480
+ "single_word": false,
1481
+ "special": true
1482
+ },
1483
+ "185": {
1484
+ "content": "<|reserved_179|>",
1485
+ "lstrip": false,
1486
+ "normalized": false,
1487
+ "rstrip": false,
1488
+ "single_word": false,
1489
+ "special": true
1490
+ },
1491
+ "186": {
1492
+ "content": "<|reserved_180|>",
1493
+ "lstrip": false,
1494
+ "normalized": false,
1495
+ "rstrip": false,
1496
+ "single_word": false,
1497
+ "special": true
1498
+ },
1499
+ "187": {
1500
+ "content": "<|reserved_181|>",
1501
+ "lstrip": false,
1502
+ "normalized": false,
1503
+ "rstrip": false,
1504
+ "single_word": false,
1505
+ "special": true
1506
+ },
1507
+ "188": {
1508
+ "content": "<|reserved_182|>",
1509
+ "lstrip": false,
1510
+ "normalized": false,
1511
+ "rstrip": false,
1512
+ "single_word": false,
1513
+ "special": true
1514
+ },
1515
+ "189": {
1516
+ "content": "<|reserved_183|>",
1517
+ "lstrip": false,
1518
+ "normalized": false,
1519
+ "rstrip": false,
1520
+ "single_word": false,
1521
+ "special": true
1522
+ },
1523
+ "190": {
1524
+ "content": "<|reserved_184|>",
1525
+ "lstrip": false,
1526
+ "normalized": false,
1527
+ "rstrip": false,
1528
+ "single_word": false,
1529
+ "special": true
1530
+ },
1531
+ "191": {
1532
+ "content": "<|reserved_185|>",
1533
+ "lstrip": false,
1534
+ "normalized": false,
1535
+ "rstrip": false,
1536
+ "single_word": false,
1537
+ "special": true
1538
+ },
1539
+ "192": {
1540
+ "content": "<|reserved_186|>",
1541
+ "lstrip": false,
1542
+ "normalized": false,
1543
+ "rstrip": false,
1544
+ "single_word": false,
1545
+ "special": true
1546
+ },
1547
+ "193": {
1548
+ "content": "<|reserved_187|>",
1549
+ "lstrip": false,
1550
+ "normalized": false,
1551
+ "rstrip": false,
1552
+ "single_word": false,
1553
+ "special": true
1554
+ },
1555
+ "194": {
1556
+ "content": "<|reserved_188|>",
1557
+ "lstrip": false,
1558
+ "normalized": false,
1559
+ "rstrip": false,
1560
+ "single_word": false,
1561
+ "special": true
1562
+ },
1563
+ "195": {
1564
+ "content": "<|reserved_189|>",
1565
+ "lstrip": false,
1566
+ "normalized": false,
1567
+ "rstrip": false,
1568
+ "single_word": false,
1569
+ "special": true
1570
+ },
1571
+ "196": {
1572
+ "content": "<|reserved_190|>",
1573
+ "lstrip": false,
1574
+ "normalized": false,
1575
+ "rstrip": false,
1576
+ "single_word": false,
1577
+ "special": true
1578
+ },
1579
+ "197": {
1580
+ "content": "<|reserved_191|>",
1581
+ "lstrip": false,
1582
+ "normalized": false,
1583
+ "rstrip": false,
1584
+ "single_word": false,
1585
+ "special": true
1586
+ },
1587
+ "198": {
1588
+ "content": "<|reserved_192|>",
1589
+ "lstrip": false,
1590
+ "normalized": false,
1591
+ "rstrip": false,
1592
+ "single_word": false,
1593
+ "special": true
1594
+ },
1595
+ "199": {
1596
+ "content": "<|reserved_193|>",
1597
+ "lstrip": false,
1598
+ "normalized": false,
1599
+ "rstrip": false,
1600
+ "single_word": false,
1601
+ "special": true
1602
+ },
1603
+ "200": {
1604
+ "content": "<|reserved_194|>",
1605
+ "lstrip": false,
1606
+ "normalized": false,
1607
+ "rstrip": false,
1608
+ "single_word": false,
1609
+ "special": true
1610
+ },
1611
+ "201": {
1612
+ "content": "<|reserved_195|>",
1613
+ "lstrip": false,
1614
+ "normalized": false,
1615
+ "rstrip": false,
1616
+ "single_word": false,
1617
+ "special": true
1618
+ },
1619
+ "202": {
1620
+ "content": "<|reserved_196|>",
1621
+ "lstrip": false,
1622
+ "normalized": false,
1623
+ "rstrip": false,
1624
+ "single_word": false,
1625
+ "special": true
1626
+ },
1627
+ "203": {
1628
+ "content": "<|reserved_197|>",
1629
+ "lstrip": false,
1630
+ "normalized": false,
1631
+ "rstrip": false,
1632
+ "single_word": false,
1633
+ "special": true
1634
+ },
1635
+ "204": {
1636
+ "content": "<|reserved_198|>",
1637
+ "lstrip": false,
1638
+ "normalized": false,
1639
+ "rstrip": false,
1640
+ "single_word": false,
1641
+ "special": true
1642
+ },
1643
+ "205": {
1644
+ "content": "<|reserved_199|>",
1645
+ "lstrip": false,
1646
+ "normalized": false,
1647
+ "rstrip": false,
1648
+ "single_word": false,
1649
+ "special": true
1650
+ },
1651
+ "206": {
1652
+ "content": "<|reserved_200|>",
1653
+ "lstrip": false,
1654
+ "normalized": false,
1655
+ "rstrip": false,
1656
+ "single_word": false,
1657
+ "special": true
1658
+ },
1659
+ "207": {
1660
+ "content": "<|reserved_201|>",
1661
+ "lstrip": false,
1662
+ "normalized": false,
1663
+ "rstrip": false,
1664
+ "single_word": false,
1665
+ "special": true
1666
+ },
1667
+ "208": {
1668
+ "content": "<|reserved_202|>",
1669
+ "lstrip": false,
1670
+ "normalized": false,
1671
+ "rstrip": false,
1672
+ "single_word": false,
1673
+ "special": true
1674
+ },
1675
+ "209": {
1676
+ "content": "<|reserved_203|>",
1677
+ "lstrip": false,
1678
+ "normalized": false,
1679
+ "rstrip": false,
1680
+ "single_word": false,
1681
+ "special": true
1682
+ },
1683
+ "210": {
1684
+ "content": "<|reserved_204|>",
1685
+ "lstrip": false,
1686
+ "normalized": false,
1687
+ "rstrip": false,
1688
+ "single_word": false,
1689
+ "special": true
1690
+ },
1691
+ "211": {
1692
+ "content": "<|reserved_205|>",
1693
+ "lstrip": false,
1694
+ "normalized": false,
1695
+ "rstrip": false,
1696
+ "single_word": false,
1697
+ "special": true
1698
+ },
1699
+ "212": {
1700
+ "content": "<|reserved_206|>",
1701
+ "lstrip": false,
1702
+ "normalized": false,
1703
+ "rstrip": false,
1704
+ "single_word": false,
1705
+ "special": true
1706
+ },
1707
+ "213": {
1708
+ "content": "<|reserved_207|>",
1709
+ "lstrip": false,
1710
+ "normalized": false,
1711
+ "rstrip": false,
1712
+ "single_word": false,
1713
+ "special": true
1714
+ },
1715
+ "214": {
1716
+ "content": "<|reserved_208|>",
1717
+ "lstrip": false,
1718
+ "normalized": false,
1719
+ "rstrip": false,
1720
+ "single_word": false,
1721
+ "special": true
1722
+ },
1723
+ "215": {
1724
+ "content": "<|reserved_209|>",
1725
+ "lstrip": false,
1726
+ "normalized": false,
1727
+ "rstrip": false,
1728
+ "single_word": false,
1729
+ "special": true
1730
+ },
1731
+ "216": {
1732
+ "content": "<|reserved_210|>",
1733
+ "lstrip": false,
1734
+ "normalized": false,
1735
+ "rstrip": false,
1736
+ "single_word": false,
1737
+ "special": true
1738
+ },
1739
+ "217": {
1740
+ "content": "<|reserved_211|>",
1741
+ "lstrip": false,
1742
+ "normalized": false,
1743
+ "rstrip": false,
1744
+ "single_word": false,
1745
+ "special": true
1746
+ },
1747
+ "218": {
1748
+ "content": "<|reserved_212|>",
1749
+ "lstrip": false,
1750
+ "normalized": false,
1751
+ "rstrip": false,
1752
+ "single_word": false,
1753
+ "special": true
1754
+ },
1755
+ "219": {
1756
+ "content": "<|reserved_213|>",
1757
+ "lstrip": false,
1758
+ "normalized": false,
1759
+ "rstrip": false,
1760
+ "single_word": false,
1761
+ "special": true
1762
+ },
1763
+ "220": {
1764
+ "content": "<|reserved_214|>",
1765
+ "lstrip": false,
1766
+ "normalized": false,
1767
+ "rstrip": false,
1768
+ "single_word": false,
1769
+ "special": true
1770
+ },
1771
+ "221": {
1772
+ "content": "<|reserved_215|>",
1773
+ "lstrip": false,
1774
+ "normalized": false,
1775
+ "rstrip": false,
1776
+ "single_word": false,
1777
+ "special": true
1778
+ },
1779
+ "222": {
1780
+ "content": "<|reserved_216|>",
1781
+ "lstrip": false,
1782
+ "normalized": false,
1783
+ "rstrip": false,
1784
+ "single_word": false,
1785
+ "special": true
1786
+ },
1787
+ "223": {
1788
+ "content": "<|reserved_217|>",
1789
+ "lstrip": false,
1790
+ "normalized": false,
1791
+ "rstrip": false,
1792
+ "single_word": false,
1793
+ "special": true
1794
+ },
1795
+ "224": {
1796
+ "content": "<|reserved_218|>",
1797
+ "lstrip": false,
1798
+ "normalized": false,
1799
+ "rstrip": false,
1800
+ "single_word": false,
1801
+ "special": true
1802
+ },
1803
+ "225": {
1804
+ "content": "<|reserved_219|>",
1805
+ "lstrip": false,
1806
+ "normalized": false,
1807
+ "rstrip": false,
1808
+ "single_word": false,
1809
+ "special": true
1810
+ },
1811
+ "226": {
1812
+ "content": "<|reserved_220|>",
1813
+ "lstrip": false,
1814
+ "normalized": false,
1815
+ "rstrip": false,
1816
+ "single_word": false,
1817
+ "special": true
1818
+ },
1819
+ "227": {
1820
+ "content": "<|reserved_221|>",
1821
+ "lstrip": false,
1822
+ "normalized": false,
1823
+ "rstrip": false,
1824
+ "single_word": false,
1825
+ "special": true
1826
+ },
1827
+ "228": {
1828
+ "content": "<|reserved_222|>",
1829
+ "lstrip": false,
1830
+ "normalized": false,
1831
+ "rstrip": false,
1832
+ "single_word": false,
1833
+ "special": true
1834
+ },
1835
+ "229": {
1836
+ "content": "<|reserved_223|>",
1837
+ "lstrip": false,
1838
+ "normalized": false,
1839
+ "rstrip": false,
1840
+ "single_word": false,
1841
+ "special": true
1842
+ },
1843
+ "230": {
1844
+ "content": "<|reserved_224|>",
1845
+ "lstrip": false,
1846
+ "normalized": false,
1847
+ "rstrip": false,
1848
+ "single_word": false,
1849
+ "special": true
1850
+ },
1851
+ "231": {
1852
+ "content": "<|reserved_225|>",
1853
+ "lstrip": false,
1854
+ "normalized": false,
1855
+ "rstrip": false,
1856
+ "single_word": false,
1857
+ "special": true
1858
+ },
1859
+ "232": {
1860
+ "content": "<|reserved_226|>",
1861
+ "lstrip": false,
1862
+ "normalized": false,
1863
+ "rstrip": false,
1864
+ "single_word": false,
1865
+ "special": true
1866
+ },
1867
+ "233": {
1868
+ "content": "<|reserved_227|>",
1869
+ "lstrip": false,
1870
+ "normalized": false,
1871
+ "rstrip": false,
1872
+ "single_word": false,
1873
+ "special": true
1874
+ },
1875
+ "234": {
1876
+ "content": "<|reserved_228|>",
1877
+ "lstrip": false,
1878
+ "normalized": false,
1879
+ "rstrip": false,
1880
+ "single_word": false,
1881
+ "special": true
1882
+ },
1883
+ "235": {
1884
+ "content": "<|reserved_229|>",
1885
+ "lstrip": false,
1886
+ "normalized": false,
1887
+ "rstrip": false,
1888
+ "single_word": false,
1889
+ "special": true
1890
+ },
1891
+ "236": {
1892
+ "content": "<|reserved_230|>",
1893
+ "lstrip": false,
1894
+ "normalized": false,
1895
+ "rstrip": false,
1896
+ "single_word": false,
1897
+ "special": true
1898
+ },
1899
+ "237": {
1900
+ "content": "<|reserved_231|>",
1901
+ "lstrip": false,
1902
+ "normalized": false,
1903
+ "rstrip": false,
1904
+ "single_word": false,
1905
+ "special": true
1906
+ },
1907
+ "238": {
1908
+ "content": "<|reserved_232|>",
1909
+ "lstrip": false,
1910
+ "normalized": false,
1911
+ "rstrip": false,
1912
+ "single_word": false,
1913
+ "special": true
1914
+ },
1915
+ "239": {
1916
+ "content": "<|reserved_233|>",
1917
+ "lstrip": false,
1918
+ "normalized": false,
1919
+ "rstrip": false,
1920
+ "single_word": false,
1921
+ "special": true
1922
+ },
1923
+ "240": {
1924
+ "content": "<|reserved_234|>",
1925
+ "lstrip": false,
1926
+ "normalized": false,
1927
+ "rstrip": false,
1928
+ "single_word": false,
1929
+ "special": true
1930
+ },
1931
+ "241": {
1932
+ "content": "<|reserved_235|>",
1933
+ "lstrip": false,
1934
+ "normalized": false,
1935
+ "rstrip": false,
1936
+ "single_word": false,
1937
+ "special": true
1938
+ },
1939
+ "242": {
1940
+ "content": "<|reserved_236|>",
1941
+ "lstrip": false,
1942
+ "normalized": false,
1943
+ "rstrip": false,
1944
+ "single_word": false,
1945
+ "special": true
1946
+ },
1947
+ "243": {
1948
+ "content": "<|reserved_237|>",
1949
+ "lstrip": false,
1950
+ "normalized": false,
1951
+ "rstrip": false,
1952
+ "single_word": false,
1953
+ "special": true
1954
+ },
1955
+ "244": {
1956
+ "content": "<|reserved_238|>",
1957
+ "lstrip": false,
1958
+ "normalized": false,
1959
+ "rstrip": false,
1960
+ "single_word": false,
1961
+ "special": true
1962
+ },
1963
+ "245": {
1964
+ "content": "<|reserved_239|>",
1965
+ "lstrip": false,
1966
+ "normalized": false,
1967
+ "rstrip": false,
1968
+ "single_word": false,
1969
+ "special": true
1970
+ },
1971
+ "246": {
1972
+ "content": "<|reserved_240|>",
1973
+ "lstrip": false,
1974
+ "normalized": false,
1975
+ "rstrip": false,
1976
+ "single_word": false,
1977
+ "special": true
1978
+ },
1979
+ "247": {
1980
+ "content": "<|reserved_241|>",
1981
+ "lstrip": false,
1982
+ "normalized": false,
1983
+ "rstrip": false,
1984
+ "single_word": false,
1985
+ "special": true
1986
+ },
1987
+ "248": {
1988
+ "content": "<|reserved_242|>",
1989
+ "lstrip": false,
1990
+ "normalized": false,
1991
+ "rstrip": false,
1992
+ "single_word": false,
1993
+ "special": true
1994
+ },
1995
+ "249": {
1996
+ "content": "<|reserved_243|>",
1997
+ "lstrip": false,
1998
+ "normalized": false,
1999
+ "rstrip": false,
2000
+ "single_word": false,
2001
+ "special": true
2002
+ },
2003
+ "250": {
2004
+ "content": "<|reserved_244|>",
2005
+ "lstrip": false,
2006
+ "normalized": false,
2007
+ "rstrip": false,
2008
+ "single_word": false,
2009
+ "special": true
2010
+ },
2011
+ "251": {
2012
+ "content": "<|reserved_245|>",
2013
+ "lstrip": false,
2014
+ "normalized": false,
2015
+ "rstrip": false,
2016
+ "single_word": false,
2017
+ "special": true
2018
+ },
2019
+ "252": {
2020
+ "content": "<|reserved_246|>",
2021
+ "lstrip": false,
2022
+ "normalized": false,
2023
+ "rstrip": false,
2024
+ "single_word": false,
2025
+ "special": true
2026
+ },
2027
+ "253": {
2028
+ "content": "<|reserved_247|>",
2029
+ "lstrip": false,
2030
+ "normalized": false,
2031
+ "rstrip": false,
2032
+ "single_word": false,
2033
+ "special": true
2034
+ },
2035
+ "254": {
2036
+ "content": "<|reserved_248|>",
2037
+ "lstrip": false,
2038
+ "normalized": false,
2039
+ "rstrip": false,
2040
+ "single_word": false,
2041
+ "special": true
2042
+ },
2043
+ "255": {
2044
+ "content": "<|reserved_249|>",
2045
+ "lstrip": false,
2046
+ "normalized": false,
2047
+ "rstrip": false,
2048
+ "single_word": false,
2049
+ "special": true
2050
+ },
2051
+ "256": {
2052
+ "content": "<0x00>",
2053
+ "lstrip": false,
2054
+ "normalized": false,
2055
+ "rstrip": false,
2056
+ "single_word": false,
2057
+ "special": true
2058
+ },
2059
+ "257": {
2060
+ "content": "<0x01>",
2061
+ "lstrip": false,
2062
+ "normalized": false,
2063
+ "rstrip": false,
2064
+ "single_word": false,
2065
+ "special": true
2066
+ },
2067
+ "258": {
2068
+ "content": "<0x02>",
2069
+ "lstrip": false,
2070
+ "normalized": false,
2071
+ "rstrip": false,
2072
+ "single_word": false,
2073
+ "special": true
2074
+ },
2075
+ "259": {
2076
+ "content": "<0x03>",
2077
+ "lstrip": false,
2078
+ "normalized": false,
2079
+ "rstrip": false,
2080
+ "single_word": false,
2081
+ "special": true
2082
+ },
2083
+ "260": {
2084
+ "content": "<0x04>",
2085
+ "lstrip": false,
2086
+ "normalized": false,
2087
+ "rstrip": false,
2088
+ "single_word": false,
2089
+ "special": true
2090
+ },
2091
+ "261": {
2092
+ "content": "<0x05>",
2093
+ "lstrip": false,
2094
+ "normalized": false,
2095
+ "rstrip": false,
2096
+ "single_word": false,
2097
+ "special": true
2098
+ },
2099
+ "262": {
2100
+ "content": "<0x06>",
2101
+ "lstrip": false,
2102
+ "normalized": false,
2103
+ "rstrip": false,
2104
+ "single_word": false,
2105
+ "special": true
2106
+ },
2107
+ "263": {
2108
+ "content": "<0x07>",
2109
+ "lstrip": false,
2110
+ "normalized": false,
2111
+ "rstrip": false,
2112
+ "single_word": false,
2113
+ "special": true
2114
+ },
2115
+ "264": {
2116
+ "content": "<0x08>",
2117
+ "lstrip": false,
2118
+ "normalized": false,
2119
+ "rstrip": false,
2120
+ "single_word": false,
2121
+ "special": true
2122
+ },
2123
+ "265": {
2124
+ "content": "<0x09>",
2125
+ "lstrip": false,
2126
+ "normalized": false,
2127
+ "rstrip": false,
2128
+ "single_word": false,
2129
+ "special": true
2130
+ },
2131
+ "266": {
2132
+ "content": "<0x0A>",
2133
+ "lstrip": false,
2134
+ "normalized": false,
2135
+ "rstrip": false,
2136
+ "single_word": false,
2137
+ "special": true
2138
+ },
2139
+ "267": {
2140
+ "content": "<0x0B>",
2141
+ "lstrip": false,
2142
+ "normalized": false,
2143
+ "rstrip": false,
2144
+ "single_word": false,
2145
+ "special": true
2146
+ },
2147
+ "268": {
2148
+ "content": "<0x0C>",
2149
+ "lstrip": false,
2150
+ "normalized": false,
2151
+ "rstrip": false,
2152
+ "single_word": false,
2153
+ "special": true
2154
+ },
2155
+ "269": {
2156
+ "content": "<0x0D>",
2157
+ "lstrip": false,
2158
+ "normalized": false,
2159
+ "rstrip": false,
2160
+ "single_word": false,
2161
+ "special": true
2162
+ },
2163
+ "270": {
2164
+ "content": "<0x0E>",
2165
+ "lstrip": false,
2166
+ "normalized": false,
2167
+ "rstrip": false,
2168
+ "single_word": false,
2169
+ "special": true
2170
+ },
2171
+ "271": {
2172
+ "content": "<0x0F>",
2173
+ "lstrip": false,
2174
+ "normalized": false,
2175
+ "rstrip": false,
2176
+ "single_word": false,
2177
+ "special": true
2178
+ },
2179
+ "272": {
2180
+ "content": "<0x10>",
2181
+ "lstrip": false,
2182
+ "normalized": false,
2183
+ "rstrip": false,
2184
+ "single_word": false,
2185
+ "special": true
2186
+ },
2187
+ "273": {
2188
+ "content": "<0x11>",
2189
+ "lstrip": false,
2190
+ "normalized": false,
2191
+ "rstrip": false,
2192
+ "single_word": false,
2193
+ "special": true
2194
+ },
2195
+ "274": {
2196
+ "content": "<0x12>",
2197
+ "lstrip": false,
2198
+ "normalized": false,
2199
+ "rstrip": false,
2200
+ "single_word": false,
2201
+ "special": true
2202
+ },
2203
+ "275": {
2204
+ "content": "<0x13>",
2205
+ "lstrip": false,
2206
+ "normalized": false,
2207
+ "rstrip": false,
2208
+ "single_word": false,
2209
+ "special": true
2210
+ },
2211
+ "276": {
2212
+ "content": "<0x14>",
2213
+ "lstrip": false,
2214
+ "normalized": false,
2215
+ "rstrip": false,
2216
+ "single_word": false,
2217
+ "special": true
2218
+ },
2219
+ "277": {
2220
+ "content": "<0x15>",
2221
+ "lstrip": false,
2222
+ "normalized": false,
2223
+ "rstrip": false,
2224
+ "single_word": false,
2225
+ "special": true
2226
+ },
2227
+ "278": {
2228
+ "content": "<0x16>",
2229
+ "lstrip": false,
2230
+ "normalized": false,
2231
+ "rstrip": false,
2232
+ "single_word": false,
2233
+ "special": true
2234
+ },
2235
+ "279": {
2236
+ "content": "<0x17>",
2237
+ "lstrip": false,
2238
+ "normalized": false,
2239
+ "rstrip": false,
2240
+ "single_word": false,
2241
+ "special": true
2242
+ },
2243
+ "280": {
2244
+ "content": "<0x18>",
2245
+ "lstrip": false,
2246
+ "normalized": false,
2247
+ "rstrip": false,
2248
+ "single_word": false,
2249
+ "special": true
2250
+ },
2251
+ "281": {
2252
+ "content": "<0x19>",
2253
+ "lstrip": false,
2254
+ "normalized": false,
2255
+ "rstrip": false,
2256
+ "single_word": false,
2257
+ "special": true
2258
+ },
2259
+ "282": {
2260
+ "content": "<0x1A>",
2261
+ "lstrip": false,
2262
+ "normalized": false,
2263
+ "rstrip": false,
2264
+ "single_word": false,
2265
+ "special": true
2266
+ },
2267
+ "283": {
2268
+ "content": "<0x1B>",
2269
+ "lstrip": false,
2270
+ "normalized": false,
2271
+ "rstrip": false,
2272
+ "single_word": false,
2273
+ "special": true
2274
+ },
2275
+ "284": {
2276
+ "content": "<0x1C>",
2277
+ "lstrip": false,
2278
+ "normalized": false,
2279
+ "rstrip": false,
2280
+ "single_word": false,
2281
+ "special": true
2282
+ },
2283
+ "285": {
2284
+ "content": "<0x1D>",
2285
+ "lstrip": false,
2286
+ "normalized": false,
2287
+ "rstrip": false,
2288
+ "single_word": false,
2289
+ "special": true
2290
+ },
2291
+ "286": {
2292
+ "content": "<0x1E>",
2293
+ "lstrip": false,
2294
+ "normalized": false,
2295
+ "rstrip": false,
2296
+ "single_word": false,
2297
+ "special": true
2298
+ },
2299
+ "287": {
2300
+ "content": "<0x1F>",
2301
+ "lstrip": false,
2302
+ "normalized": false,
2303
+ "rstrip": false,
2304
+ "single_word": false,
2305
+ "special": true
2306
+ },
2307
+ "288": {
2308
+ "content": "<0x20>",
2309
+ "lstrip": false,
2310
+ "normalized": false,
2311
+ "rstrip": false,
2312
+ "single_word": false,
2313
+ "special": true
2314
+ },
2315
+ "289": {
2316
+ "content": "<0x21>",
2317
+ "lstrip": false,
2318
+ "normalized": false,
2319
+ "rstrip": false,
2320
+ "single_word": false,
2321
+ "special": true
2322
+ },
2323
+ "290": {
2324
+ "content": "<0x22>",
2325
+ "lstrip": false,
2326
+ "normalized": false,
2327
+ "rstrip": false,
2328
+ "single_word": false,
2329
+ "special": true
2330
+ },
2331
+ "291": {
2332
+ "content": "<0x23>",
2333
+ "lstrip": false,
2334
+ "normalized": false,
2335
+ "rstrip": false,
2336
+ "single_word": false,
2337
+ "special": true
2338
+ },
2339
+ "292": {
2340
+ "content": "<0x24>",
2341
+ "lstrip": false,
2342
+ "normalized": false,
2343
+ "rstrip": false,
2344
+ "single_word": false,
2345
+ "special": true
2346
+ },
2347
+ "293": {
2348
+ "content": "<0x25>",
2349
+ "lstrip": false,
2350
+ "normalized": false,
2351
+ "rstrip": false,
2352
+ "single_word": false,
2353
+ "special": true
2354
+ },
2355
+ "294": {
2356
+ "content": "<0x26>",
2357
+ "lstrip": false,
2358
+ "normalized": false,
2359
+ "rstrip": false,
2360
+ "single_word": false,
2361
+ "special": true
2362
+ },
2363
+ "295": {
2364
+ "content": "<0x27>",
2365
+ "lstrip": false,
2366
+ "normalized": false,
2367
+ "rstrip": false,
2368
+ "single_word": false,
2369
+ "special": true
2370
+ },
2371
+ "296": {
2372
+ "content": "<0x28>",
2373
+ "lstrip": false,
2374
+ "normalized": false,
2375
+ "rstrip": false,
2376
+ "single_word": false,
2377
+ "special": true
2378
+ },
2379
+ "297": {
2380
+ "content": "<0x29>",
2381
+ "lstrip": false,
2382
+ "normalized": false,
2383
+ "rstrip": false,
2384
+ "single_word": false,
2385
+ "special": true
2386
+ },
2387
+ "298": {
2388
+ "content": "<0x2A>",
2389
+ "lstrip": false,
2390
+ "normalized": false,
2391
+ "rstrip": false,
2392
+ "single_word": false,
2393
+ "special": true
2394
+ },
2395
+ "299": {
2396
+ "content": "<0x2B>",
2397
+ "lstrip": false,
2398
+ "normalized": false,
2399
+ "rstrip": false,
2400
+ "single_word": false,
2401
+ "special": true
2402
+ },
2403
+ "300": {
2404
+ "content": "<0x2C>",
2405
+ "lstrip": false,
2406
+ "normalized": false,
2407
+ "rstrip": false,
2408
+ "single_word": false,
2409
+ "special": true
2410
+ },
2411
+ "301": {
2412
+ "content": "<0x2D>",
2413
+ "lstrip": false,
2414
+ "normalized": false,
2415
+ "rstrip": false,
2416
+ "single_word": false,
2417
+ "special": true
2418
+ },
2419
+ "302": {
2420
+ "content": "<0x2E>",
2421
+ "lstrip": false,
2422
+ "normalized": false,
2423
+ "rstrip": false,
2424
+ "single_word": false,
2425
+ "special": true
2426
+ },
2427
+ "303": {
2428
+ "content": "<0x2F>",
2429
+ "lstrip": false,
2430
+ "normalized": false,
2431
+ "rstrip": false,
2432
+ "single_word": false,
2433
+ "special": true
2434
+ },
2435
+ "304": {
2436
+ "content": "<0x30>",
2437
+ "lstrip": false,
2438
+ "normalized": false,
2439
+ "rstrip": false,
2440
+ "single_word": false,
2441
+ "special": true
2442
+ },
2443
+ "305": {
2444
+ "content": "<0x31>",
2445
+ "lstrip": false,
2446
+ "normalized": false,
2447
+ "rstrip": false,
2448
+ "single_word": false,
2449
+ "special": true
2450
+ },
2451
+ "306": {
2452
+ "content": "<0x32>",
2453
+ "lstrip": false,
2454
+ "normalized": false,
2455
+ "rstrip": false,
2456
+ "single_word": false,
2457
+ "special": true
2458
+ },
2459
+ "307": {
2460
+ "content": "<0x33>",
2461
+ "lstrip": false,
2462
+ "normalized": false,
2463
+ "rstrip": false,
2464
+ "single_word": false,
2465
+ "special": true
2466
+ },
2467
+ "308": {
2468
+ "content": "<0x34>",
2469
+ "lstrip": false,
2470
+ "normalized": false,
2471
+ "rstrip": false,
2472
+ "single_word": false,
2473
+ "special": true
2474
+ },
2475
+ "309": {
2476
+ "content": "<0x35>",
2477
+ "lstrip": false,
2478
+ "normalized": false,
2479
+ "rstrip": false,
2480
+ "single_word": false,
2481
+ "special": true
2482
+ },
2483
+ "310": {
2484
+ "content": "<0x36>",
2485
+ "lstrip": false,
2486
+ "normalized": false,
2487
+ "rstrip": false,
2488
+ "single_word": false,
2489
+ "special": true
2490
+ },
2491
+ "311": {
2492
+ "content": "<0x37>",
2493
+ "lstrip": false,
2494
+ "normalized": false,
2495
+ "rstrip": false,
2496
+ "single_word": false,
2497
+ "special": true
2498
+ },
2499
+ "312": {
2500
+ "content": "<0x38>",
2501
+ "lstrip": false,
2502
+ "normalized": false,
2503
+ "rstrip": false,
2504
+ "single_word": false,
2505
+ "special": true
2506
+ },
2507
+ "313": {
2508
+ "content": "<0x39>",
2509
+ "lstrip": false,
2510
+ "normalized": false,
2511
+ "rstrip": false,
2512
+ "single_word": false,
2513
+ "special": true
2514
+ },
2515
+ "314": {
2516
+ "content": "<0x3A>",
2517
+ "lstrip": false,
2518
+ "normalized": false,
2519
+ "rstrip": false,
2520
+ "single_word": false,
2521
+ "special": true
2522
+ },
2523
+ "315": {
2524
+ "content": "<0x3B>",
2525
+ "lstrip": false,
2526
+ "normalized": false,
2527
+ "rstrip": false,
2528
+ "single_word": false,
2529
+ "special": true
2530
+ },
2531
+ "316": {
2532
+ "content": "<0x3C>",
2533
+ "lstrip": false,
2534
+ "normalized": false,
2535
+ "rstrip": false,
2536
+ "single_word": false,
2537
+ "special": true
2538
+ },
2539
+ "317": {
2540
+ "content": "<0x3D>",
2541
+ "lstrip": false,
2542
+ "normalized": false,
2543
+ "rstrip": false,
2544
+ "single_word": false,
2545
+ "special": true
2546
+ },
2547
+ "318": {
2548
+ "content": "<0x3E>",
2549
+ "lstrip": false,
2550
+ "normalized": false,
2551
+ "rstrip": false,
2552
+ "single_word": false,
2553
+ "special": true
2554
+ },
2555
+ "319": {
2556
+ "content": "<0x3F>",
2557
+ "lstrip": false,
2558
+ "normalized": false,
2559
+ "rstrip": false,
2560
+ "single_word": false,
2561
+ "special": true
2562
+ },
2563
+ "320": {
2564
+ "content": "<0x40>",
2565
+ "lstrip": false,
2566
+ "normalized": false,
2567
+ "rstrip": false,
2568
+ "single_word": false,
2569
+ "special": true
2570
+ },
2571
+ "321": {
2572
+ "content": "<0x41>",
2573
+ "lstrip": false,
2574
+ "normalized": false,
2575
+ "rstrip": false,
2576
+ "single_word": false,
2577
+ "special": true
2578
+ },
2579
+ "322": {
2580
+ "content": "<0x42>",
2581
+ "lstrip": false,
2582
+ "normalized": false,
2583
+ "rstrip": false,
2584
+ "single_word": false,
2585
+ "special": true
2586
+ },
2587
+ "323": {
2588
+ "content": "<0x43>",
2589
+ "lstrip": false,
2590
+ "normalized": false,
2591
+ "rstrip": false,
2592
+ "single_word": false,
2593
+ "special": true
2594
+ },
2595
+ "324": {
2596
+ "content": "<0x44>",
2597
+ "lstrip": false,
2598
+ "normalized": false,
2599
+ "rstrip": false,
2600
+ "single_word": false,
2601
+ "special": true
2602
+ },
2603
+ "325": {
2604
+ "content": "<0x45>",
2605
+ "lstrip": false,
2606
+ "normalized": false,
2607
+ "rstrip": false,
2608
+ "single_word": false,
2609
+ "special": true
2610
+ },
2611
+ "326": {
2612
+ "content": "<0x46>",
2613
+ "lstrip": false,
2614
+ "normalized": false,
2615
+ "rstrip": false,
2616
+ "single_word": false,
2617
+ "special": true
2618
+ },
2619
+ "327": {
2620
+ "content": "<0x47>",
2621
+ "lstrip": false,
2622
+ "normalized": false,
2623
+ "rstrip": false,
2624
+ "single_word": false,
2625
+ "special": true
2626
+ },
2627
+ "328": {
2628
+ "content": "<0x48>",
2629
+ "lstrip": false,
2630
+ "normalized": false,
2631
+ "rstrip": false,
2632
+ "single_word": false,
2633
+ "special": true
2634
+ },
2635
+ "329": {
2636
+ "content": "<0x49>",
2637
+ "lstrip": false,
2638
+ "normalized": false,
2639
+ "rstrip": false,
2640
+ "single_word": false,
2641
+ "special": true
2642
+ },
2643
+ "330": {
2644
+ "content": "<0x4A>",
2645
+ "lstrip": false,
2646
+ "normalized": false,
2647
+ "rstrip": false,
2648
+ "single_word": false,
2649
+ "special": true
2650
+ },
2651
+ "331": {
2652
+ "content": "<0x4B>",
2653
+ "lstrip": false,
2654
+ "normalized": false,
2655
+ "rstrip": false,
2656
+ "single_word": false,
2657
+ "special": true
2658
+ },
2659
+ "332": {
2660
+ "content": "<0x4C>",
2661
+ "lstrip": false,
2662
+ "normalized": false,
2663
+ "rstrip": false,
2664
+ "single_word": false,
2665
+ "special": true
2666
+ },
2667
+ "333": {
2668
+ "content": "<0x4D>",
2669
+ "lstrip": false,
2670
+ "normalized": false,
2671
+ "rstrip": false,
2672
+ "single_word": false,
2673
+ "special": true
2674
+ },
2675
+ "334": {
2676
+ "content": "<0x4E>",
2677
+ "lstrip": false,
2678
+ "normalized": false,
2679
+ "rstrip": false,
2680
+ "single_word": false,
2681
+ "special": true
2682
+ },
2683
+ "335": {
2684
+ "content": "<0x4F>",
2685
+ "lstrip": false,
2686
+ "normalized": false,
2687
+ "rstrip": false,
2688
+ "single_word": false,
2689
+ "special": true
2690
+ },
2691
+ "336": {
2692
+ "content": "<0x50>",
2693
+ "lstrip": false,
2694
+ "normalized": false,
2695
+ "rstrip": false,
2696
+ "single_word": false,
2697
+ "special": true
2698
+ },
2699
+ "337": {
2700
+ "content": "<0x51>",
2701
+ "lstrip": false,
2702
+ "normalized": false,
2703
+ "rstrip": false,
2704
+ "single_word": false,
2705
+ "special": true
2706
+ },
2707
+ "338": {
2708
+ "content": "<0x52>",
2709
+ "lstrip": false,
2710
+ "normalized": false,
2711
+ "rstrip": false,
2712
+ "single_word": false,
2713
+ "special": true
2714
+ },
2715
+ "339": {
2716
+ "content": "<0x53>",
2717
+ "lstrip": false,
2718
+ "normalized": false,
2719
+ "rstrip": false,
2720
+ "single_word": false,
2721
+ "special": true
2722
+ },
2723
+ "340": {
2724
+ "content": "<0x54>",
2725
+ "lstrip": false,
2726
+ "normalized": false,
2727
+ "rstrip": false,
2728
+ "single_word": false,
2729
+ "special": true
2730
+ },
2731
+ "341": {
2732
+ "content": "<0x55>",
2733
+ "lstrip": false,
2734
+ "normalized": false,
2735
+ "rstrip": false,
2736
+ "single_word": false,
2737
+ "special": true
2738
+ },
2739
+ "342": {
2740
+ "content": "<0x56>",
2741
+ "lstrip": false,
2742
+ "normalized": false,
2743
+ "rstrip": false,
2744
+ "single_word": false,
2745
+ "special": true
2746
+ },
2747
+ "343": {
2748
+ "content": "<0x57>",
2749
+ "lstrip": false,
2750
+ "normalized": false,
2751
+ "rstrip": false,
2752
+ "single_word": false,
2753
+ "special": true
2754
+ },
2755
+ "344": {
2756
+ "content": "<0x58>",
2757
+ "lstrip": false,
2758
+ "normalized": false,
2759
+ "rstrip": false,
2760
+ "single_word": false,
2761
+ "special": true
2762
+ },
2763
+ "345": {
2764
+ "content": "<0x59>",
2765
+ "lstrip": false,
2766
+ "normalized": false,
2767
+ "rstrip": false,
2768
+ "single_word": false,
2769
+ "special": true
2770
+ },
2771
+ "346": {
2772
+ "content": "<0x5A>",
2773
+ "lstrip": false,
2774
+ "normalized": false,
2775
+ "rstrip": false,
2776
+ "single_word": false,
2777
+ "special": true
2778
+ },
2779
+ "347": {
2780
+ "content": "<0x5B>",
2781
+ "lstrip": false,
2782
+ "normalized": false,
2783
+ "rstrip": false,
2784
+ "single_word": false,
2785
+ "special": true
2786
+ },
2787
+ "348": {
2788
+ "content": "<0x5C>",
2789
+ "lstrip": false,
2790
+ "normalized": false,
2791
+ "rstrip": false,
2792
+ "single_word": false,
2793
+ "special": true
2794
+ },
2795
+ "349": {
2796
+ "content": "<0x5D>",
2797
+ "lstrip": false,
2798
+ "normalized": false,
2799
+ "rstrip": false,
2800
+ "single_word": false,
2801
+ "special": true
2802
+ },
2803
+ "350": {
2804
+ "content": "<0x5E>",
2805
+ "lstrip": false,
2806
+ "normalized": false,
2807
+ "rstrip": false,
2808
+ "single_word": false,
2809
+ "special": true
2810
+ },
2811
+ "351": {
2812
+ "content": "<0x5F>",
2813
+ "lstrip": false,
2814
+ "normalized": false,
2815
+ "rstrip": false,
2816
+ "single_word": false,
2817
+ "special": true
2818
+ },
2819
+ "352": {
2820
+ "content": "<0x60>",
2821
+ "lstrip": false,
2822
+ "normalized": false,
2823
+ "rstrip": false,
2824
+ "single_word": false,
2825
+ "special": true
2826
+ },
2827
+ "353": {
2828
+ "content": "<0x61>",
2829
+ "lstrip": false,
2830
+ "normalized": false,
2831
+ "rstrip": false,
2832
+ "single_word": false,
2833
+ "special": true
2834
+ },
2835
+ "354": {
2836
+ "content": "<0x62>",
2837
+ "lstrip": false,
2838
+ "normalized": false,
2839
+ "rstrip": false,
2840
+ "single_word": false,
2841
+ "special": true
2842
+ },
2843
+ "355": {
2844
+ "content": "<0x63>",
2845
+ "lstrip": false,
2846
+ "normalized": false,
2847
+ "rstrip": false,
2848
+ "single_word": false,
2849
+ "special": true
2850
+ },
2851
+ "356": {
2852
+ "content": "<0x64>",
2853
+ "lstrip": false,
2854
+ "normalized": false,
2855
+ "rstrip": false,
2856
+ "single_word": false,
2857
+ "special": true
2858
+ },
2859
+ "357": {
2860
+ "content": "<0x65>",
2861
+ "lstrip": false,
2862
+ "normalized": false,
2863
+ "rstrip": false,
2864
+ "single_word": false,
2865
+ "special": true
2866
+ },
2867
+ "358": {
2868
+ "content": "<0x66>",
2869
+ "lstrip": false,
2870
+ "normalized": false,
2871
+ "rstrip": false,
2872
+ "single_word": false,
2873
+ "special": true
2874
+ },
2875
+ "359": {
2876
+ "content": "<0x67>",
2877
+ "lstrip": false,
2878
+ "normalized": false,
2879
+ "rstrip": false,
2880
+ "single_word": false,
2881
+ "special": true
2882
+ },
2883
+ "360": {
2884
+ "content": "<0x68>",
2885
+ "lstrip": false,
2886
+ "normalized": false,
2887
+ "rstrip": false,
2888
+ "single_word": false,
2889
+ "special": true
2890
+ },
2891
+ "361": {
2892
+ "content": "<0x69>",
2893
+ "lstrip": false,
2894
+ "normalized": false,
2895
+ "rstrip": false,
2896
+ "single_word": false,
2897
+ "special": true
2898
+ },
2899
+ "362": {
2900
+ "content": "<0x6A>",
2901
+ "lstrip": false,
2902
+ "normalized": false,
2903
+ "rstrip": false,
2904
+ "single_word": false,
2905
+ "special": true
2906
+ },
2907
+ "363": {
2908
+ "content": "<0x6B>",
2909
+ "lstrip": false,
2910
+ "normalized": false,
2911
+ "rstrip": false,
2912
+ "single_word": false,
2913
+ "special": true
2914
+ },
2915
+ "364": {
2916
+ "content": "<0x6C>",
2917
+ "lstrip": false,
2918
+ "normalized": false,
2919
+ "rstrip": false,
2920
+ "single_word": false,
2921
+ "special": true
2922
+ },
2923
+ "365": {
2924
+ "content": "<0x6D>",
2925
+ "lstrip": false,
2926
+ "normalized": false,
2927
+ "rstrip": false,
2928
+ "single_word": false,
2929
+ "special": true
2930
+ },
2931
+ "366": {
2932
+ "content": "<0x6E>",
2933
+ "lstrip": false,
2934
+ "normalized": false,
2935
+ "rstrip": false,
2936
+ "single_word": false,
2937
+ "special": true
2938
+ },
2939
+ "367": {
2940
+ "content": "<0x6F>",
2941
+ "lstrip": false,
2942
+ "normalized": false,
2943
+ "rstrip": false,
2944
+ "single_word": false,
2945
+ "special": true
2946
+ },
2947
+ "368": {
2948
+ "content": "<0x70>",
2949
+ "lstrip": false,
2950
+ "normalized": false,
2951
+ "rstrip": false,
2952
+ "single_word": false,
2953
+ "special": true
2954
+ },
2955
+ "369": {
2956
+ "content": "<0x71>",
2957
+ "lstrip": false,
2958
+ "normalized": false,
2959
+ "rstrip": false,
2960
+ "single_word": false,
2961
+ "special": true
2962
+ },
2963
+ "370": {
2964
+ "content": "<0x72>",
2965
+ "lstrip": false,
2966
+ "normalized": false,
2967
+ "rstrip": false,
2968
+ "single_word": false,
2969
+ "special": true
2970
+ },
2971
+ "371": {
2972
+ "content": "<0x73>",
2973
+ "lstrip": false,
2974
+ "normalized": false,
2975
+ "rstrip": false,
2976
+ "single_word": false,
2977
+ "special": true
2978
+ },
2979
+ "372": {
2980
+ "content": "<0x74>",
2981
+ "lstrip": false,
2982
+ "normalized": false,
2983
+ "rstrip": false,
2984
+ "single_word": false,
2985
+ "special": true
2986
+ },
2987
+ "373": {
2988
+ "content": "<0x75>",
2989
+ "lstrip": false,
2990
+ "normalized": false,
2991
+ "rstrip": false,
2992
+ "single_word": false,
2993
+ "special": true
2994
+ },
2995
+ "374": {
2996
+ "content": "<0x76>",
2997
+ "lstrip": false,
2998
+ "normalized": false,
2999
+ "rstrip": false,
3000
+ "single_word": false,
3001
+ "special": true
3002
+ },
3003
+ "375": {
3004
+ "content": "<0x77>",
3005
+ "lstrip": false,
3006
+ "normalized": false,
3007
+ "rstrip": false,
3008
+ "single_word": false,
3009
+ "special": true
3010
+ },
3011
+ "376": {
3012
+ "content": "<0x78>",
3013
+ "lstrip": false,
3014
+ "normalized": false,
3015
+ "rstrip": false,
3016
+ "single_word": false,
3017
+ "special": true
3018
+ },
3019
+ "377": {
3020
+ "content": "<0x79>",
3021
+ "lstrip": false,
3022
+ "normalized": false,
3023
+ "rstrip": false,
3024
+ "single_word": false,
3025
+ "special": true
3026
+ },
3027
+ "378": {
3028
+ "content": "<0x7A>",
3029
+ "lstrip": false,
3030
+ "normalized": false,
3031
+ "rstrip": false,
3032
+ "single_word": false,
3033
+ "special": true
3034
+ },
3035
+ "379": {
3036
+ "content": "<0x7B>",
3037
+ "lstrip": false,
3038
+ "normalized": false,
3039
+ "rstrip": false,
3040
+ "single_word": false,
3041
+ "special": true
3042
+ },
3043
+ "380": {
3044
+ "content": "<0x7C>",
3045
+ "lstrip": false,
3046
+ "normalized": false,
3047
+ "rstrip": false,
3048
+ "single_word": false,
3049
+ "special": true
3050
+ },
3051
+ "381": {
3052
+ "content": "<0x7D>",
3053
+ "lstrip": false,
3054
+ "normalized": false,
3055
+ "rstrip": false,
3056
+ "single_word": false,
3057
+ "special": true
3058
+ },
3059
+ "382": {
3060
+ "content": "<0x7E>",
3061
+ "lstrip": false,
3062
+ "normalized": false,
3063
+ "rstrip": false,
3064
+ "single_word": false,
3065
+ "special": true
3066
+ },
3067
+ "383": {
3068
+ "content": "<0x7F>",
3069
+ "lstrip": false,
3070
+ "normalized": false,
3071
+ "rstrip": false,
3072
+ "single_word": false,
3073
+ "special": true
3074
+ },
3075
+ "384": {
3076
+ "content": "<0x80>",
3077
+ "lstrip": false,
3078
+ "normalized": false,
3079
+ "rstrip": false,
3080
+ "single_word": false,
3081
+ "special": true
3082
+ },
3083
+ "385": {
3084
+ "content": "<0x81>",
3085
+ "lstrip": false,
3086
+ "normalized": false,
3087
+ "rstrip": false,
3088
+ "single_word": false,
3089
+ "special": true
3090
+ },
3091
+ "386": {
3092
+ "content": "<0x82>",
3093
+ "lstrip": false,
3094
+ "normalized": false,
3095
+ "rstrip": false,
3096
+ "single_word": false,
3097
+ "special": true
3098
+ },
3099
+ "387": {
3100
+ "content": "<0x83>",
3101
+ "lstrip": false,
3102
+ "normalized": false,
3103
+ "rstrip": false,
3104
+ "single_word": false,
3105
+ "special": true
3106
+ },
3107
+ "388": {
3108
+ "content": "<0x84>",
3109
+ "lstrip": false,
3110
+ "normalized": false,
3111
+ "rstrip": false,
3112
+ "single_word": false,
3113
+ "special": true
3114
+ },
3115
+ "389": {
3116
+ "content": "<0x85>",
3117
+ "lstrip": false,
3118
+ "normalized": false,
3119
+ "rstrip": false,
3120
+ "single_word": false,
3121
+ "special": true
3122
+ },
3123
+ "390": {
3124
+ "content": "<0x86>",
3125
+ "lstrip": false,
3126
+ "normalized": false,
3127
+ "rstrip": false,
3128
+ "single_word": false,
3129
+ "special": true
3130
+ },
3131
+ "391": {
3132
+ "content": "<0x87>",
3133
+ "lstrip": false,
3134
+ "normalized": false,
3135
+ "rstrip": false,
3136
+ "single_word": false,
3137
+ "special": true
3138
+ },
3139
+ "392": {
3140
+ "content": "<0x88>",
3141
+ "lstrip": false,
3142
+ "normalized": false,
3143
+ "rstrip": false,
3144
+ "single_word": false,
3145
+ "special": true
3146
+ },
3147
+ "393": {
3148
+ "content": "<0x89>",
3149
+ "lstrip": false,
3150
+ "normalized": false,
3151
+ "rstrip": false,
3152
+ "single_word": false,
3153
+ "special": true
3154
+ },
3155
+ "394": {
3156
+ "content": "<0x8A>",
3157
+ "lstrip": false,
3158
+ "normalized": false,
3159
+ "rstrip": false,
3160
+ "single_word": false,
3161
+ "special": true
3162
+ },
3163
+ "395": {
3164
+ "content": "<0x8B>",
3165
+ "lstrip": false,
3166
+ "normalized": false,
3167
+ "rstrip": false,
3168
+ "single_word": false,
3169
+ "special": true
3170
+ },
3171
+ "396": {
3172
+ "content": "<0x8C>",
3173
+ "lstrip": false,
3174
+ "normalized": false,
3175
+ "rstrip": false,
3176
+ "single_word": false,
3177
+ "special": true
3178
+ },
3179
+ "397": {
3180
+ "content": "<0x8D>",
3181
+ "lstrip": false,
3182
+ "normalized": false,
3183
+ "rstrip": false,
3184
+ "single_word": false,
3185
+ "special": true
3186
+ },
3187
+ "398": {
3188
+ "content": "<0x8E>",
3189
+ "lstrip": false,
3190
+ "normalized": false,
3191
+ "rstrip": false,
3192
+ "single_word": false,
3193
+ "special": true
3194
+ },
3195
+ "399": {
3196
+ "content": "<0x8F>",
3197
+ "lstrip": false,
3198
+ "normalized": false,
3199
+ "rstrip": false,
3200
+ "single_word": false,
3201
+ "special": true
3202
+ },
3203
+ "400": {
3204
+ "content": "<0x90>",
3205
+ "lstrip": false,
3206
+ "normalized": false,
3207
+ "rstrip": false,
3208
+ "single_word": false,
3209
+ "special": true
3210
+ },
3211
+ "401": {
3212
+ "content": "<0x91>",
3213
+ "lstrip": false,
3214
+ "normalized": false,
3215
+ "rstrip": false,
3216
+ "single_word": false,
3217
+ "special": true
3218
+ },
3219
+ "402": {
3220
+ "content": "<0x92>",
3221
+ "lstrip": false,
3222
+ "normalized": false,
3223
+ "rstrip": false,
3224
+ "single_word": false,
3225
+ "special": true
3226
+ },
3227
+ "403": {
3228
+ "content": "<0x93>",
3229
+ "lstrip": false,
3230
+ "normalized": false,
3231
+ "rstrip": false,
3232
+ "single_word": false,
3233
+ "special": true
3234
+ },
3235
+ "404": {
3236
+ "content": "<0x94>",
3237
+ "lstrip": false,
3238
+ "normalized": false,
3239
+ "rstrip": false,
3240
+ "single_word": false,
3241
+ "special": true
3242
+ },
3243
+ "405": {
3244
+ "content": "<0x95>",
3245
+ "lstrip": false,
3246
+ "normalized": false,
3247
+ "rstrip": false,
3248
+ "single_word": false,
3249
+ "special": true
3250
+ },
3251
+ "406": {
3252
+ "content": "<0x96>",
3253
+ "lstrip": false,
3254
+ "normalized": false,
3255
+ "rstrip": false,
3256
+ "single_word": false,
3257
+ "special": true
3258
+ },
3259
+ "407": {
3260
+ "content": "<0x97>",
3261
+ "lstrip": false,
3262
+ "normalized": false,
3263
+ "rstrip": false,
3264
+ "single_word": false,
3265
+ "special": true
3266
+ },
3267
+ "408": {
3268
+ "content": "<0x98>",
3269
+ "lstrip": false,
3270
+ "normalized": false,
3271
+ "rstrip": false,
3272
+ "single_word": false,
3273
+ "special": true
3274
+ },
3275
+ "409": {
3276
+ "content": "<0x99>",
3277
+ "lstrip": false,
3278
+ "normalized": false,
3279
+ "rstrip": false,
3280
+ "single_word": false,
3281
+ "special": true
3282
+ },
3283
+ "410": {
3284
+ "content": "<0x9A>",
3285
+ "lstrip": false,
3286
+ "normalized": false,
3287
+ "rstrip": false,
3288
+ "single_word": false,
3289
+ "special": true
3290
+ },
3291
+ "411": {
3292
+ "content": "<0x9B>",
3293
+ "lstrip": false,
3294
+ "normalized": false,
3295
+ "rstrip": false,
3296
+ "single_word": false,
3297
+ "special": true
3298
+ },
3299
+ "412": {
3300
+ "content": "<0x9C>",
3301
+ "lstrip": false,
3302
+ "normalized": false,
3303
+ "rstrip": false,
3304
+ "single_word": false,
3305
+ "special": true
3306
+ },
3307
+ "413": {
3308
+ "content": "<0x9D>",
3309
+ "lstrip": false,
3310
+ "normalized": false,
3311
+ "rstrip": false,
3312
+ "single_word": false,
3313
+ "special": true
3314
+ },
3315
+ "414": {
3316
+ "content": "<0x9E>",
3317
+ "lstrip": false,
3318
+ "normalized": false,
3319
+ "rstrip": false,
3320
+ "single_word": false,
3321
+ "special": true
3322
+ },
3323
+ "415": {
3324
+ "content": "<0x9F>",
3325
+ "lstrip": false,
3326
+ "normalized": false,
3327
+ "rstrip": false,
3328
+ "single_word": false,
3329
+ "special": true
3330
+ },
3331
+ "416": {
3332
+ "content": "<0xA0>",
3333
+ "lstrip": false,
3334
+ "normalized": false,
3335
+ "rstrip": false,
3336
+ "single_word": false,
3337
+ "special": true
3338
+ },
3339
+ "417": {
3340
+ "content": "<0xA1>",
3341
+ "lstrip": false,
3342
+ "normalized": false,
3343
+ "rstrip": false,
3344
+ "single_word": false,
3345
+ "special": true
3346
+ },
3347
+ "418": {
3348
+ "content": "<0xA2>",
3349
+ "lstrip": false,
3350
+ "normalized": false,
3351
+ "rstrip": false,
3352
+ "single_word": false,
3353
+ "special": true
3354
+ },
3355
+ "419": {
3356
+ "content": "<0xA3>",
3357
+ "lstrip": false,
3358
+ "normalized": false,
3359
+ "rstrip": false,
3360
+ "single_word": false,
3361
+ "special": true
3362
+ },
3363
+ "420": {
3364
+ "content": "<0xA4>",
3365
+ "lstrip": false,
3366
+ "normalized": false,
3367
+ "rstrip": false,
3368
+ "single_word": false,
3369
+ "special": true
3370
+ },
3371
+ "421": {
3372
+ "content": "<0xA5>",
3373
+ "lstrip": false,
3374
+ "normalized": false,
3375
+ "rstrip": false,
3376
+ "single_word": false,
3377
+ "special": true
3378
+ },
3379
+ "422": {
3380
+ "content": "<0xA6>",
3381
+ "lstrip": false,
3382
+ "normalized": false,
3383
+ "rstrip": false,
3384
+ "single_word": false,
3385
+ "special": true
3386
+ },
3387
+ "423": {
3388
+ "content": "<0xA7>",
3389
+ "lstrip": false,
3390
+ "normalized": false,
3391
+ "rstrip": false,
3392
+ "single_word": false,
3393
+ "special": true
3394
+ },
3395
+ "424": {
3396
+ "content": "<0xA8>",
3397
+ "lstrip": false,
3398
+ "normalized": false,
3399
+ "rstrip": false,
3400
+ "single_word": false,
3401
+ "special": true
3402
+ },
3403
+ "425": {
3404
+ "content": "<0xA9>",
3405
+ "lstrip": false,
3406
+ "normalized": false,
3407
+ "rstrip": false,
3408
+ "single_word": false,
3409
+ "special": true
3410
+ },
3411
+ "426": {
3412
+ "content": "<0xAA>",
3413
+ "lstrip": false,
3414
+ "normalized": false,
3415
+ "rstrip": false,
3416
+ "single_word": false,
3417
+ "special": true
3418
+ },
3419
+ "427": {
3420
+ "content": "<0xAB>",
3421
+ "lstrip": false,
3422
+ "normalized": false,
3423
+ "rstrip": false,
3424
+ "single_word": false,
3425
+ "special": true
3426
+ },
3427
+ "428": {
3428
+ "content": "<0xAC>",
3429
+ "lstrip": false,
3430
+ "normalized": false,
3431
+ "rstrip": false,
3432
+ "single_word": false,
3433
+ "special": true
3434
+ },
3435
+ "429": {
3436
+ "content": "<0xAD>",
3437
+ "lstrip": false,
3438
+ "normalized": false,
3439
+ "rstrip": false,
3440
+ "single_word": false,
3441
+ "special": true
3442
+ },
3443
+ "430": {
3444
+ "content": "<0xAE>",
3445
+ "lstrip": false,
3446
+ "normalized": false,
3447
+ "rstrip": false,
3448
+ "single_word": false,
3449
+ "special": true
3450
+ },
3451
+ "431": {
3452
+ "content": "<0xAF>",
3453
+ "lstrip": false,
3454
+ "normalized": false,
3455
+ "rstrip": false,
3456
+ "single_word": false,
3457
+ "special": true
3458
+ },
3459
+ "432": {
3460
+ "content": "<0xB0>",
3461
+ "lstrip": false,
3462
+ "normalized": false,
3463
+ "rstrip": false,
3464
+ "single_word": false,
3465
+ "special": true
3466
+ },
3467
+ "433": {
3468
+ "content": "<0xB1>",
3469
+ "lstrip": false,
3470
+ "normalized": false,
3471
+ "rstrip": false,
3472
+ "single_word": false,
3473
+ "special": true
3474
+ },
3475
+ "434": {
3476
+ "content": "<0xB2>",
3477
+ "lstrip": false,
3478
+ "normalized": false,
3479
+ "rstrip": false,
3480
+ "single_word": false,
3481
+ "special": true
3482
+ },
3483
+ "435": {
3484
+ "content": "<0xB3>",
3485
+ "lstrip": false,
3486
+ "normalized": false,
3487
+ "rstrip": false,
3488
+ "single_word": false,
3489
+ "special": true
3490
+ },
3491
+ "436": {
3492
+ "content": "<0xB4>",
3493
+ "lstrip": false,
3494
+ "normalized": false,
3495
+ "rstrip": false,
3496
+ "single_word": false,
3497
+ "special": true
3498
+ },
3499
+ "437": {
3500
+ "content": "<0xB5>",
3501
+ "lstrip": false,
3502
+ "normalized": false,
3503
+ "rstrip": false,
3504
+ "single_word": false,
3505
+ "special": true
3506
+ },
3507
+ "438": {
3508
+ "content": "<0xB6>",
3509
+ "lstrip": false,
3510
+ "normalized": false,
3511
+ "rstrip": false,
3512
+ "single_word": false,
3513
+ "special": true
3514
+ },
3515
+ "439": {
3516
+ "content": "<0xB7>",
3517
+ "lstrip": false,
3518
+ "normalized": false,
3519
+ "rstrip": false,
3520
+ "single_word": false,
3521
+ "special": true
3522
+ },
3523
+ "440": {
3524
+ "content": "<0xB8>",
3525
+ "lstrip": false,
3526
+ "normalized": false,
3527
+ "rstrip": false,
3528
+ "single_word": false,
3529
+ "special": true
3530
+ },
3531
+ "441": {
3532
+ "content": "<0xB9>",
3533
+ "lstrip": false,
3534
+ "normalized": false,
3535
+ "rstrip": false,
3536
+ "single_word": false,
3537
+ "special": true
3538
+ },
3539
+ "442": {
3540
+ "content": "<0xBA>",
3541
+ "lstrip": false,
3542
+ "normalized": false,
3543
+ "rstrip": false,
3544
+ "single_word": false,
3545
+ "special": true
3546
+ },
3547
+ "443": {
3548
+ "content": "<0xBB>",
3549
+ "lstrip": false,
3550
+ "normalized": false,
3551
+ "rstrip": false,
3552
+ "single_word": false,
3553
+ "special": true
3554
+ },
3555
+ "444": {
3556
+ "content": "<0xBC>",
3557
+ "lstrip": false,
3558
+ "normalized": false,
3559
+ "rstrip": false,
3560
+ "single_word": false,
3561
+ "special": true
3562
+ },
3563
+ "445": {
3564
+ "content": "<0xBD>",
3565
+ "lstrip": false,
3566
+ "normalized": false,
3567
+ "rstrip": false,
3568
+ "single_word": false,
3569
+ "special": true
3570
+ },
3571
+ "446": {
3572
+ "content": "<0xBE>",
3573
+ "lstrip": false,
3574
+ "normalized": false,
3575
+ "rstrip": false,
3576
+ "single_word": false,
3577
+ "special": true
3578
+ },
3579
+ "447": {
3580
+ "content": "<0xBF>",
3581
+ "lstrip": false,
3582
+ "normalized": false,
3583
+ "rstrip": false,
3584
+ "single_word": false,
3585
+ "special": true
3586
+ },
3587
+ "448": {
3588
+ "content": "<0xC0>",
3589
+ "lstrip": false,
3590
+ "normalized": false,
3591
+ "rstrip": false,
3592
+ "single_word": false,
3593
+ "special": true
3594
+ },
3595
+ "449": {
3596
+ "content": "<0xC1>",
3597
+ "lstrip": false,
3598
+ "normalized": false,
3599
+ "rstrip": false,
3600
+ "single_word": false,
3601
+ "special": true
3602
+ },
3603
+ "450": {
3604
+ "content": "<0xC2>",
3605
+ "lstrip": false,
3606
+ "normalized": false,
3607
+ "rstrip": false,
3608
+ "single_word": false,
3609
+ "special": true
3610
+ },
3611
+ "451": {
3612
+ "content": "<0xC3>",
3613
+ "lstrip": false,
3614
+ "normalized": false,
3615
+ "rstrip": false,
3616
+ "single_word": false,
3617
+ "special": true
3618
+ },
3619
+ "452": {
3620
+ "content": "<0xC4>",
3621
+ "lstrip": false,
3622
+ "normalized": false,
3623
+ "rstrip": false,
3624
+ "single_word": false,
3625
+ "special": true
3626
+ },
3627
+ "453": {
3628
+ "content": "<0xC5>",
3629
+ "lstrip": false,
3630
+ "normalized": false,
3631
+ "rstrip": false,
3632
+ "single_word": false,
3633
+ "special": true
3634
+ },
3635
+ "454": {
3636
+ "content": "<0xC6>",
3637
+ "lstrip": false,
3638
+ "normalized": false,
3639
+ "rstrip": false,
3640
+ "single_word": false,
3641
+ "special": true
3642
+ },
3643
+ "455": {
3644
+ "content": "<0xC7>",
3645
+ "lstrip": false,
3646
+ "normalized": false,
3647
+ "rstrip": false,
3648
+ "single_word": false,
3649
+ "special": true
3650
+ },
3651
+ "456": {
3652
+ "content": "<0xC8>",
3653
+ "lstrip": false,
3654
+ "normalized": false,
3655
+ "rstrip": false,
3656
+ "single_word": false,
3657
+ "special": true
3658
+ },
3659
+ "457": {
3660
+ "content": "<0xC9>",
3661
+ "lstrip": false,
3662
+ "normalized": false,
3663
+ "rstrip": false,
3664
+ "single_word": false,
3665
+ "special": true
3666
+ },
3667
+ "458": {
3668
+ "content": "<0xCA>",
3669
+ "lstrip": false,
3670
+ "normalized": false,
3671
+ "rstrip": false,
3672
+ "single_word": false,
3673
+ "special": true
3674
+ },
3675
+ "459": {
3676
+ "content": "<0xCB>",
3677
+ "lstrip": false,
3678
+ "normalized": false,
3679
+ "rstrip": false,
3680
+ "single_word": false,
3681
+ "special": true
3682
+ },
3683
+ "460": {
3684
+ "content": "<0xCC>",
3685
+ "lstrip": false,
3686
+ "normalized": false,
3687
+ "rstrip": false,
3688
+ "single_word": false,
3689
+ "special": true
3690
+ },
3691
+ "461": {
3692
+ "content": "<0xCD>",
3693
+ "lstrip": false,
3694
+ "normalized": false,
3695
+ "rstrip": false,
3696
+ "single_word": false,
3697
+ "special": true
3698
+ },
3699
+ "462": {
3700
+ "content": "<0xCE>",
3701
+ "lstrip": false,
3702
+ "normalized": false,
3703
+ "rstrip": false,
3704
+ "single_word": false,
3705
+ "special": true
3706
+ },
3707
+ "463": {
3708
+ "content": "<0xCF>",
3709
+ "lstrip": false,
3710
+ "normalized": false,
3711
+ "rstrip": false,
3712
+ "single_word": false,
3713
+ "special": true
3714
+ },
3715
+ "464": {
3716
+ "content": "<0xD0>",
3717
+ "lstrip": false,
3718
+ "normalized": false,
3719
+ "rstrip": false,
3720
+ "single_word": false,
3721
+ "special": true
3722
+ },
3723
+ "465": {
3724
+ "content": "<0xD1>",
3725
+ "lstrip": false,
3726
+ "normalized": false,
3727
+ "rstrip": false,
3728
+ "single_word": false,
3729
+ "special": true
3730
+ },
3731
+ "466": {
3732
+ "content": "<0xD2>",
3733
+ "lstrip": false,
3734
+ "normalized": false,
3735
+ "rstrip": false,
3736
+ "single_word": false,
3737
+ "special": true
3738
+ },
3739
+ "467": {
3740
+ "content": "<0xD3>",
3741
+ "lstrip": false,
3742
+ "normalized": false,
3743
+ "rstrip": false,
3744
+ "single_word": false,
3745
+ "special": true
3746
+ },
3747
+ "468": {
3748
+ "content": "<0xD4>",
3749
+ "lstrip": false,
3750
+ "normalized": false,
3751
+ "rstrip": false,
3752
+ "single_word": false,
3753
+ "special": true
3754
+ },
3755
+ "469": {
3756
+ "content": "<0xD5>",
3757
+ "lstrip": false,
3758
+ "normalized": false,
3759
+ "rstrip": false,
3760
+ "single_word": false,
3761
+ "special": true
3762
+ },
3763
+ "470": {
3764
+ "content": "<0xD6>",
3765
+ "lstrip": false,
3766
+ "normalized": false,
3767
+ "rstrip": false,
3768
+ "single_word": false,
3769
+ "special": true
3770
+ },
3771
+ "471": {
3772
+ "content": "<0xD7>",
3773
+ "lstrip": false,
3774
+ "normalized": false,
3775
+ "rstrip": false,
3776
+ "single_word": false,
3777
+ "special": true
3778
+ },
3779
+ "472": {
3780
+ "content": "<0xD8>",
3781
+ "lstrip": false,
3782
+ "normalized": false,
3783
+ "rstrip": false,
3784
+ "single_word": false,
3785
+ "special": true
3786
+ },
3787
+ "473": {
3788
+ "content": "<0xD9>",
3789
+ "lstrip": false,
3790
+ "normalized": false,
3791
+ "rstrip": false,
3792
+ "single_word": false,
3793
+ "special": true
3794
+ },
3795
+ "474": {
3796
+ "content": "<0xDA>",
3797
+ "lstrip": false,
3798
+ "normalized": false,
3799
+ "rstrip": false,
3800
+ "single_word": false,
3801
+ "special": true
3802
+ },
3803
+ "475": {
3804
+ "content": "<0xDB>",
3805
+ "lstrip": false,
3806
+ "normalized": false,
3807
+ "rstrip": false,
3808
+ "single_word": false,
3809
+ "special": true
3810
+ },
3811
+ "476": {
3812
+ "content": "<0xDC>",
3813
+ "lstrip": false,
3814
+ "normalized": false,
3815
+ "rstrip": false,
3816
+ "single_word": false,
3817
+ "special": true
3818
+ },
3819
+ "477": {
3820
+ "content": "<0xDD>",
3821
+ "lstrip": false,
3822
+ "normalized": false,
3823
+ "rstrip": false,
3824
+ "single_word": false,
3825
+ "special": true
3826
+ },
3827
+ "478": {
3828
+ "content": "<0xDE>",
3829
+ "lstrip": false,
3830
+ "normalized": false,
3831
+ "rstrip": false,
3832
+ "single_word": false,
3833
+ "special": true
3834
+ },
3835
+ "479": {
3836
+ "content": "<0xDF>",
3837
+ "lstrip": false,
3838
+ "normalized": false,
3839
+ "rstrip": false,
3840
+ "single_word": false,
3841
+ "special": true
3842
+ },
3843
+ "480": {
3844
+ "content": "<0xE0>",
3845
+ "lstrip": false,
3846
+ "normalized": false,
3847
+ "rstrip": false,
3848
+ "single_word": false,
3849
+ "special": true
3850
+ },
3851
+ "481": {
3852
+ "content": "<0xE1>",
3853
+ "lstrip": false,
3854
+ "normalized": false,
3855
+ "rstrip": false,
3856
+ "single_word": false,
3857
+ "special": true
3858
+ },
3859
+ "482": {
3860
+ "content": "<0xE2>",
3861
+ "lstrip": false,
3862
+ "normalized": false,
3863
+ "rstrip": false,
3864
+ "single_word": false,
3865
+ "special": true
3866
+ },
3867
+ "483": {
3868
+ "content": "<0xE3>",
3869
+ "lstrip": false,
3870
+ "normalized": false,
3871
+ "rstrip": false,
3872
+ "single_word": false,
3873
+ "special": true
3874
+ },
3875
+ "484": {
3876
+ "content": "<0xE4>",
3877
+ "lstrip": false,
3878
+ "normalized": false,
3879
+ "rstrip": false,
3880
+ "single_word": false,
3881
+ "special": true
3882
+ },
3883
+ "485": {
3884
+ "content": "<0xE5>",
3885
+ "lstrip": false,
3886
+ "normalized": false,
3887
+ "rstrip": false,
3888
+ "single_word": false,
3889
+ "special": true
3890
+ },
3891
+ "486": {
3892
+ "content": "<0xE6>",
3893
+ "lstrip": false,
3894
+ "normalized": false,
3895
+ "rstrip": false,
3896
+ "single_word": false,
3897
+ "special": true
3898
+ },
3899
+ "487": {
3900
+ "content": "<0xE7>",
3901
+ "lstrip": false,
3902
+ "normalized": false,
3903
+ "rstrip": false,
3904
+ "single_word": false,
3905
+ "special": true
3906
+ },
3907
+ "488": {
3908
+ "content": "<0xE8>",
3909
+ "lstrip": false,
3910
+ "normalized": false,
3911
+ "rstrip": false,
3912
+ "single_word": false,
3913
+ "special": true
3914
+ },
3915
+ "489": {
3916
+ "content": "<0xE9>",
3917
+ "lstrip": false,
3918
+ "normalized": false,
3919
+ "rstrip": false,
3920
+ "single_word": false,
3921
+ "special": true
3922
+ },
3923
+ "490": {
3924
+ "content": "<0xEA>",
3925
+ "lstrip": false,
3926
+ "normalized": false,
3927
+ "rstrip": false,
3928
+ "single_word": false,
3929
+ "special": true
3930
+ },
3931
+ "491": {
3932
+ "content": "<0xEB>",
3933
+ "lstrip": false,
3934
+ "normalized": false,
3935
+ "rstrip": false,
3936
+ "single_word": false,
3937
+ "special": true
3938
+ },
3939
+ "492": {
3940
+ "content": "<0xEC>",
3941
+ "lstrip": false,
3942
+ "normalized": false,
3943
+ "rstrip": false,
3944
+ "single_word": false,
3945
+ "special": true
3946
+ },
3947
+ "493": {
3948
+ "content": "<0xED>",
3949
+ "lstrip": false,
3950
+ "normalized": false,
3951
+ "rstrip": false,
3952
+ "single_word": false,
3953
+ "special": true
3954
+ },
3955
+ "494": {
3956
+ "content": "<0xEE>",
3957
+ "lstrip": false,
3958
+ "normalized": false,
3959
+ "rstrip": false,
3960
+ "single_word": false,
3961
+ "special": true
3962
+ },
3963
+ "495": {
3964
+ "content": "<0xEF>",
3965
+ "lstrip": false,
3966
+ "normalized": false,
3967
+ "rstrip": false,
3968
+ "single_word": false,
3969
+ "special": true
3970
+ },
3971
+ "496": {
3972
+ "content": "<0xF0>",
3973
+ "lstrip": false,
3974
+ "normalized": false,
3975
+ "rstrip": false,
3976
+ "single_word": false,
3977
+ "special": true
3978
+ },
3979
+ "497": {
3980
+ "content": "<0xF1>",
3981
+ "lstrip": false,
3982
+ "normalized": false,
3983
+ "rstrip": false,
3984
+ "single_word": false,
3985
+ "special": true
3986
+ },
3987
+ "498": {
3988
+ "content": "<0xF2>",
3989
+ "lstrip": false,
3990
+ "normalized": false,
3991
+ "rstrip": false,
3992
+ "single_word": false,
3993
+ "special": true
3994
+ },
3995
+ "499": {
3996
+ "content": "<0xF3>",
3997
+ "lstrip": false,
3998
+ "normalized": false,
3999
+ "rstrip": false,
4000
+ "single_word": false,
4001
+ "special": true
4002
+ },
4003
+ "500": {
4004
+ "content": "<0xF4>",
4005
+ "lstrip": false,
4006
+ "normalized": false,
4007
+ "rstrip": false,
4008
+ "single_word": false,
4009
+ "special": true
4010
+ },
4011
+ "501": {
4012
+ "content": "<0xF5>",
4013
+ "lstrip": false,
4014
+ "normalized": false,
4015
+ "rstrip": false,
4016
+ "single_word": false,
4017
+ "special": true
4018
+ },
4019
+ "502": {
4020
+ "content": "<0xF6>",
4021
+ "lstrip": false,
4022
+ "normalized": false,
4023
+ "rstrip": false,
4024
+ "single_word": false,
4025
+ "special": true
4026
+ },
4027
+ "503": {
4028
+ "content": "<0xF7>",
4029
+ "lstrip": false,
4030
+ "normalized": false,
4031
+ "rstrip": false,
4032
+ "single_word": false,
4033
+ "special": true
4034
+ },
4035
+ "504": {
4036
+ "content": "<0xF8>",
4037
+ "lstrip": false,
4038
+ "normalized": false,
4039
+ "rstrip": false,
4040
+ "single_word": false,
4041
+ "special": true
4042
+ },
4043
+ "505": {
4044
+ "content": "<0xF9>",
4045
+ "lstrip": false,
4046
+ "normalized": false,
4047
+ "rstrip": false,
4048
+ "single_word": false,
4049
+ "special": true
4050
+ },
4051
+ "506": {
4052
+ "content": "<0xFA>",
4053
+ "lstrip": false,
4054
+ "normalized": false,
4055
+ "rstrip": false,
4056
+ "single_word": false,
4057
+ "special": true
4058
+ },
4059
+ "507": {
4060
+ "content": "<0xFB>",
4061
+ "lstrip": false,
4062
+ "normalized": false,
4063
+ "rstrip": false,
4064
+ "single_word": false,
4065
+ "special": true
4066
+ },
4067
+ "508": {
4068
+ "content": "<0xFC>",
4069
+ "lstrip": false,
4070
+ "normalized": false,
4071
+ "rstrip": false,
4072
+ "single_word": false,
4073
+ "special": true
4074
+ },
4075
+ "509": {
4076
+ "content": "<0xFD>",
4077
+ "lstrip": false,
4078
+ "normalized": false,
4079
+ "rstrip": false,
4080
+ "single_word": false,
4081
+ "special": true
4082
+ },
4083
+ "510": {
4084
+ "content": "<0xFE>",
4085
+ "lstrip": false,
4086
+ "normalized": false,
4087
+ "rstrip": false,
4088
+ "single_word": false,
4089
+ "special": true
4090
+ },
4091
+ "511": {
4092
+ "content": "<0xFF>",
4093
+ "lstrip": false,
4094
+ "normalized": false,
4095
+ "rstrip": false,
4096
+ "single_word": false,
4097
+ "special": true
4098
+ }
4099
+ },
4100
+ "bos_token": "<|endoftext|>",
4101
+ "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% else %}{{ eos_token }}{% endif %}",
4102
+ "clean_up_tokenization_spaces": false,
4103
+ "eos_token": "<|endoftext|>",
4104
+ "extra_special_tokens": {},
4105
+ "model_max_length": 1000000000000000019884624838656,
4106
+ "pad_token": "<|endoftext|>",
4107
+ "tokenizer_class": "PreTrainedTokenizerFast"
4108
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff