update

e1.0/training_args.bin → data/lid.176.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2242d8f4d8bbe20dba8cda642bc4100bc10ef653bd023b0914fdb43cdee70494
-size 5496
+oid sha256:7e69ec5451bc261cc7844e49e4792a85d7f09c06789ec800fc4a44aec362764e
+size 131266198

e1.0/added_tokens.json

@@ -1,24 +0,0 @@
-{
-  "</tool_call>": 151658,
-  "<tool_call>": 151657,
-  "<|box_end|>": 151649,
-  "<|box_start|>": 151648,
-  "<|endoftext|>": 151643,
-  "<|file_sep|>": 151664,
-  "<|fim_middle|>": 151660,
-  "<|fim_pad|>": 151662,
-  "<|fim_prefix|>": 151659,
-  "<|fim_suffix|>": 151661,
-  "<|im_end|>": 151645,
-  "<|im_start|>": 151644,
-  "<|image_pad|>": 151655,
-  "<|object_ref_end|>": 151647,
-  "<|object_ref_start|>": 151646,
-  "<|quad_end|>": 151651,
-  "<|quad_start|>": 151650,
-  "<|repo_name|>": 151663,
-  "<|video_pad|>": 151656,
-  "<|vision_end|>": 151653,
-  "<|vision_pad|>": 151654,
-  "<|vision_start|>": 151652
-}

e1.0/rng_state.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f2e155718eaa0de9768e57816e6e6874ba931ec4489afb1c2874db91c97f7a96
-size 14244

e1.0/scheduler.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c18e03b611645587dc7dfcef40d2a439b057a25a19eb36711814cbaafecaa43b
-size 1064

e1.0/special_tokens_map.json

@@ -1,19 +0,0 @@
-{
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
-  ],
-  "eos_token": "æĵļ",
-  "pad_token": "å¥ł"
-}

e1.0/tokenizer_config.json

@@ -197,12 +197,11 @@
   "bos_token": null,
   "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "æĵļ",
+  "eos_token": "<|im_end|>",
   "errors": "replace",
-  "model_max_length": 4096,
-  "pad_token": "å¥ł",
-  "padding_side": "right",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null
-}
+}

e1.0/generation_config.json → generation_config.json

@@ -1,14 +1,14 @@
 {
   "bos_token_id": 151643,
+  "pad_token_id": 151643,
   "do_sample": true,
   "eos_token_id": [
     151645,
     151643
   ],
-  "pad_token_id": 151643,
-  "repetition_penalty": 1.05,
+  "repetition_penalty": 1.1,
   "temperature": 0.7,
-  "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.45.1"
+  "top_k": 20,
+  "transformers_version": "4.37.0"
 }

model_chat.py

@@ -0,0 +1,103 @@
+import torch, sys
+import transformers
+
+try: model_path = sys.argv[1]
+except: model_path = "e1.0"
+
+print(f"Loading {model_path} ...")
+
+model = transformers.AutoModelForCausalLM.from_pretrained(
+    model_path, 
+    device_map = "auto",
+    torch_dtype = torch.bfloat16,
+)
+tokenizer = transformers.AutoTokenizer.from_pretrained(".")
+
+from qwen_vocab import old2new, new2old
+STOP_WORDS = "<|im_end|> <|endoftext|>".split()
+
+
+def map_tids(map_dict, tids):
+    return [ map_dict[x] for x in tids if x in map_dict ]
+
+
+class KeywordsStoppingCriteria(transformers.StoppingCriteria):
+    def __init__(self, str):
+        self.keyword_ids = tokenizer.encode(str)
+        self.keyword_ids = map_tids(old2new, self.keyword_ids)
+        self.keyword_len = len(self.keyword_ids)
+
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        last_token_ids = input_ids[0][-self.keyword_len:]
+        return last_token_ids.tolist() == self.keyword_ids
+
+stop_criteria_list = transformers.StoppingCriteriaList(
+    [ KeywordsStoppingCriteria(x) for x in STOP_WORDS ]
+)
+
+
+def get_answer(q):
+    if len(q) < 3: return "..."
+
+    prompt = f"<|im_start|>user\n{q}<|im_end|>\n<|im_start|>assistant"
+    old_tids = tokenizer.encode(prompt)
+
+    new_tids = map_tids(old2new, old_tids)
+    new_old_tids = map_tids(new2old, new_tids)
+
+    new_prompt = tokenizer.decode(new_old_tids)
+
+    if new_old_tids != old_tids:
+        print(f"!!! Cảnh báo sự trimm vocab làm mất thông tin !!!")
+        print(f"!!! old prompt: {prompt}")
+        print(f"!!! new prompt: {new_prompt}")
+
+    inputs = tokenizer(new_prompt, return_tensors="pt").to(model.device)
+
+    assert inputs["input_ids"][0].tolist() == new_old_tids
+
+    for i, x in enumerate(new_tids):
+        inputs["input_ids"][0][i] = x
+
+    with torch.no_grad():
+        output_ids = model.generate(
+            **inputs,
+            max_new_tokens=512,
+            temperature=0.3,
+            top_p=1.0, top_k=30, do_sample=True,
+            repetition_penalty=1.1,
+            stopping_criteria=stop_criteria_list,
+            pad_token_id=tokenizer.pad_token_id,
+        )
+
+    answer_tids = output_ids[0][len(inputs["input_ids"][0]) : ] # bỏ đi prompt tokens
+    old_tids = map_tids(new2old, answer_tids.tolist())
+
+    # print(prompt, answer_tids, old_tids) # DEBUG
+    return tokenizer.decode(old_tids)\
+        .split("<|im_end|>")[0].split("<end_of_turn>")[0].strip()
+
+
+from utils import *
+while True:
+    # bỏ qua lỗi utf-8 encoding trong trường hợp nhập text từ console
+    try: q = input(f"Bạn: {GREEN}").encode('utf-8', 'ignore').decode('utf-8', 'ignore')
+    except Exception as e: print(f"{RESET}{e}"); q = ""
+
+    reset_timer(timer=model_path)
+    a = get_answer(q).strip()
+    print(f"{RESET}Bot: {RED}{a}{RESET}")
+    measure_time("timespent", timer=model_path)
+
+'''
+python3 model_chat.py ../Qwen2.5-1.5B-Instruct__trimm_vocab
+
+python3 model_chat.py ../Qwen2.5-1.5B-Instruct
+
+số tuổi của An trừ đi số tuổi của Lan là 3, An 10 tuổi hỏi Lan mấy tuổi?
+
+ai tạo ra bạn
+
+Bạn: tạo ra một câu hoàn chỉnh với từ "thực hiện"
+Bot: Thì ra, việc thực hiện kế hoạch của chúng ta cần được lên lịch cụ thể.
+'''

qwen__1000__20000/README.md

@@ -0,0 +1,25 @@
+```sh
+head -n 3 tokens_kept__alphabet_long.jsonl
+#  token                                         token_id  count
+[  " opportunities"                               , 10488, 89723]
+[  " technologies"                                , 14310, 47144]
+[  " Pennsylvania"                                , 19771, 42463]
+```
+
+```sh
+## Sau khi cắt tỉa bằng máy và lọc sơ bằng tay
+wc -l tokens_kept__*
+
+    214 tokens_kept__alphabet_long.jsonl
+  32569 tokens_kept__alphabet_short.jsonl
+      0 tokens_kept__ascii_long.jsonl
+  18297 tokens_kept__ascii_short.jsonl
+  35092 tokens_kept__english.jsonl
+   2536 tokens_kept__others.jsonl
+  88708 total
+```
+
+Nhờ việc loại bỏ những tokens không thuộc hệ ngôn ngữ tiếng Anh + Việt (latin based), và loại bỏ những tokens có count thấp,
+giảm được số lượng vocab của qwen từ 151k xuống ~80k (gần 1/2).
+
+Việc phân loại các tokens vào từng mục như alphabet vs ascii vs other, long vs short giúp người xem cảm nhận tổng thể bộ vocab tốt hơn. Và có thể cắt tỉa thêm nếu muốn.

qwen__1000__20000/tokens_kept__alphabet_long.jsonl

@@ -0,0 +1,1270 @@
+[  "               "                              , 394, 1864062]
+[  "                   "                          , 503, 877768]
+[  "                       "                      , 664, 459973]
+[  "             "                                , 1060, 424626]
+[  "                           "                  , 999, 224408]
+[  "                 "                            , 1698, 208892]
+[  "                     "                        , 2549, 132437]
+[  "                               "              , 1383, 127402]
+[  "              "                               , 2290, 109992]
+[  "                "                             , 338, 91889]
+[  " opportunities"                               , 10488, 89723]
+[  "                         "                    , 3502, 85578]
+[  "                                   "          , 1920, 76197]
+[  "                  "                           , 4293, 71937]
+[  "                    "                         , 3824, 60093]
+[  "AttributeValue"                               , 78554, 55891]
+[  "                             "                , 4597, 55246]
+[  "                      "                       , 5180, 53152]
+[  "                                       "      , 2559, 47465]
+[  " technologies"                                , 14310, 47144]
+[  " Pennsylvania"                                , 19771, 42463]
+[  " Philadelphia"                                , 19335, 42285]
+[  " Massachusetts"                               , 21393, 41709]
+[  " assertEquals"                                , 10037, 40942]
+[  " manufacturing"                               , 14919, 39609]
+[  "                        "                     , 5108, 38546]
+[  "                                 "            , 6656, 38291]
+[  "                          "                   , 5968, 37049]
+[  " representing"                                , 14064, 36785]
+[  " demonstrated"                                , 20459, 36607]
+[  " possibilities"                               , 23607, 35454]
+[  " difficulties"                                , 26038, 33119]
+[  "                              "               , 7561, 31163]
+[  " capabilities"                                , 16928, 29946]
+[  "                                           "  , 3344, 29758]
+[  " Christianity"                                , 31086, 29623]
+[  " universities"                                , 23106, 28213]
+[  "                            "                 , 6526, 26887]
+[  " organisation"                                , 21424, 24950]
+[  " responsibilities"                            , 27323, 24793]
+[  "                                     "        , 8333, 24514]
+[  " experiencing"                                , 24084, 24359]
+[  "                                               "                                                                                                  , 4569, 23329]
+[  " characterized"                               , 31871, 23320]
+[  " establishing"                                , 30592, 22350]
+[  "                                "             , 786, 20751]
+[  " participating"                               , 23528, 20502]
+[  " organisations"                               , 28433, 19963]
+[  " participated"                                , 30308, 18878]
+[  "                                         "    , 10503, 17817]
+[  " sustainability"                              , 40229, 17549]
+[  " investigated"                                , 26219, 17500]
+[  " contributing"                                , 28720, 17064]
+[  " implementing"                                , 24931, 16822]
+[  "                                  "           , 9699, 16251]
+[  " manufactured"                                , 27548, 15692]
+[  " Technologies"                                , 24561, 15517]
+[  " accompanying"                                , 23534, 15405]
+[  " commissioned"                                , 43124, 14345]
+[  "                                    "         , 10589, 14321]
+[  " administered"                                , 36918, 13813]
+[  " dependencies"                                , 19543, 13716]
+[  " neighbourhood"                               , 39342, 13544]
+[  "                                                   "                                                                                              , 6374, 13412]
+[  "                                      "       , 11070, 13366]
+[  " conditioning"                                , 34021, 13277]
+[  " disabilities"                                , 35701, 12821]
+[  " grandchildren"                               , 49751, 12621]
+[  " coordination"                                , 37693, 11705]
+[  "                                             ", 13463, 11484]
+[  " personalities"                               , 43808, 11330]
+[  " demonstrating"                               , 44196, 10946]
+[  " similarities"                                , 42774, 10812]
+[  " backgroundColor"                             , 12694, 10156]
+[  "                                        "     , 12841, 10145]
+[  " personalized"                                , 34549, 10067]
+[  " Indianapolis"                                , 41351, 9877]
+[  " strengthened"                                , 59762, 9699]
+[  " StringBuilder"                               , 11410, 9661]
+[  " borderRadius"                                , 19208, 9538]
+[  "                                                       "                                                                                          , 8136, 9318]
+[  " Palestinians"                                , 33134, 9209]
+[  " highlighting"                                , 38586, 8921]
+[  " concentrating"                               , 75287, 8908]
+[  " communicated"                                , 57935, 8863]
+[  " Manufacturing"                               , 41077, 8719]
+[  " BorderRadius"                                , 31594, 8716]
+[  " constructing"                                , 49353, 8680]
+[  " incorporating"                               , 51813, 8623]
+[  " StatelessWidget"                             , 36798, 8591]
+[  " breathtaking"                                , 56092, 8512]
+[  " photographed"                                , 56203, 8499]
+[  " MainAxisAlignment"                           , 33214, 8442]
+[  " cryptocurrency"                              , 26037, 8405]
+[  "                                          "   , 14265, 8324]
+[  "                                                 "                                                                                                , 17648, 8273]
+[  "                                                                "                                                                                 , 5238, 8239]
+[  "                                            " , 15429, 8239]
+[  " acknowledging"                               , 60608, 8231]
+[  " mainAxisAlignment"                           , 33692, 8169]
+[  " strawberries"                                , 75103, 8149]
+[  " StatefulWidget"                              , 37642, 7915]
+[  " spokesperson"                                , 27903, 7906]
+[  " contemplating"                               , 78242, 7727]
+[  " contaminated"                                , 51573, 7567]
+[  " aforementioned"                              , 45652, 7540]
+[  " neighbouring"                                , 60927, 7526]
+[  "                                                           "                                                                                      , 11950, 7491]
+[  " uncertainties"                               , 68767, 7475]
+[  " contemplated"                                , 92339, 7440]
+[  " ArgumentError"                               , 76884, 7415]
+[  " BoxDecoration"                               , 36093, 7335]
+[  " broadcasting"                                , 48827, 7314]
+[  " MessageLookup"                               , 98774, 7267]
+[  " IllegalArgumentException"                    , 21374, 7266]
+[  " interpreting"                                , 65644, 7196]
+[  " missionaries"                                , 85216, 7015]
+[  " biodiversity"                                , 72019, 6968]
+[  " MERCHANTABILITY"                             , 7935, 6715]
+[  " intimidating"                                , 63030, 6694]
+[  " Presbyterian"                                , 85364, 6670]
+[  "                                              "                                                                                                   , 17264, 6632]
+[  " defaultValue"                                , 20163, 6548]
+[  " vulnerabilities"                             , 51127, 6495]
+[  " probabilities"                               , 48216, 6414]
+[  " savedInstanceState"                          , 14724, 6358]
+[  " interviewing"                                , 64716, 6188]
+[  " acknowledgment"                              , 77470, 6144]
+[  " laboratories"                                , 69660, 6123]
+[  " CrossAxisAlignment"                          , 43289, 6121]
+[  " experimenting"                               , 59157, 6120]
+[  " crossAxisAlignment"                          , 43695, 6079]
+[  " Universities"                                , 74798, 6060]
+[  " distributing"                                , 49712, 6014]
+[  " Organisation"                                , 46743, 5934]
+[  " findViewById"                                , 10241, 5918]
+[  " anticipating"                                , 85259, 5879]
+[  "                                                                                                                                "                 , 56940, 5863]
+[  " beneficiaries"                               , 60568, 5855]
+[  " cancellationToken"                           , 33133, 5847]
+[  " Newfoundland"                                , 88296, 5840]
+[  " globalization"                               , 79969, 5786]
+[  " specializing"                                , 57294, 5781]
+[  " volunteering"                                , 69338, 5713]
+[  "                                                     "                                                                                            , 20974, 5655]
+[  " Scandinavian"                                , 84574, 5649]
+[  " Sustainability"                              , 88712, 5617]
+[  " impoverished"                                , 87298, 5613]
+[  " Johannesburg"                                , 85541, 5579]
+[  " accelerating"                                , 68641, 5577]
+[  " autogenerated"                               , 81833, 5532]
+[  " groundbreaking"                              , 63855, 5523]
+[  " facilitating"                                , 67265, 5497]
+[  " inequalities"                                , 92234, 5487]
+[  "ToolStripMenuItem"                            , 16779, 5477]
+[  " restructuring"                               , 67927, 5451]
+[  " complexities"                                , 83040, 5426]
+[  "SerializedName"                               , 24471, 5415]
+[  "                                                "                                                                                                 , 18749, 5414]
+[  "ViewController"                               , 6632, 5367]
+[  " initialization"                              , 17465, 5233]
+[  "NotFoundException"                            , 14578, 5210]
+[  " collaborated"                                , 77074, 5204]
+[  " cybersecurity"                               , 61442, 5200]
+[  " coordinating"                                , 65415, 5174]
+[  " Broadcasting"                                , 63460, 5160]
+[  " RecyclerView"                                , 22562, 5143]
+[  " interdisciplinary"                           , 87319, 5096]
+[  " Jacksonville"                                , 47946, 5092]
+[  " interconnected"                              , 82316, 5074]
+[  " deficiencies"                                , 71846, 5036]
+[  " manipulating"                                , 62514, 5028]
+[  " Opportunities"                               , 65785, 4981]
+[  " ActiveRecord"                                , 17780, 4951]
+[  " RuntimeException"                            , 21247, 4918]
+[  " supplemented"                                , 88704, 4902]
+[  " multicultural"                               , 74316, 4855]
+[  " Neuroscience"                                , 84779, 4819]
+[  "                                                  "                                                                                               , 21509, 4816]
+[  " differentiated"                              , 88042, 4810]
+[  " firefighters"                                , 52106, 4798]
+[  " Saskatchewan"                                , 65153, 4780]
+[  " underestimated"                              , 86036, 4739]
+[  " collaborating"                               , 72201, 4699]
+[  " transmitting"                                , 77668, 4696]
+[  " TextEditingController"                       , 45998, 4668]
+[  "                                                               "                                                                                  , 15270, 4641]
+[  " municipalities"                              , 58489, 4606]
+[  " marginalized"                                , 75720, 4526]
+[  " recommending"                                , 64674, 4500]
+[  " authenticated"                               , 37260, 4484]
+[  "PropertyChanged"                              , 24075, 4393]
+[  " hospitalized"                                , 69240, 4369]
+[  " predetermined"                               , 85910, 4261]
+[  "                                                                   "                                                                              , 19273, 4240]
+[  " electronically"                              , 70770, 4228]
+[  " disadvantaged"                               , 79847, 4218]
+[  " Laboratories"                                , 77617, 4207]
+[  " documentaries"                               , 87366, 4200]
+[  " mutableListOf"                               , 76253, 4195]
+[  " abnormalities"                               , 74715, 4182]
+[  " ArgumentNullException"                       , 29335, 4169]
+[  " illustrating"                                , 92912, 4166]
+[  " methodologies"                               , 80798, 4095]
+[  " appropriated"                                , 86413, 4041]
+[  " acknowledgement"                             , 66923, 4016]
+[  " policymakers"                                , 69878, 3986]
+[  " incarcerated"                                , 80124, 3984]
+[  " encountering"                                , 91272, 3959]
+[  " repositories"                                , 49657, 3947]
+[  " CancellationToken"                           , 37237, 3927]
+[  "EventListener"                                , 7765, 3927]
+[  " neuroscience"                                , 91948, 3925]
+[  " trajectories"                                , 85548, 3923]
+[  " headquartered"                               , 80196, 3888]
+[  "                                                         "                                                                                        , 26065, 3866]
+[  " discontinued"                                , 64159, 3840]
+[  "BackgroundColor"                              , 29546, 3814]
+[  "ExecutionContext"                             , 60576, 3782]
+[  " assertNotNull"                               , 44328, 3724]
+[  " accumulating"                                , 91585, 3697]
+[  " LayoutInflater"                              , 28569, 3696]
+[  " dysfunctional"                               , 87704, 3672]
+[  "                                                    "                                                                                             , 24348, 3652]
+[  " orchestrated"                                , 92067, 3647]
+[  "ChangeListener"                               , 31668, 3641]
+[  " administering"                               , 85561, 3626]
+[  "ConfigureAwait"                               , 49684, 3618]
+[  " debilitating"                                , 91790, 3586]
+[  " Shutterstock"                                , 84042, 3562]
+[  " decentralized"                               , 47963, 3509]
+[  "                                                      "                                                                                           , 25773, 3504]
+[  "ancellationToken"                             , 23927, 3501]
+[  " InputDecoration"                             , 54478, 3429]
+[  " NONINFRINGEMENT"                             , 27986, 3409]
+[  "ClickListener"                                , 7226, 3407]
+[  " setContentView"                              , 21109, 3390]
+[  " errorMessage"                                , 24943, 3372]
+[  " longstanding"                                , 73129, 3345]
+[  "                                                                           "                                                                      , 14642, 3325]
+[  " Disabilities"                                , 97449, 3309]
+[  " controversies"                               , 85793, 3283]
+[  " IllegalStateException"                       , 40827, 3265]
+[  "                                                             "                                                                                    , 31979, 3158]
+[  " startActivity"                               , 18273, 3127]
+[  " customization"                               , 48041, 3117]
+[  " GestureDetector"                             , 63132, 3101]
+[  " efficiencies"                                , 91026, 3091]
+[  "longrightarrow"                               , 55042, 3087]
+[  " dictionaries"                                , 57514, 3083]
+[  " MaterialPageRoute"                           , 55267, 3048]
+[  " SharedPreferences"                           , 35084, 2980]
+[  " cryptocurrencies"                            , 43974, 2975]
+[  " Charlottesville"                             , 63638, 2956]
+[  " AppCompatActivity"                           , 24125, 2956]
+[  " discrepancies"                               , 90267, 2953]
+[  " transitioning"                               , 72094, 2888]
+[  " inconsistencies"                             , 91822, 2836]
+[  " NotImplementedError"                         , 38199, 2819]
+[  "                                                        "                                                                                         , 28802, 2811]
+[  " provisioning"                                , 97056, 2769]
+[  "ValidationError"                              , 80553, 2764]
+[  " MainActivity"                                , 27457, 2759]
+[  " StringBuffer"                                , 29891, 2716]
+[  "NavigationBar"                                , 50849, 2703]
+[  " resourceName"                                , 98485, 2700]
+[  " EventEmitter"                                , 30004, 2688]
+[  " ResponseEntity"                              , 19803, 2676]
+[  " ResourceManager"                             , 87269, 2675]
+[  " Serializable"                                , 24859, 2669]
+[  " troubleshooting"                             , 68671, 2644]
+[  " NotImplementedException"                     , 33146, 2624]
+[  " localStorage"                                , 15566, 2582]
+[  " nanoparticles"                               , 93653, 2533]
+[  "iscrimination"                                , 85913, 2518]
+[  " affordability"                               , 77910, 2504]
+[  "authenticated"                                , 57707, 2495]
+[  " LoggerFactory"                               , 35086, 2490]
+[  " customizable"                                , 62074, 2476]
+[  "ActionPerformed"                              , 24445, 2474]
+[  "Initialization"                               , 61928, 2460]
+[  " migrationBuilder"                            , 16955, 2441]
+[  " MutableLiveData"                             , 55281, 2439]
+[  " propertyName"                                , 44276, 2432]
+[  "ynchronization"                               , 67929, 2415]
+[  " crowdfunding"                                , 76644, 2411]
+[  " SQLException"                                , 24393, 2404]
+[  " ApplicationController"                       , 30938, 2382]
+[  "ApplicationContext"                           , 19736, 2377]
+[  " invokingState"                               , 87841, 2352]
+[  " disqualified"                                , 93576, 2336]
+[  " personalised"                                , 84248, 2325]
+[  " EventHandler"                                , 31957, 2312]
+[  " RuntimeError"                                , 44543, 2309]
+[  "SuppressWarnings"                             , 21574, 2304]
+[  "                                                                       "                                                                          , 22335, 2259]
+[  " downloadable"                                , 69652, 2242]
+[  " completamente"                               , 84624, 2219]
+[  "                                                          "                                                                                       , 32678, 2211]
+[  " HttpResponse"                                , 17580, 2211]
+[  " serialVersionUID"                            , 23673, 2175]
+[  "stackoverflow"                                , 42035, 2137]
+[  " UnsupportedOperationException"               , 42978, 2133]
+[  " Internacional"                               , 95067, 2107]
+[  " LocalDateTime"                               , 47465, 2087]
+[  " ArgumentException"                           , 39808, 2080]
+[  "Authenticated"                                , 26712, 2071]
+[  "SharedPreferences"                            , 37282, 2069]
+[  " CharSequence"                                , 57457, 2067]
+[  " addCriterion"                                , 18724, 2066]
+[  "ConnectionString"                             , 40431, 2059]
+[  " EditorGUILayout"                             , 52822, 2058]
+[  " ActiveSupport"                               , 73001, 2041]
+[  " configurable"                                , 43394, 2037]
+[  " onCreateView"                                , 41438, 2023]
+[  " initialState"                                , 29427, 2012]
+[  " Elasticsearch"                               , 58887, 2000]
+[  " RoundedRectangleBorder"                      , 82957, 1986]
+[  "SerializeField"                               , 18385, 1970]
+[  "ErrorResponse"                                , 55901, 1967]
+[  " instantiated"                                , 54586, 1966]
+[  "ServiceProvider"                              , 32604, 1958]
+[  "NavController"                                , 68407, 1956]
+[  " StringComparison"                            , 47687, 1949]
+[  " recyclerView"                                , 37149, 1946]
+[  "                                                              "                                                                                   , 37083, 1928]
+[  " Dependencies"                                , 80490, 1926]
+[  " Conditioning"                                , 82392, 1916]
+[  " SimpleDateFormat"                            , 26180, 1916]
+[  " BindingFlags"                                , 58678, 1916]
+[  "ustainability"                                , 73485, 1908]
+[  " asynchronously"                              , 67781, 1885]
+[  " TextFormField"                               , 78111, 1862]
+[  "PropertyValue"                                , 68634, 1852]
+[  " MonoBehaviour"                               , 15714, 1844]
+[  "FragmentManager"                              , 37482, 1839]
+[  "toHaveBeenCalledWith"                         , 56147, 1828]
+[  " especialmente"                               , 94002, 1828]
+[  " NullPointerException"                        , 56850, 1814]
+[  " CallingConvention"                           , 58413, 1810]
+[  "                                                                     "                                                                            , 42708, 1800]
+[  " SingleChildScrollView"                       , 88554, 1798]
+[  "                                                                 "                                                                                , 38171, 1792]
+[  " InterruptedException"                        , 35620, 1782]
+[  " whistleblower"                               , 90968, 1748]
+[  " SqlParameter"                                , 21923, 1727]
+[  " InvalidOperationException"                   , 52612, 1726]
+[  " modelBuilder"                                , 37504, 1717]
+[  " BufferedReader"                              , 25568, 1702]
+[  " OutlineInputBorder"                          , 82537, 1683]
+[  "RequestMapping"                               , 14125, 1680]
+[  " TextInputType"                               , 92551, 1676]
+[  " InitializeComponent"                         , 16233, 1662]
+[  " objectMapper"                                , 83912, 1660]
+[  " RecognitionException"                        , 79316, 1659]
+[  " JsonSerializer"                              , 70919, 1658]
+[  " CircularProgressIndicator"                   , 98479, 1638]
+[  "                                                            "                                                                                     , 38484, 1638]
+[  "OperationException"                           , 24924, 1626]
+[  " LinearLayout"                                , 27146, 1619]
+[  "StringBuilder"                                , 69412, 1611]
+[  " ApiException"                                , 82793, 1608]
+[  "LayoutManager"                                , 48748, 1604]
+[  " DependencyProperty"                          , 56552, 1601]
+[  " RaisedButton"                                , 79701, 1601]
+[  " AttributeSet"                                , 57440, 1600]
+[  "ArgumentNullException"                        , 63661, 1600]
+[  " functionalities"                             , 92080, 1583]
+[  " ActionResult"                                , 16831, 1578]
+[  " keyboardType"                                , 75099, 1572]
+[  " ImmutableList"                               , 92255, 1564]
+[  "InstanceState"                                , 8846, 1532]
+[  "BagConstraints"                               , 27346, 1519]
+[  " ValidationError"                             , 53029, 1508]
+[  "HasColumnType"                                , 76257, 1498]
+[  " currentState"                                , 57878, 1488]
+[  " KeyValuePair"                                , 53604, 1478]
+[  " LinearLayoutManager"                         , 45851, 1471]
+[  "cancellationToken"                            , 95470, 1468]
+[  " CompletableFuture"                           , 91535, 1468]
+[  "FormatException"                              , 42530, 1466]
+[  "FirstOrDefault"                               , 26841, 1465]
+[  " actionPerformed"                             , 24059, 1457]
+[  " autocomplete"                                , 31705, 1449]
+[  " HttpStatusCode"                              , 49339, 1448]
+[  " MemoryStream"                                , 36598, 1441]
+[  " Infragistics"                                , 54902, 1439]
+[  "ParseException"                               , 59262, 1438]
+[  " userRepository"                              , 49508, 1431]
+[  " FirebaseAuth"                                , 51981, 1425]
+[  " sharedPreferences"                           , 61057, 1422]
+[  " FloatingActionButton"                        , 81248, 1418]
+[  "GetProcAddress"                               , 97558, 1415]
+[  "ConverterFactory"                             , 61747, 1413]
+[  " currentIndex"                                , 50040, 1405]
+[  "                                                                  "                                                                               , 47549, 1401]
+[  "backgroundColor"                              , 39989, 1395]
+[  "ArgumentException"                            , 14343, 1393]
+[  "InternalServerError"                          , 55984, 1390]
+[  " alphanumeric"                                , 98979, 1383]
+[  " initialValue"                                , 60062, 1383]
+[  "ParameterValue"                               , 89432, 1381]
+[  "AssignableFrom"                               , 65685, 1381]
+[  "toMatchSnapshot"                              , 61426, 1378]
+[  " stringBuilder"                               , 39050, 1364]
+[  " ObjectMapper"                                , 54350, 1359]
+[  " AssertionError"                              , 66226, 1357]
+[  "RuntimeException"                             , 49276, 1347]
+[  " interpolated"                                , 85664, 1343]
+[  "NotFoundError"                                , 67908, 1341]
+[  "                                                                               "                                                                  , 39484, 1336]
+[  " OutputStream"                                , 52058, 1331]
+[  "                                                                                   "                                                              , 51475, 1325]
+[  "QualifiedName"                                , 92251, 1320]
+[  "AdapterFactory"                               , 83055, 1320]
+[  " RequestMethod"                               , 29557, 1318]
+[  " instantiation"                               , 81128, 1309]
+[  "InterruptedException"                         , 36685, 1300]
+[  " ParseException"                              , 61849, 1299]
+[  " GridBagConstraints"                          , 51923, 1295]
+[  "AttributeName"                                , 50501, 1291]
+[  "ListComponent"                                , 69376, 1276]
+[  " Responsibilities"                            , 73012, 1273]
+[  "toHaveBeenCalled"                             , 33606, 1273]
+[  "AnimationFrame"                               , 51695, 1272]
+[  " initializing"                                , 57060, 1270]
+[  "StateException"                               , 35368, 1270]
+[  " onBindViewHolder"                            , 57494, 1265]
+[  " BorderLayout"                                , 48357, 1263]
+[  " internacional"                               , 85159, 1262]
+[  " DefaultValue"                                , 97780, 1262]
+[  " misrepresented"                              , 82421, 1258]
+[  " ByteArrayOutputStream"                       , 56263, 1238]
+[  "NotSupportedException"                        , 81584, 1227]
+[  "IllegalArgumentException"                     , 73284, 1222]
+[  " PreparedStatement"                           , 42058, 1219]
+[  " RoutedEventArgs"                             , 31066, 1218]
+[  " onCreateViewHolder"                          , 58164, 1217]
+[  "ItemClickListener"                            , 37329, 1215]
+[  "EntityManager"                                , 43198, 1205]
+[  " ActivatedRoute"                              , 31417, 1204]
+[  " onViewCreated"                               , 86634, 1193]
+[  "                                                                                 "                                                                , 95429, 1191]
+[  "setOnClickListener"                           , 63628, 1185]
+[  " defStyleAttr"                                , 86090, 1185]
+[  " DataGridView"                                , 30253, 1184]
+[  " FileOutputStream"                            , 42656, 1151]
+[  " letterSpacing"                               , 87999, 1148]
+[  "ExceptionHandler"                             , 61010, 1147]
+[  " BehaviorSubject"                             , 61638, 1142]
+[  "RecognitionException"                         , 92472, 1137]
+[  "ConnectionFactory"                            , 89321, 1127]
+[  " JsonResponse"                                , 38263, 1118]
+[  " concatenated"                                , 97534, 1116]
+[  " FileInputStream"                             , 40185, 1115]
+[  " ErrorMessage"                                , 41827, 1109]
+[  "OutOfRangeException"                          , 49240, 1108]
+[  " HttpServletResponse"                         , 24988, 1108]
+[  " OkHttpClient"                                , 78920, 1108]
+[  " professionnel"                               , 84405, 1105]
+[  " PendingIntent"                               , 71420, 1103]
+[  " principalmente"                              , 94337, 1088]
+[  " ActionController"                            , 73452, 1086]
+[  "creativecommons"                              , 85231, 1085]
+[  "leftrightarrow"                               , 81819, 1081]
+[  " justifyContent"                              , 22039, 1075]
+[  " AttributeError"                              , 53631, 1070]
+[  " GetComponent"                                , 27750, 1069]
+[  "NullException"                                , 23731, 1069]
+[  " uninitialized"                               , 97287, 1066]
+[  " preprocessing"                               , 63631, 1059]
+[  "ValueHandling"                                , 80498, 1059]
+[  "CastException"                                , 88908, 1059]
+[  " PackageManager"                              , 56395, 1052]
+[  "BuilderFactory"                               , 89139, 1052]
+[  "ApiModelProperty"                             , 45113, 1048]
+[  " refreshToken"                                , 74541, 1048]
+[  "RelativeLayout"                               , 98742, 1047]
+[  " ObservableCollection"                        , 44464, 1042]
+[  "NullPointerException"                         , 83833, 1036]
+[  "                                                                         "                                                                        , 46145, 1034]
+[  " Initialization"                              , 39267, 1030]
+[  "abstractmethod"                               , 64951, 1029]
+[  " dataGridViewCellStyle"                       , 26441, 1028]
+[  " currentValue"                                , 66248, 1027]
+[  " LinkedHashMap"                               , 64333, 1026]
+[  "etSocketAddress"                              , 80507, 1025]
+[  "OutOfBoundsException"                         , 64809, 1022]
+[  " httpResponse"                                , 74806, 1018]
+[  "HasColumnName"                                , 41684, 1017]
+[  " outputStream"                                , 57961, 1016]
+[  "equalsIgnoreCase"                             , 34633, 1004]
+[  "                                                                        "                                                                         , 37144, 1000]
+[  " BeautifulSoup"                               , 35910, 996]
+[  "RenderingContext"                             , 97158, 996]
+[  "HandlerContext"                               , 75226, 995]
+[  " PropertyChanged"                             , 43189, 989]
+[  " AppComponent"                                , 30492, 989]
+[  " dispensaries"                                , 99139, 988]
+[  " placeholders"                                , 78428, 986]
+[  " getItemCount"                                , 59772, 985]
+[  "toHaveBeenCalledTimes"                        , 82544, 980]
+[  " HttpServletRequest"                          , 49594, 978]
+[  " layoutParams"                                , 94644, 977]
+[  " ActionListener"                              , 40219, 974]
+[  " responseBody"                                , 98065, 972]
+[  " RequestOptions"                              , 82349, 972]
+[  "ServletResponse"                              , 16765, 972]
+[  " entityManager"                               , 59775, 971]
+[  " layoutManager"                               , 84637, 970]
+[  "                                                                      "                                                                           , 46771, 969]
+[  " redistributed"                               , 73394, 964]
+[  "unprocessable"                                , 67760, 963]
+[  " DialogResult"                                , 36510, 963]
+[  " StringWriter"                                , 84787, 961]
+[  " BitConverter"                                , 72884, 955]
+[  " LinearGradient"                              , 96492, 952]
+[  " IActionResult"                               , 34514, 950]
+[  " AtomicInteger"                               , 85038, 946]
+[  " ResourceType"                                , 97801, 946]
+[  " ConnectionState"                             , 71693, 944]
+[  "ProgressDialog"                               , 65056, 942]
+[  " defaultMessage"                              , 78193, 941]
+[  " OnPropertyChanged"                           , 49281, 937]
+[  " cannabinoids"                                , 92886, 932]
+[  " expectedResult"                              , 81441, 928]
+[  "ThanOrEqualTo"                                , 89387, 926]
+[  "                                                                    "                                                                             , 51068, 925]
+[  " BufferedImage"                               , 56170, 923]
+[  " StreamReader"                                , 46811, 920]
+[  "RefreshLayout"                                , 61966, 919]
+[  " DialogInterface"                             , 46308, 918]
+[  " SqlConnection"                               , 27981, 916]
+[  "EditingController"                            , 43164, 913]
+[  " FirebaseFirestore"                           , 92240, 912]
+[  " DateTimeOffset"                              , 78494, 905]
+[  "                                                                            "                                                                     , 55799, 903]
+[  " multiprocessing"                             , 57124, 892]
+[  " ArrayAdapter"                                , 49640, 885]
+[  " Microsystems"                                , 94661, 884]
+[  " selectedItem"                                , 70992, 874]
+[  " NotSupportedException"                       , 92990, 873]
+[  " objectAtIndex"                               , 29320, 867]
+[  "EqualityComparer"                             , 70152, 866]
+[  "URLConnection"                                , 38672, 856]
+[  "ChangedEventArgs"                             , 71365, 852]
+[  " InputStreamReader"                           , 40287, 852]
+[  "DetailsService"                               , 72382, 852]
+[  " currentPosition"                             , 84418, 843]
+[  " functionName"                                , 90519, 843]
+[  " flexDirection"                               , 29397, 842]
+[  "ErrorException"                               , 91019, 837]
+[  "legalArgumentException"                       , 97009, 833]
+[  " XMLHttpRequest"                              , 45838, 830]
+[  "TouchListener"                                , 79848, 830]
+[  " JSONException"                               , 84690, 825]
+[  "MethodManager"                                , 72130, 824]
+[  " departamento"                                , 95889, 821]
+[  "InvalidOperationException"                    , 90735, 820]
+[  "NegativeButton"                               , 75068, 819]
+[  " connectionString"                            , 62084, 818]
+[  " progressDialog"                              , 57438, 815]
+[  " ComponentFixture"                            , 28363, 815]
+[  "ContextHolder"                                , 79804, 814]
+[  " SearchResult"                                , 96597, 805]
+[  " CommonModule"                                , 37638, 804]
+[  "                                                                              "                                                                   , 81221, 804]
+[  " BaseActivity"                                , 74645, 803]
+[  " backgroundImage"                             , 67029, 800]
+[  "ResponseStatus"                               , 92663, 800]
+[  " clearTimeout"                                , 53225, 799]
+[  " errorHandler"                                , 71073, 797]
+[  " ApplicationRecord"                           , 54826, 797]
+[  " ArgumentOutOfRangeException"                 , 76521, 796]
+[  " ContentValues"                               , 55929, 795]
+[  " getDescription"                              , 54733, 793]
+[  "PositiveButton"                               , 60762, 792]
+[  "addPreferredGap"                              , 34275, 787]
+[  "NavigationView"                               , 83685, 786]
+[  " stringWithFormat"                            , 20067, 785]
+[  "OnClickListener"                              , 49670, 783]
+[  " screenHeight"                                , 77555, 783]
+[  " ConcurrentHashMap"                           , 89075, 772]
+[  " createAction"                                , 76636, 769]
+[  " ConsoleColor"                                , 58834, 768]
+[  "                                                                             "                                                                    , 58591, 765]
+[  " attributeName"                               , 93259, 761]
+[  " RemoteException"                             , 60209, 761]
+[  " PropertyValue"                               , 94338, 760]
+[  " masturbating"                                , 92007, 759]
+[  " responseData"                                , 67445, 758]
+[  "UniformLocation"                              , 51704, 758]
+[  " RouterModule"                                , 34787, 756]
+[  "EncodingException"                            , 56637, 755]
+[  " ModelAndView"                                , 35068, 753]
+[  "TargetException"                              , 72152, 750]
+[  " requestOptions"                              , 91094, 750]
+[  " MessageBoxButtons"                           , 31407, 748]
+[  " grantResults"                                , 79693, 748]
+[  " ViewController"                              , 25295, 747]
+[  " onOptionsItemSelected"                       , 63967, 746]
+[  " onBackPressed"                               , 70609, 743]
+[  " verschiedenen"                               , 88687, 742]
+[  "ConfigurationException"                       , 98192, 734]
+[  " encodeURIComponent"                          , 56737, 734]
+[  " componentDidMount"                           , 26740, 731]
+[  "SqlConnection"                                , 61864, 727]
+[  " selectedIndex"                               , 83617, 727]
+[  "                                                                                       "                                                          , 65271, 727]
+[  " NotFoundException"                           , 66646, 724]
+[  "BTTagCompound"                                , 93375, 724]
+[  " marginBottom"                                , 32036, 720]
+[  "ValueGenerationStrategy"                      , 76617, 720]
+[  "selectedIndex"                                , 74826, 720]
+[  " ByteArrayInputStream"                        , 85625, 719]
+[  " applicationContext"                          , 85299, 713]
+[  "interopRequireDefault"                        , 35364, 713]
+[  "ItemSelectedListener"                         , 57972, 709]
+[  "ForegroundColor"                              , 84997, 708]
+[  " UserRepository"                              , 60233, 705]
+[  " ErrorResponse"                               , 95724, 703]
+[  "AccessException"                              , 38008, 703]
+[  "                                                                          "                                                                       , 41056, 702]
+[  "ResourceManager"                              , 32498, 700]
+[  " ServletException"                            , 30091, 700]
+[  " FileNotFoundException"                       , 68299, 699]
+[  " ErrorHandler"                                , 88331, 697]
+[  " BitmapFactory"                               , 69896, 695]
+[  " DirectoryInfo"                               , 81688, 694]
+[  " BufferedWriter"                              , 63129, 692]
+[  "formedURLException"                           , 64771, 691]
+[  "PackageManager"                               , 89170, 689]
+[  "ISyntaxException"                             , 77493, 687]
+[  "PermissionsResult"                            , 97781, 687]
+[  "querySelector"                                , 86267, 686]
+[  "toBeInTheDocument"                            , 99039, 683]
+[  " EntityManager"                               , 48767, 680]
+[  " RedirectToAction"                            , 28177, 680]
+[  " getSystemService"                            , 91376, 680]
+[  "CheckedChangeListener"                        , 81902, 679]
+[  " getResources"                                , 58145, 676]
+[  "GestureRecognizer"                            , 28403, 675]
+[  "RequestMethod"                                , 69790, 672]
+[  " startPosition"                               , 98342, 669]
+[  " SQLiteDatabase"                              , 62436, 668]
+[  " MessageBoxIcon"                              , 34036, 667]
+[  " dataGridView"                                , 18038, 666]
+[  "AxisAlignment"                                , 14795, 665]
+[  " BrowserModule"                               , 37371, 665]
+[  " RaisePropertyChanged"                        , 89330, 663]
+[  "ActivityCreated"                              , 92377, 663]
+[  " ExecutionContext"                            , 85451, 661]
+[  " UIViewController"                            , 20784, 658]
+[  " errorCallback"                               , 89080, 658]
+[  " componentName"                               , 94142, 657]
+[  " ReactiveFormsModule"                         , 52671, 657]
+[  " onCreateOptionsMenu"                         , 67094, 656]
+[  " objectForKey"                                , 22684, 652]
+[  "StartPosition"                                , 53767, 651]
+[  " StreamWriter"                                , 57257, 647]
+[  " createElement"                               , 86105, 647]
+[  " RegexOptions"                                , 84598, 642]
+[  "ActivityResult"                               , 50100, 641]
+[  "PreparedStatement"                            , 45405, 640]
+[  "CompleteListener"                             , 60921, 639]
+[  "CollectionView"                               , 77372, 637]
+[  "SystemService"                                , 40744, 636]
+[  " gridBagConstraints"                          , 54259, 636]
+[  "addEventListener"                             , 32374, 632]
+[  "getSimpleName"                                , 73317, 631]
+[  " validationResult"                            , 96533, 628]
+[  " StringSplitOptions"                          , 79876, 623]
+[  " DecimalFormat"                               , 80796, 622]
+[  " ClassNotFoundException"                      , 65641, 622]
+[  " preparedStatement"                           , 55510, 622]
+[  "PasswordField"                                , 92528, 621]
+[  "complexContent"                               , 97715, 620]
+[  "addContainerGap"                              , 33710, 619]
+[  " HttpResponseMessage"                         , 68704, 616]
+[  "                                                                                "                                                                 , 97417, 615]
+[  "OperationContract"                            , 96107, 610]
+[  "abcdefghijklmnopqrstuvwxyz"                   , 67512, 608]
+[  "CustomAttributes"                             , 61946, 607]
+[  " WindowManager"                               , 77755, 607]
+[  " RelayCommand"                                , 76843, 606]
+[  " RequestContext"                              , 90057, 605]
+[  " IConfiguration"                              , 64490, 602]
+[  " UICollectionView"                            , 28577, 601]
+[  " viewController"                              , 52722, 596]
+[  " ValidationResult"                            , 98594, 594]
+[  " serviceProvider"                             , 98009, 593]
+[  " Informationen"                               , 48842, 592]
+[  "AlgorithmException"                           , 71386, 591]
+[  " verschiedene"                                , 95710, 590]
+[  " SolidColorBrush"                             , 91251, 587]
+[  "NumberFormatException"                        , 75128, 587]
+[  "HttpServletRequest"                           , 80966, 587]
+[  " notifyDataSetChanged"                        , 91202, 585]
+[  " PropertyInfo"                                , 94423, 585]
+[  " sessionStorage"                              , 50143, 584]
+[  "                                                                                           "                                                      , 79871, 584]
+[  " responseType"                                , 90444, 582]
+[  "SessionFactory"                               , 63215, 581]
+[  "LocalizedString"                              , 39448, 580]
+[  "SuccessListener"                              , 80805, 579]
+[  "NSMutableArray"                               , 27754, 578]
+[  "TouchableOpacity"                             , 27067, 574]
+[  " HttpResponseRedirect"                        , 53030, 574]
+[  "orianCalendar"                                , 63793, 574]
+[  " fragmentManager"                             , 94532, 573]
+[  "Longrightarrow"                               , 27642, 573]
+[  " jsonResponse"                                , 90465, 573]
+[  "TestingModule"                                , 32078, 571]
+[  " loadChildren"                                , 63035, 566]
+[  " onActivityResult"                            , 78840, 565]
+[  " RelativeLayout"                              , 69228, 560]
+[  " emailAddress"                                , 89713, 559]
+[  "ComputedStyle"                                , 89381, 557]
+[  "getConnection"                                , 52414, 556]
+[  " getConnection"                               , 65313, 555]
+[  " handleChange"                                , 38655, 551]
+[  "ABCDEFGHIJKLMNOPQRSTUVWXYZ"                   , 75902, 546]
+[  " initialValues"                               , 93483, 542]
+[  " GetEnumerator"                               , 46479, 542]
+[  "RoutingModule"                                , 36539, 540]
+[  " clearInterval"                               , 45585, 537]
+[  "ObjectContext"                                , 60504, 535]
+[  " ModelRenderer"                               , 81010, 531]
+[  " NavController"                               , 55762, 526]
+[  "LayoutInflater"                               , 39112, 523]
+[  "SelectionMode"                                , 97572, 522]
+[  " doInBackground"                              , 71840, 521]
+[  " WebDriverWait"                               , 63646, 521]
+[  "PreferredSize"                                , 53405, 520]
+[  "ProducesResponseType"                         , 90620, 520]
+[  "WithEmailAndPassword"                         , 84361, 518]
+[  " setSupportActionBar"                         , 83048, 518]
+[  "UIApplication"                                , 28982, 517]
+[  " NSDictionary"                                , 28896, 516]
+[  " mRecyclerView"                               , 91523, 516]
+[  " paddingRight"                                , 70015, 515]
+[  "JSONException"                                , 54368, 512]
+[  " collectionView"                              , 26535, 510]
+[  " ConfigurationManager"                        , 55831, 507]
+[  " ProgressDialog"                              , 61831, 506]
+[  " SetLastError"                                , 99002, 505]
+[  "                                                                                               "                                                  , 86766, 505]
+[  " locationManager"                             , 68732, 502]
+[  " ServiceException"                            , 72107, 500]
+[  " IllegalAccessException"                      , 88780, 498]
+[  " startActivityForResult"                      , 73627, 497]
+[  "RowAtIndexPath"                               , 33058, 496]
+[  "ApplicationBuilder"                           , 75342, 496]
+[  " SceneManager"                                , 53592, 496]
+[  "InvalidArgumentException"                     , 55608, 493]
+[  "ServletRequest"                               , 13525, 492]
+[  " BroadcastReceiver"                           , 96064, 491]
+[  " paddingBottom"                               , 60665, 487]
+[  " UserController"                              , 58704, 483]
+[  " handleMessage"                               , 96840, 483]
+[  "DirectoryName"                                , 84283, 474]
+[  "oredProcedure"                                , 45517, 474]
+[  "NameValuePair"                                , 70033, 473]
+[  " UITableViewCell"                             , 27330, 472]
+[  "VertexAttribArray"                            , 64961, 470]
+[  "PropertyDescriptor"                           , 84918, 469]
+[  "createElement"                                , 58008, 466]
+[  " OnClickListener"                             , 67307, 466]
+[  "PointerException"                             , 39618, 465]
+[  " AppRoutingModule"                            , 43207, 465]
+[  "userRepository"                               , 85741, 462]
+[  "LayoutConstraint"                             , 44940, 460]
+[  "ArgsConstructor"                              , 25007, 459]
+[  " PropertyChangedEventArgs"                    , 92116, 457]
+[  " restTemplate"                                , 79829, 457]
+[  " ApplicationContext"                          , 91518, 456]
+[  " createContext"                               , 83674, 456]
+[  "setBackground"                                , 90525, 455]
+[  "GetEnumerator"                                , 75065, 453]
+[  " StartCoroutine"                              , 45212, 452]
+[  " isEqualToString"                             , 25522, 452]
+[  "getRepository"                                , 41332, 450]
+[  "SocketAddress"                                , 69455, 450]
+[  "ThemeProvider"                                , 67866, 449]
+[  " BaseController"                              , 55461, 448]
+[  " ServiceProvider"                             , 87617, 448]
+[  " PriorityQueue"                               , 84102, 446]
+[  "ClassNotFoundException"                       , 61070, 444]
+[  "HostException"                                , 96682, 444]
+[  " KeyboardInterrupt"                           , 84188, 442]
+[  "ManagedObject"                                , 41055, 442]
+[  "HasForeignKey"                                , 74313, 440]
+[  "FileNotFoundException"                        , 64691, 436]
+[  " handleSubmit"                                , 42265, 435]
+[  "HomeAsUpEnabled"                              , 95643, 435]
+[  " contentValues"                               , 81418, 434]
+[  "ApiController"                                , 68617, 433]
+[  " Initializing"                                , 94312, 431]
+[  "GeneratedValue"                               , 36106, 430]
+[  " categoryName"                                , 89610, 429]
+[  "RegularExpression"                            , 90402, 429]
+[  " AudioManager"                                , 95881, 429]
+[  " HttpURLConnection"                           , 94327, 428]
+[  " borderBottom"                                , 55206, 426]
+[  " searchString"                                , 94040, 421]
+[  " NoSuchElementException"                      , 94449, 420]
+[  " FileNotFoundError"                           , 95841, 419]
+[  " correctamente"                               , 58507, 418]
+[  " WaitForSeconds"                              , 44878, 417]
+[  " createSelector"                              , 82372, 417]
+[  "ValueCollection"                              , 86715, 415]
+[  " SplashScreen"                                , 90324, 414]
+[  " DateTimeKind"                                , 96209, 414]
+[  " HomeController"                              , 60878, 413]
+[  " cancelButton"                                , 44026, 411]
+[  " getContentPane"                              , 53576, 411]
+[  "ElementException"                             , 62738, 410]
+[  "NavigationItemSelectedListener"               , 98767, 410]
+[  "InterfaceOrientation"                         , 64479, 409]
+[  "ServletContext"                               , 78701, 409]
+[  " defaultProps"                                , 79297, 406]
+[  "ModelCreating"                                , 93392, 406]
+[  "RequiredMixin"                                , 95658, 406]
+[  "HttpException"                                , 58499, 405]
+[  "UserController"                               , 70735, 405]
+[  " DriverManager"                               , 63508, 403]
+[  "FormattedMessage"                             , 86742, 400]
+[  " principalTable"                              , 58651, 398]
+[  " ReferentialAction"                           , 58770, 398]
+[  "setVisibility"                                , 95792, 397]
+[  " MySqlCommand"                                , 58307, 396]
+[  " principalColumn"                             , 58801, 395]
+[  "PickerController"                             , 42911, 394]
+[  "ActivityIndicator"                            , 57143, 393]
+[  " StringTokenizer"                             , 71601, 390]
+[  " configparser"                                , 97437, 389]
+[  " HttpClientModule"                            , 53649, 389]
+[  "LocalizedMessage"                             , 98375, 387]
+[  " mapStateToProps"                             , 30190, 387]
+[  " mapDispatchToProps"                          , 31641, 387]
+[  " NotImplemented"                              , 21548, 385]
+[  "CursorPosition"                               , 91672, 382]
+[  "collectionView"                               , 42440, 382]
+[  " ResourceBundle"                              , 95361, 382]
+[  " HomeComponent"                               , 56852, 381]
+[  "UITableViewCell"                              , 52138, 381]
+[  " NSMutableArray"                              , 33268, 380]
+[  "ValidateAntiForgeryToken"                     , 50826, 380]
+[  " ActivityCompat"                              , 92444, 379]
+[  " onPostExecute"                               , 83163, 378]
+[  " BrowserAnimationsModule"                     , 71436, 378]
+[  " FirebaseDatabase"                            , 68474, 377]
+[  "TexParameteri"                                , 62374, 376]
+[  " primaryStage"                                , 89750, 375]
+[  "encodeURIComponent"                           , 97683, 372]
+[  "entityManager"                                , 47112, 370]
+[  "HomeController"                               , 63572, 369]
+[  "beginTransaction"                             , 58110, 368]
+[  " UsersController"                             , 96427, 362]
+[  "ModelProperty"                                , 40082, 359]
+[  "FirstResponder"                               , 52170, 358]
+[  " confirmPassword"                             , 88118, 358]
+[  " randomNumber"                                , 87720, 358]
+[  " ApplicationUser"                             , 93143, 357]
+[  " initWithFrame"                               , 30067, 357]
+[  " completionHandler"                           , 45780, 356]
+[  " QApplication"                                , 62449, 356]
+[  "RectTransform"                                , 79930, 355]
+[  "CompatActivity"                               , 14736, 355]
+[  "BarController"                                , 61876, 354]
+[  "compileComponents"                            , 43998, 353]
+[  " responseObject"                              , 69040, 352]
+[  "DataSetChanged"                               , 41641, 352]
+[  " ParameterDirection"                          , 65927, 352]
+[  " AuthenticationService"                       , 94733, 351]
+[  "PasswordEncoder"                              , 76579, 348]
+[  " SharedModule"                                , 83639, 347]
+[  " LoginComponent"                              , 59754, 347]
+[  " ShoppingCart"                                , 90907, 346]
+[  " AsyncCallback"                               , 91436, 345]
+[  "PathComponent"                                , 60068, 343]
+[  " setDescription"                              , 68325, 342]
+[  " GenerationType"                              , 55700, 342]
+[  " openFileDialog"                              , 97371, 342]
+[  " getArguments"                                , 97879, 341]
+[  " jdbcTemplate"                                , 87317, 340]
+[  " LoginActivity"                               , 98346, 340]
+[  " DatabaseReference"                           , 80046, 335]
+[  "DialogContent"                                , 80329, 329]
+[  " getSupportFragmentManager"                   , 87435, 329]
+[  "AffineTransform"                              , 61872, 326]
+[  "assertInstanceOf"                             , 63180, 325]
+[  "StringEncoding"                               , 59746, 324]
+[  "setDescription"                               , 63542, 322]
+[  "getElementsByTagName"                         , 70620, 322]
+[  " TimeInterval"                                , 98548, 320]
+[  " containerView"                               , 96961, 319]
+[  " referencedColumnName"                        , 82347, 319]
+[  "externalActionCode"                           , 90098, 319]
+[  " autoComplete"                                , 72486, 319]
+[  "WithIdentifier"                               , 45428, 319]
+[  " dataSnapshot"                                , 64193, 318]
+[  " ListViewItem"                                , 95659, 317]
+[  "stateProvider"                                , 79349, 317]
+[  " digitalWrite"                                , 64797, 317]
+[  " GameController"                              , 96896, 317]
+[  "BoundingClientRect"                           , 63113, 316]
+[  "DefaultCloseOperation"                        , 55382, 316]
+[  "ManagedObjectContext"                         , 68501, 315]
+[  " isAuthenticated"                             , 70926, 313]
+[  "AttribPointer"                                , 78299, 313]
+[  "getApplication"                               , 91869, 311]
+[  " ApplicationException"                        , 95818, 310]
+[  "BarButtonItem"                                , 38284, 307]
+[  "outedEventArgs"                               , 29072, 307]
+[  "IllegalAccessException"                       , 79851, 307]
+[  " textDecoration"                              , 91945, 306]
+[  " tableLayoutPanel"                            , 97755, 304]
+[  " UIApplication"                               , 34904, 302]
+[  "UIViewController"                             , 63748, 301]
+[  "InstantiationException"                       , 96901, 300]
+[  "CoreApplication"                              , 84479, 299]
+[  " componentWillUnmount"                        , 71576, 297]
+[  " componentWill"                               , 38845, 296]
+[  "FontAwesomeIcon"                              , 60320, 296]
+[  "UICollectionView"                             , 55571, 296]
+[  "DataGridViewTextBoxColumn"                    , 64930, 294]
+[  " MatButtonModule"                             , 93692, 293]
+[  " sourceMappingURL"                            , 44695, 293]
+[  " currentPlayer"                               , 90068, 293]
+[  " RandomForest"                                , 90714, 292]
+[  " RectTransform"                               , 96697, 292]
+[  " windowHeight"                                , 88547, 292]
+[  "handleRequest"                                , 91237, 290]
+[  " URLWithString"                               , 59579, 290]
+[  "HorizontalAlignment"                          , 85419, 288]
+[  "routeProvider"                                , 83660, 286]
+[  " getMenuInflater"                             , 91000, 285]
+[  " initComponents"                              , 52056, 283]
+[  " InvalidArgumentException"                    , 80518, 283]
+[  " cellForRowAtIndexPath"                       , 65889, 283]
+[  "viewController"                               , 95412, 282]
+[  "realDonaldTrump"                              , 73405, 280]
+[  " ObjectOutputStream"                          , 97786, 276]
+[  " htmlspecialchars"                            , 31437, 275]
+[  "DOMContentLoaded"                             , 77507, 273]
+[  "paredStatement"                               , 22422, 273]
+[  "NSMutableDictionary"                          , 47618, 272]
+[  " defaultstate"                                , 54458, 269]
+[  " MySqlConnection"                             , 81987, 268]
+[  " MouseEventArgs"                              , 81904, 267]
+[  "iationException"                              , 74027, 267]
+[  "HideInInspector"                              , 87158, 266]
+[  " invokevirtual"                               , 69825, 265]
+[  "NotificationCenter"                           , 92757, 262]
+[  " appendString"                                , 96340, 261]
+[  " combineReducers"                             , 57943, 261]
+[  " getSupportActionBar"                         , 72828, 260]
+[  " DevComponents"                               , 41225, 260]
+[  " sessionFactory"                              , 70476, 259]
+[  "getDescription"                               , 94813, 258]
+[  " dateFormatter"                               , 69776, 258]
+[  "ModelAttribute"                               , 83887, 257]
+[  "ResultsController"                            , 81321, 257]
+[  " stringBuffer"                                , 60707, 257]
+[  " OpenFileDialog"                              , 94086, 257]
+[  "getStringExtra"                               , 86681, 257]
+[  " onDataChange"                                , 98908, 256]
+[  "SingleOrDefault"                              , 87331, 255]
+[  "ostringstream"                                , 59006, 253]
+[  " ProductService"                              , 93198, 253]
+[  "estureRecognizer"                             , 81920, 252]
+[  " columnHeader"                                , 95624, 251]
+[  " BusinessException"                           , 96994, 251]
+[  "getMockBuilder"                               , 91424, 251]
+[  " mouseClicked"                                , 85739, 250]
+[  " ApiController"                               , 87826, 246]
+[  " MetroFramework"                              , 51355, 244]
+[  " AsyncStorage"                                , 71621, 243]
+[  "classCallCheck"                               , 80198, 241]
+[  " defaultCenter"                               , 55646, 241]
+[  " modelAndView"                                , 80675, 241]
+[  " ConfigureServices"                           , 96608, 239]
+[  " SqlDataAdapter"                              , 46843, 238]
+[  " gratuitement"                                , 91934, 237]
+[  " ControllerBase"                              , 79024, 236]
+[  "SupportFragmentManager"                       , 60546, 235]
+[  "TableViewCell"                                , 76348, 234]
+[  " MessageBoxButton"                            , 97603, 233]
+[  " FindObjectOfType"                            , 91422, 232]
+[  "getStatusCode"                                , 76537, 232]
+[  " UIScrollView"                                , 73652, 229]
+[  " ValueEventListener"                          , 98145, 224]
+[  " respondsToSelector"                          , 62293, 224]
+[  "SupportActionBar"                             , 45230, 224]
+[  "NSNotificationCenter"                         , 55985, 224]
+[  "MouseListener"                                , 69609, 223]
+[  " componentDidUpdate"                          , 85220, 223]
+[  " ApplicationDbContext"                        , 73224, 218]
+[  "ViewControllerAnimated"                       , 57840, 217]
+[  "ManagerInterface"                             , 85255, 214]
+[  "foregroundColor"                              , 81144, 214]
+[  " SqlDataReader"                               , 65511, 213]
+[  " sharedInstance"                              , 51958, 213]
+[  " MethodInvocation"                            , 66672, 210]
+[  " AssemblyVersion"                             , 65675, 210]
+[  " registrazione"                               , 95835, 207]
+[  " sharedApplication"                           , 60318, 206]
+[  "NSUserDefaults"                               , 54438, 205]
+[  " defaultManager"                              , 98920, 205]
+[  " NSMutableDictionary"                         , 51905, 204]
+[  " productService"                              , 98995, 202]
+[  "TableWidgetItem"                              , 74093, 202]
+[  " setBackgroundColor"                          , 79874, 200]
+[  "RestController"                               , 40052, 197]
+[  " initWithTitle"                               , 72730, 195]
+[  " applyMiddleware"                             , 83364, 194]
+[  " TouchableOpacity"                            , 63743, 193]
+[  " SelectListItem"                              , 95883, 193]
+[  " CGAffineTransform"                           , 68789, 192]
+[  "responseObject"                               , 86006, 192]
+[  " paddingHorizontal"                           , 84098, 191]
+[  "createQueryBuilder"                           , 88085, 191]
+[  " OnTriggerEnter"                              , 64104, 190]
+[  " standardUserDefaults"                        , 55989, 189]
+[  "ContentLoaded"                                , 72759, 188]
+[  "ContentAlignment"                             , 75956, 188]
+[  "SupportedException"                           , 53597, 187]
+[  " setBackground"                               , 70655, 186]
+[  " JpaRepository"                               , 65618, 186]
+[  " HeaderComponent"                             , 93112, 184]
+[  "ParticleSystem"                               , 98454, 184]
+[  "WebpackPlugin"                                , 61186, 183]
+[  " IHttpActionResult"                           , 91457, 183]
+[  " forControlEvents"                            , 48801, 180]
+[  "UIBarButtonItem"                              , 75793, 179]
+[  " navigationController"                        , 65462, 179]
+[  " DropDownList"                                , 98755, 179]
+[  "readystatechange"                             , 59811, 178]
+[  " UITableViewController"                       , 96726, 176]
+[  " componentWillMount"                          , 78138, 175]
+[  " reuseIdentifier"                             , 52660, 175]
+[  " activeClassName"                             , 98122, 174]
+[  "WillDisappear"                                , 80491, 174]
+[  " removeObject"                                , 81622, 173]
+[  "ElementsByTagName"                            , 61289, 173]
+[  " PureComponent"                               , 89088, 173]
+[  " UIBarButtonItem"                             , 53502, 171]
+[  "AllowAnonymous"                               , 99109, 170]
+[  " isKindOfClass"                               , 48511, 169]
+[  " XCTAssertEqual"                              , 32550, 168]
+[  "wingConstants"                                , 65657, 168]
+[  "ActivityIndicatorView"                        , 89293, 168]
+[  " numberOfRowsInSection"                       , 50150, 168]
+[  "AttributedString"                             , 41805, 167]
+[  " integerValue"                                , 70613, 166]
+[  " setDefaultCloseOperation"                    , 89017, 166]
+[  "navigationBar"                                , 82653, 166]
+[  " XCTAssertTrue"                               , 79462, 165]
+[  " dequeueReusableCellWithIdentifier"           , 72805, 165]
+[  "SelfPermission"                               , 77855, 164]
+[  "UIImagePickerController"                      , 98196, 164]
+[  " QTableWidgetItem"                            , 90483, 163]
+[  " SpringApplication"                           , 91751, 162]
+[  " onChangeText"                                , 60868, 162]
+[  "AdminController"                              , 98751, 161]
+[  " HttpNotFound"                                , 73568, 160]
+[  "longleftrightarrow"                           , 87459, 159]
+[  "aterangepicker"                               , 85107, 159]
+[  " JsonRequestBehavior"                         , 69831, 158]
+[  " NSLayoutConstraint"                          , 59582, 158]
+[  "MERCHANTABILITY"                              , 64769, 157]
+[  " ArrayCollection"                             , 98502, 156]
+[  " DateFormatter"                               , 97953, 154]
+[  " NSLocalizedString"                           , 60272, 154]
+[  "DefaultCellStyle"                             , 85562, 153]
+[  " BrowserRouter"                               , 62485, 152]
+[  " numberOfRows"                                , 44910, 151]
+[  " UIImagePickerController"                     , 77733, 150]
+[  " presentViewController"                       , 82448, 149]
+[  "BrowserRouter"                                , 63188, 148]
+[  " UIEdgeInsets"                                , 69257, 148]
+[  "AdapterManager"                               , 96087, 148]
+[  " GetLastError"                                , 79362, 147]
+[  "BuilderInterface"                             , 78538, 145]
+[  "UsingEncoding"                                , 92721, 145]
+[  "abcdefghijklmnop"                             , 65102, 145]
+[  " stringByAppending"                           , 59261, 145]
+[  "ScrollIndicator"                              , 98437, 145]
+[  "SharedPointer"                                , 99054, 145]
+[  " DispatchQueue"                               , 47851, 143]
+[  "TextBoxColumn"                                , 15116, 140]
+[  " CircularProgress"                            , 78197, 140]
+[  "AnimationsModule"                             , 64179, 139]
+[  "shouldReceive"                                , 85708, 139]
+[  " pushViewController"                          , 62059, 137]
+[  " UICollectionViewCell"                        , 79613, 136]
+[  " dictionaryWith"                              , 84748, 136]
+[  " UINavigationController"                      , 66067, 134]
+[  " HttpStatusCodeResult"                        , 80202, 134]
+[  "stringLiteral"                                , 59305, 133]
+[  " AssemblyTitle"                               , 89590, 132]
+[  " htmlentities"                                , 62998, 132]
+[  " cancelButtonTitle"                           , 76902, 131]
+[  " bindActionCreators"                          , 78220, 131]
+[  " AssemblyDescription"                         , 91391, 129]
+[  "getActiveSheet"                               , 49732, 127]
+[  " removeFromSuperview"                         , 77180, 126]
+[  "StartupScript"                                , 97799, 126]
+[  " animateWithDuration"                         , 81456, 126]
+[  " viewWillAppear"                              , 65665, 125]
+[  "expectException"                              , 87714, 125]
+[  "InTheDocument"                                , 91863, 124]
+[  " colorWithRed"                                , 61180, 124]
+[  " otherButtonTitles"                           , 81187, 124]
+[  " initWithStyle"                               , 90188, 123]
+[  "TreeWidgetItem"                               , 95321, 122]
+[  " AssemblyProduct"                             , 92435, 121]
+[  "ControlEvents"                                , 44949, 121]
+[  " AssemblyFileVersion"                         , 98143, 120]
+[  "StackNavigator"                               , 85317, 118]
+[  "objectManager"                                , 98047, 117]
+[  " UITableViewDataSource"                       , 79768, 115]
+[  " didReceiveMemoryWarning"                     , 38141, 115]
+[  " CLLocationCoordinate"                        , 97363, 115]
+[  "ReuseIdentifier"                              , 50885, 113]
+[  " NSAttributedString"                          , 93237, 112]
+[  " didSelectRowAtIndexPath"                     , 95607, 111]
+[  " systemFontOfSize"                            , 51816, 109]
+[  "NSNotification"                               , 42813, 109]
+[  "CriticalSection"                              , 85687, 107]
+[  "INavigationController"                        , 51217, 107]
+[  " buttonWithType"                              , 83824, 106]
+[  " UITableViewDelegate"                         , 82057, 104]
+[  " kennenlernen"                                , 88862, 104]
+[  " initWithNibName"                             , 96708, 103]
+[  " UIAlertController"                           , 53255, 101]
+[  "MBProgressHUD"                                , 91273, 100]
+[  "IntervalSince"                                , 88974, 99]
+[  " dataGridViewTextBoxColumn"                   , 94425, 98]
+[  " AssemblyCopyright"                           , 94017, 98]
+[  "ChildScrollView"                              , 87779, 97]
+[  " NotificationCenter"                          , 83335, 97]
+[  " UIAlertAction"                               , 55610, 96]
+[  "yntaxException"                               , 76493, 95]
+[  " AssemblyCompany"                             , 93891, 94]
+[  " createStackNavigator"                        , 98820, 94]
+[  "istringstream"                                , 82416, 93]
+[  "andFilterWhere"                               , 84697, 93]
+[  " stripslashes"                                , 63223, 90]
+[  "createCommand"                                , 97380, 88]
+[  " RuntimeObject"                               , 53659, 88]
+[  "HTTPRequestOperation"                         , 93787, 88]
+[  " PropelException"                             , 85293, 87]
+[  "ActionCreators"                               , 69759, 86]
+[  " STDMETHODCALLTYPE"                           , 50125, 85]
+[  "RuntimeObject"                                , 61062, 85]
+[  " forIndexPath"                                , 98656, 84]
+[  "navbarSupportedContent"                       , 75722, 84]
+[  "OptionsItemSelected"                          , 41781, 83]
+[  " setBackgroundImage"                          , 95266, 79]
+[  "apGestureRecognizer"                          , 54614, 79]
+[  "GenerationStrategy"                           , 76545, 79]
+[  "HeaderInSection"                              , 95893, 79]
+[  " numberWithInt"                               , 98277, 76]
+[  "AllArgsConstructor"                           , 64522, 76]
+[  " useNewUrlParser"                             , 92630, 75]
+[  "ABCDEFGHIJKLMNOP"                             , 74445, 75]
+[  "DispatchToProps"                              , 30564, 75]
+[  " RuntimeMethod"                               , 38778, 75]
+[  " UserDefaults"                                , 61792, 74]
+[  "CloseOperation"                               , 55159, 74]
+[  "InteractionEnabled"                           , 61230, 74]
+[  " HtmlWebpackPlugin"                           , 92415, 73]
+[  "igrationBuilder"                              , 52255, 73]
+[  " UITapGestureRecognizer"                      , 79885, 72]
+[  " UIPickerView"                                , 92231, 72]
+[  "WritableDatabase"                             , 83899, 72]
+[  " componentDid"                                , 23807, 71]
+[  " navigationOptions"                           , 99121, 70]
+[  " setTitleColor"                               , 77138, 70]
+[  " stringstream"                                , 80927, 70]
+[  "htmlspecialchars"                             , 79596, 68]
+[  " alertController"                             , 72681, 68]
+[  " UIStoryboard"                                , 91942, 67]
+[  "ibBundleOrNil"                                , 89390, 67]
+[  " NSTextAlignment"                             , 74234, 67]
+[  "BootApplication"                              , 39541, 66]
+[  "latesAutoresizingMaskIntoConstraints"         , 60107, 66]
+[  " glGetUniformLocation"                        , 92720, 65]
+[  " fontWithName"                                , 97118, 64]
+[  " stringByAppendingString"                     , 96510, 64]
+[  " AssemblyTrademark"                           , 95827, 63]
+[  " awakeFromNib"                                , 74949, 63]
+[  "OptionsResolver"                              , 63235, 61]
+[  "TransparentColor"                             , 92685, 61]
+[  "NavigatorMove"                                , 91003, 61]
+[  " gettimeofday"                                , 88579, 60]
+[  "TouchUpInside"                                , 55092, 57]
+[  "stantiateViewController"                      , 82783, 56]
+[  "NSAttributedString"                           , 95561, 54]
+[  " forCellReuseIdentifier"                      , 93856, 54]
+[  "UIStoryboardSegue"                            , 89450, 54]
+[  " RoundedRectangle"                            , 77715, 52]
+[  "UIAlertAction"                                , 56439, 52]
+[  "interopRequire"                               , 32899, 52]
+[  "autoreleasepool"                              , 87422, 51]
+[  " cellForRowAt"                                , 82655, 51]
+[  " preferredStyle"                              , 63437, 50]
+[  "MASConstraintMaker"                           , 58974, 50]
+[  " queryInterface"                              , 94915, 50]
+[  "NoArgsConstructor"                            , 65024, 49]
+[  "queueReusableCell"                            , 38559, 48]
+[  "ReceiveMemoryWarning"                         , 24087, 48]
+[  "AntiForgeryToken"                             , 50419, 48]
+[  " DataGridViewCellStyle"                       , 75263, 45]
+[  " applicationWill"                             , 92367, 44]
+[  "SetBranchAddress"                             , 65134, 44]
+[  "exampleInputEmail"                            , 82001, 43]
+[  "getClientOriginal"                            , 74885, 43]
+[  "navbarDropdown"                               , 98311, 43]
+[  " UIStoryboardSegue"                           , 69160, 40]
+[  "AutoresizingMask"                             , 57349, 37]
+[  "ICollectionView"                              , 22719, 37]
+[  "AsStringAsync"                                , 86717, 35]
+[  "CanBeConverted"                               , 78872, 35]
+[  " destinationViewController"                   , 91471, 34]
+[  "pushViewController"                           , 97271, 33]
+[  "redirectToRoute"                              , 89279, 33]
+[  " prostituerade"                               , 42723, 32]
+[  " NSStringFromClass"                           , 86361, 31]
+[  "queueReusable"                                , 37185, 30]
+[  " sourceMapping"                               , 44584, 29]
+[  "ableViewController"                           , 94906, 29]
+[  "upportInitialize"                             , 12682, 28]
+[  " dequeueReusableCell"                         , 65382, 27]
+[  "UIApplicationDelegate"                        , 92657, 25]
+[  "AutoresizingMaskIntoConstraints"              , 58491, 22]
+[  "HeadersHeight"                                , 45141, 20]
+[  " QStringLiteral"                              , 70697, 20]
+[  "exampleModalLabel"                            , 82403, 19]
+[  "VisualStyleBackColor"                         , 13715, 19]
+[  "importDefault"                                , 86479, 19]
+[  "ERCHANTABILITY"                               , 7643, 18]
+[  "InternalEnumerator"                           , 57094, 18]
+[  " prostituerte"                                , 50617, 18]
+[  " BlueprintReadOnly"                           , 93125, 18]
+[  "chartInstance"                                , 96246, 17]
+[  "CppCodeGenWriteBarrier"                       , 38936, 16]
+[  "FunctionFlags"                                , 55153, 14]
+[  " ForCanBeConvertedToForeach"                  , 79271, 11]
+[  "MemoryWarning"                                , 24082, 10]
+[  "CppTypeDefinition"                            , 65135, 10]
+[  " RouteServiceProvider"                        , 85702, 8]
+[  "ImageRelation"                                , 90489, 8]
+[  "SupportedContent"                             , 75584, 7]
+[  "matchCondition"                               , 23972, 7]
+[  " InternalEnumerator"                          , 75609, 6]
+[  "ParallelGroup"                                , 18420, 6]
+[  "HeadersHeightSizeMode"                        , 46119, 5]
+[  "CallableWrapper"                              , 84872, 4]
+[  " HinderedRotor"                               , 95065, 4]
+[  "PropertyParams"                               , 85884, 4]
+[  "StoryboardSegue"                              , 49369, 4]
+[  "SequentialGroup"                              , 23394, 3]
+[  "SmartyHeaderCode"                             , 98068, 2]
+[  "BracketAccess"                                , 93552, 2]
+[  "methodVisitor"                                , 24962, 1]
+[  "ConstraintMaker"                              , 58739, 1]
+[  "GameObjectWithTag"                            , 84962, 1]

qwen__1000__20000/tokens_kept__ascii_long.jsonl

@@ -0,0 +1,933 @@
+[  "----------------------------------------------------------------"                                                                                 , 3513, 148442]
+[  "................................................................"                                                                                 , 42269, 110580]
+[  "................................"             , 16562, 92961]
+[  "................"                             , 7895, 65071]
+[  "........................"                     , 56241, 63140]
+[  " ................................................................"                                                                                , 86072, 61645]
+[  " ----------------------------------------------------------------"                                                                                , 8466, 38421]
+[  " ................................"            , 81372, 33893]
+[  ".springframework"                             , 4029, 31488]
+[  " ================================================================="                                                                               , 16168, 29960]
+[  "(BuildContext"                                , 20941, 21375]
+[  "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"                                                                                 , 79083, 20525]
+[  " ------------------------------------------------"                                                                                                , 18036, 19699]
+[  " ----------------"                            , 14393, 19050]
+[  "================================================================"                                                                                 , 8152, 18550]
+[  "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"             , 24616, 18022]
+[  "            \n"                               , 3374, 16956]
+[  " ================================================="                                                                                               , 28585, 15867]
+[  "------------------------------------------------"                                                                                                 , 9229, 14193]
+[  ".assertEquals"                                , 14745, 13854]
+[  "--------------------------------------------------------------------------------"                                                                 , 43449, 12751]
+[  "----------------"                             , 771, 11636]
+[  "----------------------------------------------------------------------------"                                                                     , 22349, 11578]
+[  " ................"                            , 88914, 10498]
+[  " ================================================================================="                                                               , 65900, 10239]
+[  "------------\n"                               , 56339, 8999]
+[  ".printStackTrace"                             , 8228, 8867]
+[  "(savedInstanceState"                          , 14998, 8842]
+[  ".Configuration"                               , 17334, 8709]
+[  "================================"             , 3058, 8611]
+[  "------------------------------------------------------------------------------------------------"                                                 , 98474, 8486]
+[  " ================================="           , 35774, 8395]
+[  ".getElementById"                              , 4749, 7746]
+[  "--------------------------------"             , 1408, 7602]
+[  ".setOnClickListener"                          , 13359, 7363]
+[  ".ComponentModel"                              , 7104, 7318]
+[  ".preventDefault"                              , 12191, 7100]
+[  ".findViewById"                                , 11638, 6901]
+[  "                \n"                           , 6360, 6820]
+[  ".elasticsearch"                               , 53078, 6685]
+[  ".Serialization"                               , 27255, 6676]
+[  "_configuration"                               , 35726, 6483]
+[  "-inflammatory"                                , 66495, 6067]
+[  " --------------------------------"            , 19728, 5972]
+[  "-------------"                                , 19528, 5790]
+[  "================================================"                                                                                                 , 18782, 5754]
+[  "=============="                               , 38603, 5672]
+[  "\tassertEquals"                               , 21729, 5366]
+[  ".IsNullOrEmpty"                               , 14984, 5344]
+[  ".configuration"                               , 31453, 5144]
+[  ".createElement"                               , 9732, 5095]
+[  "________________"                             , 7686, 4805]
+[  ";;;;;;;;;;;;;;;;"                             , 71409, 4719]
+[  "==============="                              , 44908, 4697]
+[  ".RegisterType"                                , 81703, 4584]
+[  ".ResponseWriter"                              , 37508, 4309]
+[  "                    \n"                       , 10683, 4227]
+[  ".setDescription"                              , 65478, 4179]
+[  ".getAttribute"                                , 19146, 4174]
+[  ".addEventListener"                            , 10634, 4123]
+[  "\n\n\n\n\n\n\n\n\n\n\n\n\n\n"                 , 66376, 4104]
+[  "----------------------------"                 , 27814, 4093]
+[  "---------------"                              , 38645, 3939]
+[  "============="                                , 35366, 3841]
+[  ".currentTimeMillis"                           , 23813, 3728]
+[  "..............."                              , 135535, 3725]
+[  ".AnchorStyles"                                , 13926, 3723]
+[  "________________________________"             , 17466, 3713]
+[  " ------------"                                , 32558, 3611]
+[  ".recyclerview"                                , 35729, 3553]
+[  "_notification"                                , 34296, 3506]
+[  ".LayoutInflater"                              , 26246, 3485]
+[  ".setAttribute"                                , 13094, 3433]
+[  " ========================================================================="                                                                       , 38368, 3398]
+[  " ================="                           , 24818, 3384]
+[  ".querySelector"                               , 8583, 3362]
+[  "(milliseconds"                                , 86146, 3349]
+[  ".assertNotNull"                               , 38722, 3332]
+[  "(HaveOccurred"                                , 83844, 3223]
+[  ".EventHandler"                                , 10537, 3154]
+[  "\n            \n"                             , 18574, 3070]
+[  ".serialization"                               , 76845, 3001]
+[  "--------------"                               , 42091, 2963]
+[  ".presentation"                                , 51996, 2920]
+[  ".Serializable"                                , 23713, 2891]
+[  ".setVisibility"                               , 16542, 2847]
+[  ".LayoutParams"                                , 29219, 2829]
+[  ".AppCompatActivity"                           , 23762, 2805]
+[  ".OnClickListener"                             , 15447, 2797]
+[  ".tableLayoutPanel"                            , 22149, 2717]
+[  ".ServiceModel"                                , 83158, 2601]
+[  ".GetComponent"                                , 13952, 2581]
+[  ".setObjectName"                               , 48092, 2563]
+[  "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"             , 26921, 2504]
+[  " --------------------------------------------------------------------------------"                                                                , 39373, 2496]
+[  " ----------------------------------------------------------------------------------------------------------------"                                , 66207, 2487]
+[  ".querySelectorAll"                            , 28444, 2485]
+[  "_subscription"                                , 59838, 2434]
+[  "/apimachinery"                                , 72644, 2397]
+[  ".LoggerFactory"                               , 35828, 2380]
+[  ".dependencies"                                , 85059, 2377]
+[  ".RecyclerView"                                , 35633, 2357]
+[  "================"                             , 1518, 2348]
+[  ".Notification"                                , 49329, 2329]
+[  " '../../../../"                               , 37449, 2314]
+[  "(configuration"                               , 48724, 2313]
+[  ".IsNullOrWhiteSpace"                          , 36575, 2252]
+[  "_distribution"                                , 41465, 2252]
+[  ".assertRaises"                                , 36049, 2211]
+[  " __________________"                          , 43841, 2169]
+[  ".detectChanges"                               , 36561, 2168]
+[  ".CodeAnalysis"                                , 58505, 2153]
+[  " développement"                               , 81520, 2149]
+[  ".notification"                                , 45803, 2107]
+[  ".documentation"                               , 82769, 2089]
+[  ".Authentication"                              , 40673, 2080]
+[  "_confirmation"                                , 37681, 2078]
+[  ".VisualStudio"                                , 51681, 2067]
+[  ".ISupportInitialize"                          , 12737, 2062]
+[  "################################################################################"                                                                 , 40486, 2057]
+[  "################################################################"                                                                                 , 13067, 2050]
+[  "            \r\n"                             , 16075, 2049]
+[  "(MethodImplOptions"                           , 64600, 2042]
+[  ".spaceBetween"                                , 83765, 2036]
+[  ".UseVisualStyleBackColor"                     , 13777, 2026]
+[  " investigación"                               , 97524, 2017]
+[  " ----------------------------------------------------------------------------"                                                                    , 23852, 2016]
+[  "############################################################################"                                                                     , 32041, 2004]
+[  "\t\t\t\t\t\t\t\t\t\t\t\t\t"                   , 17546, 1996]
+[  ".addActionListener"                           , 22155, 1979]
+[  " ************************************************************************"                                                                        , 10534, 1967]
+[  ".StatusInternalServerError"                   , 66760, 1965]
+[  ".subscription"                                , 73393, 1952]
+[  "--------------------------------------------------------------------------\n"                                                                     , 25885, 1948]
+[  "================================================================================"                                                                 , 63394, 1936]
+[  "################################"             , 5026, 1925]
+[  ".InteropServices"                             , 38046, 1916]
+[  ".currentState"                                , 69560, 1872]
+[  ".InvariantCulture"                            , 44603, 1871]
+[  ".EntityFrameworkCore"                         , 27820, 1864]
+[  ".ObjectMapper"                                , 89408, 1845]
+[  "                        \n"                   , 15865, 1830]
+[  ".nativeElement"                               , 48829, 1830]
+[  "             \n"                              , 28247, 1809]
+[  ".GraphicsUnit"                                , 12174, 1801]
+[  ".backgroundColor"                             , 15451, 1799]
+[  ".StatusBadRequest"                            , 69497, 1795]
+[  ".ResumeLayout"                                , 15535, 1792]
+[  ".SuspendLayout"                               , 15541, 1786]
+[  ".hasOwnProperty"                              , 24920, 1771]
+[  "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"                                                                                 , 79518, 1767]
+[  ".authentication"                              , 45186, 1758]
+[  "              \n"                             , 26285, 1733]
+[  "################"                             , 2431, 1729]
+[  "_authentication"                              , 89465, 1727]
+[  " ------------------------------------------------------------------------------------------------"                                                , 59809, 1723]
+[  ".Globalization"                               , 41069, 1701]
+[  ".getParameter"                                , 18134, 1691]
+[  ".getColumnIndex"                              , 40414, 1688]
+[  ".githubusercontent"                           , 50927, 1644]
+[  ".getComponent"                                , 61857, 1643]
+[  ".organization"                                , 69224, 1621]
+[  "_transactions"                                , 68182, 1611]
+[  "/********************************************************"                                                                                        , 66159, 1584]
+[  "#----------------------------------------------------------------------------"                                                                    , 60915, 1581]
+[  " ################"                            , 37615, 1580]
+[  "_relationship"                                , 69031, 1566]
+[  ".FirstOrDefault"                              , 29620, 1558]
+[  "\t\t\t\t\t\t\t\t\t\t\t\t\t\t"                 , 23292, 1548]
+[  "@SuppressWarnings"                            , 34432, 1544]
+[  ".ArgumentParser"                              , 37966, 1542]
+[  " //----------------------------------------------------------------"                                                                              , 35619, 1534]
+[  " particulière"                                , 136547, 1521]
+[  ".addAttribute"                                , 25234, 1520]
+[  "@RequestMapping"                              , 20378, 1513]
+[  "_organization"                                , 82352, 1512]
+[  "_registration"                                , 49101, 1497]
+[  "/documentation"                               , 85586, 1494]
+[  ".isDebugEnabled"                              , 78367, 1492]
+[  ".JsonProperty"                                , 74469, 1489]
+[  "_dependencies"                                , 71841, 1485]
+[  ".Authorization"                               , 53786, 1485]
+[  "(UnmanagedType"                               , 68751, 1482]
+[  ".ToolStripMenuItem"                           , 22261, 1472]
+[  " **************************************************************************"                                                                      , 17535, 1449]
+[  ".authorization"                               , 63988, 1440]
+[  ".getConnection"                               , 30454, 1438]
+[  ".createParallelGroup"                         , 18439, 1438]
+[  " ################################"            , 54301, 1435]
+[  ".checkNotNull"                                , 97209, 1425]
+[  "---------------\n"                            , 62732, 1422]
+[  ".readFileSync"                                , 45346, 1400]
+[  "(grammarAccess"                               , 91517, 1391]
+[  ".Infrastructure"                              , 40435, 1388]
+[  "_notifications"                               , 67546, 1387]
+[  " construcción"                                , 140155, 1350]
+[  "(CancellationToken"                           , 81349, 1332]
+[  ".getElementsByTagName"                        , 29715, 1325]
+[  " =============================================================="                                                                                  , 70662, 1321]
+[  "-registration"                                , 98698, 1320]
+[  ".DateTimeField"                               , 34237, 1314]
+[  "(Configuration"                               , 45443, 1310]
+[  "################################################################################\n"                                                               , 90730, 1302]
+[  "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"               , 30711, 1292]
+[  ".equalsIgnoreCase"                            , 30494, 1289]
+[  ".Organization"                                , 94048, 1274]
+[  "-----------\n\n"                              , 54210, 1273]
+[  "(notification"                                , 54880, 1272]
+[  ".currentThread"                               , 48629, 1260]
+[  "################################################"                                                                                                 , 29854, 1255]
+[  ".notifications"                               , 74915, 1253]
+[  ".isNullOrEmpty"                               , 87644, 1230]
+[  "#----------------------------------------------------------------"                                                                                , 75510, 1230]
+[  ".experimental"                                , 87447, 1229]
+[  ".authenticate"                                , 48798, 1225]
+[  ".startActivity"                               , 57331, 1221]
+[  ".subscriptions"                               , 85147, 1219]
+[  "_NOTIFICATION"                                , 54241, 1217]
+[  "________________________________________________________________"                                                                                 , 46932, 1209]
+[  "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"             , 65648, 1208]
+[  "(PropertyName"                                , 89929, 1204]
+[  " -----------------------------------------------------------------------------\n"                                                                 , 51814, 1199]
+[  "********************************"             , 1689, 1193]
+[  "###############################################################################\n"                                                                , 79448, 1189]
+[  ".IntegerField"                                , 33618, 1185]
+[  ".setBackground"                               , 21471, 1175]
+[  "--------------\n"                             , 44955, 1174]
+[  "~~~~~~~~~~~~~~~~"                             , 32543, 1174]
+[  "****************************************************************"                                                                                 , 4077, 1170]
+[  ".dispatchEvent"                               , 62082, 1163]
+[  ".getAbsolutePath"                             , 52954, 1162]
+[  "/Subthreshold"                                , 57844, 1152]
+[  "               \n"                            , 26546, 1149]
+[  ".DeserializeObject"                           , 50553, 1140]
+[  ".CompilerServices"                            , 43295, 1137]
+[  " ****************************************************************"                                                                                , 22292, 1137]
+[  " \"../../../../"                              , 63392, 1135]
+[  ".setImageResource"                            , 41913, 1133]
+[  "        \n        \n"                         , 16885, 1131]
+[  " __________________________________"          , 81008, 1126]
+[  "                \r\n"                         , 27365, 1124]
+[  "(HttpServletRequest"                          , 22861, 1123]
+[  ".splitContainer"                              , 38701, 1122]
+[  "#================================================================"                                                                                , 72004, 1120]
+[  ".componentInstance"                           , 38276, 1115]
+[  "----------------------------------------------------------------------"                                                                           , 53264, 1106]
+[  ".GetEnumerator"                               , 41404, 1105]
+[  "//----------------------------------------------------------------"                                                                               , 9888, 1096]
+[  ".prepareStatement"                            , 29704, 1096]
+[  ".OrdinalIgnoreCase"                           , 69649, 1093]
+[  " ****************************************************************************"                                                                    , 28644, 1088]
+[  " organización"                                , 140051, 1083]
+[  "\n                \n"                         , 34135, 1079]
+[  "****************"                             , 893, 1068]
+[  ".Experimental"                                , 90320, 1065]
+[  "(propertyName"                                , 69196, 1062]
+[  "----------------------------------------------------------------------\n"                                                                         , 62117, 1059]
+[  ".stopPropagation"                             , 42084, 1056]
+[  ".onViewCreated"                               , 93527, 1046]
+[  "(HttpStatusCode"                              , 43722, 1043]
+[  "\t               "                            , 20835, 1040]
+[  " propriétaire"                                , 143448, 1038]
+[  ".setTextColor"                                , 46732, 1037]
+[  ".MILLISECONDS"                                , 88152, 1023]
+[  ".currentTarget"                               , 37010, 1019]
+[  ".AutoScaleMode"                               , 19422, 1016]
+[  "////////////////////////////////////////////////////////////////////////////////"                                                                 , 28226, 1015]
+[  ".LinearLayoutManager"                         , 60128, 1006]
+[  "/configuration"                               , 89442, 1006]
+[  "-------------\n"                              , 65810, 1004]
+[  "<KeyValuePair"                                , 82958, 1003]
+[  " ------------------------------------------------------------"                                                                                    , 86632, 999]
+[  ".CurrentCulture"                              , 80066, 996]
+[  ".removeEventListener"                         , 38149, 987]
+[  "                            \n"               , 25435, 986]
+[  ".SerializeObject"                             , 56036, 980]
+[  ".dataGridView"                                , 20402, 980]
+[  ".defineProperty"                              , 24802, 980]
+[  ".RegularExpressions"                          , 47543, 978]
+[  "                  \n"                         , 36577, 978]
+[  ".DataAnnotations"                             , 27694, 975]
+[  ".createSequentialGroup"                       , 23416, 965]
+[  "_representation"                              , 94263, 964]
+[  ".requireNonNull"                              , 72660, 962]
+[  ".defaultValue"                                , 98865, 961]
+[  ".getJSONObject"                               , 47841, 960]
+[  "_verification"                                , 84245, 958]
+[  ".newArrayList"                                , 87931, 948]
+[  ".DependencyInjection"                         , 58473, 944]
+[  ".getSystemService"                            , 68720, 940]
+[  ".SystemColors"                                , 24091, 937]
+[  ".AttributeSet"                                , 81801, 935]
+[  ".repositories"                                , 62907, 931]
+[  ".minecraftforge"                              , 54075, 927]
+[  "****************************************************************************"                                                                     , 33518, 925]
+[  ".PerformLayout"                               , 22778, 922]
+[  ".intersection"                                , 97075, 908]
+[  "########################################################################"                                                                         , 69061, 904]
+[  ".CreateInstance"                              , 77653, 901]
+[  ".LocalDateTime"                               , 78310, 896]
+[  ".SelectedIndex"                               , 24642, 894]
+[  ".getSimpleName"                               , 60673, 892]
+[  ".SuppressLint"                                , 87691, 886]
+[  ".executeUpdate"                               , 38912, 885]
+[  "_classification"                              , 80872, 885]
+[  ".AppendFormat"                                , 60467, 882]
+[  " comunicación"                                , 138332, 879]
+[  ".showMessageDialog"                           , 32668, 879]
+[  ".AddWithValue"                                , 23764, 876]
+[  ".documentElement"                             , 43540, 873]
+[  "********************"                         , 31947, 873]
+[  ".SQLException"                                , 41821, 869]
+[  ".ContextCompat"                               , 93564, 867]
+[  ".ReadAsStringAsync"                           , 88136, 864]
+[  ".SelectedItem"                                , 34517, 864]
+[  ".LogInformation"                              , 89920, 860]
+[  ".BooleanField"                                , 43743, 860]
+[  ".registration"                                , 73836, 858]
+[  "%%%%%%%%%%%%%%%%"                             , 16395, 855]
+[  "                      \n"                     , 52720, 853]
+[  ".setAlignment"                                , 77669, 851]
+[  ".Cryptography"                                , 53858, 850]
+[  "            \n\n"                             , 39865, 848]
+[  "--------------------"                         , 97704, 844]
+[  "==============\n"                             , 40722, 840]
+[  ".executeQuery"                                , 29946, 839]
+[  ".SimpleDateFormat"                            , 45919, 836]
+[  "************************"                     , 13439, 831]
+[  ".autoconfigure"                               , 46121, 824]
+[  ".localStorage"                                , 48107, 824]
+[  ".SharedPreferences"                           , 62977, 823]
+[  "/notification"                                , 96542, 820]
+[  " ----------------------------------------------------------------------------\n"                                                                  , 63538, 819]
+[  ".createComponent"                             , 40017, 817]
+[  ".assertAlmostEqual"                           , 87600, 817]
+[  ".getResources"                                , 48528, 815]
+[  ".HttpServletRequest"                          , 33212, 811]
+[  ".ExecuteNonQuery"                             , 32910, 808]
+[  "=-=-=-=-=-=-=-=-"                             , 95523, 808]
+[  "=============\n"                              , 47616, 804]
+[  ".instructions"                                , 80599, 802]
+[  " veröffentlicht"                              , 141871, 802]
+[  ".setSelection"                                , 68810, 802]
+[  ".transactions"                                , 80262, 798]
+[  ".ActionListener"                              , 30964, 795]
+[  "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"           , 48180, 794]
+[  " ********************************"            , 22958, 792]
+[  "############################################################"                                                                                     , 97844, 792]
+[  "########################"                     , 84299, 791]
+[  " ################################################################"                                                                                , 48497, 791]
+[  "                 \n"                          , 41636, 789]
+[  " ################################################"                                                                                                , 67161, 789]
+[  "<IActionResult"                               , 35291, 784]
+[  " //------------------------------------------------"                                                                                              , 66004, 782]
+[  " ########################"                    , 96515, 777]
+[  "////////////////"                             , 1970, 776]
+[  "************************************************************************"                                                                         , 5596, 775]
+[  "\n                    \n"                     , 59101, 774]
+[  ".errorMessage"                                , 57407, 772]
+[  "------------------------------"               , 85126, 771]
+[  ".beginTransaction"                            , 55230, 770]
+[  "===============\n"                            , 57879, 768]
+[  "****************************"                 , 35766, 765]
+[  " =============================================================================\n"                                                                 , 82049, 764]
+[  "_requirements"                                , 89632, 761]
+[  " ************************"                    , 54176, 761]
+[  "@RestController"                              , 41552, 760]
+[  "(getApplicationContext"                       , 31567, 760]
+[  ".PreparedStatement"                           , 71343, 759]
+[  "@Configuration"                               , 58446, 757]
+[  "(errorMessage"                                , 70130, 757]
+[  " características"                             , 92454, 750]
+[  "////////////////////////////////////////////////////////////////"                                                                                 , 9822, 750]
+[  ".getDescription"                              , 57464, 748]
+[  "\tassertNotNull"                              , 88121, 742]
+[  "/authentication"                              , 90577, 740]
+[  ".singletonList"                               , 93938, 740]
+[  "-notification"                                , 90724, 739]
+[  ".Repositories"                                , 46658, 736]
+[  ".fromCharCode"                                , 68865, 736]
+[  ".RequestMapping"                              , 37372, 735]
+[  ".setParameter"                                , 55192, 731]
+[  ".DataGridView"                                , 11914, 730]
+[  ".relationship"                                , 77917, 729]
+[  "(LayoutInflater"                              , 54403, 726]
+[  ".getTransaction"                              , 59280, 726]
+[  ".getSelection"                                , 51034, 726]
+[  ".onOptionsItemSelected"                       , 68467, 725]
+[  ".getBoundingClientRect"                       , 74305, 723]
+[  ".GetDirectoryName"                            , 91465, 722]
+[  "########################################################"                                                                                         , 90038, 721]
+[  "_CONFIGURATION"                               , 70752, 719]
+[  "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"         , 58667, 719]
+[  "\t\t               "                          , 46256, 718]
+[  "                                \n"           , 33641, 718]
+[  ".ComponentPlacement"                          , 34573, 716]
+[  ".Abstractions"                                , 93970, 715]
+[  ".getStringExtra"                              , 57959, 714]
+[  ".OutputStream"                                , 64300, 713]
+[  " //////////////////////////////////////////////////////////////////////////"                                                                      , 71508, 712]
+[  ".DataGridViewTextBoxColumn"                   , 23228, 709]
+[  ".localPosition"                               , 50329, 708]
+[  ".ApplyResources"                              , 57273, 707]
+[  ".ErrorMessage"                                , 73628, 703]
+[  "_authenticated"                               , 64913, 701]
+[  " '../../../../../"                            , 94645, 700]
+[  "(Notification"                                , 96731, 698]
+[  ".insertBefore"                                , 57367, 696]
+[  ".getProperties"                               , 90802, 691]
+[  ".ContentAlignment"                            , 32594, 690]
+[  ".CreateDirectory"                             , 71297, 689]
+[  ".setForeground"                               , 39084, 689]
+[  ".PropertyType"                                , 95718, 688]
+[  "(DialogInterface"                             , 48937, 688]
+[  " ########################################################################"                                                                        , 93858, 687]
+[  ".isSuccessful"                                , 71699, 687]
+[  "_intersection"                                , 82558, 685]
+[  "//================================================================"                                                                               , 23161, 681]
+[  "(MainActivity"                                , 43471, 677]
+[  ".notifyDataSetChanged"                        , 56861, 676]
+[  ".MainActivity"                                , 93537, 674]
+[  "\tStringBuilder"                              , 50396, 674]
+[  "                   \n"                        , 40337, 670]
+[  ".getInputStream"                              , 52064, 670]
+[  ")findViewById"                                , 35289, 668]
+[  "============\n"                               , 44132, 664]
+[  "/*----------------------------------------------------------------"                                                                               , 29865, 663]
+[  ".addComponent"                                , 65309, 661]
+[  ".selectedIndex"                               , 56744, 660]
+[  "////////////////////////////////////////////////////////////////////////////"                                                                     , 27183, 658]
+[  " participación"                               , 140594, 657]
+[  ".BufferedReader"                              , 47590, 652]
+[  "////////////////////////////////"             , 3898, 651]
+[  ".setBackgroundColor"                          , 60137, 651]
+[  ".removeListener"                              , 98797, 648]
+[  ".LinearLayout"                                , 38073, 648]
+[  " ############"                                , 76527, 647]
+[  "(CharSequence"                                , 72646, 642]
+[  ".ExecuteReader"                               , 38089, 631]
+[  ".\n\n\n\n\n\n\n\n\n\n\n\n"                    , 85545, 631]
+[  "                       \n"                    , 51124, 629]
+[  "<HTMLInputElement"                            , 93173, 624]
+[  " ********************************************************"                                                                                        , 65195, 624]
+[  "[SerializeField"                              , 64842, 619]
+[  "                          \n"                 , 69877, 614]
+[  "\t\t           "                              , 31044, 611]
+[  ".AddSingleton"                                , 96797, 610]
+[  "\n        \n        \n"                       , 79133, 609]
+[  "############################"                 , 88100, 609]
+[  " ############################################################################"                                                                    , 86200, 608]
+[  "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"       , 73531, 607]
+[  "\t                   "                        , 34513, 607]
+[  "/*******************************************************************************\n"                                                               , 36713, 606]
+[  ".bootstrapcdn"                                , 32393, 603]
+[  " //////////////////"                          , 46470, 603]
+[  "/****************************************************************"                                                                                , 20159, 603]
+[  "(getResources"                                , 47843, 600]
+[  ".configureTestingModule"                      , 37618, 599]
+[  " ----------------------------------------------------------------------\n"                                                                        , 89694, 599]
+[  ".AddComponent"                                , 67351, 598]
+[  "//------------------------------------------------------------------------------\n"                                                               , 45814, 596]
+[  "                                                \n"                                                                                               , 94947, 596]
+[  "########################################"     , 82578, 595]
+[  "<ActionResult"                                , 57364, 592]
+[  ".getElementsByClassName"                      , 33736, 589]
+[  ".GetCurrentMethod"                            , 90288, 589]
+[  ".SaveChangesAsync"                            , 47907, 585]
+[  ".DropDownList"                                , 75579, 582]
+[  "_capabilities"                                , 92092, 581]
+[  ".setStyleSheet"                               , 81727, 580]
+[  ".writeFileSync"                               , 76867, 579]
+[  ".WriteAllText"                                , 99022, 577]
+[  ".PackageManager"                              , 84816, 574]
+[  "+-+-+-+-+-+-+-+-"                             , 72713, 572]
+[  ".djangoproject"                               , 63521, 566]
+[  ".HttpServletResponse"                         , 39248, 566]
+[  " -------------------------------------------------------------------------\n"                                                                     , 80550, 563]
+[  ".getStatusCode"                               , 93761, 563]
+[  ".TableLayoutPanel"                            , 58615, 562]
+[  "********************************************************"                                                                                         , 53147, 562]
+[  "                     \n"                      , 55447, 561]
+[  "      \n      \n"                             , 93004, 557]
+[  ".visitVarInsn"                                , 55165, 555]
+[  "//--------------------------------------------------------------------------------"                                                               , 50561, 553]
+[  ".FileNotFoundException"                       , 61549, 552]
+[  "                                            \n"                                                                                                   , 77993, 552]
+[  ".parentElement"                               , 55265, 550]
+[  ".responseText"                                , 38017, 549]
+[  ".Transactional"                               , 56787, 549]
+[  "++++++++++++++++++++++++++++++++"             , 88155, 549]
+[  "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"     , 96991, 546]
+[  "@implementation"                              , 16578, 543]
+[  "_instructions"                                , 82427, 542]
+[  ".ColumnStyles"                                , 71388, 541]
+[  ".createStatement"                             , 50306, 540]
+[  ".ConnectionString"                            , 67478, 538]
+[  "****************************************"     , 40072, 536]
+[  ".getSharedPreferences"                        , 98432, 535]
+[  ".communication"                               , 89845, 535]
+[  ".DoesNotExist"                                , 73141, 535]
+[  "            \t"                               , 35557, 531]
+[  ".ForegroundColor"                             , 73919, 529]
+[  ".SelectSingleNode"                            , 82425, 526]
+[  ".ColumnHeader"                                , 44831, 526]
+[  ".FlatAppearance"                              , 43620, 525]
+[  "_SelectedIndexChanged"                        , 47040, 523]
+[  ".setPreferredSize"                            , 69783, 522]
+[  ".InputStreamReader"                           , 56591, 522]
+[  ".checkSelfPermission"                         , 86883, 516]
+[  ".FormattingEnabled"                           , 43522, 512]
+[  ".FileOutputStream"                            , 70537, 511]
+[  ".SerializedName"                              , 76091, 510]
+[  ".RemoveEmptyEntries"                          , 90772, 510]
+[  "////////////////////////////////////////////////////////"                                                                                         , 60908, 509]
+[  "/************************************************************************"                                                                        , 11370, 509]
+[  ".FileInputStream"                             , 64372, 508]
+[  ".RestController"                              , 52352, 506]
+[  "////////////////////////////////////////////////////////////////////////"                                                                         , 35109, 506]
+[  "@JsonProperty"                                , 97858, 506]
+[  "************************************************"                                                                                                 , 46576, 506]
+[  " ---------------------------------------------------------------------------\n"                                                                   , 78757, 498]
+[  ".createObject"                                , 81041, 497]
+[  "(localStorage"                                , 38535, 492]
+[  " --------------------------------------------------------------------------\n"                                                                    , 97942, 491]
+[  ".offsetHeight"                                , 76243, 490]
+[  ".setCellValue"                                , 63886, 490]
+[  ".assertIsInstance"                            , 83930, 489]
+[  ".AutoScaleDimensions"                         , 31373, 482]
+[  "(resourceName"                                , 71077, 481]
+[  "                         \n"                  , 75228, 479]
+[  " ****************"                            , 46846, 479]
+[  " --------------------"                        , 90781, 479]
+[  ".ToolStripItem"                               , 45615, 474]
+[  ".DialogResult"                                , 56434, 472]
+[  ".DialogInterface"                             , 71001, 470]
+[  ".requestFocus"                                , 72580, 468]
+[  ".getSelectedItem"                             , 54772, 468]
+[  " /****************************************************************"                                                                               , 98989, 468]
+[  ".InnerException"                              , 84146, 465]
+[  ".ModelSerializer"                             , 72649, 457]
+[  " -------------"                               , 96803, 455]
+[  ".getOutputStream"                             , 66116, 449]
+[  " Unterstützung"                               , 141786, 447]
+[  ".OnItemClickListener"                         , 79827, 445]
+[  "//----------------------------------------------------------------------------"                                                                   , 18012, 444]
+[  "                    \r\n"                     , 45575, 443]
+[  ".preprocessing"                               , 58964, 443]
+[  "-transitional"                                , 91330, 443]
+[  ".currentIndex"                                , 83993, 443]
+[  ".ComponentResourceManager"                    , 39603, 441]
+[  ".ParseException"                              , 71258, 439]
+[  "@Transactional"                               , 52910, 439]
+[  ".getCurrentUser"                              , 76430, 438]
+[  " ****************************************"    , 61360, 437]
+[  ".onActivityResult"                            , 92488, 435]
+[  ".myapplication"                               , 79989, 433]
+[  ".FormBorderStyle"                             , 45114, 432]
+[  ".initializeApp"                               , 82735, 430]
+[  ".createServer"                                , 77290, 430]
+[  ".FragmentManager"                             , 97055, 429]
+[  "                           \n"                , 71779, 428]
+[  "////////////////////////////////////////////////"                                                                                                 , 27405, 428]
+[  ".Localization"                                , 69548, 426]
+[  ".getExternalStorage"                          , 97182, 425]
+[  ".setPositiveButton"                           , 96968, 424]
+[  "            \n            \n"                 , 34642, 423]
+[  ".JSONException"                               , 68446, 423]
+[  "\t             "                              , 66526, 422]
+[  ".handleChange"                                , 35709, 421]
+[  ".ResponseEntity"                              , 67347, 421]
+[  "\t            "                               , 46600, 419]
+[  ".ValidationError"                             , 82714, 419]
+[  ".setRequestHeader"                            , 71199, 418]
+[  ".setToolTipText"                              , 89596, 418]
+[  "                    \n                    \n" , 88804, 417]
+[  "\tsynchronized"                               , 86889, 417]
+[  "++++++++++++++++"                             , 45153, 416]
+[  ".DecimalField"                                , 86397, 415]
+[  ".WindowManager"                               , 96965, 414]
+[  ".ResponseBody"                                , 74101, 414]
+[  "//------------------------------------------------"                                                                                               , 30889, 414]
+[  ".clientHeight"                                , 74311, 413]
+[  "(connectionString"                            , 77721, 412]
+[  " ------------------------------------------------------------------------\n"                                                                      , 96581, 410]
+[  "_crossentropy"                                , 88246, 410]
+[  "////////////////////////////////////////////////////////////////////////////////////////////////"                                                 , 60629, 409]
+[  ".getJSONArray"                                , 84367, 406]
+[  "\t                       "                    , 50440, 406]
+[  ".CreateCommand"                               , 79811, 402]
+[  ".setContentType"                              , 64609, 402]
+[  ".setHorizontal"                               , 35502, 401]
+[  " ============================================================================\n"                                                                  , 89318, 401]
+[  "_CheckedChanged"                              , 51911, 400]
+[  ".SelectedValue"                               , 31216, 400]
+[  "@ResponseBody"                                , 67712, 397]
+[  ".removeAttribute"                             , 78620, 397]
+[  ".columnHeader"                                , 66618, 396]
+[  ".ArrayAdapter"                                , 78954, 392]
+[  ".ServletException"                            , 53628, 390]
+[  "                              \n"             , 86770, 389]
+[  "                                        \n"   , 56596, 387]
+[  "                                    \n"       , 45807, 384]
+[  ".getContentPane"                              , 72620, 382]
+[  "(BitConverter"                                , 93170, 381]
+[  "//------------------------------------------------------------------------------\n\n"                                                             , 87567, 376]
+[  "    \n    \n    \n"                           , 41693, 374]
+[  ".BackgroundColor"                             , 73432, 366]
+[  ".setImageBitmap"                              , 94977, 366]
+[  ".visualization"                               , 94080, 366]
+[  ".getClassName"                                , 81964, 365]
+[  ".setHorizontalAlignment"                      , 69959, 364]
+[  ".SingleOrDefault"                             , 80994, 364]
+[  ".MiddleCenter"                                , 54408, 363]
+[  ".localization"                                , 76763, 361]
+[  ".StoredProcedure"                             , 49621, 360]
+[  "\r\n            \r\n"                         , 79654, 358]
+[  ".HorizontalAlignment"                         , 60587, 358]
+[  "(\"================"                          , 90204, 356]
+[  " -------------------------------------------------------------------------"                                                                       , 90915, 353]
+[  ".ManyToManyField"                             , 78543, 349]
+[  "(\"----------------"                          , 62191, 347]
+[  " //----------------"                          , 75473, 344]
+[  ".toJSONString"                                , 82086, 344]
+[  ".MouseEventHandler"                           , 97863, 342]
+[  ".SelectedItems"                               , 91077, 341]
+[  ".NumericUpDown"                               , 65932, 340]
+[  ".flowLayoutPanel"                             , 86614, 340]
+[  "//****************************************************************"                                                                               , 55263, 339]
+[  ".createTextNode"                              , 52850, 334]
+[  ".SpringBootTest"                              , 71204, 333]
+[  ".toolStripSeparator"                          , 64448, 333]
+[  ".setOnItemClickListener"                      , 84597, 332]
+[  "(ErrorMessage"                                , 54980, 332]
+[  ".visitMethodInsn"                             , 82356, 331]
+[  ".AddParameter"                                , 95481, 331]
+[  ".AutoComplete"                                , 82196, 329]
+[  "//-----------------------------------------------------------------------------\n"                                                                , 44142, 327]
+[  "\tStringBuffer"                               , 75281, 323]
+[  ".PictureBoxSizeMode"                          , 69812, 322]
+[  ".AutoSizeMode"                                , 83800, 320]
+[  " \n            \n"                            , 83913, 319]
+[  " //--------------------------------"          , 73538, 319]
+[  ".setCancelable"                               , 98847, 318]
+[  ".numericUpDown"                               , 71592, 318]
+[  ".navigateByUrl"                               , 95332, 314]
+[  ".DataGridViewCellStyle"                       , 50386, 314]
+[  ".RequestMethod"                               , 68940, 314]
+[  "@SpringBootTest"                              , 73216, 312]
+[  "<ApplicationUser"                             , 85152, 312]
+[  "                \n\n"                         , 71248, 311]
+[  ".FirebaseAuth"                                , 98242, 311]
+[  ".navigationController"                        , 29452, 311]
+[  ".PathVariable"                                , 75646, 310]
+[  ".setBackgroundResource"                       , 67962, 309]
+[  ".DropDownStyle"                               , 82900, 309]
+[  "(\"--------------------------------"          , 78346, 309]
+[  ".StretchImage"                                , 97070, 308]
+[  " automáticamente"                             , 83587, 307]
+[  ".openConnection"                              , 87122, 306]
+[  ".setLayoutManager"                            , 67633, 306]
+[  ".BackgroundImage"                             , 35332, 306]
+[  " /************************************************************************"                                                                       , 97005, 305]
+[  ".CheckedChanged"                              , 72118, 304]
+[  ".ToolStripButton"                             , 51013, 301]
+[  ".defaultProps"                                , 57778, 299]
+[  ".GeneratedValue"                              , 62300, 295]
+[  ".addMouseListener"                            , 78132, 295]
+[  ".onreadystatechange"                          , 64656, 294]
+[  "@AllArgsConstructor"                          , 66885, 288]
+[  "////////////////////////////////////////////////////////////"                                                                                     , 64575, 288]
+[  " //================================================================"                                                                              , 85993, 287]
+[  "(IServiceCollection"                          , 92099, 283]
+[  ".setLayoutParams"                             , 86399, 282]
+[  ":initComponents"                              , 56373, 282]
+[  ".ComboBoxStyle"                               , 87585, 282]
+[  ".StartPosition"                               , 59734, 281]
+[  ".dataGridViewTextBoxColumn"                   , 46977, 281]
+[  ".setHorizontalGroup"                          , 55324, 280]
+[  ".ExecuteScalar"                               , 81569, 280]
+[  " //////////////////////////////////"          , 60609, 280]
+[  "@NoArgsConstructor"                           , 67214, 279]
+[  "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"             , 33009, 279]
+[  "    \n    \n    \n    \n"                     , 80719, 278]
+[  "////////////////////////////////////////////////////////////////////////////////\n"                                                               , 58869, 277]
+[  ".StylePriority"                               , 59913, 277]
+[  ".RelativeLayout"                              , 96658, 276]
+[  "            \t\t"                             , 69562, 276]
+[  ".AbsoluteConstraints"                         , 63576, 276]
+[  ".setVerticalGroup"                            , 55328, 275]
+[  ".SpringApplication"                           , 59350, 273]
+[  ".getSelectionModel"                           , 81303, 273]
+[  "-INFRINGEMENT"                                , 94673, 272]
+[  ".AddTransient"                                , 98003, 271]
+[  ".SelectedIndexChanged"                        , 77089, 270]
+[  ".DataPropertyName"                            , 51337, 270]
+[  ".EntityManager"                               , 80694, 270]
+[  ".OrderByDescending"                           , 91300, 269]
+[  "\t\t\t           "                            , 52078, 269]
+[  " ************************************************"                                                                                                , 71836, 268]
+[  "                \n                \n"         , 59649, 265]
+[  "(SQLException"                                , 94985, 265]
+[  "(MigrationBuilder"                            , 53321, 265]
+[  ".DropDownItems"                               , 88187, 264]
+[  "(targetEntity"                                , 64632, 263]
+[  ".EntityFramework"                             , 25896, 261]
+[  "                               \n"            , 90306, 260]
+[  ".VideoCapture"                                , 94369, 259]
+[  "           \r\n"                              , 64783, 259]
+[  "\tBufferedReader"                             , 76936, 257]
+[  ".mousePosition"                               , 81666, 257]
+[  ".DataBindings"                                , 83501, 254]
+[  "\tPreparedStatement"                          , 59337, 254]
+[  ".SceneManagement"                             , 64421, 254]
+[  ".getElementsBy"                               , 19483, 251]
+[  "********************************************************************************"                                                                 , 61693, 249]
+[  " |--------------------------------------------------------------------------\n"                                                                   , 61250, 248]
+[  ".setWindowTitle"                              , 98275, 246]
+[  "\n                        \n"                 , 92163, 246]
+[  ".getColumnModel"                              , 97719, 246]
+[  ".BackgroundImageLayout"                       , 65568, 246]
+[  ".SwingConstants"                              , 85060, 245]
+[  "//****************************************************************************"                                                                   , 64226, 245]
+[  ".RequestParam"                                , 78062, 244]
+[  "@SpringBootApplication"                       , 62261, 242]
+[  "                        \r\n"                 , 65317, 242]
+[  "//------------------------------------------------------------------------------------------------"                                               , 56342, 240]
+[  ".DataVisualization"                           , 88972, 239]
+[  "//---------------------------------------------------------------------------\n"                                                                  , 57307, 237]
+[  ".toolStripMenuItem"                           , 84904, 233]
+[  ".AddModelError"                               , 84225, 229]
+[  ".SpringBootApplication"                       , 61923, 228]
+[  "@RequestParam"                                , 94320, 228]
+[  "/*================================================================"                                                                               , 44438, 228]
+[  "\\DependencyInjection"                        , 84236, 228]
+[  "              \r\n"                           , 89160, 228]
+[  ".TextAlignment"                               , 64495, 227]
+[  ".scrollHeight"                                , 90859, 225]
+[  ".getLongitude"                                , 97370, 225]
+[  "(ExpectedConditions"                          , 97348, 225]
+[  ".AllArgsConstructor"                          , 70657, 224]
+[  ".GenerationType"                              , 72372, 223]
+[  ":UIControlStateNormal"                        , 34029, 223]
+[  ".HtmlControls"                                , 98715, 223]
+[  ".NoArgsConstructor"                           , 72393, 221]
+[  "        \n    \n"                             , 88998, 220]
+[  " *----------------------------------------------------------------"                                                                               , 77149, 219]
+[  ".navigationBar"                               , 79953, 217]
+[  "/****************************************"    , 72961, 216]
+[  " //////////////////////////////////////////////////////////////////////"                                                                          , 95180, 216]
+[  ".handleSubmit"                                , 55389, 216]
+[  "\t\t                   "                      , 71400, 212]
+[  "//--------------------------------"           , 48256, 211]
+[  "(mapStateToProps"                             , 40250, 210]
+[  "\tstartActivity"                              , 95828, 209]
+[  "\t              "                             , 95805, 209]
+[  " ********************************************************************************"                                                                , 76550, 209]
+[  ".navigationItem"                              , 53013, 208]
+[  ".fromFunction"                                , 86168, 208]
+[  ".destroyAllWindows"                           , 87234, 207]
+[  " participação"                                , 140294, 207]
+[  ".DisplayStyle"                                , 93440, 207]
+[  ".FormStartPosition"                           , 65095, 206]
+[  " unterstützen"                                , 143562, 206]
+[  ".getWritableDatabase"                         , 96091, 206]
+[  "(IConfiguration"                              , 95426, 206]
+[  ".isAuthenticated"                             , 73528, 206]
+[  ".setCharacter"                                , 94119, 205]
+[  ".collectionView"                              , 65708, 205]
+[  "                             \n"              , 96017, 205]
+[  "_ValueChanged"                                , 98913, 204]
+[  "/********************************"            , 25275, 204]
+[  ".ToolStripSeparator"                          , 63149, 204]
+[  "////////////////////////////////////////////////////////////////////"                                                                             , 43200, 203]
+[  ".labelControl"                                , 42282, 203]
+[  ".cornerRadius"                                , 42007, 200]
+[  "<SpriteRenderer"                              , 77408, 200]
+[  "                \t"                           , 60123, 200]
+[  ".asInstanceOf"                                , 21942, 199]
+[  ".SelectCommand"                               , 90042, 198]
+[  ".SplitContainer"                              , 92398, 196]
+[  "------+------+"                               , 72757, 194]
+[  "*******************************************************************************/\n"                                                               , 79403, 192]
+[  "               \r\n"                          , 91205, 189]
+[  ".BorderFactory"                               , 85985, 188]
+[  " ******************************************************************************/\n"                                                               , 79371, 188]
+[  ".toolStripButton"                             , 66572, 188]
+[  ".getOwnProperty"                              , 61012, 186]
+[  "        \n        \n        \n"               , 73363, 184]
+[  "_HandleTypeDef"                               , 78906, 182]
+[  "@GeneratedValue"                              , 84781, 179]
+[  ".bindingNavigator"                            , 45605, 178]
+[  ".getOwnPropertyDescriptor"                    , 89124, 177]
+[  "////////////////////////////////////////////////////////////////////////////////\n\n"                                                             , 98320, 176]
+[  "//----------------"                           , 61683, 176]
+[  "/************************************************************************************************"                                                , 86544, 175]
+[  "\t                 "                          , 96094, 175]
+[  "////////////////////////////////////////////////////"                                                                                             , 82845, 174]
+[  ".CenterScreen"                                , 69862, 173]
+[  "/*----------------------------------------------------------------------------"                                                                   , 57906, 173]
+[  ".FindGameObjectWithTag"                       , 85295, 173]
+[  "//----------------------------------------------------------------------------------------------------------------"                               , 89180, 172]
+[  ".MouseAdapter"                                , 97900, 171]
+[  ".DriverManager"                               , 96569, 170]
+[  ".groupControl"                                , 79650, 170]
+[  ".setPrototypeOf"                              , 98085, 169]
+[  "\t\t\t\t           "                          , 74904, 167]
+[  ".dateTimePicker"                              , 77005, 165]
+[  "\\HttpFoundation"                             , 45256, 162]
+[  "/*------------------------------------------------"                                                                                               , 73062, 162]
+[  ".XtraPrinting"                                , 49905, 162]
+[  "/****************************************************************************\n"                                                                  , 58007, 161]
+[  ".contentOffset"                               , 87103, 161]
+[  ".LabelControl"                                , 63271, 160]
+[  ".ImageTransparentColor"                       , 93751, 152]
+[  " ******************************************************************************/\n\n"                                                             , 93682, 151]
+[  "/****************"                            , 72300, 150]
+[  "//================================================"                                                                                               , 76507, 148]
+[  ".sharedInstance"                              , 64269, 147]
+[  ".DataGridViewAutoSize"                        , 93294, 147]
+[  "möglichkeiten"                                , 142432, 146]
+[  "_RETURNTRANSFER"                              , 71165, 145]
+[  ".ConnectionStrings"                           , 79111, 143]
+[  ".bindingNavigatorMove"                        , 80225, 140]
+[  "//************************************************************************"                                                                       , 65080, 139]
+[  ".JpaRepository"                               , 66795, 139]
+[  ".DefaultCellStyle"                            , 97418, 138]
+[  ":UIControlEventTouchUpInside"                 , 57828, 136]
+[  ".BindingSource"                               , 82167, 136]
+[  ".getElementsByName"                           , 93584, 136]
+[  ".sendRedirect"                                , 82573, 136]
+[  "\t                "                           , 73912, 134]
+[  ".getRequestDispatcher"                        , 70176, 133]
+[  ".setViewportView"                             , 97027, 132]
+[  "/******************************************************************************\n"                                                                , 53196, 131]
+[  "/********************************************************************************"                                                                , 75632, 130]
+[  ".layoutControlItem"                           , 41953, 130]
+[  "//================================================================================"                                                               , 94303, 129]
+[  "        \r\n        \r\n"                     , 73867, 128]
+[  "/////////////////////////////////////////////////////////////////////////////\n"                                                                  , 83894, 127]
+[  ".DataGridViewColumnHeadersHeightSizeMode"     , 72203, 127]
+[  ".DisplayMember"                               , 87638, 127]
+[  "-----------\r\n"                              , 84877, 127]
+[  ".DateTimePicker"                              , 58723, 126]
+[  ".ColumnHeadersHeightSizeMode"                 , 72129, 125]
+[  "\\Configuration"                              , 83023, 124]
+[  ".findByIdAndUpdate"                           , 93784, 123]
+[  ".dequeueReusableCell"                         , 56968, 122]
+[  "/************************"                    , 62622, 121]
+[  "|--------------------------------------------------------------------------\n"                                                                    , 42080, 120]
+[  "//----------------------------------------------------------------------------\n"                                                                 , 78938, 120]
+[  ".TextImageRelation"                           , 91813, 120]
+[  "\t\t\t               "                        , 76639, 119]
+[  "/****************************************************************************"                                                                    , 22565, 118]
+[  ".DataGridViewContentAlignment"                , 94556, 116]
+[  ".AddInParameter"                              , 91382, 116]
+[  ".DataGridViewTriState"                        , 94937, 114]
+[  "\\OptionsResolver"                            , 80641, 114]
+[  ".opendaylight"                                , 96304, 113]
+[  "(ConfigurationManager"                        , 98048, 113]
+[  "_________________\n\n"                        , 88252, 113]
+[  ".DataGridViewColumn"                          , 51863, 112]
+[  "=findViewById"                                , 84891, 112]
+[  "\\GeneratedValue"                             , 89328, 111]
+[  "///////////////////////////////////////////////////////////////////////////////\n"                                                                , 76737, 108]
+[  " Üniversitesi"                                , 133698, 107]
+[  "/************************************************"                                                                                                , 78371, 106]
+[  "(withIdentifier"                              , 44311, 105]
+[  ".bunifuFlatButton"                            , 95234, 105]
+[  ".EventSystems"                                , 86410, 103]
+[  "\t\t            "                             , 83268, 100]
+[  "\\FrameworkBundle"                            , 76399, 97]
+[  ".MixedReality"                                , 67861, 96]
+[  "-----------*/\n"                              , 81580, 95]
+[  ".textAlignment"                               , 55183, 95]
+[  " ******************************************************************************\n"                                                                , 85785, 92]
+[  ".bottomAnchor"                                , 72807, 91]
+[  ")initWithFrame"                               , 91752, 91]
+[  " funcionários"                                , 143768, 91]
+[  ".localizedDescription"                        , 73053, 90]
+[  "\t                           "                , 75305, 90]
+[  "\t\t                       "                  , 94779, 89]
+[  ":NSLocalizedString"                           , 93095, 89]
+[  ".layoutControl"                               , 61661, 88]
+[  "_OscInitStruct"                               , 96199, 87]
+[  "/*****************************************************************************\n"                                                                 , 68132, 83]
+[  ":UIButtonTypeCustom"                          , 98159, 79]
+[  ".translatesAutoresizingMaskIntoConstraints"   , 62470, 78]
+[  "\\Repositories"                               , 57283, 78]
+[  ".userInteractionEnabled"                      , 90531, 76]
+[  ".simpleButton"                                , 96813, 76]
+[  "//---------------------------------------------------------------------------\n\n"                                                                , 94464, 75]
+[  "\\Notifications"                              , 92232, 74]
+[  ")didReceiveMemoryWarning"                     , 55504, 72]
+[  ".CustomButton"                                , 85437, 70]
+[  " /*----------------------------------------------------------------"                                                                              , 67549, 70]
+[  "                    \t"                       , 96800, 67]
+[  "/******************************************************************************/\n"                                                               , 84999, 66]
+[  ".SimpleButton"                                , 89344, 64]
+[  ".rightBarButtonItem"                          , 92917, 62]
+[  ".numberOfLines"                               , 94324, 59]
+[  ".clipsToBounds"                               , 94172, 58]
+[  ":UIControlState"                              , 30959, 56]
+[  "_InitStructure"                               , 60099, 56]
+[  "/***************************************************************************\n"                                                                   , 70988, 53]
+[  ".masksToBounds"                               , 81993, 51]
+[  ":UIControlEvent"                              , 51468, 49]
+[  "]initWithFrame"                               , 59595, 46]
+[  ")prepareForSegue"                             , 92782, 46]
+[  ".leadingAnchor"                               , 84845, 46]
+[  "(QStringLiteral"                              , 98249, 43]
+[  ".VisibleIndex"                                , 91683, 43]
+[  ".trailingAnchor"                              , 88896, 43]
+[  "_makeConstraints"                             , 63137, 41]
+[  ".didReceiveMemoryWarning"                     , 65074, 37]
+[  "établissement"                                , 137914, 32]
+[  ".barDockControl"                              , 96000, 32]
+[  ":UIButtonType"                                , 86261, 30]
+[  "(UIAlertAction"                               , 96759, 28]
+[  ".EditorButton"                                , 82778, 28]
+[  ".LayoutControlItem"                           , 82943, 26]
+[  "//--------------------------------------------------------------\n"                                                                               , 81042, 24]
+[  " __________________\n\n"                      , 79364, 23]
+[  "_regeneration"                                , 74820, 23]
+[  "(withDuration"                                , 92897, 21]
+[  "características"                              , 135524, 17]
+[  "<|endoftext|>"                                , 151643, 13]
+[  ".updateDynamic"                               , 49110, 9]
+[  ".faceVertexUvs"                               , 92595, 8]
+[  "(RuntimeObject"                               , 84631, 8]
+[  "-------------</"                              , 36642, 6]
+[  ".panelControl"                                , 82849, 4]
+[  "_StaticFields"                                , 37735, 2]
+[  "_InternalArray"                               , 72128, 2]

qwen__1000__20000/tokens_removed__alphabet.jsonl

@@ -0,0 +1,212 @@
+[  " prze"                                        , 30329, 997]
+[  " fwd"                                         , 91004, 989]
+[  " werk"                                        , 58618, 986]
+[  " uw"                                          , 37343, 986]
+[  " btw"                                         , 89857, 978]
+[  " wyst"                                        , 87564, 976]
+[  " Wohn"                                        , 52795, 976]
+[  " sez"                                         , 78472, 976]
+[  " fsm"                                         , 83138, 972]
+[  " jspb"                                        , 63770, 970]
+[  " JAVA"                                        , 85560, 967]
+[  " clf"                                         , 46079, 955]
+[  " jdbc"                                        , 85902, 951]
+[  " zaj"                                         , 81347, 950]
+[  " jogo"                                        , 66134, 948]
+[  " kaz"                                         , 90725, 947]
+[  " vw"                                          , 74086, 945]
+[  " WX"                                          , 77441, 937]
+[  " awk"                                         , 19976, 927]
+[  " wei"                                         , 83904, 921]
+[  " jorn"                                        , 71349, 903]
+[  " JIT"                                         , 96187, 901]
+[  " wiel"                                        , 63979, 897]
+[  " zb"                                          , 74355, 887]
+[  " Deze"                                        , 87836, 880]
+[  " fert"                                        , 21207, 879]
+[  " jc"                                          , 72818, 871]
+[  " dzi"                                         , 51026, 870]
+[  " jsx"                                         , 80095, 869]
+[  " jm"                                          , 95667, 863]
+[  " PREF"                                        , 98002, 860]
+[  " juga"                                        , 42369, 857]
+[  " zach"                                        , 96812, 844]
+[  " zien"                                        , 79407, 841]
+[  " einf"                                        , 31287, 838]
+[  " Zahl"                                        , 82745, 836]
+[  " darf"                                        , 87231, 834]
+[  " pj"                                          , 72613, 831]
+[  " qw"                                          , 41178, 822]
+[  " csrf"                                        , 35522, 819]
+[  " jmp"                                         , 90369, 819]
+[  " jl"                                          , 84710, 815]
+[  " vrij"                                        , 69605, 812]
+[  " mz"                                          , 95392, 809]
+[  " zab"                                         , 73071, 807]
+[  " koji"                                        , 59371, 804]
+[  " wz"                                          , 62920, 802]
+[  " czy"                                         , 35950, 798]
+[  " blij"                                        , 73280, 795]
+[  " Utf"                                         , 43432, 792]
+[  " prez"                                        , 79942, 784]
+[  " gfx"                                         , 39478, 783]
+[  " kf"                                          , 85580, 778]
+[  " pz"                                          , 97102, 777]
+[  " Wich"                                        , 76589, 763]
+[  " zlib"                                        , 81218, 761]
+[  " intf"                                        , 93706, 755]
+[  " seja"                                        , 52596, 740]
+[  " wget"                                        , 64508, 740]
+[  " Zus"                                         , 58485, 737]
+[  " sdf"                                         , 50157, 737]
+[  " Dzi"                                         , 134288, 737]
+[  " jd"                                          , 73376, 736]
+[  " cljs"                                        , 26284, 723]
+[  " krij"                                        , 69437, 721]
+[  " rij"                                         , 69345, 719]
+[  " iw"                                          , 72268, 714]
+[  " filt"                                        , 71854, 712]
+[  " woll"                                        , 73184, 710]
+[  " CFG"                                         , 46825, 704]
+[  " rodz"                                        , 79541, 697]
+[  " seiz"                                        , 33113, 696]
+[  " WCS"                                         , 98351, 687]
+[  " afs"                                         , 66713, 687]
+[  " jb"                                          , 87118, 681]
+[  " nfs"                                         , 77472, 673]
+[  " Netz"                                        , 92283, 673]
+[  " wich"                                        , 70367, 670]
+[  " cpf"                                         , 63976, 664]
+[  " jint"                                        , 52467, 658]
+[  " fds"                                         , 93734, 655]
+[  " Zust"                                        , 93716, 653]
+[  " JNI"                                         , 70829, 651]
+[  " bfs"                                         , 66564, 648]
+[  " lz"                                          , 95982, 640]
+[  " feas"                                        , 31892, 630]
+[  " JMP"                                         , 88251, 628]
+[  " Fauc"                                        , 96362, 618]
+[  " NRF"                                         , 72010, 618]
+[  " JAXB"                                        , 60640, 615]
+[  " ajud"                                        , 86059, 614]
+[  " FUNC"                                        , 43352, 611]
+[  " hdf"                                         , 97045, 610]
+[  " moz"                                         , 95339, 608]
+[  " hWnd"                                        , 58579, 606]
+[  " ucz"                                         , 87426, 602]
+[  " cj"                                          , 90997, 601]
+[  " hwnd"                                        , 52832, 600]
+[  " zx"                                          , 71322, 597]
+[  " dof"                                         , 91760, 595]
+[  " azi"                                         , 74062, 594]
+[  " Swal"                                        , 59073, 586]
+[  " wym"                                         , 71241, 582]
+[  " frei"                                        , 83423, 577]
+[  " FLAC"                                        , 89099, 577]
+[  " CPF"                                         , 45450, 569]
+[  " yaz"                                         , 78114, 561]
+[  " jt"                                          , 91076, 550]
+[  " koje"                                        , 96499, 549]
+[  " zmq"                                         , 90740, 545]
+[  " MILF"                                        , 73298, 540]
+[  " CSRF"                                        , 78595, 531]
+[  " zun"                                         , 98175, 530]
+[  " jede"                                        , 94572, 525]
+[  " BFS"                                         , 90351, 515]
+[  " vej"                                         , 92460, 509]
+[  " GFX"                                         , 76155, 503]
+[  " zg"                                          , 92058, 499]
+[  " zes"                                         , 137584, 498]
+[  " vfs"                                         , 92941, 481]
+[  " vaz"                                         , 143854, 480]
+[  " Frm"                                         , 91995, 477]
+[  " kz"                                          , 80955, 460]
+[  " wnd"                                         , 80432, 459]
+[  " zost"                                        , 95926, 459]
+[  " FLT"                                         , 96061, 446]
+[  " helf"                                        , 74403, 444]
+[  " newY"                                        , 80384, 443]
+[  " jit"                                         , 84573, 442]
+[  " azt"                                         , 97894, 438]
+[  " fz"                                          , 92649, 436]
+[  " wsp"                                         , 62507, 435]
+[  " pwm"                                         , 64551, 434]
+[  " fov"                                         , 93473, 416]
+[  " swal"                                        , 49354, 412]
+[  " CWE"                                         , 50860, 409]
+[  " newX"                                        , 88803, 408]
+[  " szy"                                         , 91765, 396]
+[  " hjem"                                        , 66985, 393]
+[  " dzie"                                        , 49673, 390]
+[  " licz"                                        , 84521, 367]
+[  " szcz"                                        , 75573, 367]
+[  " hoje"                                        , 73158, 366]
+[  " wsz"                                         , 38391, 360]
+[  " nije"                                        , 86493, 356]
+[  " zza"                                         , 97649, 344]
+[  " zend"                                        , 78568, 340]
+[  " wcs"                                         , 96668, 337]
+[  " Www"                                         , 81210, 335]
+[  " jika"                                        , 70879, 332]
+[  " lawy"                                        , 11010, 328]
+[  " MOZ"                                         , 63744, 317]
+[  " saja"                                        , 78786, 305]
+[  " fica"                                        , 95127, 297]
+[  " menj"                                        , 41374, 295]
+[  " HWND"                                        , 60384, 285]
+[  " zf"                                          , 99048, 281]
+[  " wlan"                                        , 78353, 279]
+[  " BUF"                                         , 73178, 277]
+[  " bfd"                                         , 94047, 275]
+[  " fark"                                        , 129369, 274]
+[  " znaj"                                        , 67715, 266]
+[  " fabs"                                        , 40728, 248]
+[  " EIF"                                         , 27695, 246]
+[  " rozp"                                        , 140775, 238]
+[  " zale"                                        , 142463, 232]
+[  " zoek"                                        , 73800, 226]
+[  " UIF"                                         , 44404, 214]
+[  " zob"                                         , 128165, 203]
+[  " ZEND"                                        , 92370, 202]
+[  " DWC"                                         , 70259, 198]
+[  " wyb"                                         , 143557, 192]
+[  " narz"                                        , 140251, 152]
+[  " pObj"                                        , 59952, 142]
+[  " milf"                                        , 25399, 134]
+[  " setw"                                        , 50063, 133]
+[  " szko"                                        , 140013, 126]
+[  " oczy"                                        , 138959, 122]
+[  " zwy"                                         , 143597, 122]
+[  " weap"                                        , 6158, 119]
+[  " wxT"                                         , 74197, 113]
+[  " bbw"                                         , 40760, 105]
+[  " frau"                                        , 37031, 100]
+[  " jaki"                                        , 139537, 93]
+[  " mpz"                                         , 71228, 84]
+[  " sprz"                                        , 127977, 83]
+[  " JADX"                                        , 88508, 83]
+[  " urz"                                         , 143475, 76]
+[  " fkk"                                         , 60798, 71]
+[  " atof"                                        , 64957, 68]
+[  " PdfP"                                        , 76516, 59]
+[  " ffm"                                         , 75892, 54]
+[  " jadx"                                        , 57543, 54]
+[  " Milf"                                        , 51558, 52]
+[  " mpfr"                                        , 82214, 50]
+[  " wyja"                                        , 142442, 48]
+[  " rfl"                                         , 69335, 42]
+[  " znal"                                        , 141024, 40]
+[  " zro"                                         , 135396, 38]
+[  " Wroc"                                        , 139221, 37]
+[  " Oczy"                                        , 141295, 33]
+[  " wygl"                                        , 133563, 30]
+[  " zwyk"                                        , 143598, 30]
+[  " Bbw"                                         , 85597, 25]
+[  " najb"                                        , 143244, 22]
+[  " ksz"                                         , 141368, 22]
+[  " nrw"                                         , 76506, 15]
+[  " uwag"                                        , 134907, 12]
+[  " wzgl"                                        , 127904, 7]
+[  " wida"                                        , 142268, 5]
+[  " RTVF"                                        , 87578, 4]
+[  " RTWF"                                        , 62685, 1]

qwen_vocab.py

@@ -0,0 +1,99 @@
+import os, sys, glob, json
+from utils_lang import *
+from transformers import AutoTokenizer
+
+def get_kept_tids():
+    # Keep all special tokens
+    kept_tids = set( x for x in range(151643, 151664 + 1) )
+
+    # '''
+    tokenizer = AutoTokenizer.from_pretrained(".")
+
+    canbe_vi_kept = 0
+    is_ascii_kept = 0
+
+    for tid in range(0, tokenizer.vocab_size):
+        token = tokenizer.decode(tid)
+
+        if vietnamese_syllable_ratio(token) > 0.8:
+            canbe_vi_kept += 1
+            kept_tids.add(tid)
+
+        if len(token) <= 2 and canbe_vietnamese(token):
+            canbe_vi_kept += 1
+            kept_tids.add(tid)            
+
+        if len(token) <= 2 and is_ascii(token):
+            is_ascii_kept += 1
+            kept_tids.add(tid)
+
+    print(">>> canbe_vi_kept", canbe_vi_kept)
+    print(">>> is_ascii_kept", is_ascii_kept)
+    # '''
+
+    kept_filenames = glob.glob("qwen__1000__20000/tokens_kept__*.jsonl")
+
+    for filename in kept_filenames:
+        for line in open(filename, "rt"):
+            token, tid, count = json.loads(line)
+            kept_tids.add(tid)
+
+    kept_tids = list( kept_tids )
+    kept_tids.sort()
+
+    print("new_qwen_vocab", len(kept_tids))
+    return kept_tids
+
+
+kept_tids = get_kept_tids()
+
+# old vs new vocab mapping
+old2new = {}
+new2old = {}
+
+for new_tid, old_tid in enumerate( kept_tids ):
+    old2new[ old_tid ] = new_tid
+    new2old[ new_tid ] = old_tid
+
+
+STRANGE_TOKENS = set()
+
+def old2new_tid(x, tokenizer):
+    global STRANGE_TOKENS
+
+    if x in old2new:
+        return old2new[x]
+
+    else:
+        token = tokenizer.decode(x)
+        if contains_unwanted(token):
+            return None
+
+        words = re.findall(r'[a-z]+', token, flags = re.IGNORECASE)
+
+        if len(words) > 1:
+            print(">>>", words)
+
+        if len(words) == 1:
+            tids = tokenizer.encode(words[0])
+            if len(tids) == 1 and tids[0] in old2new:
+                return old2new[tids[0]]
+
+        msg = f">>> old2new_tid error: id {x}, token '{token}'"
+        if token not in STRANGE_TOKENS:
+            print(msg)
+            STRANGE_TOKENS.add( token )
+
+        # assert False, msg
+        return None
+
+    assert False, "Không thể tới bước này, có lỗi ở phần code trên"
+
+
+if __name__ == "__main__":
+
+    n = len(kept_tids)
+    nn = round(n / 64) * 64
+
+    print("kept_tids", n)
+    print(n, nn) # 76138 => 76160 (làm tròn để chia hết cho 64)

tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

utils.py

@@ -0,0 +1,57 @@
+import time, os
+import subprocess
+import re
+
+LOCATION = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+## Các màu hay dùng
+BLACK   = '\033[30m';   WHITE  = '\033[97m'
+RED     = '\033[91m';   YELLOW = '\033[33m'
+GREEN   = '\033[32m';   CYAN   = '\033[36m'
+BLUE    = '\033[94m';   GREY   = '\033[37m'
+MAGENTA = '\033[95m';   RESET  = '\033[0m'
+
+
+def num_procs():
+    return os.cpu_count() - 2
+
+TIMER_STARTED_AT = { "default": time.time() }
+def reset_timer(timer="default"):
+    global TIMER_STARTED_AT
+    TIMER_STARTED_AT[timer] = time.time()
+
+
+def measure_time(message="", timer="default", color=YELLOW):
+    total = time.time() - TIMER_STARTED_AT[timer]
+    total = pretty_num(total)
+
+    message = message.strip()
+    if len(message) > 0: 
+        message = " " + message
+
+    print(f"{color}{timer}:{message} {total} seconds{RESET}")
+
+
+def count_words(x):
+    return len(x.split())
+
+def pretty_num(x):
+    return round(x*100)/100
+
+def mkdirs(path):
+    splits = path.split("/")
+    for i in range(0, len(splits)):
+        x = "/".join(splits[ : i + 1])
+        cmd = f"mkdir -p {x}"
+        print(cmd)
+        subprocess.run(cmd, shell = True)
+
+
+if __name__ == "__main__":
+
+    reset_timer(timer="my timer")
+
+    s = "chào cả nhà, cả nhà khỏe không ạ?"
+    print(f"{RED}{s}{RESET} có {CYAN}{count_words(s)} từ")
+
+    measure_time("tổng thời gian chạy", timer="my timer")

utils_lang.py

@@ -0,0 +1,290 @@
+import os, re, subprocess
+from utils import LOCATION
+import fasttext # pip install fasttext
+
+## Fasttext detect lang
+LANGID_FILENAME = 'lid.176.bin'
+LANGID_FILEPATH = f"{LOCATION}/data/{LANGID_FILENAME}"
+
+if not os.path.exists(LANGID_FILEPATH):
+    cmd = f"wget https://dl.fbaipublicfiles.com/fasttext/supervised-models/{LANGID_FILENAME}; mv {LANGID_FILENAME} {LANGID_FILEPATH}"
+    subprocess.run(cmd, shell=True)
+
+FASTTEXT_MODEL = fasttext.load_model(LANGID_FILEPATH)
+WORD_RE = re.compile(r'\w+\s')
+
+def detect_lang(text, check_words = False):
+    if check_words:
+        # Chỉ kiểm tra ngôn ngữ với những từ được phân tách rõ ràng
+        words = re.findall(WORD_RE, text)
+        text = " ".join(words)
+
+    rs = FASTTEXT_MODEL.f.predict(text, 1, 0.0, 'strict')
+    if rs: return rs[0][-1].split('__')[-1]
+    else:  return None
+
+assert detect_lang("hello vietnam") == "en"
+assert detect_lang( "chào nước mỹ 123sfd http://adf4| tôi là ") == "vi"
+
+
+_oa = ord('a')
+_oz = ord('z')
+_os = ord(' ')
+###
+def is_alphabet(token):
+    for c in token.lower():
+        o = ord(c)
+        if o != _os and (_oa > o or o > _oz):
+            return False
+    return True
+
+def mostly_alphabet(text):
+    lower_text = text.lower()
+    tokens = re.findall(r'[a-z]+', lower_text)
+    n = 0
+    for x in tokens:
+        n += len(x)
+    return n / len(text) > 0.8
+
+
+import nltk # pip install nltk
+nltk.download('words')
+en_words = set( nltk.corpus.words.words() )
+###
+def is_english_word(token):
+    x = token.strip().lower()
+    if x in en_words: return True
+    if len(x) > 1 and x[-1] == 's' and x[:-1] in en_words: return True # số nhiều
+    return False
+
+assert(is_english_word("car"))
+assert(is_english_word("cars"))
+
+
+'''
+https://www.regular-expressions.info/unicode.html
+https://stackoverflow.com/questions/38615740/regular-expression-to-accept-all-thai-characters-and-english-letters-in-python#answer-72440821
+https://pypi.org/project/regex/
+'''
+import regex
+
+def contains_cjk(token):
+    for char in token:
+        o = ord(char)
+        if min_cjk <= o and o <= max_cjk:
+            return True
+    return False
+
+def is_ascii(token):
+    for char in token:
+        if ord(char) > 255:
+            return False
+    return True
+
+unwanted_langs = '''
+\p{Arabic}
+\p{Armenian}
+\p{Bengali}
+\p{Bopomofo}
+\p{Braille}
+\p{Buhid}
+\p{Cherokee}
+\p{Cyrillic}
+\p{Devanagari}
+\p{Ethiopic}
+\p{Georgian}
+\p{Greek}
+\p{Gujarati}
+\p{Gurmukhi}
+\p{Hanunoo}
+\p{Hebrew}
+\p{Hiragana}
+\p{Inherited}
+\p{Kannada}
+\p{Katakana}
+\p{Khmer}
+\p{Lao}
+\p{Limbu}
+\p{Malayalam}
+\p{Mongolian}
+\p{Myanmar}
+\p{Ogham}
+\p{Oriya}
+\p{Runic}
+\p{Sinhala}
+\p{Syriac}
+\p{Tagalog}
+\p{Tagbanwa}
+\p{TaiLe}
+\p{Tamil}
+\p{Telugu}
+\p{Thaana}
+\p{Thai}
+\p{Tibetan}
+\p{Yi}
+'''.strip().split()
+
+unwanted_langs_re = regex.compile(f'[{"".join(unwanted_langs)}]+')
+
+# https://emoji-python.readthedocs.io/en/stable/
+from emoji import emoji_count # python -m pip install emoji --upgrade
+
+def contains_emoji(token):
+    return emoji_count(token) > 0
+
+def contains_unwanted(token):
+    if contains_cjk(token):
+        return True
+
+    m = regex.findall(unwanted_langs_re, token)
+    for x in m:
+        for c in x:
+            if ord(c) > 255: # not ascii
+                return True
+    return False
+
+
+import unicodedata
+none_tone_syllables = set("khum toap chuyn mien khuech khun nun lin dong buan kay nhuyen ruet duet khoap lum nhit khoanh kheng lieng xuap thia luot truynh khoay thy quap vien chieng duya luech mai moam nguyech chuych phuan hach xuyen chum phoeng duyu giech suya loam choam bich biu phuya trai doa khim xom phec cunh buch quep rien meu muoi lunh uap penh xach chuang triec luo khi soi choo muu phun chuen meng oai nghenh toen que troam tap gheu cheu chuam rao khuynh nec phu ruom xuech thoac tuon hech bien ghi duyp ping khoi bap koai suyp thoam chieu gienh noe sap cau loap chay day pinh chuo cieu goao duon min nuyn mia nen nang thoao rech nam troanh lit toong ngha thuonh toac hooc tuat genh cup phia linh chich nhoai giua meo ngoat quyu chic xeu lep vic kat chuyech niet soeng khe hao nganh nhuon thau nuoch chuyenh tup tac boet piep vot choanh goam xec tuay doc dac bech oeo dang hoach loen set goon pooc noon toat phuyn nia nghau uyt kong nhoem kich bic xuyec coang nget thuam tuem khuyenh khoeo lien ging boo chot nghin voang man sut koa che quan uang moon ung hien nghit sun hue kien quai ghec hiu puya kuan ngoac goan suen thuy luenh tre dec moe thoo men ning quac soat nhim nuang nheo ngit ngheo nguyet bup doan pha nuot mach khoang bay thup nghienh troang hoang tem guyet nhiep phoai nung khoe hoai phuen kec oach ben koam ghech vuo beo choach phoanh quem thao giec giuong pic treng eng ngoe xoc huet khiem xoep roon nguen huonh xoe chinh ghet duang bum vich ngu niu phoem guo hiep chep ghip vuat beu tric chue kuop chat rim phoc goap lai piu noang bep moang tron luyp lia nge xenh chuet try soem duong guong trua huy vum ghem mon quing vuong dut dop giuoc hap viem nem vue luep ruem piet thiet tunh soc ron quiet khuam doe hoanh nghuoi cuya giai nieng giuc nhut phau phiec xuong guang pim puat sit vep pin rut nghinh rom nheng troeo diem kuoi xoang tai theu suou boch uyn van pen doat luc pang reo senh vunh sach liec quyenh phuyu bun huynh troi deng ret ghang xuom phy chiet quao sen pau tuep guech lac nghoanh giue booc roach toai doo nhuang khui nuyt hiem nhap vooc rang khuyn puong thoay mom vuya kao ceng minh quyet ram xien nhac chooc gen nau ngum guoi ghup thoat voon quong poac trop vuyet doang tuo soe vai buc nuat giom ren henh kuat nghom trinh duonh nac tren nhiet pong num pac guom loach chac ghep gioan luan poe son tet riep doen huon nhuc tinh muom cheng khuay nhin heo leo thec poc yen koay lonh gioay hich nguo phuoc xai mooc loanh xieng dim yem ruou vieng liet tuyt duen nay nguenh huay khet luong ghit uot dien truen dao hec pap poem nguan hoo gem conh phoang lop xuyet ong huenh nguong coo sum ghap thing nhien menh xem phim ghim nghac ruech kan nhet tuya hang quu rung chach xuyt uem muou noo phet tuyn chy nghang toeng hem tic tay piem pui but hoch nghe trum gioen uenh rum tat giao chuyec giuu khoeng nguyem ngieu much kiep nhuem tuang soach coan boi khuong gung bon ming khot toanh xia guc gium quia nhon gin phuyen thuynh phing oanh khoet choong khen oay choen trut nghut guych khuc uoi het hieu siec gut map quam giuam khit truyenh tuet uat phon kung khoat phuom chuom luyen nghuyen buon cat ngien nhuo triu vuoc dai nguem ngoot sip kenh tech troc noa ruen ngoec tuu ach cooc tom siem trin chuc rec giooc nom kin dieng coom khuen xen choap hung xop geo ranh phut puou huyu nghap nghao nich chuong xun nhoc nhoeng quang phot muop ngom choon huech manh koi nhech puu sot ruang siet khenh sop truan giong han phuc thuem giach buay san tuenh loac ngoa xiep buat huem quon nhiu giuyt lieu suap xoeo duynh nhui boac pien cuoc nghiem tung xao nhuen oen ghieu xan neo luynh pat pheu quech ruc buoc goeng khing munh cenh uoc mip vap puan nhoang chiec soang yet hienh seu khoem ganh miet khuop coem duan uong geng ich ham phem moem ngiu daa liu koang rin xuac huac guop nguu kanh roai seo ruan ghuan kam riet tuych koach voom tring voat buou thich roeng kuon nin lut nguay nguyen trup boom chit xuoc tieng nhoi noonh noam khiec thieu nhum vip giuop vem gheng ghop giay hep moo biep oem duc xip thenh song quoat quyng poo goach vun vuom boen huat duyenh kon nhenh quay xoo muonh diec nuou loet khuyen giuyn lem nghoc xinh gianh luyt loi suat giem khuenh kueng kut khuot mot hing let doem tenh thoang mienh vuec huoi suon phich tuoc sech ruch piec guem ruy lun uet chun sieu toep kim phup ngai oong soap ngoai sua keu bec kia ing truay ngy muot truou ria cun sing xanh nim quenh hay beng duoi gie khoep doi ngan nuop pao ngoach chuap giat lan deo xim hinh oat puyt nooc chuyt cim thuyet phiep tao gio troat mui noan troa poanh quoam nghech noen tooc cuo goe trieng moi xac nhuenh kooc dam hom lat roam thuyu loon sau ruat nginh dach suc nao truyu roi thuot ngho peu nghiep chonh hoe suyu ten leng vet bui tuou coai noao soao quen hoam nghiec trim quy khuom mua gooc nhoach put pue tuynh tuap cui xoec goat cuyen doam truoc hiec pun nghiu ving thoe gich rip vom ghien khu bat kic kuc ghan ruot nghat ken luay nhuu vui sui buoi loong reng boem toet bit khoach uech guyen ruoc them khiu roen nha soong gam dap dom cuoi choeo koong diu luoc bunh mam xing nhe huot kit lach binh ngen gian cung din gieng koem vin kai khanh toi gac puyu nguyeng och thuya dieu moanh ghiec khoo vao choom venh voen hau khoen guou khieng chec nhoet giuoi voan gan ghia khao voay ngoong mong nhach nguou kep huyet cot kui queo duech phip phinh tuong vec xit ghanh tuyenh bin pet dooc quim voi pit nghoeo huyep lap tut ruyn poay doao phuat duo dau dip vieu doch coa ngao chau hoon koom roo quoong hat buyet khooc heu ngia phooc mic nuc pia duat kap pum nghip tiep poai boat nuoi nua thoon trang quyeu hoeng ngang thi ngam noom gop oam gech nhoap muo sac mung duenh khia nup quach gao noi nhat truo xong khuyng luu chiem rooc bim hong khip king sem sep rua thuon pec oap punh chet phua gum nhieu due nghuy phuu ngooc vuy uya giap huyenh poen luyu chao vup muy kha dich hieng nan loom khuoi luap mue khat ngiem yep xiet nghi duyt xich nong khem kuong chuay hoao thuong siep goanh quuy loec phoong xiec khuu kienh roat puyn ghiu rieu goai cuu xuyenh dunh soac ngoen chan vuych tanh mum rep truyet chop phic nhoe giup nip loch phat huu vang ngui nue phuo khech khuyet thuyen nghen say mup sec voai ngoam truop roep moa nga xuc nghoi ngung yeu luy lam niem vong chai nuoc guat buyen gioong cing thuc soen dot thet cuon khueu tru luyet dem ngoi via nhom cen noay lot niec giang ninh thun vop buyu huan nhu phien chut duu khunh ngot nhoay ruyen phom xoam teng quau sat nghoe guy quet nghanh hop nghec huyen xup khap vung phum toe chuat xuych buang pay thoeng kheu nuyu nguot cuou biet khom nien tranh phay phuot quooc suec nieu hua trich giuyp trit kiet ngong nhem tip muc phep xuy thuyn xum nhang xoac koo nhoa nhoanh voe luych nuem thuech khien hoac goet truyp ngoang thot vech con khoec muyng miem xunh huang duyet chen ghong duch phang uyp ngin gioe boao anh trung xung lic luang tret xoa gang canh tan xang tang bom suoi long thuenh bot nghich thong ngech giu chi huyech xoat kua nguy hut doong vach nhoong hai tit uay thon kinh bam huong cac xiu troai gioai noat quynh hit phit xoan goong liem gioang doach phoam quinh pieng giau mep oec thien truc choeng guan vac vat kach nham khan nhuot nhoam nhuan ngiep loang guon poong noeng thieng hot pheo boon tien dua phach giui runh gap hic loay luet voanh den chanh thui buong ching boang gai choay loem nach troon phoo vim nuon huych trem poap xat tuop tri phong onh hun sien ghy huya pam tran vinh uep nhoat luop thuet tiu suyen duop hac thuoi cin nuyen bing gioanh ghiem vuou nghieng truyn trieu ngoet oon bet tray rem quoc tich nuyp phoi sim liep nhup trec bong triem gui xech choao teu kot lec keng quoan roao som lim nhuych yec soo puyet mop roan nhung bieng boanh khon phieng toeo xuam nhua leu then thut thep thuay len thoong phuyet cua guu nguech ech xui truyt hoan cinh trenh tua buop khy quat sic moong khonh nghia kheo duay pai kiem neu thiec bop roa quyn nuay hoc voa giop bia chua goang kac noong koe cuan gion huen ngich quin mao ngoem phop khai neng nhop oet coi phac qua suyn sue ngoc nghem sao ngieng thuych cec kang buyp get tuot thac choet mieng poang gioi luon gau nuech tronh khua goen viep nguop truych ngo det got guen mec gip phui pich heng nuya gioem voo sung giuom suech chem phai ghin khoon doom puc peo khoop hoet ruoi quoet nghien the nhunh quo hanh chunh trui nhue chup phieu luou xuang kuy nhoeo suop loao khuon tha xap nguon doet git trau soanh ghu cuong vam queng ton loe nguom nhiec choac chuenh nonh thiep veng pop nhun vuot vuu nech nhoan trep tuyu triep muya huam puom thoc tiem moai muyen vuen uyet xoeng monh khuang toon cop cien lung ghom gia troong tuen kun soet nuen hoay ruynh kuyen pieu cut viet oom xuon pho koc huop chong duych boa rot bieu xoai nhip cam nhieng boai xuay quoang cach ngoeo nguoi nho guoc thung bang muoc thuat muyn nhuop soai ghai phi dung nhy nhan hoap boach coon gheo von giuy suy tin tuy roang dech giut soon bue nghuong suay chuop sang tui muong khoai nghiet xep khop lau chuya cheo gho gha xua ngim diep lip xiem sieng lue nhanh luat trien suu lech goem goay huom ngoanh thoai luen phoen xuyp phuam nghy quuyet chung loc noc lui quyeng hoat kom chien luya rit sam ting nghic tuyep guyn tonh buot oeng uom gionh gic nuong khut hoen thang chuech xoon buyng cuc cuang xuyn hoep treo hin kho triet phoan xuya ngoao trip quop xut mieu poeng lang koanh khuoc nhay thoan duyn dup thoi guay khuyt nguoc phoe cham ghiep huyn khuyem keo viu giac vuyen xieu bao trom cao chuoc tue xoi khoao doeng thuen nhao vuop kue yeng not thic ciu choang ngay bienh doac xoom lich muat nghach nguat ngoay boeng ngoan uym choan trap phap ngoap suang pem chiep rai khic suyet pot thoen giuon boong roem lanh nguya huc guyu dit khieu phoa danh uan nop ran thonh khoa doon pheng sinh duot enh ghiet khuyech puo ngau ngeo nhoep nguych tuoi gep cuy gham nat soa doay truenh nhot trac gim loai coe cho siu pach rue nhoac diet xau giet nhuyet nhic mut xuenh tho chuec thai phuang choem uych thuap chuyem bai ric gioc tuam hui khung gieu nghung nhi voch khec kop xoap nguyt khuan vau xoet khuet phiem trot sup duem khang nuom gua may ngue ngheng roong ngheu uonh cuom ang hip nghieu gioao phung chang hoi choai nuy lao ceo uyen bac can xoay puoc tam cong muan nghep pip vut ruyt ooc phen guet khep ngun bem chuyep roet chu thuan khuych quoac xam trooc pan true nguyn chuyen trat vua rat mat nghu rieng sia uop nghan truon suan moan ciec buen suom soan poom thuoc vuoi nhep cap loeo troan viec thanh cha nhuom nenh giung phiu thoanh khuat uou doap guynh tim luoi quych toang nheu moc denh khuyec vuan voet kieng thay top gonh rinh xuynh cai kiec nghong coong vuon phuy truong xuen tot seng xoen thoeo nghuech cem phan chuu inh nuo quiu teo rong nguynh phiet khuyp cum xue ngut truot xic unh ruop phoac moeng oao gue ngi rop thim quoi nanh sooc truya duom roay vanh chuy run gong nhuoc bau veo nic nhen choc dui tuyem choat rac luom thum xuot chuynh riu gioa bua ruyu buu nhuoi choe gon toa quec lon thach qui vuc met puch quyem thop nep uoy mich chui nai tong kham lay xoanh toc ngop huo khoam nghuen chim nghing ghen soeo moach cuop quoay khach guyt goc luyn pung thuo tuyp dan luonh cay chuoi loat tau truu puyen guot thap dic xoong don vay tiec coc gay ruya ruon nguet sonh phoat koon ghot khinh thom boe khuo quich deu truom moac truat vuay ruong ghac quyen duyen gieo xuet thiem boan poam buyt ruenh ding xot mim xuan khue ghenh quym noai benh nuu banh hum toay puy tuyen donh dep tro nhuong moat khac mac vuet uen sin troen khoac ruyet rui pua suynh poon kip ngeu kieu lup loa nhai thoet puoi tia him phuem khuou miu buoy xuop xuch queu huap chom pom tun muyt ngua khuyu khoc tra noanh xet tria nui ven nhiem suong tach hia thuop cuot buo puon huyng ghun ngiet phonh chuon nguac choi bung quanh mun nhoen tiet duou trech kuyng thech vuem biem nuan gun renh cang ruych ghay nguc duam gec xoem loan panh treu goa lua chia sai quot tieu nghot hon buom gach buynh lenh rup coam ray truy bip vuyt tep thooc huou truyen guyp tuan ngap dinh guya muych roac koen koonh riec nhuynh roap kau nhia soam poa quit oac giuot ponh net ginh bach kuo uon toach that hoem suot vuang ngup phanh ruu chuan pham kiu hup suoc xeng ket khau xooc doanh nhec hoong cit muon dun goi khay mem pup rach moen rich rap ngach duoc uyu dia thuou buya com suam pech guym sich thuang nghim phao vit chenh ring voong nhich phech chiu nghai peng noet quip loo ghieng noem toem khich khuya tham nghuyet tum thua theo nuyet tuc ghach thoa coen thu thuu moom miec khoong khuy kem riem thue tuyet thinh oep chech than khoan looc ngham chuot oang soay cia duy thoach hunh loeng nap poat pep thuom nhuat roc room muen buy phuynh nut hoa moay trun toam khiep poi troe dat phuong phuon giuo puot tuom mang xoach biec mit gom nhinh tram noac phin xin quoanh ruo troac lom voc xuat voam ngac phenh ngenh chin nhuech non muay chuyet ling nguyu chip thip buyn toec ngat xuyu trong trach doai kech dum quoai roanh giam thit tec roe huyt tuyech goac xon boc ghat gien mech thiu nhuam khiet giun sanh theng nghet nhuay ghe choa xay oan cuat pon nhuyn quuyen xeo xuoi ban mau miep nhong truonh chon phuoi quyt too gup nhuyu hen kum thin giot truoi muet troet hiet nhuy nuynh ghich khup trao trunh goo huoc poan giep uam quuay khong phue niep khin rau ghinh nhooc uynh toan veu phe buet ngon nit tuech suyt gat suo xuu nhau reu chap gi ".strip().split())
+def none_tone_vietnamese(s): # tieng_viet_khong_dau
+    s = re.sub(u'Đ', 'D', s)
+    s = re.sub(u'đ', 'd', s)
+    s = unicodedata.normalize('NFKD', s).encode('ASCII', 'ignore')
+    return s.decode("utf-8")
+
+def vietnamese_syllable_ratio(text):
+    none_tone_text = none_tone_vietnamese(text)
+    tokens = re.findall(r'[a-z]+', none_tone_text)
+
+    if len(tokens) == 0:
+        return 0
+
+    syllable_count = 0
+
+    for token in tokens:
+        if token in none_tone_syllables:
+            syllable_count += 1
+
+    return syllable_count / len(tokens)
+
+vi_chars = { 'ớ', 'ờ', 'ụ', 'ẫ', 'ổ', 'ậ', 'ẵ', 'â', 'ặ', 'ễ', 'ọ', 'ẩ', 'ỹ', 'ẽ', 'ủ', 'ạ', 'ấ', 'ư', 'ả', 'ỉ', 
+'ỗ', 'ồ', 'ứ', 'đ', 'ự', 'è', 'ý', 'ế', 'ỵ', 'ũ', 'ắ', 'ẻ', 'ể', 'ợ', 'ệ', 'ẳ', 'ộ', 'à', 'õ', 'ĩ', 'ằ', 'ẹ', 'ỳ', 
+'é', 'ử', 'ị', 'ở', 'ỡ', 'ê', 'ầ', 'ò', 'ề', 'ố', 'ỷ', 'ă', 'ì', 'ữ', 'ơ', 'ã', 'ỏ', 'ừ', 'ù', 'ú', 'á','ô', 'í', 'ó', 
+'a', 'b', 'c', 'd', 'e', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'x', 'y',
+' ', '"', "'", ".", ",", ";", "_", "—", "-" }
+# vi_word_re = re.compile(f'[0-9a-z{"".join(vi_chars)}]+', re.IGNORECASE)
+
+def canbe_vietnamese(token):
+    for c in token.lower():
+        if c not in vi_chars:
+            return False
+    return True
+'''
+def canbe_vietnamese(token):
+    count = 0
+    for c in token.lower():
+        if c in vi_chars:
+            count += 1
+    return count / len(token) >= 0.8
+'''
+
+'''
+The 4E00—9FFF range covers CJK Unified Ideographs (CJK=Chinese, Japanese and Korean). 
+There are a number of lower ranges that relate, to some degree, to CJK:
+
+31C0—31EF CJK Strokes
+31F0—31FF Katakana Phonetic Extensions
+3200—32FF Enclosed CJK Letters and Months
+3300—33FF CJK Compatibility
+3400—4DBF CJK Unified Ideographs Extension A
+4DC0—4DFF Yijing Hexagram Symbols
+4E00—9FFF CJK Unified Ideographs 
+'''
+
+min_cjk = 11935
+# min_cjk = ord('\u31c0')
+
+max_cjk = 64055
+# max_cjk = ord('\u9fff')
+
+if __name__ ==  "__main__":
+
+    unwanted = """
+ทรูวิชั่นส์asdf, ầds tiến lên
+게시판
+활
+⽗
+臘
+怒
+辰
+⺟
+旅
+里
+拓
+見
+嘆
+有
+县
+は
+""".strip().split("\n")
+
+    for x in unwanted:
+        if not contains_unwanted(x):
+         print(x)
+         print(min_cjk, max_cjk)
+         for c in x:
+            print(ord(c), c)
+
+
+    emoji_samples = """
+🈯
+🈲
+🈹
+🌇
+🌓
+🍘
+🎑
+🎿
+🏏
+🏒
+🏩
+🏯
+🐀
+👝
+💹
+💺
+📟
+📪
+📼
+🔀🔂
+🔃
+🔇
+🔓
+🔢
+🔤🔩
+🕖
+🕚
+🕜
+🕝
+🕞
+🕠🕢
+🌍,
+ 😂, 😃,
+ 😂
+    """.strip().split("\n")
+    
+    for x in emoji_samples:
+        if not contains_emoji(x):
+            print(x, emoji_count(x))
+
+
+    vi_samples = """
+    hê hê
+    an
+    " VIỆT
+
+    """.strip().split("\n")
+
+    for x in vi_samples:
+        if not canbe_vietnamese(x):
+            print(x)