tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 0,
      "content": "<s>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 1,
      "content": "<pad>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 2,
      "content": "</s>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 3,
      "content": "<unk>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 4,
      "content": "<mask>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 124,
      "content": "[PAD]",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    }
  ],
  "normalizer": null,
  "pre_tokenizer": {
    "type": "Whitespace"
  },
  "post_processor": null,
  "decoder": null,
  "model": {
    "type": "BPE",
    "dropout": null,
    "unk_token": null,
    "continuing_subword_prefix": null,
    "end_of_word_suffix": null,
    "fuse_unk": false,
    "byte_fallback": false,
    "ignore_merges": false,
    "vocab": {
      "<s>": 0,
      "<pad>": 1,
      "</s>": 2,
      "<unk>": 3,
      "<mask>": 4,
      "(": 5,
      ")": 6,
      ",": 7,
      "0": 8,
      "1": 9,
      ":": 10,
      "<": 11,
      "=": 12,
      ">": 13,
      "A": 14,
      "C": 15,
      "D": 16,
      "E": 17,
      "F": 18,
      "G": 19,
      "H": 20,
      "I": 21,
      "L": 22,
      "M": 23,
      "N": 24,
      "O": 25,
      "P": 26,
      "R": 27,
      "S": 28,
      "T": 29,
      "U": 30,
      "a": 31,
      "e": 32,
      "f": 33,
      "g": 34,
      "i": 35,
      "l": 36,
      "m": 37,
      "n": 38,
      "p": 39,
      "r": 40,
      "s": 41,
      "t": 42,
      "u": 43,
      "v": 44,
      "y": 45,
      "{": 46,
      "}": 47,
      "{{": 48,
      "}}": 49,
      "am": 50,
      "nam": 51,
      "name": 52,
      "pe": 53,
      "ty": 54,
      "type": 55,
      "va": 56,
      "lu": 57,
      "valu": 58,
      "value": 59,
      "RE": 60,
      "ar": 61,
      "par": 62,
      "}}(": 63,
      "am0": 64,
      "type0": 65,
      "param0": 66,
      "CE": 67,
      "DU": 68,
      "OCE": 69,
      "PR": 70,
      "et": 71,
      "fu": 72,
      "rn": 73,
      "ret": 74,
      "urn": 75,
      "}})": 76,
      "DURE": 77,
      "OCEDURE": 78,
      "PROCEDURE": 79,
      "fun": 80,
      "return": 81,
      "AL": 82,
      "CAL": 83,
      "EN": 84,
      "type1": 85,
      "}}()": 86,
      "}}({{": 87,
      "CALL": 88,
      "({{": 89,
      ",{{": 90,
      "<{{": 91,
      ">)": 92,
      "AI": 93,
      "AS": 94,
      "ARE": 95,
      "CL": 96,
      "DE": 97,
      "DO": 98,
      "GN": 99,
      "HEN": 100,
      "IF": 101,
      "IGN": 102,
      "MAI": 103,
      "RN": 104,
      "SIGN": 105,
      "TU": 106,
      "THEN": 107,
      "ai": 108,
      "gs": 109,
      "if": 110,
      "mai": 111,
      "}},{{": 112,
      "}}>)": 113,
      "var": 114,
      "RETU": 115,
      "args": 116,
      "END": 117,
      "ASSIGN": 118,
      "CLARE": 119,
      "DECLARE": 120,
      "MAIN": 121,
      "main": 122,
      "RETURN": 123
    },
    "merges": [
      "{ {",
      "} }",
      "a m",
      "n am",
      "nam e",
      "p e",
      "t y",
      "ty pe",
      "v a",
      "l u",
      "va lu",
      "valu e",
      "R E",
      "a r",
      "p ar",
      "}} (",
      "am 0",
      "type 0",
      "par am0",
      "C E",
      "D U",
      "O CE",
      "P R",
      "e t",
      "f u",
      "r n",
      "r et",
      "u rn",
      "}} )",
      "DU RE",
      "OCE DURE",
      "PR OCEDURE",
      "fu n",
      "ret urn",
      "A L",
      "C AL",
      "E N",
      "type 1",
      "}}( )",
      "}}( {{",
      "CAL L",
      "( {{",
      ", {{",
      "< {{",
      "> )",
      "A I",
      "A S",
      "A RE",
      "C L",
      "D E",
      "D O",
      "G N",
      "H EN",
      "I F",
      "I GN",
      "M AI",
      "R N",
      "S IGN",
      "T U",
      "T HEN",
      "a i",
      "g s",
      "i f",
      "m ai",
      "}} ,{{",
      "}} >)",
      "va r",
      "RE TU",
      "ar gs",
      "EN D",
      "AS SIGN",
      "CL ARE",
      "DE CLARE",
      "MAI N",
      "mai n",
      "RETU RN"
    ]
  }
}