Upload 8 files

Browse files

Files changed (8) hide show

added_tokens.json +687 -0
config.json +62 -0
generation_config.json +16 -0
source.spm +0 -0
special_tokens_map.json +5 -0
target.spm +0 -0
tokenizer_config.json +12 -0
vocab.json +0 -0

added_tokens.json ADDED Viewed

	@@ -0,0 +1,687 @@

+{
+  "&&": 65033,
+  "<<": 65021,
+  "><": 65022,
+  ">>": 65008,
+  ">>>": 65020,
+  ">>>>>>>>": 65031,
+  ">>>>>>>>>>": 65030,
+  ">“": 65036,
+  ">】": 65003,
+  "|": 65001,
+  "|<": 65037,
+  "||": 65019,
+  "|||": 65028,
+  "|熾": 65348,
+  "Đú": 65041,
+  "Đạ": 65163,
+  "Đỏ": 65374,
+  "đạ": 65364,
+  "đặ": 65045,
+  "ướ": 65676,
+  "ưở": 65376,
+  "ượ": 65042,
+  "əι": 65394,
+  "ʚ᭄": 65393,
+  "̄『": 65024,
+  "এ᭄": 65383,
+  "๓": 65381,
+  "༊᭄": 65391,
+  "ཏ": 65389,
+  "ོ": 65390,
+  "ღ": 65392,
+  "ỉđị": 65365,
+  "ốđá": 65271,
+  "ụđể": 65375,
+  "‘": 65014,
+  "’": 65299,
+  "’”": 65300,
+  "“": 65005,
+  "“<": 65038,
+  "“‘": 65298,
+  "““": 65313,
+  "“”": 65320,
+  "“嗖”": 65297,
+  "“嗖嗖”": 65339,
+  "“岐": 65434,
+  "“擢": 65306,
+  "“溟": 65413,
+  "“琉": 65431,
+  "“瓒": 65600,
+  "“竽”": 65459,
+  "“缥缈": 65436,
+  "“雍": 65358,
+  "“韬": 65440,
+  "“鲲鹏": 65427,
+  "“鹊": 65437,
+  "”": 65006,
+  "”“": 65015,
+  "”凫": 65330,
+  "”炆": 65328,
+  "”苇": 65305,
+  "”雍": 65356,
+  "‰": 65013,
+  "‱": 65011,
+  "′|┛": 65035,
+  "⁣": 65111,
+  "₫": 65032,
+  "↑": 65017,
+  "→": 65004,
+  "↓": 65536,
+  "∑": 65026,
+  "√": 65016,
+  "≤": 65012,
+  "≥": 65007,
+  "≧▽≦": 65018,
+  "┗|": 65034,
+  "┗|`": 65532,
+  "╹▽╹": 65025,
+  "△": 65027,
+  "▽": 65023,
+  "◆": 65040,
+  "★": 65010,
+  "★★★★★": 65114,
+  "★★★★★★": 65104,
+  "★★★★★★★": 65115,
+  "☆☆": 65303,
+  "♀": 65009,
+  "【】": 65039,
+  "】【": 65002,
+  "つロ": 65029,
+  "シ": 65280,
+  "ジ": 65281,
+  "ヽ": 65529,
+  "䜌": 65634,
+  "丨": 65118,
+  "丨樰儛": 65264,
+  "丨灬": 65258,
+  "丨羲": 65117,
+  "丿": 65136,
+  "乄": 65066,
+  "乄楓": 65251,
+  "亓": 65665,
+  "亹": 65630,
+  "仄": 65092,
+  "仄仄": 65093,
+  "仩塰尐嗲": 65247,
+  "伱": 65078,
+  "佝偻": 65327,
+  "佞": 65357,
+  "佪": 65141,
+  "俪俪": 65137,
+  "倏": 65177,
+  "倨": 65314,
+  "偑褚": 65227,
+  "傕": 65609,
+  "儁": 65558,
+  "儁乂": 65615,
+  "儛": 65237,
+  "儷": 65229,
+  "兖": 65164,
+  "凖": 65377,
+  "凨": 65230,
+  "剡": 65182,
+  "劼": 65277,
+  "卩灬": 65226,
+  "卮": 65352,
+  "厷": 65385,
+  "叡": 65673,
+  "叱詫": 65246,
+  "吒": 65178,
+  "吡": 65097,
+  "吥噯": 65122,
+  "吽叵": 65074,
+  "呔瀦": 65255,
+  "咣": 65326,
+  "哕": 65543,
+  "唳": 65219,
+  "唵": 65073,
+  "唻": 65533,
+  "喆": 65288,
+  "嗖嗖": 65317,
+  "嗫嚅": 65325,
+  "嘤嘤嘤": 65572,
+  "噫": 65274,
+  "嚯": 65525,
+  "囍": 65216,
+  "囖": 65487,
+  "坂": 65588,
+  "垕": 65421,
+  "垚垚": 65053,
+  "埖": 65071,
+  "埸": 65537,
+  "堇": 65502,
+  "墠": 65582,
+  "壊娚魜": 65143,
+  "夙烨": 65501,
+  "姞": 65087,
+  "姹": 65171,
+  "姽婳": 65102,
+  "娆": 65043,
+  "娑": 65345,
+  "娟娟": 65158,
+  "娲”": 65426,
+  "娿朶": 65262,
+  "婧": 65659,
+  "婲": 65239,
+  "婳": 65133,
+  "婵娟”": 65460,
+  "嫑": 65145,
+  "嫑忈": 65120,
+  "嫑烎": 65124,
+  "嫚": 65291,
+  "嫦": 65172,
+  "嬿": 65541,
+  "孒": 65261,
+  "宓": 65612,
+  "宙": 65185,
+  "尐": 65101,
+  "尐尐": 65284,
+  "尐菟": 65079,
+  "尴": 65491,
+  "屮": 65236,
+  "岚": 65046,
+  "岚峪": 65311,
+  "岚鲲": 65272,
+  "岫瑶": 65059,
+  "岷": 65602,
+  "峥": 65482,
+  "峥嵘": 65064,
+  "峯": 65096,
+  "峸姒涳": 65085,
+  "嵇": 65474,
+  "嵋": 65455,
+  "嶷": 65362,
+  "嶽": 65654,
+  "巂": 65618,
+  "巍巍": 65260,
+  "巭": 65123,
+  "巽": 65165,
+  "帙": 65337,
+  "帜": 65347,
+  "帻": 65684,
+  "幄”": 65422,
+  "廮": 65633,
+  "彘": 65208,
+  "彡": 65095,
+  "彧": 65506,
+  "彽": 65266,
+  "徕": 65308,
+  "徭": 65675,
+  "忻": 65657,
+  "怫": 65293,
+  "怼": 65075,
+  "恹": 65359,
+  "惇": 65547,
+  "愔愔": 65488,
+  "愠": 65316,
+  "愺": 65127,
+  "懐": 65100,
+  "懿萱": 65388,
+  "懿鞷畿矲魉肐嚪": 65099,
+  "戕": 65340,
+  "戢": 65668,
+  "戥锌": 65511,
+  "抔": 65335,
+  "抟": 65448,
+  "挲": 65295,
+  "捭阖": 65516,
+  "揟": 65627,
+  "摈": 65534,
+  "擞": 65294,
+  "斄": 65596,
+  "旌": 65369,
+  "旒": 65402,
+  "旖旎": 65201,
+  "旡": 65150,
+  "旲": 65289,
+  "昀": 65238,
+  "昕妤": 65671,
+  "昙栀": 65148,
+  "昝": 65655,
+  "昮": 65275,
+  "昱": 65481,
+  "晔": 65500,
+  "晖": 65198,
+  "暝": 65467,
+  "曌": 65142,
+  "曜焱": 65189,
+  "杓": 65050,
+  "柒": 65058,
+  "栀": 65077,
+  "栾璇": 65403,
+  "桀": 65430,
+  "桀桀": 65498,
+  "桀桀桀": 65184,
+  "桀桀桀桀桀桀桀桀桀桀桀桀桀桀桀": 65400,
+  "梓潼": 65586,
+  "梼杌": 65408,
+  "梼杌”": 65409,
+  "棠钗": 65218,
+  "榀婲刄": 65252,
+  "槊": 65538,
+  "槿": 65160,
+  "樯橹": 65576,
+  "樾": 65057,
+  "檎": 65371,
+  "歆": 65682,
+  "歙砚": 65472,
+  "殂": 65483,
+  "殇": 65068,
+  "殍": 65578,
+  "殚": 65580,
+  "殤": 65225,
+  "氅": 65175,
+  "氤氲": 65116,
+  "氿": 65159,
+  "汕": 65589,
+  "汾": 65622,
+  "沅": 65647,
+  "沅澧": 65646,
+  "沔": 65569,
+  "沚": 65542,
+  "沝沝": 65051,
+  "沵嗰卟": 65110,
+  "泓": 65442,
+  "泗": 65555,
+  "泠泠": 65445,
+  "泤": 65047,
+  "泫": 65527,
+  "泱": 65144,
+  "泷恬": 65656,
+  "泸": 65620,
+  "泾": 65625,
+  "洄": 65346,
+  "洭": 65648,
+  "洮": 65601,
+  "洮洮": 65672,
+  "洹": 65194,
+  "浈": 65649,
+  "浔": 65135,
+  "涞": 65585,
+  "涟": 65296,
+  "涿": 65553,
+  "淄": 65463,
+  "淝": 65638,
+  "淦岚": 65664,
+  "淼": 65170,
+  "淼淼": 65149,
+  "淼焱垚": 65153,
+  "渃": 65304,
+  "渑": 65202,
+  "湫": 65112,
+  "溏": 65495,
+  "溟": 65428,
+  "滁": 65128,
+  "滹沱": 65584,
+  "漓”": 65433,
+  "漪": 65342,
+  "漳": 65624,
+  "潼": 65568,
+  "澹": 65282,
+  "澹澹": 65180,
+  "濮": 65283,
+  "濯": 65322,
+  "瀦": 65240,
+  "灬": 65076,
+  "炁": 65089,
+  "炜": 65056,
+  "焱焱": 65199,
+  "煅": 65378,
+  "煊": 65139,
+  "煜”": 65420,
+  "煜煜": 65452,
+  "煭": 65485,
+  "熾": 65223,
+  "燔": 65539,
+  "燚燚": 65200,
+  "爰": 65146,
+  "爻": 65190,
+  "牂": 65652,
+  "牂牁": 65617,
+  "牍": 65212,
+  "牖": 65590,
+  "犼": 65191,
+  "犼”": 65415,
+  "狍": 65552,
+  "狞": 65315,
+  "狰": 65210,
+  "狻猊": 65372,
+  "猇": 65683,
+  "猞": 65591,
+  "猼訑": 65209,
+  "玎珰": 65187,
+  "玑": 65292,
+  "玖掵": 65253,
+  "珏": 65179,
+  "珐": 65524,
+  "珪": 65192,
+  "琊": 65119,
+  "琏": 65341,
+  "琰": 65509,
+  "琲": 65660,
+  "瑶”": 65412,
+  "瑶瑶": 65063,
+  "璇": 65176,
+  "璇玑": 65195,
+  "璇玑缰": 65220,
+  "璟": 65505,
+  "璿": 65233,
+  "瓒": 65592,
+  "瓘": 65545,
+  "甬": 65573,
+  "瘗": 65540,
+  "皖": 65276,
+  "眦": 65361,
+  "睚眦": 65549,
+  "睥睨": 65156,
+  "睨": 65060,
+  "瞿": 65417,
+  "矍铄": 65332,
+  "矓篂篁": 65234,
+  "矝": 65131,
+  "砚": 65458,
+  "砥砺": 65606,
+  "碣": 65561,
+  "祁": 65248,
+  "祁汜": 65510,
+  "祗": 65301,
+  "祧": 65581,
+  "祯": 65287,
+  "祺": 65499,
+  "祺韬": 65513,
+  "禹嫒": 65669,
+  "禺": 65650,
+  "秩": 65193,
+  "穰": 65674,
+  "窅": 65515,
+  "窸窸窣窣": 65344,
+  "竦": 65605,
+  "笄": 65595,
+  "笪": 65667,
+  "笳": 65594,
+  "筠": 65479,
+  "筠筠": 65213,
+  "筱": 65090,
+  "筱筱": 65152,
+  "箓": 65526,
+  "箚": 65628,
+  "篁": 65523,
+  "篆": 65468,
+  "篾": 65360,
+  "簌": 65329,
+  "簟": 65680,
+  "粤灬": 65147,
+  "粱": 65368,
+  "粳": 65449,
+  "粼粼": 65531,
+  "糍": 65496,
+  "綄": 65263,
+  "繇": 65598,
+  "纇": 65106,
+  "绉绉": 65608,
+  "绛皓": 65173,
+  "绛阙": 65181,
+  "绡": 65443,
+  "绶|": 65583,
+  "缃": 65446,
+  "缥": 65517,
+  "缨”": 65441,
+  "缯": 65637,
+  "缱绻": 65530,
+  "罡": 65125,
+  "罡缰": 65221,
+  "羌氐": 65603,
+  "羲”": 65425,
+  "羿": 65366,
+  "翊": 65503,
+  "翎”": 65414,
+  "翙翙": 65399,
+  "翥": 65497,
+  "翦": 65666,
+  "耒": 65621,
+  "胤": 65480,
+  "胤禛": 65507,
+  "胪": 65551,
+  "脩": 65616,
+  "腭": 65310,
+  "膂": 65613,
+  "膑": 65565,
+  "臧": 65571,
+  "臾": 65439,
+  "艿": 65447,
+  "芈": 65478,
+  "芊": 65113,
+  "芊芊": 65052,
+  "芍陂": 65640,
+  "芷": 65242,
+  "苋": 65435,
+  "苌": 65662,
+  "茕茕": 65061,
+  "茗": 65126,
+  "荀彧": 65548,
+  "荑": 65519,
+  "荔": 65557,
+  "荟": 65639,
+  "莒": 65636,
+  "莜": 65084,
+  "莴笋": 65438,
+  "莹莹": 65183,
+  "菖": 65444,
+  "菟": 65560,
+  "菡": 65154,
+  "菿": 65070,
+  "萱": 65138,
+  "萱|": 65486,
+  "萱”": 65411,
+  "萱萱": 65214,
+  "蒹葭": 65105,
+  "蒹葭”": 65432,
+  "蒺藜": 65611,
+  "蓁蓁": 65577,
+  "蓐": 65475,
+  "蓑": 65130,
+  "蓦": 65134,
+  "蓧": 65231,
+  "蓼": 65174,
+  "蔺": 65203,
+  "蕖": 65450,
+  "蕟蕟蕟蕟蕟蕟": 65098,
+  "蕲": 65644,
+  "蘰": 65055,
+  "蘼": 65062,
+  "虬": 65333,
+  "蚓": 65269,
+  "蚩": 65108,
+  "蚯蚓”": 65453,
+  "蛟”": 65416,
+  "蛟缳": 65217,
+  "蛟蛟": 65235,
+  "蜉蝣": 65157,
+  "蜮": 65302,
+  "蝂丄": 65245,
+  "蝮": 65354,
+  "蝰": 65465,
+  "螭": 65367,
+  "蠡": 65456,
+  "衢”": 65641,
+  "衤": 65279,
+  "衲": 65086,
+  "衹": 65514,
+  "衿": 65109,
+  "袔": 65493,
+  "袪": 65069,
+  "裀": 65678,
+  "裾": 65349,
+  "褚": 65353,
+  "覌": 65243,
+  "觞": 65082,
+  "觥": 65380,
+  "訞": 65080,
+  "訾": 65286,
+  "誮": 65094,
+  "誾": 65492,
+  "謸氕": 65088,
+  "讙": 65206,
+  "诔": 65103,
+  "谆谆": 65535,
+  "谑": 65323,
+  "谗": 65518,
+  "谡": 65599,
+  "谥": 65554,
+  "谪": 65067,
+  "谯": 65544,
+  "豊": 65268,
+  "貐": 65270,
+  "貔貅": 65204,
+  "貔貅”": 65406,
+  "貔貅鲲": 65397,
+  "貔貔": 65081,
+  "貔龛": 65685,
+  "贲": 65587,
+  "赧": 65334,
+  "跁萢": 65244,
+  "跄跄": 65490,
+  "跹": 65370,
+  "踟蹰": 65312,
+  "蹉": 65670,
+  "蹉跎": 65677,
+  "蹙": 65331,
+  "軒轅": 65259,
+  "轪": 65643,
+  "轲": 65168,
+  "轵": 65631,
+  "轸": 65562,
+  "轼": 65461,
+  "轼暧": 65508,
+  "辇": 65521,
+  "辎": 65579,
+  "辕”": 65457,
+  "迩": 65083,
+  "逄": 65466,
+  "逡": 65319,
+  "逺": 65091,
+  "遐迩": 65373,
+  "邛": 65619,
+  "邯": 65623,
+  "邳": 65559,
+  "邶": 65477,
+  "邾": 65642,
+  "郃": 65607,
+  "郜": 65661,
+  "郢": 65564,
+  "郦": 65476,
+  "郿": 65610,
+  "鄡": 65635,
+  "鄱": 65454,
+  "鄳": 65645,
+  "酆": 65285,
+  "酎": 65351,
+  "酞": 65470,
+  "酤": 65350,
+  "酹": 65065,
+  "醮": 65614,
+  "釭": 65570,
+  "釭”": 65574,
+  "鈺彡": 65107,
+  "鈽": 65241,
+  "銃": 65256,
+  "鍀": 65267,
+  "鎏": 65167,
+  "鎩": 65048,
+  "鎵尐": 65232,
+  "鐜": 65129,
+  "鑫祺": 65512,
+  "钊": 65471,
+  "钭": 65658,
+  "钲": 65593,
+  "铮": 65307,
+  "铮铮": 65318,
+  "铿": 65321,
+  "锱铢": 65520,
+  "锵": 65489,
+  "镂": 65169,
+  "镔": 65653,
+  "镡": 65651,
+  "閪": 65132,
+  "閽": 65257,
+  "闕": 65224,
+  "闫": 65343,
+  "闾": 65473,
+  "阆": 65626,
+  "阈": 65222,
+  "阊阖": 65401,
+  "阙": 65044,
+  "阚": 65278,
+  "阮馥": 65273,
+  "隗": 65405,
+  "雒": 65632,
+  "雩": 65049,
+  "霁": 65186,
+  "霁琨": 65504,
+  "霆": 65324,
+  "霏": 65484,
+  "霏霏": 65563,
+  "鞅": 65418,
+  "鞅”": 65419,
+  "鞯": 65604,
+  "韬": 65379,
+  "顁": 65121,
+  "颀": 65309,
+  "颉": 65211,
+  "颍": 65556,
+  "颔": 65355,
+  "颙": 65207,
+  "颭": 65679,
+  "飒": 65054,
+  "馍": 65451,
+  "馗": 65290,
+  "驤": 65228,
+  "驺": 65681,
+  "骊": 65663,
+  "骓": 65550,
+  "骝": 65567,
+  "骥": 65429,
+  "骧": 65188,
+  "髍": 65151,
+  "髫": 65597,
+  "髯": 65522,
+  "鬃骥": 65566,
+  "鬄": 65546,
+  "鬘": 65575,
+  "鬟": 65166,
+  "魃": 65407,
+  "魈": 65363,
+  "魈”": 65410,
+  "魍魉魓": 65215,
+  "魑": 65072,
+  "魖": 65254,
+  "鯤": 65398,
+  "鯤鵬": 65249,
+  "鲫": 65464,
+  "鲲”": 65424,
+  "鲲臻": 65404,
+  "鲲遨": 65494,
+  "鲲鲲": 65395,
+  "鲲鹏": 65162,
+  "鲵": 65462,
+  "鵬": 65265,
+  "鵸鵌": 65205,
+  "鷇": 65197,
+  "鸾翎": 65469,
+  "鹄": 65155,
+  "鹏鲲": 65396,
+  "鹏鹏": 65161,
+  "鹑": 65196,
+  "鹯": 65629,
+  "麟”": 65423,
+  "黒皕玍萿": 65250,
+  "黾": 65336,
+  "齑": 65338,
+  "龗": 65140,
+  "ꦿএ": 65384,
+  "ꦿଓ": 65528,
+  "ꦿེ࿐": 65382,
+  "꧁厷": 65386,
+  "꧂": 65387
+}

config.json ADDED Viewed

	@@ -0,0 +1,62 @@

+{
+  "_name_or_path": "/kaggle/input/marianmt-expanded-vocab/opus-mt-zh-vi-finetuned-translation-to-target/checkpoint-52500",
+  "activation_dropout": 0.0,
+  "activation_function": "swish",
+  "add_bias_logits": false,
+  "add_final_layer_norm": false,
+  "architectures": [
+    "MarianMTModel"
+  ],
+  "attention_dropout": 0.0,
+  "bad_words_ids": [
+    [
+      65000
+    ]
+  ],
+  "bos_token_id": 0,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "d_model": 512,
+  "decoder_attention_heads": 8,
+  "decoder_ffn_dim": 2048,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 6,
+  "decoder_start_token_id": 65000,
+  "decoder_vocab_size": 65686,
+  "dropout": 0.1,
+  "encoder_attention_heads": 8,
+  "encoder_ffn_dim": 2048,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 6,
+  "eos_token_id": 0,
+  "extra_pos_embeddings": 65686,
+  "forced_eos_token_id": 0,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "marian",
+  "normalize_before": false,
+  "normalize_embedding": false,
+  "num_beams": 6,
+  "num_hidden_layers": 6,
+  "pad_token_id": 65000,
+  "renormalize_logits": true,
+  "scale_embedding": true,
+  "share_encoder_decoder_embeddings": true,
+  "static_position_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.33.0",
+  "use_cache": true,
+  "vocab_size": 65686
+}

generation_config.json ADDED Viewed

	@@ -0,0 +1,16 @@

+{
+  "bad_words_ids": [
+    [
+      65000
+    ]
+  ],
+  "bos_token_id": 0,
+  "decoder_start_token_id": 65000,
+  "eos_token_id": 0,
+  "forced_eos_token_id": 0,
+  "max_length": 512,
+  "num_beams": 6,
+  "pad_token_id": 65000,
+  "renormalize_logits": true,
+  "transformers_version": "4.33.0"
+}

source.spm ADDED Viewed

Binary file (750 kB). View file

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,5 @@

+{
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

target.spm ADDED Viewed

Binary file (766 kB). View file

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,12 @@

+{
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "separate_vocabs": false,
+  "source_lang": "zho",
+  "sp_model_kwargs": {},
+  "target_lang": "vie",
+  "tokenizer_class": "MarianTokenizer",
+  "unk_token": "<unk>"
+}

vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff