Hugging Face's logo

Marialab
/
finetuned-seamless-m4T-large-1000-step

Feature Extraction
Transformers
Safetensors
Arabic
seamless_m4t_v2
Generated from Trainer
Inference Endpoints
Model card Files Community
Marialab commited on
Commit
2bdf07b
·
verified ·
1 Parent(s): 4ff8cad

End of training

Browse files
Files changed (14) hide show
  1. .gitattributes +1 -0
  2. README.md +86 -0
  3. added_tokens.json +100 -0
  4. config.json +124 -0
  5. generation_config.json +186 -0
  6. model-00001-of-00002.safetensors +3 -0
  7. model-00002-of-00002.safetensors +3 -0
  8. model.safetensors.index.json +0 -0
  9. preprocessor_config.json +111 -0
  10. sentencepiece.bpe.model +3 -0
  11. special_tokens_map.json +148 -0
  12. tokenizer.json +3 -0
  13. tokenizer_config.json +939 -0
  14. training_args.bin +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ language:
4
+ - ar
5
+ license: cc-by-nc-4.0
6
+ base_model: facebook/seamless-m4t-large
7
+ tags:
8
+ - generated_from_trainer
9
+ datasets:
10
+ - darija-c
11
+ metrics:
12
+ - bleu
13
+ model-index:
14
+ - name: Finetuned seamless-m4t-large for darija speech translation
15
+ results: []
16
+ ---
17
+
18
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
19
+ should probably proofread and complete it, then remove this comment. -->
20
+
21
+ # Finetuned seamless-m4t-large for darija speech translation
22
+
23
+ This model is a fine-tuned version of [facebook/seamless-m4t-large](https://huggingface.co/facebook/seamless-m4t-large) on the Darija-C dataset.
24
+ It achieves the following results on the evaluation set:
25
+ - Loss: 1.5449
26
+ - Bleu: 0.1562
27
+
28
+ ## Model description
29
+
30
+ More information needed
31
+
32
+ ## Intended uses & limitations
33
+
34
+ More information needed
35
+
36
+ ## Training and evaluation data
37
+
38
+ More information needed
39
+
40
+ ## Training procedure
41
+
42
+ ### Training hyperparameters
43
+
44
+ The following hyperparameters were used during training:
45
+ - learning_rate: 1e-05
46
+ - train_batch_size: 16
47
+ - eval_batch_size: 4
48
+ - seed: 42
49
+ - optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
50
+ - lr_scheduler_type: linear
51
+ - lr_scheduler_warmup_steps: 100
52
+ - training_steps: 1000
53
+ - mixed_precision_training: Native AMP
54
+
55
+ ### Training results
56
+
57
+ | Training Loss | Epoch | Step | Validation Loss | Bleu |
58
+ |:-------------:|:-----:|:----:|:---------------:|:------:|
59
+ | 10.1923 | 12.5 | 50 | 7.8012 | 0.0 |
60
+ | 7.0388 | 25.0 | 100 | 4.9095 | 0.0023 |
61
+ | 4.1866 | 37.5 | 150 | 3.0521 | 0.0276 |
62
+ | 3.1409 | 50.0 | 200 | 2.4410 | 0.0653 |
63
+ | 2.581 | 62.5 | 250 | 2.1347 | 0.0795 |
64
+ | 2.2369 | 75.0 | 300 | 2.0097 | 0.0765 |
65
+ | 2.171 | 87.5 | 350 | 1.9385 | 0.0637 |
66
+ | 2.0371 | 100.0 | 400 | 1.8743 | 0.0907 |
67
+ | 1.9672 | 112.5 | 450 | 1.8273 | 0.0943 |
68
+ | 1.9463 | 125.0 | 500 | 1.7789 | 0.1028 |
69
+ | 1.8821 | 137.5 | 550 | 1.7346 | 0.1137 |
70
+ | 1.8424 | 150.0 | 600 | 1.7088 | 0.1208 |
71
+ | 1.836 | 162.5 | 650 | 1.6618 | 0.1470 |
72
+ | 1.757 | 175.0 | 700 | 1.6311 | 0.1347 |
73
+ | 1.7532 | 187.5 | 750 | 1.6137 | 0.1561 |
74
+ | 1.8539 | 200.0 | 800 | 1.5977 | 0.1184 |
75
+ | 1.7176 | 212.5 | 850 | 1.5758 | 0.1205 |
76
+ | 1.6944 | 225.0 | 900 | 1.5604 | 0.1396 |
77
+ | 1.7136 | 237.5 | 950 | 1.5521 | 0.1562 |
78
+ | 1.7354 | 250.0 | 1000 | 1.5449 | 0.1562 |
79
+
80
+
81
+ ### Framework versions
82
+
83
+ - Transformers 4.47.1
84
+ - Pytorch 2.5.1+cu121
85
+ - Datasets 3.2.0
86
+ - Tokenizers 0.21.0
added_tokens.json ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "__afr__": 256001,
3
+ "__amh__": 256002,
4
+ "__arb__": 256003,
5
+ "__ary__": 256004,
6
+ "__arz__": 256005,
7
+ "__asm__": 256006,
8
+ "__azj__": 256007,
9
+ "__bel__": 256008,
10
+ "__ben__": 256009,
11
+ "__bos__": 256010,
12
+ "__bul__": 256011,
13
+ "__cat__": 256012,
14
+ "__ceb__": 256013,
15
+ "__ces__": 256014,
16
+ "__ckb__": 256015,
17
+ "__cmn_Hant__": 256017,
18
+ "__cmn__": 256016,
19
+ "__cym__": 256018,
20
+ "__dan__": 256019,
21
+ "__deu__": 256020,
22
+ "__ell__": 256021,
23
+ "__eng__": 256022,
24
+ "__est__": 256023,
25
+ "__eus__": 256024,
26
+ "__fin__": 256025,
27
+ "__fra__": 256026,
28
+ "__fuv__": 256027,
29
+ "__gaz__": 256028,
30
+ "__gle__": 256029,
31
+ "__glg__": 256030,
32
+ "__guj__": 256031,
33
+ "__heb__": 256032,
34
+ "__hin__": 256033,
35
+ "__hrv__": 256034,
36
+ "__hun__": 256035,
37
+ "__hye__": 256036,
38
+ "__ibo__": 256037,
39
+ "__ind__": 256038,
40
+ "__isl__": 256039,
41
+ "__ita__": 256040,
42
+ "__jav__": 256041,
43
+ "__jpn__": 256042,
44
+ "__kan__": 256043,
45
+ "__kat__": 256044,
46
+ "__kaz__": 256045,
47
+ "__khk__": 256046,
48
+ "__khm__": 256047,
49
+ "__kir__": 256048,
50
+ "__kor__": 256049,
51
+ "__lao__": 256050,
52
+ "__lit__": 256051,
53
+ "__lug__": 256052,
54
+ "__luo__": 256053,
55
+ "__lvs__": 256054,
56
+ "__mai__": 256055,
57
+ "__mal__": 256056,
58
+ "__mar__": 256057,
59
+ "__mkd__": 256058,
60
+ "__mlt__": 256059,
61
+ "__mni__": 256060,
62
+ "__mya__": 256061,
63
+ "__nld__": 256062,
64
+ "__nno__": 256063,
65
+ "__nob__": 256064,
66
+ "__npi__": 256065,
67
+ "__nya__": 256066,
68
+ "__ory__": 256067,
69
+ "__pan__": 256068,
70
+ "__pbt__": 256069,
71
+ "__pes__": 256070,
72
+ "__pol__": 256071,
73
+ "__por__": 256072,
74
+ "__ron__": 256073,
75
+ "__rus__": 256074,
76
+ "__sat__": 256075,
77
+ "__slk__": 256076,
78
+ "__slv__": 256077,
79
+ "__sna__": 256078,
80
+ "__snd__": 256079,
81
+ "__som__": 256080,
82
+ "__spa__": 256081,
83
+ "__srp__": 256082,
84
+ "__swe__": 256083,
85
+ "__swh__": 256084,
86
+ "__tam__": 256085,
87
+ "__tel__": 256086,
88
+ "__tgk__": 256087,
89
+ "__tgl__": 256088,
90
+ "__tha__": 256089,
91
+ "__tur__": 256090,
92
+ "__ukr__": 256091,
93
+ "__urd__": 256092,
94
+ "__uzn__": 256093,
95
+ "__vie__": 256094,
96
+ "__yor__": 256095,
97
+ "__yue__": 256096,
98
+ "__zlm__": 256097,
99
+ "__zul__": 256098
100
+ }
config.json ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "facebook/hf-seamless-m4t-large",
3
+ "activation_dropout": 0.0,
4
+ "activation_function": "relu",
5
+ "adaptor_dropout": 0.1,
6
+ "adaptor_kernel_size": 8,
7
+ "adaptor_stride": 8,
8
+ "add_adapter": true,
9
+ "architectures": [
10
+ "SeamlessM4Tv2Model"
11
+ ],
12
+ "attention_dropout": 0.1,
13
+ "bos_token_id": 2,
14
+ "char_vocab_size": 10943,
15
+ "conv_depthwise_kernel_size": 31,
16
+ "decoder_attention_heads": 16,
17
+ "decoder_ffn_dim": 8192,
18
+ "decoder_layerdrop": 0.05,
19
+ "decoder_layers": 24,
20
+ "decoder_start_token_id": 3,
21
+ "dropout": 0.1,
22
+ "encoder_attention_heads": 16,
23
+ "encoder_ffn_dim": 8192,
24
+ "encoder_layerdrop": 0.05,
25
+ "encoder_layers": 24,
26
+ "eos_token_id": 3,
27
+ "feature_projection_input_dim": 160,
28
+ "hidden_size": 1024,
29
+ "initializer_range": 0.02,
30
+ "is_encoder_decoder": true,
31
+ "lang_embed_dim": 256,
32
+ "layer_norm_eps": 1e-05,
33
+ "leaky_relu_slope": 0.1,
34
+ "left_max_position_embeddings": 64,
35
+ "max_new_tokens": 256,
36
+ "max_position_embeddings": 1024,
37
+ "max_source_positions": 4096,
38
+ "model_type": "seamless_m4t_v2",
39
+ "num_adapter_layers": 1,
40
+ "num_attention_heads": 16,
41
+ "num_conv_pos_embedding_groups": 16,
42
+ "num_conv_pos_embeddings": 128,
43
+ "num_hidden_layers": 24,
44
+ "pad_token_id": 0,
45
+ "position_embeddings_type": "relative",
46
+ "resblock_dilation_sizes": [
47
+ [
48
+ 1,
49
+ 3,
50
+ 5
51
+ ],
52
+ [
53
+ 1,
54
+ 3,
55
+ 5
56
+ ],
57
+ [
58
+ 1,
59
+ 3,
60
+ 5
61
+ ]
62
+ ],
63
+ "resblock_kernel_sizes": [
64
+ 3,
65
+ 7,
66
+ 11
67
+ ],
68
+ "right_max_position_embeddings": 8,
69
+ "rotary_embedding_base": 10000,
70
+ "sampling_rate": 16000,
71
+ "scale_embedding": true,
72
+ "speech_encoder_attention_heads": 16,
73
+ "speech_encoder_chunk_size": 20000,
74
+ "speech_encoder_dropout": 0.0,
75
+ "speech_encoder_hidden_act": "swish",
76
+ "speech_encoder_intermediate_size": 4096,
77
+ "speech_encoder_layerdrop": 0.1,
78
+ "speech_encoder_layers": 24,
79
+ "speech_encoder_left_chunk_num": 128,
80
+ "spkr_embed_dim": 256,
81
+ "t2u_bos_token_id": 0,
82
+ "t2u_decoder_attention_heads": 16,
83
+ "t2u_decoder_ffn_dim": 8192,
84
+ "t2u_decoder_layers": 6,
85
+ "t2u_decoder_start_token_id": 2,
86
+ "t2u_encoder_attention_heads": 16,
87
+ "t2u_encoder_ffn_dim": 8192,
88
+ "t2u_encoder_layers": 6,
89
+ "t2u_eos_token_id": 2,
90
+ "t2u_max_new_tokens": 1024,
91
+ "t2u_max_position_embeddings": 2048,
92
+ "t2u_pad_token_id": 1,
93
+ "t2u_variance_pred_dropout": 0.5,
94
+ "t2u_variance_predictor_embed_dim": 1024,
95
+ "t2u_variance_predictor_hidden_dim": 256,
96
+ "t2u_variance_predictor_kernel_size": 3,
97
+ "t2u_vocab_size": 10082,
98
+ "torch_dtype": "float32",
99
+ "transformers_version": "4.47.1",
100
+ "unit_embed_dim": 1280,
101
+ "unit_hifi_gan_vocab_size": 10000,
102
+ "upsample_initial_channel": 512,
103
+ "upsample_kernel_sizes": [
104
+ 11,
105
+ 8,
106
+ 8,
107
+ 4,
108
+ 4
109
+ ],
110
+ "upsample_rates": [
111
+ 5,
112
+ 4,
113
+ 4,
114
+ 2,
115
+ 2
116
+ ],
117
+ "use_cache": true,
118
+ "var_pred_dropout": 0.5,
119
+ "variance_predictor_kernel_size": 3,
120
+ "vocab_size": 256102,
121
+ "vocoder_num_langs": 36,
122
+ "vocoder_num_spkrs": 200,
123
+ "vocoder_offset": 4
124
+ }
generation_config.json ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 2,
3
+ "decoder_start_token_id": 3,
4
+ "eos_token_id": 3,
5
+ "max_new_tokens": 256,
6
+ "pad_token_id": 0,
7
+ "t2u_lang_code_to_id": {
8
+ "arb": 10043,
9
+ "ben": 10044,
10
+ "cat": 10045,
11
+ "ces": 10046,
12
+ "cmn": 10047,
13
+ "cym": 10048,
14
+ "dan": 10049,
15
+ "deu": 10050,
16
+ "eng": 10051,
17
+ "est": 10052,
18
+ "fin": 10053,
19
+ "fra": 10054,
20
+ "hin": 10055,
21
+ "ind": 10056,
22
+ "ita": 10057,
23
+ "jpn": 10058,
24
+ "kan": 10059,
25
+ "kor": 10060,
26
+ "mlt": 10061,
27
+ "nld": 10062,
28
+ "pes": 10063,
29
+ "pol": 10064,
30
+ "por": 10065,
31
+ "ron": 10066,
32
+ "rus": 10067,
33
+ "slk": 10068,
34
+ "spa": 10069,
35
+ "swe": 10070,
36
+ "swh": 10071,
37
+ "tam": 10072,
38
+ "tel": 10073,
39
+ "tgl": 10074,
40
+ "tha": 10075,
41
+ "tur": 10076,
42
+ "ukr": 10077,
43
+ "urd": 10078,
44
+ "uzn": 10079,
45
+ "vie": 10080
46
+ },
47
+ "text_decoder_lang_to_code_id": {
48
+ "afr": 256001,
49
+ "amh": 256002,
50
+ "arb": 256003,
51
+ "ary": 256004,
52
+ "arz": 256005,
53
+ "asm": 256006,
54
+ "azj": 256007,
55
+ "bel": 256008,
56
+ "ben": 256009,
57
+ "bos": 256010,
58
+ "bul": 256011,
59
+ "cat": 256012,
60
+ "ceb": 256013,
61
+ "ces": 256014,
62
+ "ckb": 256015,
63
+ "cmn": 256016,
64
+ "cmn_Hant": 256017,
65
+ "cym": 256018,
66
+ "dan": 256019,
67
+ "deu": 256020,
68
+ "ell": 256021,
69
+ "eng": 256022,
70
+ "est": 256023,
71
+ "eus": 256024,
72
+ "fin": 256025,
73
+ "fra": 256026,
74
+ "fuv": 256027,
75
+ "gaz": 256028,
76
+ "gle": 256029,
77
+ "glg": 256030,
78
+ "guj": 256031,
79
+ "heb": 256032,
80
+ "hin": 256033,
81
+ "hrv": 256034,
82
+ "hun": 256035,
83
+ "hye": 256036,
84
+ "ibo": 256037,
85
+ "ind": 256038,
86
+ "isl": 256039,
87
+ "ita": 256040,
88
+ "jav": 256041,
89
+ "jpn": 256042,
90
+ "kan": 256043,
91
+ "kat": 256044,
92
+ "kaz": 256045,
93
+ "khk": 256046,
94
+ "khm": 256047,
95
+ "kir": 256048,
96
+ "kor": 256049,
97
+ "lao": 256050,
98
+ "lit": 256051,
99
+ "lug": 256052,
100
+ "luo": 256053,
101
+ "lvs": 256054,
102
+ "mai": 256055,
103
+ "mal": 256056,
104
+ "mar": 256057,
105
+ "mkd": 256058,
106
+ "mlt": 256059,
107
+ "mni": 256060,
108
+ "mya": 256061,
109
+ "nld": 256062,
110
+ "nno": 256063,
111
+ "nob": 256064,
112
+ "npi": 256065,
113
+ "nya": 256066,
114
+ "ory": 256067,
115
+ "pan": 256068,
116
+ "pbt": 256069,
117
+ "pes": 256070,
118
+ "pol": 256071,
119
+ "por": 256072,
120
+ "ron": 256073,
121
+ "rus": 256074,
122
+ "sat": 256075,
123
+ "slk": 256076,
124
+ "slv": 256077,
125
+ "sna": 256078,
126
+ "snd": 256079,
127
+ "som": 256080,
128
+ "spa": 256081,
129
+ "srp": 256082,
130
+ "swe": 256083,
131
+ "swh": 256084,
132
+ "tam": 256085,
133
+ "tel": 256086,
134
+ "tgk": 256087,
135
+ "tgl": 256088,
136
+ "tha": 256089,
137
+ "tur": 256090,
138
+ "ukr": 256091,
139
+ "urd": 256092,
140
+ "uzn": 256093,
141
+ "vie": 256094,
142
+ "yor": 256095,
143
+ "yue": 256096,
144
+ "zlm": 256097,
145
+ "zul": 256098
146
+ },
147
+ "transformers_version": "4.47.1",
148
+ "vocoder_lang_code_to_id": {
149
+ "arb": 0,
150
+ "ben": 1,
151
+ "cat": 2,
152
+ "ces": 3,
153
+ "cmn": 4,
154
+ "cym": 5,
155
+ "dan": 6,
156
+ "deu": 7,
157
+ "eng": 8,
158
+ "est": 9,
159
+ "fin": 10,
160
+ "fra": 11,
161
+ "hin": 12,
162
+ "ind": 13,
163
+ "ita": 14,
164
+ "jpn": 15,
165
+ "kor": 16,
166
+ "mlt": 17,
167
+ "nld": 18,
168
+ "pes": 19,
169
+ "pol": 20,
170
+ "por": 21,
171
+ "ron": 22,
172
+ "rus": 23,
173
+ "slk": 24,
174
+ "spa": 25,
175
+ "swe": 26,
176
+ "swh": 27,
177
+ "tel": 28,
178
+ "tgl": 29,
179
+ "tha": 30,
180
+ "tur": 31,
181
+ "ukr": 32,
182
+ "urd": 33,
183
+ "uzn": 34,
184
+ "vie": 35
185
+ }
186
+ }
model-00001-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7baddc1ea2f60046e1d4b47f2fc5ab1b4efa197a548f2b2e25ff79dddcb1a6a1
3
+ size 4998786488
model-00002-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c45c56c88c34731e3d2873194011565f66e227c73640953c40dfb5b7de06ee8d
3
+ size 4238039316
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
preprocessor_config.json ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "feature_extractor_type": "SeamlessM4TFeatureExtractor",
3
+ "feature_size": 80,
4
+ "language_code": [
5
+ "__afr__",
6
+ "__amh__",
7
+ "__arb__",
8
+ "__ary__",
9
+ "__arz__",
10
+ "__asm__",
11
+ "__azj__",
12
+ "__bel__",
13
+ "__ben__",
14
+ "__bos__",
15
+ "__bul__",
16
+ "__cat__",
17
+ "__ceb__",
18
+ "__ces__",
19
+ "__ckb__",
20
+ "__cmn__",
21
+ "__cmn_Hant__",
22
+ "__cym__",
23
+ "__dan__",
24
+ "__deu__",
25
+ "__ell__",
26
+ "__eng__",
27
+ "__est__",
28
+ "__eus__",
29
+ "__fin__",
30
+ "__fra__",
31
+ "__fuv__",
32
+ "__gaz__",
33
+ "__gle__",
34
+ "__glg__",
35
+ "__guj__",
36
+ "__heb__",
37
+ "__hin__",
38
+ "__hrv__",
39
+ "__hun__",
40
+ "__hye__",
41
+ "__ibo__",
42
+ "__ind__",
43
+ "__isl__",
44
+ "__ita__",
45
+ "__jav__",
46
+ "__jpn__",
47
+ "__kan__",
48
+ "__kat__",
49
+ "__kaz__",
50
+ "__khk__",
51
+ "__khm__",
52
+ "__kir__",
53
+ "__kor__",
54
+ "__lao__",
55
+ "__lit__",
56
+ "__lug__",
57
+ "__luo__",
58
+ "__lvs__",
59
+ "__mai__",
60
+ "__mal__",
61
+ "__mar__",
62
+ "__mkd__",
63
+ "__mlt__",
64
+ "__mni__",
65
+ "__mya__",
66
+ "__nld__",
67
+ "__nno__",
68
+ "__nob__",
69
+ "__npi__",
70
+ "__nya__",
71
+ "__ory__",
72
+ "__pan__",
73
+ "__pbt__",
74
+ "__pes__",
75
+ "__pol__",
76
+ "__por__",
77
+ "__ron__",
78
+ "__rus__",
79
+ "__sat__",
80
+ "__slk__",
81
+ "__slv__",
82
+ "__sna__",
83
+ "__snd__",
84
+ "__som__",
85
+ "__spa__",
86
+ "__srp__",
87
+ "__swe__",
88
+ "__swh__",
89
+ "__tam__",
90
+ "__tel__",
91
+ "__tgk__",
92
+ "__tgl__",
93
+ "__tha__",
94
+ "__tur__",
95
+ "__ukr__",
96
+ "__urd__",
97
+ "__uzn__",
98
+ "__vie__",
99
+ "__yor__",
100
+ "__yue__",
101
+ "__zlm__",
102
+ "__zul__"
103
+ ],
104
+ "num_mel_bins": 80,
105
+ "padding_side": "right",
106
+ "padding_value": 0.0,
107
+ "processor_class": "SeamlessM4TProcessor",
108
+ "return_attention_mask": true,
109
+ "sampling_rate": 16000,
110
+ "stride": 2
111
+ }
sentencepiece.bpe.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:026a76827537db9f1348e4d5aaa127bb10a2f2ff633243f3a52d16be82d73f9d
3
+ size 5165809
special_tokens_map.json ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<pad>",
4
+ "<unk>",
5
+ "<s>",
6
+ "</s>",
7
+ "__afr__",
8
+ "__amh__",
9
+ "__arb__",
10
+ "__ary__",
11
+ "__arz__",
12
+ "__asm__",
13
+ "__azj__",
14
+ "__bel__",
15
+ "__ben__",
16
+ "__bos__",
17
+ "__bul__",
18
+ "__cat__",
19
+ "__ceb__",
20
+ "__ces__",
21
+ "__ckb__",
22
+ "__cmn__",
23
+ "__cmn_Hant__",
24
+ "__cym__",
25
+ "__dan__",
26
+ "__deu__",
27
+ "__ell__",
28
+ "__eng__",
29
+ "__est__",
30
+ "__eus__",
31
+ "__fin__",
32
+ "__fra__",
33
+ "__fuv__",
34
+ "__gaz__",
35
+ "__gle__",
36
+ "__glg__",
37
+ "__guj__",
38
+ "__heb__",
39
+ "__hin__",
40
+ "__hrv__",
41
+ "__hun__",
42
+ "__hye__",
43
+ "__ibo__",
44
+ "__ind__",
45
+ "__isl__",
46
+ "__ita__",
47
+ "__jav__",
48
+ "__jpn__",
49
+ "__kan__",
50
+ "__kat__",
51
+ "__kaz__",
52
+ "__khk__",
53
+ "__khm__",
54
+ "__kir__",
55
+ "__kor__",
56
+ "__lao__",
57
+ "__lit__",
58
+ "__lug__",
59
+ "__luo__",
60
+ "__lvs__",
61
+ "__mai__",
62
+ "__mal__",
63
+ "__mar__",
64
+ "__mkd__",
65
+ "__mlt__",
66
+ "__mni__",
67
+ "__mya__",
68
+ "__nld__",
69
+ "__nno__",
70
+ "__nob__",
71
+ "__npi__",
72
+ "__nya__",
73
+ "__ory__",
74
+ "__pan__",
75
+ "__pbt__",
76
+ "__pes__",
77
+ "__pol__",
78
+ "__por__",
79
+ "__ron__",
80
+ "__rus__",
81
+ "__sat__",
82
+ "__slk__",
83
+ "__slv__",
84
+ "__sna__",
85
+ "__snd__",
86
+ "__som__",
87
+ "__spa__",
88
+ "__srp__",
89
+ "__swe__",
90
+ "__swh__",
91
+ "__tam__",
92
+ "__tel__",
93
+ "__tgk__",
94
+ "__tgl__",
95
+ "__tha__",
96
+ "__tur__",
97
+ "__ukr__",
98
+ "__urd__",
99
+ "__uzn__",
100
+ "__vie__",
101
+ "__yor__",
102
+ "__yue__",
103
+ "__zlm__",
104
+ "__zul__"
105
+ ],
106
+ "bos_token": {
107
+ "content": "<s>",
108
+ "lstrip": false,
109
+ "normalized": false,
110
+ "rstrip": false,
111
+ "single_word": false
112
+ },
113
+ "cls_token": {
114
+ "content": "<s>",
115
+ "lstrip": false,
116
+ "normalized": false,
117
+ "rstrip": false,
118
+ "single_word": false
119
+ },
120
+ "eos_token": {
121
+ "content": "</s>",
122
+ "lstrip": false,
123
+ "normalized": false,
124
+ "rstrip": false,
125
+ "single_word": false
126
+ },
127
+ "pad_token": {
128
+ "content": "<pad>",
129
+ "lstrip": false,
130
+ "normalized": false,
131
+ "rstrip": false,
132
+ "single_word": false
133
+ },
134
+ "sep_token": {
135
+ "content": "</s>",
136
+ "lstrip": false,
137
+ "normalized": false,
138
+ "rstrip": false,
139
+ "single_word": false
140
+ },
141
+ "unk_token": {
142
+ "content": "<unk>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false
147
+ }
148
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2b7d5b556246fb04c88b7417fd7abbe3f30cc14d29a861785c76c403a3b704b0
3
+ size 31219299
tokenizer_config.json ADDED
@@ -0,0 +1,939 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": true,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<pad>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<unk>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "<s>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "</s>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "256001": {
37
+ "content": "__afr__",
38
+ "lstrip": true,
39
+ "normalized": false,
40
+ "rstrip": true,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "256002": {
45
+ "content": "__amh__",
46
+ "lstrip": true,
47
+ "normalized": false,
48
+ "rstrip": true,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "256003": {
53
+ "content": "__arb__",
54
+ "lstrip": true,
55
+ "normalized": false,
56
+ "rstrip": true,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "256004": {
61
+ "content": "__ary__",
62
+ "lstrip": true,
63
+ "normalized": false,
64
+ "rstrip": true,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "256005": {
69
+ "content": "__arz__",
70
+ "lstrip": true,
71
+ "normalized": false,
72
+ "rstrip": true,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "256006": {
77
+ "content": "__asm__",
78
+ "lstrip": true,
79
+ "normalized": false,
80
+ "rstrip": true,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "256007": {
85
+ "content": "__azj__",
86
+ "lstrip": true,
87
+ "normalized": false,
88
+ "rstrip": true,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "256008": {
93
+ "content": "__bel__",
94
+ "lstrip": true,
95
+ "normalized": false,
96
+ "rstrip": true,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "256009": {
101
+ "content": "__ben__",
102
+ "lstrip": true,
103
+ "normalized": false,
104
+ "rstrip": true,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "256010": {
109
+ "content": "__bos__",
110
+ "lstrip": true,
111
+ "normalized": false,
112
+ "rstrip": true,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "256011": {
117
+ "content": "__bul__",
118
+ "lstrip": true,
119
+ "normalized": false,
120
+ "rstrip": true,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "256012": {
125
+ "content": "__cat__",
126
+ "lstrip": true,
127
+ "normalized": false,
128
+ "rstrip": true,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "256013": {
133
+ "content": "__ceb__",
134
+ "lstrip": true,
135
+ "normalized": false,
136
+ "rstrip": true,
137
+ "single_word": false,
138
+ "special": true
139
+ },
140
+ "256014": {
141
+ "content": "__ces__",
142
+ "lstrip": true,
143
+ "normalized": false,
144
+ "rstrip": true,
145
+ "single_word": false,
146
+ "special": true
147
+ },
148
+ "256015": {
149
+ "content": "__ckb__",
150
+ "lstrip": true,
151
+ "normalized": false,
152
+ "rstrip": true,
153
+ "single_word": false,
154
+ "special": true
155
+ },
156
+ "256016": {
157
+ "content": "__cmn__",
158
+ "lstrip": true,
159
+ "normalized": false,
160
+ "rstrip": true,
161
+ "single_word": false,
162
+ "special": true
163
+ },
164
+ "256017": {
165
+ "content": "__cmn_Hant__",
166
+ "lstrip": true,
167
+ "normalized": false,
168
+ "rstrip": true,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
+ "256018": {
173
+ "content": "__cym__",
174
+ "lstrip": true,
175
+ "normalized": false,
176
+ "rstrip": true,
177
+ "single_word": false,
178
+ "special": true
179
+ },
180
+ "256019": {
181
+ "content": "__dan__",
182
+ "lstrip": true,
183
+ "normalized": false,
184
+ "rstrip": true,
185
+ "single_word": false,
186
+ "special": true
187
+ },
188
+ "256020": {
189
+ "content": "__deu__",
190
+ "lstrip": true,
191
+ "normalized": false,
192
+ "rstrip": true,
193
+ "single_word": false,
194
+ "special": true
195
+ },
196
+ "256021": {
197
+ "content": "__ell__",
198
+ "lstrip": true,
199
+ "normalized": false,
200
+ "rstrip": true,
201
+ "single_word": false,
202
+ "special": true
203
+ },
204
+ "256022": {
205
+ "content": "__eng__",
206
+ "lstrip": true,
207
+ "normalized": false,
208
+ "rstrip": true,
209
+ "single_word": false,
210
+ "special": true
211
+ },
212
+ "256023": {
213
+ "content": "__est__",
214
+ "lstrip": true,
215
+ "normalized": false,
216
+ "rstrip": true,
217
+ "single_word": false,
218
+ "special": true
219
+ },
220
+ "256024": {
221
+ "content": "__eus__",
222
+ "lstrip": true,
223
+ "normalized": false,
224
+ "rstrip": true,
225
+ "single_word": false,
226
+ "special": true
227
+ },
228
+ "256025": {
229
+ "content": "__fin__",
230
+ "lstrip": true,
231
+ "normalized": false,
232
+ "rstrip": true,
233
+ "single_word": false,
234
+ "special": true
235
+ },
236
+ "256026": {
237
+ "content": "__fra__",
238
+ "lstrip": true,
239
+ "normalized": false,
240
+ "rstrip": true,
241
+ "single_word": false,
242
+ "special": true
243
+ },
244
+ "256027": {
245
+ "content": "__fuv__",
246
+ "lstrip": true,
247
+ "normalized": false,
248
+ "rstrip": true,
249
+ "single_word": false,
250
+ "special": true
251
+ },
252
+ "256028": {
253
+ "content": "__gaz__",
254
+ "lstrip": true,
255
+ "normalized": false,
256
+ "rstrip": true,
257
+ "single_word": false,
258
+ "special": true
259
+ },
260
+ "256029": {
261
+ "content": "__gle__",
262
+ "lstrip": true,
263
+ "normalized": false,
264
+ "rstrip": true,
265
+ "single_word": false,
266
+ "special": true
267
+ },
268
+ "256030": {
269
+ "content": "__glg__",
270
+ "lstrip": true,
271
+ "normalized": false,
272
+ "rstrip": true,
273
+ "single_word": false,
274
+ "special": true
275
+ },
276
+ "256031": {
277
+ "content": "__guj__",
278
+ "lstrip": true,
279
+ "normalized": false,
280
+ "rstrip": true,
281
+ "single_word": false,
282
+ "special": true
283
+ },
284
+ "256032": {
285
+ "content": "__heb__",
286
+ "lstrip": true,
287
+ "normalized": false,
288
+ "rstrip": true,
289
+ "single_word": false,
290
+ "special": true
291
+ },
292
+ "256033": {
293
+ "content": "__hin__",
294
+ "lstrip": true,
295
+ "normalized": false,
296
+ "rstrip": true,
297
+ "single_word": false,
298
+ "special": true
299
+ },
300
+ "256034": {
301
+ "content": "__hrv__",
302
+ "lstrip": true,
303
+ "normalized": false,
304
+ "rstrip": true,
305
+ "single_word": false,
306
+ "special": true
307
+ },
308
+ "256035": {
309
+ "content": "__hun__",
310
+ "lstrip": true,
311
+ "normalized": false,
312
+ "rstrip": true,
313
+ "single_word": false,
314
+ "special": true
315
+ },
316
+ "256036": {
317
+ "content": "__hye__",
318
+ "lstrip": true,
319
+ "normalized": false,
320
+ "rstrip": true,
321
+ "single_word": false,
322
+ "special": true
323
+ },
324
+ "256037": {
325
+ "content": "__ibo__",
326
+ "lstrip": true,
327
+ "normalized": false,
328
+ "rstrip": true,
329
+ "single_word": false,
330
+ "special": true
331
+ },
332
+ "256038": {
333
+ "content": "__ind__",
334
+ "lstrip": true,
335
+ "normalized": false,
336
+ "rstrip": true,
337
+ "single_word": false,
338
+ "special": true
339
+ },
340
+ "256039": {
341
+ "content": "__isl__",
342
+ "lstrip": true,
343
+ "normalized": false,
344
+ "rstrip": true,
345
+ "single_word": false,
346
+ "special": true
347
+ },
348
+ "256040": {
349
+ "content": "__ita__",
350
+ "lstrip": true,
351
+ "normalized": false,
352
+ "rstrip": true,
353
+ "single_word": false,
354
+ "special": true
355
+ },
356
+ "256041": {
357
+ "content": "__jav__",
358
+ "lstrip": true,
359
+ "normalized": false,
360
+ "rstrip": true,
361
+ "single_word": false,
362
+ "special": true
363
+ },
364
+ "256042": {
365
+ "content": "__jpn__",
366
+ "lstrip": true,
367
+ "normalized": false,
368
+ "rstrip": true,
369
+ "single_word": false,
370
+ "special": true
371
+ },
372
+ "256043": {
373
+ "content": "__kan__",
374
+ "lstrip": true,
375
+ "normalized": false,
376
+ "rstrip": true,
377
+ "single_word": false,
378
+ "special": true
379
+ },
380
+ "256044": {
381
+ "content": "__kat__",
382
+ "lstrip": true,
383
+ "normalized": false,
384
+ "rstrip": true,
385
+ "single_word": false,
386
+ "special": true
387
+ },
388
+ "256045": {
389
+ "content": "__kaz__",
390
+ "lstrip": true,
391
+ "normalized": false,
392
+ "rstrip": true,
393
+ "single_word": false,
394
+ "special": true
395
+ },
396
+ "256046": {
397
+ "content": "__khk__",
398
+ "lstrip": true,
399
+ "normalized": false,
400
+ "rstrip": true,
401
+ "single_word": false,
402
+ "special": true
403
+ },
404
+ "256047": {
405
+ "content": "__khm__",
406
+ "lstrip": true,
407
+ "normalized": false,
408
+ "rstrip": true,
409
+ "single_word": false,
410
+ "special": true
411
+ },
412
+ "256048": {
413
+ "content": "__kir__",
414
+ "lstrip": true,
415
+ "normalized": false,
416
+ "rstrip": true,
417
+ "single_word": false,
418
+ "special": true
419
+ },
420
+ "256049": {
421
+ "content": "__kor__",
422
+ "lstrip": true,
423
+ "normalized": false,
424
+ "rstrip": true,
425
+ "single_word": false,
426
+ "special": true
427
+ },
428
+ "256050": {
429
+ "content": "__lao__",
430
+ "lstrip": true,
431
+ "normalized": false,
432
+ "rstrip": true,
433
+ "single_word": false,
434
+ "special": true
435
+ },
436
+ "256051": {
437
+ "content": "__lit__",
438
+ "lstrip": true,
439
+ "normalized": false,
440
+ "rstrip": true,
441
+ "single_word": false,
442
+ "special": true
443
+ },
444
+ "256052": {
445
+ "content": "__lug__",
446
+ "lstrip": true,
447
+ "normalized": false,
448
+ "rstrip": true,
449
+ "single_word": false,
450
+ "special": true
451
+ },
452
+ "256053": {
453
+ "content": "__luo__",
454
+ "lstrip": true,
455
+ "normalized": false,
456
+ "rstrip": true,
457
+ "single_word": false,
458
+ "special": true
459
+ },
460
+ "256054": {
461
+ "content": "__lvs__",
462
+ "lstrip": true,
463
+ "normalized": false,
464
+ "rstrip": true,
465
+ "single_word": false,
466
+ "special": true
467
+ },
468
+ "256055": {
469
+ "content": "__mai__",
470
+ "lstrip": true,
471
+ "normalized": false,
472
+ "rstrip": true,
473
+ "single_word": false,
474
+ "special": true
475
+ },
476
+ "256056": {
477
+ "content": "__mal__",
478
+ "lstrip": true,
479
+ "normalized": false,
480
+ "rstrip": true,
481
+ "single_word": false,
482
+ "special": true
483
+ },
484
+ "256057": {
485
+ "content": "__mar__",
486
+ "lstrip": true,
487
+ "normalized": false,
488
+ "rstrip": true,
489
+ "single_word": false,
490
+ "special": true
491
+ },
492
+ "256058": {
493
+ "content": "__mkd__",
494
+ "lstrip": true,
495
+ "normalized": false,
496
+ "rstrip": true,
497
+ "single_word": false,
498
+ "special": true
499
+ },
500
+ "256059": {
501
+ "content": "__mlt__",
502
+ "lstrip": true,
503
+ "normalized": false,
504
+ "rstrip": true,
505
+ "single_word": false,
506
+ "special": true
507
+ },
508
+ "256060": {
509
+ "content": "__mni__",
510
+ "lstrip": true,
511
+ "normalized": false,
512
+ "rstrip": true,
513
+ "single_word": false,
514
+ "special": true
515
+ },
516
+ "256061": {
517
+ "content": "__mya__",
518
+ "lstrip": true,
519
+ "normalized": false,
520
+ "rstrip": true,
521
+ "single_word": false,
522
+ "special": true
523
+ },
524
+ "256062": {
525
+ "content": "__nld__",
526
+ "lstrip": true,
527
+ "normalized": false,
528
+ "rstrip": true,
529
+ "single_word": false,
530
+ "special": true
531
+ },
532
+ "256063": {
533
+ "content": "__nno__",
534
+ "lstrip": true,
535
+ "normalized": false,
536
+ "rstrip": true,
537
+ "single_word": false,
538
+ "special": true
539
+ },
540
+ "256064": {
541
+ "content": "__nob__",
542
+ "lstrip": true,
543
+ "normalized": false,
544
+ "rstrip": true,
545
+ "single_word": false,
546
+ "special": true
547
+ },
548
+ "256065": {
549
+ "content": "__npi__",
550
+ "lstrip": true,
551
+ "normalized": false,
552
+ "rstrip": true,
553
+ "single_word": false,
554
+ "special": true
555
+ },
556
+ "256066": {
557
+ "content": "__nya__",
558
+ "lstrip": true,
559
+ "normalized": false,
560
+ "rstrip": true,
561
+ "single_word": false,
562
+ "special": true
563
+ },
564
+ "256067": {
565
+ "content": "__ory__",
566
+ "lstrip": true,
567
+ "normalized": false,
568
+ "rstrip": true,
569
+ "single_word": false,
570
+ "special": true
571
+ },
572
+ "256068": {
573
+ "content": "__pan__",
574
+ "lstrip": true,
575
+ "normalized": false,
576
+ "rstrip": true,
577
+ "single_word": false,
578
+ "special": true
579
+ },
580
+ "256069": {
581
+ "content": "__pbt__",
582
+ "lstrip": true,
583
+ "normalized": false,
584
+ "rstrip": true,
585
+ "single_word": false,
586
+ "special": true
587
+ },
588
+ "256070": {
589
+ "content": "__pes__",
590
+ "lstrip": true,
591
+ "normalized": false,
592
+ "rstrip": true,
593
+ "single_word": false,
594
+ "special": true
595
+ },
596
+ "256071": {
597
+ "content": "__pol__",
598
+ "lstrip": true,
599
+ "normalized": false,
600
+ "rstrip": true,
601
+ "single_word": false,
602
+ "special": true
603
+ },
604
+ "256072": {
605
+ "content": "__por__",
606
+ "lstrip": true,
607
+ "normalized": false,
608
+ "rstrip": true,
609
+ "single_word": false,
610
+ "special": true
611
+ },
612
+ "256073": {
613
+ "content": "__ron__",
614
+ "lstrip": true,
615
+ "normalized": false,
616
+ "rstrip": true,
617
+ "single_word": false,
618
+ "special": true
619
+ },
620
+ "256074": {
621
+ "content": "__rus__",
622
+ "lstrip": true,
623
+ "normalized": false,
624
+ "rstrip": true,
625
+ "single_word": false,
626
+ "special": true
627
+ },
628
+ "256075": {
629
+ "content": "__sat__",
630
+ "lstrip": true,
631
+ "normalized": false,
632
+ "rstrip": true,
633
+ "single_word": false,
634
+ "special": true
635
+ },
636
+ "256076": {
637
+ "content": "__slk__",
638
+ "lstrip": true,
639
+ "normalized": false,
640
+ "rstrip": true,
641
+ "single_word": false,
642
+ "special": true
643
+ },
644
+ "256077": {
645
+ "content": "__slv__",
646
+ "lstrip": true,
647
+ "normalized": false,
648
+ "rstrip": true,
649
+ "single_word": false,
650
+ "special": true
651
+ },
652
+ "256078": {
653
+ "content": "__sna__",
654
+ "lstrip": true,
655
+ "normalized": false,
656
+ "rstrip": true,
657
+ "single_word": false,
658
+ "special": true
659
+ },
660
+ "256079": {
661
+ "content": "__snd__",
662
+ "lstrip": true,
663
+ "normalized": false,
664
+ "rstrip": true,
665
+ "single_word": false,
666
+ "special": true
667
+ },
668
+ "256080": {
669
+ "content": "__som__",
670
+ "lstrip": true,
671
+ "normalized": false,
672
+ "rstrip": true,
673
+ "single_word": false,
674
+ "special": true
675
+ },
676
+ "256081": {
677
+ "content": "__spa__",
678
+ "lstrip": true,
679
+ "normalized": false,
680
+ "rstrip": true,
681
+ "single_word": false,
682
+ "special": true
683
+ },
684
+ "256082": {
685
+ "content": "__srp__",
686
+ "lstrip": true,
687
+ "normalized": false,
688
+ "rstrip": true,
689
+ "single_word": false,
690
+ "special": true
691
+ },
692
+ "256083": {
693
+ "content": "__swe__",
694
+ "lstrip": true,
695
+ "normalized": false,
696
+ "rstrip": true,
697
+ "single_word": false,
698
+ "special": true
699
+ },
700
+ "256084": {
701
+ "content": "__swh__",
702
+ "lstrip": true,
703
+ "normalized": false,
704
+ "rstrip": true,
705
+ "single_word": false,
706
+ "special": true
707
+ },
708
+ "256085": {
709
+ "content": "__tam__",
710
+ "lstrip": true,
711
+ "normalized": false,
712
+ "rstrip": true,
713
+ "single_word": false,
714
+ "special": true
715
+ },
716
+ "256086": {
717
+ "content": "__tel__",
718
+ "lstrip": true,
719
+ "normalized": false,
720
+ "rstrip": true,
721
+ "single_word": false,
722
+ "special": true
723
+ },
724
+ "256087": {
725
+ "content": "__tgk__",
726
+ "lstrip": true,
727
+ "normalized": false,
728
+ "rstrip": true,
729
+ "single_word": false,
730
+ "special": true
731
+ },
732
+ "256088": {
733
+ "content": "__tgl__",
734
+ "lstrip": true,
735
+ "normalized": false,
736
+ "rstrip": true,
737
+ "single_word": false,
738
+ "special": true
739
+ },
740
+ "256089": {
741
+ "content": "__tha__",
742
+ "lstrip": true,
743
+ "normalized": false,
744
+ "rstrip": true,
745
+ "single_word": false,
746
+ "special": true
747
+ },
748
+ "256090": {
749
+ "content": "__tur__",
750
+ "lstrip": true,
751
+ "normalized": false,
752
+ "rstrip": true,
753
+ "single_word": false,
754
+ "special": true
755
+ },
756
+ "256091": {
757
+ "content": "__ukr__",
758
+ "lstrip": true,
759
+ "normalized": false,
760
+ "rstrip": true,
761
+ "single_word": false,
762
+ "special": true
763
+ },
764
+ "256092": {
765
+ "content": "__urd__",
766
+ "lstrip": true,
767
+ "normalized": false,
768
+ "rstrip": true,
769
+ "single_word": false,
770
+ "special": true
771
+ },
772
+ "256093": {
773
+ "content": "__uzn__",
774
+ "lstrip": true,
775
+ "normalized": false,
776
+ "rstrip": true,
777
+ "single_word": false,
778
+ "special": true
779
+ },
780
+ "256094": {
781
+ "content": "__vie__",
782
+ "lstrip": true,
783
+ "normalized": false,
784
+ "rstrip": true,
785
+ "single_word": false,
786
+ "special": true
787
+ },
788
+ "256095": {
789
+ "content": "__yor__",
790
+ "lstrip": true,
791
+ "normalized": false,
792
+ "rstrip": true,
793
+ "single_word": false,
794
+ "special": true
795
+ },
796
+ "256096": {
797
+ "content": "__yue__",
798
+ "lstrip": true,
799
+ "normalized": false,
800
+ "rstrip": true,
801
+ "single_word": false,
802
+ "special": true
803
+ },
804
+ "256097": {
805
+ "content": "__zlm__",
806
+ "lstrip": true,
807
+ "normalized": false,
808
+ "rstrip": true,
809
+ "single_word": false,
810
+ "special": true
811
+ },
812
+ "256098": {
813
+ "content": "__zul__",
814
+ "lstrip": true,
815
+ "normalized": false,
816
+ "rstrip": true,
817
+ "single_word": false,
818
+ "special": true
819
+ }
820
+ },
821
+ "additional_special_tokens": [
822
+ "<pad>",
823
+ "<unk>",
824
+ "<s>",
825
+ "</s>",
826
+ "__afr__",
827
+ "__amh__",
828
+ "__arb__",
829
+ "__ary__",
830
+ "__arz__",
831
+ "__asm__",
832
+ "__azj__",
833
+ "__bel__",
834
+ "__ben__",
835
+ "__bos__",
836
+ "__bul__",
837
+ "__cat__",
838
+ "__ceb__",
839
+ "__ces__",
840
+ "__ckb__",
841
+ "__cmn__",
842
+ "__cmn_Hant__",
843
+ "__cym__",
844
+ "__dan__",
845
+ "__deu__",
846
+ "__ell__",
847
+ "__eng__",
848
+ "__est__",
849
+ "__eus__",
850
+ "__fin__",
851
+ "__fra__",
852
+ "__fuv__",
853
+ "__gaz__",
854
+ "__gle__",
855
+ "__glg__",
856
+ "__guj__",
857
+ "__heb__",
858
+ "__hin__",
859
+ "__hrv__",
860
+ "__hun__",
861
+ "__hye__",
862
+ "__ibo__",
863
+ "__ind__",
864
+ "__isl__",
865
+ "__ita__",
866
+ "__jav__",
867
+ "__jpn__",
868
+ "__kan__",
869
+ "__kat__",
870
+ "__kaz__",
871
+ "__khk__",
872
+ "__khm__",
873
+ "__kir__",
874
+ "__kor__",
875
+ "__lao__",
876
+ "__lit__",
877
+ "__lug__",
878
+ "__luo__",
879
+ "__lvs__",
880
+ "__mai__",
881
+ "__mal__",
882
+ "__mar__",
883
+ "__mkd__",
884
+ "__mlt__",
885
+ "__mni__",
886
+ "__mya__",
887
+ "__nld__",
888
+ "__nno__",
889
+ "__nob__",
890
+ "__npi__",
891
+ "__nya__",
892
+ "__ory__",
893
+ "__pan__",
894
+ "__pbt__",
895
+ "__pes__",
896
+ "__pol__",
897
+ "__por__",
898
+ "__ron__",
899
+ "__rus__",
900
+ "__sat__",
901
+ "__slk__",
902
+ "__slv__",
903
+ "__sna__",
904
+ "__snd__",
905
+ "__som__",
906
+ "__spa__",
907
+ "__srp__",
908
+ "__swe__",
909
+ "__swh__",
910
+ "__tam__",
911
+ "__tel__",
912
+ "__tgk__",
913
+ "__tgl__",
914
+ "__tha__",
915
+ "__tur__",
916
+ "__ukr__",
917
+ "__urd__",
918
+ "__uzn__",
919
+ "__vie__",
920
+ "__yor__",
921
+ "__yue__",
922
+ "__zlm__",
923
+ "__zul__"
924
+ ],
925
+ "bos_token": "<s>",
926
+ "clean_up_tokenization_spaces": true,
927
+ "cls_token": "<s>",
928
+ "eos_token": "</s>",
929
+ "extra_special_tokens": {},
930
+ "model_max_length": 1000000000000000019884624838656,
931
+ "pad_token": "<pad>",
932
+ "processor_class": "SeamlessM4TProcessor",
933
+ "sep_token": "</s>",
934
+ "sp_model_kwargs": {},
935
+ "src_lang": "__eng__",
936
+ "tgt_lang": "__fra__",
937
+ "tokenizer_class": "SeamlessM4TTokenizer",
938
+ "unk_token": "<unk>"
939
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa7ae520ae2ba620c34b4d3b33c9011d4063f7f65387083c65f2132468b04f50
3
+ size 5560