change hebrew code from iw (deprecated) to he
#12
by
altryne
- opened
- added_tokens.json +1 -1
- generation_config.json +1 -1
added_tokens.json
CHANGED
@@ -30,6 +30,7 @@
|
|
30 |
"<|gu|>": 50333,
|
31 |
"<|haw|>": 50352,
|
32 |
"<|ha|>": 50354,
|
|
|
33 |
"<|hi|>": 50276,
|
34 |
"<|hr|>": 50291,
|
35 |
"<|ht|>": 50339,
|
@@ -38,7 +39,6 @@
|
|
38 |
"<|id|>": 50275,
|
39 |
"<|is|>": 50311,
|
40 |
"<|it|>": 50274,
|
41 |
-
"<|iw|>": 50279,
|
42 |
"<|ja|>": 50266,
|
43 |
"<|jw|>": 50356,
|
44 |
"<|ka|>": 50329,
|
|
|
30 |
"<|gu|>": 50333,
|
31 |
"<|haw|>": 50352,
|
32 |
"<|ha|>": 50354,
|
33 |
+
"<|he|>": 50279,
|
34 |
"<|hi|>": 50276,
|
35 |
"<|hr|>": 50291,
|
36 |
"<|ht|>": 50339,
|
|
|
39 |
"<|id|>": 50275,
|
40 |
"<|is|>": 50311,
|
41 |
"<|it|>": 50274,
|
|
|
42 |
"<|ja|>": 50266,
|
43 |
"<|jw|>": 50356,
|
44 |
"<|ka|>": 50329,
|
generation_config.json
CHANGED
@@ -47,6 +47,7 @@
|
|
47 |
"<|gu|>": 50333,
|
48 |
"<|haw|>": 50352,
|
49 |
"<|ha|>": 50354,
|
|
|
50 |
"<|hi|>": 50276,
|
51 |
"<|hr|>": 50291,
|
52 |
"<|ht|>": 50339,
|
@@ -55,7 +56,6 @@
|
|
55 |
"<|id|>": 50275,
|
56 |
"<|is|>": 50311,
|
57 |
"<|it|>": 50274,
|
58 |
-
"<|iw|>": 50279,
|
59 |
"<|ja|>": 50266,
|
60 |
"<|jw|>": 50356,
|
61 |
"<|ka|>": 50329,
|
|
|
47 |
"<|gu|>": 50333,
|
48 |
"<|haw|>": 50352,
|
49 |
"<|ha|>": 50354,
|
50 |
+
"<|he|>": 50279,
|
51 |
"<|hi|>": 50276,
|
52 |
"<|hr|>": 50291,
|
53 |
"<|ht|>": 50339,
|
|
|
56 |
"<|id|>": 50275,
|
57 |
"<|is|>": 50311,
|
58 |
"<|it|>": 50274,
|
|
|
59 |
"<|ja|>": 50266,
|
60 |
"<|jw|>": 50356,
|
61 |
"<|ka|>": 50329,
|