serenarolloh
commited on
Commit
•
e7585a4
1
Parent(s):
f7a88cc
Training done
Browse files- added_tokens.json +2 -0
- tokenizer.json +18 -0
- tokenizer_config.json +16 -0
added_tokens.json
CHANGED
@@ -2,6 +2,7 @@
|
|
2 |
"</s_customer>": 57548,
|
3 |
"</s_customer_address>": 57575,
|
4 |
"</s_customer_name>": 57550,
|
|
|
5 |
"</s_customer_pin>": 57552,
|
6 |
"</s_customer_reference>": 57577,
|
7 |
"</s_date>": 57530,
|
@@ -29,6 +30,7 @@
|
|
29 |
"<s_customer>": 57547,
|
30 |
"<s_customer_address>": 57574,
|
31 |
"<s_customer_name>": 57549,
|
|
|
32 |
"<s_customer_pin>": 57551,
|
33 |
"<s_customer_reference>": 57576,
|
34 |
"<s_date>": 57529,
|
|
|
2 |
"</s_customer>": 57548,
|
3 |
"</s_customer_address>": 57575,
|
4 |
"</s_customer_name>": 57550,
|
5 |
+
"</s_customer_phone_no>": 57579,
|
6 |
"</s_customer_pin>": 57552,
|
7 |
"</s_customer_reference>": 57577,
|
8 |
"</s_date>": 57530,
|
|
|
30 |
"<s_customer>": 57547,
|
31 |
"<s_customer_address>": 57574,
|
32 |
"<s_customer_name>": 57549,
|
33 |
+
"<s_customer_phone_no>": 57578,
|
34 |
"<s_customer_pin>": 57551,
|
35 |
"<s_customer_reference>": 57576,
|
36 |
"<s_date>": 57529,
|
tokenizer.json
CHANGED
@@ -565,6 +565,24 @@
|
|
565 |
"rstrip": false,
|
566 |
"normalized": true,
|
567 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
568 |
}
|
569 |
],
|
570 |
"normalizer": {
|
|
|
565 |
"rstrip": false,
|
566 |
"normalized": true,
|
567 |
"special": false
|
568 |
+
},
|
569 |
+
{
|
570 |
+
"id": 57578,
|
571 |
+
"content": "<s_customer_phone_no>",
|
572 |
+
"single_word": false,
|
573 |
+
"lstrip": false,
|
574 |
+
"rstrip": false,
|
575 |
+
"normalized": true,
|
576 |
+
"special": false
|
577 |
+
},
|
578 |
+
{
|
579 |
+
"id": 57579,
|
580 |
+
"content": "</s_customer_phone_no>",
|
581 |
+
"single_word": false,
|
582 |
+
"lstrip": false,
|
583 |
+
"rstrip": false,
|
584 |
+
"normalized": true,
|
585 |
+
"special": false
|
586 |
}
|
587 |
],
|
588 |
"normalizer": {
|
tokenizer_config.json
CHANGED
@@ -487,6 +487,22 @@
|
|
487 |
"rstrip": false,
|
488 |
"single_word": false,
|
489 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
490 |
}
|
491 |
},
|
492 |
"additional_special_tokens": [
|
|
|
487 |
"rstrip": false,
|
488 |
"single_word": false,
|
489 |
"special": false
|
490 |
+
},
|
491 |
+
"57578": {
|
492 |
+
"content": "<s_customer_phone_no>",
|
493 |
+
"lstrip": false,
|
494 |
+
"normalized": true,
|
495 |
+
"rstrip": false,
|
496 |
+
"single_word": false,
|
497 |
+
"special": false
|
498 |
+
},
|
499 |
+
"57579": {
|
500 |
+
"content": "</s_customer_phone_no>",
|
501 |
+
"lstrip": false,
|
502 |
+
"normalized": true,
|
503 |
+
"rstrip": false,
|
504 |
+
"single_word": false,
|
505 |
+
"special": false
|
506 |
}
|
507 |
},
|
508 |
"additional_special_tokens": [
|