Update tokenization_dart.py
Browse files- tokenization_dart.py +0 -9
tokenization_dart.py
CHANGED
@@ -37,20 +37,11 @@ PROMPT_TEMPLATE = (
|
|
37 |
"{{ '</character>' }}"
|
38 |
|
39 |
"{{ '<general>' }}"
|
40 |
-
# length token
|
41 |
-
"{% if 'length' not in messages or messages['length'] is none %}"
|
42 |
-
"{{ '<|long|>' }}"
|
43 |
-
"{% else %}"
|
44 |
-
"{{ messages['length'] }}"
|
45 |
-
"{% endif %}"
|
46 |
-
|
47 |
-
# general token
|
48 |
"{% if 'general' not in messages or messages['general'] is none %}"
|
49 |
"{{ '' }}"
|
50 |
"{% else %}"
|
51 |
"{{ messages['general'] }}"
|
52 |
"{% endif %}"
|
53 |
-
"{{ '<|input_end|>' }}"
|
54 |
).strip()
|
55 |
# fmt: on
|
56 |
|
|
|
37 |
"{{ '</character>' }}"
|
38 |
|
39 |
"{{ '<general>' }}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
40 |
"{% if 'general' not in messages or messages['general'] is none %}"
|
41 |
"{{ '' }}"
|
42 |
"{% else %}"
|
43 |
"{{ messages['general'] }}"
|
44 |
"{% endif %}"
|
|
|
45 |
).strip()
|
46 |
# fmt: on
|
47 |
|