p1atdev commited on
Commit
de6058f
1 Parent(s): 2c29184

Update tokenization_dart.py

Browse files
Files changed (1) hide show
  1. tokenization_dart.py +0 -9
tokenization_dart.py CHANGED
@@ -37,20 +37,11 @@ PROMPT_TEMPLATE = (
37
  "{{ '</character>' }}"
38
 
39
  "{{ '<general>' }}"
40
- # length token
41
- "{% if 'length' not in messages or messages['length'] is none %}"
42
- "{{ '<|long|>' }}"
43
- "{% else %}"
44
- "{{ messages['length'] }}"
45
- "{% endif %}"
46
-
47
- # general token
48
  "{% if 'general' not in messages or messages['general'] is none %}"
49
  "{{ '' }}"
50
  "{% else %}"
51
  "{{ messages['general'] }}"
52
  "{% endif %}"
53
- "{{ '<|input_end|>' }}"
54
  ).strip()
55
  # fmt: on
56
 
 
37
  "{{ '</character>' }}"
38
 
39
  "{{ '<general>' }}"
 
 
 
 
 
 
 
 
40
  "{% if 'general' not in messages or messages['general'] is none %}"
41
  "{{ '' }}"
42
  "{% else %}"
43
  "{{ messages['general'] }}"
44
  "{% endif %}"
 
45
  ).strip()
46
  # fmt: on
47