rubend18 commited on
Commit
e37d689
1 Parent(s): bec5cd0

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +18 -30
app.py CHANGED
@@ -4,36 +4,24 @@ import tiktoken
4
  encodings = tiktoken.list_encoding_names()
5
  encodings.reverse()
6
 
7
- def function(input, encoding):
8
- tokenizer = tiktoken.get_encoding(encoding)
9
- tokens = tokenizer.encode(input)
10
  return len(tokens)
11
-
12
- value1 = gr.Textbox(lines=6, label="Input", placeholder="Input the text...")
13
- value2 = gr.Dropdown(
14
- label="Encoding",
15
- choices=encodings,
16
- default="cl100k_base",
17
- description="The encoding to use. (GPT-3.5 and GPT-4 use cl100k_base as their encoding.)"
18
- )
19
- value3 = gr.Textbox(lines=3, label="Output", placeholder="Output...")
20
-
21
- examples = [
22
- ["The only way to do great work is to love what you do. - Steve Jobs"],
23
- ["In the end, we will remember not the words of our enemies, but the silence of our friends. - Martin Luther King Jr."],
24
- ["Success is not final, failure is not fatal: It is the courage to continue that counts. - Winston Churchill"],
25
- ["The greatest glory in living lies not in never falling, but in rising every time we fall. - Nelson Mandela"],
26
- ["The best and most beautiful things in the world cannot be seen or even touched - they must be felt with the heart. - Helen Keller"]
27
- ]
28
 
29
- demo = gr.Interface(
30
- fn=function,
31
- inputs=[value1, value2],
32
- outputs=value3,
33
- title="ChatGPT Token Calculator",
34
- examples=examples,
35
- description="Calculate the number of tokens in a text string.o",
36
- allow_flagging="never"
37
- )
38
 
39
- demo.launch(debug=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  encodings = tiktoken.list_encoding_names()
5
  encodings.reverse()
6
 
7
+
8
+ def calc(input: str, encoding: str) -> int:
9
+ tokens = tiktoken.get_encoding(encoding).encode(input)
10
  return len(tokens)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
 
 
 
 
 
 
 
 
 
 
12
 
13
+ gr.Interface(
14
+ calc,
15
+ inputs=[
16
+ gr.Text(label="Input", lines=6, placeholder="Enter text here..."),
17
+ gr.Dropdown(
18
+ label="Encoding",
19
+ choices=encodings,
20
+ value="cl100k_base",
21
+ info="The encoding to use. (GPT-3.5 and GPT-4 use cl100k_base as their encoding.)",
22
+ ),
23
+ ],
24
+ outputs=gr.Number(label="Output"),
25
+ title="Tiktoken Calculator",
26
+ allow_flagging="never",
27
+ ).launch()