UNIST-Eunchan
commited on
Commit
•
b48fa68
1
Parent(s):
d872090
Update app.py
Browse files
app.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
import transformers
|
2 |
import streamlit as st
|
3 |
-
|
4 |
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
|
5 |
import json
|
6 |
|
@@ -33,6 +33,7 @@ def infer(input_ids, max_length, temperature, top_k, top_p):
|
|
33 |
|
34 |
|
35 |
def chunking(book_text):
|
|
|
36 |
segments = []
|
37 |
#sentences, token_lens
|
38 |
current_segment = ""
|
|
|
1 |
import transformers
|
2 |
import streamlit as st
|
3 |
+
from nltk import sent_tokenize
|
4 |
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
|
5 |
import json
|
6 |
|
|
|
33 |
|
34 |
|
35 |
def chunking(book_text):
|
36 |
+
sentences = sent_tokenize(book_text)
|
37 |
segments = []
|
38 |
#sentences, token_lens
|
39 |
current_segment = ""
|