Spaces:
Running
Running
File size: 670 Bytes
86a83a2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
"""
Take a given list of sentences and turn it into a numpy array, where each
number corresponds to a word. Padding is used (number 0) to ensure fixed length
of sentences.
"""
from __future__ import print_function, unicode_literals
import example_helper
import json
from torchmoji.sentence_tokenizer import SentenceTokenizer
with open('../model/vocabulary.json', 'r') as f:
vocabulary = json.load(f)
st = SentenceTokenizer(vocabulary, 30)
test_sentences = [
'\u2014 -- \u203c !!\U0001F602',
'Hello world!',
'This is a sample tweet #example',
]
tokens, infos, stats = st.tokenize_sentences(test_sentences)
print(tokens)
print(infos)
print(stats)
|