Spaces:
Running
Running
""" Creates a vocabulary from a tsv file. | |
""" | |
import codecs | |
import example_helper | |
from torchmoji.create_vocab import VocabBuilder | |
from torchmoji.word_generator import TweetWordGenerator | |
with codecs.open('../../twitterdata/tweets.2016-09-01', 'rU', 'utf-8') as stream: | |
wg = TweetWordGenerator(stream) | |
vb = VocabBuilder(wg) | |
vb.count_all_words() | |
vb.save_vocab() | |