johnowhitaker commited on
Commit
afb9463
1 Parent(s): 764af30

Adding transformers part - will it download the model each time? Hope not

Browse files
Files changed (1) hide show
  1. app.py +24 -2
app.py CHANGED
@@ -2,9 +2,32 @@ import streamlit as st
2
  import pandas as pd
3
  import twint
4
  import nest_asyncio
5
-
6
  import multiprocessing.pool
7
  import functools
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
 
9
  # https://stackoverflow.com/questions/492519/timeout-on-a-function-call
10
  def timeout(max_timeout):
@@ -45,7 +68,6 @@ def get_tweets(username, limit=500, save_name=None):
45
 
46
  st.title('Test')
47
 
48
- st.title(get_tweets('johnowhitaker', 20).shape)
49
 
50
  with st.form("my_form"):
51
  st.write("Inside the form")
 
2
  import pandas as pd
3
  import twint
4
  import nest_asyncio
 
5
  import multiprocessing.pool
6
  import functools
7
+ from transformers import AutoModelForSequenceClassification
8
+ from transformers import TFAutoModelForSequenceClassification
9
+ from transformers import AutoTokenizer
10
+ import numpy as np
11
+ from scipy.special import softmax
12
+ import csv
13
+ import urllib.request
14
+ import IPython.display as ipd
15
+
16
+ # Preprocess text (username and link placeholders)
17
+ def preprocess(text):
18
+ new_text = []
19
+
20
+
21
+ for t in text.split(" "):
22
+ t = '@user' if t.startswith('@') and len(t) > 1 else t
23
+ t = 'http' if t.startswith('http') else t
24
+ new_text.append(t)
25
+ return " ".join(new_text)
26
+
27
+ MODEL = "cardiffnlp/twitter-roberta-base-sentiment"
28
+ tokenizer = AutoTokenizer.from_pretrained(MODEL)
29
+ model = AutoModelForSequenceClassification.from_pretrained(MODEL)
30
+ model.save_pretrained(MODEL)
31
 
32
  # https://stackoverflow.com/questions/492519/timeout-on-a-function-call
33
  def timeout(max_timeout):
 
68
 
69
  st.title('Test')
70
 
 
71
 
72
  with st.form("my_form"):
73
  st.write("Inside the form")