test hotdog app

app.py

@@ -1,20 +1,490 @@
+from ast import arg
 import streamlit as st
-from transformers import pipeline
-from PIL import Image
+import pandas as pd
+import PIL
+import re
+from io import StringIO
+import boto3
+from urlextract import URLExtract
+import time
+# from joblib import dump, load
 
-pipeline = pipeline(task="image-classification", model="julien-c/hotdog-not-hotdog")
+import joblib
 
-st.title("Hot Dog? Or Not?")
+from bokeh.models.widgets import Div
 
-file_name = st.file_uploader("Upload a hot dog candidate image")
+import email
+#from ipyfilechooser import FileChooser
 
-if file_name is not None:
-    col1, col2 = st.columns(2)
+#from IPython.display import display
+from io import BytesIO
+from bs4 import BeautifulSoup
+import matplotlib.pyplot as plt
+import numpy as np
+import timeit
 
-    image = Image.open(file_name)
-    col1.image(image, use_column_width=True)
-    predictions = pipeline(image)
 
-    col2.header("Probabilities")
-    for p in predictions:
-        col2.subheader(f"{ p['label'] }: { round(p['score'] * 100, 1)}%")
+def table_data():
+    # creating table data
+    field = [
+        'Data Scientist',
+        'Dataset',
+        'Algorithm',
+        'Framework',
+        'Ensemble',
+        'Domain',
+        'Model Size'
+    ]
+
+    data = [
+        'Chen Song',
+        'Internal + Campaign monitor',
+        'Random Forest',
+        'Sci-kit learn',
+        'Bootstrapping',
+        'Bootstrapping Aggregation',
+        '4 KB'
+    ]
+
+    data = {
+        'Field': field,
+        'Data': data
+    }
+
+    df = pd.DataFrame.from_dict(data)
+
+    return df
+
+
+def url_button(button_name, url):
+    if st.button(button_name):
+        js = """window.open('{url}')""".format(url=url)  # New tab or window
+        html = '<img src onerror="{}">'.format(js)
+        div = Div(text=html)
+        st.bokeh_chart(div)
+
+
+def get_industry_code_dict(training_dataset):
+    training_dataset['industry_code'] = training_dataset['industry'].astype(
+        'category')
+    cat_columns = training_dataset.select_dtypes(['category']).columns
+    training_dataset[cat_columns] = training_dataset[cat_columns].apply(
+        lambda x: x.cat.codes)
+    industry_code_dict = dict(
+        zip(training_dataset.industry, training_dataset.industry_code))
+    return industry_code_dict
+
+def parse_email(uploaded_file):
+    parsed_email = []
+    efile = open(uploaded_file.name,'r')
+    emailstr = ""
+    for i, line in enumerate(efile):
+        emailstr += line
+    
+    b = email.message_from_string(emailstr)
+    for part in b.walk():
+        if part.get_content_type():
+            body = str(part.get_payload())
+            soup = BeautifulSoup(body)
+            paragraphs = soup.find_all('body')
+            for paragraph in paragraphs:
+                parsed_email.append(paragraph.text)
+    return parsed_email
+
+#def email_upload():
+#    print("Please upload your email (In HTML Format)")
+#     upload = FileUpload(accept='.html', multiple=True)
+#     display(upload)
+#     return upload
+#    fc = FileChooser()
+#    display(fc)
+#    return fc
+
+
+# New - In-Use
+def email_extractor(email_uploaded):
+    parse = parse_email(email_uploaded)
+
+    email_text = ''.join(parse).strip()
+
+    # extract the email body using string manipulation functions
+    email_body_start_index = email_text.find('Bright Apps LLC')
+    email_body_end_index = email_text.find('To read more')
+    email_body = email_text[email_body_start_index:email_body_end_index].strip()
+    
+    # get rid of non-text elements
+    email_body = email_body.replace('\n', '')
+    email_body = email_body.replace('\t', '')
+    email_body = email_body.replace('\r', '')
+    email_body = email_body.replace('</b>', '')
+    email_body = email_body.replace('<b>', '')
+    email_body = email_body.replace('\xa0', '')
+    
+    # find length of URLs if any
+    extractor = URLExtract()
+    urls = extractor.find_urls(email_body)
+    url_cnt = len(urls)
+    
+    # remove URLs and get character count
+    body = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', email_body)    
+    sep = '©'
+    body = body.split(sep, 1)[0]
+    character_cnt = sum(not chr.isspace() for chr in body)
+    
+    return email_body, character_cnt, url_cnt
+
+
+# extract email body from parse email
+def email_body_extractor(email_data):
+    # email_data = parsed_email.data[0]
+    emailstr = email_data.decode("utf-8")
+    b = email.message_from_string(emailstr)
+    body = ""
+
+    if b.is_multipart():
+        for part in b.walk():
+            ctype = part.get_content_type()
+            cdispo = str(part.get('Content-Disposition'))
+
+            # skip any text/plain (txt) attachments
+            if ctype == 'text/plain' and 'attachment' not in cdispo:
+                body = part.get_payload()  # decode
+                break
+    # not multipart - i.e. plain text, no attachments, keeping fingers crossed
+    else:
+        body = b.get_payload()
+    # Remove escape sequences
+    body = body.replace('\n', '')
+    body = body.replace('\t', '')
+    body = body.replace('\r', '')
+    body = body.replace('</b>', '')
+    body = body.replace('<b>', '')
+
+    # Extract urls in the email body and get url counts
+    extractor = URLExtract()
+    urls = extractor.find_urls(body)
+    url_cnt = len(urls)
+    # Remove urls
+    body = re.sub(
+        r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', body)
+    sep = '©'
+    body = body.split(sep, 1)[0]
+    character_cnt = sum(not chr.isspace() for chr in body)
+
+    return body, character_cnt, url_cnt
+
+
+def add_bg_from_url():
+    st.markdown(
+        f"""
+         <style>
+         .stApp {{
+             background-image: linear-gradient(#45eff5,#1C8D99);
+             background-attachment: fixed;
+             background-size: cover
+            
+         }}
+         </style>
+         """,
+        unsafe_allow_html=True
+    )
+
+add_bg_from_url()
+#linear-gradient(0deg,#010405 0,#061c2c 55%,#0a3144 75%,#0f4d60)
+
+st.markdown("# Character Count: Email Industry")
+
+
+stats_col1, stats_col2, stats_col3, stats_col4 = st.columns([1, 1, 1, 1])
+
+with stats_col1:
+    st.caption("Production: Ready")
+with stats_col2:
+    st.caption("Accuracy: 85%")
+with stats_col3:
+    st.caption("Speed: 16.89 ms")
+with stats_col4:
+    st.caption("Industry: Email")
+
+
+with st.sidebar:
+    
+    with st.expander('Model Description', expanded=False):
+        img = PIL.Image.open("figures/ModelCC.png")
+        st.image(img)
+        st.markdown('Finding the correct length for an email campaign to maximize user engagement can be an ambiguous task. The Loxz Character Count Model allows you to predict the correct length of your emails for a particular industry and a particular type of email. Using these inputs and trained on an extensive proprietary data set from the Loxz family digital archive, the models incorporate real-world and synthetic data to find the optimized character counts. We applied the random forest algorithm in this model. Bootstrapping was also ensembled in the algorithm which effectively prevents overfitting by reducing variance. The model achieves an 86% accuracy on the test set. This inference-based ML model will help the campaign engineers start with an acceptable length and zero in on the best character count, maximizing engagement in their campaign.')
+
+    with st.expander('Model Information', expanded=False):
+        hide_table_row_index = """
+            <style>
+            thead tr th:first-child {display:none}
+            tbody th {display:none}
+            </style>
+            """
+        st.markdown(hide_table_row_index, unsafe_allow_html=True)
+        st.table(table_data())
+
+    url_button('Model Homepage', 'https://www.loxz.com/#/models/CTA')
+    # url_button('Full Report','https://resources.loxz.com/reports/realtime-ml-character-count-model')
+    url_button('Amazon Market Place', 'https://aws.amazon.com/marketplace')
+
+
+industry_lists = [
+    'Retail',
+    'Software and Technology',
+    'Hospitality',
+    'Academic and Education',
+    'Healthcare',
+    'Energy',
+    'Real Estate',
+    'Entertainment',
+    'Finance and Banking'
+]
+
+campaign_types = [
+    'Promotional',
+    'Transactional',
+    'Webinar',
+    'Survey',
+    'Newsletter',
+    'Engagement',
+    'Usage_and_Consumption',
+    'Review_Request',
+    'Product_Announcement',
+    'Abandoned_Cart'
+]
+
+target_variables = [
+    'conversion_rate',
+    'click_to_open_rate'
+]
+
+uploaded_file = st.file_uploader(
+    "Please upload your email (In HTML Format)", type=["html"])
+
+if uploaded_file is None:
+    # upload_img = PIL.Image.open(uploaded_file)
+    upload_img = None
+# else:
+    # upload_img = None
+
+
+industry = st.selectbox(
+    'Please select your industry',
+    industry_lists,
+    index=6
+)
+
+campaign = st.selectbox(
+    'Please select your campaign type',
+    campaign_types,
+    index=5
+)
+
+target = st.selectbox(
+    'Please select your target variable',
+    target_variables,
+    index=1
+)
+
+st.markdown("""---""")
+
+#char_reco_preference = st.selectbox(
+#    'Do you want to increase or decrease your character count in the email?',
+#    ["Increase", "Decrease"],
+#    index=1)
+
+
+def get_files_from_aws(bucket, prefix):
+    """
+        get files from aws s3 bucket
+    bucket (STRING): bucket name
+    prefix (STRING): file location in s3 bucket
+    """
+    s3_client = boto3.client('s3',
+                             aws_access_key_id=st.secrets["aws_id"],
+                             aws_secret_access_key=st.secrets["aws_key"])
+
+    file_obj = s3_client.get_object(Bucket=bucket, Key=prefix)
+    body = file_obj['Body']
+    string = body.read().decode('utf-8')
+
+    df = pd.read_csv(StringIO(string))
+
+    return df
+
+
+# st.info([industry,campaign,target,char_reco_preference])
+
+
+if st.button('Generate Predictions'):
+    start_time = time.time()
+    if uploaded_file is None:
+        st.error('Please upload a email (HTML format)')
+    else:
+        placeholder = st.empty()
+        placeholder.text('Loading Data')
+
+        # Starting predictions
+        model = joblib.load('models/models.sav')
+        # Generate Email Data
+        email_data = get_files_from_aws(
+            'emailcampaigntrainingdata', 'trainingdata/email_dataset_training.csv')
+        acc_data = get_files_from_aws(
+            'emailcampaigntrainingdata', 'trainingdata/email_dataset_training_raw.csv')
+
+        email_data_ = email_data[["email_body", "industry", "campaign_type",
+                                  "character_cnt", "url_cnt", "Open_Rate", "Click_Through_Rate"]]
+        email_data_ = email_data_.rename(
+            {'Open_Rate': 'Click-to-open_Rate', 'Click_Through_Rate': 'Conversion_Rate'})
+        df_email_data = email_data_.rename(
+            columns={'Open_Rate': 'Click-to-open_Rate', 'Click_Through_Rate': 'Conversion_Rate'})
+
+        # Dataset:
+        training_dataset = get_files_from_aws(
+            'emailcampaigntrainingdata', 'modelCC/training.csv')
+        # X_test = get_files_from_aws('emailcampaigntrainingdata','modelCC/Xtest.csv')
+        # Y_test = get_files_from_aws('emailcampaigntrainingdata','modelCC/ytest.csv')
+
+        # print("Getting Data Time: %s seconds" % (time.time() - start_time))
+
+        industry_code_dict = get_industry_code_dict(email_data)
+        #uploaded_file = FileChooser(uploaded_file)
+        #bytes_data = uploaded_file.getvalue()
+
+        email_body, character_cnt, url_cnt = email_extractor(uploaded_file)
+
+        # Start the prediction
+        # Need to solve X test issue
+
+        # y_pred = model.predict(X_test)
+        df_uploaded = pd.DataFrame(
+            columns=['character_cnt', "url_cnt", "industry"])
+        df_uploaded.loc[0] = [character_cnt, url_cnt, industry]
+        df_uploaded["industry_code"] = industry_code_dict.get(industry)
+        df_uploaded_test = df_uploaded[[
+            "industry_code", "character_cnt", "url_cnt"]]
+        predicted_rate = model.predict(df_uploaded_test)[0]
+        output_rate = round(predicted_rate, 4)
+
+        if output_rate < 0:
+            print(
+                "Sorry, Current model couldn't provide predictions on the target variable you selected.")
+        else:
+            st.markdown('#### Current Character Count in Your Email is: <span style="color:blue">{}</span>'.format(
+                character_cnt), unsafe_allow_html=True)
+            # st.info('The model predicts that it achieves a {} of {}%'.format(target, str(round(output_rate*100,2))))
+            if target == 'conversion_rate':
+                target_vis = 'Click_Through_Rate'
+            else:
+                target_vis = 'Open_Rate'
+                
+            st.markdown('#### The model predicts that it achieves a <span style="color:blue">{}</span> of <span style="color:blue">{}</span>%'.format(
+                target_vis, str(round(output_rate*100, 3))), unsafe_allow_html=True)
+            selected_industry_code = industry_code_dict.get(industry)
+
+            if target == "click_to_open_rate":
+                selected_variable = "Open_Rate"
+            if target == "conversion_rate":
+                selected_variable = "Click_Through_Rate"
+
+            df_reco = training_dataset[[
+                "industry_code", "character_cnt", "url_cnt", selected_variable]]
+            df_reco = df_reco[df_reco["industry_code"]
+                              == selected_industry_code]
+            df_reco[selected_variable] = df_reco[selected_variable].apply(
+                lambda x: round(x, 3))
+            df_reco_sort = df_reco.sort_values(by=[selected_variable])
+            df_reco = df_reco.drop_duplicates(subset=selected_variable)
+
+            #preference = char_reco_preference
+            #if preference == "Increase":
+            #    df_reco_opt = df_reco[(df_reco[selected_variable] > output_rate) & (
+            #        df_reco["character_cnt"] > character_cnt) & (df_reco["character_cnt"] <= (1.5*character_cnt))]
+            #    df_reco_opt_rank = df_reco_opt.nlargest(3, [selected_variable])
+            # decrease character reco
+            #if preference == "Decrease":
+            #    df_reco_opt = df_reco[(df_reco[selected_variable] > output_rate) & (
+            #        df_reco["character_cnt"] < character_cnt)]
+            #    df_reco_opt_rank = df_reco_opt.nlargest(3, [selected_variable])
+
+
+            # split into two dataframes of higher and lower character_cnt (added apr 2023)
+            char_cnt_uploaded = character_cnt
+
+            df_reco_opt1 = df_reco[(df_reco[selected_variable] > output_rate) & (df_reco["character_cnt"] > char_cnt_uploaded) & (df_reco["character_cnt"] <= (1.5*char_cnt_uploaded))]
+            df_reco_opt2 = df_reco[(df_reco[selected_variable] > output_rate) & (df_reco["character_cnt"] < char_cnt_uploaded) & (df_reco["character_cnt"] >= (char_cnt_uploaded/2))]
+
+            # drop duplicates of character_cnt keeping the row with the highest output_rate
+            df_reco_opt1 = df_reco_opt1.sort_values(by=[selected_variable], ascending=False).drop_duplicates(subset=["character_cnt"])
+            df_reco_opt2 = df_reco_opt2.sort_values(by=[selected_variable], ascending=False).drop_duplicates(subset=["character_cnt"])
+            
+            # get top 2 largest in higher and lower dataframe
+            df_reco_opt_rank1 = df_reco_opt1.nlargest(2, [selected_variable])
+            df_reco_opt_rank2 = df_reco_opt2.nlargest(2, [selected_variable])
+
+            df_reco_opt_rank = pd.concat([df_reco_opt_rank1, df_reco_opt_rank2])
+            df_reco_opt_rank = df_reco_opt_rank.nlargest(3,[selected_variable])
+
+            if selected_variable == "Open_Rate":
+                selected_variable = "Click-to-Open_Rate"
+            if selected_variable == "Click_Through_Rate":
+                selected_variable = "Conversion_Rate"
+
+            st.markdown('#### To get higher, <span style="color:blue">{}</span>, the model recommends the following options:'.format(
+                selected_variable), unsafe_allow_html=True)
+            if len(df_reco_opt_rank) == 0:
+                st.markdown('#### You ve already achieved the highest, <span style="color:blue">{}</span>, with the current character count!'.format(
+                    selected_variable), unsafe_allow_html=True)
+            else:
+                #for _, row in df_reco_opt_rank.iterrows():
+                #    Character_Count = row[1]
+                #    selected_variable = row[3]
+                    # print(f"·Number of Characters: {int(Character_Count)}, Target Rate: {round(selected_variable, 3)*100}", "%")
+                #    st.markdown('Number of Characters: {}, Target Rate: {}'.format(
+                #        int(Character_Count), round(selected_variable*100, 3)))
+
+                chars = []
+                sel_var_values = []
+
+                for _, row in df_reco_opt_rank.iterrows():
+                    Character_Count = row[1]
+                    selected_variable_number = row[3]
+                    chars.append(int(Character_Count))
+                    sel_var_values.append(round(selected_variable_number, 3)*100)
+                    st.write(f"·Number of Characters: {int(Character_Count)}, Target Rate: {round(round(selected_variable_number, 3)*100, 3)}", "%")
+                st.write("\n")
+
+                if len(chars) > 1:
+                    #fig = plt.figure()
+                    #ax = fig.add_axes([0,0,1,1])
+                    fig, ax = plt.subplots(figsize=(10,4))
+                    bars = ax.barh(np.arange(len(chars)), sel_var_values, height=0.175, color='#0F4D60')
+                    
+                    #ax.bar_label(bars)
+
+                    ax.set_yticks(np.arange(len(chars)))
+                    ax.set_yticklabels(np.array(chars), fontsize=14)
+                    ax.set_title('Character Counts vs. Target Variable Rates', fontsize=18)
+                    ax.set_ylabel('Character Counts', fontsize=16)
+                    ax.set_xlabel('Target Rates %', fontsize=16)
+                    
+                    for i, bar in enumerate(bars):
+                        rounded_value = round(sel_var_values[i], 2)
+                        ax.text(bar.get_width() + 0.3, bar.get_y() + bar.get_height()/2, str(rounded_value) + '%', ha='left', va='center', fontsize=12, fontweight='bold')
+                    
+                    ax.margins(0.1,0.05)
+                    
+                    biggest_bar_index = np.argmax(sel_var_values)
+                    bars[biggest_bar_index].set_color('#00BF93')
+                    
+                    st.plotly_chart(fig, use_container_width=True)
+
+                    st.write("\n")
+                    #st.write(np.array(chars))
+                chars_out = dict(zip(chars, sel_var_values))
+                sorted_chars_out = sorted(chars_out.items(), key=lambda x: x[1], reverse=True)
+                
+
+        placeholder.empty()
+        #st.write(time.time() - start_time)

requirements.txt

@@ -1,5 +1,5 @@
-transformers
-torch
+# transformers
+# torch
 streamlit
 altair<5
 pickle5