Spaces:
Runtime error
Runtime error
import time | |
import random | |
import logging | |
import streamlit as st | |
import pandas as pd | |
from dotenv import load_dotenv | |
import utils | |
import db | |
import modeling | |
import plots | |
def set_if_not_in_session_state(key, value): | |
"""Helper function to initialize a session state variable if it doesn't exist.""" | |
if key not in st.session_state: | |
st.session_state[key] = value | |
def initialize(): | |
"""Initialization function to set up logging, load environment variables, and initialize session state variables.""" | |
load_dotenv() | |
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO) | |
keys = ['selected_rating', 'collect_data', 'gender_value', 'expert_value', 'show_launch', 'user_id', 'statements', 'current_statement', 'db'] | |
values = [0, None, None, None, True, random.randint(1, 999_999_999), None, None, None] | |
for key, value in zip(keys, values): | |
set_if_not_in_session_state(key, value) | |
connect_to_database() | |
def connect_to_database(): | |
"""Establishes a connection to the database.""" | |
if st.session_state.db is None: | |
credentials_dict = db.load_credentials() | |
connection_attempts = 0 | |
while st.session_state.db is None and connection_attempts < 3: | |
st.session_state.db = db.connect_to_db(credentials_dict) | |
if st.session_state.db is None: | |
logging.info('Retrying to connect to db...') | |
connection_attempts += 1 | |
time.sleep(1) | |
else: | |
retrieve_statements() | |
def retrieve_statements(): | |
"""Retrieves statements from the database.""" | |
retrieval_attempts = 0 | |
while st.session_state.statements is None and retrieval_attempts < 3: | |
st.session_state.statements = db.get_statements_from_db(st.session_state.db) | |
st.session_state.current_statement = db.pick_random(st.session_state.statements) | |
if st.session_state.statements is None: | |
logging.info('Retrying to retrieve statements from db...') | |
retrieval_attempts += 1 | |
time.sleep(1) | |
def get_user_consent(): | |
st.markdown(""" | |
### Support Future Research | |
Additionally, we kindly ask for your agreement to collect anonymous data from your app usage in order to improve future research. | |
You may choose to agree or decline this data collection. | |
""") | |
collect_data_options = ['Yes, I agree and want to support and help improve this research', 'No'] | |
collect_data_input = st.radio( | |
label='You may choose to agree or decline this data collection.', | |
options=collect_data_options, | |
horizontal=True, | |
label_visibility='collapsed' | |
) | |
return collect_data_options.index(collect_data_input) == 0 | |
def get_user_info(): | |
gender_options = ['[Please select]', 'Female', 'Male', 'Other'] | |
gender_input = st.selectbox( | |
label='Please select your gender', | |
options=gender_options, | |
) | |
gender_value = gender_options.index(gender_input) | |
expert_options = [ | |
'[Please select]', | |
'No, I do not have a background in social or behavioral sciences', | |
'Yes, I have either studied social or behavioral sciences or I am currently a student in this field', | |
'Yes, I have either worked as a researcher in the field of social or behavioral sciences or I have had past experience as a researcher in this area' | |
] | |
expert_input = st.selectbox( | |
label='Please indicate whether you have any experience or educational background in social or behavioral sciences (e.g., psychology)', | |
options=expert_options, | |
) | |
expert_value = expert_options.index(expert_input) | |
return expert_value, gender_value | |
def get_user_rating(placeholder): | |
with placeholder: | |
with st.container(): | |
st.markdown(f""" | |
### How desirable is the following statement? | |
To support future research, rate the following statement according to whether it is socially desirable or undesirable. | |
Is it socially desirable or undesirable to endorse the following statement? | |
#### <center>\"{st.session_state.current_statement.capitalize()}\"</center> | |
""", unsafe_allow_html=True) | |
rating_options = ['[Please select]', 'Very undesirable', 'Undesirable', 'Neutral', 'Desirable', 'Very desirable'] | |
selected_rating = st.selectbox( | |
label='Rate the statement above according to whether it is socially desirable or undesirable.', | |
options=rating_options, | |
key='selection' | |
) | |
suitability_options = ['No, I\'m just playing around', 'Yes, my input can help improve this research'] | |
research_suitability = st.radio( | |
label='Is your input suitable for research purposes?', | |
options=suitability_options, | |
horizontal=True | |
) | |
st.session_state.collect_data_optout = st.checkbox( | |
label='Don\'t ask me to rate further statements.', | |
value=False | |
) | |
st.session_state.item_rating = rating_options.index(selected_rating) | |
st.session_state.suitability_rating = suitability_options.index(research_suitability) | |
def handle_acceptance(collect_data_value, expert_value, gender_value, message): | |
if st.button(label='Accept Disclaimer', type='primary', use_container_width=True): | |
if collect_data_value and not (expert_value > 0 and gender_value > 0): | |
message.error('Please answer the questions above!') | |
else: | |
st.session_state.expert_value = expert_value | |
st.session_state.gender_value = gender_value | |
st.session_state.show_launch = False | |
st.session_state.collect_data = collect_data_value | |
st.experimental_rerun() | |
def show_launch(placeholder): | |
with placeholder: | |
with st.container(): | |
st.divider() | |
st.markdown(""" | |
## Before Using the App | |
### Disclaimer | |
This application is provided as-is, without any warranty or guarantee of any kind, expressed or implied. It is intended for educational, non-commercial use only. | |
The developers of this app shall not be held liable for any damages or losses incurred from its use. By using this application, you agree to the terms and conditions | |
outlined herein and acknowledge that any commercial use or reliance on its functionality is strictly prohibited. | |
""", unsafe_allow_html=True) | |
collect_data_value = False | |
if st.session_state.db: | |
collect_data_value = get_user_consent() | |
expert_value, gender_value = (0, 0) | |
if collect_data_value: | |
expert_value, gender_value = get_user_info() | |
message = st.empty() | |
handle_acceptance(collect_data_value, expert_value, gender_value, message) | |
def show_summary(placeholder): | |
with placeholder: | |
with st.container(): | |
st.markdown(""" | |
## What is the focus of this research? | |
Certain biases can affect how people respond to surveys and psychological questionnaires. | |
For example, survey respondents may attempt to conceal socially undesirable traits (e.g., | |
being ill-tempered) and endorse statements that cast them in a favorable manner (e.g., | |
being cooperative). | |
Developers of psychological questionnaires hence sometimes aim to ensure that questions | |
are neutral, or that a subset of questions is equally (un)desirable. In the past, human | |
judges have been tasked with quantifying item desirability. In contrast, the research | |
underlying this web application demonstrates that large language models (LLMs) can | |
achieve this too! | |
""") | |
def handle_demo_input(): | |
if st.session_state.collect_data: | |
if st.session_state.item_rating > 0: | |
st.session_state.sentiment, st.session_state.desirability = modeling.score_text(st.session_state.input_text) | |
payload = { | |
'user_id': st.session_state.user_id, | |
'gender_value': st.session_state.gender_value, | |
'expert_value': st.session_state.expert_value, | |
'statement': st.session_state.current_statement, | |
'rating': st.session_state.item_rating, | |
'suitability': st.session_state.suitability_rating, | |
'input_text': st.session_state.input_text, | |
'sentiment': st.session_state.sentiment, | |
'desirability': st.session_state.desirability, | |
} | |
write_to_db_success = db.write_to_db(st.session_state.db, payload) | |
if st.session_state.collect_data_optout: | |
st.session_state.collect_data = False | |
if write_to_db_success: | |
st.session_state.current_statement = db.pick_random(st.session_state.statements) | |
st.session_state.selection = '[Please select]' | |
else: | |
return None | |
else: | |
st.session_state.sentiment, st.session_state.desirability = modeling.score_text(st.session_state.input_text) | |
if st.session_state.db: | |
payload = { | |
'user_id': st.session_state.user_id, | |
'gender_value': None, | |
'expert_value': None, | |
'statement': None, | |
'rating': None, | |
'suitability': None, | |
'input_text': st.session_state.input_text, | |
'sentiment': st.session_state.sentiment, | |
'desirability': st.session_state.desirability, | |
} | |
write_to_db_success = db.write_to_db(st.session_state.db, payload) | |
def show_demo(placeholder): | |
with placeholder: | |
with st.container(): | |
st.divider() | |
st.markdown(""" | |
## Try it yourself! | |
Use the text field below to enter a statement that might be part of a psychological | |
questionnaire (e.g., "I love a good fight."). Your input will be processed by | |
language models, returning a machine-based estimate of item sentiment (i.e., valence) | |
and desirability. | |
""") | |
modeling.load_model() | |
if 'sentiment' in st.session_state and 'desirability' in st.session_state: | |
plots.show_scores( | |
sentiment=st.session_state.sentiment, | |
desirability=st.session_state.desirability, | |
input_text=st.session_state.input_text | |
) | |
st.session_state.input_text = st.text_input( | |
label='Item text/statement:', | |
value='I love a good fight.', | |
placeholder='Enter item text' | |
) | |
user_rating_placeholder = st.empty() | |
if st.session_state.collect_data: | |
get_user_rating(user_rating_placeholder) | |
if st.button( | |
label='Evaluate Item Text', | |
on_click=handle_demo_input, | |
type='primary', | |
use_container_width=True | |
): | |
if st.session_state.collect_data and st.session_state.item_rating == 0: | |
st.error('Please rate the statement presented above!') | |
def show_data(placeholder): | |
with placeholder: | |
with st.container(): | |
st.divider() | |
st.markdown(""" | |
## Explore the data | |
Figures show the accuarcy in precitions of human-rated item desirability by | |
the sentiment model (left) and the desirability model (right), using | |
`test`-partition data only. | |
""") | |
show_covariates = st.checkbox('Show covariates', value=True) | |
if show_covariates: | |
option = st.selectbox('Group by', options=list(utils.covariate_columns.values())) | |
else: | |
option = None | |
if 'df' not in st.session_state: | |
utils.load_data() | |
plot = plots.scatter_plot(st.session_state.df, option) | |
st.plotly_chart(plot, theme=None, use_container_width=True) | |
def main(): | |
st.set_page_config(page_title='Machine-Based Item Desirability Ratings') | |
# Define the logo and its link | |
logo_url = "logo-130x130.svg" | |
logo_link = "https://example.com/" | |
col1, col2 = st.columns([2, 5]) | |
with col1: | |
st.image('logo-130x130.svg') | |
with col2: | |
st.markdown("# Machine-Based Item Desirability Ratings") | |
st.markdown(""" | |
This web application showcases the process of obtaining item desirability ratings using natural language processing (AI), accompanying the paper "Expanding the Methodological Toolbox: Machine-Based Item Desirability Ratings as an Alternative to Human-Based Ratings". | |
π Paper: https://doi.org/10.1016/j.paid.2023.112307 | |
π Preprint: https://psyarxiv.com/7bpeq | |
πΎ Data: https://osf.io/67mkz/ | |
ποΈ Cite:<br> Hommel, B. E. (2023). Expanding the methodological toolbox: Machine-based item desirability ratings as an alternative to human-based ratings. Personality and Individual Differences, 213, 112307. https://doi.org/10.1016/j.paid.2023.112307 | |
#οΈβ£ Twitter: https://twitter.com/BjoernHommel | |
The web application is maintained by [magnolia psychometrics](https://www.magnolia-psychometrics.com/). | |
""", unsafe_allow_html=True) | |
placeholder_launch = st.empty() | |
placeholder_summary = st.empty() | |
placeholder_demo = st.empty() | |
placeholder_data = st.empty() | |
if st.session_state.show_launch is True: | |
show_launch(placeholder_launch) | |
else: | |
placeholder_launch = st.empty() | |
show_summary(placeholder_summary) | |
show_demo(placeholder_demo) | |
show_data(placeholder_data) | |
if __name__ == '__main__': | |
initialize() | |
main() |