File size: 14,298 Bytes
28183db
 
 
164cb45
a746976
28183db
51082bd
164cb45
28183db
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
021a052
28183db
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
164cb45
28183db
 
 
 
a746976
28183db
 
51082bd
28183db
 
 
 
 
 
 
 
 
 
a746976
28183db
 
 
 
 
 
 
 
 
8dfe29e
 
28183db
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
228ea6c
28183db
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5286542
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28183db
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
021a052
 
 
28183db
 
 
 
 
 
 
 
021a052
 
 
 
 
28183db
 
8dfe29e
 
 
 
 
 
 
 
 
 
 
 
 
 
28183db
06aaef2
 
 
 
 
 
 
 
 
 
28183db
 
8dfe29e
28183db
021a052
28183db
 
 
 
228ea6c
28183db
 
 
 
 
 
 
4ebac20
 
28183db
62f4be8
28183db
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
import time
import random
import logging
import streamlit as st
import pandas as pd

from dotenv import load_dotenv

import utils
import db
import modeling
import plots

def set_if_not_in_session_state(key, value):
    """Helper function to initialize a session state variable if it doesn't exist."""
    if key not in st.session_state:
        st.session_state[key] = value

def initialize():
    """Initialization function to set up logging, load environment variables, and initialize session state variables."""
    load_dotenv()
    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
    
    keys = ['selected_rating', 'collect_data', 'gender_value', 'expert_value', 'show_launch', 'user_id', 'statements', 'current_statement', 'db']
    values = [0, None, None, None, True, random.randint(1, 999_999_999), None, None, None]
    
    for key, value in zip(keys, values):
        set_if_not_in_session_state(key, value)

    connect_to_database()

def connect_to_database():
    """Establishes a connection to the database."""
    if st.session_state.db is None:
        credentials_dict = db.load_credentials()
        connection_attempts = 0

        while st.session_state.db is None and connection_attempts < 3:
            st.session_state.db = db.connect_to_db(credentials_dict)
            if st.session_state.db is None:
                logging.info('Retrying to connect to db...')
                connection_attempts += 1
                time.sleep(1)
    else:
        retrieve_statements()

def retrieve_statements():
    """Retrieves statements from the database."""
    retrieval_attempts = 0

    while st.session_state.statements is None and retrieval_attempts < 3:
        st.session_state.statements = db.get_statements_from_db(st.session_state.db)
        st.session_state.current_statement = db.pick_random(st.session_state.statements)
        if st.session_state.statements is None:
            logging.info('Retrying to retrieve statements from db...')
            retrieval_attempts += 1
            time.sleep(1)

def get_user_consent():
    st.markdown("""
        ### Support Future Research
        Additionally, we kindly ask for your agreement to collect anonymous data from your app usage in order to improve future research. 
        You may choose to agree or decline this data collection.
    """)

    collect_data_options = ['Yes, I agree and want to support and help improve this research', 'No']
    collect_data_input = st.radio(
        label='You may choose to agree or decline this data collection.',
        options=collect_data_options,
        horizontal=True,
        label_visibility='collapsed'
    )
    return collect_data_options.index(collect_data_input) == 0


def get_user_info():
    gender_options = ['[Please select]', 'Female', 'Male', 'Other']
    gender_input = st.selectbox(
        label='Please select your gender',
        options=gender_options,
    )
    gender_value = gender_options.index(gender_input)

    expert_options = [
        '[Please select]', 
        'No, I do not have a background in social or behavioral sciences', 
        'Yes, I have either studied social or behavioral sciences or I am currently a student in this field', 
        'Yes, I have either worked as a researcher in the field of social or behavioral sciences or I have had past experience as a researcher in this area'
    ]
    expert_input = st.selectbox(
        label='Please indicate whether you have any experience or educational background in social or behavioral sciences (e.g., psychology)',
        options=expert_options,
    )
    expert_value = expert_options.index(expert_input)
    
    return expert_value, gender_value

def get_user_rating(placeholder):

    with placeholder:
        with st.container():
            st.markdown(f"""
                ### How desirable is the following statement?
                To support future research, rate the following statement according to whether it is socially desirable or undesirable.
                Is it socially desirable or undesirable to endorse the following statement?
                #### <center>\"{st.session_state.current_statement.capitalize()}\"</center>
            """, unsafe_allow_html=True)
            
            rating_options = ['[Please select]', 'Very undesirable', 'Undesirable', 'Neutral', 'Desirable', 'Very desirable']
            
            selected_rating = st.selectbox(
                label='Rate the statement above according to whether it is socially desirable or undesirable.',
                options=rating_options,
                key='selection'
            )

            suitability_options = ['No, I\'m just playing around', 'Yes, my input can help improve this research']

            research_suitability = st.radio(
                label='Is your input suitable for research purposes?',
                options=suitability_options,
                horizontal=True
            )

            st.session_state.collect_data_optout = st.checkbox(
                label='Don\'t ask me to rate further statements.',
                value=False
            )

            st.session_state.item_rating = rating_options.index(selected_rating)
            st.session_state.suitability_rating = suitability_options.index(research_suitability)

def handle_acceptance(collect_data_value, expert_value, gender_value, message):
    if st.button(label='Accept Disclaimer', type='primary', use_container_width=True):
        if collect_data_value and not (expert_value > 0 and gender_value > 0):
            message.error('Please answer the questions above!')
        else:
            st.session_state.expert_value = expert_value
            st.session_state.gender_value = gender_value
            st.session_state.show_launch = False
            st.session_state.collect_data = collect_data_value
            st.experimental_rerun()

def show_launch(placeholder):
    with placeholder:
        with st.container():
            st.divider()
            st.markdown("""
                ## Before Using the App
                ### Disclaimer
                This application is provided as-is, without any warranty or guarantee of any kind, expressed or implied. It is intended for educational, non-commercial use only. 
                The developers of this app shall not be held liable for any damages or losses incurred from its use. By using this application, you agree to the terms and conditions 
                outlined herein and acknowledge that any commercial use or reliance on its functionality is strictly prohibited.
            """, unsafe_allow_html=True)

            collect_data_value = False
            if st.session_state.db:
                collect_data_value = get_user_consent()

            expert_value, gender_value = (0, 0)
            if collect_data_value:
                expert_value, gender_value = get_user_info()

            message = st.empty()

            handle_acceptance(collect_data_value, expert_value, gender_value, message)

def show_summary(placeholder):
    with placeholder:
        with st.container():
            st.markdown("""
                ## What is the focus of this research?
                Certain biases can affect how people respond to surveys and psychological questionnaires. 
                For example, survey respondents may attempt to conceal socially undesirable traits (e.g., 
                being ill-tempered) and endorse statements that cast them in a favorable manner (e.g., 
                being cooperative).

                Developers of psychological questionnaires hence sometimes aim to ensure that questions 
                are neutral, or that a subset of questions is equally (un)desirable. In the past, human 
                judges have been tasked with quantifying item desirability. In contrast, the research 
                underlying this web application demonstrates that large language models (LLMs) can 
                achieve this too!
            """)

def handle_demo_input():
    
    if st.session_state.collect_data:
        if st.session_state.item_rating > 0:

            st.session_state.sentiment, st.session_state.desirability = modeling.score_text(st.session_state.input_text)

            payload = {
                'user_id': st.session_state.user_id,
                'gender_value': st.session_state.gender_value,
                'expert_value': st.session_state.expert_value,
                'statement': st.session_state.current_statement,
                'rating': st.session_state.item_rating,
                'suitability': st.session_state.suitability_rating,
                'input_text': st.session_state.input_text,
                'sentiment': st.session_state.sentiment,
                'desirability': st.session_state.desirability,
            }
            write_to_db_success = db.write_to_db(st.session_state.db, payload)

            if st.session_state.collect_data_optout:
                st.session_state.collect_data = False

            if write_to_db_success:
                st.session_state.current_statement = db.pick_random(st.session_state.statements)
                st.session_state.selection = '[Please select]'
        else:
            return None
    else:
        st.session_state.sentiment, st.session_state.desirability = modeling.score_text(st.session_state.input_text)

        if st.session_state.db:
            payload = {
                'user_id': st.session_state.user_id,
                'gender_value': None,
                'expert_value': None,
                'statement': None,
                'rating': None,
                'suitability': None,
                'input_text': st.session_state.input_text,
                'sentiment': st.session_state.sentiment,
                'desirability': st.session_state.desirability,
            }

        write_to_db_success = db.write_to_db(st.session_state.db, payload)
        

def show_demo(placeholder):
    with placeholder:
        with st.container():
            st.divider()            
            st.markdown("""
                ## Try it yourself!
                Use the text field below to enter a statement that might be part of a psychological 
                questionnaire (e.g., "I love a good fight."). Your input will be processed by 
                language models, returning a machine-based estimate of item sentiment (i.e., valence) 
                and desirability.

            """)
            modeling.load_model()

            if 'sentiment' in st.session_state and 'desirability' in st.session_state:                    
                plots.show_scores(
                    sentiment=st.session_state.sentiment, 
                    desirability=st.session_state.desirability, 
                    input_text=st.session_state.input_text
                )

            st.session_state.input_text = st.text_input(
                label='Item text/statement:',
                value='I love a good fight.',
                placeholder='Enter item text'
            )        

            user_rating_placeholder = st.empty()

            if st.session_state.collect_data:                
                get_user_rating(user_rating_placeholder)

            if st.button(
                label='Evaluate Item Text',
                on_click=handle_demo_input,
                type='primary',
                use_container_width=True
            ):
                if st.session_state.collect_data and st.session_state.item_rating == 0:
                    st.error('Please rate the statement presented above!')

def show_data(placeholder):
    with placeholder:
        with st.container():
            st.divider()   
            st.markdown("""
                ## Explore the data
                Figures show the accuarcy in precitions of human-rated item desirability by 
                the sentiment model (left) and the desirability model (right), using 
                `test`-partition data only. 
            """)

            show_covariates = st.checkbox('Show covariates', value=True)
            if show_covariates:
                option = st.selectbox('Group by', options=list(utils.covariate_columns.values()))
            else:
                option = None
            
            if 'df' not in st.session_state:
                utils.load_data()

            plot = plots.scatter_plot(st.session_state.df, option)
            st.plotly_chart(plot, theme=None, use_container_width=True)

def main():
    st.set_page_config(page_title='Machine-Based Item Desirability Ratings')


    # Define the logo and its link
    logo_url = "logo-130x130.svg"
    logo_link = "https://example.com/"

    col1, col2 = st.columns([2, 5])
    with col1:
        st.image('logo-130x130.svg')        

    with col2:
        st.markdown("# Machine-Based Item Desirability Ratings")        
        
    st.markdown("""
        This web application showcases the process of obtaining item desirability ratings using natural language processing (AI), accompanying the paper "Expanding the Methodological Toolbox: Machine-Based Item Desirability Ratings as an Alternative to Human-Based Ratings".
        
        πŸ“– Paper: https://doi.org/10.1016/j.paid.2023.112307
        
        πŸ“‹ Preprint: https://psyarxiv.com/7bpeq

        πŸ’Ύ Data: https://osf.io/67mkz/

        πŸ–ŠοΈ Cite:<br> Hommel, B. E. (2023). Expanding the methodological toolbox: Machine-based item desirability ratings as an alternative to human-based ratings. Personality and Individual Differences, 213, 112307. https://doi.org/10.1016/j.paid.2023.112307
        
        

        The web application is maintained by [magnolia psychometrics](https://www.magnolia-psychometrics.com/).
    """, unsafe_allow_html=True)

    placeholder_launch = st.empty()
    placeholder_summary = st.empty()
    placeholder_demo = st.empty()
    placeholder_data = st.empty()

    if st.session_state.show_launch is True:
        show_launch(placeholder_launch)
    else:
        placeholder_launch = st.empty()
        show_summary(placeholder_summary)
        show_demo(placeholder_demo)
        show_data(placeholder_data)


if __name__ == '__main__':

    initialize()
    main()