init app

app.py

@@ -0,0 +1,24 @@
+import streamlit as st
+
+from beluga import load_model, process_emotions, generate_prompt
+from emodeepface import check_image_rotation, process_photo
+
+# Title webpage
+st.title("Affective Journaling Assistant")
+
+# User instructions
+st.write("""
+Welcome to the Affective Journaling Assistant!
+
+For a tailored journaling experience, we analyze your facial expressions to gauge your emotions.
+To proceed:
+
+1. Ensure the image is well-lit and of high quality.
+2. Your face should be fully visible without obstructions (e.g., no sunglasses or hats).
+3. By uploading, you acknowledge and consent to our data processing.
+
+Let's get started!
+""")
+
+# User image upload
+file_name = st.file_uploader("Please upload your photo.")

beluga.py

@@ -0,0 +1,150 @@
+# import required libraries
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+def load_model():
+    # define the tokenizer for preprocessing any inputs
+    tokenizer = AutoTokenizer.from_pretrained("stabilityai/StableBeluga-7B", use_fast=False)
+    # define the LLM model for the generating output
+    model = AutoModelForCausalLM.from_pretrained("stabilityai/StableBeluga-7B", torch_dtype=torch.float16, low_cpu_mem_usage=True, device_map="auto")
+    return (model, tokenizer)
+
+def process_emotions(model, tokenizer, emotion_data):
+    # ensure emotion_data is a dictionary
+    if not isinstance(emotion_data, dict):
+        raise ValueError("Expected emotion_data to be a dictionary.")
+    # convert emotion_data dictionary to string format
+    emotions = ', '.join(f"{k}: {v:.2f}%" for k, v in emotion_data.items())
+    # prompt-engineering instead of fine-tuning
+    system_prompt = """### System:
+    You are StableBeluga, an AI programmed to follow instructions with high accuracy. Your current task is to assist with enhancing human emotional regulation and intelligence. You will receive information about the emotional ratios of a client. Your task is to identify the one or two emotions with the highest values from the information provided, and create a one or two-sentence original journal prompt for the client's self-reflection and emotional exploration. Do not discuss the emotions or their scores, and do not include any other introductory or explanatory text. Only provide the journal prompt, and ensure it is based on the emotions with the highest values. Here are examples of the desired format:
+
+    Example 1: 'Reflect on a recent situation when you felt anger. What was the trigger and how did it affect your actions?'
+
+    Example 2: 'Write about a moment when you felt sad. What thoughts or events led to this emotion and how did it impact your day?'
+
+    Provide a prompt following this format, based on the emotions with the highest values from the data provided.
+    """
+    # user input will come from deepface FER results
+    prompt = f"{system_prompt}### User: {emotions}\n\n### Assistant:\n"
+    # preprocess the inputs
+    inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+    output = model.generate(**inputs, do_sample=True, top_p=0.95, top_k=0, max_new_tokens=256)
+    result = tokenizer.decode(output[0], skip_special_tokens=True)
+    # return decoded output
+    return result
+
+def generate_prompt(result):
+    # split the text by the "Assistant:" label
+    response = result.split("### Assistant:")[1].strip()
+    # check if response includes uneeded prompt indifier (e.g., Journal prompt: ..., etc.)
+    if ":" in response:
+        split_response = response.split(":", 1)
+        response = split_response[1].strip()
+    return response
+  
+
+
+# sample data
+# message = { 'angry': 13.207298517227173,
+#  'disgust': 0.12863066513091326,
+#  'fear': 0.11500244727358222,
+#  'happy': 18.36708039045334,
+#  'sad': 62.15316653251648,
+#  'surprise': 0.025137534248642623,
+#  'neutral': 6.003682315349579 }
+
+# prompt engineering data
+
+# prompt #1
+# ### System:
+# You are StableBeluga, an AI programmed to follow instructions with high accuracy. Your current task is to assist with enhancing human emotional regulation and intelligence. You will receive information about the emotional ratios of a client. Based on that information, please provide exactly one original and creative journal prompt to facilitate the client's self-reflection and emotional exploration.
+
+# ### User: {'angry': 13.207298517227173, 'disgust': 0.12863066513091326, 'fear': 0.11500244727358222, 'happy': 18.36708039045334, 'sad': 62.15316653251648, 'surprise': 0.025137534248642623, 'neutral': 6.003682315349579}
+
+# ### Assistant:
+#  Here is a journal prompt designed to help the client explore the different emotions they are experiencing and gain a better understanding of their current emotional state:
+
+# List three significant events or experiences that have been recent triggers for each of the emotions you observe in your chart. Then, for each of those events or experiences, describe how the event makes you feel and if it leads to an increased or decreased ratio of that emotion. This reflection will enable the client to understand their emotions better and perhaps find ways to improve their emotional regulation.
+
+
+
+# prompt #2
+# ### System:
+# You are StableBeluga, an AI programmed to follow instructions with high accuracy. Your current task is to assist with enhancing human emotional regulation and intelligence. You will receive information about the emotional ratios of a client. Please identify the one or two emotions with the highest values. Based on those emotions, provide exactly one original and creative journal prompt to facilitate the client's self-reflection and emotional exploration. Do not create prompts based on emotions with lower values.
+
+# ### User: {'angry': 13.207298517227173, 'disgust': 0.12863066513091326, 'fear': 0.11500244727358222, 'happy': 18.36708039045334, 'sad': 62.15316653251648, 'surprise': 0.025137534248642623, 'neutral': 6.003682315349579}
+
+# ### Assistant:
+#  Based on the emotions with the highest values in the client's emotional ratios, it appears that 'happy' and 'sad' are particularly prominent in their emotional landscape. To facilitate their self-reflection and emotional exploration, an original and creative journal prompt could be:
+
+# Prompt:
+
+# What makes you feel happy, and how can you incorporate that into your daily routine? How can you cultivate a more balanced emotional state by fostering positive emotions like happiness, even when faced with the inevitable challenges and negative emotions like sadness?    
+
+
+
+# prompt #3
+# ### System:
+# You are StableBeluga, an AI programmed to follow instructions with high accuracy. Your current task is to assist with enhancing human emotional regulation and intelligence. You will receive information about the emotional ratios of a client. Identify the one or two emotions with the highest values. Without discussing or reiterating these emotions or their scores, create exactly one original and creative journal prompt to facilitate the client's self-reflection and emotional exploration. Focus solely on creating the prompt.
+
+# ### User: {'angry': 13.207298517227173, 'disgust': 0.12863066513091326, 'fear': 0.11500244727358222, 'happy': 18.36708039045334, 'sad': 62.15316653251648, 'surprise': 0.025137534248642623, 'neutral': 6.003682315349579}
+
+# ### Assistant:
+#  To enhance your emotional regulation and intelligence, consider exploring the emotions with the highest values in your current emotional state. Consider journaling about your thoughts and feelings surrounding these dominant emotions. This could be done by asking yourself the following original and creative journal prompt:
+
+# What are the roots or origins of the dominant emotions in my life, and how do they impact my overall well-being and relationships? How can I shift my perspective to gain more control over these emotions and find a more balanced emotional state?
+
+
+
+# prompt #4
+### System:
+# You are StableBeluga, an AI programmed to follow instructions with high accuracy. Your current task is to assist with enhancing human emotional regulation and intelligence. You will receive information about the emotional ratios of a client. Using the one or two emotions with the highest values, create a one or two-sentence journal prompt for the client's self-reflection and emotional exploration. Do not discuss the emotions or their scores, only provide the prompt.
+
+# ### User: {'angry': 13.207298517227173, 'disgust': 0.12863066513091326, 'fear': 0.11500244727358222, 'happy': 18.36708039045334, 'sad': 62.15316653251648, 'surprise': 0.025137534248642623, 'neutral': 6.003682315349579}
+
+# ### Assistant:
+#  Journal prompt: When does anger typically arise for you, and how can understanding your triggers and reactions help you manage and cope with such situations more effectively in the future?
+
+
+
+# prompt #5
+# ### System:
+# You are StableBeluga, an AI programmed to follow instructions with high accuracy. Your current task is to assist with enhancing human emotional regulation and intelligence. You will receive information about the emotional ratios of a client. Using the one or two emotions with the highest values, create a one or two-sentence journal prompt for the client's self-reflection and emotional exploration. Do not discuss the emotions or their scores, only provide the prompt. Here are some examples of what the prompt should look like: 
+
+# Example 1: 'Reflect on a recent situation when you felt anger. What was the trigger and how did it affect your actions?'
+# Example 2: 'Write about a moment when you felt sad. What thoughts or events led to this emotion and how did it impact your day?'
+
+# Please provide a similar prompt based on the emotions provided.
+
+# ### User: {'angry': 13.207298517227173, 'disgust': 0.12863066513091326, 'fear': 0.11500244727358222, 'happy': 18.36708039045334, 'sad': 62.15316653251648, 'surprise': 0.025137534248642623, 'neutral': 6.003682315349579}
+
+# output 1
+# ### Assistant:
+#  Based on the provided emotional ratios:
+
+# "Write about a recent experience when you felt surprised. Describe the event and how it made you feel."
+
+# output 2
+# Assistant:\n Journal prompt based on the emotions provided: Reflect on a situation when you felt more than one strong emotion, like surprise and fear. Identify the trigger and describe how your emotions influenced each other and the actions you took in response.
+
+
+
+# # prompt #6
+# ### System:
+# You are StableBeluga, an AI programmed to follow instructions with high accuracy. Your current task is to assist with enhancing human emotional regulation and intelligence. You will receive information about the emotional ratios of a client. Your task is to identify the one or two emotions with the highest values from the information provided, and create a one or two-sentence original journal prompt for the client's self-reflection and emotional exploration. Do not discuss the emotions or their scores, and do not include any other introductory or explanatory text. Only provide the journal prompt, and ensure it is based on the emotions with the highest values. Here are examples of the desired format:
+
+# Example 1: 'Reflect on a recent situation when you felt anger. What was the trigger and how did it affect your actions?'
+# Example 2: 'Write about a moment when you felt sad. What thoughts or events led to this emotion and how did it impact your day?'
+
+# Provide a prompt following this format, based on the emotions with the highest values from the data provided.
+
+# ### User: angry: 13.21%, disgust: 0.13%, fear: 0.12%, happy: 18.37%, sad: 62.15%, surprise: 0.03%, neutral: 6.00%
+
+# # output 1
+# ### Assistant:
+#  Journal prompt: Think back to a time when you experienced sadness and explore the events that led to that emotion, as well as the impact it had on your thoughts and actions.
+# # output 2
+# Assistant:\n Journal Prompt: In a recent day, think back to a situation where you felt extreme happiness. Describe the events or people that led to this emotion, and how it made you feel throughout the day.
+# # output 3
+# Assistant:\n Reflect on a time when you experienced a high percentage of anger, and explore the events or circumstances that contributed to its manifestation.

emodeepface.py

@@ -0,0 +1,55 @@
+from deepface import DeepFace
+import numpy as np
+from PIL import Image, ExifTags
+
+# used to raise custom exceptions
+class ImageProcessingError(Exception):
+    pass
+
+# used to ensure upright photo orientation for face detection
+def check_image_rotation(image_path):
+    try:
+        image = Image.open(image_path)
+        for orientation in ExifTags.TAGS.keys():
+            if ExifTags.TAGS[orientation] == 'Orientation':
+                break
+        exif = image._getexif()
+        if exif is not None:
+            if orientation in exif:
+                if exif[orientation] == 3:
+                    image = image.rotate(180, expand=True)
+                elif exif[orientation] == 6:
+                    image = image.rotate(270, expand=True)
+                elif exif[orientation] == 8:
+                    image = image.rotate(90, expand=True)
+        return image
+    except (AttributeError, KeyError, IndexError):
+        # If the orientation tag is not found or any other error occurs, return the original image
+        return Image.open(image_path)
+
+# used to process photo through deepface emotion model
+def process_photo(file_name):
+    backends = ['opencv', 'mtcnn', 'retinaface', 'mediapipe', 'ssd']
+    attempt = 0
+
+    if file_name is not None:
+        image = check_image_rotation(file_name)
+        image_data = np.array(image)
+
+        while attempt < len(backends):
+            try:
+                predictions = DeepFace.analyze(image_data, actions=['emotion'], detector_backend=backends[attempt])
+                if len(predictions) > 1:
+                    faces = [(face, face['region']['w'] * face['region']['h']) for face in predictions]
+                    new_predictions = sorted(faces, key=lambda x: x[1], reverse=True)[0][0]
+                    emotion_dict = new_predictions['emotion']
+                    return emotion_dict
+                return predictions['emotion']
+            except Exception as e:
+                if attempt == len(backends) - 1:
+                    error_message = f"Failed to analyze image after attempting all detector backends available. Please upload a new image."
+                    raise ImageProcessingError(error_message)
+                else:
+                    # log the error message for each failed backend here:
+                    print(f"Retrying with backend `{backends[attempt+1]}` due to error: {str(e)}")
+                    attempt += 1

requirements.txt

@@ -0,0 +1,5 @@
+accelerate
+deepface
+sentencepiece
+torch
+transformers