Upload 3 files

app.py

@@ -0,0 +1,173 @@
+import streamlit as st
+import os
+import threading
+import pyautogui
+import numpy as np
+import cv2
+import pyaudio
+import wave
+import keyboard
+from pywinauto.application import Application
+import pygetwindow as gw
+from logic import analyze_with_audio_video
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# Audio settings
+FORMAT = pyaudio.paInt16
+CHANNELS = 1
+RATE = 44100
+CHUNK = 1024
+
+# File paths
+audio_filename = "output.wav"
+video_filename = "output.mp4"
+
+# Initialize Streamlit
+st.set_page_config(page_title="T.A.P.A.S", page_icon=":camera:", layout="wide")
+st.title("T.A.P.A.S - Technical Assistance Platform for Advanced Solution")
+
+# Initialize session state for outputs
+if 'outputs' not in st.session_state or not isinstance(st.session_state.outputs, dict):
+    st.session_state.outputs = {}
+
+if 'current_session' not in st.session_state:
+    st.session_state.current_session = 'Session 1'
+
+def cleanup_files():
+    """Deletes old files before a new recording session starts."""
+    files_to_delete = [audio_filename, video_filename]
+    for file in files_to_delete:
+        if os.path.exists(file):
+            os.remove(file)
+            print(f"Deleted old file: {file}")
+
+def record_audio(filename, stop_event):
+    audio = pyaudio.PyAudio()
+    stream = audio.open(format=FORMAT, channels=CHANNELS,
+                        rate=RATE, input=True,
+                        frames_per_buffer=CHUNK)
+    frames = []
+
+    while not stop_event.is_set():
+        data = stream.read(CHUNK)
+        frames.append(data)
+
+    stream.stop_stream()
+    audio.terminate()
+
+    with wave.open(filename, 'wb') as wf:
+        wf.setnchannels(CHANNELS)
+        wf.setsampwidth(audio.get_sample_size(FORMAT))
+        wf.setframerate(RATE)
+        wf.writeframes(b''.join(frames))
+
+def record_screen(filename, stop_event, mouse_positions):
+    screen_size = pyautogui.size()
+    fourcc = cv2.VideoWriter_fourcc(*"mp4v")
+    out = cv2.VideoWriter(filename, fourcc, 8, (screen_size.width, screen_size.height))
+
+    while not stop_event.is_set():
+        img = pyautogui.screenshot()
+        frame = np.array(img)
+        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        # Capture mouse cursor
+        x, y = pyautogui.position()
+        cv2.circle(frame, (x, y), 10, (0, 255, 0), -1)
+        out.write(frame)
+        mouse_positions.append((x, y))  # Track mouse positions
+
+    out.release()
+
+def minimize_browser():
+    browser_window = None
+    for window in gw.getAllTitles():
+        if "chrome" in window.lower() or "firefox" in window.lower() or "edge" in window.lower():
+            browser_window = window
+            break
+
+    if browser_window:
+        app = Application().connect(title_re=browser_window)
+        app.window(title_re=browser_window).minimize()
+    else:
+        print("Browser window not found.")
+
+def main():
+    stop_event = threading.Event()
+
+    # Sidebar for session selection
+    with st.sidebar:
+        st.title("Sessions")
+        session_name = st.text_input("New Session Name", "")
+        if st.button("Start New Session") and session_name:
+            st.session_state.current_session = session_name
+            st.session_state.outputs[session_name] = []
+        session_names = list(st.session_state.outputs.keys())
+        if session_names:
+            session_selection = st.selectbox("Choose a session", session_names)
+            if session_selection:
+                st.session_state.current_session = session_selection
+
+    st.header(f"Current Session: {st.session_state.current_session}")
+
+    # Initialize the current session's outputs if it doesn't exist
+    if st.session_state.current_session not in st.session_state.outputs:
+        st.session_state.outputs[st.session_state.current_session] = []
+
+    col1, col2 = st.columns(2)
+    with col1:
+        start_button = st.button("Start")
+    with col2:
+        stop_button = st.button("Stop")
+
+    if start_button:
+        minimize_browser()
+        cleanup_files()
+
+        audio_thread = threading.Thread(target=record_audio, args=(audio_filename, stop_event))
+        mouse_positions = []
+        screen_thread = threading.Thread(target=record_screen, args=(video_filename, stop_event, mouse_positions))
+
+        audio_thread.start()
+        screen_thread.start()
+
+        st.write("Recording started. Press 'q' or click 'Stop' to stop.")
+
+        while True:
+            if keyboard.is_pressed('q') or stop_button:
+                stop_event.set()
+                break
+
+        audio_thread.join()
+        screen_thread.join()
+
+        if not os.path.exists(audio_filename):
+            st.error("Audio file was not created!")
+            return
+        if not os.path.exists(video_filename):
+            st.error("Video file was not created!")
+            return
+
+        # Analyze the video and audio files together
+        result = analyze_with_audio_video(video_filename, audio_filename)
+        st.session_state.outputs[st.session_state.current_session].append(result)
+
+    # Text input for additional queries
+    additional_query = st.text_input("Type your query here if you're not satisfied with the solution:")
+
+    if st.button("Submit Query") and additional_query:
+        # Process the additional query (this would involve sending it to the model)
+        result = analyze_with_audio_video(video_filename, audio_filename)
+        st.session_state.outputs[st.session_state.current_session].append(f"Query: {additional_query}\n{result}")
+
+    # Display all outputs for the current session
+    for output in st.session_state.outputs[st.session_state.current_session]:
+        st.markdown(f"""
+            <div style="background-color: darkgray; border-radius: 10px; padding: 10px; margin-bottom: 10px; color: black;">
+                <i class="fas fa-check-circle"></i> {output}
+            </div>
+            """, unsafe_allow_html=True)
+
+if __name__ == "__main__":
+    main()

logic.py

@@ -0,0 +1,35 @@
+import google.generativeai as genai
+from dotenv import load_dotenv
+import os
+import time
+
+load_dotenv()
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+genai.configure(api_key=GOOGLE_API_KEY)
+
+code_model = genai.GenerativeModel('gemini-1.5-flash')
+
+# Function to analyze video and audio together
+def analyze_with_audio_video(video_file, audio_file):
+    # Upload the video and audio files
+    my_video_file = genai.upload_file(path=video_file)
+    my_audio_file = genai.upload_file(path=audio_file)
+    
+    while my_video_file.state.name == "PROCESSING" or my_audio_file.state.name == "PROCESSING":
+        time.sleep(5)
+        my_video_file = genai.get_file(my_video_file.name)
+        my_audio_file = genai.get_file(my_audio_file.name)
+    
+    # Example prompt directly referencing user issues
+    prompt = f"""
+    The user is reporting an error in the code. Analyze the video '{my_video_file.name}' along with the audio '{my_audio_file.name}'.
+    Get the issue from the audio file and refer it to the video file and provide the solution
+    1. **Explanation of the Error**: Pinpoint the error mentioned by the user.
+    2. **Approach to Solve the Error**: Provide a step-by-step guide to resolve the error, using both video and audio data.
+    3. **Corrected Code**: Provide the corrected code, addressing the identified error.
+    4. **Summary**: Summarize the solution, ensuring clarity on how the error was fixed.
+    """
+
+    # Request content generation from Gemini model
+    response = code_model.generate_content([my_video_file, my_audio_file, prompt])
+    return response.text

requirements.txt

@@ -0,0 +1,26 @@
+streamlit==1.38.0
+opencv-python==4.10.0.84
+opencv-python-headless
+PyAudio==0.2.14
+numpy==1.26.4
+load-dotenv==0.1.0
+python-dotenv==1.0.1
+PyAutoGUI==0.9.54
+keyboard==0.13.5
+pywinauto==0.6.8
+PyGetWindow==0.0.9
+wave
+google-ai-generativelanguage==0.6.6
+google-api-core==2.19.2
+google-api-python-client==2.143.0
+google-auth==2.34.0
+google-auth-httplib2==0.2.0
+google-cloud-aiplatform==1.64.0
+google-cloud-bigquery==3.25.0
+google-cloud-core==2.4.1
+google-cloud-resource-manager==1.12.5
+google-cloud-storage==2.18.2
+google-crc32c==1.5.0
+google-generativeai==0.7.2
+google-resumable-media==2.7.2
+googleapis-common-protos==1.65.0