whyumesh commited on
Commit
a8653f0
·
verified ·
1 Parent(s): dcb78d4

Upload 3 files

Browse files
Files changed (3) hide show
  1. app.py +173 -0
  2. logic.py +35 -0
  3. requirements.txt +26 -0
app.py ADDED
@@ -0,0 +1,173 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import os
3
+ import threading
4
+ import pyautogui
5
+ import numpy as np
6
+ import cv2
7
+ import pyaudio
8
+ import wave
9
+ import keyboard
10
+ from pywinauto.application import Application
11
+ import pygetwindow as gw
12
+ from logic import analyze_with_audio_video
13
+ from dotenv import load_dotenv
14
+
15
+ load_dotenv()
16
+
17
+ # Audio settings
18
+ FORMAT = pyaudio.paInt16
19
+ CHANNELS = 1
20
+ RATE = 44100
21
+ CHUNK = 1024
22
+
23
+ # File paths
24
+ audio_filename = "output.wav"
25
+ video_filename = "output.mp4"
26
+
27
+ # Initialize Streamlit
28
+ st.set_page_config(page_title="T.A.P.A.S", page_icon=":camera:", layout="wide")
29
+ st.title("T.A.P.A.S - Technical Assistance Platform for Advanced Solution")
30
+
31
+ # Initialize session state for outputs
32
+ if 'outputs' not in st.session_state or not isinstance(st.session_state.outputs, dict):
33
+ st.session_state.outputs = {}
34
+
35
+ if 'current_session' not in st.session_state:
36
+ st.session_state.current_session = 'Session 1'
37
+
38
+ def cleanup_files():
39
+ """Deletes old files before a new recording session starts."""
40
+ files_to_delete = [audio_filename, video_filename]
41
+ for file in files_to_delete:
42
+ if os.path.exists(file):
43
+ os.remove(file)
44
+ print(f"Deleted old file: {file}")
45
+
46
+ def record_audio(filename, stop_event):
47
+ audio = pyaudio.PyAudio()
48
+ stream = audio.open(format=FORMAT, channels=CHANNELS,
49
+ rate=RATE, input=True,
50
+ frames_per_buffer=CHUNK)
51
+ frames = []
52
+
53
+ while not stop_event.is_set():
54
+ data = stream.read(CHUNK)
55
+ frames.append(data)
56
+
57
+ stream.stop_stream()
58
+ audio.terminate()
59
+
60
+ with wave.open(filename, 'wb') as wf:
61
+ wf.setnchannels(CHANNELS)
62
+ wf.setsampwidth(audio.get_sample_size(FORMAT))
63
+ wf.setframerate(RATE)
64
+ wf.writeframes(b''.join(frames))
65
+
66
+ def record_screen(filename, stop_event, mouse_positions):
67
+ screen_size = pyautogui.size()
68
+ fourcc = cv2.VideoWriter_fourcc(*"mp4v")
69
+ out = cv2.VideoWriter(filename, fourcc, 8, (screen_size.width, screen_size.height))
70
+
71
+ while not stop_event.is_set():
72
+ img = pyautogui.screenshot()
73
+ frame = np.array(img)
74
+ frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
75
+ # Capture mouse cursor
76
+ x, y = pyautogui.position()
77
+ cv2.circle(frame, (x, y), 10, (0, 255, 0), -1)
78
+ out.write(frame)
79
+ mouse_positions.append((x, y)) # Track mouse positions
80
+
81
+ out.release()
82
+
83
+ def minimize_browser():
84
+ browser_window = None
85
+ for window in gw.getAllTitles():
86
+ if "chrome" in window.lower() or "firefox" in window.lower() or "edge" in window.lower():
87
+ browser_window = window
88
+ break
89
+
90
+ if browser_window:
91
+ app = Application().connect(title_re=browser_window)
92
+ app.window(title_re=browser_window).minimize()
93
+ else:
94
+ print("Browser window not found.")
95
+
96
+ def main():
97
+ stop_event = threading.Event()
98
+
99
+ # Sidebar for session selection
100
+ with st.sidebar:
101
+ st.title("Sessions")
102
+ session_name = st.text_input("New Session Name", "")
103
+ if st.button("Start New Session") and session_name:
104
+ st.session_state.current_session = session_name
105
+ st.session_state.outputs[session_name] = []
106
+ session_names = list(st.session_state.outputs.keys())
107
+ if session_names:
108
+ session_selection = st.selectbox("Choose a session", session_names)
109
+ if session_selection:
110
+ st.session_state.current_session = session_selection
111
+
112
+ st.header(f"Current Session: {st.session_state.current_session}")
113
+
114
+ # Initialize the current session's outputs if it doesn't exist
115
+ if st.session_state.current_session not in st.session_state.outputs:
116
+ st.session_state.outputs[st.session_state.current_session] = []
117
+
118
+ col1, col2 = st.columns(2)
119
+ with col1:
120
+ start_button = st.button("Start")
121
+ with col2:
122
+ stop_button = st.button("Stop")
123
+
124
+ if start_button:
125
+ minimize_browser()
126
+ cleanup_files()
127
+
128
+ audio_thread = threading.Thread(target=record_audio, args=(audio_filename, stop_event))
129
+ mouse_positions = []
130
+ screen_thread = threading.Thread(target=record_screen, args=(video_filename, stop_event, mouse_positions))
131
+
132
+ audio_thread.start()
133
+ screen_thread.start()
134
+
135
+ st.write("Recording started. Press 'q' or click 'Stop' to stop.")
136
+
137
+ while True:
138
+ if keyboard.is_pressed('q') or stop_button:
139
+ stop_event.set()
140
+ break
141
+
142
+ audio_thread.join()
143
+ screen_thread.join()
144
+
145
+ if not os.path.exists(audio_filename):
146
+ st.error("Audio file was not created!")
147
+ return
148
+ if not os.path.exists(video_filename):
149
+ st.error("Video file was not created!")
150
+ return
151
+
152
+ # Analyze the video and audio files together
153
+ result = analyze_with_audio_video(video_filename, audio_filename)
154
+ st.session_state.outputs[st.session_state.current_session].append(result)
155
+
156
+ # Text input for additional queries
157
+ additional_query = st.text_input("Type your query here if you're not satisfied with the solution:")
158
+
159
+ if st.button("Submit Query") and additional_query:
160
+ # Process the additional query (this would involve sending it to the model)
161
+ result = analyze_with_audio_video(video_filename, audio_filename)
162
+ st.session_state.outputs[st.session_state.current_session].append(f"Query: {additional_query}\n{result}")
163
+
164
+ # Display all outputs for the current session
165
+ for output in st.session_state.outputs[st.session_state.current_session]:
166
+ st.markdown(f"""
167
+ <div style="background-color: darkgray; border-radius: 10px; padding: 10px; margin-bottom: 10px; color: black;">
168
+ <i class="fas fa-check-circle"></i> {output}
169
+ </div>
170
+ """, unsafe_allow_html=True)
171
+
172
+ if __name__ == "__main__":
173
+ main()
logic.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import google.generativeai as genai
2
+ from dotenv import load_dotenv
3
+ import os
4
+ import time
5
+
6
+ load_dotenv()
7
+ GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
8
+ genai.configure(api_key=GOOGLE_API_KEY)
9
+
10
+ code_model = genai.GenerativeModel('gemini-1.5-flash')
11
+
12
+ # Function to analyze video and audio together
13
+ def analyze_with_audio_video(video_file, audio_file):
14
+ # Upload the video and audio files
15
+ my_video_file = genai.upload_file(path=video_file)
16
+ my_audio_file = genai.upload_file(path=audio_file)
17
+
18
+ while my_video_file.state.name == "PROCESSING" or my_audio_file.state.name == "PROCESSING":
19
+ time.sleep(5)
20
+ my_video_file = genai.get_file(my_video_file.name)
21
+ my_audio_file = genai.get_file(my_audio_file.name)
22
+
23
+ # Example prompt directly referencing user issues
24
+ prompt = f"""
25
+ The user is reporting an error in the code. Analyze the video '{my_video_file.name}' along with the audio '{my_audio_file.name}'.
26
+ Get the issue from the audio file and refer it to the video file and provide the solution
27
+ 1. **Explanation of the Error**: Pinpoint the error mentioned by the user.
28
+ 2. **Approach to Solve the Error**: Provide a step-by-step guide to resolve the error, using both video and audio data.
29
+ 3. **Corrected Code**: Provide the corrected code, addressing the identified error.
30
+ 4. **Summary**: Summarize the solution, ensuring clarity on how the error was fixed.
31
+ """
32
+
33
+ # Request content generation from Gemini model
34
+ response = code_model.generate_content([my_video_file, my_audio_file, prompt])
35
+ return response.text
requirements.txt ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ streamlit==1.38.0
2
+ opencv-python==4.10.0.84
3
+ opencv-python-headless
4
+ PyAudio==0.2.14
5
+ numpy==1.26.4
6
+ load-dotenv==0.1.0
7
+ python-dotenv==1.0.1
8
+ PyAutoGUI==0.9.54
9
+ keyboard==0.13.5
10
+ pywinauto==0.6.8
11
+ PyGetWindow==0.0.9
12
+ wave
13
+ google-ai-generativelanguage==0.6.6
14
+ google-api-core==2.19.2
15
+ google-api-python-client==2.143.0
16
+ google-auth==2.34.0
17
+ google-auth-httplib2==0.2.0
18
+ google-cloud-aiplatform==1.64.0
19
+ google-cloud-bigquery==3.25.0
20
+ google-cloud-core==2.4.1
21
+ google-cloud-resource-manager==1.12.5
22
+ google-cloud-storage==2.18.2
23
+ google-crc32c==1.5.0
24
+ google-generativeai==0.7.2
25
+ google-resumable-media==2.7.2
26
+ googleapis-common-protos==1.65.0