Spaces:
Running
Running
Merge deployment-updates into main, taking all changes from deployment-updates
Browse files
README.md
CHANGED
@@ -5,7 +5,8 @@ colorFrom: blue
|
|
5 |
colorTo: purple
|
6 |
sdk: docker
|
7 |
pinned: false
|
8 |
-
|
|
|
9 |
---
|
10 |
|
11 |
# Multimodal Live API - Web console
|
|
|
5 |
colorTo: purple
|
6 |
sdk: docker
|
7 |
pinned: false
|
8 |
+
license: apache-2.0
|
9 |
+
short_description: Gemini Multimodal Live API Console
|
10 |
---
|
11 |
|
12 |
# Multimodal Live API - Web console
|
src/components/control-tray/ControlTray.tsx
CHANGED
@@ -244,24 +244,22 @@ function ControlTray({
|
|
244 |
ref={connectButtonRef}
|
245 |
className={cn("action-button connect-toggle", { connected })}
|
246 |
onClick={async () => {
|
247 |
-
console.log('
|
248 |
try {
|
249 |
if (connected) {
|
250 |
-
console.log('๐ด
|
251 |
await disconnect();
|
252 |
-
console.log('โ
Disconnected successfully');
|
253 |
} else {
|
254 |
-
console.log('๐ Starting connection...');
|
255 |
-
console.log('๐ฑ Device info:', { isIOSDevice, isSafari });
|
256 |
|
257 |
-
|
258 |
-
console.log('๐ Calling connect()...');
|
259 |
await connect();
|
260 |
-
console.log('โ
Connected successfully');
|
261 |
}
|
262 |
} catch (err) {
|
263 |
-
console.error('โ
|
264 |
-
// Here you could add UI feedback about the error
|
265 |
}
|
266 |
}}
|
267 |
>
|
|
|
244 |
ref={connectButtonRef}
|
245 |
className={cn("action-button connect-toggle", { connected })}
|
246 |
onClick={async () => {
|
247 |
+
console.log('๐ฎ ControlTray: Connection button clicked');
|
248 |
try {
|
249 |
if (connected) {
|
250 |
+
console.log('๐ด ControlTray: Initiating disconnect...');
|
251 |
await disconnect();
|
252 |
+
console.log('โ
ControlTray: Disconnected successfully');
|
253 |
} else {
|
254 |
+
console.log('๐ ControlTray: Starting connection process...');
|
255 |
+
console.log('๐ฑ ControlTray: Device info:', { isIOSDevice, isSafari });
|
256 |
|
257 |
+
console.log('๐ ControlTray: Calling LiveAPIContext.connect()...');
|
|
|
258 |
await connect();
|
259 |
+
console.log('โ
ControlTray: Connected successfully via LiveAPIContext');
|
260 |
}
|
261 |
} catch (err) {
|
262 |
+
console.error('โ ControlTray: Connection error:', err);
|
|
|
263 |
}
|
264 |
}}
|
265 |
>
|
src/contexts/LiveAPIContext.tsx
CHANGED
@@ -28,7 +28,6 @@ export const LiveAPIProvider: FC<LiveAPIProviderProps> = ({
|
|
28 |
url = `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`,
|
29 |
children,
|
30 |
}) => {
|
31 |
-
console.log('๐ Initializing LiveAPIProvider with URL:', url);
|
32 |
const liveAPI = useLiveAPI({ url });
|
33 |
|
34 |
return (
|
@@ -39,9 +38,9 @@ export const LiveAPIProvider: FC<LiveAPIProviderProps> = ({
|
|
39 |
};
|
40 |
|
41 |
export const useLiveAPIContext = () => {
|
|
|
42 |
const context = useContext(LiveAPIContext);
|
43 |
if (!context) {
|
44 |
-
console.error('โ LiveAPIContext used outside of LiveAPIProvider');
|
45 |
throw new Error("useLiveAPIContext must be used within a LiveAPIProvider");
|
46 |
}
|
47 |
console.log('โ
LiveAPIContext successfully retrieved');
|
|
|
28 |
url = `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`,
|
29 |
children,
|
30 |
}) => {
|
|
|
31 |
const liveAPI = useLiveAPI({ url });
|
32 |
|
33 |
return (
|
|
|
38 |
};
|
39 |
|
40 |
export const useLiveAPIContext = () => {
|
41 |
+
console.log('๐ฏ LiveAPIContext: Hook being accessed');
|
42 |
const context = useContext(LiveAPIContext);
|
43 |
if (!context) {
|
|
|
44 |
throw new Error("useLiveAPIContext must be used within a LiveAPIProvider");
|
45 |
}
|
46 |
console.log('โ
LiveAPIContext successfully retrieved');
|
src/lib/multimodal-live-client.ts
CHANGED
@@ -89,20 +89,21 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
|
|
89 |
connect(config: LiveConfig): Promise<boolean> {
|
90 |
console.log('๐ Attempting WebSocket connection to:', this.url);
|
91 |
this.config = config;
|
|
|
92 |
|
93 |
const ws = new WebSocket(this.url);
|
94 |
|
95 |
ws.addEventListener("message", async (evt: MessageEvent) => {
|
96 |
console.log('๐จ Received WebSocket message:', evt.data instanceof Blob ? 'Blob data' : evt.data);
|
97 |
if (evt.data instanceof Blob) {
|
|
|
98 |
this.receive(evt.data);
|
99 |
} else {
|
100 |
-
console.log("
|
101 |
}
|
102 |
});
|
103 |
return new Promise((resolve, reject) => {
|
104 |
const onError = (ev: Event) => {
|
105 |
-
console.error('โ WebSocket connection error:', ev);
|
106 |
this.disconnect(ws);
|
107 |
const message = `Could not connect to "${this.url}"`;
|
108 |
this.log(`server.${ev.type}`, message);
|
@@ -112,10 +113,10 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
|
|
112 |
ws.addEventListener("open", (ev: Event) => {
|
113 |
console.log('โ
WebSocket connection opened successfully');
|
114 |
if (!this.config) {
|
115 |
-
console.error('โ Invalid config provided to connect()');
|
116 |
reject("Invalid config sent to `connect(config)`");
|
117 |
return;
|
118 |
}
|
|
|
119 |
this.log(`client.${ev.type}`, `connected to socket`);
|
120 |
this.emit("open");
|
121 |
|
@@ -124,13 +125,12 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
|
|
124 |
const setupMessage: SetupMessage = {
|
125 |
setup: this.config,
|
126 |
};
|
127 |
-
console.log('๐ค Sending setup message:', setupMessage);
|
128 |
this._sendDirect(setupMessage);
|
129 |
this.log("client.send", "setup");
|
130 |
|
131 |
ws.removeEventListener("error", onError);
|
132 |
ws.addEventListener("close", (ev: CloseEvent) => {
|
133 |
-
console.log(
|
134 |
this.disconnect(ws);
|
135 |
let reason = ev.reason || "";
|
136 |
if (reason.toLowerCase().includes("error")) {
|
@@ -176,17 +176,20 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
|
|
176 |
)) as LiveIncomingMessage;
|
177 |
console.log('๐ฅ Received message:', response);
|
178 |
if (isToolCallMessage(response)) {
|
|
|
179 |
this.log("server.toolCall", response);
|
180 |
this.emit("toolcall", response.toolCall);
|
181 |
return;
|
182 |
}
|
183 |
if (isToolCallCancellationMessage(response)) {
|
|
|
184 |
this.log("receive.toolCallCancellation", response);
|
185 |
this.emit("toolcallcancellation", response.toolCallCancellation);
|
186 |
return;
|
187 |
}
|
188 |
|
189 |
if (isSetupCompleteMessage(response)) {
|
|
|
190 |
this.log("server.send", "setupComplete");
|
191 |
this.emit("setupcomplete");
|
192 |
return;
|
|
|
89 |
connect(config: LiveConfig): Promise<boolean> {
|
90 |
console.log('๐ Attempting WebSocket connection to:', this.url);
|
91 |
this.config = config;
|
92 |
+
console.log('๐ MultimodalLiveClient: Starting WebSocket connection to:', this.url);
|
93 |
|
94 |
const ws = new WebSocket(this.url);
|
95 |
|
96 |
ws.addEventListener("message", async (evt: MessageEvent) => {
|
97 |
console.log('๐จ Received WebSocket message:', evt.data instanceof Blob ? 'Blob data' : evt.data);
|
98 |
if (evt.data instanceof Blob) {
|
99 |
+
console.log('๐ฉ MultimodalLiveClient: Received blob message');
|
100 |
this.receive(evt.data);
|
101 |
} else {
|
102 |
+
console.log("non blob message", evt);
|
103 |
}
|
104 |
});
|
105 |
return new Promise((resolve, reject) => {
|
106 |
const onError = (ev: Event) => {
|
|
|
107 |
this.disconnect(ws);
|
108 |
const message = `Could not connect to "${this.url}"`;
|
109 |
this.log(`server.${ev.type}`, message);
|
|
|
113 |
ws.addEventListener("open", (ev: Event) => {
|
114 |
console.log('โ
WebSocket connection opened successfully');
|
115 |
if (!this.config) {
|
|
|
116 |
reject("Invalid config sent to `connect(config)`");
|
117 |
return;
|
118 |
}
|
119 |
+
console.log('โจ MultimodalLiveClient: WebSocket connection established');
|
120 |
this.log(`client.${ev.type}`, `connected to socket`);
|
121 |
this.emit("open");
|
122 |
|
|
|
125 |
const setupMessage: SetupMessage = {
|
126 |
setup: this.config,
|
127 |
};
|
|
|
128 |
this._sendDirect(setupMessage);
|
129 |
this.log("client.send", "setup");
|
130 |
|
131 |
ws.removeEventListener("error", onError);
|
132 |
ws.addEventListener("close", (ev: CloseEvent) => {
|
133 |
+
console.log(ev);
|
134 |
this.disconnect(ws);
|
135 |
let reason = ev.reason || "";
|
136 |
if (reason.toLowerCase().includes("error")) {
|
|
|
176 |
)) as LiveIncomingMessage;
|
177 |
console.log('๐ฅ Received message:', response);
|
178 |
if (isToolCallMessage(response)) {
|
179 |
+
console.log('๐ ๏ธ MultimodalLiveClient: Received tool call');
|
180 |
this.log("server.toolCall", response);
|
181 |
this.emit("toolcall", response.toolCall);
|
182 |
return;
|
183 |
}
|
184 |
if (isToolCallCancellationMessage(response)) {
|
185 |
+
console.log('๐ซ MultimodalLiveClient: Received tool call cancellation');
|
186 |
this.log("receive.toolCallCancellation", response);
|
187 |
this.emit("toolcallcancellation", response.toolCallCancellation);
|
188 |
return;
|
189 |
}
|
190 |
|
191 |
if (isSetupCompleteMessage(response)) {
|
192 |
+
console.log('๐ MultimodalLiveClient: Setup complete received');
|
193 |
this.log("server.send", "setupComplete");
|
194 |
this.emit("setupcomplete");
|
195 |
return;
|