Spaces:
Running
Running
File size: 3,394 Bytes
7f2a14a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 |
/**
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { type FunctionDeclaration, SchemaType } from "@google/generative-ai";
import { useEffect, useRef, useState, memo } from "react";
import vegaEmbed from "vega-embed";
import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
import { ToolCall } from "../../multimodal-live-types";
const declaration: FunctionDeclaration = {
name: "render_altair",
description: "Displays an altair graph in json format.",
parameters: {
type: SchemaType.OBJECT,
properties: {
json_graph: {
type: SchemaType.STRING,
description:
"JSON STRING representation of the graph to render. Must be a string, not a json object",
},
},
required: ["json_graph"],
},
};
function AltairComponent() {
const [jsonString, setJSONString] = useState<string>("");
const { client, setConfig } = useLiveAPIContext();
useEffect(() => {
setConfig({
model: "models/gemini-2.0-flash-exp",
generationConfig: {
responseModalities: "audio",
speechConfig: {
voiceConfig: { prebuiltVoiceConfig: { voiceName: "Aoede" } },
},
},
systemInstruction: {
parts: [
{
text: 'You are my helpful assistant. Any time I ask you for a graph call the "render_altair" function I have provided you. Dont ask for additional information just make your best judgement.',
},
],
},
tools: [
// there is a free-tier quota for search
{ googleSearch: {} },
{ functionDeclarations: [declaration] },
],
});
}, [setConfig]);
useEffect(() => {
const onToolCall = (toolCall: ToolCall) => {
console.log(`got toolcall`, toolCall);
const fc = toolCall.functionCalls.find(
(fc) => fc.name === declaration.name,
);
if (fc) {
const str = (fc.args as any).json_graph;
setJSONString(str);
}
// send data for the response of your tool call
// in this case Im just saying it was successful
if (toolCall.functionCalls.length) {
setTimeout(
() =>
client.sendToolResponse({
functionResponses: toolCall.functionCalls.map((fc) => ({
response: { output: { success: true } },
id: fc.id,
})),
}),
200,
);
}
};
client.on("toolcall", onToolCall);
return () => {
client.off("toolcall", onToolCall);
};
}, [client]);
const embedRef = useRef<HTMLDivElement>(null);
useEffect(() => {
if (embedRef.current && jsonString) {
vegaEmbed(embedRef.current, JSON.parse(jsonString));
}
}, [embedRef, jsonString]);
return <div className="vega-embed" ref={embedRef} />;
}
export const Altair = memo(AltairComponent);
|