import {useCallback, useEffect, useRef, useState} from 'react'; import { Canvas, createPortal, extend, useFrame, useThree, } from '@react-three/fiber'; import ThreeMeshUI from 'three-mesh-ui'; import {ARButton, XR, Hands, XREvent} from '@react-three/xr'; import {TextGeometry} from 'three/examples/jsm/geometries/TextGeometry.js'; import {TranslationSentences} from '../types/StreamingTypes'; import Button from './Button'; import {RoomState} from '../types/RoomState'; import ThreeMeshUIText, {ThreeMeshUITextType} from './ThreeMeshUIText'; import {BLACK, WHITE} from './Colors'; /** * Using `?url` at the end of this import tells vite this is a static asset, and * provides us a URL to the hashed version of the file when the project is built. * See: https://vitejs.dev/guide/assets.html#explicit-url-imports */ import robotoFontFamilyJson from '../assets/RobotoMono-Regular-msdf.json?url'; import robotoFontTexture from '../assets/RobotoMono-Regular.png'; import {getURLParams} from '../URLParams'; import TextBlocks, {CHARS_PER_LINE} from './TextBlocks'; import {BufferedSpeechPlayer} from '../createBufferedSpeechPlayer'; import {CURSOR_BLINK_INTERVAL_MS} from '../cursorBlinkInterval'; // Adds on react JSX for add-on libraries to react-three-fiber extend(ThreeMeshUI); extend({TextGeometry}); async function fetchSupportedCharSet(): Promise> { try { const response = await fetch(robotoFontFamilyJson); const fontFamily = await response.json(); return new Set(fontFamily.info.charset); } catch (e) { console.error('Failed to fetch supported XR charset', e); return new Set(); } } let supportedCharSet = new Set(); fetchSupportedCharSet().then((result) => (supportedCharSet = result)); // This component wraps any children so it is positioned relative to the camera, rather than from the origin function CameraLinkedObject({children}) { const camera = useThree((state) => state.camera); return createPortal(<>{children}, camera); } function ThreeMeshUIComponents({ translationSentences, skipARIntro, roomState, animateTextDisplay, }: XRConfigProps & {skipARIntro: boolean}) { // The "loop" for re-rendering required for threemeshUI useFrame(() => { ThreeMeshUI.update(); }); const [started, setStarted] = useState(skipARIntro); return ( <> {getURLParams().ARTranscriptionType === 'single_block' ? ( ) : ( )} {skipARIntro ? null : ( )} ); } // Original UI that just uses a single block to render 6 lines in a panel function TranscriptPanelSingleBlock({ animateTextDisplay, started, translationSentences, roomState, }: { animateTextDisplay: boolean; started: boolean; translationSentences: TranslationSentences; roomState: RoomState | null; }) { const textRef = useRef(); const [didReceiveTranslationSentences, setDidReceiveTranslationSentences] = useState(false); const hasActiveTranscoders = (roomState?.activeTranscoders ?? 0) > 0; const [cursorBlinkOn, setCursorBlinkOn] = useState(false); // Normally we don't setState in render, but here we need to for computed state, and this if statement assures it won't loop infinitely if (!didReceiveTranslationSentences && translationSentences.length > 0) { setDidReceiveTranslationSentences(true); } const width = 1; const height = 0.3; const fontSize = 0.03; useEffect(() => { if (animateTextDisplay && hasActiveTranscoders) { const interval = setInterval(() => { setCursorBlinkOn((prev) => !prev); }, CURSOR_BLINK_INTERVAL_MS); return () => clearInterval(interval); } else { setCursorBlinkOn(false); } }, [animateTextDisplay, hasActiveTranscoders]); useEffect(() => { if (textRef.current != null) { const initialPrompt = 'Welcome to the presentation. We are excited to share with you the work we have been doing... Our model can now translate languages in less than 2 second latency.'; // These are rough ratios based on spot checking const maxLines = 6; const charsPerLine = 55; const transcriptSentences: string[] = didReceiveTranslationSentences ? translationSentences : [initialPrompt]; // The transcript is an array of sentences. For each sentence we break this down into an array of words per line. // This is needed so we can "scroll" through without changing the order of words in the transcript const linesToDisplay = transcriptSentences.flatMap((sentence, idx) => { const blinkingCursor = cursorBlinkOn && idx === transcriptSentences.length - 1 ? '|' : ' '; const words = sentence.concat(blinkingCursor).split(/\s+/); // Here we break each sentence up with newlines so all words per line fit within the panel return words.reduce( (wordChunks, currentWord) => { const filteredWord = [...currentWord] .filter((c) => { if (supportedCharSet.has(c)) { return true; } console.error( `Unsupported char ${c} - make sure this is supported in the font family msdf file`, ); return false; }) .join(''); const lastLineSoFar = wordChunks[wordChunks.length - 1]; const charCount = lastLineSoFar.length + filteredWord.length + 1; if (charCount <= charsPerLine) { wordChunks[wordChunks.length - 1] = lastLineSoFar + ' ' + filteredWord; } else { wordChunks.push(filteredWord); } return wordChunks; }, [''], ); }); // Only keep the last maxLines so new text keeps scrolling up from the bottom linesToDisplay.splice(0, linesToDisplay.length - maxLines); textRef.current.set({content: linesToDisplay.join('\n')}); } }, [ translationSentences, textRef, didReceiveTranslationSentences, cursorBlinkOn, ]); const opacity = started ? 1 : 0; return ( ); } // Splits up the lines into separate blocks to treat each one separately. // This allows changing of opacity, animating per line, changing height / width per line etc function TranscriptPanelBlocks({ animateTextDisplay, translationSentences, }: { animateTextDisplay: boolean; translationSentences: TranslationSentences; }) { const [didReceiveTranslationSentences, setDidReceiveTranslationSentences] = // Currently causing issues with displaying dummy text, skip over useState(false); // Normally we don't setState in render, but here we need to for computed state, and this if statement assures it won't loop infinitely if (!didReceiveTranslationSentences && translationSentences.length > 0) { setDidReceiveTranslationSentences(true); } const initialPrompt = 'Listening...'; const transcriptSentences: string[] = didReceiveTranslationSentences ? translationSentences : [initialPrompt]; // The transcript is an array of sentences. For each sentence we break this down into an array of words per line. // This is needed so we can "scroll" through without changing the order of words in the transcript const sentenceLines = transcriptSentences.map((sentence) => { const words = sentence.split(/\s+/); // Here we break each sentence up with newlines so all words per line fit within the panel return words.reduce( (wordChunks, currentWord) => { const filteredWord = [...currentWord] .filter((c) => { if (supportedCharSet.has(c)) { return true; } console.error( `Unsupported char ${c} - make sure this is supported in the font family msdf file`, ); return false; }) .join(''); const lastLineSoFar = wordChunks[wordChunks.length - 1]; const charCount = lastLineSoFar.length + filteredWord.length + 1; if (charCount <= CHARS_PER_LINE) { wordChunks[wordChunks.length - 1] = lastLineSoFar + ' ' + filteredWord; } else { wordChunks.push(filteredWord); } return wordChunks; }, [''], ); }); return ( ); } function IntroPanel({started, setStarted}) { const width = 0.5; const height = 0.4; const padding = 0.03; // Kind of hacky but making the panel disappear by moving it completely off the camera view. // If we try to remove elements we end up throwing and stopping the experience // opacity=0 also runs into weird bugs where not everything is invisible const xCoordinate = started ? 1000000 : 0; const commonArgs = { backgroundColor: WHITE, width, height, padding, backgroundOpacity: 1, textAlign: 'center', fontFamily: robotoFontFamilyJson, fontTexture: robotoFontTexture, }; return ( <>