improvements

src/app/games/index.ts

@@ -13,6 +13,6 @@ import { game as nexus } from "./nexus"
 
 export const games = { pirates, city, dungeon, doom, vernian, enchanters, flamenco, pharaoh, tensor, nexus}
 
-export const defaultGame: GameType = "tensor"
+export const defaultGame: GameType = "dungeon"
 
 export const getGame = (type?: GameType) => games[type || defaultGame] || games[defaultGame]

src/app/main.tsx

@@ -24,6 +24,7 @@ import { getBackground } from "@/app/queries/getBackground"
 import { getDialogue } from "@/app/queries/getDialogue"
 import { getActionnables } from "@/app/queries/getActionnables"
 import { Engine, EngineType, defaultEngine, engines, getEngine } from "./engines"
+import { normalizeActionnables } from "@/lib/normalizeActionnables"
 
 const getInitialRenderedScene = (): RenderedScene => ({
   renderId: "",
@@ -74,10 +75,11 @@ export default function Main() {
         prompt: game.getScenePrompt(nextSituation).join(", "),
 
         // ACTIONNABLES
-        actionnables: (Array.isArray(nextActionnables) && nextActionnables.length
+        actionnables: normalizeActionnables(
+          Array.isArray(nextActionnables) && nextActionnables.length
           ? nextActionnables
           : game.initialActionnables
-        ).slice(0, 10) // too many can slow us down it seems
+        )
       })
 
       console.log("got the first version of our scene!", newRendered)
@@ -170,7 +172,7 @@ export default function Main() {
       let newDialogue = ""
       try {
         newDialogue = await getDialogue({ game, situation, actionnable })
-        console.log(`newDialogue:`, newDialogue)
+        // console.log(`newDialogue:`, newDialogue)
         setDialogue(newDialogue)
       } catch (err) {
         console.log(`failed to generate dialogue (but it's only a nice to have, so..)`)
@@ -197,6 +199,8 @@ export default function Main() {
 
   const clickables = Array.from(new Set(rendered.segments.map(s => s.label)).values())
 
+  // console.log("segments:", rendered.segments)
+
   const handleToggleDebug = (isToggledOn: boolean) => {
     const current = new URLSearchParams(Array.from(searchParams.entries()))
     current.set("debug", `${isToggledOn}`)

src/app/queries/getActionnables.ts

@@ -4,6 +4,7 @@ import { parseJsonList } from "@/lib/parseJsonList"
 
 import { getBase } from "./getBase"
 import { predict } from "./predict"
+import { normalizeActionnables } from "@/lib/normalizeActionnables"
 
 export const getActionnables = async ({
   game,
@@ -55,7 +56,7 @@ export const getActionnables = async ({
     }
   }
   
-  let result = []
+  let result: string[] = []
 
   try {
     result = parseJsonList(rawStringOutput)
@@ -68,33 +69,17 @@ export const getActionnables = async ({
 
     try {
       const sanitized = rawStringOutput.replaceAll("[", "").replaceAll("]", "")
-      result = (JSON.parse(`[${sanitized}]`) as string[]).map(item =>
-        // clean the words to remove any punctuation
-        item.replace(/\W/g, '').trim()
-      )
+      result = (JSON.parse(`[${sanitized}]`) as string[])
 
       if (!result.length) {
         throw new Error("no actionnables")
       }
     } catch (err) {
       console.log("failed to repair and recover a valid JSON! Using a generic fallback..")
-
-      return [
-        "door",
-        "rock",
-        "window",
-        "table",
-        "ground",
-        "sky",
-        "object",
-        "tree",
-        "wall",
-        "floor"
-      ]
       
       // throw new Error("failed to parse the actionnables")
     }
   }
 
-  return  result
+  return normalizeActionnables(result)
 }

src/components/business/cartesian-image.tsx

@@ -1,7 +1,6 @@
 import { useRef } from "react"
 import { SceneEventHandler } from "./types"
 import { RenderedScene } from "@/app/types"
-import { useImageDimension } from "@/lib/useImageDimension"
 
 export function CartesianImage({
   rendered,
@@ -15,13 +14,11 @@ export function CartesianImage({
   debug?: boolean
 }) {
 
-  const maskDimension = useImageDimension(rendered.maskUrl)
-
   const ref = useRef<HTMLImageElement>(null)
   const handleEvent = async (event: React.MouseEvent<HTMLImageElement, MouseEvent>, isClick: boolean) => {
 
     if (!ref.current) {
-      console.log("element isn't ready")
+      // console.log("element isn't ready")
       return
     }
 
@@ -35,14 +32,10 @@ export function CartesianImage({
 
     // then we convert them to relative coordinates
     const relativeX = containerX / boundingRect.width
-    const relativey = containerY / boundingRect.height
-
-    // finally, we convert back to coordinates within the input image
-    const imageX = relativeX * maskDimension.width
-    const imageY = relativey * maskDimension.height
+    const relativeY = containerY / boundingRect.height
 
     const eventType = isClick ? "click" : "hover"
-    onEvent(eventType, imageX, imageY)
+    onEvent(eventType, relativeX, relativeY)
   }
 
   if (!rendered.assetUrl) {
@@ -50,7 +43,11 @@ export function CartesianImage({
   }
   return (
     <div
-      className={className}
+      className={[
+        "h-[512px]",
+        className
+      ].join(" ")
+      }
     >
       <img
         src={rendered.assetUrl || undefined}

src/components/business/cartesian-video.tsx

@@ -1,7 +1,6 @@
 import { useRef } from "react"
 import { SceneEventHandler } from "./types"
 import { RenderedScene } from "@/app/types"
-import { useImageDimension } from "@/lib/useImageDimension"
 
 export function CartesianVideo({
   rendered,
@@ -14,13 +13,11 @@ export function CartesianVideo({
   className?: string
   debug?: boolean
 }) {
-  const maskDimension = useImageDimension(rendered.maskUrl)
-
   const ref = useRef<HTMLVideoElement>(null)
   const handleEvent = (event: React.MouseEvent<HTMLVideoElement, MouseEvent>, isClick: boolean) => {
 
     if (!ref.current) {
-      console.log("element isn't ready")
+      // console.log("element isn't ready")
       return
     }
 
@@ -34,15 +31,10 @@ export function CartesianVideo({
 
     // then we convert them to relative coordinates
     const relativeX = containerX / boundingRect.width
-    const relativey = containerY / boundingRect.height
-
-    // finally, we convert back to coordinates within the input image
-    // TODO: go read the video size
-    const contentX = relativeX * maskDimension.width
-    const contentY = relativey * maskDimension.height
+    const relativeY = containerY / boundingRect.height
 
     const eventType = isClick ? "click" : "hover"
-    onEvent(eventType, contentX, contentY)
+    onEvent(eventType, relativeX, relativeY)
   }
 
   return (

src/components/business/renderer.tsx

@@ -8,6 +8,7 @@ import { CartesianImage } from "./cartesian-image"
 import { SceneEventHandler, SceneEventType } from "./types"
 import { CartesianVideo } from "./cartesian-video"
 import { SphericalImage } from "./spherical-image"
+import { useImageDimension } from "@/lib/useImageDimension"
 
 export const Renderer = ({
   rendered,
@@ -33,6 +34,7 @@ export const Renderer = ({
   const [progressPercent, setProcessPercent] = useState(0)
   const progressRef = useRef(0)
   const isLoadingRef = useRef(isLoading)
+  const maskDimension = useImageDimension(rendered.maskUrl)
 
   useEffect(() => {
     if (!rendered.maskUrl) {
@@ -104,7 +106,8 @@ export const Renderer = ({
     return closestSegment;
   }
 
-  const handleMouseEvent: SceneEventHandler = async (type: SceneEventType, x: number, y: number) => {
+  // note: coordinates must be between 0 and 1
+  const handleMouseEvent: SceneEventHandler = async (type: SceneEventType, relativeX: number, relativeY: number) => {
     if (!contextRef.current) return; // Return early if mask image has not been loaded yet
   
     if (isLoading) {
@@ -120,13 +123,16 @@ export const Renderer = ({
       return
     }
 
-    const newSegment = getSegmentAt(x, y)
+    const imageX = relativeX * maskDimension.width
+    const imageY = relativeY * maskDimension.height
+    
+    const newSegment = getSegmentAt(imageX, imageY)
 
     if (actionnable !== newSegment.label) {
       if (newSegment.label) {
-        console.log(`User is hovering "${newSegment.label}"`)
+        // console.log(`User is hovering "${newSegment.label}"`)
       } else {
-        console.log(`Nothing in the area`)
+        // console.log(`Nothing in the area`)
       }
 
       // update the actionnable immediately, so we can show the hand / finger cursor pointer
@@ -179,8 +185,8 @@ export const Renderer = ({
           "relative border-2 border-gray-50 rounded-xl overflow-hidden",
           engine.type === "cartesian_video"
           || engine.type === "cartesian_image"
-            ? " w-[1024px] h-[512px]" // w-[1024px] h-[512px]"
-            : "w-full h-screen",
+            ? " w-full" // w-[1024px] h-[512px]"
+            : "w-full",
             
           isLoading
             ? "cursor-wait"

src/components/business/spherical-image.tsx

@@ -1,10 +1,14 @@
 import { useEffect, useRef, useState } from "react"
-import { PluginConstructor } from "@photo-sphere-viewer/core"
+import { PanoramaPosition, PluginConstructor, Point, Position, SphericalPosition, Viewer } from "@photo-sphere-viewer/core"
 import { LensflarePlugin, ReactPhotoSphereViewer } from "react-photo-sphere-viewer"
 
 import { RenderedScene } from "@/app/types"
 
 import { SceneEventHandler } from "./types"
+import { useImageDimension } from "@/lib/useImageDimension"
+import { lightSourceNames } from "@/lib/lightSourceNames"
+
+type PhotoSpherePlugin = (PluginConstructor | [PluginConstructor, any])
 
 export function SphericalImage({
   rendered,
@@ -17,19 +21,24 @@ export function SphericalImage({
   className?: string
   debug?: boolean
 }) {
-  const sceneConfig = JSON.stringify({ rendered, debug })
+
+
+  const imageDimension = useImageDimension(rendered.assetUrl)
+  const maskDimension = useImageDimension(rendered.maskUrl)
+
+  const sceneConfig = JSON.stringify({ rendered, debug, imageDimension, maskDimension })
   const [lastSceneConfig, setLastSceneConfig] = useState<string>(sceneConfig)
-  const ref = useRef<{
-    needsUpdate: () => void
-    setPanorama: (src: string, options: Record<string, any>) => Promise<void>
-    setOptions: (options: Record<string, any>) => void
-  }>(null)
+  const rootContainerRef = useRef<HTMLDivElement>(null)
+  const viewerContainerRef = useRef<HTMLElement>()
+  const viewerRef = useRef<Viewer>()
+  const [mouseMoved, setMouseMoved] = useState<boolean>(false)
+
+  const defaultZoomLvl = 1 // 0 = 180 fov, 100 = 1 fov
 
   const options = {
-    defaultZoomLvl: 1,
-    overlay: rendered.maskUrl || undefined,
-    overlayOpacity: debug ? 0.5 : 0,
+    defaultZoomLvl,
     fisheye: false, // ..no!
+    overlayOpacity: debug ? 0.5 : 0,
     /*
     panoData: {
       fullWidth: 2000,
@@ -45,91 +54,194 @@ export function SphericalImage({
     */
   }
 
+
   useEffect(() => {
     const task = async () => {
       // console.log("SphericalImage: useEffect")
       if (sceneConfig !== lastSceneConfig) {
         // console.log("SphericalImage: scene config changed!")
     
-        if (!ref.current) {
+        if (!viewerRef.current) {
           // console.log("SphericalImage: no ref!")
           setLastSceneConfig(sceneConfig)
           return
         }
+        const viewer = viewerRef.current
+
+        const newOptions = {
+          ...options,
+        }
+
+        const lensflares: { id: string; position: SphericalPosition; type: number }[] = []
+          
+        if (maskDimension.width && imageDimension.width) {
+
+          // console.log("rendered.segments:", rendered.segments)
+   
+          rendered.segments
+            .filter(segment => lightSourceNames.includes(segment.label))
+            .forEach(light => {
+            // console.log("light detected", light)
+            const [x1, y1, x2, y2] = light.box
+            const [centerX, centerY] = [(x1 + x2) / 2, (y1 + y2) / 2]
+            // console.log("center:", { centerX, centerY })
+            const [relativeX, relativeY] = [centerX / maskDimension.width, centerY/ maskDimension.height]
+            // console.log("relative:", { relativeX, relativeY})
+
+            const panoramaPosition: PanoramaPosition = {
+              textureX: relativeX * imageDimension.width,
+              textureY: relativeY * imageDimension.height
+            }
+            // console.log("panoramaPosition:", panoramaPosition)
+
+            const position = viewer.dataHelper.textureCoordsToSphericalCoords(panoramaPosition)
+            // console.log("sphericalPosition:", position)
+            if ( // make sure coordinates are valid
+              !isNaN(position.pitch) && isFinite(position.pitch) &&
+              !isNaN(position.yaw) && isFinite(position.yaw)) {
+              lensflares.push({
+                id: `flare_${lensflares.length}`,
+                position,
+                type: 0,
+              })
+            }
+          })
+        }
+
+        // console.log("lensflares:", lensflares)
+        const lensFlarePlugin = viewer.getPlugin<LensflarePlugin>("lensflare")
+        lensFlarePlugin.setLensflares(lensflares)
 
         // console.log("SphericalImage: calling setOptions")
         // console.log("SphericalImage: changing the panorama to: " + rendered.assetUrl.slice(0, 120))
       
-        await ref.current.setPanorama(rendered.assetUrl, {
-          ...options,
+        await viewer.setPanorama(rendered.assetUrl, {
+          ...newOptions,
           showLoader: false,
         })
-        ref.current.setOptions(options)
+
+        // TODO we should separate all those updates, probaby
+        viewer.setOptions(newOptions)
+        viewer.setOverlay(rendered.maskUrl || undefined)
+
         // console.log("SphericalImage: asking to re-render")
-        ref.current.needsUpdate()
+        viewerRef.current.needsUpdate()
 
         setLastSceneConfig(sceneConfig)
       }
     }
     task()
-  }, [sceneConfig, rendered.assetUrl, ref.current])
+  }, [sceneConfig, rendered.assetUrl, viewerRef.current, maskDimension.width, imageDimension])
 
-  const plugins: (PluginConstructor | [PluginConstructor, any])[] = [
+  const handleEvent = async (event: React.MouseEvent<HTMLDivElement, MouseEvent>, isClick: boolean) => {
+    const rootContainer = rootContainerRef.current
+    const viewer = viewerRef.current
+    const viewerContainer = viewerContainerRef.current
 
-    // for the lensflare plugin we are also gonna need aw ay to determine the position
-    [LensflarePlugin, {
-      lensflares: [
-        {
-          id: 'sun',
-          position: { yaw: '145deg', pitch: '2deg' },
-          type: 0,
-        }
-      ]
-    }]
-  ]
+    /*
+    if (isClick) console.log(`handleEvent(${isClick})`, {
+      " imageDimension.width": imageDimension.width,
+      "rendered.maskUrl": rendered.maskUrl
+    })
+    */
+  
+    if (!viewer || !rootContainer || !viewerContainer || !imageDimension.width || !rendered.maskUrl) {
+      return
+    }
+
+    const containerRect = viewerContainer.getBoundingClientRect()
+    // if (isClick) console.log("containerRect:", containerRect)
+
+    const containerY = event.clientY - containerRect.top
+    // console.log("containerY:", containerY)
+    
+    const position: Position = viewer.getPosition()
+
+    const viewerPosition: Point = viewer.dataHelper.sphericalCoordsToViewerCoords(position)
+    // if (isClick) console.log("viewerPosition:", viewerPosition)
+
+    // we want to ignore events that are happening in the toolbar
+    // note that we will probably hide this toolbar at some point,
+    // to implement our own UI
+    if (isClick && containerY > (containerRect.height - 40)) {
+      // console.log("we are in the toolbar.. ignoring the click")
+      return
+    }
+
+    const panoramaPosition: PanoramaPosition = viewer.dataHelper.sphericalCoordsToTextureCoords(position)
+
+    if (typeof panoramaPosition.textureX !== "number" || typeof panoramaPosition.textureY !== "number") {
+      return
+    }
+    
+    const relativeX = panoramaPosition.textureX / imageDimension.width
+    const relativeY = panoramaPosition.textureY / imageDimension.height
+
+    onEvent(isClick ? "click" : "hover", relativeX, relativeY)
+  }
 
   if (!rendered.assetUrl) {
     return null
   }
 
   return (
-    <ReactPhotoSphereViewer
-      src={rendered.assetUrl}
-      ref={ref}
-      container=""
-      containerClass={className}
-      // 
-      height="100vh"
-      width="100%"
-      // plugins={plugins}
-
-      {...options}
-   
-      onClick={(data, instance) => {
-        console.log("on click:")
-        const position = data.target.getPosition()
-        console.log("position:", position)
+    <div
+      ref={rootContainerRef}
+      onMouseMove={(event) => {
+        handleEvent(event, false)
+        setMouseMoved(true)
       }}
-
-      onReady={(instance) => {
-        console.log("spherical image display is ready")
-        /*
-        const markersPlugs = instance.getPlugin(MarkersPlugin);
-        if (!markersPlugs)
-          return;
-        markersPlugs.addMarker({
-          id: "imageLayer2",
-          imageLayer: "drone.png",
-          size: { width: 220, height: 220 },
-          position: { yaw: '130.5deg', pitch: '-0.1deg' },
-          tooltip: "Image embedded in the scene"
-        });
-        markersPlugs.addEventListener("select-marker", () => {
-          console.log("asd");
-        });
-        */
+      onMouseUp={(event) => {
+        if (!mouseMoved) {
+          handleEvent(event, true)
+        }
+        setMouseMoved(false)
       }}
+      onMouseDown={() => {
+        setMouseMoved(false)
+      }}
+      >
+      <ReactPhotoSphereViewer
+        src={rendered.assetUrl}
+        container=""
+        containerClass={className}
+        
+        height="60vh"
+        width="100%"
+        
+        // to access a plugin we must use viewer.getPlugin()
+        plugins={[[LensflarePlugin, { lensflares: [] }]]}
+
+        {...options}
+
+        // note: photo sphere viewer performs an aggressive caching of our callbacks,
+        // so we aggressively disable it by using a ref
+        onClick={() => {
+          // nothing to do here
+        }}
+    
+        onReady={(instance) => {
+          viewerRef.current = instance
+          viewerContainerRef.current = instance.container
+
+          /*
+          const markersPlugs = instance.getPlugin(MarkersPlugin);
+          if (!markersPlugs)
+            return;
+          markersPlugs.addMarker({
+            id: "imageLayer2",
+            imageLayer: "drone.png",
+            size: { width: 220, height: 220 },
+            position: { yaw: '130.5deg', pitch: '-0.1deg' },
+            tooltip: "Image embedded in the scene"
+          });
+          markersPlugs.addEventListener("select-marker", () => {
+            console.log("asd");
+          });
+          */
+        }}
 
-    />
+      />
+    </div>
   )
 }

src/lib/getCoordinatesFromMousePosition.ts

@@ -9,31 +9,38 @@ export function getCoordinatesFromMousePosition({
   imageHeight,
   fov,
 }: {
-  mouseX: number
-  mouseY: number
-  containerWidth: number
-  containerHeight: number
-  currentYaw: number
-  currentPitch: number
-  imageWidth: number
-  imageHeight: number
-  fov: number
+  mouseX: number,
+  mouseY: number,
+  containerWidth: number,
+  containerHeight: number,
+  currentYaw: number,
+  currentPitch: number,
+  imageWidth: number,
+  imageHeight: number,
+  fov: number,
 }): { x: number; y: number } {
-  // Convert mouse position to relative to the viewer's center
+  // Considering the full width/height of FOV
   const relativeX = 2 * (mouseX / containerWidth) - 1;
   const relativeY = 2 * (mouseY / containerHeight) - 1;
 
-  // Calculate angle differences (in degrees)
-  const deltaYaw = relativeX * fov / 2;
-  const deltaPitch = relativeY * fov / 2;
+  // yaw varies with FOV over width and pitch over height
+  const deltaYaw = relativeX * fov * (Math.PI / 180);
+  const deltaPitch = relativeY * fov * (Math.PI / 180);
+  
+  let newYaw = currentYaw + deltaYaw;
+  while (newYaw < 0) newYaw += 2 * Math.PI;
+  while (newYaw > 2 * Math.PI) newYaw -= 2 * Math.PI;
 
-  // Calculate new yaw and pitch
-  const newYaw = currentYaw + deltaYaw;
-  const newPitch = currentPitch + deltaPitch;
+  let newPitch = currentPitch + deltaPitch;
+  if (newPitch < -Math.PI / 2) newPitch = -Math.PI / 2;
+  if (newPitch > Math.PI / 2) newPitch = Math.PI / 2;
 
-  // Now convert these yaw, pitch back to (x, y) on the image
-  const x = ((newYaw + 180) / 360) * imageWidth;
-  const y = ((newPitch + 90) / 180) * imageHeight;
+  // Changing origin for the yaw rotation to bring image center to screen center
+  newYaw = (newYaw + Math.PI) % (2 * Math.PI);
+  
+  // Image X corresponds to Yaw and Y corresponds to Pitch
+  const x = ((newYaw / (2 * Math.PI)) * imageWidth) % imageWidth;
+  const y = (((newPitch + Math.PI / 2) / Math.PI) * imageHeight) % imageHeight;
 
   return { x, y };
 }

src/lib/lightSourceNames.ts

@@ -0,0 +1,11 @@
+export const lightSourceNames = [
+  "window",
+  "light",
+  "sun",
+  "torch",
+  "fire",
+  "lights",
+  "torches",
+  "fires",
+  "fireplace"
+]

src/lib/normalizeActionnables.ts

@@ -0,0 +1,42 @@
+import { lightSourceNames } from "./lightSourceNames"
+
+export function normalizeActionnables(rawActionnables: string[]) {
+
+  const tmp = rawActionnables.map(item =>
+    // clean the words to remove any punctuation
+    item.replace(/\W/g, '').trim()
+  )
+
+  const deduplicated = new Set<string>([
+    ...tmp,
+    // in case result is too small, we add a reserve of useful words here
+    "door",
+    "rock",
+    "window",
+    "table",
+    "ground",
+    "sky",
+    "object",
+    "tree",
+    "wall",
+    "floor"
+    // but we still only want 10 here
+    ].slice(0, 10)
+  )
+
+  console.log("deduplicated:", deduplicated)
+
+  let actionnables = Array.from(deduplicated.values())
+
+  // if we are missing a light source, we add one (the generic "light")
+  if (!actionnables.some(actionnable => lightSourceNames.includes(actionnable))) {
+    actionnables.push("light")
+  }
+
+  // if ground surfaces aren't in the list, we add at least one (the most generic)
+  // if (!actionnables.includes("floor") || !actionnables.includes("ground")) {
+  //   actionnables.push("floor")
+  // }
+
+  return actionnables
+}