fix

src/app/main.tsx

@@ -270,7 +270,7 @@ export default function Main() {
       </div>
 
       <div className={[
-        "flex flex-col w-full pt-4 space-y-2 px-2 text-gray-50 dark:text-gray-50",
+        "flex flex-col w-full pt-4 space-y-3 px-2 text-gray-50 dark:text-gray-50",
         getGame(gameRef.current).className // apply the game theme
       ].join(" ")}>
         <div className="flex flex-row">
@@ -286,7 +286,9 @@ export default function Main() {
             {i < (clickables.length - 1) ? <div>,</div> : null}
           </div>)}
         </div>
-        <p className="text-xl">You are looking at: <span className="font-bold">{hoveredActionnable || "nothing"}</span></p>
+        <div className="text-xl p-3 rounded-xl backdrop-blur-sm bg-white/30">
+          You are looking at: <span className="font-bold">{hoveredActionnable || "nothing"}</span>
+        </div>
         <Renderer
           rendered={rendered}
           onUserAction={handleUserAction}
@@ -295,7 +297,7 @@ export default function Main() {
           game={game}
           engine={engine}
         />
-        <p className="text-xl">{dialogue}</p>
+        <div className="text-xl rounded-xl backdrop-blur-sm bg-white/30">{dialogue}</div>
       </div>
     </div>
   )

src/app/types.ts

@@ -44,6 +44,6 @@ export interface RenderedScene {
   status: RenderedSceneStatus
   assetUrl: string 
   error: string
-  maskBase64: string
+  maskUrl: string
   segments: ImageSegment[]
 }

src/components/business/cartesian-image.tsx

@@ -1,24 +1,21 @@
 import { ForwardedRef, forwardRef } from "react"
 
 import { SceneEventHandler } from "./types"
+import { RenderedScene } from "@/app/types"
 
 export const CartesianImage = forwardRef(({
-  src,
-  width,
-  height,
+  rendered,
   onEvent,
   className,
 }: {
-  src: string
-  width: number | string
-  height: number | string
+  rendered: RenderedScene
   onEvent: SceneEventHandler
   className?: string
-}, ref: ForwardedRef<HTMLImageElement>) => {
+}, ref: ForwardedRef<HTMLDivElement>) => {
 
-  const handleEvent = (event: React.MouseEvent<HTMLImageElement, MouseEvent>, isClick: boolean) => {
+  const handleEvent = (event: React.MouseEvent<HTMLDivElement, MouseEvent>, isClick: boolean) => {
 
-    const element = ((ref as any)?.current) as HTMLImageElement
+    const element = ((ref as any)?.current) as HTMLDivElement
 
     if (!element) {
       console.log("element isn't ready")
@@ -26,23 +23,32 @@ export const CartesianImage = forwardRef(({
     }
 
     const boundingRect = element.getBoundingClientRect()
-    const x = event.clientX - boundingRect.left
-    const y = event.clientY - boundingRect.top
+    const x = (event.clientX - boundingRect.left) * 1.03
+    const y = (event.clientY - boundingRect.top) //* 1.03
 
     const eventType = isClick ? "click" : "hover"
     onEvent(eventType, x, y)
   }
 
+  if (!rendered.assetUrl) {
+    return null
+  }
   return (
-    <img
-      src={src}
-      ref={ref}
-      width={width}
-      height={height}
+    <div
       className={className}
       onMouseUp={(event) => handleEvent(event, true)}
       onMouseMove={(event) => handleEvent(event, false)}
-    />
+    >
+      <img
+        src={rendered.assetUrl || undefined}
+        ref={ref as any}
+        className="absolute"
+      />
+      <img
+        src={rendered.maskUrl || undefined}
+        className="absolute opacity-50"
+      />
+    </div>
   )
 })
 

src/components/business/cartesian-video.tsx

@@ -1,24 +1,21 @@
 import { ForwardedRef, forwardRef } from "react"
 
 import { SceneEventHandler } from "./types"
+import { RenderedScene } from "@/app/types"
 
 export const CartesianVideo = forwardRef(({
-  src,
-  width,
-  height,
+  rendered,
   onEvent,
   className,
 }: {
-  src: string
-  width: number | string
-  height: number | string
+  rendered: RenderedScene
   onEvent: SceneEventHandler
   className?: string
-}, ref: ForwardedRef<HTMLVideoElement>) => {
+}, ref: ForwardedRef<HTMLDivElement>) => {
 
-  const handleEvent = (event: React.MouseEvent<HTMLVideoElement, MouseEvent>, isClick: boolean) => {
+  const handleEvent = (event: React.MouseEvent<HTMLDivElement, MouseEvent>, isClick: boolean) => {
 
-    const element = ((ref as any)?.current) as HTMLVideoElement
+    const element = ((ref as any)?.current) as HTMLDivElement
 
     if (!element) {
       console.log("element isn't ready")
@@ -34,18 +31,24 @@ export const CartesianVideo = forwardRef(({
   }
 
   return (
-    <video
-      src={src}
-      ref={ref}
+    <div
+    className={className}
+    ref={ref}
+    onMouseUp={(event) => handleEvent(event, true)}
+    onMouseMove={(event) => handleEvent(event, false)}
+  >
+   <video
+      src={rendered.assetUrl || undefined}
       muted
       autoPlay
       loop
-      width={width}
-      height={height}
-      className={className}
-      onMouseUp={(event) => handleEvent(event, true)}
-      onMouseMove={(event) => handleEvent(event, false)}
+      className="absolute"
     />
+    <img
+      src={rendered.maskUrl || undefined}
+      className="absolute opacity-50"
+    />
+  </div>
   )
 })
 

src/components/business/renderer.tsx

@@ -10,11 +10,7 @@ import { CartesianVideo } from "./cartesian-video"
 import { SphericalImage } from "./spherical-image"
 
 export const Renderer = ({
-  rendered: {
-    assetUrl = "",
-    maskBase64 = "",
-    segments = []
-  },
+  rendered,
   onUserAction,
   onUserHover,
   isLoading,
@@ -29,34 +25,30 @@ export const Renderer = ({
   engine: Engine
 }) => {
   const timeoutRef = useRef<any>()
-  const imgRef = useRef<HTMLImageElement | HTMLVideoElement | null>(null)
+  const viewRef = useRef<HTMLDivElement>()
   const canvasRef = useRef<HTMLCanvasElement | null>(null)
   const contextRef = useRef<CanvasRenderingContext2D | null>(null)
   const [actionnable, setActionnable] = useState<string>("")
   const [progressPercent, setProcessPercent] = useState(0)
   const progressRef = useRef(0)
   const isLoadingRef = useRef(isLoading)
-  const maskUrl = `data:image/png;base64,${maskBase64}`
 
   useEffect(() => {
-    if (maskBase64) {
-      // console.log("maskBase64:", maskBase64)
-      const img = new Image();
-      img.onload = function () {
-        canvasRef.current = document.createElement('canvas');
-        canvasRef.current.width = img.width;
-        canvasRef.current.height = img.height;
-        contextRef.current = canvasRef.current.getContext('2d', {
-          willReadFrequently: true
-        });
-        contextRef.current!.drawImage(img, 0, 0, img.width, img.height);
-      }
-      img.src = "data:image/png;base64," + maskBase64;
-    } else {
-      // console.log("error, no maskBase64 detected!")
+    if (!rendered.maskUrl) {
+      return
     }
-  }, [maskBase64]);
-
+    const img = new Image()
+    img.onload = function () {
+      canvasRef.current = document.createElement('canvas')
+      canvasRef.current.width = img.width
+      canvasRef.current.height = img.height
+      contextRef.current = canvasRef.current.getContext('2d', {
+        willReadFrequently: true
+      })
+      contextRef.current!.drawImage(img, 0, 0, img.width, img.height)
+    }
+    img.src = rendered.maskUrl
+  }, [rendered.maskUrl])
 
   const getPixelColor = (x: number, y: number) => {
     if (!contextRef.current) {
@@ -74,7 +66,7 @@ export const Renderer = ({
 
   const getSegmentAt = (x: number, y: number): ImageSegment => {
     if (!contextRef.current) throw new Error("Unable to get context from canvas");
-    if (!maskBase64) throw new Error("Mask is undefined");
+    if (!rendered.maskUrl) throw new Error("Mask is undefined");
 
     let closestSegment: ImageSegment = {
       id: 0,
@@ -92,7 +84,7 @@ export const Renderer = ({
   
     let minDistance = Infinity;
         
-    segments.forEach(segment => {
+    rendered.segments.forEach(segment => {
       const segmentColor = segment.color.slice(0,3); // get the RGB part only
 
       const distance = Math.sqrt(
@@ -122,7 +114,7 @@ export const Renderer = ({
     // sometimes we generate an image, but the segmentation fails
     // so if we click anywhere bug there are no segments,
     // we inform the rest of the app by passing nothing
-    if (type === "click" && segments.length == 0) {
+    if (type === "click" && rendered.segments.length == 0) {
       onUserAction("nothing, to trigger a scene reload")
       return
     }
@@ -149,7 +141,7 @@ export const Renderer = ({
     } else {
       // only trigger hover events if there are segments,
       // otherwise it's best to stay silent
-      if (segments.length) {
+      if (rendered.segments.length) {
         onUserHover(actionnable)
       }
     }
@@ -177,40 +169,7 @@ export const Renderer = ({
     if (isLoading) {
       timeoutRef.current = setInterval(updateProgressBar, 200)
     }
-  }, [isLoading, assetUrl, engine?.type])
-
-
-  /*
-        <img
-          src={assetUrl}
-          ref={imgRef}
-          width="1024px"
-          height="512px"
-          className={
-            [
-              "absolute top-0 left-0",
-              actionnable ? "cursor-pointer" : ""
-            ].join(" ")
-          }
-          onMouseDown={(event) => handleMouseEvent(event, true)}
-          onMouseMove={handleMouseEvent}
-        />
-
-        <img
-          src={"data:image/png;base64," + maskBase64}
-          ref={imgRef}
-          width="1024px"
-          height="512px"
-          className={
-            [
-              "absolute top-0 left-0 opacity-30",
-              actionnable ? "cursor-pointer" : ""
-            ].join(" ")
-          }
-          onMouseDown={(event) => handleMouseEvent(event, true)}
-          onMouseMove={handleMouseEvent}
-        />
-        */
+  }, [isLoading, rendered.assetUrl, engine?.type])
 
   return (
     <div className="w-full pt-2">
@@ -219,7 +178,7 @@ export const Renderer = ({
           "relative border-2 border-gray-50 rounded-xl overflow-hidden",
           engine.type === "cartesian_video"
           || engine.type === "cartesian_image"
-            ? "w-[1024px] h-[512px]"
+            ? "w-full h-screen" // w-[1024px] h-[512px]"
             : "w-full h-[800px]",
             
           isLoading
@@ -228,27 +187,21 @@ export const Renderer = ({
             ? "cursor-pointer"
             : ""
           ].join(" ")}>
-        {!assetUrl ?
-          null
-        : engine.type === "cartesian_video"
+        {engine.type === "cartesian_video"
           ? <CartesianVideo
-              src={assetUrl}
-              ref={imgRef as any}
-              width="1024px"
-              height="512px"
+              rendered={rendered}
+              ref={viewRef as any}
               onEvent={handleMouseEvent}
             />
           : engine.type === "spherical_image"
           ? <SphericalImage
-              src={assetUrl}
-              ref={imgRef as any}
+              rendered={rendered}
+              ref={viewRef as any}
               onEvent={handleMouseEvent}
             />
           : <CartesianImage
-              src={assetUrl}
-              ref={imgRef as any}
-              width="1024px"
-              height="512px"
+              rendered={rendered}
+              ref={viewRef as any}
               onEvent={handleMouseEvent}
             />
         }

src/components/business/spherical-image.tsx

@@ -2,13 +2,14 @@ import { ForwardedRef, forwardRef } from "react"
 import { ReactPhotoSphereViewer } from "react-photo-sphere-viewer"
 
 import { SceneEventHandler } from "./types"
+import { RenderedScene } from "@/app/types"
 
 export const SphericalImage = forwardRef(({
-  src,
+  rendered,
   onEvent,
   className,
 }: {
-  src: string
+  rendered: RenderedScene
   onEvent: SceneEventHandler
   className?: string
 }, ref: ForwardedRef<HTMLImageElement>) => {
@@ -16,7 +17,7 @@ export const SphericalImage = forwardRef(({
 
   return (
     <ReactPhotoSphereViewer
-      src={src}
+      src={rendered.assetUrl}
       // ref={ref}
       container=""
       containerClass={className}