adjustment

src/app/render.ts

@@ -72,7 +72,7 @@ export async function newRender({
           prompt,
           // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
           nbFrames, // when nbFrames is 1, we will only generate static images
-          nbSteps: isForVideo ? 25 : 35, // 20 = fast, 30 = better, 50 = best
+          nbSteps: isForVideo ? 30 : 35, // 20 = fast, 30 = better, 50 = best
           actionnables,
           segmentation: "firstframe", // one day we will remove this param, to make it automatic
           width: isForVideo ? 576 : 1024,
@@ -121,7 +121,7 @@ export async function getRender(renderId: string) {
   }
 
   try {
-    console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
+    // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
     const res = await fetch(`${apiUrl}/render/${renderId}`, {
       method: "GET",
       headers: {

src/components/business/cartesian-image.tsx

@@ -1,9 +1,9 @@
-import { ForwardedRef, forwardRef } from "react"
-
+import { useRef } from "react"
 import { SceneEventHandler } from "./types"
 import { RenderedScene } from "@/app/types"
+import { useImageDimension } from "@/lib/useImageDimension"
 
-export const CartesianImage = forwardRef(({
+export function CartesianImage({
   rendered,
   onEvent,
   className,
@@ -13,23 +13,36 @@ export const CartesianImage = forwardRef(({
   onEvent: SceneEventHandler
   className?: string
   debug?: boolean
-}, ref: ForwardedRef<HTMLDivElement>) => {
+}) {
 
-  const handleEvent = (event: React.MouseEvent<HTMLDivElement, MouseEvent>, isClick: boolean) => {
+  const maskDimension = useImageDimension(rendered.maskUrl)
 
-    const element = ((ref as any)?.current) as HTMLDivElement
+  const ref = useRef<HTMLImageElement>(null)
+  const handleEvent = async (event: React.MouseEvent<HTMLImageElement, MouseEvent>, isClick: boolean) => {
 
-    if (!element) {
+    if (!ref.current) {
       console.log("element isn't ready")
       return
     }
 
-    const boundingRect = element.getBoundingClientRect()
-    const x = (event.clientX - boundingRect.left) * 1.03
-    const y = (event.clientY - boundingRect.top) //* 1.03
+    const boundingRect = ref.current.getBoundingClientRect()
+    // const x = (event.clientX - boundingRect.left) * 1.03
+    // const y = (event.clientY - boundingRect.top) //* 1.03
+
+    // those X and Y will be based on the current size of the container, which might be variable
+    const containerX = event.clientX - boundingRect.left
+    const containerY = event.clientY - boundingRect.top
+
+    // then we convert them to relative coordinates
+    const relativeX = containerX / boundingRect.width
+    const relativey = containerY / boundingRect.height
+
+    // finally, we convert back to coordinates within the input image
+    const imageX = relativeX * maskDimension.width
+    const imageY = relativey * maskDimension.height
 
     const eventType = isClick ? "click" : "hover"
-    onEvent(eventType, x, y)
+    onEvent(eventType, imageX, imageY)
   }
 
   if (!rendered.assetUrl) {
@@ -38,20 +51,18 @@ export const CartesianImage = forwardRef(({
   return (
     <div
       className={className}
-      onMouseUp={(event) => handleEvent(event, true)}
-      onMouseMove={(event) => handleEvent(event, false)}
     >
       <img
         src={rendered.assetUrl || undefined}
-        ref={ref as any}
+        ref={ref}
         className="absolute"
+        onMouseUp={(event) => handleEvent(event, true)}
+        onMouseMove={(event) => handleEvent(event, false)}
       />
       {debug && <img
         src={rendered.maskUrl || undefined}
-        className="absolute opacity-50"
+        className="absolute opacity-50 pointer-events-none"
       />}
     </div>
   )
-})
-
-CartesianImage.displayName = "CartesianImage"
+}

src/components/business/cartesian-video.tsx

@@ -1,9 +1,9 @@
-import { ForwardedRef, forwardRef } from "react"
-
+import { useRef } from "react"
 import { SceneEventHandler } from "./types"
 import { RenderedScene } from "@/app/types"
+import { useImageDimension } from "@/lib/useImageDimension"
 
-export const CartesianVideo = forwardRef(({
+export function CartesianVideo({
   rendered,
   onEvent,
   className,
@@ -13,49 +13,58 @@ export const CartesianVideo = forwardRef(({
   onEvent: SceneEventHandler
   className?: string
   debug?: boolean
-}, ref: ForwardedRef<HTMLDivElement>) => {
-
-  const handleEvent = (event: React.MouseEvent<HTMLDivElement, MouseEvent>, isClick: boolean) => {
+}) {
+  const maskDimension = useImageDimension(rendered.maskUrl)
 
-    const element = ((ref as any)?.current) as HTMLDivElement
+  const ref = useRef<HTMLVideoElement>(null)
+  const handleEvent = (event: React.MouseEvent<HTMLVideoElement, MouseEvent>, isClick: boolean) => {
 
-    if (!element) {
+    if (!ref.current) {
       console.log("element isn't ready")
       return
     }
 
-    const boundingRect = element.getBoundingClientRect()
-    const x = event.clientX - boundingRect.left
-    const y = event.clientY - boundingRect.top
+    const boundingRect = ref.current.getBoundingClientRect()
+    // const x = (event.clientX - boundingRect.left) * 1.03
+    // const y = (event.clientY - boundingRect.top) //* 1.03
+
+    // those X and Y will be based on the current size of the container, which might be variable
+    const containerX = event.clientX - boundingRect.left
+    const containerY = event.clientY - boundingRect.top
+
+    // then we convert them to relative coordinates
+    const relativeX = containerX / boundingRect.width
+    const relativey = containerY / boundingRect.height
+
+    // finally, we convert back to coordinates within the input image
+    // TODO: go read the video size
+    const contentX = relativeX * maskDimension.width
+    const contentY = relativey * maskDimension.height
 
     const eventType = isClick ? "click" : "hover"
-    onEvent(eventType, x, y)
+    onEvent(eventType, contentX, contentY)
   }
 
   return (
-    <div
-    className={className}
-    ref={ref}
-    onMouseUp={(event) => handleEvent(event, true)}
-    onMouseMove={(event) => handleEvent(event, false)}
-  >
-   <video
-      src={rendered.assetUrl || undefined}
-      muted
-      autoPlay
-      loop
-      width="1024px"
-      height="512px"
-      className="absolute"
-    />
-    {debug && <img
-      src={rendered.maskUrl || undefined}
-      className="absolute opacity-50"
-      width="1024px"
-      height="512px"
-    />}
-  </div>
+    <div className={className}>
+      <video
+        src={rendered.assetUrl || undefined}
+        ref={ref}
+        onMouseUp={(event) => handleEvent(event, true)}
+        onMouseMove={(event) => handleEvent(event, false)}
+        className="absolute"
+        muted
+        autoPlay
+        loop
+        width="1024px"
+        height="512px"
+      />
+      {debug && <img
+        src={rendered.maskUrl || undefined}
+        className="absolute opacity-50 pointer-events-none"
+        width="1024px"
+        height="512px"
+      />}
+    </div>
   )
-})
-
-CartesianVideo.displayName = "CartesianVideo"
+}

src/components/business/renderer.tsx

@@ -27,7 +27,6 @@ export const Renderer = ({
   debug: boolean
 }) => {
   const timeoutRef = useRef<any>()
-  const viewRef = useRef<HTMLDivElement>()
   const canvasRef = useRef<HTMLCanvasElement | null>(null)
   const contextRef = useRef<CanvasRenderingContext2D | null>(null)
   const [actionnable, setActionnable] = useState<string>("")
@@ -192,20 +191,17 @@ export const Renderer = ({
         {engine.type === "cartesian_video"
           ? <CartesianVideo
               rendered={rendered}
-              ref={viewRef as any}
               onEvent={handleMouseEvent}
               debug={debug}
             />
           : engine.type === "spherical_image"
           ? <SphericalImage
               rendered={rendered}
-              ref={viewRef as any}
               onEvent={handleMouseEvent}
               debug={debug}
             />
           : <CartesianImage
               rendered={rendered}
-              ref={viewRef as any}
               onEvent={handleMouseEvent}
               debug={debug}
             />

src/components/business/spherical-image.tsx

@@ -1,10 +1,9 @@
-import { ForwardedRef, forwardRef } from "react"
 import { ReactPhotoSphereViewer } from "react-photo-sphere-viewer"
 
 import { SceneEventHandler } from "./types"
 import { RenderedScene } from "@/app/types"
 
-export const SphericalImage = forwardRef(({
+export function SphericalImage({
   rendered,
   onEvent,
   className,
@@ -14,7 +13,7 @@ export const SphericalImage = forwardRef(({
   onEvent: SceneEventHandler
   className?: string
   debug?: boolean
-}, ref: ForwardedRef<HTMLImageElement>) => {
+}) {
 
   if (!rendered.assetUrl) {
     return null
@@ -32,6 +31,21 @@ export const SphericalImage = forwardRef(({
 
       defaultZoomLvl={1}
 
+      overlay={rendered.maskUrl || undefined}
+      overlayOpacity={debug ? 0.5 : 0}
+
+      panoData={{
+        fullWidth: 1300,
+        fullHeight: 700,
+        croppedWidth: 1024,
+        croppedHeight: 512,
+        croppedX: 0,
+        croppedY: 120,
+        // poseHeading: 0, // 0 to 360
+         posePitch: 0, // -90 to 90
+        // poseRoll: 0, // -180 to 180
+      }}
+
       onClick={(data, instance) => {
         console.log("on click:")
         const position = data.target.getPosition()
@@ -59,6 +73,4 @@ export const SphericalImage = forwardRef(({
 
     />
   )
-})
-
-SphericalImage.displayName = "SphericalImage"
+}

src/lib/getImageDimension.ts

@@ -0,0 +1,16 @@
+export interface ImageDimension {
+  width: number
+  height: number
+}
+
+export async function getImageDimension(src: string): Promise<ImageDimension> {
+  if (!src) {
+    return { width: 0, height: 0 }
+  }
+  const img = new Image()
+  img.src = src
+  await img.decode()
+  const width = img.width
+  const height = img.height
+  return { width, height }
+}

src/lib/useImageDimension.ts

@@ -0,0 +1,20 @@
+import { useEffect, useState } from "react"
+
+import { ImageDimension, getImageDimension } from "./getImageDimension"
+
+export function useImageDimension(src: string) {
+  const [dimension, setDimension] = useState<ImageDimension>({
+    width: 0,
+    height: 0,
+  })
+
+  useEffect(() => {
+    const compute = async () => {
+      const newDimension = await getImageDimension(src)
+      setDimension(newDimension)
+    }
+    compute()
+  }, [src])
+
+  return dimension
+}