working on the new game engine

src/app/engines.ts

@@ -1,28 +1,76 @@
 
-export type EngineType = "image" | "video"
+export type EngineType =
+  | "cartesian_image"
+  | "cartesian_video"
+  | "spherical_image"
+  | "spherical_video"
+  | "spherical_stereogram"
+  | "spherical_stereovideo"
 
 export interface Engine {
   type: EngineType
   label: string
   modelName: string
   modelUrl: string
+  visible: boolean
+  enabled: boolean
 }
 
 export const engines: Record<string, Engine> = {
-  image: {
-    type: "image",
-    label: "Image",
-    modelName: "SDXL 1.0",
+  cartesian_image: {
+    type: "cartesian_image",
+    label: "Cartesian image",
+    visible: true,
+    enabled: true,
+    modelName: "SDXL",
     modelUrl: "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
   },
-  video: {
-    type: "video",
-    label: "Video",
-    modelName: "Zeroscope V2 576w",
+  cartesian_video: {
+    type: "cartesian_video",
+    label: "Cartesian video",
+    visible: false,
+    enabled: false,
+    modelName: "Zeroscope",
     modelUrl: "https://huggingface.co/cerspense/zeroscope_v2_576w",
-  }
+  },
+  spherical_image: {
+    type: "spherical_image",
+    label: "Spherical image",
+    visible: true,
+    enabled: true,
+    modelName: "SDXL 360",
+    modelUrl: "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
+  },
+
+  spherical_video: {
+    type: "spherical_video",
+    label: "Spherical video",
+    visible: false,
+    enabled: false,
+    modelName: "",
+    modelUrl: "",
+  },
+
+  spherical_stereogram: {
+    type: "spherical_stereogram",
+    label: "Spherical stereogram",
+    visible: false,
+    enabled: false,
+    modelName: "SDXL 360",
+    modelUrl: "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
+  },
+
+  // A.K.A. the Holy Graal
+  spherical_stereovideo: {
+      type: "spherical_stereovideo",
+      label: "Spherical stereovideo",
+      visible: false,
+      enabled: false,
+      modelName: "",
+      modelUrl: "",
+    }
 }
 
-export const defaultEngine: EngineType = "image"
+export const defaultEngine: EngineType = "cartesian_image"
 
 export const getEngine = (type?: EngineType): Engine => engines[type || defaultEngine] || engines[defaultEngine]

src/app/games/city.ts

@@ -44,6 +44,7 @@ const initialActionnables = [
 export const game: Game = {
   title: "City",
   type: "city",
+  engine: "cartesian_image",
   className:  edu.className,
   initialSituation,
   initialActionnables,

src/app/games/doom.ts

@@ -17,6 +17,7 @@ const initialActionnables = [
 export const game: Game = {
   title: "Doom",
   type: "doom",
+  engine: "cartesian_image",
   className: orbitron.className,
   initialSituation,
   initialActionnables,

src/app/games/dungeon.ts

@@ -48,6 +48,7 @@ const initialActionnables = [
 export const game: Game = {
   title: "Dungeon",
   type: "dungeon",
+  engine: "cartesian_image",
   className: amatic.className,
   initialSituation,
   initialActionnables,

src/app/games/enchanters.ts

@@ -2,7 +2,7 @@ import { macondo } from "@/lib/fonts"
 import { Game } from "./types"
 
 const initialSituation = [
-  `looking at a beautiful medieval castle on a lake, with a metallic gate, during golden house, surrounded by mountain, with a flying dragon visible afar`,
+  `looking at a beautiful medieval castle on a lake, with a metallic gate, during golden hour, surrounded by mountain, with a flying dragon visible afar`,
 ].join(", ")
 
 const initialActionnables = [
@@ -18,6 +18,7 @@ const initialActionnables = [
 export const game: Game = {
   title: "Enchanters",
   type: "enchanters",
+  engine: "spherical_image",
   className: macondo.className,
   initialSituation,
   initialActionnables,

src/app/games/pirates.ts

@@ -55,6 +55,7 @@ const initialSituation = [
 export const game: Game = {
   title: "Pirates",
   type: "pirates",
+  engine: "cartesian_image",
   className: lugrasimo.className,
   initialSituation,
   initialActionnables,

src/app/games/types.ts

@@ -1,3 +1,4 @@
+import { EngineType } from "../engines"
 
 export type GameType = "pirates" | "city" | "dungeon" | "doom" | "vernian" | "enchanters"
 
@@ -9,6 +10,7 @@ export interface Scene {
 export interface Game {
   title: string
   type: GameType
+  engine: EngineType
   className: string
   initialSituation: string
   initialActionnables: string[]

src/app/games/vernian.ts

@@ -19,17 +19,18 @@ const initialActionnables = [
 export const game: Game = {
   title: "Vernian",
   type: "vernian",
+  engine: "spherical_image",
   className: imfell.className,
   initialSituation,
   initialActionnables,
   getScenePrompt: (situation?: string) => [
     `Screenshot from a videogame`,
-    `steam punk`,
+    `steam punk decor`,
     `jules verne architecture and design`,
     `mysterious machines and mechanisms`,
     `first person`,
-    `unreal engine`,
     situation || initialSituation,
+    `unreal engine`,
   ]
 }
 

src/app/main.tsx

@@ -14,7 +14,7 @@ import {
   SelectValue,
 } from "@/components/ui/select"
 
-import { render } from "./render"
+import { newRender, getRender } from "./render"
 
 import { RenderedScene } from "./types"
 import { Game, GameType } from "./games/types"
@@ -24,14 +24,18 @@ import { getDialogue } from "@/app/queries/getDialogue"
 import { getActionnables } from "@/app/queries/getActionnables"
 import { Engine, EngineType, defaultEngine, engines, getEngine } from "./engines"
 
+const getInitialRenderedScene = (): RenderedScene => ({
+  renderId: "",
+  status: "pending",
+  assetUrl: "", 
+  error: "",
+  maskBase64: "",
+  segments: []
+})
 export default function Main() {
   const [isPending, startTransition] = useTransition()
-  const [rendered, setRendered] = useState<RenderedScene>({
-    assetUrl: "", 
-    error: "",
-    maskBase64: "",
-    segments:[]
-  })
+  const [rendered, setRendered] = useState<RenderedScene>(getInitialRenderedScene())
+  const renderedRef = useRef<RenderedScene>()
   const router = useRouter()
   const pathname = usePathname()
   const searchParams = useSearchParams()
@@ -41,24 +45,22 @@ export default function Main() {
   const [game, setGame] = useState<Game>(getGame(gameRef.current))
 
   const requestedEngine = (searchParams.get('engine') as EngineType) || defaultEngine
-  // const engineRef = useRef<EngineType>(requestedEngine)
   const [engine, setEngine] = useState<Engine>(getEngine(requestedEngine))
 
   const [situation, setSituation] = useState("")
   const [scene, setScene] = useState("")
   const [dialogue, setDialogue] = useState("")
   const [hoveredActionnable, setHoveredActionnable] = useState("")
-  const [isLoading, setLoading] = useState(true)
 
-  const loadNextScene = async (nextSituation?: string, nextActionnables?: string[]) => {
-    // console.log(`update view..`)
-    setLoading(true)
+  const loopRef = useRef<any>(null)
 
+  const loadNextScene = async (nextSituation?: string, nextActionnables?: string[]) => {
+    
     await startTransition(async () => {
       console.log("Rendering a scene for " + game.type)
 
       // console.log(`rendering scene..`)
-      const newRendered = await render({
+      renderedRef.current = await newRender({
         engine,
 
         // SCENE PROMPT
@@ -71,30 +73,74 @@ export default function Main() {
         ).slice(0, 6) // too many can slow us down it seems
       })
 
+      console.log("got the first version of our scene!", renderedRef.current)
+
       // detect if type game type changed while we were busy
       if (game?.type !== gameRef?.current) {
         console.log("game type changed! aborting..")
         return
-      } 
+      }
 
-      if (newRendered.assetUrl) {
-        // console.log(`got a new url: ${newRendered.assetUrl}`)
-        setScene(scene)
+      setScene(scene)
 
-        setRendered(newRendered)
-        setLoading(false)
-      }
+      setRendered(renderedRef.current)
     })
   }
 
+  const checkRenderedLoop = async () => {
+    // console.log("checkRenderedLoop! rendered:", renderedRef.current)
+    clearTimeout(loopRef.current)
+    if (!renderedRef.current?.renderId || renderedRef.current?.status !== "pending") {
+      // console.log("let's try again in a moments")
+      loopRef.current = setTimeout(() => checkRenderedLoop(), 200)
+      return
+    } 
+
+    // console.log("checking rendering..")
+    await startTransition(async () => {
+      // console.log(`getting latest updated scene..`)
+      try {
+        if (!renderedRef.current?.renderId) {
+          throw new Error(`missing renderId`)
+        }
+
+      
+        // console.log(`calling getRender(${renderedRef.current.renderId})`)
+        const newRendered = await getRender(renderedRef.current.renderId)
+        // console.log(`got latest updated scene:`, renderedRef.current)
+
+        // detect if type game type changed while we were busy
+        if (game?.type !== gameRef?.current) {
+          console.log("game type changed! aborting..")
+          return
+        }
+
+    
+        const before = JSON.stringify(renderedRef.current)
+        const after = JSON.stringify(newRendered)
+
+        if (after !== before) {
+          console.log("updating scene..")
+          renderedRef.current = newRendered
+          setRendered(renderedRef.current)
+        }
+      } catch (err) {
+        console.error(err)
+      }
+
+      clearTimeout(loopRef.current)
+      loopRef.current = setTimeout(() => checkRenderedLoop(), 1000)
+    })
+  }
+    
   useEffect(() => {
     loadNextScene()
+    checkRenderedLoop()
   }, [])
 
   const handleUserAction = async (actionnable: string) => {
     console.log("user actionnable:", actionnable)
   
-    setLoading(true)
 
     // TODO: ask Llama2 what to do about it
     // we need a frame and some actionnables,
@@ -128,7 +174,6 @@ export default function Main() {
           // todo we could also use useEffect
       } catch (err) {
         console.error(`failed to get one of the mandatory entites: ${err}`)
-        setLoading(false)
       }
     })
   }
@@ -138,13 +183,16 @@ export default function Main() {
   const handleSelectGame = (newGameType: GameType) => {
     gameRef.current = newGameType
     setGame(getGame(newGameType))
+    /*
     setRendered({
+      renderId: "",
+      status: "pending",
       assetUrl: "", 
       error: "",
       maskBase64: "",
       segments:[]
     })
-    setLoading(true)
+    */
 
     const current = new URLSearchParams(Array.from(searchParams.entries()))
     current.set("game", newGameType)
@@ -186,7 +234,7 @@ export default function Main() {
     <div
       className="flex flex-col w-full max-w-5xl"
     >
-      <div className="flex flex-row w-full justify-between items-center px-2 py-2 border-b-1 border-gray-50 bg-gray-800">
+      <div className="flex flex-row w-full justify-between items-center px-2 py-2 border-b-1 border-gray-50 dark:border-gray-50 bg-gray-800 dark:bg-gray-800">
         <div className="flex flex-row items-center space-x-3 font-mono">
           <label className="flex text-sm">Select a story:</label>
           <Select
@@ -211,8 +259,10 @@ export default function Main() {
               <SelectValue className="text-sm" placeholder="Type" />
             </SelectTrigger>
             <SelectContent>
-              {Object.entries(engines).map(([key, engine]) =>
-              <SelectItem key={key} value={key} disabled={key === "video"}>{engine.label} ({engine.modelName})</SelectItem>
+              {Object.entries(engines)
+                .filter(([_, engine]) => engine.visible)
+                .map(([key, engine]) =>
+              <SelectItem key={key} value={key} disabled={!engine.enabled}>{engine.label} ({engine.modelName})</SelectItem>
               )}
             </SelectContent>
           </Select>
@@ -220,12 +270,17 @@ export default function Main() {
       </div>
 
       <div className={[
-        "flex flex-col w-full pt-4 space-y-3 px-2",
+        "flex flex-col w-full pt-4 space-y-3 px-2 text-gray-50 dark:text-gray-50",
         getGame(gameRef.current).className // apply the game theme
       ].join(" ")}>
         <p className="text-xl">A stable diffusion exploration game. Click on an item to explore a new scene!</p>
         <div className="flex flex-row">
-          <div className="text-xl mr-2">🔎 Clickable items:</div>
+          <div className="text-xl mr-2">
+            {rendered.segments.length
+              ? <span>🔎 Clickable items:</span>
+              : <span>⌛ Loading clickable items..</span>
+            }
+          </div>
           {clickables.map((clickable, i) => 
           <div key={i} className="flex flex-row text-xl mr-2">
             <div className="">{clickable}</div>
@@ -237,7 +292,7 @@ export default function Main() {
           rendered={rendered}
           onUserAction={handleUserAction}
           onUserHover={setHoveredActionnable}
-          isLoading={isLoading}
+          isLoading={rendered.status === "pending"}
           game={game}
           engine={engine}
         />

src/app/render.ts

@@ -9,10 +9,9 @@ import { Engine, EngineType } from "./engines"
 // so we have to add it ourselves if needed
 const apiUrl = process.env.RENDERING_ENGINE_API
 
-
 const cacheDurationInSec = 30 * 60 // 30 minutes
 
-export async function render({
+export async function newRender({
   prompt,
   actionnables = [],
   engine,
@@ -30,20 +29,39 @@ export async function render({
     throw new Error(`cannot call the rendering API without actionnables, aborting..`)
   }
 
-  const nbFrames = engine.type === "video" ? 8 : 1
+  const nbFrames = engine.type === "cartesian_video" ? 8 : 1
 
   const cacheKey = `render/${JSON.stringify({ prompt, actionnables, nbFrames, type: engine.type })}`
 
-  return await Gorgon.get(cacheKey, async () => {
+  // return await Gorgon.get(cacheKey, async () => {
+
     let defaulResult: RenderedScene = {
+      renderId: "",
+      status: "error",
       assetUrl: "",
       maskBase64: "",
-      error: "",
+      error: "failed to fetch the data",
       segments: []
     }
 
     try {
-      console.log(`calling ${apiUrl}/render with prompt: ${prompt}`)
+      // console.log(`calling POST ${apiUrl}/render with prompt: ${prompt}`)
+
+      const isForVideo = nbFrames > 1
+
+  
+      console.log("REQUEST:", JSON.stringify({
+        prompt,
+        // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
+        nbFrames, // when nbFrames is 1, we will only generate static images
+        nbSteps: isForVideo ? 20 : 30, // 20 = fast, 30 = better, 50 = best
+        actionnables,
+        segmentation: "firstframe", // one day we will remove this param, to make it automatic
+        width: isForVideo ? 576 : 1024,
+        height: isForVideo ? 320 : 512,
+      }))
+    
+
       const res = await fetch(`${apiUrl}/render`, {
         method: "POST",
         headers: {
@@ -55,9 +73,11 @@ export async function render({
           prompt,
           // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
           nbFrames, // when nbFrames is 1, we will only generate static images
-          nbSteps: 20,
+          nbSteps: isForVideo ? 12 : 30, // 20 = fast, 30 = better, 50 = best
           actionnables,
           segmentation: "firstframe", // one day we will remove this param, to make it automatic
+          width: isForVideo ? 576 : 1024,
+          height: isForVideo ? 320 : 512,
         }),
         cache: 'no-store',
       // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
@@ -79,8 +99,60 @@ export async function render({
       return response
     } catch (err) {
       console.error(err)
-      Gorgon.clear(cacheKey)
+      // Gorgon.clear(cacheKey)
       return defaulResult
     }
-  }, cacheDurationInSec * 1000)
+
+  // }, cacheDurationInSec * 1000)
+}
+
+export async function getRender(renderId: string) {
+  if (!renderId) {
+    console.error(`cannot call the rendering API without a renderId, aborting..`)
+    throw new Error(`cannot call the rendering API without a renderId, aborting..`)
+  }
+
+  let defaulResult: RenderedScene = {
+    renderId: "",
+    status: "error",
+    assetUrl: "",
+    maskBase64: "",
+    error: "failed to fetch the data",
+    segments: []
+  }
+
+  try {
+    console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
+    const res = await fetch(`${apiUrl}/render/${renderId}`, {
+      method: "GET",
+      headers: {
+        Accept: "application/json",
+        "Content-Type": "application/json",
+        // Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
+      },
+      cache: 'no-store',
+    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+    // next: { revalidate: 1 }
+    })
+
+    // console.log("res:", res)
+    // The return value is *not* serialized
+    // You can return Date, Map, Set, etc.
+    
+    // Recommendation: handle errors
+    if (res.status !== 200) {
+      // This will activate the closest `error.js` Error Boundary
+      throw new Error('Failed to fetch data')
+    }
+    
+    const response = (await res.json()) as RenderedScene
+    // console.log("response:", response)
+    return response
+  } catch (err) {
+    console.error(err)
+    // Gorgon.clear(cacheKey)
+    return defaulResult
+  }
+
+  // }, cacheDurationInSec * 1000)
 }

src/app/types.ts

@@ -1,3 +1,5 @@
+export type ProjectionMode = 'cartesian' | 'spherical'
+
 export interface RenderRequest {
   prompt: string
 
@@ -21,6 +23,11 @@ export interface RenderRequest {
   nbSteps: number // min: 1, max: 50
 
   seed: number
+
+  width: number // fixed at 1024 for now
+  height: number // fixed at 512 for now
+
+  projection: ProjectionMode
 }
 
 export interface ImageSegment {
@@ -31,7 +38,10 @@ export interface ImageSegment {
   score: number 
 }
 
+export type RenderedSceneStatus = 'pending' | 'completed' | 'error'
 export interface RenderedScene {
+  renderId: string
+  status: RenderedSceneStatus
   assetUrl: string 
   error: string
   maskBase64: string

src/components/business/cartesian-image.tsx

@@ -0,0 +1,47 @@
+import { ForwardedRef, forwardRef } from "react"
+
+import { SceneEventHandler } from "./types"
+
+export const CartesianImage = forwardRef(({
+  src,
+  width,
+  height,
+  onEvent,
+  className,
+}: {
+  src: string
+  width: number | string
+  height: number | string
+  onEvent: SceneEventHandler
+  className?: string
+}, ref: ForwardedRef<HTMLImageElement>) => {
+
+  const handleEvent = (event: React.MouseEvent<HTMLImageElement, MouseEvent>, isClick: boolean) => {
+
+    const element = ((ref as any)?.current) as HTMLImageElement
+
+    if (!element) {
+      console.log("element isn't ready")
+      return
+    }
+
+    const boundingRect = element.getBoundingClientRect()
+    const x = event.clientX - boundingRect.left
+    const y = event.clientY - boundingRect.top
+
+    const eventType = isClick ? "click" : "hover"
+    onEvent(eventType, x, y)
+  }
+
+  return (
+    <img
+      src={src}
+      ref={ref}
+      width={width}
+      height={height}
+      className={className}
+      onMouseUp={(event) => handleEvent(event, true)}
+      onMouseMove={(event) => handleEvent(event, false)}
+    />
+  )
+})

src/components/business/cartesian-video.tsx

@@ -0,0 +1,50 @@
+import { ForwardedRef, forwardRef } from "react"
+
+import { SceneEventHandler } from "./types"
+
+export const CartesianVideo = forwardRef(({
+  src,
+  width,
+  height,
+  onEvent,
+  className,
+}: {
+  src: string
+  width: number | string
+  height: number | string
+  onEvent: SceneEventHandler
+  className?: string
+}, ref: ForwardedRef<HTMLVideoElement>) => {
+
+  const handleEvent = (event: React.MouseEvent<HTMLVideoElement, MouseEvent>, isClick: boolean) => {
+
+    const element = ((ref as any)?.current) as HTMLVideoElement
+
+    if (!element) {
+      console.log("element isn't ready")
+      return
+    }
+
+    const boundingRect = element.getBoundingClientRect()
+    const x = event.clientX - boundingRect.left
+    const y = event.clientY - boundingRect.top
+
+    const eventType = isClick ? "click" : "hover"
+    onEvent(eventType, x, y)
+  }
+
+  return (
+    <video
+      src={src}
+      ref={ref}
+      muted
+      autoPlay
+      loop
+      width={width}
+      height={height}
+      className={className}
+      onMouseUp={(event) => handleEvent(event, true)}
+      onMouseMove={(event) => handleEvent(event, false)}
+    />
+  )
+})

src/components/business/renderer.tsx

@@ -4,6 +4,10 @@ import { ImageSegment, RenderedScene } from "@/app/types"
 import { ProgressBar } from "../misc/progress"
 import { Game } from "@/app/games/types"
 import { Engine, EngineType } from "@/app/engines"
+import { CartesianImage } from "./cartesian-image"
+import { SceneEventHandler, SceneEventType } from "./types"
+import { CartesianVideo } from "./cartesian-video"
+import { SphericalImage } from "./spherical-image"
 
 export const Renderer = ({
   rendered: {
@@ -106,26 +110,22 @@ export const Renderer = ({
     return closestSegment;
   }
 
-  const handleMouseEvent = async (event: React.MouseEvent, isClickEvent: boolean = false) => {
+  const handleMouseEvent: SceneEventHandler = async (type: SceneEventType, x: number, y: number) => {
     if (!contextRef.current) return; // Return early if mask image has not been loaded yet
   
     if (isLoading) {
       // we ignore all user interactions
-      return false
+      return
     }
 
     // sometimes we generate an image, but the segmentation fails
     // so if we click anywhere bug there are no segments,
     // we inform the rest of the app by passing nothing
-    if (isClickEvent && segments.length == 0) {
+    if (type === "click" && segments.length == 0) {
       onUserAction("nothing, to trigger a scene reload")
       return
     }
 
-    const boundingRect = imgRef.current!.getBoundingClientRect();
-    const x = event.clientX - boundingRect.left;
-    const y = event.clientY - boundingRect.top;
-  
     const newSegment = getSegmentAt(x, y)
 
     if (actionnable !== newSegment.label) {
@@ -139,7 +139,7 @@ export const Renderer = ({
       setActionnable(newSegment.label)
     }
 
-    if (isClickEvent) {
+    if (type === "click") {
       if (!newSegment.label) {
         return
       }
@@ -212,49 +212,43 @@ export const Renderer = ({
         */
 
   return (
-    <div className={[
-      "w-full py-8",
-      // isLoading ? "animate-pulse" : ""
-    ].join(" ")
-    }>
-      <div className="relative w-[1024px] h-[512px] border-2 border-gray-50 rounded-xl overflow-hidden">
+    <div className="w-full py-8">
+      <div
+        className={[
+          "relative w-full h-[800px] border-2 border-gray-50 rounded-xl overflow-hidden",
+          isLoading
+            ? "cursor-wait"
+            : actionnable
+            ? "cursor-pointer"
+            : ""
+          ].join(" ")}>
         {!assetUrl ?
           null
-        : engine.type === "video"
-          ? <video
-            src={assetUrl}
-            ref={imgRef as any}
-            muted
-            autoPlay
-            loop
-            width="1024px"
-            height="512px"
-
-            className={
-              [
-               //  "border-1 border-gray-50",
-              //  "absolute top-0 left-0",
-                actionnable && !isLoading ? "cursor-pointer" : ""
-              ].join(" ")
-            }
-            onMouseDown={(event) => handleMouseEvent(event, true)}
-            onMouseMove={handleMouseEvent}
-          />
-          : <img
-          src={assetUrl}
-          // src={"data:image/png;base64," + maskBase64}
-          ref={imgRef as any}
-          width="1024px"
-          height="512px"
-          className={
-            [
-            //  "absolute top-0 left-0",
-              actionnable && !isLoading ? "cursor-pointer" : ""
-            ].join(" ")
-          }
-          onMouseDown={(event) => handleMouseEvent(event, true)}
-          onMouseMove={handleMouseEvent}
-        />}
+        : engine.type === "cartesian_video"
+          ? <CartesianVideo
+              src={assetUrl}
+              ref={imgRef as any}
+              width="1024px"
+              height="512px"
+              onEvent={handleMouseEvent}
+            />
+          : engine.type === "spherical_image"
+          ? <SphericalImage
+              src={assetUrl}
+              ref={imgRef as any}
+              width="1024px"
+              height="512px"
+              onEvent={handleMouseEvent}
+            />
+          : <CartesianImage
+              src={assetUrl}
+              ref={imgRef as any}
+              width="1024px"
+              height="512px"
+              onEvent={handleMouseEvent}
+            />
+        }
+
       </div>
 
       {isLoading

src/components/business/spherical-image.tsx

@@ -0,0 +1,62 @@
+import { ForwardedRef, forwardRef } from "react"
+import { ReactPhotoSphereViewer } from "react-photo-sphere-viewer"
+// import { CubemapAdapter } from "react-photo-sphere-viewer"
+
+import { SceneEventHandler } from "./types"
+
+export const SphericalImage = forwardRef(({
+  src,
+  width,
+  height,
+  onEvent,
+  className,
+}: {
+  src: string
+  width: number | string
+  height: number | string
+  onEvent: SceneEventHandler
+  className?: string
+}, ref: ForwardedRef<HTMLImageElement>) => {
+
+
+  return (
+    <ReactPhotoSphereViewer
+      src={src}
+      // ref={ref}
+      container=""
+      containerClass={className}
+      // 
+      height="800px"
+     //  height={'100vh'}
+      width="100%"
+
+      defaultZoomLvl={1}
+
+      onClick={(data, instance) => {
+        console.log("on click:")
+        const position = data.target.getPosition()
+        console.log("position:", position)
+      }}
+
+      onReady={(instance) => {
+        console.log("spherical image display is ready")
+        /*
+        const markersPlugs = instance.getPlugin(MarkersPlugin);
+        if (!markersPlugs)
+          return;
+        markersPlugs.addMarker({
+          id: "imageLayer2",
+          imageLayer: "drone.png",
+          size: { width: 220, height: 220 },
+          position: { yaw: '130.5deg', pitch: '-0.1deg' },
+          tooltip: "Image embedded in the scene"
+        });
+        markersPlugs.addEventListener("select-marker", () => {
+          console.log("asd");
+        });
+        */
+      }}
+
+    />
+  )
+})

src/components/business/types.ts

@@ -0,0 +1,3 @@
+export type SceneEventType = 'hover' | 'click' | 'back'
+
+export type SceneEventHandler = (type: SceneEventType, x: number, y: number) => Promise<void>