update

package.json

@@ -55,12 +55,14 @@
     "pick": "^0.0.1",
     "postcss": "8.4.26",
     "react": "18.2.0",
+    "react-circular-menu": "^2.4.2",
     "react-circular-progressbar": "^2.1.0",
     "react-day-picker": "^8.8.0",
     "react-dnd": "^16.0.1",
     "react-dnd-html5-backend": "^16.0.1",
     "react-dom": "18.2.0",
     "react-photo-sphere-viewer": "^3.3.5-psv5.1.4",
+    "styled-components": "^6.0.7",
     "tailwind-merge": "^1.13.2",
     "tailwindcss": "3.3.3",
     "tailwindcss-animate": "^1.0.6",

src/app/main.tsx

@@ -315,8 +315,8 @@ export default function Main() {
         newEvent = <>🔎 You are holding <span className="font-bold">&quot;{item.name}&quot;</span> and looking around, wondering how to use it.</>
         newEventString = `User is holding "${item.name}" from their inventory and wonder how they can use it.`
       } else {
-        newEvent = <>🔎 You are looking at the scene, looking for clues.</>
-        newEventString = `User is looking at the scene, looking for clues.`
+        newEvent = <>🔎 You are looking at the scene, searching for clues.</>
+        newEventString = `User is looking at the scene, searching for clues.`
       }
     } else if (event === "HoveringActionnable") {
       if (item) {

src/app/queries/getActionnables.ts

@@ -6,6 +6,27 @@ import { getBase } from "./getBase"
 import { predict } from "./predict"
 import { normalizeActionnables } from "@/lib/normalizeActionnables"
 
+const parseActionnablesOrThrow = (input: string) => {
+  let result: string[] = []
+  try {
+    result = parseJsonList(input)
+
+    if (!result.length) {
+      throw new Error("no actionnables")
+    }
+  } catch (err) {
+    console.log("failed to find a valid JSON! attempting method 2..")
+
+    const sanitized = input.replaceAll("[", "").replaceAll("]", "")
+    result = (JSON.parse(`[${sanitized}]`) as string[])
+
+    if (!result.length) {
+      throw new Error("no actionnables")
+    }
+  }
+  return result
+}
+
 export const getActionnables = async ({
   game,
   situation = "",
@@ -39,43 +60,21 @@ export const getActionnables = async ({
   ])
 
   let rawStringOutput = ""
-  
+  let result: string[] = []
+
   try {
     rawStringOutput = await predict(prompt)
+    result = parseActionnablesOrThrow(rawStringOutput)
   } catch (err) {
     console.log(`prediction of the actionnables failed, trying again..`)
     try {
-      rawStringOutput = await predict(prompt)
+      rawStringOutput = await predict(prompt+".")
+      result = parseActionnablesOrThrow(rawStringOutput)
     } catch (err) {
-      console.error(`prediction of the actionnables failed again!`)
-      throw new Error(`failed to generate the actionnables ${err}`)
+      console.error(`prediction of the actionnables failed again! going to use default value`)
+      console.log("for reference, rawStringOutput was: ", rawStringOutput)
     }
   }
   
-  let result: string[] = []
-
-  try {
-    result = parseJsonList(rawStringOutput)
-
-    if (!result.length) {
-      throw new Error("no actionnables")
-    }
-  } catch (err) {
-    console.log("failed to find a valid JSON! attempting method 2..")
-
-    try {
-      const sanitized = rawStringOutput.replaceAll("[", "").replaceAll("]", "")
-      result = (JSON.parse(`[${sanitized}]`) as string[])
-
-      if (!result.length) {
-        throw new Error("no actionnables")
-      }
-    } catch (err) {
-      console.log("failed to repair and recover a valid JSON! Using a generic fallback..")
-      
-      // throw new Error("failed to parse the actionnables")
-    }
-  }
-
   return normalizeActionnables(result)
 }

src/app/queries/getBackground.ts

@@ -27,20 +27,21 @@ export const getBackground = async ({
   })
 
   const basePrompt = initialPrompt !== currentPrompt
-    ? `You must imagine the most plausible next scene, based on where the player was located before and is now, and also what the player did before and are doing now.
-Here is the original scene in which the user was located at first, which will inform you about the general settings to follow (you must respect this): "${initialPrompt}".`
+    ? `You must imagine a very short caption for a background photo image, based on current and past situation.
+Here is the original scene in which the user was located at first, which will inform you about the general game mood to follow (you must respect this): "${initialPrompt}".`
     : ""
 
   const prompt = createLlamaPrompt([
     {
       role: "system",
       content: [
-        `You are the AI game master of a role video game.`,
+        `You are the photo director of a role video game.`,
         basePrompt,
-        `You are going to receive new information about the current whereabouts of the player.`,
-        `Please write a photo caption for the next plausible scene to display in intricate details: the environment, lights, era, characters, objects, textures, light etc.`,
-        `You MUST include the following important objects that the user can click on: ${newActionnables}.`,
-        `As this is a caption be synthetic: describe things, but don't comment on them. Be straight to the point, and do not say things like "As the player clicks on.." or "the scene shifts to" (the best is not not mention the player at all)`
+        `You are going to receive new information about the current activity of the player.`,
+        `Please write in a single sentence a photo caption for the next plausible scene, using a few words for each of those categories: the environment, era, characters, objects, textures, lighting.`,
+        `Separate each of those category descriptions using a comma.`,
+        `You MUST mention the following important objects that the user can click on: ${newActionnables}.`,
+        `Be brief in your caption don't add your own comments. Be straight to the point, and never reply things like "As the player approaches.." or "As the player clicks.." or "the scene shifts to.." (the best is not not mention the player at all)`
       ].filter(item => item).join("\n")
     },
     {
@@ -63,5 +64,6 @@ Here is the original scene in which the user was located at first, which will in
     }
   }
 
-  return result
+  const tmp = result.split("Caption:").pop() || result
+  return tmp.replaceAll("\n", ", ")
 }

src/app/queries/getDialogue.ts

@@ -31,8 +31,8 @@ export const getDialogue = async ({
   */
 
   const basePrompt = initialPrompt !== currentPrompt
-    ? `You must imagine the most plausible next dialogue line from the game master, based on where the player was located before and is now, and also what the player did before and are doing now.
-Here is the original scene in which the user was located at first, which will inform you about the general settings to follow (you must respect this): "${initialPrompt}".`
+    ? `You must imagine the most plausible next dialogue line from the game master, based on current and past situation.
+Here is the original situation, which will inform you about the general game mood to follow (you must respect this): "${initialPrompt}".`
     : ""
 
   const prompt = createLlamaPrompt([

src/components/renderer/index.tsx

@@ -11,6 +11,8 @@ import { SphericalImage } from "./spherical-image"
 import { useImageDimension } from "@/lib/useImageDimension"
 import { useDrop } from "react-dnd"
 import { formatActionnableName } from "@/lib/formatActionnableName"
+import { SceneTooltip } from "./scene-tooltip"
+import { SceneMenu } from "./scene-menu"
 
 export const SceneRenderer = ({
   rendered,
@@ -28,6 +30,7 @@ export const SceneRenderer = ({
   debug: boolean
 }) => {
   const timeoutRef = useRef<any>()
+  const containerRef = useRef<HTMLDivElement>(null)
   const canvasRef = useRef<HTMLCanvasElement | null>(null)
   const contextRef = useRef<CanvasRenderingContext2D | null>(null)
   const [actionnable, setActionnable] = useState<string>("")
@@ -37,6 +40,17 @@ export const SceneRenderer = ({
   const isLoadingRef = useRef(isLoading)
   const maskDimension = useImageDimension(rendered.maskUrl)
 
+  const [isHover, setHover] = useState(false)
+
+  const tooltipTimeoutRef = useRef<ReturnType<typeof setTimeout>>()
+  const menuTimeoutRef = useRef<ReturnType<typeof setTimeout>>()
+  const [isTooltipVisible, setTooltipVisible] = useState(false)
+  const [isMenuVisible, setMenuVisible] = useState(false)
+  const [tooltipX, setTooltipX] = useState(0)
+  const [tooltipY, setTooltipY] = useState(0)
+  const [menuX, setMenuX] = useState(0)
+  const [menuY, setMenuY] = useState(0)
+
   const [{ isOver, canDrop }, drop] = useDrop({
     accept: "item",
     drop: (): DropZoneTarget => ({
@@ -122,19 +136,21 @@ export const SceneRenderer = ({
 
   // note: coordinates must be between 0 and 1
   const handleMouseEvent: MouseEventHandler = async (type: MouseEventType, relativeX: number, relativeY: number) => {
-    if (!contextRef.current) return; // Return early if mask image has not been loaded yet
-    if (!rendered.maskUrl) return;
     
-    if (isLoading) {
-      // we ignore all user interactions
-      return
-    }
+    const noMenu = !containerRef.current
+    const noContext = !contextRef.current
+    const noSegmentationMask = !rendered.maskUrl
+    const noSegmentsToClickOn = rendered.segments.length == 0
+
+    const mustAbort =
+     noMenu
+     || noContext
+     || noSegmentationMask
+     || noSegmentsToClickOn
+     || isLoading
 
-    // sometimes we generate an image, but the segmentation fails
-    // so if we click anywhere bug there are no segments,
-    // we inform the rest of the app by passing nothing
-    if (type === "click" && rendered.segments.length == 0) {
-      onEvent("ClickOnNothing")
+    if (mustAbort) {
+      // if (type === "click") { onEvent("ClickOnNothing") }
       return
     }
 
@@ -154,21 +170,77 @@ export const SceneRenderer = ({
       setActionnable(actionnableRef.current = newSegment.label)
     }
 
+    const container = containerRef.current
+    const containerBox = container.getBoundingClientRect()
+
+    const absoluteMouseX = containerBox.left + relativeX * container.clientWidth
+    const absoluteMouseY = containerBox.top + relativeY * container.clientHeight
+    
+    clearTimeout(tooltipTimeoutRef.current)
+    clearTimeout(menuTimeoutRef.current)
+    setTooltipVisible(false)
+    setMenuVisible(false)
+    setTooltipX(absoluteMouseX)
+    setTooltipY(absoluteMouseY)
+    setMenuX(absoluteMouseX)
+    setMenuY(absoluteMouseY)
+
+    
     if (type === "click") {
+      setMenuVisible(false)
       if (!newSegment.label) {
+        // setMenuVisible(false)
         return
       }
+
+      setTooltipVisible(true)
+      setMenuVisible(true)
+
       console.log("User clicked on " + newSegment.label)
       onEvent("ClickOnActionnable", actionnable)
-    } else {
-      // only trigger hover events if there are segments,
-      // otherwise it's best to stay silent
-      if (rendered.segments.length) {
-        if (actionnable) {
-          onEvent("HoveringActionnable", actionnable)
-        } else {
-          onEvent("HoveringNothing")
-        }
+    } else { // hover
+      if (actionnable) {
+        setHover(true)
+
+        tooltipTimeoutRef.current = setTimeout(() => {
+          if (tooltipTimeoutRef.current) {
+            clearTimeout(tooltipTimeoutRef.current)
+            tooltipTimeoutRef.current = undefined
+            setTooltipVisible(true)
+          }
+        }, 400)
+
+        menuTimeoutRef.current = setTimeout(() => {
+          if (menuTimeoutRef.current) {
+            clearTimeout(menuTimeoutRef.current)
+            menuTimeoutRef.current = undefined
+            setMenuVisible(true)
+          }
+        }, 500)
+
+        onEvent("HoveringActionnable", actionnable)
+      } else {
+        setHover(false)
+        onEvent("HoveringNothing")
+
+        /*
+        tooltipTimeoutRef.current = setTimeout(() => {
+          if (tooltipTimeoutRef.current) {
+            setTooltipVisible(false)
+            clearTimeout(tooltipTimeoutRef.current)
+            tooltipTimeoutRef.current = undefined
+          }
+        }, 500)
+
+        menuTimeoutRef.current = setTimeout(() => {
+          if (menuTimeoutRef.current) {
+            setMenuVisible(false)
+            clearTimeout(menuTimeoutRef.current)
+            menuTimeoutRef.current = undefined
+          }
+        }, 500)
+        */
+
       }
     }
   };
@@ -200,17 +272,20 @@ export const SceneRenderer = ({
   return (
     <div className="w-full pt-2" ref={drop}>
       <div
+        ref={containerRef}
         className={[
           "relative border-2 border-gray-50 rounded-xl overflow-hidden min-h-[512px]",
           engine.type === "cartesian_video"
           || engine.type === "cartesian_image"
-            ? " w-full" // w-[1024px] h-[512px]"
+            ? "w-full" // w-[1024px] h-[512px]"
             : "w-full",
             
           isLoading
             ? "cursor-wait"
             : actionnable
-            ? "cursor-pointer"
+            ? isHover
+            ? "cursor-crosshair"
+            : "cursor-crosshair"
             : ""
           ].join(" ")}>
         {engine.type === "cartesian_video"
@@ -234,6 +309,22 @@ export const SceneRenderer = ({
 
       </div>
 
+      <SceneTooltip
+        isVisible={isTooltipVisible && !isLoading}
+        x={tooltipX}
+        y={tooltipY}>
+        {actionnable}
+      </SceneTooltip>
+
+      {/*
+      <SceneMenu
+        actions={["Go here", "Interact"]}
+        isVisible={isMenuVisible && !isLoading}
+        x={menuX}
+        y={menuY}
+      />
+      */}
+
       {isLoading
       ? <div className="fixed flex w-20 h-20 bottom-8 right-0 mr-8 z-50">
           <ProgressBar

src/components/renderer/scene-menu.tsx

@@ -0,0 +1,43 @@
+export function SceneMenu({
+  actions,
+  isVisible,
+  x,
+  y,
+}: {
+  actions: string[]
+  isVisible: boolean
+  x: number
+  y: number
+}) {
+  return (
+    <div className={[
+      `z-20 fixed flex flex-col w-24 pt-8 px-2 pb-2`,
+      `translate-x-[-50%] translate-y-[-20px]`,
+      isVisible ? "" : "",
+      isVisible ? "" : "pointer-events-none"
+    ].join(" ")}
+    style={{
+      top: `${y}px`,
+      left: `${x}px`,
+    }}
+    >
+    {actions.map((action, i) =>
+      <div
+      key={action}
+      className={[
+        `flex items-center justify-center px-2 py-1 cursor-pointer`
+      ].join(" ")}>
+        <div
+          className={[
+            `transition-all duration-150`,
+            isVisible ? "opacity-100 scale-100" : "scale-0 opacity-0 pointer-events-none",
+            `flex items-center justify-center rounded-full h-8 px-4`,
+            `hover:bg-gray-50 bg-gray-100 hover:border-gray-800 border-gray-300 border`,
+            `rounded-2xl text-gray-800 text-md`,
+          ].join(" ")}>
+          {action}
+        </div>
+    </div>)}
+    </div>
+  )
+}

src/components/renderer/scene-tooltip.tsx

@@ -0,0 +1,41 @@
+import { ReactNode } from "react"
+
+export function SceneTooltip({
+  children,
+  isVisible,
+  x,
+  y,
+}: {
+  children: ReactNode
+  isVisible: boolean
+  x: number
+  y: number
+}) {
+  return (
+    <div className={[
+      `z-10 fixed flex flex-col space-y-2 w-24 h-16 px-2`,
+      `translate-x-[-50%] translate-y-[-40px]`,
+      isVisible ? "cursor-pointer" : "",
+      "pointer-events-none"
+    ].join(" ")}
+    style={{
+      top: `${y}px`,
+      left: `${x}px`,
+    }}
+    >
+      <div
+        className={[
+          `transition-all duration-150`,
+          isVisible ? "opacity-100 scale-100" : "scale-0 opacity-0 pointer-events-none",
+          `flex items-center justify-center rounded-full h-8 px-4`,
+          `text-gray-50 text-xl`,
+          `cursor-pointer capitalize`
+        ].join(" ")}
+        style={{
+          textShadow: "#000 0px 0px 1px, #000 0px 0px 1px, #000 0px 0px 1px"
+        }}>
+        {children}
+      </div>
+    </div>
+  )
+}