debugging..

src/app/games/dungeon.ts

@@ -51,7 +51,7 @@ export const game: Game = {
   initialActionnables,
   getScenePrompt: (situation?: string) => [
     `screenshot from an adventure videogame`,
-        // `first-person footage`,
+    // `first-person footage`,
     situation || initialSituation,
     `medieval`,
     `unreal engine`,

src/app/main.tsx

@@ -18,6 +18,7 @@ import { RenderedScene } from "./types"
 import { predict } from "./predict"
 import { GameType } from "./games/types"
 import { defaultGame, games, getGame } from "./games"
+import { getPrompts } from "./prompts"
 
 export default function Main() {
   const [isPending, startTransition] = useTransition()
@@ -80,56 +81,42 @@ export default function Main() {
 
   const handleUserAction = async (actionnable: string) => {
     console.log("user actionnable:", actionnable)
- 
+  
+    setLoading(true)
+
     // TODO: ask Llama2 what to do about it
     // we need a frame and some actionnables,
     // perhaps even some music or sound effects
 
     await startTransition(async () => {
- 
-      setLoading(true)
 
       const game = getGame(ref.current)
-      const initialPrompt = [...game.getScenePrompt()].join(", ")
 
-      const currentPrompt = situation
-        ? [...game.getScenePrompt(situation)].join(", ")
-        : initialPrompt
+      const prompts = getPrompts(game, situation, actionnable)
+
+      console.log("prompts:", prompts)
 
       try {
-        const basePrompt = [
-          `QUESTION: You are the AI game master of a role video game.`,
-          initialPrompt !== currentPrompt ? `The initial scene of the game was this: "${initialPrompt}".` : '',
-          `The player is currently in this scene: "${currentPrompt}".`,
-          `The player has just clicked on "${actionnable}".`
-        ]
 
         console.log("ask the LLM to invent next steps..")
 
-        const rawSituation = await predict([
-          ...basePrompt,
-          `Please describe the new scene to display in intricate details: the environment, lights, era, characters, objects, textures, light etc. You must include important objects, that the user can click on (eg. characters, doors, vehicles, useful objects).\nANSWER:`
-        ].join(" "))
+        const rawSituation = await predict(prompts.situationPrompt)
 
         console.log(`rawSituation: `, rawSituation)
 
         if (!rawSituation) {
           throw new Error("failed to generate the situation")
         }
-        const newSituation = `${rawSituation.split("QUESTION:")[0] || ""}`
+        const newSituation = `${rawSituation || ""}`
         if (!newSituation) {
           throw new Error("failed to parse the situation")
         }
 
         console.log(`newSituation: `, newSituation)
 
-        const rawActionnables = await predict([
-          ...basePrompt,
-          `Here are the 4 most important objects visible in this scene, that the user can click on. The list is in JSON (list of strings). You must list basic name of things (eg. "parrot", "chest", "spaceship", "glass", "door", "person", "window", "light", "knob", "button" etc..) \nJSON = [`
-        ].join(" "))
+        const rawActionnables = await predict(prompts.actionnablesPrompt)
         console.log(`rawActionnables: `, rawActionnables)
 
- 
         if (!rawActionnables) {
           throw new Error("failed to generate the actionnables")
         }
@@ -152,17 +139,13 @@ export default function Main() {
  
         console.log(`newActionnables: `, newActionnables)
 
- 
-        const rawDialogue = await predict([
-          ...basePrompt,
-          `As a game master, what should you say next? (Only reply with 2 sentences, please).\nANSWER:`
-        ].join(" "))
+        const rawDialogue = await predict(prompts.dialoguePrompt)
         console.log(`rawDialogue: `, rawDialogue)
 
         if (!rawDialogue) {
           throw new Error("failed to generate the dialogue")
         }
-        const newDialogue = `${rawDialogue.split("QUESTION:")[0] || ""}`
+        const newDialogue = `${rawDialogue || ""}`
         if (!newDialogue) {
             throw new Error("failed to parse the dialogue")
         }

src/app/predict.ts

@@ -24,6 +24,7 @@ export async function predict(inputs: string) {
       if (
         instructions.includes("</s>") || 
         instructions.includes("<s>") ||
+        instructions.includes("</SYS>") ||
         instructions.includes("<|end|>") ||
         instructions.includes("<|assistant|>")
       ) {
@@ -40,6 +41,7 @@ export async function predict(inputs: string) {
     .replaceAll("<|end|>", "")
     .replaceAll("<s>", "")
     .replaceAll("</s>", "")
+    .replaceAll("</SYS>", "")
     .replaceAll("<|assistant|>", "")
     .replaceAll('""', '"')
   )

src/app/prompts.ts

@@ -0,0 +1,83 @@
+import { Game } from "./games/types"
+
+
+export const getPrompts = (game: Game, situation: string = "", actionnable: string = "") => {
+
+  const initialPrompt = [...game.getScenePrompt()].join(", ")
+
+  const currentPrompt = situation
+    ? [...game.getScenePrompt(situation)].join(", ")
+    : initialPrompt
+
+  const userSituationPrompt = [
+    `[PROMPT] Player is currently in "${currentPrompt}". Player clicked on the "${actionnable}".`
+  ]
+
+  const baseSituationPromptIfWeHaveHistory = initialPrompt !== currentPrompt
+    ? ` You must imagine the most plausible next scene, based on where the player was located before and is now, and also what the player did before and are doing now.
+Here is the original scene in which the user was located at first, which will inform you about the general settings to follow (you must respect this): "${initialPrompt}".`
+    : ""
+
+  const situationSystemPrompt = [
+    `[INST] <<SYS>>
+You are the AI game master of a role video game.${baseSituationPromptIfWeHaveHistory}
+You are going to receive new information about the current whereabouts of the player.
+Please describe the next plausible scene to display in intricate details: the environment, lights, era, characters, objects, textures, light etc. You must include important objects, that the user can click on (eg. characters, doors, vehicles, useful objects).
+<</SYS>>`,
+  ]
+
+  const situationPrompt = [
+    ...situationSystemPrompt,
+    ...userSituationPrompt,
+  ].join(" ")
+
+  const actionnablesSystemPrompt = [
+    `[INST] <<SYS>>
+You are an API endpoint that can return a list of objects thare are in a scene.
+You must list basic name of things (eg. "parrot", "chest", "spaceship", "glass", "door", "person", "window", "light", "knob", "button" <etc styleName=""></etc>
+The answer must be a JSON array, ie. a list of quoted strings.
+<</SYS>>`,
+  ]
+
+  const actionnablesPrompt = [
+    ...actionnablesSystemPrompt,
+    currentPrompt
+  ].join(" ")
+
+  const baseDialoguePromptIfWeHaveHistory = initialPrompt !== currentPrompt
+  ? `for your information, the initial game panel and scene was: "${initialPrompt}".`
+  : ""
+
+  const dialogueSystemPrompt = [
+    `[INST] <<SYS>>
+You are the AI game master of a role video game.
+You are going to receive new information about the current whereabouts and action of the player.
+${baseDialoguePromptIfWeHaveHistory}
+You must imagine a funny response to speak in reaction to what the player did, like in some old point and click video games.
+Please limit yourself to only a 1 or 2 sentences, please.
+<</SYS>>`
+  ]
+
+  const dialoguePrompt = [
+    ...dialogueSystemPrompt,
+    ...userSituationPrompt,
+  ].join(" ")
+
+  const prompts = {
+    initialPrompt,
+    currentPrompt,
+    userSituationPrompt,
+    baseSituationPromptIfWeHaveHistory,
+    situationSystemPrompt,
+    situationPrompt,
+    actionnablesSystemPrompt,
+    actionnablesPrompt,
+    baseDialoguePromptIfWeHaveHistory,
+    dialogueSystemPrompt,
+    dialoguePrompt,
+  }
+
+  return prompts
+}
+
+

src/components/business/image-renderer.tsx

@@ -1,6 +1,7 @@
 import { useEffect, useRef, useState } from "react"
 
 import { ImageSegment, RenderedScene } from "@/app/types"
+import { ProgressBar } from "../misc/progress"
 
 export const ImageRenderer = ({
   rendered: {
@@ -21,6 +22,7 @@ export const ImageRenderer = ({
   const canvasRef = useRef<HTMLCanvasElement | null>(null)
   const contextRef = useRef<CanvasRenderingContext2D | null>(null)
   const [actionnable, setActionnable] = useState<string>("")
+  const [progressPercent, setProcessPercent] = useState(0)
 
   useEffect(() => {
     if (maskBase64) {
@@ -86,7 +88,7 @@ export const ImageRenderer = ({
       if(distance < minDistance) {
         minDistance = distance;
         closestSegment = segment;
-        console.log(`${distance} -> ${segment.label}: score = ${segment.score}`)
+        // console.log(`${distance} -> ${segment.label}: score = ${segment.score}`)
       }
     });
 
@@ -127,9 +129,36 @@ export const ImageRenderer = ({
     }
   };
 
+  useEffect(() => {
+    let progress = 0
+
+    // note: when everything is fine, it takes about 45 seconds to render a new scene
+
+    const computeProgress = async () => {
+      console.log("'computing progress")
+      if (!isLoading && assetUrl) {
+        console.log("Finished loading!")
+        setProcessPercent(100)
+        return
+      }
+
+      console.log("still loading..")
+
+      setTimeout(() => {
+        setProcessPercent(progress++)
+      }, 1000)
+
+    }
+
+    computeProgress()
+  }, [isLoading, assetUrl])
+
   if (!assetUrl) {
     return <div className="flex w-full h-screen items-center justify-center text-center">
-      <div>Generating a new panel..</div>
+      <ProgressBar
+        text="⌛"
+        progressPercentage={progressPercent}
+      />
     </div>
   }
 

src/components/misc/progress.tsx

@@ -0,0 +1,57 @@
+"use client"
+
+import { CircularProgressbar } from "react-circular-progressbar"
+import "react-circular-progressbar/dist/styles.css"
+
+export function ProgressBar ({
+  className,
+  progressPercentage,
+  text
+}: {
+  className?: string
+  progressPercentage?: number
+  text?: string
+}) {
+  return (
+    <div className={className}>
+      <CircularProgressbar
+        // doc: https://www.npmjs.com/package/react-circular-progressbar
+
+        value={progressPercentage || 0}
+
+        // Text to display inside progressbar. Default: ''.
+        text={text || ""}   
+
+        // Width of circular line relative to total width of component, a value from 0-100. Default: 8.
+        strokeWidth={8}
+
+        /*
+          // As a convenience, you can use buildStyles to configure the most common style changes:
+
+          styles={buildStyles({
+            // Rotation of path and trail, in number of turns (0-1)
+            rotation: 0.25,
+
+            // Whether to use rounded or flat corners on the ends - can use 'butt' or 'round'
+            strokeLinecap: 'butt',
+
+            // Text size
+            textSize: '16px',
+
+            // How long animation takes to go from one percentage to another, in seconds
+            pathTransitionDuration: 0.5,
+
+            // Can specify path transition in more detail, or remove it entirely
+            // pathTransition: 'none',
+
+            // Colors
+            pathColor: `rgba(62, 152, 199, ${percentage / 100})`,
+            textColor: '#f88',
+            trailColor: '#d6d6d6',
+            backgroundColor: '#3e98c7',
+          })}
+          */
+      />
+    </div>
+  )
+}