up

src/app/engine/caption.ts

@@ -0,0 +1,54 @@
+"use server"
+
+import { ImageAnalysisRequest, ImageAnalysisResponse } from "@/types"
+
+const apiUrl = `${process.env.RENDERING_ENGINE_API || ""}`
+
+export async function see({
+  prompt,
+  imageBase64
+}: {
+  prompt: string
+  imageBase64: string
+}): Promise<string> {
+  if (!prompt) {
+    console.error(`cannot call the API without an image, aborting..`)
+    throw new Error(`cannot call the API without an image, aborting..`)
+  }
+
+  try {
+    const request = {
+      prompt,
+      image: imageBase64
+
+    } as ImageAnalysisRequest
+
+    console.log(`calling ${apiUrl}/analyze called with: `, {
+      prompt: request.prompt,
+      image: request.image.slice(0, 20)
+    })
+
+    const res = await fetch(`${apiUrl}/analyze`, {
+      method: "POST",
+      headers: {
+        Accept: "application/json",
+        "Content-Type": "application/json",
+        // Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
+      },
+      body: JSON.stringify(request),
+      cache: 'no-store',
+    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+    // next: { revalidate: 1 }
+    })
+
+    if (res.status !== 200) {
+      throw new Error('Failed to fetch data')
+    }
+
+    const response = (await res.json()) as ImageAnalysisResponse
+    return response.result
+  } catch (err) {
+    console.error(err)
+    return ""
+  }
+}

src/app/engine/render.ts

@@ -29,6 +29,7 @@ export async function newRender({
     renderId: "",
     status: "error",
     assetUrl: "",
+    alt: prompt || "",
     maskUrl: "",
     error: "failed to fetch the data",
     segments: []
@@ -58,7 +59,7 @@ export async function newRender({
         // no need to upscale right now as we generate tiny panels
         // maybe later we can provide an "export" button to PDF
         upscalingFactor: 2,
-
+        analyze: true,
         cache: "ignore"
       } as Partial<RenderRequest>),
       cache: 'no-store',
@@ -95,6 +96,7 @@ export async function getRender(renderId: string) {
     renderId: "",
     status: "error",
     assetUrl: "",
+    alt: "",
     maskUrl: "",
     error: "failed to fetch the data",
     segments: []

src/app/interface/panel/index.tsx

@@ -11,6 +11,7 @@ import { useStore } from "@/app/store"
 import { cn } from "@/lib/utils"
 import { getInitialRenderedScene } from "@/lib/getInitialRenderedScene"
 import { Progress } from "@/app/interface/progress"
+import { see } from "@/app/engine/caption"
 // import { Bubble } from "./bubble"
 
 export function Panel({
@@ -34,6 +35,10 @@ export function Panel({
   const panels = useStore(state => state.panels)
   const prompt = panels[panel] || ""
 
+  // const setCaption = useStore(state => state.setCaption)
+  // const captions = useStore(state => state.captions)
+  // const caption = captions[panel] || ""
+
   const [_isPending, startTransition] = useTransition()
   const [rendered, setRendered] = useState<RenderedScene>(getInitialRenderedScene())
   const renderedRef = useRef<RenderedScene>()
@@ -72,6 +77,7 @@ export function Panel({
           renderId: "",
           status: "error",
           assetUrl: "",
+          alt: "",
           maskUrl: "",
           error: "failed to fetch the data",
           segments: []
@@ -128,6 +134,32 @@ export function Panel({
     }
   }, [])
 
+  /*
+  doing the captionning from the browser is expensive
+  a simpler solution is to caption directly during SDXL generation
+
+  useEffect(() => {
+    if (!rendered.assetUrl) { return }
+    // the asset url can evolve with time (link to a better resolution image)
+    // however it would be costly to ask for the caption, the low resolution is enough for the semantic resolution
+    // so we just do nothing if we already have the caption
+    if (caption) { return }
+    startTransition(async () => {
+      try {
+        const newCaption = await see({
+          prompt: "please caption the following image",
+          imageBase64: rendered.assetUrl 
+        })
+        if (newCaption) {
+          setCaption(newCaption)
+        }
+      } catch (err) {
+        console.error(`failed to generate the caption:`, err)
+      }
+    })
+  }, [rendered.assetUrl, caption])
+  */
+
   if (isLoading) {
     return (
       <div className={cn(
@@ -152,6 +184,7 @@ export function Panel({
       {rendered.assetUrl && <img
         src={rendered.assetUrl}
         className="w-full h-full object-cover"
+        alt={rendered.alt}
       />}
 
       {/*<Bubble className="absolute top-4 left-4">

src/app/layouts/index.tsx

@@ -1,3 +1,5 @@
+"use client"
+
 import { Panel } from "@/app/interface/panel"
 import { pick } from "@/lib/pick"
 

src/app/store/index.ts

@@ -1,4 +1,5 @@
 "use client"
+
 import { create } from "zustand"
 
 import { FontName } from "@/lib/fonts"
@@ -10,6 +11,7 @@ export const useStore = create<{
   font: FontName
   preset: Preset
   panels: string[]
+  captions: Record<string, string>
   layout: LayoutName
   zoomLevel: number
   isGeneratingLogic: boolean
@@ -21,6 +23,7 @@ export const useStore = create<{
   setPreset: (preset: Preset) => void
   setPanels: (panels: string[]) => void
   setLayout: (layout: LayoutName) => void
+  setCaption: (panelId: number, caption: string) => void
   setZoomLevel: (zoomLevel: number) => void
   setGeneratingLogic: (isGeneratingLogic: boolean) => void
   setGeneratingImages: (panelId: number, value: boolean) => void
@@ -30,16 +33,49 @@ export const useStore = create<{
   font: "cartoonist",
   preset: getPreset("japanese_manga"),
   panels: [],
+  captions: {},
   layout: getRandomLayoutName(),
   zoomLevel: 50,
   isGeneratingLogic: false,
   panelGenerationStatus: {},
   isGeneratingText: false,
   atLeastOnePanelIsBusy: false,
-  setPrompt: (prompt: string) => set({ prompt }),
-  setFont: (font: FontName) => set({ font }),
-  setPreset: (preset: Preset) => set({ preset }),
+  setPrompt: (prompt: string) => {
+    const existingPrompt = get().prompt
+    if (prompt === existingPrompt) { return }
+    set({
+      prompt,
+      panels: [],
+      captions: {},
+    })
+  },
+  setFont: (font: FontName) => {
+    const existingFont = get().font
+    if (font === existingFont) { return }
+    set({
+      font,
+      panels: [],
+      captions: {}
+    })
+  },
+  setPreset: (preset: Preset) => {
+    const existingPreset = get().preset
+    if (preset.label === existingPreset.label) { return }
+    set({
+      preset,
+      panels: [],
+      captions: {}
+    })
+  },
   setPanels: (panels: string[]) => set({ panels }),
+  setCaption: (panelId: number, caption: string) => {
+    set({
+      captions: {
+        ...get().captions,
+        [panelId]: caption
+      }
+    })
+  },
   setLayout: (layout: LayoutName) => set({ layout }),
   setZoomLevel: (zoomLevel: number) =>  set({ zoomLevel }),
   setGeneratingLogic: (isGeneratingLogic: boolean) => set({ isGeneratingLogic }),

src/lib/getInitialRenderedScene.ts

@@ -4,6 +4,7 @@ export const getInitialRenderedScene = (): RenderedScene => ({
   renderId: "",
   status: "pending",
   assetUrl: "", 
+  alt: "",
   error: "",
   maskUrl: "",
   segments: []

src/types.ts

@@ -42,6 +42,8 @@ export interface RenderRequest {
   cache: CacheMode
 
   wait: boolean // wait until the job is completed
+
+  analyze: boolean // analyze the image to generate a caption (optional)
 }
 
 export interface ImageSegment {
@@ -61,8 +63,18 @@ export interface RenderedScene {
   renderId: string
   status: RenderedSceneStatus
   assetUrl: string 
+  alt: string
   error: string
   maskUrl: string
   segments: ImageSegment[]
 }
 
+export interface ImageAnalysisRequest {
+  image: string // in base64
+  prompt: string
+}
+
+export interface ImageAnalysisResponse {
+  result: string
+  error?: string
+}