making progress on the AI Stories Factory

.env

@@ -1,2 +1,9 @@
 HF_API_TOKEN="<USE YOUR OWN>"
 MICROSERVICE_API_SECRET_TOKEN="<USE YOUR OWN>"
+
+AI_TUBE_URL="https://aitube.at"
+# AI_TUBE_URL="http://localhost:3000"
+
+AI_TUBE_API_SECRET_JWT_KEY=""
+AI_TUBE_API_SECRET_JWT_ISSUER=""
+AI_TUBE_API_SECRET_JWT_AUDIENCE=""

package-lock.json

@@ -54,6 +54,7 @@
         "typescript": "5.4.5",
         "usehooks-ts": "^2.14.0",
         "uuid": "^9.0.1",
+        "yaml": "^2.4.1",
         "zustand": "^4.5.1"
       }
     },

package.json

@@ -55,6 +55,7 @@
     "typescript": "5.4.5",
     "usehooks-ts": "^2.14.0",
     "uuid": "^9.0.1",
+    "yaml": "^2.4.1",
     "zustand": "^4.5.1"
   }
 }

src/app/layout.tsx

@@ -1,4 +1,4 @@
-import { cn } from '@/lib/utils'
+import { cn } from '@/lib/utils/cn'
 import './globals.css'
 import type { Metadata } from 'next'
 import { inter, salsa } from './fonts'

src/app/main.tsx

@@ -6,12 +6,13 @@ import { Card, CardContent, CardHeader } from '@/components/ui/card'
 import { Button } from '@/components/ui/button'
 import { InputField } from '@/components/form/input-field'
 import { Toaster } from '@/components/ui/sonner'
-import { cn } from '@/lib/utils'
+import { cn } from '@/lib/utils/cn'
 
 import { useStore } from './store'
 import { TextareaField } from '@/components/form/textarea-field'
 import { DeviceFrameset } from 'react-device-frameset'
 import 'react-device-frameset/styles/marvel-devices.min.css'
+import { generateClap } from './server/aitube/generateClap'
 
 export function Main() {
   const [_isPending, startTransition] = useTransition()
@@ -39,7 +40,23 @@ export function Main() {
   const isBusy = status === "generating" || hasPendingTasks
 
   const handleSubmit = async () => {
+    const prompt = storyPromptDraft
 
+    setStatus("generating")
+    setStoryPrompt(prompt)
+
+    startTransition(async () => {
+      console.log(`handleSubmit(): generating a clap using prompt = "${prompt}" `)
+
+      try {
+        const clap = await generateClap({ prompt })
+
+        console.log(`handleSubmit(): received a clap = `, clap)
+        setStatus("finished")
+      } catch (err) {
+        setStatus("error")
+      }
+    })
   }
 
   return (

src/app/page.tsx

@@ -4,7 +4,7 @@ import { useEffect, useState } from "react"
 import Head from "next/head"
 import Script from "next/script"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 import { Main } from "./main"
 

src/app/server/aitube/generateClap.ts

@@ -0,0 +1,44 @@
+"use server"
+
+import { parseClap } from "@/lib/clap/parseClap"
+import { ClapProject } from "@/lib/clap/types"
+
+import { aitubeApiUrl } from "../config"
+
+export async function generateClap({
+  prompt = "",
+}: {
+  prompt: string
+}): Promise<ClapProject> {
+  if (!prompt) { throw new Error(`please provide a prompt`) }
+
+  // AiTube Stories is nice, but we also need to leave some compute for AiTube Live and AiTube Gaming
+  const height = 1024
+  const width = 512
+
+  // console.log(`calling `+ gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict")
+
+  // remember: a space needs to be public for the classic fetch() to work
+  const res = await fetch(aitubeApiUrl, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      // TODO pass the JWT so that only the AI Stories Factory can call the API
+      // Authorization: `Bearer ${hfApiToken}`,
+    },
+    body: JSON.stringify({
+      prompt,
+      width,
+      height
+    }),
+    cache: "no-store",
+    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+    // next: { revalidate: 1 }
+  })
+
+  const blob = await res.blob()
+
+  const clap = await parseClap(blob)
+  
+  return clap
+}

src/app/server/config.ts

@@ -1,2 +1,5 @@
 
 export const serverHuggingfaceApiKey = `${process.env.HF_API_TOKEN || ""}`
+
+export const aitubeUrl = `${process.env.AI_TUBE_URL || "" }`
+export const aitubeApiUrl = + aitubeUrl + (aitubeUrl.endsWith("/") ? "" : "/") + "api/"

src/app/server/services/background.ts

@@ -1,50 +0,0 @@
-"use server"
-
-import { BackgroundRemovalParams } from "@/types"
-
-import { addBase64HeaderToPng } from "../utils/addBase64HeaderToPng"
-
-const gradioApi = `https://jbilcke-hf-background-removal-api.hf.space`
-const microserviceApiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
-
-export async function removeBackground({
-  imageAsBase64,
-}: BackgroundRemovalParams): Promise<string> {
-
-  // remember: a space needs to be public for the classic fetch() to work
-  const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      // Authorization: `Bearer ${hfApiToken}`,
-    },
-    body: JSON.stringify({
-      fn_index: 0, // <- is it 0 or 1?
-      data: [
-        microserviceApiKey,
-        imageAsBase64,
-      ],
-    }),
-    cache: "no-store",
-    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-    // next: { revalidate: 1 }
-  })
-
-  const { data } = await res.json()
-  
-  // console.log("data:", data)
-  // Recommendation: handle errors
-  if (res.status !== 200 || !Array.isArray(data)) {
-    // This will activate the closest `error.js` Error Boundary
-    throw new Error(`Failed to fetch data (status: ${res.status})`)
-  }
-  // console.log("data:", data.slice(0, 50))
-
-  const base64Content = (data?.[0] || "") as string
-
-  if (!base64Content) {
-    throw new Error(`invalid response (no content)`)
-  }
-
-  return addBase64HeaderToPng(base64Content)
-}

src/app/server/services/inpaint.ts

@@ -1,107 +0,0 @@
-"use server"
-
-import { InpaintingParams } from "@/types"
-
-import { addBase64HeaderToPng } from "../utils/addBase64HeaderToPng"
-import { segmentToInpaintingMask } from "../utils/segmentToInpaintingMask"
-
-const gradioApi = `https://jbilcke-hf-inpainting-api.hf.space`
-const microserviceApiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
-
-export async function inpaint({
-  imageAsBase64,
-  maskAsBase64,
-  positivePrompt = "",
-  negativePrompt = "",
-  guidanceScale = 7.5,
-  numInferenceSteps = 20,
-  strength = 0.99,
-  scheduler = "EulerDiscreteScheduler"
-}: InpaintingParams): Promise<string> {
-
-  const posPrompt = [
-    positivePrompt,
-    "clean",
-    "high-resolution",
-    "8k",
-    "best quality",
-    "masterpiece",
-    "crisp",
-    "sharp",
-    "intricate details"
-  ].join(", ")
-  
-  const negPrompt = [
-    negativePrompt,
-    "pixelated",
-    "pixels",
-    "noise",
-    "blur",
-    "motion blur",
-    "lowres",
-    "oversmooth",
-    "longbody",
-    "bad anatomy",
-    "bad hands",
-    "missing fingers",
-    "extra digit",
-    "fewer digits",
-    "cropped",
-    "worst quality",
-    "low quality",
-    "artificial",
-    "unrealistic",
-    "watermark",
-    "trademark",
-    "error",
-    "mistake"
-  ].join(", ")
-
-  // the segmentation mask is a RGB color one (that's how we can split layers)
-  // so we first convert it to either black or white
-  const inpaintingMaskAsBase64 = await segmentToInpaintingMask(maskAsBase64)
-  
-  // remember: a space needs to be public for the classic fetch() to work
-  const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      // Authorization: `Bearer ${hfApiToken}`,
-    },
-    body: JSON.stringify({
-      fn_index: 0, // <- is it 0 or 1?
-      data: [
-        microserviceApiKey,
-        imageAsBase64, 	// blob in 'parameter_5' Image component		
-        inpaintingMaskAsBase64,
-        posPrompt,
-        negPrompt,
-        guidanceScale,
-        numInferenceSteps,
-        strength,
-        scheduler,
-      ],
-    }),
-    cache: "no-store",
-    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-    // next: { revalidate: 1 }
-  })
-
-  const { data } = await res.json()
-  
-  // console.log("data:", data)
-  // Recommendation: handle errors
-  if (res.status !== 200 || !Array.isArray(data)) {
-    // This will activate the closest `error.js` Error Boundary
-    throw new Error(`Failed to fetch data (status: ${res.status})`)
-  }
-  // console.log("data:", data.slice(0, 50))
-
-  const base64Content = (data?.[0] || "") as string
-
-  if (!base64Content) {
-    throw new Error(`invalid response (no content)`)
-  }
-
-  return addBase64HeaderToPng(base64Content)
-}

src/app/server/services/segment.ts

@@ -1,138 +0,0 @@
-"use server"
-
-import { SemanticLayer, SemanticLayers } from "@/lib/config"
-
-import { addBase64HeaderToPng } from "../utils/addBase64HeaderToPng"
-import { segmentsToInpaintingMasks } from "../utils/segmentsToInpaintingMasks"
-import { alphaToWhite } from "../utils/alphaToWhite"
-
-const gradioApi = `https://jbilcke-hf-segmentation-api.hf.space`
-const microserviceApiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
-
-export type SegmentationResult = {
-  id: number
-  box: number[] // [299.63092041015625, 111.72967529296875, 661.6744384765625, 692.8449096679688],
-  label: string
-  score: number
-  color: number[] // [0.8506358185563304, 0.9904733533937202, 0.32005103765589715, 1.0]
-}
-
-type SegmentationApiResponse = {
-  data: SegmentationResult[]
-  bitmap: string // base64 png
-}
-
-export type SegmentationResults = {
-  data: Partial<Record<SemanticLayer, SegmentationResult>>
-  bitmap: string // base64 png
-}
-
-export async function segment({
-  imageAsBase64,
-  layers,
-}:  {
-  imageAsBase64: string
-  layers: SemanticLayers
-}): Promise<SemanticLayers> {
-
-  const emptyResponse: SemanticLayers = {}
-
-  Object.entries(layers).forEach(([key, value]) => {
-    emptyResponse[key as SemanticLayer] = ""
-  })
-
-  // console.log(`calling `+ gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict")
-
-  const detectionPrompt = Object.keys(layers).map(x => x.trim().toLowerCase()).join(" . ")
-
-  // min 0, max 1, value 0.3, step 0.001
-  const boxThreshold = 0.3
-
-  // min 0, max 1, value 0.25, step 0.001
-  const textThreshold = 0.25
-
-  // min 0, max 1, value 0.8, step 0.001
-  const iouThreshold = 0.8
-
-  // SAM is finicky, it doesn't work on images with an alpha channel
-  // so we first need to remove that
-  let imageToSegmentInBase64 = ""
-  imageToSegmentInBase64 = imageAsBase64
-  /*
-  try {
-    imageToSegmentInBase64 = await alphaToWhite(imageAsBase64)
-  } catch (err) {
-    console.error(`failed to get a valid imageToSegmentInBase64:`, err)
-    return emptyResponse
-  }
-  */
-
-  // remember: a space needs to be public for the classic fetch() to work
-  const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      // Authorization: `Bearer ${hfApiToken}`,
-    },
-    body: JSON.stringify({
-      fn_index: 0, // <- is it 0 or 1?
-      data: [
-        microserviceApiKey,
-        imageToSegmentInBase64,
-        detectionPrompt,
-        boxThreshold,
-        textThreshold,
-        iouThreshold,
-      ],
-    }),
-    cache: "no-store",
-    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-    // next: { revalidate: 1 }
-  })
-
-  const layeredResults = {} as Partial<Record<SemanticLayer, SegmentationResult>>
-
-  const { data } = await res.json()
-  
-  // console.log("data:", data)
-  // Recommendation: handle errors
-  if (res.status !== 200 || !Array.isArray(data)) {
-    // This will activate the closest `error.js` Error Boundary
-    console.error(`Failed to fetch data (${res.status} error: ${res.statusText})`, res)
-    return emptyResponse
-  }
-  // console.log("data:", data.slice(0, 50))
-
-  let apiResponse: SegmentationApiResponse = {
-    data: [],
-    bitmap: ""
-  }
-  
-  try {
-    apiResponse = JSON.parse((data?.[0] || "{}")) as SegmentationApiResponse
-  } catch (err) {
-    console.error(`Failed to parse api response`, err)
-    return emptyResponse
-  }
-
-  // console.log("segmentation", segmentation)
-  // console.log("segmentation.data:", segmentation.data)
-  const items = [...(apiResponse.data || [])]
-  // console.log("items:", items)
-
-  const bitmap = apiResponse.bitmap ? addBase64HeaderToPng(apiResponse.bitmap) : ""
-
-  Object.entries(layers).forEach(([key, value]) => {
-    const match = items.find(x => `${key || ""}`.trim().toLowerCase() === `${x.label || ""}`.trim().toLowerCase())
-    if (match) {
-      layeredResults[key as SemanticLayer] = match
-    }
-  })
-
-  const maskLayers = await segmentsToInpaintingMasks({
-    data: layeredResults,
-    bitmap,
-  })
-
-  return maskLayers
-}

src/app/server/services/stableCascade.ts

@@ -1,67 +0,0 @@
-"use server"
-
-import { generateSeed } from "@/lib/generateSeed"
-import { StableCascadeParams } from "@/types"
-
-import { addBase64HeaderToPng } from "../utils/addBase64HeaderToPng"
-
-const gradioApi = `https://jbilcke-hf-stable-cascade-api.hf.space`
-const microserviceApiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
-
-export async function stableCascade({
-  prompt,
-  negativePrompt,
-  guidanceScale,
-  nbPriorInferenceSteps,
-  nbDecoderInferenceSteps,
-  seed,
-  width,
-  height,
-}: StableCascadeParams): Promise<string> {
-
-  // console.log(`calling `+ gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict")
-
-  // remember: a space needs to be public for the classic fetch() to work
-  const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      // Authorization: `Bearer ${hfApiToken}`,
-    },
-    body: JSON.stringify({
-      fn_index: 0, // <- is it 0 or 1?
-      data: [
-        microserviceApiKey,
-        prompt,
-        negativePrompt,
-        height,
-        width,
-        guidanceScale,
-        seed || generateSeed(),
-        nbPriorInferenceSteps,
-        nbDecoderInferenceSteps
-      ],
-    }),
-    cache: "no-store",
-    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-    // next: { revalidate: 1 }
-  })
-
-  const { data } = await res.json()
-  
-  // console.log("data:", data)
-  // Recommendation: handle errors
-  if (res.status !== 200 || !Array.isArray(data)) {
-    // This will activate the closest `error.js` Error Boundary
-    throw new Error(`Failed to fetch data (status: ${res.status})`)
-  }
-  // console.log("data:", data.slice(0, 50))
-
-  const base64Content = (data?.[0] || "") as string
-
-  if (!base64Content) {
-    throw new Error(`invalid response (no content)`)
-  }
-
-  return addBase64HeaderToPng(base64Content)
-}

src/app/server/services/upscale.ts

@@ -1,105 +0,0 @@
-"use server"
-
-import { generateSeed } from "@/lib/generateSeed"
-import { UpscalingParams } from "@/types"
-
-import { addBase64HeaderToPng } from "../utils/addBase64HeaderToPng"
-
-const gradioApi = `https://jbilcke-hf-image-upscaling-api.hf.space`
-const microserviceApiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
-
-export async function upscale({
-  imageAsBase64,
-  prompt,
-  negativePrompt,
-  scaleFactor,
-  nbSteps,
-  seed,
-}: UpscalingParams): Promise<string> {
-
-  const addedPrompt = [
-    "clean",
-    "high-resolution",
-    "8k",
-    "best quality",
-    "masterpiece",
-    "crisp",
-    "sharp",
-    "intricate details"
-  ].join(", ")
-  
-  const negPrompt = [
-    negativePrompt,
-    "pixelated",
-    "pixels",
-    "noise",
-    "blur",
-    "motion blur",
-    "lowres",
-    "oversmooth",
-    "longbody",
-    "bad anatomy",
-    "bad hands",
-    "missing fingers",
-    "extra digit",
-    "fewer digits",
-    "cropped",
-    "worst quality",
-    "low quality",
-    "artificial",
-    "unrealistic",
-    "watermark",
-    "trademark",
-    "error",
-    "mistake"
-  ].join(", ")
-
-  const conditioningScale = 1.4
-  const classifierFreeGuidance = 9.5
-  
-  // remember: a space needs to be public for the classic fetch() to work
-  const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      // Authorization: `Bearer ${hfApiToken}`,
-    },
-    body: JSON.stringify({
-      fn_index: 0, // <- is it 0 or 1?
-      data: [
-        microserviceApiKey,
-        imageAsBase64, 	// blob in 'parameter_5' Image component		
-				prompt, // string  in 'Prompt' Textbox component		
-				addedPrompt, // string  in 'Added Prompt' Textbox component		
-				negPrompt, // string  in 'Negative Prompt' Textbox component		
-				nbSteps, // number (numeric value between 10 and 50) in 'Denoise Steps' Slider component		
-				scaleFactor, // number (numeric value between 1 and 4) in 'Upsample Scale' Slider component		
-				conditioningScale, // number (numeric value between 0.5 and 1.5) in 'Conditioning Scale' Slider component		
-				classifierFreeGuidance, // number (numeric value between 0.1 and 10.0) in 'Classier-free Guidance' Slider component		
-				seed || generateSeed(), // number (numeric value between -1 and 2147483647) in 'Seed' Slider component
-      ],
-    }),
-    cache: "no-store",
-    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-    // next: { revalidate: 1 }
-  })
-
-  const { data } = await res.json()
- 
-  // Recommendation: handle errors
-  if (res.status !== 200 || !Array.isArray(data)) {
-    // This will activate the closest `error.js` Error Boundary
-    throw new Error(`Failed to fetch data (status: ${res.status})`)
-  }
-  // console.log("data:", data.slice(0, 50))
-
-  const base64Content = (data?.[0] || "") as string
-
-  if (!base64Content) {
-    throw new Error(`invalid response (no content)`)
-  }
- 
-  // console.log("upscaling base64Content:", addBase64HeaderToPng(base64Content).slice(0, 50))
-
-  return addBase64HeaderToPng(base64Content)
-}

src/app/server/utils/alphaToWhite.ts

@@ -1,34 +0,0 @@
-import Jimp from 'jimp';
-
-/**
- * Convert a PNG with an alpha channel to a PNG with a white background
- * 
- * this also makes sure the image is 1024x1024, as the segmentation algorithm is finicky
- * and will fail if this is not respected
- * @param dataUri 
- * @returns 
- */
-export async function alphaToWhite(dataUri: string): Promise<string> {
-
-
-  // strip off the "data:image/png;base64," part
-  const base64Data = dataUri.replace(/^data:image\/\w+;base64,/, "");
-  if (!base64Data) {
-    throw new Error(`invalid image, cannot convert from alpha to white background`)
-  }
-
-  // convert base64 to buffer
-  const imageData = Buffer.from(base64Data, 'base64');
-
-  // read the image using Jimp
-  let img = await Jimp.read(imageData);
-
-  img = img.background(0xFFFFFFFF).resize(1024, 1024);
-  
-  return new Promise((resolve, reject) => {
-    img.getBuffer(Jimp.MIME_PNG, (err, buffer) => {
-      if (err) reject(err);
-      else resolve(`data:${Jimp.MIME_PNG};base64,${buffer.toString('base64')}`);
-    });
-  });
-}

src/app/server/utils/segmentToInpaintingMask.ts

@@ -1,35 +0,0 @@
-"use server"
-
-import Jimp from 'jimp';
-
-/**
- * Converts a segment mask (colored pixels over white)
- * into an inpainting mask (black pixels over white)
- * 
- * @param pngBase64 
- * @returns 
- */
-export async function segmentToInpaintingMask(pngBase64: string) {
-    const black = 0x00000000;
-    const white = 0xFFFFFFFF;
-
-    // strip off the "data:image/png;base64," part
-    const base64Data = pngBase64.replace(/^data:image\/\w+;base64,/, "");
-
-    // convert base64 to buffer
-    const imageData = Buffer.from(base64Data, 'base64');
-
-    // read the image using Jimp
-    const image = await Jimp.read(imageData);
-
-    image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y, idx) => {
-        const currentColor = image.getPixelColor(x, y);
-        if (currentColor !== white) {
-            image.bitmap.data[idx] = black;
-        }
-    });
-
-    // get base64 data
-    const base64Image = await image.getBase64Async(Jimp.MIME_PNG);
-    return "data:image/png;base64," + base64Image.split(",")[1];
-}

src/app/server/utils/segmentsToInpaintingMasks.ts

@@ -1,78 +0,0 @@
-import Jimp from "jimp"
-
-import { SemanticLayer, SemanticLayers } from "@/lib/config"
-
-import { SegmentationResults } from "../segment"
-
-function getEuclideanDistance(color1: number[], color2: number[]): number {
-  return Math.sqrt(
-    Math.pow(color1[0] - color2[0], 2) +
-    Math.pow(color1[1] - color2[1], 2) +
-    Math.pow(color1[2] - color2[2], 2)
-  );
-}
-
-export async function segmentsToInpaintingMasks(segmentationResults: SegmentationResults): Promise<SemanticLayers> {
-  const image = await Jimp.read(Buffer.from(segmentationResults.bitmap.replace(/^data:image\/\w+;base64,/, ""), 'base64'));
-
-  const resultImages: Partial<Record<SemanticLayer, Jimp>> = {}
-    // Convert all result images to base64 strings
-    const base64Images: SemanticLayers = {}
-
-  for (let layer in segmentationResults.data) {
-    resultImages[layer as SemanticLayer] = new Jimp(image)
-    base64Images[layer as SemanticLayer] = ""
-  }
-
-  // Iterate through each pixel in the image
-  image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y, idx) => {
-    // Get the color of the current pixel
-    const color = Jimp.intToRGBA(image.getPixelColor(x, y));
-    const currentColor = [color.r / 255, color.g / 255, color.b / 255];
-
-    // Determine which category the color belongs to
-    let minDistance = Infinity;
-    let closestLayer: SemanticLayer | null = null;
-
-    for (let layer in segmentationResults.data) {
-      const layerColor = segmentationResults.data[layer as SemanticLayer]!.color;
-      const distance = getEuclideanDistance(currentColor, layerColor);
-
-      if(distance < minDistance) {
-        minDistance = distance;
-        closestLayer = layer as SemanticLayer;
-      }
-    };
-
-    if (!closestLayer) {
-      return;
-    }
-
-    // Set the color of the pixel in the corresponding result image to black, and white in others
-    for (let layer in resultImages) {
-      // used to guarantee the !.bitmap
-      if (!resultImages[layer as SemanticLayer]?.bitmap) {
-        continue
-      }
-
-      for (let i = 0; i < 4; i++) {
-        if (layer === closestLayer) {
-          if(i < 3)
-            resultImages[layer as SemanticLayer]!.bitmap.data[idx + i] = 0x00; // set rgb channels to black
-          else
-            resultImages[layer as SemanticLayer]!.bitmap.data[idx + i] = 0xFF; // set alpha channel to maximum
-        } else {
-          resultImages[layer as SemanticLayer]!.bitmap.data[idx + i] = 0xFF;   // set rgba channels to white
-        }
-      }
-    }
-  });
-
-  // Convert all result images to base64 strings
-  for (let layer in resultImages) {
-    const base64Image = await resultImages[layer as SemanticLayer]!.getBase64Async(Jimp.MIME_PNG);
-    base64Images[layer as SemanticLayer] = "data:image/png;base64," + base64Image.split(",")[1];
-  }
-
-  return base64Images;
-}

src/components/form/input-field.tsx

@@ -2,7 +2,7 @@ import { ComponentProps } from "react";
 
 import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
-import { cn } from "@/lib/utils";
+import { cn } from "@/lib/utils/cn";
 
 export function InputField({
   label,

src/components/form/select-field.tsx

@@ -2,7 +2,7 @@ import { ComponentProps } from "react";
 
 import { Select } from "@/components/ui/select";
 import { Label } from "@/components/ui/label";
-import { cn } from "@/lib/utils";
+import { cn } from "@/lib/utils/cn";
 
 export function SelectField({
   label,

src/components/form/slider-field.tsx

@@ -2,7 +2,7 @@ import { ComponentProps } from "react";
 
 import { Slider } from "@/components/ui/slider";
 import { Label } from "@/components/ui/label";
-import { cn } from "@/lib/utils";
+import { cn } from "@/lib/utils/cn";
 
 export function SliderField({
   label,

src/components/form/textarea-field.tsx

@@ -2,7 +2,7 @@ import { ComponentProps } from "react";
 
 import { Textarea } from "../ui/textarea";
 import { Label } from "@/components/ui/label";
-import { cn } from "@/lib/utils";
+import { cn } from "@/lib/utils/cn";
 
 export function TextareaField({
   label,

src/components/ui/accordion.tsx

@@ -4,7 +4,7 @@ import * as React from "react"
 import * as AccordionPrimitive from "@radix-ui/react-accordion"
 import { ChevronDown } from "lucide-react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Accordion = AccordionPrimitive.Root
 

src/components/ui/alert.tsx

@@ -1,7 +1,7 @@
 import * as React from "react"
 import { cva, type VariantProps } from "class-variance-authority"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const alertVariants = cva(
   "relative w-full rounded-lg border border-stone-200 p-4 [&:has(svg)]:pl-11 [&>svg+div]:translate-y-[-3px] [&>svg]:absolute [&>svg]:left-4 [&>svg]:top-4 [&>svg]:text-stone-950 dark:border-stone-800 dark:[&>svg]:text-stone-50",

src/components/ui/avatar.tsx

@@ -3,7 +3,7 @@
 import * as React from "react"
 import * as AvatarPrimitive from "@radix-ui/react-avatar"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Avatar = React.forwardRef<
   React.ElementRef<typeof AvatarPrimitive.Root>,

src/components/ui/badge.tsx

@@ -1,7 +1,7 @@
 import * as React from "react"
 import { cva, type VariantProps } from "class-variance-authority"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const badgeVariants = cva(
   "inline-flex items-center rounded-full border border-stone-200 px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-stone-400 focus:ring-offset-2 dark:border-stone-800 dark:focus:ring-stone-800",

src/components/ui/button.tsx

@@ -2,7 +2,7 @@ import * as React from "react"
 import { Slot } from "@radix-ui/react-slot"
 import { cva, type VariantProps } from "class-variance-authority"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const buttonVariants = cva(
   "inline-flex items-center justify-center rounded-md text-sm font-medium ring-offset-white transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-stone-400 focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 dark:ring-offset-stone-950 dark:focus-visible:ring-stone-800",

src/components/ui/card.tsx

@@ -1,6 +1,6 @@
 import * as React from "react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Card = React.forwardRef<
   HTMLDivElement,

src/components/ui/checkbox.tsx

@@ -4,7 +4,7 @@ import * as React from "react"
 import * as CheckboxPrimitive from "@radix-ui/react-checkbox"
 import { Check } from "lucide-react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Checkbox = React.forwardRef<
   React.ElementRef<typeof CheckboxPrimitive.Root>,

src/components/ui/command.tsx

@@ -5,7 +5,7 @@ import { DialogProps } from "@radix-ui/react-dialog"
 import { Command as CommandPrimitive } from "cmdk"
 import { Search } from "lucide-react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 import { Dialog, DialogContent } from "@/components/ui/dialog"
 
 const Command = React.forwardRef<

src/components/ui/dialog.tsx

@@ -4,7 +4,7 @@ import * as React from "react"
 import * as DialogPrimitive from "@radix-ui/react-dialog"
 import { X } from "lucide-react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Dialog = DialogPrimitive.Root
 

src/components/ui/dropdown-menu.tsx

@@ -4,7 +4,7 @@ import * as React from "react"
 import * as DropdownMenuPrimitive from "@radix-ui/react-dropdown-menu"
 import { Check, ChevronRight, Circle } from "lucide-react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const DropdownMenu = DropdownMenuPrimitive.Root
 

src/components/ui/input.tsx

@@ -1,6 +1,6 @@
 import * as React from "react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 export interface InputProps
   extends React.InputHTMLAttributes<HTMLInputElement> {}

src/components/ui/label.tsx

@@ -4,7 +4,7 @@ import * as React from "react"
 import * as LabelPrimitive from "@radix-ui/react-label"
 import { cva, type VariantProps } from "class-variance-authority"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const labelVariants = cva(
   "text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"

src/components/ui/menubar.tsx

@@ -4,7 +4,7 @@ import * as React from "react"
 import * as MenubarPrimitive from "@radix-ui/react-menubar"
 import { Check, ChevronRight, Circle } from "lucide-react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const MenubarMenu = MenubarPrimitive.Menu
 

src/components/ui/popover.tsx

@@ -3,7 +3,7 @@
 import * as React from "react"
 import * as PopoverPrimitive from "@radix-ui/react-popover"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Popover = PopoverPrimitive.Root
 

src/components/ui/select.tsx

@@ -4,7 +4,7 @@ import * as React from "react"
 import * as SelectPrimitive from "@radix-ui/react-select"
 import { Check, ChevronDown } from "lucide-react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Select = SelectPrimitive.Root
 

src/components/ui/separator.tsx

@@ -3,7 +3,7 @@
 import * as React from "react"
 import * as SeparatorPrimitive from "@radix-ui/react-separator"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Separator = React.forwardRef<
   React.ElementRef<typeof SeparatorPrimitive.Root>,

src/components/ui/slider.tsx

@@ -3,7 +3,7 @@
 import * as React from "react"
 import * as SliderPrimitive from "@radix-ui/react-slider"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Slider = React.forwardRef<
   React.ElementRef<typeof SliderPrimitive.Root>,

src/components/ui/switch.tsx

@@ -3,7 +3,7 @@
 import * as React from "react"
 import * as SwitchPrimitives from "@radix-ui/react-switch"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Switch = React.forwardRef<
   React.ElementRef<typeof SwitchPrimitives.Root>,

src/components/ui/table.tsx

@@ -1,6 +1,6 @@
 import * as React from "react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Table = React.forwardRef<
   HTMLTableElement,

src/components/ui/tabs.tsx

@@ -3,7 +3,7 @@
 import * as React from "react"
 import * as TabsPrimitive from "@radix-ui/react-tabs"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const Tabs = TabsPrimitive.Root
 

src/components/ui/textarea.tsx

@@ -1,6 +1,6 @@
 import * as React from "react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 export interface TextareaProps
   extends React.TextareaHTMLAttributes<HTMLTextAreaElement> {}

src/components/ui/toast.tsx

@@ -3,7 +3,7 @@ import * as ToastPrimitives from "@radix-ui/react-toast"
 import { cva, type VariantProps } from "class-variance-authority"
 import { X } from "lucide-react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const ToastProvider = ToastPrimitives.Provider
 

src/components/ui/tooltip.tsx

@@ -3,7 +3,7 @@
 import * as React from "react"
 import * as TooltipPrimitive from "@radix-ui/react-tooltip"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const TooltipProvider = TooltipPrimitive.Provider
 

src/components/ui/vertical-slider.tsx

@@ -3,7 +3,7 @@
 import * as React from "react"
 import * as SliderPrimitive from "@radix-ui/react-slider"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils/cn"
 
 const VerticalSlider = React.forwardRef<
   React.ElementRef<typeof SliderPrimitive.Root>,

src/lib/base64/blobToDataUri.ts

@@ -0,0 +1,21 @@
+export async function blobToDataUri(blob: Blob, defaultContentType = ""): Promise<string> {
+  if (typeof window === "undefined") {
+    const arrayBuffer = await blob.arrayBuffer()
+    let buffer = Buffer.from(arrayBuffer)
+    return "data:" + (defaultContentType || blob.type) + ';base64,' + buffer.toString('base64');
+  } else {
+    return new Promise<string>((resolve, reject) => {
+      const reader = new FileReader()
+      reader.onload = _e => {
+        let dataUri = `${reader.result as string || ""}`
+        if (defaultContentType) {
+          dataUri = dataUri.replace("application/octet-stream", defaultContentType)
+        }
+        resolve(dataUri)
+      }
+      reader.onerror = _e => reject(reader.error)
+      reader.onabort = _e => reject(new Error("Read aborted"))
+      reader.readAsDataURL(blob)
+    });
+  }
+}

src/lib/base64/dataUriToBlob.ts

@@ -0,0 +1,15 @@
+
+export function dataUriToBlob(dataURI = "", defaultContentType = ""): Blob {
+  dataURI = dataURI.replace(/^data:/, '');
+
+  const type = dataURI.match(/(?:image|application|video|audio|text)\/[^;]+/)?.[0] || defaultContentType;
+  const base64 = dataURI.replace(/^[^,]+,/, '');
+  const arrayBuffer = new ArrayBuffer(base64.length);
+  const typedArray = new Uint8Array(arrayBuffer);
+
+  for (let i = 0; i < base64.length; i++) {
+    typedArray[i] = base64.charCodeAt(i);
+  }
+
+  return new Blob([arrayBuffer], { type });
+}

src/lib/clap/clap-specification-draft.md

@@ -0,0 +1,162 @@
+# CLAP Format Specification
+
+- Status: DRAFT
+- Document revision: 0.0.1
+- Last updated: Feb 6th, 2024
+- Author(s): Julian BILCKE (@flngr)
+
+## BEFORE YOU READ
+
+The CLAP format spec is experimental and not finished yet!
+There might be inconsistencies, unnecessary redundancies or blatant omissions.
+
+## What are CLAP files?
+
+The CLAP format (.clap) is a file format designed for AI video projects.
+
+It preserves prompts and assets into the same container, making it easier to share an AI video project between different people or applications.
+
+## Structure
+
+A CLAP is an array of objects serialized into a YAML text string, then finally compressed using gzip to a binary file.
+
+The file extension is `.clap`
+The mime type is `application/x-yaml`
+
+There can be 5 different types of objects:
+
+- one HEADER
+- one METADATA
+- zero, one or more MODEL(s)
+- zero, one or more SCENE(s)
+- zero, one or more SEGMENT(s)
+
+This can be represented in javascript like this:
+
+```javascript
+[
+  clapHeader, // one metadata object
+  clapMeta, // one metadata object
+  ...clapModels, // optional array of models
+  ...clapScenes, // optional array of scenes
+  ...clapSegments // optional array of segments
+]
+```
+
+## Header
+
+The HEADER provides information about how to decode a CLAP.
+
+Knowing in advance the number of models, scenes and segments helps the decoder parsing the information,
+and in some implementation, help with debugging, logging, and provisioning memory usage.
+
+However in the future, it is possible that a different scheme is used, in order to support streaming.
+
+Either by recognizing the shape of each object (fields), or by using a specific field eg. a `_type`.
+
+```typescript
+{
+  // used to know which format version is used.
+  // CLAP is still in development and the format is not fully specified yet,
+  // during the period most .clap file will have the "clap-0" format
+  format: "clap-0"
+  
+  numberOfModels: number // integer
+  numberOfScenes: number // integer
+  numberOfSegments: number // integer
+}
+```
+
+## Metadata
+
+```typescript
+{
+  id: string // "<a valid UUID V4>"
+  title: string // "project title"
+  description: string // "project description"
+  licence: string // "information about licensing"
+
+  // this provides information about the image ratio
+  // this might be removed in the final spec, as this
+  // can be re-computed from width and height
+  orientation: "landscape" | "vertical" | "square"
+
+  // the expected duration of the project
+  durationInMs: number
+
+  // the suggested width and height of the video
+  // note that this is just an indicator,
+  // and might be superseeded by the application reading the .clap file
+  width: number // integer between 256 and 8192 (value in pixels)
+  height: number // integer between 256 and 8192 (value in pixels)
+
+  // name of the suggested video model to use
+  // note that this is just an indicator,
+  // and might be superseeded by the application reading the .clap file
+  defaultVideoModel: string
+
+  // additional prompt to use in the video generation
+  // this helps adding some magic touch and flair to the videos,
+  // but perhaps the field should be renamed
+  extraPositivePrompt: string
+
+  // the screenplay (script) of the video
+  screenplay: string
+
+  // whether to loop the content by default or not
+  isLoop: boolean
+  
+  // helper to indicate whether the .clap might contain interactive elements
+  isInteractive: boolean
+}
+```
+
+## Models
+
+Before talking about models, first we should describe the concept of entity:
+
+in a story, an entity is something (person, place, vehicle, animal, robot, alien, object) with a name, a description of the appearance, an age, mileage or quality, an origin, and so on.
+
+An example could be "a giant magical school bus, with appearance of a cat with wheels, and which talks"
+
+The CLAP model would be an instance (an interpretation) of this entity, where we would assign it an identity:
+- a name and age
+- a visual style (a photo of the magic school bus cat)
+- a voice style
+- and maybe other things eg. an origin or background story
+
+As you can see, it can be difficult to create clearly separated categories, like "vehicule", "character", or "location"
+(the magical cat bus could turn into a location in some scene, a speaking character in another etc)
+
+This is why there is a common schema for all models:
+
+```typescript
+{
+  id: string
+  category: ClapSegmentCategory
+  triggerName: string
+  label: string
+  description: string
+  author: string
+  thumbnailUrl: string
+  seed: number
+
+  assetSourceType: ClapAssetSource
+  assetUrl: string
+  
+  age: number
+  gender: ClapModelGender
+  region: ClapModelRegion
+  appearance: ClapModelAppearance
+  voiceVendor: ClapVoiceVendor
+  voiceId: string
+}
+```
+
+## Atomic types
+
+...
+
+## TO BE CONTINUED
+
+(you can read "./types.ts" for more information)

src/lib/clap/clapToDataUri.ts

@@ -0,0 +1,11 @@
+
+import { blobToDataUri } from "@/lib/base64/blobToDataUri"
+
+import { serializeClap } from "./serializeClap"
+import { ClapProject } from "./types"
+
+export async function clapToDataUri(clap: ClapProject): Promise<string> {
+  const archive = await serializeClap(clap)
+  const dataUri = await blobToDataUri(archive, "application/x-gzip")
+  return dataUri
+}