update the API client

next.config.js

@@ -4,7 +4,8 @@ const nextConfig = {
 
   experimental: {
     serverActions: {
-      bodySizeLimit: '16mb'
+      // a clap file can be quite large - but that's OK
+      bodySizeLimit: '32mb'
     }
   }
 }

package-lock.json

@@ -9,7 +9,7 @@
       "version": "0.0.0",
       "dependencies": {
         "@aitube/clap": "0.0.10",
-        "@aitube/client": "0.0.12",
+        "@aitube/client": "0.0.15",
         "@radix-ui/react-accordion": "^1.1.2",
         "@radix-ui/react-avatar": "^1.0.4",
         "@radix-ui/react-checkbox": "^1.0.4",
@@ -77,9 +77,12 @@
       }
     },
     "node_modules/@aitube/client": {
-      "version": "0.0.12",
-      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.12.tgz",
-      "integrity": "sha512-b/QFTtAKwr7H5dMSco+iXhwJRpPw/sT487EGpNjDbuQamIJ3FqdlVMTC/c5jdX8meFp+m35n/dY58Iy39Lle5A==",
+      "version": "0.0.15",
+      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.15.tgz",
+      "integrity": "sha512-lGmdsBqjNVStBxZSH+Iig/nOyPdSpqpqU6M0OvOBMTwR4rohSvIQ7TnFJGvoc4WEFciNoCc6Vg6Q5W99ovG+fg==",
+      "dependencies": {
+        "query-string": "^9.0.0"
+      },
       "peerDependencies": {
         "@aitube/clap": "0.0.10"
       }
@@ -196,9 +199,9 @@
       }
     },
     "node_modules/@floating-ui/dom": {
-      "version": "1.6.4",
-      "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.4.tgz",
-      "integrity": "sha512-0G8R+zOvQsAG1pg2Q99P21jiqxqGBW1iRe/iXHsBRBxnpXKFI8QwbB4x5KmYLggNO5m34IQgOIu9SCRfR/WWiQ==",
+      "version": "1.6.5",
+      "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.5.tgz",
+      "integrity": "sha512-Nsdud2X65Dz+1RHjAIP0t8z5e2ff/IRbei6BqFrl1urT8sDVzM1HMQ+R0XcU5ceRfyO3I6ayeqIfh+6Wb8LGTw==",
       "dependencies": {
         "@floating-ui/core": "^1.0.0",
         "@floating-ui/utils": "^0.2.0"
@@ -3125,9 +3128,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001615",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001615.tgz",
-      "integrity": "sha512-1IpazM5G3r38meiae0bHRnPhz+CBQ3ZLqbQMtrg+AsTPKAXgW38JNsXkyZ+v8waCsDmPq87lmfun5Q2AGysNEQ==",
+      "version": "1.0.30001616",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001616.tgz",
+      "integrity": "sha512-RHVYKov7IcdNjVHJFNY/78RdG4oGVjbayxv8u5IO74Wv7Hlq4PnJE6mo/OjFijjVFNy5ijnCt6H3IIo4t+wfEw==",
       "funding": [
         {
           "type": "opencollective",
@@ -5585,9 +5588,9 @@
       }
     },
     "node_modules/minipass": {
-      "version": "7.0.4",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
-      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.0.tgz",
+      "integrity": "sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==",
       "engines": {
         "node": ">=16 || 14 >=14.17"
       }

package.json

@@ -10,7 +10,7 @@
   },
   "dependencies": {
     "@aitube/clap": "0.0.10",
-    "@aitube/client": "0.0.12",
+    "@aitube/client": "0.0.15",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-avatar": "^1.0.4",
     "@radix-ui/react-checkbox": "^1.0.4",

src/app/main.tsx

@@ -1,7 +1,7 @@
 "use client"
 
 import React, { useEffect, useRef, useTransition } from 'react'
-import { IoMdPhoneLandscape, IoMdPhonePortrait } from 'react-icons/io'
+import { IoMdPhonePortrait } from 'react-icons/io'
 import { ClapProject } from '@aitube/clap'
 import Image from 'next/image'
 import { useFilePicker } from 'use-file-picker'
@@ -15,6 +15,7 @@ import { TextareaField } from '@/components/form/textarea-field'
 import { cn } from '@/lib/utils/cn'
 
 import { createClap } from './server/aitube/createClap'
+import { editClapEntities } from './server/aitube/editClapEntities'
 import { editClapDialogues } from './server/aitube/editClapDialogues'
 import { editClapStoryboards } from './server/aitube/editClapStoryboards'
 import { exportClapToVideo } from './server/aitube/exportClapToVideo'
@@ -34,9 +35,12 @@ export function Main() {
   const promptDraft = useRef("")
   promptDraft.current = storyPromptDraft
   const storyPrompt = useStore(s => s.storyPrompt)
+  const mainCharacterImage = useStore(s => s.mainCharacterImage)
+  const mainCharacterVoice = useStore(s => s.mainCharacterVoice)
   const orientation = useStore(s => s.orientation)
   const status = useStore(s => s.status)
   const storyGenerationStatus = useStore(s => s.storyGenerationStatus)
+  const assetGenerationStatus = useStore(s => s.assetGenerationStatus)
   const voiceGenerationStatus = useStore(s => s.voiceGenerationStatus)
   const imageGenerationStatus = useStore(s => s.imageGenerationStatus)
   const videoGenerationStatus = useStore(s => s.videoGenerationStatus)
@@ -45,11 +49,14 @@ export function Main() {
   const currentVideoOrientation = useStore(s => s.currentVideoOrientation)
   const setStoryPromptDraft = useStore(s => s.setStoryPromptDraft)
   const setStoryPrompt = useStore(s => s.setStoryPrompt)
+  const setMainCharacterImage = useStore(s => s.setMainCharacterImage)
+  const setMainCharacterVoice = useStore(s => s.setMainCharacterVoice)
   const setStatus = useStore(s => s.setStatus)
   const toggleOrientation = useStore(s => s.toggleOrientation)
   const error = useStore(s => s.error)
   const setError = useStore(s => s.setError)
   const setStoryGenerationStatus = useStore(s => s.setStoryGenerationStatus)
+  const setAssetGenerationStatus = useStore(s => s.setAssetGenerationStatus)
   const setVoiceGenerationStatus = useStore(s => s.setVoiceGenerationStatus)
   const setImageGenerationStatus = useStore(s => s.setImageGenerationStatus)
   const setVideoGenerationStatus = useStore(s => s.setVideoGenerationStatus)
@@ -64,6 +71,7 @@ export function Main() {
 
   const hasPendingTasks =
     storyGenerationStatus === "generating" ||
+    assetGenerationStatus === "generating" ||
     voiceGenerationStatus === "generating" ||
     imageGenerationStatus === "generating" ||
     videoGenerationStatus === "generating"
@@ -99,7 +107,7 @@ export function Main() {
 
       let clap: ClapProject | undefined = undefined
       try {
-        setProgress(5)
+        setProgress(1)
 
         setStatus("generating")
         setStoryGenerationStatus("generating")
@@ -127,12 +135,53 @@ export function Main() {
         return
       }
 
+      console.log("-------- GENERATED STORY --------")
+      console.table(clap.segments, [
+        // 'startTimeInMs',
+        'endTimeInMs',
+        // 'track',
+        'category',
+        'prompt'
+      ])
+
+ 
+      try {
+        setProgress(10)
+        setAssetGenerationStatus("generating")
+        clap = await editClapEntities({ clap })
+
+        if (!clap) { throw new Error(`failed to edit the entities`) }
+
+        console.log(`handleSubmit(): received a clap with entities = `, clap)
+        setCurrentClap(clap)
+        setAssetGenerationStatus("finished")
+      } catch (err) {
+        setAssetGenerationStatus("error")
+        setStatus("error")
+        setError(`${err}`)
+        return
+      }
+      if (!clap) {
+        return
+      }
+      
+
+      /*
+      if (mainCharacterImage) {
+        console.log("handleSubmit(): User specified a main character image")
+        // various strategies here, for instance we can assume that the first character is the main character,
+        // or maybe a more reliable way is to count the number of occurrences.
+        // there is a risk of misgendering, so ideally we should add some kind of UI to do this,
+        // such as a list of characters.
+      }
+      */
+
       // TODO Julian
       console.log("handleSubmit(): TODO Julian: generate images in parallel of the dialogue using Promise.all()")
       // this is not trivial to do btw, since we will have to merge the clap together
       // (this could be a helper function inside @aitube/clap)
       try {
-        setProgress(25)
+        setProgress(40)
         setImageGenerationStatus("generating")
         clap = await editClapStoryboards({ clap })
 
@@ -153,7 +202,7 @@ export function Main() {
 
       
       try {
-        setProgress(50)
+        setProgress(60)
         setVoiceGenerationStatus("generating")
         clap = await editClapDialogues({ clap })
 
@@ -174,7 +223,7 @@ export function Main() {
 
       let assetUrl = ""
       try {
-        setProgress(75)
+        setProgress(80)
         setVideoGenerationStatus("generating")
         assetUrl = await exportClapToVideo({ clap })
 
@@ -328,6 +377,9 @@ export function Main() {
                   
                  
                     {/*
+
+                    TODO: To finish by Julian a bit later
+
                     <div className="
                       flex flex-col
                       
@@ -335,7 +387,18 @@ export function Main() {
                       transition-all duration-200 ease-in-out
                       space-y-2 md:space-y-4
                     ">
-                      put menu here
+                       <Input
+                          type="file"
+                          className=""
+                          onChange={async (e: React.ChangeEvent<HTMLInputElement>) => {
+                            if (e.target.files && e.target.files.length > 0) {
+                              const file = e.target.files[0];
+                              const newImageBase64 = await fileToBase64(file)
+                              setMainCharacterImage(newImageBase64)
+                            }
+                          }}
+                          accept="image/*"
+                        />
                     </div>
                     */}
 
@@ -504,6 +567,7 @@ export function Main() {
                     <p className="text-base text-white/70">{isBusy
                       ? (
                         storyGenerationStatus === "generating" ? "Enhancing the story.."
+                        : assetGenerationStatus === "generating" ? "Creating characters.."
                         : imageGenerationStatus === "generating" ? "Generating storyboards.."
                         : voiceGenerationStatus === "generating" ? "Generating voices.."
                         : videoGenerationStatus === "generating" ? "Assembling final video.."

src/app/server/aitube/editClapEntities.ts

@@ -0,0 +1,23 @@
+"use server"
+
+import { ClapProject } from "@aitube/clap"
+import { editClapEntities as apiEditClapEntities, ClapCompletionMode, ClapEntityPrompt } from "@aitube/client"
+
+import { getToken } from "./getToken"
+
+export async function editClapEntities({
+  clap,
+  entityPrompts = []
+}: {
+  clap: ClapProject
+  entityPrompts?: ClapEntityPrompt[]
+}): Promise<ClapProject> {
+  const newClap: ClapProject = await apiEditClapEntities({
+    clap,
+    entityPrompts,
+    completionMode: ClapCompletionMode.FULL,
+    token: await getToken()
+  })
+
+  return newClap
+}

src/app/store.ts

@@ -20,6 +20,7 @@ export const useStore = create<{
 
   status: GlobalStatus
   storyGenerationStatus: TaskStatus
+  assetGenerationStatus: TaskStatus
   voiceGenerationStatus: TaskStatus
   imageGenerationStatus: TaskStatus
   videoGenerationStatus: TaskStatus
@@ -39,6 +40,7 @@ export const useStore = create<{
   setStoryPrompt: (storyPrompt: string) => void
   setStatus: (status: GlobalStatus) => void
   setStoryGenerationStatus: (storyGenerationStatus: TaskStatus) => void
+  setAssetGenerationStatus: (assetGenerationStatus: TaskStatus) => void
   setVoiceGenerationStatus: (voiceGenerationStatus: TaskStatus) => void
   setImageGenerationStatus: (imageGenerationStatus: TaskStatus) => void
   setVideoGenerationStatus: (videoGenerationStatus: TaskStatus) => void
@@ -59,6 +61,7 @@ export const useStore = create<{
   orientation: VideoOrientation.PORTRAIT,
   status: "idle",
   storyGenerationStatus: "idle",
+  assetGenerationStatus: "idle",
   voiceGenerationStatus: "idle",
   imageGenerationStatus: "idle",
   videoGenerationStatus: "idle",
@@ -91,6 +94,7 @@ export const useStore = create<{
   setStoryPrompt: (storyPrompt: string) => { set({ storyPrompt }) },
   setStatus: (status: GlobalStatus) => { set({ status }) },
   setStoryGenerationStatus: (storyGenerationStatus: TaskStatus) => { set({ storyGenerationStatus }) },
+  setAssetGenerationStatus: (assetGenerationStatus: TaskStatus) => { set({ assetGenerationStatus }) },
   setVoiceGenerationStatus: (voiceGenerationStatus: TaskStatus) => { set({ voiceGenerationStatus }) },
   setImageGenerationStatus: (imageGenerationStatus: TaskStatus) => { set({ imageGenerationStatus }) },
   setVideoGenerationStatus: (videoGenerationStatus: TaskStatus) => { set({ videoGenerationStatus }) },