preparing a system to have multiple workers

package-lock.json

@@ -12,13 +12,15 @@
         "@gradio/client": "^0.1.3",
         "@huggingface/inference": "^2.5.2",
         "@types/express": "^4.17.17",
+        "@types/uuid": "^9.0.2",
         "fluent-ffmpeg": "^2.1.2",
         "fs-extra": "^11.1.1",
         "node-fetch": "^3.3.1",
         "node-media-server": "^2.6.2",
         "puppeteer": "^20.7.4",
         "temp-dir": "^3.0.0",
-        "ts-node": "^10.9.1"
+        "ts-node": "^10.9.1",
+        "uuid": "^9.0.0"
       }
     },
     "node_modules/@babel/code-frame": {
@@ -315,6 +317,11 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/uuid": {
+      "version": "9.0.2",
+      "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.2.tgz",
+      "integrity": "sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ=="
+    },
     "node_modules/@types/yauzl": {
       "version": "2.10.0",
       "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz",
@@ -2361,6 +2368,14 @@
         "node": ">= 0.4.0"
       }
     },
+    "node_modules/uuid": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
+      "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
     "node_modules/v8-compile-cache-lib": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",

package.json

@@ -16,12 +16,14 @@
     "@gradio/client": "^0.1.3",
     "@huggingface/inference": "^2.5.2",
     "@types/express": "^4.17.17",
+    "@types/uuid": "^9.0.2",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^11.1.1",
     "node-fetch": "^3.3.1",
     "node-media-server": "^2.6.2",
     "puppeteer": "^20.7.4",
     "temp-dir": "^3.0.0",
-    "ts-node": "^10.9.1"
+    "ts-node": "^10.9.1",
+    "uuid": "^9.0.0"
   }
 }

src/callZeroscope.mts

@@ -22,10 +22,7 @@ export const callZeroscope = async (prompt: string, options?: {
     nbSteps, // 10, (numeric value between 10 and 50) in 'Number of inference steps' Slider component
   ]) as any
   
-  const { name, orig_name } = rawResponse?.data?.[0]?.[0] as { name: string, orig_name: string }
+  const { name } = rawResponse?.data?.[0]?.[0] as { name: string, orig_name: string }
 
-  return {
-    url: `${videoSpaceApiUrl}/file=${name}`,
-    name: orig_name
-  }
+  return `${videoSpaceApiUrl}/file=${name}`
 }

src/downloadVideo.mts

@@ -3,9 +3,7 @@ import fs from 'node:fs'
 
 import tmpDir from 'temp-dir'
 
-export const downloadVideo = async (remoteUrl: string, options?: { fileName: string; }): Promise<string> => {
-
-  const fileName = options?.fileName || `${Date.now()}.mp4`
+export const downloadVideo = async (remoteUrl: string, fileName: string): Promise<string> => {
 
   const filePath = path.resolve(tmpDir, fileName)
 

src/generateVideo.mts

@@ -1,10 +0,0 @@
-import { callZeroscope } from './callZeroscope.mts'
-import { downloadVideo } from './downloadVideo.mts'
-
-export const generateVideo = async (prompt: string): Promise<string> => {
-  const { url } = await callZeroscope(prompt)
-
-  const rawVideo = await downloadVideo(url)
-
-  return rawVideo
-}

src/index.mts

@@ -1,3 +1,4 @@
+import { v4 as uuid } from 'uuid'
 import { generateVideo } from './generateVideo.mts'
 import { upscaleVideo } from './upscaleVideo.mts'
 import { keepVideo } from './keepVideo.mts'
@@ -5,15 +6,20 @@ import { keepVideo } from './keepVideo.mts'
 import { demoPrompts } from './prompts.mts'
 import { getStats } from './getStats.mts'
 import { enhanceVideo } from './enhanceVideo.mts'
+import { callZeroscope } from './callZeroscope.mts'
+import { downloadVideo } from './downloadVideo.mts'
 
 const main = async () => {
   console.log('Generating ideas..')
   const ideas = demoPrompts
 
-  console.log('Generating videos..')
+  console.log('Generating videos sequences..')
+  const instanceId = process.env.WEBTV_WORKER_INSTANCE_ID || '0'
 
   for (const { input, captions } of ideas) {
-    console.log(`\nVideo to generate: ${input}`)
+    console.log(`\nVideo sequence to generate: ${input}`)
+
+    const sequenceId = uuid()
 
     const silentVideos: string[] = []
 
@@ -21,30 +27,34 @@ const main = async () => {
     const videoDurationInSecs = 3
 
     for (const caption of captions) {
-      console.log(`- generating video.. prompt: ${caption}`)
+      console.log(`- generating shot: ${caption}`)
       try {
-        const rawVideo = await generateVideo(caption)
+        const generatedVideoUrl = await callZeroscope(caption)
+
+        const videoName = `inst_${instanceId}_seq_${sequenceId}_shot_${Date.now()}.mp4`
+
+        console.log(`- downloading ${videoName} from ${generatedVideoUrl}`)
+        await downloadVideo(generatedVideoUrl, videoName)
 
-        console.log(`- downloaded ${rawVideo}`)
+        console.log(`- downloaded ${videoName}`)
 
         console.log('- upscaling video..')
       
-        let upscaledVideo = rawVideo
         try {
-          upscaledVideo = await upscaleVideo(rawVideo, caption)
+          await upscaleVideo(videoName, caption)
         } catch (err) {
           // upscaling is finicky, if it fails we try again
           console.log('- trying again to upscale video..')
-          upscaledVideo = await upscaleVideo(rawVideo, caption)
+          await upscaleVideo(videoName, caption)
         }
 
         console.log('- enhancing video..')
-        const enhancedVideo = await enhanceVideo(upscaledVideo)
+        await enhanceVideo(videoName)
 
         console.log('- saving final video..')
-        await keepVideo(enhancedVideo, process.env.WEBTV_VIDEO_STORAGE_PATH_NEXT)
+        await keepVideo(videoName, process.env.WEBTV_VIDEO_STORAGE_PATH_NEXT)
 
-        silentVideos.push(enhancedVideo)
+        silentVideos.push(videoName)
 
         console.log('- done!')
       } catch (err) {

src/prompts.mts

@@ -1,6 +1,8 @@
 
 export interface Scene {
+  id?: string // TODO: make it mandatory
   input: string
+  audio?: string // TODO: make it mandatory
   captions: string[]
 }
 
@@ -70,7 +72,7 @@ The output must be in JSON (here is the type signature:
 - Input: a movie about a police car chase in Los Angeles (NOT a 3D render!), in term of total timeline film starts at golden hour, then sunset, then night. There is a light rain, the ground is wet. We can see a police car going at speed after a red lambo (to help the AI generator, you must mention police car and the rend lambo in ALL videos shots!). The police car has its lights on, reflecting on the wet ground. The police chase the lambo across all los angeles, including some iconic places. More police cars should join the chase, and even a helicopter at some point. We should see the light of the helicopter ont he ground, like in movies. Everything should be enveloped in a mist, haze, fog, wet ground. The camera should alternate between point of view to give it an intense, fast paced style (drone shots, helicopter shot etc). Please always mention (for each video) the time era and period, clothes of the characters, colors and textures, background, camera positions (note: add the keywords "cinematic", "imax", "movie", "film grain" to help the AI generator)
 */
 
-export const demoPrompts = [
+export const demoPrompts: Scene[] = [
   {
     "input": "Slow-mo footage of various objects falling and exploding or being hit by bullets. Very colorful, dramatic lighting, mesmerizing.",
     "captions": [

src/upscaleVideo.mts

@@ -59,7 +59,9 @@ export async function upscaleVideo(fileName: string, prompt: string) {
   const tmpFileName = `${fileName}_xl`
 
   // console.log('downloading file from space..')
-  await downloadVideo(upscaledFileUrl, { fileName: tmpFileName })
+  console.log(`- downloading ${fileName} from ${upscaledFileUrl}`)
+
+  await downloadVideo(upscaledFileUrl, tmpFileName)
 
   const tmpFilePath = path.join(tmpDir, tmpFileName)
   const filePath = path.join(tmpDir, fileName)

start.sh

@@ -5,7 +5,7 @@ bash ./scripts/init.sh
 
 # ------ background processes ---------
 
-# starts the streaming server
+# starts the streaming server first, otherwise ffmpeg won't find it
 node ./media-server.js &
 
 sleep 1
@@ -23,4 +23,6 @@ bash scripts/stream.sh &
 
 sleep 1
 
-npm run start
+# here we have the possibility of using multiple workers
+# buf if we do that, we will have to split the workload (split the prompts)
+WEBTV_WORKER_INSTANCE_ID=1 npm run start