up

.env

@@ -25,6 +25,9 @@ VIDEO_HOTSHOT_XL_API_GRADIO="https://jbilcke-hf-hotshot-xl-server-1.hf.space"
 VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL="cloneofsimo/hotshot-xl-lora-controlnet"
 VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL_VERSION="75e26ffd033a59a78954a3d675632f47f7f8470402aec51c255b9f9b7b62568b"
 
+INTERPOLATION_API_REPLICATE_MODEL="zsxkib/st-mfnet"
+INTERPOLATION_API_REPLICATE_MODEL_VERSION="faa7693430b0a4ac95d1b8e25165673c1d7a7263537a7c4bb9be82a3e2d130fb"
+
 # ----------- CENSORSHIP -------
 ENABLE_CENSORSHIP=""
 FINGERPRINT_KEY=""

src/app/interface/generate/index.tsx

@@ -15,6 +15,7 @@ import { getSDXLModels } from "@/app/server/actions/models"
 import { HotshotImageInferenceSize, Post, QualityLevel, QualityOption, SDXLModel } from "@/types"
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
 import { TooltipProvider } from "@radix-ui/react-tooltip"
+import { interpolate } from "@/app/server/actions/interpolate"
 
 const qualityOptions = [
   {
@@ -27,6 +28,8 @@ const qualityOptions = [
   }
 ] as QualityOption[]
 
+type Stage = "generate" | "interpolate" | "finished"
+
 export function Generate() {
   const router = useRouter()
   const pathname = usePathname()
@@ -49,12 +52,14 @@ export function Generate() {
 
   const [communityRoll, setCommunityRoll] = useState<Post[]>([])
 
+  const [stage, setStage] = useState<Stage>("generate")
+  
   const [qualityLevel, setQualityLevel] = useState<QualityLevel>("low")
   
   const { progressPercent, remainingTimeInSec } = useCountdown({
     isActive: isLocked,
     timerId: runs, // everytime we change this, the timer will reset
-    durationInSec: 80, // it usually takes 40 seconds, but there might be lag
+    durationInSec: /*stage === "interpolate" ? 30 :*/ 80, // it usually takes 40 seconds, but there might be lag
     onEnd: () => {}
   })
   
@@ -83,8 +88,9 @@ export function Generate() {
     if (!promptDraft) { return }
 
     setShowModels(false)
-    setRuns(runs + 1)
+    setRuns(runsRef.current + 1)
     setLocked(true)
+    setStage("generate")
 
     scrollRef.current?.scroll({
       top: 0,
@@ -118,44 +124,73 @@ export function Generate() {
         size
       }
 
-      let newAssetUrl = ""
+      let rawAssetUrl = ""
       try {
         // console.log("starting transition, calling generateAnimation")
-        newAssetUrl = await generateAnimation(params)
-        setAssetUrl(newAssetUrl)
+        rawAssetUrl = await generateAnimation(params)
+
+        if (!rawAssetUrl) {
+          throw new Error("invalid asset url")
+        }
+
+        setAssetUrl(rawAssetUrl)
 
       } catch (err) {
         console.log("generation failed! probably just a Gradio failure, so let's just run the round robin again!")
         
         try {
-          newAssetUrl = await generateAnimation(params)
-          setAssetUrl(newAssetUrl)
+          rawAssetUrl = await generateAnimation(params)
         } catch (err) {
           console.error(`generation failed again! ${err}`)
         }
-      } finally {
+      } 
+
+      if (!rawAssetUrl) {
+        console.log("failed to generate the video, aborting")
         setLocked(false)
+        return
+      }
+      
+      setAssetUrl(rawAssetUrl)
 
-        if (newAssetUrl) {
-          try {
-            const post = await postToCommunity({
-              prompt: promptDraft,
-              model: huggingFaceLora,
-              assetUrl: newAssetUrl,
-            })
-            console.log("successfully submitted to the community!", post)
-
-            // now you got a read/write object
-            const current = new URLSearchParams(Array.from(searchParams.entries()))
-            current.set("postId", post.postId.trim())
-            current.set("prompt", post.prompt.trim())
-            current.set("model", post.model.trim())
-            const search = current.toString()
-            router.push(`${pathname}${search ? `?${search}` : ""}`)
-          } catch (err) {
-            console.error(`not a blocker, but we failed to post to the community (reason: ${err})`)
-          }
+
+      let assetUrl = rawAssetUrl
+        
+      setStage("interpolate")
+      setRuns(runsRef.current + 1)
+
+      try {
+        assetUrl = await interpolate(rawAssetUrl)
+
+        if (!assetUrl) {
+          throw new Error("invalid interpolated asset url")
         }
+
+        setAssetUrl(assetUrl)
+      } catch (err) {
+        console.log(`failed to interpolate the video, but this is not a blocker: ${err}`)
+      }
+
+      setLocked(false)
+      setStage("generate")
+        
+      try {
+        const post = await postToCommunity({
+          prompt: promptDraft,
+          model: huggingFaceLora,
+          assetUrl,
+        })
+        console.log("successfully submitted to the community!", post)
+
+        // now you got a read/write object
+        const current = new URLSearchParams(Array.from(searchParams.entries()))
+        current.set("postId", post.postId.trim())
+        current.set("prompt", post.prompt.trim())
+        current.set("model", post.model.trim())
+        const search = current.toString()
+        router.push(`${pathname}${search ? `?${search}` : ""}`)
+      } catch (err) {
+        console.error(`not a blocker, but we failed to post to the community (reason: ${err})`)
       }
     })
   }
@@ -323,7 +358,7 @@ export function Generate() {
                 `space-y-3 md:space-y-6`,
                 `items-center`,
               )}>
-                {assetUrl.startsWith("data:video/mp4")
+                {assetUrl.startsWith("data:video/mp4") || assetUrl.endsWith(".mp4")
                  ? <video
                     muted
                     autoPlay
@@ -420,7 +455,10 @@ export function Generate() {
                   disabled={isLocked}
                   onClick={handleSubmit}
                   >
-                  {isLocked ? `Loading..` : "Generate"}
+                  {isLocked
+                    ? (stage === "generate" ? `Generating..` : `Smoothing..`)
+                    : "Generate"
+                  }
                 </animated.button>
               </div>
             </div>

src/app/server/actions/interpolate.ts

@@ -0,0 +1,81 @@
+"use server"
+
+import Replicate from "replicate"
+
+import { sleep } from "@/lib/sleep"
+
+const replicateToken = `${process.env.AUTH_REPLICATE_API_TOKEN || ""}`
+const replicateModel = `${process.env.INTERPOLATION_API_REPLICATE_MODEL || ""}`
+const replicateModelVersion = `${process.env.INTERPOLATION_API_REPLICATE_MODEL_VERSION || ""}`
+
+export async function interpolate(input: string): Promise<string> {
+  if (!replicateToken) {
+    throw new Error(`you need to configure your AUTH_REPLICATE_API_TOKEN in order to use interpolation`)
+  }
+  if (!replicateModel) {
+    throw new Error(`you need to configure your INTERPOLATION_API_REPLICATE_MODEL in order to use interpolation`)
+  }
+
+  if (!replicateModelVersion) {
+    throw new Error(`you need to configure your INTERPOLATION_API_REPLICATE_MODEL_VERSION in order to use interpolation`)
+  }
+  const replicate = new Replicate({ auth: replicateToken })
+
+  const prediction = await replicate.predictions.create({
+    version: replicateModelVersion,
+    input: {
+      mp4: input,
+      framerate_multiplier: 8,
+      keep_original_duration: true,
+    }
+  })
+
+  let res: Response
+  let pollingCount = 0
+  do {
+    // This is normally a fast model, so let's check every 4 seconds
+    await sleep(4000)
+
+    res = await fetch(`https://api.replicate.com/v1/predictions/${prediction.id}`, {
+      method: "GET",
+      headers: {
+        Authorization: `Token ${replicateToken}`,
+      },
+      cache: 'no-store',
+    })
+
+    // console.log("res:", res)
+
+    /*
+    try {
+      const text = await res.text()
+      console.log("res.text:", text)
+    } catch (err) {
+      console.error("res.text() error:", err)
+    }
+    */
+
+    if (res.status === 200) {
+      try {
+        const response = (await res.json()) as any
+        // console.log("response:", response)
+        const error = `${response?.error || ""}`
+        if (error) {
+          throw new Error(error)
+        }
+        if (response.status === "succeeded") {
+          return response.output[1]
+        }
+      } catch (err) {
+        console.error("res.json() error:", err)
+      }
+    }
+
+    pollingCount++
+
+    // To prevent indefinite polling, we can stop after a certain number
+    if (pollingCount >= 20) {
+      throw new Error('Request time out.')
+    }
+  } while (true)
+}