jbilcke-hf HF staff commited on
Commit
6c5a5b1
1 Parent(s): 2c51dfe

let's add more backends

Browse files
Files changed (4) hide show
  1. .env +4 -0
  2. README.md +26 -0
  3. package-lock.json +120 -120
  4. src/app/server/actions/animation.ts +92 -1
.env CHANGED
@@ -1,4 +1,8 @@
1
 
 
 
 
 
2
  # TODO: those could be
3
  # VIDEO_HOTSHOT_XL_API_OFFICIAL
4
  # VIDEO_HOTSHOT_XL_API_NODE
 
1
 
2
+
3
+
4
+ AUTH_REPLICATE_API_TOKEN="<YOUR SECRET>"
5
+
6
  # TODO: those could be
7
  # VIDEO_HOTSHOT_XL_API_OFFICIAL
8
  # VIDEO_HOTSHOT_XL_API_NODE
README.md CHANGED
@@ -9,3 +9,29 @@ app_port: 3000
9
  ---
10
 
11
  # Hotshot-XL Text-to-GIF
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  ---
10
 
11
  # Hotshot-XL Text-to-GIF
12
+
13
+
14
+ ## Setup
15
+
16
+ If you run the app locally you need to create a `.env.local` file
17
+ (If you deploy to Hugging Face, just set the environment variable from the settings)
18
+
19
+ ### Video rendering engine
20
+
21
+ Note: the app is in heavy development, not all backends are supported
22
+
23
+ Set `VIDEO_ENGINE` to one of:
24
+
25
+ - `VIDEO_ENGINE="VIDEO_HOTSHOT_XL_API_GRADIO"`
26
+ - `VIDEO_ENGINE="VIDEO_HOTSHOT_XL_API_REPLICATE"`
27
+ - `VIDEO_ENGINE="VIDEO_HOTSHOT_XL_API_NODE"` <- not working yet
28
+ - `VIDEO_ENGINE="VIDEO_HOTSHOT_XL_API_OFFICIAL"` <- not working yet
29
+
30
+
31
+ ### Authentication
32
+
33
+ If you intent to use a special provider (eg. Replicate) you need to setup your token
34
+
35
+ - `AUTH_REPLICATE_API_TOKEN="<YOUR SECRET>"`
36
+
37
+
package-lock.json CHANGED
@@ -2324,6 +2324,126 @@
2324
  "node": ">= 10"
2325
  }
2326
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2327
  "node_modules/@nicolo-ribaudo/chokidar-2": {
2328
  "version": "2.1.8-no-fsevents.3",
2329
  "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/chokidar-2/-/chokidar-2-2.1.8-no-fsevents.3.tgz",
@@ -9634,126 +9754,6 @@
9634
  "optional": true
9635
  }
9636
  }
9637
- },
9638
- "node_modules/@next/swc-darwin-x64": {
9639
- "version": "13.4.10",
9640
- "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-13.4.10.tgz",
9641
- "integrity": "sha512-ngXhUBbcZIWZWqNbQSNxQrB9T1V+wgfCzAor2olYuo/YpaL6mUYNUEgeBMhr8qwV0ARSgKaOp35lRvB7EmCRBg==",
9642
- "cpu": [
9643
- "x64"
9644
- ],
9645
- "optional": true,
9646
- "os": [
9647
- "darwin"
9648
- ],
9649
- "engines": {
9650
- "node": ">= 10"
9651
- }
9652
- },
9653
- "node_modules/@next/swc-linux-arm64-gnu": {
9654
- "version": "13.4.10",
9655
- "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-13.4.10.tgz",
9656
- "integrity": "sha512-SjCZZCOmHD4uyM75MVArSAmF5Y+IJSGroPRj2v9/jnBT36SYFTORN8Ag/lhw81W9EeexKY/CUg2e9mdebZOwsg==",
9657
- "cpu": [
9658
- "arm64"
9659
- ],
9660
- "optional": true,
9661
- "os": [
9662
- "linux"
9663
- ],
9664
- "engines": {
9665
- "node": ">= 10"
9666
- }
9667
- },
9668
- "node_modules/@next/swc-linux-arm64-musl": {
9669
- "version": "13.4.10",
9670
- "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-13.4.10.tgz",
9671
- "integrity": "sha512-F+VlcWijX5qteoYIOxNiBbNE8ruaWuRlcYyIRK10CugqI/BIeCDzEDyrHIHY8AWwbkTwe6GRHabMdE688Rqq4Q==",
9672
- "cpu": [
9673
- "arm64"
9674
- ],
9675
- "optional": true,
9676
- "os": [
9677
- "linux"
9678
- ],
9679
- "engines": {
9680
- "node": ">= 10"
9681
- }
9682
- },
9683
- "node_modules/@next/swc-linux-x64-gnu": {
9684
- "version": "13.4.10",
9685
- "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-13.4.10.tgz",
9686
- "integrity": "sha512-WDv1YtAV07nhfy3i1visr5p/tjiH6CeXp4wX78lzP1jI07t4PnHHG1WEDFOduXh3WT4hG6yN82EQBQHDi7hBrQ==",
9687
- "cpu": [
9688
- "x64"
9689
- ],
9690
- "optional": true,
9691
- "os": [
9692
- "linux"
9693
- ],
9694
- "engines": {
9695
- "node": ">= 10"
9696
- }
9697
- },
9698
- "node_modules/@next/swc-linux-x64-musl": {
9699
- "version": "13.4.10",
9700
- "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-13.4.10.tgz",
9701
- "integrity": "sha512-zFkzqc737xr6qoBgDa3AwC7jPQzGLjDlkNmt/ljvQJ/Veri5ECdHjZCUuiTUfVjshNIIpki6FuP0RaQYK9iCRg==",
9702
- "cpu": [
9703
- "x64"
9704
- ],
9705
- "optional": true,
9706
- "os": [
9707
- "linux"
9708
- ],
9709
- "engines": {
9710
- "node": ">= 10"
9711
- }
9712
- },
9713
- "node_modules/@next/swc-win32-arm64-msvc": {
9714
- "version": "13.4.10",
9715
- "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-13.4.10.tgz",
9716
- "integrity": "sha512-IboRS8IWz5mWfnjAdCekkl8s0B7ijpWeDwK2O8CdgZkoCDY0ZQHBSGiJ2KViAG6+BJVfLvcP+a2fh6cdyBr9QQ==",
9717
- "cpu": [
9718
- "arm64"
9719
- ],
9720
- "optional": true,
9721
- "os": [
9722
- "win32"
9723
- ],
9724
- "engines": {
9725
- "node": ">= 10"
9726
- }
9727
- },
9728
- "node_modules/@next/swc-win32-ia32-msvc": {
9729
- "version": "13.4.10",
9730
- "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-13.4.10.tgz",
9731
- "integrity": "sha512-bSA+4j8jY4EEiwD/M2bol4uVEu1lBlgsGdvM+mmBm/BbqofNBfaZ2qwSbwE2OwbAmzNdVJRFRXQZ0dkjopTRaQ==",
9732
- "cpu": [
9733
- "ia32"
9734
- ],
9735
- "optional": true,
9736
- "os": [
9737
- "win32"
9738
- ],
9739
- "engines": {
9740
- "node": ">= 10"
9741
- }
9742
- },
9743
- "node_modules/@next/swc-win32-x64-msvc": {
9744
- "version": "13.4.10",
9745
- "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-13.4.10.tgz",
9746
- "integrity": "sha512-g2+tU63yTWmcVQKDGY0MV1PjjqgZtwM4rB1oVVi/v0brdZAcrcTV+04agKzWtvWroyFz6IqtT0MoZJA7PNyLVw==",
9747
- "cpu": [
9748
- "x64"
9749
- ],
9750
- "optional": true,
9751
- "os": [
9752
- "win32"
9753
- ],
9754
- "engines": {
9755
- "node": ">= 10"
9756
- }
9757
  }
9758
  }
9759
  }
 
2324
  "node": ">= 10"
2325
  }
2326
  },
2327
+ "node_modules/@next/swc-darwin-x64": {
2328
+ "version": "13.4.10",
2329
+ "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-13.4.10.tgz",
2330
+ "integrity": "sha512-ngXhUBbcZIWZWqNbQSNxQrB9T1V+wgfCzAor2olYuo/YpaL6mUYNUEgeBMhr8qwV0ARSgKaOp35lRvB7EmCRBg==",
2331
+ "cpu": [
2332
+ "x64"
2333
+ ],
2334
+ "optional": true,
2335
+ "os": [
2336
+ "darwin"
2337
+ ],
2338
+ "engines": {
2339
+ "node": ">= 10"
2340
+ }
2341
+ },
2342
+ "node_modules/@next/swc-linux-arm64-gnu": {
2343
+ "version": "13.4.10",
2344
+ "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-13.4.10.tgz",
2345
+ "integrity": "sha512-SjCZZCOmHD4uyM75MVArSAmF5Y+IJSGroPRj2v9/jnBT36SYFTORN8Ag/lhw81W9EeexKY/CUg2e9mdebZOwsg==",
2346
+ "cpu": [
2347
+ "arm64"
2348
+ ],
2349
+ "optional": true,
2350
+ "os": [
2351
+ "linux"
2352
+ ],
2353
+ "engines": {
2354
+ "node": ">= 10"
2355
+ }
2356
+ },
2357
+ "node_modules/@next/swc-linux-arm64-musl": {
2358
+ "version": "13.4.10",
2359
+ "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-13.4.10.tgz",
2360
+ "integrity": "sha512-F+VlcWijX5qteoYIOxNiBbNE8ruaWuRlcYyIRK10CugqI/BIeCDzEDyrHIHY8AWwbkTwe6GRHabMdE688Rqq4Q==",
2361
+ "cpu": [
2362
+ "arm64"
2363
+ ],
2364
+ "optional": true,
2365
+ "os": [
2366
+ "linux"
2367
+ ],
2368
+ "engines": {
2369
+ "node": ">= 10"
2370
+ }
2371
+ },
2372
+ "node_modules/@next/swc-linux-x64-gnu": {
2373
+ "version": "13.4.10",
2374
+ "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-13.4.10.tgz",
2375
+ "integrity": "sha512-WDv1YtAV07nhfy3i1visr5p/tjiH6CeXp4wX78lzP1jI07t4PnHHG1WEDFOduXh3WT4hG6yN82EQBQHDi7hBrQ==",
2376
+ "cpu": [
2377
+ "x64"
2378
+ ],
2379
+ "optional": true,
2380
+ "os": [
2381
+ "linux"
2382
+ ],
2383
+ "engines": {
2384
+ "node": ">= 10"
2385
+ }
2386
+ },
2387
+ "node_modules/@next/swc-linux-x64-musl": {
2388
+ "version": "13.4.10",
2389
+ "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-13.4.10.tgz",
2390
+ "integrity": "sha512-zFkzqc737xr6qoBgDa3AwC7jPQzGLjDlkNmt/ljvQJ/Veri5ECdHjZCUuiTUfVjshNIIpki6FuP0RaQYK9iCRg==",
2391
+ "cpu": [
2392
+ "x64"
2393
+ ],
2394
+ "optional": true,
2395
+ "os": [
2396
+ "linux"
2397
+ ],
2398
+ "engines": {
2399
+ "node": ">= 10"
2400
+ }
2401
+ },
2402
+ "node_modules/@next/swc-win32-arm64-msvc": {
2403
+ "version": "13.4.10",
2404
+ "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-13.4.10.tgz",
2405
+ "integrity": "sha512-IboRS8IWz5mWfnjAdCekkl8s0B7ijpWeDwK2O8CdgZkoCDY0ZQHBSGiJ2KViAG6+BJVfLvcP+a2fh6cdyBr9QQ==",
2406
+ "cpu": [
2407
+ "arm64"
2408
+ ],
2409
+ "optional": true,
2410
+ "os": [
2411
+ "win32"
2412
+ ],
2413
+ "engines": {
2414
+ "node": ">= 10"
2415
+ }
2416
+ },
2417
+ "node_modules/@next/swc-win32-ia32-msvc": {
2418
+ "version": "13.4.10",
2419
+ "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-13.4.10.tgz",
2420
+ "integrity": "sha512-bSA+4j8jY4EEiwD/M2bol4uVEu1lBlgsGdvM+mmBm/BbqofNBfaZ2qwSbwE2OwbAmzNdVJRFRXQZ0dkjopTRaQ==",
2421
+ "cpu": [
2422
+ "ia32"
2423
+ ],
2424
+ "optional": true,
2425
+ "os": [
2426
+ "win32"
2427
+ ],
2428
+ "engines": {
2429
+ "node": ">= 10"
2430
+ }
2431
+ },
2432
+ "node_modules/@next/swc-win32-x64-msvc": {
2433
+ "version": "13.4.10",
2434
+ "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-13.4.10.tgz",
2435
+ "integrity": "sha512-g2+tU63yTWmcVQKDGY0MV1PjjqgZtwM4rB1oVVi/v0brdZAcrcTV+04agKzWtvWroyFz6IqtT0MoZJA7PNyLVw==",
2436
+ "cpu": [
2437
+ "x64"
2438
+ ],
2439
+ "optional": true,
2440
+ "os": [
2441
+ "win32"
2442
+ ],
2443
+ "engines": {
2444
+ "node": ">= 10"
2445
+ }
2446
+ },
2447
  "node_modules/@nicolo-ribaudo/chokidar-2": {
2448
  "version": "2.1.8-no-fsevents.3",
2449
  "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/chokidar-2/-/chokidar-2-2.1.8-no-fsevents.3.tgz",
 
9754
  "optional": true
9755
  }
9756
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9757
  }
9758
  }
9759
  }
src/app/server/actions/animation.ts CHANGED
@@ -1,6 +1,10 @@
1
  "use server"
2
 
 
 
 
3
  import { ImageInferenceSize } from "@/types"
 
4
 
5
  const videoEngine = `${process.env.VIDEO_ENGINE || ""}`
6
 
@@ -8,6 +12,10 @@ const officialApi = `${process.env.VIDEO_HOTSHOT_XL_API_OFFICIAL || ""}`
8
  const nodeApi = `${process.env.VIDEO_HOTSHOT_XL_API_NODE || ""}`
9
  const gradioApi = `${process.env.VIDEO_HOTSHOT_XL_API_GRADIO || ""}`
10
 
 
 
 
 
11
  export async function generateAnimation({
12
  prompt,
13
  size,
@@ -21,9 +29,92 @@ export async function generateAnimation({
21
  throw new Error(`prompt is too short!`)
22
  }
23
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  try {
25
 
26
- if (videoEngine === "VIDEO_HOTSHOT_XL_API_NODE") {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  // TODO: support other API to avoid duplicate work?
28
  // (are the other API supporting custom LoRAs?)
29
  const res = await fetch(nodeApi, {
 
1
  "use server"
2
 
3
+ import Replicate from "replicate"
4
+
5
+ import { generateSeed } from "@/lib/generateSeed"
6
  import { ImageInferenceSize } from "@/types"
7
+ import { sleep } from "@/lib/sleep"
8
 
9
  const videoEngine = `${process.env.VIDEO_ENGINE || ""}`
10
 
 
12
  const nodeApi = `${process.env.VIDEO_HOTSHOT_XL_API_NODE || ""}`
13
  const gradioApi = `${process.env.VIDEO_HOTSHOT_XL_API_GRADIO || ""}`
14
 
15
+ const replicateToken = `${process.env.AUTH_REPLICATE_API_TOKEN || ""}`
16
+ const replicateModel = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL || ""}`
17
+ const replicateModelVersion = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL_VERSION || ""}`
18
+
19
  export async function generateAnimation({
20
  prompt,
21
  size,
 
29
  throw new Error(`prompt is too short!`)
30
  }
31
 
32
+ // pimp themy prompt
33
+ prompt = [
34
+ "beautiful",
35
+ prompt,
36
+ "hd"
37
+ ].join(", ")
38
+
39
+ const negativePrompt = [
40
+ "cropped",
41
+ "dark",
42
+ "underexposed",
43
+ "overexposed",
44
+ "watermark",
45
+ "watermarked",
46
+ ]
47
+
48
  try {
49
 
50
+ if (videoEngine === "VIDEO_HOTSHOT_XL_API_REPLICATE") {
51
+ if (!replicateToken) {
52
+ throw new Error(`you need to configure your AUTH_REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
53
+ }
54
+ if (!replicateModel) {
55
+ throw new Error(`you need to configure your RENDERING_REPLICATE_API_MODEL in order to use the REPLICATE rendering engine`)
56
+ }
57
+
58
+ if (!replicateModelVersion) {
59
+ throw new Error(`you need to configure your REPLICATE_API_MODEL_VERSION in order to use the REPLICATE rendering engine`)
60
+ }
61
+ const replicate = new Replicate({ auth: replicateToken })
62
+
63
+ const [width, height] = size.split("x")
64
+
65
+ // console.log("Calling replicate..")
66
+ const seed = generateSeed()
67
+ const prediction = await replicate.predictions.create({
68
+ version: replicateModelVersion,
69
+ input: {
70
+ prompt,
71
+ negative_prompt: negativePrompt,
72
+ hf_lora_url: lora,
73
+ width,
74
+ height,
75
+
76
+ // those are used to create an upsampling or downsampling
77
+ // original_width: width,
78
+ // original_height: height,
79
+ // target_width: width,
80
+ // target_height: height,
81
+
82
+ video_length: 8, // nb frames
83
+ video_duration: 1000, // video duration in ms
84
+ // seed
85
+ }
86
+ })
87
+
88
+ // console.log("prediction:", prediction)
89
+
90
+ // no need to reply straight away as images take time to generate, this isn't instantaneous
91
+ // also our friends at Replicate won't like it if we spam them with requests
92
+ await sleep(5000)
93
+
94
+
95
+ const res = await fetch(`https://api.replicate.com/v1/predictions/${prediction.id}`, {
96
+ method: "GET",
97
+ headers: {
98
+ Authorization: `Token ${replicateToken}`,
99
+ },
100
+ cache: 'no-store',
101
+ // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
102
+ // next: { revalidate: 1 }
103
+ })
104
+
105
+ // Recommendation: handle errors
106
+ if (res.status !== 200) {
107
+ // This will activate the closest `error.js` Error Boundary
108
+ throw new Error('Failed to fetch data')
109
+ }
110
+
111
+ const response = (await res.json()) as any
112
+ const error = `${response?.error || ""}`
113
+ if (error) {
114
+ throw new Error(error)
115
+ }
116
+ return `${response?.output || ""}`
117
+ } else if (videoEngine === "VIDEO_HOTSHOT_XL_API_NODE") {
118
  // TODO: support other API to avoid duplicate work?
119
  // (are the other API supporting custom LoRAs?)
120
  const res = await fetch(nodeApi, {