Spaces:
Runtime error
Runtime error
Duplicate from DGSpitzer/TXT-2-IMG-2-MUSIC-2-VIDEO
Browse filesCo-authored-by: DGSpitzer <DGSpitzer@users.noreply.huggingface.co>
- .gitattributes +33 -0
- README.md +14 -0
- app.py +478 -0
- app_old.py +467 -0
- requirements.txt +5 -0
.gitattributes
ADDED
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
6 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
7 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
8 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
9 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
10 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
11 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
12 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
13 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
14 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
15 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
16 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
17 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
18 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
19 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
20 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
21 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
22 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
23 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
24 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
25 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
26 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
27 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
28 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
29 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
30 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
31 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
32 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
33 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
title: Txt 2 Img 2 Music 2 Video
|
3 |
+
emoji: ☯️🎨🎸🎞
|
4 |
+
colorFrom: yellow
|
5 |
+
colorTo: red
|
6 |
+
sdk: gradio
|
7 |
+
sdk_version: 3.8.2
|
8 |
+
app_file: app.py
|
9 |
+
pinned: false
|
10 |
+
license: apache-2.0
|
11 |
+
duplicated_from: DGSpitzer/TXT-2-IMG-2-MUSIC-2-VIDEO
|
12 |
+
---
|
13 |
+
|
14 |
+
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
app.py
ADDED
@@ -0,0 +1,478 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from PIL import Image
|
2 |
+
import numpy as np
|
3 |
+
import gradio as gr
|
4 |
+
import paddlehub as hub
|
5 |
+
import urllib
|
6 |
+
import cv2
|
7 |
+
|
8 |
+
import io
|
9 |
+
from os import path
|
10 |
+
from pydub import AudioSegment
|
11 |
+
import moviepy.video.io.ImageSequenceClip
|
12 |
+
from moviepy.editor import *
|
13 |
+
import mutagen
|
14 |
+
from mutagen.mp3 import MP3
|
15 |
+
|
16 |
+
#text_to_img = gr.Blocks.load(name="spaces/DGSpitzer/ERNIE-ViLG", api_key = os.environ.get("hf"))
|
17 |
+
text_to_img = gr.Blocks.load(name="spaces/PaddlePaddle/ERNIE-ViLG")
|
18 |
+
img_to_text = gr.Blocks.load(name="spaces/pharma/CLIP-Interrogator")
|
19 |
+
text_to_music = gr.Interface.load("spaces/fffiloni/text-2-music")
|
20 |
+
|
21 |
+
model = hub.Module(name='ernie_vilg', ak = os.environ.get("ak"), sk = os.environ.get("sk"))
|
22 |
+
#model = hub.Module(name='ernie_vilg')
|
23 |
+
language_translation_model = hub.Module(name='baidu_translate')
|
24 |
+
language_recognition_model = hub.Module(name='baidu_language_recognition')
|
25 |
+
|
26 |
+
style_list = ['古风', '油画', '水彩', '卡通', '二次元', '浮世绘', '蒸汽波艺术', 'low poly', '像素风格', '概念艺术', '未来主义', '赛博朋克', '写实风格', '洛丽塔风格', '巴洛克风格', '超现实主义', '探索无限']
|
27 |
+
style_list_EN = ['Chinese Ancient Style', 'Oil painting', 'Watercolor', 'Cartoon', 'Anime', 'Ukiyoe', 'Vaporwave', 'low poly', 'Pixel Style', 'Conceptual Art', 'Futurism', 'Cyberpunk', 'Realistic style', 'Lolita style', 'Baroque style', 'Surrealism', '']
|
28 |
+
|
29 |
+
tips = {"en": "Tips: The input text will be translated into Chinese for generation",
|
30 |
+
"jp": "ヒント: 入力テキストは生成のために中国語に翻訳されます",
|
31 |
+
"kor": "힌트: 입력 텍스트는 생성을 위해 중국어로 번역됩니다"}
|
32 |
+
|
33 |
+
count = 0
|
34 |
+
|
35 |
+
def translate_language_example(text_prompts, style_indx):
|
36 |
+
return translate_language(text_prompts)
|
37 |
+
|
38 |
+
def translate_language(text_prompts):
|
39 |
+
global count
|
40 |
+
try:
|
41 |
+
count += 1
|
42 |
+
tips_text = None
|
43 |
+
language_code = language_recognition_model.recognize(text_prompts)
|
44 |
+
if language_code != 'zh':
|
45 |
+
text_prompts = language_translation_model.translate(text_prompts, language_code, 'zh')
|
46 |
+
except Exception as e:
|
47 |
+
error_text = str(e)
|
48 |
+
return {status_text:error_text, language_tips_text:gr.update(visible=False)}
|
49 |
+
if language_code in tips:
|
50 |
+
tips_text = tips[language_code]
|
51 |
+
else:
|
52 |
+
tips_text = tips['en']
|
53 |
+
if language_code == 'zh':
|
54 |
+
return {language_tips_text:gr.update(visible=False), translated_language:text_prompts, trigger_component: gr.update(value=count, visible=False)}
|
55 |
+
else:
|
56 |
+
return {language_tips_text:gr.update(visible=True, value=tips_text), translated_language:text_prompts, trigger_component: gr.update(value=count, visible=False)}
|
57 |
+
|
58 |
+
|
59 |
+
def get_result(text_prompts, style_indx):
|
60 |
+
#results = text_to_img(text_prompts, style_indx, fn_index=3)
|
61 |
+
print(text_prompts)
|
62 |
+
#results = text_to_img(text_prompts, text_prompts, fn_index=3)
|
63 |
+
print(style_indx)
|
64 |
+
try:
|
65 |
+
style = style_list[style_indx]
|
66 |
+
|
67 |
+
#results = text_to_img(text_prompts, style_indx, fn_index=3)
|
68 |
+
|
69 |
+
results = model.generate_image(
|
70 |
+
text_prompts=text_prompts, style=style, visualization=False, topk=1)
|
71 |
+
except Exception as e:
|
72 |
+
error_text = str(e)
|
73 |
+
return {video_result:None, status_text:error_text}
|
74 |
+
|
75 |
+
#print("Ernie Vilg Output: " + str(results[:1]))
|
76 |
+
print("Ernie Vilg Output test: " + str(results))
|
77 |
+
|
78 |
+
#image_output = results[:1]
|
79 |
+
image_output = results[:1]
|
80 |
+
|
81 |
+
print("file name: " + image_output[0].filename)
|
82 |
+
|
83 |
+
# Encode your PIL Image as a JPEG without writing to disk
|
84 |
+
imagefile = "imageoutput.png"
|
85 |
+
#img_np = np.array(image_output[0])
|
86 |
+
#img_nparray= cv2.cvtColor(img_np, cv2.COLOR_BGR2RGBA)
|
87 |
+
#img_blue_correction = Image.fromarray(img_nparray)
|
88 |
+
#img_blue_correction.save(imagefile, img_blue_correction.format)
|
89 |
+
image_output[0].save(imagefile, image_output[0].format)
|
90 |
+
|
91 |
+
interrogate_prompt = img_to_text(imagefile, fn_index=1)[0]
|
92 |
+
print(interrogate_prompt)
|
93 |
+
music_output = get_music(interrogate_prompt + ", " + style_list_EN[style_indx])
|
94 |
+
|
95 |
+
video_merged = merge_video(music_output, image_output)
|
96 |
+
return {video_result:video_merged, status_text:'Success'}
|
97 |
+
|
98 |
+
def get_music(prompt):
|
99 |
+
|
100 |
+
result = text_to_music(prompt, fn_index=0)
|
101 |
+
|
102 |
+
print(f"""—————
|
103 |
+
NEW RESULTS
|
104 |
+
prompt : {prompt}
|
105 |
+
music : {result}
|
106 |
+
———————
|
107 |
+
""")
|
108 |
+
|
109 |
+
url = result
|
110 |
+
save_as = "file.mp3"
|
111 |
+
|
112 |
+
data = urllib.request.urlopen(url)
|
113 |
+
|
114 |
+
f = open(save_as,'wb')
|
115 |
+
f.write(data.read())
|
116 |
+
f.close()
|
117 |
+
|
118 |
+
#wave_file="file.wav"
|
119 |
+
|
120 |
+
#sound = AudioSegment.from_mp3(save_as)
|
121 |
+
#sound.export(wave_file, format="wav")
|
122 |
+
|
123 |
+
return save_as
|
124 |
+
|
125 |
+
def merge_video(music, img_list):
|
126 |
+
#Convert to mp3
|
127 |
+
#music.export("audio.mp3", format="mp3")
|
128 |
+
print('wav audio converted to mp3 audio' )
|
129 |
+
print('now getting duration of this mp3 audio' )
|
130 |
+
#getting audio clip's duration
|
131 |
+
audio_length = int(MP3(music).info.length)
|
132 |
+
print('Audio length is :',audio_length)
|
133 |
+
|
134 |
+
file_name = 'video_no_audio.mp4'
|
135 |
+
fps = 12
|
136 |
+
slide_time = audio_length
|
137 |
+
fourcc = cv2.VideoWriter.fourcc(*'MJPG')
|
138 |
+
out = cv2.VideoWriter(file_name, fourcc, fps, (1024, 1024))
|
139 |
+
|
140 |
+
for image in img_list:
|
141 |
+
cv_img = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
|
142 |
+
for _ in range(slide_time * fps):
|
143 |
+
#cv_img = cv2.resize(np.array(cv_img), (1024, 1024))
|
144 |
+
out.write(cv_img)
|
145 |
+
|
146 |
+
out.release()
|
147 |
+
|
148 |
+
|
149 |
+
#String a list of images into a video and write to memory
|
150 |
+
print('video clip created successfully from images')
|
151 |
+
|
152 |
+
# loading video file
|
153 |
+
print('Starting video and audio merge')
|
154 |
+
videoclip = VideoFileClip(file_name) #("/content/gdrive/My Drive/AI/my_video1.mp4")
|
155 |
+
print('loading video-clip')
|
156 |
+
|
157 |
+
# loading audio file
|
158 |
+
audioclip = AudioFileClip(music) #.subclip(0, 15)
|
159 |
+
print('loading mp3-format audio')
|
160 |
+
# adding audio to the video clip
|
161 |
+
mergedclip = videoclip.set_audio(audioclip)
|
162 |
+
print('video and audio merged successfully')
|
163 |
+
|
164 |
+
#Getting size and frame count of merged video file
|
165 |
+
print('Getting size and frame count of merged video file')
|
166 |
+
duration = mergedclip.duration
|
167 |
+
frame_count = mergedclip.fps
|
168 |
+
print('duration is:',duration)
|
169 |
+
print('frame count :', frame_count)
|
170 |
+
mergedclip.to_videofile('mergedvideo.mp4')
|
171 |
+
return 'mergedvideo.mp4'
|
172 |
+
|
173 |
+
title="文生图生音乐视频 Text to Image to Music to Video"
|
174 |
+
|
175 |
+
description="An AI art generation pipeline, which supports text-to-image-to-music task."
|
176 |
+
|
177 |
+
css = """
|
178 |
+
.gradio-container {
|
179 |
+
font-family: 'IBM Plex Sans', sans-serif;
|
180 |
+
}
|
181 |
+
.gr-button {
|
182 |
+
color: white;
|
183 |
+
border-color: black;
|
184 |
+
background: black;
|
185 |
+
}
|
186 |
+
input[type='range'] {
|
187 |
+
accent-color: black;
|
188 |
+
}
|
189 |
+
.dark input[type='range'] {
|
190 |
+
accent-color: #dfdfdf;
|
191 |
+
}
|
192 |
+
.container {
|
193 |
+
max-width: 730px;
|
194 |
+
margin: auto;
|
195 |
+
padding-top: 1.5rem;
|
196 |
+
}
|
197 |
+
#gallery {
|
198 |
+
min-height: 22rem;
|
199 |
+
margin-bottom: 15px;
|
200 |
+
margin-left: auto;
|
201 |
+
margin-right: auto;
|
202 |
+
border-bottom-right-radius: .5rem !important;
|
203 |
+
border-bottom-left-radius: .5rem !important;
|
204 |
+
}
|
205 |
+
#gallery>div>.h-full {
|
206 |
+
min-height: 20rem;
|
207 |
+
}
|
208 |
+
.details:hover {
|
209 |
+
text-decoration: underline;
|
210 |
+
}
|
211 |
+
.gr-button {
|
212 |
+
white-space: nowrap;
|
213 |
+
}
|
214 |
+
.gr-button:focus {
|
215 |
+
border-color: rgb(147 197 253 / var(--tw-border-opacity));
|
216 |
+
outline: none;
|
217 |
+
box-shadow: var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow, 0 0 #0000);
|
218 |
+
--tw-border-opacity: 1;
|
219 |
+
--tw-ring-offset-shadow: var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);
|
220 |
+
--tw-ring-shadow: var(--tw-ring-inset) 0 0 0 calc(3px var(--tw-ring-offset-width)) var(--tw-ring-color);
|
221 |
+
--tw-ring-color: rgb(191 219 254 / var(--tw-ring-opacity));
|
222 |
+
--tw-ring-opacity: .5;
|
223 |
+
}
|
224 |
+
.footer {
|
225 |
+
margin-bottom: 45px;
|
226 |
+
margin-top: 35px;
|
227 |
+
text-align: center;
|
228 |
+
border-bottom: 1px solid #e5e5e5;
|
229 |
+
}
|
230 |
+
.footer>p {
|
231 |
+
font-size: .8rem;
|
232 |
+
display: inline-block;
|
233 |
+
padding: 0 10px;
|
234 |
+
transform: translateY(10px);
|
235 |
+
background: white;
|
236 |
+
}
|
237 |
+
.dark .footer {
|
238 |
+
border-color: #303030;
|
239 |
+
}
|
240 |
+
.dark .footer>p {
|
241 |
+
background: #0b0f19;
|
242 |
+
}
|
243 |
+
.prompt h4{
|
244 |
+
margin: 1.25em 0 .25em 0;
|
245 |
+
font-weight: bold;
|
246 |
+
font-size: 115%;
|
247 |
+
}
|
248 |
+
"""
|
249 |
+
|
250 |
+
block = gr.Blocks(css=css)
|
251 |
+
|
252 |
+
examples = [
|
253 |
+
[
|
254 |
+
'蒙娜丽莎,赛博朋克,宝丽来,33毫米',
|
255 |
+
'蒸汽波艺术(Vaporwave)'
|
256 |
+
],
|
257 |
+
[
|
258 |
+
'一条由闪电制成的令人敬畏的龙',
|
259 |
+
'概念艺术(Conceptual Art)'
|
260 |
+
],
|
261 |
+
[
|
262 |
+
'An awesome dragon made of lightning',
|
263 |
+
'概念艺术(Conceptual Art)'
|
264 |
+
],
|
265 |
+
[
|
266 |
+
'嫦娥在时代广场,戏曲',
|
267 |
+
'写实风格(Realistic style)'
|
268 |
+
],
|
269 |
+
[
|
270 |
+
'Peking Opera at New York',
|
271 |
+
'探索无限(Explore infinity)'
|
272 |
+
],
|
273 |
+
[
|
274 |
+
'古风少女',
|
275 |
+
'水彩(Watercolor)'
|
276 |
+
],
|
277 |
+
[
|
278 |
+
'辐射游戏角色',
|
279 |
+
'探索无限(Explore infinity)'
|
280 |
+
],
|
281 |
+
[
|
282 |
+
'Fallout game character',
|
283 |
+
'探索无限(Explore infinity)'
|
284 |
+
],
|
285 |
+
[
|
286 |
+
'Traditional Chinese Painting',
|
287 |
+
'古风(Ancient Style)'
|
288 |
+
],
|
289 |
+
[
|
290 |
+
'原神游戏截图,pixiv, 二次元绘画作品',
|
291 |
+
'二次元(Anime)'
|
292 |
+
],
|
293 |
+
[
|
294 |
+
'Genshin Impact Game Screenshot, pixiv, Anime Painting Artworks',
|
295 |
+
'二次元(Anime)'
|
296 |
+
],
|
297 |
+
[
|
298 |
+
'原神角色设定, 哪吒, pixiv, 二次元绘画',
|
299 |
+
'二次元(Anime)'
|
300 |
+
],
|
301 |
+
[
|
302 |
+
'Genshin Impact Character Design, Harry Potter, pixiv, Anime Painting',
|
303 |
+
'二次元(Anime)'
|
304 |
+
],
|
305 |
+
[
|
306 |
+
'巨狼,飘雪,蓝色大片烟雾,毛发细致,烟雾缭绕,高清,3d,cg感,侧面照',
|
307 |
+
'探索无限(Explore infinity)'
|
308 |
+
],
|
309 |
+
[
|
310 |
+
'汉服少女,中国山水画,青山绿水,溪水长流,古风,科技都市,丹青水墨,中国风',
|
311 |
+
'赛博朋克(Cyberpunk)'
|
312 |
+
],
|
313 |
+
[
|
314 |
+
'戴着墨镜的赛博朋克女孩肖像,在夕阳下的城市中, 油画风格',
|
315 |
+
'赛博朋克(Cyberpunk)'
|
316 |
+
],
|
317 |
+
[
|
318 |
+
'Portrait of a cyberpunk girl with sunglasses, in the city sunset, oil painting',
|
319 |
+
'赛博朋克(Cyberpunk)'
|
320 |
+
],
|
321 |
+
[
|
322 |
+
'暗黑破坏神',
|
323 |
+
'探索无限(Explore infinity)'
|
324 |
+
],
|
325 |
+
[
|
326 |
+
'火焰,凤凰,少女,未来感,高清,3d,精致面容,cg感,古风,唯美,毛发细致,上半身立绘',
|
327 |
+
'探索无限(Explore infinity)'
|
328 |
+
],
|
329 |
+
[
|
330 |
+
'浮世绘日本科幻哑光绘画,概念艺术,动漫风格神道寺禅园英雄动作序列,包豪斯',
|
331 |
+
'探索无限(Explore infinity)'
|
332 |
+
],
|
333 |
+
[
|
334 |
+
'一只猫坐在椅子上,戴着一副墨镜,海盗风格',
|
335 |
+
'探索无限(Explore infinity)'
|
336 |
+
],
|
337 |
+
[
|
338 |
+
'稲妻で作られた畏敬の念を抱かせる竜、コンセプトアート',
|
339 |
+
'油画(Oil painting)'
|
340 |
+
],
|
341 |
+
[
|
342 |
+
'번개로 만든 경외스러운 용, 개념 예술',
|
343 |
+
'油画(Oil painting)'
|
344 |
+
],
|
345 |
+
[
|
346 |
+
'梵高猫头鹰',
|
347 |
+
'蒸汽波艺术(Vaporwave)'
|
348 |
+
],
|
349 |
+
[
|
350 |
+
'萨尔瓦多·达利描绘古代文明的超现实主义梦幻油画',
|
351 |
+
'写实风格(Realistic style)'
|
352 |
+
],
|
353 |
+
[
|
354 |
+
'夕阳日落时,阳光落在云层上,海面波涛汹涌,风景,胶片感',
|
355 |
+
'探索无限(Explore infinity)'
|
356 |
+
],
|
357 |
+
[
|
358 |
+
'Sunset, the sun falls on the clouds, the sea is rough, the scenery is filmy',
|
359 |
+
'油画(Oil painting)'
|
360 |
+
],
|
361 |
+
[
|
362 |
+
'夕日が沈むと、雲の上に太陽の光が落ち、海面は波が荒く、風景、フィルム感',
|
363 |
+
'油画(Oil painting)'
|
364 |
+
],
|
365 |
+
[
|
366 |
+
'석양이 질 때 햇빛이 구름 위에 떨어지고, 해수면의 파도가 용솟음치며, 풍경, 필름감',
|
367 |
+
'油画(Oil painting)'
|
368 |
+
],
|
369 |
+
]
|
370 |
+
|
371 |
+
with block:
|
372 |
+
gr.HTML(
|
373 |
+
"""
|
374 |
+
<div style="text-align: center; max-width: 650px; margin: 0 auto;">
|
375 |
+
<div
|
376 |
+
style="
|
377 |
+
display: inline-flex;
|
378 |
+
gap: 0.8rem;
|
379 |
+
font-size: 1.75rem;
|
380 |
+
margin-bottom: 10px;
|
381 |
+
margin-left: 220px;
|
382 |
+
justify-content: center;
|
383 |
+
"
|
384 |
+
>
|
385 |
+
</div>
|
386 |
+
<div
|
387 |
+
style="
|
388 |
+
display: inline-flex;
|
389 |
+
align-items: center;
|
390 |
+
gap: 0.8rem;
|
391 |
+
font-size: 1.75rem;
|
392 |
+
margin-bottom: 10px;
|
393 |
+
justify-content: center;
|
394 |
+
">
|
395 |
+
<h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 15px;">文生图生音乐视频</h1>
|
396 |
+
</div>
|
397 |
+
<div
|
398 |
+
style="
|
399 |
+
display: inline-flex;
|
400 |
+
align-items: center;
|
401 |
+
gap: 0.8rem;
|
402 |
+
font-size: 1.75rem;
|
403 |
+
margin-bottom: 10px;
|
404 |
+
justify-content: center;
|
405 |
+
">
|
406 |
+
<h1 style="font-weight: 900; margin-bottom: 7px;">Text to Image to Music to Video</h1>
|
407 |
+
</div>
|
408 |
+
<p style="margin-bottom: 10px; font-size: 94%">
|
409 |
+
Powered by <a href="https://huggingface.co/spaces/PaddlePaddle/ERNIE-ViLG" target="_blank">ERNIE-ViLG 2.0</a>, <a href="https://huggingface.co/spaces/Mubert/Text-to-Music" target="_blank">Mubert AI</a>, <a href="https://huggingface.co/spaces/pharma/CLIP-Interrogator" target="_blank">CLIP Interrogator</a> and fffiloni's <a href="https://huggingface.co/spaces/fffiloni/img-to-music" target="_blank">Image to Music</a> projects
|
410 |
+
</p>
|
411 |
+
</div>
|
412 |
+
"""
|
413 |
+
)
|
414 |
+
with gr.Group():
|
415 |
+
with gr.Box():
|
416 |
+
with gr.Row().style(mobile_collapse=False, equal_height=True):
|
417 |
+
text = gr.Textbox(
|
418 |
+
label="Prompt",
|
419 |
+
show_label=False,
|
420 |
+
max_lines=1,
|
421 |
+
placeholder="Enter your prompt, multiple languages are supported now.",
|
422 |
+
).style(
|
423 |
+
border=(True, False, True, True),
|
424 |
+
rounded=(True, False, False, True),
|
425 |
+
container=False,
|
426 |
+
)
|
427 |
+
|
428 |
+
btn = gr.Button("Generate image").style(
|
429 |
+
margin=False,
|
430 |
+
rounded=(False, True, True, False),
|
431 |
+
)
|
432 |
+
language_tips_text = gr.Textbox(label="language tips", show_label=False, visible=False, max_lines=1)
|
433 |
+
styles = gr.Dropdown(label="风格(style)", choices=['古风(Ancient Style)', '油画(Oil painting)', '水彩(Watercolor)',
|
434 |
+
'卡通(Cartoon)', '二次元(Anime)', '浮世绘(Ukiyoe)', '蒸汽波艺术(Vaporwave)', 'low poly',
|
435 |
+
'像素风格(Pixel Style)', '概念艺术(Conceptual Art)', '未来主义(Futurism)', '赛博朋克(Cyberpunk)', '写实风格(Realistic style)',
|
436 |
+
'洛丽塔风格(Lolita style)', '巴洛克风格(Baroque style)', '超现实主义(Surrealism)', '探索无限(Explore infinity)'], value='探索无限(Explore infinity)', type="index")
|
437 |
+
status_text = gr.Textbox(
|
438 |
+
label="处理状态(Process status)",
|
439 |
+
show_label=True,
|
440 |
+
max_lines=1,
|
441 |
+
interactive=False
|
442 |
+
)
|
443 |
+
|
444 |
+
video_result = gr.Video(type=None, label='Final Merged video')
|
445 |
+
|
446 |
+
trigger_component = gr.Textbox(vaule="", visible=False) # This component is used for triggering inference funtion.
|
447 |
+
translated_language = gr.Textbox(vaule="", visible=False)
|
448 |
+
|
449 |
+
|
450 |
+
ex = gr.Examples(examples=examples, fn=translate_language_example, inputs=[text, styles], outputs=[language_tips_text, status_text, trigger_component, translated_language], cache_examples=False)
|
451 |
+
ex.dataset.headers = [""]
|
452 |
+
|
453 |
+
|
454 |
+
text.submit(translate_language, inputs=[text], outputs=[language_tips_text, status_text, trigger_component, translated_language])
|
455 |
+
btn.click(translate_language, inputs=[text], outputs=[language_tips_text, status_text, trigger_component, translated_language])
|
456 |
+
trigger_component.change(fn=get_result, inputs=[translated_language, styles], outputs=[video_result, status_text])
|
457 |
+
|
458 |
+
|
459 |
+
gr.Markdown(
|
460 |
+
"""
|
461 |
+
### <u>[Prompt Tutorial 公式教程...](https://github.com/PaddlePaddle/PaddleHub/blob/develop/modules/image/text_to_image/ernie_vilg/README.md#四-prompt-指南)([Explore more...](https://github.com/PaddlePaddle/PaddleHub/blob/develop/modules/image/text_to_image/ernie_vilg/README.md#四-prompt-指南))</u>
|
462 |
+
"""
|
463 |
+
)
|
464 |
+
gr.Markdown(
|
465 |
+
"""
|
466 |
+
Space by [@DGSpitzer](https://www.youtube.com/channel/UCzzsYBF4qwtMwJaPJZ5SuPg)❤️ [@大谷的游戏创作小屋](https://space.bilibili.com/176003)
|
467 |
+
[![Twitter Follow](https://img.shields.io/twitter/follow/DGSpitzer?label=%40DGSpitzer&style=social)](https://twitter.com/DGSpitzer)
|
468 |
+
![visitors](https://visitor-badge.glitch.me/badge?page_id=dgspitzer_txt2img2video)
|
469 |
+
"""
|
470 |
+
)
|
471 |
+
gr.HTML('''
|
472 |
+
<div class="footer">
|
473 |
+
<p>Model:<a href="https://github.com/PaddlePaddle/PaddleHub" style="text-decoration: underline;" target="_blank">PaddleHub</a> and <a href="https://wenxin.baidu.com/ernie-vilg" style="text-decoration: underline;" target="_blank">文心大模型</a>
|
474 |
+
</p>
|
475 |
+
</div>
|
476 |
+
''')
|
477 |
+
|
478 |
+
block.queue(concurrency_count=128).launch()
|
app_old.py
ADDED
@@ -0,0 +1,467 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from PIL import Image
|
2 |
+
import numpy as np
|
3 |
+
import gradio as gr
|
4 |
+
import paddlehub as hub
|
5 |
+
import urllib
|
6 |
+
import cv2
|
7 |
+
|
8 |
+
import io
|
9 |
+
from os import path
|
10 |
+
from pydub import AudioSegment
|
11 |
+
import moviepy.video.io.ImageSequenceClip
|
12 |
+
from moviepy.editor import *
|
13 |
+
import mutagen
|
14 |
+
from mutagen.mp3 import MP3
|
15 |
+
|
16 |
+
img_to_text = gr.Blocks.load(name="spaces/pharma/CLIP-Interrogator")
|
17 |
+
text_to_music = gr.Interface.load("spaces/fffiloni/text-2-music")
|
18 |
+
|
19 |
+
model = hub.Module(name='ernie_vilg', ak = os.environ.get("ak"), sk = os.environ.get("sk"))
|
20 |
+
#model = hub.Module(name='ernie_vilg')
|
21 |
+
language_translation_model = hub.Module(name='baidu_translate')
|
22 |
+
language_recognition_model = hub.Module(name='baidu_language_recognition')
|
23 |
+
|
24 |
+
style_list = ['古风', '油画', '水彩', '卡通', '二次元', '浮世绘', '蒸汽波艺术', 'low poly', '像素风格', '概念艺术', '未来主义', '赛博朋克', '写实风格', '洛丽塔风格', '巴洛克风格', '超现实主义', '探索无限']
|
25 |
+
style_list_EN = ['Chinese Ancient Style', 'Oil painting', 'Watercolor', 'Cartoon', 'Anime', 'Ukiyoe', 'Vaporwave', 'low poly', 'Pixel Style', 'Conceptual Art', 'Futurism', 'Cyberpunk', 'Realistic style', 'Lolita style', 'Baroque style', 'Surrealism', '']
|
26 |
+
|
27 |
+
tips = {"en": "Tips: The input text will be translated into Chinese for generation",
|
28 |
+
"jp": "ヒント: 入力テキストは生成のために中国語に翻訳されます",
|
29 |
+
"kor": "힌트: 입력 텍스트는 생성을 위해 중국어로 번역됩니다"}
|
30 |
+
|
31 |
+
count = 0
|
32 |
+
|
33 |
+
def translate_language_example(text_prompts, style_indx):
|
34 |
+
return translate_language(text_prompts)
|
35 |
+
|
36 |
+
def translate_language(text_prompts):
|
37 |
+
global count
|
38 |
+
try:
|
39 |
+
count += 1
|
40 |
+
tips_text = None
|
41 |
+
language_code = language_recognition_model.recognize(text_prompts)
|
42 |
+
if language_code != 'zh':
|
43 |
+
text_prompts = language_translation_model.translate(text_prompts, language_code, 'zh')
|
44 |
+
except Exception as e:
|
45 |
+
error_text = str(e)
|
46 |
+
return {status_text:error_text, language_tips_text:gr.update(visible=False)}
|
47 |
+
if language_code in tips:
|
48 |
+
tips_text = tips[language_code]
|
49 |
+
else:
|
50 |
+
tips_text = tips['en']
|
51 |
+
if language_code == 'zh':
|
52 |
+
return {language_tips_text:gr.update(visible=False), translated_language:text_prompts, trigger_component: gr.update(value=count, visible=False)}
|
53 |
+
else:
|
54 |
+
return {language_tips_text:gr.update(visible=True, value=tips_text), translated_language:text_prompts, trigger_component: gr.update(value=count, visible=False)}
|
55 |
+
|
56 |
+
|
57 |
+
def get_result(text_prompts, style_indx):
|
58 |
+
try:
|
59 |
+
style = style_list[style_indx]
|
60 |
+
results = model.generate_image(
|
61 |
+
text_prompts=text_prompts, style=style, visualization=False, topk=1)
|
62 |
+
except Exception as e:
|
63 |
+
error_text = str(e)
|
64 |
+
return {video_result:None, status_text:error_text}
|
65 |
+
|
66 |
+
image_output = results[:1]
|
67 |
+
|
68 |
+
print("Ernie Vilg Output: " + str(results[:1]))
|
69 |
+
print("Ernie Vilg Output test: " + str(results))
|
70 |
+
print("file name: " + image_output[0].filename)
|
71 |
+
|
72 |
+
# Encode your PIL Image as a JPEG without writing to disk
|
73 |
+
imagefile = "imageoutput.png"
|
74 |
+
#img_np = np.array(image_output[0])
|
75 |
+
#img_nparray= cv2.cvtColor(img_np, cv2.COLOR_BGR2RGBA)
|
76 |
+
#img_blue_correction = Image.fromarray(img_nparray)
|
77 |
+
#img_blue_correction.save(imagefile, img_blue_correction.format)
|
78 |
+
image_output[0].save(imagefile, image_output[0].format)
|
79 |
+
|
80 |
+
interrogate_prompt = img_to_text(imagefile, fn_index=1)[0]
|
81 |
+
print(interrogate_prompt)
|
82 |
+
music_output = get_music(interrogate_prompt + ", " + style_list_EN[style_indx])
|
83 |
+
|
84 |
+
video_merged = merge_video(music_output, image_output)
|
85 |
+
return {video_result:video_merged, status_text:'Success'}
|
86 |
+
|
87 |
+
def get_music(prompt):
|
88 |
+
|
89 |
+
result = text_to_music(prompt, fn_index=0)
|
90 |
+
|
91 |
+
print(f"""—————
|
92 |
+
NEW RESULTS
|
93 |
+
prompt : {prompt}
|
94 |
+
music : {result}
|
95 |
+
———————
|
96 |
+
""")
|
97 |
+
|
98 |
+
url = result
|
99 |
+
save_as = "file.mp3"
|
100 |
+
|
101 |
+
data = urllib.request.urlopen(url)
|
102 |
+
|
103 |
+
f = open(save_as,'wb')
|
104 |
+
f.write(data.read())
|
105 |
+
f.close()
|
106 |
+
|
107 |
+
#wave_file="file.wav"
|
108 |
+
|
109 |
+
#sound = AudioSegment.from_mp3(save_as)
|
110 |
+
#sound.export(wave_file, format="wav")
|
111 |
+
|
112 |
+
return save_as
|
113 |
+
|
114 |
+
def merge_video(music, img_list):
|
115 |
+
#Convert to mp3
|
116 |
+
#music.export("audio.mp3", format="mp3")
|
117 |
+
print('wav audio converted to mp3 audio' )
|
118 |
+
print('now getting duration of this mp3 audio' )
|
119 |
+
#getting audio clip's duration
|
120 |
+
audio_length = int(MP3(music).info.length)
|
121 |
+
print('Audio length is :',audio_length)
|
122 |
+
|
123 |
+
file_name = 'video_no_audio.mp4'
|
124 |
+
fps = 12
|
125 |
+
slide_time = audio_length
|
126 |
+
fourcc = cv2.VideoWriter.fourcc(*'MJPG')
|
127 |
+
out = cv2.VideoWriter(file_name, fourcc, fps, (1024, 1024))
|
128 |
+
|
129 |
+
for image in img_list:
|
130 |
+
cv_img = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
|
131 |
+
for _ in range(slide_time * fps):
|
132 |
+
#cv_img = cv2.resize(np.array(cv_img), (1024, 1024))
|
133 |
+
out.write(cv_img)
|
134 |
+
|
135 |
+
out.release()
|
136 |
+
|
137 |
+
|
138 |
+
#String a list of images into a video and write to memory
|
139 |
+
print('video clip created successfully from images')
|
140 |
+
|
141 |
+
# loading video file
|
142 |
+
print('Starting video and audio merge')
|
143 |
+
videoclip = VideoFileClip(file_name) #("/content/gdrive/My Drive/AI/my_video1.mp4")
|
144 |
+
print('loading video-clip')
|
145 |
+
|
146 |
+
# loading audio file
|
147 |
+
audioclip = AudioFileClip(music) #.subclip(0, 15)
|
148 |
+
print('loading mp3-format audio')
|
149 |
+
# adding audio to the video clip
|
150 |
+
mergedclip = videoclip.set_audio(audioclip)
|
151 |
+
print('video and audio merged successfully')
|
152 |
+
|
153 |
+
#Getting size and frame count of merged video file
|
154 |
+
print('Getting size and frame count of merged video file')
|
155 |
+
duration = mergedclip.duration
|
156 |
+
frame_count = mergedclip.fps
|
157 |
+
print('duration is:',duration)
|
158 |
+
print('frame count :', frame_count)
|
159 |
+
mergedclip.to_videofile('mergedvideo.mp4')
|
160 |
+
return 'mergedvideo.mp4'
|
161 |
+
|
162 |
+
title="文生图生音乐视频 Text to Image to Music to Video"
|
163 |
+
|
164 |
+
description="An AI art generation pipeline, which supports text-to-image-to-music task."
|
165 |
+
|
166 |
+
css = """
|
167 |
+
.gradio-container {
|
168 |
+
font-family: 'IBM Plex Sans', sans-serif;
|
169 |
+
}
|
170 |
+
.gr-button {
|
171 |
+
color: white;
|
172 |
+
border-color: black;
|
173 |
+
background: black;
|
174 |
+
}
|
175 |
+
input[type='range'] {
|
176 |
+
accent-color: black;
|
177 |
+
}
|
178 |
+
.dark input[type='range'] {
|
179 |
+
accent-color: #dfdfdf;
|
180 |
+
}
|
181 |
+
.container {
|
182 |
+
max-width: 730px;
|
183 |
+
margin: auto;
|
184 |
+
padding-top: 1.5rem;
|
185 |
+
}
|
186 |
+
#gallery {
|
187 |
+
min-height: 22rem;
|
188 |
+
margin-bottom: 15px;
|
189 |
+
margin-left: auto;
|
190 |
+
margin-right: auto;
|
191 |
+
border-bottom-right-radius: .5rem !important;
|
192 |
+
border-bottom-left-radius: .5rem !important;
|
193 |
+
}
|
194 |
+
#gallery>div>.h-full {
|
195 |
+
min-height: 20rem;
|
196 |
+
}
|
197 |
+
.details:hover {
|
198 |
+
text-decoration: underline;
|
199 |
+
}
|
200 |
+
.gr-button {
|
201 |
+
white-space: nowrap;
|
202 |
+
}
|
203 |
+
.gr-button:focus {
|
204 |
+
border-color: rgb(147 197 253 / var(--tw-border-opacity));
|
205 |
+
outline: none;
|
206 |
+
box-shadow: var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow, 0 0 #0000);
|
207 |
+
--tw-border-opacity: 1;
|
208 |
+
--tw-ring-offset-shadow: var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);
|
209 |
+
--tw-ring-shadow: var(--tw-ring-inset) 0 0 0 calc(3px var(--tw-ring-offset-width)) var(--tw-ring-color);
|
210 |
+
--tw-ring-color: rgb(191 219 254 / var(--tw-ring-opacity));
|
211 |
+
--tw-ring-opacity: .5;
|
212 |
+
}
|
213 |
+
.footer {
|
214 |
+
margin-bottom: 45px;
|
215 |
+
margin-top: 35px;
|
216 |
+
text-align: center;
|
217 |
+
border-bottom: 1px solid #e5e5e5;
|
218 |
+
}
|
219 |
+
.footer>p {
|
220 |
+
font-size: .8rem;
|
221 |
+
display: inline-block;
|
222 |
+
padding: 0 10px;
|
223 |
+
transform: translateY(10px);
|
224 |
+
background: white;
|
225 |
+
}
|
226 |
+
.dark .footer {
|
227 |
+
border-color: #303030;
|
228 |
+
}
|
229 |
+
.dark .footer>p {
|
230 |
+
background: #0b0f19;
|
231 |
+
}
|
232 |
+
.prompt h4{
|
233 |
+
margin: 1.25em 0 .25em 0;
|
234 |
+
font-weight: bold;
|
235 |
+
font-size: 115%;
|
236 |
+
}
|
237 |
+
"""
|
238 |
+
|
239 |
+
block = gr.Blocks(css=css)
|
240 |
+
|
241 |
+
examples = [
|
242 |
+
[
|
243 |
+
'蒙娜丽莎,赛博朋克,宝丽来,33毫米',
|
244 |
+
'蒸汽波艺术(Vaporwave)'
|
245 |
+
],
|
246 |
+
[
|
247 |
+
'一条由闪电制成的令人敬畏的龙',
|
248 |
+
'概念艺术(Conceptual Art)'
|
249 |
+
],
|
250 |
+
[
|
251 |
+
'An awesome dragon made of lightning',
|
252 |
+
'概念艺术(Conceptual Art)'
|
253 |
+
],
|
254 |
+
[
|
255 |
+
'嫦娥在时代广场,戏曲',
|
256 |
+
'写实风格(Realistic style)'
|
257 |
+
],
|
258 |
+
[
|
259 |
+
'Peking Opera at New York',
|
260 |
+
'探索无限(Explore infinity)'
|
261 |
+
],
|
262 |
+
[
|
263 |
+
'古风少女',
|
264 |
+
'水彩(Watercolor)'
|
265 |
+
],
|
266 |
+
[
|
267 |
+
'辐射游戏角色',
|
268 |
+
'探索无限(Explore infinity)'
|
269 |
+
],
|
270 |
+
[
|
271 |
+
'Fallout game character',
|
272 |
+
'探索无限(Explore infinity)'
|
273 |
+
],
|
274 |
+
[
|
275 |
+
'Traditional Chinese Painting',
|
276 |
+
'古风(Ancient Style)'
|
277 |
+
],
|
278 |
+
[
|
279 |
+
'原神游戏截图,pixiv, 二次元绘画作品',
|
280 |
+
'二次元(Anime)'
|
281 |
+
],
|
282 |
+
[
|
283 |
+
'Genshin Impact Game Screenshot, pixiv, Anime Painting Artworks',
|
284 |
+
'二次元(Anime)'
|
285 |
+
],
|
286 |
+
[
|
287 |
+
'原神角色设定, 哪吒, pixiv, 二次元绘画',
|
288 |
+
'二次元(Anime)'
|
289 |
+
],
|
290 |
+
[
|
291 |
+
'Genshin Impact Character Design, Harry Potter, pixiv, Anime Painting',
|
292 |
+
'二次元(Anime)'
|
293 |
+
],
|
294 |
+
[
|
295 |
+
'巨狼,飘雪,蓝色大片烟雾,毛发细致,烟雾缭绕,高清,3d,cg感,侧面照',
|
296 |
+
'探索无限(Explore infinity)'
|
297 |
+
],
|
298 |
+
[
|
299 |
+
'汉服少女,中国山水画,青山绿水,溪水长流,古风,科技都市,丹青水墨,中国风',
|
300 |
+
'赛博朋克(Cyberpunk)'
|
301 |
+
],
|
302 |
+
[
|
303 |
+
'戴着墨镜的赛博朋克女孩肖像,在夕阳下的城市中, 油画风格',
|
304 |
+
'赛博朋克(Cyberpunk)'
|
305 |
+
],
|
306 |
+
[
|
307 |
+
'Portrait of a cyberpunk girl with sunglasses, in the city sunset, oil painting',
|
308 |
+
'赛博朋克(Cyberpunk)'
|
309 |
+
],
|
310 |
+
[
|
311 |
+
'暗黑破坏神',
|
312 |
+
'探索无限(Explore infinity)'
|
313 |
+
],
|
314 |
+
[
|
315 |
+
'火焰,凤凰,少女,未来感,高清,3d,精致面容,cg感,古风,唯美,毛发细致,上半身立绘',
|
316 |
+
'探索无限(Explore infinity)'
|
317 |
+
],
|
318 |
+
[
|
319 |
+
'浮世绘日本科幻哑光绘画,概念艺术,动漫风格神道寺禅园英雄动作序列,包豪斯',
|
320 |
+
'探索无限(Explore infinity)'
|
321 |
+
],
|
322 |
+
[
|
323 |
+
'一只猫坐在椅子上,戴着一副墨镜,海盗风格',
|
324 |
+
'探索无限(Explore infinity)'
|
325 |
+
],
|
326 |
+
[
|
327 |
+
'稲妻で作られた畏敬の念を抱かせる竜、コンセプトアート',
|
328 |
+
'油画(Oil painting)'
|
329 |
+
],
|
330 |
+
[
|
331 |
+
'번개로 만든 경외스러운 용, 개념 예술',
|
332 |
+
'油画(Oil painting)'
|
333 |
+
],
|
334 |
+
[
|
335 |
+
'梵高猫头鹰',
|
336 |
+
'蒸汽波艺术(Vaporwave)'
|
337 |
+
],
|
338 |
+
[
|
339 |
+
'萨尔瓦多·达利描绘古代文明的超现实主义梦幻油画',
|
340 |
+
'写实风格(Realistic style)'
|
341 |
+
],
|
342 |
+
[
|
343 |
+
'夕阳日落时,阳光落在云层上,海面波涛汹涌,风景,胶片感',
|
344 |
+
'探索无限(Explore infinity)'
|
345 |
+
],
|
346 |
+
[
|
347 |
+
'Sunset, the sun falls on the clouds, the sea is rough, the scenery is filmy',
|
348 |
+
'油画(Oil painting)'
|
349 |
+
],
|
350 |
+
[
|
351 |
+
'夕日が沈むと、雲の上に太陽の光が落ち、海面は波が荒く、風景、フィルム感',
|
352 |
+
'油画(Oil painting)'
|
353 |
+
],
|
354 |
+
[
|
355 |
+
'석양이 질 때 햇빛이 구름 위에 떨어지고, 해수면의 파도가 용솟음치며, 풍경, 필름감',
|
356 |
+
'油画(Oil painting)'
|
357 |
+
],
|
358 |
+
]
|
359 |
+
|
360 |
+
with block:
|
361 |
+
gr.HTML(
|
362 |
+
"""
|
363 |
+
<div style="text-align: center; max-width: 650px; margin: 0 auto;">
|
364 |
+
<div
|
365 |
+
style="
|
366 |
+
display: inline-flex;
|
367 |
+
gap: 0.8rem;
|
368 |
+
font-size: 1.75rem;
|
369 |
+
margin-bottom: 10px;
|
370 |
+
margin-left: 220px;
|
371 |
+
justify-content: center;
|
372 |
+
"
|
373 |
+
>
|
374 |
+
</div>
|
375 |
+
<div
|
376 |
+
style="
|
377 |
+
display: inline-flex;
|
378 |
+
align-items: center;
|
379 |
+
gap: 0.8rem;
|
380 |
+
font-size: 1.75rem;
|
381 |
+
margin-bottom: 10px;
|
382 |
+
justify-content: center;
|
383 |
+
">
|
384 |
+
<h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 15px;">文生图生音乐视频</h1>
|
385 |
+
</div>
|
386 |
+
<div
|
387 |
+
style="
|
388 |
+
display: inline-flex;
|
389 |
+
align-items: center;
|
390 |
+
gap: 0.8rem;
|
391 |
+
font-size: 1.75rem;
|
392 |
+
margin-bottom: 10px;
|
393 |
+
justify-content: center;
|
394 |
+
">
|
395 |
+
<h1 style="font-weight: 900; margin-bottom: 7px;">Text to Image to Music to Video</h1>
|
396 |
+
</div>
|
397 |
+
<p style="margin-bottom: 10px; font-size: 94%">
|
398 |
+
Powered by <a href="https://huggingface.co/spaces/PaddlePaddle/ERNIE-ViLG" target="_blank">ERNIE-ViLG 2.0</a>, <a href="https://huggingface.co/spaces/Mubert/Text-to-Music" target="_blank">Mubert AI</a>, <a href="https://huggingface.co/spaces/pharma/CLIP-Interrogator" target="_blank">CLIP Interrogator</a> and fffiloni's <a href="https://huggingface.co/spaces/fffiloni/img-to-music" target="_blank">Image to Music</a> projects
|
399 |
+
</p>
|
400 |
+
</div>
|
401 |
+
"""
|
402 |
+
)
|
403 |
+
with gr.Group():
|
404 |
+
with gr.Box():
|
405 |
+
with gr.Row().style(mobile_collapse=False, equal_height=True):
|
406 |
+
text = gr.Textbox(
|
407 |
+
label="Prompt",
|
408 |
+
show_label=False,
|
409 |
+
max_lines=1,
|
410 |
+
placeholder="Enter your prompt, multiple languages are supported now.",
|
411 |
+
).style(
|
412 |
+
border=(True, False, True, True),
|
413 |
+
rounded=(True, False, False, True),
|
414 |
+
container=False,
|
415 |
+
)
|
416 |
+
|
417 |
+
btn = gr.Button("Generate image").style(
|
418 |
+
margin=False,
|
419 |
+
rounded=(False, True, True, False),
|
420 |
+
)
|
421 |
+
language_tips_text = gr.Textbox(label="language tips", show_label=False, visible=False, max_lines=1)
|
422 |
+
styles = gr.Dropdown(label="风格(style)", choices=['古风(Ancient Style)', '油画(Oil painting)', '水彩(Watercolor)',
|
423 |
+
'卡通(Cartoon)', '二次元(Anime)', '浮世绘(Ukiyoe)', '蒸汽波艺术(Vaporwave)', 'low poly',
|
424 |
+
'像素风格(Pixel Style)', '概念艺术(Conceptual Art)', '未来主义(Futurism)', '赛博朋克(Cyberpunk)', '写实风格(Realistic style)',
|
425 |
+
'洛丽塔风格(Lolita style)', '巴���克风格(Baroque style)', '超现实主义(Surrealism)', '探索无限(Explore infinity)'], value='探索无限(Explore infinity)', type="index")
|
426 |
+
status_text = gr.Textbox(
|
427 |
+
label="处理状态(Process status)",
|
428 |
+
show_label=True,
|
429 |
+
max_lines=1,
|
430 |
+
interactive=False
|
431 |
+
)
|
432 |
+
|
433 |
+
video_result = gr.Video(type=None, label='Final Merged video')
|
434 |
+
|
435 |
+
trigger_component = gr.Textbox(vaule="", visible=False) # This component is used for triggering inference funtion.
|
436 |
+
translated_language = gr.Textbox(vaule="", visible=False)
|
437 |
+
|
438 |
+
|
439 |
+
ex = gr.Examples(examples=examples, fn=translate_language_example, inputs=[text, styles], outputs=[language_tips_text, status_text, trigger_component, translated_language], cache_examples=False)
|
440 |
+
ex.dataset.headers = [""]
|
441 |
+
|
442 |
+
|
443 |
+
text.submit(translate_language, inputs=[text], outputs=[language_tips_text, status_text, trigger_component, translated_language])
|
444 |
+
btn.click(translate_language, inputs=[text], outputs=[language_tips_text, status_text, trigger_component, translated_language])
|
445 |
+
trigger_component.change(fn=get_result, inputs=[translated_language, styles], outputs=[video_result, status_text])
|
446 |
+
|
447 |
+
|
448 |
+
gr.Markdown(
|
449 |
+
"""
|
450 |
+
### <u>[Prompt Tutorial 公式教程...](https://github.com/PaddlePaddle/PaddleHub/blob/develop/modules/image/text_to_image/ernie_vilg/README.md#四-prompt-指南)([Explore more...](https://github.com/PaddlePaddle/PaddleHub/blob/develop/modules/image/text_to_image/ernie_vilg/README.md#四-prompt-指南))</u>
|
451 |
+
"""
|
452 |
+
)
|
453 |
+
gr.Markdown(
|
454 |
+
"""
|
455 |
+
Space by [@DGSpitzer](https://www.youtube.com/channel/UCzzsYBF4qwtMwJaPJZ5SuPg)❤️ [@大谷的游戏创作小屋](https://space.bilibili.com/176003)
|
456 |
+
[![Twitter Follow](https://img.shields.io/twitter/follow/DGSpitzer?label=%40DGSpitzer&style=social)](https://twitter.com/DGSpitzer)
|
457 |
+
![visitors](https://visitor-badge.glitch.me/badge?page_id=dgspitzer_txt2img2video)
|
458 |
+
"""
|
459 |
+
)
|
460 |
+
gr.HTML('''
|
461 |
+
<div class="footer">
|
462 |
+
<p>Model:<a href="https://github.com/PaddlePaddle/PaddleHub" style="text-decoration: underline;" target="_blank">PaddleHub</a> and <a href="https://wenxin.baidu.com/ernie-vilg" style="text-decoration: underline;" target="_blank">文心大模型</a>
|
463 |
+
</p>
|
464 |
+
</div>
|
465 |
+
''')
|
466 |
+
|
467 |
+
block.queue(concurrency_count=128).launch()
|
requirements.txt
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
paddlepaddle
|
2 |
+
paddlehub
|
3 |
+
requests
|
4 |
+
moviepy
|
5 |
+
mutagen
|