kevinwang676 commited on
Commit
7bbebbe
1 Parent(s): 7062004

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +23 -0
  2. .gitignore +174 -0
  3. .ipynb_checkpoints/Untitled-checkpoint.ipynb +6 -0
  4. .ipynb_checkpoints/app-checkpoint.py +608 -0
  5. .ipynb_checkpoints/app_sadtalker-checkpoint.py +111 -0
  6. ChatGLM2-SadTalker-VC-1/.flake8 +21 -0
  7. ChatGLM2-SadTalker-VC-1/.gitattributes +52 -0
  8. ChatGLM2-SadTalker-VC-1/.gitignore +159 -0
  9. ChatGLM2-SadTalker-VC-1/.ipynb_checkpoints/requirements-checkpoint.txt +12 -0
  10. ChatGLM2-SadTalker-VC-1/Dockerfile +59 -0
  11. ChatGLM2-SadTalker-VC-1/LICENSE +21 -0
  12. ChatGLM2-SadTalker-VC-1/README.md +15 -0
  13. ChatGLM2-SadTalker-VC-1/packages.txt +2 -0
  14. ChatGLM2-SadTalker-VC-1/requirements.txt +12 -0
  15. LICENSE +21 -0
  16. README.md +268 -13
  17. Untitled.ipynb +0 -0
  18. __pycache__/commons.cpython-310.pyc +0 -0
  19. __pycache__/mel_processing.cpython-310.pyc +0 -0
  20. __pycache__/models.cpython-310.pyc +0 -0
  21. __pycache__/modules.cpython-310.pyc +0 -0
  22. __pycache__/tts_voice.cpython-310.pyc +0 -0
  23. __pycache__/utils.cpython-310.pyc +0 -0
  24. app.py +608 -0
  25. app_sadtalker.py +111 -0
  26. checkpoint/__init__.py +0 -0
  27. checkpoint/freevc-24.pth +3 -0
  28. checkpoints/SadTalker_V0.0.2_256.safetensors +3 -0
  29. checkpoints/SadTalker_V0.0.2_512.safetensors +3 -0
  30. checkpoints/mapping_00109-model.pth.tar +3 -0
  31. checkpoints/mapping_00229-model.pth.tar +3 -0
  32. cog.yaml +35 -0
  33. commons.py +171 -0
  34. configs/.ipynb_checkpoints/freevc-24-checkpoint.json +54 -0
  35. configs/freevc-24.json +54 -0
  36. docs/FAQ.md +46 -0
  37. docs/best_practice.md +94 -0
  38. docs/changlelog.md +29 -0
  39. docs/example_crop.gif +3 -0
  40. docs/example_crop_still.gif +3 -0
  41. docs/example_full.gif +3 -0
  42. docs/example_full_crop.gif +0 -0
  43. docs/example_full_enhanced.gif +3 -0
  44. docs/face3d.md +48 -0
  45. docs/free_view_result.gif +3 -0
  46. docs/install.md +47 -0
  47. docs/resize_good.gif +3 -0
  48. docs/resize_no.gif +3 -0
  49. docs/sadtalker_logo.png +0 -0
  50. docs/using_ref_video.gif +3 -0
.gitattributes CHANGED
@@ -33,3 +33,26 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ docs/example_crop.gif filter=lfs diff=lfs merge=lfs -text
37
+ docs/example_crop_still.gif filter=lfs diff=lfs merge=lfs -text
38
+ docs/example_full.gif filter=lfs diff=lfs merge=lfs -text
39
+ docs/example_full_enhanced.gif filter=lfs diff=lfs merge=lfs -text
40
+ docs/free_view_result.gif filter=lfs diff=lfs merge=lfs -text
41
+ docs/resize_good.gif filter=lfs diff=lfs merge=lfs -text
42
+ docs/resize_no.gif filter=lfs diff=lfs merge=lfs -text
43
+ docs/using_ref_video.gif filter=lfs diff=lfs merge=lfs -text
44
+ examples/driven_audio/chinese_news.wav filter=lfs diff=lfs merge=lfs -text
45
+ examples/driven_audio/deyu.wav filter=lfs diff=lfs merge=lfs -text
46
+ examples/driven_audio/eluosi.wav filter=lfs diff=lfs merge=lfs -text
47
+ examples/driven_audio/fayu.wav filter=lfs diff=lfs merge=lfs -text
48
+ examples/driven_audio/imagine.wav filter=lfs diff=lfs merge=lfs -text
49
+ examples/driven_audio/japanese.wav filter=lfs diff=lfs merge=lfs -text
50
+ examples/ref_video/WDA_AlexandriaOcasioCortez_000.mp4 filter=lfs diff=lfs merge=lfs -text
51
+ examples/ref_video/WDA_KatieHill_000.mp4 filter=lfs diff=lfs merge=lfs -text
52
+ examples/source_image/art_16.png filter=lfs diff=lfs merge=lfs -text
53
+ examples/source_image/art_17.png filter=lfs diff=lfs merge=lfs -text
54
+ examples/source_image/art_3.png filter=lfs diff=lfs merge=lfs -text
55
+ examples/source_image/art_4.png filter=lfs diff=lfs merge=lfs -text
56
+ examples/source_image/art_5.png filter=lfs diff=lfs merge=lfs -text
57
+ examples/source_image/art_8.png filter=lfs diff=lfs merge=lfs -text
58
+ examples/source_image/art_9.png filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # poetry
98
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102
+ #poetry.lock
103
+
104
+ # pdm
105
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106
+ #pdm.lock
107
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108
+ # in version control.
109
+ # https://pdm.fming.dev/#use-with-ide
110
+ .pdm.toml
111
+
112
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113
+ __pypackages__/
114
+
115
+ # Celery stuff
116
+ celerybeat-schedule
117
+ celerybeat.pid
118
+
119
+ # SageMath parsed files
120
+ *.sage.py
121
+
122
+ # Environments
123
+ .env
124
+ .venv
125
+ env/
126
+ venv/
127
+ ENV/
128
+ env.bak/
129
+ venv.bak/
130
+
131
+ # Spyder project settings
132
+ .spyderproject
133
+ .spyproject
134
+
135
+ # Rope project settings
136
+ .ropeproject
137
+
138
+ # mkdocs documentation
139
+ /site
140
+
141
+ # mypy
142
+ .mypy_cache/
143
+ .dmypy.json
144
+ dmypy.json
145
+
146
+ # Pyre type checker
147
+ .pyre/
148
+
149
+ # pytype static type analyzer
150
+ .pytype/
151
+
152
+ # Cython debug symbols
153
+ cython_debug/
154
+
155
+ # PyCharm
156
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
159
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
160
+ .idea/
161
+
162
+ examples/results/*
163
+ gfpgan/*
164
+ checkpoints/*
165
+ assets/*
166
+ results/*
167
+ Dockerfile
168
+ start_docker.sh
169
+ start.sh
170
+
171
+ checkpoints
172
+
173
+ # Mac
174
+ .DS_Store
.ipynb_checkpoints/Untitled-checkpoint.ipynb ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [],
3
+ "metadata": {},
4
+ "nbformat": 4,
5
+ "nbformat_minor": 5
6
+ }
.ipynb_checkpoints/app-checkpoint.py ADDED
@@ -0,0 +1,608 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, sys
2
+ import tempfile
3
+ import gradio as gr
4
+ from src.gradio_demo import SadTalker
5
+ # from src.utils.text2speech import TTSTalker
6
+ from huggingface_hub import snapshot_download
7
+
8
+ import torch
9
+ import librosa
10
+ from scipy.io.wavfile import write
11
+ from transformers import WavLMModel
12
+
13
+ import utils
14
+ from models import SynthesizerTrn
15
+ from mel_processing import mel_spectrogram_torch
16
+ from speaker_encoder.voice_encoder import SpeakerEncoder
17
+
18
+ import time
19
+ from textwrap import dedent
20
+
21
+ import mdtex2html
22
+ from loguru import logger
23
+ from transformers import AutoModel, AutoTokenizer
24
+
25
+ from tts_voice import tts_order_voice
26
+ import edge_tts
27
+ import tempfile
28
+ import anyio
29
+
30
+
31
+ def get_source_image(image):
32
+ return image
33
+
34
+ try:
35
+ import webui # in webui
36
+ in_webui = True
37
+ except:
38
+ in_webui = False
39
+
40
+
41
+ def toggle_audio_file(choice):
42
+ if choice == False:
43
+ return gr.update(visible=True), gr.update(visible=False)
44
+ else:
45
+ return gr.update(visible=False), gr.update(visible=True)
46
+
47
+ def ref_video_fn(path_of_ref_video):
48
+ if path_of_ref_video is not None:
49
+ return gr.update(value=True)
50
+ else:
51
+ return gr.update(value=False)
52
+
53
+ def download_model():
54
+ REPO_ID = 'vinthony/SadTalker-V002rc'
55
+ snapshot_download(repo_id=REPO_ID, local_dir='./checkpoints', local_dir_use_symlinks=True)
56
+
57
+ def sadtalker_demo():
58
+
59
+ download_model()
60
+
61
+ sad_talker = SadTalker(lazy_load=True)
62
+ # tts_talker = TTSTalker()
63
+
64
+ download_model()
65
+ sad_talker = SadTalker(lazy_load=True)
66
+
67
+
68
+ # ChatGLM2 & FreeVC
69
+
70
+ '''
71
+ def get_wavlm():
72
+ os.system('gdown https://drive.google.com/uc?id=12-cB34qCTvByWT-QtOcZaqwwO21FLSqU')
73
+ shutil.move('WavLM-Large.pt', 'wavlm')
74
+ '''
75
+
76
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
77
+
78
+ smodel = SpeakerEncoder('speaker_encoder/ckpt/pretrained_bak_5805000.pt')
79
+
80
+ print("Loading FreeVC(24k)...")
81
+ hps = utils.get_hparams_from_file("configs/freevc-24.json")
82
+ freevc_24 = SynthesizerTrn(
83
+ hps.data.filter_length // 2 + 1,
84
+ hps.train.segment_size // hps.data.hop_length,
85
+ **hps.model).to(device)
86
+ _ = freevc_24.eval()
87
+ _ = utils.load_checkpoint("checkpoint/freevc-24.pth", freevc_24, None)
88
+
89
+ print("Loading WavLM for content...")
90
+ cmodel = WavLMModel.from_pretrained("microsoft/wavlm-large").to(device)
91
+
92
+ def convert(model, src, tgt):
93
+ with torch.no_grad():
94
+ # tgt
95
+ wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate)
96
+ wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20)
97
+ if model == "FreeVC" or model == "FreeVC (24kHz)":
98
+ g_tgt = smodel.embed_utterance(wav_tgt)
99
+ g_tgt = torch.from_numpy(g_tgt).unsqueeze(0).to(device)
100
+ else:
101
+ wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).to(device)
102
+ mel_tgt = mel_spectrogram_torch(
103
+ wav_tgt,
104
+ hps.data.filter_length,
105
+ hps.data.n_mel_channels,
106
+ hps.data.sampling_rate,
107
+ hps.data.hop_length,
108
+ hps.data.win_length,
109
+ hps.data.mel_fmin,
110
+ hps.data.mel_fmax
111
+ )
112
+ # src
113
+ wav_src, _ = librosa.load(src, sr=hps.data.sampling_rate)
114
+ wav_src = torch.from_numpy(wav_src).unsqueeze(0).to(device)
115
+ c = cmodel(wav_src).last_hidden_state.transpose(1, 2).to(device)
116
+ # infer
117
+ if model == "FreeVC":
118
+ audio = freevc.infer(c, g=g_tgt)
119
+ elif model == "FreeVC-s":
120
+ audio = freevc_s.infer(c, mel=mel_tgt)
121
+ else:
122
+ audio = freevc_24.infer(c, g=g_tgt)
123
+ audio = audio[0][0].data.cpu().float().numpy()
124
+ if model == "FreeVC" or model == "FreeVC-s":
125
+ write("out.wav", hps.data.sampling_rate, audio)
126
+ else:
127
+ write("out.wav", 24000, audio)
128
+ out = "out.wav"
129
+ return out
130
+
131
+ # GLM2
132
+
133
+ language_dict = tts_order_voice
134
+
135
+ # fix timezone in Linux
136
+ os.environ["TZ"] = "Asia/Shanghai"
137
+ try:
138
+ time.tzset() # type: ignore # pylint: disable=no-member
139
+ except Exception:
140
+ # Windows
141
+ logger.warning("Windows, cant run time.tzset()")
142
+
143
+ # model_name = "THUDM/chatglm2-6b"
144
+ model_name = "THUDM/chatglm2-6b-int4"
145
+
146
+ RETRY_FLAG = False
147
+
148
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
149
+
150
+ # model = AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda()
151
+
152
+ # 4/8 bit
153
+ # model = AutoModel.from_pretrained("THUDM/chatglm2-6b", trust_remote_code=True).quantize(4).cuda()
154
+
155
+ has_cuda = torch.cuda.is_available()
156
+
157
+ # has_cuda = False # force cpu
158
+
159
+ if has_cuda:
160
+ model_glm = (
161
+ AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda().half()
162
+ ) # 3.92G
163
+ else:
164
+ model_glm = AutoModel.from_pretrained(
165
+ model_name, trust_remote_code=True
166
+ ).float() # .float() .half().float()
167
+
168
+ model_glm = model_glm.eval()
169
+
170
+ _ = """Override Chatbot.postprocess"""
171
+
172
+
173
+ def postprocess(self, y):
174
+ if y is None:
175
+ return []
176
+ for i, (message, response) in enumerate(y):
177
+ y[i] = (
178
+ None if message is None else mdtex2html.convert((message)),
179
+ None if response is None else mdtex2html.convert(response),
180
+ )
181
+ return y
182
+
183
+
184
+ gr.Chatbot.postprocess = postprocess
185
+
186
+
187
+ def parse_text(text):
188
+ """copy from https://github.com/GaiZhenbiao/ChuanhuChatGPT/"""
189
+ lines = text.split("\n")
190
+ lines = [line for line in lines if line != ""]
191
+ count = 0
192
+ for i, line in enumerate(lines):
193
+ if "```" in line:
194
+ count += 1
195
+ items = line.split("`")
196
+ if count % 2 == 1:
197
+ lines[i] = f'<pre><code class="language-{items[-1]}">'
198
+ else:
199
+ lines[i] = "<br></code></pre>"
200
+ else:
201
+ if i > 0:
202
+ if count % 2 == 1:
203
+ line = line.replace("`", r"\`")
204
+ line = line.replace("<", "&lt;")
205
+ line = line.replace(">", "&gt;")
206
+ line = line.replace(" ", "&nbsp;")
207
+ line = line.replace("*", "&ast;")
208
+ line = line.replace("_", "&lowbar;")
209
+ line = line.replace("-", "&#45;")
210
+ line = line.replace(".", "&#46;")
211
+ line = line.replace("!", "&#33;")
212
+ line = line.replace("(", "&#40;")
213
+ line = line.replace(")", "&#41;")
214
+ line = line.replace("$", "&#36;")
215
+ lines[i] = "<br>" + line
216
+ text = "".join(lines)
217
+ return text
218
+
219
+
220
+ def predict(
221
+ RETRY_FLAG, input, chatbot, max_length, top_p, temperature, history, past_key_values
222
+ ):
223
+ try:
224
+ chatbot.append((parse_text(input), ""))
225
+ except Exception as exc:
226
+ logger.error(exc)
227
+ logger.debug(f"{chatbot=}")
228
+ _ = """
229
+ if chatbot:
230
+ chatbot[-1] = (parse_text(input), str(exc))
231
+ yield chatbot, history, past_key_values
232
+ # """
233
+ yield chatbot, history, past_key_values
234
+
235
+ for response, history, past_key_values in model_glm.stream_chat(
236
+ tokenizer,
237
+ input,
238
+ history,
239
+ past_key_values=past_key_values,
240
+ return_past_key_values=True,
241
+ max_length=max_length,
242
+ top_p=top_p,
243
+ temperature=temperature,
244
+ ):
245
+ chatbot[-1] = (parse_text(input), parse_text(response))
246
+ # chatbot[-1][-1] = parse_text(response)
247
+
248
+ yield chatbot, history, past_key_values, parse_text(response)
249
+
250
+
251
+ def trans_api(input, max_length=4096, top_p=0.8, temperature=0.2):
252
+ if max_length < 10:
253
+ max_length = 4096
254
+ if top_p < 0.1 or top_p > 1:
255
+ top_p = 0.85
256
+ if temperature <= 0 or temperature > 1:
257
+ temperature = 0.01
258
+ try:
259
+ res, _ = model_glm.chat(
260
+ tokenizer,
261
+ input,
262
+ history=[],
263
+ past_key_values=None,
264
+ max_length=max_length,
265
+ top_p=top_p,
266
+ temperature=temperature,
267
+ )
268
+ # logger.debug(f"{res=} \n{_=}")
269
+ except Exception as exc:
270
+ logger.error(f"{exc=}")
271
+ res = str(exc)
272
+
273
+ return res
274
+
275
+
276
+ def reset_user_input():
277
+ return gr.update(value="")
278
+
279
+
280
+ def reset_state():
281
+ return [], [], None, ""
282
+
283
+
284
+ # Delete last turn
285
+ def delete_last_turn(chat, history):
286
+ if chat and history:
287
+ chat.pop(-1)
288
+ history.pop(-1)
289
+ return chat, history
290
+
291
+
292
+ # Regenerate response
293
+ def retry_last_answer(
294
+ user_input, chatbot, max_length, top_p, temperature, history, past_key_values
295
+ ):
296
+ if chatbot and history:
297
+ # Removing the previous conversation from chat
298
+ chatbot.pop(-1)
299
+ # Setting up a flag to capture a retry
300
+ RETRY_FLAG = True
301
+ # Getting last message from user
302
+ user_input = history[-1][0]
303
+ # Removing bot response from the history
304
+ history.pop(-1)
305
+
306
+ yield from predict(
307
+ RETRY_FLAG, # type: ignore
308
+ user_input,
309
+ chatbot,
310
+ max_length,
311
+ top_p,
312
+ temperature,
313
+ history,
314
+ past_key_values,
315
+ )
316
+
317
+ # print
318
+
319
+ def print(text):
320
+ return text
321
+
322
+ # TTS
323
+
324
+ async def text_to_speech_edge(text, language_code):
325
+ voice = language_dict[language_code]
326
+ communicate = edge_tts.Communicate(text, voice)
327
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp_file:
328
+ tmp_path = tmp_file.name
329
+
330
+ await communicate.save(tmp_path)
331
+
332
+ return tmp_path
333
+
334
+
335
+ with gr.Blocks(title="ChatGLM2-6B-int4", theme=gr.themes.Soft(text_size="sm"), analytics_enabled=False) as demo:
336
+ gr.HTML("<center>"
337
+ "<h1>📺💕🎶 - ChatGLM2+声音克隆+视频对话:和喜欢的角色畅所欲言吧!</h1>"
338
+ "</center>")
339
+ gr.Markdown("## <center>🥳 - ChatGLM2+FreeVC+SadTalker,为您打造沉浸式的视频对话体验,支持中英双语</center>")
340
+ gr.Markdown("## <center>🌊 - 更多精彩应用,尽在[滔滔AI](http://www.talktalkai.com);滔滔AI,为爱滔滔!💕</center>")
341
+ gr.Markdown("### <center>⭐ - 如果您喜欢这个程序,欢迎给我的[GitHub项目](https://github.com/KevinWang676/ChatGLM2-Voice-Cloning)点赞支持!</center>")
342
+
343
+ with gr.Tab("🍻 - ChatGLM2聊天区"):
344
+ with gr.Accordion("📒 相关信息", open=False):
345
+ _ = f""" ChatGLM2的可选参数信息:
346
+ * Low temperature: responses will be more deterministic and focused; High temperature: responses more creative.
347
+ * Suggested temperatures -- translation: up to 0.3; chatting: > 0.4
348
+ * Top P controls dynamic vocabulary selection based on context.\n
349
+ 如果您想让ChatGLM2进行角色扮演并与之对话,请先输入恰当的提示词,如“请你扮演成动漫角色蜡笔小新并和我进行对话”;您也可以为ChatGLM2提供自定义的角色设定\n
350
+ 当您使用声音克隆功能时,请先在此程序的对应位置上传一段您喜欢的音频
351
+ """
352
+ gr.Markdown(dedent(_))
353
+ chatbot = gr.Chatbot(height=300)
354
+ with gr.Row():
355
+ with gr.Column(scale=4):
356
+ with gr.Column(scale=12):
357
+ user_input = gr.Textbox(
358
+ label="请在此处和GLM2聊天 (按回车键即可发送)",
359
+ placeholder="聊点什么吧",
360
+ )
361
+ RETRY_FLAG = gr.Checkbox(value=False, visible=False)
362
+ with gr.Column(min_width=32, scale=1):
363
+ with gr.Row():
364
+ submitBtn = gr.Button("开始和GLM2交流吧", variant="primary")
365
+ deleteBtn = gr.Button("删除最新一轮对话", variant="secondary")
366
+ retryBtn = gr.Button("重新生成最新一轮对话", variant="secondary")
367
+
368
+ with gr.Accordion("🔧 更多设置", open=False):
369
+ with gr.Row():
370
+ emptyBtn = gr.Button("清空所有聊天记录")
371
+ max_length = gr.Slider(
372
+ 0,
373
+ 32768,
374
+ value=8192,
375
+ step=1.0,
376
+ label="Maximum length",
377
+ interactive=True,
378
+ )
379
+ top_p = gr.Slider(
380
+ 0, 1, value=0.85, step=0.01, label="Top P", interactive=True
381
+ )
382
+ temperature = gr.Slider(
383
+ 0.01, 1, value=0.95, step=0.01, label="Temperature", interactive=True
384
+ )
385
+
386
+
387
+ with gr.Row():
388
+ test1 = gr.Textbox(label="GLM2的最新回答 (可编辑)", lines = 3)
389
+ with gr.Column():
390
+ language = gr.Dropdown(choices=list(language_dict.keys()), value="普通话 (中国大陆)-Xiaoxiao-女", label="请选择文本对应的语言及您喜欢的说话人")
391
+ tts_btn = gr.Button("生成对应的音频吧", variant="primary")
392
+ output_audio = gr.Audio(type="filepath", label="为您生成的音频", interactive=False)
393
+
394
+ tts_btn.click(text_to_speech_edge, inputs=[test1, language], outputs=[output_audio])
395
+
396
+ with gr.Row():
397
+ model_choice = gr.Dropdown(choices=["FreeVC", "FreeVC-s", "FreeVC (24kHz)"], value="FreeVC (24kHz)", label="Model", visible=False)
398
+ audio1 = output_audio
399
+ audio2 = gr.Audio(label="请上传您喜欢的声音进行声音克隆", type='filepath')
400
+ clone_btn = gr.Button("开始AI声音克隆吧", variant="primary")
401
+ audio_cloned = gr.Audio(label="为您生成的专属声音克隆音频", type='filepath')
402
+
403
+ clone_btn.click(convert, inputs=[model_choice, audio1, audio2], outputs=[audio_cloned])
404
+
405
+ history = gr.State([])
406
+ past_key_values = gr.State(None)
407
+
408
+ user_input.submit(
409
+ predict,
410
+ [
411
+ RETRY_FLAG,
412
+ user_input,
413
+ chatbot,
414
+ max_length,
415
+ top_p,
416
+ temperature,
417
+ history,
418
+ past_key_values,
419
+ ],
420
+ [chatbot, history, past_key_values, test1],
421
+ show_progress="full",
422
+ )
423
+ submitBtn.click(
424
+ predict,
425
+ [
426
+ RETRY_FLAG,
427
+ user_input,
428
+ chatbot,
429
+ max_length,
430
+ top_p,
431
+ temperature,
432
+ history,
433
+ past_key_values,
434
+ ],
435
+ [chatbot, history, past_key_values, test1],
436
+ show_progress="full",
437
+ api_name="predict",
438
+ )
439
+ submitBtn.click(reset_user_input, [], [user_input])
440
+
441
+ emptyBtn.click(
442
+ reset_state, outputs=[chatbot, history, past_key_values, test1], show_progress="full"
443
+ )
444
+
445
+ retryBtn.click(
446
+ retry_last_answer,
447
+ inputs=[
448
+ user_input,
449
+ chatbot,
450
+ max_length,
451
+ top_p,
452
+ temperature,
453
+ history,
454
+ past_key_values,
455
+ ],
456
+ # outputs = [chatbot, history, last_user_message, user_message]
457
+ outputs=[chatbot, history, past_key_values, test1],
458
+ )
459
+ deleteBtn.click(delete_last_turn, [chatbot, history], [chatbot, history])
460
+
461
+ with gr.Accordion("📔 提示词示例", open=False):
462
+ etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
463
+ examples = gr.Examples(
464
+ examples=[
465
+ ["Explain the plot of Cinderella in a sentence."],
466
+ [
467
+ "How long does it take to become proficient in French, and what are the best methods for retaining information?"
468
+ ],
469
+ ["What are some common mistakes to avoid when writing code?"],
470
+ ["Build a prompt to generate a beautiful portrait of a horse"],
471
+ ["Suggest four metaphors to describe the benefits of AI"],
472
+ ["Write a pop song about leaving home for the sandy beaches."],
473
+ ["Write a summary demonstrating my ability to tame lions"],
474
+ ["鲁迅和周树人什么关系"],
475
+ ["从前有一头牛,这头牛后面有什么?"],
476
+ ["正无穷大加一大于正无穷大吗?"],
477
+ ["正无穷大加正无穷大大于正无穷大吗?"],
478
+ ["-2的平方根等于什么"],
479
+ ["树上有5只鸟,猎人开枪打死了一只。树上还有几只鸟?"],
480
+ ["树上有11只鸟,猎人开枪打死了一只。树上还有几只鸟?提示:需考虑鸟可能受惊吓飞走。"],
481
+ ["鲁迅和周树人什么关系 用英文回答"],
482
+ ["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"],
483
+ [f"{etext} 翻成中文,列出3个版本"],
484
+ [f"{etext} \n 翻成中文,保留原意,但使用文学性的语言。不要写解释。列出3个版本"],
485
+ ["js 判断一个数是不是质数"],
486
+ ["js 实现python 的 range(10)"],
487
+ ["js 实现python 的 [*(range(10)]"],
488
+ ["假定 1 + 2 = 4, 试求 7 + 8"],
489
+ ["Erkläre die Handlung von Cinderella in einem Satz."],
490
+ ["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch"],
491
+ ],
492
+ inputs=[user_input],
493
+ examples_per_page=30,
494
+ )
495
+
496
+ with gr.Accordion("For Chat/Translation API", open=False, visible=False):
497
+ input_text = gr.Text()
498
+ tr_btn = gr.Button("Go", variant="primary")
499
+ out_text = gr.Text()
500
+ tr_btn.click(
501
+ trans_api,
502
+ [input_text, max_length, top_p, temperature],
503
+ out_text,
504
+ # show_progress="full",
505
+ api_name="tr",
506
+ )
507
+ _ = """
508
+ input_text.submit(
509
+ trans_api,
510
+ [input_text, max_length, top_p, temperature],
511
+ out_text,
512
+ show_progress="full",
513
+ api_name="tr1",
514
+ )
515
+ # """
516
+ with gr.Tab("📺 - 视频聊天区"):
517
+ with gr.Row().style(equal_height=False):
518
+ with gr.Column(variant='panel'):
519
+ with gr.Tabs(elem_id="sadtalker_source_image"):
520
+ with gr.TabItem('图片上传'):
521
+ with gr.Row():
522
+ source_image = gr.Image(label="请上传一张您喜欢角色的图片", source="upload", type="filepath", elem_id="img2img_image").style(width=512)
523
+
524
+
525
+ with gr.Tabs(elem_id="sadtalker_driven_audio"):
526
+ with gr.TabItem('💡您还可以将视频下载到本地'):
527
+
528
+ with gr.Row():
529
+ driven_audio = audio_cloned
530
+ driven_audio_no = gr.Audio(label="Use IDLE mode, no audio is required", source="upload", type="filepath", visible=False)
531
+
532
+ with gr.Column():
533
+ use_idle_mode = gr.Checkbox(label="Use Idle Animation", visible=False)
534
+ length_of_audio = gr.Number(value=5, label="The length(seconds) of the generated video.", visible=False)
535
+ use_idle_mode.change(toggle_audio_file, inputs=use_idle_mode, outputs=[driven_audio, driven_audio_no]) # todo
536
+
537
+ with gr.Row():
538
+ ref_video = gr.Video(label="Reference Video", source="upload", type="filepath", elem_id="vidref", visible=False).style(width=512)
539
+
540
+ with gr.Column():
541
+ use_ref_video = gr.Checkbox(label="Use Reference Video", visible=False)
542
+ ref_info = gr.Radio(['pose', 'blink','pose+blink', 'all'], value='pose', label='Reference Video',info="How to borrow from reference Video?((fully transfer, aka, video driving mode))", visible=False)
543
+
544
+ ref_video.change(ref_video_fn, inputs=ref_video, outputs=[use_ref_video]) # todo
545
+
546
+
547
+ with gr.Column(variant='panel'):
548
+ with gr.Tabs(elem_id="sadtalker_checkbox"):
549
+ with gr.TabItem('视频设置'):
550
+ with gr.Column(variant='panel'):
551
+ # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
552
+ # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
553
+ with gr.Row():
554
+ pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0, visible=False) #
555
+ exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1, visible=False) #
556
+ blink_every = gr.Checkbox(label="use eye blink", value=True, visible=False)
557
+
558
+ with gr.Row():
559
+ size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model?", visible=False) #
560
+ preprocess_type = gr.Radio(['crop', 'full'], value='crop', label='是否聚焦角色面部', info="crop:视频会聚焦角色面部;full:视频会显示图片全貌")
561
+
562
+ with gr.Row():
563
+ is_still_mode = gr.Checkbox(label="静态模式 (开启静态模式,角色的面部动作会减少;默认开启)", value=True)
564
+ facerender = gr.Radio(['facevid2vid','pirender'], value='facevid2vid', label='facerender', info="which face render?", visible=False)
565
+
566
+ with gr.Row():
567
+ batch_size = gr.Slider(label="Batch size (数值越大,生成速度越快;若显卡性能好,可增大数值)", step=1, maximum=32, value=2)
568
+ enhancer = gr.Checkbox(label="GFPGAN as Face enhancer", value=True, visible=False)
569
+
570
+ submit = gr.Button('开始视频聊天吧', elem_id="sadtalker_generate", variant='primary')
571
+
572
+ with gr.Tabs(elem_id="sadtalker_genearted"):
573
+ gen_video = gr.Video(label="为您生成的专属视频", format="mp4").style(width=256)
574
+
575
+
576
+
577
+ submit.click(
578
+ fn=sad_talker.test,
579
+ inputs=[source_image,
580
+ driven_audio,
581
+ preprocess_type,
582
+ is_still_mode,
583
+ enhancer,
584
+ batch_size,
585
+ size_of_image,
586
+ pose_style,
587
+ facerender,
588
+ exp_weight,
589
+ use_ref_video,
590
+ ref_video,
591
+ ref_info,
592
+ use_idle_mode,
593
+ length_of_audio,
594
+ blink_every
595
+ ],
596
+ outputs=[gen_video]
597
+ )
598
+ gr.Markdown("### <center>注意❗:请不要生成会对个人以及组织造成侵害的内容,此程序仅供科研、学习及个人娱乐使用。</center>")
599
+ gr.Markdown("<center>💡- 如何使用此程序:输入您对ChatGLM的提问后,依次点击“开始和GLM2交流吧”、“生成对应的音频吧”、“开始AI声音克隆吧”、“开始视频聊天吧”三个按键即可;使用声音克隆功能时,请先上传一段您喜欢的音频</center>")
600
+ gr.HTML('''
601
+ <div class="footer">
602
+ <p>🌊🏞️🎶 - 江水东流急,滔滔无尽声。 明·顾璘
603
+ </p>
604
+ </div>
605
+ ''')
606
+
607
+
608
+ demo.queue().launch(show_error=True, debug=True)
.ipynb_checkpoints/app_sadtalker-checkpoint.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, sys
2
+ import gradio as gr
3
+ from src.gradio_demo import SadTalker
4
+
5
+
6
+ try:
7
+ import webui # in webui
8
+ in_webui = True
9
+ except:
10
+ in_webui = False
11
+
12
+
13
+ def toggle_audio_file(choice):
14
+ if choice == False:
15
+ return gr.update(visible=True), gr.update(visible=False)
16
+ else:
17
+ return gr.update(visible=False), gr.update(visible=True)
18
+
19
+ def ref_video_fn(path_of_ref_video):
20
+ if path_of_ref_video is not None:
21
+ return gr.update(value=True)
22
+ else:
23
+ return gr.update(value=False)
24
+
25
+ def sadtalker_demo(checkpoint_path='checkpoints', config_path='src/config', warpfn=None):
26
+
27
+ sad_talker = SadTalker(checkpoint_path, config_path, lazy_load=True)
28
+
29
+ with gr.Blocks(analytics_enabled=False) as sadtalker_interface:
30
+ gr.Markdown("<div align='center'> <h2> 😭 SadTalker: Learning Realistic 3D Motion Coefficients for Stylized Audio-Driven Single Image Talking Face Animation (CVPR 2023) </span> </h2> \
31
+ <a style='font-size:18px;color: #efefef' href='https://arxiv.org/abs/2211.12194'>Arxiv</a> &nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
32
+ <a style='font-size:18px;color: #efefef' href='https://sadtalker.github.io'>Homepage</a> &nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
33
+ <a style='font-size:18px;color: #efefef' href='https://github.com/Winfredy/SadTalker'> Github </div>")
34
+
35
+ with gr.Row().style(equal_height=False):
36
+ with gr.Column(variant='panel'):
37
+ with gr.Tabs(elem_id="sadtalker_source_image"):
38
+ with gr.TabItem('Upload image'):
39
+ with gr.Row():
40
+ source_image = gr.Image(label="Source image", source="upload", type="filepath", elem_id="img2img_image").style(width=512)
41
+
42
+ with gr.Tabs(elem_id="sadtalker_driven_audio"):
43
+ with gr.TabItem('Upload OR TTS'):
44
+ with gr.Column(variant='panel'):
45
+ driven_audio = gr.Audio(label="Input audio", source="upload", type="filepath")
46
+
47
+ if sys.platform != 'win32' and not in_webui:
48
+ from src.utils.text2speech import TTSTalker
49
+ tts_talker = TTSTalker()
50
+ with gr.Column(variant='panel'):
51
+ input_text = gr.Textbox(label="Generating audio from text", lines=5, placeholder="please enter some text here, we genreate the audio from text using @Coqui.ai TTS.")
52
+ tts = gr.Button('Generate audio',elem_id="sadtalker_audio_generate", variant='primary')
53
+ tts.click(fn=tts_talker.test, inputs=[input_text], outputs=[driven_audio])
54
+
55
+ with gr.Column(variant='panel'):
56
+ with gr.Tabs(elem_id="sadtalker_checkbox"):
57
+ with gr.TabItem('Settings'):
58
+ gr.Markdown("need help? please visit our [best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md) for more detials")
59
+ with gr.Column(variant='panel'):
60
+ # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
61
+ # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
62
+ pose_style = gr.Slider(minimum=0, maximum=46, step=1, label="Pose style", value=0) #
63
+ size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model?") #
64
+ preprocess_type = gr.Radio(['crop', 'resize','full', 'extcrop', 'extfull'], value='crop', label='preprocess', info="How to handle input image?")
65
+ is_still_mode = gr.Checkbox(label="Still Mode (fewer hand motion, works with preprocess `full`)")
66
+ batch_size = gr.Slider(label="batch size in generation", step=1, maximum=10, value=2)
67
+ enhancer = gr.Checkbox(label="GFPGAN as Face enhancer")
68
+ submit = gr.Button('Generate', elem_id="sadtalker_generate", variant='primary')
69
+
70
+ with gr.Tabs(elem_id="sadtalker_genearted"):
71
+ gen_video = gr.Video(label="Generated video", format="mp4").style(width=256)
72
+
73
+ if warpfn:
74
+ submit.click(
75
+ fn=warpfn(sad_talker.test),
76
+ inputs=[source_image,
77
+ driven_audio,
78
+ preprocess_type,
79
+ is_still_mode,
80
+ enhancer,
81
+ batch_size,
82
+ size_of_image,
83
+ pose_style
84
+ ],
85
+ outputs=[gen_video]
86
+ )
87
+ else:
88
+ submit.click(
89
+ fn=sad_talker.test,
90
+ inputs=[source_image,
91
+ driven_audio,
92
+ preprocess_type,
93
+ is_still_mode,
94
+ enhancer,
95
+ batch_size,
96
+ size_of_image,
97
+ pose_style
98
+ ],
99
+ outputs=[gen_video]
100
+ )
101
+
102
+ return sadtalker_interface
103
+
104
+
105
+ if __name__ == "__main__":
106
+
107
+ demo = sadtalker_demo()
108
+ demo.queue()
109
+ demo.launch(share=True)
110
+
111
+
ChatGLM2-SadTalker-VC-1/.flake8 ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [flake8]
2
+ ignore =
3
+ # E203 whitespace before ':'
4
+ E203
5
+ D203,
6
+ # line too long
7
+ E501
8
+ per-file-ignores =
9
+ # imported but unused
10
+ # __init__.py: F401
11
+ test_*.py: F401
12
+ exclude =
13
+ .git,
14
+ __pycache__,
15
+ docs/source/conf.py,
16
+ old,
17
+ build,
18
+ dist,
19
+ .venv
20
+ pad*.py
21
+ max-complexity = 25
ChatGLM2-SadTalker-VC-1/.gitattributes ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tflite filter=lfs diff=lfs merge=lfs -text
29
+ *.tgz filter=lfs diff=lfs merge=lfs -text
30
+ *.wasm filter=lfs diff=lfs merge=lfs -text
31
+ *.xz filter=lfs diff=lfs merge=lfs -text
32
+ *.zip filter=lfs diff=lfs merge=lfs -text
33
+ *.zst filter=lfs diff=lfs merge=lfs -text
34
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
35
+ checkpoints/BFM_Fitting/01_MorphableModel.mat filter=lfs diff=lfs merge=lfs -text
36
+ checkpoints/BFM_Fitting/BFM09_model_info.mat filter=lfs diff=lfs merge=lfs -text
37
+ checkpoints/facevid2vid_00189-model.pth.tar filter=lfs diff=lfs merge=lfs -text
38
+ checkpoints/mapping_00229-model.pth.tar filter=lfs diff=lfs merge=lfs -text
39
+ checkpoints/shape_predictor_68_face_landmarks.dat filter=lfs diff=lfs merge=lfs -text
40
+ examples/driven_audio/chinese_news.wav filter=lfs diff=lfs merge=lfs -text
41
+ examples/driven_audio/deyu.wav filter=lfs diff=lfs merge=lfs -text
42
+ examples/driven_audio/eluosi.wav filter=lfs diff=lfs merge=lfs -text
43
+ examples/driven_audio/fayu.wav filter=lfs diff=lfs merge=lfs -text
44
+ examples/driven_audio/imagine.wav filter=lfs diff=lfs merge=lfs -text
45
+ examples/driven_audio/japanese.wav filter=lfs diff=lfs merge=lfs -text
46
+ examples/source_image/art_16.png filter=lfs diff=lfs merge=lfs -text
47
+ examples/source_image/art_17.png filter=lfs diff=lfs merge=lfs -text
48
+ examples/source_image/art_3.png filter=lfs diff=lfs merge=lfs -text
49
+ examples/source_image/art_4.png filter=lfs diff=lfs merge=lfs -text
50
+ examples/source_image/art_5.png filter=lfs diff=lfs merge=lfs -text
51
+ examples/source_image/art_8.png filter=lfs diff=lfs merge=lfs -text
52
+ examples/source_image/art_9.png filter=lfs diff=lfs merge=lfs -text
ChatGLM2-SadTalker-VC-1/.gitignore ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # poetry
98
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102
+ #poetry.lock
103
+
104
+ # pdm
105
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106
+ #pdm.lock
107
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108
+ # in version control.
109
+ # https://pdm.fming.dev/#use-with-ide
110
+ .pdm.toml
111
+
112
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113
+ __pypackages__/
114
+
115
+ # Celery stuff
116
+ celerybeat-schedule
117
+ celerybeat.pid
118
+
119
+ # SageMath parsed files
120
+ *.sage.py
121
+
122
+ # Environments
123
+ .env
124
+ .venv
125
+ env/
126
+ venv/
127
+ ENV/
128
+ env.bak/
129
+ venv.bak/
130
+
131
+ # Spyder project settings
132
+ .spyderproject
133
+ .spyproject
134
+
135
+ # Rope project settings
136
+ .ropeproject
137
+
138
+ # mkdocs documentation
139
+ /site
140
+
141
+ # mypy
142
+ .mypy_cache/
143
+ .dmypy.json
144
+ dmypy.json
145
+
146
+ # Pyre type checker
147
+ .pyre/
148
+
149
+ # pytype static type analyzer
150
+ .pytype/
151
+
152
+ # Cython debug symbols
153
+ cython_debug/
154
+
155
+ results/
156
+ checkpoints/
157
+ gradio_cached_examples/
158
+ gfpgan/
159
+ start.sh
ChatGLM2-SadTalker-VC-1/.ipynb_checkpoints/requirements-checkpoint.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ scipy
2
+ transformers
3
+ librosa==0.8.1
4
+ webrtcvad==2.0.10
5
+ protobuf
6
+ cpm_kernels
7
+ mdtex2html
8
+ sentencepiece
9
+ accelerate
10
+ loguru
11
+ edge_tts
12
+ altair
ChatGLM2-SadTalker-VC-1/Dockerfile ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04
2
+ ENV DEBIAN_FRONTEND=noninteractive
3
+ RUN apt-get update && \
4
+ apt-get upgrade -y && \
5
+ apt-get install -y --no-install-recommends \
6
+ git \
7
+ zip \
8
+ unzip \
9
+ git-lfs \
10
+ wget \
11
+ curl \
12
+ # ffmpeg \
13
+ ffmpeg \
14
+ x264 \
15
+ # python build dependencies \
16
+ build-essential \
17
+ libssl-dev \
18
+ zlib1g-dev \
19
+ libbz2-dev \
20
+ libreadline-dev \
21
+ libsqlite3-dev \
22
+ libncursesw5-dev \
23
+ xz-utils \
24
+ tk-dev \
25
+ libxml2-dev \
26
+ libxmlsec1-dev \
27
+ libffi-dev \
28
+ liblzma-dev && \
29
+ apt-get clean && \
30
+ rm -rf /var/lib/apt/lists/*
31
+
32
+ RUN useradd -m -u 1000 user
33
+ USER user
34
+ ENV HOME=/home/user \
35
+ PATH=/home/user/.local/bin:${PATH}
36
+ WORKDIR ${HOME}/app
37
+
38
+ RUN curl https://pyenv.run | bash
39
+ ENV PATH=${HOME}/.pyenv/shims:${HOME}/.pyenv/bin:${PATH}
40
+ ENV PYTHON_VERSION=3.10.9
41
+ RUN pyenv install ${PYTHON_VERSION} && \
42
+ pyenv global ${PYTHON_VERSION} && \
43
+ pyenv rehash && \
44
+ pip install --no-cache-dir -U pip setuptools wheel
45
+
46
+ RUN pip install --no-cache-dir -U torch==1.12.1 torchvision==0.13.1
47
+ COPY --chown=1000 requirements.txt /tmp/requirements.txt
48
+ RUN pip install --no-cache-dir -U -r /tmp/requirements.txt
49
+
50
+ COPY --chown=1000 . ${HOME}/app
51
+ RUN ls -a
52
+ ENV PYTHONPATH=${HOME}/app \
53
+ PYTHONUNBUFFERED=1 \
54
+ GRADIO_ALLOW_FLAGGING=never \
55
+ GRADIO_NUM_PORTS=1 \
56
+ GRADIO_SERVER_NAME=0.0.0.0 \
57
+ GRADIO_THEME=huggingface \
58
+ SYSTEM=spaces
59
+ CMD ["python", "app.py"]
ChatGLM2-SadTalker-VC-1/LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2023 Tencent AI Lab
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
ChatGLM2-SadTalker-VC-1/README.md ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: ChatGLM2-SadTalker
3
+ emoji: 📺
4
+ colorFrom: purple
5
+ colorTo: green
6
+ sdk: gradio
7
+ sdk_version: 3.23.0
8
+ app_file: app.py
9
+ pinned: false
10
+ license: mit
11
+ duplicated_from: kevinwang676/ChatGLM2-SadTalker-VC
12
+ ---
13
+
14
+
15
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
ChatGLM2-SadTalker-VC-1/packages.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ ffmpeg
2
+ libsndfile1
ChatGLM2-SadTalker-VC-1/requirements.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ scipy
2
+ transformers
3
+ librosa==0.8.1
4
+ webrtcvad==2.0.10
5
+ protobuf
6
+ cpm_kernels
7
+ mdtex2html
8
+ sentencepiece
9
+ accelerate
10
+ loguru
11
+ edge_tts
12
+ altair
LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2023 Tencent AI Lab
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
README.md CHANGED
@@ -1,13 +1,268 @@
1
- ---
2
- title: ChatGLM2 SadTalker
3
- emoji: 📚
4
- colorFrom: blue
5
- colorTo: blue
6
- sdk: gradio
7
- sdk_version: 3.37.0
8
- app_file: app.py
9
- pinned: false
10
- license: mit
11
- ---
12
-
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <div align="center">
2
+
3
+ <img src='https://user-images.githubusercontent.com/4397546/229094115-862c747e-7397-4b54-ba4a-bd368bfe2e0f.png' width='500px'/>
4
+
5
+
6
+ <!--<h2> 😭 SadTalker: <span style="font-size:12px">Learning Realistic 3D Motion Coefficients for Stylized Audio-Driven Single Image Talking Face Animation </span> </h2> -->
7
+
8
+ <a href='https://arxiv.org/abs/2211.12194'><img src='https://img.shields.io/badge/ArXiv-PDF-red'></a> &nbsp; <a href='https://sadtalker.github.io'><img src='https://img.shields.io/badge/Project-Page-Green'></a> &nbsp; [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Winfredy/SadTalker/blob/main/quick_demo.ipynb) &nbsp; [![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/vinthony/SadTalker) &nbsp; [![sd webui-colab](https://img.shields.io/badge/Automatic1111-Colab-green)](https://colab.research.google.com/github/camenduru/stable-diffusion-webui-colab/blob/main/video/stable/stable_diffusion_1_5_video_webui_colab.ipynb) &nbsp; [![Replicate](https://replicate.com/cjwbw/sadtalker/badge)](https://replicate.com/cjwbw/sadtalker)
9
+
10
+ <div>
11
+ <a target='_blank'>Wenxuan Zhang <sup>*,1,2</sup> </a>&emsp;
12
+ <a href='https://vinthony.github.io/' target='_blank'>Xiaodong Cun <sup>*,2</a>&emsp;
13
+ <a href='https://xuanwangvc.github.io/' target='_blank'>Xuan Wang <sup>3</sup></a>&emsp;
14
+ <a href='https://yzhang2016.github.io/' target='_blank'>Yong Zhang <sup>2</sup></a>&emsp;
15
+ <a href='https://xishen0220.github.io/' target='_blank'>Xi Shen <sup>2</sup></a>&emsp; </br>
16
+ <a href='https://yuguo-xjtu.github.io/' target='_blank'>Yu Guo<sup>1</sup> </a>&emsp;
17
+ <a href='https://scholar.google.com/citations?hl=zh-CN&user=4oXBp9UAAAAJ' target='_blank'>Ying Shan <sup>2</sup> </a>&emsp;
18
+ <a target='_blank'>Fei Wang <sup>1</sup> </a>&emsp;
19
+ </div>
20
+ <br>
21
+ <div>
22
+ <sup>1</sup> Xi'an Jiaotong University &emsp; <sup>2</sup> Tencent AI Lab &emsp; <sup>3</sup> Ant Group &emsp;
23
+ </div>
24
+ <br>
25
+ <i><strong><a href='https://arxiv.org/abs/2211.12194' target='_blank'>CVPR 2023</a></strong></i>
26
+ <br>
27
+ <br>
28
+
29
+
30
+ ![sadtalker](https://user-images.githubusercontent.com/4397546/222490039-b1f6156b-bf00-405b-9fda-0c9a9156f991.gif)
31
+
32
+ <b>TL;DR: &nbsp;&nbsp;&nbsp;&nbsp;&nbsp; single portrait image 🙎‍♂️ &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;+ &nbsp;&nbsp;&nbsp;&nbsp;&nbsp; audio 🎤 &nbsp;&nbsp;&nbsp;&nbsp;&nbsp; = &nbsp;&nbsp;&nbsp;&nbsp;&nbsp; talking head video 🎞.</b>
33
+
34
+ <br>
35
+
36
+ </div>
37
+
38
+
39
+
40
+ ## 🔥 Highlight
41
+
42
+ - 🔥 The extension of the [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) is online. Checkout more details [here](docs/webui_extension.md).
43
+
44
+ https://user-images.githubusercontent.com/4397546/231495639-5d4bb925-ea64-4a36-a519-6389917dac29.mp4
45
+
46
+ - 🔥 `full image mode` is online! checkout [here](https://github.com/Winfredy/SadTalker#full-bodyimage-generation) for more details.
47
+
48
+ | still+enhancer in v0.0.1 | still + enhancer in v0.0.2 | [input image @bagbag1815](https://twitter.com/bagbag1815/status/1642754319094108161) |
49
+ |:--------------------: |:--------------------: | :----: |
50
+ | <video src="https://user-images.githubusercontent.com/48216707/229484996-5d7be64f-2553-4c9e-a452-c5cf0b8ebafe.mp4" type="video/mp4"> </video> | <video src="https://user-images.githubusercontent.com/4397546/230717873-355b7bf3-d3de-49f9-a439-9220e623fce7.mp4" type="video/mp4"> </video> | <img src='./examples/source_image/full_body_2.png' width='380'>
51
+
52
+ - 🔥 Several new mode, eg, `still mode`, `reference mode`, `resize mode` are online for better and custom applications.
53
+
54
+ - 🔥 Happy to see more community demos at [bilibili](https://search.bilibili.com/all?keyword=sadtalker&from_source=webtop_search&spm_id_from=333.1007&search_source=3
55
+ ), [Youtube](https://www.youtube.com/results?search_query=sadtalker&sp=CAM%253D) and [twitter #sadtalker](https://twitter.com/search?q=%23sadtalker&src=typed_query).
56
+
57
+ ## 📋 Changelog (Previous changelog can be founded [here](docs/changlelog.md))
58
+
59
+ - __[2023.06.12]__: add more new features in WEBUI extension, see the discussion [here](https://github.com/OpenTalker/SadTalker/discussions/386).
60
+
61
+ - __[2023.06.05]__: release a new 512 beta face model. Fixed some bugs and improve the performance.
62
+
63
+ - __[2023.04.15]__: Adding automatic1111 colab by @camenduru, thanks for this awesome colab: [![sd webui-colab](https://img.shields.io/badge/Automatic1111-Colab-green)](https://colab.research.google.com/github/camenduru/stable-diffusion-webui-colab/blob/main/video/stable/stable_diffusion_1_5_video_webui_colab.ipynb).
64
+
65
+ - __[2023.04.12]__: adding a more detailed sd-webui installation document, fixed reinstallation problem.
66
+
67
+ - __[2023.04.12]__: Fixed the sd-webui safe issues becasue of the 3rd packages, optimize the output path in `sd-webui-extension`.
68
+
69
+ - __[2023.04.08]__: ❗️❗️❗️ In v0.0.2, we add a logo watermark to the generated video to prevent abusing since it is very realistic.
70
+
71
+ - __[2023.04.08]__: v0.0.2, full image animation, adding baidu driver for download checkpoints. Optimizing the logic about enhancer.
72
+
73
+
74
+ ## 🚧 TODO: See the Discussion https://github.com/OpenTalker/SadTalker/issues/280
75
+
76
+ ## If you have any problem, please view our [FAQ](docs/FAQ.md) before opening an issue.
77
+
78
+
79
+
80
+ ## ⚙️ 1. Installation.
81
+
82
+ Tutorials from communities: [中文windows教程](https://www.bilibili.com/video/BV1Dc411W7V6/) | [日本語コース](https://br-d.fanbox.cc/posts/5685086?utm_campaign=manage_post_page&utm_medium=share&utm_source=twitter)
83
+
84
+ ### Linux:
85
+
86
+ 1. Installing [anaconda](https://www.anaconda.com/), python and git.
87
+
88
+ 2. Creating the env and install the requirements.
89
+ ```bash
90
+ git clone https://github.com/Winfredy/SadTalker.git
91
+
92
+ cd SadTalker
93
+
94
+ conda create -n sadtalker python=3.8
95
+
96
+ conda activate sadtalker
97
+
98
+ pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113
99
+
100
+ conda install ffmpeg
101
+
102
+ pip install -r requirements.txt
103
+
104
+ ### tts is optional for gradio demo.
105
+ ### pip install TTS
106
+
107
+ ```
108
+ ### Windows ([中文windows教程](https://www.bilibili.com/video/BV1Dc411W7V6/)):
109
+
110
+ 1. Install [Python 3.10.6](https://www.python.org/downloads/windows/), checking "Add Python to PATH".
111
+ 2. Install [git](https://git-scm.com/download/win) manually (OR `scoop install git` via [scoop](https://scoop.sh/)).
112
+ 3. Install `ffmpeg`, following [this instruction](https://www.wikihow.com/Install-FFmpeg-on-Windows) (OR using `scoop install ffmpeg` via [scoop](https://scoop.sh/)).
113
+ 4. Download our SadTalker repository, for example by running `git clone https://github.com/Winfredy/SadTalker.git`.
114
+ 5. Download the `checkpoint` and `gfpgan` [below↓](https://github.com/Winfredy/SadTalker#-2-download-trained-models).
115
+ 5. Run `start.bat` from Windows Explorer as normal, non-administrator, user, a gradio WebUI demo will be started.
116
+
117
+ ### Macbook:
118
+
119
+ More tips about installnation on Macbook and the Docker file can be founded [here](docs/install.md)
120
+
121
+ ## 📥 2. Download Trained Models.
122
+
123
+ You can run the following script to put all the models in the right place.
124
+
125
+ ```bash
126
+ bash scripts/download_models.sh
127
+ ```
128
+
129
+ Other alternatives:
130
+ > we also provide an offline patch (`gfpgan/`), thus, no model will be downloaded when generating.
131
+
132
+ **Google Driver**: download our pre-trained model from [ this link (main checkpoints)](https://drive.google.com/file/d/1gwWh45pF7aelNP_P78uDJL8Sycep-K7j/view?usp=sharing) and [ gfpgan (offline patch)](https://drive.google.com/file/d/19AIBsmfcHW6BRJmeqSFlG5fL445Xmsyi?usp=sharing)
133
+
134
+ **Github Release Page**: download all the files from the [lastest github release page](https://github.com/Winfredy/SadTalker/releases), and then, put it in ./checkpoints.
135
+
136
+ **百度云盘**: we provided the downloaded model in [checkpoints, 提取码: sadt.](https://pan.baidu.com/s/1P4fRgk9gaSutZnn8YW034Q?pwd=sadt) And [gfpgan, 提取码: sadt.](https://pan.baidu.com/s/1kb1BCPaLOWX1JJb9Czbn6w?pwd=sadt)
137
+
138
+
139
+
140
+ <details><summary>Model Details</summary>
141
+
142
+
143
+ Model explains:
144
+
145
+ ##### New version
146
+ | Model | Description
147
+ | :--- | :----------
148
+ |checkpoints/mapping_00229-model.pth.tar | Pre-trained MappingNet in Sadtalker.
149
+ |checkpoints/mapping_00109-model.pth.tar | Pre-trained MappingNet in Sadtalker.
150
+ |checkpoints/SadTalker_V0.0.2_256.safetensors | packaged sadtalker checkpoints of old version, 256 face render).
151
+ |checkpoints/SadTalker_V0.0.2_512.safetensors | packaged sadtalker checkpoints of old version, 512 face render).
152
+ |gfpgan/weights | Face detection and enhanced models used in `facexlib` and `gfpgan`.
153
+
154
+
155
+ ##### Old version
156
+ | Model | Description
157
+ | :--- | :----------
158
+ |checkpoints/auido2exp_00300-model.pth | Pre-trained ExpNet in Sadtalker.
159
+ |checkpoints/auido2pose_00140-model.pth | Pre-trained PoseVAE in Sadtalker.
160
+ |checkpoints/mapping_00229-model.pth.tar | Pre-trained MappingNet in Sadtalker.
161
+ |checkpoints/mapping_00109-model.pth.tar | Pre-trained MappingNet in Sadtalker.
162
+ |checkpoints/facevid2vid_00189-model.pth.tar | Pre-trained face-vid2vid model from [the reappearance of face-vid2vid](https://github.com/zhanglonghao1992/One-Shot_Free-View_Neural_Talking_Head_Synthesis).
163
+ |checkpoints/epoch_20.pth | Pre-trained 3DMM extractor in [Deep3DFaceReconstruction](https://github.com/microsoft/Deep3DFaceReconstruction).
164
+ |checkpoints/wav2lip.pth | Highly accurate lip-sync model in [Wav2lip](https://github.com/Rudrabha/Wav2Lip).
165
+ |checkpoints/shape_predictor_68_face_landmarks.dat | Face landmark model used in [dilb](http://dlib.net/).
166
+ |checkpoints/BFM | 3DMM library file.
167
+ |checkpoints/hub | Face detection models used in [face alignment](https://github.com/1adrianb/face-alignment).
168
+ |gfpgan/weights | Face detection and enhanced models used in `facexlib` and `gfpgan`.
169
+
170
+ The final folder will be shown as:
171
+
172
+ <img width="331" alt="image" src="https://user-images.githubusercontent.com/4397546/232511411-4ca75cbf-a434-48c5-9ae0-9009e8316484.png">
173
+
174
+
175
+ </details>
176
+
177
+ ## 🔮 3. Quick Start ([Best Practice](docs/best_practice.md)).
178
+
179
+ ### WebUI Demos:
180
+
181
+ **Online**: [Huggingface](https://huggingface.co/spaces/vinthony/SadTalker) | [SDWebUI-Colab](https://colab.research.google.com/github/camenduru/stable-diffusion-webui-colab/blob/main/video/stable/stable_diffusion_1_5_video_webui_colab.ipynb) | [Colab](https://colab.research.google.com/github/Winfredy/SadTalker/blob/main/quick_demo.ipynb)
182
+
183
+ **Local Autiomatic1111 stable-diffusion webui extension**: please refer to [Autiomatic1111 stable-diffusion webui docs](docs/webui_extension.md).
184
+
185
+ **Local gradio demo(highly recommanded!)**: Similar to our [hugging-face demo](https://huggingface.co/spaces/vinthony/SadTalker) can be run by:
186
+
187
+ ```bash
188
+ ## you need manually install TTS(https://github.com/coqui-ai/TTS) via `pip install tts` in advanced.
189
+ python app.py
190
+ ```
191
+
192
+ **Local gradio demo(highly recommanded!)**:
193
+
194
+ - windows: just double click `webui.bat`, the requirements will be installed automatically.
195
+ - Linux/Mac OS: run `bash webui.sh` to start the webui.
196
+
197
+
198
+ ### Manually usages:
199
+
200
+ ##### Animating a portrait image from default config:
201
+ ```bash
202
+ python inference.py --driven_audio <audio.wav> \
203
+ --source_image <video.mp4 or picture.png> \
204
+ --enhancer gfpgan
205
+ ```
206
+ The results will be saved in `results/$SOME_TIMESTAMP/*.mp4`.
207
+
208
+ ##### Full body/image Generation:
209
+
210
+ Using `--still` to generate a natural full body video. You can add `enhancer` to improve the quality of the generated video.
211
+
212
+ ```bash
213
+ python inference.py --driven_audio <audio.wav> \
214
+ --source_image <video.mp4 or picture.png> \
215
+ --result_dir <a file to store results> \
216
+ --still \
217
+ --preprocess full \
218
+ --enhancer gfpgan
219
+ ```
220
+
221
+ More examples and configuration and tips can be founded in the [ >>> best practice documents <<<](docs/best_practice.md).
222
+
223
+ ## 🛎 Citation
224
+
225
+ If you find our work useful in your research, please consider citing:
226
+
227
+ ```bibtex
228
+ @article{zhang2022sadtalker,
229
+ title={SadTalker: Learning Realistic 3D Motion Coefficients for Stylized Audio-Driven Single Image Talking Face Animation},
230
+ author={Zhang, Wenxuan and Cun, Xiaodong and Wang, Xuan and Zhang, Yong and Shen, Xi and Guo, Yu and Shan, Ying and Wang, Fei},
231
+ journal={arXiv preprint arXiv:2211.12194},
232
+ year={2022}
233
+ }
234
+ ```
235
+
236
+
237
+
238
+ ## 💗 Acknowledgements
239
+
240
+ Facerender code borrows heavily from [zhanglonghao's reproduction of face-vid2vid](https://github.com/zhanglonghao1992/One-Shot_Free-View_Neural_Talking_Head_Synthesis) and [PIRender](https://github.com/RenYurui/PIRender). We thank the authors for sharing their wonderful code. In training process, We also use the model from [Deep3DFaceReconstruction](https://github.com/microsoft/Deep3DFaceReconstruction) and [Wav2lip](https://github.com/Rudrabha/Wav2Lip). We thank for their wonderful work.
241
+
242
+ See also these wonderful 3rd libraries we use:
243
+
244
+ - **Face Utils**: https://github.com/xinntao/facexlib
245
+ - **Face Enhancement**: https://github.com/TencentARC/GFPGAN
246
+ - **Image/Video Enhancement**:https://github.com/xinntao/Real-ESRGAN
247
+
248
+ ## 🥂 Extensions:
249
+
250
+ - [SadTalker-Video-Lip-Sync](https://github.com/Zz-ww/SadTalker-Video-Lip-Sync) from [@Zz-ww](https://github.com/Zz-ww): SadTalker for Video Lip Editing
251
+
252
+ ## 🥂 Related Works
253
+ - [StyleHEAT: One-Shot High-Resolution Editable Talking Face Generation via Pre-trained StyleGAN (ECCV 2022)](https://github.com/FeiiYin/StyleHEAT)
254
+ - [CodeTalker: Speech-Driven 3D Facial Animation with Discrete Motion Prior (CVPR 2023)](https://github.com/Doubiiu/CodeTalker)
255
+ - [VideoReTalking: Audio-based Lip Synchronization for Talking Head Video Editing In the Wild (SIGGRAPH Asia 2022)](https://github.com/vinthony/video-retalking)
256
+ - [DPE: Disentanglement of Pose and Expression for General Video Portrait Editing (CVPR 2023)](https://github.com/Carlyx/DPE)
257
+ - [3D GAN Inversion with Facial Symmetry Prior (CVPR 2023)](https://github.com/FeiiYin/SPI/)
258
+ - [T2M-GPT: Generating Human Motion from Textual Descriptions with Discrete Representations (CVPR 2023)](https://github.com/Mael-zys/T2M-GPT)
259
+
260
+ ## 📢 Disclaimer
261
+
262
+ This is not an official product of Tencent. This repository can only be used for personal/research/non-commercial purposes.
263
+
264
+ LOGO: color and font suggestion: [ChatGPT](ai.com), logo font:[Montserrat Alternates
265
+ ](https://fonts.google.com/specimen/Montserrat+Alternates?preview.text=SadTalker&preview.text_type=custom&query=mont).
266
+
267
+ All the copyright of the demo images and audio are from communities users or the geneartion from stable diffusion. Free free to contact us if you feel uncomfortable.
268
+
Untitled.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
__pycache__/commons.cpython-310.pyc ADDED
Binary file (5.96 kB). View file
 
__pycache__/mel_processing.cpython-310.pyc ADDED
Binary file (3.41 kB). View file
 
__pycache__/models.cpython-310.pyc ADDED
Binary file (11.2 kB). View file
 
__pycache__/modules.cpython-310.pyc ADDED
Binary file (10 kB). View file
 
__pycache__/tts_voice.cpython-310.pyc ADDED
Binary file (1.8 kB). View file
 
__pycache__/utils.cpython-310.pyc ADDED
Binary file (9.98 kB). View file
 
app.py ADDED
@@ -0,0 +1,608 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, sys
2
+ import tempfile
3
+ import gradio as gr
4
+ from src.gradio_demo import SadTalker
5
+ # from src.utils.text2speech import TTSTalker
6
+ from huggingface_hub import snapshot_download
7
+
8
+ import torch
9
+ import librosa
10
+ from scipy.io.wavfile import write
11
+ from transformers import WavLMModel
12
+
13
+ import utils
14
+ from models import SynthesizerTrn
15
+ from mel_processing import mel_spectrogram_torch
16
+ from speaker_encoder.voice_encoder import SpeakerEncoder
17
+
18
+ import time
19
+ from textwrap import dedent
20
+
21
+ import mdtex2html
22
+ from loguru import logger
23
+ from transformers import AutoModel, AutoTokenizer
24
+
25
+ from tts_voice import tts_order_voice
26
+ import edge_tts
27
+ import tempfile
28
+ import anyio
29
+
30
+
31
+ def get_source_image(image):
32
+ return image
33
+
34
+ try:
35
+ import webui # in webui
36
+ in_webui = True
37
+ except:
38
+ in_webui = False
39
+
40
+
41
+ def toggle_audio_file(choice):
42
+ if choice == False:
43
+ return gr.update(visible=True), gr.update(visible=False)
44
+ else:
45
+ return gr.update(visible=False), gr.update(visible=True)
46
+
47
+ def ref_video_fn(path_of_ref_video):
48
+ if path_of_ref_video is not None:
49
+ return gr.update(value=True)
50
+ else:
51
+ return gr.update(value=False)
52
+
53
+ def download_model():
54
+ REPO_ID = 'vinthony/SadTalker-V002rc'
55
+ snapshot_download(repo_id=REPO_ID, local_dir='./checkpoints', local_dir_use_symlinks=True)
56
+
57
+ def sadtalker_demo():
58
+
59
+ download_model()
60
+
61
+ sad_talker = SadTalker(lazy_load=True)
62
+ # tts_talker = TTSTalker()
63
+
64
+ download_model()
65
+ sad_talker = SadTalker(lazy_load=True)
66
+
67
+
68
+ # ChatGLM2 & FreeVC
69
+
70
+ '''
71
+ def get_wavlm():
72
+ os.system('gdown https://drive.google.com/uc?id=12-cB34qCTvByWT-QtOcZaqwwO21FLSqU')
73
+ shutil.move('WavLM-Large.pt', 'wavlm')
74
+ '''
75
+
76
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
77
+
78
+ smodel = SpeakerEncoder('speaker_encoder/ckpt/pretrained_bak_5805000.pt')
79
+
80
+ print("Loading FreeVC(24k)...")
81
+ hps = utils.get_hparams_from_file("configs/freevc-24.json")
82
+ freevc_24 = SynthesizerTrn(
83
+ hps.data.filter_length // 2 + 1,
84
+ hps.train.segment_size // hps.data.hop_length,
85
+ **hps.model).to(device)
86
+ _ = freevc_24.eval()
87
+ _ = utils.load_checkpoint("checkpoint/freevc-24.pth", freevc_24, None)
88
+
89
+ print("Loading WavLM for content...")
90
+ cmodel = WavLMModel.from_pretrained("microsoft/wavlm-large").to(device)
91
+
92
+ def convert(model, src, tgt):
93
+ with torch.no_grad():
94
+ # tgt
95
+ wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate)
96
+ wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20)
97
+ if model == "FreeVC" or model == "FreeVC (24kHz)":
98
+ g_tgt = smodel.embed_utterance(wav_tgt)
99
+ g_tgt = torch.from_numpy(g_tgt).unsqueeze(0).to(device)
100
+ else:
101
+ wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).to(device)
102
+ mel_tgt = mel_spectrogram_torch(
103
+ wav_tgt,
104
+ hps.data.filter_length,
105
+ hps.data.n_mel_channels,
106
+ hps.data.sampling_rate,
107
+ hps.data.hop_length,
108
+ hps.data.win_length,
109
+ hps.data.mel_fmin,
110
+ hps.data.mel_fmax
111
+ )
112
+ # src
113
+ wav_src, _ = librosa.load(src, sr=hps.data.sampling_rate)
114
+ wav_src = torch.from_numpy(wav_src).unsqueeze(0).to(device)
115
+ c = cmodel(wav_src).last_hidden_state.transpose(1, 2).to(device)
116
+ # infer
117
+ if model == "FreeVC":
118
+ audio = freevc.infer(c, g=g_tgt)
119
+ elif model == "FreeVC-s":
120
+ audio = freevc_s.infer(c, mel=mel_tgt)
121
+ else:
122
+ audio = freevc_24.infer(c, g=g_tgt)
123
+ audio = audio[0][0].data.cpu().float().numpy()
124
+ if model == "FreeVC" or model == "FreeVC-s":
125
+ write("out.wav", hps.data.sampling_rate, audio)
126
+ else:
127
+ write("out.wav", 24000, audio)
128
+ out = "out.wav"
129
+ return out
130
+
131
+ # GLM2
132
+
133
+ language_dict = tts_order_voice
134
+
135
+ # fix timezone in Linux
136
+ os.environ["TZ"] = "Asia/Shanghai"
137
+ try:
138
+ time.tzset() # type: ignore # pylint: disable=no-member
139
+ except Exception:
140
+ # Windows
141
+ logger.warning("Windows, cant run time.tzset()")
142
+
143
+ # model_name = "THUDM/chatglm2-6b"
144
+ model_name = "THUDM/chatglm2-6b-int4"
145
+
146
+ RETRY_FLAG = False
147
+
148
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
149
+
150
+ # model = AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda()
151
+
152
+ # 4/8 bit
153
+ # model = AutoModel.from_pretrained("THUDM/chatglm2-6b", trust_remote_code=True).quantize(4).cuda()
154
+
155
+ has_cuda = torch.cuda.is_available()
156
+
157
+ # has_cuda = False # force cpu
158
+
159
+ if has_cuda:
160
+ model_glm = (
161
+ AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda().half()
162
+ ) # 3.92G
163
+ else:
164
+ model_glm = AutoModel.from_pretrained(
165
+ model_name, trust_remote_code=True
166
+ ).float() # .float() .half().float()
167
+
168
+ model_glm = model_glm.eval()
169
+
170
+ _ = """Override Chatbot.postprocess"""
171
+
172
+
173
+ def postprocess(self, y):
174
+ if y is None:
175
+ return []
176
+ for i, (message, response) in enumerate(y):
177
+ y[i] = (
178
+ None if message is None else mdtex2html.convert((message)),
179
+ None if response is None else mdtex2html.convert(response),
180
+ )
181
+ return y
182
+
183
+
184
+ gr.Chatbot.postprocess = postprocess
185
+
186
+
187
+ def parse_text(text):
188
+ """copy from https://github.com/GaiZhenbiao/ChuanhuChatGPT/"""
189
+ lines = text.split("\n")
190
+ lines = [line for line in lines if line != ""]
191
+ count = 0
192
+ for i, line in enumerate(lines):
193
+ if "```" in line:
194
+ count += 1
195
+ items = line.split("`")
196
+ if count % 2 == 1:
197
+ lines[i] = f'<pre><code class="language-{items[-1]}">'
198
+ else:
199
+ lines[i] = "<br></code></pre>"
200
+ else:
201
+ if i > 0:
202
+ if count % 2 == 1:
203
+ line = line.replace("`", r"\`")
204
+ line = line.replace("<", "&lt;")
205
+ line = line.replace(">", "&gt;")
206
+ line = line.replace(" ", "&nbsp;")
207
+ line = line.replace("*", "&ast;")
208
+ line = line.replace("_", "&lowbar;")
209
+ line = line.replace("-", "&#45;")
210
+ line = line.replace(".", "&#46;")
211
+ line = line.replace("!", "&#33;")
212
+ line = line.replace("(", "&#40;")
213
+ line = line.replace(")", "&#41;")
214
+ line = line.replace("$", "&#36;")
215
+ lines[i] = "<br>" + line
216
+ text = "".join(lines)
217
+ return text
218
+
219
+
220
+ def predict(
221
+ RETRY_FLAG, input, chatbot, max_length, top_p, temperature, history, past_key_values
222
+ ):
223
+ try:
224
+ chatbot.append((parse_text(input), ""))
225
+ except Exception as exc:
226
+ logger.error(exc)
227
+ logger.debug(f"{chatbot=}")
228
+ _ = """
229
+ if chatbot:
230
+ chatbot[-1] = (parse_text(input), str(exc))
231
+ yield chatbot, history, past_key_values
232
+ # """
233
+ yield chatbot, history, past_key_values
234
+
235
+ for response, history, past_key_values in model_glm.stream_chat(
236
+ tokenizer,
237
+ input,
238
+ history,
239
+ past_key_values=past_key_values,
240
+ return_past_key_values=True,
241
+ max_length=max_length,
242
+ top_p=top_p,
243
+ temperature=temperature,
244
+ ):
245
+ chatbot[-1] = (parse_text(input), parse_text(response))
246
+ # chatbot[-1][-1] = parse_text(response)
247
+
248
+ yield chatbot, history, past_key_values, parse_text(response)
249
+
250
+
251
+ def trans_api(input, max_length=4096, top_p=0.8, temperature=0.2):
252
+ if max_length < 10:
253
+ max_length = 4096
254
+ if top_p < 0.1 or top_p > 1:
255
+ top_p = 0.85
256
+ if temperature <= 0 or temperature > 1:
257
+ temperature = 0.01
258
+ try:
259
+ res, _ = model_glm.chat(
260
+ tokenizer,
261
+ input,
262
+ history=[],
263
+ past_key_values=None,
264
+ max_length=max_length,
265
+ top_p=top_p,
266
+ temperature=temperature,
267
+ )
268
+ # logger.debug(f"{res=} \n{_=}")
269
+ except Exception as exc:
270
+ logger.error(f"{exc=}")
271
+ res = str(exc)
272
+
273
+ return res
274
+
275
+
276
+ def reset_user_input():
277
+ return gr.update(value="")
278
+
279
+
280
+ def reset_state():
281
+ return [], [], None, ""
282
+
283
+
284
+ # Delete last turn
285
+ def delete_last_turn(chat, history):
286
+ if chat and history:
287
+ chat.pop(-1)
288
+ history.pop(-1)
289
+ return chat, history
290
+
291
+
292
+ # Regenerate response
293
+ def retry_last_answer(
294
+ user_input, chatbot, max_length, top_p, temperature, history, past_key_values
295
+ ):
296
+ if chatbot and history:
297
+ # Removing the previous conversation from chat
298
+ chatbot.pop(-1)
299
+ # Setting up a flag to capture a retry
300
+ RETRY_FLAG = True
301
+ # Getting last message from user
302
+ user_input = history[-1][0]
303
+ # Removing bot response from the history
304
+ history.pop(-1)
305
+
306
+ yield from predict(
307
+ RETRY_FLAG, # type: ignore
308
+ user_input,
309
+ chatbot,
310
+ max_length,
311
+ top_p,
312
+ temperature,
313
+ history,
314
+ past_key_values,
315
+ )
316
+
317
+ # print
318
+
319
+ def print(text):
320
+ return text
321
+
322
+ # TTS
323
+
324
+ async def text_to_speech_edge(text, language_code):
325
+ voice = language_dict[language_code]
326
+ communicate = edge_tts.Communicate(text, voice)
327
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp_file:
328
+ tmp_path = tmp_file.name
329
+
330
+ await communicate.save(tmp_path)
331
+
332
+ return tmp_path
333
+
334
+
335
+ with gr.Blocks(title="ChatGLM2-6B-int4", theme=gr.themes.Soft(text_size="sm"), analytics_enabled=False) as demo:
336
+ gr.HTML("<center>"
337
+ "<h1>📺💕🎶 - ChatGLM2+声音克隆+视频对话:和喜欢的角色畅所欲言吧!</h1>"
338
+ "</center>")
339
+ gr.Markdown("## <center>🥳 - ChatGLM2+FreeVC+SadTalker,为您打造沉浸式的视频对话体验,支持中英双语</center>")
340
+ gr.Markdown("## <center>🌊 - 更多精彩应用,尽在[滔滔AI](http://www.talktalkai.com);滔滔AI,为爱滔滔!💕</center>")
341
+ gr.Markdown("### <center>⭐ - 如果您喜欢这个程序,欢迎给我的[GitHub项目](https://github.com/KevinWang676/ChatGLM2-Voice-Cloning)点赞支持!</center>")
342
+
343
+ with gr.Tab("🍻 - ChatGLM2聊天区"):
344
+ with gr.Accordion("📒 相关信息", open=False):
345
+ _ = f""" ChatGLM2的可选参数信息:
346
+ * Low temperature: responses will be more deterministic and focused; High temperature: responses more creative.
347
+ * Suggested temperatures -- translation: up to 0.3; chatting: > 0.4
348
+ * Top P controls dynamic vocabulary selection based on context.\n
349
+ 如果您想让ChatGLM2进行角色扮演并与之对话,请先输入恰当的提示词,如“请你扮演成动漫角色蜡笔小新并和我进行对话”;您也可以为ChatGLM2提供自定义的角色设定\n
350
+ 当您使用声音克隆功能时,请先在此程序的对应位置上传一段您喜欢的音频
351
+ """
352
+ gr.Markdown(dedent(_))
353
+ chatbot = gr.Chatbot(height=300)
354
+ with gr.Row():
355
+ with gr.Column(scale=4):
356
+ with gr.Column(scale=12):
357
+ user_input = gr.Textbox(
358
+ label="请在此处和GLM2聊天 (按回车键即可发送)",
359
+ placeholder="聊点什么吧",
360
+ )
361
+ RETRY_FLAG = gr.Checkbox(value=False, visible=False)
362
+ with gr.Column(min_width=32, scale=1):
363
+ with gr.Row():
364
+ submitBtn = gr.Button("开始和GLM2交流吧", variant="primary")
365
+ deleteBtn = gr.Button("删除最新一轮对话", variant="secondary")
366
+ retryBtn = gr.Button("重新生成最新一轮对话", variant="secondary")
367
+
368
+ with gr.Accordion("🔧 更多设置", open=False):
369
+ with gr.Row():
370
+ emptyBtn = gr.Button("清空所有聊天记录")
371
+ max_length = gr.Slider(
372
+ 0,
373
+ 32768,
374
+ value=8192,
375
+ step=1.0,
376
+ label="Maximum length",
377
+ interactive=True,
378
+ )
379
+ top_p = gr.Slider(
380
+ 0, 1, value=0.85, step=0.01, label="Top P", interactive=True
381
+ )
382
+ temperature = gr.Slider(
383
+ 0.01, 1, value=0.95, step=0.01, label="Temperature", interactive=True
384
+ )
385
+
386
+
387
+ with gr.Row():
388
+ test1 = gr.Textbox(label="GLM2的最新回答 (可编辑)", lines = 3)
389
+ with gr.Column():
390
+ language = gr.Dropdown(choices=list(language_dict.keys()), value="普通话 (中国大陆)-Xiaoxiao-女", label="请选择文本对应的语言及您喜欢的说话人")
391
+ tts_btn = gr.Button("生成对应的音频吧", variant="primary")
392
+ output_audio = gr.Audio(type="filepath", label="为您生成的音频", interactive=False)
393
+
394
+ tts_btn.click(text_to_speech_edge, inputs=[test1, language], outputs=[output_audio])
395
+
396
+ with gr.Row():
397
+ model_choice = gr.Dropdown(choices=["FreeVC", "FreeVC-s", "FreeVC (24kHz)"], value="FreeVC (24kHz)", label="Model", visible=False)
398
+ audio1 = output_audio
399
+ audio2 = gr.Audio(label="请上传您喜欢的声音进行声音克隆", type='filepath')
400
+ clone_btn = gr.Button("开始AI声音克隆吧", variant="primary")
401
+ audio_cloned = gr.Audio(label="为您生成的专属声音克隆音频", type='filepath')
402
+
403
+ clone_btn.click(convert, inputs=[model_choice, audio1, audio2], outputs=[audio_cloned])
404
+
405
+ history = gr.State([])
406
+ past_key_values = gr.State(None)
407
+
408
+ user_input.submit(
409
+ predict,
410
+ [
411
+ RETRY_FLAG,
412
+ user_input,
413
+ chatbot,
414
+ max_length,
415
+ top_p,
416
+ temperature,
417
+ history,
418
+ past_key_values,
419
+ ],
420
+ [chatbot, history, past_key_values, test1],
421
+ show_progress="full",
422
+ )
423
+ submitBtn.click(
424
+ predict,
425
+ [
426
+ RETRY_FLAG,
427
+ user_input,
428
+ chatbot,
429
+ max_length,
430
+ top_p,
431
+ temperature,
432
+ history,
433
+ past_key_values,
434
+ ],
435
+ [chatbot, history, past_key_values, test1],
436
+ show_progress="full",
437
+ api_name="predict",
438
+ )
439
+ submitBtn.click(reset_user_input, [], [user_input])
440
+
441
+ emptyBtn.click(
442
+ reset_state, outputs=[chatbot, history, past_key_values, test1], show_progress="full"
443
+ )
444
+
445
+ retryBtn.click(
446
+ retry_last_answer,
447
+ inputs=[
448
+ user_input,
449
+ chatbot,
450
+ max_length,
451
+ top_p,
452
+ temperature,
453
+ history,
454
+ past_key_values,
455
+ ],
456
+ # outputs = [chatbot, history, last_user_message, user_message]
457
+ outputs=[chatbot, history, past_key_values, test1],
458
+ )
459
+ deleteBtn.click(delete_last_turn, [chatbot, history], [chatbot, history])
460
+
461
+ with gr.Accordion("📔 提示词示例", open=False):
462
+ etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
463
+ examples = gr.Examples(
464
+ examples=[
465
+ ["Explain the plot of Cinderella in a sentence."],
466
+ [
467
+ "How long does it take to become proficient in French, and what are the best methods for retaining information?"
468
+ ],
469
+ ["What are some common mistakes to avoid when writing code?"],
470
+ ["Build a prompt to generate a beautiful portrait of a horse"],
471
+ ["Suggest four metaphors to describe the benefits of AI"],
472
+ ["Write a pop song about leaving home for the sandy beaches."],
473
+ ["Write a summary demonstrating my ability to tame lions"],
474
+ ["鲁迅和周树人什么关系"],
475
+ ["从前有一头牛,这头牛后面有什么?"],
476
+ ["正无穷大加一大于正无穷大吗?"],
477
+ ["正无穷大加正无穷大大于正无穷大吗?"],
478
+ ["-2的平方根等于什么"],
479
+ ["树上有5只鸟,猎人开枪打死了一只。树上还有几只鸟?"],
480
+ ["树上有11只鸟,猎人开枪打死了一只。树上还有几只鸟?提示:需考虑鸟可能受惊吓飞走。"],
481
+ ["鲁迅和周树人什么关系 用英文回答"],
482
+ ["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"],
483
+ [f"{etext} 翻成中文,列出3个版本"],
484
+ [f"{etext} \n 翻成中文,保留原意,但使用文学性的语言。不要写解释。列出3个版本"],
485
+ ["js 判断一个数是不是质数"],
486
+ ["js 实现python 的 range(10)"],
487
+ ["js 实现python 的 [*(range(10)]"],
488
+ ["假定 1 + 2 = 4, 试求 7 + 8"],
489
+ ["Erkläre die Handlung von Cinderella in einem Satz."],
490
+ ["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch"],
491
+ ],
492
+ inputs=[user_input],
493
+ examples_per_page=30,
494
+ )
495
+
496
+ with gr.Accordion("For Chat/Translation API", open=False, visible=False):
497
+ input_text = gr.Text()
498
+ tr_btn = gr.Button("Go", variant="primary")
499
+ out_text = gr.Text()
500
+ tr_btn.click(
501
+ trans_api,
502
+ [input_text, max_length, top_p, temperature],
503
+ out_text,
504
+ # show_progress="full",
505
+ api_name="tr",
506
+ )
507
+ _ = """
508
+ input_text.submit(
509
+ trans_api,
510
+ [input_text, max_length, top_p, temperature],
511
+ out_text,
512
+ show_progress="full",
513
+ api_name="tr1",
514
+ )
515
+ # """
516
+ with gr.Tab("📺 - 视频聊天区"):
517
+ with gr.Row().style(equal_height=False):
518
+ with gr.Column(variant='panel'):
519
+ with gr.Tabs(elem_id="sadtalker_source_image"):
520
+ with gr.TabItem('图片上传'):
521
+ with gr.Row():
522
+ source_image = gr.Image(label="请上传一张您喜欢角色的图片", source="upload", type="filepath", elem_id="img2img_image").style(width=512)
523
+
524
+
525
+ with gr.Tabs(elem_id="sadtalker_driven_audio"):
526
+ with gr.TabItem('💡您还可以将视频下载到本地'):
527
+
528
+ with gr.Row():
529
+ driven_audio = audio_cloned
530
+ driven_audio_no = gr.Audio(label="Use IDLE mode, no audio is required", source="upload", type="filepath", visible=False)
531
+
532
+ with gr.Column():
533
+ use_idle_mode = gr.Checkbox(label="Use Idle Animation", visible=False)
534
+ length_of_audio = gr.Number(value=5, label="The length(seconds) of the generated video.", visible=False)
535
+ use_idle_mode.change(toggle_audio_file, inputs=use_idle_mode, outputs=[driven_audio, driven_audio_no]) # todo
536
+
537
+ with gr.Row():
538
+ ref_video = gr.Video(label="Reference Video", source="upload", type="filepath", elem_id="vidref", visible=False).style(width=512)
539
+
540
+ with gr.Column():
541
+ use_ref_video = gr.Checkbox(label="Use Reference Video", visible=False)
542
+ ref_info = gr.Radio(['pose', 'blink','pose+blink', 'all'], value='pose', label='Reference Video',info="How to borrow from reference Video?((fully transfer, aka, video driving mode))", visible=False)
543
+
544
+ ref_video.change(ref_video_fn, inputs=ref_video, outputs=[use_ref_video]) # todo
545
+
546
+
547
+ with gr.Column(variant='panel'):
548
+ with gr.Tabs(elem_id="sadtalker_checkbox"):
549
+ with gr.TabItem('视频设置'):
550
+ with gr.Column(variant='panel'):
551
+ # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
552
+ # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
553
+ with gr.Row():
554
+ pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0, visible=False) #
555
+ exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1, visible=False) #
556
+ blink_every = gr.Checkbox(label="use eye blink", value=True, visible=False)
557
+
558
+ with gr.Row():
559
+ size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model?", visible=False) #
560
+ preprocess_type = gr.Radio(['crop', 'full'], value='crop', label='是否聚焦角色面部', info="crop:视频会聚焦角色面部;full:视频会显示图片全貌")
561
+
562
+ with gr.Row():
563
+ is_still_mode = gr.Checkbox(label="静态模式 (开启静态模式,角色的面部动作会减少;默认开启)", value=True)
564
+ facerender = gr.Radio(['facevid2vid','pirender'], value='facevid2vid', label='facerender', info="which face render?", visible=False)
565
+
566
+ with gr.Row():
567
+ batch_size = gr.Slider(label="Batch size (数值越大,生成速度越快;若显卡性能好,可增大数值)", step=1, maximum=32, value=2)
568
+ enhancer = gr.Checkbox(label="GFPGAN as Face enhancer", value=True, visible=False)
569
+
570
+ submit = gr.Button('开始视频聊天吧', elem_id="sadtalker_generate", variant='primary')
571
+
572
+ with gr.Tabs(elem_id="sadtalker_genearted"):
573
+ gen_video = gr.Video(label="为您生成的专属视频", format="mp4").style(width=256)
574
+
575
+
576
+
577
+ submit.click(
578
+ fn=sad_talker.test,
579
+ inputs=[source_image,
580
+ driven_audio,
581
+ preprocess_type,
582
+ is_still_mode,
583
+ enhancer,
584
+ batch_size,
585
+ size_of_image,
586
+ pose_style,
587
+ facerender,
588
+ exp_weight,
589
+ use_ref_video,
590
+ ref_video,
591
+ ref_info,
592
+ use_idle_mode,
593
+ length_of_audio,
594
+ blink_every
595
+ ],
596
+ outputs=[gen_video]
597
+ )
598
+ gr.Markdown("### <center>注意❗:请不要生成会对个人以及组织造成侵害的内容,此程序仅供科研、学习及个人娱乐使用。</center>")
599
+ gr.Markdown("<center>💡- 如何使用此程序:输入您对ChatGLM的提问后,依次点击“开始和GLM2交流吧”、“生成对应的音频吧”、“开始AI声音克隆吧”、“开始视频聊天吧”三个按键即可;使用声音克隆功能时,请先上传一段您喜欢的音频</center>")
600
+ gr.HTML('''
601
+ <div class="footer">
602
+ <p>🌊🏞️🎶 - 江水东流急,滔滔无尽声。 明·顾璘
603
+ </p>
604
+ </div>
605
+ ''')
606
+
607
+
608
+ demo.queue().launch(show_error=True, debug=True)
app_sadtalker.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, sys
2
+ import gradio as gr
3
+ from src.gradio_demo import SadTalker
4
+
5
+
6
+ try:
7
+ import webui # in webui
8
+ in_webui = True
9
+ except:
10
+ in_webui = False
11
+
12
+
13
+ def toggle_audio_file(choice):
14
+ if choice == False:
15
+ return gr.update(visible=True), gr.update(visible=False)
16
+ else:
17
+ return gr.update(visible=False), gr.update(visible=True)
18
+
19
+ def ref_video_fn(path_of_ref_video):
20
+ if path_of_ref_video is not None:
21
+ return gr.update(value=True)
22
+ else:
23
+ return gr.update(value=False)
24
+
25
+ def sadtalker_demo(checkpoint_path='checkpoints', config_path='src/config', warpfn=None):
26
+
27
+ sad_talker = SadTalker(checkpoint_path, config_path, lazy_load=True)
28
+
29
+ with gr.Blocks(analytics_enabled=False) as sadtalker_interface:
30
+ gr.Markdown("<div align='center'> <h2> 😭 SadTalker: Learning Realistic 3D Motion Coefficients for Stylized Audio-Driven Single Image Talking Face Animation (CVPR 2023) </span> </h2> \
31
+ <a style='font-size:18px;color: #efefef' href='https://arxiv.org/abs/2211.12194'>Arxiv</a> &nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
32
+ <a style='font-size:18px;color: #efefef' href='https://sadtalker.github.io'>Homepage</a> &nbsp;&nbsp;&nbsp;&nbsp;&nbsp; \
33
+ <a style='font-size:18px;color: #efefef' href='https://github.com/Winfredy/SadTalker'> Github </div>")
34
+
35
+ with gr.Row().style(equal_height=False):
36
+ with gr.Column(variant='panel'):
37
+ with gr.Tabs(elem_id="sadtalker_source_image"):
38
+ with gr.TabItem('Upload image'):
39
+ with gr.Row():
40
+ source_image = gr.Image(label="Source image", source="upload", type="filepath", elem_id="img2img_image").style(width=512)
41
+
42
+ with gr.Tabs(elem_id="sadtalker_driven_audio"):
43
+ with gr.TabItem('Upload OR TTS'):
44
+ with gr.Column(variant='panel'):
45
+ driven_audio = gr.Audio(label="Input audio", source="upload", type="filepath")
46
+
47
+ if sys.platform != 'win32' and not in_webui:
48
+ from src.utils.text2speech import TTSTalker
49
+ tts_talker = TTSTalker()
50
+ with gr.Column(variant='panel'):
51
+ input_text = gr.Textbox(label="Generating audio from text", lines=5, placeholder="please enter some text here, we genreate the audio from text using @Coqui.ai TTS.")
52
+ tts = gr.Button('Generate audio',elem_id="sadtalker_audio_generate", variant='primary')
53
+ tts.click(fn=tts_talker.test, inputs=[input_text], outputs=[driven_audio])
54
+
55
+ with gr.Column(variant='panel'):
56
+ with gr.Tabs(elem_id="sadtalker_checkbox"):
57
+ with gr.TabItem('Settings'):
58
+ gr.Markdown("need help? please visit our [best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md) for more detials")
59
+ with gr.Column(variant='panel'):
60
+ # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
61
+ # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
62
+ pose_style = gr.Slider(minimum=0, maximum=46, step=1, label="Pose style", value=0) #
63
+ size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model?") #
64
+ preprocess_type = gr.Radio(['crop', 'resize','full', 'extcrop', 'extfull'], value='crop', label='preprocess', info="How to handle input image?")
65
+ is_still_mode = gr.Checkbox(label="Still Mode (fewer hand motion, works with preprocess `full`)")
66
+ batch_size = gr.Slider(label="batch size in generation", step=1, maximum=10, value=2)
67
+ enhancer = gr.Checkbox(label="GFPGAN as Face enhancer")
68
+ submit = gr.Button('Generate', elem_id="sadtalker_generate", variant='primary')
69
+
70
+ with gr.Tabs(elem_id="sadtalker_genearted"):
71
+ gen_video = gr.Video(label="Generated video", format="mp4").style(width=256)
72
+
73
+ if warpfn:
74
+ submit.click(
75
+ fn=warpfn(sad_talker.test),
76
+ inputs=[source_image,
77
+ driven_audio,
78
+ preprocess_type,
79
+ is_still_mode,
80
+ enhancer,
81
+ batch_size,
82
+ size_of_image,
83
+ pose_style
84
+ ],
85
+ outputs=[gen_video]
86
+ )
87
+ else:
88
+ submit.click(
89
+ fn=sad_talker.test,
90
+ inputs=[source_image,
91
+ driven_audio,
92
+ preprocess_type,
93
+ is_still_mode,
94
+ enhancer,
95
+ batch_size,
96
+ size_of_image,
97
+ pose_style
98
+ ],
99
+ outputs=[gen_video]
100
+ )
101
+
102
+ return sadtalker_interface
103
+
104
+
105
+ if __name__ == "__main__":
106
+
107
+ demo = sadtalker_demo()
108
+ demo.queue()
109
+ demo.launch(share=True)
110
+
111
+
checkpoint/__init__.py ADDED
File without changes
checkpoint/freevc-24.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b39a86fefbc9ec6e30be8d26ee2a6aa5ffe6d235f6ab15773d01cdf348e5b20
3
+ size 472644351
checkpoints/SadTalker_V0.0.2_256.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c211f5d6de003516bf1bbda9f47049a4c9c99133b1ab565c6961e5af16477bff
3
+ size 725066984
checkpoints/SadTalker_V0.0.2_512.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e063f7ff5258240bdb0f7690783a7b1374e6a4a81ce8fa33456f4cd49694340
3
+ size 725066984
checkpoints/mapping_00109-model.pth.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84a8642468a3fcfdd9ab6be955267043116c2bec2284686a5262f1eaf017f64c
3
+ size 155779231
checkpoints/mapping_00229-model.pth.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62a1e06006cc963220f6477438518ed86e9788226c62ae382ddc42fbcefb83f1
3
+ size 155521183
cog.yaml ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ build:
2
+ gpu: true
3
+ cuda: "11.3"
4
+ python_version: "3.8"
5
+ system_packages:
6
+ - "ffmpeg"
7
+ - "libgl1-mesa-glx"
8
+ - "libglib2.0-0"
9
+ python_packages:
10
+ - "torch==1.12.1"
11
+ - "torchvision==0.13.1"
12
+ - "torchaudio==0.12.1"
13
+ - "joblib==1.1.0"
14
+ - "scikit-image==0.19.3"
15
+ - "basicsr==1.4.2"
16
+ - "facexlib==0.3.0"
17
+ - "resampy==0.3.1"
18
+ - "pydub==0.25.1"
19
+ - "scipy==1.10.1"
20
+ - "kornia==0.6.8"
21
+ - "face_alignment==1.3.5"
22
+ - "imageio==2.19.3"
23
+ - "imageio-ffmpeg==0.4.7"
24
+ - "librosa==0.9.2" #
25
+ - "tqdm==4.65.0"
26
+ - "yacs==0.1.8"
27
+ - "gfpgan==1.3.8"
28
+ - "dlib-bin==19.24.1"
29
+ - "av==10.0.0"
30
+ - "trimesh==3.9.20"
31
+ run:
32
+ - mkdir -p /root/.cache/torch/hub/checkpoints/ && wget --output-document "/root/.cache/torch/hub/checkpoints/s3fd-619a316812.pth" "https://www.adrianbulat.com/downloads/python-fan/s3fd-619a316812.pth"
33
+ - mkdir -p /root/.cache/torch/hub/checkpoints/ && wget --output-document "/root/.cache/torch/hub/checkpoints/2DFAN4-cd938726ad.zip" "https://www.adrianbulat.com/downloads/python-fan/2DFAN4-cd938726ad.zip"
34
+
35
+ predict: "predict.py:Predictor"
commons.py ADDED
@@ -0,0 +1,171 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import numpy as np
3
+ import torch
4
+ from torch import nn
5
+ from torch.nn import functional as F
6
+
7
+
8
+ def init_weights(m, mean=0.0, std=0.01):
9
+ classname = m.__class__.__name__
10
+ if classname.find("Conv") != -1:
11
+ m.weight.data.normal_(mean, std)
12
+
13
+
14
+ def get_padding(kernel_size, dilation=1):
15
+ return int((kernel_size*dilation - dilation)/2)
16
+
17
+
18
+ def convert_pad_shape(pad_shape):
19
+ l = pad_shape[::-1]
20
+ pad_shape = [item for sublist in l for item in sublist]
21
+ return pad_shape
22
+
23
+
24
+ def intersperse(lst, item):
25
+ result = [item] * (len(lst) * 2 + 1)
26
+ result[1::2] = lst
27
+ return result
28
+
29
+
30
+ def kl_divergence(m_p, logs_p, m_q, logs_q):
31
+ """KL(P||Q)"""
32
+ kl = (logs_q - logs_p) - 0.5
33
+ kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q)
34
+ return kl
35
+
36
+
37
+ def rand_gumbel(shape):
38
+ """Sample from the Gumbel distribution, protect from overflows."""
39
+ uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
40
+ return -torch.log(-torch.log(uniform_samples))
41
+
42
+
43
+ def rand_gumbel_like(x):
44
+ g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
45
+ return g
46
+
47
+
48
+ def slice_segments(x, ids_str, segment_size=4):
49
+ ret = torch.zeros_like(x[:, :, :segment_size])
50
+ for i in range(x.size(0)):
51
+ idx_str = ids_str[i]
52
+ idx_end = idx_str + segment_size
53
+ ret[i] = x[i, :, idx_str:idx_end]
54
+ return ret
55
+
56
+
57
+ def rand_slice_segments(x, x_lengths=None, segment_size=4):
58
+ b, d, t = x.size()
59
+ if x_lengths is None:
60
+ x_lengths = t
61
+ ids_str_max = x_lengths - segment_size + 1
62
+ ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
63
+ ret = slice_segments(x, ids_str, segment_size)
64
+ return ret, ids_str
65
+
66
+
67
+ def rand_spec_segments(x, x_lengths=None, segment_size=4):
68
+ b, d, t = x.size()
69
+ if x_lengths is None:
70
+ x_lengths = t
71
+ ids_str_max = x_lengths - segment_size
72
+ ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
73
+ ret = slice_segments(x, ids_str, segment_size)
74
+ return ret, ids_str
75
+
76
+
77
+ def get_timing_signal_1d(
78
+ length, channels, min_timescale=1.0, max_timescale=1.0e4):
79
+ position = torch.arange(length, dtype=torch.float)
80
+ num_timescales = channels // 2
81
+ log_timescale_increment = (
82
+ math.log(float(max_timescale) / float(min_timescale)) /
83
+ (num_timescales - 1))
84
+ inv_timescales = min_timescale * torch.exp(
85
+ torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment)
86
+ scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
87
+ signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
88
+ signal = F.pad(signal, [0, 0, 0, channels % 2])
89
+ signal = signal.view(1, channels, length)
90
+ return signal
91
+
92
+
93
+ def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
94
+ b, channels, length = x.size()
95
+ signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
96
+ return x + signal.to(dtype=x.dtype, device=x.device)
97
+
98
+
99
+ def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
100
+ b, channels, length = x.size()
101
+ signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
102
+ return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
103
+
104
+
105
+ def subsequent_mask(length):
106
+ mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
107
+ return mask
108
+
109
+
110
+ @torch.jit.script
111
+ def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
112
+ n_channels_int = n_channels[0]
113
+ in_act = input_a + input_b
114
+ t_act = torch.tanh(in_act[:, :n_channels_int, :])
115
+ s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
116
+ acts = t_act * s_act
117
+ return acts
118
+
119
+
120
+ def convert_pad_shape(pad_shape):
121
+ l = pad_shape[::-1]
122
+ pad_shape = [item for sublist in l for item in sublist]
123
+ return pad_shape
124
+
125
+
126
+ def shift_1d(x):
127
+ x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
128
+ return x
129
+
130
+
131
+ def sequence_mask(length, max_length=None):
132
+ if max_length is None:
133
+ max_length = length.max()
134
+ x = torch.arange(max_length, dtype=length.dtype, device=length.device)
135
+ return x.unsqueeze(0) < length.unsqueeze(1)
136
+
137
+
138
+ def generate_path(duration, mask):
139
+ """
140
+ duration: [b, 1, t_x]
141
+ mask: [b, 1, t_y, t_x]
142
+ """
143
+ device = duration.device
144
+
145
+ b, _, t_y, t_x = mask.shape
146
+ cum_duration = torch.cumsum(duration, -1)
147
+
148
+ cum_duration_flat = cum_duration.view(b * t_x)
149
+ path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
150
+ path = path.view(b, t_x, t_y)
151
+ path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
152
+ path = path.unsqueeze(1).transpose(2,3) * mask
153
+ return path
154
+
155
+
156
+ def clip_grad_value_(parameters, clip_value, norm_type=2):
157
+ if isinstance(parameters, torch.Tensor):
158
+ parameters = [parameters]
159
+ parameters = list(filter(lambda p: p.grad is not None, parameters))
160
+ norm_type = float(norm_type)
161
+ if clip_value is not None:
162
+ clip_value = float(clip_value)
163
+
164
+ total_norm = 0
165
+ for p in parameters:
166
+ param_norm = p.grad.data.norm(norm_type)
167
+ total_norm += param_norm.item() ** norm_type
168
+ if clip_value is not None:
169
+ p.grad.data.clamp_(min=-clip_value, max=clip_value)
170
+ total_norm = total_norm ** (1. / norm_type)
171
+ return total_norm
configs/.ipynb_checkpoints/freevc-24-checkpoint.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train": {
3
+ "log_interval": 200,
4
+ "eval_interval": 10000,
5
+ "seed": 1234,
6
+ "epochs": 10000,
7
+ "learning_rate": 2e-4,
8
+ "betas": [0.8, 0.99],
9
+ "eps": 1e-9,
10
+ "batch_size": 64,
11
+ "fp16_run": false,
12
+ "lr_decay": 0.999875,
13
+ "segment_size": 8640,
14
+ "init_lr_ratio": 1,
15
+ "warmup_epochs": 0,
16
+ "c_mel": 45,
17
+ "c_kl": 1.0,
18
+ "use_sr": true,
19
+ "max_speclen": 128,
20
+ "port": "8008"
21
+ },
22
+ "data": {
23
+ "training_files":"filelists/train.txt",
24
+ "validation_files":"filelists/val.txt",
25
+ "max_wav_value": 32768.0,
26
+ "sampling_rate": 16000,
27
+ "filter_length": 1280,
28
+ "hop_length": 320,
29
+ "win_length": 1280,
30
+ "n_mel_channels": 80,
31
+ "mel_fmin": 0.0,
32
+ "mel_fmax": null
33
+ },
34
+ "model": {
35
+ "inter_channels": 192,
36
+ "hidden_channels": 192,
37
+ "filter_channels": 768,
38
+ "n_heads": 2,
39
+ "n_layers": 6,
40
+ "kernel_size": 3,
41
+ "p_dropout": 0.1,
42
+ "resblock": "1",
43
+ "resblock_kernel_sizes": [3,7,11],
44
+ "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
45
+ "upsample_rates": [10,6,4,2],
46
+ "upsample_initial_channel": 512,
47
+ "upsample_kernel_sizes": [16,16,4,4],
48
+ "n_layers_q": 3,
49
+ "use_spectral_norm": false,
50
+ "gin_channels": 256,
51
+ "ssl_dim": 1024,
52
+ "use_spk": true
53
+ }
54
+ }
configs/freevc-24.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train": {
3
+ "log_interval": 200,
4
+ "eval_interval": 10000,
5
+ "seed": 1234,
6
+ "epochs": 10000,
7
+ "learning_rate": 2e-4,
8
+ "betas": [0.8, 0.99],
9
+ "eps": 1e-9,
10
+ "batch_size": 64,
11
+ "fp16_run": false,
12
+ "lr_decay": 0.999875,
13
+ "segment_size": 8640,
14
+ "init_lr_ratio": 1,
15
+ "warmup_epochs": 0,
16
+ "c_mel": 45,
17
+ "c_kl": 1.0,
18
+ "use_sr": true,
19
+ "max_speclen": 128,
20
+ "port": "8008"
21
+ },
22
+ "data": {
23
+ "training_files":"filelists/train.txt",
24
+ "validation_files":"filelists/val.txt",
25
+ "max_wav_value": 32768.0,
26
+ "sampling_rate": 16000,
27
+ "filter_length": 1280,
28
+ "hop_length": 320,
29
+ "win_length": 1280,
30
+ "n_mel_channels": 80,
31
+ "mel_fmin": 0.0,
32
+ "mel_fmax": null
33
+ },
34
+ "model": {
35
+ "inter_channels": 192,
36
+ "hidden_channels": 192,
37
+ "filter_channels": 768,
38
+ "n_heads": 2,
39
+ "n_layers": 6,
40
+ "kernel_size": 3,
41
+ "p_dropout": 0.1,
42
+ "resblock": "1",
43
+ "resblock_kernel_sizes": [3,7,11],
44
+ "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
45
+ "upsample_rates": [10,6,4,2],
46
+ "upsample_initial_channel": 512,
47
+ "upsample_kernel_sizes": [16,16,4,4],
48
+ "n_layers_q": 3,
49
+ "use_spectral_norm": false,
50
+ "gin_channels": 256,
51
+ "ssl_dim": 1024,
52
+ "use_spk": true
53
+ }
54
+ }
docs/FAQ.md ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ ## Frequency Asked Question
3
+
4
+ **Q: `ffmpeg` is not recognized as an internal or external command**
5
+
6
+ In Linux, you can install the ffmpeg via `conda install ffmpeg`. Or on Mac OS X, try to install ffmpeg via `brew install ffmpeg`. On windows, make sure you have `ffmpeg` in the `%PATH%` as suggested in [#54](https://github.com/Winfredy/SadTalker/issues/54), then, following [this](https://www.geeksforgeeks.org/how-to-install-ffmpeg-on-windows/) installation to install `ffmpeg`.
7
+
8
+ **Q: Running Requirments.**
9
+
10
+ Please refer to the discussion here: https://github.com/Winfredy/SadTalker/issues/124#issuecomment-1508113989
11
+
12
+
13
+ **Q: ModuleNotFoundError: No module named 'ai'**
14
+
15
+ please check the checkpoint's size of the `epoch_20.pth`. (https://github.com/Winfredy/SadTalker/issues/167, https://github.com/Winfredy/SadTalker/issues/113)
16
+
17
+ **Q: Illegal Hardware Error: Mac M1**
18
+
19
+ please reinstall the `dlib` by `pip install dlib` individually. (https://github.com/Winfredy/SadTalker/issues/129, https://github.com/Winfredy/SadTalker/issues/109)
20
+
21
+
22
+ **Q: FileNotFoundError: [Errno 2] No such file or directory: checkpoints\BFM_Fitting\similarity_Lm3D_all.mat**
23
+
24
+ Make sure you have downloaded the checkpoints and gfpgan as [here](https://github.com/Winfredy/SadTalker#-2-download-trained-models) and placed them in the right place.
25
+
26
+ **Q: RuntimeError: unexpected EOF, expected 237192 more bytes. The file might be corrupted.**
27
+
28
+ The files are not automatically downloaded. Please update the code and download the gfpgan folders as [here](https://github.com/Winfredy/SadTalker#-2-download-trained-models).
29
+
30
+ **Q: CUDA out of memory error**
31
+
32
+ please refer to https://stackoverflow.com/questions/73747731/runtimeerror-cuda-out-of-memory-how-setting-max-split-size-mb
33
+
34
+ ```
35
+ # windows
36
+ set PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128
37
+ python inference.py ...
38
+
39
+ # linux
40
+ export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128
41
+ python inference.py ...
42
+ ```
43
+
44
+ **Q: Error while decoding stream #0:0: Invalid data found when processing input [mp3float @ 0000015037628c00] Header missing**
45
+
46
+ Our method only support wav or mp3 files as input, please make sure the feeded audios are in these formats.
docs/best_practice.md ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Best Practice and Tips for configuration
2
+
3
+ > Our model only works on REAL person's photo or the portrait image similar to REAL person. The anime talking head genreation method will be released in future.
4
+
5
+ Advanced confiurations for `inference.py`:
6
+
7
+ | Name | Configuration | default | Explaination |
8
+ |:------------- |:------------- |:----- | :------------- |
9
+ | Enhance Mode | `--enhancer` | None | Using `gfpgan` or `RestoreFormer` to enhance the generated face via face restoration network
10
+ | Background Enhancer | `--background_enhancer` | None | Using `realesrgan` to enhance the full video.
11
+ | Still Mode | ` --still` | False | Using the same pose parameters as the original image, fewer head motion.
12
+ | Expressive Mode | `--expression_scale` | 1.0 | a larger value will make the expression motion stronger.
13
+ | save path | `--result_dir` |`./results` | The file will be save in the newer location.
14
+ | preprocess | `--preprocess` | `crop` | Run and produce the results in the croped input image. Other choices: `resize`, where the images will be resized to the specific resolution. `full` Run the full image animation, use with `--still` to get better results.
15
+ | ref Mode (eye) | `--ref_eyeblink` | None | A video path, where we borrow the eyeblink from this reference video to provide more natural eyebrow movement.
16
+ | ref Mode (pose) | `--ref_pose` | None | A video path, where we borrow the pose from the head reference video.
17
+ | 3D Mode | `--face3dvis` | False | Need additional installation. More details to generate the 3d face can be founded [here](docs/face3d.md).
18
+ | free-view Mode | `--input_yaw`,<br> `--input_pitch`,<br> `--input_roll` | None | Genearting novel view or free-view 4D talking head from a single image. More details can be founded [here](https://github.com/Winfredy/SadTalker#generating-4d-free-view-talking-examples-from-audio-and-a-single-image).
19
+
20
+
21
+ ### About `--preprocess`
22
+
23
+ Our method automatically handle the input images via `crop`, `resize` and `full`.
24
+
25
+ In `crop` mode, we only generate the croped image via the facial keypoints and generated the facial anime avator. The animation of both expression and head pose are realistic.
26
+
27
+ > still mode will stop the eyeblink and head pose movement.
28
+
29
+ | [input image @bagbag1815](https://twitter.com/bagbag1815/status/1642754319094108161) | crop | crop w/still |
30
+ |:--------------------: |:--------------------: | :----: |
31
+ | <img src='../examples/source_image/full_body_2.png' width='380'> | ![full_body_2](example_crop.gif) | ![full_body_2](example_crop_still.gif) |
32
+
33
+
34
+ In `resize` mode, we resize the whole images to generate the fully talking head video. Thus, an image similar to the ID photo can be produced. ⚠️ It will produce bad results for full person images.
35
+
36
+
37
+
38
+
39
+ | <img src='../examples/source_image/full_body_2.png' width='380'> | <img src='../examples/source_image/full4.jpeg' width='380'> |
40
+ |:--------------------: |:--------------------: |
41
+ | ❌ not suitable for resize mode | ✅ good for resize mode |
42
+ | <img src='resize_no.gif'> | <img src='resize_good.gif' width='380'> |
43
+
44
+ In `full` mode, our model will automatically process the croped region and paste back to the original image. Remember to use `--still` to keep the original head pose.
45
+
46
+ | input | `--still` | `--still` & `enhancer` |
47
+ |:--------------------: |:--------------------: | :--:|
48
+ | <img src='../examples/source_image/full_body_2.png' width='380'> | <img src='./example_full.gif' width='380'> | <img src='./example_full_enhanced.gif' width='380'>
49
+
50
+
51
+ ### About `--enhancer`
52
+
53
+ For better facial quality, we intergate [gfpgan](https://github.com/TencentARC/GFPGAN) and [real-esrgan](https://github.com/xinntao/Real-ESRGAN) for different purpose. Just adding `--enhancer <gfpgan or RestoreFormer>` or `--background_enhancer <realesrgan>` for the enhancement of the face and the full image.
54
+
55
+ ```bash
56
+ # make sure above packages are available:
57
+ pip install gfpgan
58
+ pip install realesrgan
59
+ ```
60
+
61
+ ### About `--face3dvis`
62
+
63
+ This flag indicate that we can generated the 3d-rendered face and it's 3d facial landmarks. More details can be founded [here](face3d.md).
64
+
65
+ | Input | Animated 3d face |
66
+ |:-------------: | :-------------: |
67
+ | <img src='../examples/source_image/art_0.png' width='200px'> | <video src="https://user-images.githubusercontent.com/4397546/226856847-5a6a0a4d-a5ec-49e2-9b05-3206db65e8e3.mp4"></video> |
68
+
69
+ > Kindly ensure to activate the audio as the default audio playing is incompatible with GitHub.
70
+
71
+
72
+
73
+ #### reference eye-link mode.
74
+
75
+ | Input, w/ reference video , reference video |
76
+ |:-------------: |
77
+ | ![free_view](using_ref_video.gif)|
78
+ | If the reference video is shorter than the input audio, we will loop the reference video .
79
+
80
+
81
+
82
+ #### Generating 4D free-view talking examples from audio and a single image
83
+
84
+ We use `input_yaw`, `input_pitch`, `input_roll` to control head pose. For example, `--input_yaw -20 30 10` means the input head yaw degree changes from -20 to 30 and then changes from 30 to 10.
85
+ ```bash
86
+ python inference.py --driven_audio <audio.wav> \
87
+ --source_image <video.mp4 or picture.png> \
88
+ --result_dir <a file to store results> \
89
+ --input_yaw -20 30 10
90
+ ```
91
+
92
+ | Results, Free-view results, Novel view results |
93
+ |:-------------: |
94
+ | ![free_view](free_view_result.gif)|
docs/changlelog.md ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## changelogs
2
+
3
+
4
+ - __[2023.04.06]__: stable-diffiusion webui extension is release.
5
+
6
+ - __[2023.04.03]__: Enable TTS in huggingface and gradio local demo.
7
+
8
+ - __[2023.03.30]__: Launch beta version of the full body mode.
9
+
10
+ - __[2023.03.30]__: Launch new feature: through using reference videos, our algorithm can generate videos with more natural eye blinking and some eyebrow movement.
11
+
12
+ - __[2023.03.29]__: `resize mode` is online by `python infererence.py --preprocess resize`! Where we can produce a larger crop of the image as discussed in https://github.com/Winfredy/SadTalker/issues/35.
13
+
14
+ - __[2023.03.29]__: local gradio demo is online! `python app.py` to start the demo. New `requirments.txt` is used to avoid the bugs in `librosa`.
15
+
16
+ - __[2023.03.28]__: Online demo is launched in [![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/vinthony/SadTalker), thanks AK!
17
+
18
+ - __[2023.03.22]__: Launch new feature: generating the 3d face animation from a single image. New applications about it will be updated.
19
+
20
+ - __[2023.03.22]__: Launch new feature: `still mode`, where only a small head pose will be produced via `python inference.py --still`.
21
+
22
+ - __[2023.03.18]__: Support `expression intensity`, now you can change the intensity of the generated motion: `python inference.py --expression_scale 1.3 (some value > 1)`.
23
+
24
+ - __[2023.03.18]__: Reconfig the data folders, now you can download the checkpoint automatically using `bash scripts/download_models.sh`.
25
+ - __[2023.03.18]__: We have offically integrate the [GFPGAN](https://github.com/TencentARC/GFPGAN) for face enhancement, using `python inference.py --enhancer gfpgan` for better visualization performance.
26
+ - __[2023.03.14]__: Specify the version of package `joblib` to remove the errors in using `librosa`, [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Winfredy/SadTalker/blob/main/quick_demo.ipynb) is online!
27
+ - __[2023.03.06]__: Solve some bugs in code and errors in installation
28
+ - __[2023.03.03]__: Release the test code for audio-driven single image animation!
29
+ - __[2023.02.28]__: SadTalker has been accepted by CVPR 2023!
docs/example_crop.gif ADDED

Git LFS Details

  • SHA256: da08306e3e6355928887e74057ee4221f9d877d8536341d907e29fe35e078b45
  • Pointer size: 132 Bytes
  • Size of remote file: 1.55 MB
docs/example_crop_still.gif ADDED

Git LFS Details

  • SHA256: 667c7531ed0a4d97a3ca9b15f79eea655b93dc40eda94498aa43b9e6a48c49aa
  • Pointer size: 132 Bytes
  • Size of remote file: 1.25 MB
docs/example_full.gif ADDED

Git LFS Details

  • SHA256: 2d1a2b8f5ed7b942a8625a5767828c1bc47568165a187079fbbb8492ed57301b
  • Pointer size: 132 Bytes
  • Size of remote file: 1.46 MB
docs/example_full_crop.gif ADDED
docs/example_full_enhanced.gif ADDED

Git LFS Details

  • SHA256: 906ca893e72854021c7715f784dc3fe219bbe67b73ff461e6ba8374f0d3b4712
  • Pointer size: 132 Bytes
  • Size of remote file: 5.78 MB
docs/face3d.md ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## 3D Face visualization
2
+
3
+ We use pytorch3d to visualize the produced 3d face from a single image.
4
+
5
+ Since it is not easy to install, we produce a new install guidence here:
6
+
7
+ ```bash
8
+ git clone https://github.com/Winfredy/SadTalker.git
9
+ cd SadTalker
10
+ conda create -n sadtalker3d python=3.8
11
+ source activate sadtalker3d
12
+
13
+ conda install ffmpeg
14
+ conda install -c fvcore -c iopath -c conda-forge fvcore iopath
15
+ conda install libgcc gmp
16
+
17
+ pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu113
18
+
19
+ # insintall pytorch3d
20
+ pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py38_cu113_pyt1110/download.html
21
+
22
+ pip install -r requirements3d.txt
23
+
24
+ ### install gpfgan for enhancer
25
+ pip install git+https://github.com/TencentARC/GFPGAN
26
+
27
+
28
+ ### when occurs gcc version problem `from pytorch import _C` from pytorch3d, add the anaconda path to LD_LIBRARY_PATH
29
+ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/$YOUR_ANACONDA_PATH/lib/
30
+
31
+ ```
32
+
33
+
34
+ Then, generating the result via:
35
+
36
+ ```bash
37
+
38
+
39
+ python inference.py --driven_audio <audio.wav> \
40
+ --source_image <video.mp4 or picture.png> \
41
+ --result_dir <a file to store results> \
42
+ --face3dvis
43
+
44
+ ```
45
+
46
+ Then, the results will be given in the folders with the file name of `face3d.mp4`.
47
+
48
+ More applications about 3d face will be released.
docs/free_view_result.gif ADDED

Git LFS Details

  • SHA256: 035a7fba6800964254728f82fec47fe5c91458183e19a7506dd54d89940af40f
  • Pointer size: 132 Bytes
  • Size of remote file: 5.61 MB
docs/install.md ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ ### Mac (Tested on M1 Mac OS 13.3)
3
+
4
+ ```
5
+ git clone https://github.com/Winfredy/SadTalker.git
6
+
7
+ cd SadTalker
8
+
9
+ conda create -n sadtalker python=3.8
10
+
11
+ conda activate sadtalker
12
+
13
+ # install pytorch 2.0
14
+ pip install torch torchvision torchaudio
15
+
16
+ conda install ffmpeg
17
+
18
+ pip install -r requirements.txt
19
+
20
+ pip install dlib # mac need to install the original dlib.
21
+
22
+ ```
23
+
24
+
25
+
26
+ ### Windows Native
27
+
28
+ - Make sure you have `ffmpeg` in the `%PATH%` as suggested in [#54](https://github.com/Winfredy/SadTalker/issues/54), following [this](https://www.geeksforgeeks.org/how-to-install-ffmpeg-on-windows/) installation to install `ffmpeg`.
29
+
30
+
31
+ ### Windows WSL
32
+ - Make sure the environment: `export LD_LIBRARY_PATH=/usr/lib/wsl/lib:$LD_LIBRARY_PATH`
33
+
34
+
35
+ ### Docker installnation
36
+
37
+ A dockerfile are also provided by [@thegenerativegeneration](https://github.com/thegenerativegeneration) in [docker hub](https://hub.docker.com/repository/docker/wawa9000/sadtalker), which can be used directly as:
38
+
39
+ ```bash
40
+ docker run --gpus "all" --rm -v $(pwd):/host_dir wawa9000/sadtalker \
41
+ --driven_audio /host_dir/deyu.wav \
42
+ --source_image /host_dir/image.jpg \
43
+ --expression_scale 1.0 \
44
+ --still \
45
+ --result_dir /host_dir
46
+ ```
47
+
docs/resize_good.gif ADDED

Git LFS Details

  • SHA256: ada6f2ea847e71c2a963882fd83f6b54193f4fe7c402f9f20698632b15bbdc0c
  • Pointer size: 132 Bytes
  • Size of remote file: 1.73 MB
docs/resize_no.gif ADDED

Git LFS Details

  • SHA256: c7702f0be5c87c8977bf3c4a73ea4d27e90d0a5a3015816abb880cfd8f75c6ac
  • Pointer size: 132 Bytes
  • Size of remote file: 2.14 MB
docs/sadtalker_logo.png ADDED
docs/using_ref_video.gif ADDED

Git LFS Details

  • SHA256: 9bb68ae077a6c009e7d30a36d34c30bf1310a073ab3c7d9cc1b5c9abe285e888
  • Pointer size: 132 Bytes
  • Size of remote file: 8.11 MB