import argparse import cv2 import glob import mimetypes import numpy as np import os import shutil import subprocess import torch from basicsr.archs.rrdbnet_arch import RRDBNet from basicsr.utils.download_util import load_file_from_url from os import path as osp from tqdm import tqdm from realesrgan import RealESRGANer from realesrgan.archs.srvgg_arch import SRVGGNetCompact try: import ffmpeg except ImportError: import pip pip.main(['install', '--user', 'ffmpeg-python']) import ffmpeg def get_video_meta_info(video_path): ret = {} probe = ffmpeg.probe(video_path) video_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'video'] has_audio = any(stream['codec_type'] == 'audio' for stream in probe['streams']) ret['width'] = video_streams[0]['width'] ret['height'] = video_streams[0]['height'] ret['fps'] = eval(video_streams[0]['avg_frame_rate']) ret['audio'] = ffmpeg.input(video_path).audio if has_audio else None ret['nb_frames'] = int(video_streams[0]['nb_frames']) return ret def get_sub_video(args, num_process, process_idx): if num_process == 1: return args.input meta = get_video_meta_info(args.input) duration = int(meta['nb_frames'] / meta['fps']) part_time = duration // num_process print(f'duration: {duration}, part_time: {part_time}') os.makedirs(osp.join(args.output, f'{args.video_name}_inp_tmp_videos'), exist_ok=True) out_path = osp.join(args.output, f'{args.video_name}_inp_tmp_videos', f'{process_idx:03d}.mp4') cmd = [ args.ffmpeg_bin, f'-i {args.input}', '-ss', f'{part_time * process_idx}', f'-to {part_time * (process_idx + 1)}' if process_idx != num_process - 1 else '', '-async 1', out_path, '-y' ] print(' '.join(cmd)) subprocess.call(' '.join(cmd), shell=True) return out_path class Reader: def __init__(self, args, total_workers=1, worker_idx=0): self.args = args input_type = mimetypes.guess_type(args.input)[0] self.input_type = 'folder' if input_type is None else input_type self.paths = [] # for image&folder type self.audio = None self.input_fps = None if self.input_type.startswith('video'): video_path = get_sub_video(args, total_workers, worker_idx) self.stream_reader = ( ffmpeg.input(video_path).output('pipe:', format='rawvideo', pix_fmt='bgr24', loglevel='error').run_async( pipe_stdin=True, pipe_stdout=True, cmd=args.ffmpeg_bin)) meta = get_video_meta_info(video_path) self.width = meta['width'] self.height = meta['height'] self.input_fps = meta['fps'] self.audio = meta['audio'] self.nb_frames = meta['nb_frames'] else: if self.input_type.startswith('image'): self.paths = [args.input] else: paths = sorted(glob.glob(os.path.join(args.input, '*'))) tot_frames = len(paths) num_frame_per_worker = tot_frames // total_workers + (1 if tot_frames % total_workers else 0) self.paths = paths[num_frame_per_worker * worker_idx:num_frame_per_worker * (worker_idx + 1)] self.nb_frames = len(self.paths) assert self.nb_frames > 0, 'empty folder' from PIL import Image tmp_img = Image.open(self.paths[0]) self.width, self.height = tmp_img.size self.idx = 0 def get_resolution(self): return self.height, self.width def get_fps(self): if self.args.fps is not None: return self.args.fps elif self.input_fps is not None: return self.input_fps return 24 def get_audio(self): return self.audio def __len__(self): return self.nb_frames def get_frame_from_stream(self): img_bytes = self.stream_reader.stdout.read(self.width * self.height * 3) # 3 bytes for one pixel if not img_bytes: return None img = np.frombuffer(img_bytes, np.uint8).reshape([self.height, self.width, 3]) return img def get_frame_from_list(self): if self.idx >= self.nb_frames: return None img = cv2.imread(self.paths[self.idx]) self.idx += 1 return img def get_frame(self): if self.input_type.startswith('video'): return self.get_frame_from_stream() else: return self.get_frame_from_list() def close(self): if self.input_type.startswith('video'): self.stream_reader.stdin.close() self.stream_reader.wait() class Writer: def __init__(self, args, audio, height, width, video_save_path, fps): out_width, out_height = int(width * args.outscale), int(height * args.outscale) if out_height > 2160: print('You are generating video that is larger than 4K, which will be very slow due to IO speed.', 'We highly recommend to decrease the outscale(aka, -s).') if audio is not None: self.stream_writer = ( ffmpeg.input('pipe:', format='rawvideo', pix_fmt='bgr24', s=f'{out_width}x{out_height}', framerate=fps).output( audio, video_save_path, pix_fmt='yuv420p', vcodec='libx264', loglevel='error', acodec='copy').overwrite_output().run_async( pipe_stdin=True, pipe_stdout=True, cmd=args.ffmpeg_bin)) else: self.stream_writer = ( ffmpeg.input('pipe:', format='rawvideo', pix_fmt='bgr24', s=f'{out_width}x{out_height}', framerate=fps).output( video_save_path, pix_fmt='yuv420p', vcodec='libx264', loglevel='error').overwrite_output().run_async( pipe_stdin=True, pipe_stdout=True, cmd=args.ffmpeg_bin)) def write_frame(self, frame): frame = frame.astype(np.uint8).tobytes() self.stream_writer.stdin.write(frame) def close(self): self.stream_writer.stdin.close() self.stream_writer.wait() def inference_video(args, video_save_path, device=None, total_workers=1, worker_idx=0): # ---------------------- determine models according to model names ---------------------- # args.model_name = args.model_name.split('.pth')[0] if args.model_name == 'RealESRGAN_x4plus': # x4 RRDBNet model model = RRDBNet(num_in_ch=3, num_out_ch=3, num_feat=64, num_block=23, num_grow_ch=32, scale=4) netscale = 4 file_url = ['https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth'] elif args.model_name == 'RealESRNet_x4plus': # x4 RRDBNet model model = RRDBNet(num_in_ch=3, num_out_ch=3, num_feat=64, num_block=23, num_grow_ch=32, scale=4) netscale = 4 file_url = ['https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.1/RealESRNet_x4plus.pth'] elif args.model_name == 'RealESRGAN_x4plus_anime_6B': # x4 RRDBNet model with 6 blocks model = RRDBNet(num_in_ch=3, num_out_ch=3, num_feat=64, num_block=6, num_grow_ch=32, scale=4) netscale = 4 file_url = ['https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth'] elif args.model_name == 'RealESRGAN_x2plus': # x2 RRDBNet model model = RRDBNet(num_in_ch=3, num_out_ch=3, num_feat=64, num_block=23, num_grow_ch=32, scale=2) netscale = 2 file_url = ['https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.1/RealESRGAN_x2plus.pth'] elif args.model_name == 'realesr-animevideov3': # x4 VGG-style model (XS size) model = SRVGGNetCompact(num_in_ch=3, num_out_ch=3, num_feat=64, num_conv=16, upscale=4, act_type='prelu') netscale = 4 file_url = ['https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.5.0/realesr-animevideov3.pth'] elif args.model_name == 'realesr-general-x4v3': # x4 VGG-style model (S size) model = SRVGGNetCompact(num_in_ch=3, num_out_ch=3, num_feat=64, num_conv=32, upscale=4, act_type='prelu') netscale = 4 file_url = [ 'https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.5.0/realesr-general-wdn-x4v3.pth', 'https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.5.0/realesr-general-x4v3.pth' ] # ---------------------- determine model paths ---------------------- # model_path = os.path.join('weights', args.model_name + '.pth') if not os.path.isfile(model_path): ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) for url in file_url: # model_path will be updated model_path = load_file_from_url( url=url, model_dir=os.path.join(ROOT_DIR, 'weights'), progress=True, file_name=None) # use dni to control the denoise strength dni_weight = None if args.model_name == 'realesr-general-x4v3' and args.denoise_strength != 1: wdn_model_path = model_path.replace('realesr-general-x4v3', 'realesr-general-wdn-x4v3') model_path = [model_path, wdn_model_path] dni_weight = [args.denoise_strength, 1 - args.denoise_strength] # restorer upsampler = RealESRGANer( scale=netscale, model_path=model_path, dni_weight=dni_weight, model=model, tile=args.tile, tile_pad=args.tile_pad, pre_pad=args.pre_pad, half=not args.fp32, device=device, ) if 'anime' in args.model_name and args.face_enhance: print('face_enhance is not supported in anime models, we turned this option off for you. ' 'if you insist on turning it on, please manually comment the relevant lines of code.') args.face_enhance = False if args.face_enhance: # Use GFPGAN for face enhancement from gfpgan import GFPGANer face_enhancer = GFPGANer( model_path='https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth', upscale=args.outscale, arch='clean', channel_multiplier=2, bg_upsampler=upsampler) # TODO support custom device else: face_enhancer = None reader = Reader(args, total_workers, worker_idx) audio = reader.get_audio() height, width = reader.get_resolution() fps = reader.get_fps() writer = Writer(args, audio, height, width, video_save_path, fps) pbar = tqdm(total=len(reader), unit='frame', desc='inference') while True: img = reader.get_frame() if img is None: break try: if args.face_enhance: _, _, output = face_enhancer.enhance(img, has_aligned=False, only_center_face=False, paste_back=True) else: output, _ = upsampler.enhance(img, outscale=args.outscale) except RuntimeError as error: print('Error', error) print('If you encounter CUDA out of memory, try to set --tile with a smaller number.') else: writer.write_frame(output) torch.cuda.synchronize(device) pbar.update(1) reader.close() writer.close() def run(args): args.video_name = osp.splitext(os.path.basename(args.input))[0] video_save_path = osp.join(args.output, f'{args.video_name}_{args.suffix}.mp4') if args.extract_frame_first: tmp_frames_folder = osp.join(args.output, f'{args.video_name}_inp_tmp_frames') os.makedirs(tmp_frames_folder, exist_ok=True) os.system(f'ffmpeg -i {args.input} -qscale:v 1 -qmin 1 -qmax 1 -vsync 0 {tmp_frames_folder}/frame%08d.png') args.input = tmp_frames_folder num_gpus = torch.cuda.device_count() num_process = num_gpus * args.num_process_per_gpu if num_process == 1: inference_video(args, video_save_path) return ctx = torch.multiprocessing.get_context('spawn') pool = ctx.Pool(num_process) os.makedirs(osp.join(args.output, f'{args.video_name}_out_tmp_videos'), exist_ok=True) pbar = tqdm(total=num_process, unit='sub_video', desc='inference') for i in range(num_process): sub_video_save_path = osp.join(args.output, f'{args.video_name}_out_tmp_videos', f'{i:03d}.mp4') pool.apply_async( inference_video, args=(args, sub_video_save_path, torch.device(i % num_gpus), num_process, i), callback=lambda arg: pbar.update(1)) pool.close() pool.join() # combine sub videos # prepare vidlist.txt with open(f'{args.output}/{args.video_name}_vidlist.txt', 'w') as f: for i in range(num_process): f.write(f'file \'{args.video_name}_out_tmp_videos/{i:03d}.mp4\'\n') cmd = [ args.ffmpeg_bin, '-f', 'concat', '-safe', '0', '-i', f'{args.output}/{args.video_name}_vidlist.txt', '-c', 'copy', f'{video_save_path}' ] print(' '.join(cmd)) subprocess.call(cmd) shutil.rmtree(osp.join(args.output, f'{args.video_name}_out_tmp_videos')) if osp.exists(osp.join(args.output, f'{args.video_name}_inp_tmp_videos')): shutil.rmtree(osp.join(args.output, f'{args.video_name}_inp_tmp_videos')) os.remove(f'{args.output}/{args.video_name}_vidlist.txt') def main(): """Inference demo for Real-ESRGAN. It mainly for restoring anime videos. """ parser = argparse.ArgumentParser() parser.add_argument('-i', '--input', type=str, default='inputs', help='Input video, image or folder') parser.add_argument( '-n', '--model_name', type=str, default='realesr-animevideov3', help=('Model names: realesr-animevideov3 | RealESRGAN_x4plus_anime_6B | RealESRGAN_x4plus | RealESRNet_x4plus |' ' RealESRGAN_x2plus | realesr-general-x4v3' 'Default:realesr-animevideov3')) parser.add_argument('-o', '--output', type=str, default='results', help='Output folder') parser.add_argument( '-dn', '--denoise_strength', type=float, default=0.5, help=('Denoise strength. 0 for weak denoise (keep noise), 1 for strong denoise ability. ' 'Only used for the realesr-general-x4v3 model')) parser.add_argument('-s', '--outscale', type=float, default=4, help='The final upsampling scale of the image') parser.add_argument('--suffix', type=str, default='out', help='Suffix of the restored video') parser.add_argument('-t', '--tile', type=int, default=0, help='Tile size, 0 for no tile during testing') parser.add_argument('--tile_pad', type=int, default=10, help='Tile padding') parser.add_argument('--pre_pad', type=int, default=0, help='Pre padding size at each border') parser.add_argument('--face_enhance', action='store_true', help='Use GFPGAN to enhance face') parser.add_argument( '--fp32', action='store_true', help='Use fp32 precision during inference. Default: fp16 (half precision).') parser.add_argument('--fps', type=float, default=None, help='FPS of the output video') parser.add_argument('--ffmpeg_bin', type=str, default='ffmpeg', help='The path to ffmpeg') parser.add_argument('--extract_frame_first', action='store_true') parser.add_argument('--num_process_per_gpu', type=int, default=1) parser.add_argument( '--alpha_upsampler', type=str, default='realesrgan', help='The upsampler for the alpha channels. Options: realesrgan | bicubic') parser.add_argument( '--ext', type=str, default='auto', help='Image extension. Options: auto | jpg | png, auto means using the same extension as inputs') args = parser.parse_args() args.input = args.input.rstrip('/').rstrip('\\') os.makedirs(args.output, exist_ok=True) if mimetypes.guess_type(args.input)[0] is not None and mimetypes.guess_type(args.input)[0].startswith('video'): is_video = True else: is_video = False if is_video and args.input.endswith('.flv'): mp4_path = args.input.replace('.flv', '.mp4') os.system(f'ffmpeg -i {args.input} -codec copy {mp4_path}') args.input = mp4_path if args.extract_frame_first and not is_video: args.extract_frame_first = False run(args) if args.extract_frame_first: tmp_frames_folder = osp.join(args.output, f'{args.video_name}_inp_tmp_frames') shutil.rmtree(tmp_frames_folder) if __name__ == '__main__': main()