import os import tempfile import gradio import argparse import math import torch import numpy as np import trimesh import copy import functools from scipy.spatial.transform import Rotation from dust3r.inference import inference from dust3r.model import AsymmetricCroCo3DStereo from dust3r.image_pairs import make_pairs from dust3r.utils.image import load_images, rgb from dust3r.utils.device import to_numpy from dust3r.viz import add_scene_cam, CAM_COLORS, OPENGL, pts3d_to_trimesh, cat_meshes from dust3r.cloud_opt import global_aligner, GlobalAlignerMode import matplotlib.pyplot as pl pl.ion() torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 batch_size = 1 def run_dust3r_inference(args, filelist, schedule, niter, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, scenegraph_type, winsize, refid): tmpdirname = tempfile.mkdtemp(suffix='dust3r_gradio_demo') if args.tmp_dir is not None: tmp_path = args.tmp_dir os.makedirs(tmp_path, exist_ok=True) tempfile.tempdir = tmp_path if args.server_name is not None: server_name = args.server_name else: server_name = '0.0.0.0' if args.local_network else '127.0.0.1' if args.weights is not None: weights_path = args.weights else: weights_path = "naver/" + args.model_name model = AsymmetricCroCo3DStereo.from_pretrained(weights_path).to(args.device) recon_fun = functools.partial(get_reconstructed_scene, tmpdirname, model, args.device, args.image_size) model_from_scene_fun = functools.partial(get_3D_model_from_scene, tmpdirname) # Run the reconstruction function scene, outfile, imgs = recon_fun(filelist, schedule, niter, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, scenegraph_type, winsize, refid) # Return the result return outfile, imgs def main_demo(tmpdirname, model, device, image_size, server_name, server_port, silent=False): recon_fun = functools.partial(get_reconstructed_scene, tmpdirname, model, device, silent, image_size) model_from_scene_fun = functools.partial(get_3D_model_from_scene, tmpdirname, silent) with gradio.Blocks(css=""".gradio-container {margin: 0 !important; min-width: 100%};""", title="DUSt3R Demo") as demo: # scene state is save so that you can change conf_thr, cam_size... without rerunning the inference scene = gradio.State(None) gradio.HTML('

DUSt3R Demo

') with gradio.Column(): inputfiles = gradio.File(file_count="multiple") with gradio.Row(): schedule = gradio.Dropdown(["linear", "cosine"], value='linear', label="schedule", info="For global alignment!") niter = gradio.Number(value=300, precision=0, minimum=0, maximum=5000, label="num_iterations", info="For global alignment!") scenegraph_type = gradio.Dropdown(["complete", "swin", "oneref"], value='complete', label="Scenegraph", info="Define how to make pairs", interactive=True) winsize = gradio.Slider(label="Scene Graph: Window Size", value=1, minimum=1, maximum=1, step=1, visible=False) refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, maximum=0, step=1, visible=False) run_btn = gradio.Button("Run") with gradio.Row(): # adjust the confidence threshold min_conf_thr = gradio.Slider(label="min_conf_thr", value=3.0, minimum=1.0, maximum=20, step=0.1) # adjust the camera size in the output pointcloud cam_size = gradio.Slider(label="cam_size", value=0.05, minimum=0.001, maximum=0.1, step=0.001) with gradio.Row(): as_pointcloud = gradio.Checkbox(value=False, label="As pointcloud") # two post process implemented mask_sky = gradio.Checkbox(value=False, label="Mask sky") clean_depth = gradio.Checkbox(value=True, label="Clean-up depthmaps") transparent_cams = gradio.Checkbox(value=False, label="Transparent cameras") outmodel = gradio.Model3D() outgallery = gradio.Gallery(label='rgb,depth,confidence', columns=3, height="100%") # events scenegraph_type.change(set_scenegraph_options, inputs=[inputfiles, winsize, refid, scenegraph_type], outputs=[winsize, refid]) inputfiles.change(set_scenegraph_options, inputs=[inputfiles, winsize, refid, scenegraph_type], outputs=[winsize, refid]) run_btn.click(fn=recon_fun, inputs=[inputfiles, schedule, niter, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, scenegraph_type, winsize, refid], outputs=[scene, outmodel, outgallery]) min_conf_thr.release(fn=model_from_scene_fun, inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size], outputs=outmodel) cam_size.change(fn=model_from_scene_fun, inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size], outputs=outmodel) as_pointcloud.change(fn=model_from_scene_fun, inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size], outputs=outmodel) mask_sky.change(fn=model_from_scene_fun, inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size], outputs=outmodel) clean_depth.change(fn=model_from_scene_fun, inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size], outputs=outmodel) transparent_cams.change(model_from_scene_fun, inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size], outputs=outmodel) demo.launch(share=False, server_name=server_name, server_port=server_port) # Gradio interface components inputfiles = gradio.File(label="Input Images", file_count="multiple") schedule = gradio.Dropdown(["linear", "cosine"], label="Schedule") niter = gradio.Number(label="Number of Iterations", min=0, max=5000, step=1) min_conf_thr = gradio.Slider(label="Minimum Confidence Threshold", min=1.0, max=20.0, step=0.1, default=3.0) as_pointcloud = gradio.Checkbox(label="As Pointcloud", default=False) mask_sky = gradio.Checkbox(label="Mask Sky", default=False) clean_depth = gradio.Checkbox(label="Clean-up Depthmaps", default=True) transparent_cams = gradio.Checkbox(label="Transparent Cameras", default=False) cam_size = gradio.Slider(label="Camera Size", min=0.001, max=0.1, step=0.001, default=0.05) scenegraph_type = gradio.Dropdown(["complete", "swin", "oneref"], label="Scene Graph Type") winsize = gradio.Slider(label="Window Size", min=1, max=1, step=1, default=1) refid = gradio.Slider(label="Reference ID", min=0, max=0, step=1, default=0) # Function to connect Gradio inputs to your main logic def run_inference(filelist, schedule, niter, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, scenegraph_type, winsize, refid): args = None # You need to define your args here outfile, imgs = run_dust3r_inference(args, filelist, schedule, niter, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, scenegraph_type, winsize, refid) return imgs # Launch the Gradio interface iface = gradio.Interface( fn=run_inference, inputs=[inputfiles, schedule, niter, min_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, scenegraph_type, winsize, refid], outputs=gradio.Gallery(label="Output Images", columns=3), title="DUSt3R Demo", description="Reconstruct 3D scenes from input images using DUSt3R.", server_name='0.0.0.0', server_port=7860 ) iface.launch(share=True)