import numpy as np import torch import torch.nn.functional as F import math import cv2 from scipy.stats import qmc from easydict import EasyDict as edict from ..representations.octree import DfsOctree def intrinsics_to_projection( intrinsics: torch.Tensor, near: float, far: float, ) -> torch.Tensor: """ OpenCV intrinsics to OpenGL perspective matrix Args: intrinsics (torch.Tensor): [3, 3] OpenCV intrinsics matrix near (float): near plane to clip far (float): far plane to clip Returns: (torch.Tensor): [4, 4] OpenGL perspective matrix """ fx, fy = intrinsics[0, 0], intrinsics[1, 1] cx, cy = intrinsics[0, 2], intrinsics[1, 2] ret = torch.zeros((4, 4), dtype=intrinsics.dtype, device=intrinsics.device) ret[0, 0] = 2 * fx ret[1, 1] = 2 * fy ret[0, 2] = 2 * cx - 1 ret[1, 2] = - 2 * cy + 1 ret[2, 2] = far / (far - near) ret[2, 3] = near * far / (near - far) ret[3, 2] = 1. return ret def render(viewpoint_camera, octree : DfsOctree, pipe, bg_color : torch.Tensor, scaling_modifier = 1.0, used_rank = None, colors_overwrite = None, aux=None, halton_sampler=None): """ Render the scene. Background tensor (bg_color) must be on GPU! """ # lazy import if 'OctreeTrivecRasterizer' not in globals(): from diffoctreerast import OctreeVoxelRasterizer, OctreeGaussianRasterizer, OctreeTrivecRasterizer, OctreeDecoupolyRasterizer # Set up rasterization configuration tanfovx = math.tan(viewpoint_camera.FoVx * 0.5) tanfovy = math.tan(viewpoint_camera.FoVy * 0.5) raster_settings = edict( image_height=int(viewpoint_camera.image_height), image_width=int(viewpoint_camera.image_width), tanfovx=tanfovx, tanfovy=tanfovy, bg=bg_color, scale_modifier=scaling_modifier, viewmatrix=viewpoint_camera.world_view_transform, projmatrix=viewpoint_camera.full_proj_transform, sh_degree=octree.active_sh_degree, campos=viewpoint_camera.camera_center, with_distloss=pipe.with_distloss, jitter=pipe.jitter, debug=pipe.debug, ) positions = octree.get_xyz if octree.primitive == "voxel": densities = octree.get_density elif octree.primitive == "gaussian": opacities = octree.get_opacity elif octree.primitive == "trivec": trivecs = octree.get_trivec densities = octree.get_density raster_settings.density_shift = octree.density_shift elif octree.primitive == "decoupoly": decoupolys_V, decoupolys_g = octree.get_decoupoly densities = octree.get_density raster_settings.density_shift = octree.density_shift else: raise ValueError(f"Unknown primitive {octree.primitive}") depths = octree.get_depth # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. colors_precomp = None shs = octree.get_features if octree.primitive in ["voxel", "gaussian"] and colors_overwrite is not None: colors_precomp = colors_overwrite shs = None ret = edict() if octree.primitive == "voxel": renderer = OctreeVoxelRasterizer(raster_settings=raster_settings) rgb, depth, alpha, distloss = renderer( positions = positions, densities = densities, shs = shs, colors_precomp = colors_precomp, depths = depths, aabb = octree.aabb, aux = aux, ) ret['rgb'] = rgb ret['depth'] = depth ret['alpha'] = alpha ret['distloss'] = distloss elif octree.primitive == "gaussian": renderer = OctreeGaussianRasterizer(raster_settings=raster_settings) rgb, depth, alpha = renderer( positions = positions, opacities = opacities, shs = shs, colors_precomp = colors_precomp, depths = depths, aabb = octree.aabb, aux = aux, ) ret['rgb'] = rgb ret['depth'] = depth ret['alpha'] = alpha elif octree.primitive == "trivec": raster_settings.used_rank = used_rank if used_rank is not None else trivecs.shape[1] renderer = OctreeTrivecRasterizer(raster_settings=raster_settings) rgb, depth, alpha, percent_depth = renderer( positions = positions, trivecs = trivecs, densities = densities, shs = shs, colors_precomp = colors_precomp, colors_overwrite = colors_overwrite, depths = depths, aabb = octree.aabb, aux = aux, halton_sampler = halton_sampler, ) ret['percent_depth'] = percent_depth ret['rgb'] = rgb ret['depth'] = depth ret['alpha'] = alpha elif octree.primitive == "decoupoly": raster_settings.used_rank = used_rank if used_rank is not None else decoupolys_V.shape[1] renderer = OctreeDecoupolyRasterizer(raster_settings=raster_settings) rgb, depth, alpha = renderer( positions = positions, decoupolys_V = decoupolys_V, decoupolys_g = decoupolys_g, densities = densities, shs = shs, colors_precomp = colors_precomp, depths = depths, aabb = octree.aabb, aux = aux, ) ret['rgb'] = rgb ret['depth'] = depth ret['alpha'] = alpha return ret class OctreeRenderer: """ Renderer for the Voxel representation. Args: rendering_options (dict): Rendering options. """ def __init__(self, rendering_options={}) -> None: try: import diffoctreerast except ImportError: print("\033[93m[WARNING] diffoctreerast is not installed. The renderer will be disabled.\033[0m") self.unsupported = True else: self.unsupported = False self.pipe = edict({ "with_distloss": False, "with_aux": False, "scale_modifier": 1.0, "used_rank": None, "jitter": False, "debug": False, }) self.rendering_options = edict({ "resolution": None, "near": None, "far": None, "ssaa": 1, "bg_color": 'random', }) self.halton_sampler = qmc.Halton(2, scramble=False) self.rendering_options.update(rendering_options) self.bg_color = None def render( self, octree: DfsOctree, extrinsics: torch.Tensor, intrinsics: torch.Tensor, colors_overwrite: torch.Tensor = None, ) -> edict: """ Render the octree. Args: octree (Octree): octree extrinsics (torch.Tensor): (4, 4) camera extrinsics intrinsics (torch.Tensor): (3, 3) camera intrinsics colors_overwrite (torch.Tensor): (N, 3) override color Returns: edict containing: color (torch.Tensor): (3, H, W) rendered color depth (torch.Tensor): (H, W) rendered depth alpha (torch.Tensor): (H, W) rendered alpha distloss (Optional[torch.Tensor]): (H, W) rendered distance loss percent_depth (Optional[torch.Tensor]): (H, W) rendered percent depth aux (Optional[edict]): auxiliary tensors """ resolution = self.rendering_options["resolution"] near = self.rendering_options["near"] far = self.rendering_options["far"] ssaa = self.rendering_options["ssaa"] if self.unsupported: image = np.zeros((512, 512, 3), dtype=np.uint8) text_bbox = cv2.getTextSize("Unsupported", cv2.FONT_HERSHEY_SIMPLEX, 2, 3)[0] origin = (512 - text_bbox[0]) // 2, (512 - text_bbox[1]) // 2 image = cv2.putText(image, "Unsupported", origin, cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 3, cv2.LINE_AA) return { 'color': torch.tensor(image, dtype=torch.float32).permute(2, 0, 1) / 255, } if self.rendering_options["bg_color"] == 'random': self.bg_color = torch.zeros(3, dtype=torch.float32, device="cuda") if np.random.rand() < 0.5: self.bg_color += 1 else: self.bg_color = torch.tensor(self.rendering_options["bg_color"], dtype=torch.float32, device="cuda") if self.pipe["with_aux"]: aux = { 'grad_color2': torch.zeros((octree.num_leaf_nodes, 3), dtype=torch.float32, requires_grad=True, device="cuda") + 0, 'contributions': torch.zeros((octree.num_leaf_nodes, 1), dtype=torch.float32, requires_grad=True, device="cuda") + 0, } for k in aux.keys(): aux[k].requires_grad_() aux[k].retain_grad() else: aux = None view = extrinsics perspective = intrinsics_to_projection(intrinsics, near, far) camera = torch.inverse(view)[:3, 3] focalx = intrinsics[0, 0] focaly = intrinsics[1, 1] fovx = 2 * torch.atan(0.5 / focalx) fovy = 2 * torch.atan(0.5 / focaly) camera_dict = edict({ "image_height": resolution * ssaa, "image_width": resolution * ssaa, "FoVx": fovx, "FoVy": fovy, "znear": near, "zfar": far, "world_view_transform": view.T.contiguous(), "projection_matrix": perspective.T.contiguous(), "full_proj_transform": (perspective @ view).T.contiguous(), "camera_center": camera }) # Render render_ret = render(camera_dict, octree, self.pipe, self.bg_color, aux=aux, colors_overwrite=colors_overwrite, scaling_modifier=self.pipe.scale_modifier, used_rank=self.pipe.used_rank, halton_sampler=self.halton_sampler) if ssaa > 1: render_ret.rgb = F.interpolate(render_ret.rgb[None], size=(resolution, resolution), mode='bilinear', align_corners=False, antialias=True).squeeze() render_ret.depth = F.interpolate(render_ret.depth[None, None], size=(resolution, resolution), mode='bilinear', align_corners=False, antialias=True).squeeze() render_ret.alpha = F.interpolate(render_ret.alpha[None, None], size=(resolution, resolution), mode='bilinear', align_corners=False, antialias=True).squeeze() if hasattr(render_ret, 'percent_depth'): render_ret.percent_depth = F.interpolate(render_ret.percent_depth[None, None], size=(resolution, resolution), mode='bilinear', align_corners=False, antialias=True).squeeze() ret = edict({ 'color': render_ret.rgb, 'depth': render_ret.depth, 'alpha': render_ret.alpha, }) if self.pipe["with_distloss"] and 'distloss' in render_ret: ret['distloss'] = render_ret.distloss if self.pipe["with_aux"]: ret['aux'] = aux if hasattr(render_ret, 'percent_depth'): ret['percent_depth'] = render_ret.percent_depth return ret