mesh_recon/systems/neus_videonvs.py

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch_efficient_distloss import flatten_eff_distloss

import pytorch_lightning as pl
from pytorch_lightning.utilities.rank_zero import rank_zero_info, rank_zero_debug

import models
from models.utils import cleanup
from models.ray_utils import get_rays
import systems
from systems.base import BaseSystem
from systems.criterions import PSNR, binary_cross_entropy

import pdb


def ranking_loss(error, penalize_ratio=0.7, extra_weights=None, type="mean"):
    # error, indices = torch.sort(error)
    # # only sum relatively small errors
    # s_error = torch.index_select(
    #     error, 0, index=indices[: int(penalize_ratio * indices.shape[0])]
    # )
    # if extra_weights is not None:
    #     weights = torch.index_select(
    #         extra_weights, 0, index=indices[: int(penalize_ratio * indices.shape[0])]
    #     )
    #     s_error = s_error * weights

    if type == "mean":
        return torch.mean(error)
    elif type == "sum":
        return torch.sum(error)


@systems.register("videonvs-neus-system")
class PinholeNeuSSystem(BaseSystem):
    """
    Two ways to print to console:
    1. self.print: correctly handle progress bar
    2. rank_zero_info: use the logging module
    """

    def prepare(self):
        self.criterions = {"psnr": PSNR()}
        self.train_num_samples = self.config.model.train_num_rays * (
            self.config.model.num_samples_per_ray
            + self.config.model.get("num_samples_per_ray_bg", 0)
        )
        self.train_num_rays = self.config.model.train_num_rays
        self.cos = torch.nn.CosineSimilarity(dim=-1, eps=1e-6)

    def forward(self, batch):
        return self.model(batch["rays"])

    def preprocess_data(self, batch, stage):
        if "index" in batch:  # validation / testing
            index = batch["index"]
        else:
            if self.config.model.batch_image_sampling:
                index = torch.randint(
                    0,
                    len(self.dataset.all_images),
                    size=(self.train_num_rays,),
                    device=self.dataset.all_images.device,
                )
            else:
                index = torch.randint(
                    0,
                    len(self.dataset.all_images),
                    size=(1,),
                    device=self.dataset.all_images.device,
                )
        if stage in ["train"]:
            c2w = self.dataset.all_c2w[index]
            x = torch.randint(
                0,
                self.dataset.w,
                size=(self.train_num_rays,),
                device=self.dataset.all_images.device,
            )
            y = torch.randint(
                0,
                self.dataset.h,
                size=(self.train_num_rays,),
                device=self.dataset.all_images.device,
            )
            if self.dataset.directions.ndim == 3:  # (H, W, 3)
                directions = self.dataset.directions[y, x]
                # origins = self.dataset.origins[y, x]
            elif self.dataset.directions.ndim == 4:  # (N, H, W, 3)
                directions = self.dataset.directions[index, y, x]
                # origins = self.dataset.origins[index, y, x]
            rays_o, rays_d = get_rays(directions, c2w)
            rgb = (
                self.dataset.all_images[index, y, x]
                .view(-1, self.dataset.all_images.shape[-1])
                .to(self.rank)
            )
            normal = (
                self.dataset.all_normals_world[index, y, x]
                .view(-1, self.dataset.all_normals_world.shape[-1])
                .to(self.rank)
            )
            fg_mask = self.dataset.all_fg_masks[index, y, x].view(-1).to(self.rank)
            rgb_mask = self.dataset.all_rgb_masks[index, y, x].view(-1).to(self.rank)
            # view_weights = self.dataset.view_weights[index, y, x].view(-1).to(self.rank)
            view_weights = None
        else:
            c2w = self.dataset.all_c2w[index][0]
            if self.dataset.directions.ndim == 3:  # (H, W, 3)
                directions = self.dataset.directions
                # origins = self.dataset.origins
            elif self.dataset.directions.ndim == 4:  # (N, H, W, 3)
                directions = self.dataset.directions[index][0]
                # origins = self.dataset.origins[index][0]
            rays_o, rays_d = get_rays(directions, c2w)
            rgb = (
                self.dataset.all_images[index]
                .view(-1, self.dataset.all_images.shape[-1])
                .to(self.rank)
            )
            normal = (
                self.dataset.all_normals_world[index]
                .view(-1, self.dataset.all_images.shape[-1])
                .to(self.rank)
            )
            fg_mask = self.dataset.all_fg_masks[index].view(-1).to(self.rank)
            rgb_mask = self.dataset.all_rgb_masks[index].view(-1).to(self.rank)
            view_weights = None

        cosines = self.cos(rays_d, normal)
        rays = torch.cat([rays_o, F.normalize(rays_d, p=2, dim=-1)], dim=-1)

        if stage in ["train"]:
            if self.config.model.background_color == "white":
                self.model.background_color = torch.ones(
                    (3,), dtype=torch.float32, device=self.rank
                )
            elif self.config.model.background_color == "black":
                self.model.background_color = torch.zeros(
                    (3,), dtype=torch.float32, device=self.rank
                )
            elif self.config.model.background_color == "random":
                self.model.background_color = torch.rand(
                    (3,), dtype=torch.float32, device=self.rank
                )
            else:
                raise NotImplementedError
        else:
            self.model.background_color = torch.ones(
                (3,), dtype=torch.float32, device=self.rank
            )

        if self.dataset.apply_mask:
            rgb = rgb * fg_mask[..., None] + self.model.background_color * (
                1 - fg_mask[..., None]
            )

        batch.update(
            {
                "rays": rays,
                "rgb": rgb,
                "normal": normal,
                "fg_mask": fg_mask,
                "rgb_mask": rgb_mask,
                "cosines": cosines,
                "view_weights": view_weights,
            }
        )

    def training_step(self, batch, batch_idx):
        out = self(batch)

        cosines = batch["cosines"]
        fg_mask = batch["fg_mask"]
        rgb_mask = batch["rgb_mask"]
        view_weights = batch["view_weights"]

        cosines[cosines > -0.1] = 0
        mask = (fg_mask > 0) & (cosines < -0.1)
        rgb_mask = out["rays_valid_full"][..., 0] & (rgb_mask > 0)

        grad_cosines = self.cos(batch["rays"][..., 3:], out["comp_normal"]).detach()
        # grad_cosines = cosines

        loss = 0.0

        # update train_num_rays
        if self.config.model.dynamic_ray_sampling:
            train_num_rays = int(
                self.train_num_rays
                * (self.train_num_samples / out["num_samples_full"].sum().item())
            )
            self.train_num_rays = min(
                int(self.train_num_rays * 0.9 + train_num_rays * 0.1),
                self.config.model.max_train_num_rays,
            )

        erros_rgb_mse = F.mse_loss(
            out["comp_rgb_full"][rgb_mask], batch["rgb"][rgb_mask], reduction="none"
        )
        # erros_rgb_mse = erros_rgb_mse * torch.exp(grad_cosines.abs())[:, None][rgb_mask] / torch.exp(grad_cosines.abs()[rgb_mask]).sum()
        # loss_rgb_mse = ranking_loss(erros_rgb_mse.sum(dim=1), penalize_ratio=0.7, type='sum')
        # loss_rgb_mse = ranking_loss(
        #     erros_rgb_mse.sum(dim=1), penalize_ratio=0.7, type="mean"
        # )
        loss_rgb_mse = ranking_loss(
            erros_rgb_mse.sum(dim=1), penalize_ratio=1.0, type="mean"
        )
        self.log("train/loss_rgb_mse", loss_rgb_mse, prog_bar=True, rank_zero_only=True)
        loss += loss_rgb_mse * self.C(self.config.system.loss.lambda_rgb_mse)

        loss_rgb_l1 = F.l1_loss(
            out["comp_rgb_full"][rgb_mask], batch["rgb"][rgb_mask], reduction="none"
        )
        loss_rgb_l1 = ranking_loss(
            loss_rgb_l1.sum(dim=1),
            extra_weights=1.0,
            penalize_ratio=1.0,
        )
        self.log("train/loss_rgb", loss_rgb_l1)
        loss += loss_rgb_l1 * self.C(self.config.system.loss.lambda_rgb_l1)

        normal_errors = 1 - F.cosine_similarity(
            out["comp_normal"], batch["normal"], dim=1
        )
        # normal_errors = normal_errors * cosines.abs() / cosines.abs().sum()
        normal_errors = (
            normal_errors * torch.exp(cosines.abs()) / torch.exp(cosines.abs()).sum()
        )
        loss_normal = ranking_loss(
            normal_errors[mask],
            penalize_ratio=0.7,
            #    extra_weights=view_weights[mask],
            type="sum",
        )
        self.log("train/loss_normal", loss_normal, prog_bar=True, rank_zero_only=True)
        loss += loss_normal * self.C(self.config.system.loss.lambda_normal)

        loss_eikonal = (
            (torch.linalg.norm(out["sdf_grad_samples"], ord=2, dim=-1) - 1.0) ** 2
        ).mean()
        self.log("train/loss_eikonal", loss_eikonal, prog_bar=True, rank_zero_only=True)
        loss += loss_eikonal * self.C(self.config.system.loss.lambda_eikonal)

        opacity = torch.clamp(out["opacity"].squeeze(-1), 1.0e-3, 1.0 - 1.0e-3)
        loss_mask = binary_cross_entropy(
            opacity, batch["fg_mask"].float(), reduction="none"
        )
        loss_mask = ranking_loss(loss_mask, penalize_ratio=1.0, extra_weights=1.0)
        self.log("train/loss_mask", loss_mask, prog_bar=True, rank_zero_only=True)
        loss += loss_mask * (
            self.C(self.config.system.loss.lambda_mask)
            if self.dataset.has_mask
            else 0.0
        )

        loss_opaque = binary_cross_entropy(opacity, opacity)
        self.log("train/loss_opaque", loss_opaque)
        loss += loss_opaque * self.C(self.config.system.loss.lambda_opaque)

        loss_sparsity = torch.exp(
            -self.config.system.loss.sparsity_scale * out["random_sdf"].abs()
        ).mean()
        self.log(
            "train/loss_sparsity", loss_sparsity, prog_bar=True, rank_zero_only=True
        )
        loss += loss_sparsity * self.C(self.config.system.loss.lambda_sparsity)

        if self.C(self.config.system.loss.lambda_curvature) > 0:
            assert (
                "sdf_laplace_samples" in out
            ), "Need geometry.grad_type='finite_difference' to get SDF Laplace samples"
            loss_curvature = out["sdf_laplace_samples"].abs().mean()
            self.log("train/loss_curvature", loss_curvature)
            loss += loss_curvature * self.C(self.config.system.loss.lambda_curvature)

        # distortion loss proposed in MipNeRF360
        # an efficient implementation from https://github.com/sunset1995/torch_efficient_distloss
        if self.C(self.config.system.loss.lambda_distortion) > 0:
            loss_distortion = flatten_eff_distloss(
                out["weights"], out["points"], out["intervals"], out["ray_indices"]
            )
            self.log("train/loss_distortion", loss_distortion)
            loss += loss_distortion * self.C(self.config.system.loss.lambda_distortion)

        if (
            self.config.model.learned_background
            and self.C(self.config.system.loss.lambda_distortion_bg) > 0
        ):
            loss_distortion_bg = flatten_eff_distloss(
                out["weights_bg"],
                out["points_bg"],
                out["intervals_bg"],
                out["ray_indices_bg"],
            )
            self.log("train/loss_distortion_bg", loss_distortion_bg)
            loss += loss_distortion_bg * self.C(
                self.config.system.loss.lambda_distortion_bg
            )

        if self.C(self.config.system.loss.lambda_3d_normal_smooth) > 0:
            if "random_sdf_grad" not in out:
                raise ValueError(
                    "random_sdf_grad is required for normal smooth loss, no normal is found in the output."
                )
            if "normal_perturb" not in out:
                raise ValueError(
                    "normal_perturb is required for normal smooth loss, no normal_perturb is found in the output."
                )
            normals_3d = out["random_sdf_grad"]
            normals_perturb_3d = out["normal_perturb"]
            loss_3d_normal_smooth = (normals_3d - normals_perturb_3d).abs().mean()
            self.log(
                "train/loss_3d_normal_smooth", loss_3d_normal_smooth, prog_bar=True
            )

            loss += loss_3d_normal_smooth * self.C(
                self.config.system.loss.lambda_3d_normal_smooth
            )

        losses_model_reg = self.model.regularizations(out)
        for name, value in losses_model_reg.items():
            self.log(f"train/loss_{name}", value)
            loss_ = value * self.C(self.config.system.loss[f"lambda_{name}"])
            loss += loss_

        self.log("train/inv_s", out["inv_s"], prog_bar=True)

        for name, value in self.config.system.loss.items():
            if name.startswith("lambda"):
                self.log(f"train_params/{name}", self.C(value))

        self.log("train/num_rays", float(self.train_num_rays), prog_bar=True)

        return {"loss": loss}

    """
    # aggregate outputs from different devices (DP)
    def training_step_end(self, out):
        pass
    """

    """
    # aggregate outputs from different iterations
    def training_epoch_end(self, out):
        pass
    """

    def validation_step(self, batch, batch_idx):
        out = self(batch)
        psnr = self.criterions["psnr"](
            out["comp_rgb_full"].to(batch["rgb"]), batch["rgb"]
        )
        W, H = self.dataset.img_wh
        self.save_image_grid(
            f"it{self.global_step}-{batch['index'][0].item()}.png",
            [
                {
                    "type": "rgb",
                    "img": batch["rgb"].view(H, W, 3),
                    "kwargs": {"data_format": "HWC"},
                },
                {
                    "type": "rgb",
                    "img": out["comp_rgb_full"].view(H, W, 3),
                    "kwargs": {"data_format": "HWC"},
                },
            ]
            + (
                [
                    {
                        "type": "rgb",
                        "img": out["comp_rgb_bg"].view(H, W, 3),
                        "kwargs": {"data_format": "HWC"},
                    },
                    {
                        "type": "rgb",
                        "img": out["comp_rgb"].view(H, W, 3),
                        "kwargs": {"data_format": "HWC"},
                    },
                ]
                if self.config.model.learned_background
                else []
            )
            + [
                {"type": "grayscale", "img": out["depth"].view(H, W), "kwargs": {}},
                {
                    "type": "rgb",
                    "img": out["comp_normal"].view(H, W, 3),
                    "kwargs": {"data_format": "HWC", "data_range": (-1, 1)},
                },
            ],
        )
        return {"psnr": psnr, "index": batch["index"]}

    """
    # aggregate outputs from different devices when using DP
    def validation_step_end(self, out):
        pass
    """

    def validation_epoch_end(self, out):
        out = self.all_gather(out)
        if self.trainer.is_global_zero:
            out_set = {}
            for step_out in out:
                # DP
                if step_out["index"].ndim == 1:
                    out_set[step_out["index"].item()] = {"psnr": step_out["psnr"]}
                # DDP
                else:
                    for oi, index in enumerate(step_out["index"]):
                        out_set[index[0].item()] = {"psnr": step_out["psnr"][oi]}
            psnr = torch.mean(torch.stack([o["psnr"] for o in out_set.values()]))
            self.log("val/psnr", psnr, prog_bar=True, rank_zero_only=True)
        self.export()

    def test_step(self, batch, batch_idx):
        out = self(batch)
        psnr = self.criterions["psnr"](
            out["comp_rgb_full"].to(batch["rgb"]), batch["rgb"]
        )
        W, H = self.dataset.img_wh
        self.save_image_grid(
            f"it{self.global_step}-test/{batch['index'][0].item()}.png",
            [
                {
                    "type": "rgb",
                    "img": batch["rgb"].view(H, W, 3),
                    "kwargs": {"data_format": "HWC"},
                },
                {
                    "type": "rgb",
                    "img": out["comp_rgb_full"].view(H, W, 3),
                    "kwargs": {"data_format": "HWC"},
                },
            ]
            + (
                [
                    {
                        "type": "rgb",
                        "img": out["comp_rgb_bg"].view(H, W, 3),
                        "kwargs": {"data_format": "HWC"},
                    },
                    {
                        "type": "rgb",
                        "img": out["comp_rgb"].view(H, W, 3),
                        "kwargs": {"data_format": "HWC"},
                    },
                ]
                if self.config.model.learned_background
                else []
            )
            + [
                {"type": "grayscale", "img": out["depth"].view(H, W), "kwargs": {}},
                {
                    "type": "rgb",
                    "img": out["comp_normal"].view(H, W, 3),
                    "kwargs": {"data_format": "HWC", "data_range": (-1, 1)},
                },
            ],
        )
        return {"psnr": psnr, "index": batch["index"]}

    def test_epoch_end(self, out):
        """
        Synchronize devices.
        Generate image sequence using test outputs.
        """
        out = self.all_gather(out)
        if self.trainer.is_global_zero:
            out_set = {}
            for step_out in out:
                # DP
                if step_out["index"].ndim == 1:
                    out_set[step_out["index"].item()] = {"psnr": step_out["psnr"]}
                # DDP
                else:
                    for oi, index in enumerate(step_out["index"]):
                        out_set[index[0].item()] = {"psnr": step_out["psnr"][oi]}
            psnr = torch.mean(torch.stack([o["psnr"] for o in out_set.values()]))
            self.log("test/psnr", psnr, prog_bar=True, rank_zero_only=True)

            self.save_img_sequence(
                f"it{self.global_step}-test",
                f"it{self.global_step}-test",
                "(\d+)\.png",
                save_format="mp4",
                fps=30,
            )

            self.export()

    def export(self):
        mesh = self.model.export(self.config.export)
        self.save_mesh(
            f"it{self.global_step}-{self.config.model.geometry.isosurface.method}{self.config.model.geometry.isosurface.resolution}.obj",
            ortho_scale=self.config.export.ortho_scale,
            **mesh,
        )