display/display_sample.py

# TRI-VIDAR - Copyright 2022 Toyota Research Institute.  All rights reserved.

import json

import numpy as np
from camviz import BBox3D
from camviz import Camera as CameraCV
from camviz import Draw

from vidar.geometry.camera import Camera
from vidar.geometry.pose import Pose
from vidar.utils.data import make_batch, fold_batch, modrem
from vidar.utils.flip import flip_batch
from vidar.utils.viz import viz_depth, viz_optical_flow, viz_semantic


def change_key(dic, c, n):
    steps = sorted(dic.keys())
    return steps[(steps.index(c) + n) % len(steps)]


def display_sample(data, flip=False):

    tasks = ['rgb', 'depth', 'fwd_optical_flow', 'bwd_optical_flow','semantic']
    cam_colors = ['red', 'blu', 'gre', 'yel', 'mag', 'cya'] * 100

    data = make_batch(data)
    if flip:
        data = flip_batch(data)
    data = fold_batch(data)

    rgb = data['rgb']
    intrinsics = data['intrinsics']
    depth = data['depth']
    pose = data['pose']

    pose = Pose.from_dict(pose, to_global=True)
    cam = Camera.from_dict(intrinsics, rgb, pose)

    num_cams = rgb[0].shape[0]
    wh = rgb[0].shape[-2:][::-1]

    keys = [key for key in tasks if key in data.keys()]

    points = {}
    for key, val in cam.items():
        points[key] = cam[key].reconstruct_depth_map(
            depth[key], to_world=True).reshape(num_cams, 3, -1).permute(0, 2, 1)

    draw = Draw((wh[0] * 4, wh[1] * 3), width=2100)
    draw.add2DimageGrid('cam', (0.0, 0.0, 0.5, 1.0), n=(3, 2), res=wh)
    draw.add3Dworld('wld', (0.5, 0.0, 1.0, 1.0), pose=cam[0].Tcw.T[0])

    draw.addTexture('cam', n=num_cams)
    draw.addBuffer3f('lidar', 1000000, n=num_cams)
    draw.addBuffer3f('color', 1000000, n=num_cams)

    with_bbox3d = 'bbox3d' in data
    if with_bbox3d:
        bbox3d_corners = [[BBox3D(b) for b in bb] for bb in data['bbox3d']['corners']]

    with_pointcache = 'pointcache' in data
    if with_pointcache:
        pointcache = np.concatenate([np.concatenate(pp, 0) for pp in data['pointcache']['points']], 0)
        draw.addBufferf('pointcache', pointcache[:, :3])

    camcv = []
    for i in range(num_cams):
        camcv.append({key: CameraCV.from_vidar(val, i) for key, val in cam.items()})

    t, k = 0, 0
    key = keys[k]
    change = True
    color = True

    while draw.input():
        if draw.SPACE:
            color = not color
            change = True
        if draw.RIGHT:
            change = True
            k = (k + 1) % len(keys)
            while t not in data[keys[k]].keys():
                k = (k + 1) % len(keys)
            key = keys[k]
        if draw.LEFT:
            change = True
            k = (k - 1) % len(keys)
            while t not in data[keys[k]].keys():
                k = (k - 1) % len(keys)
            key = keys[k]
        if draw.UP:
            change = True
            t = change_key(data[key], t, 1)
            while t not in data[keys[k]].keys():
                t = change_key(data[key], t, 1)
        if draw.DOWN:
            change = True
            t = change_key(data[key], t, -1)
            while t not in data[keys[k]].keys():
                t = change_key(data[key], t, -1)
        if change:
            change = False
            for i in range(num_cams):
                img = data[key][t][i]
                if key == 'depth':
                    img = viz_depth(img, filter_zeros=True)
                elif key in ['fwd_optical_flow', 'bwd_optical_flow']:
                    img = viz_optical_flow(img)
                elif key == 'semantic':
                    ontology = json.load(open('vidar/datasets/ontologies/%s.json' % data['tag'][0]))
                    img = viz_semantic(img, ontology)
                draw.updTexture('cam%d' % i, img)
                draw.updBufferf('lidar%d' % i, points[t][i])
                draw.updBufferf('color%d' % i, data['rgb'][t][i])

        draw.clear()
        for i in range(num_cams):
            draw['cam%d%d' % modrem(i, 2)].image('cam%d' % i)
            draw['wld'].size(1).color(cam_colors[i]).points('lidar%d' % i, ('color%d' % i) if color else None)
            for cam_key, cam_val in camcv[i].items():
                clr = cam_colors[i] if cam_key == t else 'gra'
                tex = 'cam%d' % i if cam_key == t else None
                draw['wld'].object(cam_val, color=clr, tex=tex)
            if with_bbox3d:
                [[draw['wld'].object(b) for b in bb] for bb in bbox3d_corners]
            if with_pointcache:
                draw['wld'].color('whi').points('pointcache')

        draw.update(30)