cubercnn/data/generate_depth_maps.py

import numpy as np
import torchvision.transforms as transforms
import torch
from PIL import Image
from depth.metric_depth.zoedepth.models.builder import build_model
from depth.metric_depth.zoedepth.utils.config import get_config

def depth_of_images(image, model):
    """
    This function takes in a list of images and returns the depth of the images"""
    # Born out of Issue 36. 
    # Allows  the user to set up own test files to infer on (Create a folder my_test and add subfolder input and output in the metric_depth directory before running this script.)
    # Make sure you have the necessary libraries
    # Code by @1ssb

    # Global settings
    DATASET = 'nyu' # Lets not pick a fight with the model's dataloader

    color_image = Image.fromarray(image).convert('RGB')
    original_width, original_height = color_image.size
    image_tensor = transforms.ToTensor()(color_image).unsqueeze(0).to('cuda' if torch.cuda.is_available() else 'cpu')

    # input as bx3xhxw (unnormalized image)
    pred_o = model(image_tensor, dataset=DATASET)
    if isinstance(pred_o, dict):
        pred = pred_o.get('metric_depth', pred_o.get('out'))
        features = pred_o.get('depth_features', None)
    elif isinstance(pred_o, (list, tuple)):
        pred = pred[-1]
    pred = pred.squeeze().detach().cpu().numpy()

    # Resize color image and depth to final size
    resized_pred = Image.fromarray(pred).resize((original_width, original_height), Image.NEAREST)

    # resized_pred is the image shaped to the original image size, depth is in meters
    return np.array(resized_pred)

def setup_depth_model(model_name, pretrained_resource):
    DATASET = 'nyu' # Lets not pick a fight with the model's dataloader
    config = get_config(model_name, "eval", DATASET)
    config.pretrained_resource = pretrained_resource
    model = build_model(config).to('cuda' if torch.cuda.is_available() else 'cpu')
    model.eval()
    return model

def init_dataset():
    ''' dataloader stuff.
     I'm not sure what the difference between the omni3d dataset and load omni3D json functions are. this is a 3rd alternative to this. The train script calls something similar to this.'''
    cfg, filter_settings = get_config_and_filter_settings()

    dataset_names = ['SUNRGBD_train','SUNRGBD_val','SUNRGBD_test']
    dataset_paths_to_json = ['datasets/Omni3D/'+dataset_name+'.json' for dataset_name in dataset_names]
    # for dataset_name in dataset_names:
    #     simple_register(dataset_name, filter_settings, filter_empty=True)

    # Get Image and annotations
    datasets = data.Omni3D(dataset_paths_to_json, filter_settings=filter_settings)
    data.register_and_store_model_metadata(datasets, cfg.OUTPUT_DIR, filter_settings)


    thing_classes = MetadataCatalog.get('omni3d_model').thing_classes
    dataset_id_to_contiguous_id = MetadataCatalog.get('omni3d_model').thing_dataset_id_to_contiguous_id

    infos = datasets.dataset['info']

    dataset_id_to_unknown_cats = {}
    possible_categories = set(i for i in range(cfg.MODEL.ROI_HEADS.NUM_CLASSES + 1))
    
    dataset_id_to_src = {}

    for info in infos:
        dataset_id = info['id']
        known_category_training_ids = set()
        
        if not dataset_id in dataset_id_to_src:
            dataset_id_to_src[dataset_id] = info['source']

        for id in info['known_category_ids']:
            if id in dataset_id_to_contiguous_id:
                known_category_training_ids.add(dataset_id_to_contiguous_id[id])
        
        # determine and store the unknown categories.
        unknown_categories = possible_categories - known_category_training_ids
        dataset_id_to_unknown_cats[dataset_id] = unknown_categories

    return datasets

if __name__ == '__main__':
    import os
    from detectron2.data.catalog import MetadataCatalog

    from cubercnn import data
    from priors import get_config_and_filter_settings

    import torch.nn.functional as F

    from tqdm import tqdm
    # datasets = init_dataset()

    os.makedirs('datasets/depth_maps', exist_ok=True)

    depth_model = 'zoedepth'
    pretrained_resource = 'local::depth/checkpoints/depth_anything_metric_depth_indoor.pt'
    model = setup_depth_model(depth_model, pretrained_resource)
    for img_id in tqdm(os.listdir('datasets/coco_examples')):
        file_path = 'coco_examples/'+img_id
    # for img_id, img_info in track(datasets.imgs.items()):
    #     file_path = img_info['file_path']
        img = np.array(Image.open('datasets/'+file_path))
        depth = depth_of_images(img, model)
        np.savez_compressed(f'datasets/depth_maps/{img_id}.npz', depth=depth)