import torch from detectron2.utils.logger import setup_logger setup_logger() from detectron2.config import get_cfg import detectron2.data.transforms as T from detectron2.checkpoint import DetectionCheckpointer from detectron2.modeling import build_model import numpy as np import cv2 import os import argparse import time import h5py import pickle import gradio as gr import fiftyone as fo from fiftyone import ViewField as F import tqdm torch.manual_seed(0) np.random.seed(0) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False from vos.detection.modeling.regnet import build_regnet_fpn_backbone import core.metadata as metadata from utils_clustering import * fullName2ab_dict = {'PASCAL-VOC':"voc", 'BDD100K':"bdd", 'KITTI':"kitti", 'Speed signs':"speed", 'NuScenes':"nu"} ab2FullName_dict = {'voc':"PASCAL-VOC", 'bdd':"BDD100K", 'kitti':"KITTI", 'speed':"Speed signs", 'nu':"NuScenes"} class Detectron2Monitor(): def __init__(self, id, backbone): self.id, self.label_list = self._get_label_list(id) self.backbone = backbone self.cfg, self.device, self.model = self._get_model() self.label_dict = {i:label for i, label in enumerate(self.label_list)} self.eval_list = ["ID-voc-OOD-coco", "OOD-open", "voc-val"] if self.id == "voc" else ["ID-bdd-OOD-coco", "OOD-open", "voc-ood", f"{self.id}-val"] def _get_label_list(self, id): id = fullName2ab_dict[id] if id == 'voc': label_list = metadata.VOC_THING_CLASSES elif id == 'bdd': label_list = metadata.BDD_THING_CLASSES elif id == 'kitti': label_list = metadata.KITTI_THING_CLASSES elif id == 'speed' or id == 'prescan': label_list = metadata.SPEED_THING_CLASSES else: label_list = metadata.NU_THING_CLASSES return id, label_list def _get_model(self): cfg = get_cfg() cfg.merge_from_file(f"/home/hugo/bdd100k-monitoring/monitoringObjectDetection/vanilla_{self.backbone}.yaml") cfg.MODEL.WEIGHTS = f"models/model_final_{self.backbone}_{self.id}.pth" cfg.MODEL.DEVICE='cuda' cfg.MODEL.ROI_HEADS.NUM_CLASSES = len(self.label_list) model = build_model(cfg) model.eval() checkpointer = DetectionCheckpointer(model) checkpointer.load(cfg.MODEL.WEIGHTS) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) return cfg, device, model def _inference(self, model, inputs): with torch.no_grad(): images = model.preprocess_image(inputs) features = model.backbone(images.tensor) proposals, _ = model.proposal_generator(images, features, None) # RPN features_ = [features[f] for f in model.roi_heads.box_in_features] box_features = model.roi_heads.box_pooler(features_, [x.proposal_boxes for x in proposals]) box_features = model.roi_heads.box_head(box_features) # features of all 1k candidates predictions = model.roi_heads.box_predictor(box_features) pred_instances, pred_inds = model.roi_heads.box_predictor.inference(predictions, proposals) pred_instances = model.roi_heads.forward_with_given_boxes(features, pred_instances) # output boxes, masks, scores, etc pred_instances = model._postprocess(pred_instances, inputs, images.image_sizes) # scale box to orig size # features of the proposed boxes feats = box_features[pred_inds].cpu().numpy() return pred_instances, feats def _save_features(self, feats_npy, dataset_view, file_path): features_idx_dict = {cls:[] for cls in self.label_list} for sample in tqdm.tqdm(dataset_view, desc="Saving features"): for detection in sample.prediction.detections: label_pred = detection.label feature_idx = detection.feature_idx features_idx_dict[label_pred].append(feature_idx) feats_dict = {cls:feats_npy[features_idx_dict[cls]] for cls in self.label_list} if not os.path.exists(file_path): os.makedirs(os.path.dirname(file_path), exist_ok=True) with open(file_path, 'wb') as f: pickle.dump(feats_dict, f) def _extract(self, dataset_name): dataset = fo.load_dataset(dataset_name) aug = T.AugmentationList([T.ResizeShortestEdge( [self.cfg.INPUT.MIN_SIZE_TEST, self.cfg.INPUT.MIN_SIZE_TEST], self.cfg.INPUT.MAX_SIZE_TEST), ] ) i = 0 feats_list = [] for sample in tqdm.tqdm(dataset, desc="Extracting features"): image = cv2.imread(sample.filepath) height, width = image.shape[:2] input = T.AugInput(image) transform = aug(input) image = input.image image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)).to(self.device) inputs = [{"image": image, "height": height, "width": width}] preds, feats = self._inference(self.model, inputs) boxes = preds[0]["instances"].pred_boxes.tensor.cpu().detach().numpy() classes = preds[0]["instances"].pred_classes.cpu().detach().numpy() scores = preds[0]["instances"].scores.cpu().detach().numpy() feats_list.extend(feats) if i == 1000: np.save('feats.npy', feats_list) detections = [] for score, label, box in zip(scores, classes, boxes): x1, y1, x2, y2 = box rel_box = [x1/width, y1/height, (x2 - x1) / width, (y2 - y1) / height] label = self.label_dict[label] detections.append( fo.Detection( label=label, bounding_box=rel_box, confidence=score, feature_idx=i ), ) i += 1 sample["prediction"] = fo.Detections(detections=detections) sample.save() feats_npy = np.array(feats_list) with h5py.File(f'feats_{self.id}-train_{self.backbone}.h5', 'w') as f: dset = f.create_dataset(f"feats_{self.id}-train_{self.backbone}", data=feats_npy) if dataset.name.endswith(("train", "val")): results = dataset.evaluate_detections( "prediction", gt_field="detections", eval_key="eval", compute_mAP=True) # results.print_report() # print("mAP: ", results.mAP()) tp_prediction_view = dataset.filter_labels("prediction", F("eval") == "tp") self._save_features(feats_npy, tp_prediction_view, f"train_feats/{self.id}/{self.backbone}/{self.id}-train_feats_tp_dict.pickle") if dataset.name.endswith("val"): fp_prediction_view = dataset.filter_labels("prediction", F("eval") == "fp") self._save_features(feats_npy, fp_prediction_view, f"val_feats/{self.id}/{self.backbone}/{self.dataset_name}_feats_fp_dict.pickle") else: self._save_features(feats_npy, dataset, f"val_feats/{self.id}/{self.backbone}/{self.dataset_name}_feats_fp_dict.pickle") def _construct(self, clustering_algo, nb_clusters=4, eps=5, min_samples=10): with open(f"/home/hugo/bdd100k-monitoring/train_feats/{self.id}/{self.backbone}/{self.id}-train_feats_tp_dict.pickle", 'rb') as f: feats_dict = pickle.load(f) dir_path = f'monitors/{self.id}/{self.backbone}/{clustering_algo}' if not os.path.exists(dir_path): os.makedirs(dir_path) monitor_dict = {} for class_, fts in tqdm.tqdm(feats_dict.items(), desc="Constructing monitors"): if clustering_algo == "kmeans": clusters = k_means_cluster(fts, nb_clusters) elif clustering_algo == "spectral": clusters = spectral_cluster(fts, nb_clusters) elif clustering_algo == "dbscan": clusters = dbscan_cluster(fts, eps, min_samples) dims = fts.shape[1] box_list = [] for cl_id, points in clusters.items(): box = Box() box.build(dims, points) box_list.append(box) monitor = Monitor(good_ref=box_list) monitor_dict[class_] = monitor if clustering_algo == "dbscan": with open(f"monitors/{self.id}/{self.backbone}/{clustering_algo}/eps{eps}_min_samples{min_samples}.pkl" , 'wb') as f: pickle.dump(monitor_dict, f) else: with open(f"monitors/{self.id}/{self.backbone}/{clustering_algo}/{nb_clusters}.pkl" , 'wb') as f: pickle.dump(monitor_dict, f) def _load_monitors(self, clustering_algo, nb_clusters, eps=5, min_samples=10): if clustering_algo == "dbscan": with open(f"monitors/{self.id}/{self.backbone}/{clustering_algo}/eps{eps}_min_samples{min_samples}.pkl", 'rb') as f: monitors_dict = pickle.load(f) else: with open(f"monitors/{self.id}/{self.backbone}/{clustering_algo}/{nb_clusters}.pkl", 'rb') as f: monitors_dict = pickle.load(f) return monitors_dict def _evaluate(self, monitors_dict): dataset_name = f"{self.id}-val" with open(f'val_feats/{self.id}/{self.backbone}/{dataset_name}_feats_tp_dict.pickle', 'rb') as f: feats_tp_dict = pickle.load(f) with open(f'val_feats/{self.id}/{self.backbone}/{dataset_name}_feats_fp_dict.pickle', 'rb') as f: feats_fp_dict = pickle.load(f) # monitors_dict = self._load_monitors(clustering_algo, nb_clusters, eps, min_samples) # make verdicts on ID data data_tp = [] data_fp = [] accept_sum = {"tp": 0, "fp": 0} reject_sum = {"tp": 0, "fp": 0} for label in tqdm.tqdm(self.label_list, desc="Evaluation on ID data"): if label in monitors_dict: verdict = monitors_dict[label].make_verdicts(feats_tp_dict[label]) data_tp.append([label, len(verdict), np.sum(verdict)/len(verdict)]) accept_sum["tp"] += np.sum(verdict) reject_sum["tp"] += len(verdict) - np.sum(verdict) verdict = monitors_dict[label].make_verdicts(feats_fp_dict[label]) data_fp.append([label, len(verdict), (len(verdict)-np.sum(verdict))/len(verdict)]) accept_sum["fp"] += np.sum(verdict) reject_sum["fp"] += len(verdict) - np.sum(verdict) TPR = round((accept_sum['tp'] / (reject_sum['tp'] + accept_sum['tp'])*100), 2) FPR = round((accept_sum['fp'] / (reject_sum['fp'] + accept_sum['fp'])*100), 2) id_name = ab2FullName_dict[self.id] df_id = pd.DataFrame([[id_name, f"{TPR}%", f"{FPR}%"]], columns=["Dataset", "TPR", "FPR"]) data_ood = [] i = 0 self.eval_list.remove(dataset_name) for dataset_name in tqdm.tqdm(self.eval_list, desc="Evaluation on OOD data"): accept_sum = {"tp": 0, "fp": 0} reject_sum = {"tp": 0, "fp": 0} with open(f'val_feats/{self.id}/{self.backbone}/{dataset_name}_feats_fp_dict.pickle', 'rb') as f: feats_fp_dict = pickle.load(f) for label in self.label_list: if label in monitors_dict: verdict = monitors_dict[label].make_verdicts(feats_fp_dict[label]) accept_sum["fp"] += np.sum(verdict) reject_sum["fp"] += len(verdict) - np.sum(verdict) FPR = round((accept_sum['fp'] / (reject_sum['fp'] + accept_sum['fp'])*100), 2) data_ood.append([dataset_name, str(FPR)+"%"]) i += 1 # prepare dataframes df_ood = pd.DataFrame(data_ood, columns=["Dataset", "FPR"]) df_ood["Dataset"] = ["COCO", "Open Images"] if self.id == "voc" else ["COCO", "Open Images", "VOC-OOD"] return df_id, df_ood def _enlarge(self, monitors_dict, delta): for label, monitor in monitors_dict.items(): for i in range(len(monitor.good_ref)): monitor.good_ref[i].ivals = monitor.good_ref[i].ivals*np.array([1-delta, 1+delta]) monitors_dict[label] = monitor return monitors_dict def fx_gradio(id, backbone, progress=gr.Progress(track_tqdm=True)): detectron2monitor = Detectron2Monitor(id, backbone) t0 = time.time() detectron2monitor._extract(f"{detectron2monitor.id}-train") minutes, seconds = divmod(time.time()-t0, 60) return f"Total feature extraction time: {int(minutes):02d}:{int(seconds):02d}" def construct_gradio(id, backbone, clustering_algo, nb_clusters, eps, min_samples, progress=gr.Progress(track_tqdm=True)): detection2monitor = Detectron2Monitor(id, backbone) t0 = time.time() detection2monitor._construct(clustering_algo, nb_clusters, eps, min_samples) minutes, seconds = divmod(time.time()-t0, 60) return f"Total monitor construction time: {int(minutes):02d}:{int(seconds):02d}" def fx_eval_gradio(id, backbone, progress=gr.Progress(track_tqdm=True)): detectron2monitor = Detectron2Monitor(id, backbone) t0 = time.time() for dataset_name in tqdm.tqdm(detectron2monitor.eval_list, desc="Evaluation data preparation"): detectron2monitor._extract(dataset_name) minutes, seconds = divmod(time.time()-t0, 60) return f"Total evaluation data preparation time: {int(minutes):02d}:{int(seconds):02d}" def eval_gradio(id, backbone, clustering_algo, nb_clusters, eps, min_samples, progress=gr.Progress(track_tqdm=True)): detectron2monitor = Detectron2Monitor(id, backbone) monitors_dict = detectron2monitor._load_monitors(clustering_algo, nb_clusters, eps, min_samples) df_id, df_ood = detectron2monitor._evaluate(monitors_dict) return df_id, df_ood def enlarge_gradio(id, backbone, clustering_algo, nb_clusters, eps, min_samples, delta, progress=gr.Progress(track_tqdm=True)): detectron2monitor = Detectron2Monitor(id, backbone) monitors_dict = detectron2monitor._load_monitors(clustering_algo, nb_clusters, eps, min_samples) monitors_dict_enlarge = detectron2monitor._enlarge(monitors_dict, delta) df_id, df_ood = detectron2monitor._evaluate(monitors_dict_enlarge) # if clustering_algo == "dbscan": # with open(f"monitors/{id}/{backbone}/{clustering_algo}/eps{eps}_min_samples{min_samples}_delta{delta}.pkl" , 'wb') as f: # pickle.dump(monitors_dict, f) # else: # with open(f"monitors/{id}/{backbone}/{clustering_algo}/{nb_clusters}_delta{delta}.pkl" , 'wb') as f: # pickle.dump(monitors_dict, f) # return f"Monitors enlarged by {delta*100}%" return df_id, df_ood with gr.Blocks(theme='soft') as demo: gr.Markdown("# Monitor enlargment utility") id = gr.Radio(['PASCAL-VOC', 'BDD100K', 'KITTI', 'Speed signs', 'NuScenes'], label="Dataset") backbone = gr.Radio(['regnet', 'resnet'], label="Backbone") clustering_algo = gr.Dropdown(['kmeans', 'spectral', 'dbscan', 'opticals'], label="Clustering algorithm") with gr.Row(): nb_clusters = gr.Number(value=5, label="Number of clusters", precision=0) eps = gr.Number(value=5, label="Epsilon", precision=0) min_samples = gr.Number(value=10, label="Minimum samples", precision=0) delta = gr.Slider(minimum=0, maximum=2.5, step=0.05, label="Delta") with gr.Row(): with gr.Group("Original monitors"): eval_id = gr.Dataframe(type="pandas", label="ID performance") eavl_ood = gr.Dataframe(type="pandas", label="OOD performance") eval_btn = gr.Button("Monitor Evaluation") with gr.Column("Enlarged monitors"): eval_id2 = gr.Dataframe(type="pandas", label="ID performance") eavl_ood2 = gr.Dataframe(type="pandas", label="OOD performance") enlarge_btn = gr.Button("Monitor Enlargement") eval_btn.click(fn=eval_gradio, inputs=[id, backbone, clustering_algo, nb_clusters, eps, min_samples], outputs=[eval_id, eavl_ood]) enlarge_btn.click(fn=enlarge_gradio, inputs=[id, backbone, clustering_algo, nb_clusters, eps, min_samples, delta], outputs=[eval_id2, eavl_ood2]) demo.queue().launch()