from typing import Dict import cv2 import numpy as np import tensorflow as tf from PIL import Image from tensorflow import keras RESOLUTION = 224 crop_layer = keras.layers.CenterCrop(RESOLUTION, RESOLUTION) norm_layer = keras.layers.Normalization( mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], variance=[(0.229 * 255) ** 2, (0.224 * 255) ** 2, (0.225 * 255) ** 2], ) rescale_layer = keras.layers.Rescaling(scale=1.0 / 127.5, offset=-1) def preprocess_image(orig_image: Image, model_type: str, size=RESOLUTION): """Image preprocessing utility.""" # Turn the image into a numpy array and add batch dim. image = np.array(orig_image) image = tf.expand_dims(image, 0) # If model type is vit rescale the image to [-1, 1]. if model_type == "original_vit": image = rescale_layer(image) # Resize the image using bicubic interpolation. resize_size = int((256 / 224) * size) image = tf.image.resize(image, (resize_size, resize_size), method="bicubic") # Crop the image. preprocessed_image = crop_layer(image) # If model type is DeiT or DINO normalize the image. if model_type != "original_vit": image = norm_layer(preprocessed_image) return orig_image, preprocessed_image.numpy() def attention_rollout_map( image: Image, attention_score_dict: Dict[str, np.ndarray], model_type: str ): """Computes attention rollout results. Reference: https://arxiv.org/abs/2005.00928 Code copied and modified from here: https://github.com/jeonsworld/ViT-pytorch/blob/main/visualize_attention_map.ipynb """ num_cls_tokens = 2 if "distilled" in model_type else 1 # Stack the individual attention matrices from individual transformer blocks. attn_mat = tf.stack( [attention_score_dict[k] for k in attention_score_dict.keys()] ) attn_mat = tf.squeeze(attn_mat, axis=1) # Average the attention weights across all heads. attn_mat = tf.reduce_mean(attn_mat, axis=1) # To account for residual connections, we add an identity matrix to the # attention matrix and re-normalize the weights. residual_attn = tf.eye(attn_mat.shape[1]) aug_attn_mat = attn_mat + residual_attn aug_attn_mat = ( aug_attn_mat / tf.reduce_sum(aug_attn_mat, axis=-1)[..., None] ) aug_attn_mat = aug_attn_mat.numpy() # Recursively multiply the weight matrices. joint_attentions = np.zeros(aug_attn_mat.shape) joint_attentions[0] = aug_attn_mat[0] for n in range(1, aug_attn_mat.shape[0]): joint_attentions[n] = np.matmul( aug_attn_mat[n], joint_attentions[n - 1] ) # Attention from the output token to the input space. v = joint_attentions[-1] grid_size = int(np.sqrt(aug_attn_mat.shape[-1])) mask = v[0, num_cls_tokens:].reshape(grid_size, grid_size) mask = cv2.resize(mask / mask.max(), image.size)[..., np.newaxis] result = (mask * image).astype("uint8") return result