import argparse import torch import torch.nn.functional as F import torchvision.transforms.functional as TF from torchvision import transforms from PIL import Image import skimage.morphology, skimage.io import cv2 import numpy as np import random from transformers import StoppingCriteria, StoppingCriteriaList from copy import deepcopy from medomni.common.config import Config from medomni.common.dist_utils import get_rank from medomni.common.registry import registry import torchio as tio import nibabel as nib from scipy import ndimage, misc import time import ipdb # Function to parse command line arguments def parse_args(): parser = argparse.ArgumentParser(description="Demo") parser.add_argument("--cfg-path", required=True, help="path to configuration file.") parser.add_argument( "--options", nargs="+", help="override some settings in the used config, the key-value pair in xxx=yyy format will be merged into config file (deprecate), change to --cfg-options instead.", ) args = parser.parse_args() return args def seg_2d_process(image_path, pred_mask, img_size=224): image = cv2.imread(image_path[0]) if pred_mask.sum() != 0: labels = skimage.morphology.label(pred_mask) labelCount = np.bincount(labels.ravel()) largest_label = np.argmax(labelCount[1:]) + 1 pred_mask[labels != largest_label] = 0 pred_mask[labels == largest_label] = 255 pred_mask = pred_mask.astype(np.uint8) contours, _ = cv2.findContours(pred_mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) if contours: contours = np.vstack(contours) binary_array = np.zeros((img_size, img_size)) binary_array = cv2.drawContours(binary_array, contours, -1, 255, thickness=cv2.FILLED) binary_array = cv2.resize(binary_array, (image.shape[1], image.shape[0]), interpolation = cv2.INTER_NEAREST) / 255 image = [Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))] mask = [binary_array] else: image = [Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))] mask = [np.zeros((image.shape[1], image.shape[0]))] else: mask = [np.zeros((image.shape[1], image.shape[0]))] image = [Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))] # output_image = cv2.drawContours(binary_array, contours, -1, (110, 0, 255), 2) # output_image_pil = Image.fromarray(cv2.cvtColor(output_image, cv2.COLOR_BGR2RGB)) return image, mask def seg_3d_process(image_path, seg_mask): img = nib.load(image_path[0]).get_fdata() image = window_scan(img).transpose(2,0,1).astype(np.uint8) if seg_mask.sum() != 0: seg_mask = resize_back_volume_abd(seg_mask, image.shape).astype(np.uint8) image_slices = [] contour_slices = [] for i in range(seg_mask.shape[0]): slice_img = np.fliplr(np.rot90(image[i])) slice_mask = np.fliplr(np.rot90(seg_mask[i])) contours, _ = cv2.findContours(slice_mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) image_slices.append(Image.fromarray(slice_img)) if contours: binary_array = np.zeros(seg_mask.shape[1:]) binary_array = cv2.drawContours(binary_array, contours, -1, 255, thickness=cv2.FILLED) / 255 binary_array = cv2.resize(binary_array, slice_img.shape, interpolation = cv2.INTER_NEAREST) contour_slices.append(binary_array) else: contour_slices.append(np.zeros_like(slice_img)) else: image_slices = [] contour_slices = [] slice_img = np.fliplr(np.rot90(image[i])) image_slices.append(Image.fromarray(slice_img)) contour_slices.append(np.zeros_like(slice_img)) return image_slices, contour_slices def det_2d_process(image_path, box): image_slices = [] image = cv2.imread(image_path[0]) if box is not None: hi,wd,_ = image.shape color = tuple(np.random.random(size=3) * 256) x1, y1, x2, y2 = int(box[0]*wd), int(box[1]*hi), int(box[2]*wd), int(box[3]*hi) image = cv2.rectangle(image, (x1, y1), (x2, y2), color, 10) image_slices.append(Image.fromarray(image)) return image_slices def window_scan(scan, window_center=50, window_width=400): """ Apply windowing to a scan. Parameters: scan (numpy.ndarray): 3D numpy array of the CT scan window_center (int): The center of the window window_width (int): The width of the window Returns: numpy.ndarray: Windowed CT scan """ lower_bound = window_center - (window_width // 2) upper_bound = window_center + (window_width // 2) windowed_scan = np.clip(scan, lower_bound, upper_bound) windowed_scan = (windowed_scan - lower_bound) / (upper_bound - lower_bound) windowed_scan = (windowed_scan * 255).astype(np.uint8) return windowed_scan def task_seg_2d(model, preds, hidden_states, image): token_mask = preds == model.seg_token_idx_2d indices = torch.where(token_mask == True)[0].cpu().numpy() feats = model.model_seg_2d.encoder(image.unsqueeze(0)[:, 0]) last_feats = feats[-1] target_states = [hidden_states[ind][-1] for ind in indices] if target_states: target_states = torch.cat(target_states).squeeze() seg_states = model.text2seg_2d(target_states).unsqueeze(0) last_feats = last_feats + seg_states.unsqueeze(-1).unsqueeze(-1) last_feats = model.text2seg_2d_gn(last_feats) feats[-1] = last_feats seg_feats = model.model_seg_2d.decoder(*feats) seg_preds = model.model_seg_2d.segmentation_head(seg_feats) seg_probs = F.sigmoid(seg_preds) seg_mask = seg_probs.to(dtype=torch.float32).cpu().squeeze().numpy() >= 0.5 return seg_mask else: return None def task_seg_3d(model, preds, hidden_states, img_embeds_list): new_img_embeds_list = deepcopy(img_embeds_list) token_mask = preds == model.seg_token_idx_3d indices = torch.where(token_mask == True)[0].cpu().numpy() target_states = [hidden_states[ind][-1] for ind in indices] if target_states: target_states = torch.cat(target_states).squeeze().unsqueeze(0) seg_states = model.text2seg_3d(target_states) last_feats = new_img_embeds_list[-1] last_feats = last_feats + seg_states.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) last_feats = model.text2seg_3d_gn(last_feats) new_img_embeds_list[-1] = last_feats seg_preds = model.visual_encoder_3d(encoder_only=False, x_=new_img_embeds_list) seg_probs = F.sigmoid(seg_preds) seg_mask = seg_probs.to(dtype=torch.float32).cpu().squeeze().numpy() >= 0.5 return seg_mask def task_det_2d(model, preds, hidden_states): token_mask = preds == model.det_token_idx indices = torch.where(token_mask == True)[0].cpu().numpy() target_states = [hidden_states[ind][-1] for ind in indices] if target_states: target_states = torch.cat(target_states).squeeze() det_states = model.text_det(target_states).detach().cpu() return det_states.numpy() return torch.zeros_like(indices) class StoppingCriteriaSub(StoppingCriteria): def __init__(self, stops=[]): super().__init__() self.stops = stops def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor): for stop in self.stops: if torch.all((stop == input_ids[0][-len(stop):])).item(): return True return False def resize_back_volume_abd(img, target_size): desired_depth = target_size[0] desired_width = target_size[1] desired_height = target_size[2] current_depth = img.shape[0] # [d, w, h] current_width = img.shape[1] current_height = img.shape[2] depth = current_depth / desired_depth width = current_width / desired_width height = current_height / desired_height depth_factor = 1 / depth width_factor = 1 / width height_factor = 1 / height img = ndimage.zoom(img, (depth_factor, width_factor, height_factor), order=0) return img def resize_volume_abd(img): img[img<=-200] = -200 img[img>=300] = 300 desired_depth = 64 desired_width = 192 desired_height = 192 current_width = img.shape[0] # [w, h, d] current_height = img.shape[1] current_depth = img.shape[2] depth = current_depth / desired_depth width = current_width / desired_width height = current_height / desired_height depth_factor = 1 / depth width_factor = 1 / width height_factor = 1 / height img = ndimage.zoom(img, (width_factor, height_factor, depth_factor), order=0) return img def load_and_preprocess_image(image): mean = (0.48145466, 0.4578275, 0.40821073) std = (0.26862954, 0.26130258, 0.27577711) transform = transforms.Compose([ transforms.Resize([224, 224]), transforms.ToTensor(), transforms.Normalize(mean, std) ]) image = transform(image).type(torch.bfloat16).unsqueeze(0) return image def load_and_preprocess_volume(image): img = nib.load(image).get_fdata() image = torch.from_numpy(resize_volume_abd(img)).permute(2,0,1) transform = tio.Compose([ tio.ZNormalization(masking_method=tio.ZNormalization.mean), ]) image = transform(image.unsqueeze(0)).type(torch.bfloat16) return image def read_image(image_path): if image_path.endswith(('.jpg', '.jpeg', '.png')): return load_and_preprocess_image(Image.open(image_path).convert('RGB')) elif image_path.endswith('.nii.gz'): return load_and_preprocess_volume(image_path) else: raise ValueError("Unsupported file format") def generate(model, image_path, image, context, modal, task, num_imgs, prompt, num_beams, do_sample, min_length, top_p, repetition_penalty, length_penalty, temperature): if task == 'report generation' or task == 'classification': prompt = '' + context + '' + prompt img_embeds, atts_img, img_embeds_list = model.encode_img(image.unsqueeze(0), [modal]) placeholder = [''] * 9 prefix = '###Human:' + ''.join([f'' + ''.join(placeholder) + f'' for i in range(num_imgs)]) img_embeds, atts_img = model.prompt_wrap(img_embeds, atts_img, [prefix], [num_imgs]) prompt += '###Assistant:' prompt_tokens = model.llama_tokenizer(prompt, return_tensors="pt", add_special_tokens=False).to(image.device) new_img_embeds, new_atts_img = model.prompt_concat(img_embeds, atts_img, prompt_tokens) outputs = model.llama_model.generate( inputs_embeds=new_img_embeds, max_new_tokens=450, stopping_criteria=StoppingCriteriaList([StoppingCriteriaSub(stops=[ torch.tensor([835]).type(torch.bfloat16).to(image.device), torch.tensor([2277, 29937]).type(torch.bfloat16).to(image.device) ])]), num_beams=num_beams, do_sample=do_sample, min_length=min_length, top_p=top_p, repetition_penalty=repetition_penalty, length_penalty=length_penalty, temperature=temperature, output_hidden_states=True, return_dict_in_generate=True, ) hidden_states = outputs.hidden_states preds = outputs.sequences[0] output_image = None seg_mask_2d = None seg_mask_3d = None if sum(preds == model.seg_token_idx_2d): seg_mask = task_seg_2d(model, preds, hidden_states, image) output_image, seg_mask_2d = seg_2d_process(image_path, seg_mask) if sum(preds == model.seg_token_idx_3d): seg_mask = task_seg_3d(model, preds, hidden_states, img_embeds_list) output_image, seg_mask_3d = seg_3d_process(image_path, seg_mask) if sum(preds == model.det_token_idx): det_box = task_det_2d(model, preds, hidden_states) output_image = det_2d_process(image_path, det_box) if preds[0] == 0: # Remove unknown token at the beginning preds = preds[1:] if preds[0] == 1: # Remove start token at the beginning preds = preds[1:] output_text = model.llama_tokenizer.decode(preds, add_special_tokens=False) output_text = output_text.split('###')[0].split('Assistant:')[-1].strip() if 'mel' in output_text and modal == 'derm': output_text = 'The main diagnosis is melanoma.' return output_image, seg_mask_2d, seg_mask_3d, output_text def generate_predictions(model, images, context, prompt, modality, task, num_beams, do_sample, min_length, top_p, repetition_penalty, length_penalty, temperature, device): num_imgs = len(images) modal = modality.lower() image_tensors = [read_image(img).to(device) for img in images] if modality == 'ct': time.sleep(2) else: time.sleep(1) image_tensor = torch.cat(image_tensors) with torch.autocast(device): with torch.no_grad(): generated_image, seg_mask_2d, seg_mask_3d, output_text = generate(model, images, image_tensor, context, modal, task, num_imgs, prompt, num_beams, do_sample, min_length, top_p, repetition_penalty, length_penalty, temperature) return seg_mask_2d, seg_mask_3d, output_text