import json import os from functools import cache from pickle import load import gradio as gr import matplotlib.pyplot as plt import numpy as np import PIL.Image as Image import torch from huggingface_hub import hf_hub_download from safetensors.torch import load_file from msma import ( ScoreFlow, build_model_from_config, build_model_from_pickle, config_presets, ) @cache def load_model(modeldir, preset="edm2-img64-s-fid", device="cpu"): modeldir = f"{modeldir}/{preset}" with open(f"{modeldir}/config.json", "rb") as f: model_params = json.load(f) scorenet = build_model_from_pickle(preset=preset) model = ScoreFlow(scorenet, **model_params['PatchFlow']) model.flow.load_state_dict(torch.load(f"{modeldir}/flow.pt")) print("Loaded:", model_params) return model.to(device) @cache def load_model_from_hub(preset, device): cache_dir = "/tmp/" if 'DNNLIB_CACHE_DIR' in os.environ: cache_dir = os.environ["DNNLIB_CACHE_DIR"] for fname in ['config.json', 'gmm.pkl', 'refscores.npz', 'model.safetensors' ]: cached_fname = hf_hub_download( repo_id="ahsanMah/localizing-edm", subfolder=preset, filename=fname, cache_dir=cache_dir, ) modeldir = os.path.dirname(cached_fname) print("HF Cache Dir:", modeldir) with open(f"{modeldir}/config.json", "rb") as f: model_params = json.load(f) print("Loaded:", model_params) hf_checkpoint = f"{modeldir}/model.safetensors" model = build_model_from_config(model_params) model.load_state_dict(load_file(hf_checkpoint), strict=True) model = model.eval().requires_grad_(False) model.to(device) return model, modeldir @cache def load_reference_scores(model_dir): with np.load(f"{model_dir}/refscores.npz", "rb") as f: ref_nll = f["arr_0"] return ref_nll def compute_gmm_likelihood(x_score, model_dir): with open(f"{model_dir}/gmm.pkl", "rb") as f: clf = load(f) nll = -clf.score(x_score) ref_nll = load_reference_scores(model_dir) percentile = (ref_nll < nll).mean() * 100 return nll, percentile, ref_nll def plot_against_reference(nll, ref_nll): fig, ax = plt.subplots() ax.hist(ref_nll, label="Reference Scores", bins=25) ax.axvline(nll, label="Image Score", c="red", ls="--") plt.legend() fig.tight_layout() return fig def plot_heatmap(img: Image, heatmap: np.array): fig, ax = plt.subplots() cmap = plt.get_cmap("gist_heat") h = -heatmap[0, 0].copy() qmin, qmax = np.quantile(h, 0.8), np.quantile(h, 0.999) h = np.clip(h, a_min=qmin, a_max=qmax) h = (h - h.min()) / (h.max() - h.min()) h = cmap(h, bytes=True)[:, :, :3] h = Image.fromarray(h).resize(img.size, resample=Image.Resampling.BILINEAR) im = Image.blend(img, h, alpha=0.6) return im @torch.no_grad def run_inference(model, img): img = torch.nn.functional.interpolate(img, size=64, mode="bilinear") score_norms = model.scorenet(img) score_norms = score_norms.square().sum(dim=(2, 3, 4)) ** 0.5 img_likelihood = model(img).cpu().numpy() score_norms = score_norms.cpu().numpy() return img_likelihood, score_norms def localize_anomalies(input_img, preset="edm2-img64-s-fid", load_from_hub=False): orig_size = input_img.size device = "cuda" if torch.cuda.is_available() else "cpu" # img = center_crop_imagenet(64, img) input_img = input_img.resize(size=(64, 64), resample=Image.Resampling.LANCZOS) with torch.inference_mode(): img = np.array(input_img) img = torch.from_numpy(img).permute(2, 0, 1).unsqueeze(0) img = img.float().to(device) if load_from_hub: model, modeldir = load_model_from_hub(preset=preset, device=device) else: model = load_model(modeldir="models", preset=preset, device=device) modeldir = f"models/{preset}" img_likelihood, score_norms = run_inference(model, img) nll, pct, ref_nll = compute_gmm_likelihood( score_norms, model_dir=modeldir ) outstr = f"Anomaly score: {nll:.3f} / {pct:.2f} percentile" histplot = plot_against_reference(nll, ref_nll) heatmapplot = plot_heatmap(input_img, img_likelihood) heatmapplot = heatmapplot.resize(orig_size) return outstr, heatmapplot, histplot def build_demo(inference_fn): demo = gr.Interface( fn=inference_fn, inputs=[ gr.Image(type="pil", label="Input Image"), gr.Dropdown( choices=config_presets.keys(), label="Score Model Preset", info="The preset of the underlying score estimator. These are the EDM2 diffusion models from Karras et.al.", ), gr.Checkbox( label="HuggingFace Hub", value=True, info="Load a pretrained model from HuggingFace. Uncheck to use a model from `models` directory.", ), ], outputs=[ gr.Text( label="Estimated global outlier scores - Percentiles with respect to Imagenette Scores" ), gr.Image(label="Anomaly Heatmap", min_width=160), gr.Plot(label="Comparing to Imagenette"), ], examples=[ ["samples/duckelephant.jpeg", "edm2-img64-s-fid", True], ["samples/sharkhorse.jpeg", "edm2-img64-s-fid", True], ["samples/goldfish.jpeg", "edm2-img64-s-fid", True], ], ) return demo demo = build_demo(localize_anomalies) if __name__ == "__main__": demo.launch()