.

app.py

@@ -0,0 +1,104 @@
+import numpy as np
+import streamlit as st
+from PIL import Image
+import torch
+import clip
+from torchray.attribution.grad_cam import grad_cam
+from miniclip.imageWrangle import heatmap, min_max_norm, torch_to_rgba
+
+st.set_page_config(layout="wide")
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+@st.cache(show_spinner=True, allow_output_mutation=True)
+def get_model():
+    return clip.load("RN50", device=device, jit=False)
+
+
+# OPTIONS:
+
+st.sidebar.header('Options')
+alpha = st.sidebar.radio("select alpha", [0.5, 0.7, 0.8], index=1)
+layer = st.sidebar.selectbox("select saliency layer", ['layer4.2.relu'], index=0)
+
+st.header("Saliency Map demo for CLIP")
+st.write(
+    "a quick experiment by [Hendrik Strobelt](http://hendrik.strobelt.com) ([MIT-IBM Watson AI Lab](https://mitibmwatsonailab.mit.edu/)) ")
+with st.beta_expander('1. Upload Image', expanded=True):
+    imageFile = st.file_uploader("Select a file:", type=[".jpg", ".png", ".jpeg"])
+
+# st.write("### (2) Enter some desriptive texts.")
+with st.beta_expander('2. Write Descriptions', expanded=True):
+    textarea = st.text_area("Descriptions seperated by semicolon", "a car; a dog; a cat")
+    prefix = st.text_input("(optional) Prefix all descriptions with: ", "an image of")
+
+if imageFile:
+    st.markdown("<hr style='border:black solid;'>", unsafe_allow_html=True)
+    image_raw = Image.open(imageFile)
+    model, preprocess = get_model()
+
+    # preprocess image:
+    image = preprocess(image_raw).unsqueeze(0).to(device)
+
+    # preprocess text
+    prefix = prefix.strip()
+    if len(prefix) > 0:
+        categories = [f"{prefix} {x.strip()}" for x in textarea.split(';')]
+    else:
+        categories = [x.strip() for x in textarea.split(';')]
+    text = clip.tokenize(categories).to(device)
+    # st.write(text)
+    # with st.echo():
+    with torch.no_grad():
+        image_features = model.encode_image(image)
+        text_features = model.encode_text(text)
+        image_features_norm = image_features.norm(dim=-1, keepdim=True)
+        image_features_new = image_features / image_features_norm
+        text_features_norm = text_features.norm(dim=-1, keepdim=True)
+        text_features_new = text_features / text_features_norm
+        logit_scale = model.logit_scale.exp()
+        logits_per_image = logit_scale * image_features_new @ text_features_new.t()
+        probs = logits_per_image.softmax(dim=-1).cpu().numpy().tolist()
+
+    saliency = grad_cam(model.visual, image.type(model.dtype), image_features, saliency_layer=layer)
+    hm = heatmap(image[0], saliency[0][0,].detach().type(torch.float32).cpu(), alpha=alpha)
+
+    collect_images = []
+    for i in range(len(categories)):
+        # mutliply the normalized text embedding with image norm to get approx image embedding
+        text_prediction = (text_features_new[[i]] * image_features_norm)
+        saliency = grad_cam(model.visual, image.type(model.dtype), text_prediction, saliency_layer=layer)
+        hm = heatmap(image[0], saliency[0][0,].detach().type(torch.float32).cpu(), alpha=alpha)
+        collect_images.append(hm)
+    logits = logits_per_image.cpu().numpy().tolist()[0]
+    st.write("### Grad Cam for text embeddings")
+    st.image(collect_images,
+             width=256,
+             caption=[f"{x} - {str(round(y, 3))}/{str(round(l, 2))}" for (x, y, l) in
+                      zip(categories, probs[0], logits)])
+
+    st.write("### Original Image and Grad Cam for image embedding")
+    st.image([Image.fromarray((torch_to_rgba(image[0]).numpy() * 255.).astype(np.uint8)), hm],
+             caption=["original", "image gradcam"])  # , caption="Grad Cam for original embedding")
+
+    # st.image(imageFile)
+
+
+# @st.cache
+def get_readme():
+    with open('README.md') as f:
+        return "\n".join([x.strip() for x in f.readlines()])
+
+
+st.markdown("<hr style='border:black solid;'>", unsafe_allow_html=True)
+with st.beta_expander('Description', expanded=True):
+    st.markdown(get_readme(), unsafe_allow_html=True)
+
+hide_streamlit_style = """
+<style>
+#MainMenu {visibility: hidden;}
+footer {visibility: hidden;}
+</style>
+
+"""
+st.markdown(hide_streamlit_style, unsafe_allow_html=True)

miniclip/imageWrangle.py

@@ -0,0 +1,57 @@
+from PIL import Image
+import numpy as np
+from streamlit.logger import update_formatter
+import torch
+from matplotlib import cm
+
+
+
+def min_max_norm(array):
+    lim = [array.min(), array.max()]
+    array = array - lim[0] 
+    array.mul_(1 / (1.e-10+ (lim[1] - lim[0])))
+    # array = torch.clamp(array, min=0, max=1)
+    return array
+
+def torch_to_rgba(img):
+    img = min_max_norm(img)
+    rgba_im = img.permute(1, 2, 0).cpu()
+    if rgba_im.shape[2] == 3:
+        rgba_im = torch.cat((rgba_im, torch.ones(*rgba_im.shape[:2], 1)), dim=2)
+    assert rgba_im.shape[2] == 4
+    return rgba_im
+
+
+def numpy_to_image(img, size):
+    """
+    takes a [0..1] normalized rgba input and returns resized image as [0...255] rgba image
+    """
+    resized = Image.fromarray((img*255.).astype(np.uint8)).resize((size, size))
+    return resized
+
+def upscale_pytorch(img:np.array, size):
+    torch_img = torch.from_numpy(img).unsqueeze(0).permute(0,3,1,2)
+    print(torch_img)
+    upsampler = torch.nn.Upsample(size=size)    
+    return upsampler(torch_img)[0].permute(1,2,0).cpu().numpy()
+
+
+def heatmap(image:torch.Tensor, heatmap: torch.Tensor, size=None, alpha=.6):
+    if not size:
+        size = image.shape[1]
+    # print(heatmap)
+    # print(min_max_norm(heatmap))
+
+    img = torch_to_rgba(image).numpy() # [0...1] rgba numpy "image"
+    hm = cm.hot(min_max_norm(heatmap).numpy()) # [0...1] rgba numpy "image"
+
+    # print(hm.shape, hm)
+ #
+
+    img = np.array(numpy_to_image(img,size))
+    hm = np.array(numpy_to_image(hm, size))
+    # hm = upscale_pytorch(hm, size)
+    # print (hm) 
+
+    return Image.fromarray((alpha * hm + (1-alpha)*img).astype(np.uint8))
+    # return Image.fromarray(hm)

requirements.txt

@@ -0,0 +1,7 @@
+numpy~=1.20.1
+streamlit~=0.78.0
+torch~=1.7.1
+pillow~=8.1.2
+torchray~=1.0.0.2
+matplotlib~=3.3.4
+git+https://github.com/openai/CLIP.git