File size: 4,913 Bytes
7ebfeb9
afda258
 
cf05f8b
22ed06b
 
623b4fb
931c795
e82dfb2
7fcb6d2
c60fb3f
eba7622
afda258
8875dbc
f385ddd
 
7ebfeb9
8875dbc
f385ddd
 
afda258
dee2758
f385ddd
 
fc6f52f
f385ddd
ec31839
8ba8a00
 
 
4c7d9fb
7ebfeb9
97fceae
 
30e7fb3
a4593c9
8ba8a00
97fceae
 
 
ea41590
3049213
97fceae
9cda1d3
97fceae
eba7622
8ba8a00
9e98a91
 
 
 
174fd3c
 
1c1d514
 
2422226
 
8ba8a00
9e98a91
 
 
174fd3c
 
1c1d514
 
9e98a91
 
ea41590
97fceae
ea41590
97fceae
ea41590
97fceae
ea41590
f385ddd
e5a0b2d
c26aed2
97fceae
f103c3f
97fceae
 
9e98a91
 
 
0a2f651
c9254be
ae697d5
 
97fceae
 
 
ea41590
cc88e44
8ba8a00
a4593c9
 
 
97fceae
 
cc88e44
97fceae
8875dbc
623b4fb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
import gradio as gr
from PIL import Image
import clipGPT
import vitGPT
import skimage.io as io
import PIL.Image
import difflib
import ViTCoAtt
import cnnrnn
from build_vocab import Vocabulary
import pickle

# Caption generation functions
def generate_caption_clipgpt(image, max_tokens, temperature):
    caption = clipGPT.generate_caption_clipgpt(image, max_tokens, temperature)
    return caption

def generate_caption_vitgpt(image, max_tokens, temperature):
    caption = vitGPT.generate_caption(image, max_tokens, temperature)
    return caption

def generate_caption_vitCoAtt(image):
    caption = ViTCoAtt.CaptionSampler.main(image)
    return caption

def generate_caption_cnnrnn(image):
    with open('Image_features_ecoder_decoder.pickle', 'rb') as f:
        Xnet_features = pickle.load(f)
        image = Xnet_features[image]
        caption = cnnrnn.get_result(image)
    return caption


with gr.Row():
    image = gr.Image(label="Upload Chest X-ray", type="pil", height='50',width='50')   
                     
 
with gr.Row():
    with gr.Column(): # Column for dropdowns and model choice
        max_tokens = gr.Dropdown(list(range(50, 101)), label="Max Tokens", value=75)
        temperature = gr.Slider(0.5, 0.9, step=0.1, label="Temperature", value=0.9)
        imgID = gr.Dropdown(["1","2","3","4"], label="Choose the ID of the image selected")

    model_choice = gr.Radio(["CLIP-GPT2", "ViT-GPT2", "ViT-CoAttention", "Baseline Model CNN-RNN"], label="Select Model") 
    generate_button = gr.Button("Generate Caption") 
    

caption = gr.Textbox(label="Generated Caption")
real_caption = gr.Textbox(label="Actual Caption")

def getCaption(imgID):   
    real_captions = {"1" : "No acute cardiopulmonary abnormality. 2. Stable bilateral emphysematous and lower lobe fibrotic changes. Bilateral emphysematous again noted and lower lobe fibrotic changes. Postsurgical changes of the chest including CABG procedure, stable. Stable valve artifact. There are no focal areas of consolidation. No large pleural effusions. No evidence of pneumothorax. Degenerative changes noted of the visualized thoracic spine. Nodular right lower lobe opacity, XXXX nipple XXXX. Contour abnormality of the posterior aspect of the right 7th rib again noted, stable.", 
                     "2":"Hypoinflation with bibasilar focal atelectasis. Lung volumes are XXXX. XXXX opacities are present in both lung bases. A hiatal hernia is present. Heart and pulmonary XXXX are normal.",
                     "3":"No evidence of acute cardiopulmonary process. The XXXX examination consists of frontal and lateral radiographs of the chest. External monitor leads XXXX the thorax. The cardiomediastinal contours are within normal limits. Pulmonary vascularity is within normal limits. No focal consolidation, pleural effusion, or pneumothorax identified. The visualized osseous structures and upper abdomen are unremarkable.",
                     "4":"Negative chest . The lungs are clear. The cardiomediastinal silhouette is within normal limits. No pneumothorax or pleural effusion.",
                    "5": "No Actual Caption",
                    "6": "No Actual Caption"}

    return real_captions[imgID]
    
def getImageID(imgID):
    imgIDs = {"1":"/content/drive/MyDrive/cnn-rnn/NLMCXR_png/CXR412_IM-2056_0",
          "2":"/content/drive/MyDrive/cnn-rnn/NLMCXR_png/CXR545_IM-2149_0",
          "3":"/content/drive/MyDrive/cnn-rnn/NLMCXR_png/CXR3044_IM-1418_0",
          "4":"/content/drive/MyDrive/cnn-rnn/NLMCXR_png/CXR3587_IM-1765_0"}
    return imgIDs[imgID]
    
def predict(img, model_name, max_tokens, temperature, imgID):
    if model_name == "CLIP-GPT2":
        return generate_caption_clipgpt(img, max_tokens, temperature), getCaption(imgID)
    elif model_name == "ViT-GPT2":
        return generate_caption_vitgpt(img, max_tokens, temperature), getCaption(imgID)
    elif model_name == "ViT-CoAttention":
        return generate_caption_vitCoAtt(img),  getCaption(imgID)
    elif model_name == "Baseline Model CNN-RNN":
        img = getImageID(imgID)
        return generate_caption_cnnrnn(img), getCaption(imgID)
    else:
        return "select a model","select an image"   



#main call
        
examples = [[f"example{i}.jpg"] for i in range(1,7)]

description= "You can generate captions by uploading an X-Ray and selecting a model of your choice below. Please select the number of Max Tokens and Temperature setting, if you are testing CLIP GPT2 and VIT GPT2 Models"
title = "MedViT: A Vision Transformer-Driven Method for Generating Medical Reports 🏥🤖"

interface = gr.Interface(
        fn=predict,
        inputs = [image, model_choice, max_tokens, temperature, imgID],
        theme="sudeepshouche/minimalist",
        outputs=[caption,real_caption],
        examples = examples,
        title = title,
        description = description
    )


interface.launch(debug=True)