import streamlit as st import tifffile import pydicom from scipy.ndimage import zoom import torch from core.models.dani_model import dani_model import numpy as np from PIL import Image import base64 import time # Funzione per convertire un'immagine in base64 def image_to_base64(image_path): with open(image_path, "rb") as img_file: return base64.b64encode(img_file.read()).decode() st.markdown(""" """, unsafe_allow_html=True) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Dati di esempio predefiniti esempi = { "Frontal ➔ Lateral": {'Frontal': 'FtoL.png', 'Lateral': 'LfromF.png'}, "Frontal ➔ Report": {'Frontal': '31d9847f-987fcf63-704f7496-d2b21eb8-63cd973e.tiff', 'Report': 'Small bilateral pleural effusions, left greater than right.'}, "Frontal ➔ Lateral + Report": {'Frontal': '81bca127-0c416084-67f8033c-ecb26476-6d1ecf60.tiff', 'Lateral': 'd52a0c5c-bb7104b0-b1d821a5-959984c3-33c04ccb.tiff', 'Report': 'No acute intrathoracic process. Heart Size is normal. Lungs are clear. No pneumothorax'}, "Lateral ➔ Frontal": {'Lateral': 'LtoF.png', 'Frontal': 'FfromL.png'}, "Lateral ➔ Report": {'Lateral': 'd52a0c5c-bb7104b0-b1d821a5-959984c3-33c04ccb.tiff', 'Report': 'no acute cardiopulmonary process. if concern for injury persists, a dedicated rib series with markers would be necessary to ensure no rib fractures.'}, "Lateral ➔ Frontal + Report": {'Lateral': 'reald52a0c5c-bb7104b0-b1d821a5-959984c3-33c04ccb.tiff', 'Frontal': 'ab37274f-b4c1fc04-e2ff24b4-4a130ba3-cd167968.tiff', 'Report': 'No acute intrathoracic process. If there is strong concern for rib fracture, a dedicated rib series may be performed.'}, "Report ➔ Frontal": {'Report': 'Left lung opacification which may reflect pneumonia superimposed on metastatic disease.', 'Frontal': '02aa804e-bde0afdd-112c0b34-7bc16630-4e384014.tiff'}, "Report ➔ Lateral": {'Report': 'Bilateral pleural effusions, cardiomegaly and mild edema suggest fluid overload.', 'Lateral': '489faba7-a9dc5f1d-fd7241d6-9638d855-eaa952b1.tiff'}, "Report ➔ Frontal + Lateral": {'Report': 'No acute intrathoracic process. The lungs are clean and heart is normal size.', 'Frontal': 'f27ba7cd-44486c2e-29f3e890-f2b9f94e-84110448.tiff', 'Lateral': 'b20c9570-de77944a-b8604ba0-73305a7b-d608a72b.tiff'}, "Frontal + Lateral ➔ Report": {'Frontal': '95856dd1-5878b5b1-9c104817-760c0122-6187946f.tiff', 'Lateral': '3723d912-71940d69-4fef2dd2-27af5a7b-127ba20c.tiff', 'Report': 'Opacities in the right upper or middle lobe, maybe early pneumonia.'}, "Frontal + Report ➔ Lateral": {'Frontal': 'e7f21453-7956d79a-44e44614-fae8ff16-d174d1a0.tiff', 'Report': 'No focal consolidation.', 'Lateral': '8037e6b9-06367464-a4ccd63a-5c5c5a81-ce3e7ffc.tiff'}, "Lateral + Report ➔ Frontal": {'Lateral': '02c66644-b1883a91-54aed0e7-62d25460-398f9865.tiff', 'Report': 'No evidence of acute cardiopulmonary process.', 'Frontal': 'b1f169f1-12177dd5-2fa1c4b1-7b816311-85d769e9.tiff'} } # CSS per personalizzare il tema st.markdown(""" """, unsafe_allow_html=True) # Sostituisci questo con il link dell'immagine online logo_1_path = "./DEMO/Loghi/Logo_UCBM.png" # Sostituisci con il percorso del primo logo logo_2_path = "./DEMO/Loghi/Logo UmU.png" # Sostituisci con il percorso del secondo logo logo_3_path = "./DEMO/Loghi/Logo COSBI.png" # Sostituisci con il percorso del terzo logo logo_4_path = "./DEMO/Loghi/logo trasparent.png" # Sostituisci con il percorso del quarto logo # Converti le immagini in base64 logo_1_base64 = image_to_base64(logo_1_path) logo_2_base64 = image_to_base64(logo_2_path) logo_3_base64 = image_to_base64(logo_3_path) logo_4_base64 = image_to_base64(logo_4_path) # CSS per posizionare i loghi in basso a destra e renderli piccoli st.markdown(f""" """, unsafe_allow_html=True) # Inizializzazione dello stato della sessione if 'step' not in st.session_state: st.session_state['step'] = 1 if 'selected_option' not in st.session_state: st.session_state['selected_option'] = None if 'frontal_file' not in st.session_state: st.session_state['frontal_file'] = None if 'lateral_file' not in st.session_state: st.session_state['lateral_file'] = None if 'report' not in st.session_state: st.session_state['report'] = "" if 'inputs' not in st.session_state: st.session_state['inputs'] = None if 'outputs' not in st.session_state: st.session_state['outputs'] = None if 'frontal' not in st.session_state: st.session_state['frontal'] = None if 'lateral' not in st.session_state: st.session_state['lateral'] = None if 'report' not in st.session_state: st.session_state['report'] = "" if 'generate' not in st.session_state: st.session_state['generate'] = False # Inizializza inference_tester solo una volta if 'inference_tester' not in st.session_state: st.session_state['inference_tester'] = 1 # Usa inference_tester dalla sessione inference_tester = st.session_state['inference_tester'] st.markdown('

MedCoDi-M

', unsafe_allow_html=True) if st.session_state['step'] == 1: # Breve descrizione del lavoro st.markdown("""
This work introduces MedCoDi-M, a novel multi-prompt foundation model for multi-modal medical data generation. In this demo, you will be able to perform various generation tasks including frontal and lateral chest X-rays and clinical report generation. MedCoDi-M enables flexible, any-to-any generation across different medical data modalities, utilizing contrastive learning and a modular approach for enhanced performance.
""", unsafe_allow_html=True) # lasciamo un po' di spazio st.markdown('
', unsafe_allow_html=True) # Immagine con didascalia migliorata e con dimensione della caption aumentata image_path = "./DEMO/Loghi/model_final.png" # Sostituisci con il percorso della tua immagine st.image(image_path, caption='', use_container_width=True) # Caption con dimensione del testo migliorata st.markdown("""
Framework of MedCoDi-M: This demo allows you to generate frontal and lateral chest X-rays, as well as medical reports, through the MedCoDi-M model.
""", unsafe_allow_html=True) # lasciamo un po' di spazio st.markdown('
', unsafe_allow_html=True) # Bottone con testo "Try it out" if st.button("Try it out!"): st.session_state['step'] = 2 st.rerun() # Fase 1: Selezione dell'opzione if st.session_state['step'] == 2: # Opzioni disponibili options = [ "Frontal ➔ Lateral", "Frontal ➔ Report", "Frontal ➔ Lateral + Report", "Lateral ➔ Frontal", "Lateral ➔ Report", "Lateral ➔ Frontal + Report", "Report ➔ Frontal", "Report ➔ Lateral", "Report ➔ Frontal + Lateral", "Frontal + Lateral ➔ Report", "Frontal + Report ➔ Lateral", "Lateral + Report ➔ Frontal" ] # Messaggio di selezione con dimensione aumentata st.markdown( "

Select the type of generation you want to perform:

", unsafe_allow_html=True) # Aumentare la dimensione di "Please select an option:" st.markdown( "

Please select an option:

", unsafe_allow_html=True) # Reset esplicito del valore di `selectbox` in caso di reset st.session_state['selected_option'] = st.selectbox( "", options, key='selectbox_option', index=0) # Rimuoviamo il testo dal selectbox st.markdown('
', unsafe_allow_html=True) # Creiamo colonne per i pulsanti col1, col2, col3 = st.columns(3) # Pulsante per provare un esempio with col1: if st.button("Try an example"): st.session_state['step'] = 5 # Passa al passo 5 st.rerun() # Pulsante per tornare all'inizio with col2: if st.button("Return to the beginning"): # Ripristina lo stato della sessione st.session_state['step'] = 1 st.session_state['selected_option'] = None st.session_state['selected_option2'] = None st.session_state['frontal_file'] = None st.session_state['lateral_file'] = None st.session_state['report'] = "" st.rerun() # Fase 2: Caricamento file if st.session_state['step'] == 3: st.markdown( f"

You selected: {st.session_state['selected_option']}. Now, please upload the required files below:

", unsafe_allow_html=True) # Carica l'immagine frontale if "Frontal" in st.session_state['selected_option'].split(" ➔")[0]: st.markdown("
Load the Frontal X-ray in DICOM format
", unsafe_allow_html=True) st.session_state['frontal_file'] = st.file_uploader("", type=["dcm"]) # Carica l'immagine laterale if "Lateral" in st.session_state['selected_option'].split(" ➔")[0]: st.markdown("
Load the Lateral X-ray in DICOM format
", unsafe_allow_html=True) st.session_state['lateral_file'] = st.file_uploader("", type=["dcm"]) # Inserisci il report clinico if "Report" in st.session_state['selected_option'].split(" ➔")[0]: st.markdown("
Type the clinical report
", unsafe_allow_html=True) st.session_state['report'] = st.text_area("", value=st.session_state['report']) # lasciamo un po' di spazio st.markdown('
', unsafe_allow_html=True) # Creare colonne per allineare i pulsanti in orizzontale col1, col2 = st.columns(2) with col1: if st.button("Start Generation"): frontal = None lateral = None report = None # Dato che questo step è velocissimo, prima di procedere mettiamo una finta barra di caricamento di 3 secondi with st.spinner("Preprocessing the data..."): time.sleep(3) # Controllo che i file necessari siano stati caricati if "Frontal" in st.session_state['selected_option'].split(" ➔")[0] and not st.session_state['frontal_file']: st.error("Load the Frontal image.") elif "Lateral" in st.session_state['selected_option'].split(" ➔")[0] and not st.session_state['lateral_file']: st.error("Load the Lateral image.") elif "Report" in st.session_state['selected_option'].split(" ➔")[0] and not st.session_state['report']: st.error("Type the clinical report.") else: st.write(f"Execution of: {st.session_state['selected_option']}") # Carica l'immagine e avvia l'inferenza if st.session_state['frontal_file']: dicom = pydicom.dcmread(st.session_state['frontal_file']) image = dicom.pixel_array if dicom.PhotometricInterpretation == 'MONOCHROME1': image = (2 ** dicom.BitsStored - 1) - image if dicom.ImagerPixelSpacing != [0.139, 0.139]: zoom_factor = [0.139 / dicom.ImagerPixelSpacing[0], 0.139 / dicom.ImagerPixelSpacing[1]] image = zoom(image, zoom_factor) image = image / (2 ** dicom.BitsStored - 1) # Se l'immagine non è quadrata, facciamo padding if image.shape[0] != image.shape[1]: diff = abs(image.shape[0] - image.shape[1]) pad_size = diff // 2 if image.shape[0] > image.shape[1]: padded_image = np.pad(image, ((0, 0), (pad_size, pad_size))) else: padded_image = np.pad(image, ((pad_size, pad_size), (0, 0))) # Resizing a 256x256 e a 512x512 zoom_factor = [256 / padded_image.shape[0], 256 / padded_image.shape[1]] image_256 = zoom(padded_image, zoom_factor) frontal = image_256 if frontal.dtype != np.uint8: frontal2 = (255 * (frontal - frontal.min()) / (frontal.max() - frontal.min())).astype(np.uint8) frontal = torch.tensor(frontal, dtype=torch.float32).unsqueeze(0).unsqueeze(0) frontal2 = Image.fromarray(frontal2) st.write("Frontal Image loaded successfully!") # Mostra l'immagine caricata st.image(frontal2, caption="Frontal Image Loaded", use_container_width=True) if st.session_state['lateral_file']: dicom = pydicom.dcmread(st.session_state['lateral_file']) image = dicom.pixel_array if dicom.PhotometricInterpretation == 'MONOCHROME1': image = (2 ** dicom.BitsStored - 1) - image if dicom.ImagerPixelSpacing != [0.139, 0.139]: zoom_factor = [0.139 / dicom.ImagerPixelSpacing[0], 0.139 / dicom.ImagerPixelSpacing[1]] image = zoom(image, zoom_factor) image = image / (2 ** dicom.BitsStored - 1) # Se l'immagine non è quadrata, facciamo padding if image.shape[0] != image.shape[1]: diff = abs(image.shape[0] - image.shape[1]) pad_size = diff // 2 if image.shape[0] > image.shape[1]: padded_image = np.pad(image, ((0, 0), (pad_size, pad_size))) else: padded_image = np.pad(image, ((pad_size, pad_size), (0, 0))) # Resizing a 256x256 e a 512x512 zoom_factor = [256 / padded_image.shape[0], 256 / padded_image.shape[1]] image_256 = zoom(padded_image, zoom_factor) lateral = image_256 if lateral.dtype != np.uint8: lateral2 = (255 * (lateral - lateral.min()) / (lateral.max() - lateral.min())).astype(np.uint8) lateral = torch.tensor(lateral, dtype=torch.float32).unsqueeze(0).unsqueeze(0) lateral2 = Image.Frontalmarray(lateral2) st.write("Lateral Image loaded successfully!") st.image(lateral2, caption="Lateral Image Loaded", use_container_width=True) if st.session_state['report']: report = st.session_state['report'] st.write(f"Loaded Report: {report}") inputs = [] if "Frontal" in st.session_state['selected_option'].split(" ➔")[0]: inputs.append('frontal') if "Lateral" in st.session_state['selected_option'].split(" ➔")[0]: inputs.append('lateral') if "Report" in st.session_state['selected_option'].split(" ➔")[0]: inputs.append('text') # Ora vediamo cosa c'è dopo la freccia outputs = [] if "Frontal" in st.session_state['selected_option'].split(" ➔")[1]: outputs.append('frontal') if "Lateral" in st.session_state['selected_option'].split(" ➔")[1]: outputs.append('lateral') if "Report" in st.session_state['selected_option'].split(" ➔")[1]: outputs.append('text') # Ultima cosa che va fatta è passare allo step 4, prima di farlo però, tutte le variabili che ci servono # devono essere salvate nello stato della sessione st.session_state['inputs'] = inputs st.session_state['outputs'] = outputs st.session_state['frontal'] = frontal st.session_state['lateral'] = lateral st.session_state['report'] = report st.session_state['generate'] = True st.session_state['step'] = 4 st.rerun() with col2: if st.button("Return to the beginning"): # Ripristina lo stato della sessione st.session_state['step'] = 1 st.session_state['selected_option'] = None st.session_state['selected_option2'] = None st.session_state['frontal_file'] = None st.session_state['lateral_file'] = None st.session_state['report'] = "" st.rerun() if st.session_state['step'] == 4: st.write("Generation completed successfully!") st.session_state['generate'] = False if st.button("Return to the beginning"): # Ripristina lo stato della sessione st.session_state['generate'] = False st.session_state['step'] = 1 st.session_state['selected_option'] = None st.session_state['frontal_file'] = None st.session_state['lateral_file'] = None st.session_state['report'] = "" st.session_state['inputs'] = None st.session_state['outputs'] = None st.session_state['frontal'] = None st.session_state['lateral'] = None st.session_state['report'] = "" st.rerun() if st.session_state['step'] == 5: st.markdown( f"

You selected: {st.session_state['selected_option']}

", unsafe_allow_html=True) inputs = [] if "Frontal" in st.session_state['selected_option'].split(" ➔")[0]: inputs.append('Frontal') if "Lateral" in st.session_state['selected_option'].split(" ➔")[0]: inputs.append('Lateral') if "Report" in st.session_state['selected_option'].split(" ➔")[0]: inputs.append('Report') outputs = [] if "Frontal" in st.session_state['selected_option'].split(" ➔")[1]: outputs.append('Frontal') if "Lateral" in st.session_state['selected_option'].split(" ➔")[1]: outputs.append('Lateral') if "Report" in st.session_state['selected_option'].split(" ➔")[1]: outputs.append('Report') esempio = esempi[st.session_state['selected_option']] # Mostra i file associati all'esempio st.markdown( "

INPUT:

", unsafe_allow_html=True) # Colonne per gli INPUTS input_cols = st.columns(len(inputs)) for idx, inp in enumerate(inputs): with input_cols[idx]: if inp == 'Frontal': path = "./DEMO/ESEMPI/" + esempio['Frontal'] print(path) if path.endswith(".tiff"): im = tifffile.imread(path) im = np.clip(im, 0, 1) elif path.endswith(".png"): im = Image.open(path) st.image(im, caption="Frontal Image") elif inp == 'Lateral': path = "./DEMO/ESEMPI/" + esempio['Lateral'] if path.endswith(".tiff"): im = tifffile.imread(path) im = np.clip(im, 0, 1) elif path.endswith(".png"): im = Image.open(path) st.image(im, caption="Lateral Image") elif inp == 'Report': st.markdown( f"

Report: {esempio['Report']}

", unsafe_allow_html=True ) st.markdown( "

OUTPUT:

", unsafe_allow_html=True) # Colonne per gli OUTPUTS output_cols = st.columns(len(outputs)) for idx, out in enumerate(outputs): with output_cols[idx]: if out == 'Frontal': path = "./DEMO/ESEMPI/" + esempio['Frontal'] if path.endswith(".tiff"): im = tifffile.imread(path) # facciamo clamp tra 0 e 1 im = np.clip(im, 0, 1) elif path.endswith(".png"): im = Image.open(path) st.image(im, caption="Frontal Image") elif out == 'Lateral': path = "./DEMO/ESEMPI/" + esempio['Lateral'] if path.endswith(".tiff"): im = tifffile.imread(path) # facciamo clamp tra 0 e 1 im = np.clip(im, 0, 1) elif path.endswith(".png"): im = Image.open(path) st.image(im, caption="Lateral Image") elif out == 'Report': st.markdown( f"

Report: {esempio['Report']}

", unsafe_allow_html=True ) # Pulsante per tornare all'inizio if st.button("Return to the beginning"): # Ripristina lo stato della sessione st.session_state['step'] = 1 st.session_state['selected_option'] = None st.session_state['selected_option2'] = None st.session_state['frontal_file'] = None st.session_state['lateral_file'] = None st.session_state['report'] = "" st.rerun()