import glob import tempfile from decimal import Decimal from pathlib import Path from typing import List, Dict, Any import gradio as gr from PIL import Image import open_clip import torch import os import pandas as pd import numpy as np from gradio import processing_utils, utils from download_example_images import read_actor_files, save_images_to_folder DEFAULT_INITIAL_NAME = "John Doe" PROMPTS = [ '{0}', 'an image of {0}', 'a photo of {0}', '{0} on a photo', 'a photo of a person named {0}', 'a person named {0}', 'a man named {0}', 'a woman named {0}', 'the name of the person is {0}', 'a photo of a person with the name {0}', '{0} at a gala', 'a photo of the celebrity {0}', 'actor {0}', 'actress {0}', 'a colored photo of {0}', 'a black and white photo of {0}', 'a cool photo of {0}', 'a cropped photo of {0}', 'a cropped image of {0}', '{0} in a suit', '{0} in a dress' ] OPEN_CLIP_MODEL_NAMES = ['ViT-B-32', 'ViT-B-16', 'ViT-L-14'] NUM_TOTAL_NAMES = 1_000 SEED = 42 MIN_NUM_CORRECT_PROMPT_PREDS = 1 EDAMPLE_IMAGE_DIR = './example_images/' IMG_BATCHSIZE = 16 DEVICE = "cuda" if torch.cuda.is_available() else "cpu" EXAMPLE_IMAGE_URLS = read_actor_files(EDAMPLE_IMAGE_DIR) save_images_to_folder(os.path.join(EDAMPLE_IMAGE_DIR, 'images'), EXAMPLE_IMAGE_URLS) MODELS = {} for model_name in OPEN_CLIP_MODEL_NAMES: dataset = 'LAION400M' model, _, preprocess = open_clip.create_model_and_transforms( model_name, pretrained=f'{dataset.lower()}_e32' ) model = model.eval() MODELS[f'OpenClip {model_name} trained on {dataset}'] = { 'model_instance': model, 'preprocessing': preprocess, 'model_name': model_name, 'prompt_text_embeddings': torch.load(f'./prompt_text_embeddings/{model_name}_prompt_text_embeddings.pt') } FULL_NAMES_DF = pd.read_csv('full_names.csv', index_col=0) LAION_MEMBERSHIP_OCCURENCE = pd.read_csv('laion_membership_occurence_count.csv', index_col=0) EXAMPLE_ACTORS_BY_MODEL = { "ViT-B-32": ["T._J._Thyne"], "ViT-B-16": ["Barbara_Schöneberger", "Carolin_Kebekus"], "ViT-L-14": ["Max_Giermann", "Nicole_De_Boer"] } EXAMPLES = [] for model_name, person_names in EXAMPLE_ACTORS_BY_MODEL.items(): for name in person_names: image_folder = os.path.join("./example_images/images/", name) for dd_model_name in MODELS.keys(): if model_name not in dd_model_name: continue EXAMPLES.append([ dd_model_name, name.replace("_", " "), [[x.format(name.replace("_", " ")) for x in PROMPTS]], [os.path.join(image_folder, x) for x in os.listdir(image_folder)] ]) LICENSE_DETAILS = """
Example Images License Information ### Barbara Schöneberger | Image Name | Image Url | Author | License | |----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------|--------------| | Barbara_Schöneberger_0.jpg | [https://upload.wikimedia.org/wikipedia/commons/1/1d/Barbara_Sch%C3%B6neberger_-_Deutscher_Radiopreis_Hamburg_2016_13.jpg](https://upload.wikimedia.org/wikipedia/commons/1/1d/Barbara_Sch%C3%B6neberger_-_Deutscher_Radiopreis_Hamburg_2016_13.jpg) | Frank Schwichtenberg | CC-BY-SA-3.0 | | Barbara_Schöneberger_1.jpg | [https://upload.wikimedia.org/wikipedia/commons/9/9d/Barbara_Sch%C3%B6neberger_%282007%29.jpg](https://upload.wikimedia.org/wikipedia/commons/9/9d/Barbara_Sch%C3%B6neberger_%282007%29.jpg) | Pottschalk | CC-BY-SA-3.0 | | Barbara_Schöneberger_2.jpg | [https://upload.wikimedia.org/wikipedia/commons/f/f0/Barbara_Sch%C3%B6neberger_-_Deutscher_Radiopreis_Hamburg_2016_03.jpg](https://upload.wikimedia.org/wikipedia/commons/f/f0/Barbara_Sch%C3%B6neberger_-_Deutscher_Radiopreis_Hamburg_2016_03.jpg) | Frank Schwichtenberg | CC-BY-SA-3.0 | | Barbara_Schöneberger_3.jpg | [https://upload.wikimedia.org/wikipedia/commons/f/fa/Barbara_Sch%C3%B6neberger_-_Deutscher_Radiopreis_Hamburg_2016_12.jpg](https://upload.wikimedia.org/wikipedia/commons/f/fa/Barbara_Sch%C3%B6neberger_-_Deutscher_Radiopreis_Hamburg_2016_12.jpg) | Frank Schwichtenberg | CC-BY-SA-3.0 | | Barbara_Schöneberger_4.jpg | [https://upload.wikimedia.org/wikipedia/commons/0/0a/Barbara_Sch%C3%B6neberger_-_Deutscher_Radiopreis_Hamburg_2016_01.jpg](https://upload.wikimedia.org/wikipedia/commons/0/0a/Barbara_Sch%C3%B6neberger_-_Deutscher_Radiopreis_Hamburg_2016_01.jpg) | Frank Schwichtenberg | CC-BY-SA-3.0 | ### Carolin Kebekus | Image Name | Image Url | Author | License | |-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------|--------------| | Carolin_Kebekus_0.jpg | [https://upload.wikimedia.org/wikipedia/commons/c/ce/Carolin_Kebekus_-_2019102193318_2019-04-12_Radio_Regenbogen_Award_2019_-_Sven_-_1D_X_MK_II_-_0905_-_AK8I0075.jpg](https://upload.wikimedia.org/wikipedia/commons/c/ce/Carolin_Kebekus_-_2019102193318_2019-04-12_Radio_Regenbogen_Award_2019_-_Sven_-_1D_X_MK_II_-_0905_-_AK8I0075.jpg) | Sven Mandel | CC-BY-SA-4.0 | | Carolin_Kebekus_1.jpg | [https://upload.wikimedia.org/wikipedia/commons/4/45/Carolin-Kebekus-Bonn.jpg](https://upload.wikimedia.org/wikipedia/commons/4/45/Carolin-Kebekus-Bonn.jpg) | Superbass | CC-BY-SA-3.0 | | Carolin_Kebekus_2.jpg | [https://upload.wikimedia.org/wikipedia/commons/4/45/Carolin-Kebekus-Bonn.jpg](https://upload.wikimedia.org/wikipedia/commons/4/45/Carolin-Kebekus-Bonn.jpg) | Sven Mandel | CC-BY-SA-4.0 | | Carolin_Kebekus_3.jpg | [https://upload.wikimedia.org/wikipedia/commons/0/02/Carolin_Kebekus-5848.jpg](https://upload.wikimedia.org/wikipedia/commons/0/02/Carolin_Kebekus-5848.jpg) | Harald Krichel | CC-BY-SA-3.0 | | Carolin_Kebekus_4.jpg | [https://upload.wikimedia.org/wikipedia/commons/e/e1/2021-09-16-Carolin_Kebekus_Deutscher_Fernsehpreis_2021_-3757.jpg](https://upload.wikimedia.org/wikipedia/commons/e/e1/2021-09-16-Carolin_Kebekus_Deutscher_Fernsehpreis_2021_-3757.jpg) | Superbass | CC-BY-SA-4.0 | ### Max Giermann | Image Name | Image Url | Author | License | |--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------|--------------| | Max_Giermann_0.jpg | [https://upload.wikimedia.org/wikipedia/commons/4/4b/2018-01-26-DFP_2018-7513.jpg](https://upload.wikimedia.org/wikipedia/commons/4/4b/2018-01-26-DFP_2018-7513.jpg) | Superbass | CC-BY-SA-4.0 | | Max_Giermann_1.jpg | [https://upload.wikimedia.org/wikipedia/commons/f/f6/Deutscher_Fernsehpreis_2012_-_Max_Giermann.jpg](https://upload.wikimedia.org/wikipedia/commons/f/f6/Deutscher_Fernsehpreis_2012_-_Max_Giermann.jpg) | JCS | CC-BY-3.0 | | Max_Giermann_2.jpg | [https://upload.wikimedia.org/wikipedia/commons/1/1c/Hessischer_Filmpreis_2017_-_Max_Giermann_2.JPG](https://upload.wikimedia.org/wikipedia/commons/1/1c/Hessischer_Filmpreis_2017_-_Max_Giermann_2.JPG) | JCS | CC-BY-3.0 | | Max_Giermann_3.jpg | [https://upload.wikimedia.org/wikipedia/commons/1/1d/Max_Giermann_%28extra_3%29_01.jpg](https://upload.wikimedia.org/wikipedia/commons/1/1d/Max_Giermann_%28extra_3%29_01.jpg) | Frank Schwichtenberg | CC-BY-SA-3.0 | | Max_Giermann_4.jpg | [https://upload.wikimedia.org/wikipedia/commons/8/85/Max_Giermann_%28extra_3%29_03.jpg](https://upload.wikimedia.org/wikipedia/commons/8/85/Max_Giermann_%28extra_3%29_03.jpg) | Frank Schwichtenberg | CC-BY-SA-3.0 | ### Nicole De Boer | Image Name | Image Url | Author | License | |----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------------| | Nicole_De_Boer_0.jpg | [https://upload.wikimedia.org/wikipedia/commons/0/03/Praha%2C_Lhotka%2C_KC_Novodvorsk%C3%A1%2C_CzechTREK_2013_%2827%29.jpg](https://upload.wikimedia.org/wikipedia/commons/0/03/Praha%2C_Lhotka%2C_KC_Novodvorsk%C3%A1%2C_CzechTREK_2013_%2827%29.jpg) | Harold | CC-BY-SA-3.0 | | Nicole_De_Boer_1.jpg | [https://upload.wikimedia.org/wikipedia/commons/d/db/Nicole_DeBoer_at_Toronto_Comicon_1.jpg](https://upload.wikimedia.org/wikipedia/commons/d/db/Nicole_DeBoer_at_Toronto_Comicon_1.jpg) | Tabercil | CC-BY-SA-3.0 | | Nicole_De_Boer_2.jpg | [https://upload.wikimedia.org/wikipedia/commons/4/4b/Nicole_de_Boer_at_Toronto_Comicon_2_%28cropped%29.jpg](https://upload.wikimedia.org/wikipedia/commons/4/4b/Nicole_de_Boer_at_Toronto_Comicon_2_%28cropped%29.jpg) | Tabercil | CC-BY-SA-3.0 | | Nicole_De_Boer_3.jpg | [https://upload.wikimedia.org/wikipedia/commons/b/b9/Nicole_de_boer_LFCC2015.jpg](https://upload.wikimedia.org/wikipedia/commons/b/b9/Nicole_de_boer_LFCC2015.jpg) | Dazzoboy | CC-BY-SA-4.0 | | Nicole_De_Boer_4.jpg | [https://upload.wikimedia.org/wikipedia/commons/9/90/Nicole_de_Boer_at_Toronto_Comicon_2.jpg](https://upload.wikimedia.org/wikipedia/commons/9/90/Nicole_de_Boer_at_Toronto_Comicon_2.jpg) | Tabercil | CC-BY-SA-3.0 | ### T. J. Thyne | Image Name | Image Url | Author | License | |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|--------------| | T._J._Thyne_0.jpg | [https://live.staticflickr.com/7036/6837850246_c09a148d70_o.jpg](https://live.staticflickr.com/7036/6837850246_c09a148d70_o.jpg) | Genevieve | CC-BY-2.0 | | T._J._Thyne_1.jpg | [https://live.staticflickr.com/3273/5705869811_d9ff808383_o.jpg](https://live.staticflickr.com/3273/5705869811_d9ff808383_o.jpg) | Genevieve | CC-BY-2.0 | | T._J._Thyne_2.jpg | [https://upload.wikimedia.org/wikipedia/commons/d/d8/TJThyneFanExpo2017.jpg](https://upload.wikimedia.org/wikipedia/commons/d/d8/TJThyneFanExpo2017.jpg) | Christian Dahl-Lacroix | CC-BY-SA-4.0 | | T._J._Thyne_3.jpg | [https://live.staticflickr.com/7041/6984629777_8a415b72d9_b.jpg](https://live.staticflickr.com/7041/6984629777_8a415b72d9_b.jpg) | Genevieve | CC-BY-2.0 | | T._J._Thyne_4.jpg | [https://live.staticflickr.com/7042/6837821654_d65ab80913_b.jpg](https://live.staticflickr.com/7042/6837821654_d65ab80913_b.jpg) | Genevieve | CC-BY-2.0 |
""" CORRECT_RESULT_INTERPRETATION = """

{0} is in the Training Data!

The name of {0} has been correctly predicted for {1} out of {2} prompts. This means that {0} was in the training data and was used to train the model. Keep in mind that the probability of correctly predicting the name for {3} by chance {4} times with {5} possible names for the model to choose from, is only (1 ⁄ {5}){6} = {7}%. """ INDECISIVE_RESULT_INTERPRETATION = """

{0} might be in the Training Data!

For none of the {1} prompts the majority vote for the name of {0} was correct. However, while the majority votes are not correct, the name of {0} was correctly predicted {2} times for {3}. This is an indication that the model has seen {0} during training. A different selection of images might have a clearer result. Keep in mind that the probability that the name is correctly predicted by chance {2} times for {3} is (1 ⁄ {4}){2} = {5}%. """ INCORRECT_RESULT_INTERPRETATION = """

{0} is most likely not in the Training Data!

The name of {0} has not been correctly predicted for any of the {1} prompts. This is an indication that {0} has most likely not been used for training the model. """ OCCURENCE_INFORMATION = """

According to our analysis {0} appeared {1} times among 400 million image-text pairs in the LAION-400M training dataset. """ CSS = """ .footer { margin-bottom: 45px; margin-top: 35px; text-align: center; border-bottom: 1px solid #e5e5e5; } #file_upload { max-height: 250px; overflow-y: auto !important; } .footer>p { font-size: .8rem; display: inline-block; padding: 0 10px; transform: translateY(10px); background: white; } .dark .footer { border-color: #303030; } .dark .footer>p { background: #0b0f19; } .acknowledgments h4{ margin: 1.25em 0 .25em 0; font-weight: bold; font-size: 115%; } """ # monkey patch the update function of the Files component since otherwise it is not possible to access the original # file name def preprocess( self, x: List[Dict[str, Any]] | None ) -> bytes | tempfile._TemporaryFileWrapper | List[ bytes | tempfile._TemporaryFileWrapper ] | None: """ Parameters: x: List of JSON objects with filename as 'name' property and base64 data as 'data' property Returns: File objects in requested format """ if x is None: return None def process_single_file(f) -> bytes | tempfile._TemporaryFileWrapper: file_name, orig_name, data, is_file = ( f["name"] if "name" in f.keys() else f["orig_name"], f["orig_name"], f["data"], f.get("is_file", False), ) if self.type == "file": if is_file: temp_file_path = self.make_temp_copy_if_needed(file_name) file = tempfile.NamedTemporaryFile(delete=False) file.name = temp_file_path file.orig_name = os.path.basename(orig_name.replace(self.hash_file(file_name), "")) # type: ignore else: file = processing_utils.decode_base64_to_file( data, file_path=file_name ) file.orig_name = file_name # type: ignore self.temp_files.add(str(utils.abspath(file.name))) return file elif ( self.type == "binary" or self.type == "bytes" ): # "bytes" is included for backwards compatibility if is_file: with open(file_name, "rb") as file_data: return file_data.read() return processing_utils.decode_base64_to_binary(data)[0] else: raise ValueError( "Unknown type: " + str(self.type) + ". Please choose from: 'file', 'bytes'." ) if self.file_count == "single": if isinstance(x, list): return process_single_file(x[0]) else: return process_single_file(x) else: if isinstance(x, list): return [process_single_file(f) for f in x] else: return process_single_file(x) gr.Files.preprocess = preprocess @torch.no_grad() def calculate_text_embeddings(model_name, prompts): tokenizer = open_clip.get_tokenizer(MODELS[model_name]['model_name']) context_vecs = open_clip.tokenize(prompts) model_instance = MODELS[model_name]['model_instance'] model_instance = model_instance.to(DEVICE) context_vecs = context_vecs.to(DEVICE) text_features = model_instance.encode_text(context_vecs, normalize=True).cpu() model_instance = model_instance.cpu() context_vecs = context_vecs.cpu() return text_features @torch.no_grad() def calculate_image_embeddings(model_name, images): preprocessing = MODELS[model_name]['preprocessing'] model_instance = MODELS[model_name]['model_instance'] # load the given images user_imgs = [] for tmp_file_img in images: img = Image.open(tmp_file_img.name) # preprocess the images user_imgs.append(preprocessing(img)) # calculate the image embeddings image_embeddings = [] model_instance = model_instance.to(DEVICE) for batch_idx in range(0, len(user_imgs), IMG_BATCHSIZE): imgs = user_imgs[batch_idx:batch_idx + IMG_BATCHSIZE] imgs = torch.stack(imgs) imgs = imgs.to(DEVICE) emb = model_instance.encode_image(imgs, normalize=True).cpu() image_embeddings.append(emb) imgs = imgs.cpu() model_instance = model_instance.cpu() return torch.cat(image_embeddings) def get_possible_names(true_name): possible_names = FULL_NAMES_DF possible_names['full_names'] = FULL_NAMES_DF['first_name'].astype(str) + ' ' + FULL_NAMES_DF['last_name'].astype( str) possible_names = possible_names[possible_names['full_names'] != true_name] # sample the same amount of male and female names sampled_names = possible_names.groupby('sex').sample(int(NUM_TOTAL_NAMES / 2), random_state=42) # shuffle the rows randomly sampled_names = sampled_names.sample(frac=1) # get only the full names since we don't need first and last name and gender anymore possible_full_names = sampled_names['full_names'] return possible_full_names def round_to_first_digit(value: Decimal): tmp = np.format_float_positional(value) prob_str = [] for c in str(tmp): if c in ("0", "."): prob_str.append(c) else: prob_str.append(c) break return "".join(prob_str) def get_majority_predictions(predictions: pd.Series, values_only=False, counts_only=False, value=None): """Takes a series of predictions and returns the unique values and the number of prediction occurrences in descending order.""" values, counts = np.unique(predictions, return_counts=True) descending_counts_indices = counts.argsort()[::-1] values, counts = values[descending_counts_indices], counts[descending_counts_indices] idx_most_often_pred_names = np.argwhere(counts == counts.max()).flatten() if values_only: return values[idx_most_often_pred_names] elif counts_only: return counts[idx_most_often_pred_names] elif value is not None: if value not in values: return [0] # return how often the values appears in the predictions return counts[np.where(values == value)[0]] else: return values[idx_most_often_pred_names], counts[idx_most_often_pred_names] def on_submit_btn_click(model_name, true_name, prompts, images): # assert that the name is in the prompts assert prompts.iloc[0].str.contains(true_name).sum() == len(prompts.T) # calculate the image embeddings img_embeddings = calculate_image_embeddings(model_name, images) # calculate the text embeddings of the populated prompts user_text_emb = calculate_text_embeddings(model_name, prompts.values[0].tolist()) # get the indices of the possible names possible_names = get_possible_names(true_name) # get the text embeddings of the possible names prompt_text_embeddings = MODELS[model_name]['prompt_text_embeddings'] text_embeddings_used_for_prediction = prompt_text_embeddings.index_select(1, torch.tensor(possible_names.index.values)) # add the true name and the text embeddings to the possible names names_used_for_prediction = pd.concat([possible_names, pd.Series(true_name)], ignore_index=True) text_embeddings_used_for_prediction = torch.cat([text_embeddings_used_for_prediction, user_text_emb.unsqueeze(1)], dim=1) # calculate the similarity of the images and the given texts with torch.no_grad(): logits_per_image = MODELS[model_name][ 'model_instance' ].logit_scale.exp().cpu() * img_embeddings @ text_embeddings_used_for_prediction.swapaxes(-1, -2) preds = logits_per_image.argmax(-1) # get the predicted names for each prompt predicted_names = [] for pred in preds: predicted_names.append(names_used_for_prediction.iloc[pred]) predicted_names = np.array(predicted_names) # convert the predictions into a dataframe name_predictions = pd.DataFrame(predicted_names).T.reset_index().rename( columns={i: f'Prompt {i + 1}' for i in range(len(predicted_names))} ).rename(columns={'index': 'Image'}) # add the image names name_predictions['Image'] = [x.orig_name for x in images] # get the majority votes majority_preds = name_predictions[[f'Prompt {i + 1}' for i in range(len(PROMPTS))]].apply( lambda x: get_majority_predictions(x, values_only=True) ) # get how often the majority name was predicted majority_preds_counts = name_predictions[[f'Prompt {i + 1}' for i in range(len(PROMPTS))]].apply( lambda x: get_majority_predictions(x, counts_only=True) ).apply(lambda x: x[0]) # get how often the correct name was predicted - even if no majority true_name_preds_counts = name_predictions[[f'Prompt {i + 1}' for i in range(len(PROMPTS))]].apply( lambda x: get_majority_predictions(x, value=true_name) ).apply(lambda x: x[0]) # convert the majority preds to a series of lists if it is a dataframe majority_preds = majority_preds.T.squeeze().apply(lambda x: [x]) if len(majority_preds) == 1 else majority_preds # create the results dataframe for display result = pd.concat( [name_predictions, pd.concat([pd.Series({'Image': 'Correct Name Predictions'}), true_name_preds_counts]).to_frame().T], ignore_index=True ) result = pd.concat( [result, pd.concat([pd.Series({'Image': 'Majority Vote'}), majority_preds]).to_frame().T], ignore_index=True ) result = pd.concat( [result, pd.concat([pd.Series({'Image': 'Majority Vote Counts'}), majority_preds_counts]).to_frame().T], ignore_index=True ) result = result.set_index('Image') # check whether there is only one majority vote. If not, display Not Applicable result.loc['Majority Vote'] = result.loc['Majority Vote'].apply( lambda x: x[0] if len(x) == 1 else "N/A") # check whether the majority prediction is the correct name result.loc['Correct Majority Prediction'] = result.apply(lambda x: x['Majority Vote'] == true_name, axis=0) result = result[[f'Prompt {i + 1}' for i in range(len(PROMPTS))]].sort_values( ['Correct Name Predictions', 'Majority Vote Counts', "Correct Majority Prediction"], axis=1, ascending=False ) predictions = result.loc[[x.orig_name for x in images]] prediction_results = result.loc[['Correct Name Predictions', 'Majority Vote', 'Correct Majority Prediction']] # if there are correct predictions num_correct_maj_preds = prediction_results.loc['Correct Majority Prediction'].sum() num_correct_name_preds = result.loc['Correct Name Predictions'].max() if num_correct_maj_preds > 0: interpretation = CORRECT_RESULT_INTERPRETATION.format( true_name, num_correct_maj_preds, len(PROMPTS), prediction_results.columns[0], len(images), len(possible_names), predictions.iloc[:, 0].value_counts()[true_name], round_to_first_digit( ( (Decimal(1) / Decimal(len(possible_names))) ** predictions.iloc[:, 0].value_counts()[true_name] ) * Decimal(100) ) ) elif num_correct_name_preds > 0: interpretation = INDECISIVE_RESULT_INTERPRETATION.format( true_name, len(PROMPTS), num_correct_name_preds, prediction_results.columns[result.loc['Correct Name Predictions'].to_numpy().argmax()], len(possible_names), round_to_first_digit( ( (Decimal(1) / Decimal(len(possible_names))) ** Decimal(num_correct_name_preds) ) * Decimal(100) ) ) else: interpretation = INCORRECT_RESULT_INTERPRETATION.format( true_name, len(PROMPTS) ) if true_name.lower() in LAION_MEMBERSHIP_OCCURENCE['name'].str.lower().values: row = LAION_MEMBERSHIP_OCCURENCE[LAION_MEMBERSHIP_OCCURENCE['name'].str.lower() == true_name.lower()] interpretation = interpretation + OCCURENCE_INFORMATION.format(true_name, row['count'].values[0]) return predictions.reset_index(), prediction_results.reset_index(names=[""]), interpretation def populate_prompts(name): return [[x.format(name) for x in PROMPTS]] def load_uploaded_imgs(images): if images is None: return None imgs = [] for file_wrapper in images: img = Image.open(file_wrapper.name) imgs.append((img, file_wrapper.orig_name)) return imgs block = gr.Blocks(css=CSS) with block as demo: gr.HTML( """

Does CLIP Know My Face?

Want to know whether you were used to train a CLIP model? Below you can choose a model, enter your name and upload some pictures. If the model correctly predicts your name for multiple images, it is very likely that you were part of the training data. Pick some of the examples below and try it out!

Details and further analysis can be found in the paper Does CLIP Know My Face? .

""" ) with gr.Row(): with gr.Box(): gr.Markdown("## Inputs") with gr.Column(): model_dd = gr.Dropdown(label="CLIP Model", choices=list(MODELS.keys()), value=list(MODELS.keys())[0]) true_name = gr.Textbox(label='Name of Person:', lines=1, value=DEFAULT_INITIAL_NAME) prompts = gr.Dataframe( value=[[x.format(DEFAULT_INITIAL_NAME) for x in PROMPTS]], label='Prompts Used (hold shift to scroll sideways):', interactive=False ) true_name.change(fn=populate_prompts, inputs=[true_name], outputs=prompts, show_progress=True, status_tracker=None) uploaded_imgs = gr.Files(label='Upload Images:', file_types=['image'], elem_id='file_upload').style() image_gallery = gr.Gallery(label='Images Used:', show_label=True, elem_id="image_gallery").style(grid=[5]) uploaded_imgs.change(load_uploaded_imgs, inputs=uploaded_imgs, outputs=image_gallery) submit_btn = gr.Button(value='Submit') with gr.Box(): gr.Markdown("## Outputs") prediction_df = gr.Dataframe(label="Prediction Output (hold shift to scroll sideways):", interactive=False) result_df = gr.DataFrame(label="Result (hold shift to scroll sideways):", interactive=False) interpretation = gr.HTML() submit_btn.click(on_submit_btn_click, inputs=[model_dd, true_name, prompts, uploaded_imgs], outputs=[prediction_df, result_df, interpretation]) gr.Examples( examples=EXAMPLES, inputs=[model_dd, true_name, prompts, uploaded_imgs], outputs=[prediction_df, result_df, interpretation], fn=on_submit_btn_click, cache_examples=True ) gr.Markdown(LICENSE_DETAILS) gr.HTML( """

Gradio Demo by AIML@TU Darmstadt

Created by Dominik Hintersdorf at AIML Lab.

""" ) if __name__ == "__main__": demo.launch()