import gradio as gr from transformers import AutoProcessor, AutoModelForCausalLM from PIL import Image import torch from peft import PeftModel import numpy as np import os from unittest.mock import patch from transformers.dynamic_module_utils import get_imports def fixed_get_imports(filename: str | os.PathLike) -> list[str]: if not str(filename).endswith("modeling_florence2.py"): return get_imports(filename) imports = get_imports(filename) imports.remove("flash_attn") return imports device = torch.device("cuda" if torch.cuda.is_available() else "cpu") torch_dtype = torch.float32 # Load the fine-tuned base model with patch("transformers.dynamic_module_utils.get_imports", fixed_get_imports): caption_model = AutoModelForCausalLM.from_pretrained('microsoft/Florence-2-base-ft', trust_remote_code=True, revision='refs/pr/6', torch_dtype=torch_dtype).to(device) with patch("transformers.dynamic_module_utils.get_imports", fixed_get_imports): model = AutoModelForCausalLM.from_pretrained('byh711/FLODA-deepfake', trust_remote_code=True, torch_dtype=torch_dtype).to(device) processor = AutoProcessor.from_pretrained('byh711/FLODA-deepfake', trust_remote_code=True) model.eval() def caption_generate(task_prompt, text_input=None, image=None): if isinstance(image, np.ndarray): image = Image.fromarray(image) if text_input is None: prompt = task_prompt else: prompt = task_prompt + text_input inputs = processor(text=prompt, images=image, return_tensors="pt").to(device, torch_dtype) generated_ids = caption_model.generate( input_ids=inputs["input_ids"], pixel_values=inputs["pixel_values"], max_new_tokens=1024, num_beams=3 ) generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0] parsed_answer = processor.post_process_generation(generated_text, task=task_prompt, image_size=(image.width, image.height)) return parsed_answer[task_prompt][1:-1] def run_example(task_prompt, text_input=None, image=None): if text_input is None: prompt = task_prompt else: prompt = task_prompt + text_input if isinstance(image, np.ndarray): image = Image.fromarray(image) image = image.convert("RGB") inputs = processor(text=prompt, images=image, return_tensors="pt").to(device) inputs = {k: v.to(torch_dtype) if v.is_floating_point() else v for k, v in inputs.items()} generated_ids = model.generate( input_ids=inputs["input_ids"], pixel_values=inputs["pixel_values"], max_new_tokens=1024, num_beams=3 ) generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0] result = processor.post_process_generation(generated_text, task=task_prompt, image_size=(image.width, image.height))[''] if result.lower() == "yes": return "This is a real image." elif result.lower() == "no": return "This is a fake image." else: return f"Uncertain. Model output: {result}" # Define the Gradio interface css = """ body { background-color: #1e1e2e; color: #d4d4dc; font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; } #output { height: 500px; overflow: auto; border: 1px solid #444; background-color: #282c34; color: #f1f1f1; padding: 10px; } .gr-button { background-color: #3a3f51; border: none; color: #ffffff; padding: 10px 20px; text-align: center; font-size: 14px; cursor: pointer; transition: 0.3s; } .gr-button:hover { background-color: #4b5263; } .gr-textbox { background-color: #2e2e38; border: 1px solid #555; color: #ffffff; } .gr-markdown { color: #d4d4dc; } """ js_func = """ function refresh() { const url = new URL(window.location); if (url.searchParams.get('__theme') !== 'dark') { url.searchParams.set('__theme', 'dark'); window.location.href = url.href; } } """ TITLE = "# FLODA: Vision-Language Models for Deepfake Detection" DESCRIPTION = """ FLODA (FLorence-2 Optimized for Deepfake Assessment) is an advanced deepfake detection model leveraging the power of [Florence-2](https://huggingface.co/microsoft/Florence-2-base-ft). FLODA combines image captioning with authenticity assessment in a single end-to-end architecture, demonstrating superior performance compared to existing benchmarks. Learn more about FLODA in the published paper [here](https://github.com/byh711/FLODA). """ with gr.Blocks(js=js_func, css=css) as demo: gr.Markdown(TITLE) gr.Markdown(DESCRIPTION) with gr.Tab(label="FLODA: Deepfake Detection"): with gr.Row(): with gr.Column(): input_img = gr.Image(label="Input Picture", type="numpy") submit_btn = gr.Button(value="Submit") with gr.Column(): output_text = gr.Textbox(label="Output Text") submit_btn.click( fn=lambda image: run_example("", text_input=None, image=image), inputs=[input_img], outputs=[output_text] ) demo.launch(debug=True)