import gradio as gr import os import zipfile import json from io import BytesIO import base64 from PIL import Image import uuid import tempfile import numpy as np import time # Function to save dataset to zip def save_dataset_to_zip(dataset_name, dataset): temp_dir = tempfile.mkdtemp() dataset_path = os.path.join(temp_dir, dataset_name) os.makedirs(dataset_path, exist_ok=True) images_dir = os.path.join(dataset_path, 'images') os.makedirs(images_dir, exist_ok=True) annotations = [] for idx, entry in enumerate(dataset): image_data = entry['image'] prompt = entry['prompt'] # Save image to images directory image_filename = f"{uuid.uuid4().hex}.png" image_path = os.path.join(images_dir, image_filename) # Decode the base64 image data image = Image.open(BytesIO(base64.b64decode(image_data.split(",")[1]))) image.save(image_path) # Add annotation annotations.append({ 'file_name': os.path.join('images', image_filename), 'text': prompt }) # Save annotations to JSONL file annotations_path = os.path.join(dataset_path, 'annotations.jsonl') with open(annotations_path, 'w') as f: for ann in annotations: f.write(json.dumps(ann) + '\n') # Create a zip file with the dataset_name as the top-level folder zip_buffer = BytesIO() with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zipf: for root, dirs, files in os.walk(dataset_path): for file in files: abs_file = os.path.join(root, file) rel_file = os.path.relpath(abs_file, temp_dir) zipf.write(abs_file, rel_file) zip_buffer.seek(0) return zip_buffer # Function to load dataset from zip def load_dataset_from_zip(zip_file_path): temp_dir = tempfile.mkdtemp() try: with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: zip_ref.extractall(temp_dir) # Get dataset name from zip file name dataset_name_guess = os.path.splitext(os.path.basename(zip_file_path))[0] dataset_path = os.path.join(temp_dir, dataset_name_guess) if os.path.exists(dataset_path): dataset_name = dataset_name_guess else: # If the dataset_name directory doesn't exist, try to find the top-level directory entries = [entry for entry in os.listdir(temp_dir) if os.path.isdir(os.path.join(temp_dir, entry))] if entries: dataset_name = entries[0] dataset_path = os.path.join(temp_dir, dataset_name) else: # Files are directly in temp_dir dataset_name = dataset_name_guess dataset_path = temp_dir annotations_path = os.path.join(dataset_path, 'annotations.jsonl') dataset = [] if os.path.exists(annotations_path): with open(annotations_path, 'r') as f: for line in f: ann = json.loads(line) file_name = ann['file_name'] prompt = ann['text'] image_path = os.path.join(dataset_path, file_name) # Read image and convert to base64 with open(image_path, 'rb') as img_f: image_bytes = img_f.read() encoded = base64.b64encode(image_bytes).decode() mime_type = "image/png" image_data = f"data:{mime_type};base64,{encoded}" dataset.append({ 'image': image_data, 'prompt': prompt }) else: # If annotations file not found return None, [] return dataset_name, dataset except Exception as e: print(f"Error loading dataset: {e}") return None, [] # Function to display dataset as HTML def display_dataset_html(dataset, page_number=0, items_per_page=2): if dataset: start_idx = page_number * items_per_page end_idx = start_idx + items_per_page dataset_slice = dataset[start_idx:end_idx] html_content = '''
''' for idx_offset, entry in enumerate(dataset_slice): idx = start_idx + idx_offset image_data = entry['image'] prompt = entry['prompt'] # Decode base64 image data to numpy array image_bytes = base64.b64decode(image_data.split(",")[1]) image = Image.open(BytesIO(image_bytes)) # Compress image image.thumbnail((100, 100)) # Resize image to 100x100 pixels buffered = BytesIO() image.save(buffered, format="PNG") img_str = base64.b64encode(buffered.getvalue()).decode('utf-8') img_data = f"data:image/png;base64,{img_str}" html_content += f"""
'''
{idx}
Image {idx}
{prompt}
""" html_content += '
' return html_content else: return "
No entries in dataset.
" # Interface with gr.Blocks() as demo: gr.Markdown("

Dataset Creator

") gr.Markdown("You must create/upload a dataset before selecting one") datasets = gr.State({}) current_dataset_name = gr.State("") current_page_number = gr.State(0) dataset_html = gr.HTML() # Define dataset_html here # Top-level components with gr.Column(): dataset_selector = gr.Dropdown(label="Select Dataset", interactive=True) message_box = gr.Textbox(interactive=False, label="Message") # Tabs with gr.Tabs(): with gr.TabItem("Create / Upload Dataset"): with gr.Row(): with gr.Column(): gr.Markdown("### Create a New Dataset") dataset_name_input = gr.Textbox(label="New Dataset Name") create_button = gr.Button("Create Dataset") with gr.Column(): gr.Markdown("### Upload Existing Dataset") upload_input = gr.File(label="Upload Dataset Zip", type="filepath", file_types=['.zip']) upload_button = gr.Button("Upload Dataset") def create_dataset(name, datasets): if not name: return gr.update(), "Please enter a dataset name." if name in datasets: return gr.update(), f"Dataset '{name}' already exists." datasets[name] = [] return gr.update(choices=list(datasets.keys()), value=name), f"Dataset '{name}' created." create_button.click( create_dataset, inputs=[dataset_name_input, datasets], outputs=[dataset_selector, message_box] ) def upload_dataset(zip_file_path, datasets): if not zip_file_path: return gr.update(), "Please upload a zip file." dataset_name, dataset = load_dataset_from_zip(zip_file_path) if dataset_name is None: return gr.update(), "Failed to load dataset from zip file." if dataset_name in datasets: return gr.update(), f"Dataset '{dataset_name}' already exists." datasets[dataset_name] = dataset return gr.update(choices=list(datasets.keys()), value=dataset_name), f"Dataset '{dataset_name}' uploaded." upload_button.click( upload_dataset, inputs=[upload_input, datasets], outputs=[dataset_selector, message_box] ) with gr.TabItem("Add Entry"): with gr.Row(): image_input = gr.Image(label="Upload Image", type="numpy") prompt_input = gr.Textbox(label="Prompt") add_button = gr.Button("Add Entry") def add_entry(image_data, prompt, current_dataset_name, datasets): if not current_dataset_name: return datasets, gr.update(), gr.update(), "No dataset selected." if image_data is None or not prompt: return datasets, gr.update(), gr.update(), "Please provide both an image and a prompt." # Convert image_data to base64 image = Image.fromarray(image_data.astype('uint8')) buffered = BytesIO() image.save(buffered, format="PNG") img_str = base64.b64encode(buffered.getvalue()).decode('utf-8') img_data = f"data:image/png;base64,{img_str}" datasets[current_dataset_name].append({'image': img_data, 'prompt': prompt}) dataset = datasets[current_dataset_name] # Reset page number to 0 and refresh HTML page_number = 0 dataset = datasets[current_dataset_name] html_content = display_dataset_html(dataset, page_number=page_number) return datasets, page_number, gr.update(value=html_content), f"Entry added to dataset '{current_dataset_name}'." add_button.click( add_entry, inputs=[image_input, prompt_input, current_dataset_name, datasets], outputs=[datasets, current_page_number, dataset_html, message_box] ) with gr.TabItem("Edit / Delete Entry"): with gr.Column(): selected_image = gr.Image(label="Selected Image", interactive=False, type="numpy") selected_prompt = gr.Textbox(label="Current Prompt", interactive=False) # Define entry_selector here entry_selector = gr.Dropdown(label="Select Entry to Edit/Delete") new_prompt_input = gr.Textbox(label="New Prompt (for Edit)") with gr.Row(): edit_button = gr.Button("Edit Entry") delete_button = gr.Button("Delete Entry") def update_selected_entry(entry_option, current_dataset_name, datasets): if not current_dataset_name or not entry_option: return gr.update(), gr.update() index = int(entry_option.split(":")[0]) entry = datasets[current_dataset_name][index] image_data = entry['image'] prompt = entry['prompt'] # Decode base64 image data to numpy array image_bytes = base64.b64decode(image_data.split(",")[1]) image = Image.open(BytesIO(image_bytes)) image_array = np.array(image) return gr.update(value=image_array), gr.update(value=prompt) entry_selector.change( update_selected_entry, inputs=[entry_selector, current_dataset_name, datasets], outputs=[selected_image, selected_prompt] ) def edit_entry(entry_option, new_prompt, current_dataset_name, datasets, current_page_number): if not current_dataset_name: return datasets, gr.update(), gr.update(), gr.update(), f"No dataset selected." if not entry_option or not new_prompt.strip(): return datasets, gr.update(), gr.update(), gr.update(), f"Please select an entry and provide a new prompt." index = int(entry_option.split(":")[0]) datasets[current_dataset_name][index]['prompt'] = new_prompt dataset = datasets[current_dataset_name] html_content = display_dataset_html(dataset, page_number=current_page_number) # Update entry_selector options entry_options = [f"{idx}: {entry['prompt'][:30]}" for idx, entry in enumerate(dataset)] return datasets, gr.update(value=html_content), gr.update(choices=entry_options), gr.update(value=""), f"Entry {index} updated." edit_button.click( edit_entry, inputs=[entry_selector, new_prompt_input, current_dataset_name, datasets, current_page_number], outputs=[datasets, dataset_html, entry_selector, new_prompt_input, message_box] ) def delete_entry(entry_option, current_dataset_name, datasets, current_page_number): if not current_dataset_name: return datasets, gr.update(), gr.update(), gr.update(), gr.update(), "No dataset selected." if not entry_option: return datasets, gr.update(), gr.update(), gr.update(), gr.update(), "Please select an entry to delete." index = int(entry_option.split(":")[0]) del datasets[current_dataset_name][index] dataset = datasets[current_dataset_name] html_content = display_dataset_html(dataset, page_number=current_page_number) # Update entry_selector options entry_options = [f"{idx}: {entry['prompt'][:30]}" for idx, entry in enumerate(dataset)] return datasets, gr.update(value=html_content), gr.update(choices=entry_options), gr.update(value=None), f"Entry {index} deleted." delete_button.click( delete_entry, inputs=[entry_selector, current_dataset_name, datasets, current_page_number], outputs=[datasets, dataset_html, entry_selector, selected_image, message_box] ) # Function to update entry_selector options def update_entry_selector(current_dataset_name, datasets): if current_dataset_name in datasets: dataset = datasets[current_dataset_name] entry_options = [f"{idx}: {entry['prompt'][:30]}" for idx, entry in enumerate(dataset)] return gr.update(choices=entry_options) else: return gr.update(choices=[]) # Update entry_selector when dataset is selected dataset_selector.change( update_entry_selector, inputs=[current_dataset_name, datasets], outputs=[entry_selector] ) # Also update entry_selector when an entry is added in "Add Entry" tab add_button.click( update_entry_selector, inputs=[current_dataset_name, datasets], outputs=[entry_selector] ) with gr.TabItem("Download Dataset"): download_button = gr.Button("Download Dataset") download_output = gr.File(label="Download Zip", interactive=False) def download_dataset(current_dataset_name, datasets): if not current_dataset_name: return None, "No dataset selected." if not datasets[current_dataset_name]: return None, "Dataset is empty." zip_buffer = save_dataset_to_zip(current_dataset_name, datasets[current_dataset_name]) # Write zip_buffer to a temporary file temp_dir = tempfile.mkdtemp() zip_path = os.path.join(temp_dir, f"{current_dataset_name}.zip") with open(zip_path, 'wb') as f: f.write(zip_buffer.getvalue()) return zip_path, f"Dataset '{current_dataset_name}' is ready for download." download_button.click( download_dataset, inputs=[current_dataset_name, datasets], outputs=[download_output, message_box] ) def select_dataset(dataset_name, datasets): if dataset_name in datasets: dataset = datasets[dataset_name] html_content = display_dataset_html(dataset, page_number=0) return dataset_name, 0, gr.update(value=html_content), f"Dataset '{dataset_name}' selected." else: return "", 0, gr.update(value="
Select a dataset.
"), "" dataset_selector.change( select_dataset, inputs=[dataset_selector, datasets], outputs=[current_dataset_name, current_page_number, dataset_html, message_box] ) # Dataset Viewer and Pagination Controls at the Bottom with gr.Column(): gr.Markdown("### Dataset Viewer") dataset_viewer = gr.HTML() # Use dataset_viewer instead of dataset_html with gr.Row(): prev_button = gr.Button("Previous Page") next_button = gr.Button("Next Page") def change_page(action, current_page_number, datasets, current_dataset_name): if not current_dataset_name: return current_page_number, gr.update(), "No dataset selected." dataset = datasets[current_dataset_name] total_pages = (len(dataset) - 1) // 5 + 1 if action == "next": if current_page_number + 1 < total_pages: current_page_number += 1 elif action == "prev": if current_page_number > 0: current_page_number -= 1 html_content = display_dataset_html(dataset, page_number=current_page_number) return current_page_number, gr.update(value=html_content), "" prev_button.click( fn=lambda current_page_number, datasets, current_dataset_name: change_page("prev", current_page_number, datasets, current_dataset_name), inputs=[current_page_number, datasets, current_dataset_name], outputs=[current_page_number, dataset_viewer, message_box] ) next_button.click( fn=lambda current_page_number, datasets, current_dataset_name: change_page("next", current_page_number, datasets, current_dataset_name), inputs=[current_page_number, datasets, current_dataset_name], outputs=[current_page_number, dataset_viewer, message_box] ) # Initialize dataset_selector def initialize_components(datasets): return gr.update(choices=list(datasets.keys())) demo.load( initialize_components, inputs=[datasets], outputs=[dataset_selector] ) # Hide dataset_html dataset_html.visible = False # Update all components when a dataset is selected def update_all_components(current_dataset_name, datasets): while current_dataset_name not in datasets: time.sleep(0.1) # Wait until dataset is loaded dataset = datasets[current_dataset_name] html_content = display_dataset_html(dataset, page_number=0) entry_options = [f"{idx}: {entry['prompt'][:30]}" for idx, entry in enumerate(dataset)] return gr.update(value=html_content), gr.update(choices=entry_options) dataset_selector.change( update_all_components, inputs=[current_dataset_name, datasets], outputs=[dataset_viewer, entry_selector] ) # Update all components when an entry is added def update_all_components_after_add(current_dataset_name, datasets): while current_dataset_name not in datasets: time.sleep(0.1) # Wait until dataset is loaded dataset = datasets[current_dataset_name] html_content = display_dataset_html(dataset, page_number=0) entry_options = [f"{idx}: {entry['prompt'][:30]}" for idx, entry in enumerate(dataset)] return gr.update(value=html_content), gr.update(choices=entry_options) add_button.click( update_all_components_after_add, inputs=[current_dataset_name, datasets], outputs=[dataset_viewer, entry_selector] ) # Update all components when an entry is edited def update_all_components_after_edit(current_dataset_name, datasets): while current_dataset_name not in datasets: time.sleep(0.1) # Wait until dataset is loaded dataset = datasets[current_dataset_name] html_content = display_dataset_html(dataset, page_number=0) entry_options = [f"{idx}: {entry['prompt'][:30]}" for idx, entry in enumerate(dataset)] return gr.update(value=html_content), gr.update(choices=entry_options) edit_button.click( update_all_components_after_edit, inputs=[current_dataset_name, datasets], outputs=[dataset_viewer, entry_selector] ) # Update all components when an entry is deleted def update_all_components_after_delete(current_dataset_name, datasets): while current_dataset_name not in datasets: time.sleep(0.1) # Wait until dataset is loaded dataset = datasets[current_dataset_name] html_content = display_dataset_html(dataset, page_number=0) entry_options = [f"{idx}: {entry['prompt'][:30]}" for idx, entry in enumerate(dataset)] return gr.update(value=html_content), gr.update(choices=entry_options) delete_button.click( update_all_components_after_delete, inputs=[current_dataset_name, datasets], outputs=[dataset_viewer, entry_selector] ) demo.launch()