app.py

import gradio as gr
import plotly.graph_objs as go
import trimesh
import numpy as np
from PIL import Image, ImageDraw
import torch
from diffusers import StableDiffusionPipeline, StableDiffusionInpaintPipeline
import os
import matplotlib.pyplot as plt

# Load the Stable Diffusion model for text-to-image generation and inpainting
device = "cuda" if torch.cuda.is_available() else "cpu"
pipeline = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(device)
pipeline_inpaint = StableDiffusionInpaintPipeline.from_pretrained(
    "runwayml/stable-diffusion-inpainting", 
    torch_dtype=torch.float16
).to(device)

# Get the current directory
current_dir = os.getcwd()
# Default object file path
DEFAULT_OBJ_FILE = os.path.join(current_dir, "female.obj")
# Temporary texture file path
TEMP_TEXTURE_FILE = os.path.join(current_dir, "generated_texture.png")
# File path to save the 2D image
OUTPUT_IMAGE_FILE = os.path.join(current_dir, "output_image.png")
DEFAULT_GLB_FILE = os.path.join(current_dir, "vroid_girl1.glb")


def apply_texture(mesh, texture_file, uv_scale):
    """
    Applies the texture to the mesh with UV scaling to make triangles/rectangles smaller or larger.
    """
    texture_image = Image.open(texture_file)
    uv_coords = mesh.visual.uv

    # Apply scaling to UV coordinates to make the mapping finer or coarser
    uv_coords = np.clip(uv_coords * uv_scale, 0, 1)

    # Get the size of the texture image
    img_width, img_height = texture_image.size
    texture_array = np.array(texture_image)

    # Prepare to store the colors per face
    face_colors = []

    for face in mesh.faces:
        uv_face = uv_coords[face]
        pixel_coords = np.round(uv_face * np.array([img_width - 1, img_height - 1])).astype(int)

        # Ensure the UV coordinates are within the bounds of the texture image
        valid_coords = np.all((pixel_coords[:, 0] >= 0) & (pixel_coords[:, 0] < img_width) &
                              (pixel_coords[:, 1] >= 0) & (pixel_coords[:, 1] < img_height))

        if valid_coords:
            # Get the average color for the face from the corresponding UV points in the texture
            face_color = np.mean(texture_array[pixel_coords[:, 1], pixel_coords[:, 0]], axis=0)
            face_colors.append(face_color / 255.0)  # Normalize to [0, 1]
        else:
            # Assign a default color (e.g., gray) if UV coordinates are not valid
            face_colors.append([0.5, 0.5, 0.5])  # Default to gray if no texture is applied

    # Ensure no face is left unpainted
    face_colors = np.array(face_colors)
    if len(face_colors) < len(mesh.faces):
        face_colors = np.pad(face_colors, ((0, len(mesh.faces) - len(face_colors)), (0, 0)), 'constant', constant_values=0.5)

    return face_colors


def load_glb_file(filename):
    trimesh_scene = trimesh.load(filename)
    if isinstance(trimesh_scene, trimesh.Scene):
        mesh = trimesh_scene.dump(concatenate=True)
    else:
        mesh = trimesh_scene
    return mesh

def generate_clothing_image(prompt, num_inference_steps):
    """
    Generates the clothing texture based on the provided prompt and number of inference steps.
    """
    image = pipeline(prompt, num_inference_steps=num_inference_steps).images[0]
    image.save(TEMP_TEXTURE_FILE)
    return TEMP_TEXTURE_FILE, image

def generate_uv_specific_texture(prompt, uv_map_file, num_inference_steps=50):
    """
    Generates a texture for the 3D model using a given prompt and UV map.
    
    Args:
        prompt (str): The prompt for the diffusion model to generate the texture.
        uv_map_file (str): Path to the UV map file.
        num_inference_steps (int): The number of iterations/steps for the diffusion process.
        
    Returns:
        (str, PIL.Image): The path to the generated texture file and the generated texture image.
    """
    # Load UV map as a mask
    uv_map = Image.open(uv_map_file)
    
    # Generate texture based on UV map and the provided prompt
    image = pipeline_inpaint(
        prompt=prompt, 
        image=uv_map, 
        mask_image=uv_map, 
        num_inference_steps=num_inference_steps  # Set custom number of inference steps
    ).images[0]
    
    # Save the generated texture
    image.save(TEMP_TEXTURE_FILE)
    
    return TEMP_TEXTURE_FILE, image

def display_3d_object(obj_file, texture_file, light_intensity, ambient_intensity, color, uv_scale, transparency):
    """
    Displays the 3D object with applied texture or color, with support for UV scaling and transparency.
    """
    file_extension = obj_file.split('.')[-1].lower()

    # Load mesh
    if file_extension == 'obj':
        mesh = trimesh.load(obj_file)
    elif file_extension == 'glb':
        mesh = load_glb_file(obj_file)
    else:
        raise ValueError("Unsupported file format. Please upload a .obj or .glb file.")

    # Apply texture or color
    if texture_file:
        face_colors = apply_texture(mesh, texture_file, uv_scale)
    else:
        face_colors = np.array([color] * len(mesh.faces))  # Use a single color for all faces if no texture

    # Define lighting settings
    fig = go.Figure(data=[
        go.Mesh3d(
            x=mesh.vertices[:, 0],
            y=mesh.vertices[:, 1],
            z=mesh.vertices[:, 2],
            i=mesh.faces[:, 0],
            j=mesh.faces[:, 1],
            k=mesh.faces[:, 2],
            facecolor=face_colors if texture_file else None,
            color=color if not texture_file else None,
            opacity=transparency,  # Adjustable transparency
            lighting=dict(
                ambient=ambient_intensity,
                diffuse=light_intensity,
                specular=0.8,  # Fine-tuned specular to avoid excessive shininess
                roughness=0.3,
                fresnel=0.1
            ),
            lightposition=dict(
                x=100,
                y=200,
                z=300
            )
        )
    ])
    fig.update_layout(scene=dict(aspectmode='data'))

    return fig

def clear_texture():
    """
    Clears the texture preview and removes the texture file, allowing for a reset.
    """
    if os.path.exists(TEMP_TEXTURE_FILE):
        os.remove(TEMP_TEXTURE_FILE)
    return None

def restore_original(obj_file):
    """
    Restores the original 3D object without any applied texture.
    """
    return display_3d_object(obj_file, None, 0.8, 0.5, "#D3D3D3", 1.0, 1.0)  # Default settings for restoration

def update_texture_display(prompt, texture_file, num_inference_steps):
    """
    Update the texture display either by generating a texture from the prompt
    or by displaying an uploaded texture.
    """
    if prompt:
        # Generate new texture based on the prompt using the Stable Diffusion pipeline
        texture_path, image = generate_clothing_image(prompt, num_inference_steps)
        return image
    elif texture_file:
        # Display the uploaded texture file
        return Image.open(texture_file)
    return None

with gr.Blocks() as demo:
    gr.Markdown("## 3D Object Viewer with Custom Texture, UV Scale, Transparency, Color, and Adjustable Lighting")

    with gr.Row():
        with gr.Column(scale=1):
            gr.Markdown("### Texture Options")
            prompt_input = gr.Textbox(label="Enter a Prompt to Generate Texture", placeholder="Type a prompt...")
            num_inference_steps_slider = gr.Slider(minimum=5, maximum=100, step=1, value=10, label="Num Inference Steps")
            generate_button = gr.Button("Generate Texture")
            texture_file = gr.File(label="Upload Texture file (PNG or JPG, optional)", type="filepath")
            texture_preview = gr.Image(label="Texture Preview", visible=True)

            gr.Markdown("### Mapping, Lighting & Color Settings")
            uv_scale_slider = gr.Slider(minimum=0.1, maximum=5, step=0.1, value=1.0, label="UV Mapping Scale (Make smaller/bigger)")
            light_intensity_slider = gr.Slider(minimum=0, maximum=1, step=0.1, value=0.8, label="Light Intensity")
            ambient_intensity_slider = gr.Slider(minimum=0, maximum=1, step=0.1, value=0.5, label="Ambient Intensity")
            transparency_slider = gr.Slider(minimum=0.1, maximum=1.0, step=0.1, value=1.0, label="Transparency (1.0 is fully opaque)")
            color_picker = gr.ColorPicker(value="#D3D3D3", label="Object Color")
            submit_button = gr.Button("Submit")
            restore_button = gr.Button("Restore")
            clear_button = gr.Button("Clear")
            obj_file = gr.File(label="Upload OBJ or GLB file", value=DEFAULT_OBJ_FILE, type='filepath')

        with gr.Column(scale=2):
            display = gr.Plot(label="3D Viewer")

    def update_display(file, texture, uv_scale, light_intensity, ambient_intensity, transparency, color, num_inference_steps):
        texture_to_use = TEMP_TEXTURE_FILE if os.path.exists(TEMP_TEXTURE_FILE) else texture
        return display_3d_object(file, texture_to_use, light_intensity, ambient_intensity, color, uv_scale, transparency)

    submit_button.click(fn=update_display, inputs=[obj_file, texture_file, uv_scale_slider, light_intensity_slider, ambient_intensity_slider, transparency_slider, color_picker, num_inference_steps_slider], outputs=display)
    generate_button.click(fn=update_texture_display, inputs=[prompt_input, texture_file, num_inference_steps_slider], outputs=texture_preview)
    restore_button.click(fn=restore_original, inputs=[obj_file], outputs=display)
    clear_button.click(fn=clear_texture, outputs=texture_preview)
    texture_file.change(fn=update_texture_display, inputs=[prompt_input, texture_file, num_inference_steps_slider], outputs=texture_preview)

    demo.load(fn=update_display, inputs=[obj_file, texture_file, uv_scale_slider, light_intensity_slider, ambient_intensity_slider, transparency_slider, color_picker, num_inference_steps_slider], outputs=display)

    gr.Examples(
        examples=[[DEFAULT_OBJ_FILE, None],[DEFAULT_GLB_FILE, None]],
        inputs=[obj_file, texture_file],
        label="Example Files"
    )

demo.launch(debug=True)