Create app.py

app.py

@@ -0,0 +1,227 @@
+from PIL import Image
+import torch
+
+from point_e.diffusion.configs import DIFFUSION_CONFIGS, diffusion_from_config
+from point_e.diffusion.sampler import PointCloudSampler
+from point_e.models.download import load_checkpoint
+from point_e.models.configs import MODEL_CONFIGS, model_from_config
+from point_e.util.plotting import plot_point_cloud
+from point_e.util.pc_to_mesh import marching_cubes_mesh
+
+import skimage.measure
+
+from pyntcloud import PyntCloud
+import matplotlib.colors
+import plotly.graph_objs as go
+
+import trimesh
+
+import gradio as gr
+
+
+state = ""
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+def set_state(s):
+    print(s)
+    global state
+    state = s
+
+def get_state():
+    return state
+
+set_state('Creating txt2mesh model...')
+t2m_name = 'base40M-textvec' # 'base40M' 
+t2m_model = model_from_config(MODEL_CONFIGS[t2m_name], device)
+t2m_model.eval()
+base_diffusion_t2m = diffusion_from_config(DIFFUSION_CONFIGS[t2m_name])
+
+set_state('Downloading txt2mesh checkpoint...')
+t2m_model.load_state_dict(load_checkpoint(t2m_name, device))
+
+
+def load_img2mesh_model(model_name):
+    set_state(f'Creating img2mesh model {model_name}...')
+    i2m_name = model_name
+    i2m_model = model_from_config(MODEL_CONFIGS[i2m_name], device)
+    i2m_model.eval()
+    base_diffusion_i2m = diffusion_from_config(DIFFUSION_CONFIGS[i2m_name])
+
+    set_state(f'Downloading img2mesh checkpoint {model_name}...')
+    i2m_model.load_state_dict(load_checkpoint(i2m_name, device))
+
+    return i2m_model, base_diffusion_i2m
+
+img2mesh_model_name = 'base40M' #'base300M' #'base1B'
+img2mesh_model, base_diffusion_i2m = load_img2mesh_model(img2mesh_model_name)
+
+
+set_state('Creating upsample model...')
+upsampler_model = model_from_config(MODEL_CONFIGS['upsample'], device)
+upsampler_model.eval()
+upsampler_diffusion = diffusion_from_config(DIFFUSION_CONFIGS['upsample'])
+
+set_state('Downloading upsampler checkpoint...')
+upsampler_model.load_state_dict(load_checkpoint('upsample', device))
+
+set_state('Creating SDF model...')
+sdf_name = 'sdf'
+sdf_model = model_from_config(MODEL_CONFIGS[sdf_name], device)
+sdf_model.eval()
+
+set_state('Loading SDF model...')
+sdf_model.load_state_dict(load_checkpoint(sdf_name, device))
+
+set_state('')
+
+def get_sampler(model_name, txt2obj, guidance_scale):
+
+    global img2mesh_model_name
+    global base_diffusion_i2m
+    global img2mesh_model
+    if model_name != img2mesh_model_name:
+        img2mesh_model_name = model_name
+        img2mesh_model, base_diffusion_i2m = load_img2mesh_model(model_name)
+
+    return PointCloudSampler(
+            device=device,
+            models=[t2m_model, upsampler_model],
+            diffusions=[base_diffusion_t2m if txt2obj else base_diffusion_i2m, upsampler_diffusion],
+            num_points=[1024, 4096 - 1024],
+            aux_channels=['R', 'G', 'B'],
+            guidance_scale=[guidance_scale, 0.0 if txt2obj else guidance_scale],
+            model_kwargs_key_filter=('texts', '') if txt2obj else ("*",)
+        )
+
+def generate(model_name, input, guidance_scale, grid_size):
+
+    set_state('Entered generate function...')
+
+    if isinstance(input, Image.Image):
+        input = prepare_img(input)
+
+    # if input is a string, it's a text prompt
+    sampler = get_sampler(model_name, txt2obj=True if isinstance(input, str) else False, guidance_scale=guidance_scale)
+
+    # Produce a sample from the model.
+    set_state('Sampling...')
+    samples = None
+    kw_args = dict(texts=[input]) if isinstance(input, str) else dict(images=[input])
+    for x in sampler.sample_batch_progressive(batch_size=1, model_kwargs=kw_args):
+        samples = x
+
+    set_state('Converting to point cloud...')
+    pc = sampler.output_to_point_clouds(samples)[0]
+
+    set_state('Converting to mesh...')
+    save_ply(pc, 'output.ply', grid_size)
+
+    set_state('')
+
+    return pc_to_plot(pc), ply_to_obj('output.ply', 'output.obj'), gr.update(value='output.obj', visible=True)
+
+def prepare_img(img):
+
+    w, h = img.size
+    if w > h:
+        img = img.crop(((w-h)/2, 0, (w+h)/2, h))
+    else:
+        img = img.crop((0, (h-w)/2, w, (h+w)/2))
+
+    # resize to 256x256
+    img = img.resize((256, 256))
+    
+    return img
+
+def pc_to_plot(pc):
+
+    return go.Figure(
+        data=[
+            go.Scatter3d(
+                x=pc.coords[:,0], y=pc.coords[:,1], z=pc.coords[:,2], 
+                mode='markers',
+                marker=dict(
+                  size=2,
+                  color=['rgb({},{},{})'.format(r,g,b) for r,g,b in zip(pc.channels["R"], pc.channels["G"], pc.channels["B"])],
+              )
+            )
+        ],
+        layout=dict(
+            scene=dict(xaxis=dict(visible=False), yaxis=dict(visible=False), zaxis=dict(visible=False))
+        ),
+    )
+
+def ply_to_obj(ply_file, obj_file):
+    mesh = trimesh.load(ply_file)
+    mesh.export(obj_file)
+
+    return obj_file
+
+def save_ply(pc, file_name, grid_size):
+
+    # Produce a mesh (with vertex colors)
+    mesh = marching_cubes_mesh(
+        pc=pc,
+        model=sdf_model,
+        batch_size=4096,
+        grid_size=grid_size, # increase to 128 for resolution used in evals
+        progress=True,
+    )
+
+    # Write the mesh to a PLY file to import into some other program.
+    with open(file_name, 'wb') as f:
+        mesh.write_ply(f)
+
+
+with gr.Blocks() as app:
+    gr.Markdown("## Point-E text-to-3D Demo")
+    gr.Markdown("This is a demo for [Point-E: A System for Generating 3D Point Clouds from Complex Prompts](https://arxiv.org/abs/2212.08751) by OpenAI. Check out the [GitHub repo](https://github.com/openai/point-e) for more information.")
+
+    with gr.Row():
+        with gr.Column():
+            with gr.Tab("Text to 3D"):
+                prompt = gr.Textbox(label="Prompt", placeholder="A cactus in a pot")
+                btn_generate_txt2obj = gr.Button(value="Generate")
+            with gr.Tab("Image to 3D"):
+                img = gr.Image(label="Image")
+                btn_generate_img2obj = gr.Button(value="Generate")
+            with gr.Accordion("Advanced settings", open=False):
+                dropdown_models = gr.Dropdown(label="Model", value="base40M", choices=["base40M", "base300M", "base1B"])
+                guidance_scale = gr.Slider(label="Guidance scale", value=3.0, minimum=3.0, maximum=10.0, step=1.0)
+                grid_size = gr.Slider(label="Grid size", value=32, minimum=16, maximum=128, step=16)
+
+            state_info = state_info = gr.Textbox(label="State", show_label=False).style(container=False)
+
+        with gr.Column():
+            plot = gr.Plot(label="Point cloud")
+            # btn_pc_to_obj = gr.Button(value="Convert to OBJ", visible=False)
+            model_3d = gr.Model3D(value=None)
+            file_out = gr.File(label="Obj file", visible=False)
+            
+
+
+        # inputs = [dropdown_models, prompt, img, guidance_scale, grid_size]
+        outputs = [plot, model_3d, file_out]
+
+        prompt.submit(generate, inputs=[dropdown_models, prompt, guidance_scale, grid_size], outputs=outputs)
+        btn_generate_txt2obj.click(generate, inputs=[dropdown_models, prompt, guidance_scale, grid_size], outputs=outputs)
+        btn_generate_img2obj.click(generate, inputs=[dropdown_models, img, guidance_scale, grid_size], outputs=outputs)
+        # btn_pc_to_obj.click(ply_to_obj, inputs=plot, outputs=[model_3d, file_out])
+
+    gr.HTML("""
+    <div style="border-top: 1px solid #303030;">
+      <br>
+      <p>Space by:<br>
+      <a href="https://twitter.com/hahahahohohe"><img src="https://img.shields.io/twitter/follow/hahahahohohe?label=%40anzorq&style=social" alt="Twitter Follow"></a><br>
+      <a href="https://github.com/qunash"><img alt="GitHub followers" src="https://img.shields.io/github/followers/qunash?style=social" alt="Github Follow"></a></p><br>
+      <a href="https://www.buymeacoffee.com/anzorq" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 35px !important;width: 120px !important;" ></a><br><br>
+      <p><img src="https://visitor-badge.glitch.me/badge?page_id=anzorq.point-e_demo" alt="visitors"></p>
+    </div>
+    """)
+
+    app.load(get_state, inputs=[], outputs=state_info, every=0.5, show_progress=False)
+
+
+app.queue()
+# app.launch(debug=True, share=True, height=768)
+app.launch(debug=True)