attempt 3d reconstruction

README.md

@@ -1,10 +1,10 @@
 ---
-title: Dpt Depth Estimation
+title: Dpt Depth Estimation + 3D
 emoji: ⚡
 colorFrom: blue
 colorTo: red
 sdk: gradio
-sdk_version: 2.8.13
+sdk_version: 2.9.1
 app_file: app.py
 pinned: false
 ---

app.py

@@ -3,6 +3,7 @@ from transformers import DPTFeatureExtractor, DPTForDepthEstimation
 import torch
 import numpy as np
 from PIL import Image
+import open3d as o3d
 
 torch.hub.download_url_to_file('http://images.cocodataset.org/val2017/000000039769.jpg', 'cats.jpg')
 
@@ -26,21 +27,49 @@ def process_image(image):
                         align_corners=False,
                  ).squeeze()
     output = prediction.cpu().numpy()
-    formatted = (output * 255 / np.max(output)).astype('uint8')
-    img = Image.fromarray(formatted)
-    return img
+    depth_image = (output * 255 / np.max(output)).astype('uint8')
+    # create_obj(formatted, "test.obj")
+    create_obj_2(np.array(image), depth_image)
+    # img = Image.fromarray(formatted)
+    return "output.gltf"
     
-    return result
+    # return result
+
+    # gradio.inputs.Image3D(self, label=None, optional=False)
+
+def create_obj_2(rgb_image, depth_image):
+    depth_o3d = o3d.geometry.Image(depth_image)
+    image_o3d = o3d.geometry.Image(rgb_image)
+    rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(image_o3d, depth_o3d)
+    w = int(depth_image.shape[0])
+    h = int(depth_image.shape[1])
+
+    FOV = np.pi/4
+    camera_intrinsic = o3d.camera.PinholeCameraIntrinsic()
+    camera_intrinsic.set_intrinsics(w, h, w*0.5, h*0.5, w*0.5, h*0.5 )
+
+    pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image,camera_intrinsic)
+    print('normals')
+    pcd.normals = o3d.utility.Vector3dVector(np.zeros((1, 3)))  # invalidate existing normals
+    pcd.estimate_normals()
+    # pcd.orient_normals_consistent_tangent_plane(100)
+    print('run Poisson surface reconstruction')
+    with o3d.utility.VerbosityContextManager(o3d.utility.VerbosityLevel.Debug) as cm:
+        mesh, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd, depth=9)
+    print(mesh)
+    o3d.io.write_triangle_mesh("output.gltf",mesh,write_triangle_uvs=True)
+    return "output.gltf"
     
-title = "Interactive demo: DPT"
+title = "Interactive demo: DPT + 3D"
 description = "Demo for Intel's DPT, a Dense Prediction Transformer for state-of-the-art dense prediction tasks such as semantic segmentation and depth estimation."
 examples =[['cats.jpg']]
 
 iface = gr.Interface(fn=process_image, 
                      inputs=gr.inputs.Image(type="pil"), 
-                     outputs=gr.outputs.Image(type="pil", label="predicted depth"),
+                     outputs=gr.outputs.Image3D(label="predicted depth", clear_color=[1.0,1.0,1.0,1.0]),
                      title=title,
                      description=description,
                      examples=examples,
+                     allow_flagging="never",
                      enable_queue=True)
 iface.launch(debug=True)

packages.txt

@@ -0,0 +1 @@
+libgl1-mesa-glx

requirements.txt

@@ -1,4 +1,7 @@
 torch
 git+https://github.com/nielsrogge/transformers.git@add_dpt_redesign#egg=transformers
 numpy
-Pillow
+Pillow
+gradio>=2.9.1
+jinja2
+open3d