From 639681b1c9de895cac7af940c8b066e4c6e59f74 Mon Sep 17 00:00:00 2001 From: yushiang Date: Sun, 8 Dec 2019 15:19:45 +0000 Subject: [PATCH 1/2] add raw depth option --- pyrender/constants.py | 3 ++- pyrender/renderer.py | 28 +++++++++++++++++----------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/pyrender/constants.py b/pyrender/constants.py index 711b547..64763ef 100644 --- a/pyrender/constants.py +++ b/pyrender/constants.py @@ -51,7 +51,8 @@ class RenderFlags(object): """Render the color buffer with the alpha channel enabled.""" FLAT = 4096 """Render the color buffer flat, with no lighting computations.""" - + USE_RAW_DEPTH = 8192 + """Disable depth output preprocessing""" class TextAlign: """Text alignment options for captions. diff --git a/pyrender/renderer.py b/pyrender/renderer.py index 9e48d53..19a35a2 100644 --- a/pyrender/renderer.py +++ b/pyrender/renderer.py @@ -1117,18 +1117,24 @@ def _read_main_framebuffer(self, scene, flags): depth_im = np.frombuffer(depth_buf, dtype=np.float32) depth_im = depth_im.reshape((height, width)) depth_im = np.flip(depth_im, axis=0) - inf_inds = (depth_im == 1.0) - depth_im = 2.0 * depth_im - 1.0 - z_near = scene.main_camera_node.camera.znear - z_far = scene.main_camera_node.camera.zfar - noninf = np.logical_not(inf_inds) - if z_far is None: - depth_im[noninf] = 2 * z_near / (1.0 - depth_im[noninf]) + + + if not (flags & RenderFlags.USE_RAW_DEPTH): + inf_inds = (depth_im == 1.0) + depth_im = 2.0 * depth_im - 1.0 + z_near = scene.main_camera_node.camera.znear + z_far = scene.main_camera_node.camera.zfar + noninf = np.logical_not(inf_inds) + if z_far is None: + depth_im[noninf] = 2 * z_near / (1.0 - depth_im[noninf]) + else: + depth_im[noninf] = ((2.0 * z_near * z_far) / + (z_far + z_near - depth_im[noninf] * + (z_far - z_near))) + depth_im[inf_inds] = 0.0 else: - depth_im[noninf] = ((2.0 * z_near * z_far) / - (z_far + z_near - depth_im[noninf] * - (z_far - z_near))) - depth_im[inf_inds] = 0.0 + pass + # Resize for macos if needed if sys.platform == 'darwin': From f3a94c3a8d9d380feeb9c6b64b2bf60203d12f50 Mon Sep 17 00:00:00 2001 From: yushiang Date: Sun, 8 Dec 2019 16:54:09 +0000 Subject: [PATCH 2/2] add example --- examples/unproj_example.py | 114 +++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 examples/unproj_example.py diff --git a/examples/unproj_example.py b/examples/unproj_example.py new file mode 100644 index 0000000..d4c7e27 --- /dev/null +++ b/examples/unproj_example.py @@ -0,0 +1,114 @@ +"""Examples of using pyrender for viewing and offscreen rendering. +""" +import pyglet +pyglet.options['shadow_window'] = False +import os +import numpy as np +import trimesh + +import pyrender +from pyrender import PerspectiveCamera,\ + DirectionalLight, SpotLight, PointLight,\ + MetallicRoughnessMaterial,\ + Primitive, Mesh, Node, Scene,\ + Viewer, OffscreenRenderer + +def unproj( inv_mvp , raw_depth): + im_z = raw_depth[:].copy() + + height, width = raw_depth.shape + im_y, im_x = np.indices((height, width)) + + # opengl is bottom-left + im_y = height-1 - im_y + + x0 = 0 + y0 = 0 + x_nd = (1.0*im_x - x0) / (width/2.0)-1 + y_nd = (1.0*im_y - y0) / (height/2.0)-1 + + # https://www.khronos.org/registry/OpenGL-Refpages/gl2.1/xhtml/gluUnProject.xml + # 2*depth-1 + x = x_nd + y = y_nd + z = im_z*2-1 + w = np.ones(x.shape) + + xyzw = np.array([x[:],y[:],z[:],w]) + + valid_mask = im_z!=1 + xyzw=xyzw[:,valid_mask] + + xyzw = inv_mvp @ xyzw + xyz = xyzw[:3,:].copy() + xyz = xyz / xyzw[3] + xyz = np.transpose(xyz) + # N,3 + return xyz + +if __name__ =='__main__': + + #============================================================================== + # Mesh creation + #============================================================================== + + # Drill trimesh + drill_trimesh = trimesh.load('./models/drill.obj') + drill_trimesh.visual.vertex_colors = np.array((1,0,0,1)) + drill_trimesh.visual.face_colors = np.array((1,0,0,1)) + drill_mesh = Mesh.from_trimesh(drill_trimesh) + + #drill_pose = np.eye(4) + #drill_pose[0,3] = 0.1 + #drill_pose[2,3] = -np.min(drill_trimesh.vertices[:,2]) + + + #============================================================================== + # Camera creation + #============================================================================== + + cam = PerspectiveCamera(yfov=(np.pi / 3.0), aspectRatio=1.0*640/480, znear=0.1, zfar=6) + cam_pose = np.array([ + [0.0, -np.sqrt(2)/2, np.sqrt(2)/2, 0.5], + [1.0, 0.0, 0.0, 0.0], + [0.0, np.sqrt(2)/2, np.sqrt(2)/2, 0.4], + [0.0, 0.0, 0.0, 1.0] + ]) + + proj = cam.get_projection_matrix() + mvp = proj @ np.linalg.inv(cam_pose) + inv_mvp = np.linalg.inv(mvp) + #============================================================================== + # Scene creation + #============================================================================== + + scene = Scene(ambient_light=np.array([0.02, 0.02, 0.02, 1.0])) + cam_node = scene.add(cam, pose=cam_pose) + + scene.main_camera_node= cam_node + + #============================================================================== + drill_node = scene.add(drill_mesh) + r = OffscreenRenderer(viewport_width=640, viewport_height=480) + + rf = pyrender.RenderFlags.NONE + rf |= pyrender.RenderFlags.USE_RAW_DEPTH + color, raw_depth = r.render(scene, flags=rf) + r.delete() + + # unproject to get point cloud + pcd = unproj( inv_mvp , raw_depth) + + #============================================================================== + #------------------------------------------------------------------------------ + # Creating meshes from point clouds + #------------------------------------------------------------------------------ + points_mesh = Mesh.from_points(pcd, colors=np.array((0.0,1.0,0.0,1.0))) + pcd_node = scene.add(points_mesh) + + #============================================================================== + # Using the viewer with a pre-specified camera + #============================================================================== + v = Viewer(scene, render_flags={'point_size':5}) + +