diff --git a/hy3dgen/texgen/differentiable_renderer/camera_utils.py b/hy3dgen/texgen/differentiable_renderer/camera_utils.py index 289710a..2a4dd0e 100755 --- a/hy3dgen/texgen/differentiable_renderer/camera_utils.py +++ b/hy3dgen/texgen/differentiable_renderer/camera_utils.py @@ -44,13 +44,14 @@ def transform_pos(mtx, pos, keepdim=False): return torch.matmul(posw, t_mtx.t())[None, ...] -def get_mv_matrix(elev, azim, camera_distance, center=None): +def get_mv_matrix(elev, azim, camera_distance, center=None, pan_x=0.0, pan_y=0.0): elev = -elev azim += 90 elev_rad = math.radians(elev) azim_rad = math.radians(azim) + # Calculate base camera position camera_position = np.array([camera_distance * math.cos(elev_rad) * math.cos(azim_rad), camera_distance * math.cos(elev_rad) * math.sin(azim_rad), @@ -61,15 +62,22 @@ def get_mv_matrix(elev, azim, camera_distance, center=None): else: center = np.array(center) + # Calculate view direction lookat = center - camera_position lookat = lookat / np.linalg.norm(lookat) + # Calculate up and right vectors up = np.array([0, 0, 1.0]) right = np.cross(lookat, up) right = right / np.linalg.norm(right) up = np.cross(right, lookat) up = up / np.linalg.norm(up) + # Apply panning by moving camera position and center + pan_offset = (right * pan_x + up * pan_y) + camera_position += pan_offset + + # Create camera matrix c2w = np.concatenate( [np.stack([right, up, -lookat], axis=-1), camera_position[:, None]], axis=-1) diff --git a/hy3dgen/texgen/differentiable_renderer/mesh_render.py b/hy3dgen/texgen/differentiable_renderer/mesh_render.py index 342d2f8..9db3751 100755 --- a/hy3dgen/texgen/differentiable_renderer/mesh_render.py +++ b/hy3dgen/texgen/differentiable_renderer/mesh_render.py @@ -151,6 +151,8 @@ class MeshRender(): (2 / 512) * max(self.default_resolution[0], self.default_resolution[1])) self.bake_mode = bake_mode + self.tex = None + self.raster_mode = raster_mode if self.raster_mode == 'cr': import custom_rasterizer as cr @@ -442,10 +444,11 @@ class MeshRender(): bg_color=[1, 1, 1], use_abs_coor=False, normalize_rgb=True, - return_type='th' - ): - - pos_camera, pos_clip = self.get_pos_from_mvp(elev, azim, camera_distance, center) + return_type='th', + pan_x=0.0, + pan_y=0.0 + ): + pos_camera, pos_clip = self.get_pos_from_mvp(elev, azim, camera_distance, center, pan_y=pan_y, pan_x=pan_x) if resolution is None: resolution = self.default_resolution if isinstance(resolution, (int, float)): @@ -499,7 +502,7 @@ class MeshRender(): image = image.cpu().numpy() * 255 image = Image.fromarray(image.astype(np.uint8)) - return image + return image, visible_mask def convert_normal_map(self, image): # blue is front, red is left, green is top @@ -520,13 +523,16 @@ class MeshRender(): return Image.fromarray(image) - def get_pos_from_mvp(self, elev, azim, camera_distance, center): + def get_pos_from_mvp(self, elev, azim, camera_distance, center, pan_y=0.0, pan_x=0.0): proj = self.camera_proj_mat r_mv = get_mv_matrix( elev=elev, azim=azim, camera_distance=self.camera_distance if camera_distance is None else camera_distance, - center=center) + center=center, + pan_x=pan_x, + pan_y=pan_y + ) pos_camera = transform_pos(r_mv, self.vtx_pos, keepdim=True) pos_clip = transform_pos(proj, pos_camera) @@ -540,9 +546,11 @@ class MeshRender(): camera_distance=None, center=None, resolution=None, - return_type='th' + return_type='th', + pan_x=0.0, + pan_y=0.0 ): - pos_camera, pos_clip = self.get_pos_from_mvp(elev, azim, camera_distance, center) + pos_camera, pos_clip = self.get_pos_from_mvp(elev, azim, camera_distance, center, pan_y=pan_y, pan_x=pan_x) if resolution is None: resolution = self.default_resolution diff --git a/nodes.py b/nodes.py index 843fbff..dc7ef96 100644 --- a/nodes.py +++ b/nodes.py @@ -340,7 +340,109 @@ class Hy3DRenderMultiView: def render_normal_multiview(self, camera_elevs, camera_azims, use_abs_coor=True): normal_maps = [] for elev, azim in zip(camera_elevs, camera_azims): - normal_map = self.render.render_normal( + normal_map, _ = self.render.render_normal( + elev, azim, use_abs_coor=use_abs_coor, return_type='th') + normal_maps.append(normal_map) + + return normal_maps + + def render_position_multiview(self, camera_elevs, camera_azims): + position_maps = [] + for elev, azim in zip(camera_elevs, camera_azims): + position_map = self.render.render_position( + elev, azim, return_type='th') + position_maps.append(position_map) + + return position_maps + +class Hy3DRenderSingleView: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "mesh": ("HY3DMESH",), + "render_type": (["normal", "depth"], {"default": "normal"}), + "render_size": ("INT", {"default": 1024, "min": 64, "max": 4096, "step": 16}), + "camera_type": (["orth", "perspective"], {"default": "orth"}), + "camera_distance": ("FLOAT", {"default": 1.45, "min": 0.1, "max": 10.0, "step": 0.001}), + "pan_x": ("FLOAT", {"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01}), + "pan_y": ("FLOAT", {"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01}), + "ortho_scale": ("FLOAT", {"default": 1.2, "min": 0.1, "max": 10.0, "step": 0.001}), + "azimuth": ("FLOAT", {"default": 0, "min": -360, "max": 360, "step": 1}), + "elevation": ("FLOAT", {"default": 0, "min": -360, "max": 360, "step": 1}), + "bg_color": ("STRING", {"default": "0, 0, 0", "tooltip": "Color as RGB values in range 0-255, separated by commas."}), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("image", ) + FUNCTION = "process" + CATEGORY = "Hunyuan3DWrapper" + + def process(self, mesh, render_type, camera_type, ortho_scale, camera_distance, pan_x, pan_y, render_size, azimuth, elevation, bg_color): + + from .hy3dgen.texgen.differentiable_renderer.mesh_render import MeshRender + + bg_color = [int(x.strip())/255.0 for x in bg_color.split(",")] + + self.render = MeshRender( + default_resolution=render_size, + texture_size=1024, + camera_distance=camera_distance, + camera_type=camera_type, + ortho_scale=ortho_scale, + filter_mode='linear' + ) + + self.render.load_mesh(mesh) + + if render_type == "normal": + normals, mask = self.render.render_normal( + elevation, + azimuth, + camera_distance=camera_distance, + center=None, + resolution=render_size, + bg_color=[0, 0, 0], + use_abs_coor=False, + pan_x=pan_x, + pan_y=pan_y + ) + + normals = 2.0 * normals - 1.0 # Map [0,1] to [-1,1] + normals = normals / (torch.norm(normals, dim=-1, keepdim=True) + 1e-6) + # Remap axes for standard normal map convention + image = torch.zeros_like(normals) + image[..., 0] = normals[..., 0] # View right to R + image[..., 1] = normals[..., 1] # View up to G + image[..., 2] = -normals[..., 2] # View forward (negated) to B + + image = (image + 1) * 0.5 + + #mask = mask.cpu().float() + masked_image = image * mask + + bg_color = torch.tensor(bg_color, dtype=torch.float32, device=image.device) + bg = bg_color.view(1, 1, 3) * (1.0 - mask) + final_image = masked_image + bg + elif render_type == "depth": + depth = self.render.render_depth( + elevation, + azimuth, + camera_distance=camera_distance, + center=None, + resolution=render_size, + pan_x=pan_x, + pan_y=pan_y + ) + final_image = depth.unsqueeze(0).repeat(1, 1, 1, 3).cpu().float() + + return (final_image,) + + def render_normal_multiview(self, camera_elevs, camera_azims, use_abs_coor=True): + normal_maps = [] + for elev, azim in zip(camera_elevs, camera_azims): + normal_map, _ = self.render.render_normal( elev, azim, use_abs_coor=use_abs_coor, return_type='th') normal_maps.append(normal_map) @@ -976,7 +1078,8 @@ NODE_CLASS_MAPPINGS = { "Hy3DGetMeshPBRTextures": Hy3DGetMeshPBRTextures, "Hy3DSetMeshPBRTextures": Hy3DSetMeshPBRTextures, "Hy3DSetMeshPBRAttributes": Hy3DSetMeshPBRAttributes, - "Hy3DVAEDecode": Hy3DVAEDecode + "Hy3DVAEDecode": Hy3DVAEDecode, + "Hy3DRenderSingleView": Hy3DRenderSingleView } NODE_DISPLAY_NAME_MAPPINGS = { "Hy3DModelLoader": "Hy3DModelLoader", @@ -1000,5 +1103,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "Hy3DGetMeshPBRTextures": "Hy3D Get Mesh PBR Textures", "Hy3DSetMeshPBRTextures": "Hy3D Set Mesh PBR Textures", "Hy3DSetMeshPBRAttributes": "Hy3D Set Mesh PBR Attributes", - "Hy3DVAEDecode": "Hy3D VAE Decode" + "Hy3DVAEDecode": "Hy3D VAE Decode", + "Hy3DRenderSingleView": "Hy3D Render SingleView" }