diff --git a/omni/docker/Dockerfile b/omni/docker/Dockerfile index 60b53af..30ad7d9 100644 --- a/omni/docker/Dockerfile +++ b/omni/docker/Dockerfile @@ -14,6 +14,7 @@ COPY ./patches/raylight_for_multi_arc.patch /tmp/ COPY ./patches/xinference_device_utils.patch /tmp/ COPY ./patches/comfyui_for_multi_arc.patch /tmp/ COPY ./patches/comfyui_voxcpm_for_xpu.patch /tmp/ +COPY ./patches/comfyui_hunyuan3d_for_xpu.patch /tmp/ # Add Intel oneAPI repo and PPA for GPU support @@ -67,6 +68,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO cd raylight && \ git checkout ff8e90ba1f2c2d23e3ac23746910ddfb523fc8f1 && \ git apply /tmp/raylight_for_multi_arc.patch && \ + pip install ray==2.49.2 && \ pip install -r requirements.txt && \ cd .. && \ git clone https://github.com/yolain/ComfyUI-Easy-Use.git comfyui-easy-use && \ @@ -83,14 +85,31 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO git checkout 044dd93c0effc9090fb279117de5db4cd90242a0 && \ git apply /tmp/comfyui_voxcpm_for_xpu.patch && \ pip install -r requirements.txt && \ + cd .. && \ + git clone https://github.com/visualbruno/ComfyUI-Hunyuan3d-2-1.git && \ + cd ComfyUI-Hunyuan3d-2-1 && \ + git checkout 9d7ef32509101495a7840b3ae8e718c8d1183305 && \ + git apply /tmp/comfyui_hunyuan3d_for_xpu.patch && \ + pip install bigdl-core==2.4.0b1 rembg realesrgan && \ + pip install -r requirements.txt && \ + cd hy3dpaint/custom_rasterizer && \ + python setup.py install && \ + cd ../DifferentiableRenderer && \ + python setup.py install && \ + cd /llm/ComfyUI/custom_nodes && \ # Install Xinference pip install "xinference[transformers]" && \ patch /usr/local/lib/python3.10/dist-packages/xinference/device_utils.py < /tmp/xinference_device_utils.patch && \ pip install kokoro Jinja2==3.1.6 jieba ordered-set pypinyin cn2an pypinyin-dict && \ # Clean rm -rf /tmp/* +RUN cd /llm/ComfyUI/custom_nodes && \ + git clone https://github.com/kijai/ComfyUI-KJNodes.git && \ + cd ComfyUI-KJNodes && \ + pip install -r requirements.txt COPY ./workflows/* /llm/ComfyUI/user/default/workflows/ COPY ./example_inputs/* /llm/ComfyUI/input/ +COPY ./tools/* /llm/tools/ WORKDIR /llm/ComfyUI diff --git a/omni/example_inputs/hunyuan3d_input.png b/omni/example_inputs/hunyuan3d_input.png new file mode 100644 index 0000000..00fda1d Binary files /dev/null and b/omni/example_inputs/hunyuan3d_input.png differ diff --git a/omni/patches/comfyui_hunyuan3d_for_xpu.patch b/omni/patches/comfyui_hunyuan3d_for_xpu.patch new file mode 100644 index 0000000..ab11071 --- /dev/null +++ b/omni/patches/comfyui_hunyuan3d_for_xpu.patch @@ -0,0 +1,894 @@ +diff --git a/hy3dpaint/DifferentiableRenderer/MeshRender.py b/hy3dpaint/DifferentiableRenderer/MeshRender.py +index 5dd9d2a..a45e4e2 100644 +--- a/hy3dpaint/DifferentiableRenderer/MeshRender.py ++++ b/hy3dpaint/DifferentiableRenderer/MeshRender.py +@@ -333,7 +333,7 @@ class MeshRender: + raster_mode="cr", + shader_type="face", + use_opengl=False, +- device="cuda", ++ device="xpu", + ortho_scale=1.0 + ): + """ +diff --git a/hy3dpaint/DifferentiableRenderer/compile_mesh_painter.sh b/hy3dpaint/DifferentiableRenderer/compile_mesh_painter.sh +index cf24fb3..b9bae56 100644 +--- a/hy3dpaint/DifferentiableRenderer/compile_mesh_painter.sh ++++ b/hy3dpaint/DifferentiableRenderer/compile_mesh_painter.sh +@@ -1 +1 @@ +-c++ -O3 -Wall -shared -std=c++11 -fPIC `python -m pybind11 --includes` mesh_inpaint_processor.cpp -o mesh_inpaint_processor`python3-config --extension-suffix` +\ No newline at end of file ++c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` mesh_inpaint_processor.cpp -o mesh_inpaint_processor`python3.10-config --extension-suffix` +\ No newline at end of file +diff --git a/hy3dpaint/custom_rasterizer/custom_rasterizer/render.py b/hy3dpaint/custom_rasterizer/custom_rasterizer/render.py +index 9d06b51..82bdbf3 100644 +--- a/hy3dpaint/custom_rasterizer/custom_rasterizer/render.py ++++ b/hy3dpaint/custom_rasterizer/custom_rasterizer/render.py +@@ -18,9 +18,13 @@ import torch + + def rasterize(pos, tri, resolution, clamp_depth=torch.zeros(0), use_depth_prior=0): + assert pos.device == tri.device ++ pos_cpu = pos[0].cpu() ++ tri_cpu = tri.cpu() + findices, barycentric = custom_rasterizer_kernel.rasterize_image( +- pos[0], tri, clamp_depth, resolution[1], resolution[0], 1e-6, use_depth_prior ++ pos_cpu, tri_cpu, clamp_depth, resolution[1], resolution[0], 1e-6, use_depth_prior + ) ++ findices = findices.to(pos.device) ++ barycentric = barycentric.to(pos.device) + return findices, barycentric + + +diff --git a/hy3dpaint/custom_rasterizer/lib/custom_rasterizer_kernel/rasterizer.cpp b/hy3dpaint/custom_rasterizer/lib/custom_rasterizer_kernel/rasterizer.cpp +index 4529d7e..7ae1fd5 100644 +--- a/hy3dpaint/custom_rasterizer/lib/custom_rasterizer_kernel/rasterizer.cpp ++++ b/hy3dpaint/custom_rasterizer/lib/custom_rasterizer_kernel/rasterizer.cpp +@@ -128,8 +128,8 @@ std::vector rasterize_image(torch::Tensor V, torch::Tensor F, tor + int device_id = V.get_device(); + if (device_id == -1) + return rasterize_image_cpu(V, F, D, width, height, occlusion_truncation, use_depth_prior); +- else +- return rasterize_image_gpu(V, F, D, width, height, occlusion_truncation, use_depth_prior); ++ // else ++ // return rasterize_image_gpu(V, F, D, width, height, occlusion_truncation, use_depth_prior); + } + + PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { +diff --git a/hy3dpaint/custom_rasterizer/lib/custom_rasterizer_kernel/rasterizer.h b/hy3dpaint/custom_rasterizer/lib/custom_rasterizer_kernel/rasterizer.h +index a1fa8ff..38ad470 100644 +--- a/hy3dpaint/custom_rasterizer/lib/custom_rasterizer_kernel/rasterizer.h ++++ b/hy3dpaint/custom_rasterizer/lib/custom_rasterizer_kernel/rasterizer.h +@@ -4,16 +4,16 @@ + #include + #include + #include +-#include // For CUDA context ++// #include // For CUDA context + #include + #define INT64 uint64_t + #define MAXINT 2147483647 + +-__host__ __device__ inline float calculateSignedArea2(float* a, float* b, float* c) { ++inline float calculateSignedArea2(float* a, float* b, float* c) { + return ((c[0] - a[0]) * (b[1] - a[1]) - (b[0] - a[0]) * (c[1] - a[1])); + } + +-__host__ __device__ inline void calculateBarycentricCoordinate(float* a, float* b, float* c, float* p, ++inline void calculateBarycentricCoordinate(float* a, float* b, float* c, float* p, + float* barycentric) + { + float beta_tri = calculateSignedArea2(a, p, c); +@@ -34,14 +34,14 @@ __host__ __device__ inline void calculateBarycentricCoordinate(float* a, float* + barycentric[2] = gamma; + } + +-__host__ __device__ inline bool isBarycentricCoordInBounds(float* barycentricCoord) { ++inline bool isBarycentricCoordInBounds(float* barycentricCoord) { + return barycentricCoord[0] >= 0.0 && barycentricCoord[0] <= 1.0 && + barycentricCoord[1] >= 0.0 && barycentricCoord[1] <= 1.0 && + barycentricCoord[2] >= 0.0 && barycentricCoord[2] <= 1.0; + } + +-std::vector rasterize_image_gpu(torch::Tensor V, torch::Tensor F, torch::Tensor D, +- int width, int height, float occlusion_truncation, int use_depth_prior); ++// std::vector rasterize_image_gpu(torch::Tensor V, torch::Tensor F, torch::Tensor D, ++// int width, int height, float occlusion_truncation, int use_depth_prior); + + std::vector> build_hierarchy(std::vector view_layer_positions, std::vector view_layer_normals, int num_level, int resolution); + +diff --git a/hy3dpaint/custom_rasterizer/setup.py b/hy3dpaint/custom_rasterizer/setup.py +index 15192e9..06f34fe 100644 +--- a/hy3dpaint/custom_rasterizer/setup.py ++++ b/hy3dpaint/custom_rasterizer/setup.py +@@ -18,12 +18,12 @@ from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CppExtensio + + # build custom rasterizer + +-custom_rasterizer_module = CUDAExtension( ++custom_rasterizer_module = CppExtension( + "custom_rasterizer_kernel", + [ + "lib/custom_rasterizer_kernel/rasterizer.cpp", + "lib/custom_rasterizer_kernel/grid_neighbor.cpp", +- "lib/custom_rasterizer_kernel/rasterizer_gpu.cu", ++ # "lib/custom_rasterizer_kernel/rasterizer_gpu.cu", + ], + ) + +diff --git a/hy3dpaint/hunyuanpaintpbr/pipeline.py b/hy3dpaint/hunyuanpaintpbr/pipeline.py +index 8f2ca9c..53e9843 100644 +--- a/hy3dpaint/hunyuanpaintpbr/pipeline.py ++++ b/hy3dpaint/hunyuanpaintpbr/pipeline.py +@@ -242,7 +242,7 @@ class HunyuanPaintPipeline(StableDiffusionPipeline): + if img.shape[2] > 3: + alpha = img[:, :, 3:] + img = img[:, :, :3] * alpha + bg_c * (1 - alpha) +- img = torch.from_numpy(img).permute(2, 0, 1).unsqueeze(0).contiguous().half().to("cuda") ++ img = torch.from_numpy(img).permute(2, 0, 1).unsqueeze(0).contiguous().half().to("xpu") + view_imgs.append(img) + view_imgs = torch.cat(view_imgs, dim=0) + images_tensor.append(view_imgs.unsqueeze(0)) +@@ -713,7 +713,7 @@ class HunyuanPaintPipeline(StableDiffusionPipeline): + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[0] +diff --git a/hy3dpaint/hunyuanpaintpbr/unet/attn_processor.py b/hy3dpaint/hunyuanpaintpbr/unet/attn_processor.py +index 37ebde0..e614bd4 100644 +--- a/hy3dpaint/hunyuanpaintpbr/unet/attn_processor.py ++++ b/hy3dpaint/hunyuanpaintpbr/unet/attn_processor.py +@@ -687,7 +687,7 @@ class SelfAttnProcessor2_0(BaseAttnProcessor): + + # Device management (if needed) + if multiple_devices: +- device = torch.device("cuda:0") if token == "albedo" else torch.device("cuda:1") ++ device = torch.device("xpu:0") if token == "albedo" else torch.device("xpu:1") + for attr in [f"to_q{token_suffix}", f"to_k{token_suffix}", f"to_v{token_suffix}", f"to_out{token_suffix}"]: + getattr(target, attr).to(device) + +@@ -747,7 +747,7 @@ class SelfAttnProcessor2_0(BaseAttnProcessor): + # Process each PBR setting + results = [] + for token, pbr_hs in zip(self.pbr_setting, pbr_hidden_states): +- processed_hs = rearrange(pbr_hs, "b n_pbrs n l c -> (b n_pbrs n) l c").to("cuda:0") ++ processed_hs = rearrange(pbr_hs, "b n_pbrs n l c -> (b n_pbrs n) l c").to("xpu:0") + result = self.process_single(attn, processed_hs, None, attention_mask, temb, token, False) + results.append(result) + +diff --git a/hy3dpaint/hunyuanpaintpbr/unet/model.py b/hy3dpaint/hunyuanpaintpbr/unet/model.py +index f98ca7e..e62db2f 100644 +--- a/hy3dpaint/hunyuanpaintpbr/unet/model.py ++++ b/hy3dpaint/hunyuanpaintpbr/unet/model.py +@@ -143,7 +143,7 @@ class HunyuanPaint(pl.LightningModule): + self.register_buffer("sqrt_recipm1_alphas_cumprod", torch.sqrt(1.0 / alphas_cumprod - 1).float()) + + def on_fit_start(self): +- device = torch.device(f"cuda:{self.local_rank}") ++ device = torch.device(f"xpu:{self.local_rank}") + self.pipeline.to(device) + if self.global_rank == 0: + os.makedirs(os.path.join(self.logdir, "images_val"), exist_ok=True) +diff --git a/hy3dpaint/textureGenPipeline.py b/hy3dpaint/textureGenPipeline.py +index 05b43ab..2ce8a2f 100644 +--- a/hy3dpaint/textureGenPipeline.py ++++ b/hy3dpaint/textureGenPipeline.py +@@ -48,7 +48,7 @@ def quick_convert_with_obj2gltf(obj_path: str, glb_path: str) -> bool: + + class Hunyuan3DPaintConfig: + def __init__(self, resolution, camera_azims, camera_elevs, view_weights, ortho_scale, texture_size): +- self.device = "cuda" ++ self.device = "xpu" + + cfg_path = os.path.join( + os.path.dirname(__file__), "cfgs", "hunyuan-paint-pbr.yaml" +@@ -105,7 +105,7 @@ class Hunyuan3DPaintPipeline: + #self.load_models() + + def load_models(self): +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + #self.models["super_model"] = imageSuperNet(self.config) + self.models["multiview_model"] = multiviewDiffusionNet(self.config) + print("Models Loaded.") +@@ -176,7 +176,7 @@ class Hunyuan3DPaintPipeline: + # image_style = [image.convert("RGB") for image in image_style] + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + + ########### Multiview ########## + print('Generating MultiViews PBR ...') +@@ -283,7 +283,7 @@ class Hunyuan3DPaintPipeline: + del self.model + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + gc.collect() + + def load_mesh(self, mesh): +diff --git a/hy3dpaint/utils/multiview_utils.py b/hy3dpaint/utils/multiview_utils.py +index 77c0fdf..af1608c 100644 +--- a/hy3dpaint/utils/multiview_utils.py ++++ b/hy3dpaint/utils/multiview_utils.py +@@ -25,6 +25,61 @@ from diffusers import EulerAncestralDiscreteScheduler, DDIMScheduler, UniPCMulti + from ..hunyuanpaintpbr.pipeline import HunyuanPaintPipeline + + ++def get_or_download_model_path(): ++ env_var_name = "_LLM_SCALER_OMNI_HY3D_PATH" ++ model_sub_path_in_repo = "hunyuan3d-paintpbr-v2-1" ++ ++ base_download_dir = os.getenv(env_var_name, "/llm/models/Hunyuan3D-2.1") ++ ++ if base_download_dir is None: ++ raise ValueError( ++ f"Environment variable '{env_var_name}' is not set. " ++ f"Please set it to the desired base directory for model downloads and storage." ++ ) ++ ++ os.makedirs(base_download_dir, exist_ok=True) ++ ++ expected_full_model_path = os.path.join(base_download_dir, model_sub_path_in_repo) ++ ++ if os.path.exists(expected_full_model_path) and os.path.isdir( ++ expected_full_model_path ++ ): ++ print(f"Model found at designated path: {expected_full_model_path}") ++ model_path = expected_full_model_path ++ else: ++ print( ++ f"Model not found at {expected_full_model_path}. Attempting to download..." ++ ) ++ try: ++ downloaded_repo_root_path = huggingface_hub.snapshot_download( ++ repo_id="tencent/Hunyuan3D-2.1", ++ allow_patterns=[ ++ f"{model_sub_path_in_repo}/*", ++ ], ++ local_dir=base_download_dir, ++ local_dir_use_symlinks=False, ++ ) ++ ++ model_path = os.path.join(downloaded_repo_root_path, model_sub_path_in_repo) ++ ++ if not (os.path.exists(model_path) and os.path.isdir(model_path)): ++ raise RuntimeError( ++ f"Download completed, but expected model directory not found at {model_path}. " ++ f"This might indicate an issue with allow_patterns or the remote repository structure." ++ ) ++ ++ print(f"Model downloaded successfully to: {model_path}") ++ ++ except HfHubHTTPError as e: ++ print(f"Error downloading model from Hugging Face Hub: {e}") ++ raise ++ except Exception as e: ++ print(f"An unexpected error occurred during model download: {e}") ++ raise ++ ++ return model_path ++ ++ + class multiviewDiffusionNet: + def __init__(self, config) -> None: + self.device = config.device +@@ -35,13 +90,8 @@ class multiviewDiffusionNet: + self.cfg = cfg + self.mode = self.cfg.model.params.stable_diffusion_config.custom_pipeline[2:] + +- model_path = huggingface_hub.snapshot_download( +- repo_id=config.multiview_pretrained_path, +- allow_patterns=["hunyuan3d-paintpbr-v2-1/*"], +- ) ++ model_path = get_or_download_model_path() + +- model_path = os.path.join(model_path, "hunyuan3d-paintpbr-v2-1") +- + pipeline = HunyuanPaintPipeline.from_pretrained( + model_path, + torch_dtype=torch.float16 +@@ -77,17 +127,17 @@ class multiviewDiffusionNet: + def forward_one(self, input_images, control_images, prompt=None, custom_view_size=None, resize_input=False, num_steps=10, guidance_scale=3.0, seed=0): + self.seed_everything(seed) + custom_view_size = custom_view_size if custom_view_size is not None else self.pipeline.view_size +- ++ + if not isinstance(input_images, List): + input_images = [input_images] +- ++ + if not resize_input: + input_images = [ + input_image.resize((self.pipeline.view_size, self.pipeline.view_size)) for input_image in input_images + ] + else: + input_images = [input_image.resize((custom_view_size, custom_view_size)) for input_image in input_images] +- ++ + for i in range(len(control_images)): + control_images[i] = control_images[i].resize((custom_view_size, custom_view_size)) + if control_images[i].mode == "L": +diff --git a/hy3dshape/hy3dshape/models/autoencoders/model.py b/hy3dshape/hy3dshape/models/autoencoders/model.py +index 3d782f5..c6d081c 100644 +--- a/hy3dshape/hy3dshape/models/autoencoders/model.py ++++ b/hy3dshape/hy3dshape/models/autoencoders/model.py +@@ -123,7 +123,7 @@ class VectsetVAE(nn.Module): + cls, + ckpt_path, + config_path, +- device='cuda', ++ device='xpu', + dtype=torch.float16, + use_safetensors=None, + **kwargs, +@@ -157,7 +157,7 @@ class VectsetVAE(nn.Module): + def from_pretrained( + cls, + model_path, +- device='cuda', ++ device='xpu', + dtype=torch.float16, + use_safetensors=False, + variant='fp16', +diff --git a/hy3dshape/hy3dshape/models/conditioner.py b/hy3dshape/hy3dshape/models/conditioner.py +index d0d848c..8a8eff3 100644 +--- a/hy3dshape/hy3dshape/models/conditioner.py ++++ b/hy3dshape/hy3dshape/models/conditioner.py +@@ -93,8 +93,9 @@ class ImageEncoder(nn.Module): + low, high = value_range + image = (image - low) / (high - low) + +- image = image.to(self.model.device, dtype=self.model.dtype) ++ image = image.to(device="cpu", dtype=self.model.dtype) + inputs = self.transform(image) ++ inputs = inputs.to(self.model.device) + outputs = self.model(inputs) + + last_hidden_state = outputs.last_hidden_state +diff --git a/hy3dshape/hy3dshape/models/denoisers/hunyuandit.py b/hy3dshape/hy3dshape/models/denoisers/hunyuandit.py +index b4b3b50..15f0466 100644 +--- a/hy3dshape/hy3dshape/models/denoisers/hunyuandit.py ++++ b/hy3dshape/hy3dshape/models/denoisers/hunyuandit.py +@@ -209,26 +209,26 @@ class CrossAttention(nn.Module): + q = self.q_norm(q) + k = self.k_norm(k) + +- with torch.backends.cuda.sdp_kernel( +- enable_flash=True, +- enable_math=False, +- enable_mem_efficient=True +- ): +- q, k, v = map(lambda t: rearrange(t, 'b n h d -> b h n d', h=self.num_heads), (q, k, v)) +- context = F.scaled_dot_product_attention( +- q, k, v +- ).transpose(1, 2).reshape(b, s1, -1) ++ # with torch.backends.cuda.sdp_kernel( ++ # enable_flash=True, ++ # enable_math=False, ++ # enable_mem_efficient=True ++ # ): ++ q, k, v = map(lambda t: rearrange(t, 'b n h d -> b h n d', h=self.num_heads), (q, k, v)) ++ context = F.scaled_dot_product_attention( ++ q, k, v ++ ).transpose(1, 2).reshape(b, s1, -1) + + if self.with_dca: +- with torch.backends.cuda.sdp_kernel( +- enable_flash=True, +- enable_math=False, +- enable_mem_efficient=True +- ): +- k_dca, v_dca = map(lambda t: rearrange(t, 'b n h d -> b h n d', h=self.num_heads), +- (k_dca, v_dca)) +- context_dca = F.scaled_dot_product_attention( +- q, k_dca, v_dca).transpose(1, 2).reshape(b, s1, -1) ++ # with torch.backends.cuda.sdp_kernel( ++ # enable_flash=True, ++ # enable_math=False, ++ # enable_mem_efficient=True ++ # ): ++ k_dca, v_dca = map(lambda t: rearrange(t, 'b n h d -> b h n d', h=self.num_heads), ++ (k_dca, v_dca)) ++ context_dca = F.scaled_dot_product_attention( ++ q, k_dca, v_dca).transpose(1, 2).reshape(b, s1, -1) + + context = context + self.dca_weight * context_dca + +@@ -286,13 +286,13 @@ class Attention(nn.Module): + q = self.q_norm(q) # [b, h, s, d] + k = self.k_norm(k) # [b, h, s, d] + +- with torch.backends.cuda.sdp_kernel( +- enable_flash=True, +- enable_math=False, +- enable_mem_efficient=True +- ): +- x = F.scaled_dot_product_attention(q, k, v) +- x = x.transpose(1, 2).reshape(B, N, -1) ++ # with torch.backends.cuda.sdp_kernel( ++ # enable_flash=True, ++ # enable_math=False, ++ # enable_mem_efficient=True ++ # ): ++ x = F.scaled_dot_product_attention(q, k, v) ++ x = x.transpose(1, 2).reshape(B, N, -1) + + x = self.out_proj(x) + return x +diff --git a/hy3dshape/hy3dshape/models/diffusion/flow_matching_sit.py b/hy3dshape/hy3dshape/models/diffusion/flow_matching_sit.py +index 9813b74..e2d52a5 100644 +--- a/hy3dshape/hy3dshape/models/diffusion/flow_matching_sit.py ++++ b/hy3dshape/hy3dshape/models/diffusion/flow_matching_sit.py +@@ -254,13 +254,13 @@ class Diffuser(pl.LightningModule): + pl.seed_everything(self.trainer.global_rank) + + def forward(self, batch): +- with torch.autocast(device_type="cuda", dtype=torch.bfloat16): #float32 for text ++ with torch.autocast(device_type="xpu", dtype=torch.bfloat16): #float32 for text + contexts = self.cond_stage_model(image=batch.get('image'), text=batch.get('text'), mask=batch.get('mask')) + # t5_text = contexts['t5_text']['prompt_embeds'] + # nan_count = torch.isnan(t5_text).sum() + # if nan_count > 0: + # print("t5_text has %d NaN values"%(nan_count)) +- with torch.autocast(device_type="cuda", dtype=torch.float16): ++ with torch.autocast(device_type="xpu", dtype=torch.float16): + with torch.no_grad(): + latents = self.first_stage_model.encode(batch[self.first_stage_key], sample_posterior=True) + latents = self.z_scale_factor * latents +@@ -287,7 +287,7 @@ class Diffuser(pl.LightningModule): + # else: + # mesh.export(f"check_{time.time()}.glb") + +- with torch.autocast(device_type="cuda", dtype=torch.bfloat16): ++ with torch.autocast(device_type="xpu", dtype=torch.bfloat16): + loss = self.transport.training_losses(self.model, latents, dict(contexts=contexts))["loss"].mean() + return loss + +@@ -322,7 +322,7 @@ class Diffuser(pl.LightningModule): + generator = torch.Generator().manual_seed(0) + + with self.ema_scope("Sample"): +- with torch.amp.autocast(device_type='cuda'): ++ with torch.amp.autocast(device_type='xpu'): + try: + self.pipeline.device = self.device + self.pipeline.dtype = self.dtype +diff --git a/hy3dshape/hy3dshape/pipelines.py b/hy3dshape/hy3dshape/pipelines.py +index 924c1a8..50330d1 100644 +--- a/hy3dshape/hy3dshape/pipelines.py ++++ b/hy3dshape/hy3dshape/pipelines.py +@@ -144,7 +144,7 @@ class Hunyuan3DDiTPipeline: + cls, + ckpt_path, + config_path, +- device='cuda', ++ device='xpu', + dtype=torch.float16, + use_safetensors=None, + attention_mode="sdpa", +@@ -208,7 +208,7 @@ class Hunyuan3DDiTPipeline: + def from_pretrained( + cls, + model_path, +- device='cuda', ++ device='xpu', + dtype=torch.float16, + use_safetensors=False, + variant='fp16', +@@ -245,7 +245,7 @@ class Hunyuan3DDiTPipeline: + scheduler, + conditioner, + image_processor, +- device='cuda', ++ device='xpu', + dtype=torch.float16, + **kwargs + ): +@@ -338,7 +338,7 @@ class Hunyuan3DDiTPipeline: + return torch.device(module._hf_hook.execution_device) + return self.device + +- def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = "cuda"): ++ def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = "xpu"): + r""" + Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared + to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` +diff --git a/nodes.py b/nodes.py +index 8e6680f..057247c 100644 +--- a/nodes.py ++++ b/nodes.py +@@ -35,6 +35,9 @@ from .hy3dshape.hy3dshape.models.autoencoders import ShapeVAE + + from .hy3dshape.hy3dshape.meshlib import postprocessmesh + ++from .xpu_convert import convert_to_xpu ++convert_to_xpu() ++ + from spandrel import ModelLoader, ImageModelDescriptor + + import folder_paths +@@ -204,7 +207,7 @@ def _convert_texture_format(tex: Union[np.ndarray, torch.Tensor, Image.Image], + + def convert_ndarray_to_pil(texture): + texture_size = len(texture) +- tex = _convert_texture_format(texture,(texture_size, texture_size),"cuda") ++ tex = _convert_texture_format(texture,(texture_size, texture_size),"xpu") + tex = tex.cpu().numpy() + processed_texture = (tex * 255).astype(np.uint8) + pil_texture = Image.fromarray(processed_texture) +@@ -308,7 +311,7 @@ class Hy3DMeshGenerator: + #del vae + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + gc.collect() + + return (latents,) +@@ -389,7 +392,7 @@ class Hy3DMeshGenerator: + # #del vae + + # mm.soft_empty_cache() +- # torch.cuda.empty_cache() ++ # torch.xpu.empty_cache() + # gc.collect() + + # return (latents,) +@@ -532,7 +535,7 @@ class Hy3DInPaint: + del pipeline + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + gc.collect() + + return (texture_tensor, texture_mr_tensor, trimesh, output_glb_path) +@@ -702,7 +705,7 @@ class Hy3D21VAEDecode: + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + + vae.to(device) + +@@ -731,7 +734,7 @@ class Hy3D21VAEDecode: + del vae + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + gc.collect() + + return (mesh_output, ) +@@ -1350,7 +1353,7 @@ class Hy3D21MeshGenerationBatch: + mesh_output.export(output_glb_path, file_type=file_format) + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + gc.collect() + else: + print(f'Skipping file {file}') +@@ -1361,7 +1364,7 @@ class Hy3D21MeshGenerationBatch: + del vae + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + gc.collect() + + return (input_folder, output_folder, processed_input_images, processed_output_meshes, ) +@@ -1573,7 +1576,7 @@ class Hy3D21GenerateMultiViewsBatch: + del paint_pipeline + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + gc.collect() + else: + print(f'Skipping {file}') +@@ -1588,7 +1591,7 @@ class Hy3D21GenerateMultiViewsBatch: + print('Nothing to process') + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + gc.collect() + + return (processed_meshes, ) +@@ -1833,7 +1836,7 @@ class Hy3DBakeMultiViewsWithMetaData: + del pipeline + + mm.soft_empty_cache() +- torch.cuda.empty_cache() ++ torch.xpu.empty_cache() + gc.collect() + + return (texture_tensor, texture_mr_tensor, trimesh, output_glb_path) +diff --git a/requirements.txt b/requirements.txt +index b0eec02..306fef6 100644 +--- a/requirements.txt ++++ b/requirements.txt +@@ -1,5 +1,5 @@ + trimesh +-pymeshlab ++pymeshlab==2022.2.post3 + pygltflib + xatlas + open3d +diff --git a/xpu_convert.py b/xpu_convert.py +new file mode 100644 +index 0000000..f408557 +--- /dev/null ++++ b/xpu_convert.py +@@ -0,0 +1,263 @@ ++import torch ++from torch import nn ++import torch.nn.functional as F ++from diffusers.models.attention_processor import AttnProcessor2_0, Attention ++from typing import Callable, List, Optional, Tuple, Union ++import math ++ ++_original_layer_norm_forward = nn.LayerNorm.forward ++ ++def _new_layer_norm_forward(self, hidden_states: torch.Tensor): ++ if ( ++ hidden_states.device.type == 'xpu' and ++ hidden_states.dtype in (torch.float, torch.half) and ++ self.weight is not None ++ ): ++ try: ++ import bigdl_core ++ hidden_size = math.prod(self.normalized_shape) ++ x_2d = hidden_states.reshape(-1, hidden_size).contiguous() ++ output = bigdl_core.layer_norm(x_2d, self.weight, self.bias, self.eps) ++ return output.reshape(hidden_states.shape) ++ ++ except ImportError: ++ return _original_layer_norm_forward(self, hidden_states) ++ else: ++ print(hidden_states.dtype) ++ return _original_layer_norm_forward(self, hidden_states) ++ ++ ++_original_F_sdpa = F.scaled_dot_product_attention ++def chunk_scaled_dot_product_attention( ++ query, ++ key, ++ value, ++ attn_mask=None, ++ dropout_p=0.0, ++ is_causal=False, ++ scale=None, ++ chunk_size=1024, ++): ++ if chunk_size is None or query.size(2) <= chunk_size: ++ return _original_F_sdpa( ++ query, key, value, attn_mask, dropout_p, is_causal, scale=scale ++ ) ++ ++ if scale is not None: ++ return _original_F_sdpa( ++ query, key, value, attn_mask, dropout_p, is_causal, scale=scale ++ ) ++ ++ if is_causal: ++ warnings.warn("Chunked computation may not work correctly with causal attention. " ++ "Consider setting chunk_size=None for causal attention.") ++ ++ if dropout_p > 0: ++ warnings.warn("Dropout is applied independently to each chunk, which may " ++ "result in slightly different behavior compared to non-chunked version.") ++ ++ Lq = query.size(2) ++ query_chunks = torch.split(query, chunk_size, dim=2) ++ ++ mask_chunks = None ++ if attn_mask is not None: ++ split_dim = -2 if attn_mask.dim() >= 2 else 0 ++ if attn_mask.size(split_dim) == 1: ++ mask_chunks = [attn_mask] * len(query_chunks) ++ elif attn_mask.size(split_dim) == Lq: ++ mask_chunks = torch.split(attn_mask, chunk_size, dim=split_dim) ++ else: ++ raise ValueError(f"Attention mask size {attn_mask.size()} is incompatible " ++ f"with query size {query.size()} for chunked computation") ++ else: ++ mask_chunks = [None] * len(query_chunks) ++ ++ output_chunks = [] ++ ++ for q_chunk, m_chunk in zip(query_chunks, mask_chunks): ++ chunk_output = F.scaled_dot_product_attention( ++ q_chunk, key, value, ++ attn_mask=m_chunk, ++ dropout_p=dropout_p, ++ is_causal=is_causal ++ ) ++ output_chunks.append(chunk_output) ++ ++ return torch.cat(output_chunks, dim=2) ++ ++def chunked_diffusers_attention_processor_call( ++ self, ++ attn: Attention, ++ hidden_states: torch.Tensor, ++ encoder_hidden_states: Optional[torch.Tensor] = None, ++ attention_mask: Optional[torch.Tensor] = None, ++ temb: Optional[torch.Tensor] = None, ++ *args, ++ **kwargs, ++) -> torch.Tensor: ++ if len(args) > 0 or kwargs.get("scale", None) is not None: ++ deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." ++ deprecate("scale", "1.0.0", deprecation_message) ++ ++ residual = hidden_states ++ if attn.spatial_norm is not None: ++ hidden_states = attn.spatial_norm(hidden_states, temb) ++ ++ input_ndim = hidden_states.ndim ++ ++ if input_ndim == 4: ++ batch_size, channel, height, width = hidden_states.shape ++ hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) ++ ++ batch_size, sequence_length, _ = ( ++ hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape ++ ) ++ ++ if attention_mask is not None: ++ attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) ++ # scaled_dot_product_attention expects attention_mask shape to be ++ # (batch, heads, source_length, target_length) ++ attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) ++ ++ if attn.group_norm is not None: ++ hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) ++ ++ query = attn.to_q(hidden_states) ++ ++ if encoder_hidden_states is None: ++ encoder_hidden_states = hidden_states ++ elif attn.norm_cross: ++ encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) ++ ++ key = attn.to_k(encoder_hidden_states) ++ value = attn.to_v(encoder_hidden_states) ++ ++ inner_dim = key.shape[-1] ++ head_dim = inner_dim // attn.heads ++ ++ query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) ++ ++ key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) ++ value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) ++ ++ if attn.norm_q is not None: ++ query = attn.norm_q(query) ++ if attn.norm_k is not None: ++ key = attn.norm_k(key) ++ ++ # the output of sdp = (batch, num_heads, seq_len, head_dim) ++ # TODO: add support for attn.scale when we move to Torch 2.1 ++ hidden_states = chunk_scaled_dot_product_attention( ++ query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ++ ) ++ ++ hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) ++ hidden_states = hidden_states.to(query.dtype) ++ ++ # linear proj ++ hidden_states = attn.to_out[0](hidden_states) ++ # dropout ++ hidden_states = attn.to_out[1](hidden_states) ++ ++ if input_ndim == 4: ++ hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) ++ ++ if attn.residual_connection: ++ hidden_states = hidden_states + residual ++ ++ hidden_states = hidden_states / attn.rescale_output_factor ++ ++ return hidden_states ++ ++from realesrgan import RealESRGANer ++def process_on_xpu(self): ++ self.output = self.model(self.img.to("xpu")) ++ ++import torch ++import torch.nn.functional as F ++import functools # Used to preserve function metadata like docstrings ++ ++# Global variables to store the original function and patch status ++_original_interpolate_func = None ++_is_interpolate_patched = False ++ ++ ++def patch_xpu_interpolate_to_cpu(): ++ """ ++ patches torch.nn.functional.interpolate. If an input tensor is on an XPU device, ++ it will be moved to CPU for interpolation, and the result will be moved back ++ to the original XPU device. ++ """ ++ global _original_interpolate_func, _is_interpolate_patched ++ ++ if _is_interpolate_patched: ++ print("torch.nn.functional.interpolate is already patched for XPU. Skipping.") ++ return ++ ++ # Store the original function ++ _original_interpolate_func = F.interpolate ++ ++ @functools.wraps(_original_interpolate_func) ++ def _custom_interpolate(input_tensor, *args, **kwargs): ++ """ ++ Custom wrapper for interpolate. Moves XPU tensors to CPU for computation. ++ """ ++ ++ if input_tensor.device.type == "xpu": ++ # print( ++ # f"Intercepted interpolate call for XPU tensor at device {input_tensor.device}. Moving to CPU for computation." ++ # ) ++ original_device = input_tensor.device ++ ++ # Move input to CPU ++ input_on_cpu = input_tensor.to("cpu") ++ ++ # Call the original interpolate function on CPU ++ result_on_cpu = _original_interpolate_func(input_on_cpu, *args, **kwargs) ++ ++ # Move the result back to the original XPU device ++ result_on_xpu = result_on_cpu.to(original_device) ++ # print( ++ # f"Interpolation completed on CPU, result moved back to {original_device}." ++ # ) ++ return result_on_xpu ++ else: ++ # If not an XPU tensor, just call the original function directly ++ return _original_interpolate_func(input_tensor, *args, **kwargs) ++ ++ # Replace the original function with our custom one ++ F.interpolate = _custom_interpolate ++ _is_interpolate_patched = True ++ print( ++ "Successfully patched torch.nn.functional.interpolate to handle XPU tensors on CPU." ++ ) ++ ++ ++def unpatch_xpu_interpolate_to_cpu(): ++ """ ++ Restores the original torch.nn.functional.interpolate function if it was patched. ++ """ ++ global _original_interpolate_func, _is_interpolate_patched ++ ++ if not _is_interpolate_patched: ++ print( ++ "torch.nn.functional.interpolate is not currently patched. Skipping unpatch." ++ ) ++ return ++ ++ if _original_interpolate_func is not None: ++ F.interpolate = _original_interpolate_func ++ _original_interpolate_func = None ++ _is_interpolate_patched = False ++ print("Successfully unpatched torch.nn.functional.interpolate.") ++ else: ++ print("Error: Could not unpatch. Original function reference missing.") ++ ++def convert_to_xpu(): ++ nn.LayerNorm.forward = _new_layer_norm_forward ++ # AttnProcessor2_0.__call__ = chunked_diffusers_attention_processor_call ++ F.scaled_dot_product_attention = chunk_scaled_dot_product_attention ++ RealESRGANer.process = process_on_xpu ++ # patch_xpu_interpolate_to_cpu() ++ ++ print("Converted to XPU compatible functions.") diff --git a/omni/tools/download_hy3d_pbr_model.py b/omni/tools/download_hy3d_pbr_model.py new file mode 100644 index 0000000..a94644e --- /dev/null +++ b/omni/tools/download_hy3d_pbr_model.py @@ -0,0 +1,76 @@ +import os +import huggingface_hub +from huggingface_hub.utils import HfHubHTTPError + + +def get_or_download_model_path(): + env_var_name = "_LLM_SCALER_OMNI_HY3D_PATH" + model_sub_path_in_repo = "hunyuan3d-paintpbr-v2-1" + + base_download_dir = os.getenv(env_var_name, "/llm/models/Hunyuan3D-2.1") + + if base_download_dir is None: + raise ValueError( + f"Environment variable '{env_var_name}' is not set. " + f"Please set it to the desired base directory for model downloads and storage." + ) + + os.makedirs(base_download_dir, exist_ok=True) + + expected_full_model_path = os.path.join( + base_download_dir, model_sub_path_in_repo + ) + + if os.path.exists(expected_full_model_path) and os.path.isdir( + expected_full_model_path + ): + print(f"Model found at designated path: {expected_full_model_path}") + model_path = expected_full_model_path + else: + print( + f"Model not found at {expected_full_model_path}. Attempting to download..." + ) + try: + downloaded_repo_root_path = huggingface_hub.snapshot_download( + repo_id="tencent/Hunyuan3D-2.1", + allow_patterns=[ + f"{model_sub_path_in_repo}/*", + ], + local_dir=base_download_dir, + local_dir_use_symlinks=False, + ) + + model_path = os.path.join( + downloaded_repo_root_path, model_sub_path_in_repo + ) + + if not (os.path.exists(model_path) and os.path.isdir(model_path)): + raise RuntimeError( + f"Download completed, but expected model directory not found at {model_path}. " + f"This might indicate an issue with allow_patterns or the remote repository structure." + ) + + print(f"Model downloaded successfully to: {model_path}") + + except HfHubHTTPError as e: + print(f"Error downloading model from Hugging Face Hub: {e}") + raise + except Exception as e: + print(f"An unexpected error occurred during model download: {e}") + raise + + return model_path + + +if __name__ == "__main__": + + try: + final_model_dir = get_or_download_model_path() + print(f"\nFinal model directory to use: {final_model_dir}") + + except ValueError as e: + print(f"Configuration error: {e}") + except RuntimeError as e: + print(f"Model download/verification error: {e}") + except Exception as e: + print(f"An unexpected error occurred: {e}") diff --git a/omni/tools/fix_degradations.sh b/omni/tools/fix_degradations.sh new file mode 100644 index 0000000..e6e6930 --- /dev/null +++ b/omni/tools/fix_degradations.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# ============================================================================== +# Script Name: patch_degradations.sh +# Description: Replaces a specific import statement in basicsr's degradations.py. +# This is often needed to fix compatibility issues with newer +# PyTorch/torchvision versions where rgb_to_grayscale was moved. +# Requirements: sudo access +# ============================================================================== + +# --- Configuration Variables --- +FILE_PATH="/usr/local/lib/python3.10/dist-packages/basicsr/data/degradations.py" +OLD_STRING="from torchvision.transforms.functional_tensor import rgb_to_grayscale" +NEW_STRING="from torchvision.transforms.functional import rgb_to_grayscale" +BACKUP_EXTENSION=".bak" # Extension for the backup file + +# --- Script Start --- +echo "--------------------------------------------------------" +echo "Starting patch script for basicsr/data/degradations.py" +echo "--------------------------------------------------------" +echo "Target file: $FILE_PATH" + +# 1. Check if the target file exists +if [ ! -f "$FILE_PATH" ]; then + echo "Error: Target file not found at '$FILE_PATH'." + echo "Please ensure basicsr is installed for Python 3.10." + echo "Exiting." + exit 1 +fi + +# 2. Check if the original string exists in the file +# Using grep -q for quiet mode, -F for fixed string match +if ! grep -qF "$OLD_STRING" "$FILE_PATH"; then + echo "The original string to be replaced was not found in '$FILE_PATH'." + echo "It's possible the file has already been patched or has a different version." + echo "No changes made. Exiting gracefully." + exit 0 +fi + +# 3. Create a backup of the original file +BACKUP_FILE="${FILE_PATH}${BACKUP_EXTENSION}" +echo "Creating a backup of the original file to: $BACKUP_FILE" +sudo cp "$FILE_PATH" "$BACKUP_FILE" +if [ $? -ne 0 ]; then + echo "Error: Failed to create backup file. Please check permissions." + echo "Exiting without making changes." + exit 1 +fi +echo "Backup created successfully." + +# 4. Perform the replacement using sed +echo "Replacing string in '$FILE_PATH'..." +echo " Old: '$OLD_STRING'" +echo " New: '$NEW_STRING'" +echo "(Requires sudo password)" + +# Using '|' as a delimiter for sed's substitute command (s///) +# because the strings contain '/' characters. +# -i flag for in-place editing. +sudo sed -i "s|$OLD_STRING|$NEW_STRING|g" "$FILE_PATH" + +if [ $? -ne 0 ]; then + echo "Error: The 'sed' command failed. Please check permissions or script syntax." + echo "Original file might be corrupted. Check '$BACKUP_FILE'." + echo "Exiting." + exit 1 +fi + +# 5. Verify the replacement +echo "Verification:" +if grep -qF "$NEW_STRING" "$FILE_PATH" && ! grep -qF "$OLD_STRING" "$FILE_PATH"; then + echo " Success! The string has been replaced correctly." + echo " Old string no longer found." + echo " New string found." +else + echo " Warning: Verification failed. The replacement might not be complete or correct." + echo " Please check the file '$FILE_PATH' manually." + echo " You can restore the original file from '$BACKUP_FILE' if needed." + exit 1 +fi + +echo "--------------------------------------------------------" +echo "Patch completed successfully!" +echo "If you encounter any issues, you can restore the file:" +echo "sudo mv \"$BACKUP_FILE\" \"$FILE_PATH\"" +echo "--------------------------------------------------------" + +exit 0 \ No newline at end of file diff --git a/omni/tools/prepare_hy3d.sh b/omni/tools/prepare_hy3d.sh new file mode 100644 index 0000000..03377a0 --- /dev/null +++ b/omni/tools/prepare_hy3d.sh @@ -0,0 +1,4 @@ +export _LLM_SCALER_OMNI_HY3D_PATH="/llm/models/Hunyuan3D-2.1" +python download_hy3d_pbr_model.py +bash replace_cuda_to_xpu.sh /llm/models/Hunyuan3D-2.1/hunyuan3d-paintpbr-v2-1/unet/attn_processor.py +bash fix_degradations.sh \ No newline at end of file diff --git a/omni/workflows/3d_hunyuan3d.json b/omni/workflows/3d_hunyuan3d.json new file mode 100644 index 0000000..1027d0f --- /dev/null +++ b/omni/workflows/3d_hunyuan3d.json @@ -0,0 +1,1450 @@ +{ + "id": "9b0be1ef-2103-481f-8c22-df026fcfedef", + "revision": 0, + "last_node_id": 56, + "last_link_id": 132, + "nodes": [ + { + "id": 31, + "type": "SetNode", + "pos": [ + -1054.7391357421875, + 682.4335327148438 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "STRING", + "type": "STRING", + "link": 36 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_OutputMeshName", + "properties": { + "previousName": "OutputMeshName", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "OutputMeshName" + ] + }, + { + "id": 34, + "type": "GetNode", + "pos": [ + -296.3294982910156, + 523.9727783203125 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 81 + ] + } + ], + "title": "Get_NumberOfFaces", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "NumberOfFaces" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 41, + "type": "GetNode", + "pos": [ + 87.23178100585938, + 519.231201171875 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 85 + ] + } + ], + "title": "Get_OutputMeshName", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "OutputMeshName" + ] + }, + { + "id": 44, + "type": "Hy3D21ExportMesh", + "pos": [ + 334.2273254394531, + 678.4039916992188 + ], + "size": [ + 283.5542907714844, + 106 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "trimesh", + "type": "TRIMESH", + "link": 84 + }, + { + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": 85 + } + ], + "outputs": [ + { + "name": "glb_path", + "type": "STRING", + "links": null + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "b25dbd8bc1adb6e1e699c3a2740d3a2358ee4752", + "Node name for S&R": "Hy3D21ExportMesh", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "3D/Hy3D", + "glb", + true + ] + }, + { + "id": 27, + "type": "Preview3D", + "pos": [ + 685.949951171875, + 1533.4534912109375 + ], + "size": [ + 649.873046875, + 600.7554321289062 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "camera_info", + "shape": 7, + "type": "LOAD3D_CAMERA", + "link": null + }, + { + "name": "model_file", + "type": "STRING", + "widget": { + "name": "model_file" + }, + "link": 119 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "Preview3D", + "widget_ue_connectable": {}, + "Camera Info": { + "position": { + "x": 0.16040607929891837, + "y": 3.7969502133215416, + "z": 4.006855879069825 + }, + "target": { + "x": 0, + "y": 2.5, + "z": 0 + }, + "zoom": 1, + "cameraType": "perspective" + }, + "Last Time Model File": "Hy21_Mesh.glb" + }, + "widgets_values": [ + "Hy21_Mesh.glb", + "" + ] + }, + { + "id": 51, + "type": "PreviewImage", + "pos": [ + 1025.20166015625, + 1194.26611328125 + ], + "size": [ + 298.908935546875, + 277.8708190917969 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 112 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PreviewImage", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 32, + "type": "StringConstant", + "pos": [ + -1394.16357421875, + 686.4265747070312 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 36 + ] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-KJNodes", + "ver": "1.1.2", + "Node name for S&R": "StringConstant", + "cnr_id": "comfyui-kjnodes", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Hy21_Mesh" + ] + }, + { + "id": 4, + "type": "Hy3D21VAELoader", + "pos": [ + -733.929931640625, + 675.7706298828125 + ], + "size": [ + 279.4253845214844, + 58 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "vae_config", + "shape": 7, + "type": "HY3D21VAECONFIG", + "link": null + } + ], + "outputs": [ + { + "name": "vae", + "type": "HY3DVAE", + "links": [ + 61 + ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "e439689e4b67fb2af5f487ee26ef3a710be92658", + "Node name for S&R": "Hy3D21VAELoader", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "hunyuan3d-vae-v2-1.ckpt" + ] + }, + { + "id": 45, + "type": "Hy3D21MeshUVWrap", + "pos": [ + -687.3154296875, + 1183.163818359375 + ], + "size": [ + 239.62557983398438, + 26 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "trimesh", + "type": "TRIMESH", + "link": 86 + } + ], + "outputs": [ + { + "name": "trimesh", + "type": "TRIMESH", + "links": [ + 87 + ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "b25dbd8bc1adb6e1e699c3a2740d3a2358ee4752", + "Node name for S&R": "Hy3D21MeshUVWrap", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 19, + "type": "Hy3D21CameraConfig", + "pos": [ + -791.797119140625, + 1300.020263671875 + ], + "size": [ + 382.7560729980469, + 133.63636779785156 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "camera_config", + "type": "HY3D21CAMERA", + "links": [ + 17, + 19 + ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "e439689e4b67fb2af5f487ee26ef3a710be92658", + "Node name for S&R": "Hy3D21CameraConfig", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "0, 90, 180, 270, 0, 180", + "0, 0, 0, 0, 90, -90", + "1, 0.5, 1, 0.5, 1, 1", + 1.1000000000000003 + ] + }, + { + "id": 30, + "type": "INTConstant", + "pos": [ + -1363.2161865234375, + 531.6890869140625 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 35 + ] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-KJNodes", + "ver": "1.1.2", + "Node name for S&R": "INTConstant", + "cnr_id": "comfyui-kjnodes", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 200000 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 29, + "type": "SetNode", + "pos": [ + -1054.739013671875, + 532.6876220703125 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 35 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_NumberOfFaces", + "properties": { + "previousName": "NumberOfFaces", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "NumberOfFaces" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 14, + "type": "Hy3D21LoadImageWithTransparency", + "pos": [ + -1304.341064453125, + 844.59814453125 + ], + "size": [ + 462.2257385253906, + 601.9747314453125 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [] + }, + { + "name": "mask", + "type": "MASK", + "links": null + }, + { + "name": "image_with_alpha", + "type": "IMAGE", + "links": [ + 125, + 126 + ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "19e77b34abcadf80a782bc34ec447b8b366773d1", + "Node name for S&R": "Hy3D21LoadImageWithTransparency", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "demo.png", + "image" + ] + }, + { + "id": 55, + "type": "UpscaleModelLoader", + "pos": [ + -298.4969177246094, + 1319.7991943359375 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "UPSCALE_MODEL", + "type": "UPSCALE_MODEL", + "links": [ + 120, + 127 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.65", + "Node name for S&R": "UpscaleModelLoader" + }, + "widgets_values": [ + "RealESRGAN_x4plus.pth" + ] + }, + { + "id": 54, + "type": "ImageUpscaleWithModel", + "pos": [ + -185.4017333984375, + 1508.4267578125 + ], + "size": [ + 221.98202514648438, + 46 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "upscale_model", + "type": "UPSCALE_MODEL", + "link": 120 + }, + { + "name": "image", + "type": "IMAGE", + "link": 128 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 129, + 130 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.65", + "Node name for S&R": "ImageUpscaleWithModel" + } + }, + { + "id": 56, + "type": "ImageUpscaleWithModel", + "pos": [ + -196.5100860595703, + 1613.7398681640625 + ], + "size": [ + 221.98202514648438, + 46 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "upscale_model", + "type": "UPSCALE_MODEL", + "link": 127 + }, + { + "name": "image", + "type": "IMAGE", + "link": 131 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 132 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.65", + "Node name for S&R": "ImageUpscaleWithModel" + } + }, + { + "id": 50, + "type": "PreviewImage", + "pos": [ + 714.6932983398438, + 1196.8720703125 + ], + "size": [ + 279.83984375, + 287.4053955078125 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 111 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PreviewImage", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 49, + "type": "Hy3DInPaint", + "pos": [ + 393.9945983886719, + 1476.14453125 + ], + "size": [ + 270.072265625, + 138 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "HY3DPIPELINE", + "link": 103 + }, + { + "name": "albedo", + "type": "NPARRAY", + "link": 104 + }, + { + "name": "albedo_mask", + "type": "NPARRAY", + "link": 105 + }, + { + "name": "mr", + "type": "NPARRAY", + "link": 106 + }, + { + "name": "mr_mask", + "type": "NPARRAY", + "link": 107 + }, + { + "name": "output_mesh_name", + "type": "STRING", + "widget": { + "name": "output_mesh_name" + }, + "link": 108 + } + ], + "outputs": [ + { + "name": "albedo", + "type": "IMAGE", + "links": [ + 111 + ] + }, + { + "name": "mr", + "type": "IMAGE", + "links": [ + 112 + ] + }, + { + "name": "trimesh", + "type": "TRIMESH", + "links": [] + }, + { + "name": "output_glb_path", + "type": "STRING", + "links": [ + 119 + ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "b25dbd8bc1adb6e1e699c3a2740d3a2358ee4752", + "Node name for S&R": "Hy3DInPaint", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "" + ] + }, + { + "id": 33, + "type": "GetNode", + "pos": [ + 180.6302947998047, + 1327.0037841796875 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 108 + ] + } + ], + "title": "Get_OutputMeshName", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "OutputMeshName" + ] + }, + { + "id": 21, + "type": "Hy3DBakeMultiViews", + "pos": [ + 93.5582504272461, + 1478.854248046875 + ], + "size": [ + 247.41854858398438, + 146 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "HY3DPIPELINE", + "link": 18 + }, + { + "name": "camera_config", + "type": "HY3D21CAMERA", + "link": 19 + }, + { + "name": "albedo", + "type": "IMAGE", + "link": 129 + }, + { + "name": "mr", + "type": "IMAGE", + "link": 132 + } + ], + "outputs": [ + { + "name": "pipeline", + "type": "HY3DPIPELINE", + "links": [ + 103 + ] + }, + { + "name": "albedo", + "type": "NPARRAY", + "links": [ + 104 + ] + }, + { + "name": "albedo_mask", + "type": "NPARRAY", + "links": [ + 105 + ] + }, + { + "name": "mr", + "type": "NPARRAY", + "links": [ + 106 + ] + }, + { + "name": "mr_mask", + "type": "NPARRAY", + "links": [ + 107 + ] + }, + { + "name": "albedo_texture", + "type": "IMAGE", + "links": null + }, + { + "name": "mr_texture", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "e439689e4b67fb2af5f487ee26ef3a710be92658", + "Node name for S&R": "Hy3DBakeMultiViews", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 20, + "type": "Hy3DMultiViewsGenerator", + "pos": [ + -591.0275268554688, + 1542.135498046875 + ], + "size": [ + 338.5425720214844, + 322 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "trimesh", + "type": "TRIMESH", + "link": 87 + }, + { + "name": "camera_config", + "type": "HY3D21CAMERA", + "link": 17 + }, + { + "name": "image", + "type": "IMAGE", + "link": 126 + } + ], + "outputs": [ + { + "name": "pipeline", + "type": "HY3DPIPELINE", + "links": [ + 18 + ] + }, + { + "name": "albedo", + "type": "IMAGE", + "links": [ + 128 + ] + }, + { + "name": "mr", + "type": "IMAGE", + "links": [ + 131 + ] + }, + { + "name": "positions", + "type": "IMAGE", + "links": null + }, + { + "name": "normals", + "type": "IMAGE", + "links": null + }, + { + "name": "camera_config", + "type": "HY3D21CAMERA", + "links": null + }, + { + "name": "metadata", + "type": "HY3D21METADATA", + "links": null + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "19e77b34abcadf80a782bc34ec447b8b366773d1", + "Node name for S&R": "Hy3DMultiViewsGenerator", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 768, + 15, + 3, + 4096, + false, + 834564175769408, + "randomize" + ] + }, + { + "id": 43, + "type": "Hy3D21PostprocessMesh", + "pos": [ + -61.86928176879883, + 679.3651733398438 + ], + "size": [ + 341.3726501464844, + 154 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "trimesh", + "type": "TRIMESH", + "link": 80 + }, + { + "name": "max_facenum", + "type": "INT", + "widget": { + "name": "max_facenum" + }, + "link": 81 + } + ], + "outputs": [ + { + "name": "trimesh", + "type": "TRIMESH", + "links": [ + 84, + 86 + ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "b25dbd8bc1adb6e1e699c3a2740d3a2358ee4752", + "Node name for S&R": "Hy3D21PostprocessMesh", + "widget_ue_connectable": {} + }, + "widgets_values": [ + true, + true, + true, + 40000, + false + ] + }, + { + "id": 37, + "type": "Hy3DMeshGenerator", + "pos": [ + -746.8765258789062, + 816.26318359375 + ], + "size": [ + 306.1265563964844, + 178 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 125 + } + ], + "outputs": [ + { + "name": "latents", + "type": "HY3DLATENT", + "links": [ + 62 + ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "f82cfc7e7198169fd39b4ffee510ef34ce842076", + "Node name for S&R": "Hy3DMeshGenerator", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "hunyuan3d-dit-v2-1.ckpt", + 50, + 5, + 982602593805038, + "randomize", + "sdpa" + ] + }, + { + "id": 9, + "type": "Hy3D21VAEDecode", + "pos": [ + -400.6885681152344, + 674.083740234375 + ], + "size": [ + 288.7496032714844, + 266 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "HY3DVAE", + "link": 61 + }, + { + "name": "latents", + "type": "HY3DLATENT", + "link": 62 + }, + { + "name": "pipeline", + "type": "HY3D21PIPELINE", + "link": null + } + ], + "outputs": [ + { + "name": "trimesh", + "type": "TRIMESH", + "links": [ + 80 + ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Hunyuan3d-2-1", + "ver": "e439689e4b67fb2af5f487ee26ef3a710be92658", + "Node name for S&R": "Hy3D21VAEDecode", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 1.01, + 384, + 8000, + 0, + "mc", + true, + false + ] + }, + { + "id": 46, + "type": "SaveImage", + "pos": [ + -151.80267333984375, + 1741.975341796875 + ], + "size": [ + 270, + 270 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 130 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "SaveImage", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "MV" + ] + } + ], + "links": [ + [ + 17, + 19, + 0, + 20, + 1, + "HY3D21CAMERA" + ], + [ + 18, + 20, + 0, + 21, + 0, + "HY3DPIPELINE" + ], + [ + 19, + 19, + 0, + 21, + 1, + "HY3D21CAMERA" + ], + [ + 35, + 30, + 0, + 29, + 0, + "*" + ], + [ + 36, + 32, + 0, + 31, + 0, + "*" + ], + [ + 61, + 4, + 0, + 9, + 0, + "HY3DVAE" + ], + [ + 62, + 37, + 0, + 9, + 1, + "HY3DLATENT" + ], + [ + 80, + 9, + 0, + 43, + 0, + "TRIMESH" + ], + [ + 81, + 34, + 0, + 43, + 1, + "INT" + ], + [ + 84, + 43, + 0, + 44, + 0, + "TRIMESH" + ], + [ + 85, + 41, + 0, + 44, + 1, + "STRING" + ], + [ + 86, + 43, + 0, + 45, + 0, + "TRIMESH" + ], + [ + 87, + 45, + 0, + 20, + 0, + "TRIMESH" + ], + [ + 103, + 21, + 0, + 49, + 0, + "HY3DPIPELINE" + ], + [ + 104, + 21, + 1, + 49, + 1, + "NPARRAY" + ], + [ + 105, + 21, + 2, + 49, + 2, + "NPARRAY" + ], + [ + 106, + 21, + 3, + 49, + 3, + "NPARRAY" + ], + [ + 107, + 21, + 4, + 49, + 4, + "NPARRAY" + ], + [ + 108, + 33, + 0, + 49, + 5, + "STRING" + ], + [ + 111, + 49, + 0, + 50, + 0, + "IMAGE" + ], + [ + 112, + 49, + 1, + 51, + 0, + "IMAGE" + ], + [ + 119, + 49, + 3, + 27, + 1, + "STRING" + ], + [ + 120, + 55, + 0, + 54, + 0, + "UPSCALE_MODEL" + ], + [ + 125, + 14, + 2, + 37, + 0, + "IMAGE" + ], + [ + 126, + 14, + 2, + 20, + 2, + "IMAGE" + ], + [ + 127, + 55, + 0, + 56, + 0, + "UPSCALE_MODEL" + ], + [ + 128, + 20, + 1, + 54, + 1, + "IMAGE" + ], + [ + 129, + 54, + 0, + 21, + 2, + "IMAGE" + ], + [ + 130, + 54, + 0, + 46, + 0, + "IMAGE" + ], + [ + 131, + 20, + 2, + 56, + 1, + "IMAGE" + ], + [ + 132, + 56, + 0, + 21, + 3, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Mesh Generation", + "bounding": [ + -774.5154418945312, + 445.6306457519531, + 1402.297119140625, + 583.0671997070312 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Mesh Texturing", + "bounding": [ + -793.2161254882812, + 1114.864013671875, + 2147.495849609375, + 1034.4346923828125 + ], + "color": "#8A8", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6934334949441355, + "offset": [ + 1560.2682051816885, + -846.2662614838155 + ] + }, + "frontendVersion": "1.27.10", + "ue_links": [], + "links_added_by_ue": [], + "groupNodes": {}, + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file