From e5577ac9fd04905c7c63b8dc45c09b1520e0c7cd Mon Sep 17 00:00:00 2001 From: CrazyBoyM Date: Mon, 5 Aug 2024 20:38:43 +0800 Subject: [PATCH] fix hunyuandit controlnet --- comfy/controlnet.py | 1 - comfy/ldm/hydit/controlnet.py | 2 +- comfy/ldm/hydit/models.py | 2 -- comfy_extras/nodes_hunyuan.py | 2 -- 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/comfy/controlnet.py b/comfy/controlnet.py index 386012632..9c10bd3e1 100644 --- a/comfy/controlnet.py +++ b/comfy/controlnet.py @@ -472,7 +472,6 @@ def load_controlnet_hunyuandit(controlnet_data): logging.debug("unexpected controlnet keys: {}".format(unexpected)) latent_format = comfy.latent_formats.HunyuanDit() - # latent_format.shift_factor = 0 #SD3 controlnet weirdness control = ControlNetWarperHunyuanDiT(control_model, compression_ratio=1, latent_format=latent_format, load_device=load_device, manual_cast_dtype=manual_cast_dtype) return control diff --git a/comfy/ldm/hydit/controlnet.py b/comfy/ldm/hydit/controlnet.py index 1c3c8b1cd..b149744fc 100644 --- a/comfy/ldm/hydit/controlnet.py +++ b/comfy/ldm/hydit/controlnet.py @@ -226,13 +226,13 @@ class HunYuanControlNet(nn.Module): _, _, oh, ow = x.shape th, tw = oh // self.patch_size, ow // self.patch_size + # Get image RoPE embedding according to `reso`lution. freqs_cis_img = calc_rope(x, self.patch_size, self.hidden_size // self.num_heads) # (cos_cis_img, sin_cis_img) # ========================= Build time and image embedding ========================= t = self.t_embedder(t, dtype=self.dtype) x = self.x_embedder(x) - # Get image RoPE embedding according to `reso`lution. # ========================= Concatenate all extra vectors ========================= # Build text tokens with pooling diff --git a/comfy/ldm/hydit/models.py b/comfy/ldm/hydit/models.py index 552e49904..9a1f3733f 100644 --- a/comfy/ldm/hydit/models.py +++ b/comfy/ldm/hydit/models.py @@ -372,8 +372,6 @@ class HunYuanDiT(nn.Module): if layer > self.depth // 2: if controls is not None: skip = skips.pop() + controls.pop() - print(f"###### skip sum: {skip.sum()} ######") - # breakpoint() else: skip = skips.pop() x = block(x, c, text_states, freqs_cis_img, skip) # (N, L, D) diff --git a/comfy_extras/nodes_hunyuan.py b/comfy_extras/nodes_hunyuan.py index 0b40d9464..babb2e80c 100644 --- a/comfy_extras/nodes_hunyuan.py +++ b/comfy_extras/nodes_hunyuan.py @@ -57,8 +57,6 @@ class ControlNetApplyAdvancedHunYuan: c_net = cnets[prev_cnet] else: c_net = control_net.copy().set_cond_hint(control_hint, strength, (start_percent, end_percent), vae) - # c_net.set_extra_arg('encoder_hidden_states_t5', d.get('conditioning_mt5xl', None)) - # c_net.set_extra_arg('text_embedding_mask_t5', d.get('attention_mask_mt5xl', None)) c_net.set_previous_controlnet(prev_cnet) cnets[prev_cnet] = c_net