From 0a6b0081176c6233015ec00d004c534c088ddcb0 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 4 Aug 2024 10:03:33 -0400 Subject: [PATCH 1/8] Fix issue with some custom nodes. --- comfy/model_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/model_base.py b/comfy/model_base.py index 94f4d333c..d19f5697a 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -84,6 +84,7 @@ class BaseModel(torch.nn.Module): if comfy.model_management.force_channels_last(): self.diffusion_model.to(memory_format=torch.channels_last) logging.debug("using channels last mode for diffusion model") + logging.info("model weight dtype {}, manual cast: {}".format(self.get_dtype(), self.manual_cast_dtype)) self.model_type = model_type self.model_sampling = model_sampling(model_config, model_type) @@ -94,7 +95,6 @@ class BaseModel(torch.nn.Module): self.concat_keys = () logging.info("model_type {}".format(model_type.name)) logging.debug("adm {}".format(self.adm_channels)) - logging.info("model weight dtype {}, manual cast: {}".format(self.get_dtype(), self.manual_cast_dtype)) self.memory_usage_factor = model_config.memory_usage_factor def apply_model(self, x, t, c_concat=None, c_crossattn=None, control=None, transformer_options={}, **kwargs): From 3b71f84b5051905be8f3311abeb39d725743d15b Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 4 Aug 2024 15:45:43 -0400 Subject: [PATCH 2/8] ONNX tracing fixes. --- comfy/ldm/aura/mmdit.py | 6 ++---- comfy/ldm/common_dit.py | 8 ++++++++ comfy/ldm/flux/model.py | 8 +++----- comfy/ldm/modules/diffusionmodules/mmdit.py | 5 ++--- comfy/model_detection.py | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 comfy/ldm/common_dit.py diff --git a/comfy/ldm/aura/mmdit.py b/comfy/ldm/aura/mmdit.py index 9956d3638..cd9a42185 100644 --- a/comfy/ldm/aura/mmdit.py +++ b/comfy/ldm/aura/mmdit.py @@ -9,6 +9,7 @@ import torch.nn.functional as F from comfy.ldm.modules.attention import optimized_attention import comfy.ops +import comfy.ldm.common_dit def modulate(x, shift, scale): return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) @@ -407,10 +408,7 @@ class MMDiT(nn.Module): def patchify(self, x): B, C, H, W = x.size() - pad_h = (self.patch_size - H % self.patch_size) % self.patch_size - pad_w = (self.patch_size - W % self.patch_size) % self.patch_size - - x = torch.nn.functional.pad(x, (0, pad_w, 0, pad_h), mode='circular') + x = comfy.ldm.common_dit.pad_to_patch_size(x, (self.patch_size, self.patch_size)) x = x.view( B, C, diff --git a/comfy/ldm/common_dit.py b/comfy/ldm/common_dit.py new file mode 100644 index 000000000..990025521 --- /dev/null +++ b/comfy/ldm/common_dit.py @@ -0,0 +1,8 @@ +import torch + +def pad_to_patch_size(img, patch_size=(2, 2), padding_mode="circular"): + if padding_mode == "circular" and torch.jit.is_tracing() or torch.jit.is_scripting(): + padding_mode = "reflect" + pad_h = (patch_size[0] - img.shape[-2] % patch_size[0]) % patch_size[0] + pad_w = (patch_size[1] - img.shape[-1] % patch_size[1]) % patch_size[1] + return torch.nn.functional.pad(img, (0, pad_w, 0, pad_h), mode=padding_mode) diff --git a/comfy/ldm/flux/model.py b/comfy/ldm/flux/model.py index e7931c16d..db6cf3d22 100644 --- a/comfy/ldm/flux/model.py +++ b/comfy/ldm/flux/model.py @@ -15,6 +15,7 @@ from .layers import ( ) from einops import rearrange, repeat +import comfy.ldm.common_dit @dataclass class FluxParams: @@ -42,7 +43,7 @@ class Flux(nn.Module): self.dtype = dtype params = FluxParams(**kwargs) self.params = params - self.in_channels = params.in_channels + self.in_channels = params.in_channels * 2 * 2 self.out_channels = self.in_channels if params.hidden_size % params.num_heads != 0: raise ValueError( @@ -125,10 +126,7 @@ class Flux(nn.Module): def forward(self, x, timestep, context, y, guidance, **kwargs): bs, c, h, w = x.shape patch_size = 2 - pad_h = (patch_size - h % 2) % patch_size - pad_w = (patch_size - w % 2) % patch_size - - x = torch.nn.functional.pad(x, (0, pad_w, 0, pad_h), mode='circular') + x = comfy.ldm.common_dit.pad_to_patch_size(x, (patch_size, patch_size)) img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch_size, pw=patch_size) diff --git a/comfy/ldm/modules/diffusionmodules/mmdit.py b/comfy/ldm/modules/diffusionmodules/mmdit.py index ea1b5aa05..491a58a20 100644 --- a/comfy/ldm/modules/diffusionmodules/mmdit.py +++ b/comfy/ldm/modules/diffusionmodules/mmdit.py @@ -9,6 +9,7 @@ from .. import attention from einops import rearrange, repeat from .util import timestep_embedding import comfy.ops +import comfy.ldm.common_dit def default(x, y): if x is not None: @@ -111,9 +112,7 @@ class PatchEmbed(nn.Module): # f"Input width ({W}) should be divisible by patch size ({self.patch_size[1]})." # ) if self.dynamic_img_pad: - pad_h = (self.patch_size[0] - H % self.patch_size[0]) % self.patch_size[0] - pad_w = (self.patch_size[1] - W % self.patch_size[1]) % self.patch_size[1] - x = torch.nn.functional.pad(x, (0, pad_w, 0, pad_h), mode=self.padding_mode) + x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size, padding_mode=self.padding_mode) x = self.proj(x) if self.flatten: x = x.flatten(2).transpose(1, 2) # NCHW -> NLC diff --git a/comfy/model_detection.py b/comfy/model_detection.py index dda9797b7..c47119686 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -131,7 +131,7 @@ def detect_unet_config(state_dict, key_prefix): if '{}double_blocks.0.img_attn.norm.key_norm.scale'.format(key_prefix) in state_dict_keys: #Flux dit_config = {} dit_config["image_model"] = "flux" - dit_config["in_channels"] = 64 + dit_config["in_channels"] = 16 dit_config["vec_in_dim"] = 768 dit_config["context_in_dim"] = 4096 dit_config["hidden_size"] = 3072 From ddb6a9f47cd2e680aa821f320d52e909f0a03fc3 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 4 Aug 2024 15:59:02 -0400 Subject: [PATCH 3/8] Set the step in EmptySD3LatentImage to 16. These models work better when the res is a multiple of 16. --- comfy_extras/nodes_sd3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comfy_extras/nodes_sd3.py b/comfy_extras/nodes_sd3.py index 0aafa2426..ae9b85981 100644 --- a/comfy_extras/nodes_sd3.py +++ b/comfy_extras/nodes_sd3.py @@ -27,8 +27,8 @@ class EmptySD3LatentImage: @classmethod def INPUT_TYPES(s): - return {"required": { "width": ("INT", {"default": 1024, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}), - "height": ("INT", {"default": 1024, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}), + return {"required": { "width": ("INT", {"default": 1024, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}), + "height": ("INT", {"default": 1024, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}), "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096})}} RETURN_TYPES = ("LATENT",) FUNCTION = "generate" From 7afa985fbafc15b2b603a4428917f4a600560699 Mon Sep 17 00:00:00 2001 From: Silver <65376327+silveroxides@users.noreply.github.com> Date: Sun, 4 Aug 2024 23:10:02 +0200 Subject: [PATCH 4/8] Correct spelling 'token_weight_pars_t5' to 'token_weight_pairs_t5' (#4200) --- comfy/text_encoders/flux.py | 4 ++-- comfy/text_encoders/sd3_clip.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/comfy/text_encoders/flux.py b/comfy/text_encoders/flux.py index 0590741bb..ee26f560d 100644 --- a/comfy/text_encoders/flux.py +++ b/comfy/text_encoders/flux.py @@ -52,9 +52,9 @@ class FluxClipModel(torch.nn.Module): def encode_token_weights(self, token_weight_pairs): token_weight_pairs_l = token_weight_pairs["l"] - token_weight_pars_t5 = token_weight_pairs["t5xxl"] + token_weight_pairs_t5 = token_weight_pairs["t5xxl"] - t5_out, t5_pooled = self.t5xxl.encode_token_weights(token_weight_pars_t5) + t5_out, t5_pooled = self.t5xxl.encode_token_weights(token_weight_pairs_t5) l_out, l_pooled = self.clip_l.encode_token_weights(token_weight_pairs_l) return t5_out, l_pooled diff --git a/comfy/text_encoders/sd3_clip.py b/comfy/text_encoders/sd3_clip.py index 143d884cb..549c068e9 100644 --- a/comfy/text_encoders/sd3_clip.py +++ b/comfy/text_encoders/sd3_clip.py @@ -81,7 +81,7 @@ class SD3ClipModel(torch.nn.Module): def encode_token_weights(self, token_weight_pairs): token_weight_pairs_l = token_weight_pairs["l"] token_weight_pairs_g = token_weight_pairs["g"] - token_weight_pars_t5 = token_weight_pairs["t5xxl"] + token_weight_pairs_t5 = token_weight_pairs["t5xxl"] lg_out = None pooled = None out = None @@ -108,7 +108,7 @@ class SD3ClipModel(torch.nn.Module): pooled = torch.cat((l_pooled, g_pooled), dim=-1) if self.t5xxl is not None: - t5_out, t5_pooled = self.t5xxl.encode_token_weights(token_weight_pars_t5) + t5_out, t5_pooled = self.t5xxl.encode_token_weights(token_weight_pairs_t5) if lg_out is not None: out = torch.cat([lg_out, t5_out], dim=-2) else: From 78e133d0415784924cd2674e2ee48f3eeca8a2aa Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 4 Aug 2024 21:59:42 -0400 Subject: [PATCH 5/8] Support simple diffusers Flux loras. --- comfy/lora.py | 8 ++++++++ comfy/utils.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/comfy/lora.py b/comfy/lora.py index fdc128c09..04e8861c9 100644 --- a/comfy/lora.py +++ b/comfy/lora.py @@ -288,4 +288,12 @@ def model_lora_keys_unet(model, key_map={}): key_lora = k[len("diffusion_model."):-len(".weight")] key_map["base_model.model.{}".format(key_lora)] = k #official hunyuan lora format + if isinstance(model, comfy.model_base.Flux): #Diffusers lora Flux + diffusers_keys = comfy.utils.flux_to_diffusers(model.model_config.unet_config, output_prefix="diffusion_model.") + for k in diffusers_keys: + if k.endswith(".weight"): + to = diffusers_keys[k] + key_lora = "transformer.{}".format(k[:-len(".weight")]) #simpletrainer and probably regular diffusers flux lora format + key_map[key_lora] = to + return key_map diff --git a/comfy/utils.py b/comfy/utils.py index 06e09170a..ec7d36607 100644 --- a/comfy/utils.py +++ b/comfy/utils.py @@ -415,6 +415,59 @@ def auraflow_to_diffusers(mmdit_config, output_prefix=""): return key_map +def flux_to_diffusers(mmdit_config, output_prefix=""): + n_double_layers = mmdit_config.get("depth", 0) + n_single_layers = mmdit_config.get("depth_single_blocks", 0) + hidden_size = mmdit_config.get("hidden_size", 0) + + key_map = {} + for index in range(n_double_layers): + prefix_from = "transformer_blocks.{}".format(index) + prefix_to = "{}double_blocks.{}".format(output_prefix, index) + + for end in ("weight", "bias"): + k = "{}.attn.".format(prefix_from) + qkv = "{}.img_attn.qkv.{}".format(prefix_to, end) + key_map["{}to_q.{}".format(k, end)] = (qkv, (0, 0, hidden_size)) + key_map["{}to_k.{}".format(k, end)] = (qkv, (0, hidden_size, hidden_size)) + key_map["{}to_v.{}".format(k, end)] = (qkv, (0, hidden_size * 2, hidden_size)) + + block_map = {"attn.to_out.0.weight": "img_attn.proj.weight", + "attn.to_out.0.bias": "img_attn.proj.bias", + } + + for k in block_map: + key_map["{}.{}".format(prefix_from, k)] = "{}.{}".format(prefix_to, block_map[k]) + + for index in range(n_single_layers): + prefix_from = "single_transformer_blocks.{}".format(index) + prefix_to = "{}single_blocks.{}".format(output_prefix, index) + + for end in ("weight", "bias"): + k = "{}.attn.".format(prefix_from) + qkv = "{}.linear1.{}".format(prefix_to, end) + key_map["{}to_q.{}".format(k, end)] = (qkv, (0, 0, hidden_size)) + key_map["{}to_k.{}".format(k, end)] = (qkv, (0, hidden_size, hidden_size)) + key_map["{}to_v.{}".format(k, end)] = (qkv, (0, hidden_size * 2, hidden_size)) + key_map["{}proj_mlp.{}".format(k, end)] = (qkv, (0, hidden_size * 3, hidden_size)) + + block_map = {#TODO + } + + for k in block_map: + key_map["{}.{}".format(prefix_from, k)] = "{}.{}".format(prefix_to, block_map[k]) + + MAP_BASIC = { #TODO + } + + for k in MAP_BASIC: + if len(k) > 2: + key_map[k[1]] = ("{}{}".format(output_prefix, k[0]), None, k[2]) + else: + key_map[k[1]] = "{}{}".format(output_prefix, k[0]) + + return key_map + def repeat_to_batch_size(tensor, batch_size, dim=0): if tensor.shape[dim] > batch_size: return tensor.narrow(dim, 0, batch_size) From a178e25912b01abf436eba1cfaab316ba02d272d Mon Sep 17 00:00:00 2001 From: a-One-Fan <100067309+a-One-Fan@users.noreply.github.com> Date: Mon, 5 Aug 2024 08:26:20 +0300 Subject: [PATCH 6/8] Fix Flux FP64 math on XPU (#4210) --- comfy/ldm/flux/math.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/ldm/flux/math.py b/comfy/ldm/flux/math.py index 88c2b6bb4..136ce2aa8 100644 --- a/comfy/ldm/flux/math.py +++ b/comfy/ldm/flux/math.py @@ -14,7 +14,7 @@ def attention(q: Tensor, k: Tensor, v: Tensor, pe: Tensor) -> Tensor: def rope(pos: Tensor, dim: int, theta: int) -> Tensor: assert dim % 2 == 0 - if comfy.model_management.is_device_mps(pos.device): + if comfy.model_management.is_device_mps(pos.device) or comfy.model_management.is_intel_xpu(): device = torch.device("cpu") else: device = pos.device From 33e5203a2a7bc90dc4c6577ed645456abc530155 Mon Sep 17 00:00:00 2001 From: bymyself Date: Mon, 5 Aug 2024 09:25:28 -0700 Subject: [PATCH 7/8] Don't cache index.html (#4211) --- server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server.py b/server.py index 23ca2fd33..1c07f9781 100644 --- a/server.py +++ b/server.py @@ -127,7 +127,11 @@ class PromptServer(): @routes.get("/") async def get_root(request): - return web.FileResponse(os.path.join(self.web_root, "index.html")) + response = web.FileResponse(os.path.join(self.web_root, "index.html")) + response.headers['Cache-Control'] = 'no-cache' + response.headers["Pragma"] = "no-cache" + response.headers["Expires"] = "0" + return response @routes.get("/embeddings") def get_embeddings(self): From e545a636baae052000abb1250a69e1cac32b2bae Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 5 Aug 2024 12:31:12 -0400 Subject: [PATCH 8/8] This probably doesn't work anymore. --- README.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/README.md b/README.md index a542ed4d6..d5ded7297 100644 --- a/README.md +++ b/README.md @@ -165,20 +165,6 @@ You can install ComfyUI in Apple Mac silicon (M1 or M2) with any recent macOS ve ```pip install torch-directml``` Then you can launch ComfyUI with: ```python main.py --directml``` -### I already have another UI for Stable Diffusion installed do I really have to install all of these dependencies? - -You don't. If you have another UI installed and working with its own python venv you can use that venv to run ComfyUI. You can open up your favorite terminal and activate it: - -```source path_to_other_sd_gui/venv/bin/activate``` - -or on Windows: - -With Powershell: ```"path_to_other_sd_gui\venv\Scripts\Activate.ps1"``` - -With cmd.exe: ```"path_to_other_sd_gui\venv\Scripts\activate.bat"``` - -And then you can use that terminal to run ComfyUI without installing any dependencies. Note that the venv folder might be called something else depending on the SD UI. - # Running ```python main.py```