From cf6e1efb69a17cdaad509e665ad53b0e031b4807 Mon Sep 17 00:00:00 2001 From: freakabcd Date: Tue, 14 May 2024 05:22:22 +1000 Subject: [PATCH 01/20] Show message on error when loading wf from file (works on drag and drop) (#3466) --- web/scripts/app.js | 48 +++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/web/scripts/app.js b/web/scripts/app.js index 7ed262cb2..a516be704 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -2157,6 +2157,14 @@ export class ComfyApp { api.dispatchEvent(new CustomEvent("promptQueued", { detail: { number, batchCount } })); } + showErrorOnFileLoad(file) { + this.ui.dialog.show( + $el("div", [ + $el("p", {textContent: `Unable to find workflow in ${file.name}`}) + ]).outerHTML + ); + } + /** * Loads workflow data from the specified file * @param {File} file @@ -2164,27 +2172,27 @@ export class ComfyApp { async handleFile(file) { if (file.type === "image/png") { const pngInfo = await getPngMetadata(file); - if (pngInfo) { - if (pngInfo.workflow) { - await this.loadGraphData(JSON.parse(pngInfo.workflow)); - } else if (pngInfo.prompt) { - this.loadApiJson(JSON.parse(pngInfo.prompt)); - } else if (pngInfo.parameters) { - importA1111(this.graph, pngInfo.parameters); - } + if (pngInfo?.workflow) { + await this.loadGraphData(JSON.parse(pngInfo.workflow)); + } else if (pngInfo?.prompt) { + this.loadApiJson(JSON.parse(pngInfo.prompt)); + } else if (pngInfo?.parameters) { + importA1111(this.graph, pngInfo.parameters); + } else { + this.showErrorOnFileLoad(file); } } else if (file.type === "image/webp") { const pngInfo = await getWebpMetadata(file); - if (pngInfo) { - if (pngInfo.workflow) { - this.loadGraphData(JSON.parse(pngInfo.workflow)); - } else if (pngInfo.Workflow) { - this.loadGraphData(JSON.parse(pngInfo.Workflow)); // Support loading workflows from that webp custom node. - } else if (pngInfo.prompt) { - this.loadApiJson(JSON.parse(pngInfo.prompt)); - } else if (pngInfo.Prompt) { - this.loadApiJson(JSON.parse(pngInfo.Prompt)); // Support loading prompts from that webp custom node. - } + // Support loading workflows from that webp custom node. + const workflow = pngInfo?.workflow || pngInfo?.Workflow; + const prompt = pngInfo?.prompt || pngInfo?.Prompt; + + if (workflow) { + this.loadGraphData(JSON.parse(workflow)); + } else if (prompt) { + this.loadApiJson(JSON.parse(prompt)); + } else { + this.showErrorOnFileLoad(file); } } else if (file.type === "application/json" || file.name?.endsWith(".json")) { const reader = new FileReader(); @@ -2205,7 +2213,11 @@ export class ComfyApp { await this.loadGraphData(JSON.parse(info.workflow)); } else if (info.prompt) { this.loadApiJson(JSON.parse(info.prompt)); + } else { + this.showErrorOnFileLoad(file); } + } else { + this.showErrorOnFileLoad(file); } } From 2de3b69b3074028fd0f850f801f4ca023489c692 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 13 May 2024 21:54:11 -0400 Subject: [PATCH 02/20] Support saving some more modelspec types. --- comfy_extras/nodes_model_merging.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/comfy_extras/nodes_model_merging.py b/comfy_extras/nodes_model_merging.py index 8c5dc9859..bb15112f4 100644 --- a/comfy_extras/nodes_model_merging.py +++ b/comfy_extras/nodes_model_merging.py @@ -175,9 +175,14 @@ def save_checkpoint(model, clip=None, vae=None, clip_vision=None, filename_prefi enable_modelspec = True if isinstance(model.model, comfy.model_base.SDXL): - metadata["modelspec.architecture"] = "stable-diffusion-xl-v1-base" + if isinstance(model.model, comfy.model_base.SDXL_instructpix2pix): + metadata["modelspec.architecture"] = "stable-diffusion-xl-v1-edit" + else: + metadata["modelspec.architecture"] = "stable-diffusion-xl-v1-base" elif isinstance(model.model, comfy.model_base.SDXLRefiner): metadata["modelspec.architecture"] = "stable-diffusion-xl-v1-refiner" + elif isinstance(model.model, comfy.model_base.SVD_img2vid): + metadata["modelspec.architecture"] = "stable-video-diffusion-img2vid-v1" else: enable_modelspec = False From b0ab31d06c5df98b094d8f38db5cda4e5aec47eb Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 14 May 2024 12:47:31 -0400 Subject: [PATCH 03/20] Refactor attention upcasting code part 1. --- comfy/ldm/modules/attention.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index d51a2fae1..de66db4f8 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -22,9 +22,9 @@ ops = comfy.ops.disable_weight_init # CrossAttn precision handling if args.dont_upcast_attention: logging.info("disabling upcasting of attention") - _ATTN_PRECISION = "fp16" + _ATTN_PRECISION = None else: - _ATTN_PRECISION = "fp32" + _ATTN_PRECISION = torch.float32 def exists(val): @@ -85,7 +85,7 @@ class FeedForward(nn.Module): def Normalize(in_channels, dtype=None, device=None): return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True, dtype=dtype, device=device) -def attention_basic(q, k, v, heads, mask=None): +def attention_basic(q, k, v, heads, mask=None, attn_precision=None): b, _, dim_head = q.shape dim_head //= heads scale = dim_head ** -0.5 @@ -101,7 +101,7 @@ def attention_basic(q, k, v, heads, mask=None): ) # force cast to fp32 to avoid overflowing - if _ATTN_PRECISION =="fp32": + if attn_precision == torch.float32: sim = einsum('b i d, b j d -> b i j', q.float(), k.float()) * scale else: sim = einsum('b i d, b j d -> b i j', q, k) * scale @@ -135,7 +135,7 @@ def attention_basic(q, k, v, heads, mask=None): return out -def attention_sub_quad(query, key, value, heads, mask=None): +def attention_sub_quad(query, key, value, heads, mask=None, attn_precision=None): b, _, dim_head = query.shape dim_head //= heads @@ -146,7 +146,7 @@ def attention_sub_quad(query, key, value, heads, mask=None): key = key.unsqueeze(3).reshape(b, -1, heads, dim_head).permute(0, 2, 3, 1).reshape(b * heads, dim_head, -1) dtype = query.dtype - upcast_attention = _ATTN_PRECISION =="fp32" and query.dtype != torch.float32 + upcast_attention = attn_precision == torch.float32 and query.dtype != torch.float32 if upcast_attention: bytes_per_token = torch.finfo(torch.float32).bits//8 else: @@ -195,7 +195,7 @@ def attention_sub_quad(query, key, value, heads, mask=None): hidden_states = hidden_states.unflatten(0, (-1, heads)).transpose(1,2).flatten(start_dim=2) return hidden_states -def attention_split(q, k, v, heads, mask=None): +def attention_split(q, k, v, heads, mask=None, attn_precision=None): b, _, dim_head = q.shape dim_head //= heads scale = dim_head ** -0.5 @@ -214,10 +214,12 @@ def attention_split(q, k, v, heads, mask=None): mem_free_total = model_management.get_free_memory(q.device) - if _ATTN_PRECISION =="fp32": + if attn_precision == torch.float32: element_size = 4 + upcast = True else: element_size = q.element_size() + upcast = False gb = 1024 ** 3 tensor_size = q.shape[0] * q.shape[1] * k.shape[1] * element_size @@ -251,7 +253,7 @@ def attention_split(q, k, v, heads, mask=None): slice_size = q.shape[1] // steps if (q.shape[1] % steps) == 0 else q.shape[1] for i in range(0, q.shape[1], slice_size): end = i + slice_size - if _ATTN_PRECISION =="fp32": + if upcast: with torch.autocast(enabled=False, device_type = 'cuda'): s1 = einsum('b i d, b j d -> b i j', q[:, i:end].float(), k.float()) * scale else: @@ -302,7 +304,7 @@ try: except: pass -def attention_xformers(q, k, v, heads, mask=None): +def attention_xformers(q, k, v, heads, mask=None, attn_precision=None): b, _, dim_head = q.shape dim_head //= heads if BROKEN_XFORMERS: @@ -334,7 +336,7 @@ def attention_xformers(q, k, v, heads, mask=None): ) return out -def attention_pytorch(q, k, v, heads, mask=None): +def attention_pytorch(q, k, v, heads, mask=None, attn_precision=None): b, _, dim_head = q.shape dim_head //= heads q, k, v = map( @@ -409,9 +411,9 @@ class CrossAttention(nn.Module): v = self.to_v(context) if mask is None: - out = optimized_attention(q, k, v, self.heads) + out = optimized_attention(q, k, v, self.heads, attn_precision=_ATTN_PRECISION) else: - out = optimized_attention_masked(q, k, v, self.heads, mask) + out = optimized_attention_masked(q, k, v, self.heads, mask, attn_precision=_ATTN_PRECISION) return self.to_out(out) From bb4940d837f0cfd338ff64776b084303be066c67 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 14 May 2024 15:18:00 -0400 Subject: [PATCH 04/20] Only enable attention upcasting on models that actually need it. --- README.md | 6 ---- comfy/cli_args.py | 1 - comfy/ldm/modules/attention.py | 28 ++++++++----------- .../modules/diffusionmodules/openaimodel.py | 4 ++- comfy/supported_models.py | 12 ++++++++ 5 files changed, 27 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 312468a98..80de21bcd 100644 --- a/README.md +++ b/README.md @@ -207,12 +207,6 @@ To use a textual inversion concepts/embeddings in a text prompt put them in the ```embedding:embedding_filename.pt``` -## How to increase generation speed? - -On non Nvidia hardware you can set this command line setting to disable the upcasting to fp32 in some cross attention operations which will increase your speed. Note that this will very likely give you black images on SD2.x models. If you use xformers or pytorch attention this option does not do anything. - -```--dont-upcast-attention``` - ## How to show high-quality previews? Use ```--preview-method auto``` to enable previews. diff --git a/comfy/cli_args.py b/comfy/cli_args.py index 569c79380..2759f4e94 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -51,7 +51,6 @@ cm_group = parser.add_mutually_exclusive_group() cm_group.add_argument("--cuda-malloc", action="store_true", help="Enable cudaMallocAsync (enabled by default for torch 2.0 and up).") cm_group.add_argument("--disable-cuda-malloc", action="store_true", help="Disable cudaMallocAsync.") -parser.add_argument("--dont-upcast-attention", action="store_true", help="Disable upcasting of attention. Can boost speed but increase the chances of black images.") fp_group = parser.add_mutually_exclusive_group() fp_group.add_argument("--force-fp32", action="store_true", help="Force fp32 (If this makes your GPU work better please report it).") diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index de66db4f8..2515bac5e 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -19,14 +19,6 @@ from comfy.cli_args import args import comfy.ops ops = comfy.ops.disable_weight_init -# CrossAttn precision handling -if args.dont_upcast_attention: - logging.info("disabling upcasting of attention") - _ATTN_PRECISION = None -else: - _ATTN_PRECISION = torch.float32 - - def exists(val): return val is not None @@ -386,10 +378,11 @@ def optimized_attention_for_device(device, mask=False, small_input=False): class CrossAttention(nn.Module): - def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=ops): + def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., attn_precision=None, dtype=None, device=None, operations=ops): super().__init__() inner_dim = dim_head * heads context_dim = default(context_dim, query_dim) + self.attn_precision = attn_precision self.heads = heads self.dim_head = dim_head @@ -411,15 +404,15 @@ class CrossAttention(nn.Module): v = self.to_v(context) if mask is None: - out = optimized_attention(q, k, v, self.heads, attn_precision=_ATTN_PRECISION) + out = optimized_attention(q, k, v, self.heads, attn_precision=self.attn_precision) else: - out = optimized_attention_masked(q, k, v, self.heads, mask, attn_precision=_ATTN_PRECISION) + out = optimized_attention_masked(q, k, v, self.heads, mask, attn_precision=self.attn_precision) return self.to_out(out) class BasicTransformerBlock(nn.Module): def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True, ff_in=False, inner_dim=None, - disable_self_attn=False, disable_temporal_crossattention=False, switch_temporal_ca_to_sa=False, dtype=None, device=None, operations=ops): + disable_self_attn=False, disable_temporal_crossattention=False, switch_temporal_ca_to_sa=False, attn_precision=None, dtype=None, device=None, operations=ops): super().__init__() self.ff_in = ff_in or inner_dim is not None @@ -434,7 +427,7 @@ class BasicTransformerBlock(nn.Module): self.disable_self_attn = disable_self_attn self.attn1 = CrossAttention(query_dim=inner_dim, heads=n_heads, dim_head=d_head, dropout=dropout, - context_dim=context_dim if self.disable_self_attn else None, dtype=dtype, device=device, operations=operations) # is a self-attention if not self.disable_self_attn + context_dim=context_dim if self.disable_self_attn else None, attn_precision=attn_precision, dtype=dtype, device=device, operations=operations) # is a self-attention if not self.disable_self_attn self.ff = FeedForward(inner_dim, dim_out=dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device, operations=operations) if disable_temporal_crossattention: @@ -448,7 +441,7 @@ class BasicTransformerBlock(nn.Module): context_dim_attn2 = context_dim self.attn2 = CrossAttention(query_dim=inner_dim, context_dim=context_dim_attn2, - heads=n_heads, dim_head=d_head, dropout=dropout, dtype=dtype, device=device, operations=operations) # is self-attn if context is none + heads=n_heads, dim_head=d_head, dropout=dropout, attn_precision=attn_precision, dtype=dtype, device=device, operations=operations) # is self-attn if context is none self.norm2 = operations.LayerNorm(inner_dim, dtype=dtype, device=device) self.norm1 = operations.LayerNorm(inner_dim, dtype=dtype, device=device) @@ -588,7 +581,7 @@ class SpatialTransformer(nn.Module): def __init__(self, in_channels, n_heads, d_head, depth=1, dropout=0., context_dim=None, disable_self_attn=False, use_linear=False, - use_checkpoint=True, dtype=None, device=None, operations=ops): + use_checkpoint=True, attn_precision=None, dtype=None, device=None, operations=ops): super().__init__() if exists(context_dim) and not isinstance(context_dim, list): context_dim = [context_dim] * depth @@ -606,7 +599,7 @@ class SpatialTransformer(nn.Module): self.transformer_blocks = nn.ModuleList( [BasicTransformerBlock(inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim[d], - disable_self_attn=disable_self_attn, checkpoint=use_checkpoint, dtype=dtype, device=device, operations=operations) + disable_self_attn=disable_self_attn, checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=dtype, device=device, operations=operations) for d in range(depth)] ) if not use_linear: @@ -662,6 +655,7 @@ class SpatialVideoTransformer(SpatialTransformer): disable_self_attn=False, disable_temporal_crossattention=False, max_time_embed_period: int = 10000, + attn_precision=None, dtype=None, device=None, operations=ops ): super().__init__( @@ -674,6 +668,7 @@ class SpatialVideoTransformer(SpatialTransformer): context_dim=context_dim, use_linear=use_linear, disable_self_attn=disable_self_attn, + attn_precision=attn_precision, dtype=dtype, device=device, operations=operations ) self.time_depth = time_depth @@ -703,6 +698,7 @@ class SpatialVideoTransformer(SpatialTransformer): inner_dim=time_mix_inner_dim, disable_self_attn=disable_self_attn, disable_temporal_crossattention=disable_temporal_crossattention, + attn_precision=attn_precision, dtype=dtype, device=device, operations=operations ) for _ in range(self.depth) diff --git a/comfy/ldm/modules/diffusionmodules/openaimodel.py b/comfy/ldm/modules/diffusionmodules/openaimodel.py index d782eff31..1f5a4ded2 100644 --- a/comfy/ldm/modules/diffusionmodules/openaimodel.py +++ b/comfy/ldm/modules/diffusionmodules/openaimodel.py @@ -431,6 +431,7 @@ class UNetModel(nn.Module): video_kernel_size=None, disable_temporal_crossattention=False, max_ddpm_temb_period=10000, + attn_precision=None, device=None, operations=ops, ): @@ -550,13 +551,14 @@ class UNetModel(nn.Module): disable_self_attn=disable_self_attn, disable_temporal_crossattention=disable_temporal_crossattention, max_time_embed_period=max_ddpm_temb_period, + attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations ) else: return SpatialTransformer( ch, num_heads, dim_head, depth=depth, context_dim=context_dim, disable_self_attn=disable_self_attn, use_linear=use_linear_in_transformer, - use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations + use_checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations ) def get_resblock( diff --git a/comfy/supported_models.py b/comfy/supported_models.py index b3b69e05b..6ca32e8ee 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -65,6 +65,12 @@ class SD20(supported_models_base.BASE): "use_temporal_attention": False, } + unet_extra_config = { + "num_heads": -1, + "num_head_channels": 64, + "attn_precision": torch.float32, + } + latent_format = latent_formats.SD15 def model_type(self, state_dict, prefix=""): @@ -276,6 +282,12 @@ class SVD_img2vid(supported_models_base.BASE): "use_temporal_resblock": True } + unet_extra_config = { + "num_heads": -1, + "num_head_channels": 64, + "attn_precision": torch.float32, + } + clip_vision_prefix = "conditioner.embedders.0.open_clip.model.visual." latent_format = latent_formats.SD15 From ec6f16adb607fa8d14b26670106e1a09d8401e20 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 14 May 2024 18:02:27 -0400 Subject: [PATCH 05/20] Fix SAG. --- comfy/ldm/modules/attention.py | 6 ++++-- comfy_extras/nodes_sag.py | 10 +++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index 2515bac5e..1d5cf0da5 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -420,6 +420,7 @@ class BasicTransformerBlock(nn.Module): inner_dim = dim self.is_res = inner_dim == dim + self.attn_precision = attn_precision if self.ff_in: self.norm_in = operations.LayerNorm(dim, dtype=dtype, device=device) @@ -427,7 +428,7 @@ class BasicTransformerBlock(nn.Module): self.disable_self_attn = disable_self_attn self.attn1 = CrossAttention(query_dim=inner_dim, heads=n_heads, dim_head=d_head, dropout=dropout, - context_dim=context_dim if self.disable_self_attn else None, attn_precision=attn_precision, dtype=dtype, device=device, operations=operations) # is a self-attention if not self.disable_self_attn + context_dim=context_dim if self.disable_self_attn else None, attn_precision=self.attn_precision, dtype=dtype, device=device, operations=operations) # is a self-attention if not self.disable_self_attn self.ff = FeedForward(inner_dim, dim_out=dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device, operations=operations) if disable_temporal_crossattention: @@ -441,7 +442,7 @@ class BasicTransformerBlock(nn.Module): context_dim_attn2 = context_dim self.attn2 = CrossAttention(query_dim=inner_dim, context_dim=context_dim_attn2, - heads=n_heads, dim_head=d_head, dropout=dropout, attn_precision=attn_precision, dtype=dtype, device=device, operations=operations) # is self-attn if context is none + heads=n_heads, dim_head=d_head, dropout=dropout, attn_precision=self.attn_precision, dtype=dtype, device=device, operations=operations) # is self-attn if context is none self.norm2 = operations.LayerNorm(inner_dim, dtype=dtype, device=device) self.norm1 = operations.LayerNorm(inner_dim, dtype=dtype, device=device) @@ -471,6 +472,7 @@ class BasicTransformerBlock(nn.Module): extra_options["n_heads"] = self.n_heads extra_options["dim_head"] = self.d_head + extra_options["attn_precision"] = self.attn_precision if self.ff_in: x_skip = x diff --git a/comfy_extras/nodes_sag.py b/comfy_extras/nodes_sag.py index 69084e91d..8d786db57 100644 --- a/comfy_extras/nodes_sag.py +++ b/comfy_extras/nodes_sag.py @@ -5,12 +5,12 @@ import math from einops import rearrange, repeat import os -from comfy.ldm.modules.attention import optimized_attention, _ATTN_PRECISION +from comfy.ldm.modules.attention import optimized_attention import comfy.samplers # from comfy/ldm/modules/attention.py # but modified to return attention scores as well as output -def attention_basic_with_sim(q, k, v, heads, mask=None): +def attention_basic_with_sim(q, k, v, heads, mask=None, attn_precision=None): b, _, dim_head = q.shape dim_head //= heads scale = dim_head ** -0.5 @@ -26,7 +26,7 @@ def attention_basic_with_sim(q, k, v, heads, mask=None): ) # force cast to fp32 to avoid overflowing - if _ATTN_PRECISION =="fp32": + if attn_precision == torch.float32: sim = einsum('b i d, b j d -> b i j', q.float(), k.float()) * scale else: sim = einsum('b i d, b j d -> b i j', q, k) * scale @@ -121,13 +121,13 @@ class SelfAttentionGuidance: if 1 in cond_or_uncond: uncond_index = cond_or_uncond.index(1) # do the entire attention operation, but save the attention scores to attn_scores - (out, sim) = attention_basic_with_sim(q, k, v, heads=heads) + (out, sim) = attention_basic_with_sim(q, k, v, heads=heads, attn_precision=extra_options["attn_precision"]) # when using a higher batch size, I BELIEVE the result batch dimension is [uc1, ... ucn, c1, ... cn] n_slices = heads * b attn_scores = sim[n_slices * uncond_index:n_slices * (uncond_index+1)] return out else: - return optimized_attention(q, k, v, heads=heads) + return optimized_attention(q, k, v, heads=heads, attn_precision=extra_options["attn_precision"]) def post_cfg_function(args): nonlocal attn_scores From 2d4164271634476627aae31fbec251ca748a0ae0 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 15 May 2024 02:40:06 -0400 Subject: [PATCH 06/20] Fix lowvram dora issue. --- comfy/model_patcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index c38b2f79b..35ede5eef 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -16,7 +16,7 @@ def apply_weight_decompose(dora_scale, weight): .transpose(0, 1) ) - return weight * (dora_scale / weight_norm) + return weight * (dora_scale / weight_norm).type(weight.dtype) def set_model_options_patch_replace(model_options, patch, name, block_name, number, transformer_index=None): to = model_options["transformer_options"].copy() From 58f8388020ba6ab5a913beb742a6312914d640b2 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 16 May 2024 00:11:01 -0400 Subject: [PATCH 07/20] More proper fix for #3484. --- folder_paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/folder_paths.py b/folder_paths.py index 489795002..234b73409 100644 --- a/folder_paths.py +++ b/folder_paths.py @@ -258,7 +258,7 @@ def get_save_image_path(filename_prefix, output_dir, image_width=0, image_height raise Exception(err) try: - counter = max(filter(lambda a: a[1][:-1] == filename and a[1][-1] == "_", map(map_filename, os.listdir(full_output_folder))))[0] + 1 + counter = max(filter(lambda a: os.path.normcase(a[1][:-1]) == os.path.normcase(filename) and a[1][-1] == "_", map(map_filename, os.listdir(full_output_folder))))[0] + 1 except ValueError: counter = 1 except FileNotFoundError: From 46daf0a9a7c02e7652e60f3a7af898e2f6594c2e Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 16 May 2024 04:09:41 -0400 Subject: [PATCH 08/20] Add debug options to force on and off attention upcasting. --- comfy/cli_args.py | 5 +++++ comfy/ldm/modules/attention.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/comfy/cli_args.py b/comfy/cli_args.py index 2759f4e94..b8ac9bc69 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -95,6 +95,11 @@ attn_group.add_argument("--use-pytorch-cross-attention", action="store_true", he parser.add_argument("--disable-xformers", action="store_true", help="Disable xformers.") +upcast = parser.add_mutually_exclusive_group() +upcast.add_argument("--force-upcast-attention", action="store_true", help="Force enable attention upcasting, please report if it fixes black images.") +upcast.add_argument("--dont-upcast-attention", action="store_true", help="Disable all upcasting of attention. Should be unnecessary except for debugging.") + + vram_group = parser.add_mutually_exclusive_group() vram_group.add_argument("--gpu-only", action="store_true", help="Store and run everything (text encoders/CLIP models, etc... on the GPU).") vram_group.add_argument("--highvram", action="store_true", help="By default models will be unloaded to CPU memory after being used. This option keeps them in GPU memory.") diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index 1d5cf0da5..426530867 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -19,6 +19,14 @@ from comfy.cli_args import args import comfy.ops ops = comfy.ops.disable_weight_init + +def get_attn_precision(attn_precision): + if args.dont_upcast_attention: + return None + if attn_precision is None and args.force_upcast_attention: + return torch.float32 + return attn_precision + def exists(val): return val is not None @@ -78,6 +86,8 @@ def Normalize(in_channels, dtype=None, device=None): return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True, dtype=dtype, device=device) def attention_basic(q, k, v, heads, mask=None, attn_precision=None): + attn_precision = get_attn_precision(attn_precision) + b, _, dim_head = q.shape dim_head //= heads scale = dim_head ** -0.5 @@ -128,6 +138,8 @@ def attention_basic(q, k, v, heads, mask=None, attn_precision=None): def attention_sub_quad(query, key, value, heads, mask=None, attn_precision=None): + attn_precision = get_attn_precision(attn_precision) + b, _, dim_head = query.shape dim_head //= heads @@ -188,6 +200,8 @@ def attention_sub_quad(query, key, value, heads, mask=None, attn_precision=None) return hidden_states def attention_split(q, k, v, heads, mask=None, attn_precision=None): + attn_precision = get_attn_precision(attn_precision) + b, _, dim_head = q.shape dim_head //= heads scale = dim_head ** -0.5 From 19300655ddaeb1287a2ecf427cf64c0766e8a999 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 17 May 2024 00:31:32 -0400 Subject: [PATCH 09/20] Don't automatically switch to lowvram mode on GPUs with low memory. --- comfy/model_management.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index 7b54b2567..21ae8d291 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -119,10 +119,6 @@ def get_total_memory(dev=None, torch_total_too=False): total_vram = get_total_memory(get_torch_device()) / (1024 * 1024) total_ram = psutil.virtual_memory().total / (1024 * 1024) logging.info("Total VRAM {:0.0f} MB, total RAM {:0.0f} MB".format(total_vram, total_ram)) -if not args.normalvram and not args.cpu: - if lowvram_available and total_vram <= 4096: - logging.warning("Trying to enable lowvram mode because your GPU seems to have 4GB or less. If you don't want this use: --normalvram") - set_vram_to = VRAMState.LOW_VRAM try: OOM_EXCEPTION = torch.cuda.OutOfMemoryError @@ -451,9 +447,7 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False): model_size = loaded_model.model_memory_required(torch_dev) current_free_mem = get_free_memory(torch_dev) lowvram_model_memory = int(max(64 * (1024 * 1024), (current_free_mem - 1024 * (1024 * 1024)) / 1.3 )) - if model_size > (current_free_mem - inference_memory): #only switch to lowvram if really necessary - vram_set_state = VRAMState.LOW_VRAM - else: + if model_size <= (current_free_mem - inference_memory): #only switch to lowvram if really necessary lowvram_model_memory = 0 if vram_set_state == VRAMState.NO_VRAM: From 91590adf04e644304267a7a296710b4f7ae80bb2 Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Fri, 17 May 2024 18:16:08 +0100 Subject: [PATCH 10/20] Add webcam node (#3497) * Add webcam node * unused import --- comfy_extras/nodes_webcam.py | 33 ++++++++ nodes.py | 1 + web/extensions/core/webcamCapture.js | 120 +++++++++++++++++++++++++++ 3 files changed, 154 insertions(+) create mode 100644 comfy_extras/nodes_webcam.py create mode 100644 web/extensions/core/webcamCapture.js diff --git a/comfy_extras/nodes_webcam.py b/comfy_extras/nodes_webcam.py new file mode 100644 index 000000000..32a0ba2f6 --- /dev/null +++ b/comfy_extras/nodes_webcam.py @@ -0,0 +1,33 @@ +import nodes +import folder_paths + +MAX_RESOLUTION = nodes.MAX_RESOLUTION + + +class WebcamCapture(nodes.LoadImage): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": ("WEBCAM", {}), + "width": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}), + "height": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}), + "capture_on_queue": ("BOOLEAN", {"default": True}), + } + } + RETURN_TYPES = ("IMAGE",) + FUNCTION = "load_capture" + + CATEGORY = "image" + + def load_capture(s, image, **kwargs): + return super().load_image(folder_paths.get_annotated_filepath(image)) + + +NODE_CLASS_MAPPINGS = { + "WebcamCapture": WebcamCapture, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "WebcamCapture": "Webcam Capture", +} \ No newline at end of file diff --git a/nodes.py b/nodes.py index 37e7d7345..34821ca3f 100644 --- a/nodes.py +++ b/nodes.py @@ -1961,6 +1961,7 @@ def init_custom_nodes(): "nodes_align_your_steps.py", "nodes_attention_multiply.py", "nodes_advanced_samplers.py", + "nodes_webcam.py", ] import_failed = [] diff --git a/web/extensions/core/webcamCapture.js b/web/extensions/core/webcamCapture.js new file mode 100644 index 000000000..ea1f597e2 --- /dev/null +++ b/web/extensions/core/webcamCapture.js @@ -0,0 +1,120 @@ +import { app } from "../../scripts/app.js"; +import { api } from "../../scripts/api.js"; + +const WEBCAM_READY = Symbol(); + +app.registerExtension({ + name: "Comfy.WebcamCapture", + getCustomWidgets(app) { + return { + WEBCAM(node, inputName) { + let res; + node[WEBCAM_READY] = new Promise((resolve) => (res = resolve)); + + const container = document.createElement("div"); + container.style.background = "rgba(0,0,0,0.25)"; + container.style.textAlign = "center"; + + const video = document.createElement("video"); + video.style.height = video.style.width = "100%"; + + const loadVideo = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false }); + container.replaceChildren(video); + + setTimeout(() => res(video), 500); // Fallback as loadedmetadata doesnt fire sometimes? + video.addEventListener("loadedmetadata", () => res(video), false); + video.srcObject = stream; + video.play(); + } catch (error) { + const label = document.createElement("div"); + label.style.color = "red"; + label.style.overflow = "auto"; + label.style.maxHeight = "100%"; + label.style.whiteSpace = "pre-wrap"; + label.textContent = "Unable to load webcam, please ensure access is granted:\n" + error.message; + container.replaceChildren(label); + } + }; + + loadVideo(); + + return { widget: node.addDOMWidget(inputName, "WEBCAM", container) }; + }, + }; + }, + nodeCreated(node) { + if ((node.type, node.constructor.comfyClass !== "WebcamCapture")) return; + + let video; + const camera = node.widgets.find((w) => w.name === "image"); + const w = node.widgets.find((w) => w.name === "width"); + const h = node.widgets.find((w) => w.name === "height"); + const captureOnQueue = node.widgets.find((w) => w.name === "capture_on_queue"); + + const canvas = document.createElement("canvas"); + + const capture = () => { + canvas.width = w.value; + canvas.height = h.value; + const ctx = canvas.getContext("2d"); + ctx.drawImage(video, 0, 0, w.value, h.value); + const data = canvas.toDataURL("image/png"); + + const img = new Image(); + img.onload = () => { + node.imgs = [img]; + app.graph.setDirtyCanvas(true); + requestAnimationFrame(() => { + node.setSizeForImage?.(); + }); + }; + img.src = data; + }; + + const btn = node.addWidget("button", "waiting for camera...", "capture", capture); + btn.disabled = true; + btn.serializeValue = () => undefined; + + camera.serializeValue = async () => { + if (captureOnQueue.value) { + capture(); + } else if (!node.imgs?.length) { + const err = `No webcam image captured`; + alert(err); + throw new Error(err); + } + + // Upload image to temp storage + const blob = await new Promise((r) => canvas.toBlob(r)); + const name = `${+new Date()}.png`; + const file = new File([blob], name); + const body = new FormData(); + body.append("image", file); + body.append("subfolder", "webcam"); + body.append("type", "temp"); + const resp = await api.fetchApi("/upload/image", { + method: "POST", + body, + }); + if (resp.status !== 200) { + const err = `Error uploading camera image: ${resp.status} - ${resp.statusText}`; + alert(err); + throw new Error(err); + } + return `webcam/${name} [temp]`; + }; + + node[WEBCAM_READY].then((v) => { + video = v; + // If width isnt specified then use video output resolution + if (!w.value) { + w.value = video.videoWidth || 640; + h.value = video.videoHeight || 480; + } + btn.disabled = false; + btn.label = "capture"; + }); + }, +}); From 1c4af5918a5ffb022606e56ab2d31e510dcccec2 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 17 May 2024 14:02:09 -0400 Subject: [PATCH 11/20] Better error message if the webcam node doesn't work. --- web/extensions/core/webcamCapture.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/web/extensions/core/webcamCapture.js b/web/extensions/core/webcamCapture.js index ea1f597e2..dd5725bd4 100644 --- a/web/extensions/core/webcamCapture.js +++ b/web/extensions/core/webcamCapture.js @@ -33,7 +33,13 @@ app.registerExtension({ label.style.overflow = "auto"; label.style.maxHeight = "100%"; label.style.whiteSpace = "pre-wrap"; - label.textContent = "Unable to load webcam, please ensure access is granted:\n" + error.message; + + if (window.isSecureContext) { + label.textContent = "Unable to load webcam, please ensure access is granted:\n" + error.message; + } else { + label.textContent = "Unable to load webcam. A secure context is required, if you are not accessing ComfyUI on localhost (127.0.0.1) you will have to enable TLS (https)\n\n" + error.message; + } + container.replaceChildren(label); } }; From 98f828fad98643d4f8f31e355cc79a1fd42e1bb8 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 18 May 2024 09:36:26 -0400 Subject: [PATCH 12/20] Remove unnecessary code. --- comfy/ldm/modules/attention.py | 11 ++--------- comfy/ldm/modules/diffusionmodules/model.py | 1 - 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index 426530867..88ee2f32d 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -318,11 +318,7 @@ def attention_xformers(q, k, v, heads, mask=None, attn_precision=None): return attention_pytorch(q, k, v, heads, mask) q, k, v = map( - lambda t: t.unsqueeze(3) - .reshape(b, -1, heads, dim_head) - .permute(0, 2, 1, 3) - .reshape(b * heads, -1, dim_head) - .contiguous(), + lambda t: t.reshape(b, -1, heads, dim_head), (q, k, v), ) @@ -335,10 +331,7 @@ def attention_xformers(q, k, v, heads, mask=None, attn_precision=None): out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=mask) out = ( - out.unsqueeze(0) - .reshape(b, heads, -1, dim_head) - .permute(0, 2, 1, 3) - .reshape(b, -1, heads * dim_head) + out.reshape(b, -1, heads * dim_head) ) return out diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.py index fabc5c5e5..04eb83b21 100644 --- a/comfy/ldm/modules/diffusionmodules/model.py +++ b/comfy/ldm/modules/diffusionmodules/model.py @@ -3,7 +3,6 @@ import math import torch import torch.nn as nn import numpy as np -from einops import rearrange from typing import Optional, Any import logging From 0bdc2b15c75426af75a326d5966ad47aab5b76d3 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 18 May 2024 10:11:44 -0400 Subject: [PATCH 13/20] Cleanup. --- comfy/ldm/modules/attention.py | 10 +++------- comfy/ldm/modules/diffusionmodules/openaimodel.py | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index 88ee2f32d..aa74b6323 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -6,7 +6,7 @@ from einops import rearrange, repeat from typing import Optional, Any import logging -from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding +from .diffusionmodules.util import AlphaBlender, timestep_embedding from .sub_quadratic_attention import efficient_dot_product_attention from comfy import model_management @@ -454,15 +454,11 @@ class BasicTransformerBlock(nn.Module): self.norm1 = operations.LayerNorm(inner_dim, dtype=dtype, device=device) self.norm3 = operations.LayerNorm(inner_dim, dtype=dtype, device=device) - self.checkpoint = checkpoint self.n_heads = n_heads self.d_head = d_head self.switch_temporal_ca_to_sa = switch_temporal_ca_to_sa def forward(self, x, context=None, transformer_options={}): - return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint) - - def _forward(self, x, context=None, transformer_options={}): extra_options = {} block = transformer_options.get("block", None) block_index = transformer_options.get("block_index", 0) @@ -629,7 +625,7 @@ class SpatialTransformer(nn.Module): x = self.norm(x) if not self.use_linear: x = self.proj_in(x) - x = rearrange(x, 'b c h w -> b (h w) c').contiguous() + x = x.movedim(1, -1).flatten(1, 2).contiguous() if self.use_linear: x = self.proj_in(x) for i, block in enumerate(self.transformer_blocks): @@ -637,7 +633,7 @@ class SpatialTransformer(nn.Module): x = block(x, context=context[i], transformer_options=transformer_options) if self.use_linear: x = self.proj_out(x) - x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w).contiguous() + x = x.reshape(x.shape[0], h, w, x.shape[-1]).movedim(-1, 1).contiguous() if not self.use_linear: x = self.proj_out(x) return x + x_in diff --git a/comfy/ldm/modules/diffusionmodules/openaimodel.py b/comfy/ldm/modules/diffusionmodules/openaimodel.py index 1f5a4ded2..ba8fc2c4a 100644 --- a/comfy/ldm/modules/diffusionmodules/openaimodel.py +++ b/comfy/ldm/modules/diffusionmodules/openaimodel.py @@ -258,7 +258,7 @@ class ResBlock(TimestepBlock): else: if emb_out is not None: if self.exchange_temb_dims: - emb_out = rearrange(emb_out, "b t c ... -> b c t ...") + emb_out = emb_out.movedim(1, 2) h = h + emb_out h = self.out_layers(h) return self.skip_connection(x) + h From f37a47110bef97186b8392830d5c49148a06ecef Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 19 May 2024 11:45:36 -0400 Subject: [PATCH 14/20] Make --preview-method auto default to the fast latent2rgb previews. --- latent_preview.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/latent_preview.py b/latent_preview.py index 4dbcbf455..12927647a 100644 --- a/latent_preview.py +++ b/latent_preview.py @@ -63,8 +63,6 @@ def get_previewer(device, latent_format): if method == LatentPreviewMethod.Auto: method = LatentPreviewMethod.Latent2RGB - if taesd_decoder_path: - method = LatentPreviewMethod.TAESD if method == LatentPreviewMethod.TAESD: if taesd_decoder_path: From 4ae1515f14d338a347fdf8f647e062f5bc17d196 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 19 May 2024 17:42:35 -0400 Subject: [PATCH 15/20] Slightly faster latent2rgb previews. --- latent_preview.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/latent_preview.py b/latent_preview.py index 12927647a..05d750e2c 100644 --- a/latent_preview.py +++ b/latent_preview.py @@ -37,12 +37,13 @@ class Latent2RGBPreviewer(LatentPreviewer): self.latent_rgb_factors = torch.tensor(latent_rgb_factors, device="cpu") def decode_latent_to_preview(self, x0): - latent_image = x0[0].permute(1, 2, 0).cpu() @ self.latent_rgb_factors + self.latent_rgb_factors = self.latent_rgb_factors.to(dtype=x0.dtype, device=x0.device) + latent_image = x0[0].permute(1, 2, 0) @ self.latent_rgb_factors latents_ubyte = (((latent_image + 1) / 2) .clamp(0, 1) # change scale from -1..1 to 0..1 .mul(0xFF) # to 0..255 - .byte()).cpu() + ).to(device="cpu", dtype=torch.uint8, non_blocking=True) return Image.fromarray(latents_ubyte.numpy()) From 11a2ad5110a96ac5895e889d455c227d1b80dd30 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 19 May 2024 17:58:03 -0400 Subject: [PATCH 16/20] Fix controlnet not upcasting on models that have it enabled. --- comfy/cldm/cldm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/comfy/cldm/cldm.py b/comfy/cldm/cldm.py index 5eee5a51c..28076dd92 100644 --- a/comfy/cldm/cldm.py +++ b/comfy/cldm/cldm.py @@ -52,6 +52,7 @@ class ControlNet(nn.Module): adm_in_channels=None, transformer_depth_middle=None, transformer_depth_output=None, + attn_precision=None, device=None, operations=comfy.ops.disable_weight_init, **kwargs, @@ -202,7 +203,7 @@ class ControlNet(nn.Module): SpatialTransformer( ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim, disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer, - use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations + use_checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations ) ) self.input_blocks.append(TimestepEmbedSequential(*layers)) @@ -262,7 +263,7 @@ class ControlNet(nn.Module): mid_block += [SpatialTransformer( # always uses a self-attn ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim, disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer, - use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations + use_checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations ), ResBlock( ch, From 09e069ae6ca700cabb885662a83a05227b4f79a5 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 20 May 2024 06:22:29 -0400 Subject: [PATCH 17/20] Log the pytorch version. --- comfy/model_management.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/comfy/model_management.py b/comfy/model_management.py index 21ae8d291..b21928628 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -120,6 +120,11 @@ total_vram = get_total_memory(get_torch_device()) / (1024 * 1024) total_ram = psutil.virtual_memory().total / (1024 * 1024) logging.info("Total VRAM {:0.0f} MB, total RAM {:0.0f} MB".format(total_vram, total_ram)) +try: + logging.info("pytorch version: {}".format(torch.version.__version__)) +except: + pass + try: OOM_EXCEPTION = torch.cuda.OutOfMemoryError except: From 4bc1884478a14dea2d64a34f9beb1c52a2d16b09 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" <128333288+ltdrdata@users.noreply.github.com> Date: Mon, 20 May 2024 19:58:46 +0900 Subject: [PATCH 18/20] Provide a better error message when attempting to execute the workflow with a missing node. (#3517) --- execution.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/execution.py b/execution.py index 47d58b9d7..86bde1c46 100644 --- a/execution.py +++ b/execution.py @@ -622,8 +622,17 @@ def full_type_name(klass): def validate_prompt(prompt): outputs = set() for x in prompt: + if 'class_type' not in prompt[x]: + error = { + "type": "invalid_prompt", + "message": f"Cannot execute due to a missing node", + "details": f"Node ID '#{x}'", + "extra_info": {} + } + return (False, error, [], []) + class_ = nodes.NODE_CLASS_MAPPINGS[prompt[x]['class_type']] - if hasattr(class_, 'OUTPUT_NODE') and class_.OUTPUT_NODE == True: + if hasattr(class_, 'OUTPUT_NODE') and class_.OUTPUT_NODE is True: outputs.add(x) if len(outputs) == 0: From 276f8fce9f5a80b500947fb5745a4dde9e84622d Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 20 May 2024 07:03:06 -0400 Subject: [PATCH 19/20] Print error when node is missing. --- execution.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/execution.py b/execution.py index 86bde1c46..76225a962 100644 --- a/execution.py +++ b/execution.py @@ -625,13 +625,23 @@ def validate_prompt(prompt): if 'class_type' not in prompt[x]: error = { "type": "invalid_prompt", - "message": f"Cannot execute due to a missing node", + "message": f"Cannot execute because a node is missing the class_type property.", + "details": f"Node ID '#{x}'", + "extra_info": {} + } + return (False, error, [], []) + + class_type = prompt[x]['class_type'] + class_ = nodes.NODE_CLASS_MAPPINGS.get(class_type, None) + if class_ is None: + error = { + "type": "invalid_prompt", + "message": f"Cannot execute because node {class_type} does not exist.", "details": f"Node ID '#{x}'", "extra_info": {} } return (False, error, [], []) - class_ = nodes.NODE_CLASS_MAPPINGS[prompt[x]['class_type']] if hasattr(class_, 'OUTPUT_NODE') and class_.OUTPUT_NODE is True: outputs.add(x) From 1900e5119f70d6db0677fe91194050be3c4476c4 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 20 May 2024 08:19:54 -0400 Subject: [PATCH 20/20] Fix potential issue. --- comfy/ldm/modules/attention.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index aa74b6323..74a2fd99a 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -625,7 +625,7 @@ class SpatialTransformer(nn.Module): x = self.norm(x) if not self.use_linear: x = self.proj_in(x) - x = x.movedim(1, -1).flatten(1, 2).contiguous() + x = x.movedim(1, 3).flatten(1, 2).contiguous() if self.use_linear: x = self.proj_in(x) for i, block in enumerate(self.transformer_blocks): @@ -633,7 +633,7 @@ class SpatialTransformer(nn.Module): x = block(x, context=context[i], transformer_options=transformer_options) if self.use_linear: x = self.proj_out(x) - x = x.reshape(x.shape[0], h, w, x.shape[-1]).movedim(-1, 1).contiguous() + x = x.reshape(x.shape[0], h, w, x.shape[-1]).movedim(3, 1).contiguous() if not self.use_linear: x = self.proj_out(x) return x + x_in