From d43cff21055fdd0054acba075cea19740bca69da Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Sat, 8 Jul 2023 01:42:33 +0900 Subject: [PATCH 1/9] bugfix: image widget's was mis-aligned when node has multiline widget --- web/scripts/app.js | 6 +++++- web/scripts/widgets.js | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/web/scripts/app.js b/web/scripts/app.js index 09310c7f8..bf424058f 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -368,7 +368,11 @@ export class ComfyApp { shiftY = w.last_y; if (w.computeSize) { shiftY += w.computeSize()[1] + 4; - } else { + } + else if(w.computedHeight) { + shiftY += w.computedHeight; + } + else { shiftY += LiteGraph.NODE_WIDGET_HEIGHT + 4; } } else { diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js index dfa26aef4..89d4a2e38 100644 --- a/web/scripts/widgets.js +++ b/web/scripts/widgets.js @@ -129,6 +129,7 @@ function addMultilineWidget(node, name, opts, app) { w.y = y; if (w.type === "customtext") { y += freeSpace; + w.computedHeight = freeSpace - multi.length*4; } else if (w.computeSize) { y += w.computeSize()[1] + 4; } else { From 9caab9380de707518b9efc7e4dde8d998fb0b9d9 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" <128333288+ltdrdata@users.noreply.github.com> Date: Sat, 8 Jul 2023 15:36:48 +0900 Subject: [PATCH 2/9] fix: Image.ANTIALIAS is no longer available. (#847) * modify deprecated api call * prevent breaking old Pillow users * change LANCZOS to BILINEAR --- latent_preview.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/latent_preview.py b/latent_preview.py index 1d143339c..833e6822e 100644 --- a/latent_preview.py +++ b/latent_preview.py @@ -3,7 +3,6 @@ from PIL import Image, ImageOps from io import BytesIO import struct import numpy as np - from comfy.cli_args import args, LatentPreviewMethod from comfy.taesd.taesd import TAESD import folder_paths @@ -16,7 +15,13 @@ class LatentPreviewer: def decode_latent_to_preview_image(self, preview_format, x0): preview_image = self.decode_latent_to_preview(x0) - preview_image = ImageOps.contain(preview_image, (MAX_PREVIEW_RESOLUTION, MAX_PREVIEW_RESOLUTION), Image.ANTIALIAS) + + if hasattr(Image, 'Resampling'): + resampling = Image.Resampling.BILINEAR + else: + resampling = Image.ANTIALIAS + + preview_image = ImageOps.contain(preview_image, (MAX_PREVIEW_RESOLUTION, MAX_PREVIEW_RESOLUTION), resampling) preview_type = 1 if preview_format == "JPEG": From a9a4ba7574c06ce9364afaffb09d0435b5ff19c9 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 8 Jul 2023 22:16:40 -0400 Subject: [PATCH 3/9] Fix merging not working when model2 of model merge node was a merge. --- comfy/sd.py | 189 ++++++++++++++++------------ comfy_extras/nodes_model_merging.py | 12 +- 2 files changed, 114 insertions(+), 87 deletions(-) diff --git a/comfy/sd.py b/comfy/sd.py index 76eaa5b59..526dc5311 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -206,7 +206,7 @@ class ModelPatcher: def __init__(self, model, load_device, offload_device, size=0): self.size = size self.model = model - self.patches = [] + self.patches = {} self.backup = {} self.model_options = {"transformer_options":{}} self.model_size() @@ -227,7 +227,10 @@ class ModelPatcher: def clone(self): n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size) - n.patches = self.patches[:] + n.patches = {} + for k in self.patches: + n.patches[k] = self.patches[k][:] + n.model_options = copy.deepcopy(self.model_options) n.model_keys = self.model_keys return n @@ -295,12 +298,28 @@ class ModelPatcher: return self.model.get_dtype() def add_patches(self, patches, strength_patch=1.0, strength_model=1.0): - p = {} + p = set() for k in patches: if k in self.model_keys: - p[k] = patches[k] - self.patches += [(strength_patch, p, strength_model)] - return p.keys() + p.add(k) + current_patches = self.patches.get(k, []) + current_patches.append((strength_patch, patches[k], strength_model)) + self.patches[k] = current_patches + + return list(p) + + def get_key_patches(self, filter_prefix=None): + model_sd = self.model_state_dict() + p = {} + for k in model_sd: + if filter_prefix is not None: + if not k.startswith(filter_prefix): + continue + if k in self.patches: + p[k] = [model_sd[k]] + self.patches[k] + else: + p[k] = (model_sd[k],) + return p def model_state_dict(self, filter_prefix=None): sd = self.model.state_dict() @@ -313,85 +332,93 @@ class ModelPatcher: def patch_model(self): model_sd = self.model_state_dict() - for p in self.patches: - for k in p[1]: - v = p[1][k] - key = k - if key not in model_sd: - print("could not patch. key doesn't exist in model:", k) - continue + for key in self.patches: + if key not in model_sd: + print("could not patch. key doesn't exist in model:", k) + continue - weight = model_sd[key] - if key not in self.backup: - self.backup[key] = weight.clone() + weight = model_sd[key] - alpha = p[0] - strength_model = p[2] + if key not in self.backup: + self.backup[key] = weight.clone() - if strength_model != 1.0: - weight *= strength_model - - if len(v) == 1: - w1 = v[0] - if w1.shape != weight.shape: - print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape)) - else: - weight += alpha * w1.type(weight.dtype).to(weight.device) - elif len(v) == 4: #lora/locon - mat1 = v[0] - mat2 = v[1] - if v[2] is not None: - alpha *= v[2] / mat2.shape[0] - if v[3] is not None: - #locon mid weights, hopefully the math is fine because I didn't properly test it - final_shape = [mat2.shape[1], mat2.shape[0], v[3].shape[2], v[3].shape[3]] - mat2 = torch.mm(mat2.transpose(0, 1).flatten(start_dim=1).float(), v[3].transpose(0, 1).flatten(start_dim=1).float()).reshape(final_shape).transpose(0, 1) - weight += (alpha * torch.mm(mat1.flatten(start_dim=1).float(), mat2.flatten(start_dim=1).float())).reshape(weight.shape).type(weight.dtype).to(weight.device) - elif len(v) == 8: #lokr - w1 = v[0] - w2 = v[1] - w1_a = v[3] - w1_b = v[4] - w2_a = v[5] - w2_b = v[6] - t2 = v[7] - dim = None - - if w1 is None: - dim = w1_b.shape[0] - w1 = torch.mm(w1_a.float(), w1_b.float()) - - if w2 is None: - dim = w2_b.shape[0] - if t2 is None: - w2 = torch.mm(w2_a.float(), w2_b.float()) - else: - w2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float(), w2_b.float(), w2_a.float()) - - if len(w2.shape) == 4: - w1 = w1.unsqueeze(2).unsqueeze(2) - if v[2] is not None and dim is not None: - alpha *= v[2] / dim - - weight += alpha * torch.kron(w1.float(), w2.float()).reshape(weight.shape).type(weight.dtype).to(weight.device) - else: #loha - w1a = v[0] - w1b = v[1] - if v[2] is not None: - alpha *= v[2] / w1b.shape[0] - w2a = v[3] - w2b = v[4] - if v[5] is not None: #cp decomposition - t1 = v[5] - t2 = v[6] - m1 = torch.einsum('i j k l, j r, i p -> p r k l', t1.float(), w1b.float(), w1a.float()) - m2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float(), w2b.float(), w2a.float()) - else: - m1 = torch.mm(w1a.float(), w1b.float()) - m2 = torch.mm(w2a.float(), w2b.float()) - - weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype).to(weight.device) + weight[:] = self.calculate_weight(self.patches[key], weight.clone(), key) return self.model + + def calculate_weight(self, patches, weight, key): + for p in patches: + alpha = p[0] + v = p[1] + strength_model = p[2] + + if strength_model != 1.0: + weight *= strength_model + + if isinstance(v, list): + v = (self.calculate_weight(v[1:], v[0].clone(), key), ) + + if len(v) == 1: + w1 = v[0] + if w1.shape != weight.shape: + print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape)) + else: + weight += alpha * w1.type(weight.dtype).to(weight.device) + elif len(v) == 4: #lora/locon + mat1 = v[0] + mat2 = v[1] + if v[2] is not None: + alpha *= v[2] / mat2.shape[0] + if v[3] is not None: + #locon mid weights, hopefully the math is fine because I didn't properly test it + final_shape = [mat2.shape[1], mat2.shape[0], v[3].shape[2], v[3].shape[3]] + mat2 = torch.mm(mat2.transpose(0, 1).flatten(start_dim=1).float(), v[3].transpose(0, 1).flatten(start_dim=1).float()).reshape(final_shape).transpose(0, 1) + weight += (alpha * torch.mm(mat1.flatten(start_dim=1).float(), mat2.flatten(start_dim=1).float())).reshape(weight.shape).type(weight.dtype).to(weight.device) + elif len(v) == 8: #lokr + w1 = v[0] + w2 = v[1] + w1_a = v[3] + w1_b = v[4] + w2_a = v[5] + w2_b = v[6] + t2 = v[7] + dim = None + + if w1 is None: + dim = w1_b.shape[0] + w1 = torch.mm(w1_a.float(), w1_b.float()) + + if w2 is None: + dim = w2_b.shape[0] + if t2 is None: + w2 = torch.mm(w2_a.float(), w2_b.float()) + else: + w2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float(), w2_b.float(), w2_a.float()) + + if len(w2.shape) == 4: + w1 = w1.unsqueeze(2).unsqueeze(2) + if v[2] is not None and dim is not None: + alpha *= v[2] / dim + + weight += alpha * torch.kron(w1.float(), w2.float()).reshape(weight.shape).type(weight.dtype).to(weight.device) + else: #loha + w1a = v[0] + w1b = v[1] + if v[2] is not None: + alpha *= v[2] / w1b.shape[0] + w2a = v[3] + w2b = v[4] + if v[5] is not None: #cp decomposition + t1 = v[5] + t2 = v[6] + m1 = torch.einsum('i j k l, j r, i p -> p r k l', t1.float(), w1b.float(), w1a.float()) + m2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float(), w2b.float(), w2a.float()) + else: + m1 = torch.mm(w1a.float(), w1b.float()) + m2 = torch.mm(w2a.float(), w2b.float()) + + weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype).to(weight.device) + return weight + def unpatch_model(self): model_sd = self.model_state_dict() keys = list(self.backup.keys()) diff --git a/comfy_extras/nodes_model_merging.py b/comfy_extras/nodes_model_merging.py index 9bbb84da4..eae9b6fdb 100644 --- a/comfy_extras/nodes_model_merging.py +++ b/comfy_extras/nodes_model_merging.py @@ -18,9 +18,9 @@ class ModelMergeSimple: def merge(self, model1, model2, ratio): m = model1.clone() - sd = model2.model_state_dict("diffusion_model.") - for k in sd: - m.add_patches({k: (sd[k], )}, 1.0 - ratio, ratio) + kp = model2.get_key_patches("diffusion_model.") + for k in kp: + m.add_patches({k: kp[k]}, 1.0 - ratio, ratio) return (m, ) class ModelMergeBlocks: @@ -39,10 +39,10 @@ class ModelMergeBlocks: def merge(self, model1, model2, **kwargs): m = model1.clone() - sd = model2.model_state_dict("diffusion_model.") + kp = model2.get_key_patches("diffusion_model.") default_ratio = next(iter(kwargs.values())) - for k in sd: + for k in kp: ratio = default_ratio k_unet = k[len("diffusion_model."):] @@ -52,7 +52,7 @@ class ModelMergeBlocks: ratio = kwargs[arg] last_arg_size = len(arg) - m.add_patches({k: (sd[k], )}, 1.0 - ratio, ratio) + m.add_patches({k: kp[k]}, 1.0 - ratio, ratio) return (m, ) class CheckpointSave: From d3f5998218ee488dd64fc63c8d095850cccab2b9 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 9 Jul 2023 09:33:53 -0400 Subject: [PATCH 4/9] Support loading clip_g from diffusers in CLIP Loader nodes. --- comfy/sdxl_clip.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/comfy/sdxl_clip.py b/comfy/sdxl_clip.py index f676d8c81..d9298b205 100644 --- a/comfy/sdxl_clip.py +++ b/comfy/sdxl_clip.py @@ -35,6 +35,8 @@ class SDXLClipG(sd1_clip.SD1ClipModel): def load_sd(self, sd): if "text_projection" in sd: self.text_projection[:] = sd.pop("text_projection") + if "text_projection.weight" in sd: + self.text_projection[:] = sd.pop("text_projection.weight").transpose(0, 1) return super().load_sd(sd) class SDXLClipGTokenizer(sd1_clip.SD1Tokenizer): From 0ae81c03bb035d3db46b86d48227eed3998872ba Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 9 Jul 2023 09:56:03 -0400 Subject: [PATCH 5/9] Empty cache after model unloading for normal vram and lower. --- comfy/model_management.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/comfy/model_management.py b/comfy/model_management.py index 09dcaa295..e148408b8 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -238,6 +238,8 @@ def unload_model(): current_loaded_model.model_patches_to(current_loaded_model.offload_device) current_loaded_model.unpatch_model() current_loaded_model = None + if vram_state != VRAMState.HIGH_VRAM: + soft_empty_cache() if vram_state != VRAMState.HIGH_VRAM: if len(current_gpu_controlnets) > 0: From d5323d16e0c43e7cac54d334e493b71b4ead3ede Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 9 Jul 2023 13:59:09 -0400 Subject: [PATCH 6/9] latent2rgb matrix for SDXL. --- comfy/latent_formats.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py index 07937f73d..96dd7e948 100644 --- a/comfy/latent_formats.py +++ b/comfy/latent_formats.py @@ -11,10 +11,10 @@ class SD15(LatentFormat): self.scale_factor = scale_factor self.latent_rgb_factors = [ # R G B - [0.298, 0.207, 0.208], # L1 - [0.187, 0.286, 0.173], # L2 - [-0.158, 0.189, 0.264], # L3 - [-0.184, -0.271, -0.473], # L4 + [ 0.3512, 0.2297, 0.3227], + [ 0.3250, 0.4974, 0.2350], + [-0.2829, 0.1762, 0.2721], + [-0.2120, -0.2616, -0.7177] ] self.taesd_decoder_name = "taesd_decoder.pth" @@ -23,9 +23,9 @@ class SDXL(LatentFormat): self.scale_factor = 0.13025 self.latent_rgb_factors = [ #TODO: these are the factors for SD1.5, need to estimate new ones for SDXL # R G B - [0.298, 0.207, 0.208], # L1 - [0.187, 0.286, 0.173], # L2 - [-0.158, 0.189, 0.264], # L3 - [-0.184, -0.271, -0.473], # L4 + [ 0.3920, 0.4054, 0.4549], + [-0.2634, -0.0196, 0.0653], + [ 0.0568, 0.1687, -0.0755], + [-0.3112, -0.2359, -0.2076] ] self.taesd_decoder_name = "taesdxl_decoder.pth" From af15add967a8b1c3ed8060a6cb0056afcc380bee Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 9 Jul 2023 15:41:19 -0400 Subject: [PATCH 7/9] Fix annoyance with textbox unselecting in chromium. --- web/scripts/widgets.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js index 89d4a2e38..4fc5dc199 100644 --- a/web/scripts/widgets.js +++ b/web/scripts/widgets.js @@ -163,11 +163,12 @@ function addMultilineWidget(node, name, opts, app) { .multiplySelf(ctx.getTransform()) .translateSelf(margin, margin + y); + const scale = new DOMMatrix().scaleSelf(transform.a, transform.d) Object.assign(this.inputEl.style, { transformOrigin: "0 0", - transform: transform, - left: "0px", - top: "0px", + transform: scale, + left: `${transform.a + transform.e}px`, + top: `${transform.d + transform.f}px`, width: `${widgetWidth - (margin * 2)}px`, height: `${this.parent.inputHeight - (margin * 2)}px`, position: "absolute", From 6ad0a6d7e2f656af2e0ad501db6d9f5d14ae7add Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 9 Jul 2023 17:46:56 -0400 Subject: [PATCH 8/9] Don't patch weights when multiplier is zero. --- comfy/sd.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/comfy/sd.py b/comfy/sd.py index 526dc5311..4bc9a15fa 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -359,10 +359,11 @@ class ModelPatcher: if len(v) == 1: w1 = v[0] - if w1.shape != weight.shape: - print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape)) - else: - weight += alpha * w1.type(weight.dtype).to(weight.device) + if alpha != 0.0: + if w1.shape != weight.shape: + print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape)) + else: + weight += alpha * w1.type(weight.dtype).to(weight.device) elif len(v) == 4: #lora/locon mat1 = v[0] mat2 = v[1] From 606a5370908858a2b721a3d79cee44f5b73f251b Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 10 Jul 2023 10:28:38 -0400 Subject: [PATCH 9/9] Support SDXL embedding format with 2 CLIP. --- comfy/latent_formats.py | 2 +- comfy/sd1_clip.py | 11 +++++++---- comfy/sdxl_clip.py | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py index 96dd7e948..8b59cfbdc 100644 --- a/comfy/latent_formats.py +++ b/comfy/latent_formats.py @@ -21,7 +21,7 @@ class SD15(LatentFormat): class SDXL(LatentFormat): def __init__(self): self.scale_factor = 0.13025 - self.latent_rgb_factors = [ #TODO: these are the factors for SD1.5, need to estimate new ones for SDXL + self.latent_rgb_factors = [ # R G B [ 0.3920, 0.4054, 0.4549], [-0.2634, -0.0196, 0.0653], diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py index 11f26bb8f..fbd87c569 100644 --- a/comfy/sd1_clip.py +++ b/comfy/sd1_clip.py @@ -233,7 +233,7 @@ def expand_directory_list(directories): dirs.add(root) return list(dirs) -def load_embed(embedding_name, embedding_directory, embedding_size): +def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=None): if isinstance(embedding_directory, str): embedding_directory = [embedding_directory] @@ -292,13 +292,15 @@ def load_embed(embedding_name, embedding_directory, embedding_size): continue out_list.append(t.reshape(-1, t.shape[-1])) embed_out = torch.cat(out_list, dim=0) + elif embed_key is not None and embed_key in embed: + embed_out = embed[embed_key] else: values = embed.values() embed_out = next(iter(values)) return embed_out class SD1Tokenizer: - def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768): + def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768, embedding_key='clip_l'): if tokenizer_path is None: tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_tokenizer") self.tokenizer = CLIPTokenizer.from_pretrained(tokenizer_path) @@ -315,17 +317,18 @@ class SD1Tokenizer: self.max_word_length = 8 self.embedding_identifier = "embedding:" self.embedding_size = embedding_size + self.embedding_key = embedding_key def _try_get_embedding(self, embedding_name:str): ''' Takes a potential embedding name and tries to retrieve it. Returns a Tuple consisting of the embedding and any leftover string, embedding can be None. ''' - embed = load_embed(embedding_name, self.embedding_directory, self.embedding_size) + embed = load_embed(embedding_name, self.embedding_directory, self.embedding_size, self.embedding_key) if embed is None: stripped = embedding_name.strip(',') if len(stripped) < len(embedding_name): - embed = load_embed(stripped, self.embedding_directory, self.embedding_size) + embed = load_embed(stripped, self.embedding_directory, self.embedding_size, self.embedding_key) return (embed, embedding_name[len(stripped):]) return (embed, "") diff --git a/comfy/sdxl_clip.py b/comfy/sdxl_clip.py index d9298b205..d0803b10b 100644 --- a/comfy/sdxl_clip.py +++ b/comfy/sdxl_clip.py @@ -41,7 +41,7 @@ class SDXLClipG(sd1_clip.SD1ClipModel): class SDXLClipGTokenizer(sd1_clip.SD1Tokenizer): def __init__(self, tokenizer_path=None, embedding_directory=None): - super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1280) + super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1280, embedding_key='clip_g') class SDXLTokenizer(sd1_clip.SD1Tokenizer):