diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py index 07937f73d..8b59cfbdc 100644 --- a/comfy/latent_formats.py +++ b/comfy/latent_formats.py @@ -11,21 +11,21 @@ class SD15(LatentFormat): self.scale_factor = scale_factor self.latent_rgb_factors = [ # R G B - [0.298, 0.207, 0.208], # L1 - [0.187, 0.286, 0.173], # L2 - [-0.158, 0.189, 0.264], # L3 - [-0.184, -0.271, -0.473], # L4 + [ 0.3512, 0.2297, 0.3227], + [ 0.3250, 0.4974, 0.2350], + [-0.2829, 0.1762, 0.2721], + [-0.2120, -0.2616, -0.7177] ] self.taesd_decoder_name = "taesd_decoder.pth" class SDXL(LatentFormat): def __init__(self): self.scale_factor = 0.13025 - self.latent_rgb_factors = [ #TODO: these are the factors for SD1.5, need to estimate new ones for SDXL + self.latent_rgb_factors = [ # R G B - [0.298, 0.207, 0.208], # L1 - [0.187, 0.286, 0.173], # L2 - [-0.158, 0.189, 0.264], # L3 - [-0.184, -0.271, -0.473], # L4 + [ 0.3920, 0.4054, 0.4549], + [-0.2634, -0.0196, 0.0653], + [ 0.0568, 0.1687, -0.0755], + [-0.3112, -0.2359, -0.2076] ] self.taesd_decoder_name = "taesdxl_decoder.pth" diff --git a/comfy/model_management.py b/comfy/model_management.py index 09dcaa295..e148408b8 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -238,6 +238,8 @@ def unload_model(): current_loaded_model.model_patches_to(current_loaded_model.offload_device) current_loaded_model.unpatch_model() current_loaded_model = None + if vram_state != VRAMState.HIGH_VRAM: + soft_empty_cache() if vram_state != VRAMState.HIGH_VRAM: if len(current_gpu_controlnets) > 0: diff --git a/comfy/sd.py b/comfy/sd.py index 76eaa5b59..4bc9a15fa 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -206,7 +206,7 @@ class ModelPatcher: def __init__(self, model, load_device, offload_device, size=0): self.size = size self.model = model - self.patches = [] + self.patches = {} self.backup = {} self.model_options = {"transformer_options":{}} self.model_size() @@ -227,7 +227,10 @@ class ModelPatcher: def clone(self): n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size) - n.patches = self.patches[:] + n.patches = {} + for k in self.patches: + n.patches[k] = self.patches[k][:] + n.model_options = copy.deepcopy(self.model_options) n.model_keys = self.model_keys return n @@ -295,12 +298,28 @@ class ModelPatcher: return self.model.get_dtype() def add_patches(self, patches, strength_patch=1.0, strength_model=1.0): - p = {} + p = set() for k in patches: if k in self.model_keys: - p[k] = patches[k] - self.patches += [(strength_patch, p, strength_model)] - return p.keys() + p.add(k) + current_patches = self.patches.get(k, []) + current_patches.append((strength_patch, patches[k], strength_model)) + self.patches[k] = current_patches + + return list(p) + + def get_key_patches(self, filter_prefix=None): + model_sd = self.model_state_dict() + p = {} + for k in model_sd: + if filter_prefix is not None: + if not k.startswith(filter_prefix): + continue + if k in self.patches: + p[k] = [model_sd[k]] + self.patches[k] + else: + p[k] = (model_sd[k],) + return p def model_state_dict(self, filter_prefix=None): sd = self.model.state_dict() @@ -313,85 +332,94 @@ class ModelPatcher: def patch_model(self): model_sd = self.model_state_dict() - for p in self.patches: - for k in p[1]: - v = p[1][k] - key = k - if key not in model_sd: - print("could not patch. key doesn't exist in model:", k) - continue + for key in self.patches: + if key not in model_sd: + print("could not patch. key doesn't exist in model:", k) + continue - weight = model_sd[key] - if key not in self.backup: - self.backup[key] = weight.clone() + weight = model_sd[key] - alpha = p[0] - strength_model = p[2] + if key not in self.backup: + self.backup[key] = weight.clone() - if strength_model != 1.0: - weight *= strength_model + weight[:] = self.calculate_weight(self.patches[key], weight.clone(), key) + return self.model - if len(v) == 1: - w1 = v[0] + def calculate_weight(self, patches, weight, key): + for p in patches: + alpha = p[0] + v = p[1] + strength_model = p[2] + + if strength_model != 1.0: + weight *= strength_model + + if isinstance(v, list): + v = (self.calculate_weight(v[1:], v[0].clone(), key), ) + + if len(v) == 1: + w1 = v[0] + if alpha != 0.0: if w1.shape != weight.shape: print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape)) else: weight += alpha * w1.type(weight.dtype).to(weight.device) - elif len(v) == 4: #lora/locon - mat1 = v[0] - mat2 = v[1] - if v[2] is not None: - alpha *= v[2] / mat2.shape[0] - if v[3] is not None: - #locon mid weights, hopefully the math is fine because I didn't properly test it - final_shape = [mat2.shape[1], mat2.shape[0], v[3].shape[2], v[3].shape[3]] - mat2 = torch.mm(mat2.transpose(0, 1).flatten(start_dim=1).float(), v[3].transpose(0, 1).flatten(start_dim=1).float()).reshape(final_shape).transpose(0, 1) - weight += (alpha * torch.mm(mat1.flatten(start_dim=1).float(), mat2.flatten(start_dim=1).float())).reshape(weight.shape).type(weight.dtype).to(weight.device) - elif len(v) == 8: #lokr - w1 = v[0] - w2 = v[1] - w1_a = v[3] - w1_b = v[4] - w2_a = v[5] - w2_b = v[6] - t2 = v[7] - dim = None + elif len(v) == 4: #lora/locon + mat1 = v[0] + mat2 = v[1] + if v[2] is not None: + alpha *= v[2] / mat2.shape[0] + if v[3] is not None: + #locon mid weights, hopefully the math is fine because I didn't properly test it + final_shape = [mat2.shape[1], mat2.shape[0], v[3].shape[2], v[3].shape[3]] + mat2 = torch.mm(mat2.transpose(0, 1).flatten(start_dim=1).float(), v[3].transpose(0, 1).flatten(start_dim=1).float()).reshape(final_shape).transpose(0, 1) + weight += (alpha * torch.mm(mat1.flatten(start_dim=1).float(), mat2.flatten(start_dim=1).float())).reshape(weight.shape).type(weight.dtype).to(weight.device) + elif len(v) == 8: #lokr + w1 = v[0] + w2 = v[1] + w1_a = v[3] + w1_b = v[4] + w2_a = v[5] + w2_b = v[6] + t2 = v[7] + dim = None - if w1 is None: - dim = w1_b.shape[0] - w1 = torch.mm(w1_a.float(), w1_b.float()) + if w1 is None: + dim = w1_b.shape[0] + w1 = torch.mm(w1_a.float(), w1_b.float()) - if w2 is None: - dim = w2_b.shape[0] - if t2 is None: - w2 = torch.mm(w2_a.float(), w2_b.float()) - else: - w2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float(), w2_b.float(), w2_a.float()) - - if len(w2.shape) == 4: - w1 = w1.unsqueeze(2).unsqueeze(2) - if v[2] is not None and dim is not None: - alpha *= v[2] / dim - - weight += alpha * torch.kron(w1.float(), w2.float()).reshape(weight.shape).type(weight.dtype).to(weight.device) - else: #loha - w1a = v[0] - w1b = v[1] - if v[2] is not None: - alpha *= v[2] / w1b.shape[0] - w2a = v[3] - w2b = v[4] - if v[5] is not None: #cp decomposition - t1 = v[5] - t2 = v[6] - m1 = torch.einsum('i j k l, j r, i p -> p r k l', t1.float(), w1b.float(), w1a.float()) - m2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float(), w2b.float(), w2a.float()) + if w2 is None: + dim = w2_b.shape[0] + if t2 is None: + w2 = torch.mm(w2_a.float(), w2_b.float()) else: - m1 = torch.mm(w1a.float(), w1b.float()) - m2 = torch.mm(w2a.float(), w2b.float()) + w2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float(), w2_b.float(), w2_a.float()) + + if len(w2.shape) == 4: + w1 = w1.unsqueeze(2).unsqueeze(2) + if v[2] is not None and dim is not None: + alpha *= v[2] / dim + + weight += alpha * torch.kron(w1.float(), w2.float()).reshape(weight.shape).type(weight.dtype).to(weight.device) + else: #loha + w1a = v[0] + w1b = v[1] + if v[2] is not None: + alpha *= v[2] / w1b.shape[0] + w2a = v[3] + w2b = v[4] + if v[5] is not None: #cp decomposition + t1 = v[5] + t2 = v[6] + m1 = torch.einsum('i j k l, j r, i p -> p r k l', t1.float(), w1b.float(), w1a.float()) + m2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float(), w2b.float(), w2a.float()) + else: + m1 = torch.mm(w1a.float(), w1b.float()) + m2 = torch.mm(w2a.float(), w2b.float()) + + weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype).to(weight.device) + return weight - weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype).to(weight.device) - return self.model def unpatch_model(self): model_sd = self.model_state_dict() keys = list(self.backup.keys()) diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py index 11f26bb8f..fbd87c569 100644 --- a/comfy/sd1_clip.py +++ b/comfy/sd1_clip.py @@ -233,7 +233,7 @@ def expand_directory_list(directories): dirs.add(root) return list(dirs) -def load_embed(embedding_name, embedding_directory, embedding_size): +def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=None): if isinstance(embedding_directory, str): embedding_directory = [embedding_directory] @@ -292,13 +292,15 @@ def load_embed(embedding_name, embedding_directory, embedding_size): continue out_list.append(t.reshape(-1, t.shape[-1])) embed_out = torch.cat(out_list, dim=0) + elif embed_key is not None and embed_key in embed: + embed_out = embed[embed_key] else: values = embed.values() embed_out = next(iter(values)) return embed_out class SD1Tokenizer: - def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768): + def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768, embedding_key='clip_l'): if tokenizer_path is None: tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_tokenizer") self.tokenizer = CLIPTokenizer.from_pretrained(tokenizer_path) @@ -315,17 +317,18 @@ class SD1Tokenizer: self.max_word_length = 8 self.embedding_identifier = "embedding:" self.embedding_size = embedding_size + self.embedding_key = embedding_key def _try_get_embedding(self, embedding_name:str): ''' Takes a potential embedding name and tries to retrieve it. Returns a Tuple consisting of the embedding and any leftover string, embedding can be None. ''' - embed = load_embed(embedding_name, self.embedding_directory, self.embedding_size) + embed = load_embed(embedding_name, self.embedding_directory, self.embedding_size, self.embedding_key) if embed is None: stripped = embedding_name.strip(',') if len(stripped) < len(embedding_name): - embed = load_embed(stripped, self.embedding_directory, self.embedding_size) + embed = load_embed(stripped, self.embedding_directory, self.embedding_size, self.embedding_key) return (embed, embedding_name[len(stripped):]) return (embed, "") diff --git a/comfy/sdxl_clip.py b/comfy/sdxl_clip.py index f676d8c81..d0803b10b 100644 --- a/comfy/sdxl_clip.py +++ b/comfy/sdxl_clip.py @@ -35,11 +35,13 @@ class SDXLClipG(sd1_clip.SD1ClipModel): def load_sd(self, sd): if "text_projection" in sd: self.text_projection[:] = sd.pop("text_projection") + if "text_projection.weight" in sd: + self.text_projection[:] = sd.pop("text_projection.weight").transpose(0, 1) return super().load_sd(sd) class SDXLClipGTokenizer(sd1_clip.SD1Tokenizer): def __init__(self, tokenizer_path=None, embedding_directory=None): - super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1280) + super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1280, embedding_key='clip_g') class SDXLTokenizer(sd1_clip.SD1Tokenizer): diff --git a/comfy_extras/nodes_model_merging.py b/comfy_extras/nodes_model_merging.py index 9bbb84da4..eae9b6fdb 100644 --- a/comfy_extras/nodes_model_merging.py +++ b/comfy_extras/nodes_model_merging.py @@ -18,9 +18,9 @@ class ModelMergeSimple: def merge(self, model1, model2, ratio): m = model1.clone() - sd = model2.model_state_dict("diffusion_model.") - for k in sd: - m.add_patches({k: (sd[k], )}, 1.0 - ratio, ratio) + kp = model2.get_key_patches("diffusion_model.") + for k in kp: + m.add_patches({k: kp[k]}, 1.0 - ratio, ratio) return (m, ) class ModelMergeBlocks: @@ -39,10 +39,10 @@ class ModelMergeBlocks: def merge(self, model1, model2, **kwargs): m = model1.clone() - sd = model2.model_state_dict("diffusion_model.") + kp = model2.get_key_patches("diffusion_model.") default_ratio = next(iter(kwargs.values())) - for k in sd: + for k in kp: ratio = default_ratio k_unet = k[len("diffusion_model."):] @@ -52,7 +52,7 @@ class ModelMergeBlocks: ratio = kwargs[arg] last_arg_size = len(arg) - m.add_patches({k: (sd[k], )}, 1.0 - ratio, ratio) + m.add_patches({k: kp[k]}, 1.0 - ratio, ratio) return (m, ) class CheckpointSave: diff --git a/latent_preview.py b/latent_preview.py index 1d143339c..833e6822e 100644 --- a/latent_preview.py +++ b/latent_preview.py @@ -3,7 +3,6 @@ from PIL import Image, ImageOps from io import BytesIO import struct import numpy as np - from comfy.cli_args import args, LatentPreviewMethod from comfy.taesd.taesd import TAESD import folder_paths @@ -16,7 +15,13 @@ class LatentPreviewer: def decode_latent_to_preview_image(self, preview_format, x0): preview_image = self.decode_latent_to_preview(x0) - preview_image = ImageOps.contain(preview_image, (MAX_PREVIEW_RESOLUTION, MAX_PREVIEW_RESOLUTION), Image.ANTIALIAS) + + if hasattr(Image, 'Resampling'): + resampling = Image.Resampling.BILINEAR + else: + resampling = Image.ANTIALIAS + + preview_image = ImageOps.contain(preview_image, (MAX_PREVIEW_RESOLUTION, MAX_PREVIEW_RESOLUTION), resampling) preview_type = 1 if preview_format == "JPEG": diff --git a/web/scripts/app.js b/web/scripts/app.js index 09310c7f8..bf424058f 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -368,7 +368,11 @@ export class ComfyApp { shiftY = w.last_y; if (w.computeSize) { shiftY += w.computeSize()[1] + 4; - } else { + } + else if(w.computedHeight) { + shiftY += w.computedHeight; + } + else { shiftY += LiteGraph.NODE_WIDGET_HEIGHT + 4; } } else { diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js index dfa26aef4..4fc5dc199 100644 --- a/web/scripts/widgets.js +++ b/web/scripts/widgets.js @@ -129,6 +129,7 @@ function addMultilineWidget(node, name, opts, app) { w.y = y; if (w.type === "customtext") { y += freeSpace; + w.computedHeight = freeSpace - multi.length*4; } else if (w.computeSize) { y += w.computeSize()[1] + 4; } else { @@ -162,11 +163,12 @@ function addMultilineWidget(node, name, opts, app) { .multiplySelf(ctx.getTransform()) .translateSelf(margin, margin + y); + const scale = new DOMMatrix().scaleSelf(transform.a, transform.d) Object.assign(this.inputEl.style, { transformOrigin: "0 0", - transform: transform, - left: "0px", - top: "0px", + transform: scale, + left: `${transform.a + transform.e}px`, + top: `${transform.d + transform.f}px`, width: `${widgetWidth - (margin * 2)}px`, height: `${this.parent.inputHeight - (margin * 2)}px`, position: "absolute",