mirror of
https://git.datalinker.icu/comfyanonymous/ComfyUI
synced 2026-11-22 04:47:03 +08:00
Merge branch 'comfyanonymous:master' into master
This commit is contained in:
commit
f8311a62ad
@ -11,21 +11,21 @@ class SD15(LatentFormat):
|
|||||||
self.scale_factor = scale_factor
|
self.scale_factor = scale_factor
|
||||||
self.latent_rgb_factors = [
|
self.latent_rgb_factors = [
|
||||||
# R G B
|
# R G B
|
||||||
[0.298, 0.207, 0.208], # L1
|
[ 0.3512, 0.2297, 0.3227],
|
||||||
[0.187, 0.286, 0.173], # L2
|
[ 0.3250, 0.4974, 0.2350],
|
||||||
[-0.158, 0.189, 0.264], # L3
|
[-0.2829, 0.1762, 0.2721],
|
||||||
[-0.184, -0.271, -0.473], # L4
|
[-0.2120, -0.2616, -0.7177]
|
||||||
]
|
]
|
||||||
self.taesd_decoder_name = "taesd_decoder.pth"
|
self.taesd_decoder_name = "taesd_decoder.pth"
|
||||||
|
|
||||||
class SDXL(LatentFormat):
|
class SDXL(LatentFormat):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.scale_factor = 0.13025
|
self.scale_factor = 0.13025
|
||||||
self.latent_rgb_factors = [ #TODO: these are the factors for SD1.5, need to estimate new ones for SDXL
|
self.latent_rgb_factors = [
|
||||||
# R G B
|
# R G B
|
||||||
[0.298, 0.207, 0.208], # L1
|
[ 0.3920, 0.4054, 0.4549],
|
||||||
[0.187, 0.286, 0.173], # L2
|
[-0.2634, -0.0196, 0.0653],
|
||||||
[-0.158, 0.189, 0.264], # L3
|
[ 0.0568, 0.1687, -0.0755],
|
||||||
[-0.184, -0.271, -0.473], # L4
|
[-0.3112, -0.2359, -0.2076]
|
||||||
]
|
]
|
||||||
self.taesd_decoder_name = "taesdxl_decoder.pth"
|
self.taesd_decoder_name = "taesdxl_decoder.pth"
|
||||||
|
|||||||
@ -238,6 +238,8 @@ def unload_model():
|
|||||||
current_loaded_model.model_patches_to(current_loaded_model.offload_device)
|
current_loaded_model.model_patches_to(current_loaded_model.offload_device)
|
||||||
current_loaded_model.unpatch_model()
|
current_loaded_model.unpatch_model()
|
||||||
current_loaded_model = None
|
current_loaded_model = None
|
||||||
|
if vram_state != VRAMState.HIGH_VRAM:
|
||||||
|
soft_empty_cache()
|
||||||
|
|
||||||
if vram_state != VRAMState.HIGH_VRAM:
|
if vram_state != VRAMState.HIGH_VRAM:
|
||||||
if len(current_gpu_controlnets) > 0:
|
if len(current_gpu_controlnets) > 0:
|
||||||
|
|||||||
50
comfy/sd.py
50
comfy/sd.py
@ -206,7 +206,7 @@ class ModelPatcher:
|
|||||||
def __init__(self, model, load_device, offload_device, size=0):
|
def __init__(self, model, load_device, offload_device, size=0):
|
||||||
self.size = size
|
self.size = size
|
||||||
self.model = model
|
self.model = model
|
||||||
self.patches = []
|
self.patches = {}
|
||||||
self.backup = {}
|
self.backup = {}
|
||||||
self.model_options = {"transformer_options":{}}
|
self.model_options = {"transformer_options":{}}
|
||||||
self.model_size()
|
self.model_size()
|
||||||
@ -227,7 +227,10 @@ class ModelPatcher:
|
|||||||
|
|
||||||
def clone(self):
|
def clone(self):
|
||||||
n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size)
|
n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size)
|
||||||
n.patches = self.patches[:]
|
n.patches = {}
|
||||||
|
for k in self.patches:
|
||||||
|
n.patches[k] = self.patches[k][:]
|
||||||
|
|
||||||
n.model_options = copy.deepcopy(self.model_options)
|
n.model_options = copy.deepcopy(self.model_options)
|
||||||
n.model_keys = self.model_keys
|
n.model_keys = self.model_keys
|
||||||
return n
|
return n
|
||||||
@ -295,12 +298,28 @@ class ModelPatcher:
|
|||||||
return self.model.get_dtype()
|
return self.model.get_dtype()
|
||||||
|
|
||||||
def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):
|
def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):
|
||||||
p = {}
|
p = set()
|
||||||
for k in patches:
|
for k in patches:
|
||||||
if k in self.model_keys:
|
if k in self.model_keys:
|
||||||
p[k] = patches[k]
|
p.add(k)
|
||||||
self.patches += [(strength_patch, p, strength_model)]
|
current_patches = self.patches.get(k, [])
|
||||||
return p.keys()
|
current_patches.append((strength_patch, patches[k], strength_model))
|
||||||
|
self.patches[k] = current_patches
|
||||||
|
|
||||||
|
return list(p)
|
||||||
|
|
||||||
|
def get_key_patches(self, filter_prefix=None):
|
||||||
|
model_sd = self.model_state_dict()
|
||||||
|
p = {}
|
||||||
|
for k in model_sd:
|
||||||
|
if filter_prefix is not None:
|
||||||
|
if not k.startswith(filter_prefix):
|
||||||
|
continue
|
||||||
|
if k in self.patches:
|
||||||
|
p[k] = [model_sd[k]] + self.patches[k]
|
||||||
|
else:
|
||||||
|
p[k] = (model_sd[k],)
|
||||||
|
return p
|
||||||
|
|
||||||
def model_state_dict(self, filter_prefix=None):
|
def model_state_dict(self, filter_prefix=None):
|
||||||
sd = self.model.state_dict()
|
sd = self.model.state_dict()
|
||||||
@ -313,26 +332,34 @@ class ModelPatcher:
|
|||||||
|
|
||||||
def patch_model(self):
|
def patch_model(self):
|
||||||
model_sd = self.model_state_dict()
|
model_sd = self.model_state_dict()
|
||||||
for p in self.patches:
|
for key in self.patches:
|
||||||
for k in p[1]:
|
|
||||||
v = p[1][k]
|
|
||||||
key = k
|
|
||||||
if key not in model_sd:
|
if key not in model_sd:
|
||||||
print("could not patch. key doesn't exist in model:", k)
|
print("could not patch. key doesn't exist in model:", k)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
weight = model_sd[key]
|
weight = model_sd[key]
|
||||||
|
|
||||||
if key not in self.backup:
|
if key not in self.backup:
|
||||||
self.backup[key] = weight.clone()
|
self.backup[key] = weight.clone()
|
||||||
|
|
||||||
|
weight[:] = self.calculate_weight(self.patches[key], weight.clone(), key)
|
||||||
|
return self.model
|
||||||
|
|
||||||
|
def calculate_weight(self, patches, weight, key):
|
||||||
|
for p in patches:
|
||||||
alpha = p[0]
|
alpha = p[0]
|
||||||
|
v = p[1]
|
||||||
strength_model = p[2]
|
strength_model = p[2]
|
||||||
|
|
||||||
if strength_model != 1.0:
|
if strength_model != 1.0:
|
||||||
weight *= strength_model
|
weight *= strength_model
|
||||||
|
|
||||||
|
if isinstance(v, list):
|
||||||
|
v = (self.calculate_weight(v[1:], v[0].clone(), key), )
|
||||||
|
|
||||||
if len(v) == 1:
|
if len(v) == 1:
|
||||||
w1 = v[0]
|
w1 = v[0]
|
||||||
|
if alpha != 0.0:
|
||||||
if w1.shape != weight.shape:
|
if w1.shape != weight.shape:
|
||||||
print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
|
print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
|
||||||
else:
|
else:
|
||||||
@ -391,7 +418,8 @@ class ModelPatcher:
|
|||||||
m2 = torch.mm(w2a.float(), w2b.float())
|
m2 = torch.mm(w2a.float(), w2b.float())
|
||||||
|
|
||||||
weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype).to(weight.device)
|
weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype).to(weight.device)
|
||||||
return self.model
|
return weight
|
||||||
|
|
||||||
def unpatch_model(self):
|
def unpatch_model(self):
|
||||||
model_sd = self.model_state_dict()
|
model_sd = self.model_state_dict()
|
||||||
keys = list(self.backup.keys())
|
keys = list(self.backup.keys())
|
||||||
|
|||||||
@ -233,7 +233,7 @@ def expand_directory_list(directories):
|
|||||||
dirs.add(root)
|
dirs.add(root)
|
||||||
return list(dirs)
|
return list(dirs)
|
||||||
|
|
||||||
def load_embed(embedding_name, embedding_directory, embedding_size):
|
def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=None):
|
||||||
if isinstance(embedding_directory, str):
|
if isinstance(embedding_directory, str):
|
||||||
embedding_directory = [embedding_directory]
|
embedding_directory = [embedding_directory]
|
||||||
|
|
||||||
@ -292,13 +292,15 @@ def load_embed(embedding_name, embedding_directory, embedding_size):
|
|||||||
continue
|
continue
|
||||||
out_list.append(t.reshape(-1, t.shape[-1]))
|
out_list.append(t.reshape(-1, t.shape[-1]))
|
||||||
embed_out = torch.cat(out_list, dim=0)
|
embed_out = torch.cat(out_list, dim=0)
|
||||||
|
elif embed_key is not None and embed_key in embed:
|
||||||
|
embed_out = embed[embed_key]
|
||||||
else:
|
else:
|
||||||
values = embed.values()
|
values = embed.values()
|
||||||
embed_out = next(iter(values))
|
embed_out = next(iter(values))
|
||||||
return embed_out
|
return embed_out
|
||||||
|
|
||||||
class SD1Tokenizer:
|
class SD1Tokenizer:
|
||||||
def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768):
|
def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768, embedding_key='clip_l'):
|
||||||
if tokenizer_path is None:
|
if tokenizer_path is None:
|
||||||
tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_tokenizer")
|
tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_tokenizer")
|
||||||
self.tokenizer = CLIPTokenizer.from_pretrained(tokenizer_path)
|
self.tokenizer = CLIPTokenizer.from_pretrained(tokenizer_path)
|
||||||
@ -315,17 +317,18 @@ class SD1Tokenizer:
|
|||||||
self.max_word_length = 8
|
self.max_word_length = 8
|
||||||
self.embedding_identifier = "embedding:"
|
self.embedding_identifier = "embedding:"
|
||||||
self.embedding_size = embedding_size
|
self.embedding_size = embedding_size
|
||||||
|
self.embedding_key = embedding_key
|
||||||
|
|
||||||
def _try_get_embedding(self, embedding_name:str):
|
def _try_get_embedding(self, embedding_name:str):
|
||||||
'''
|
'''
|
||||||
Takes a potential embedding name and tries to retrieve it.
|
Takes a potential embedding name and tries to retrieve it.
|
||||||
Returns a Tuple consisting of the embedding and any leftover string, embedding can be None.
|
Returns a Tuple consisting of the embedding and any leftover string, embedding can be None.
|
||||||
'''
|
'''
|
||||||
embed = load_embed(embedding_name, self.embedding_directory, self.embedding_size)
|
embed = load_embed(embedding_name, self.embedding_directory, self.embedding_size, self.embedding_key)
|
||||||
if embed is None:
|
if embed is None:
|
||||||
stripped = embedding_name.strip(',')
|
stripped = embedding_name.strip(',')
|
||||||
if len(stripped) < len(embedding_name):
|
if len(stripped) < len(embedding_name):
|
||||||
embed = load_embed(stripped, self.embedding_directory, self.embedding_size)
|
embed = load_embed(stripped, self.embedding_directory, self.embedding_size, self.embedding_key)
|
||||||
return (embed, embedding_name[len(stripped):])
|
return (embed, embedding_name[len(stripped):])
|
||||||
return (embed, "")
|
return (embed, "")
|
||||||
|
|
||||||
|
|||||||
@ -35,11 +35,13 @@ class SDXLClipG(sd1_clip.SD1ClipModel):
|
|||||||
def load_sd(self, sd):
|
def load_sd(self, sd):
|
||||||
if "text_projection" in sd:
|
if "text_projection" in sd:
|
||||||
self.text_projection[:] = sd.pop("text_projection")
|
self.text_projection[:] = sd.pop("text_projection")
|
||||||
|
if "text_projection.weight" in sd:
|
||||||
|
self.text_projection[:] = sd.pop("text_projection.weight").transpose(0, 1)
|
||||||
return super().load_sd(sd)
|
return super().load_sd(sd)
|
||||||
|
|
||||||
class SDXLClipGTokenizer(sd1_clip.SD1Tokenizer):
|
class SDXLClipGTokenizer(sd1_clip.SD1Tokenizer):
|
||||||
def __init__(self, tokenizer_path=None, embedding_directory=None):
|
def __init__(self, tokenizer_path=None, embedding_directory=None):
|
||||||
super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1280)
|
super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1280, embedding_key='clip_g')
|
||||||
|
|
||||||
|
|
||||||
class SDXLTokenizer(sd1_clip.SD1Tokenizer):
|
class SDXLTokenizer(sd1_clip.SD1Tokenizer):
|
||||||
|
|||||||
@ -18,9 +18,9 @@ class ModelMergeSimple:
|
|||||||
|
|
||||||
def merge(self, model1, model2, ratio):
|
def merge(self, model1, model2, ratio):
|
||||||
m = model1.clone()
|
m = model1.clone()
|
||||||
sd = model2.model_state_dict("diffusion_model.")
|
kp = model2.get_key_patches("diffusion_model.")
|
||||||
for k in sd:
|
for k in kp:
|
||||||
m.add_patches({k: (sd[k], )}, 1.0 - ratio, ratio)
|
m.add_patches({k: kp[k]}, 1.0 - ratio, ratio)
|
||||||
return (m, )
|
return (m, )
|
||||||
|
|
||||||
class ModelMergeBlocks:
|
class ModelMergeBlocks:
|
||||||
@ -39,10 +39,10 @@ class ModelMergeBlocks:
|
|||||||
|
|
||||||
def merge(self, model1, model2, **kwargs):
|
def merge(self, model1, model2, **kwargs):
|
||||||
m = model1.clone()
|
m = model1.clone()
|
||||||
sd = model2.model_state_dict("diffusion_model.")
|
kp = model2.get_key_patches("diffusion_model.")
|
||||||
default_ratio = next(iter(kwargs.values()))
|
default_ratio = next(iter(kwargs.values()))
|
||||||
|
|
||||||
for k in sd:
|
for k in kp:
|
||||||
ratio = default_ratio
|
ratio = default_ratio
|
||||||
k_unet = k[len("diffusion_model."):]
|
k_unet = k[len("diffusion_model."):]
|
||||||
|
|
||||||
@ -52,7 +52,7 @@ class ModelMergeBlocks:
|
|||||||
ratio = kwargs[arg]
|
ratio = kwargs[arg]
|
||||||
last_arg_size = len(arg)
|
last_arg_size = len(arg)
|
||||||
|
|
||||||
m.add_patches({k: (sd[k], )}, 1.0 - ratio, ratio)
|
m.add_patches({k: kp[k]}, 1.0 - ratio, ratio)
|
||||||
return (m, )
|
return (m, )
|
||||||
|
|
||||||
class CheckpointSave:
|
class CheckpointSave:
|
||||||
|
|||||||
@ -3,7 +3,6 @@ from PIL import Image, ImageOps
|
|||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
import struct
|
import struct
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
from comfy.cli_args import args, LatentPreviewMethod
|
from comfy.cli_args import args, LatentPreviewMethod
|
||||||
from comfy.taesd.taesd import TAESD
|
from comfy.taesd.taesd import TAESD
|
||||||
import folder_paths
|
import folder_paths
|
||||||
@ -16,7 +15,13 @@ class LatentPreviewer:
|
|||||||
|
|
||||||
def decode_latent_to_preview_image(self, preview_format, x0):
|
def decode_latent_to_preview_image(self, preview_format, x0):
|
||||||
preview_image = self.decode_latent_to_preview(x0)
|
preview_image = self.decode_latent_to_preview(x0)
|
||||||
preview_image = ImageOps.contain(preview_image, (MAX_PREVIEW_RESOLUTION, MAX_PREVIEW_RESOLUTION), Image.ANTIALIAS)
|
|
||||||
|
if hasattr(Image, 'Resampling'):
|
||||||
|
resampling = Image.Resampling.BILINEAR
|
||||||
|
else:
|
||||||
|
resampling = Image.ANTIALIAS
|
||||||
|
|
||||||
|
preview_image = ImageOps.contain(preview_image, (MAX_PREVIEW_RESOLUTION, MAX_PREVIEW_RESOLUTION), resampling)
|
||||||
|
|
||||||
preview_type = 1
|
preview_type = 1
|
||||||
if preview_format == "JPEG":
|
if preview_format == "JPEG":
|
||||||
|
|||||||
@ -368,7 +368,11 @@ export class ComfyApp {
|
|||||||
shiftY = w.last_y;
|
shiftY = w.last_y;
|
||||||
if (w.computeSize) {
|
if (w.computeSize) {
|
||||||
shiftY += w.computeSize()[1] + 4;
|
shiftY += w.computeSize()[1] + 4;
|
||||||
} else {
|
}
|
||||||
|
else if(w.computedHeight) {
|
||||||
|
shiftY += w.computedHeight;
|
||||||
|
}
|
||||||
|
else {
|
||||||
shiftY += LiteGraph.NODE_WIDGET_HEIGHT + 4;
|
shiftY += LiteGraph.NODE_WIDGET_HEIGHT + 4;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@ -129,6 +129,7 @@ function addMultilineWidget(node, name, opts, app) {
|
|||||||
w.y = y;
|
w.y = y;
|
||||||
if (w.type === "customtext") {
|
if (w.type === "customtext") {
|
||||||
y += freeSpace;
|
y += freeSpace;
|
||||||
|
w.computedHeight = freeSpace - multi.length*4;
|
||||||
} else if (w.computeSize) {
|
} else if (w.computeSize) {
|
||||||
y += w.computeSize()[1] + 4;
|
y += w.computeSize()[1] + 4;
|
||||||
} else {
|
} else {
|
||||||
@ -162,11 +163,12 @@ function addMultilineWidget(node, name, opts, app) {
|
|||||||
.multiplySelf(ctx.getTransform())
|
.multiplySelf(ctx.getTransform())
|
||||||
.translateSelf(margin, margin + y);
|
.translateSelf(margin, margin + y);
|
||||||
|
|
||||||
|
const scale = new DOMMatrix().scaleSelf(transform.a, transform.d)
|
||||||
Object.assign(this.inputEl.style, {
|
Object.assign(this.inputEl.style, {
|
||||||
transformOrigin: "0 0",
|
transformOrigin: "0 0",
|
||||||
transform: transform,
|
transform: scale,
|
||||||
left: "0px",
|
left: `${transform.a + transform.e}px`,
|
||||||
top: "0px",
|
top: `${transform.d + transform.f}px`,
|
||||||
width: `${widgetWidth - (margin * 2)}px`,
|
width: `${widgetWidth - (margin * 2)}px`,
|
||||||
height: `${this.parent.inputHeight - (margin * 2)}px`,
|
height: `${this.parent.inputHeight - (margin * 2)}px`,
|
||||||
position: "absolute",
|
position: "absolute",
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user