mirror of
https://git.datalinker.icu/comfyanonymous/ComfyUI
synced 2026-11-22 00:27:08 +08:00
Merge remote-tracking branch 'origin/master' into newmenu
This commit is contained in:
commit
114369c853
@ -11,7 +11,7 @@ This ui will let you design and execute advanced stable diffusion pipelines usin
|
|||||||
|
|
||||||
## Features
|
## Features
|
||||||
- Nodes/graph/flowchart interface to experiment and create complex Stable Diffusion workflows without needing to code anything.
|
- Nodes/graph/flowchart interface to experiment and create complex Stable Diffusion workflows without needing to code anything.
|
||||||
- Fully supports SD1.x, SD2.x, [SDXL](https://comfyanonymous.github.io/ComfyUI_examples/sdxl/) and [Stable Video Diffusion](https://comfyanonymous.github.io/ComfyUI_examples/video/)
|
- Fully supports SD1.x, SD2.x, [SDXL](https://comfyanonymous.github.io/ComfyUI_examples/sdxl/), [Stable Video Diffusion](https://comfyanonymous.github.io/ComfyUI_examples/video/) and [Stable Cascade](https://comfyanonymous.github.io/ComfyUI_examples/stable_cascade/)
|
||||||
- Asynchronous Queue system
|
- Asynchronous Queue system
|
||||||
- Many optimizations: Only re-executes the parts of the workflow that changes between executions.
|
- Many optimizations: Only re-executes the parts of the workflow that changes between executions.
|
||||||
- Command line option: ```--lowvram``` to make it work on GPUs with less than 3GB vram (enabled automatically on GPUs with low vram)
|
- Command line option: ```--lowvram``` to make it work on GPUs with less than 3GB vram (enabled automatically on GPUs with low vram)
|
||||||
|
|||||||
@ -97,7 +97,7 @@ class CLIPTextModel_(torch.nn.Module):
|
|||||||
x = self.embeddings(input_tokens)
|
x = self.embeddings(input_tokens)
|
||||||
mask = None
|
mask = None
|
||||||
if attention_mask is not None:
|
if attention_mask is not None:
|
||||||
mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], attention_mask.shape[-1], attention_mask.shape[-1])
|
mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, attention_mask.shape[-1], attention_mask.shape[-1])
|
||||||
mask = mask.masked_fill(mask.to(torch.bool), float("-inf"))
|
mask = mask.masked_fill(mask.to(torch.bool), float("-inf"))
|
||||||
|
|
||||||
causal_mask = torch.empty(x.shape[1], x.shape[1], dtype=x.dtype, device=x.device).fill_(float("-inf")).triu_(1)
|
causal_mask = torch.empty(x.shape[1], x.shape[1], dtype=x.dtype, device=x.device).fill_(float("-inf")).triu_(1)
|
||||||
|
|||||||
@ -2,7 +2,8 @@ import torch
|
|||||||
from torch import nn
|
from torch import nn
|
||||||
from .ldm.modules.attention import CrossAttention
|
from .ldm.modules.attention import CrossAttention
|
||||||
from inspect import isfunction
|
from inspect import isfunction
|
||||||
|
import comfy.ops
|
||||||
|
ops = comfy.ops.manual_cast
|
||||||
|
|
||||||
def exists(val):
|
def exists(val):
|
||||||
return val is not None
|
return val is not None
|
||||||
@ -22,7 +23,7 @@ def default(val, d):
|
|||||||
class GEGLU(nn.Module):
|
class GEGLU(nn.Module):
|
||||||
def __init__(self, dim_in, dim_out):
|
def __init__(self, dim_in, dim_out):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.proj = nn.Linear(dim_in, dim_out * 2)
|
self.proj = ops.Linear(dim_in, dim_out * 2)
|
||||||
|
|
||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
x, gate = self.proj(x).chunk(2, dim=-1)
|
x, gate = self.proj(x).chunk(2, dim=-1)
|
||||||
@ -35,14 +36,14 @@ class FeedForward(nn.Module):
|
|||||||
inner_dim = int(dim * mult)
|
inner_dim = int(dim * mult)
|
||||||
dim_out = default(dim_out, dim)
|
dim_out = default(dim_out, dim)
|
||||||
project_in = nn.Sequential(
|
project_in = nn.Sequential(
|
||||||
nn.Linear(dim, inner_dim),
|
ops.Linear(dim, inner_dim),
|
||||||
nn.GELU()
|
nn.GELU()
|
||||||
) if not glu else GEGLU(dim, inner_dim)
|
) if not glu else GEGLU(dim, inner_dim)
|
||||||
|
|
||||||
self.net = nn.Sequential(
|
self.net = nn.Sequential(
|
||||||
project_in,
|
project_in,
|
||||||
nn.Dropout(dropout),
|
nn.Dropout(dropout),
|
||||||
nn.Linear(inner_dim, dim_out)
|
ops.Linear(inner_dim, dim_out)
|
||||||
)
|
)
|
||||||
|
|
||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
@ -57,11 +58,12 @@ class GatedCrossAttentionDense(nn.Module):
|
|||||||
query_dim=query_dim,
|
query_dim=query_dim,
|
||||||
context_dim=context_dim,
|
context_dim=context_dim,
|
||||||
heads=n_heads,
|
heads=n_heads,
|
||||||
dim_head=d_head)
|
dim_head=d_head,
|
||||||
|
operations=ops)
|
||||||
self.ff = FeedForward(query_dim, glu=True)
|
self.ff = FeedForward(query_dim, glu=True)
|
||||||
|
|
||||||
self.norm1 = nn.LayerNorm(query_dim)
|
self.norm1 = ops.LayerNorm(query_dim)
|
||||||
self.norm2 = nn.LayerNorm(query_dim)
|
self.norm2 = ops.LayerNorm(query_dim)
|
||||||
|
|
||||||
self.register_parameter('alpha_attn', nn.Parameter(torch.tensor(0.)))
|
self.register_parameter('alpha_attn', nn.Parameter(torch.tensor(0.)))
|
||||||
self.register_parameter('alpha_dense', nn.Parameter(torch.tensor(0.)))
|
self.register_parameter('alpha_dense', nn.Parameter(torch.tensor(0.)))
|
||||||
@ -87,17 +89,18 @@ class GatedSelfAttentionDense(nn.Module):
|
|||||||
|
|
||||||
# we need a linear projection since we need cat visual feature and obj
|
# we need a linear projection since we need cat visual feature and obj
|
||||||
# feature
|
# feature
|
||||||
self.linear = nn.Linear(context_dim, query_dim)
|
self.linear = ops.Linear(context_dim, query_dim)
|
||||||
|
|
||||||
self.attn = CrossAttention(
|
self.attn = CrossAttention(
|
||||||
query_dim=query_dim,
|
query_dim=query_dim,
|
||||||
context_dim=query_dim,
|
context_dim=query_dim,
|
||||||
heads=n_heads,
|
heads=n_heads,
|
||||||
dim_head=d_head)
|
dim_head=d_head,
|
||||||
|
operations=ops)
|
||||||
self.ff = FeedForward(query_dim, glu=True)
|
self.ff = FeedForward(query_dim, glu=True)
|
||||||
|
|
||||||
self.norm1 = nn.LayerNorm(query_dim)
|
self.norm1 = ops.LayerNorm(query_dim)
|
||||||
self.norm2 = nn.LayerNorm(query_dim)
|
self.norm2 = ops.LayerNorm(query_dim)
|
||||||
|
|
||||||
self.register_parameter('alpha_attn', nn.Parameter(torch.tensor(0.)))
|
self.register_parameter('alpha_attn', nn.Parameter(torch.tensor(0.)))
|
||||||
self.register_parameter('alpha_dense', nn.Parameter(torch.tensor(0.)))
|
self.register_parameter('alpha_dense', nn.Parameter(torch.tensor(0.)))
|
||||||
@ -126,14 +129,14 @@ class GatedSelfAttentionDense2(nn.Module):
|
|||||||
|
|
||||||
# we need a linear projection since we need cat visual feature and obj
|
# we need a linear projection since we need cat visual feature and obj
|
||||||
# feature
|
# feature
|
||||||
self.linear = nn.Linear(context_dim, query_dim)
|
self.linear = ops.Linear(context_dim, query_dim)
|
||||||
|
|
||||||
self.attn = CrossAttention(
|
self.attn = CrossAttention(
|
||||||
query_dim=query_dim, context_dim=query_dim, dim_head=d_head)
|
query_dim=query_dim, context_dim=query_dim, dim_head=d_head, operations=ops)
|
||||||
self.ff = FeedForward(query_dim, glu=True)
|
self.ff = FeedForward(query_dim, glu=True)
|
||||||
|
|
||||||
self.norm1 = nn.LayerNorm(query_dim)
|
self.norm1 = ops.LayerNorm(query_dim)
|
||||||
self.norm2 = nn.LayerNorm(query_dim)
|
self.norm2 = ops.LayerNorm(query_dim)
|
||||||
|
|
||||||
self.register_parameter('alpha_attn', nn.Parameter(torch.tensor(0.)))
|
self.register_parameter('alpha_attn', nn.Parameter(torch.tensor(0.)))
|
||||||
self.register_parameter('alpha_dense', nn.Parameter(torch.tensor(0.)))
|
self.register_parameter('alpha_dense', nn.Parameter(torch.tensor(0.)))
|
||||||
@ -201,11 +204,11 @@ class PositionNet(nn.Module):
|
|||||||
self.position_dim = fourier_freqs * 2 * 4 # 2 is sin&cos, 4 is xyxy
|
self.position_dim = fourier_freqs * 2 * 4 # 2 is sin&cos, 4 is xyxy
|
||||||
|
|
||||||
self.linears = nn.Sequential(
|
self.linears = nn.Sequential(
|
||||||
nn.Linear(self.in_dim + self.position_dim, 512),
|
ops.Linear(self.in_dim + self.position_dim, 512),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
nn.Linear(512, 512),
|
ops.Linear(512, 512),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
nn.Linear(512, out_dim),
|
ops.Linear(512, out_dim),
|
||||||
)
|
)
|
||||||
|
|
||||||
self.null_positive_feature = torch.nn.Parameter(
|
self.null_positive_feature = torch.nn.Parameter(
|
||||||
@ -215,16 +218,15 @@ class PositionNet(nn.Module):
|
|||||||
|
|
||||||
def forward(self, boxes, masks, positive_embeddings):
|
def forward(self, boxes, masks, positive_embeddings):
|
||||||
B, N, _ = boxes.shape
|
B, N, _ = boxes.shape
|
||||||
dtype = self.linears[0].weight.dtype
|
masks = masks.unsqueeze(-1)
|
||||||
masks = masks.unsqueeze(-1).to(dtype)
|
positive_embeddings = positive_embeddings
|
||||||
positive_embeddings = positive_embeddings.to(dtype)
|
|
||||||
|
|
||||||
# embedding position (it may includes padding as placeholder)
|
# embedding position (it may includes padding as placeholder)
|
||||||
xyxy_embedding = self.fourier_embedder(boxes.to(dtype)) # B*N*4 --> B*N*C
|
xyxy_embedding = self.fourier_embedder(boxes) # B*N*4 --> B*N*C
|
||||||
|
|
||||||
# learnable null embedding
|
# learnable null embedding
|
||||||
positive_null = self.null_positive_feature.view(1, 1, -1)
|
positive_null = self.null_positive_feature.to(device=boxes.device, dtype=boxes.dtype).view(1, 1, -1)
|
||||||
xyxy_null = self.null_position_feature.view(1, 1, -1)
|
xyxy_null = self.null_position_feature.to(device=boxes.device, dtype=boxes.dtype).view(1, 1, -1)
|
||||||
|
|
||||||
# replace padding with learnable null embedding
|
# replace padding with learnable null embedding
|
||||||
positive_embeddings = positive_embeddings * \
|
positive_embeddings = positive_embeddings * \
|
||||||
@ -251,7 +253,7 @@ class Gligen(nn.Module):
|
|||||||
def func(x, extra_options):
|
def func(x, extra_options):
|
||||||
key = extra_options["transformer_index"]
|
key = extra_options["transformer_index"]
|
||||||
module = self.module_list[key]
|
module = self.module_list[key]
|
||||||
return module(x, objs)
|
return module(x, objs.to(device=x.device, dtype=x.dtype))
|
||||||
return func
|
return func
|
||||||
|
|
||||||
def set_position(self, latent_image_shape, position_params, device):
|
def set_position(self, latent_image_shape, position_params, device):
|
||||||
|
|||||||
@ -37,11 +37,41 @@ class SDXL(LatentFormat):
|
|||||||
class SD_X4(LatentFormat):
|
class SD_X4(LatentFormat):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.scale_factor = 0.08333
|
self.scale_factor = 0.08333
|
||||||
|
self.latent_rgb_factors = [
|
||||||
|
[-0.2340, -0.3863, -0.3257],
|
||||||
|
[ 0.0994, 0.0885, -0.0908],
|
||||||
|
[-0.2833, -0.2349, -0.3741],
|
||||||
|
[ 0.2523, -0.0055, -0.1651]
|
||||||
|
]
|
||||||
|
|
||||||
class SC_Prior(LatentFormat):
|
class SC_Prior(LatentFormat):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.scale_factor = 1.0
|
self.scale_factor = 1.0
|
||||||
|
self.latent_rgb_factors = [
|
||||||
|
[-0.0326, -0.0204, -0.0127],
|
||||||
|
[-0.1592, -0.0427, 0.0216],
|
||||||
|
[ 0.0873, 0.0638, -0.0020],
|
||||||
|
[-0.0602, 0.0442, 0.1304],
|
||||||
|
[ 0.0800, -0.0313, -0.1796],
|
||||||
|
[-0.0810, -0.0638, -0.1581],
|
||||||
|
[ 0.1791, 0.1180, 0.0967],
|
||||||
|
[ 0.0740, 0.1416, 0.0432],
|
||||||
|
[-0.1745, -0.1888, -0.1373],
|
||||||
|
[ 0.2412, 0.1577, 0.0928],
|
||||||
|
[ 0.1908, 0.0998, 0.0682],
|
||||||
|
[ 0.0209, 0.0365, -0.0092],
|
||||||
|
[ 0.0448, -0.0650, -0.1728],
|
||||||
|
[-0.1658, -0.1045, -0.1308],
|
||||||
|
[ 0.0542, 0.1545, 0.1325],
|
||||||
|
[-0.0352, -0.1672, -0.2541]
|
||||||
|
]
|
||||||
|
|
||||||
class SC_B(LatentFormat):
|
class SC_B(LatentFormat):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.scale_factor = 1.0
|
self.scale_factor = 1.0
|
||||||
|
self.latent_rgb_factors = [
|
||||||
|
[ 0.1121, 0.2006, 0.1023],
|
||||||
|
[-0.2093, -0.0222, -0.0195],
|
||||||
|
[-0.3087, -0.1535, 0.0366],
|
||||||
|
[ 0.0290, -0.1574, -0.4078]
|
||||||
|
]
|
||||||
|
|||||||
@ -84,7 +84,7 @@ class GlobalResponseNorm(nn.Module):
|
|||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
Gx = torch.norm(x, p=2, dim=(1, 2), keepdim=True)
|
Gx = torch.norm(x, p=2, dim=(1, 2), keepdim=True)
|
||||||
Nx = Gx / (Gx.mean(dim=-1, keepdim=True) + 1e-6)
|
Nx = Gx / (Gx.mean(dim=-1, keepdim=True) + 1e-6)
|
||||||
return self.gamma.to(x.device) * (x * Nx) + self.beta.to(x.device) + x
|
return self.gamma.to(device=x.device, dtype=x.dtype) * (x * Nx) + self.beta.to(device=x.device, dtype=x.dtype) + x
|
||||||
|
|
||||||
|
|
||||||
class ResBlock(nn.Module):
|
class ResBlock(nn.Module):
|
||||||
|
|||||||
95
comfy/ldm/cascade/stage_c_coder.py
Normal file
95
comfy/ldm/cascade/stage_c_coder.py
Normal file
@ -0,0 +1,95 @@
|
|||||||
|
"""
|
||||||
|
This file is part of ComfyUI.
|
||||||
|
Copyright (C) 2024 Stability AI
|
||||||
|
|
||||||
|
This program is free software: you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU General Public License as published by
|
||||||
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
"""
|
||||||
|
import torch
|
||||||
|
import torchvision
|
||||||
|
from torch import nn
|
||||||
|
|
||||||
|
|
||||||
|
# EfficientNet
|
||||||
|
class EfficientNetEncoder(nn.Module):
|
||||||
|
def __init__(self, c_latent=16):
|
||||||
|
super().__init__()
|
||||||
|
self.backbone = torchvision.models.efficientnet_v2_s().features.eval()
|
||||||
|
self.mapper = nn.Sequential(
|
||||||
|
nn.Conv2d(1280, c_latent, kernel_size=1, bias=False),
|
||||||
|
nn.BatchNorm2d(c_latent, affine=False), # then normalize them to have mean 0 and std 1
|
||||||
|
)
|
||||||
|
self.mean = nn.Parameter(torch.tensor([0.485, 0.456, 0.406]))
|
||||||
|
self.std = nn.Parameter(torch.tensor([0.229, 0.224, 0.225]))
|
||||||
|
|
||||||
|
def forward(self, x):
|
||||||
|
x = x * 0.5 + 0.5
|
||||||
|
x = (x - self.mean.view([3,1,1])) / self.std.view([3,1,1])
|
||||||
|
o = self.mapper(self.backbone(x))
|
||||||
|
return o
|
||||||
|
|
||||||
|
|
||||||
|
# Fast Decoder for Stage C latents. E.g. 16 x 24 x 24 -> 3 x 192 x 192
|
||||||
|
class Previewer(nn.Module):
|
||||||
|
def __init__(self, c_in=16, c_hidden=512, c_out=3):
|
||||||
|
super().__init__()
|
||||||
|
self.blocks = nn.Sequential(
|
||||||
|
nn.Conv2d(c_in, c_hidden, kernel_size=1), # 16 channels to 512 channels
|
||||||
|
nn.GELU(),
|
||||||
|
nn.BatchNorm2d(c_hidden),
|
||||||
|
|
||||||
|
nn.Conv2d(c_hidden, c_hidden, kernel_size=3, padding=1),
|
||||||
|
nn.GELU(),
|
||||||
|
nn.BatchNorm2d(c_hidden),
|
||||||
|
|
||||||
|
nn.ConvTranspose2d(c_hidden, c_hidden // 2, kernel_size=2, stride=2), # 16 -> 32
|
||||||
|
nn.GELU(),
|
||||||
|
nn.BatchNorm2d(c_hidden // 2),
|
||||||
|
|
||||||
|
nn.Conv2d(c_hidden // 2, c_hidden // 2, kernel_size=3, padding=1),
|
||||||
|
nn.GELU(),
|
||||||
|
nn.BatchNorm2d(c_hidden // 2),
|
||||||
|
|
||||||
|
nn.ConvTranspose2d(c_hidden // 2, c_hidden // 4, kernel_size=2, stride=2), # 32 -> 64
|
||||||
|
nn.GELU(),
|
||||||
|
nn.BatchNorm2d(c_hidden // 4),
|
||||||
|
|
||||||
|
nn.Conv2d(c_hidden // 4, c_hidden // 4, kernel_size=3, padding=1),
|
||||||
|
nn.GELU(),
|
||||||
|
nn.BatchNorm2d(c_hidden // 4),
|
||||||
|
|
||||||
|
nn.ConvTranspose2d(c_hidden // 4, c_hidden // 4, kernel_size=2, stride=2), # 64 -> 128
|
||||||
|
nn.GELU(),
|
||||||
|
nn.BatchNorm2d(c_hidden // 4),
|
||||||
|
|
||||||
|
nn.Conv2d(c_hidden // 4, c_hidden // 4, kernel_size=3, padding=1),
|
||||||
|
nn.GELU(),
|
||||||
|
nn.BatchNorm2d(c_hidden // 4),
|
||||||
|
|
||||||
|
nn.Conv2d(c_hidden // 4, c_out, kernel_size=1),
|
||||||
|
)
|
||||||
|
|
||||||
|
def forward(self, x):
|
||||||
|
return (self.blocks(x) - 0.5) * 2.0
|
||||||
|
|
||||||
|
class StageC_coder(nn.Module):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.previewer = Previewer()
|
||||||
|
self.encoder = EfficientNetEncoder()
|
||||||
|
|
||||||
|
def encode(self, x):
|
||||||
|
return self.encoder(x)
|
||||||
|
|
||||||
|
def decode(self, x):
|
||||||
|
return self.previewer(x)
|
||||||
@ -114,7 +114,12 @@ def attention_basic(q, k, v, heads, mask=None):
|
|||||||
mask = repeat(mask, 'b j -> (b h) () j', h=h)
|
mask = repeat(mask, 'b j -> (b h) () j', h=h)
|
||||||
sim.masked_fill_(~mask, max_neg_value)
|
sim.masked_fill_(~mask, max_neg_value)
|
||||||
else:
|
else:
|
||||||
sim += mask
|
if len(mask.shape) == 2:
|
||||||
|
bs = 1
|
||||||
|
else:
|
||||||
|
bs = mask.shape[0]
|
||||||
|
mask = mask.reshape(bs, -1, mask.shape[-2], mask.shape[-1]).expand(b, heads, -1, -1).reshape(-1, mask.shape[-2], mask.shape[-1])
|
||||||
|
sim.add_(mask)
|
||||||
|
|
||||||
# attention, what we cannot get enough of
|
# attention, what we cannot get enough of
|
||||||
sim = sim.softmax(dim=-1)
|
sim = sim.softmax(dim=-1)
|
||||||
@ -165,6 +170,13 @@ def attention_sub_quad(query, key, value, heads, mask=None):
|
|||||||
if query_chunk_size is None:
|
if query_chunk_size is None:
|
||||||
query_chunk_size = 512
|
query_chunk_size = 512
|
||||||
|
|
||||||
|
if mask is not None:
|
||||||
|
if len(mask.shape) == 2:
|
||||||
|
bs = 1
|
||||||
|
else:
|
||||||
|
bs = mask.shape[0]
|
||||||
|
mask = mask.reshape(bs, -1, mask.shape[-2], mask.shape[-1]).expand(b, heads, -1, -1).reshape(-1, mask.shape[-2], mask.shape[-1])
|
||||||
|
|
||||||
hidden_states = efficient_dot_product_attention(
|
hidden_states = efficient_dot_product_attention(
|
||||||
query,
|
query,
|
||||||
key,
|
key,
|
||||||
@ -223,6 +235,13 @@ def attention_split(q, k, v, heads, mask=None):
|
|||||||
raise RuntimeError(f'Not enough memory, use lower resolution (max approx. {max_res}x{max_res}). '
|
raise RuntimeError(f'Not enough memory, use lower resolution (max approx. {max_res}x{max_res}). '
|
||||||
f'Need: {mem_required/64/gb:0.1f}GB free, Have:{mem_free_total/gb:0.1f}GB free')
|
f'Need: {mem_required/64/gb:0.1f}GB free, Have:{mem_free_total/gb:0.1f}GB free')
|
||||||
|
|
||||||
|
if mask is not None:
|
||||||
|
if len(mask.shape) == 2:
|
||||||
|
bs = 1
|
||||||
|
else:
|
||||||
|
bs = mask.shape[0]
|
||||||
|
mask = mask.reshape(bs, -1, mask.shape[-2], mask.shape[-1]).expand(b, heads, -1, -1).reshape(-1, mask.shape[-2], mask.shape[-1])
|
||||||
|
|
||||||
# print("steps", steps, mem_required, mem_free_total, modifier, q.element_size(), tensor_size)
|
# print("steps", steps, mem_required, mem_free_total, modifier, q.element_size(), tensor_size)
|
||||||
first_op_done = False
|
first_op_done = False
|
||||||
cleared_cache = False
|
cleared_cache = False
|
||||||
|
|||||||
@ -474,15 +474,11 @@ class StableCascade_B(BaseModel):
|
|||||||
|
|
||||||
clip_text_pooled = kwargs["pooled_output"]
|
clip_text_pooled = kwargs["pooled_output"]
|
||||||
if clip_text_pooled is not None:
|
if clip_text_pooled is not None:
|
||||||
out['clip_text_pooled'] = comfy.conds.CONDRegular(clip_text_pooled)
|
out['clip'] = comfy.conds.CONDRegular(clip_text_pooled)
|
||||||
|
|
||||||
#size of prior doesn't really matter if zeros because it gets resized but I still want it to get batched
|
#size of prior doesn't really matter if zeros because it gets resized but I still want it to get batched
|
||||||
prior = kwargs.get("stable_cascade_prior", torch.zeros((1, 16, (noise.shape[2] * 4) // 42, (noise.shape[3] * 4) // 42), dtype=noise.dtype, layout=noise.layout, device=noise.device))
|
prior = kwargs.get("stable_cascade_prior", torch.zeros((1, 16, (noise.shape[2] * 4) // 42, (noise.shape[3] * 4) // 42), dtype=noise.dtype, layout=noise.layout, device=noise.device))
|
||||||
|
|
||||||
out["effnet"] = comfy.conds.CONDRegular(prior)
|
out["effnet"] = comfy.conds.CONDRegular(prior)
|
||||||
out["sca"] = comfy.conds.CONDRegular(torch.zeros((1,)))
|
out["sca"] = comfy.conds.CONDRegular(torch.zeros((1,)))
|
||||||
|
|
||||||
cross_attn = kwargs.get("cross_attn", None)
|
|
||||||
if cross_attn is not None:
|
|
||||||
out['clip'] = comfy.conds.CONDCrossAttn(cross_attn)
|
|
||||||
return out
|
return out
|
||||||
|
|||||||
@ -499,7 +499,7 @@ def unet_dtype(device=None, model_params=0, supported_dtypes=[torch.float16, tor
|
|||||||
if should_use_fp16(device=device, model_params=model_params, manual_cast=True):
|
if should_use_fp16(device=device, model_params=model_params, manual_cast=True):
|
||||||
if torch.float16 in supported_dtypes:
|
if torch.float16 in supported_dtypes:
|
||||||
return torch.float16
|
return torch.float16
|
||||||
if should_use_bf16(device):
|
if should_use_bf16(device, model_params=model_params, manual_cast=True):
|
||||||
if torch.bfloat16 in supported_dtypes:
|
if torch.bfloat16 in supported_dtypes:
|
||||||
return torch.bfloat16
|
return torch.bfloat16
|
||||||
return torch.float32
|
return torch.float32
|
||||||
@ -720,9 +720,9 @@ def should_use_fp16(device=None, model_params=0, prioritize_performance=True, ma
|
|||||||
if FORCE_FP16:
|
if FORCE_FP16:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
if device is not None: #TODO
|
if device is not None:
|
||||||
if is_device_mps(device):
|
if is_device_mps(device):
|
||||||
return False
|
return True
|
||||||
|
|
||||||
if FORCE_FP32:
|
if FORCE_FP32:
|
||||||
return False
|
return False
|
||||||
@ -730,8 +730,11 @@ def should_use_fp16(device=None, model_params=0, prioritize_performance=True, ma
|
|||||||
if directml_enabled:
|
if directml_enabled:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if cpu_mode() or mps_mode():
|
if mps_mode():
|
||||||
return False #TODO ?
|
return True
|
||||||
|
|
||||||
|
if cpu_mode():
|
||||||
|
return False
|
||||||
|
|
||||||
if is_intel_xpu():
|
if is_intel_xpu():
|
||||||
return True
|
return True
|
||||||
@ -771,10 +774,24 @@ def should_use_fp16(device=None, model_params=0, prioritize_performance=True, ma
|
|||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def should_use_bf16(device=None):
|
def should_use_bf16(device=None, model_params=0, prioritize_performance=True, manual_cast=False):
|
||||||
|
if device is not None:
|
||||||
|
if is_device_cpu(device): #TODO ? bf16 works on CPU but is extremely slow
|
||||||
|
return False
|
||||||
|
|
||||||
|
if device is not None: #TODO not sure about mps bf16 support
|
||||||
|
if is_device_mps(device):
|
||||||
|
return False
|
||||||
|
|
||||||
if FORCE_FP32:
|
if FORCE_FP32:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
if directml_enabled:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if cpu_mode() or mps_mode():
|
||||||
|
return False
|
||||||
|
|
||||||
if is_intel_xpu():
|
if is_intel_xpu():
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@ -785,6 +802,13 @@ def should_use_bf16(device=None):
|
|||||||
if props.major >= 8:
|
if props.major >= 8:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
bf16_works = torch.cuda.is_bf16_supported()
|
||||||
|
|
||||||
|
if bf16_works or manual_cast:
|
||||||
|
free_model_memory = (get_free_memory() * 0.9 - minimum_inference_memory())
|
||||||
|
if (not prioritize_performance) or model_params * 4 > free_model_memory:
|
||||||
|
return True
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def soft_empty_cache(force=False):
|
def soft_empty_cache(force=False):
|
||||||
|
|||||||
@ -136,23 +136,46 @@ class ModelSamplingContinuousEDM(torch.nn.Module):
|
|||||||
class StableCascadeSampling(ModelSamplingDiscrete):
|
class StableCascadeSampling(ModelSamplingDiscrete):
|
||||||
def __init__(self, model_config=None):
|
def __init__(self, model_config=None):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.num_timesteps = 1000
|
|
||||||
cosine_s=8e-3
|
if model_config is not None:
|
||||||
self.cosine_s = torch.tensor([cosine_s])
|
sampling_settings = model_config.sampling_settings
|
||||||
sigmas = torch.empty((self.num_timesteps), dtype=torch.float32)
|
else:
|
||||||
|
sampling_settings = {}
|
||||||
|
|
||||||
|
self.set_parameters(sampling_settings.get("shift", 1.0))
|
||||||
|
|
||||||
|
def set_parameters(self, shift=1.0, cosine_s=8e-3):
|
||||||
|
self.shift = shift
|
||||||
|
self.cosine_s = torch.tensor(cosine_s)
|
||||||
self._init_alpha_cumprod = torch.cos(self.cosine_s / (1 + self.cosine_s) * torch.pi * 0.5) ** 2
|
self._init_alpha_cumprod = torch.cos(self.cosine_s / (1 + self.cosine_s) * torch.pi * 0.5) ** 2
|
||||||
|
|
||||||
|
#This part is just for compatibility with some schedulers in the codebase
|
||||||
|
self.num_timesteps = 10000
|
||||||
|
sigmas = torch.empty((self.num_timesteps), dtype=torch.float32)
|
||||||
for x in range(self.num_timesteps):
|
for x in range(self.num_timesteps):
|
||||||
t = x / self.num_timesteps
|
t = (x + 1) / self.num_timesteps
|
||||||
sigmas[x] = self.sigma(t)
|
sigmas[x] = self.sigma(t)
|
||||||
|
|
||||||
self.set_sigmas(sigmas)
|
self.set_sigmas(sigmas)
|
||||||
|
|
||||||
def sigma(self, timestep):
|
def sigma(self, timestep):
|
||||||
alpha_cumprod = (torch.cos((timestep + self.cosine_s) / (1 + self.cosine_s) * torch.pi * 0.5) ** 2 / self._init_alpha_cumprod).clamp(0.0001, 0.9999)
|
alpha_cumprod = (torch.cos((timestep + self.cosine_s) / (1 + self.cosine_s) * torch.pi * 0.5) ** 2 / self._init_alpha_cumprod)
|
||||||
|
|
||||||
|
if self.shift != 1.0:
|
||||||
|
var = alpha_cumprod
|
||||||
|
logSNR = (var/(1-var)).log()
|
||||||
|
logSNR += 2 * torch.log(1.0 / torch.tensor(self.shift))
|
||||||
|
alpha_cumprod = logSNR.sigmoid()
|
||||||
|
|
||||||
|
alpha_cumprod = alpha_cumprod.clamp(0.0001, 0.9999)
|
||||||
return ((1 - alpha_cumprod) / alpha_cumprod) ** 0.5
|
return ((1 - alpha_cumprod) / alpha_cumprod) ** 0.5
|
||||||
|
|
||||||
def timestep(self, sigma):
|
def timestep(self, sigma):
|
||||||
return super().timestep(sigma) / 1000.0
|
var = 1 / ((sigma * sigma) + 1)
|
||||||
|
var = var.clamp(0, 1.0)
|
||||||
|
s, min_var = self.cosine_s.to(var.device), self._init_alpha_cumprod.to(var.device)
|
||||||
|
t = (((var * min_var) ** 0.5).acos() / (torch.pi * 0.5)) * (1 + s) - s
|
||||||
|
return t
|
||||||
|
|
||||||
def percent_to_sigma(self, percent):
|
def percent_to_sigma(self, percent):
|
||||||
if percent <= 0.0:
|
if percent <= 0.0:
|
||||||
|
|||||||
65
comfy/sd.py
65
comfy/sd.py
@ -4,6 +4,7 @@ from enum import Enum
|
|||||||
from comfy import model_management
|
from comfy import model_management
|
||||||
from .ldm.models.autoencoder import AutoencoderKL, AutoencodingEngine
|
from .ldm.models.autoencoder import AutoencoderKL, AutoencodingEngine
|
||||||
from .ldm.cascade.stage_a import StageA
|
from .ldm.cascade.stage_a import StageA
|
||||||
|
from .ldm.cascade.stage_c_coder import StageC_coder
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
@ -137,8 +138,11 @@ class CLIP:
|
|||||||
tokens = self.tokenize(text)
|
tokens = self.tokenize(text)
|
||||||
return self.encode_from_tokens(tokens)
|
return self.encode_from_tokens(tokens)
|
||||||
|
|
||||||
def load_sd(self, sd):
|
def load_sd(self, sd, full_model=False):
|
||||||
return self.cond_stage_model.load_sd(sd)
|
if full_model:
|
||||||
|
return self.cond_stage_model.load_state_dict(sd, strict=False)
|
||||||
|
else:
|
||||||
|
return self.cond_stage_model.load_sd(sd)
|
||||||
|
|
||||||
def get_sd(self):
|
def get_sd(self):
|
||||||
return self.cond_stage_model.state_dict()
|
return self.cond_stage_model.state_dict()
|
||||||
@ -158,6 +162,7 @@ class VAE:
|
|||||||
self.memory_used_encode = lambda shape, dtype: (1767 * shape[2] * shape[3]) * model_management.dtype_size(dtype) #These are for AutoencoderKL and need tweaking (should be lower)
|
self.memory_used_encode = lambda shape, dtype: (1767 * shape[2] * shape[3]) * model_management.dtype_size(dtype) #These are for AutoencoderKL and need tweaking (should be lower)
|
||||||
self.memory_used_decode = lambda shape, dtype: (2178 * shape[2] * shape[3] * 64) * model_management.dtype_size(dtype)
|
self.memory_used_decode = lambda shape, dtype: (2178 * shape[2] * shape[3] * 64) * model_management.dtype_size(dtype)
|
||||||
self.downscale_ratio = 8
|
self.downscale_ratio = 8
|
||||||
|
self.upscale_ratio = 8
|
||||||
self.latent_channels = 4
|
self.latent_channels = 4
|
||||||
self.process_input = lambda image: image * 2.0 - 1.0
|
self.process_input = lambda image: image * 2.0 - 1.0
|
||||||
self.process_output = lambda image: torch.clamp((image + 1.0) / 2.0, min=0.0, max=1.0)
|
self.process_output = lambda image: torch.clamp((image + 1.0) / 2.0, min=0.0, max=1.0)
|
||||||
@ -176,11 +181,31 @@ class VAE:
|
|||||||
elif "vquantizer.codebook.weight" in sd: #VQGan: stage a of stable cascade
|
elif "vquantizer.codebook.weight" in sd: #VQGan: stage a of stable cascade
|
||||||
self.first_stage_model = StageA()
|
self.first_stage_model = StageA()
|
||||||
self.downscale_ratio = 4
|
self.downscale_ratio = 4
|
||||||
|
self.upscale_ratio = 4
|
||||||
#TODO
|
#TODO
|
||||||
#self.memory_used_encode
|
#self.memory_used_encode
|
||||||
#self.memory_used_decode
|
#self.memory_used_decode
|
||||||
self.process_input = lambda image: image
|
self.process_input = lambda image: image
|
||||||
self.process_output = lambda image: image
|
self.process_output = lambda image: image
|
||||||
|
elif "backbone.1.0.block.0.1.num_batches_tracked" in sd: #effnet: encoder for stage c latent of stable cascade
|
||||||
|
self.first_stage_model = StageC_coder()
|
||||||
|
self.downscale_ratio = 32
|
||||||
|
self.latent_channels = 16
|
||||||
|
new_sd = {}
|
||||||
|
for k in sd:
|
||||||
|
new_sd["encoder.{}".format(k)] = sd[k]
|
||||||
|
sd = new_sd
|
||||||
|
elif "blocks.11.num_batches_tracked" in sd: #previewer: decoder for stage c latent of stable cascade
|
||||||
|
self.first_stage_model = StageC_coder()
|
||||||
|
self.latent_channels = 16
|
||||||
|
new_sd = {}
|
||||||
|
for k in sd:
|
||||||
|
new_sd["previewer.{}".format(k)] = sd[k]
|
||||||
|
sd = new_sd
|
||||||
|
elif "encoder.backbone.1.0.block.0.1.num_batches_tracked" in sd: #combined effnet and previewer for stable cascade
|
||||||
|
self.first_stage_model = StageC_coder()
|
||||||
|
self.downscale_ratio = 32
|
||||||
|
self.latent_channels = 16
|
||||||
else:
|
else:
|
||||||
#default SD1.x/SD2.x VAE parameters
|
#default SD1.x/SD2.x VAE parameters
|
||||||
ddconfig = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0}
|
ddconfig = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0}
|
||||||
@ -188,6 +213,7 @@ class VAE:
|
|||||||
if 'encoder.down.2.downsample.conv.weight' not in sd: #Stable diffusion x4 upscaler VAE
|
if 'encoder.down.2.downsample.conv.weight' not in sd: #Stable diffusion x4 upscaler VAE
|
||||||
ddconfig['ch_mult'] = [1, 2, 4]
|
ddconfig['ch_mult'] = [1, 2, 4]
|
||||||
self.downscale_ratio = 4
|
self.downscale_ratio = 4
|
||||||
|
self.upscale_ratio = 4
|
||||||
|
|
||||||
self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4)
|
self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4)
|
||||||
else:
|
else:
|
||||||
@ -213,6 +239,15 @@ class VAE:
|
|||||||
|
|
||||||
self.patcher = comfy.model_patcher.ModelPatcher(self.first_stage_model, load_device=self.device, offload_device=offload_device)
|
self.patcher = comfy.model_patcher.ModelPatcher(self.first_stage_model, load_device=self.device, offload_device=offload_device)
|
||||||
|
|
||||||
|
def vae_encode_crop_pixels(self, pixels):
|
||||||
|
x = (pixels.shape[1] // self.downscale_ratio) * self.downscale_ratio
|
||||||
|
y = (pixels.shape[2] // self.downscale_ratio) * self.downscale_ratio
|
||||||
|
if pixels.shape[1] != x or pixels.shape[2] != y:
|
||||||
|
x_offset = (pixels.shape[1] % self.downscale_ratio) // 2
|
||||||
|
y_offset = (pixels.shape[2] % self.downscale_ratio) // 2
|
||||||
|
pixels = pixels[:, x_offset:x + x_offset, y_offset:y + y_offset, :]
|
||||||
|
return pixels
|
||||||
|
|
||||||
def decode_tiled_(self, samples, tile_x=64, tile_y=64, overlap = 16):
|
def decode_tiled_(self, samples, tile_x=64, tile_y=64, overlap = 16):
|
||||||
steps = samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x, tile_y, overlap)
|
steps = samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x, tile_y, overlap)
|
||||||
steps += samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x // 2, tile_y * 2, overlap)
|
steps += samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x // 2, tile_y * 2, overlap)
|
||||||
@ -221,9 +256,9 @@ class VAE:
|
|||||||
|
|
||||||
decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).float()
|
decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).float()
|
||||||
output = self.process_output(
|
output = self.process_output(
|
||||||
(comfy.utils.tiled_scale(samples, decode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = self.downscale_ratio, output_device=self.output_device, pbar = pbar) +
|
(comfy.utils.tiled_scale(samples, decode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = self.upscale_ratio, output_device=self.output_device, pbar = pbar) +
|
||||||
comfy.utils.tiled_scale(samples, decode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = self.downscale_ratio, output_device=self.output_device, pbar = pbar) +
|
comfy.utils.tiled_scale(samples, decode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = self.upscale_ratio, output_device=self.output_device, pbar = pbar) +
|
||||||
comfy.utils.tiled_scale(samples, decode_fn, tile_x, tile_y, overlap, upscale_amount = self.downscale_ratio, output_device=self.output_device, pbar = pbar))
|
comfy.utils.tiled_scale(samples, decode_fn, tile_x, tile_y, overlap, upscale_amount = self.upscale_ratio, output_device=self.output_device, pbar = pbar))
|
||||||
/ 3.0)
|
/ 3.0)
|
||||||
return output
|
return output
|
||||||
|
|
||||||
@ -248,7 +283,7 @@ class VAE:
|
|||||||
batch_number = int(free_memory / memory_used)
|
batch_number = int(free_memory / memory_used)
|
||||||
batch_number = max(1, batch_number)
|
batch_number = max(1, batch_number)
|
||||||
|
|
||||||
pixel_samples = torch.empty((samples_in.shape[0], 3, round(samples_in.shape[2] * self.downscale_ratio), round(samples_in.shape[3] * self.downscale_ratio)), device=self.output_device)
|
pixel_samples = torch.empty((samples_in.shape[0], 3, round(samples_in.shape[2] * self.upscale_ratio), round(samples_in.shape[3] * self.upscale_ratio)), device=self.output_device)
|
||||||
for x in range(0, samples_in.shape[0], batch_number):
|
for x in range(0, samples_in.shape[0], batch_number):
|
||||||
samples = samples_in[x:x+batch_number].to(self.vae_dtype).to(self.device)
|
samples = samples_in[x:x+batch_number].to(self.vae_dtype).to(self.device)
|
||||||
pixel_samples[x:x+batch_number] = self.process_output(self.first_stage_model.decode(samples).to(self.output_device).float())
|
pixel_samples[x:x+batch_number] = self.process_output(self.first_stage_model.decode(samples).to(self.output_device).float())
|
||||||
@ -265,6 +300,7 @@ class VAE:
|
|||||||
return output.movedim(1,-1)
|
return output.movedim(1,-1)
|
||||||
|
|
||||||
def encode(self, pixel_samples):
|
def encode(self, pixel_samples):
|
||||||
|
pixel_samples = self.vae_encode_crop_pixels(pixel_samples)
|
||||||
pixel_samples = pixel_samples.movedim(-1,1)
|
pixel_samples = pixel_samples.movedim(-1,1)
|
||||||
try:
|
try:
|
||||||
memory_used = self.memory_used_encode(pixel_samples.shape, self.vae_dtype)
|
memory_used = self.memory_used_encode(pixel_samples.shape, self.vae_dtype)
|
||||||
@ -284,6 +320,7 @@ class VAE:
|
|||||||
return samples
|
return samples
|
||||||
|
|
||||||
def encode_tiled(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):
|
def encode_tiled(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):
|
||||||
|
pixel_samples = self.vae_encode_crop_pixels(pixel_samples)
|
||||||
model_management.load_model_gpu(self.patcher)
|
model_management.load_model_gpu(self.patcher)
|
||||||
pixel_samples = pixel_samples.movedim(-1,1)
|
pixel_samples = pixel_samples.movedim(-1,1)
|
||||||
samples = self.encode_tiled_(pixel_samples, tile_x=tile_x, tile_y=tile_y, overlap=overlap)
|
samples = self.encode_tiled_(pixel_samples, tile_x=tile_x, tile_y=tile_y, overlap=overlap)
|
||||||
@ -460,9 +497,6 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
|
|||||||
parameters = comfy.utils.calculate_parameters(sd, "model.diffusion_model.")
|
parameters = comfy.utils.calculate_parameters(sd, "model.diffusion_model.")
|
||||||
load_device = model_management.get_torch_device()
|
load_device = model_management.get_torch_device()
|
||||||
|
|
||||||
class WeightsLoader(torch.nn.Module):
|
|
||||||
pass
|
|
||||||
|
|
||||||
model_config = model_detection.model_config_from_unet(sd, "model.diffusion_model.")
|
model_config = model_detection.model_config_from_unet(sd, "model.diffusion_model.")
|
||||||
unet_dtype = model_management.unet_dtype(model_params=parameters, supported_dtypes=model_config.supported_inference_dtypes)
|
unet_dtype = model_management.unet_dtype(model_params=parameters, supported_dtypes=model_config.supported_inference_dtypes)
|
||||||
manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)
|
manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)
|
||||||
@ -487,14 +521,17 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
|
|||||||
vae = VAE(sd=vae_sd)
|
vae = VAE(sd=vae_sd)
|
||||||
|
|
||||||
if output_clip:
|
if output_clip:
|
||||||
w = WeightsLoader()
|
|
||||||
clip_target = model_config.clip_target()
|
clip_target = model_config.clip_target()
|
||||||
if clip_target is not None:
|
if clip_target is not None:
|
||||||
sd = model_config.process_clip_state_dict(sd)
|
clip_sd = model_config.process_clip_state_dict(sd)
|
||||||
if any(k.startswith('cond_stage_model.') for k in sd):
|
if len(clip_sd) > 0:
|
||||||
clip = CLIP(clip_target, embedding_directory=embedding_directory)
|
clip = CLIP(clip_target, embedding_directory=embedding_directory)
|
||||||
w.cond_stage_model = clip.cond_stage_model
|
m, u = clip.load_sd(clip_sd, full_model=True)
|
||||||
load_model_weights(w, sd)
|
if len(m) > 0:
|
||||||
|
print("clip missing:", m)
|
||||||
|
|
||||||
|
if len(u) > 0:
|
||||||
|
print("clip unexpected:", u)
|
||||||
else:
|
else:
|
||||||
print("no CLIP/text encoder weights in checkpoint, the text encoder model will not be loaded.")
|
print("no CLIP/text encoder weights in checkpoint, the text encoder model will not be loaded.")
|
||||||
|
|
||||||
|
|||||||
@ -40,8 +40,8 @@ class SD15(supported_models_base.BASE):
|
|||||||
state_dict['cond_stage_model.transformer.text_model.embeddings.position_ids'] = ids.round()
|
state_dict['cond_stage_model.transformer.text_model.embeddings.position_ids'] = ids.round()
|
||||||
|
|
||||||
replace_prefix = {}
|
replace_prefix = {}
|
||||||
replace_prefix["cond_stage_model."] = "cond_stage_model.clip_l."
|
replace_prefix["cond_stage_model."] = "clip_l."
|
||||||
state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix, filter_keys=True)
|
||||||
return state_dict
|
return state_dict
|
||||||
|
|
||||||
def process_clip_state_dict_for_saving(self, state_dict):
|
def process_clip_state_dict_for_saving(self, state_dict):
|
||||||
@ -72,10 +72,10 @@ class SD20(supported_models_base.BASE):
|
|||||||
|
|
||||||
def process_clip_state_dict(self, state_dict):
|
def process_clip_state_dict(self, state_dict):
|
||||||
replace_prefix = {}
|
replace_prefix = {}
|
||||||
replace_prefix["conditioner.embedders.0.model."] = "cond_stage_model.model." #SD2 in sgm format
|
replace_prefix["conditioner.embedders.0.model."] = "clip_h." #SD2 in sgm format
|
||||||
state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
replace_prefix["cond_stage_model.model."] = "clip_h."
|
||||||
|
state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix, filter_keys=True)
|
||||||
state_dict = utils.transformers_convert(state_dict, "cond_stage_model.model.", "cond_stage_model.clip_h.transformer.text_model.", 24)
|
state_dict = utils.transformers_convert(state_dict, "clip_h.", "clip_h.transformer.text_model.", 24)
|
||||||
return state_dict
|
return state_dict
|
||||||
|
|
||||||
def process_clip_state_dict_for_saving(self, state_dict):
|
def process_clip_state_dict_for_saving(self, state_dict):
|
||||||
@ -131,11 +131,10 @@ class SDXLRefiner(supported_models_base.BASE):
|
|||||||
def process_clip_state_dict(self, state_dict):
|
def process_clip_state_dict(self, state_dict):
|
||||||
keys_to_replace = {}
|
keys_to_replace = {}
|
||||||
replace_prefix = {}
|
replace_prefix = {}
|
||||||
|
replace_prefix["conditioner.embedders.0.model."] = "clip_g."
|
||||||
|
state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix, filter_keys=True)
|
||||||
|
|
||||||
state_dict = utils.transformers_convert(state_dict, "conditioner.embedders.0.model.", "cond_stage_model.clip_g.transformer.text_model.", 32)
|
state_dict = utils.transformers_convert(state_dict, "clip_g.", "clip_g.transformer.text_model.", 32)
|
||||||
keys_to_replace["conditioner.embedders.0.model.text_projection"] = "cond_stage_model.clip_g.text_projection"
|
|
||||||
keys_to_replace["conditioner.embedders.0.model.logit_scale"] = "cond_stage_model.clip_g.logit_scale"
|
|
||||||
|
|
||||||
state_dict = utils.state_dict_key_replace(state_dict, keys_to_replace)
|
state_dict = utils.state_dict_key_replace(state_dict, keys_to_replace)
|
||||||
return state_dict
|
return state_dict
|
||||||
|
|
||||||
@ -179,13 +178,13 @@ class SDXL(supported_models_base.BASE):
|
|||||||
keys_to_replace = {}
|
keys_to_replace = {}
|
||||||
replace_prefix = {}
|
replace_prefix = {}
|
||||||
|
|
||||||
replace_prefix["conditioner.embedders.0.transformer.text_model"] = "cond_stage_model.clip_l.transformer.text_model"
|
replace_prefix["conditioner.embedders.0.transformer.text_model"] = "clip_l.transformer.text_model"
|
||||||
state_dict = utils.transformers_convert(state_dict, "conditioner.embedders.1.model.", "cond_stage_model.clip_g.transformer.text_model.", 32)
|
replace_prefix["conditioner.embedders.1.model."] = "clip_g."
|
||||||
keys_to_replace["conditioner.embedders.1.model.text_projection"] = "cond_stage_model.clip_g.text_projection"
|
state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix, filter_keys=True)
|
||||||
keys_to_replace["conditioner.embedders.1.model.text_projection.weight"] = "cond_stage_model.clip_g.text_projection"
|
|
||||||
keys_to_replace["conditioner.embedders.1.model.logit_scale"] = "cond_stage_model.clip_g.logit_scale"
|
state_dict = utils.transformers_convert(state_dict, "clip_g.", "clip_g.transformer.text_model.", 32)
|
||||||
|
keys_to_replace["clip_g.text_projection.weight"] = "clip_g.text_projection"
|
||||||
|
|
||||||
state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
|
||||||
state_dict = utils.state_dict_key_replace(state_dict, keys_to_replace)
|
state_dict = utils.state_dict_key_replace(state_dict, keys_to_replace)
|
||||||
return state_dict
|
return state_dict
|
||||||
|
|
||||||
@ -316,6 +315,14 @@ class Stable_Cascade_C(supported_models_base.BASE):
|
|||||||
latent_format = latent_formats.SC_Prior
|
latent_format = latent_formats.SC_Prior
|
||||||
supported_inference_dtypes = [torch.bfloat16, torch.float32]
|
supported_inference_dtypes = [torch.bfloat16, torch.float32]
|
||||||
|
|
||||||
|
sampling_settings = {
|
||||||
|
"shift": 2.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
vae_key_prefix = ["vae."]
|
||||||
|
text_encoder_key_prefix = ["text_encoder."]
|
||||||
|
clip_vision_prefix = "clip_l_vision."
|
||||||
|
|
||||||
def process_unet_state_dict(self, state_dict):
|
def process_unet_state_dict(self, state_dict):
|
||||||
key_list = list(state_dict.keys())
|
key_list = list(state_dict.keys())
|
||||||
for y in ["weight", "bias"]:
|
for y in ["weight", "bias"]:
|
||||||
@ -348,6 +355,12 @@ class Stable_Cascade_B(Stable_Cascade_C):
|
|||||||
latent_format = latent_formats.SC_B
|
latent_format = latent_formats.SC_B
|
||||||
supported_inference_dtypes = [torch.float16, torch.bfloat16, torch.float32]
|
supported_inference_dtypes = [torch.float16, torch.bfloat16, torch.float32]
|
||||||
|
|
||||||
|
sampling_settings = {
|
||||||
|
"shift": 1.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
clip_vision_prefix = None
|
||||||
|
|
||||||
def get_model(self, state_dict, prefix="", device=None):
|
def get_model(self, state_dict, prefix="", device=None):
|
||||||
out = model_base.StableCascade_B(self, device=device)
|
out = model_base.StableCascade_B(self, device=device)
|
||||||
return out
|
return out
|
||||||
|
|||||||
@ -22,6 +22,7 @@ class BASE:
|
|||||||
sampling_settings = {}
|
sampling_settings = {}
|
||||||
latent_format = latent_formats.LatentFormat
|
latent_format = latent_formats.LatentFormat
|
||||||
vae_key_prefix = ["first_stage_model."]
|
vae_key_prefix = ["first_stage_model."]
|
||||||
|
text_encoder_key_prefix = ["cond_stage_model."]
|
||||||
supported_inference_dtypes = [torch.float16, torch.bfloat16, torch.float32]
|
supported_inference_dtypes = [torch.float16, torch.bfloat16, torch.float32]
|
||||||
|
|
||||||
manual_cast_dtype = None
|
manual_cast_dtype = None
|
||||||
@ -55,6 +56,7 @@ class BASE:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
def process_clip_state_dict(self, state_dict):
|
def process_clip_state_dict(self, state_dict):
|
||||||
|
state_dict = utils.state_dict_prefix_replace(state_dict, {k: "" for k in self.text_encoder_key_prefix}, filter_keys=True)
|
||||||
return state_dict
|
return state_dict
|
||||||
|
|
||||||
def process_unet_state_dict(self, state_dict):
|
def process_unet_state_dict(self, state_dict):
|
||||||
@ -64,7 +66,7 @@ class BASE:
|
|||||||
return state_dict
|
return state_dict
|
||||||
|
|
||||||
def process_clip_state_dict_for_saving(self, state_dict):
|
def process_clip_state_dict_for_saving(self, state_dict):
|
||||||
replace_prefix = {"": "cond_stage_model."}
|
replace_prefix = {"": self.text_encoder_key_prefix[0]}
|
||||||
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
||||||
|
|
||||||
def process_clip_vision_state_dict_for_saving(self, state_dict):
|
def process_clip_vision_state_dict_for_saving(self, state_dict):
|
||||||
@ -78,7 +80,7 @@ class BASE:
|
|||||||
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
||||||
|
|
||||||
def process_vae_state_dict_for_saving(self, state_dict):
|
def process_vae_state_dict_for_saving(self, state_dict):
|
||||||
replace_prefix = {"": "first_stage_model."}
|
replace_prefix = {"": self.vae_key_prefix[0]}
|
||||||
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
||||||
|
|
||||||
def set_inference_dtype(self, dtype, manual_cast_dtype):
|
def set_inference_dtype(self, dtype, manual_cast_dtype):
|
||||||
|
|||||||
@ -17,6 +17,10 @@ class LCM(comfy.model_sampling.EPS):
|
|||||||
|
|
||||||
return c_out * x0 + c_skip * model_input
|
return c_out * x0 + c_skip * model_input
|
||||||
|
|
||||||
|
class X0(comfy.model_sampling.EPS):
|
||||||
|
def calculate_denoised(self, sigma, model_output, model_input):
|
||||||
|
return model_output
|
||||||
|
|
||||||
class ModelSamplingDiscreteDistilled(comfy.model_sampling.ModelSamplingDiscrete):
|
class ModelSamplingDiscreteDistilled(comfy.model_sampling.ModelSamplingDiscrete):
|
||||||
original_timesteps = 50
|
original_timesteps = 50
|
||||||
|
|
||||||
@ -68,7 +72,7 @@ class ModelSamplingDiscrete:
|
|||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
return {"required": { "model": ("MODEL",),
|
return {"required": { "model": ("MODEL",),
|
||||||
"sampling": (["eps", "v_prediction", "lcm"],),
|
"sampling": (["eps", "v_prediction", "lcm", "x0"],),
|
||||||
"zsnr": ("BOOLEAN", {"default": False}),
|
"zsnr": ("BOOLEAN", {"default": False}),
|
||||||
}}
|
}}
|
||||||
|
|
||||||
@ -88,6 +92,8 @@ class ModelSamplingDiscrete:
|
|||||||
elif sampling == "lcm":
|
elif sampling == "lcm":
|
||||||
sampling_type = LCM
|
sampling_type = LCM
|
||||||
sampling_base = ModelSamplingDiscreteDistilled
|
sampling_base = ModelSamplingDiscreteDistilled
|
||||||
|
elif sampling == "x0":
|
||||||
|
sampling_type = X0
|
||||||
|
|
||||||
class ModelSamplingAdvanced(sampling_base, sampling_type):
|
class ModelSamplingAdvanced(sampling_base, sampling_type):
|
||||||
pass
|
pass
|
||||||
@ -99,6 +105,32 @@ class ModelSamplingDiscrete:
|
|||||||
m.add_object_patch("model_sampling", model_sampling)
|
m.add_object_patch("model_sampling", model_sampling)
|
||||||
return (m, )
|
return (m, )
|
||||||
|
|
||||||
|
class ModelSamplingStableCascade:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "model": ("MODEL",),
|
||||||
|
"shift": ("FLOAT", {"default": 2.0, "min": 0.0, "max": 100.0, "step":0.01}),
|
||||||
|
}}
|
||||||
|
|
||||||
|
RETURN_TYPES = ("MODEL",)
|
||||||
|
FUNCTION = "patch"
|
||||||
|
|
||||||
|
CATEGORY = "advanced/model"
|
||||||
|
|
||||||
|
def patch(self, model, shift):
|
||||||
|
m = model.clone()
|
||||||
|
|
||||||
|
sampling_base = comfy.model_sampling.StableCascadeSampling
|
||||||
|
sampling_type = comfy.model_sampling.EPS
|
||||||
|
|
||||||
|
class ModelSamplingAdvanced(sampling_base, sampling_type):
|
||||||
|
pass
|
||||||
|
|
||||||
|
model_sampling = ModelSamplingAdvanced(model.model.model_config)
|
||||||
|
model_sampling.set_parameters(shift)
|
||||||
|
m.add_object_patch("model_sampling", model_sampling)
|
||||||
|
return (m, )
|
||||||
|
|
||||||
class ModelSamplingContinuousEDM:
|
class ModelSamplingContinuousEDM:
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
@ -171,5 +203,6 @@ class RescaleCFG:
|
|||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
"ModelSamplingDiscrete": ModelSamplingDiscrete,
|
"ModelSamplingDiscrete": ModelSamplingDiscrete,
|
||||||
"ModelSamplingContinuousEDM": ModelSamplingContinuousEDM,
|
"ModelSamplingContinuousEDM": ModelSamplingContinuousEDM,
|
||||||
|
"ModelSamplingStableCascade": ModelSamplingStableCascade,
|
||||||
"RescaleCFG": RescaleCFG,
|
"RescaleCFG": RescaleCFG,
|
||||||
}
|
}
|
||||||
|
|||||||
@ -18,6 +18,7 @@
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
import nodes
|
import nodes
|
||||||
|
import comfy.utils
|
||||||
|
|
||||||
|
|
||||||
class StableCascade_EmptyLatentImage:
|
class StableCascade_EmptyLatentImage:
|
||||||
@ -29,8 +30,8 @@ class StableCascade_EmptyLatentImage:
|
|||||||
return {"required": {
|
return {"required": {
|
||||||
"width": ("INT", {"default": 1024, "min": 256, "max": nodes.MAX_RESOLUTION, "step": 8}),
|
"width": ("INT", {"default": 1024, "min": 256, "max": nodes.MAX_RESOLUTION, "step": 8}),
|
||||||
"height": ("INT", {"default": 1024, "min": 256, "max": nodes.MAX_RESOLUTION, "step": 8}),
|
"height": ("INT", {"default": 1024, "min": 256, "max": nodes.MAX_RESOLUTION, "step": 8}),
|
||||||
"compression": ("INT", {"default": 42, "min": 32, "max": 64, "step": 1}),
|
"compression": ("INT", {"default": 42, "min": 4, "max": 128, "step": 1}),
|
||||||
"batch_size": ("INT", {"default": 1, "min": 1, "max": 64})
|
"batch_size": ("INT", {"default": 1, "min": 1, "max": 4096})
|
||||||
}}
|
}}
|
||||||
RETURN_TYPES = ("LATENT", "LATENT")
|
RETURN_TYPES = ("LATENT", "LATENT")
|
||||||
RETURN_NAMES = ("stage_c", "stage_b")
|
RETURN_NAMES = ("stage_c", "stage_b")
|
||||||
@ -47,6 +48,39 @@ class StableCascade_EmptyLatentImage:
|
|||||||
"samples": b_latent,
|
"samples": b_latent,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
class StableCascade_StageC_VAEEncode:
|
||||||
|
def __init__(self, device="cpu"):
|
||||||
|
self.device = device
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": {
|
||||||
|
"image": ("IMAGE",),
|
||||||
|
"vae": ("VAE", ),
|
||||||
|
"compression": ("INT", {"default": 42, "min": 4, "max": 128, "step": 1}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("LATENT", "LATENT")
|
||||||
|
RETURN_NAMES = ("stage_c", "stage_b")
|
||||||
|
FUNCTION = "generate"
|
||||||
|
|
||||||
|
CATEGORY = "_for_testing/stable_cascade"
|
||||||
|
|
||||||
|
def generate(self, image, vae, compression):
|
||||||
|
width = image.shape[-2]
|
||||||
|
height = image.shape[-3]
|
||||||
|
out_width = (width // compression) * vae.downscale_ratio
|
||||||
|
out_height = (height // compression) * vae.downscale_ratio
|
||||||
|
|
||||||
|
s = comfy.utils.common_upscale(image.movedim(-1,1), out_width, out_height, "bicubic", "center").movedim(1,-1)
|
||||||
|
|
||||||
|
c_latent = vae.encode(s[:,:,:,:3])
|
||||||
|
b_latent = torch.zeros([c_latent.shape[0], 4, height // 4, width // 4])
|
||||||
|
return ({
|
||||||
|
"samples": c_latent,
|
||||||
|
}, {
|
||||||
|
"samples": b_latent,
|
||||||
|
})
|
||||||
|
|
||||||
class StableCascade_StageB_Conditioning:
|
class StableCascade_StageB_Conditioning:
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
@ -71,4 +105,5 @@ class StableCascade_StageB_Conditioning:
|
|||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
"StableCascade_EmptyLatentImage": StableCascade_EmptyLatentImage,
|
"StableCascade_EmptyLatentImage": StableCascade_EmptyLatentImage,
|
||||||
"StableCascade_StageB_Conditioning": StableCascade_StageB_Conditioning,
|
"StableCascade_StageB_Conditioning": StableCascade_StageB_Conditioning,
|
||||||
|
"StableCascade_StageC_VAEEncode": StableCascade_StageC_VAEEncode,
|
||||||
}
|
}
|
||||||
|
|||||||
20
nodes.py
20
nodes.py
@ -309,18 +309,7 @@ class VAEEncode:
|
|||||||
|
|
||||||
CATEGORY = "latent"
|
CATEGORY = "latent"
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def vae_encode_crop_pixels(pixels):
|
|
||||||
x = (pixels.shape[1] // 8) * 8
|
|
||||||
y = (pixels.shape[2] // 8) * 8
|
|
||||||
if pixels.shape[1] != x or pixels.shape[2] != y:
|
|
||||||
x_offset = (pixels.shape[1] % 8) // 2
|
|
||||||
y_offset = (pixels.shape[2] % 8) // 2
|
|
||||||
pixels = pixels[:, x_offset:x + x_offset, y_offset:y + y_offset, :]
|
|
||||||
return pixels
|
|
||||||
|
|
||||||
def encode(self, vae, pixels):
|
def encode(self, vae, pixels):
|
||||||
pixels = self.vae_encode_crop_pixels(pixels)
|
|
||||||
t = vae.encode(pixels[:,:,:,:3])
|
t = vae.encode(pixels[:,:,:,:3])
|
||||||
return ({"samples":t}, )
|
return ({"samples":t}, )
|
||||||
|
|
||||||
@ -336,7 +325,6 @@ class VAEEncodeTiled:
|
|||||||
CATEGORY = "_for_testing"
|
CATEGORY = "_for_testing"
|
||||||
|
|
||||||
def encode(self, vae, pixels, tile_size):
|
def encode(self, vae, pixels, tile_size):
|
||||||
pixels = VAEEncode.vae_encode_crop_pixels(pixels)
|
|
||||||
t = vae.encode_tiled(pixels[:,:,:,:3], tile_x=tile_size, tile_y=tile_size, )
|
t = vae.encode_tiled(pixels[:,:,:,:3], tile_x=tile_size, tile_y=tile_size, )
|
||||||
return ({"samples":t}, )
|
return ({"samples":t}, )
|
||||||
|
|
||||||
@ -350,14 +338,14 @@ class VAEEncodeForInpaint:
|
|||||||
CATEGORY = "latent/inpaint"
|
CATEGORY = "latent/inpaint"
|
||||||
|
|
||||||
def encode(self, vae, pixels, mask, grow_mask_by=6):
|
def encode(self, vae, pixels, mask, grow_mask_by=6):
|
||||||
x = (pixels.shape[1] // 8) * 8
|
x = (pixels.shape[1] // vae.downscale_ratio) * vae.downscale_ratio
|
||||||
y = (pixels.shape[2] // 8) * 8
|
y = (pixels.shape[2] // vae.downscale_ratio) * vae.downscale_ratio
|
||||||
mask = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(pixels.shape[1], pixels.shape[2]), mode="bilinear")
|
mask = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(pixels.shape[1], pixels.shape[2]), mode="bilinear")
|
||||||
|
|
||||||
pixels = pixels.clone()
|
pixels = pixels.clone()
|
||||||
if pixels.shape[1] != x or pixels.shape[2] != y:
|
if pixels.shape[1] != x or pixels.shape[2] != y:
|
||||||
x_offset = (pixels.shape[1] % 8) // 2
|
x_offset = (pixels.shape[1] % vae.downscale_ratio) // 2
|
||||||
y_offset = (pixels.shape[2] % 8) // 2
|
y_offset = (pixels.shape[2] % vae.downscale_ratio) // 2
|
||||||
pixels = pixels[:,x_offset:x + x_offset, y_offset:y + y_offset,:]
|
pixels = pixels[:,x_offset:x + x_offset, y_offset:y + y_offset,:]
|
||||||
mask = mask[:,:,x_offset:x + x_offset, y_offset:y + y_offset]
|
mask = mask[:,:,x_offset:x + x_offset, y_offset:y + y_offset]
|
||||||
|
|
||||||
|
|||||||
@ -24,7 +24,7 @@ export function getPngMetadata(file) {
|
|||||||
const length = dataView.getUint32(offset);
|
const length = dataView.getUint32(offset);
|
||||||
// Get the chunk type
|
// Get the chunk type
|
||||||
const type = String.fromCharCode(...pngData.slice(offset + 4, offset + 8));
|
const type = String.fromCharCode(...pngData.slice(offset + 4, offset + 8));
|
||||||
if (type === "tEXt" || type == "comf") {
|
if (type === "tEXt" || type == "comf" || type === "iTXt") {
|
||||||
// Get the keyword
|
// Get the keyword
|
||||||
let keyword_end = offset + 8;
|
let keyword_end = offset + 8;
|
||||||
while (pngData[keyword_end] !== 0) {
|
while (pngData[keyword_end] !== 0) {
|
||||||
@ -33,7 +33,7 @@ export function getPngMetadata(file) {
|
|||||||
const keyword = String.fromCharCode(...pngData.slice(offset + 8, keyword_end));
|
const keyword = String.fromCharCode(...pngData.slice(offset + 8, keyword_end));
|
||||||
// Get the text
|
// Get the text
|
||||||
const contentArraySegment = pngData.slice(keyword_end + 1, offset + 8 + length);
|
const contentArraySegment = pngData.slice(keyword_end + 1, offset + 8 + length);
|
||||||
const contentJson = Array.from(contentArraySegment).map(s=>String.fromCharCode(s)).join('')
|
const contentJson = new TextDecoder("utf-8").decode(contentArraySegment);
|
||||||
txt_chunks[keyword] = contentJson;
|
txt_chunks[keyword] = contentJson;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user