diff --git a/.github/workflows/windows_release_nightly_pytorch.yml b/.github/workflows/windows_release_nightly_pytorch.yml
index 672a7f220..fa24a985c 100644
--- a/.github/workflows/windows_release_nightly_pytorch.yml
+++ b/.github/workflows/windows_release_nightly_pytorch.yml
@@ -7,7 +7,7 @@ on:
description: 'cuda version'
required: true
type: string
- default: "121"
+ default: "124"
python_minor:
description: 'python minor version'
@@ -19,7 +19,7 @@ on:
description: 'python patch version'
required: true
type: string
- default: "2"
+ default: "3"
# push:
# branches:
# - master
@@ -49,7 +49,7 @@ jobs:
echo 'import site' >> ./python3${{ inputs.python_minor }}._pth
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
./python.exe get-pip.py
- python -m pip wheel torch torchvision torchaudio mpmath==1.3.0 --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
+ python -m pip wheel torch torchvision torchaudio mpmath==1.3.0 numpy==1.26.4 --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
ls ../temp_wheel_dir
./python.exe -s -m pip install --pre ../temp_wheel_dir/*
sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth
diff --git a/README.md b/README.md
index ba1e844b3..cf32014b2 100644
--- a/README.md
+++ b/README.md
@@ -41,29 +41,32 @@ Workflow examples can be found on the [Examples page](https://comfyanonymous.git
## Shortcuts
-| Keybind | Explanation |
-|---------------------------|--------------------------------------------------------------------------------------------------------------------|
-| Ctrl + Enter | Queue up current graph for generation |
-| Ctrl + Shift + Enter | Queue up current graph as first for generation |
-| Ctrl + Z/Ctrl + Y | Undo/Redo |
-| Ctrl + S | Save workflow |
-| Ctrl + O | Load workflow |
-| Ctrl + A | Select all nodes |
-| Alt + C | Collapse/uncollapse selected nodes |
-| Ctrl + M | Mute/unmute selected nodes |
-| Ctrl + B | Bypass selected nodes (acts like the node was removed from the graph and the wires reconnected through) |
-| Delete/Backspace | Delete selected nodes |
-| Ctrl + Delete/Backspace | Delete the current graph |
-| Space | Move the canvas around when held and moving the cursor |
-| Ctrl/Shift + Click | Add clicked node to selection |
-| Ctrl + C/Ctrl + V | Copy and paste selected nodes (without maintaining connections to outputs of unselected nodes) |
-| Ctrl + C/Ctrl + Shift + V | Copy and paste selected nodes (maintaining connections from outputs of unselected nodes to inputs of pasted nodes) |
-| Shift + Drag | Move multiple selected nodes at the same time |
-| Ctrl + D | Load default graph |
-| Q | Toggle visibility of the queue |
-| H | Toggle visibility of history |
-| R | Refresh graph |
-| Double-Click LMB | Open node quick search palette |
+| Keybind | Explanation |
+|------------------------------------|--------------------------------------------------------------------------------------------------------------------|
+| Ctrl + Enter | Queue up current graph for generation |
+| Ctrl + Shift + Enter | Queue up current graph as first for generation |
+| Ctrl + Z/Ctrl + Y | Undo/Redo |
+| Ctrl + S | Save workflow |
+| Ctrl + O | Load workflow |
+| Ctrl + A | Select all nodes |
+| Alt + C | Collapse/uncollapse selected nodes |
+| Ctrl + M | Mute/unmute selected nodes |
+| Ctrl + B | Bypass selected nodes (acts like the node was removed from the graph and the wires reconnected through) |
+| Delete/Backspace | Delete selected nodes |
+| Ctrl + Backspace | Delete the current graph |
+| Space | Move the canvas around when held and moving the cursor |
+| Ctrl/Shift + Click | Add clicked node to selection |
+| Ctrl + C/Ctrl + V | Copy and paste selected nodes (without maintaining connections to outputs of unselected nodes) |
+| Ctrl + C/Ctrl + Shift + V | Copy and paste selected nodes (maintaining connections from outputs of unselected nodes to inputs of pasted nodes) |
+| Shift + Drag | Move multiple selected nodes at the same time |
+| Ctrl + D | Load default graph |
+| Alt + `+` | Canvas Zoom in |
+| Alt + `-` | Canvas Zoom out |
+| Ctrl + Shift + LMB + Vertical drag | Canvas Zoom in/out |
+| Q | Toggle visibility of the queue |
+| H | Toggle visibility of history |
+| R | Refresh graph |
+| Double-Click LMB | Open node quick search palette |
Ctrl can also be replaced with Cmd instead for macOS users
@@ -99,7 +102,7 @@ Put your VAE in: models/vae
### AMD GPUs (Linux only)
AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version:
-```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.7```
+```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.0```
This is the command to install the nightly with ROCm 6.0 which might have some performance improvements:
@@ -133,7 +136,16 @@ After this you should have everything installed and can proceed to running Comfy
### Others:
-#### [Intel Arc](https://github.com/comfyanonymous/ComfyUI/discussions/476)
+#### Intel GPUs
+
+Intel GPU support is available for all Intel GPUs supported by Intel's Extension for Pytorch (IPEX) with the support requirements listed in the [Installation](https://intel.github.io/intel-extension-for-pytorch/index.html#installation?platform=gpu) page. Choose your platform and method of install and follow the instructions. The steps are as follows:
+
+1. Start by installing the drivers or kernel listed or newer in the Installation page of IPEX linked above for Windows and Linux if needed.
+1. Follow the instructions to install [Intel's oneAPI Basekit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit-download.html) for your platform.
+1. Install the packages for IPEX using the instructions provided in the Installation page for your platform.
+1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux and run ComfyUI normally as described above after everything is installed.
+
+Additional discussion and help can be found [here](https://github.com/comfyanonymous/ComfyUI/discussions/476).
#### Apple Mac silicon
@@ -195,20 +207,20 @@ To use a textual inversion concepts/embeddings in a text prompt put them in the
```embedding:embedding_filename.pt```
-## How to increase generation speed?
-
-Make sure you use the regular loaders/Load Checkpoint node to load checkpoints. It will auto pick the right settings depending on your GPU.
-
-You can set this command line setting to disable the upcasting to fp32 in some cross attention operations which will increase your speed. Note that this will very likely give you black images on SD2.x models. If you use xformers or pytorch attention this option does not do anything.
-
-```--dont-upcast-attention```
-
## How to show high-quality previews?
Use ```--preview-method auto``` to enable previews.
The default installation includes a fast latent preview method that's low-resolution. To enable higher-quality previews with [TAESD](https://github.com/madebyollin/taesd), download the [taesd_decoder.pth](https://github.com/madebyollin/taesd/raw/main/taesd_decoder.pth) (for SD1.x and SD2.x) and [taesdxl_decoder.pth](https://github.com/madebyollin/taesd/raw/main/taesdxl_decoder.pth) (for SDXL) models and place them in the `models/vae_approx` folder. Once they're installed, restart ComfyUI to enable high-quality previews.
+## How to use TLS/SSL?
+Generate a self-signed certificate (not appropriate for shared/production use) and key by running the command: `openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -sha256 -days 3650 -nodes -subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=CommonNameOrHostname"`
+
+Use `--tls-keyfile key.pem --tls-certfile cert.pem` to enable TLS/SSL, the app will now be accessible with `https://...` instead of `http://...`.
+
+> Note: Windows users can use [alexisrolland/docker-openssl](https://github.com/alexisrolland/docker-openssl) or one of the [3rd party binary distributions](https://wiki.openssl.org/index.php/Binaries) to run the command example above.
+
If you use a container, note that the volume mount `-v` can be a relative path so `... -v ".\:/openssl-certs" ...` would create the key & cert files in the current directory of your command prompt or powershell terminal.
+
## Support and dev channel
[Matrix space: #comfyui_space:matrix.org](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) (it's like discord but open source).
diff --git a/comfy/cldm/cldm.py b/comfy/cldm/cldm.py
index 5eee5a51c..28076dd92 100644
--- a/comfy/cldm/cldm.py
+++ b/comfy/cldm/cldm.py
@@ -52,6 +52,7 @@ class ControlNet(nn.Module):
adm_in_channels=None,
transformer_depth_middle=None,
transformer_depth_output=None,
+ attn_precision=None,
device=None,
operations=comfy.ops.disable_weight_init,
**kwargs,
@@ -202,7 +203,7 @@ class ControlNet(nn.Module):
SpatialTransformer(
ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim,
disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
- use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
+ use_checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations
)
)
self.input_blocks.append(TimestepEmbedSequential(*layers))
@@ -262,7 +263,7 @@ class ControlNet(nn.Module):
mid_block += [SpatialTransformer( # always uses a self-attn
ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,
disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer,
- use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
+ use_checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations
),
ResBlock(
ch,
diff --git a/comfy/cli_args.py b/comfy/cli_args.py
index 353bb51e7..b8ac9bc69 100644
--- a/comfy/cli_args.py
+++ b/comfy/cli_args.py
@@ -35,6 +35,8 @@ parser = argparse.ArgumentParser()
parser.add_argument("--listen", type=str, default="127.0.0.1", metavar="IP", nargs="?", const="0.0.0.0", help="Specify the IP address to listen on (default: 127.0.0.1). If --listen is provided without an argument, it defaults to 0.0.0.0. (listens on all)")
parser.add_argument("--port", type=int, default=8188, help="Set the listen port.")
+parser.add_argument("--tls-keyfile", type=str, help="Path to TLS (SSL) key file. Enables TLS, makes app accessible at https://... requires --tls-certfile to function")
+parser.add_argument("--tls-certfile", type=str, help="Path to TLS (SSL) certificate file. Enables TLS, makes app accessible at https://... requires --tls-keyfile to function")
parser.add_argument("--enable-cors-header", type=str, default=None, metavar="ORIGIN", nargs="?", const="*", help="Enable CORS (Cross-Origin Resource Sharing) with optional origin or allow all with default '*'.")
parser.add_argument("--max-upload-size", type=float, default=100, help="Set the maximum upload size in MB.")
@@ -49,7 +51,6 @@ cm_group = parser.add_mutually_exclusive_group()
cm_group.add_argument("--cuda-malloc", action="store_true", help="Enable cudaMallocAsync (enabled by default for torch 2.0 and up).")
cm_group.add_argument("--disable-cuda-malloc", action="store_true", help="Disable cudaMallocAsync.")
-parser.add_argument("--dont-upcast-attention", action="store_true", help="Disable upcasting of attention. Can boost speed but increase the chances of black images.")
fp_group = parser.add_mutually_exclusive_group()
fp_group.add_argument("--force-fp32", action="store_true", help="Force fp32 (If this makes your GPU work better please report it).")
@@ -94,6 +95,11 @@ attn_group.add_argument("--use-pytorch-cross-attention", action="store_true", he
parser.add_argument("--disable-xformers", action="store_true", help="Disable xformers.")
+upcast = parser.add_mutually_exclusive_group()
+upcast.add_argument("--force-upcast-attention", action="store_true", help="Force enable attention upcasting, please report if it fixes black images.")
+upcast.add_argument("--dont-upcast-attention", action="store_true", help="Disable all upcasting of attention. Should be unnecessary except for debugging.")
+
+
vram_group = parser.add_mutually_exclusive_group()
vram_group.add_argument("--gpu-only", action="store_true", help="Store and run everything (text encoders/CLIP models, etc... on the GPU).")
vram_group.add_argument("--highvram", action="store_true", help="By default models will be unloaded to CPU memory after being used. This option keeps them in GPU memory.")
diff --git a/comfy/k_diffusion/sampling.py b/comfy/k_diffusion/sampling.py
index 7af016829..f9b281894 100644
--- a/comfy/k_diffusion/sampling.py
+++ b/comfy/k_diffusion/sampling.py
@@ -527,6 +527,9 @@ def sample_dpmpp_2s_ancestral(model, x, sigmas, extra_args=None, callback=None,
@torch.no_grad()
def sample_dpmpp_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r=1 / 2):
"""DPM-Solver++ (stochastic)."""
+ if len(sigmas) <= 1:
+ return x
+
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
seed = extra_args.get("seed", None)
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=seed, cpu=True) if noise_sampler is None else noise_sampler
@@ -595,6 +598,8 @@ def sample_dpmpp_2m(model, x, sigmas, extra_args=None, callback=None, disable=No
@torch.no_grad()
def sample_dpmpp_2m_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, solver_type='midpoint'):
"""DPM-Solver++(2M) SDE."""
+ if len(sigmas) <= 1:
+ return x
if solver_type not in {'heun', 'midpoint'}:
raise ValueError('solver_type must be \'heun\' or \'midpoint\'')
@@ -642,6 +647,9 @@ def sample_dpmpp_2m_sde(model, x, sigmas, extra_args=None, callback=None, disabl
def sample_dpmpp_3m_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None):
"""DPM-Solver++(3M) SDE."""
+ if len(sigmas) <= 1:
+ return x
+
seed = extra_args.get("seed", None)
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=seed, cpu=True) if noise_sampler is None else noise_sampler
@@ -690,18 +698,27 @@ def sample_dpmpp_3m_sde(model, x, sigmas, extra_args=None, callback=None, disabl
@torch.no_grad()
def sample_dpmpp_3m_sde_gpu(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None):
+ if len(sigmas) <= 1:
+ return x
+
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=extra_args.get("seed", None), cpu=False) if noise_sampler is None else noise_sampler
return sample_dpmpp_3m_sde(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler)
@torch.no_grad()
def sample_dpmpp_2m_sde_gpu(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, solver_type='midpoint'):
+ if len(sigmas) <= 1:
+ return x
+
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=extra_args.get("seed", None), cpu=False) if noise_sampler is None else noise_sampler
return sample_dpmpp_2m_sde(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler, solver_type=solver_type)
@torch.no_grad()
def sample_dpmpp_sde_gpu(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r=1 / 2):
+ if len(sigmas) <= 1:
+ return x
+
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=extra_args.get("seed", None), cpu=False) if noise_sampler is None else noise_sampler
return sample_dpmpp_sde(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler, r=r)
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py
index f116efee3..93c944589 100644
--- a/comfy/ldm/modules/attention.py
+++ b/comfy/ldm/modules/attention.py
@@ -6,7 +6,7 @@ from einops import rearrange, repeat
from typing import Optional, Any
import logging
-from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding
+from .diffusionmodules.util import AlphaBlender, timestep_embedding
from .sub_quadratic_attention import efficient_dot_product_attention
from comfy import model_management
@@ -19,13 +19,14 @@ from comfy.cli_args import args
import comfy.ops
ops = comfy.ops.disable_weight_init
-# CrossAttn precision handling
-if args.dont_upcast_attention:
- logging.info("disabling upcasting of attention")
- _ATTN_PRECISION = "fp16"
-else:
- _ATTN_PRECISION = "fp32"
+FORCE_UPCAST_ATTENTION_DTYPE = model_management.force_upcast_attention_dtype()
+def get_attn_precision(attn_precision):
+ if args.dont_upcast_attention:
+ return None
+ if FORCE_UPCAST_ATTENTION_DTYPE is not None:
+ return FORCE_UPCAST_ATTENTION_DTYPE
+ return attn_precision
def exists(val):
return val is not None
@@ -85,7 +86,9 @@ class FeedForward(nn.Module):
def Normalize(in_channels, dtype=None, device=None):
return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True, dtype=dtype, device=device)
-def attention_basic(q, k, v, heads, mask=None):
+def attention_basic(q, k, v, heads, mask=None, attn_precision=None):
+ attn_precision = get_attn_precision(attn_precision)
+
b, _, dim_head = q.shape
dim_head //= heads
scale = dim_head ** -0.5
@@ -101,7 +104,7 @@ def attention_basic(q, k, v, heads, mask=None):
)
# force cast to fp32 to avoid overflowing
- if _ATTN_PRECISION =="fp32":
+ if attn_precision == torch.float32:
sim = einsum('b i d, b j d -> b i j', q.float(), k.float()) * scale
else:
sim = einsum('b i d, b j d -> b i j', q, k) * scale
@@ -135,7 +138,9 @@ def attention_basic(q, k, v, heads, mask=None):
return out
-def attention_sub_quad(query, key, value, heads, mask=None):
+def attention_sub_quad(query, key, value, heads, mask=None, attn_precision=None):
+ attn_precision = get_attn_precision(attn_precision)
+
b, _, dim_head = query.shape
dim_head //= heads
@@ -146,7 +151,7 @@ def attention_sub_quad(query, key, value, heads, mask=None):
key = key.unsqueeze(3).reshape(b, -1, heads, dim_head).permute(0, 2, 3, 1).reshape(b * heads, dim_head, -1)
dtype = query.dtype
- upcast_attention = _ATTN_PRECISION =="fp32" and query.dtype != torch.float32
+ upcast_attention = attn_precision == torch.float32 and query.dtype != torch.float32
if upcast_attention:
bytes_per_token = torch.finfo(torch.float32).bits//8
else:
@@ -195,7 +200,9 @@ def attention_sub_quad(query, key, value, heads, mask=None):
hidden_states = hidden_states.unflatten(0, (-1, heads)).transpose(1,2).flatten(start_dim=2)
return hidden_states
-def attention_split(q, k, v, heads, mask=None):
+def attention_split(q, k, v, heads, mask=None, attn_precision=None):
+ attn_precision = get_attn_precision(attn_precision)
+
b, _, dim_head = q.shape
dim_head //= heads
scale = dim_head ** -0.5
@@ -214,10 +221,12 @@ def attention_split(q, k, v, heads, mask=None):
mem_free_total = model_management.get_free_memory(q.device)
- if _ATTN_PRECISION =="fp32":
+ if attn_precision == torch.float32:
element_size = 4
+ upcast = True
else:
element_size = q.element_size()
+ upcast = False
gb = 1024 ** 3
tensor_size = q.shape[0] * q.shape[1] * k.shape[1] * element_size
@@ -251,7 +260,7 @@ def attention_split(q, k, v, heads, mask=None):
slice_size = q.shape[1] // steps if (q.shape[1] % steps) == 0 else q.shape[1]
for i in range(0, q.shape[1], slice_size):
end = i + slice_size
- if _ATTN_PRECISION =="fp32":
+ if upcast:
with torch.autocast(enabled=False, device_type = 'cuda'):
s1 = einsum('b i d, b j d -> b i j', q[:, i:end].float(), k.float()) * scale
else:
@@ -297,24 +306,30 @@ def attention_split(q, k, v, heads, mask=None):
BROKEN_XFORMERS = False
try:
x_vers = xformers.__version__
- #I think 0.0.23 is also broken (q with bs bigger than 65535 gives CUDA error)
- BROKEN_XFORMERS = x_vers.startswith("0.0.21") or x_vers.startswith("0.0.22") or x_vers.startswith("0.0.23")
+ # XFormers bug confirmed on all versions from 0.0.21 to 0.0.26 (q with bs bigger than 65535 gives CUDA error)
+ BROKEN_XFORMERS = x_vers.startswith("0.0.2") and not x_vers.startswith("0.0.20")
except:
pass
-def attention_xformers(q, k, v, heads, mask=None):
+def attention_xformers(q, k, v, heads, mask=None, attn_precision=None):
b, _, dim_head = q.shape
dim_head //= heads
+
+ disabled_xformers = False
+
if BROKEN_XFORMERS:
if b * heads > 65535:
- return attention_pytorch(q, k, v, heads, mask)
+ disabled_xformers = True
+
+ if not disabled_xformers:
+ if torch.jit.is_tracing() or torch.jit.is_scripting():
+ disabled_xformers = True
+
+ if disabled_xformers:
+ return attention_pytorch(q, k, v, heads, mask)
q, k, v = map(
- lambda t: t.unsqueeze(3)
- .reshape(b, -1, heads, dim_head)
- .permute(0, 2, 1, 3)
- .reshape(b * heads, -1, dim_head)
- .contiguous(),
+ lambda t: t.reshape(b, -1, heads, dim_head),
(q, k, v),
)
@@ -327,14 +342,11 @@ def attention_xformers(q, k, v, heads, mask=None):
out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=mask)
out = (
- out.unsqueeze(0)
- .reshape(b, heads, -1, dim_head)
- .permute(0, 2, 1, 3)
- .reshape(b, -1, heads * dim_head)
+ out.reshape(b, -1, heads * dim_head)
)
return out
-def attention_pytorch(q, k, v, heads, mask=None):
+def attention_pytorch(q, k, v, heads, mask=None, attn_precision=None):
b, _, dim_head = q.shape
dim_head //= heads
q, k, v = map(
@@ -384,10 +396,11 @@ def optimized_attention_for_device(device, mask=False, small_input=False):
class CrossAttention(nn.Module):
- def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=ops):
+ def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., attn_precision=None, dtype=None, device=None, operations=ops):
super().__init__()
inner_dim = dim_head * heads
context_dim = default(context_dim, query_dim)
+ self.attn_precision = attn_precision
self.heads = heads
self.dim_head = dim_head
@@ -409,15 +422,15 @@ class CrossAttention(nn.Module):
v = self.to_v(context)
if mask is None:
- out = optimized_attention(q, k, v, self.heads)
+ out = optimized_attention(q, k, v, self.heads, attn_precision=self.attn_precision)
else:
- out = optimized_attention_masked(q, k, v, self.heads, mask)
+ out = optimized_attention_masked(q, k, v, self.heads, mask, attn_precision=self.attn_precision)
return self.to_out(out)
class BasicTransformerBlock(nn.Module):
def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True, ff_in=False, inner_dim=None,
- disable_self_attn=False, disable_temporal_crossattention=False, switch_temporal_ca_to_sa=False, dtype=None, device=None, operations=ops):
+ disable_self_attn=False, disable_temporal_crossattention=False, switch_temporal_ca_to_sa=False, attn_precision=None, dtype=None, device=None, operations=ops):
super().__init__()
self.ff_in = ff_in or inner_dim is not None
@@ -425,6 +438,7 @@ class BasicTransformerBlock(nn.Module):
inner_dim = dim
self.is_res = inner_dim == dim
+ self.attn_precision = attn_precision
if self.ff_in:
self.norm_in = operations.LayerNorm(dim, dtype=dtype, device=device)
@@ -432,7 +446,7 @@ class BasicTransformerBlock(nn.Module):
self.disable_self_attn = disable_self_attn
self.attn1 = CrossAttention(query_dim=inner_dim, heads=n_heads, dim_head=d_head, dropout=dropout,
- context_dim=context_dim if self.disable_self_attn else None, dtype=dtype, device=device, operations=operations) # is a self-attention if not self.disable_self_attn
+ context_dim=context_dim if self.disable_self_attn else None, attn_precision=self.attn_precision, dtype=dtype, device=device, operations=operations) # is a self-attention if not self.disable_self_attn
self.ff = FeedForward(inner_dim, dim_out=dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device, operations=operations)
if disable_temporal_crossattention:
@@ -446,20 +460,16 @@ class BasicTransformerBlock(nn.Module):
context_dim_attn2 = context_dim
self.attn2 = CrossAttention(query_dim=inner_dim, context_dim=context_dim_attn2,
- heads=n_heads, dim_head=d_head, dropout=dropout, dtype=dtype, device=device, operations=operations) # is self-attn if context is none
+ heads=n_heads, dim_head=d_head, dropout=dropout, attn_precision=self.attn_precision, dtype=dtype, device=device, operations=operations) # is self-attn if context is none
self.norm2 = operations.LayerNorm(inner_dim, dtype=dtype, device=device)
self.norm1 = operations.LayerNorm(inner_dim, dtype=dtype, device=device)
self.norm3 = operations.LayerNorm(inner_dim, dtype=dtype, device=device)
- self.checkpoint = checkpoint
self.n_heads = n_heads
self.d_head = d_head
self.switch_temporal_ca_to_sa = switch_temporal_ca_to_sa
def forward(self, x, context=None, transformer_options={}):
- return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint)
-
- def _forward(self, x, context=None, transformer_options={}):
extra_options = {}
block = transformer_options.get("block", None)
block_index = transformer_options.get("block_index", 0)
@@ -476,6 +486,7 @@ class BasicTransformerBlock(nn.Module):
extra_options["n_heads"] = self.n_heads
extra_options["dim_head"] = self.d_head
+ extra_options["attn_precision"] = self.attn_precision
if self.ff_in:
x_skip = x
@@ -586,7 +597,7 @@ class SpatialTransformer(nn.Module):
def __init__(self, in_channels, n_heads, d_head,
depth=1, dropout=0., context_dim=None,
disable_self_attn=False, use_linear=False,
- use_checkpoint=True, dtype=None, device=None, operations=ops):
+ use_checkpoint=True, attn_precision=None, dtype=None, device=None, operations=ops):
super().__init__()
if exists(context_dim) and not isinstance(context_dim, list):
context_dim = [context_dim] * depth
@@ -604,7 +615,7 @@ class SpatialTransformer(nn.Module):
self.transformer_blocks = nn.ModuleList(
[BasicTransformerBlock(inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim[d],
- disable_self_attn=disable_self_attn, checkpoint=use_checkpoint, dtype=dtype, device=device, operations=operations)
+ disable_self_attn=disable_self_attn, checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=dtype, device=device, operations=operations)
for d in range(depth)]
)
if not use_linear:
@@ -625,7 +636,7 @@ class SpatialTransformer(nn.Module):
x = self.norm(x)
if not self.use_linear:
x = self.proj_in(x)
- x = rearrange(x, 'b c h w -> b (h w) c').contiguous()
+ x = x.movedim(1, 3).flatten(1, 2).contiguous()
if self.use_linear:
x = self.proj_in(x)
for i, block in enumerate(self.transformer_blocks):
@@ -633,7 +644,7 @@ class SpatialTransformer(nn.Module):
x = block(x, context=context[i], transformer_options=transformer_options)
if self.use_linear:
x = self.proj_out(x)
- x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w).contiguous()
+ x = x.reshape(x.shape[0], h, w, x.shape[-1]).movedim(3, 1).contiguous()
if not self.use_linear:
x = self.proj_out(x)
return x + x_in
@@ -660,6 +671,7 @@ class SpatialVideoTransformer(SpatialTransformer):
disable_self_attn=False,
disable_temporal_crossattention=False,
max_time_embed_period: int = 10000,
+ attn_precision=None,
dtype=None, device=None, operations=ops
):
super().__init__(
@@ -672,6 +684,7 @@ class SpatialVideoTransformer(SpatialTransformer):
context_dim=context_dim,
use_linear=use_linear,
disable_self_attn=disable_self_attn,
+ attn_precision=attn_precision,
dtype=dtype, device=device, operations=operations
)
self.time_depth = time_depth
@@ -701,6 +714,7 @@ class SpatialVideoTransformer(SpatialTransformer):
inner_dim=time_mix_inner_dim,
disable_self_attn=disable_self_attn,
disable_temporal_crossattention=disable_temporal_crossattention,
+ attn_precision=attn_precision,
dtype=dtype, device=device, operations=operations
)
for _ in range(self.depth)
diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.py
index fabc5c5e5..04eb83b21 100644
--- a/comfy/ldm/modules/diffusionmodules/model.py
+++ b/comfy/ldm/modules/diffusionmodules/model.py
@@ -3,7 +3,6 @@ import math
import torch
import torch.nn as nn
import numpy as np
-from einops import rearrange
from typing import Optional, Any
import logging
diff --git a/comfy/ldm/modules/diffusionmodules/openaimodel.py b/comfy/ldm/modules/diffusionmodules/openaimodel.py
index d782eff31..ba8fc2c4a 100644
--- a/comfy/ldm/modules/diffusionmodules/openaimodel.py
+++ b/comfy/ldm/modules/diffusionmodules/openaimodel.py
@@ -258,7 +258,7 @@ class ResBlock(TimestepBlock):
else:
if emb_out is not None:
if self.exchange_temb_dims:
- emb_out = rearrange(emb_out, "b t c ... -> b c t ...")
+ emb_out = emb_out.movedim(1, 2)
h = h + emb_out
h = self.out_layers(h)
return self.skip_connection(x) + h
@@ -431,6 +431,7 @@ class UNetModel(nn.Module):
video_kernel_size=None,
disable_temporal_crossattention=False,
max_ddpm_temb_period=10000,
+ attn_precision=None,
device=None,
operations=ops,
):
@@ -550,13 +551,14 @@ class UNetModel(nn.Module):
disable_self_attn=disable_self_attn,
disable_temporal_crossattention=disable_temporal_crossattention,
max_time_embed_period=max_ddpm_temb_period,
+ attn_precision=attn_precision,
dtype=self.dtype, device=device, operations=operations
)
else:
return SpatialTransformer(
ch, num_heads, dim_head, depth=depth, context_dim=context_dim,
disable_self_attn=disable_self_attn, use_linear=use_linear_in_transformer,
- use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
+ use_checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations
)
def get_resblock(
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 8c89adf5e..841598b73 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -162,7 +162,7 @@ class BaseModel(torch.nn.Module):
c_concat = kwargs.get("noise_concat", None)
if c_concat is not None:
- out['c_concat'] = comfy.conds.CONDNoiseShape(data)
+ out['c_concat'] = comfy.conds.CONDNoiseShape(c_concat)
return out
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index 795af828a..23358a2c0 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -357,7 +357,14 @@ def unet_config_from_diffusers_unet(state_dict, dtype=None):
'context_dim': 1024, 'num_head_channels': 64, 'transformer_depth_output': [1, 1, 1, 1, 1, 1],
'use_temporal_attention': False, 'use_temporal_resblock': False, 'disable_self_attentions': [True, False, False]}
- supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint, SSD_1B, Segmind_Vega, KOALA_700M, KOALA_1B, SD09_XS, SDXL_diffusers_ip2p]
+ SD_XS = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
+ 'adm_in_channels': None, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': [1, 1, 1],
+ 'transformer_depth': [0, 1, 1], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': -2, 'use_linear_in_transformer': False,
+ 'context_dim': 768, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 1, 1, 1, 1],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
+
+
+ supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint, SSD_1B, Segmind_Vega, KOALA_700M, KOALA_1B, SD09_XS, SD_XS, SDXL_diffusers_ip2p]
for unet_config in supported_models:
matches = True
diff --git a/comfy/model_management.py b/comfy/model_management.py
index 310ec2537..ef36a2c48 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -5,6 +5,7 @@ from comfy.cli_args import args
import comfy.utils
import torch
import sys
+import platform
class VRAMState(Enum):
DISABLED = 0 #No vram present: no need to move models to vram
@@ -83,7 +84,7 @@ def get_torch_device():
return torch.device("cpu")
else:
if is_intel_xpu():
- return torch.device("xpu")
+ return torch.device("xpu", torch.xpu.current_device())
else:
return torch.device(torch.cuda.current_device())
@@ -102,8 +103,8 @@ def get_total_memory(dev=None, torch_total_too=False):
elif is_intel_xpu():
stats = torch.xpu.memory_stats(dev)
mem_reserved = stats['reserved_bytes.all.current']
- mem_total = torch.xpu.get_device_properties(dev).total_memory
mem_total_torch = mem_reserved
+ mem_total = torch.xpu.get_device_properties(dev).total_memory
else:
stats = torch.cuda.memory_stats(dev)
mem_reserved = stats['reserved_bytes.all.current']
@@ -119,10 +120,11 @@ def get_total_memory(dev=None, torch_total_too=False):
total_vram = get_total_memory(get_torch_device()) / (1024 * 1024)
total_ram = psutil.virtual_memory().total / (1024 * 1024)
logging.info("Total VRAM {:0.0f} MB, total RAM {:0.0f} MB".format(total_vram, total_ram))
-if not args.normalvram and not args.cpu:
- if lowvram_available and total_vram <= 4096:
- logging.warning("Trying to enable lowvram mode because your GPU seems to have 4GB or less. If you don't want this use: --normalvram")
- set_vram_to = VRAMState.LOW_VRAM
+
+try:
+ logging.info("pytorch version: {}".format(torch.version.__version__))
+except:
+ pass
try:
OOM_EXCEPTION = torch.cuda.OutOfMemoryError
@@ -285,7 +287,7 @@ class LoadedModel:
else:
return self.model_memory()
- def model_load(self, lowvram_model_memory=0):
+ def model_load(self, lowvram_model_memory=0, force_patch_weights=False):
patch_model_to = self.device
self.model.model_patches_to(self.device)
@@ -295,7 +297,7 @@ class LoadedModel:
try:
if lowvram_model_memory > 0 and load_weights:
- self.real_model = self.model.patch_model_lowvram(device_to=patch_model_to, lowvram_model_memory=lowvram_model_memory)
+ self.real_model = self.model.patch_model_lowvram(device_to=patch_model_to, lowvram_model_memory=lowvram_model_memory, force_patch_weights=force_patch_weights)
else:
self.real_model = self.model.patch_model(device_to=patch_model_to, patch_weights=load_weights)
except Exception as e:
@@ -304,11 +306,16 @@ class LoadedModel:
raise e
if is_intel_xpu() and not args.disable_ipex_optimize:
- self.real_model = torch.xpu.optimize(self.real_model.eval(), inplace=True, auto_kernel_selection=True, graph_mode=True)
+ self.real_model = ipex.optimize(self.real_model.eval(), graph_mode=True, concat_linear=True)
self.weights_loaded = True
return self.real_model
+ def should_reload_model(self, force_patch_weights=False):
+ if force_patch_weights and self.model.lowvram_patch_counter > 0:
+ return True
+ return False
+
def model_unload(self, unpatch_weights=True):
self.model.unpatch_model(self.model.offload_device, unpatch_weights=unpatch_weights)
self.model.model_patches_to(self.model.offload_device)
@@ -379,7 +386,7 @@ def free_memory(memory_required, device, keep_loaded=[]):
if mem_free_torch > mem_free_total * 0.25:
soft_empty_cache()
-def load_models_gpu(models, memory_required=0):
+def load_models_gpu(models, memory_required=0, force_patch_weights=False):
global vram_state
inference_memory = minimum_inference_memory()
@@ -391,12 +398,22 @@ def load_models_gpu(models, memory_required=0):
models_already_loaded = []
for x in models:
loaded_model = LoadedModel(x)
+ loaded = None
- if loaded_model in current_loaded_models:
- index = current_loaded_models.index(loaded_model)
- current_loaded_models.insert(0, current_loaded_models.pop(index))
- models_already_loaded.append(loaded_model)
- else:
+ try:
+ loaded_model_index = current_loaded_models.index(loaded_model)
+ except:
+ loaded_model_index = None
+
+ if loaded_model_index is not None:
+ loaded = current_loaded_models[loaded_model_index]
+ if loaded.should_reload_model(force_patch_weights=force_patch_weights): #TODO: cleanup this model reload logic
+ current_loaded_models.pop(loaded_model_index).model_unload(unpatch_weights=True)
+ loaded = None
+ else:
+ models_already_loaded.append(loaded)
+
+ if loaded is None:
if hasattr(x, "model"):
logging.info(f"Requested to load {x.model.__class__.__name__}")
models_to_load.append(loaded_model)
@@ -436,15 +453,13 @@ def load_models_gpu(models, memory_required=0):
model_size = loaded_model.model_memory_required(torch_dev)
current_free_mem = get_free_memory(torch_dev)
lowvram_model_memory = int(max(64 * (1024 * 1024), (current_free_mem - 1024 * (1024 * 1024)) / 1.3 ))
- if model_size > (current_free_mem - inference_memory): #only switch to lowvram if really necessary
- vram_set_state = VRAMState.LOW_VRAM
- else:
+ if model_size <= (current_free_mem - inference_memory): #only switch to lowvram if really necessary
lowvram_model_memory = 0
if vram_set_state == VRAMState.NO_VRAM:
lowvram_model_memory = 64 * 1024 * 1024
- cur_loaded_model = loaded_model.model_load(lowvram_model_memory)
+ cur_loaded_model = loaded_model.model_load(lowvram_model_memory, force_patch_weights=force_patch_weights)
current_loaded_models.insert(0, loaded_model)
return
@@ -552,8 +567,6 @@ def text_encoder_device():
if args.gpu_only:
return get_torch_device()
elif vram_state == VRAMState.HIGH_VRAM or vram_state == VRAMState.NORMAL_VRAM:
- if is_intel_xpu():
- return torch.device("cpu")
if should_use_fp16(prioritize_performance=False):
return get_torch_device()
else:
@@ -619,6 +632,13 @@ def device_supports_non_blocking(device):
return False #pytorch bug? mps doesn't support non blocking
return True
+def device_should_use_non_blocking(device):
+ if not device_supports_non_blocking(device):
+ return False
+ return False
+ # return True #TODO: figure out why this causes memory issues on Nvidia and possibly others
+
+
def cast_to_device(tensor, device, dtype, copy=False):
device_supports_cast = False
if tensor.dtype == torch.float32 or tensor.dtype == torch.float16:
@@ -629,7 +649,7 @@ def cast_to_device(tensor, device, dtype, copy=False):
elif is_intel_xpu():
device_supports_cast = True
- non_blocking = device_supports_non_blocking(device)
+ non_blocking = device_should_use_non_blocking(device)
if device_supports_cast:
if copy:
@@ -672,6 +692,18 @@ def pytorch_attention_flash_attention():
return True
return False
+def force_upcast_attention_dtype():
+ upcast = args.force_upcast_attention
+ try:
+ if platform.mac_ver()[0] in ['14.5']: #black image bug on OSX Sonoma 14.5
+ upcast = True
+ except:
+ pass
+ if upcast:
+ return torch.float32
+ else:
+ return None
+
def get_free_memory(dev=None, torch_free_too=False):
global directml_enabled
if dev is None:
@@ -687,10 +719,10 @@ def get_free_memory(dev=None, torch_free_too=False):
elif is_intel_xpu():
stats = torch.xpu.memory_stats(dev)
mem_active = stats['active_bytes.all.current']
- mem_allocated = stats['allocated_bytes.all.current']
mem_reserved = stats['reserved_bytes.all.current']
mem_free_torch = mem_reserved - mem_active
- mem_free_total = torch.xpu.get_device_properties(dev).total_memory - mem_allocated
+ mem_free_xpu = torch.xpu.get_device_properties(dev).total_memory - mem_reserved
+ mem_free_total = mem_free_xpu + mem_free_torch
else:
stats = torch.cuda.memory_stats(dev)
mem_active = stats['active_bytes.all.current']
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py
index 657ddf843..c397ee51c 100644
--- a/comfy/model_patcher.py
+++ b/comfy/model_patcher.py
@@ -6,6 +6,8 @@ import uuid
import comfy.utils
import comfy.model_management
+from comfy.types import UnetWrapperFunction
+
def apply_weight_decompose(dora_scale, weight):
weight_norm = (
@@ -16,8 +18,28 @@ def apply_weight_decompose(dora_scale, weight):
.transpose(0, 1)
)
- return weight * (dora_scale / weight_norm)
+ return weight * (dora_scale / weight_norm).type(weight.dtype)
+def set_model_options_patch_replace(model_options, patch, name, block_name, number, transformer_index=None):
+ to = model_options["transformer_options"].copy()
+
+ if "patches_replace" not in to:
+ to["patches_replace"] = {}
+ else:
+ to["patches_replace"] = to["patches_replace"].copy()
+
+ if name not in to["patches_replace"]:
+ to["patches_replace"][name] = {}
+ else:
+ to["patches_replace"][name] = to["patches_replace"][name].copy()
+
+ if transformer_index is not None:
+ block = (block_name, number, transformer_index)
+ else:
+ block = (block_name, number)
+ to["patches_replace"][name][block] = patch
+ model_options["transformer_options"] = to
+ return model_options
class ModelPatcher:
def __init__(self, model, load_device, offload_device, size=0, current_device=None, weight_inplace_update=False):
@@ -38,6 +60,7 @@ class ModelPatcher:
self.weight_inplace_update = weight_inplace_update
self.model_lowvram = False
+ self.lowvram_patch_counter = 0
self.patches_uuid = uuid.uuid4()
def model_size(self):
@@ -96,7 +119,7 @@ class ModelPatcher:
if disable_cfg1_optimization:
self.model_options["disable_cfg1_optimization"] = True
- def set_model_unet_function_wrapper(self, unet_wrapper_function):
+ def set_model_unet_function_wrapper(self, unet_wrapper_function: UnetWrapperFunction):
self.model_options["model_function_wrapper"] = unet_wrapper_function
def set_model_denoise_mask_function(self, denoise_mask_function):
@@ -109,16 +132,7 @@ class ModelPatcher:
to["patches"][name] = to["patches"].get(name, []) + [patch]
def set_model_patch_replace(self, patch, name, block_name, number, transformer_index=None):
- to = self.model_options["transformer_options"]
- if "patches_replace" not in to:
- to["patches_replace"] = {}
- if name not in to["patches_replace"]:
- to["patches_replace"][name] = {}
- if transformer_index is not None:
- block = (block_name, number, transformer_index)
- else:
- block = (block_name, number)
- to["patches_replace"][name][block] = patch
+ self.model_options = set_model_options_patch_replace(self.model_options, patch, name, block_name, number, transformer_index=transformer_index)
def set_model_attn1_patch(self, patch):
self.set_model_patch(patch, "attn1_patch")
@@ -261,7 +275,7 @@ class ModelPatcher:
return self.model
- def patch_model_lowvram(self, device_to=None, lowvram_model_memory=0):
+ def patch_model_lowvram(self, device_to=None, lowvram_model_memory=0, force_patch_weights=False):
self.patch_model(device_to, patch_weights=False)
logging.info("loading in lowvram mode {}".format(lowvram_model_memory/(1024 * 1024)))
@@ -273,6 +287,7 @@ class ModelPatcher:
return self.model_patcher.calculate_weight(self.model_patcher.patches[self.key], weight, self.key)
mem_counter = 0
+ patch_counter = 0
for n, m in self.model.named_modules():
lowvram_weight = False
if hasattr(m, "comfy_cast_weights"):
@@ -285,9 +300,17 @@ class ModelPatcher:
if lowvram_weight:
if weight_key in self.patches:
- m.weight_function = LowVramPatch(weight_key, self)
+ if force_patch_weights:
+ self.patch_weight_to_device(weight_key)
+ else:
+ m.weight_function = LowVramPatch(weight_key, self)
+ patch_counter += 1
if bias_key in self.patches:
- m.bias_function = LowVramPatch(bias_key, self)
+ if force_patch_weights:
+ self.patch_weight_to_device(bias_key)
+ else:
+ m.bias_function = LowVramPatch(bias_key, self)
+ patch_counter += 1
m.prev_comfy_cast_weights = m.comfy_cast_weights
m.comfy_cast_weights = True
@@ -300,6 +323,7 @@ class ModelPatcher:
logging.debug("lowvram: loaded module regularly {}".format(m))
self.model_lowvram = True
+ self.lowvram_patch_counter = patch_counter
return self.model
def calculate_weight(self, patches, weight, key):
@@ -451,6 +475,7 @@ class ModelPatcher:
m.bias_function = None
self.model_lowvram = False
+ self.lowvram_patch_counter = 0
keys = list(self.backup.keys())
diff --git a/comfy/ops.py b/comfy/ops.py
index eb6507682..7ebb3dd2f 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -21,7 +21,7 @@ import comfy.model_management
def cast_bias_weight(s, input):
bias = None
- non_blocking = comfy.model_management.device_supports_non_blocking(input.device)
+ non_blocking = comfy.model_management.device_should_use_non_blocking(input.device)
if s.bias is not None:
bias = s.bias.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)
if s.bias_function is not None:
diff --git a/comfy/samplers.py b/comfy/samplers.py
index 415a35cc3..29962a916 100644
--- a/comfy/samplers.py
+++ b/comfy/samplers.py
@@ -34,7 +34,7 @@ def get_area_and_mult(conds, x_in, timestep_in):
mask = conds['mask']
assert(mask.shape[1] == x_in.shape[2])
assert(mask.shape[2] == x_in.shape[3])
- mask = mask[:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] * mask_strength
+ mask = mask[:input_x.shape[0],area[2]:area[0] + area[2],area[3]:area[1] + area[3]] * mask_strength
mask = mask.unsqueeze(1).repeat(input_x.shape[0] // mask.shape[0], input_x.shape[1], 1, 1)
else:
mask = torch.ones_like(input_x)
@@ -539,6 +539,9 @@ class KSAMPLER(Sampler):
def ksampler(sampler_name, extra_options={}, inpaint_options={}):
if sampler_name == "dpm_fast":
def dpm_fast_function(model, noise, sigmas, extra_args, callback, disable):
+ if len(sigmas) <= 1:
+ return noise
+
sigma_min = sigmas[-1]
if sigma_min == 0:
sigma_min = sigmas[-2]
@@ -547,6 +550,9 @@ def ksampler(sampler_name, extra_options={}, inpaint_options={}):
sampler_function = dpm_fast_function
elif sampler_name == "dpm_adaptive":
def dpm_adaptive_function(model, noise, sigmas, extra_args, callback, disable, **extra_options):
+ if len(sigmas) <= 1:
+ return noise
+
sigma_min = sigmas[-1]
if sigma_min == 0:
sigma_min = sigmas[-2]
diff --git a/comfy/sd.py b/comfy/sd.py
index 3919f4bfa..8044c184f 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -210,16 +210,26 @@ class VAE:
self.first_stage_model = StageC_coder()
self.downscale_ratio = 32
self.latent_channels = 16
- else:
+ elif "decoder.conv_in.weight" in sd:
#default SD1.x/SD2.x VAE parameters
ddconfig = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0}
- if 'encoder.down.2.downsample.conv.weight' not in sd: #Stable diffusion x4 upscaler VAE
+ if 'encoder.down.2.downsample.conv.weight' not in sd and 'decoder.up.3.upsample.conv.weight' not in sd: #Stable diffusion x4 upscaler VAE
ddconfig['ch_mult'] = [1, 2, 4]
self.downscale_ratio = 4
self.upscale_ratio = 4
- self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4)
+ self.latent_channels = ddconfig['z_channels'] = sd["decoder.conv_in.weight"].shape[1]
+ if 'quant_conv.weight' in sd:
+ self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4)
+ else:
+ self.first_stage_model = AutoencodingEngine(regularizer_config={'target': "comfy.ldm.models.autoencoder.DiagonalGaussianRegularizer"},
+ encoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Encoder", 'params': ddconfig},
+ decoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Decoder", 'params': ddconfig})
+ else:
+ logging.warning("WARNING: No VAE weights detected, VAE not initalized.")
+ self.first_stage_model = None
+ return
else:
self.first_stage_model = AutoencoderKL(**(config['params']))
self.first_stage_model = self.first_stage_model.eval()
@@ -408,6 +418,8 @@ def load_gligen(ckpt_path):
return comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=model_management.unet_offload_device())
def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_clip=True, embedding_directory=None, state_dict=None, config=None):
+ logging.warning("Warning: The load checkpoint with config function is deprecated and will eventually be removed, please use the other one.")
+ model, clip, vae, _ = load_checkpoint_guess_config(ckpt_path, output_vae=output_vae, output_clip=output_clip, output_clipvision=False, embedding_directory=embedding_directory, output_model=True)
#TODO: this function is a mess and should be removed eventually
if config is None:
with open(config_path, 'r') as stream:
@@ -415,81 +427,20 @@ def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_cl
model_config_params = config['model']['params']
clip_config = model_config_params['cond_stage_config']
scale_factor = model_config_params['scale_factor']
- vae_config = model_config_params['first_stage_config']
-
- fp16 = False
- if "unet_config" in model_config_params:
- if "params" in model_config_params["unet_config"]:
- unet_config = model_config_params["unet_config"]["params"]
- if "use_fp16" in unet_config:
- fp16 = unet_config.pop("use_fp16")
- if fp16:
- unet_config["dtype"] = torch.float16
-
- noise_aug_config = None
- if "noise_aug_config" in model_config_params:
- noise_aug_config = model_config_params["noise_aug_config"]
-
- model_type = model_base.ModelType.EPS
if "parameterization" in model_config_params:
if model_config_params["parameterization"] == "v":
- model_type = model_base.ModelType.V_PREDICTION
+ m = model.clone()
+ class ModelSamplingAdvanced(comfy.model_sampling.ModelSamplingDiscrete, comfy.model_sampling.V_PREDICTION):
+ pass
+ m.add_object_patch("model_sampling", ModelSamplingAdvanced(model.model.model_config))
+ model = m
- clip = None
- vae = None
+ layer_idx = clip_config.get("params", {}).get("layer_idx", None)
+ if layer_idx is not None:
+ clip.clip_layer(layer_idx)
- class WeightsLoader(torch.nn.Module):
- pass
-
- if state_dict is None:
- state_dict = comfy.utils.load_torch_file(ckpt_path)
-
- class EmptyClass:
- pass
-
- model_config = comfy.supported_models_base.BASE({})
-
- from . import latent_formats
- model_config.latent_format = latent_formats.SD15(scale_factor=scale_factor)
- model_config.unet_config = model_detection.convert_config(unet_config)
-
- if config['model']["target"].endswith("ImageEmbeddingConditionedLatentDiffusion"):
- model = model_base.SD21UNCLIP(model_config, noise_aug_config["params"], model_type=model_type)
- else:
- model = model_base.BaseModel(model_config, model_type=model_type)
-
- if config['model']["target"].endswith("LatentInpaintDiffusion"):
- model.set_inpaint()
-
- if fp16:
- model = model.half()
-
- offload_device = model_management.unet_offload_device()
- model = model.to(offload_device)
- model.load_model_weights(state_dict, "model.diffusion_model.")
-
- if output_vae:
- vae_sd = comfy.utils.state_dict_prefix_replace(state_dict, {"first_stage_model.": ""}, filter_keys=True)
- vae = VAE(sd=vae_sd, config=vae_config)
-
- if output_clip:
- w = WeightsLoader()
- clip_target = EmptyClass()
- clip_target.params = clip_config.get("params", {})
- if clip_config["target"].endswith("FrozenOpenCLIPEmbedder"):
- clip_target.clip = sd2_clip.SD2ClipModel
- clip_target.tokenizer = sd2_clip.SD2Tokenizer
- clip = CLIP(clip_target, embedding_directory=embedding_directory)
- w.cond_stage_model = clip.cond_stage_model.clip_h
- elif clip_config["target"].endswith("FrozenCLIPEmbedder"):
- clip_target.clip = sd1_clip.SD1ClipModel
- clip_target.tokenizer = sd1_clip.SD1Tokenizer
- clip = CLIP(clip_target, embedding_directory=embedding_directory)
- w.cond_stage_model = clip.cond_stage_model.clip_l
- load_clip_weights(w, state_dict)
-
- return (comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device), clip, vae)
+ return (model, clip, vae)
def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True):
sd = comfy.utils.load_torch_file(ckpt_path)
@@ -535,7 +486,11 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
clip = CLIP(clip_target, embedding_directory=embedding_directory)
m, u = clip.load_sd(clip_sd, full_model=True)
if len(m) > 0:
- logging.warning("clip missing: {}".format(m))
+ m_filter = list(filter(lambda a: ".logit_scale" not in a and ".transformer.text_projection.weight" not in a, m))
+ if len(m_filter) > 0:
+ logging.warning("clip missing: {}".format(m))
+ else:
+ logging.debug("clip missing: {}".format(m))
if len(u) > 0:
logging.debug("clip unexpected {}:".format(u))
@@ -607,7 +562,7 @@ def save_checkpoint(output_path, model, clip=None, vae=None, clip_vision=None, m
load_models.append(clip.load_model())
clip_sd = clip.get_sd()
- model_management.load_models_gpu(load_models)
+ model_management.load_models_gpu(load_models, force_patch_weights=True)
clip_vision_sd = clip_vision.get_sd() if clip_vision is not None else None
sd = model.model.state_dict_for_saving(clip_sd, vae.get_sd(), clip_vision_sd)
for k in extra_keys:
diff --git a/comfy/supported_models.py b/comfy/supported_models.py
index b3b69e05b..6ca32e8ee 100644
--- a/comfy/supported_models.py
+++ b/comfy/supported_models.py
@@ -65,6 +65,12 @@ class SD20(supported_models_base.BASE):
"use_temporal_attention": False,
}
+ unet_extra_config = {
+ "num_heads": -1,
+ "num_head_channels": 64,
+ "attn_precision": torch.float32,
+ }
+
latent_format = latent_formats.SD15
def model_type(self, state_dict, prefix=""):
@@ -276,6 +282,12 @@ class SVD_img2vid(supported_models_base.BASE):
"use_temporal_resblock": True
}
+ unet_extra_config = {
+ "num_heads": -1,
+ "num_head_channels": 64,
+ "attn_precision": torch.float32,
+ }
+
clip_vision_prefix = "conditioner.embedders.0.open_clip.model.visual."
latent_format = latent_formats.SD15
diff --git a/comfy/types.py b/comfy/types.py
new file mode 100644
index 000000000..a8a3d29fd
--- /dev/null
+++ b/comfy/types.py
@@ -0,0 +1,32 @@
+import torch
+from typing import Callable, Protocol, TypedDict, Optional, List
+
+
+class UnetApplyFunction(Protocol):
+ """Function signature protocol on comfy.model_base.BaseModel.apply_model"""
+
+ def __call__(self, x: torch.Tensor, t: torch.Tensor, **kwargs) -> torch.Tensor:
+ pass
+
+
+class UnetApplyConds(TypedDict):
+ """Optional conditions for unet apply function."""
+
+ c_concat: Optional[torch.Tensor]
+ c_crossattn: Optional[torch.Tensor]
+ control: Optional[torch.Tensor]
+ transformer_options: Optional[dict]
+
+
+class UnetParams(TypedDict):
+ # Tensor of shape [B, C, H, W]
+ input: torch.Tensor
+ # Tensor of shape [B]
+ timestep: torch.Tensor
+ c: UnetApplyConds
+ # List of [0, 1], [0], [1], ...
+ # 0 means unconditional, 1 means conditional
+ cond_or_uncond: List[int]
+
+
+UnetWrapperFunction = Callable[[UnetApplyFunction, UnetParams], torch.Tensor]
diff --git a/comfy_extras/nodes_advanced_samplers.py b/comfy_extras/nodes_advanced_samplers.py
new file mode 100644
index 000000000..d973def81
--- /dev/null
+++ b/comfy_extras/nodes_advanced_samplers.py
@@ -0,0 +1,61 @@
+import comfy.samplers
+import comfy.utils
+import torch
+import numpy as np
+from tqdm.auto import trange, tqdm
+import math
+
+
+@torch.no_grad()
+def sample_lcm_upscale(model, x, sigmas, extra_args=None, callback=None, disable=None, total_upscale=2.0, upscale_method="bislerp", upscale_steps=None):
+ extra_args = {} if extra_args is None else extra_args
+
+ if upscale_steps is None:
+ upscale_steps = max(len(sigmas) // 2 + 1, 2)
+ else:
+ upscale_steps += 1
+ upscale_steps = min(upscale_steps, len(sigmas) + 1)
+
+ upscales = np.linspace(1.0, total_upscale, upscale_steps)[1:]
+
+ orig_shape = x.size()
+ s_in = x.new_ones([x.shape[0]])
+ for i in trange(len(sigmas) - 1, disable=disable):
+ denoised = model(x, sigmas[i] * s_in, **extra_args)
+ if callback is not None:
+ callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised})
+
+ x = denoised
+ if i < len(upscales):
+ x = comfy.utils.common_upscale(x, round(orig_shape[-1] * upscales[i]), round(orig_shape[-2] * upscales[i]), upscale_method, "disabled")
+
+ if sigmas[i + 1] > 0:
+ x += sigmas[i + 1] * torch.randn_like(x)
+ return x
+
+
+class SamplerLCMUpscale:
+ upscale_methods = ["bislerp", "nearest-exact", "bilinear", "area", "bicubic"]
+
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"scale_ratio": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 20.0, "step": 0.01}),
+ "scale_steps": ("INT", {"default": -1, "min": -1, "max": 1000, "step": 1}),
+ "upscale_method": (s.upscale_methods,),
+ }
+ }
+ RETURN_TYPES = ("SAMPLER",)
+ CATEGORY = "sampling/custom_sampling/samplers"
+
+ FUNCTION = "get_sampler"
+
+ def get_sampler(self, scale_ratio, scale_steps, upscale_method):
+ if scale_steps < 0:
+ scale_steps = None
+ sampler = comfy.samplers.KSAMPLER(sample_lcm_upscale, extra_options={"total_upscale": scale_ratio, "upscale_steps": scale_steps, "upscale_method": upscale_method})
+ return (sampler, )
+
+NODE_CLASS_MAPPINGS = {
+ "SamplerLCMUpscale": SamplerLCMUpscale,
+}
diff --git a/comfy_extras/nodes_align_your_steps.py b/comfy_extras/nodes_align_your_steps.py
new file mode 100644
index 000000000..3ffe53187
--- /dev/null
+++ b/comfy_extras/nodes_align_your_steps.py
@@ -0,0 +1,53 @@
+#from: https://research.nvidia.com/labs/toronto-ai/AlignYourSteps/howto.html
+import numpy as np
+import torch
+
+def loglinear_interp(t_steps, num_steps):
+ """
+ Performs log-linear interpolation of a given array of decreasing numbers.
+ """
+ xs = np.linspace(0, 1, len(t_steps))
+ ys = np.log(t_steps[::-1])
+
+ new_xs = np.linspace(0, 1, num_steps)
+ new_ys = np.interp(new_xs, xs, ys)
+
+ interped_ys = np.exp(new_ys)[::-1].copy()
+ return interped_ys
+
+NOISE_LEVELS = {"SD1": [14.6146412293, 6.4745760956, 3.8636745985, 2.6946151520, 1.8841921177, 1.3943805092, 0.9642583904, 0.6523686016, 0.3977456272, 0.1515232662, 0.0291671582],
+ "SDXL":[14.6146412293, 6.3184485287, 3.7681790315, 2.1811480769, 1.3405244945, 0.8620721141, 0.5550693289, 0.3798540708, 0.2332364134, 0.1114188177, 0.0291671582],
+ "SVD": [700.00, 54.5, 15.886, 7.977, 4.248, 1.789, 0.981, 0.403, 0.173, 0.034, 0.002]}
+
+class AlignYourStepsScheduler:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"model_type": (["SD1", "SDXL", "SVD"], ),
+ "steps": ("INT", {"default": 10, "min": 10, "max": 10000}),
+ "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
+ }
+ }
+ RETURN_TYPES = ("SIGMAS",)
+ CATEGORY = "sampling/custom_sampling/schedulers"
+
+ FUNCTION = "get_sigmas"
+
+ def get_sigmas(self, model_type, steps, denoise):
+ total_steps = steps
+ if denoise < 1.0:
+ if denoise <= 0.0:
+ return (torch.FloatTensor([]),)
+ total_steps = round(steps * denoise)
+
+ sigmas = NOISE_LEVELS[model_type][:]
+ if (steps + 1) != len(sigmas):
+ sigmas = loglinear_interp(sigmas, steps + 1)
+
+ sigmas = sigmas[-(total_steps + 1):]
+ sigmas[-1] = 0
+ return (torch.FloatTensor(sigmas), )
+
+NODE_CLASS_MAPPINGS = {
+ "AlignYourStepsScheduler": AlignYourStepsScheduler,
+}
diff --git a/comfy_extras/nodes_attention_multiply.py b/comfy_extras/nodes_attention_multiply.py
new file mode 100644
index 000000000..4747eb395
--- /dev/null
+++ b/comfy_extras/nodes_attention_multiply.py
@@ -0,0 +1,120 @@
+
+def attention_multiply(attn, model, q, k, v, out):
+ m = model.clone()
+ sd = model.model_state_dict()
+
+ for key in sd:
+ if key.endswith("{}.to_q.bias".format(attn)) or key.endswith("{}.to_q.weight".format(attn)):
+ m.add_patches({key: (None,)}, 0.0, q)
+ if key.endswith("{}.to_k.bias".format(attn)) or key.endswith("{}.to_k.weight".format(attn)):
+ m.add_patches({key: (None,)}, 0.0, k)
+ if key.endswith("{}.to_v.bias".format(attn)) or key.endswith("{}.to_v.weight".format(attn)):
+ m.add_patches({key: (None,)}, 0.0, v)
+ if key.endswith("{}.to_out.0.bias".format(attn)) or key.endswith("{}.to_out.0.weight".format(attn)):
+ m.add_patches({key: (None,)}, 0.0, out)
+
+ return m
+
+
+class UNetSelfAttentionMultiply:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "q": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "k": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "v": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "out": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ }}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "_for_testing/attention_experiments"
+
+ def patch(self, model, q, k, v, out):
+ m = attention_multiply("attn1", model, q, k, v, out)
+ return (m, )
+
+class UNetCrossAttentionMultiply:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "q": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "k": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "v": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "out": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ }}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "_for_testing/attention_experiments"
+
+ def patch(self, model, q, k, v, out):
+ m = attention_multiply("attn2", model, q, k, v, out)
+ return (m, )
+
+class CLIPAttentionMultiply:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "clip": ("CLIP",),
+ "q": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "k": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "v": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "out": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ }}
+ RETURN_TYPES = ("CLIP",)
+ FUNCTION = "patch"
+
+ CATEGORY = "_for_testing/attention_experiments"
+
+ def patch(self, clip, q, k, v, out):
+ m = clip.clone()
+ sd = m.patcher.model_state_dict()
+
+ for key in sd:
+ if key.endswith("self_attn.q_proj.weight") or key.endswith("self_attn.q_proj.bias"):
+ m.add_patches({key: (None,)}, 0.0, q)
+ if key.endswith("self_attn.k_proj.weight") or key.endswith("self_attn.k_proj.bias"):
+ m.add_patches({key: (None,)}, 0.0, k)
+ if key.endswith("self_attn.v_proj.weight") or key.endswith("self_attn.v_proj.bias"):
+ m.add_patches({key: (None,)}, 0.0, v)
+ if key.endswith("self_attn.out_proj.weight") or key.endswith("self_attn.out_proj.bias"):
+ m.add_patches({key: (None,)}, 0.0, out)
+ return (m, )
+
+class UNetTemporalAttentionMultiply:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "self_structural": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "self_temporal": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "cross_structural": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ "cross_temporal": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ }}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "_for_testing/attention_experiments"
+
+ def patch(self, model, self_structural, self_temporal, cross_structural, cross_temporal):
+ m = model.clone()
+ sd = model.model_state_dict()
+
+ for k in sd:
+ if (k.endswith("attn1.to_out.0.bias") or k.endswith("attn1.to_out.0.weight")):
+ if '.time_stack.' in k:
+ m.add_patches({k: (None,)}, 0.0, self_temporal)
+ else:
+ m.add_patches({k: (None,)}, 0.0, self_structural)
+ elif (k.endswith("attn2.to_out.0.bias") or k.endswith("attn2.to_out.0.weight")):
+ if '.time_stack.' in k:
+ m.add_patches({k: (None,)}, 0.0, cross_temporal)
+ else:
+ m.add_patches({k: (None,)}, 0.0, cross_structural)
+ return (m, )
+
+NODE_CLASS_MAPPINGS = {
+ "UNetSelfAttentionMultiply": UNetSelfAttentionMultiply,
+ "UNetCrossAttentionMultiply": UNetCrossAttentionMultiply,
+ "CLIPAttentionMultiply": CLIPAttentionMultiply,
+ "UNetTemporalAttentionMultiply": UNetTemporalAttentionMultiply,
+}
diff --git a/comfy_extras/nodes_canny.py b/comfy_extras/nodes_canny.py
index 8138b5f73..fab2ab7ac 100644
--- a/comfy_extras/nodes_canny.py
+++ b/comfy_extras/nodes_canny.py
@@ -1,4 +1,3 @@
-#From https://github.com/kornia/kornia
import math
import torch
diff --git a/comfy_extras/nodes_clip_sdxl.py b/comfy_extras/nodes_clip_sdxl.py
index dcf8859fa..3087b917b 100644
--- a/comfy_extras/nodes_clip_sdxl.py
+++ b/comfy_extras/nodes_clip_sdxl.py
@@ -8,7 +8,7 @@ class CLIPTextEncodeSDXLRefiner:
"ascore": ("FLOAT", {"default": 6.0, "min": 0.0, "max": 1000.0, "step": 0.01}),
"width": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}),
"height": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}),
- "text": ("STRING", {"multiline": True}), "clip": ("CLIP", ),
+ "text": ("STRING", {"multiline": True, "dynamicPrompts": True}), "clip": ("CLIP", ),
}}
RETURN_TYPES = ("CONDITIONING",)
FUNCTION = "encode"
@@ -30,8 +30,8 @@ class CLIPTextEncodeSDXL:
"crop_h": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION}),
"target_width": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}),
"target_height": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}),
- "text_g": ("STRING", {"multiline": True, "default": "CLIP_G"}), "clip": ("CLIP", ),
- "text_l": ("STRING", {"multiline": True, "default": "CLIP_L"}), "clip": ("CLIP", ),
+ "text_g": ("STRING", {"multiline": True, "dynamicPrompts": True}), "clip": ("CLIP", ),
+ "text_l": ("STRING", {"multiline": True, "dynamicPrompts": True}), "clip": ("CLIP", ),
}}
RETURN_TYPES = ("CONDITIONING",)
FUNCTION = "encode"
diff --git a/comfy_extras/nodes_cond.py b/comfy_extras/nodes_cond.py
index 646fefa17..4c3a1d5bf 100644
--- a/comfy_extras/nodes_cond.py
+++ b/comfy_extras/nodes_cond.py
@@ -3,7 +3,7 @@
class CLIPTextEncodeControlnet:
@classmethod
def INPUT_TYPES(s):
- return {"required": {"clip": ("CLIP", ), "conditioning": ("CONDITIONING", ), "text": ("STRING", {"multiline": True})}}
+ return {"required": {"clip": ("CLIP", ), "conditioning": ("CONDITIONING", ), "text": ("STRING", {"multiline": True, "dynamicPrompts": True})}}
RETURN_TYPES = ("CONDITIONING",)
FUNCTION = "encode"
diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.py
index 06238f892..47f08bf60 100644
--- a/comfy_extras/nodes_custom_sampler.py
+++ b/comfy_extras/nodes_custom_sampler.py
@@ -39,8 +39,8 @@ class KarrasScheduler:
def INPUT_TYPES(s):
return {"required":
{"steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
- "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}),
- "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}),
+ "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}),
+ "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}),
"rho": ("FLOAT", {"default": 7.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
}
}
@@ -58,8 +58,8 @@ class ExponentialScheduler:
def INPUT_TYPES(s):
return {"required":
{"steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
- "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}),
- "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}),
+ "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}),
+ "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}),
}
}
RETURN_TYPES = ("SIGMAS",)
@@ -76,8 +76,8 @@ class PolyexponentialScheduler:
def INPUT_TYPES(s):
return {"required":
{"steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
- "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}),
- "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}),
+ "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}),
+ "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}),
"rho": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
}
}
@@ -107,8 +107,7 @@ class SDTurboScheduler:
def get_sigmas(self, model, steps, denoise):
start_step = 10 - int(10 * denoise)
timesteps = torch.flip(torch.arange(1, 11) * 100 - 1, (0,))[start_step:start_step + steps]
- comfy.model_management.load_models_gpu([model])
- sigmas = model.model.model_sampling.sigma(timesteps)
+ sigmas = model.get_model_object("model_sampling").sigma(timesteps)
sigmas = torch.cat([sigmas, sigmas.new_zeros([1])])
return (sigmas, )
@@ -117,8 +116,8 @@ class VPScheduler:
def INPUT_TYPES(s):
return {"required":
{"steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
- "beta_d": ("FLOAT", {"default": 19.9, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}), #TODO: fix default values
- "beta_min": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}),
+ "beta_d": ("FLOAT", {"default": 19.9, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}), #TODO: fix default values
+ "beta_min": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}),
"eps_s": ("FLOAT", {"default": 0.001, "min": 0.0, "max": 1.0, "step":0.0001, "round": False}),
}
}
@@ -140,6 +139,7 @@ class SplitSigmas:
}
}
RETURN_TYPES = ("SIGMAS","SIGMAS")
+ RETURN_NAMES = ("high_sigmas", "low_sigmas")
CATEGORY = "sampling/custom_sampling/sigmas"
FUNCTION = "get_sigmas"
@@ -149,6 +149,27 @@ class SplitSigmas:
sigmas2 = sigmas[step:]
return (sigmas1, sigmas2)
+class SplitSigmasDenoise:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"sigmas": ("SIGMAS", ),
+ "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
+ }
+ }
+ RETURN_TYPES = ("SIGMAS","SIGMAS")
+ RETURN_NAMES = ("high_sigmas", "low_sigmas")
+ CATEGORY = "sampling/custom_sampling/sigmas"
+
+ FUNCTION = "get_sigmas"
+
+ def get_sigmas(self, sigmas, denoise):
+ steps = max(sigmas.shape[-1] - 1, 0)
+ total_steps = round(steps * denoise)
+ sigmas1 = sigmas[:-(total_steps)]
+ sigmas2 = sigmas[-(total_steps + 1):]
+ return (sigmas1, sigmas2)
+
class FlipSigmas:
@classmethod
def INPUT_TYPES(s):
@@ -600,6 +621,7 @@ NODE_CLASS_MAPPINGS = {
"SamplerDPMPP_SDE": SamplerDPMPP_SDE,
"SamplerDPMAdaptative": SamplerDPMAdaptative,
"SplitSigmas": SplitSigmas,
+ "SplitSigmasDenoise": SplitSigmasDenoise,
"FlipSigmas": FlipSigmas,
"CFGGuider": CFGGuider,
diff --git a/comfy_extras/nodes_freelunch.py b/comfy_extras/nodes_freelunch.py
index 6f1d87bf3..c5ebcf26f 100644
--- a/comfy_extras/nodes_freelunch.py
+++ b/comfy_extras/nodes_freelunch.py
@@ -42,7 +42,7 @@ class FreeU:
on_cpu_devices = {}
def output_block_patch(h, hsp, transformer_options):
- scale = scale_dict.get(h.shape[1], None)
+ scale = scale_dict.get(int(h.shape[1]), None)
if scale is not None:
h[:,:h.shape[1] // 2] = h[:,:h.shape[1] // 2] * scale[0]
if hsp.device not in on_cpu_devices:
@@ -81,7 +81,7 @@ class FreeU_V2:
on_cpu_devices = {}
def output_block_patch(h, hsp, transformer_options):
- scale = scale_dict.get(h.shape[1], None)
+ scale = scale_dict.get(int(h.shape[1]), None)
if scale is not None:
hidden_mean = h.mean(1).unsqueeze(1)
B = hidden_mean.shape[0]
diff --git a/comfy_extras/nodes_model_downscale.py b/comfy_extras/nodes_model_downscale.py
index 48bcc6892..58b5073ec 100644
--- a/comfy_extras/nodes_model_downscale.py
+++ b/comfy_extras/nodes_model_downscale.py
@@ -20,8 +20,9 @@ class PatchModelAddDownscale:
CATEGORY = "_for_testing"
def patch(self, model, block_number, downscale_factor, start_percent, end_percent, downscale_after_skip, downscale_method, upscale_method):
- sigma_start = model.model.model_sampling.percent_to_sigma(start_percent)
- sigma_end = model.model.model_sampling.percent_to_sigma(end_percent)
+ model_sampling = model.get_model_object("model_sampling")
+ sigma_start = model_sampling.percent_to_sigma(start_percent)
+ sigma_end = model_sampling.percent_to_sigma(end_percent)
def input_block_patch(h, transformer_options):
if transformer_options["block"][1] == block_number:
diff --git a/comfy_extras/nodes_model_merging.py b/comfy_extras/nodes_model_merging.py
index 2a431f65d..bb15112f4 100644
--- a/comfy_extras/nodes_model_merging.py
+++ b/comfy_extras/nodes_model_merging.py
@@ -175,9 +175,14 @@ def save_checkpoint(model, clip=None, vae=None, clip_vision=None, filename_prefi
enable_modelspec = True
if isinstance(model.model, comfy.model_base.SDXL):
- metadata["modelspec.architecture"] = "stable-diffusion-xl-v1-base"
+ if isinstance(model.model, comfy.model_base.SDXL_instructpix2pix):
+ metadata["modelspec.architecture"] = "stable-diffusion-xl-v1-edit"
+ else:
+ metadata["modelspec.architecture"] = "stable-diffusion-xl-v1-base"
elif isinstance(model.model, comfy.model_base.SDXLRefiner):
metadata["modelspec.architecture"] = "stable-diffusion-xl-v1-refiner"
+ elif isinstance(model.model, comfy.model_base.SVD_img2vid):
+ metadata["modelspec.architecture"] = "stable-video-diffusion-img2vid-v1"
else:
enable_modelspec = False
@@ -262,7 +267,7 @@ class CLIPSave:
for x in extra_pnginfo:
metadata[x] = json.dumps(extra_pnginfo[x])
- comfy.model_management.load_models_gpu([clip.load_model()])
+ comfy.model_management.load_models_gpu([clip.load_model()], force_patch_weights=True)
clip_sd = clip.get_sd()
for prefix in ["clip_l.", "clip_g.", ""]:
diff --git a/comfy_extras/nodes_pag.py b/comfy_extras/nodes_pag.py
new file mode 100644
index 000000000..63f43fd62
--- /dev/null
+++ b/comfy_extras/nodes_pag.py
@@ -0,0 +1,56 @@
+#Modified/simplified version of the node from: https://github.com/pamparamm/sd-perturbed-attention
+#If you want the one with more options see the above repo.
+
+#My modified one here is more basic but has less chances of breaking with ComfyUI updates.
+
+import comfy.model_patcher
+import comfy.samplers
+
+class PerturbedAttentionGuidance:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {
+ "required": {
+ "model": ("MODEL",),
+ "scale": ("FLOAT", {"default": 3.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}),
+ }
+ }
+
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "_for_testing"
+
+ def patch(self, model, scale):
+ unet_block = "middle"
+ unet_block_id = 0
+ m = model.clone()
+
+ def perturbed_attention(q, k, v, extra_options, mask=None):
+ return v
+
+ def post_cfg_function(args):
+ model = args["model"]
+ cond_pred = args["cond_denoised"]
+ cond = args["cond"]
+ cfg_result = args["denoised"]
+ sigma = args["sigma"]
+ model_options = args["model_options"].copy()
+ x = args["input"]
+
+ if scale == 0:
+ return cfg_result
+
+ # Replace Self-attention with PAG
+ model_options = comfy.model_patcher.set_model_options_patch_replace(model_options, perturbed_attention, "attn1", unet_block, unet_block_id)
+ (pag,) = comfy.samplers.calc_cond_batch(model, [cond], x, sigma, model_options)
+
+ return cfg_result + (cond_pred - pag) * scale
+
+ m.set_model_sampler_post_cfg_function(post_cfg_function)
+
+ return (m,)
+
+NODE_CLASS_MAPPINGS = {
+ "PerturbedAttentionGuidance": PerturbedAttentionGuidance,
+}
diff --git a/comfy_extras/nodes_perpneg.py b/comfy_extras/nodes_perpneg.py
index 992e8ab62..306cf9cd0 100644
--- a/comfy_extras/nodes_perpneg.py
+++ b/comfy_extras/nodes_perpneg.py
@@ -3,9 +3,18 @@ import comfy.model_management
import comfy.sampler_helpers
import comfy.samplers
import comfy.utils
+import node_helpers
+def perp_neg(x, noise_pred_pos, noise_pred_neg, noise_pred_nocond, neg_scale, cond_scale):
+ pos = noise_pred_pos - noise_pred_nocond
+ neg = noise_pred_neg - noise_pred_nocond
-#TODO: This node should be removed and replaced with one that uses the new Guider/SamplerCustomAdvanced.
+ perp = neg - ((torch.mul(neg, pos).sum())/(torch.norm(pos)**2)) * pos
+ perp_neg = perp * neg_scale
+ cfg_result = noise_pred_nocond + cond_scale*(pos - perp_neg)
+ return cfg_result
+
+#TODO: This node should be removed, it has been replaced with PerpNegGuider
class PerpNeg:
@classmethod
def INPUT_TYPES(s):
@@ -34,12 +43,7 @@ class PerpNeg:
(noise_pred_nocond,) = comfy.samplers.calc_cond_batch(model, [nocond_processed], x, sigma, model_options)
- pos = noise_pred_pos - noise_pred_nocond
- neg = noise_pred_neg - noise_pred_nocond
- perp = neg - ((torch.mul(neg, pos).sum())/(torch.norm(pos)**2)) * pos
- perp_neg = perp * neg_scale
- cfg_result = noise_pred_nocond + cond_scale*(pos - perp_neg)
- cfg_result = x - cfg_result
+ cfg_result = x - perp_neg(x, noise_pred_pos, noise_pred_neg, noise_pred_nocond, neg_scale, cond_scale)
return cfg_result
m.set_model_sampler_cfg_function(cfg_function)
@@ -47,10 +51,52 @@ class PerpNeg:
return (m, )
+class Guider_PerpNeg(comfy.samplers.CFGGuider):
+ def set_conds(self, positive, negative, empty_negative_prompt):
+ empty_negative_prompt = node_helpers.conditioning_set_values(empty_negative_prompt, {"prompt_type": "negative"})
+ self.inner_set_conds({"positive": positive, "empty_negative_prompt": empty_negative_prompt, "negative": negative})
+
+ def set_cfg(self, cfg, neg_scale):
+ self.cfg = cfg
+ self.neg_scale = neg_scale
+
+ def predict_noise(self, x, timestep, model_options={}, seed=None):
+ positive_cond = self.conds.get("positive", None)
+ negative_cond = self.conds.get("negative", None)
+ empty_cond = self.conds.get("empty_negative_prompt", None)
+
+ out = comfy.samplers.calc_cond_batch(self.inner_model, [negative_cond, positive_cond, empty_cond], x, timestep, model_options)
+ return perp_neg(x, out[1], out[0], out[2], self.neg_scale, self.cfg)
+
+class PerpNegGuider:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"model": ("MODEL",),
+ "positive": ("CONDITIONING", ),
+ "negative": ("CONDITIONING", ),
+ "empty_conditioning": ("CONDITIONING", ),
+ "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
+ "neg_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.01}),
+ }
+ }
+
+ RETURN_TYPES = ("GUIDER",)
+
+ FUNCTION = "get_guider"
+ CATEGORY = "_for_testing"
+
+ def get_guider(self, model, positive, negative, empty_conditioning, cfg, neg_scale):
+ guider = Guider_PerpNeg(model)
+ guider.set_conds(positive, negative, empty_conditioning)
+ guider.set_cfg(cfg, neg_scale)
+ return (guider,)
+
NODE_CLASS_MAPPINGS = {
"PerpNeg": PerpNeg,
+ "PerpNegGuider": PerpNegGuider,
}
NODE_DISPLAY_NAME_MAPPINGS = {
- "PerpNeg": "Perp-Neg",
+ "PerpNeg": "Perp-Neg (DEPRECATED by PerpNegGuider)",
}
diff --git a/comfy_extras/nodes_photomaker.py b/comfy_extras/nodes_photomaker.py
index 90130142b..29d127d74 100644
--- a/comfy_extras/nodes_photomaker.py
+++ b/comfy_extras/nodes_photomaker.py
@@ -141,7 +141,7 @@ class PhotoMakerEncode:
return {"required": { "photomaker": ("PHOTOMAKER",),
"image": ("IMAGE",),
"clip": ("CLIP", ),
- "text": ("STRING", {"multiline": True, "default": "photograph of photomaker"}),
+ "text": ("STRING", {"multiline": True, "dynamicPrompts": True, "default": "photograph of photomaker"}),
}}
RETURN_TYPES = ("CONDITIONING",)
diff --git a/comfy_extras/nodes_post_processing.py b/comfy_extras/nodes_post_processing.py
index 0110b472f..68f6ef51e 100644
--- a/comfy_extras/nodes_post_processing.py
+++ b/comfy_extras/nodes_post_processing.py
@@ -5,6 +5,7 @@ from PIL import Image
import math
import comfy.utils
+import comfy.model_management
class Blend:
@@ -102,6 +103,7 @@ class Blur:
if blur_radius == 0:
return (image,)
+ image = image.to(comfy.model_management.get_torch_device())
batch_size, height, width, channels = image.shape
kernel_size = blur_radius * 2 + 1
@@ -112,7 +114,7 @@ class Blur:
blurred = F.conv2d(padded_image, kernel, padding=kernel_size // 2, groups=channels)[:,:,blur_radius:-blur_radius, blur_radius:-blur_radius]
blurred = blurred.permute(0, 2, 3, 1)
- return (blurred,)
+ return (blurred.to(comfy.model_management.intermediate_device()),)
class Quantize:
def __init__(self):
@@ -225,6 +227,7 @@ class Sharpen:
return (image,)
batch_size, height, width, channels = image.shape
+ image = image.to(comfy.model_management.get_torch_device())
kernel_size = sharpen_radius * 2 + 1
kernel = gaussian_kernel(kernel_size, sigma, device=image.device) * -(alpha*10)
@@ -239,7 +242,7 @@ class Sharpen:
result = torch.clamp(sharpened, 0, 1)
- return (result,)
+ return (result.to(comfy.model_management.intermediate_device()),)
class ImageScaleToTotalPixels:
upscale_methods = ["nearest-exact", "bilinear", "area", "bicubic", "lanczos"]
diff --git a/comfy_extras/nodes_sag.py b/comfy_extras/nodes_sag.py
index 69084e91d..8d786db57 100644
--- a/comfy_extras/nodes_sag.py
+++ b/comfy_extras/nodes_sag.py
@@ -5,12 +5,12 @@ import math
from einops import rearrange, repeat
import os
-from comfy.ldm.modules.attention import optimized_attention, _ATTN_PRECISION
+from comfy.ldm.modules.attention import optimized_attention
import comfy.samplers
# from comfy/ldm/modules/attention.py
# but modified to return attention scores as well as output
-def attention_basic_with_sim(q, k, v, heads, mask=None):
+def attention_basic_with_sim(q, k, v, heads, mask=None, attn_precision=None):
b, _, dim_head = q.shape
dim_head //= heads
scale = dim_head ** -0.5
@@ -26,7 +26,7 @@ def attention_basic_with_sim(q, k, v, heads, mask=None):
)
# force cast to fp32 to avoid overflowing
- if _ATTN_PRECISION =="fp32":
+ if attn_precision == torch.float32:
sim = einsum('b i d, b j d -> b i j', q.float(), k.float()) * scale
else:
sim = einsum('b i d, b j d -> b i j', q, k) * scale
@@ -121,13 +121,13 @@ class SelfAttentionGuidance:
if 1 in cond_or_uncond:
uncond_index = cond_or_uncond.index(1)
# do the entire attention operation, but save the attention scores to attn_scores
- (out, sim) = attention_basic_with_sim(q, k, v, heads=heads)
+ (out, sim) = attention_basic_with_sim(q, k, v, heads=heads, attn_precision=extra_options["attn_precision"])
# when using a higher batch size, I BELIEVE the result batch dimension is [uc1, ... ucn, c1, ... cn]
n_slices = heads * b
attn_scores = sim[n_slices * uncond_index:n_slices * (uncond_index+1)]
return out
else:
- return optimized_attention(q, k, v, heads=heads)
+ return optimized_attention(q, k, v, heads=heads, attn_precision=extra_options["attn_precision"])
def post_cfg_function(args):
nonlocal attn_scores
diff --git a/comfy_extras/nodes_upscale_model.py b/comfy_extras/nodes_upscale_model.py
index 2b5e49a55..52c95df23 100644
--- a/comfy_extras/nodes_upscale_model.py
+++ b/comfy_extras/nodes_upscale_model.py
@@ -37,9 +37,14 @@ class ImageUpscaleWithModel:
def upscale(self, upscale_model, image):
device = model_management.get_torch_device()
+
+ memory_required = model_management.module_size(upscale_model)
+ memory_required += (512 * 512 * 3) * image.element_size() * max(upscale_model.scale, 1.0) * 384.0 #The 384.0 is an estimate of how much some of these models take, TODO: make it more accurate
+ memory_required += image.nelement() * image.element_size()
+ model_management.free_memory(memory_required, device)
+
upscale_model.to(device)
in_img = image.movedim(-1,-3).to(device)
- free_memory = model_management.get_free_memory(device)
tile = 512
overlap = 32
diff --git a/comfy_extras/nodes_webcam.py b/comfy_extras/nodes_webcam.py
new file mode 100644
index 000000000..32a0ba2f6
--- /dev/null
+++ b/comfy_extras/nodes_webcam.py
@@ -0,0 +1,33 @@
+import nodes
+import folder_paths
+
+MAX_RESOLUTION = nodes.MAX_RESOLUTION
+
+
+class WebcamCapture(nodes.LoadImage):
+ @classmethod
+ def INPUT_TYPES(s):
+ return {
+ "required": {
+ "image": ("WEBCAM", {}),
+ "width": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}),
+ "height": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}),
+ "capture_on_queue": ("BOOLEAN", {"default": True}),
+ }
+ }
+ RETURN_TYPES = ("IMAGE",)
+ FUNCTION = "load_capture"
+
+ CATEGORY = "image"
+
+ def load_capture(s, image, **kwargs):
+ return super().load_image(folder_paths.get_annotated_filepath(image))
+
+
+NODE_CLASS_MAPPINGS = {
+ "WebcamCapture": WebcamCapture,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+ "WebcamCapture": "Webcam Capture",
+}
\ No newline at end of file
diff --git a/execution.py b/execution.py
index 35bdb77a6..76225a962 100644
--- a/execution.py
+++ b/execution.py
@@ -176,7 +176,7 @@ def recursive_execute(server, prompt, outputs, current_item, extra_data, execute
for node_id, node_outputs in outputs.items():
output_data_formatted[node_id] = [[format_value(x) for x in l] for l in node_outputs]
- logging.error("!!! Exception during processing !!!")
+ logging.error(f"!!! Exception during processing!!! {ex}")
logging.error(traceback.format_exc())
error_details = {
@@ -622,8 +622,27 @@ def full_type_name(klass):
def validate_prompt(prompt):
outputs = set()
for x in prompt:
- class_ = nodes.NODE_CLASS_MAPPINGS[prompt[x]['class_type']]
- if hasattr(class_, 'OUTPUT_NODE') and class_.OUTPUT_NODE == True:
+ if 'class_type' not in prompt[x]:
+ error = {
+ "type": "invalid_prompt",
+ "message": f"Cannot execute because a node is missing the class_type property.",
+ "details": f"Node ID '#{x}'",
+ "extra_info": {}
+ }
+ return (False, error, [], [])
+
+ class_type = prompt[x]['class_type']
+ class_ = nodes.NODE_CLASS_MAPPINGS.get(class_type, None)
+ if class_ is None:
+ error = {
+ "type": "invalid_prompt",
+ "message": f"Cannot execute because node {class_type} does not exist.",
+ "details": f"Node ID '#{x}'",
+ "extra_info": {}
+ }
+ return (False, error, [], [])
+
+ if hasattr(class_, 'OUTPUT_NODE') and class_.OUTPUT_NODE is True:
outputs.add(x)
if len(outputs) == 0:
diff --git a/folder_paths.py b/folder_paths.py
index a84524d9f..234b73409 100644
--- a/folder_paths.py
+++ b/folder_paths.py
@@ -2,7 +2,7 @@ import os
import time
import logging
-supported_pt_extensions = set(['.ckpt', '.pt', '.bin', '.pth', '.safetensors'])
+supported_pt_extensions = set(['.ckpt', '.pt', '.bin', '.pth', '.safetensors', '.pkl'])
folder_names_and_paths = {}
@@ -181,6 +181,8 @@ def get_full_path(folder_name, filename):
full_path = os.path.join(x, filename)
if os.path.isfile(full_path):
return full_path
+ elif os.path.islink(full_path):
+ logging.warning("WARNING path {} exists but doesn't link anywhere, skipping.".format(full_path))
return None
@@ -256,7 +258,7 @@ def get_save_image_path(filename_prefix, output_dir, image_width=0, image_height
raise Exception(err)
try:
- counter = max(filter(lambda a: a[1][:-1] == filename and a[1][-1] == "_", map(map_filename, os.listdir(full_output_folder))))[0] + 1
+ counter = max(filter(lambda a: os.path.normcase(a[1][:-1]) == os.path.normcase(filename) and a[1][-1] == "_", map(map_filename, os.listdir(full_output_folder))))[0] + 1
except ValueError:
counter = 1
except FileNotFoundError:
diff --git a/latent_preview.py b/latent_preview.py
index 4dbcbf455..b258fcf20 100644
--- a/latent_preview.py
+++ b/latent_preview.py
@@ -4,6 +4,7 @@ import struct
import numpy as np
from comfy.cli_args import args, LatentPreviewMethod
from comfy.taesd.taesd import TAESD
+import comfy.model_management
import folder_paths
import comfy.utils
import logging
@@ -24,9 +25,8 @@ class TAESDPreviewerImpl(LatentPreviewer):
def decode_latent_to_preview(self, x0):
x_sample = self.taesd.decode(x0[:1])[0].detach()
- x_sample = torch.clamp((x_sample + 1.0) / 2.0, min=0.0, max=1.0)
- x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2)
- x_sample = x_sample.astype(np.uint8)
+ x_sample = 255. * torch.clamp((x_sample + 1.0) / 2.0, min=0.0, max=1.0)
+ x_sample = np.moveaxis(x_sample.to(device="cpu", dtype=torch.uint8, non_blocking=comfy.model_management.device_supports_non_blocking(x_sample.device)).numpy(), 0, 2)
preview_image = Image.fromarray(x_sample)
return preview_image
@@ -37,12 +37,13 @@ class Latent2RGBPreviewer(LatentPreviewer):
self.latent_rgb_factors = torch.tensor(latent_rgb_factors, device="cpu")
def decode_latent_to_preview(self, x0):
- latent_image = x0[0].permute(1, 2, 0).cpu() @ self.latent_rgb_factors
+ self.latent_rgb_factors = self.latent_rgb_factors.to(dtype=x0.dtype, device=x0.device)
+ latent_image = x0[0].permute(1, 2, 0) @ self.latent_rgb_factors
latents_ubyte = (((latent_image + 1) / 2)
.clamp(0, 1) # change scale from -1..1 to 0..1
.mul(0xFF) # to 0..255
- .byte()).cpu()
+ ).to(device="cpu", dtype=torch.uint8, non_blocking=comfy.model_management.device_supports_non_blocking(latent_image.device))
return Image.fromarray(latents_ubyte.numpy())
@@ -63,8 +64,6 @@ def get_previewer(device, latent_format):
if method == LatentPreviewMethod.Auto:
method = LatentPreviewMethod.Latent2RGB
- if taesd_decoder_path:
- method = LatentPreviewMethod.TAESD
if method == LatentPreviewMethod.TAESD:
if taesd_decoder_path:
diff --git a/main.py b/main.py
index b3a3ebea8..a374f2b12 100644
--- a/main.py
+++ b/main.py
@@ -243,11 +243,11 @@ if __name__ == "__main__":
call_on_start = None
if args.auto_launch:
- def startup_server(address, port):
+ def startup_server(scheme, address, port):
import webbrowser
if os.name == 'nt' and address == '0.0.0.0':
address = '127.0.0.1'
- webbrowser.open(f"http://{address}:{port}")
+ webbrowser.open(f"{scheme}://{address}:{port}")
call_on_start = startup_server
try:
diff --git a/node_helpers.py b/node_helpers.py
index 8828a4ec9..43b9e829f 100644
--- a/node_helpers.py
+++ b/node_helpers.py
@@ -1,3 +1,4 @@
+from PIL import ImageFile, UnidentifiedImageError
def conditioning_set_values(conditioning, values={}):
c = []
@@ -8,3 +9,16 @@ def conditioning_set_values(conditioning, values={}):
c.append(n)
return c
+
+def pillow(fn, arg):
+ prev_value = None
+ try:
+ x = fn(arg)
+ except (OSError, UnidentifiedImageError, ValueError): #PIL issues #4472 and #2445, also fixes ComfyUI issue #3416
+ prev_value = ImageFile.LOAD_TRUNCATED_IMAGES
+ ImageFile.LOAD_TRUNCATED_IMAGES = True
+ x = fn(arg)
+ finally:
+ if prev_value is not None:
+ ImageFile.LOAD_TRUNCATED_IMAGES = prev_value
+ return x
diff --git a/nodes.py b/nodes.py
index 78e0cf116..34821ca3f 100644
--- a/nodes.py
+++ b/nodes.py
@@ -10,14 +10,14 @@ import time
import random
import logging
-from PIL import Image, ImageOps, ImageSequence
+from PIL import Image, ImageOps, ImageSequence, ImageFile
from PIL.PngImagePlugin import PngInfo
+
import numpy as np
import safetensors.torch
sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy"))
-
import comfy.diffusers_load
import comfy.samplers
import comfy.sample
@@ -47,7 +47,7 @@ MAX_RESOLUTION=16384
class CLIPTextEncode:
@classmethod
def INPUT_TYPES(s):
- return {"required": {"text": ("STRING", {"multiline": True}), "clip": ("CLIP", )}}
+ return {"required": {"text": ("STRING", {"multiline": True, "dynamicPrompts": True}), "clip": ("CLIP", )}}
RETURN_TYPES = ("CONDITIONING",)
FUNCTION = "encode"
@@ -583,8 +583,8 @@ class LoraLoader:
return {"required": { "model": ("MODEL",),
"clip": ("CLIP", ),
"lora_name": (folder_paths.get_filename_list("loras"), ),
- "strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}),
- "strength_clip": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}),
+ "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}),
+ "strength_clip": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}),
}}
RETURN_TYPES = ("MODEL", "CLIP")
FUNCTION = "load_lora"
@@ -617,7 +617,7 @@ class LoraLoaderModelOnly(LoraLoader):
def INPUT_TYPES(s):
return {"required": { "model": ("MODEL",),
"lora_name": (folder_paths.get_filename_list("loras"), ),
- "strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}),
+ "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}),
}}
RETURN_TYPES = ("MODEL",)
FUNCTION = "load_lora_model_only"
@@ -966,7 +966,7 @@ class GLIGENTextBoxApply:
return {"required": {"conditioning_to": ("CONDITIONING", ),
"clip": ("CLIP", ),
"gligen_textbox_model": ("GLIGEN", ),
- "text": ("STRING", {"multiline": True}),
+ "text": ("STRING", {"multiline": True, "dynamicPrompts": True}),
"width": ("INT", {"default": 64, "min": 8, "max": MAX_RESOLUTION, "step": 8}),
"height": ("INT", {"default": 64, "min": 8, "max": MAX_RESOLUTION, "step": 8}),
"x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}),
@@ -1456,14 +1456,29 @@ class LoadImage:
FUNCTION = "load_image"
def load_image(self, image):
image_path = folder_paths.get_annotated_filepath(image)
- img = Image.open(image_path)
+
+ img = node_helpers.pillow(Image.open, image_path)
+
output_images = []
output_masks = []
+ w, h = None, None
+
+ excluded_formats = ['MPO']
+
for i in ImageSequence.Iterator(img):
- i = ImageOps.exif_transpose(i)
+ i = node_helpers.pillow(ImageOps.exif_transpose, i)
+
if i.mode == 'I':
i = i.point(lambda i: i * (1 / 255))
image = i.convert("RGB")
+
+ if len(output_images) == 0:
+ w = image.size[0]
+ h = image.size[1]
+
+ if image.size[0] != w or image.size[1] != h:
+ continue
+
image = np.array(image).astype(np.float32) / 255.0
image = torch.from_numpy(image)[None,]
if 'A' in i.getbands():
@@ -1474,7 +1489,7 @@ class LoadImage:
output_images.append(image)
output_masks.append(mask.unsqueeze(0))
- if len(output_images) > 1:
+ if len(output_images) > 1 and img.format not in excluded_formats:
output_image = torch.cat(output_images, dim=0)
output_mask = torch.cat(output_masks, dim=0)
else:
@@ -1515,8 +1530,8 @@ class LoadImageMask:
FUNCTION = "load_image"
def load_image(self, image, channel):
image_path = folder_paths.get_annotated_filepath(image)
- i = Image.open(image_path)
- i = ImageOps.exif_transpose(i)
+ i = node_helpers.pillow(Image.open, image_path)
+ i = node_helpers.pillow(ImageOps.exif_transpose, i)
if i.getbands() != ("R", "G", "B", "A"):
if i.mode == 'I':
i = i.point(lambda i: i * (1 / 255))
@@ -1942,6 +1957,11 @@ def init_custom_nodes():
"nodes_differential_diffusion.py",
"nodes_ip2p.py",
"nodes_model_merging_model_specific.py",
+ "nodes_pag.py",
+ "nodes_align_your_steps.py",
+ "nodes_attention_multiply.py",
+ "nodes_advanced_samplers.py",
+ "nodes_webcam.py",
]
import_failed = []
diff --git a/server.py b/server.py
index 5642bd5e2..bab3b0600 100644
--- a/server.py
+++ b/server.py
@@ -11,6 +11,7 @@ import urllib
import json
import glob
import struct
+import ssl
from PIL import Image, ImageOps
from PIL.PngImagePlugin import PngInfo
from io import BytesIO
@@ -623,14 +624,22 @@ class PromptServer():
async def start(self, address, port, verbose=True, call_on_start=None):
runner = web.AppRunner(self.app, access_log=None)
await runner.setup()
- site = web.TCPSite(runner, address, port)
+ ssl_ctx = None
+ scheme = "http"
+ if args.tls_keyfile and args.tls_certfile:
+ ssl_ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_TLS_SERVER, verify_mode=ssl.CERT_NONE)
+ ssl_ctx.load_cert_chain(certfile=args.tls_certfile,
+ keyfile=args.tls_keyfile)
+ scheme = "https"
+
+ site = web.TCPSite(runner, address, port, ssl_context=ssl_ctx)
await site.start()
if verbose:
logging.info("Starting server\n")
- logging.info("To see the GUI go to: http://{}:{}".format(address, port))
+ logging.info("To see the GUI go to: {}://{}:{}".format(scheme, address, port))
if call_on_start is not None:
- call_on_start(address, port)
+ call_on_start(scheme, address, port)
def add_on_prompt_handler(self, handler):
self.on_prompt_handlers.append(handler)
diff --git a/tests-ui/utils/ezgraph.js b/tests-ui/utils/ezgraph.js
index 8a55246ee..97be7aa72 100644
--- a/tests-ui/utils/ezgraph.js
+++ b/tests-ui/utils/ezgraph.js
@@ -204,13 +204,17 @@ export class EzWidget {
convertToWidget() {
if (!this.isConvertedToInput)
throw new Error(`Widget ${this.widget.name} cannot be converted as it is already a widget.`);
- this.node.menu[`Convert ${this.widget.name} to widget`].call();
+ var menu = this.node.menu["Convert Input to Widget"].item.submenu.options;
+ var index = menu.findIndex(a => a.content == `Convert ${this.widget.name} to widget`);
+ menu[index].callback.call();
}
convertToInput() {
if (this.isConvertedToInput)
throw new Error(`Widget ${this.widget.name} cannot be converted as it is already an input.`);
- this.node.menu[`Convert ${this.widget.name} to input`].call();
+ var menu = this.node.menu["Convert Widget to Input"].item.submenu.options;
+ var index = menu.findIndex(a => a.content == `Convert ${this.widget.name} to input`);
+ menu[index].callback.call();
}
}
diff --git a/web/extensions/core/colorPalette.js b/web/extensions/core/colorPalette.js
index b8d83613d..02546782f 100644
--- a/web/extensions/core/colorPalette.js
+++ b/web/extensions/core/colorPalette.js
@@ -20,6 +20,10 @@ const colorPalettes = {
"MODEL": "#B39DDB", // light lavender-purple
"STYLE_MODEL": "#C2FFAE", // light green-yellow
"VAE": "#FF6E6E", // bright red
+ "NOISE": "#B0B0B0", // gray
+ "GUIDER": "#66FFFF", // cyan
+ "SAMPLER": "#ECB4B4", // very soft red
+ "SIGMAS": "#CDFFCD", // soft lime green
"TAESD": "#DCC274", // cheesecake
},
"litegraph_base": {
diff --git a/web/extensions/core/dynamicPrompts.js b/web/extensions/core/dynamicPrompts.js
index 599a9e685..7417361ba 100644
--- a/web/extensions/core/dynamicPrompts.js
+++ b/web/extensions/core/dynamicPrompts.js
@@ -17,7 +17,7 @@ app.registerExtension({
// Locate dynamic prompt text widgets
// Include any widgets with dynamicPrompts set to true, and customtext
const widgets = node.widgets.filter(
- (n) => (n.type === "customtext" && n.dynamicPrompts !== false) || n.dynamicPrompts
+ (n) => n.dynamicPrompts
);
for (const widget of widgets) {
// Override the serialization of the value to resolve dynamic prompts for all widgets supporting it in this node
diff --git a/web/extensions/core/keybinds.js b/web/extensions/core/keybinds.js
index cf698ea5a..ac367c116 100644
--- a/web/extensions/core/keybinds.js
+++ b/web/extensions/core/keybinds.js
@@ -21,7 +21,6 @@ app.registerExtension({
s: "#comfy-save-button",
o: "#comfy-file-input",
Backspace: "#comfy-clear-button",
- Delete: "#comfy-clear-button",
d: "#comfy-load-default-button",
};
diff --git a/web/extensions/core/maskeditor.js b/web/extensions/core/maskeditor.js
index 4f69ac760..36f7496e7 100644
--- a/web/extensions/core/maskeditor.js
+++ b/web/extensions/core/maskeditor.js
@@ -164,6 +164,41 @@ class MaskEditorDialog extends ComfyDialog {
return divElement;
}
+ createOpacitySlider(self, name, callback) {
+ const divElement = document.createElement('div');
+ divElement.id = "maskeditor-opacity-slider";
+ divElement.style.cssFloat = "left";
+ divElement.style.fontFamily = "sans-serif";
+ divElement.style.marginRight = "4px";
+ divElement.style.color = "var(--input-text)";
+ divElement.style.backgroundColor = "var(--comfy-input-bg)";
+ divElement.style.borderRadius = "8px";
+ divElement.style.borderColor = "var(--border-color)";
+ divElement.style.borderStyle = "solid";
+ divElement.style.fontSize = "15px";
+ divElement.style.height = "21px";
+ divElement.style.padding = "1px 6px";
+ divElement.style.display = "flex";
+ divElement.style.position = "relative";
+ divElement.style.top = "2px";
+ divElement.style.pointerEvents = "auto";
+ self.opacity_slider_input = document.createElement('input');
+ self.opacity_slider_input.setAttribute('type', 'range');
+ self.opacity_slider_input.setAttribute('min', '0.1');
+ self.opacity_slider_input.setAttribute('max', '1.0');
+ self.opacity_slider_input.setAttribute('step', '0.01')
+ self.opacity_slider_input.setAttribute('value', '0.7');
+ const labelElement = document.createElement("label");
+ labelElement.textContent = name;
+
+ divElement.appendChild(labelElement);
+ divElement.appendChild(self.opacity_slider_input);
+
+ self.opacity_slider_input.addEventListener("input", callback);
+
+ return divElement;
+ }
+
setlayout(imgCanvas, maskCanvas) {
const self = this;
@@ -203,6 +238,13 @@ class MaskEditorDialog extends ComfyDialog {
self.updateBrushPreview(self, null, null);
});
+ this.brush_opacity_slider = this.createOpacitySlider(self, "Opacity", (event) => {
+ self.brush_opacity = event.target.value;
+ if (self.brush_color_mode !== "negative") {
+ self.maskCanvas.style.opacity = self.brush_opacity;
+ }
+ });
+
this.colorButton = this.createLeftButton(this.getColorButtonText(), () => {
if (self.brush_color_mode === "black") {
self.brush_color_mode = "white";
@@ -237,6 +279,7 @@ class MaskEditorDialog extends ComfyDialog {
bottom_panel.appendChild(this.saveButton);
bottom_panel.appendChild(cancelButton);
bottom_panel.appendChild(this.brush_size_slider);
+ bottom_panel.appendChild(this.brush_opacity_slider);
bottom_panel.appendChild(this.colorButton);
imgCanvas.style.position = "absolute";
@@ -472,7 +515,7 @@ class MaskEditorDialog extends ComfyDialog {
else {
return {
mixBlendMode: "initial",
- opacity: "0.7",
+ opacity: this.brush_opacity,
};
}
}
@@ -538,6 +581,7 @@ class MaskEditorDialog extends ComfyDialog {
this.maskCtx.putImageData(maskData, 0, 0);
}
+ brush_opacity = 0.7;
brush_size = 10;
brush_color_mode = "black";
drawing_mode = false;
diff --git a/web/extensions/core/webcamCapture.js b/web/extensions/core/webcamCapture.js
new file mode 100644
index 000000000..dd5725bd4
--- /dev/null
+++ b/web/extensions/core/webcamCapture.js
@@ -0,0 +1,126 @@
+import { app } from "../../scripts/app.js";
+import { api } from "../../scripts/api.js";
+
+const WEBCAM_READY = Symbol();
+
+app.registerExtension({
+ name: "Comfy.WebcamCapture",
+ getCustomWidgets(app) {
+ return {
+ WEBCAM(node, inputName) {
+ let res;
+ node[WEBCAM_READY] = new Promise((resolve) => (res = resolve));
+
+ const container = document.createElement("div");
+ container.style.background = "rgba(0,0,0,0.25)";
+ container.style.textAlign = "center";
+
+ const video = document.createElement("video");
+ video.style.height = video.style.width = "100%";
+
+ const loadVideo = async () => {
+ try {
+ const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false });
+ container.replaceChildren(video);
+
+ setTimeout(() => res(video), 500); // Fallback as loadedmetadata doesnt fire sometimes?
+ video.addEventListener("loadedmetadata", () => res(video), false);
+ video.srcObject = stream;
+ video.play();
+ } catch (error) {
+ const label = document.createElement("div");
+ label.style.color = "red";
+ label.style.overflow = "auto";
+ label.style.maxHeight = "100%";
+ label.style.whiteSpace = "pre-wrap";
+
+ if (window.isSecureContext) {
+ label.textContent = "Unable to load webcam, please ensure access is granted:\n" + error.message;
+ } else {
+ label.textContent = "Unable to load webcam. A secure context is required, if you are not accessing ComfyUI on localhost (127.0.0.1) you will have to enable TLS (https)\n\n" + error.message;
+ }
+
+ container.replaceChildren(label);
+ }
+ };
+
+ loadVideo();
+
+ return { widget: node.addDOMWidget(inputName, "WEBCAM", container) };
+ },
+ };
+ },
+ nodeCreated(node) {
+ if ((node.type, node.constructor.comfyClass !== "WebcamCapture")) return;
+
+ let video;
+ const camera = node.widgets.find((w) => w.name === "image");
+ const w = node.widgets.find((w) => w.name === "width");
+ const h = node.widgets.find((w) => w.name === "height");
+ const captureOnQueue = node.widgets.find((w) => w.name === "capture_on_queue");
+
+ const canvas = document.createElement("canvas");
+
+ const capture = () => {
+ canvas.width = w.value;
+ canvas.height = h.value;
+ const ctx = canvas.getContext("2d");
+ ctx.drawImage(video, 0, 0, w.value, h.value);
+ const data = canvas.toDataURL("image/png");
+
+ const img = new Image();
+ img.onload = () => {
+ node.imgs = [img];
+ app.graph.setDirtyCanvas(true);
+ requestAnimationFrame(() => {
+ node.setSizeForImage?.();
+ });
+ };
+ img.src = data;
+ };
+
+ const btn = node.addWidget("button", "waiting for camera...", "capture", capture);
+ btn.disabled = true;
+ btn.serializeValue = () => undefined;
+
+ camera.serializeValue = async () => {
+ if (captureOnQueue.value) {
+ capture();
+ } else if (!node.imgs?.length) {
+ const err = `No webcam image captured`;
+ alert(err);
+ throw new Error(err);
+ }
+
+ // Upload image to temp storage
+ const blob = await new Promise((r) => canvas.toBlob(r));
+ const name = `${+new Date()}.png`;
+ const file = new File([blob], name);
+ const body = new FormData();
+ body.append("image", file);
+ body.append("subfolder", "webcam");
+ body.append("type", "temp");
+ const resp = await api.fetchApi("/upload/image", {
+ method: "POST",
+ body,
+ });
+ if (resp.status !== 200) {
+ const err = `Error uploading camera image: ${resp.status} - ${resp.statusText}`;
+ alert(err);
+ throw new Error(err);
+ }
+ return `webcam/${name} [temp]`;
+ };
+
+ node[WEBCAM_READY].then((v) => {
+ video = v;
+ // If width isnt specified then use video output resolution
+ if (!w.value) {
+ w.value = video.videoWidth || 640;
+ h.value = video.videoHeight || 480;
+ }
+ btn.disabled = false;
+ btn.label = "capture";
+ });
+ },
+});
diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js
index 23f51d812..f1a1d22cd 100644
--- a/web/extensions/core/widgetInputs.js
+++ b/web/extensions/core/widgetInputs.js
@@ -256,8 +256,18 @@ export function mergeIfValid(output, config2, forceUpdate, recreateWidget, confi
return { customConfig };
}
+let useConversionSubmenusSetting;
app.registerExtension({
name: "Comfy.WidgetInputs",
+ init() {
+ useConversionSubmenusSetting = app.ui.settings.addSetting({
+ id: "Comfy.NodeInputConversionSubmenus",
+ name: "Node widget/input conversion sub-menus",
+ tooltip: "In the node context menu, place the entries that convert between input/widget in sub-menus.",
+ type: "boolean",
+ defaultValue: true,
+ });
+ },
async beforeRegisterNodeDef(nodeType, nodeData, app) {
// Add menu options to conver to/from widgets
const origGetExtraMenuOptions = nodeType.prototype.getExtraMenuOptions;
@@ -292,12 +302,31 @@ app.registerExtension({
}
}
}
+
+ //Convert.. main menu
if (toInput.length) {
- options.push(...toInput, null);
+ if (useConversionSubmenusSetting.value) {
+ options.push({
+ content: "Convert Widget to Input",
+ submenu: {
+ options: toInput,
+ },
+ });
+ } else {
+ options.push(...toInput, null);
+ }
}
-
if (toWidget.length) {
- options.push(...toWidget, null);
+ if (useConversionSubmenusSetting.value) {
+ options.push({
+ content: "Convert Input to Widget",
+ submenu: {
+ options: toWidget,
+ },
+ });
+ } else {
+ options.push(...toWidget, null);
+ }
}
}
diff --git a/web/scripts/app.js b/web/scripts/app.js
index c9c754148..6f6759bb5 100644
--- a/web/scripts/app.js
+++ b/web/scripts/app.js
@@ -262,6 +262,36 @@ export class ComfyApp {
})
);
}
+
+ #addRestoreWorkflowView() {
+ const serialize = LGraph.prototype.serialize;
+ const self = this;
+ LGraph.prototype.serialize = function() {
+ const workflow = serialize.apply(this, arguments);
+
+ // Store the drag & scale info in the serialized workflow if the setting is enabled
+ if (self.enableWorkflowViewRestore.value) {
+ if (!workflow.extra) {
+ workflow.extra = {};
+ }
+ workflow.extra.ds = {
+ scale: self.canvas.ds.scale,
+ offset: self.canvas.ds.offset,
+ };
+ } else if (workflow.extra?.ds) {
+ // Clear any old view data
+ delete workflow.extra.ds;
+ }
+
+ return workflow;
+ }
+ this.enableWorkflowViewRestore = this.ui.settings.addSetting({
+ id: "Comfy.EnableWorkflowViewRestore",
+ name: "Save and restore canvas position and zoom level in workflows",
+ type: "boolean",
+ defaultValue: true
+ });
+ }
/**
* Adds special context menu handling for nodes
@@ -953,6 +983,12 @@ export class ComfyApp {
const origProcessMouseDown = LGraphCanvas.prototype.processMouseDown;
LGraphCanvas.prototype.processMouseDown = function(e) {
+ // prepare for ctrl+shift drag: zoom start
+ if(e.ctrlKey && e.shiftKey && e.buttons) {
+ self.zoom_drag_start = [e.x, e.y, this.ds.scale];
+ return;
+ }
+
const res = origProcessMouseDown.apply(this, arguments);
this.selected_group_moving = false;
@@ -973,6 +1009,26 @@ export class ComfyApp {
const origProcessMouseMove = LGraphCanvas.prototype.processMouseMove;
LGraphCanvas.prototype.processMouseMove = function(e) {
+ // handle ctrl+shift drag
+ if(e.ctrlKey && e.shiftKey && self.zoom_drag_start) {
+ // stop canvas zoom action
+ if(!e.buttons) {
+ self.zoom_drag_start = null;
+ return;
+ }
+
+ // calculate delta
+ let deltaY = e.y - self.zoom_drag_start[1];
+ let startScale = self.zoom_drag_start[2];
+
+ let scale = startScale - deltaY/100;
+
+ this.ds.changeScale(scale, [this.ds.element.width/2, this.ds.element.height/2]);
+ this.graph.change();
+
+ return;
+ }
+
const orig_selected_group = this.selected_group;
if (this.selected_group && !this.selected_group_resizing && !this.selected_group_moving) {
@@ -1059,6 +1115,20 @@ export class ComfyApp {
// Trigger onPaste
return true;
}
+
+ if((e.key === '+') && e.altKey) {
+ block_default = true;
+ let scale = this.ds.scale * 1.1;
+ this.ds.changeScale(scale, [this.ds.element.width/2, this.ds.element.height/2]);
+ this.graph.change();
+ }
+
+ if((e.key === '-') && e.altKey) {
+ block_default = true;
+ let scale = this.ds.scale * 1 / 1.1;
+ this.ds.changeScale(scale, [this.ds.element.width/2, this.ds.element.height/2]);
+ this.graph.change();
+ }
}
this.graph.change();
@@ -1465,6 +1535,7 @@ export class ComfyApp {
this.#addProcessKeyHandler();
this.#addConfigureHandler();
this.#addApiUpdateHandlers();
+ this.#addRestoreWorkflowView();
this.graph = new LGraph();
@@ -1765,6 +1836,10 @@ export class ComfyApp {
try {
this.graph.configure(graphData);
+ if (this.enableWorkflowViewRestore.value && graphData.extra?.ds) {
+ this.canvas.ds.offset = graphData.extra.ds.offset;
+ this.canvas.ds.scale = graphData.extra.ds.scale;
+ }
} catch (error) {
let errorHint = [];
// Try extracting filename to see if it was caused by an extension script
@@ -2082,6 +2157,14 @@ export class ComfyApp {
api.dispatchEvent(new CustomEvent("promptQueued", { detail: { number, batchCount } }));
}
+ showErrorOnFileLoad(file) {
+ this.ui.dialog.show(
+ $el("div", [
+ $el("p", {textContent: `Unable to find workflow in ${file.name}`})
+ ]).outerHTML
+ );
+ }
+
/**
* Loads workflow data from the specified file
* @param {File} file
@@ -2089,27 +2172,27 @@ export class ComfyApp {
async handleFile(file) {
if (file.type === "image/png") {
const pngInfo = await getPngMetadata(file);
- if (pngInfo) {
- if (pngInfo.workflow) {
- await this.loadGraphData(JSON.parse(pngInfo.workflow));
- } else if (pngInfo.prompt) {
- this.loadApiJson(JSON.parse(pngInfo.prompt));
- } else if (pngInfo.parameters) {
- importA1111(this.graph, pngInfo.parameters);
- }
+ if (pngInfo?.workflow) {
+ await this.loadGraphData(JSON.parse(pngInfo.workflow));
+ } else if (pngInfo?.prompt) {
+ this.loadApiJson(JSON.parse(pngInfo.prompt));
+ } else if (pngInfo?.parameters) {
+ importA1111(this.graph, pngInfo.parameters);
+ } else {
+ this.showErrorOnFileLoad(file);
}
} else if (file.type === "image/webp") {
const pngInfo = await getWebpMetadata(file);
- if (pngInfo) {
- if (pngInfo.workflow) {
- this.loadGraphData(JSON.parse(pngInfo.workflow));
- } else if (pngInfo.Workflow) {
- this.loadGraphData(JSON.parse(pngInfo.Workflow)); // Support loading workflows from that webp custom node.
- } else if (pngInfo.prompt) {
- this.loadApiJson(JSON.parse(pngInfo.prompt));
- } else if (pngInfo.Prompt) {
- this.loadApiJson(JSON.parse(pngInfo.Prompt)); // Support loading prompts from that webp custom node.
- }
+ // Support loading workflows from that webp custom node.
+ const workflow = pngInfo?.workflow || pngInfo?.Workflow;
+ const prompt = pngInfo?.prompt || pngInfo?.Prompt;
+
+ if (workflow) {
+ this.loadGraphData(JSON.parse(workflow));
+ } else if (prompt) {
+ this.loadApiJson(JSON.parse(prompt));
+ } else {
+ this.showErrorOnFileLoad(file);
}
} else if (file.type === "application/json" || file.name?.endsWith(".json")) {
const reader = new FileReader();
@@ -2130,7 +2213,11 @@ export class ComfyApp {
await this.loadGraphData(JSON.parse(info.workflow));
} else if (info.prompt) {
this.loadApiJson(JSON.parse(info.prompt));
+ } else {
+ this.showErrorOnFileLoad(file);
}
+ } else {
+ this.showErrorOnFileLoad(file);
}
}
@@ -2238,6 +2325,12 @@ export class ComfyApp {
await this.#invokeExtensionsAsync("refreshComboInNodes", defs);
}
+ resetView() {
+ app.canvas.ds.scale = 1;
+ app.canvas.ds.offset = [0, 0]
+ app.graph.setDirtyCanvas(true, true);
+ }
+
/**
* Clean current state
*/
diff --git a/web/scripts/ui.js b/web/scripts/ui.js
index 5ca6214eb..36fed3238 100644
--- a/web/scripts/ui.js
+++ b/web/scripts/ui.js
@@ -90,12 +90,15 @@ function dragElement(dragEl, settings) {
}).observe(dragEl);
function ensureInBounds() {
- if (dragEl.classList.contains("comfy-menu-manual-pos")) {
+ try {
newPosX = Math.min(document.body.clientWidth - dragEl.clientWidth, Math.max(0, dragEl.offsetLeft));
newPosY = Math.min(document.body.clientHeight - dragEl.clientHeight, Math.max(0, dragEl.offsetTop));
positionElement();
}
+ catch(exception){
+ // robust
+ }
}
function positionElement() {
@@ -594,16 +597,23 @@ export class ComfyUI {
if (!confirmClear.value || confirm("Clear workflow?")) {
app.clean();
app.graph.clear();
+ app.resetView();
}
}
}),
$el("button", {
id: "comfy-load-default-button", textContent: "Load Default", onclick: async () => {
if (!confirmClear.value || confirm("Load default workflow?")) {
+ app.resetView();
await app.loadGraphData()
}
}
}),
+ $el("button", {
+ id: "comfy-reset-view-button", textContent: "Reset View", onclick: async () => {
+ app.resetView();
+ }
+ }),
]);
const devMode = this.settings.addSetting({
diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js
index 678b1b8ec..6a6899705 100644
--- a/web/scripts/widgets.js
+++ b/web/scripts/widgets.js
@@ -229,7 +229,11 @@ function createIntWidget(node, inputName, inputData, app, isSeedInput) {
val,
function (v) {
const s = this.options.step / 10;
- this.value = Math.round(v / s) * s;
+ let sh = this.options.min % s;
+ if (isNaN(sh)) {
+ sh = 0;
+ }
+ this.value = Math.round((v - sh) / s) * s + sh;
},
config
),
@@ -307,7 +311,9 @@ export const ComfyWidgets = {
return { widget: node.addWidget(widgetType, inputName, val,
function (v) {
if (config.round) {
- this.value = Math.round(v/config.round)*config.round;
+ this.value = Math.round((v + Number.EPSILON)/config.round)*config.round;
+ if (this.value > config.max) this.value = config.max;
+ if (this.value < config.min) this.value = config.min;
} else {
this.value = v;
}