diff --git a/.ci/nightly/update_windows/update_comfyui_and_python_dependencies.bat b/.ci/nightly/update_windows/update_comfyui_and_python_dependencies.bat
deleted file mode 100755
index 94f5d1023..000000000
--- a/.ci/nightly/update_windows/update_comfyui_and_python_dependencies.bat
+++ /dev/null
@@ -1,3 +0,0 @@
-..\python_embeded\python.exe .\update.py ..\ComfyUI\
-..\python_embeded\python.exe -s -m pip install --upgrade --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu121 -r ../ComfyUI/requirements.txt pygit2
-pause
diff --git a/.ci/nightly/windows_base_files/run_nvidia_gpu.bat b/.ci/nightly/windows_base_files/run_nvidia_gpu.bat
deleted file mode 100755
index 8ee2f3402..000000000
--- a/.ci/nightly/windows_base_files/run_nvidia_gpu.bat
+++ /dev/null
@@ -1,2 +0,0 @@
-.\python_embeded\python.exe -s ComfyUI\main.py --windows-standalone-build --use-pytorch-cross-attention
-pause
diff --git a/.github/workflows/test-ui.yaml b/.github/workflows/test-ui.yaml
index 950691755..4b8b97934 100644
--- a/.github/workflows/test-ui.yaml
+++ b/.github/workflows/test-ui.yaml
@@ -22,5 +22,5 @@ jobs:
run: |
npm ci
npm run test:generate
- npm test
+ npm test -- --verbose
working-directory: ./tests-ui
diff --git a/.github/workflows/windows_release_nightly_pytorch.yml b/.github/workflows/windows_release_nightly_pytorch.yml
index b793f7fe2..90e09d27a 100644
--- a/.github/workflows/windows_release_nightly_pytorch.yml
+++ b/.github/workflows/windows_release_nightly_pytorch.yml
@@ -2,6 +2,24 @@ name: "Windows Release Nightly pytorch"
on:
workflow_dispatch:
+ inputs:
+ cu:
+ description: 'cuda version'
+ required: true
+ type: string
+ default: "121"
+
+ python_minor:
+ description: 'python minor version'
+ required: true
+ type: string
+ default: "12"
+
+ python_patch:
+ description: 'python patch version'
+ required: true
+ type: string
+ default: "1"
# push:
# branches:
# - master
@@ -20,21 +38,21 @@ jobs:
persist-credentials: false
- uses: actions/setup-python@v4
with:
- python-version: '3.11.6'
+ python-version: 3.${{ inputs.python_minor }}.${{ inputs.python_patch }}
- shell: bash
run: |
cd ..
cp -r ComfyUI ComfyUI_copy
- curl https://www.python.org/ftp/python/3.11.6/python-3.11.6-embed-amd64.zip -o python_embeded.zip
+ curl https://www.python.org/ftp/python/3.${{ inputs.python_minor }}.${{ inputs.python_patch }}/python-3.${{ inputs.python_minor }}.${{ inputs.python_patch }}-embed-amd64.zip -o python_embeded.zip
unzip python_embeded.zip -d python_embeded
cd python_embeded
- echo 'import site' >> ./python311._pth
+ echo 'import site' >> ./python3${{ inputs.python_minor }}._pth
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
./python.exe get-pip.py
- python -m pip wheel torch torchvision torchaudio aiohttp==3.8.5 --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu121 -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
+ python -m pip wheel torch torchvision torchaudio --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
ls ../temp_wheel_dir
./python.exe -s -m pip install --pre ../temp_wheel_dir/*
- sed -i '1i../ComfyUI' ./python311._pth
+ sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth
cd ..
git clone https://github.com/comfyanonymous/taesd
@@ -49,9 +67,10 @@ jobs:
mkdir update
cp -r ComfyUI/.ci/update_windows/* ./update/
cp -r ComfyUI/.ci/windows_base_files/* ./
- cp -r ComfyUI/.ci/nightly/update_windows/* ./update/
- cp -r ComfyUI/.ci/nightly/windows_base_files/* ./
+ echo "..\python_embeded\python.exe .\update.py ..\ComfyUI\\
+ ..\python_embeded\python.exe -s -m pip install --upgrade --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2
+ pause" > ./update/update_comfyui_and_python_dependencies.bat
cd ..
"C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma -mx=8 -mfb=64 -md=32m -ms=on -mf=BCJ2 ComfyUI_windows_portable_nightly_pytorch.7z ComfyUI_windows_portable_nightly_pytorch
diff --git a/README.md b/README.md
index d622c9072..167214c05 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ This ui will let you design and execute advanced stable diffusion pipelines usin
## Features
- Nodes/graph/flowchart interface to experiment and create complex Stable Diffusion workflows without needing to code anything.
-- Fully supports SD1.x, SD2.x and SDXL
+- Fully supports SD1.x, SD2.x, [SDXL](https://comfyanonymous.github.io/ComfyUI_examples/sdxl/) and [Stable Video Diffusion](https://comfyanonymous.github.io/ComfyUI_examples/video/)
- Asynchronous Queue system
- Many optimizations: Only re-executes the parts of the workflow that changes between executions.
- Command line option: ```--lowvram``` to make it work on GPUs with less than 3GB vram (enabled automatically on GPUs with low vram)
@@ -30,6 +30,8 @@ This ui will let you design and execute advanced stable diffusion pipelines usin
- [unCLIP Models](https://comfyanonymous.github.io/ComfyUI_examples/unclip/)
- [GLIGEN](https://comfyanonymous.github.io/ComfyUI_examples/gligen/)
- [Model Merging](https://comfyanonymous.github.io/ComfyUI_examples/model_merging/)
+- [LCM models and Loras](https://comfyanonymous.github.io/ComfyUI_examples/lcm/)
+- [SDXL Turbo](https://comfyanonymous.github.io/ComfyUI_examples/sdturbo/)
- Latent previews with [TAESD](#how-to-show-high-quality-previews)
- Starts up very fast.
- Works fully offline: will never download anything.
@@ -43,6 +45,7 @@ Workflow examples can be found on the [Examples page](https://comfyanonymous.git
|---------------------------|--------------------------------------------------------------------------------------------------------------------|
| Ctrl + Enter | Queue up current graph for generation |
| Ctrl + Shift + Enter | Queue up current graph as first for generation |
+| Ctrl + Z/Ctrl + Y | Undo/Redo |
| Ctrl + S | Save workflow |
| Ctrl + O | Load workflow |
| Ctrl + A | Select all nodes |
@@ -90,22 +93,27 @@ Put your SD checkpoints (the huge ckpt/safetensors files) in: models/checkpoints
Put your VAE in: models/vae
-Note: pytorch does not support python 3.12 yet so make sure your python version is 3.11 or earlier.
+Note: pytorch stable does not support python 3.12 yet. If you have python 3.12 you will have to use the nightly version of pytorch. If you run into issues you should try python 3.11 instead.
### AMD GPUs (Linux only)
AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version:
```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.6```
-This is the command to install the nightly with ROCm 5.7 that might have some performance improvements:
+This is the command to install the nightly with ROCm 5.7 which has a python 3.12 package and might have some performance improvements:
+
```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm5.7```
### NVIDIA
-Nvidia users should install pytorch using this command:
+Nvidia users should install stable pytorch using this command:
```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu121```
+This is the command to install pytorch nightly instead which has a python 3.12 package and might have performance improvements:
+
+```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121```
+
#### Troubleshooting
If you get the "Torch not compiled with CUDA enabled" error, uninstall torch with:
@@ -190,7 +198,7 @@ To use a textual inversion concepts/embeddings in a text prompt put them in the
Make sure you use the regular loaders/Load Checkpoint node to load checkpoints. It will auto pick the right settings depending on your GPU.
-You can set this command line setting to disable the upcasting to fp32 in some cross attention operations which will increase your speed. Note that this will very likely give you black images on SD2.x models. If you use xformers this option does not do anything.
+You can set this command line setting to disable the upcasting to fp32 in some cross attention operations which will increase your speed. Note that this will very likely give you black images on SD2.x models. If you use xformers or pytorch attention this option does not do anything.
```--dont-upcast-attention```
diff --git a/comfy/cldm/cldm.py b/comfy/cldm/cldm.py
index 9a63202ab..5eee5a51c 100644
--- a/comfy/cldm/cldm.py
+++ b/comfy/cldm/cldm.py
@@ -53,7 +53,8 @@ class ControlNet(nn.Module):
transformer_depth_middle=None,
transformer_depth_output=None,
device=None,
- operations=comfy.ops,
+ operations=comfy.ops.disable_weight_init,
+ **kwargs,
):
super().__init__()
assert use_spatial_transformer == True, "use_spatial_transformer has to be true"
@@ -140,24 +141,24 @@ class ControlNet(nn.Module):
)
]
)
- self.zero_convs = nn.ModuleList([self.make_zero_conv(model_channels, operations=operations)])
+ self.zero_convs = nn.ModuleList([self.make_zero_conv(model_channels, operations=operations, dtype=self.dtype, device=device)])
self.input_hint_block = TimestepEmbedSequential(
- operations.conv_nd(dims, hint_channels, 16, 3, padding=1),
+ operations.conv_nd(dims, hint_channels, 16, 3, padding=1, dtype=self.dtype, device=device),
nn.SiLU(),
- operations.conv_nd(dims, 16, 16, 3, padding=1),
+ operations.conv_nd(dims, 16, 16, 3, padding=1, dtype=self.dtype, device=device),
nn.SiLU(),
- operations.conv_nd(dims, 16, 32, 3, padding=1, stride=2),
+ operations.conv_nd(dims, 16, 32, 3, padding=1, stride=2, dtype=self.dtype, device=device),
nn.SiLU(),
- operations.conv_nd(dims, 32, 32, 3, padding=1),
+ operations.conv_nd(dims, 32, 32, 3, padding=1, dtype=self.dtype, device=device),
nn.SiLU(),
- operations.conv_nd(dims, 32, 96, 3, padding=1, stride=2),
+ operations.conv_nd(dims, 32, 96, 3, padding=1, stride=2, dtype=self.dtype, device=device),
nn.SiLU(),
- operations.conv_nd(dims, 96, 96, 3, padding=1),
+ operations.conv_nd(dims, 96, 96, 3, padding=1, dtype=self.dtype, device=device),
nn.SiLU(),
- operations.conv_nd(dims, 96, 256, 3, padding=1, stride=2),
+ operations.conv_nd(dims, 96, 256, 3, padding=1, stride=2, dtype=self.dtype, device=device),
nn.SiLU(),
- zero_module(operations.conv_nd(dims, 256, model_channels, 3, padding=1))
+ operations.conv_nd(dims, 256, model_channels, 3, padding=1, dtype=self.dtype, device=device)
)
self._feature_size = model_channels
@@ -205,7 +206,7 @@ class ControlNet(nn.Module):
)
)
self.input_blocks.append(TimestepEmbedSequential(*layers))
- self.zero_convs.append(self.make_zero_conv(ch, operations=operations))
+ self.zero_convs.append(self.make_zero_conv(ch, operations=operations, dtype=self.dtype, device=device))
self._feature_size += ch
input_block_chans.append(ch)
if level != len(channel_mult) - 1:
@@ -233,7 +234,7 @@ class ControlNet(nn.Module):
)
ch = out_ch
input_block_chans.append(ch)
- self.zero_convs.append(self.make_zero_conv(ch, operations=operations))
+ self.zero_convs.append(self.make_zero_conv(ch, operations=operations, dtype=self.dtype, device=device))
ds *= 2
self._feature_size += ch
@@ -275,14 +276,14 @@ class ControlNet(nn.Module):
operations=operations
)]
self.middle_block = TimestepEmbedSequential(*mid_block)
- self.middle_block_out = self.make_zero_conv(ch, operations=operations)
+ self.middle_block_out = self.make_zero_conv(ch, operations=operations, dtype=self.dtype, device=device)
self._feature_size += ch
- def make_zero_conv(self, channels, operations=None):
- return TimestepEmbedSequential(zero_module(operations.conv_nd(self.dims, channels, channels, 1, padding=0)))
+ def make_zero_conv(self, channels, operations=None, dtype=None, device=None):
+ return TimestepEmbedSequential(operations.conv_nd(self.dims, channels, channels, 1, padding=0, dtype=dtype, device=device))
def forward(self, x, hint, timesteps, context, y=None, **kwargs):
- t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(self.dtype)
+ t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(x.dtype)
emb = self.time_embed(t_emb)
guided_hint = self.input_hint_block(hint, emb, context)
@@ -294,7 +295,7 @@ class ControlNet(nn.Module):
assert y.shape[0] == x.shape[0]
emb = emb + self.label_emb(y)
- h = x.type(self.dtype)
+ h = x
for module, zero_conv in zip(self.input_blocks, self.zero_convs):
if guided_hint is not None:
h = module(h, emb, context)
diff --git a/comfy/cli_args.py b/comfy/cli_args.py
index e79b89c0f..50d7b62fa 100644
--- a/comfy/cli_args.py
+++ b/comfy/cli_args.py
@@ -55,13 +55,26 @@ fp_group = parser.add_mutually_exclusive_group()
fp_group.add_argument("--force-fp32", action="store_true", help="Force fp32 (If this makes your GPU work better please report it).")
fp_group.add_argument("--force-fp16", action="store_true", help="Force fp16.")
-parser.add_argument("--bf16-unet", action="store_true", help="Run the UNET in bf16. This should only be used for testing stuff.")
+fpunet_group = parser.add_mutually_exclusive_group()
+fpunet_group.add_argument("--bf16-unet", action="store_true", help="Run the UNET in bf16. This should only be used for testing stuff.")
+fpunet_group.add_argument("--fp16-unet", action="store_true", help="Store unet weights in fp16.")
+fpunet_group.add_argument("--fp8_e4m3fn-unet", action="store_true", help="Store unet weights in fp8_e4m3fn.")
+fpunet_group.add_argument("--fp8_e5m2-unet", action="store_true", help="Store unet weights in fp8_e5m2.")
fpvae_group = parser.add_mutually_exclusive_group()
fpvae_group.add_argument("--fp16-vae", action="store_true", help="Run the VAE in fp16, might cause black images.")
fpvae_group.add_argument("--fp32-vae", action="store_true", help="Run the VAE in full precision fp32.")
fpvae_group.add_argument("--bf16-vae", action="store_true", help="Run the VAE in bf16.")
+parser.add_argument("--cpu-vae", action="store_true", help="Run the VAE on the CPU.")
+
+fpte_group = parser.add_mutually_exclusive_group()
+fpte_group.add_argument("--fp8_e4m3fn-text-enc", action="store_true", help="Store text encoder weights in fp8 (e4m3fn variant).")
+fpte_group.add_argument("--fp8_e5m2-text-enc", action="store_true", help="Store text encoder weights in fp8 (e5m2 variant).")
+fpte_group.add_argument("--fp16-text-enc", action="store_true", help="Store text encoder weights in fp16.")
+fpte_group.add_argument("--fp32-text-enc", action="store_true", help="Store text encoder weights in fp32.")
+
+
parser.add_argument("--directml", type=int, nargs="?", metavar="DIRECTML_DEVICE", const=-1, help="Use torch-directml.")
parser.add_argument("--disable-ipex-optimize", action="store_true", help="Disables ipex.optimize when loading models with Intel GPUs.")
@@ -91,7 +104,7 @@ vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for e
parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.")
-
+parser.add_argument("--deterministic", action="store_true", help="Make pytorch use slower deterministic algorithms when it can. Note that this might not make images deterministic in all cases.")
parser.add_argument("--dont-print-server", action="store_true", help="Don't print server output.")
parser.add_argument("--quick-test-for-ci", action="store_true", help="Quick test for CI.")
diff --git a/comfy/clip_model.py b/comfy/clip_model.py
new file mode 100644
index 000000000..7397b7a26
--- /dev/null
+++ b/comfy/clip_model.py
@@ -0,0 +1,188 @@
+import torch
+from comfy.ldm.modules.attention import optimized_attention_for_device
+
+class CLIPAttention(torch.nn.Module):
+ def __init__(self, embed_dim, heads, dtype, device, operations):
+ super().__init__()
+
+ self.heads = heads
+ self.q_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)
+ self.k_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)
+ self.v_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)
+
+ self.out_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)
+
+ def forward(self, x, mask=None, optimized_attention=None):
+ q = self.q_proj(x)
+ k = self.k_proj(x)
+ v = self.v_proj(x)
+
+ out = optimized_attention(q, k, v, self.heads, mask)
+ return self.out_proj(out)
+
+ACTIVATIONS = {"quick_gelu": lambda a: a * torch.sigmoid(1.702 * a),
+ "gelu": torch.nn.functional.gelu,
+}
+
+class CLIPMLP(torch.nn.Module):
+ def __init__(self, embed_dim, intermediate_size, activation, dtype, device, operations):
+ super().__init__()
+ self.fc1 = operations.Linear(embed_dim, intermediate_size, bias=True, dtype=dtype, device=device)
+ self.activation = ACTIVATIONS[activation]
+ self.fc2 = operations.Linear(intermediate_size, embed_dim, bias=True, dtype=dtype, device=device)
+
+ def forward(self, x):
+ x = self.fc1(x)
+ x = self.activation(x)
+ x = self.fc2(x)
+ return x
+
+class CLIPLayer(torch.nn.Module):
+ def __init__(self, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations):
+ super().__init__()
+ self.layer_norm1 = operations.LayerNorm(embed_dim, dtype=dtype, device=device)
+ self.self_attn = CLIPAttention(embed_dim, heads, dtype, device, operations)
+ self.layer_norm2 = operations.LayerNorm(embed_dim, dtype=dtype, device=device)
+ self.mlp = CLIPMLP(embed_dim, intermediate_size, intermediate_activation, dtype, device, operations)
+
+ def forward(self, x, mask=None, optimized_attention=None):
+ x += self.self_attn(self.layer_norm1(x), mask, optimized_attention)
+ x += self.mlp(self.layer_norm2(x))
+ return x
+
+
+class CLIPEncoder(torch.nn.Module):
+ def __init__(self, num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations):
+ super().__init__()
+ self.layers = torch.nn.ModuleList([CLIPLayer(embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations) for i in range(num_layers)])
+
+ def forward(self, x, mask=None, intermediate_output=None):
+ optimized_attention = optimized_attention_for_device(x.device, mask=mask is not None)
+
+ if intermediate_output is not None:
+ if intermediate_output < 0:
+ intermediate_output = len(self.layers) + intermediate_output
+
+ intermediate = None
+ for i, l in enumerate(self.layers):
+ x = l(x, mask, optimized_attention)
+ if i == intermediate_output:
+ intermediate = x.clone()
+ return x, intermediate
+
+class CLIPEmbeddings(torch.nn.Module):
+ def __init__(self, embed_dim, vocab_size=49408, num_positions=77, dtype=None, device=None):
+ super().__init__()
+ self.token_embedding = torch.nn.Embedding(vocab_size, embed_dim, dtype=dtype, device=device)
+ self.position_embedding = torch.nn.Embedding(num_positions, embed_dim, dtype=dtype, device=device)
+
+ def forward(self, input_tokens):
+ return self.token_embedding(input_tokens) + self.position_embedding.weight
+
+
+class CLIPTextModel_(torch.nn.Module):
+ def __init__(self, config_dict, dtype, device, operations):
+ num_layers = config_dict["num_hidden_layers"]
+ embed_dim = config_dict["hidden_size"]
+ heads = config_dict["num_attention_heads"]
+ intermediate_size = config_dict["intermediate_size"]
+ intermediate_activation = config_dict["hidden_act"]
+
+ super().__init__()
+ self.embeddings = CLIPEmbeddings(embed_dim, dtype=torch.float32, device=device)
+ self.encoder = CLIPEncoder(num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations)
+ self.final_layer_norm = operations.LayerNorm(embed_dim, dtype=dtype, device=device)
+
+ def forward(self, input_tokens, attention_mask=None, intermediate_output=None, final_layer_norm_intermediate=True):
+ x = self.embeddings(input_tokens)
+ mask = None
+ if attention_mask is not None:
+ mask = 1.0 - attention_mask.to(x.dtype).unsqueeze(1).unsqueeze(1).expand(attention_mask.shape[0], 1, attention_mask.shape[-1], attention_mask.shape[-1])
+ mask = mask.masked_fill(mask.to(torch.bool), float("-inf"))
+
+ causal_mask = torch.empty(x.shape[1], x.shape[1], dtype=x.dtype, device=x.device).fill_(float("-inf")).triu_(1)
+ if mask is not None:
+ mask += causal_mask
+ else:
+ mask = causal_mask
+
+ x, i = self.encoder(x, mask=mask, intermediate_output=intermediate_output)
+ x = self.final_layer_norm(x)
+ if i is not None and final_layer_norm_intermediate:
+ i = self.final_layer_norm(i)
+
+ pooled_output = x[torch.arange(x.shape[0], device=x.device), input_tokens.to(dtype=torch.int, device=x.device).argmax(dim=-1),]
+ return x, i, pooled_output
+
+class CLIPTextModel(torch.nn.Module):
+ def __init__(self, config_dict, dtype, device, operations):
+ super().__init__()
+ self.num_layers = config_dict["num_hidden_layers"]
+ self.text_model = CLIPTextModel_(config_dict, dtype, device, operations)
+ self.dtype = dtype
+
+ def get_input_embeddings(self):
+ return self.text_model.embeddings.token_embedding
+
+ def set_input_embeddings(self, embeddings):
+ self.text_model.embeddings.token_embedding = embeddings
+
+ def forward(self, *args, **kwargs):
+ return self.text_model(*args, **kwargs)
+
+class CLIPVisionEmbeddings(torch.nn.Module):
+ def __init__(self, embed_dim, num_channels=3, patch_size=14, image_size=224, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.class_embedding = torch.nn.Parameter(torch.empty(embed_dim, dtype=dtype, device=device))
+
+ self.patch_embedding = operations.Conv2d(
+ in_channels=num_channels,
+ out_channels=embed_dim,
+ kernel_size=patch_size,
+ stride=patch_size,
+ bias=False,
+ dtype=dtype,
+ device=device
+ )
+
+ num_patches = (image_size // patch_size) ** 2
+ num_positions = num_patches + 1
+ self.position_embedding = torch.nn.Embedding(num_positions, embed_dim, dtype=dtype, device=device)
+
+ def forward(self, pixel_values):
+ embeds = self.patch_embedding(pixel_values).flatten(2).transpose(1, 2)
+ return torch.cat([self.class_embedding.to(embeds.device).expand(pixel_values.shape[0], 1, -1), embeds], dim=1) + self.position_embedding.weight.to(embeds.device)
+
+
+class CLIPVision(torch.nn.Module):
+ def __init__(self, config_dict, dtype, device, operations):
+ super().__init__()
+ num_layers = config_dict["num_hidden_layers"]
+ embed_dim = config_dict["hidden_size"]
+ heads = config_dict["num_attention_heads"]
+ intermediate_size = config_dict["intermediate_size"]
+ intermediate_activation = config_dict["hidden_act"]
+
+ self.embeddings = CLIPVisionEmbeddings(embed_dim, config_dict["num_channels"], config_dict["patch_size"], config_dict["image_size"], dtype=torch.float32, device=device, operations=operations)
+ self.pre_layrnorm = operations.LayerNorm(embed_dim)
+ self.encoder = CLIPEncoder(num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations)
+ self.post_layernorm = operations.LayerNorm(embed_dim)
+
+ def forward(self, pixel_values, attention_mask=None, intermediate_output=None):
+ x = self.embeddings(pixel_values)
+ x = self.pre_layrnorm(x)
+ #TODO: attention_mask?
+ x, i = self.encoder(x, mask=None, intermediate_output=intermediate_output)
+ pooled_output = self.post_layernorm(x[:, 0, :])
+ return x, i, pooled_output
+
+class CLIPVisionModelProjection(torch.nn.Module):
+ def __init__(self, config_dict, dtype, device, operations):
+ super().__init__()
+ self.vision_model = CLIPVision(config_dict, dtype, device, operations)
+ self.visual_projection = operations.Linear(config_dict["hidden_size"], config_dict["projection_dim"], bias=False)
+
+ def forward(self, *args, **kwargs):
+ x = self.vision_model(*args, **kwargs)
+ out = self.visual_projection(x[2])
+ return (x[0], x[1], out)
diff --git a/comfy/clip_vision.py b/comfy/clip_vision.py
index 9e2e03d72..4564fcfb2 100644
--- a/comfy/clip_vision.py
+++ b/comfy/clip_vision.py
@@ -1,38 +1,44 @@
-from transformers import CLIPVisionModelWithProjection, CLIPVisionConfig, modeling_utils
from .utils import load_torch_file, transformers_convert, common_upscale
import os
import torch
import contextlib
+import json
import comfy.ops
import comfy.model_patcher
import comfy.model_management
import comfy.utils
+import comfy.clip_model
+
+class Output:
+ def __getitem__(self, key):
+ return getattr(self, key)
+ def __setitem__(self, key, item):
+ setattr(self, key, item)
def clip_preprocess(image, size=224):
mean = torch.tensor([ 0.48145466,0.4578275,0.40821073], device=image.device, dtype=image.dtype)
std = torch.tensor([0.26862954,0.26130258,0.27577711], device=image.device, dtype=image.dtype)
- scale = (size / min(image.shape[1], image.shape[2]))
- image = torch.nn.functional.interpolate(image.movedim(-1, 1), size=(round(scale * image.shape[1]), round(scale * image.shape[2])), mode="bicubic", antialias=True)
- h = (image.shape[2] - size)//2
- w = (image.shape[3] - size)//2
- image = image[:,:,h:h+size,w:w+size]
+ image = image.movedim(-1, 1)
+ if not (image.shape[2] == size and image.shape[3] == size):
+ scale = (size / min(image.shape[2], image.shape[3]))
+ image = torch.nn.functional.interpolate(image, size=(round(scale * image.shape[2]), round(scale * image.shape[3])), mode="bicubic", antialias=True)
+ h = (image.shape[2] - size)//2
+ w = (image.shape[3] - size)//2
+ image = image[:,:,h:h+size,w:w+size]
image = torch.clip((255. * image), 0, 255).round() / 255.0
return (image - mean.view([3,1,1])) / std.view([3,1,1])
class ClipVisionModel():
def __init__(self, json_config):
- config = CLIPVisionConfig.from_json_file(json_config)
+ with open(json_config) as f:
+ config = json.load(f)
+
self.load_device = comfy.model_management.text_encoder_device()
offload_device = comfy.model_management.text_encoder_offload_device()
- self.dtype = torch.float32
- if comfy.model_management.should_use_fp16(self.load_device, prioritize_performance=False):
- self.dtype = torch.float16
-
- with comfy.ops.use_comfy_ops(offload_device, self.dtype):
- with modeling_utils.no_init_weights():
- self.model = CLIPVisionModelWithProjection(config)
- self.model.to(self.dtype)
+ self.dtype = comfy.model_management.text_encoder_dtype(self.load_device)
+ self.model = comfy.clip_model.CLIPVisionModelProjection(config, self.dtype, offload_device, comfy.ops.manual_cast)
+ self.model.eval()
self.patcher = comfy.model_patcher.ModelPatcher(self.model, load_device=self.load_device, offload_device=offload_device)
def load_sd(self, sd):
@@ -40,25 +46,13 @@ class ClipVisionModel():
def encode_image(self, image):
comfy.model_management.load_model_gpu(self.patcher)
- pixel_values = clip_preprocess(image.to(self.load_device))
-
- if self.dtype != torch.float32:
- precision_scope = torch.autocast
- else:
- precision_scope = lambda a, b: contextlib.nullcontext(a)
-
- with precision_scope(comfy.model_management.get_autocast_device(self.load_device), torch.float32):
- outputs = self.model(pixel_values=pixel_values, output_hidden_states=True)
-
- for k in outputs:
- t = outputs[k]
- if t is not None:
- if k == 'hidden_states':
- outputs["penultimate_hidden_states"] = t[-2].cpu()
- outputs["hidden_states"] = None
- else:
- outputs[k] = t.cpu()
+ pixel_values = clip_preprocess(image.to(self.load_device)).float()
+ out = self.model(pixel_values=pixel_values, intermediate_output=-2)
+ outputs = Output()
+ outputs["last_hidden_state"] = out[0].to(comfy.model_management.intermediate_device())
+ outputs["image_embeds"] = out[2].to(comfy.model_management.intermediate_device())
+ outputs["penultimate_hidden_states"] = out[1].to(comfy.model_management.intermediate_device())
return outputs
def convert_to_transformers(sd, prefix):
diff --git a/comfy/controlnet.py b/comfy/controlnet.py
index 098681582..8404054f3 100644
--- a/comfy/controlnet.py
+++ b/comfy/controlnet.py
@@ -1,10 +1,12 @@
import torch
import math
import os
+import contextlib
import comfy.utils
import comfy.model_management
import comfy.model_detection
import comfy.model_patcher
+import comfy.ops
import comfy.cldm.cldm
import comfy.t2i_adapter.adapter
@@ -33,16 +35,16 @@ class ControlBase:
self.cond_hint_original = None
self.cond_hint = None
self.strength = 1.0
- self.timestep_percent_range = (1.0, 0.0)
+ self.timestep_percent_range = (0.0, 1.0)
+ self.global_average_pooling = False
self.timestep_range = None
if device is None:
device = comfy.model_management.get_torch_device()
self.device = device
self.previous_controlnet = None
- self.global_average_pooling = False
- def set_cond_hint(self, cond_hint, strength=1.0, timestep_percent_range=(1.0, 0.0)):
+ def set_cond_hint(self, cond_hint, strength=1.0, timestep_percent_range=(0.0, 1.0)):
self.cond_hint_original = cond_hint
self.strength = strength
self.timestep_percent_range = timestep_percent_range
@@ -75,6 +77,7 @@ class ControlBase:
c.cond_hint_original = self.cond_hint_original
c.strength = self.strength
c.timestep_percent_range = self.timestep_percent_range
+ c.global_average_pooling = self.global_average_pooling
def inference_memory_requirements(self, dtype):
if self.previous_controlnet is not None:
@@ -127,12 +130,14 @@ class ControlBase:
return out
class ControlNet(ControlBase):
- def __init__(self, control_model, global_average_pooling=False, device=None):
+ def __init__(self, control_model, global_average_pooling=False, device=None, load_device=None, manual_cast_dtype=None):
super().__init__(device)
self.control_model = control_model
- self.control_model_wrapped = comfy.model_patcher.ModelPatcher(self.control_model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device())
+ self.load_device = load_device
+ self.control_model_wrapped = comfy.model_patcher.ModelPatcher(self.control_model, load_device=load_device, offload_device=comfy.model_management.unet_offload_device())
self.global_average_pooling = global_average_pooling
self.model_sampling_current = None
+ self.manual_cast_dtype = manual_cast_dtype
def get_control(self, x_noisy, t, cond, batched_number):
control_prev = None
@@ -146,28 +151,31 @@ class ControlNet(ControlBase):
else:
return None
+ dtype = self.control_model.dtype
+ if self.manual_cast_dtype is not None:
+ dtype = self.manual_cast_dtype
+
output_dtype = x_noisy.dtype
if self.cond_hint is None or x_noisy.shape[2] * 8 != self.cond_hint.shape[2] or x_noisy.shape[3] * 8 != self.cond_hint.shape[3]:
if self.cond_hint is not None:
del self.cond_hint
self.cond_hint = None
- self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * 8, x_noisy.shape[2] * 8, 'nearest-exact', "center").to(self.control_model.dtype).to(self.device)
+ self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * 8, x_noisy.shape[2] * 8, 'nearest-exact', "center").to(dtype).to(self.device)
if x_noisy.shape[0] != self.cond_hint.shape[0]:
self.cond_hint = broadcast_image_to(self.cond_hint, x_noisy.shape[0], batched_number)
-
context = cond['c_crossattn']
y = cond.get('y', None)
if y is not None:
- y = y.to(self.control_model.dtype)
+ y = y.to(dtype)
timestep = self.model_sampling_current.timestep(t)
x_noisy = self.model_sampling_current.calculate_input(t, x_noisy)
- control = self.control_model(x=x_noisy.to(self.control_model.dtype), hint=self.cond_hint, timesteps=timestep.float(), context=context.to(self.control_model.dtype), y=y)
+ control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.float(), context=context.to(dtype), y=y)
return self.control_merge(None, control, control_prev, output_dtype)
def copy(self):
- c = ControlNet(self.control_model, global_average_pooling=self.global_average_pooling)
+ c = ControlNet(self.control_model, global_average_pooling=self.global_average_pooling, load_device=self.load_device, manual_cast_dtype=self.manual_cast_dtype)
self.copy_to(c)
return c
@@ -198,10 +206,11 @@ class ControlLoraOps:
self.bias = None
def forward(self, input):
+ weight, bias = comfy.ops.cast_bias_weight(self, input)
if self.up is not None:
- return torch.nn.functional.linear(input, self.weight.to(input.device) + (torch.mm(self.up.flatten(start_dim=1), self.down.flatten(start_dim=1))).reshape(self.weight.shape).type(input.dtype), self.bias)
+ return torch.nn.functional.linear(input, weight + (torch.mm(self.up.flatten(start_dim=1), self.down.flatten(start_dim=1))).reshape(self.weight.shape).type(input.dtype), bias)
else:
- return torch.nn.functional.linear(input, self.weight.to(input.device), self.bias)
+ return torch.nn.functional.linear(input, weight, bias)
class Conv2d(torch.nn.Module):
def __init__(
@@ -237,16 +246,11 @@ class ControlLoraOps:
def forward(self, input):
+ weight, bias = comfy.ops.cast_bias_weight(self, input)
if self.up is not None:
- return torch.nn.functional.conv2d(input, self.weight.to(input.device) + (torch.mm(self.up.flatten(start_dim=1), self.down.flatten(start_dim=1))).reshape(self.weight.shape).type(input.dtype), self.bias, self.stride, self.padding, self.dilation, self.groups)
+ return torch.nn.functional.conv2d(input, weight + (torch.mm(self.up.flatten(start_dim=1), self.down.flatten(start_dim=1))).reshape(self.weight.shape).type(input.dtype), bias, self.stride, self.padding, self.dilation, self.groups)
else:
- return torch.nn.functional.conv2d(input, self.weight.to(input.device), self.bias, self.stride, self.padding, self.dilation, self.groups)
-
- def conv_nd(self, dims, *args, **kwargs):
- if dims == 2:
- return self.Conv2d(*args, **kwargs)
- else:
- raise ValueError(f"unsupported dimensions: {dims}")
+ return torch.nn.functional.conv2d(input, weight, bias, self.stride, self.padding, self.dilation, self.groups)
class ControlLora(ControlNet):
@@ -260,17 +264,26 @@ class ControlLora(ControlNet):
controlnet_config = model.model_config.unet_config.copy()
controlnet_config.pop("out_channels")
controlnet_config["hint_channels"] = self.control_weights["input_hint_block.0.weight"].shape[1]
- controlnet_config["operations"] = ControlLoraOps()
- self.control_model = comfy.cldm.cldm.ControlNet(**controlnet_config)
+ self.manual_cast_dtype = model.manual_cast_dtype
dtype = model.get_dtype()
- self.control_model.to(dtype)
+ if self.manual_cast_dtype is None:
+ class control_lora_ops(ControlLoraOps, comfy.ops.disable_weight_init):
+ pass
+ else:
+ class control_lora_ops(ControlLoraOps, comfy.ops.manual_cast):
+ pass
+ dtype = self.manual_cast_dtype
+
+ controlnet_config["operations"] = control_lora_ops
+ controlnet_config["dtype"] = dtype
+ self.control_model = comfy.cldm.cldm.ControlNet(**controlnet_config)
self.control_model.to(comfy.model_management.get_torch_device())
diffusion_model = model.diffusion_model
sd = diffusion_model.state_dict()
cm = self.control_model.state_dict()
for k in sd:
- weight = comfy.model_management.resolve_lowvram_weight(sd[k], diffusion_model, k)
+ weight = sd[k]
try:
comfy.utils.set_attr(self.control_model, k, weight)
except:
@@ -367,6 +380,10 @@ def load_controlnet(ckpt_path, model=None):
if controlnet_config is None:
unet_dtype = comfy.model_management.unet_dtype()
controlnet_config = comfy.model_detection.model_config_from_unet(controlnet_data, prefix, unet_dtype, True).unet_config
+ load_device = comfy.model_management.get_torch_device()
+ manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device)
+ if manual_cast_dtype is not None:
+ controlnet_config["operations"] = comfy.ops.manual_cast
controlnet_config.pop("out_channels")
controlnet_config["hint_channels"] = controlnet_data["{}input_hint_block.0.weight".format(prefix)].shape[1]
control_model = comfy.cldm.cldm.ControlNet(**controlnet_config)
@@ -395,14 +412,12 @@ def load_controlnet(ckpt_path, model=None):
missing, unexpected = control_model.load_state_dict(controlnet_data, strict=False)
print(missing, unexpected)
- control_model = control_model.to(unet_dtype)
-
global_average_pooling = False
filename = os.path.splitext(ckpt_path)[0]
if filename.endswith("_shuffle") or filename.endswith("_shuffle_fp16"): #TODO: smarter way of enabling global_average_pooling
global_average_pooling = True
- control = ControlNet(control_model, global_average_pooling=global_average_pooling)
+ control = ControlNet(control_model, global_average_pooling=global_average_pooling, load_device=load_device, manual_cast_dtype=manual_cast_dtype)
return control
class T2IAdapter(ControlBase):
diff --git a/comfy/extra_samplers/uni_pc.py b/comfy/extra_samplers/uni_pc.py
index 1a7a83929..08bf0fc9e 100644
--- a/comfy/extra_samplers/uni_pc.py
+++ b/comfy/extra_samplers/uni_pc.py
@@ -858,7 +858,7 @@ def predict_eps_sigma(model, input, sigma_in, **kwargs):
return (input - model(input, sigma_in, **kwargs)) / sigma
-def sample_unipc(model, noise, image, sigmas, sampling_function, max_denoise, extra_args=None, callback=None, disable=False, noise_mask=None, variant='bh1'):
+def sample_unipc(model, noise, image, sigmas, max_denoise, extra_args=None, callback=None, disable=False, noise_mask=None, variant='bh1'):
timesteps = sigmas.clone()
if sigmas[-1] == 0:
timesteps = sigmas[:]
diff --git a/comfy/k_diffusion/sampling.py b/comfy/k_diffusion/sampling.py
index dd6f7bbe5..761c2e0ef 100644
--- a/comfy/k_diffusion/sampling.py
+++ b/comfy/k_diffusion/sampling.py
@@ -750,3 +750,61 @@ def sample_lcm(model, x, sigmas, extra_args=None, callback=None, disable=None, n
if sigmas[i + 1] > 0:
x += sigmas[i + 1] * noise_sampler(sigmas[i], sigmas[i + 1])
return x
+
+
+
+@torch.no_grad()
+def sample_heunpp2(model, x, sigmas, extra_args=None, callback=None, disable=None, s_churn=0., s_tmin=0., s_tmax=float('inf'), s_noise=1.):
+ # From MIT licensed: https://github.com/Carzit/sd-webui-samplers-scheduler/
+ extra_args = {} if extra_args is None else extra_args
+ s_in = x.new_ones([x.shape[0]])
+ s_end = sigmas[-1]
+ for i in trange(len(sigmas) - 1, disable=disable):
+ gamma = min(s_churn / (len(sigmas) - 1), 2 ** 0.5 - 1) if s_tmin <= sigmas[i] <= s_tmax else 0.
+ eps = torch.randn_like(x) * s_noise
+ sigma_hat = sigmas[i] * (gamma + 1)
+ if gamma > 0:
+ x = x + eps * (sigma_hat ** 2 - sigmas[i] ** 2) ** 0.5
+ denoised = model(x, sigma_hat * s_in, **extra_args)
+ d = to_d(x, sigma_hat, denoised)
+ if callback is not None:
+ callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigma_hat, 'denoised': denoised})
+ dt = sigmas[i + 1] - sigma_hat
+ if sigmas[i + 1] == s_end:
+ # Euler method
+ x = x + d * dt
+ elif sigmas[i + 2] == s_end:
+
+ # Heun's method
+ x_2 = x + d * dt
+ denoised_2 = model(x_2, sigmas[i + 1] * s_in, **extra_args)
+ d_2 = to_d(x_2, sigmas[i + 1], denoised_2)
+
+ w = 2 * sigmas[0]
+ w2 = sigmas[i+1]/w
+ w1 = 1 - w2
+
+ d_prime = d * w1 + d_2 * w2
+
+
+ x = x + d_prime * dt
+
+ else:
+ # Heun++
+ x_2 = x + d * dt
+ denoised_2 = model(x_2, sigmas[i + 1] * s_in, **extra_args)
+ d_2 = to_d(x_2, sigmas[i + 1], denoised_2)
+ dt_2 = sigmas[i + 2] - sigmas[i + 1]
+
+ x_3 = x_2 + d_2 * dt_2
+ denoised_3 = model(x_3, sigmas[i + 2] * s_in, **extra_args)
+ d_3 = to_d(x_3, sigmas[i + 2], denoised_3)
+
+ w = 3 * sigmas[0]
+ w2 = sigmas[i + 1] / w
+ w3 = sigmas[i + 2] / w
+ w1 = 1 - w2 - w3
+
+ d_prime = w1 * d + w2 * d_2 + w3 * d_3
+ x = x + d_prime * dt
+ return x
diff --git a/comfy/ldm/models/autoencoder.py b/comfy/ldm/models/autoencoder.py
index d2f1d74a9..b91ec3249 100644
--- a/comfy/ldm/models/autoencoder.py
+++ b/comfy/ldm/models/autoencoder.py
@@ -8,6 +8,7 @@ from comfy.ldm.modules.distributions.distributions import DiagonalGaussianDistri
from comfy.ldm.util import instantiate_from_config
from comfy.ldm.modules.ema import LitEma
+import comfy.ops
class DiagonalGaussianRegularizer(torch.nn.Module):
def __init__(self, sample: bool = True):
@@ -161,12 +162,12 @@ class AutoencodingEngineLegacy(AutoencodingEngine):
},
**kwargs,
)
- self.quant_conv = torch.nn.Conv2d(
+ self.quant_conv = comfy.ops.disable_weight_init.Conv2d(
(1 + ddconfig["double_z"]) * ddconfig["z_channels"],
(1 + ddconfig["double_z"]) * embed_dim,
1,
)
- self.post_quant_conv = torch.nn.Conv2d(embed_dim, ddconfig["z_channels"], 1)
+ self.post_quant_conv = comfy.ops.disable_weight_init.Conv2d(embed_dim, ddconfig["z_channels"], 1)
self.embed_dim = embed_dim
def get_autoencoder_params(self) -> list:
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py
index 016795a59..3e12886b0 100644
--- a/comfy/ldm/modules/attention.py
+++ b/comfy/ldm/modules/attention.py
@@ -5,8 +5,10 @@ import torch.nn.functional as F
from torch import nn, einsum
from einops import rearrange, repeat
from typing import Optional, Any
+from functools import partial
-from .diffusionmodules.util import checkpoint
+
+from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding
from .sub_quadratic_attention import efficient_dot_product_attention
from comfy import model_management
@@ -17,6 +19,7 @@ if model_management.xformers_enabled():
from comfy.cli_args import args
import comfy.ops
+ops = comfy.ops.disable_weight_init
# CrossAttn precision handling
if args.dont_upcast_attention:
@@ -53,7 +56,7 @@ def init_(tensor):
# feedforward
class GEGLU(nn.Module):
- def __init__(self, dim_in, dim_out, dtype=None, device=None, operations=comfy.ops):
+ def __init__(self, dim_in, dim_out, dtype=None, device=None, operations=ops):
super().__init__()
self.proj = operations.Linear(dim_in, dim_out * 2, dtype=dtype, device=device)
@@ -63,7 +66,7 @@ class GEGLU(nn.Module):
class FeedForward(nn.Module):
- def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0., dtype=None, device=None, operations=comfy.ops):
+ def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0., dtype=None, device=None, operations=ops):
super().__init__()
inner_dim = int(dim * mult)
dim_out = default(dim_out, dim)
@@ -81,16 +84,6 @@ class FeedForward(nn.Module):
def forward(self, x):
return self.net(x)
-
-def zero_module(module):
- """
- Zero out the parameters of a module and return it.
- """
- for p in module.parameters():
- p.detach().zero_()
- return module
-
-
def Normalize(in_channels, dtype=None, device=None):
return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True, dtype=dtype, device=device)
@@ -111,19 +104,20 @@ def attention_basic(q, k, v, heads, mask=None):
# force cast to fp32 to avoid overflowing
if _ATTN_PRECISION =="fp32":
- with torch.autocast(enabled=False, device_type = 'cuda'):
- q, k = q.float(), k.float()
- sim = einsum('b i d, b j d -> b i j', q, k) * scale
+ sim = einsum('b i d, b j d -> b i j', q.float(), k.float()) * scale
else:
sim = einsum('b i d, b j d -> b i j', q, k) * scale
del q, k
if exists(mask):
- mask = rearrange(mask, 'b ... -> b (...)')
- max_neg_value = -torch.finfo(sim.dtype).max
- mask = repeat(mask, 'b j -> (b h) () j', h=h)
- sim.masked_fill_(~mask, max_neg_value)
+ if mask.dtype == torch.bool:
+ mask = rearrange(mask, 'b ... -> b (...)') #TODO: check if this bool part matches pytorch attention
+ max_neg_value = -torch.finfo(sim.dtype).max
+ mask = repeat(mask, 'b j -> (b h) () j', h=h)
+ sim.masked_fill_(~mask, max_neg_value)
+ else:
+ sim += mask
# attention, what we cannot get enough of
sim = sim.softmax(dim=-1)
@@ -276,9 +270,20 @@ def attention_split(q, k, v, heads, mask=None):
)
return r1
+BROKEN_XFORMERS = False
+try:
+ x_vers = xformers.__version__
+ #I think 0.0.23 is also broken (q with bs bigger than 65535 gives CUDA error)
+ BROKEN_XFORMERS = x_vers.startswith("0.0.21") or x_vers.startswith("0.0.22") or x_vers.startswith("0.0.23")
+except:
+ pass
+
def attention_xformers(q, k, v, heads, mask=None):
b, _, dim_head = q.shape
dim_head //= heads
+ if BROKEN_XFORMERS:
+ if b * heads > 65535:
+ return attention_pytorch(q, k, v, heads, mask)
q, k, v = map(
lambda t: t.unsqueeze(3)
@@ -337,8 +342,20 @@ else:
if model_management.pytorch_attention_enabled():
optimized_attention_masked = attention_pytorch
+def optimized_attention_for_device(device, mask=False):
+ if device == torch.device("cpu"): #TODO
+ if model_management.pytorch_attention_enabled():
+ return attention_pytorch
+ else:
+ return attention_basic
+ if mask:
+ return optimized_attention_masked
+
+ return optimized_attention
+
+
class CrossAttention(nn.Module):
- def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=comfy.ops):
+ def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=ops):
super().__init__()
inner_dim = dim_head * heads
context_dim = default(context_dim, query_dim)
@@ -370,53 +387,72 @@ class CrossAttention(nn.Module):
class BasicTransformerBlock(nn.Module):
- def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True,
- disable_self_attn=False, dtype=None, device=None, operations=comfy.ops):
+ def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True, ff_in=False, inner_dim=None,
+ disable_self_attn=False, disable_temporal_crossattention=False, switch_temporal_ca_to_sa=False, dtype=None, device=None, operations=ops):
super().__init__()
+
+ self.ff_in = ff_in or inner_dim is not None
+ if inner_dim is None:
+ inner_dim = dim
+
+ self.is_res = inner_dim == dim
+
+ if self.ff_in:
+ self.norm_in = operations.LayerNorm(dim, dtype=dtype, device=device)
+ self.ff_in = FeedForward(dim, dim_out=inner_dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device, operations=operations)
+
self.disable_self_attn = disable_self_attn
- self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout,
+ self.attn1 = CrossAttention(query_dim=inner_dim, heads=n_heads, dim_head=d_head, dropout=dropout,
context_dim=context_dim if self.disable_self_attn else None, dtype=dtype, device=device, operations=operations) # is a self-attention if not self.disable_self_attn
- self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device, operations=operations)
- self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim,
- heads=n_heads, dim_head=d_head, dropout=dropout, dtype=dtype, device=device, operations=operations) # is self-attn if context is none
- self.norm1 = nn.LayerNorm(dim, dtype=dtype, device=device)
- self.norm2 = nn.LayerNorm(dim, dtype=dtype, device=device)
- self.norm3 = nn.LayerNorm(dim, dtype=dtype, device=device)
+ self.ff = FeedForward(inner_dim, dim_out=dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device, operations=operations)
+
+ if disable_temporal_crossattention:
+ if switch_temporal_ca_to_sa:
+ raise ValueError
+ else:
+ self.attn2 = None
+ else:
+ context_dim_attn2 = None
+ if not switch_temporal_ca_to_sa:
+ context_dim_attn2 = context_dim
+
+ self.attn2 = CrossAttention(query_dim=inner_dim, context_dim=context_dim_attn2,
+ heads=n_heads, dim_head=d_head, dropout=dropout, dtype=dtype, device=device, operations=operations) # is self-attn if context is none
+ self.norm2 = operations.LayerNorm(inner_dim, dtype=dtype, device=device)
+
+ self.norm1 = operations.LayerNorm(inner_dim, dtype=dtype, device=device)
+ self.norm3 = operations.LayerNorm(inner_dim, dtype=dtype, device=device)
self.checkpoint = checkpoint
self.n_heads = n_heads
self.d_head = d_head
+ self.switch_temporal_ca_to_sa = switch_temporal_ca_to_sa
def forward(self, x, context=None, transformer_options={}):
return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint)
def _forward(self, x, context=None, transformer_options={}):
extra_options = {}
- block = None
- block_index = 0
- if "current_index" in transformer_options:
- extra_options["transformer_index"] = transformer_options["current_index"]
- if "block_index" in transformer_options:
- block_index = transformer_options["block_index"]
- extra_options["block_index"] = block_index
- if "original_shape" in transformer_options:
- extra_options["original_shape"] = transformer_options["original_shape"]
- if "block" in transformer_options:
- block = transformer_options["block"]
- extra_options["block"] = block
- if "cond_or_uncond" in transformer_options:
- extra_options["cond_or_uncond"] = transformer_options["cond_or_uncond"]
- if "patches" in transformer_options:
- transformer_patches = transformer_options["patches"]
- else:
- transformer_patches = {}
+ block = transformer_options.get("block", None)
+ block_index = transformer_options.get("block_index", 0)
+ transformer_patches = {}
+ transformer_patches_replace = {}
+
+ for k in transformer_options:
+ if k == "patches":
+ transformer_patches = transformer_options[k]
+ elif k == "patches_replace":
+ transformer_patches_replace = transformer_options[k]
+ else:
+ extra_options[k] = transformer_options[k]
extra_options["n_heads"] = self.n_heads
extra_options["dim_head"] = self.d_head
- if "patches_replace" in transformer_options:
- transformer_patches_replace = transformer_options["patches_replace"]
- else:
- transformer_patches_replace = {}
+ if self.ff_in:
+ x_skip = x
+ x = self.ff_in(self.norm_in(x))
+ if self.is_res:
+ x += x_skip
n = self.norm1(x)
if self.disable_self_attn:
@@ -465,31 +501,34 @@ class BasicTransformerBlock(nn.Module):
for p in patch:
x = p(x, extra_options)
- n = self.norm2(x)
-
- context_attn2 = context
- value_attn2 = None
- if "attn2_patch" in transformer_patches:
- patch = transformer_patches["attn2_patch"]
- value_attn2 = context_attn2
- for p in patch:
- n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options)
-
- attn2_replace_patch = transformer_patches_replace.get("attn2", {})
- block_attn2 = transformer_block
- if block_attn2 not in attn2_replace_patch:
- block_attn2 = block
-
- if block_attn2 in attn2_replace_patch:
- if value_attn2 is None:
+ if self.attn2 is not None:
+ n = self.norm2(x)
+ if self.switch_temporal_ca_to_sa:
+ context_attn2 = n
+ else:
+ context_attn2 = context
+ value_attn2 = None
+ if "attn2_patch" in transformer_patches:
+ patch = transformer_patches["attn2_patch"]
value_attn2 = context_attn2
- n = self.attn2.to_q(n)
- context_attn2 = self.attn2.to_k(context_attn2)
- value_attn2 = self.attn2.to_v(value_attn2)
- n = attn2_replace_patch[block_attn2](n, context_attn2, value_attn2, extra_options)
- n = self.attn2.to_out(n)
- else:
- n = self.attn2(n, context=context_attn2, value=value_attn2)
+ for p in patch:
+ n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options)
+
+ attn2_replace_patch = transformer_patches_replace.get("attn2", {})
+ block_attn2 = transformer_block
+ if block_attn2 not in attn2_replace_patch:
+ block_attn2 = block
+
+ if block_attn2 in attn2_replace_patch:
+ if value_attn2 is None:
+ value_attn2 = context_attn2
+ n = self.attn2.to_q(n)
+ context_attn2 = self.attn2.to_k(context_attn2)
+ value_attn2 = self.attn2.to_v(value_attn2)
+ n = attn2_replace_patch[block_attn2](n, context_attn2, value_attn2, extra_options)
+ n = self.attn2.to_out(n)
+ else:
+ n = self.attn2(n, context=context_attn2, value=value_attn2)
if "attn2_output_patch" in transformer_patches:
patch = transformer_patches["attn2_output_patch"]
@@ -497,7 +536,12 @@ class BasicTransformerBlock(nn.Module):
n = p(n, extra_options)
x += n
- x = self.ff(self.norm3(x)) + x
+ if self.is_res:
+ x_skip = x
+ x = self.ff(self.norm3(x))
+ if self.is_res:
+ x += x_skip
+
return x
@@ -513,13 +557,13 @@ class SpatialTransformer(nn.Module):
def __init__(self, in_channels, n_heads, d_head,
depth=1, dropout=0., context_dim=None,
disable_self_attn=False, use_linear=False,
- use_checkpoint=True, dtype=None, device=None, operations=comfy.ops):
+ use_checkpoint=True, dtype=None, device=None, operations=ops):
super().__init__()
if exists(context_dim) and not isinstance(context_dim, list):
context_dim = [context_dim] * depth
self.in_channels = in_channels
inner_dim = n_heads * d_head
- self.norm = Normalize(in_channels, dtype=dtype, device=device)
+ self.norm = operations.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True, dtype=dtype, device=device)
if not use_linear:
self.proj_in = operations.Conv2d(in_channels,
inner_dim,
@@ -565,3 +609,164 @@ class SpatialTransformer(nn.Module):
x = self.proj_out(x)
return x + x_in
+
+class SpatialVideoTransformer(SpatialTransformer):
+ def __init__(
+ self,
+ in_channels,
+ n_heads,
+ d_head,
+ depth=1,
+ dropout=0.0,
+ use_linear=False,
+ context_dim=None,
+ use_spatial_context=False,
+ timesteps=None,
+ merge_strategy: str = "fixed",
+ merge_factor: float = 0.5,
+ time_context_dim=None,
+ ff_in=False,
+ checkpoint=False,
+ time_depth=1,
+ disable_self_attn=False,
+ disable_temporal_crossattention=False,
+ max_time_embed_period: int = 10000,
+ dtype=None, device=None, operations=ops
+ ):
+ super().__init__(
+ in_channels,
+ n_heads,
+ d_head,
+ depth=depth,
+ dropout=dropout,
+ use_checkpoint=checkpoint,
+ context_dim=context_dim,
+ use_linear=use_linear,
+ disable_self_attn=disable_self_attn,
+ dtype=dtype, device=device, operations=operations
+ )
+ self.time_depth = time_depth
+ self.depth = depth
+ self.max_time_embed_period = max_time_embed_period
+
+ time_mix_d_head = d_head
+ n_time_mix_heads = n_heads
+
+ time_mix_inner_dim = int(time_mix_d_head * n_time_mix_heads)
+
+ inner_dim = n_heads * d_head
+ if use_spatial_context:
+ time_context_dim = context_dim
+
+ self.time_stack = nn.ModuleList(
+ [
+ BasicTransformerBlock(
+ inner_dim,
+ n_time_mix_heads,
+ time_mix_d_head,
+ dropout=dropout,
+ context_dim=time_context_dim,
+ # timesteps=timesteps,
+ checkpoint=checkpoint,
+ ff_in=ff_in,
+ inner_dim=time_mix_inner_dim,
+ disable_self_attn=disable_self_attn,
+ disable_temporal_crossattention=disable_temporal_crossattention,
+ dtype=dtype, device=device, operations=operations
+ )
+ for _ in range(self.depth)
+ ]
+ )
+
+ assert len(self.time_stack) == len(self.transformer_blocks)
+
+ self.use_spatial_context = use_spatial_context
+ self.in_channels = in_channels
+
+ time_embed_dim = self.in_channels * 4
+ self.time_pos_embed = nn.Sequential(
+ operations.Linear(self.in_channels, time_embed_dim, dtype=dtype, device=device),
+ nn.SiLU(),
+ operations.Linear(time_embed_dim, self.in_channels, dtype=dtype, device=device),
+ )
+
+ self.time_mixer = AlphaBlender(
+ alpha=merge_factor, merge_strategy=merge_strategy
+ )
+
+ def forward(
+ self,
+ x: torch.Tensor,
+ context: Optional[torch.Tensor] = None,
+ time_context: Optional[torch.Tensor] = None,
+ timesteps: Optional[int] = None,
+ image_only_indicator: Optional[torch.Tensor] = None,
+ transformer_options={}
+ ) -> torch.Tensor:
+ _, _, h, w = x.shape
+ x_in = x
+ spatial_context = None
+ if exists(context):
+ spatial_context = context
+
+ if self.use_spatial_context:
+ assert (
+ context.ndim == 3
+ ), f"n dims of spatial context should be 3 but are {context.ndim}"
+
+ if time_context is None:
+ time_context = context
+ time_context_first_timestep = time_context[::timesteps]
+ time_context = repeat(
+ time_context_first_timestep, "b ... -> (b n) ...", n=h * w
+ )
+ elif time_context is not None and not self.use_spatial_context:
+ time_context = repeat(time_context, "b ... -> (b n) ...", n=h * w)
+ if time_context.ndim == 2:
+ time_context = rearrange(time_context, "b c -> b 1 c")
+
+ x = self.norm(x)
+ if not self.use_linear:
+ x = self.proj_in(x)
+ x = rearrange(x, "b c h w -> b (h w) c")
+ if self.use_linear:
+ x = self.proj_in(x)
+
+ num_frames = torch.arange(timesteps, device=x.device)
+ num_frames = repeat(num_frames, "t -> b t", b=x.shape[0] // timesteps)
+ num_frames = rearrange(num_frames, "b t -> (b t)")
+ t_emb = timestep_embedding(num_frames, self.in_channels, repeat_only=False, max_period=self.max_time_embed_period).to(x.dtype)
+ emb = self.time_pos_embed(t_emb)
+ emb = emb[:, None, :]
+
+ for it_, (block, mix_block) in enumerate(
+ zip(self.transformer_blocks, self.time_stack)
+ ):
+ transformer_options["block_index"] = it_
+ x = block(
+ x,
+ context=spatial_context,
+ transformer_options=transformer_options,
+ )
+
+ x_mix = x
+ x_mix = x_mix + emb
+
+ B, S, C = x_mix.shape
+ x_mix = rearrange(x_mix, "(b t) s c -> (b s) t c", t=timesteps)
+ x_mix = mix_block(x_mix, context=time_context) #TODO: transformer_options
+ x_mix = rearrange(
+ x_mix, "(b s) t c -> (b t) s c", s=S, b=B // timesteps, c=C, t=timesteps
+ )
+
+ x = self.time_mixer(x_spatial=x, x_temporal=x_mix, image_only_indicator=image_only_indicator)
+
+ if self.use_linear:
+ x = self.proj_out(x)
+ x = rearrange(x, "b (h w) c -> b c h w", h=h, w=w)
+ if not self.use_linear:
+ x = self.proj_out(x)
+ out = x + x_in
+ return out
+
+
diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.py
index f23417fd2..cc81c1f23 100644
--- a/comfy/ldm/modules/diffusionmodules/model.py
+++ b/comfy/ldm/modules/diffusionmodules/model.py
@@ -8,6 +8,7 @@ from typing import Optional, Any
from comfy import model_management
import comfy.ops
+ops = comfy.ops.disable_weight_init
if model_management.xformers_enabled_vae():
import xformers
@@ -40,7 +41,7 @@ def nonlinearity(x):
def Normalize(in_channels, num_groups=32):
- return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
+ return ops.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
class Upsample(nn.Module):
@@ -48,7 +49,7 @@ class Upsample(nn.Module):
super().__init__()
self.with_conv = with_conv
if self.with_conv:
- self.conv = comfy.ops.Conv2d(in_channels,
+ self.conv = ops.Conv2d(in_channels,
in_channels,
kernel_size=3,
stride=1,
@@ -78,7 +79,7 @@ class Downsample(nn.Module):
self.with_conv = with_conv
if self.with_conv:
# no asymmetric padding in torch conv, must do it ourselves
- self.conv = comfy.ops.Conv2d(in_channels,
+ self.conv = ops.Conv2d(in_channels,
in_channels,
kernel_size=3,
stride=2,
@@ -105,30 +106,30 @@ class ResnetBlock(nn.Module):
self.swish = torch.nn.SiLU(inplace=True)
self.norm1 = Normalize(in_channels)
- self.conv1 = comfy.ops.Conv2d(in_channels,
+ self.conv1 = ops.Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=1,
padding=1)
if temb_channels > 0:
- self.temb_proj = comfy.ops.Linear(temb_channels,
+ self.temb_proj = ops.Linear(temb_channels,
out_channels)
self.norm2 = Normalize(out_channels)
self.dropout = torch.nn.Dropout(dropout, inplace=True)
- self.conv2 = comfy.ops.Conv2d(out_channels,
+ self.conv2 = ops.Conv2d(out_channels,
out_channels,
kernel_size=3,
stride=1,
padding=1)
if self.in_channels != self.out_channels:
if self.use_conv_shortcut:
- self.conv_shortcut = comfy.ops.Conv2d(in_channels,
+ self.conv_shortcut = ops.Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=1,
padding=1)
else:
- self.nin_shortcut = comfy.ops.Conv2d(in_channels,
+ self.nin_shortcut = ops.Conv2d(in_channels,
out_channels,
kernel_size=1,
stride=1,
@@ -245,22 +246,22 @@ class AttnBlock(nn.Module):
self.in_channels = in_channels
self.norm = Normalize(in_channels)
- self.q = comfy.ops.Conv2d(in_channels,
+ self.q = ops.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
- self.k = comfy.ops.Conv2d(in_channels,
+ self.k = ops.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
- self.v = comfy.ops.Conv2d(in_channels,
+ self.v = ops.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
- self.proj_out = comfy.ops.Conv2d(in_channels,
+ self.proj_out = ops.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
@@ -312,14 +313,14 @@ class Model(nn.Module):
# timestep embedding
self.temb = nn.Module()
self.temb.dense = nn.ModuleList([
- comfy.ops.Linear(self.ch,
+ ops.Linear(self.ch,
self.temb_ch),
- comfy.ops.Linear(self.temb_ch,
+ ops.Linear(self.temb_ch,
self.temb_ch),
])
# downsampling
- self.conv_in = comfy.ops.Conv2d(in_channels,
+ self.conv_in = ops.Conv2d(in_channels,
self.ch,
kernel_size=3,
stride=1,
@@ -388,7 +389,7 @@ class Model(nn.Module):
# end
self.norm_out = Normalize(block_in)
- self.conv_out = comfy.ops.Conv2d(block_in,
+ self.conv_out = ops.Conv2d(block_in,
out_ch,
kernel_size=3,
stride=1,
@@ -461,7 +462,7 @@ class Encoder(nn.Module):
self.in_channels = in_channels
# downsampling
- self.conv_in = comfy.ops.Conv2d(in_channels,
+ self.conv_in = ops.Conv2d(in_channels,
self.ch,
kernel_size=3,
stride=1,
@@ -506,7 +507,7 @@ class Encoder(nn.Module):
# end
self.norm_out = Normalize(block_in)
- self.conv_out = comfy.ops.Conv2d(block_in,
+ self.conv_out = ops.Conv2d(block_in,
2*z_channels if double_z else z_channels,
kernel_size=3,
stride=1,
@@ -541,7 +542,7 @@ class Decoder(nn.Module):
def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
resolution, z_channels, give_pre_end=False, tanh_out=False, use_linear_attn=False,
- conv_out_op=comfy.ops.Conv2d,
+ conv_out_op=ops.Conv2d,
resnet_op=ResnetBlock,
attn_op=AttnBlock,
**ignorekwargs):
@@ -565,7 +566,7 @@ class Decoder(nn.Module):
self.z_shape, np.prod(self.z_shape)))
# z to block_in
- self.conv_in = comfy.ops.Conv2d(z_channels,
+ self.conv_in = ops.Conv2d(z_channels,
block_in,
kernel_size=3,
stride=1,
diff --git a/comfy/ldm/modules/diffusionmodules/openaimodel.py b/comfy/ldm/modules/diffusionmodules/openaimodel.py
index 49c1e8cbb..057dd16b2 100644
--- a/comfy/ldm/modules/diffusionmodules/openaimodel.py
+++ b/comfy/ldm/modules/diffusionmodules/openaimodel.py
@@ -5,17 +5,20 @@ import numpy as np
import torch as th
import torch.nn as nn
import torch.nn.functional as F
+from einops import rearrange
+from functools import partial
from .util import (
checkpoint,
avg_pool_nd,
zero_module,
- normalization,
timestep_embedding,
+ AlphaBlender,
)
-from ..attention import SpatialTransformer
+from ..attention import SpatialTransformer, SpatialVideoTransformer, default
from comfy.ldm.util import exists
import comfy.ops
+ops = comfy.ops.disable_weight_init
class TimestepBlock(nn.Module):
"""
@@ -28,6 +31,26 @@ class TimestepBlock(nn.Module):
Apply the module to `x` given `emb` timestep embeddings.
"""
+#This is needed because accelerate makes a copy of transformer_options which breaks "transformer_index"
+def forward_timestep_embed(ts, x, emb, context=None, transformer_options={}, output_shape=None, time_context=None, num_video_frames=None, image_only_indicator=None):
+ for layer in ts:
+ if isinstance(layer, VideoResBlock):
+ x = layer(x, emb, num_video_frames, image_only_indicator)
+ elif isinstance(layer, TimestepBlock):
+ x = layer(x, emb)
+ elif isinstance(layer, SpatialVideoTransformer):
+ x = layer(x, context, time_context, num_video_frames, image_only_indicator, transformer_options)
+ if "transformer_index" in transformer_options:
+ transformer_options["transformer_index"] += 1
+ elif isinstance(layer, SpatialTransformer):
+ x = layer(x, context, transformer_options)
+ if "transformer_index" in transformer_options:
+ transformer_options["transformer_index"] += 1
+ elif isinstance(layer, Upsample):
+ x = layer(x, output_shape=output_shape)
+ else:
+ x = layer(x)
+ return x
class TimestepEmbedSequential(nn.Sequential, TimestepBlock):
"""
@@ -35,31 +58,8 @@ class TimestepEmbedSequential(nn.Sequential, TimestepBlock):
support it as an extra input.
"""
- def forward(self, x, emb, context=None, transformer_options={}, output_shape=None):
- for layer in self:
- if isinstance(layer, TimestepBlock):
- x = layer(x, emb)
- elif isinstance(layer, SpatialTransformer):
- x = layer(x, context, transformer_options)
- elif isinstance(layer, Upsample):
- x = layer(x, output_shape=output_shape)
- else:
- x = layer(x)
- return x
-
-#This is needed because accelerate makes a copy of transformer_options which breaks "current_index"
-def forward_timestep_embed(ts, x, emb, context=None, transformer_options={}, output_shape=None):
- for layer in ts:
- if isinstance(layer, TimestepBlock):
- x = layer(x, emb)
- elif isinstance(layer, SpatialTransformer):
- x = layer(x, context, transformer_options)
- transformer_options["current_index"] += 1
- elif isinstance(layer, Upsample):
- x = layer(x, output_shape=output_shape)
- else:
- x = layer(x)
- return x
+ def forward(self, *args, **kwargs):
+ return forward_timestep_embed(self, *args, **kwargs)
class Upsample(nn.Module):
"""
@@ -70,7 +70,7 @@ class Upsample(nn.Module):
upsampling occurs in the inner-two dimensions.
"""
- def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None, operations=comfy.ops):
+ def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None, operations=ops):
super().__init__()
self.channels = channels
self.out_channels = out_channels or channels
@@ -106,7 +106,7 @@ class Downsample(nn.Module):
downsampling occurs in the inner-two dimensions.
"""
- def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None, operations=comfy.ops):
+ def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None, operations=ops):
super().__init__()
self.channels = channels
self.out_channels = out_channels or channels
@@ -154,9 +154,12 @@ class ResBlock(TimestepBlock):
use_checkpoint=False,
up=False,
down=False,
+ kernel_size=3,
+ exchange_temb_dims=False,
+ skip_t_emb=False,
dtype=None,
device=None,
- operations=comfy.ops
+ operations=ops
):
super().__init__()
self.channels = channels
@@ -166,11 +169,17 @@ class ResBlock(TimestepBlock):
self.use_conv = use_conv
self.use_checkpoint = use_checkpoint
self.use_scale_shift_norm = use_scale_shift_norm
+ self.exchange_temb_dims = exchange_temb_dims
+
+ if isinstance(kernel_size, list):
+ padding = [k // 2 for k in kernel_size]
+ else:
+ padding = kernel_size // 2
self.in_layers = nn.Sequential(
- nn.GroupNorm(32, channels, dtype=dtype, device=device),
+ operations.GroupNorm(32, channels, dtype=dtype, device=device),
nn.SiLU(),
- operations.conv_nd(dims, channels, self.out_channels, 3, padding=1, dtype=dtype, device=device),
+ operations.conv_nd(dims, channels, self.out_channels, kernel_size, padding=padding, dtype=dtype, device=device),
)
self.updown = up or down
@@ -184,27 +193,31 @@ class ResBlock(TimestepBlock):
else:
self.h_upd = self.x_upd = nn.Identity()
- self.emb_layers = nn.Sequential(
- nn.SiLU(),
- operations.Linear(
- emb_channels,
- 2 * self.out_channels if use_scale_shift_norm else self.out_channels, dtype=dtype, device=device
- ),
- )
+ self.skip_t_emb = skip_t_emb
+ if self.skip_t_emb:
+ self.emb_layers = None
+ self.exchange_temb_dims = False
+ else:
+ self.emb_layers = nn.Sequential(
+ nn.SiLU(),
+ operations.Linear(
+ emb_channels,
+ 2 * self.out_channels if use_scale_shift_norm else self.out_channels, dtype=dtype, device=device
+ ),
+ )
self.out_layers = nn.Sequential(
- nn.GroupNorm(32, self.out_channels, dtype=dtype, device=device),
+ operations.GroupNorm(32, self.out_channels, dtype=dtype, device=device),
nn.SiLU(),
nn.Dropout(p=dropout),
- zero_module(
- operations.conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1, dtype=dtype, device=device)
- ),
+ operations.conv_nd(dims, self.out_channels, self.out_channels, kernel_size, padding=padding, dtype=dtype, device=device)
+ ,
)
if self.out_channels == channels:
self.skip_connection = nn.Identity()
elif use_conv:
self.skip_connection = operations.conv_nd(
- dims, channels, self.out_channels, 3, padding=1, dtype=dtype, device=device
+ dims, channels, self.out_channels, kernel_size, padding=padding, dtype=dtype, device=device
)
else:
self.skip_connection = operations.conv_nd(dims, channels, self.out_channels, 1, dtype=dtype, device=device)
@@ -230,19 +243,110 @@ class ResBlock(TimestepBlock):
h = in_conv(h)
else:
h = self.in_layers(x)
- emb_out = self.emb_layers(emb).type(h.dtype)
- while len(emb_out.shape) < len(h.shape):
- emb_out = emb_out[..., None]
+
+ emb_out = None
+ if not self.skip_t_emb:
+ emb_out = self.emb_layers(emb).type(h.dtype)
+ while len(emb_out.shape) < len(h.shape):
+ emb_out = emb_out[..., None]
if self.use_scale_shift_norm:
out_norm, out_rest = self.out_layers[0], self.out_layers[1:]
- scale, shift = th.chunk(emb_out, 2, dim=1)
- h = out_norm(h) * (1 + scale) + shift
+ h = out_norm(h)
+ if emb_out is not None:
+ scale, shift = th.chunk(emb_out, 2, dim=1)
+ h *= (1 + scale)
+ h += shift
h = out_rest(h)
else:
- h = h + emb_out
+ if emb_out is not None:
+ if self.exchange_temb_dims:
+ emb_out = rearrange(emb_out, "b t c ... -> b c t ...")
+ h = h + emb_out
h = self.out_layers(h)
return self.skip_connection(x) + h
+
+class VideoResBlock(ResBlock):
+ def __init__(
+ self,
+ channels: int,
+ emb_channels: int,
+ dropout: float,
+ video_kernel_size=3,
+ merge_strategy: str = "fixed",
+ merge_factor: float = 0.5,
+ out_channels=None,
+ use_conv: bool = False,
+ use_scale_shift_norm: bool = False,
+ dims: int = 2,
+ use_checkpoint: bool = False,
+ up: bool = False,
+ down: bool = False,
+ dtype=None,
+ device=None,
+ operations=ops
+ ):
+ super().__init__(
+ channels,
+ emb_channels,
+ dropout,
+ out_channels=out_channels,
+ use_conv=use_conv,
+ use_scale_shift_norm=use_scale_shift_norm,
+ dims=dims,
+ use_checkpoint=use_checkpoint,
+ up=up,
+ down=down,
+ dtype=dtype,
+ device=device,
+ operations=operations
+ )
+
+ self.time_stack = ResBlock(
+ default(out_channels, channels),
+ emb_channels,
+ dropout=dropout,
+ dims=3,
+ out_channels=default(out_channels, channels),
+ use_scale_shift_norm=False,
+ use_conv=False,
+ up=False,
+ down=False,
+ kernel_size=video_kernel_size,
+ use_checkpoint=use_checkpoint,
+ exchange_temb_dims=True,
+ dtype=dtype,
+ device=device,
+ operations=operations
+ )
+ self.time_mixer = AlphaBlender(
+ alpha=merge_factor,
+ merge_strategy=merge_strategy,
+ rearrange_pattern="b t -> b 1 t 1 1",
+ )
+
+ def forward(
+ self,
+ x: th.Tensor,
+ emb: th.Tensor,
+ num_video_frames: int,
+ image_only_indicator = None,
+ ) -> th.Tensor:
+ x = super().forward(x, emb)
+
+ x_mix = rearrange(x, "(b t) c h w -> b c t h w", t=num_video_frames)
+ x = rearrange(x, "(b t) c h w -> b c t h w", t=num_video_frames)
+
+ x = self.time_stack(
+ x, rearrange(emb, "(b t) ... -> b t ...", t=num_video_frames)
+ )
+ x = self.time_mixer(
+ x_spatial=x_mix, x_temporal=x, image_only_indicator=image_only_indicator
+ )
+ x = rearrange(x, "b c t h w -> (b t) c h w")
+ return x
+
+
class Timestep(nn.Module):
def __init__(self, dim):
super().__init__()
@@ -255,7 +359,10 @@ def apply_control(h, control, name):
if control is not None and name in control and len(control[name]) > 0:
ctrl = control[name].pop()
if ctrl is not None:
- h += ctrl
+ try:
+ h += ctrl
+ except:
+ print("warning control could not be applied", h.shape, ctrl.shape)
return h
class UNetModel(nn.Module):
@@ -316,8 +423,18 @@ class UNetModel(nn.Module):
adm_in_channels=None,
transformer_depth_middle=None,
transformer_depth_output=None,
+ use_temporal_resblock=False,
+ use_temporal_attention=False,
+ time_context_dim=None,
+ extra_ff_mix_layer=False,
+ use_spatial_context=False,
+ merge_strategy=None,
+ merge_factor=0.0,
+ video_kernel_size=None,
+ disable_temporal_crossattention=False,
+ max_ddpm_temb_period=10000,
device=None,
- operations=comfy.ops,
+ operations=ops,
):
super().__init__()
assert use_spatial_transformer == True, "use_spatial_transformer has to be true"
@@ -370,8 +487,12 @@ class UNetModel(nn.Module):
self.num_heads = num_heads
self.num_head_channels = num_head_channels
self.num_heads_upsample = num_heads_upsample
+ self.use_temporal_resblocks = use_temporal_resblock
self.predict_codebook_ids = n_embed is not None
+ self.default_num_video_frames = None
+ self.default_image_only_indicator = None
+
time_embed_dim = model_channels * 4
self.time_embed = nn.Sequential(
operations.Linear(model_channels, time_embed_dim, dtype=self.dtype, device=device),
@@ -408,13 +529,104 @@ class UNetModel(nn.Module):
input_block_chans = [model_channels]
ch = model_channels
ds = 1
+
+ def get_attention_layer(
+ ch,
+ num_heads,
+ dim_head,
+ depth=1,
+ context_dim=None,
+ use_checkpoint=False,
+ disable_self_attn=False,
+ ):
+ if use_temporal_attention:
+ return SpatialVideoTransformer(
+ ch,
+ num_heads,
+ dim_head,
+ depth=depth,
+ context_dim=context_dim,
+ time_context_dim=time_context_dim,
+ dropout=dropout,
+ ff_in=extra_ff_mix_layer,
+ use_spatial_context=use_spatial_context,
+ merge_strategy=merge_strategy,
+ merge_factor=merge_factor,
+ checkpoint=use_checkpoint,
+ use_linear=use_linear_in_transformer,
+ disable_self_attn=disable_self_attn,
+ disable_temporal_crossattention=disable_temporal_crossattention,
+ max_time_embed_period=max_ddpm_temb_period,
+ dtype=self.dtype, device=device, operations=operations
+ )
+ else:
+ return SpatialTransformer(
+ ch, num_heads, dim_head, depth=depth, context_dim=context_dim,
+ disable_self_attn=disable_self_attn, use_linear=use_linear_in_transformer,
+ use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
+ )
+
+ def get_resblock(
+ merge_factor,
+ merge_strategy,
+ video_kernel_size,
+ ch,
+ time_embed_dim,
+ dropout,
+ out_channels,
+ dims,
+ use_checkpoint,
+ use_scale_shift_norm,
+ down=False,
+ up=False,
+ dtype=None,
+ device=None,
+ operations=ops
+ ):
+ if self.use_temporal_resblocks:
+ return VideoResBlock(
+ merge_factor=merge_factor,
+ merge_strategy=merge_strategy,
+ video_kernel_size=video_kernel_size,
+ channels=ch,
+ emb_channels=time_embed_dim,
+ dropout=dropout,
+ out_channels=out_channels,
+ dims=dims,
+ use_checkpoint=use_checkpoint,
+ use_scale_shift_norm=use_scale_shift_norm,
+ down=down,
+ up=up,
+ dtype=dtype,
+ device=device,
+ operations=operations
+ )
+ else:
+ return ResBlock(
+ channels=ch,
+ emb_channels=time_embed_dim,
+ dropout=dropout,
+ out_channels=out_channels,
+ use_checkpoint=use_checkpoint,
+ dims=dims,
+ use_scale_shift_norm=use_scale_shift_norm,
+ down=down,
+ up=up,
+ dtype=dtype,
+ device=device,
+ operations=operations
+ )
+
for level, mult in enumerate(channel_mult):
for nr in range(self.num_res_blocks[level]):
layers = [
- ResBlock(
- ch,
- time_embed_dim,
- dropout,
+ get_resblock(
+ merge_factor=merge_factor,
+ merge_strategy=merge_strategy,
+ video_kernel_size=video_kernel_size,
+ ch=ch,
+ time_embed_dim=time_embed_dim,
+ dropout=dropout,
out_channels=mult * model_channels,
dims=dims,
use_checkpoint=use_checkpoint,
@@ -441,11 +653,9 @@ class UNetModel(nn.Module):
disabled_sa = False
if not exists(num_attention_blocks) or nr < num_attention_blocks[level]:
- layers.append(SpatialTransformer(
+ layers.append(get_attention_layer(
ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim,
- disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
- use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
- )
+ disable_self_attn=disabled_sa, use_checkpoint=use_checkpoint)
)
self.input_blocks.append(TimestepEmbedSequential(*layers))
self._feature_size += ch
@@ -454,10 +664,13 @@ class UNetModel(nn.Module):
out_ch = ch
self.input_blocks.append(
TimestepEmbedSequential(
- ResBlock(
- ch,
- time_embed_dim,
- dropout,
+ get_resblock(
+ merge_factor=merge_factor,
+ merge_strategy=merge_strategy,
+ video_kernel_size=video_kernel_size,
+ ch=ch,
+ time_embed_dim=time_embed_dim,
+ dropout=dropout,
out_channels=out_ch,
dims=dims,
use_checkpoint=use_checkpoint,
@@ -487,10 +700,14 @@ class UNetModel(nn.Module):
#num_heads = 1
dim_head = ch // num_heads if use_spatial_transformer else num_head_channels
mid_block = [
- ResBlock(
- ch,
- time_embed_dim,
- dropout,
+ get_resblock(
+ merge_factor=merge_factor,
+ merge_strategy=merge_strategy,
+ video_kernel_size=video_kernel_size,
+ ch=ch,
+ time_embed_dim=time_embed_dim,
+ dropout=dropout,
+ out_channels=None,
dims=dims,
use_checkpoint=use_checkpoint,
use_scale_shift_norm=use_scale_shift_norm,
@@ -499,15 +716,18 @@ class UNetModel(nn.Module):
operations=operations
)]
if transformer_depth_middle >= 0:
- mid_block += [SpatialTransformer( # always uses a self-attn
+ mid_block += [get_attention_layer( # always uses a self-attn
ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,
- disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer,
- use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
+ disable_self_attn=disable_middle_self_attn, use_checkpoint=use_checkpoint
),
- ResBlock(
- ch,
- time_embed_dim,
- dropout,
+ get_resblock(
+ merge_factor=merge_factor,
+ merge_strategy=merge_strategy,
+ video_kernel_size=video_kernel_size,
+ ch=ch,
+ time_embed_dim=time_embed_dim,
+ dropout=dropout,
+ out_channels=None,
dims=dims,
use_checkpoint=use_checkpoint,
use_scale_shift_norm=use_scale_shift_norm,
@@ -523,10 +743,13 @@ class UNetModel(nn.Module):
for i in range(self.num_res_blocks[level] + 1):
ich = input_block_chans.pop()
layers = [
- ResBlock(
- ch + ich,
- time_embed_dim,
- dropout,
+ get_resblock(
+ merge_factor=merge_factor,
+ merge_strategy=merge_strategy,
+ video_kernel_size=video_kernel_size,
+ ch=ch + ich,
+ time_embed_dim=time_embed_dim,
+ dropout=dropout,
out_channels=model_channels * mult,
dims=dims,
use_checkpoint=use_checkpoint,
@@ -554,19 +777,21 @@ class UNetModel(nn.Module):
if not exists(num_attention_blocks) or i < num_attention_blocks[level]:
layers.append(
- SpatialTransformer(
+ get_attention_layer(
ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim,
- disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
- use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
+ disable_self_attn=disabled_sa, use_checkpoint=use_checkpoint
)
)
if level and i == self.num_res_blocks[level]:
out_ch = ch
layers.append(
- ResBlock(
- ch,
- time_embed_dim,
- dropout,
+ get_resblock(
+ merge_factor=merge_factor,
+ merge_strategy=merge_strategy,
+ video_kernel_size=video_kernel_size,
+ ch=ch,
+ time_embed_dim=time_embed_dim,
+ dropout=dropout,
out_channels=out_ch,
dims=dims,
use_checkpoint=use_checkpoint,
@@ -584,13 +809,13 @@ class UNetModel(nn.Module):
self._feature_size += ch
self.out = nn.Sequential(
- nn.GroupNorm(32, ch, dtype=self.dtype, device=device),
+ operations.GroupNorm(32, ch, dtype=self.dtype, device=device),
nn.SiLU(),
zero_module(operations.conv_nd(dims, model_channels, out_channels, 3, padding=1, dtype=self.dtype, device=device)),
)
if self.predict_codebook_ids:
self.id_predictor = nn.Sequential(
- nn.GroupNorm(32, ch, dtype=self.dtype, device=device),
+ operations.GroupNorm(32, ch, dtype=self.dtype, device=device),
operations.conv_nd(dims, model_channels, n_embed, 1, dtype=self.dtype, device=device),
#nn.LogSoftmax(dim=1) # change to cross_entropy and produce non-normalized logits
)
@@ -605,31 +830,45 @@ class UNetModel(nn.Module):
:return: an [N x C x ...] Tensor of outputs.
"""
transformer_options["original_shape"] = list(x.shape)
- transformer_options["current_index"] = 0
+ transformer_options["transformer_index"] = 0
transformer_patches = transformer_options.get("patches", {})
+ num_video_frames = kwargs.get("num_video_frames", self.default_num_video_frames)
+ image_only_indicator = kwargs.get("image_only_indicator", self.default_image_only_indicator)
+ time_context = kwargs.get("time_context", None)
+
assert (y is not None) == (
self.num_classes is not None
), "must specify y if and only if the model is class-conditional"
hs = []
- t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(self.dtype)
+ t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(x.dtype)
emb = self.time_embed(t_emb)
if self.num_classes is not None:
assert y.shape[0] == x.shape[0]
emb = emb + self.label_emb(y)
- h = x.type(self.dtype)
+ h = x
for id, module in enumerate(self.input_blocks):
transformer_options["block"] = ("input", id)
- h = forward_timestep_embed(module, h, emb, context, transformer_options)
+ h = forward_timestep_embed(module, h, emb, context, transformer_options, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
h = apply_control(h, control, 'input')
+ if "input_block_patch" in transformer_patches:
+ patch = transformer_patches["input_block_patch"]
+ for p in patch:
+ h = p(h, transformer_options)
+
hs.append(h)
+ if "input_block_patch_after_skip" in transformer_patches:
+ patch = transformer_patches["input_block_patch_after_skip"]
+ for p in patch:
+ h = p(h, transformer_options)
transformer_options["block"] = ("middle", 0)
- h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options)
+ h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
h = apply_control(h, control, 'middle')
+
for id, module in enumerate(self.output_blocks):
transformer_options["block"] = ("output", id)
hsp = hs.pop()
@@ -646,7 +885,7 @@ class UNetModel(nn.Module):
output_shape = hs[-1].shape
else:
output_shape = None
- h = forward_timestep_embed(module, h, emb, context, transformer_options, output_shape)
+ h = forward_timestep_embed(module, h, emb, context, transformer_options, output_shape, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
h = h.type(x.dtype)
if self.predict_codebook_ids:
return self.id_predictor(h)
diff --git a/comfy/ldm/modules/diffusionmodules/upscaling.py b/comfy/ldm/modules/diffusionmodules/upscaling.py
index 709a7f52e..768a47f9c 100644
--- a/comfy/ldm/modules/diffusionmodules/upscaling.py
+++ b/comfy/ldm/modules/diffusionmodules/upscaling.py
@@ -43,8 +43,8 @@ class AbstractLowScaleModel(nn.Module):
def q_sample(self, x_start, t, noise=None):
noise = default(noise, lambda: torch.randn_like(x_start))
- return (extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start +
- extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise)
+ return (extract_into_tensor(self.sqrt_alphas_cumprod.to(x_start.device), t, x_start.shape) * x_start +
+ extract_into_tensor(self.sqrt_one_minus_alphas_cumprod.to(x_start.device), t, x_start.shape) * noise)
def forward(self, x):
return x, None
diff --git a/comfy/ldm/modules/diffusionmodules/util.py b/comfy/ldm/modules/diffusionmodules/util.py
index 0298ca99d..ac7e27173 100644
--- a/comfy/ldm/modules/diffusionmodules/util.py
+++ b/comfy/ldm/modules/diffusionmodules/util.py
@@ -13,10 +13,76 @@ import math
import torch
import torch.nn as nn
import numpy as np
-from einops import repeat
+from einops import repeat, rearrange
from comfy.ldm.util import instantiate_from_config
-import comfy.ops
+
+class AlphaBlender(nn.Module):
+ strategies = ["learned", "fixed", "learned_with_images"]
+
+ def __init__(
+ self,
+ alpha: float,
+ merge_strategy: str = "learned_with_images",
+ rearrange_pattern: str = "b t -> (b t) 1 1",
+ ):
+ super().__init__()
+ self.merge_strategy = merge_strategy
+ self.rearrange_pattern = rearrange_pattern
+
+ assert (
+ merge_strategy in self.strategies
+ ), f"merge_strategy needs to be in {self.strategies}"
+
+ if self.merge_strategy == "fixed":
+ self.register_buffer("mix_factor", torch.Tensor([alpha]))
+ elif (
+ self.merge_strategy == "learned"
+ or self.merge_strategy == "learned_with_images"
+ ):
+ self.register_parameter(
+ "mix_factor", torch.nn.Parameter(torch.Tensor([alpha]))
+ )
+ else:
+ raise ValueError(f"unknown merge strategy {self.merge_strategy}")
+
+ def get_alpha(self, image_only_indicator: torch.Tensor) -> torch.Tensor:
+ # skip_time_mix = rearrange(repeat(skip_time_mix, 'b -> (b t) () () ()', t=t), '(b t) 1 ... -> b 1 t ...', t=t)
+ if self.merge_strategy == "fixed":
+ # make shape compatible
+ # alpha = repeat(self.mix_factor, '1 -> b () t () ()', t=t, b=bs)
+ alpha = self.mix_factor.to(image_only_indicator.device)
+ elif self.merge_strategy == "learned":
+ alpha = torch.sigmoid(self.mix_factor.to(image_only_indicator.device))
+ # make shape compatible
+ # alpha = repeat(alpha, '1 -> s () ()', s = t * bs)
+ elif self.merge_strategy == "learned_with_images":
+ assert image_only_indicator is not None, "need image_only_indicator ..."
+ alpha = torch.where(
+ image_only_indicator.bool(),
+ torch.ones(1, 1, device=image_only_indicator.device),
+ rearrange(torch.sigmoid(self.mix_factor.to(image_only_indicator.device)), "... -> ... 1"),
+ )
+ alpha = rearrange(alpha, self.rearrange_pattern)
+ # make shape compatible
+ # alpha = repeat(alpha, '1 -> s () ()', s = t * bs)
+ else:
+ raise NotImplementedError()
+ return alpha
+
+ def forward(
+ self,
+ x_spatial,
+ x_temporal,
+ image_only_indicator=None,
+ ) -> torch.Tensor:
+ alpha = self.get_alpha(image_only_indicator)
+ x = (
+ alpha.to(x_spatial.dtype) * x_spatial
+ + (1.0 - alpha).to(x_spatial.dtype) * x_temporal
+ )
+ return x
+
def make_beta_schedule(schedule, n_timestep, linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3):
if schedule == "linear":
@@ -206,46 +272,6 @@ def mean_flat(tensor):
return tensor.mean(dim=list(range(1, len(tensor.shape))))
-def normalization(channels, dtype=None):
- """
- Make a standard normalization layer.
- :param channels: number of input channels.
- :return: an nn.Module for normalization.
- """
- return GroupNorm32(32, channels, dtype=dtype)
-
-
-# PyTorch 1.7 has SiLU, but we support PyTorch 1.5.
-class SiLU(nn.Module):
- def forward(self, x):
- return x * torch.sigmoid(x)
-
-
-class GroupNorm32(nn.GroupNorm):
- def forward(self, x):
- return super().forward(x.float()).type(x.dtype)
-
-
-def conv_nd(dims, *args, **kwargs):
- """
- Create a 1D, 2D, or 3D convolution module.
- """
- if dims == 1:
- return nn.Conv1d(*args, **kwargs)
- elif dims == 2:
- return comfy.ops.Conv2d(*args, **kwargs)
- elif dims == 3:
- return nn.Conv3d(*args, **kwargs)
- raise ValueError(f"unsupported dimensions: {dims}")
-
-
-def linear(*args, **kwargs):
- """
- Create a linear module.
- """
- return comfy.ops.Linear(*args, **kwargs)
-
-
def avg_pool_nd(dims, *args, **kwargs):
"""
Create a 1D, 2D, or 3D average pooling module.
diff --git a/comfy/ldm/modules/encoders/noise_aug_modules.py b/comfy/ldm/modules/encoders/noise_aug_modules.py
index b59bf204b..66767b587 100644
--- a/comfy/ldm/modules/encoders/noise_aug_modules.py
+++ b/comfy/ldm/modules/encoders/noise_aug_modules.py
@@ -15,12 +15,12 @@ class CLIPEmbeddingNoiseAugmentation(ImageConcatWithNoiseAugmentation):
def scale(self, x):
# re-normalize to centered mean and unit variance
- x = (x - self.data_mean) * 1. / self.data_std
+ x = (x - self.data_mean.to(x.device)) * 1. / self.data_std.to(x.device)
return x
def unscale(self, x):
# back to original data stats
- x = (x * self.data_std) + self.data_mean
+ x = (x * self.data_std.to(x.device)) + self.data_mean.to(x.device)
return x
def forward(self, x, noise_level=None):
diff --git a/comfy/ldm/modules/temporal_ae.py b/comfy/ldm/modules/temporal_ae.py
new file mode 100644
index 000000000..2992aeafc
--- /dev/null
+++ b/comfy/ldm/modules/temporal_ae.py
@@ -0,0 +1,245 @@
+import functools
+from typing import Callable, Iterable, Union
+
+import torch
+from einops import rearrange, repeat
+
+import comfy.ops
+ops = comfy.ops.disable_weight_init
+
+from .diffusionmodules.model import (
+ AttnBlock,
+ Decoder,
+ ResnetBlock,
+)
+from .diffusionmodules.openaimodel import ResBlock, timestep_embedding
+from .attention import BasicTransformerBlock
+
+def partialclass(cls, *args, **kwargs):
+ class NewCls(cls):
+ __init__ = functools.partialmethod(cls.__init__, *args, **kwargs)
+
+ return NewCls
+
+
+class VideoResBlock(ResnetBlock):
+ def __init__(
+ self,
+ out_channels,
+ *args,
+ dropout=0.0,
+ video_kernel_size=3,
+ alpha=0.0,
+ merge_strategy="learned",
+ **kwargs,
+ ):
+ super().__init__(out_channels=out_channels, dropout=dropout, *args, **kwargs)
+ if video_kernel_size is None:
+ video_kernel_size = [3, 1, 1]
+ self.time_stack = ResBlock(
+ channels=out_channels,
+ emb_channels=0,
+ dropout=dropout,
+ dims=3,
+ use_scale_shift_norm=False,
+ use_conv=False,
+ up=False,
+ down=False,
+ kernel_size=video_kernel_size,
+ use_checkpoint=False,
+ skip_t_emb=True,
+ )
+
+ self.merge_strategy = merge_strategy
+ if self.merge_strategy == "fixed":
+ self.register_buffer("mix_factor", torch.Tensor([alpha]))
+ elif self.merge_strategy == "learned":
+ self.register_parameter(
+ "mix_factor", torch.nn.Parameter(torch.Tensor([alpha]))
+ )
+ else:
+ raise ValueError(f"unknown merge strategy {self.merge_strategy}")
+
+ def get_alpha(self, bs):
+ if self.merge_strategy == "fixed":
+ return self.mix_factor
+ elif self.merge_strategy == "learned":
+ return torch.sigmoid(self.mix_factor)
+ else:
+ raise NotImplementedError()
+
+ def forward(self, x, temb, skip_video=False, timesteps=None):
+ b, c, h, w = x.shape
+ if timesteps is None:
+ timesteps = b
+
+ x = super().forward(x, temb)
+
+ if not skip_video:
+ x_mix = rearrange(x, "(b t) c h w -> b c t h w", t=timesteps)
+
+ x = rearrange(x, "(b t) c h w -> b c t h w", t=timesteps)
+
+ x = self.time_stack(x, temb)
+
+ alpha = self.get_alpha(bs=b // timesteps).to(x.device)
+ x = alpha * x + (1.0 - alpha) * x_mix
+
+ x = rearrange(x, "b c t h w -> (b t) c h w")
+ return x
+
+
+class AE3DConv(ops.Conv2d):
+ def __init__(self, in_channels, out_channels, video_kernel_size=3, *args, **kwargs):
+ super().__init__(in_channels, out_channels, *args, **kwargs)
+ if isinstance(video_kernel_size, Iterable):
+ padding = [int(k // 2) for k in video_kernel_size]
+ else:
+ padding = int(video_kernel_size // 2)
+
+ self.time_mix_conv = ops.Conv3d(
+ in_channels=out_channels,
+ out_channels=out_channels,
+ kernel_size=video_kernel_size,
+ padding=padding,
+ )
+
+ def forward(self, input, timesteps=None, skip_video=False):
+ if timesteps is None:
+ timesteps = input.shape[0]
+ x = super().forward(input)
+ if skip_video:
+ return x
+ x = rearrange(x, "(b t) c h w -> b c t h w", t=timesteps)
+ x = self.time_mix_conv(x)
+ return rearrange(x, "b c t h w -> (b t) c h w")
+
+
+class AttnVideoBlock(AttnBlock):
+ def __init__(
+ self, in_channels: int, alpha: float = 0, merge_strategy: str = "learned"
+ ):
+ super().__init__(in_channels)
+ # no context, single headed, as in base class
+ self.time_mix_block = BasicTransformerBlock(
+ dim=in_channels,
+ n_heads=1,
+ d_head=in_channels,
+ checkpoint=False,
+ ff_in=True,
+ )
+
+ time_embed_dim = self.in_channels * 4
+ self.video_time_embed = torch.nn.Sequential(
+ ops.Linear(self.in_channels, time_embed_dim),
+ torch.nn.SiLU(),
+ ops.Linear(time_embed_dim, self.in_channels),
+ )
+
+ self.merge_strategy = merge_strategy
+ if self.merge_strategy == "fixed":
+ self.register_buffer("mix_factor", torch.Tensor([alpha]))
+ elif self.merge_strategy == "learned":
+ self.register_parameter(
+ "mix_factor", torch.nn.Parameter(torch.Tensor([alpha]))
+ )
+ else:
+ raise ValueError(f"unknown merge strategy {self.merge_strategy}")
+
+ def forward(self, x, timesteps=None, skip_time_block=False):
+ if skip_time_block:
+ return super().forward(x)
+
+ if timesteps is None:
+ timesteps = x.shape[0]
+
+ x_in = x
+ x = self.attention(x)
+ h, w = x.shape[2:]
+ x = rearrange(x, "b c h w -> b (h w) c")
+
+ x_mix = x
+ num_frames = torch.arange(timesteps, device=x.device)
+ num_frames = repeat(num_frames, "t -> b t", b=x.shape[0] // timesteps)
+ num_frames = rearrange(num_frames, "b t -> (b t)")
+ t_emb = timestep_embedding(num_frames, self.in_channels, repeat_only=False)
+ emb = self.video_time_embed(t_emb) # b, n_channels
+ emb = emb[:, None, :]
+ x_mix = x_mix + emb
+
+ alpha = self.get_alpha().to(x.device)
+ x_mix = self.time_mix_block(x_mix, timesteps=timesteps)
+ x = alpha * x + (1.0 - alpha) * x_mix # alpha merge
+
+ x = rearrange(x, "b (h w) c -> b c h w", h=h, w=w)
+ x = self.proj_out(x)
+
+ return x_in + x
+
+ def get_alpha(
+ self,
+ ):
+ if self.merge_strategy == "fixed":
+ return self.mix_factor
+ elif self.merge_strategy == "learned":
+ return torch.sigmoid(self.mix_factor)
+ else:
+ raise NotImplementedError(f"unknown merge strategy {self.merge_strategy}")
+
+
+
+def make_time_attn(
+ in_channels,
+ attn_type="vanilla",
+ attn_kwargs=None,
+ alpha: float = 0,
+ merge_strategy: str = "learned",
+):
+ return partialclass(
+ AttnVideoBlock, in_channels, alpha=alpha, merge_strategy=merge_strategy
+ )
+
+
+class Conv2DWrapper(torch.nn.Conv2d):
+ def forward(self, input: torch.Tensor, **kwargs) -> torch.Tensor:
+ return super().forward(input)
+
+
+class VideoDecoder(Decoder):
+ available_time_modes = ["all", "conv-only", "attn-only"]
+
+ def __init__(
+ self,
+ *args,
+ video_kernel_size: Union[int, list] = 3,
+ alpha: float = 0.0,
+ merge_strategy: str = "learned",
+ time_mode: str = "conv-only",
+ **kwargs,
+ ):
+ self.video_kernel_size = video_kernel_size
+ self.alpha = alpha
+ self.merge_strategy = merge_strategy
+ self.time_mode = time_mode
+ assert (
+ self.time_mode in self.available_time_modes
+ ), f"time_mode parameter has to be in {self.available_time_modes}"
+
+ if self.time_mode != "attn-only":
+ kwargs["conv_out_op"] = partialclass(AE3DConv, video_kernel_size=self.video_kernel_size)
+ if self.time_mode not in ["conv-only", "only-last-conv"]:
+ kwargs["attn_op"] = partialclass(make_time_attn, alpha=self.alpha, merge_strategy=self.merge_strategy)
+ if self.time_mode not in ["attn-only", "only-last-conv"]:
+ kwargs["resnet_op"] = partialclass(VideoResBlock, video_kernel_size=self.video_kernel_size, alpha=self.alpha, merge_strategy=self.merge_strategy)
+
+ super().__init__(*args, **kwargs)
+
+ def get_last_layer(self, skip_time_mix=False, **kwargs):
+ if self.time_mode == "attn-only":
+ raise NotImplementedError("TODO")
+ else:
+ return (
+ self.conv_out.time_mix_conv.weight
+ if not skip_time_mix
+ else self.conv_out.weight
+ )
diff --git a/comfy/lora.py b/comfy/lora.py
index 29c59d893..5e4009b47 100644
--- a/comfy/lora.py
+++ b/comfy/lora.py
@@ -43,7 +43,7 @@ def load_lora(lora, to_load):
if mid_name is not None and mid_name in lora.keys():
mid = lora[mid_name]
loaded_keys.add(mid_name)
- patch_dict[to_load[x]] = (lora[A_name], lora[B_name], alpha, mid)
+ patch_dict[to_load[x]] = ("lora", (lora[A_name], lora[B_name], alpha, mid))
loaded_keys.add(A_name)
loaded_keys.add(B_name)
@@ -64,7 +64,7 @@ def load_lora(lora, to_load):
loaded_keys.add(hada_t1_name)
loaded_keys.add(hada_t2_name)
- patch_dict[to_load[x]] = (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2)
+ patch_dict[to_load[x]] = ("loha", (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2))
loaded_keys.add(hada_w1_a_name)
loaded_keys.add(hada_w1_b_name)
loaded_keys.add(hada_w2_a_name)
@@ -116,8 +116,19 @@ def load_lora(lora, to_load):
loaded_keys.add(lokr_t2_name)
if (lokr_w1 is not None) or (lokr_w2 is not None) or (lokr_w1_a is not None) or (lokr_w2_a is not None):
- patch_dict[to_load[x]] = (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2)
+ patch_dict[to_load[x]] = ("lokr", (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2))
+ #glora
+ a1_name = "{}.a1.weight".format(x)
+ a2_name = "{}.a2.weight".format(x)
+ b1_name = "{}.b1.weight".format(x)
+ b2_name = "{}.b2.weight".format(x)
+ if a1_name in lora:
+ patch_dict[to_load[x]] = ("glora", (lora[a1_name], lora[a2_name], lora[b1_name], lora[b2_name], alpha))
+ loaded_keys.add(a1_name)
+ loaded_keys.add(a2_name)
+ loaded_keys.add(b1_name)
+ loaded_keys.add(b2_name)
w_norm_name = "{}.w_norm".format(x)
b_norm_name = "{}.b_norm".format(x)
@@ -126,21 +137,21 @@ def load_lora(lora, to_load):
if w_norm is not None:
loaded_keys.add(w_norm_name)
- patch_dict[to_load[x]] = (w_norm,)
+ patch_dict[to_load[x]] = ("diff", (w_norm,))
if b_norm is not None:
loaded_keys.add(b_norm_name)
- patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = (b_norm,)
+ patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = ("diff", (b_norm,))
diff_name = "{}.diff".format(x)
diff_weight = lora.get(diff_name, None)
if diff_weight is not None:
- patch_dict[to_load[x]] = (diff_weight,)
+ patch_dict[to_load[x]] = ("diff", (diff_weight,))
loaded_keys.add(diff_name)
diff_bias_name = "{}.diff_b".format(x)
diff_bias = lora.get(diff_bias_name, None)
if diff_bias is not None:
- patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = (diff_bias,)
+ patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = ("diff", (diff_bias,))
loaded_keys.add(diff_bias_name)
for x in lora.keys():
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 7ba253470..b3a1fcd51 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -4,23 +4,30 @@ from comfy.ldm.modules.encoders.noise_aug_modules import CLIPEmbeddingNoiseAugme
from comfy.ldm.modules.diffusionmodules.openaimodel import Timestep
import comfy.model_management
import comfy.conds
+import comfy.ops
from enum import Enum
+import contextlib
from . import utils
class ModelType(Enum):
EPS = 1
V_PREDICTION = 2
+ V_PREDICTION_EDM = 3
-from comfy.model_sampling import EPS, V_PREDICTION, ModelSamplingDiscrete
+from comfy.model_sampling import EPS, V_PREDICTION, ModelSamplingDiscrete, ModelSamplingContinuousEDM
+
def model_sampling(model_config, model_type):
+ s = ModelSamplingDiscrete
+
if model_type == ModelType.EPS:
c = EPS
elif model_type == ModelType.V_PREDICTION:
c = V_PREDICTION
-
- s = ModelSamplingDiscrete
+ elif model_type == ModelType.V_PREDICTION_EDM:
+ c = V_PREDICTION
+ s = ModelSamplingContinuousEDM
class ModelSampling(s, c):
pass
@@ -35,9 +42,14 @@ class BaseModel(torch.nn.Module):
unet_config = model_config.unet_config
self.latent_format = model_config.latent_format
self.model_config = model_config
+ self.manual_cast_dtype = model_config.manual_cast_dtype
if not unet_config.get("disable_unet_model_creation", False):
- self.diffusion_model = UNetModel(**unet_config, device=device)
+ if self.manual_cast_dtype is not None:
+ operations = comfy.ops.manual_cast
+ else:
+ operations = comfy.ops.disable_weight_init
+ self.diffusion_model = UNetModel(**unet_config, device=device, operations=operations)
self.model_type = model_type
self.model_sampling = model_sampling(model_config, model_type)
@@ -56,6 +68,10 @@ class BaseModel(torch.nn.Module):
context = c_crossattn
dtype = self.get_dtype()
+
+ if self.manual_cast_dtype is not None:
+ dtype = self.manual_cast_dtype
+
xc = xc.to(dtype)
t = self.model_sampling.timestep(t).float()
context = context.to(dtype)
@@ -65,6 +81,7 @@ class BaseModel(torch.nn.Module):
if hasattr(extra, "to"):
extra = extra.to(dtype)
extra_conds[o] = extra
+
model_output = self.diffusion_model(xc, t, context=context, control=control, transformer_options=transformer_options, **extra_conds).float()
return self.model_sampling.calculate_denoised(sigma, model_output, x)
@@ -109,9 +126,15 @@ class BaseModel(torch.nn.Module):
cond_concat.append(blank_inpaint_image_like(noise))
data = torch.cat(cond_concat, dim=1)
out['c_concat'] = comfy.conds.CONDNoiseShape(data)
+
adm = self.encode_adm(**kwargs)
if adm is not None:
out['y'] = comfy.conds.CONDRegular(adm)
+
+ cross_attn = kwargs.get("cross_attn", None)
+ if cross_attn is not None:
+ out['c_crossattn'] = comfy.conds.CONDCrossAttn(cross_attn)
+
return out
def load_model_weights(self, sd, unet_prefix=""):
@@ -121,6 +144,7 @@ class BaseModel(torch.nn.Module):
if k.startswith(unet_prefix):
to_load[k[len(unet_prefix):]] = sd.pop(k)
+ to_load = self.model_config.process_unet_state_dict(to_load)
m, u = self.diffusion_model.load_state_dict(to_load, strict=False)
if len(m) > 0:
print("unet missing:", m)
@@ -138,11 +162,7 @@ class BaseModel(torch.nn.Module):
def state_dict_for_saving(self, clip_state_dict, vae_state_dict):
clip_state_dict = self.model_config.process_clip_state_dict_for_saving(clip_state_dict)
- unet_sd = self.diffusion_model.state_dict()
- unet_state_dict = {}
- for k in unet_sd:
- unet_state_dict[k] = comfy.model_management.resolve_lowvram_weight(unet_sd[k], self.diffusion_model, k)
-
+ unet_state_dict = self.diffusion_model.state_dict()
unet_state_dict = self.model_config.process_unet_state_dict_for_saving(unet_state_dict)
vae_state_dict = self.model_config.process_vae_state_dict_for_saving(vae_state_dict)
if self.get_dtype() == torch.float16:
@@ -157,6 +177,20 @@ class BaseModel(torch.nn.Module):
def set_inpaint(self):
self.inpaint_model = True
+ def memory_required(self, input_shape):
+ if comfy.model_management.xformers_enabled() or comfy.model_management.pytorch_attention_flash_attention():
+ dtype = self.get_dtype()
+ if self.manual_cast_dtype is not None:
+ dtype = self.manual_cast_dtype
+ #TODO: this needs to be tweaked
+ area = input_shape[0] * input_shape[2] * input_shape[3]
+ return (area * comfy.model_management.dtype_size(dtype) / 50) * (1024 * 1024)
+ else:
+ #TODO: this formula might be too aggressive since I tweaked the sub-quad and split algorithms to use less memory.
+ area = input_shape[0] * input_shape[2] * input_shape[3]
+ return (((area * 0.6) / 0.9) + 1024) * (1024 * 1024)
+
+
def unclip_adm(unclip_conditioning, device, noise_augmentor, noise_augment_merge=0.0):
adm_inputs = []
weights = []
@@ -251,3 +285,82 @@ class SDXL(BaseModel):
out.append(self.embedder(torch.Tensor([target_width])))
flat = torch.flatten(torch.cat(out)).unsqueeze(dim=0).repeat(clip_pooled.shape[0], 1)
return torch.cat((clip_pooled.to(flat.device), flat), dim=1)
+
+class SVD_img2vid(BaseModel):
+ def __init__(self, model_config, model_type=ModelType.V_PREDICTION_EDM, device=None):
+ super().__init__(model_config, model_type, device=device)
+ self.embedder = Timestep(256)
+
+ def encode_adm(self, **kwargs):
+ fps_id = kwargs.get("fps", 6) - 1
+ motion_bucket_id = kwargs.get("motion_bucket_id", 127)
+ augmentation = kwargs.get("augmentation_level", 0)
+
+ out = []
+ out.append(self.embedder(torch.Tensor([fps_id])))
+ out.append(self.embedder(torch.Tensor([motion_bucket_id])))
+ out.append(self.embedder(torch.Tensor([augmentation])))
+
+ flat = torch.flatten(torch.cat(out)).unsqueeze(dim=0)
+ return flat
+
+ def extra_conds(self, **kwargs):
+ out = {}
+ adm = self.encode_adm(**kwargs)
+ if adm is not None:
+ out['y'] = comfy.conds.CONDRegular(adm)
+
+ latent_image = kwargs.get("concat_latent_image", None)
+ noise = kwargs.get("noise", None)
+ device = kwargs["device"]
+
+ if latent_image is None:
+ latent_image = torch.zeros_like(noise)
+
+ if latent_image.shape[1:] != noise.shape[1:]:
+ latent_image = utils.common_upscale(latent_image, noise.shape[-1], noise.shape[-2], "bilinear", "center")
+
+ latent_image = utils.resize_to_batch_size(latent_image, noise.shape[0])
+
+ out['c_concat'] = comfy.conds.CONDNoiseShape(latent_image)
+
+ cross_attn = kwargs.get("cross_attn", None)
+ if cross_attn is not None:
+ out['c_crossattn'] = comfy.conds.CONDCrossAttn(cross_attn)
+
+ if "time_conditioning" in kwargs:
+ out["time_context"] = comfy.conds.CONDCrossAttn(kwargs["time_conditioning"])
+
+ out['image_only_indicator'] = comfy.conds.CONDConstant(torch.zeros((1,), device=device))
+ out['num_video_frames'] = comfy.conds.CONDConstant(noise.shape[0])
+ return out
+
+class Stable_Zero123(BaseModel):
+ def __init__(self, model_config, model_type=ModelType.EPS, device=None, cc_projection_weight=None, cc_projection_bias=None):
+ super().__init__(model_config, model_type, device=device)
+ self.cc_projection = comfy.ops.manual_cast.Linear(cc_projection_weight.shape[1], cc_projection_weight.shape[0], dtype=self.get_dtype(), device=device)
+ self.cc_projection.weight.copy_(cc_projection_weight)
+ self.cc_projection.bias.copy_(cc_projection_bias)
+
+ def extra_conds(self, **kwargs):
+ out = {}
+
+ latent_image = kwargs.get("concat_latent_image", None)
+ noise = kwargs.get("noise", None)
+
+ if latent_image is None:
+ latent_image = torch.zeros_like(noise)
+
+ if latent_image.shape[1:] != noise.shape[1:]:
+ latent_image = utils.common_upscale(latent_image, noise.shape[-1], noise.shape[-2], "bilinear", "center")
+
+ latent_image = utils.resize_to_batch_size(latent_image, noise.shape[0])
+
+ out['c_concat'] = comfy.conds.CONDNoiseShape(latent_image)
+
+ cross_attn = kwargs.get("cross_attn", None)
+ if cross_attn is not None:
+ if cross_attn.shape[-1] != 768:
+ cross_attn = self.cc_projection(cross_attn)
+ out['c_crossattn'] = comfy.conds.CONDCrossAttn(cross_attn)
+ return out
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index 4f4e0b3b7..e3af422a3 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -24,7 +24,8 @@ def calculate_transformer_depth(prefix, state_dict_keys, state_dict):
last_transformer_depth = count_blocks(state_dict_keys, transformer_prefix + '{}')
context_dim = state_dict['{}0.attn2.to_k.weight'.format(transformer_prefix)].shape[1]
use_linear_in_transformer = len(state_dict['{}1.proj_in.weight'.format(prefix)].shape) == 2
- return last_transformer_depth, context_dim, use_linear_in_transformer
+ time_stack = '{}1.time_stack.0.attn1.to_q.weight'.format(prefix) in state_dict or '{}1.time_mix_blocks.0.attn1.to_q.weight'.format(prefix) in state_dict
+ return last_transformer_depth, context_dim, use_linear_in_transformer, time_stack
return None
def detect_unet_config(state_dict, key_prefix, dtype):
@@ -57,6 +58,7 @@ def detect_unet_config(state_dict, key_prefix, dtype):
context_dim = None
use_linear_in_transformer = False
+ video_model = False
current_res = 1
count = 0
@@ -99,6 +101,7 @@ def detect_unet_config(state_dict, key_prefix, dtype):
if context_dim is None:
context_dim = out[1]
use_linear_in_transformer = out[2]
+ video_model = out[3]
else:
transformer_depth.append(0)
@@ -127,6 +130,19 @@ def detect_unet_config(state_dict, key_prefix, dtype):
unet_config["transformer_depth_middle"] = transformer_depth_middle
unet_config['use_linear_in_transformer'] = use_linear_in_transformer
unet_config["context_dim"] = context_dim
+
+ if video_model:
+ unet_config["extra_ff_mix_layer"] = True
+ unet_config["use_spatial_context"] = True
+ unet_config["merge_strategy"] = "learned_with_images"
+ unet_config["merge_factor"] = 0.0
+ unet_config["video_kernel_size"] = [3, 1, 1]
+ unet_config["use_temporal_resblock"] = True
+ unet_config["use_temporal_attention"] = True
+ else:
+ unet_config["use_temporal_resblock"] = False
+ unet_config["use_temporal_attention"] = False
+
return unet_config
def model_config_from_unet_config(unet_config):
@@ -186,17 +202,24 @@ def convert_config(unet_config):
def unet_config_from_diffusers_unet(state_dict, dtype):
match = {}
- attention_resolutions = []
+ transformer_depth = []
attn_res = 1
- for i in range(5):
- k = "down_blocks.{}.attentions.1.transformer_blocks.0.attn2.to_k.weight".format(i)
- if k in state_dict:
- match["context_dim"] = state_dict[k].shape[1]
- attention_resolutions.append(attn_res)
- attn_res *= 2
+ down_blocks = count_blocks(state_dict, "down_blocks.{}")
+ for i in range(down_blocks):
+ attn_blocks = count_blocks(state_dict, "down_blocks.{}.attentions.".format(i) + '{}')
+ for ab in range(attn_blocks):
+ transformer_count = count_blocks(state_dict, "down_blocks.{}.attentions.{}.transformer_blocks.".format(i, ab) + '{}')
+ transformer_depth.append(transformer_count)
+ if transformer_count > 0:
+ match["context_dim"] = state_dict["down_blocks.{}.attentions.{}.transformer_blocks.0.attn2.to_k.weight".format(i, ab)].shape[1]
- match["attention_resolutions"] = attention_resolutions
+ attn_res *= 2
+ if attn_blocks == 0:
+ transformer_depth.append(0)
+ transformer_depth.append(0)
+
+ match["transformer_depth"] = transformer_depth
match["model_channels"] = state_dict["conv_in.weight"].shape[0]
match["in_channels"] = state_dict["conv_in.weight"].shape[1]
@@ -208,50 +231,71 @@ def unet_config_from_diffusers_unet(state_dict, dtype):
SDXL = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
- 'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 2, 10], 'channel_mult': [1, 2, 4],
- 'transformer_depth_middle': 10, 'use_linear_in_transformer': True, 'context_dim': 2048, "num_head_channels": 64}
+ 'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 2, 2, 10, 10], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 10,
+ 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 0, 2, 2, 2, 10, 10, 10],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
SDXL_refiner = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
'num_classes': 'sequential', 'adm_in_channels': 2560, 'dtype': dtype, 'in_channels': 4, 'model_channels': 384,
- 'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 4, 4, 0], 'channel_mult': [1, 2, 4, 4],
- 'transformer_depth_middle': 4, 'use_linear_in_transformer': True, 'context_dim': 1280, "num_head_channels": 64}
+ 'num_res_blocks': [2, 2, 2, 2], 'transformer_depth': [0, 0, 4, 4, 4, 4, 0, 0], 'channel_mult': [1, 2, 4, 4], 'transformer_depth_middle': 4,
+ 'use_linear_in_transformer': True, 'context_dim': 1280, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 0, 4, 4, 4, 4, 4, 4, 0, 0, 0],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
SD21 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
- 'adm_in_channels': None, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,
- 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
- 'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024, "num_head_channels": 64}
+ 'adm_in_channels': None, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': [2, 2, 2, 2],
+ 'transformer_depth': [1, 1, 1, 1, 1, 1, 0, 0], 'channel_mult': [1, 2, 4, 4], 'transformer_depth_middle': 1, 'use_linear_in_transformer': True,
+ 'context_dim': 1024, 'num_head_channels': 64, 'transformer_depth_output': [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
SD21_uncliph = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
'num_classes': 'sequential', 'adm_in_channels': 2048, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
- 'num_res_blocks': 2, 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
- 'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024, "num_head_channels": 64}
+ 'num_res_blocks': [2, 2, 2, 2], 'transformer_depth': [1, 1, 1, 1, 1, 1, 0, 0], 'channel_mult': [1, 2, 4, 4], 'transformer_depth_middle': 1,
+ 'use_linear_in_transformer': True, 'context_dim': 1024, 'num_head_channels': 64, 'transformer_depth_output': [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
SD21_unclipl = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
'num_classes': 'sequential', 'adm_in_channels': 1536, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
- 'num_res_blocks': 2, 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
- 'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024}
+ 'num_res_blocks': [2, 2, 2, 2], 'transformer_depth': [1, 1, 1, 1, 1, 1, 0, 0], 'channel_mult': [1, 2, 4, 4], 'transformer_depth_middle': 1,
+ 'use_linear_in_transformer': True, 'context_dim': 1024, 'num_head_channels': 64, 'transformer_depth_output': [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
- SD15 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
- 'adm_in_channels': None, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,
- 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
- 'transformer_depth_middle': 1, 'use_linear_in_transformer': False, 'context_dim': 768, "num_heads": 8}
+ SD15 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False, 'adm_in_channels': None,
+ 'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': [2, 2, 2, 2], 'transformer_depth': [1, 1, 1, 1, 1, 1, 0, 0],
+ 'channel_mult': [1, 2, 4, 4], 'transformer_depth_middle': 1, 'use_linear_in_transformer': False, 'context_dim': 768, 'num_heads': 8,
+ 'transformer_depth_output': [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
SDXL_mid_cnet = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
- 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
- 'num_res_blocks': 2, 'attention_resolutions': [4], 'transformer_depth': [0, 0, 1], 'channel_mult': [1, 2, 4],
- 'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 2048, "num_head_channels": 64}
+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
+ 'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 0, 0, 1, 1], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 1,
+ 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 0, 0, 0, 0, 1, 1, 1],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
SDXL_small_cnet = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
- 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
- 'num_res_blocks': 2, 'attention_resolutions': [], 'transformer_depth': [0, 0, 0], 'channel_mult': [1, 2, 4],
- 'transformer_depth_middle': 0, 'use_linear_in_transformer': True, "num_head_channels": 64, 'context_dim': 1}
+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
+ 'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 0, 0, 0, 0], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 0,
+ 'use_linear_in_transformer': True, 'num_head_channels': 64, 'context_dim': 1, 'transformer_depth_output': [0, 0, 0, 0, 0, 0, 0, 0, 0],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
SDXL_diffusers_inpaint = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
- 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 9, 'model_channels': 320,
- 'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 2, 10], 'channel_mult': [1, 2, 4],
- 'transformer_depth_middle': 10, 'use_linear_in_transformer': True, 'context_dim': 2048, "num_head_channels": 64}
+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 9, 'model_channels': 320,
+ 'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 2, 2, 10, 10], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 10,
+ 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 0, 2, 2, 2, 10, 10, 10],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
- supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint]
+ SSD_1B = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
+ 'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 2, 2, 4, 4], 'transformer_depth_output': [0, 0, 0, 1, 1, 2, 10, 4, 4],
+ 'channel_mult': [1, 2, 4], 'transformer_depth_middle': -1, 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64,
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
+
+ Segmind_Vega = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
+ 'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 1, 1, 2, 2], 'transformer_depth_output': [0, 0, 0, 1, 1, 1, 2, 2, 2],
+ 'channel_mult': [1, 2, 4], 'transformer_depth_middle': -1, 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64,
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
+
+ supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint, SSD_1B, Segmind_Vega]
for unet_config in supported_models:
matches = True
diff --git a/comfy/model_management.py b/comfy/model_management.py
index 53582fc73..fefd3c8c9 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -28,6 +28,10 @@ total_vram = 0
lowvram_available = True
xpu_available = False
+if args.deterministic:
+ print("Using deterministic algorithms for pytorch")
+ torch.use_deterministic_algorithms(True, warn_only=True)
+
directml_enabled = False
if args.directml is not None:
import torch_directml
@@ -133,6 +137,10 @@ else:
import xformers
import xformers.ops
XFORMERS_IS_AVAILABLE = True
+ try:
+ XFORMERS_IS_AVAILABLE = xformers._has_cpp_library
+ except:
+ pass
try:
XFORMERS_VERSION = xformers.version.__version__
print("xformers version:", XFORMERS_VERSION)
@@ -178,6 +186,9 @@ except:
if is_intel_xpu():
VAE_DTYPE = torch.bfloat16
+if args.cpu_vae:
+ VAE_DTYPE = torch.float32
+
if args.fp16_vae:
VAE_DTYPE = torch.float16
elif args.bf16_vae:
@@ -210,15 +221,8 @@ if args.force_fp16:
FORCE_FP16 = True
if lowvram_available:
- try:
- import accelerate
- if set_vram_to in (VRAMState.LOW_VRAM, VRAMState.NO_VRAM):
- vram_state = set_vram_to
- except Exception as e:
- import traceback
- print(traceback.format_exc())
- print("ERROR: LOW VRAM MODE NEEDS accelerate.")
- lowvram_available = False
+ if set_vram_to in (VRAMState.LOW_VRAM, VRAMState.NO_VRAM):
+ vram_state = set_vram_to
if cpu_state != CPUState.GPU:
@@ -258,6 +262,14 @@ print("VAE dtype:", VAE_DTYPE)
current_loaded_models = []
+def module_size(module):
+ module_mem = 0
+ sd = module.state_dict()
+ for k in sd:
+ t = sd[k]
+ module_mem += t.nelement() * t.element_size()
+ return module_mem
+
class LoadedModel:
def __init__(self, model):
self.model = model
@@ -290,8 +302,20 @@ class LoadedModel:
if lowvram_model_memory > 0:
print("loading in lowvram mode", lowvram_model_memory/(1024 * 1024))
- device_map = accelerate.infer_auto_device_map(self.real_model, max_memory={0: "{}MiB".format(lowvram_model_memory // (1024 * 1024)), "cpu": "16GiB"})
- accelerate.dispatch_model(self.real_model, device_map=device_map, main_device=self.device)
+ mem_counter = 0
+ for m in self.real_model.modules():
+ if hasattr(m, "comfy_cast_weights"):
+ m.prev_comfy_cast_weights = m.comfy_cast_weights
+ m.comfy_cast_weights = True
+ module_mem = module_size(m)
+ if mem_counter + module_mem < lowvram_model_memory:
+ m.to(self.device)
+ mem_counter += module_mem
+ elif hasattr(m, "weight"): #only modules with comfy_cast_weights can be set to lowvram mode
+ m.to(self.device)
+ mem_counter += module_size(m)
+ print("lowvram: loaded module regularly", m)
+
self.model_accelerated = True
if is_intel_xpu() and not args.disable_ipex_optimize:
@@ -301,7 +325,11 @@ class LoadedModel:
def model_unload(self):
if self.model_accelerated:
- accelerate.hooks.remove_hook_from_submodules(self.real_model)
+ for m in self.real_model.modules():
+ if hasattr(m, "prev_comfy_cast_weights"):
+ m.comfy_cast_weights = m.prev_comfy_cast_weights
+ del m.prev_comfy_cast_weights
+
self.model_accelerated = False
self.model.unpatch_model(self.model.offload_device)
@@ -394,14 +422,14 @@ def load_models_gpu(models, memory_required=0):
if lowvram_available and (vram_set_state == VRAMState.LOW_VRAM or vram_set_state == VRAMState.NORMAL_VRAM):
model_size = loaded_model.model_memory_required(torch_dev)
current_free_mem = get_free_memory(torch_dev)
- lowvram_model_memory = int(max(256 * (1024 * 1024), (current_free_mem - 1024 * (1024 * 1024)) / 1.3 ))
+ lowvram_model_memory = int(max(64 * (1024 * 1024), (current_free_mem - 1024 * (1024 * 1024)) / 1.3 ))
if model_size > (current_free_mem - inference_memory): #only switch to lowvram if really necessary
vram_set_state = VRAMState.LOW_VRAM
else:
lowvram_model_memory = 0
if vram_set_state == VRAMState.NO_VRAM:
- lowvram_model_memory = 256 * 1024 * 1024
+ lowvram_model_memory = 64 * 1024 * 1024
cur_loaded_model = loaded_model.model_load(lowvram_model_memory)
current_loaded_models.insert(0, loaded_model)
@@ -426,6 +454,13 @@ def dtype_size(dtype):
dtype_size = 4
if dtype == torch.float16 or dtype == torch.bfloat16:
dtype_size = 2
+ elif dtype == torch.float32:
+ dtype_size = 4
+ else:
+ try:
+ dtype_size = dtype.itemsize
+ except: #Old pytorch doesn't have .itemsize
+ pass
return dtype_size
def unet_offload_device():
@@ -455,10 +490,30 @@ def unet_inital_load_device(parameters, dtype):
def unet_dtype(device=None, model_params=0):
if args.bf16_unet:
return torch.bfloat16
+ if args.fp16_unet:
+ return torch.float16
+ if args.fp8_e4m3fn_unet:
+ return torch.float8_e4m3fn
+ if args.fp8_e5m2_unet:
+ return torch.float8_e5m2
if should_use_fp16(device=device, model_params=model_params):
return torch.float16
return torch.float32
+# None means no manual cast
+def unet_manual_cast(weight_dtype, inference_device):
+ if weight_dtype == torch.float32:
+ return None
+
+ fp16_supported = comfy.model_management.should_use_fp16(inference_device, prioritize_performance=False)
+ if fp16_supported and weight_dtype == torch.float16:
+ return None
+
+ if fp16_supported:
+ return torch.float16
+ else:
+ return torch.float32
+
def text_encoder_offload_device():
if args.gpu_only:
return get_torch_device()
@@ -478,7 +533,33 @@ def text_encoder_device():
else:
return torch.device("cpu")
+def text_encoder_dtype(device=None):
+ if args.fp8_e4m3fn_text_enc:
+ return torch.float8_e4m3fn
+ elif args.fp8_e5m2_text_enc:
+ return torch.float8_e5m2
+ elif args.fp16_text_enc:
+ return torch.float16
+ elif args.fp32_text_enc:
+ return torch.float32
+
+ if is_device_cpu(device):
+ return torch.float16
+
+ if should_use_fp16(device, prioritize_performance=False):
+ return torch.float16
+ else:
+ return torch.float32
+
+def intermediate_device():
+ if args.gpu_only:
+ return get_torch_device()
+ else:
+ return torch.device("cpu")
+
def vae_device():
+ if args.cpu_vae:
+ return torch.device("cpu")
return get_torch_device()
def vae_offload_device():
@@ -496,6 +577,22 @@ def get_autocast_device(dev):
return dev.type
return "cuda"
+def supports_dtype(device, dtype): #TODO
+ if dtype == torch.float32:
+ return True
+ if is_device_cpu(device):
+ return False
+ if dtype == torch.float16:
+ return True
+ if dtype == torch.bfloat16:
+ return True
+ return False
+
+def device_supports_non_blocking(device):
+ if is_device_mps(device):
+ return False #pytorch bug? mps doesn't support non blocking
+ return True
+
def cast_to_device(tensor, device, dtype, copy=False):
device_supports_cast = False
if tensor.dtype == torch.float32 or tensor.dtype == torch.float16:
@@ -506,15 +603,17 @@ def cast_to_device(tensor, device, dtype, copy=False):
elif is_intel_xpu():
device_supports_cast = True
+ non_blocking = device_supports_non_blocking(device)
+
if device_supports_cast:
if copy:
if tensor.device == device:
- return tensor.to(dtype, copy=copy)
- return tensor.to(device, copy=copy).to(dtype)
+ return tensor.to(dtype, copy=copy, non_blocking=non_blocking)
+ return tensor.to(device, copy=copy, non_blocking=non_blocking).to(dtype, non_blocking=non_blocking)
else:
- return tensor.to(device).to(dtype)
+ return tensor.to(device, non_blocking=non_blocking).to(dtype, non_blocking=non_blocking)
else:
- return tensor.to(dtype).to(device, copy=copy)
+ return tensor.to(device, dtype, copy=copy, non_blocking=non_blocking)
def xformers_enabled():
global directml_enabled
@@ -579,27 +678,6 @@ def get_free_memory(dev=None, torch_free_too=False):
else:
return mem_free_total
-def batch_area_memory(area):
- if xformers_enabled() or pytorch_attention_flash_attention():
- #TODO: these formulas are copied from maximum_batch_area below
- return (area / 20) * (1024 * 1024)
- else:
- return (((area * 0.6) / 0.9) + 1024) * (1024 * 1024)
-
-def maximum_batch_area():
- global vram_state
- if vram_state == VRAMState.NO_VRAM:
- return 0
-
- memory_free = get_free_memory() / (1024 * 1024)
- if xformers_enabled() or pytorch_attention_flash_attention():
- #TODO: this needs to be tweaked
- area = 20 * memory_free
- else:
- #TODO: this formula is because AMD sucks and has memory management issues which might be fixed in the future
- area = ((memory_free - 1024) * 0.9) / (0.6)
- return int(max(area, 0))
-
def cpu_mode():
global cpu_state
return cpu_state == CPUState.CPU
@@ -689,11 +767,11 @@ def soft_empty_cache(force=False):
torch.cuda.empty_cache()
torch.cuda.ipc_collect()
-def resolve_lowvram_weight(weight, model, key):
- if weight.device == torch.device("meta"): #lowvram NOTE: this depends on the inner working of the accelerate library so it might break.
- key_split = key.split('.') # I have no idea why they don't just leave the weight there instead of using the meta device.
- op = comfy.utils.get_attr(model, '.'.join(key_split[:-1]))
- weight = op._hf_hook.weights_map[key_split[-1]]
+def unload_all_models():
+ free_memory(1e30, get_torch_device())
+
+
+def resolve_lowvram_weight(weight, model, key): #TODO: remove
return weight
#TODO: might be cleaner to put this somewhere else
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py
index 6d7a61c41..b1b5ea6a8 100644
--- a/comfy/model_patcher.py
+++ b/comfy/model_patcher.py
@@ -28,16 +28,12 @@ class ModelPatcher:
if self.size > 0:
return self.size
model_sd = self.model.state_dict()
- size = 0
- for k in model_sd:
- t = model_sd[k]
- size += t.nelement() * t.element_size()
- self.size = size
+ self.size = comfy.model_management.module_size(self.model)
self.model_keys = set(model_sd.keys())
- return size
+ return self.size
def clone(self):
- n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size, self.current_device)
+ n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size, self.current_device, weight_inplace_update=self.weight_inplace_update)
n.patches = {}
for k in self.patches:
n.patches[k] = self.patches[k][:]
@@ -52,11 +48,21 @@ class ModelPatcher:
return True
return False
- def set_model_sampler_cfg_function(self, sampler_cfg_function):
+ def memory_required(self, input_shape):
+ return self.model.memory_required(input_shape=input_shape)
+
+ def set_model_sampler_cfg_function(self, sampler_cfg_function, disable_cfg1_optimization=False):
if len(inspect.signature(sampler_cfg_function).parameters) == 3:
self.model_options["sampler_cfg_function"] = lambda args: sampler_cfg_function(args["cond"], args["uncond"], args["cond_scale"]) #Old way
else:
self.model_options["sampler_cfg_function"] = sampler_cfg_function
+ if disable_cfg1_optimization:
+ self.model_options["disable_cfg1_optimization"] = True
+
+ def set_model_sampler_post_cfg_function(self, post_cfg_function, disable_cfg1_optimization=False):
+ self.model_options["sampler_post_cfg_function"] = self.model_options.get("sampler_post_cfg_function", []) + [post_cfg_function]
+ if disable_cfg1_optimization:
+ self.model_options["disable_cfg1_optimization"] = True
def set_model_unet_function_wrapper(self, unet_wrapper_function):
self.model_options["model_function_wrapper"] = unet_wrapper_function
@@ -67,13 +73,17 @@ class ModelPatcher:
to["patches"] = {}
to["patches"][name] = to["patches"].get(name, []) + [patch]
- def set_model_patch_replace(self, patch, name, block_name, number):
+ def set_model_patch_replace(self, patch, name, block_name, number, transformer_index=None):
to = self.model_options["transformer_options"]
if "patches_replace" not in to:
to["patches_replace"] = {}
if name not in to["patches_replace"]:
to["patches_replace"][name] = {}
- to["patches_replace"][name][(block_name, number)] = patch
+ if transformer_index is not None:
+ block = (block_name, number, transformer_index)
+ else:
+ block = (block_name, number)
+ to["patches_replace"][name][block] = patch
def set_model_attn1_patch(self, patch):
self.set_model_patch(patch, "attn1_patch")
@@ -81,11 +91,11 @@ class ModelPatcher:
def set_model_attn2_patch(self, patch):
self.set_model_patch(patch, "attn2_patch")
- def set_model_attn1_replace(self, patch, block_name, number):
- self.set_model_patch_replace(patch, "attn1", block_name, number)
+ def set_model_attn1_replace(self, patch, block_name, number, transformer_index=None):
+ self.set_model_patch_replace(patch, "attn1", block_name, number, transformer_index)
- def set_model_attn2_replace(self, patch, block_name, number):
- self.set_model_patch_replace(patch, "attn2", block_name, number)
+ def set_model_attn2_replace(self, patch, block_name, number, transformer_index=None):
+ self.set_model_patch_replace(patch, "attn2", block_name, number, transformer_index)
def set_model_attn1_output_patch(self, patch):
self.set_model_patch(patch, "attn1_output_patch")
@@ -93,6 +103,12 @@ class ModelPatcher:
def set_model_attn2_output_patch(self, patch):
self.set_model_patch(patch, "attn2_output_patch")
+ def set_model_input_block_patch(self, patch):
+ self.set_model_patch(patch, "input_block_patch")
+
+ def set_model_input_block_patch_after_skip(self, patch):
+ self.set_model_patch(patch, "input_block_patch_after_skip")
+
def set_model_output_block_patch(self, patch):
self.set_model_patch(patch, "output_block_patch")
@@ -176,7 +192,7 @@ class ModelPatcher:
inplace_update = self.weight_inplace_update
if key not in self.backup:
- self.backup[key] = weight.to(device=device_to, copy=inplace_update)
+ self.backup[key] = weight.to(device=self.offload_device, copy=inplace_update)
if device_to is not None:
temp_weight = comfy.model_management.cast_to_device(weight, device_to, torch.float32, copy=True)
@@ -208,13 +224,19 @@ class ModelPatcher:
v = (self.calculate_weight(v[1:], v[0].clone(), key), )
if len(v) == 1:
+ patch_type = "diff"
+ elif len(v) == 2:
+ patch_type = v[0]
+ v = v[1]
+
+ if patch_type == "diff":
w1 = v[0]
if alpha != 0.0:
if w1.shape != weight.shape:
print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
else:
weight += alpha * comfy.model_management.cast_to_device(w1, weight.device, weight.dtype)
- elif len(v) == 4: #lora/locon
+ elif patch_type == "lora": #lora/locon
mat1 = comfy.model_management.cast_to_device(v[0], weight.device, torch.float32)
mat2 = comfy.model_management.cast_to_device(v[1], weight.device, torch.float32)
if v[2] is not None:
@@ -228,7 +250,7 @@ class ModelPatcher:
weight += (alpha * torch.mm(mat1.flatten(start_dim=1), mat2.flatten(start_dim=1))).reshape(weight.shape).type(weight.dtype)
except Exception as e:
print("ERROR", key, e)
- elif len(v) == 8: #lokr
+ elif patch_type == "lokr":
w1 = v[0]
w2 = v[1]
w1_a = v[3]
@@ -267,7 +289,7 @@ class ModelPatcher:
weight += alpha * torch.kron(w1, w2).reshape(weight.shape).type(weight.dtype)
except Exception as e:
print("ERROR", key, e)
- else: #loha
+ elif patch_type == "loha":
w1a = v[0]
w1b = v[1]
if v[2] is not None:
@@ -296,6 +318,18 @@ class ModelPatcher:
weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype)
except Exception as e:
print("ERROR", key, e)
+ elif patch_type == "glora":
+ if v[4] is not None:
+ alpha *= v[4] / v[0].shape[0]
+
+ a1 = comfy.model_management.cast_to_device(v[0].flatten(start_dim=1), weight.device, torch.float32)
+ a2 = comfy.model_management.cast_to_device(v[1].flatten(start_dim=1), weight.device, torch.float32)
+ b1 = comfy.model_management.cast_to_device(v[2].flatten(start_dim=1), weight.device, torch.float32)
+ b2 = comfy.model_management.cast_to_device(v[3].flatten(start_dim=1), weight.device, torch.float32)
+
+ weight += ((torch.mm(b2, b1) + torch.mm(torch.mm(weight.flatten(start_dim=1), a2), a1)) * alpha).reshape(weight.shape).type(weight.dtype)
+ else:
+ print("patch type not recognized", patch_type, key)
return weight
diff --git a/comfy/model_sampling.py b/comfy/model_sampling.py
index a2935d47d..cc8745c10 100644
--- a/comfy/model_sampling.py
+++ b/comfy/model_sampling.py
@@ -1,7 +1,7 @@
import torch
import numpy as np
from comfy.ldm.modules.diffusionmodules.util import make_beta_schedule
-
+import math
class EPS:
def calculate_input(self, sigma, noise):
@@ -22,10 +22,17 @@ class V_PREDICTION(EPS):
class ModelSamplingDiscrete(torch.nn.Module):
def __init__(self, model_config=None):
super().__init__()
- beta_schedule = "linear"
+
if model_config is not None:
- beta_schedule = model_config.beta_schedule
- self._register_schedule(given_betas=None, beta_schedule=beta_schedule, timesteps=1000, linear_start=0.00085, linear_end=0.012, cosine_s=8e-3)
+ sampling_settings = model_config.sampling_settings
+ else:
+ sampling_settings = {}
+
+ beta_schedule = sampling_settings.get("beta_schedule", "linear")
+ linear_start = sampling_settings.get("linear_start", 0.00085)
+ linear_end = sampling_settings.get("linear_end", 0.012)
+
+ self._register_schedule(given_betas=None, beta_schedule=beta_schedule, timesteps=1000, linear_start=linear_start, linear_end=linear_end, cosine_s=8e-3)
self.sigma_data = 1.0
def _register_schedule(self, given_betas=None, beta_schedule="linear", timesteps=1000,
@@ -65,16 +72,65 @@ class ModelSamplingDiscrete(torch.nn.Module):
def timestep(self, sigma):
log_sigma = sigma.log()
dists = log_sigma.to(self.log_sigmas.device) - self.log_sigmas[:, None]
- return dists.abs().argmin(dim=0).view(sigma.shape)
+ return dists.abs().argmin(dim=0).view(sigma.shape).to(sigma.device)
def sigma(self, timestep):
- t = torch.clamp(timestep.float(), min=0, max=(len(self.sigmas) - 1))
+ t = torch.clamp(timestep.float().to(self.log_sigmas.device), min=0, max=(len(self.sigmas) - 1))
low_idx = t.floor().long()
high_idx = t.ceil().long()
w = t.frac()
log_sigma = (1 - w) * self.log_sigmas[low_idx] + w * self.log_sigmas[high_idx]
- return log_sigma.exp()
+ return log_sigma.exp().to(timestep.device)
def percent_to_sigma(self, percent):
- return self.sigma(torch.tensor(percent * 999.0))
+ if percent <= 0.0:
+ return 999999999.9
+ if percent >= 1.0:
+ return 0.0
+ percent = 1.0 - percent
+ return self.sigma(torch.tensor(percent * 999.0)).item()
+
+class ModelSamplingContinuousEDM(torch.nn.Module):
+ def __init__(self, model_config=None):
+ super().__init__()
+ self.sigma_data = 1.0
+
+ if model_config is not None:
+ sampling_settings = model_config.sampling_settings
+ else:
+ sampling_settings = {}
+
+ sigma_min = sampling_settings.get("sigma_min", 0.002)
+ sigma_max = sampling_settings.get("sigma_max", 120.0)
+ self.set_sigma_range(sigma_min, sigma_max)
+
+ def set_sigma_range(self, sigma_min, sigma_max):
+ sigmas = torch.linspace(math.log(sigma_min), math.log(sigma_max), 1000).exp()
+
+ self.register_buffer('sigmas', sigmas) #for compatibility with some schedulers
+ self.register_buffer('log_sigmas', sigmas.log())
+
+ @property
+ def sigma_min(self):
+ return self.sigmas[0]
+
+ @property
+ def sigma_max(self):
+ return self.sigmas[-1]
+
+ def timestep(self, sigma):
+ return 0.25 * sigma.log()
+
+ def sigma(self, timestep):
+ return (timestep / 0.25).exp()
+
+ def percent_to_sigma(self, percent):
+ if percent <= 0.0:
+ return 999999999.9
+ if percent >= 1.0:
+ return 0.0
+ percent = 1.0 - percent
+
+ log_sigma_min = math.log(self.sigma_min)
+ return math.exp((math.log(self.sigma_max) - log_sigma_min) * percent + log_sigma_min)
diff --git a/comfy/ops.py b/comfy/ops.py
index 0bfb698aa..f6f85de60 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -1,40 +1,115 @@
import torch
from contextlib import contextmanager
+import comfy.model_management
-class Linear(torch.nn.Linear):
- def reset_parameters(self):
- return None
+def cast_bias_weight(s, input):
+ bias = None
+ non_blocking = comfy.model_management.device_supports_non_blocking(input.device)
+ if s.bias is not None:
+ bias = s.bias.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)
+ weight = s.weight.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)
+ return weight, bias
-class Conv2d(torch.nn.Conv2d):
- def reset_parameters(self):
- return None
-class Conv3d(torch.nn.Conv3d):
- def reset_parameters(self):
- return None
+class disable_weight_init:
+ class Linear(torch.nn.Linear):
+ comfy_cast_weights = False
+ def reset_parameters(self):
+ return None
-def conv_nd(dims, *args, **kwargs):
- if dims == 2:
- return Conv2d(*args, **kwargs)
- elif dims == 3:
- return Conv3d(*args, **kwargs)
- else:
- raise ValueError(f"unsupported dimensions: {dims}")
+ def forward_comfy_cast_weights(self, input):
+ weight, bias = cast_bias_weight(self, input)
+ return torch.nn.functional.linear(input, weight, bias)
-@contextmanager
-def use_comfy_ops(device=None, dtype=None): # Kind of an ugly hack but I can't think of a better way
- old_torch_nn_linear = torch.nn.Linear
- force_device = device
- force_dtype = dtype
- def linear_with_dtype(in_features: int, out_features: int, bias: bool = True, device=None, dtype=None):
- if force_device is not None:
- device = force_device
- if force_dtype is not None:
- dtype = force_dtype
- return Linear(in_features, out_features, bias=bias, device=device, dtype=dtype)
+ def forward(self, *args, **kwargs):
+ if self.comfy_cast_weights:
+ return self.forward_comfy_cast_weights(*args, **kwargs)
+ else:
+ return super().forward(*args, **kwargs)
- torch.nn.Linear = linear_with_dtype
- try:
- yield
- finally:
- torch.nn.Linear = old_torch_nn_linear
+ class Conv2d(torch.nn.Conv2d):
+ comfy_cast_weights = False
+ def reset_parameters(self):
+ return None
+
+ def forward_comfy_cast_weights(self, input):
+ weight, bias = cast_bias_weight(self, input)
+ return self._conv_forward(input, weight, bias)
+
+ def forward(self, *args, **kwargs):
+ if self.comfy_cast_weights:
+ return self.forward_comfy_cast_weights(*args, **kwargs)
+ else:
+ return super().forward(*args, **kwargs)
+
+ class Conv3d(torch.nn.Conv3d):
+ comfy_cast_weights = False
+ def reset_parameters(self):
+ return None
+
+ def forward_comfy_cast_weights(self, input):
+ weight, bias = cast_bias_weight(self, input)
+ return self._conv_forward(input, weight, bias)
+
+ def forward(self, *args, **kwargs):
+ if self.comfy_cast_weights:
+ return self.forward_comfy_cast_weights(*args, **kwargs)
+ else:
+ return super().forward(*args, **kwargs)
+
+ class GroupNorm(torch.nn.GroupNorm):
+ comfy_cast_weights = False
+ def reset_parameters(self):
+ return None
+
+ def forward_comfy_cast_weights(self, input):
+ weight, bias = cast_bias_weight(self, input)
+ return torch.nn.functional.group_norm(input, self.num_groups, weight, bias, self.eps)
+
+ def forward(self, *args, **kwargs):
+ if self.comfy_cast_weights:
+ return self.forward_comfy_cast_weights(*args, **kwargs)
+ else:
+ return super().forward(*args, **kwargs)
+
+
+ class LayerNorm(torch.nn.LayerNorm):
+ comfy_cast_weights = False
+ def reset_parameters(self):
+ return None
+
+ def forward_comfy_cast_weights(self, input):
+ weight, bias = cast_bias_weight(self, input)
+ return torch.nn.functional.layer_norm(input, self.normalized_shape, weight, bias, self.eps)
+
+ def forward(self, *args, **kwargs):
+ if self.comfy_cast_weights:
+ return self.forward_comfy_cast_weights(*args, **kwargs)
+ else:
+ return super().forward(*args, **kwargs)
+
+ @classmethod
+ def conv_nd(s, dims, *args, **kwargs):
+ if dims == 2:
+ return s.Conv2d(*args, **kwargs)
+ elif dims == 3:
+ return s.Conv3d(*args, **kwargs)
+ else:
+ raise ValueError(f"unsupported dimensions: {dims}")
+
+
+class manual_cast(disable_weight_init):
+ class Linear(disable_weight_init.Linear):
+ comfy_cast_weights = True
+
+ class Conv2d(disable_weight_init.Conv2d):
+ comfy_cast_weights = True
+
+ class Conv3d(disable_weight_init.Conv3d):
+ comfy_cast_weights = True
+
+ class GroupNorm(disable_weight_init.GroupNorm):
+ comfy_cast_weights = True
+
+ class LayerNorm(disable_weight_init.LayerNorm):
+ comfy_cast_weights = True
diff --git a/comfy/sample.py b/comfy/sample.py
index b3fcd1658..4b0d15c49 100644
--- a/comfy/sample.py
+++ b/comfy/sample.py
@@ -47,7 +47,8 @@ def convert_cond(cond):
temp = c[1].copy()
model_conds = temp.get("model_conds", {})
if c[0] is not None:
- model_conds["c_crossattn"] = comfy.conds.CONDCrossAttn(c[0])
+ model_conds["c_crossattn"] = comfy.conds.CONDCrossAttn(c[0]) #TODO: remove
+ temp["cross_attn"] = c[0]
temp["model_conds"] = model_conds
out.append(temp)
return out
@@ -83,7 +84,7 @@ def prepare_sampling(model, noise_shape, positive, negative, noise_mask):
real_model = None
models, inference_memory = get_additional_models(positive, negative, model.model_dtype())
- comfy.model_management.load_models_gpu([model] + models, comfy.model_management.batch_area_memory(noise_shape[0] * noise_shape[2] * noise_shape[3]) + inference_memory)
+ comfy.model_management.load_models_gpu([model] + models, model.memory_required([noise_shape[0] * 2] + list(noise_shape[1:])) + inference_memory)
real_model = model.model
return real_model, positive, negative, noise_mask, models
@@ -98,10 +99,10 @@ def sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative
sampler = comfy.samplers.KSampler(real_model, steps=steps, device=model.load_device, sampler=sampler_name, scheduler=scheduler, denoise=denoise, model_options=model.model_options)
samples = sampler.sample(noise, positive_copy, negative_copy, cfg=cfg, latent_image=latent_image, start_step=start_step, last_step=last_step, force_full_denoise=force_full_denoise, denoise_mask=noise_mask, sigmas=sigmas, callback=callback, disable_pbar=disable_pbar, seed=seed)
- samples = samples.cpu()
+ samples = samples.to(comfy.model_management.intermediate_device())
cleanup_additional_models(models)
- cleanup_additional_models(set(get_models_from_cond(positive, "control") + get_models_from_cond(negative, "control")))
+ cleanup_additional_models(set(get_models_from_cond(positive_copy, "control") + get_models_from_cond(negative_copy, "control")))
return samples
def sample_custom(model, noise, cfg, sampler, sigmas, positive, negative, latent_image, noise_mask=None, callback=None, disable_pbar=False, seed=None):
@@ -111,8 +112,8 @@ def sample_custom(model, noise, cfg, sampler, sigmas, positive, negative, latent
sigmas = sigmas.to(model.load_device)
samples = comfy.samplers.sample(real_model, noise, positive_copy, negative_copy, cfg, model.load_device, sampler, sigmas, model_options=model.model_options, latent_image=latent_image, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)
- samples = samples.cpu()
+ samples = samples.to(comfy.model_management.intermediate_device())
cleanup_additional_models(models)
- cleanup_additional_models(set(get_models_from_cond(positive, "control") + get_models_from_cond(negative, "control")))
+ cleanup_additional_models(set(get_models_from_cond(positive_copy, "control") + get_models_from_cond(negative_copy, "control")))
return samples
diff --git a/comfy/samplers.py b/comfy/samplers.py
index a839ee9e2..0453c1f6f 100644
--- a/comfy/samplers.py
+++ b/comfy/samplers.py
@@ -2,263 +2,274 @@ from .k_diffusion import sampling as k_diffusion_sampling
from .extra_samplers import uni_pc
import torch
import enum
+import collections
from comfy import model_management
import math
from comfy import model_base
import comfy.utils
import comfy.conds
+def get_area_and_mult(conds, x_in, timestep_in):
+ area = (x_in.shape[2], x_in.shape[3], 0, 0)
+ strength = 1.0
+
+ if 'timestep_start' in conds:
+ timestep_start = conds['timestep_start']
+ if timestep_in[0] > timestep_start:
+ return None
+ if 'timestep_end' in conds:
+ timestep_end = conds['timestep_end']
+ if timestep_in[0] < timestep_end:
+ return None
+ if 'area' in conds:
+ area = conds['area']
+ if 'strength' in conds:
+ strength = conds['strength']
+
+ input_x = x_in[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]]
+ if 'mask' in conds:
+ # Scale the mask to the size of the input
+ # The mask should have been resized as we began the sampling process
+ mask_strength = 1.0
+ if "mask_strength" in conds:
+ mask_strength = conds["mask_strength"]
+ mask = conds['mask']
+ assert(mask.shape[1] == x_in.shape[2])
+ assert(mask.shape[2] == x_in.shape[3])
+ mask = mask[:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] * mask_strength
+ mask = mask.unsqueeze(1).repeat(input_x.shape[0] // mask.shape[0], input_x.shape[1], 1, 1)
+ else:
+ mask = torch.ones_like(input_x)
+ mult = mask * strength
+
+ if 'mask' not in conds:
+ rr = 8
+ if area[2] != 0:
+ for t in range(rr):
+ mult[:,:,t:1+t,:] *= ((1.0/rr) * (t + 1))
+ if (area[0] + area[2]) < x_in.shape[2]:
+ for t in range(rr):
+ mult[:,:,area[0] - 1 - t:area[0] - t,:] *= ((1.0/rr) * (t + 1))
+ if area[3] != 0:
+ for t in range(rr):
+ mult[:,:,:,t:1+t] *= ((1.0/rr) * (t + 1))
+ if (area[1] + area[3]) < x_in.shape[3]:
+ for t in range(rr):
+ mult[:,:,:,area[1] - 1 - t:area[1] - t] *= ((1.0/rr) * (t + 1))
+
+ conditioning = {}
+ model_conds = conds["model_conds"]
+ for c in model_conds:
+ conditioning[c] = model_conds[c].process_cond(batch_size=x_in.shape[0], device=x_in.device, area=area)
+
+ control = conds.get('control', None)
+
+ patches = None
+ if 'gligen' in conds:
+ gligen = conds['gligen']
+ patches = {}
+ gligen_type = gligen[0]
+ gligen_model = gligen[1]
+ if gligen_type == "position":
+ gligen_patch = gligen_model.model.set_position(input_x.shape, gligen[2], input_x.device)
+ else:
+ gligen_patch = gligen_model.model.set_empty(input_x.shape, input_x.device)
+
+ patches['middle_patch'] = [gligen_patch]
+
+ cond_obj = collections.namedtuple('cond_obj', ['input_x', 'mult', 'conditioning', 'area', 'control', 'patches'])
+ return cond_obj(input_x, mult, conditioning, area, control, patches)
+
+def cond_equal_size(c1, c2):
+ if c1 is c2:
+ return True
+ if c1.keys() != c2.keys():
+ return False
+ for k in c1:
+ if not c1[k].can_concat(c2[k]):
+ return False
+ return True
+
+def can_concat_cond(c1, c2):
+ if c1.input_x.shape != c2.input_x.shape:
+ return False
+
+ def objects_concatable(obj1, obj2):
+ if (obj1 is None) != (obj2 is None):
+ return False
+ if obj1 is not None:
+ if obj1 is not obj2:
+ return False
+ return True
+
+ if not objects_concatable(c1.control, c2.control):
+ return False
+
+ if not objects_concatable(c1.patches, c2.patches):
+ return False
+
+ return cond_equal_size(c1.conditioning, c2.conditioning)
+
+def cond_cat(c_list):
+ c_crossattn = []
+ c_concat = []
+ c_adm = []
+ crossattn_max_len = 0
+
+ temp = {}
+ for x in c_list:
+ for k in x:
+ cur = temp.get(k, [])
+ cur.append(x[k])
+ temp[k] = cur
+
+ out = {}
+ for k in temp:
+ conds = temp[k]
+ out[k] = conds[0].concat(conds[1:])
+
+ return out
+
+def calc_cond_uncond_batch(model, cond, uncond, x_in, timestep, model_options):
+ out_cond = torch.zeros_like(x_in)
+ out_count = torch.ones_like(x_in) * 1e-37
+
+ out_uncond = torch.zeros_like(x_in)
+ out_uncond_count = torch.ones_like(x_in) * 1e-37
+
+ COND = 0
+ UNCOND = 1
+
+ to_run = []
+ for x in cond:
+ p = get_area_and_mult(x, x_in, timestep)
+ if p is None:
+ continue
+
+ to_run += [(p, COND)]
+ if uncond is not None:
+ for x in uncond:
+ p = get_area_and_mult(x, x_in, timestep)
+ if p is None:
+ continue
+
+ to_run += [(p, UNCOND)]
+
+ while len(to_run) > 0:
+ first = to_run[0]
+ first_shape = first[0][0].shape
+ to_batch_temp = []
+ for x in range(len(to_run)):
+ if can_concat_cond(to_run[x][0], first[0]):
+ to_batch_temp += [x]
+
+ to_batch_temp.reverse()
+ to_batch = to_batch_temp[:1]
+
+ free_memory = model_management.get_free_memory(x_in.device)
+ for i in range(1, len(to_batch_temp) + 1):
+ batch_amount = to_batch_temp[:len(to_batch_temp)//i]
+ input_shape = [len(batch_amount) * first_shape[0]] + list(first_shape)[1:]
+ if model.memory_required(input_shape) < free_memory:
+ to_batch = batch_amount
+ break
+
+ input_x = []
+ mult = []
+ c = []
+ cond_or_uncond = []
+ area = []
+ control = None
+ patches = None
+ for x in to_batch:
+ o = to_run.pop(x)
+ p = o[0]
+ input_x.append(p.input_x)
+ mult.append(p.mult)
+ c.append(p.conditioning)
+ area.append(p.area)
+ cond_or_uncond.append(o[1])
+ control = p.control
+ patches = p.patches
+
+ batch_chunks = len(cond_or_uncond)
+ input_x = torch.cat(input_x)
+ c = cond_cat(c)
+ timestep_ = torch.cat([timestep] * batch_chunks)
+
+ if control is not None:
+ c['control'] = control.get_control(input_x, timestep_, c, len(cond_or_uncond))
+
+ transformer_options = {}
+ if 'transformer_options' in model_options:
+ transformer_options = model_options['transformer_options'].copy()
+
+ if patches is not None:
+ if "patches" in transformer_options:
+ cur_patches = transformer_options["patches"].copy()
+ for p in patches:
+ if p in cur_patches:
+ cur_patches[p] = cur_patches[p] + patches[p]
+ else:
+ cur_patches[p] = patches[p]
+ else:
+ transformer_options["patches"] = patches
+
+ transformer_options["cond_or_uncond"] = cond_or_uncond[:]
+ transformer_options["sigmas"] = timestep
+
+ c['transformer_options'] = transformer_options
+
+ if 'model_function_wrapper' in model_options:
+ output = model_options['model_function_wrapper'](model.apply_model, {"input": input_x, "timestep": timestep_, "c": c, "cond_or_uncond": cond_or_uncond}).chunk(batch_chunks)
+ else:
+ output = model.apply_model(input_x, timestep_, **c).chunk(batch_chunks)
+ del input_x
+
+ for o in range(batch_chunks):
+ if cond_or_uncond[o] == COND:
+ out_cond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
+ out_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
+ else:
+ out_uncond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
+ out_uncond_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
+ del mult
+
+ out_cond /= out_count
+ del out_count
+ out_uncond /= out_uncond_count
+ del out_uncond_count
+ return out_cond, out_uncond
#The main sampling function shared by all the samplers
#Returns denoised
-def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, model_options={}, seed=None):
- def get_area_and_mult(conds, x_in, timestep_in):
- area = (x_in.shape[2], x_in.shape[3], 0, 0)
- strength = 1.0
-
- if 'timestep_start' in conds:
- timestep_start = conds['timestep_start']
- if timestep_in[0] > timestep_start:
- return None
- if 'timestep_end' in conds:
- timestep_end = conds['timestep_end']
- if timestep_in[0] < timestep_end:
- return None
- if 'area' in conds:
- area = conds['area']
- if 'strength' in conds:
- strength = conds['strength']
-
- input_x = x_in[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]]
- if 'mask' in conds:
- # Scale the mask to the size of the input
- # The mask should have been resized as we began the sampling process
- mask_strength = 1.0
- if "mask_strength" in conds:
- mask_strength = conds["mask_strength"]
- mask = conds['mask']
- assert(mask.shape[1] == x_in.shape[2])
- assert(mask.shape[2] == x_in.shape[3])
- mask = mask[:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] * mask_strength
- mask = mask.unsqueeze(1).repeat(input_x.shape[0] // mask.shape[0], input_x.shape[1], 1, 1)
- else:
- mask = torch.ones_like(input_x)
- mult = mask * strength
-
- if 'mask' not in conds:
- rr = 8
- if area[2] != 0:
- for t in range(rr):
- mult[:,:,t:1+t,:] *= ((1.0/rr) * (t + 1))
- if (area[0] + area[2]) < x_in.shape[2]:
- for t in range(rr):
- mult[:,:,area[0] - 1 - t:area[0] - t,:] *= ((1.0/rr) * (t + 1))
- if area[3] != 0:
- for t in range(rr):
- mult[:,:,:,t:1+t] *= ((1.0/rr) * (t + 1))
- if (area[1] + area[3]) < x_in.shape[3]:
- for t in range(rr):
- mult[:,:,:,area[1] - 1 - t:area[1] - t] *= ((1.0/rr) * (t + 1))
-
- conditionning = {}
- model_conds = conds["model_conds"]
- for c in model_conds:
- conditionning[c] = model_conds[c].process_cond(batch_size=x_in.shape[0], device=x_in.device, area=area)
-
- control = None
- if 'control' in conds:
- control = conds['control']
-
- patches = None
- if 'gligen' in conds:
- gligen = conds['gligen']
- patches = {}
- gligen_type = gligen[0]
- gligen_model = gligen[1]
- if gligen_type == "position":
- gligen_patch = gligen_model.model.set_position(input_x.shape, gligen[2], input_x.device)
- else:
- gligen_patch = gligen_model.model.set_empty(input_x.shape, input_x.device)
-
- patches['middle_patch'] = [gligen_patch]
-
- return (input_x, mult, conditionning, area, control, patches)
-
- def cond_equal_size(c1, c2):
- if c1 is c2:
- return True
- if c1.keys() != c2.keys():
- return False
- for k in c1:
- if not c1[k].can_concat(c2[k]):
- return False
- return True
-
- def can_concat_cond(c1, c2):
- if c1[0].shape != c2[0].shape:
- return False
-
- #control
- if (c1[4] is None) != (c2[4] is None):
- return False
- if c1[4] is not None:
- if c1[4] is not c2[4]:
- return False
-
- #patches
- if (c1[5] is None) != (c2[5] is None):
- return False
- if (c1[5] is not None):
- if c1[5] is not c2[5]:
- return False
-
- return cond_equal_size(c1[2], c2[2])
-
- def cond_cat(c_list):
- c_crossattn = []
- c_concat = []
- c_adm = []
- crossattn_max_len = 0
-
- temp = {}
- for x in c_list:
- for k in x:
- cur = temp.get(k, [])
- cur.append(x[k])
- temp[k] = cur
-
- out = {}
- for k in temp:
- conds = temp[k]
- out[k] = conds[0].concat(conds[1:])
-
- return out
-
- def calc_cond_uncond_batch(model_function, cond, uncond, x_in, timestep, max_total_area, model_options):
- out_cond = torch.zeros_like(x_in)
- out_count = torch.ones_like(x_in) * 1e-37
-
- out_uncond = torch.zeros_like(x_in)
- out_uncond_count = torch.ones_like(x_in) * 1e-37
-
- COND = 0
- UNCOND = 1
-
- to_run = []
- for x in cond:
- p = get_area_and_mult(x, x_in, timestep)
- if p is None:
- continue
-
- to_run += [(p, COND)]
- if uncond is not None:
- for x in uncond:
- p = get_area_and_mult(x, x_in, timestep)
- if p is None:
- continue
-
- to_run += [(p, UNCOND)]
-
- while len(to_run) > 0:
- first = to_run[0]
- first_shape = first[0][0].shape
- to_batch_temp = []
- for x in range(len(to_run)):
- if can_concat_cond(to_run[x][0], first[0]):
- to_batch_temp += [x]
-
- to_batch_temp.reverse()
- to_batch = to_batch_temp[:1]
-
- for i in range(1, len(to_batch_temp) + 1):
- batch_amount = to_batch_temp[:len(to_batch_temp)//i]
- if (len(batch_amount) * first_shape[0] * first_shape[2] * first_shape[3] < max_total_area):
- to_batch = batch_amount
- break
-
- input_x = []
- mult = []
- c = []
- cond_or_uncond = []
- area = []
- control = None
- patches = None
- for x in to_batch:
- o = to_run.pop(x)
- p = o[0]
- input_x += [p[0]]
- mult += [p[1]]
- c += [p[2]]
- area += [p[3]]
- cond_or_uncond += [o[1]]
- control = p[4]
- patches = p[5]
-
- batch_chunks = len(cond_or_uncond)
- input_x = torch.cat(input_x)
- c = cond_cat(c)
- timestep_ = torch.cat([timestep] * batch_chunks)
-
- if control is not None:
- c['control'] = control.get_control(input_x, timestep_, c, len(cond_or_uncond))
-
- transformer_options = {}
- if 'transformer_options' in model_options:
- transformer_options = model_options['transformer_options'].copy()
-
- if patches is not None:
- if "patches" in transformer_options:
- cur_patches = transformer_options["patches"].copy()
- for p in patches:
- if p in cur_patches:
- cur_patches[p] = cur_patches[p] + patches[p]
- else:
- cur_patches[p] = patches[p]
- else:
- transformer_options["patches"] = patches
-
- transformer_options["cond_or_uncond"] = cond_or_uncond[:]
- c['transformer_options'] = transformer_options
-
- if 'model_function_wrapper' in model_options:
- output = model_options['model_function_wrapper'](model_function, {"input": input_x, "timestep": timestep_, "c": c, "cond_or_uncond": cond_or_uncond}).chunk(batch_chunks)
- else:
- output = model_function(input_x, timestep_, **c).chunk(batch_chunks)
- del input_x
-
- for o in range(batch_chunks):
- if cond_or_uncond[o] == COND:
- out_cond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
- out_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
- else:
- out_uncond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
- out_uncond_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
- del mult
-
- out_cond /= out_count
- del out_count
- out_uncond /= out_uncond_count
- del out_uncond_count
- return out_cond, out_uncond
-
-
- max_total_area = model_management.maximum_batch_area()
- if math.isclose(cond_scale, 1.0):
- uncond = None
-
- cond, uncond = calc_cond_uncond_batch(model_function, cond, uncond, x, timestep, max_total_area, model_options)
- if "sampler_cfg_function" in model_options:
- args = {"cond": x - cond, "uncond": x - uncond, "cond_scale": cond_scale, "timestep": timestep, "input": x, "sigma": timestep}
- return x - model_options["sampler_cfg_function"](args)
+def sampling_function(model, x, timestep, uncond, cond, cond_scale, model_options={}, seed=None):
+ if math.isclose(cond_scale, 1.0) and model_options.get("disable_cfg1_optimization", False) == False:
+ uncond_ = None
else:
- return uncond + (cond - uncond) * cond_scale
+ uncond_ = uncond
+
+ cond_pred, uncond_pred = calc_cond_uncond_batch(model, cond, uncond_, x, timestep, model_options)
+ if "sampler_cfg_function" in model_options:
+ args = {"cond": x - cond_pred, "uncond": x - uncond_pred, "cond_scale": cond_scale, "timestep": timestep, "input": x, "sigma": timestep,
+ "cond_denoised": cond_pred, "uncond_denoised": uncond_pred, "model": model, "model_options": model_options}
+ cfg_result = x - model_options["sampler_cfg_function"](args)
+ else:
+ cfg_result = uncond_pred + (cond_pred - uncond_pred) * cond_scale
+
+ for fn in model_options.get("sampler_post_cfg_function", []):
+ args = {"denoised": cfg_result, "cond": cond, "uncond": uncond, "model": model, "uncond_denoised": uncond_pred, "cond_denoised": cond_pred,
+ "sigma": timestep, "model_options": model_options, "input": x}
+ cfg_result = fn(args)
+
+ return cfg_result
class CFGNoisePredictor(torch.nn.Module):
def __init__(self, model):
super().__init__()
self.inner_model = model
def apply_model(self, x, timestep, cond, uncond, cond_scale, model_options={}, seed=None):
- out = sampling_function(self.inner_model.apply_model, x, timestep, uncond, cond, cond_scale, model_options=model_options, seed=seed)
+ out = sampling_function(self.inner_model, x, timestep, uncond, cond, cond_scale, model_options=model_options, seed=seed)
return out
def forward(self, *args, **kwargs):
return self.apply_model(*args, **kwargs)
@@ -273,10 +284,7 @@ class KSamplerX0Inpaint(torch.nn.Module):
x = x * denoise_mask + (self.latent_image + self.noise * sigma.reshape([sigma.shape[0]] + [1] * (len(self.noise.shape) - 1))) * latent_mask
out = self.inner_model(x, sigma, cond=cond, uncond=uncond, cond_scale=cond_scale, model_options=model_options, seed=seed)
if denoise_mask is not None:
- out *= denoise_mask
-
- if denoise_mask is not None:
- out += self.latent_image * latent_mask
+ out = out * denoise_mask + self.latent_image * latent_mask
return out
def simple_scheduler(model, steps):
@@ -511,52 +519,69 @@ class Sampler:
class UNIPC(Sampler):
def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
- return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, sampling_function=sampling_function, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, disable=disable_pbar)
+ return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, disable=disable_pbar)
class UNIPCBH2(Sampler):
def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
- return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, sampling_function=sampling_function, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, variant='bh2', disable=disable_pbar)
+ return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, variant='bh2', disable=disable_pbar)
-KSAMPLER_NAMES = ["euler", "euler_ancestral", "heun", "dpm_2", "dpm_2_ancestral",
+KSAMPLER_NAMES = ["euler", "euler_ancestral", "heun", "heunpp2","dpm_2", "dpm_2_ancestral",
"lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu",
"dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm"]
+class KSAMPLER(Sampler):
+ def __init__(self, sampler_function, extra_options={}, inpaint_options={}):
+ self.sampler_function = sampler_function
+ self.extra_options = extra_options
+ self.inpaint_options = inpaint_options
+
+ def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
+ extra_args["denoise_mask"] = denoise_mask
+ model_k = KSamplerX0Inpaint(model_wrap)
+ model_k.latent_image = latent_image
+ if self.inpaint_options.get("random", False): #TODO: Should this be the default?
+ generator = torch.manual_seed(extra_args.get("seed", 41) + 1)
+ model_k.noise = torch.randn(noise.shape, generator=generator, device="cpu").to(noise.dtype).to(noise.device)
+ else:
+ model_k.noise = noise
+
+ if self.max_denoise(model_wrap, sigmas):
+ noise = noise * torch.sqrt(1.0 + sigmas[0] ** 2.0)
+ else:
+ noise = noise * sigmas[0]
+
+ k_callback = None
+ total_steps = len(sigmas) - 1
+ if callback is not None:
+ k_callback = lambda x: callback(x["i"], x["denoised"], x["x"], total_steps)
+
+ if latent_image is not None:
+ noise += latent_image
+
+ samples = self.sampler_function(model_k, noise, sigmas, extra_args=extra_args, callback=k_callback, disable=disable_pbar, **self.extra_options)
+ return samples
+
+
def ksampler(sampler_name, extra_options={}, inpaint_options={}):
- class KSAMPLER(Sampler):
- def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
- extra_args["denoise_mask"] = denoise_mask
- model_k = KSamplerX0Inpaint(model_wrap)
- model_k.latent_image = latent_image
- if inpaint_options.get("random", False): #TODO: Should this be the default?
- generator = torch.manual_seed(extra_args.get("seed", 41) + 1)
- model_k.noise = torch.randn(noise.shape, generator=generator, device="cpu").to(noise.dtype).to(noise.device)
- else:
- model_k.noise = noise
-
- if self.max_denoise(model_wrap, sigmas):
- noise = noise * torch.sqrt(1.0 + sigmas[0] ** 2.0)
- else:
- noise = noise * sigmas[0]
-
- k_callback = None
- total_steps = len(sigmas) - 1
- if callback is not None:
- k_callback = lambda x: callback(x["i"], x["denoised"], x["x"], total_steps)
-
+ if sampler_name == "dpm_fast":
+ def dpm_fast_function(model, noise, sigmas, extra_args, callback, disable):
sigma_min = sigmas[-1]
if sigma_min == 0:
sigma_min = sigmas[-2]
+ total_steps = len(sigmas) - 1
+ return k_diffusion_sampling.sample_dpm_fast(model, noise, sigma_min, sigmas[0], total_steps, extra_args=extra_args, callback=callback, disable=disable)
+ sampler_function = dpm_fast_function
+ elif sampler_name == "dpm_adaptive":
+ def dpm_adaptive_function(model, noise, sigmas, extra_args, callback, disable):
+ sigma_min = sigmas[-1]
+ if sigma_min == 0:
+ sigma_min = sigmas[-2]
+ return k_diffusion_sampling.sample_dpm_adaptive(model, noise, sigma_min, sigmas[0], extra_args=extra_args, callback=callback, disable=disable)
+ sampler_function = dpm_adaptive_function
+ else:
+ sampler_function = getattr(k_diffusion_sampling, "sample_{}".format(sampler_name))
- if latent_image is not None:
- noise += latent_image
- if sampler_name == "dpm_fast":
- samples = k_diffusion_sampling.sample_dpm_fast(model_k, noise, sigma_min, sigmas[0], total_steps, extra_args=extra_args, callback=k_callback, disable=disable_pbar)
- elif sampler_name == "dpm_adaptive":
- samples = k_diffusion_sampling.sample_dpm_adaptive(model_k, noise, sigma_min, sigmas[0], extra_args=extra_args, callback=k_callback, disable=disable_pbar)
- else:
- samples = getattr(k_diffusion_sampling, "sample_{}".format(sampler_name))(model_k, noise, sigmas, extra_args=extra_args, callback=k_callback, disable=disable_pbar, **extra_options)
- return samples
- return KSAMPLER
+ return KSAMPLER(sampler_function, extra_options, inpaint_options)
def wrap_model(model):
model_denoise = CFGNoisePredictor(model)
@@ -574,6 +599,13 @@ def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, model
calculate_start_end_timesteps(model, negative)
calculate_start_end_timesteps(model, positive)
+ if latent_image is not None:
+ latent_image = model.process_latent_in(latent_image)
+
+ if hasattr(model, 'extra_conds'):
+ positive = encode_model_conds(model.extra_conds, positive, noise, device, "positive", latent_image=latent_image, denoise_mask=denoise_mask)
+ negative = encode_model_conds(model.extra_conds, negative, noise, device, "negative", latent_image=latent_image, denoise_mask=denoise_mask)
+
#make sure each cond area has an opposite one with the same area
for c in positive:
create_cond_with_same_area_if_none(negative, c)
@@ -585,13 +617,6 @@ def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, model
apply_empty_x_to_equal_area(list(filter(lambda c: c.get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x])
apply_empty_x_to_equal_area(positive, negative, 'gligen', lambda cond_cnets, x: cond_cnets[x])
- if latent_image is not None:
- latent_image = model.process_latent_in(latent_image)
-
- if hasattr(model, 'extra_conds'):
- positive = encode_model_conds(model.extra_conds, positive, noise, device, "positive", latent_image=latent_image, denoise_mask=denoise_mask)
- negative = encode_model_conds(model.extra_conds, negative, noise, device, "negative", latent_image=latent_image, denoise_mask=denoise_mask)
-
extra_args = {"cond":positive, "uncond":negative, "cond_scale": cfg, "model_options": model_options, "seed":seed}
samples = sampler.sample(model_wrap, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar)
@@ -617,11 +642,11 @@ def calculate_sigmas_scheduler(model, scheduler_name, steps):
print("error invalid scheduler", self.scheduler)
return sigmas
-def sampler_class(name):
+def sampler_object(name):
if name == "uni_pc":
- sampler = UNIPC
+ sampler = UNIPC()
elif name == "uni_pc_bh2":
- sampler = UNIPCBH2
+ sampler = UNIPCBH2()
elif name == "ddim":
sampler = ksampler("euler", inpaint_options={"random": True})
else:
@@ -686,6 +711,6 @@ class KSampler:
else:
return torch.zeros_like(noise)
- sampler = sampler_class(self.sampler)
+ sampler = sampler_object(self.sampler)
- return sample(self.model, noise, positive, negative, cfg, self.device, sampler(), sigmas, self.model_options, latent_image=latent_image, denoise_mask=denoise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)
+ return sample(self.model, noise, positive, negative, cfg, self.device, sampler, sigmas, self.model_options, latent_image=latent_image, denoise_mask=denoise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)
diff --git a/comfy/sd.py b/comfy/sd.py
index 65d94f46e..220637a05 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -23,6 +23,7 @@ import comfy.model_patcher
import comfy.lora
import comfy.t2i_adapter.adapter
import comfy.supported_models_base
+import comfy.taesd.taesd
def load_model_weights(model, sd):
m, u = model.load_state_dict(sd, strict=False)
@@ -95,10 +96,7 @@ class CLIP:
load_device = model_management.text_encoder_device()
offload_device = model_management.text_encoder_offload_device()
params['device'] = offload_device
- if model_management.should_use_fp16(load_device, prioritize_performance=False):
- params['dtype'] = torch.float16
- else:
- params['dtype'] = torch.float32
+ params['dtype'] = model_management.text_encoder_dtype(load_device)
self.cond_stage_model = clip(**(params))
@@ -153,14 +151,28 @@ class CLIP:
return self.patcher.get_key_patches()
class VAE:
- def __init__(self, sd=None, device=None, config=None):
+ def __init__(self, sd=None, device=None, config=None, dtype=None):
if 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format
sd = diffusers_convert.convert_vae_state_dict(sd)
+ self.memory_used_encode = lambda shape, dtype: (1767 * shape[2] * shape[3]) * model_management.dtype_size(dtype) #These are for AutoencoderKL and need tweaking (should be lower)
+ self.memory_used_decode = lambda shape, dtype: (2178 * shape[2] * shape[3] * 64) * model_management.dtype_size(dtype)
+
if config is None:
- #default SD1.x/SD2.x VAE parameters
- ddconfig = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0}
- self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4)
+ if "decoder.mid.block_1.mix_factor" in sd:
+ encoder_config = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0}
+ decoder_config = encoder_config.copy()
+ decoder_config["video_kernel_size"] = [3, 1, 1]
+ decoder_config["alpha"] = 0.0
+ self.first_stage_model = AutoencodingEngine(regularizer_config={'target': "comfy.ldm.models.autoencoder.DiagonalGaussianRegularizer"},
+ encoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Encoder", 'params': encoder_config},
+ decoder_config={'target': "comfy.ldm.modules.temporal_ae.VideoDecoder", 'params': decoder_config})
+ elif "taesd_decoder.1.weight" in sd:
+ self.first_stage_model = comfy.taesd.taesd.TAESD()
+ else:
+ #default SD1.x/SD2.x VAE parameters
+ ddconfig = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0}
+ self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4)
else:
self.first_stage_model = AutoencoderKL(**(config['params']))
self.first_stage_model = self.first_stage_model.eval()
@@ -175,9 +187,14 @@ class VAE:
if device is None:
device = model_management.vae_device()
self.device = device
- self.offload_device = model_management.vae_offload_device()
- self.vae_dtype = model_management.vae_dtype()
+ offload_device = model_management.vae_offload_device()
+ if dtype is None:
+ dtype = model_management.vae_dtype()
+ self.vae_dtype = dtype
self.first_stage_model.to(self.vae_dtype)
+ self.output_device = model_management.intermediate_device()
+
+ self.patcher = comfy.model_patcher.ModelPatcher(self.first_stage_model, load_device=self.device, offload_device=offload_device)
def decode_tiled_(self, samples, tile_x=64, tile_y=64, overlap = 16):
steps = samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x, tile_y, overlap)
@@ -187,9 +204,9 @@ class VAE:
decode_fn = lambda a: (self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)) + 1.0).float()
output = torch.clamp((
- (comfy.utils.tiled_scale(samples, decode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = 8, pbar = pbar) +
- comfy.utils.tiled_scale(samples, decode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = 8, pbar = pbar) +
- comfy.utils.tiled_scale(samples, decode_fn, tile_x, tile_y, overlap, upscale_amount = 8, pbar = pbar))
+ (comfy.utils.tiled_scale(samples, decode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = 8, output_device=self.output_device, pbar = pbar) +
+ comfy.utils.tiled_scale(samples, decode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = 8, output_device=self.output_device, pbar = pbar) +
+ comfy.utils.tiled_scale(samples, decode_fn, tile_x, tile_y, overlap, upscale_amount = 8, output_device=self.output_device, pbar = pbar))
/ 3.0) / 2.0, min=0.0, max=1.0)
return output
@@ -200,65 +217,59 @@ class VAE:
pbar = comfy.utils.ProgressBar(steps)
encode_fn = lambda a: self.first_stage_model.encode((2. * a - 1.).to(self.vae_dtype).to(self.device)).float()
- samples = comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x, tile_y, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
- samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
- samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
+ samples = comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x, tile_y, overlap, upscale_amount = (1/8), out_channels=4, output_device=self.output_device, pbar=pbar)
+ samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = (1/8), out_channels=4, output_device=self.output_device, pbar=pbar)
+ samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = (1/8), out_channels=4, output_device=self.output_device, pbar=pbar)
samples /= 3.0
return samples
def decode(self, samples_in):
- self.first_stage_model = self.first_stage_model.to(self.device)
try:
- memory_used = (2562 * samples_in.shape[2] * samples_in.shape[3] * 64) * 1.7
- model_management.free_memory(memory_used, self.device)
+ memory_used = self.memory_used_decode(samples_in.shape, self.vae_dtype)
+ model_management.load_models_gpu([self.patcher], memory_required=memory_used)
free_memory = model_management.get_free_memory(self.device)
batch_number = int(free_memory / memory_used)
batch_number = max(1, batch_number)
- pixel_samples = torch.empty((samples_in.shape[0], 3, round(samples_in.shape[2] * 8), round(samples_in.shape[3] * 8)), device="cpu")
+ pixel_samples = torch.empty((samples_in.shape[0], 3, round(samples_in.shape[2] * 8), round(samples_in.shape[3] * 8)), device=self.output_device)
for x in range(0, samples_in.shape[0], batch_number):
samples = samples_in[x:x+batch_number].to(self.vae_dtype).to(self.device)
- pixel_samples[x:x+batch_number] = torch.clamp((self.first_stage_model.decode(samples).cpu().float() + 1.0) / 2.0, min=0.0, max=1.0)
+ pixel_samples[x:x+batch_number] = torch.clamp((self.first_stage_model.decode(samples).to(self.output_device).float() + 1.0) / 2.0, min=0.0, max=1.0)
except model_management.OOM_EXCEPTION as e:
print("Warning: Ran out of memory when regular VAE decoding, retrying with tiled VAE decoding.")
pixel_samples = self.decode_tiled_(samples_in)
- self.first_stage_model = self.first_stage_model.to(self.offload_device)
- pixel_samples = pixel_samples.cpu().movedim(1,-1)
+ pixel_samples = pixel_samples.to(self.output_device).movedim(1,-1)
return pixel_samples
def decode_tiled(self, samples, tile_x=64, tile_y=64, overlap = 16):
- self.first_stage_model = self.first_stage_model.to(self.device)
+ model_management.load_model_gpu(self.patcher)
output = self.decode_tiled_(samples, tile_x, tile_y, overlap)
- self.first_stage_model = self.first_stage_model.to(self.offload_device)
return output.movedim(1,-1)
def encode(self, pixel_samples):
- self.first_stage_model = self.first_stage_model.to(self.device)
pixel_samples = pixel_samples.movedim(-1,1)
try:
- memory_used = (2078 * pixel_samples.shape[2] * pixel_samples.shape[3]) * 1.7 #NOTE: this constant along with the one in the decode above are estimated from the mem usage for the VAE and could change.
- model_management.free_memory(memory_used, self.device)
+ memory_used = self.memory_used_encode(pixel_samples.shape, self.vae_dtype)
+ model_management.load_models_gpu([self.patcher], memory_required=memory_used)
free_memory = model_management.get_free_memory(self.device)
batch_number = int(free_memory / memory_used)
batch_number = max(1, batch_number)
- samples = torch.empty((pixel_samples.shape[0], 4, round(pixel_samples.shape[2] // 8), round(pixel_samples.shape[3] // 8)), device="cpu")
+ samples = torch.empty((pixel_samples.shape[0], 4, round(pixel_samples.shape[2] // 8), round(pixel_samples.shape[3] // 8)), device=self.output_device)
for x in range(0, pixel_samples.shape[0], batch_number):
pixels_in = (2. * pixel_samples[x:x+batch_number] - 1.).to(self.vae_dtype).to(self.device)
- samples[x:x+batch_number] = self.first_stage_model.encode(pixels_in).cpu().float()
+ samples[x:x+batch_number] = self.first_stage_model.encode(pixels_in).to(self.output_device).float()
except model_management.OOM_EXCEPTION as e:
print("Warning: Ran out of memory when regular VAE encoding, retrying with tiled VAE encoding.")
samples = self.encode_tiled_(pixel_samples)
- self.first_stage_model = self.first_stage_model.to(self.offload_device)
return samples
def encode_tiled(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):
- self.first_stage_model = self.first_stage_model.to(self.device)
+ model_management.load_model_gpu(self.patcher)
pixel_samples = pixel_samples.movedim(-1,1)
samples = self.encode_tiled_(pixel_samples, tile_x=tile_x, tile_y=tile_y, overlap=overlap)
- self.first_stage_model = self.first_stage_model.to(self.offload_device)
return samples
def get_sd(self):
@@ -424,11 +435,15 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
parameters = comfy.utils.calculate_parameters(sd, "model.diffusion_model.")
unet_dtype = model_management.unet_dtype(model_params=parameters)
+ load_device = model_management.get_torch_device()
+ manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device)
class WeightsLoader(torch.nn.Module):
pass
model_config = model_detection.model_config_from_unet(sd, "model.diffusion_model.", unet_dtype)
+ model_config.set_manual_cast(manual_cast_dtype)
+
if model_config is None:
raise RuntimeError("ERROR: Could not detect model type of: {}".format(ckpt_path))
@@ -444,6 +459,7 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
if output_vae:
vae_sd = comfy.utils.state_dict_prefix_replace(sd, {"first_stage_model.": ""}, filter_keys=True)
+ vae_sd = model_config.process_vae_state_dict(vae_sd)
vae = VAE(sd=vae_sd)
if output_clip:
@@ -460,7 +476,7 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
print("left over keys:", left_over)
if output_model:
- model_patcher = comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=model_management.unet_offload_device(), current_device=inital_load_device)
+ model_patcher = comfy.model_patcher.ModelPatcher(model, load_device=load_device, offload_device=model_management.unet_offload_device(), current_device=inital_load_device)
if inital_load_device != torch.device("cpu"):
print("loaded straight to GPU")
model_management.load_model_gpu(model_patcher)
@@ -468,20 +484,21 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
return (model_patcher, clip, vae, clipvision)
-def load_unet(unet_path): #load unet in diffusers format
- sd = comfy.utils.load_torch_file(unet_path)
+def load_unet_state_dict(sd): #load unet in diffusers format
parameters = comfy.utils.calculate_parameters(sd)
unet_dtype = model_management.unet_dtype(model_params=parameters)
+ load_device = model_management.get_torch_device()
+ manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device)
+
if "input_blocks.0.0.weight" in sd: #ldm
model_config = model_detection.model_config_from_unet(sd, "", unet_dtype)
if model_config is None:
- raise RuntimeError("ERROR: Could not detect model type of: {}".format(unet_path))
+ return None
new_sd = sd
else: #diffusers
model_config = model_detection.model_config_from_diffusers_unet(sd, unet_dtype)
if model_config is None:
- print("ERROR UNSUPPORTED UNET", unet_path)
return None
diffusers_keys = comfy.utils.unet_to_diffusers(model_config.unet_config)
@@ -493,13 +510,22 @@ def load_unet(unet_path): #load unet in diffusers format
else:
print(diffusers_keys[k], k)
offload_device = model_management.unet_offload_device()
+ model_config.set_manual_cast(manual_cast_dtype)
model = model_config.get_model(new_sd, "")
model = model.to(offload_device)
model.load_model_weights(new_sd, "")
left_over = sd.keys()
if len(left_over) > 0:
print("left over keys in unet:", left_over)
- return comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device)
+ return comfy.model_patcher.ModelPatcher(model, load_device=load_device, offload_device=offload_device)
+
+def load_unet(unet_path):
+ sd = comfy.utils.load_torch_file(unet_path)
+ model = load_unet_state_dict(sd)
+ if model is None:
+ print("ERROR UNSUPPORTED UNET", unet_path)
+ raise RuntimeError("ERROR: Could not detect model type of: {}".format(unet_path))
+ return model
def save_checkpoint(output_path, model, clip, vae, metadata=None):
model_management.load_models_gpu([model, clip.load_model()])
diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py
index 7db7ee0f4..6ffef515e 100644
--- a/comfy/sd1_clip.py
+++ b/comfy/sd1_clip.py
@@ -1,12 +1,14 @@
import os
-from transformers import CLIPTokenizer, CLIPTextModel, CLIPTextConfig, modeling_utils
+from transformers import CLIPTokenizer
import comfy.ops
import torch
import traceback
import zipfile
from . import model_management
import contextlib
+import comfy.clip_model
+import json
def gen_empty_tokens(special_tokens, length):
start_token = special_tokens.get("start", None)
@@ -37,7 +39,7 @@ class ClipTokenWeightEncoder:
out, pooled = self.encode(to_encode)
if pooled is not None:
- first_pooled = pooled[0:1].cpu()
+ first_pooled = pooled[0:1].to(model_management.intermediate_device())
else:
first_pooled = pooled
@@ -54,8 +56,8 @@ class ClipTokenWeightEncoder:
output.append(z)
if (len(output) == 0):
- return out[-1:].cpu(), first_pooled
- return torch.cat(output, dim=-2).cpu(), first_pooled
+ return out[-1:].to(model_management.intermediate_device()), first_pooled
+ return torch.cat(output, dim=-2).to(model_management.intermediate_device()), first_pooled
class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
"""Uses the CLIP transformer encoder for text (from huggingface)"""
@@ -65,31 +67,19 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
"hidden"
]
def __init__(self, version="openai/clip-vit-large-patch14", device="cpu", max_length=77,
- freeze=True, layer="last", layer_idx=None, textmodel_json_config=None, textmodel_path=None, dtype=None,
- special_tokens={"start": 49406, "end": 49407, "pad": 49407},layer_norm_hidden_state=True, config_class=CLIPTextConfig,
- model_class=CLIPTextModel, inner_name="text_model"): # clip-vit-base-patch32
+ freeze=True, layer="last", layer_idx=None, textmodel_json_config=None, dtype=None, model_class=comfy.clip_model.CLIPTextModel,
+ special_tokens={"start": 49406, "end": 49407, "pad": 49407}, layer_norm_hidden_state=True): # clip-vit-base-patch32
super().__init__()
assert layer in self.LAYERS
- self.num_layers = 12
- if textmodel_path is not None:
- self.transformer = model_class.from_pretrained(textmodel_path)
- else:
- if textmodel_json_config is None:
- textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_clip_config.json")
- config = config_class.from_json_file(textmodel_json_config)
- self.num_layers = config.num_hidden_layers
- with comfy.ops.use_comfy_ops(device, dtype):
- with modeling_utils.no_init_weights():
- self.transformer = model_class(config)
- self.inner_name = inner_name
- if dtype is not None:
- self.transformer.to(dtype)
- inner_model = getattr(self.transformer, self.inner_name)
- if hasattr(inner_model, "embeddings"):
- inner_model.embeddings.to(torch.float32)
- else:
- self.transformer.set_input_embeddings(self.transformer.get_input_embeddings().to(torch.float32))
+ if textmodel_json_config is None:
+ textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_clip_config.json")
+
+ with open(textmodel_json_config) as f:
+ config = json.load(f)
+
+ self.transformer = model_class(config, dtype, device, comfy.ops.manual_cast)
+ self.num_layers = self.transformer.num_layers
self.max_length = max_length
if freeze:
@@ -104,7 +94,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
self.layer_norm_hidden_state = layer_norm_hidden_state
if layer == "hidden":
assert layer_idx is not None
- assert abs(layer_idx) <= self.num_layers
+ assert abs(layer_idx) < self.num_layers
self.clip_layer(layer_idx)
self.layer_default = (self.layer, self.layer_idx)
@@ -115,7 +105,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
param.requires_grad = False
def clip_layer(self, layer_idx):
- if abs(layer_idx) >= self.num_layers:
+ if abs(layer_idx) > self.num_layers:
self.layer = "last"
else:
self.layer = "hidden"
@@ -170,41 +160,31 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
tokens = self.set_up_textual_embeddings(tokens, backup_embeds)
tokens = torch.LongTensor(tokens).to(device)
- if getattr(self.transformer, self.inner_name).final_layer_norm.weight.dtype != torch.float32:
- precision_scope = torch.autocast
+ attention_mask = None
+ if self.enable_attention_masks:
+ attention_mask = torch.zeros_like(tokens)
+ max_token = self.transformer.get_input_embeddings().weight.shape[0] - 1
+ for x in range(attention_mask.shape[0]):
+ for y in range(attention_mask.shape[1]):
+ attention_mask[x, y] = 1
+ if tokens[x, y] == max_token:
+ break
+
+ outputs = self.transformer(tokens, attention_mask, intermediate_output=self.layer_idx, final_layer_norm_intermediate=self.layer_norm_hidden_state)
+ self.transformer.set_input_embeddings(backup_embeds)
+
+ if self.layer == "last":
+ z = outputs[0]
else:
- precision_scope = lambda a, b: contextlib.nullcontext(a)
+ z = outputs[1]
- with precision_scope(model_management.get_autocast_device(device), torch.float32):
- attention_mask = None
- if self.enable_attention_masks:
- attention_mask = torch.zeros_like(tokens)
- max_token = self.transformer.get_input_embeddings().weight.shape[0] - 1
- for x in range(attention_mask.shape[0]):
- for y in range(attention_mask.shape[1]):
- attention_mask[x, y] = 1
- if tokens[x, y] == max_token:
- break
+ if outputs[2] is not None:
+ pooled_output = outputs[2].float()
+ else:
+ pooled_output = None
- outputs = self.transformer(input_ids=tokens, attention_mask=attention_mask, output_hidden_states=self.layer=="hidden")
- self.transformer.set_input_embeddings(backup_embeds)
-
- if self.layer == "last":
- z = outputs.last_hidden_state
- elif self.layer == "pooled":
- z = outputs.pooler_output[:, None, :]
- else:
- z = outputs.hidden_states[self.layer_idx]
- if self.layer_norm_hidden_state:
- z = getattr(self.transformer, self.inner_name).final_layer_norm(z)
-
- if hasattr(outputs, "pooler_output"):
- pooled_output = outputs.pooler_output.float()
- else:
- pooled_output = None
-
- if self.text_projection is not None and pooled_output is not None:
- pooled_output = pooled_output.float().to(self.text_projection.device) @ self.text_projection.float()
+ if self.text_projection is not None and pooled_output is not None:
+ pooled_output = pooled_output.float().to(self.text_projection.device) @ self.text_projection.float()
return z.float(), pooled_output
def encode(self, tokens):
diff --git a/comfy/sd2_clip.py b/comfy/sd2_clip.py
index 2ee0ca055..9c878d54a 100644
--- a/comfy/sd2_clip.py
+++ b/comfy/sd2_clip.py
@@ -3,13 +3,13 @@ import torch
import os
class SD2ClipHModel(sd1_clip.SDClipModel):
- def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None):
+ def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None):
if layer == "penultimate":
layer="hidden"
- layer_idx=23
+ layer_idx=-2
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd2_clip_config.json")
- super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype, special_tokens={"start": 49406, "end": 49407, "pad": 0})
+ super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"start": 49406, "end": 49407, "pad": 0})
class SD2ClipHTokenizer(sd1_clip.SDTokenizer):
def __init__(self, tokenizer_path=None, embedding_directory=None):
diff --git a/comfy/sdxl_clip.py b/comfy/sdxl_clip.py
index 673399e22..b35056bb9 100644
--- a/comfy/sdxl_clip.py
+++ b/comfy/sdxl_clip.py
@@ -3,13 +3,13 @@ import torch
import os
class SDXLClipG(sd1_clip.SDClipModel):
- def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None):
+ def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None):
if layer == "penultimate":
layer="hidden"
layer_idx=-2
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_config_bigg.json")
- super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype,
+ super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype,
special_tokens={"start": 49406, "end": 49407, "pad": 0}, layer_norm_hidden_state=False)
def load_sd(self, sd):
@@ -37,7 +37,7 @@ class SDXLTokenizer:
class SDXLClipModel(torch.nn.Module):
def __init__(self, device="cpu", dtype=None):
super().__init__()
- self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=11, device=device, dtype=dtype, layer_norm_hidden_state=False)
+ self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=-2, device=device, dtype=dtype, layer_norm_hidden_state=False)
self.clip_g = SDXLClipG(device=device, dtype=dtype)
def clip_layer(self, layer_idx):
diff --git a/comfy/supported_models.py b/comfy/supported_models.py
index fdd4ea4f5..251bf6ace 100644
--- a/comfy/supported_models.py
+++ b/comfy/supported_models.py
@@ -17,6 +17,7 @@ class SD15(supported_models_base.BASE):
"model_channels": 320,
"use_linear_in_transformer": False,
"adm_in_channels": None,
+ "use_temporal_attention": False,
}
unet_extra_config = {
@@ -56,6 +57,7 @@ class SD20(supported_models_base.BASE):
"model_channels": 320,
"use_linear_in_transformer": True,
"adm_in_channels": None,
+ "use_temporal_attention": False,
}
latent_format = latent_formats.SD15
@@ -69,6 +71,10 @@ class SD20(supported_models_base.BASE):
return model_base.ModelType.EPS
def process_clip_state_dict(self, state_dict):
+ replace_prefix = {}
+ replace_prefix["conditioner.embedders.0.model."] = "cond_stage_model.model." #SD2 in sgm format
+ state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix)
+
state_dict = utils.transformers_convert(state_dict, "cond_stage_model.model.", "cond_stage_model.clip_h.transformer.text_model.", 24)
return state_dict
@@ -88,6 +94,7 @@ class SD21UnclipL(SD20):
"model_channels": 320,
"use_linear_in_transformer": True,
"adm_in_channels": 1536,
+ "use_temporal_attention": False,
}
clip_vision_prefix = "embedder.model.visual."
@@ -100,6 +107,7 @@ class SD21UnclipH(SD20):
"model_channels": 320,
"use_linear_in_transformer": True,
"adm_in_channels": 2048,
+ "use_temporal_attention": False,
}
clip_vision_prefix = "embedder.model.visual."
@@ -112,6 +120,7 @@ class SDXLRefiner(supported_models_base.BASE):
"context_dim": 1280,
"adm_in_channels": 2560,
"transformer_depth": [0, 0, 4, 4, 4, 4, 0, 0],
+ "use_temporal_attention": False,
}
latent_format = latent_formats.SDXL
@@ -148,7 +157,8 @@ class SDXL(supported_models_base.BASE):
"use_linear_in_transformer": True,
"transformer_depth": [0, 0, 2, 2, 10, 10],
"context_dim": 2048,
- "adm_in_channels": 2816
+ "adm_in_channels": 2816,
+ "use_temporal_attention": False,
}
latent_format = latent_formats.SDXL
@@ -203,8 +213,71 @@ class SSD1B(SDXL):
"use_linear_in_transformer": True,
"transformer_depth": [0, 0, 2, 2, 4, 4],
"context_dim": 2048,
- "adm_in_channels": 2816
+ "adm_in_channels": 2816,
+ "use_temporal_attention": False,
}
+class Segmind_Vega(SDXL):
+ unet_config = {
+ "model_channels": 320,
+ "use_linear_in_transformer": True,
+ "transformer_depth": [0, 0, 1, 1, 2, 2],
+ "context_dim": 2048,
+ "adm_in_channels": 2816,
+ "use_temporal_attention": False,
+ }
-models = [SD15, SD20, SD21UnclipL, SD21UnclipH, SDXLRefiner, SDXL, SSD1B]
+class SVD_img2vid(supported_models_base.BASE):
+ unet_config = {
+ "model_channels": 320,
+ "in_channels": 8,
+ "use_linear_in_transformer": True,
+ "transformer_depth": [1, 1, 1, 1, 1, 1, 0, 0],
+ "context_dim": 1024,
+ "adm_in_channels": 768,
+ "use_temporal_attention": True,
+ "use_temporal_resblock": True
+ }
+
+ clip_vision_prefix = "conditioner.embedders.0.open_clip.model.visual."
+
+ latent_format = latent_formats.SD15
+
+ sampling_settings = {"sigma_max": 700.0, "sigma_min": 0.002}
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.SVD_img2vid(self, device=device)
+ return out
+
+ def clip_target(self):
+ return None
+
+class Stable_Zero123(supported_models_base.BASE):
+ unet_config = {
+ "context_dim": 768,
+ "model_channels": 320,
+ "use_linear_in_transformer": False,
+ "adm_in_channels": None,
+ "use_temporal_attention": False,
+ "in_channels": 8,
+ }
+
+ unet_extra_config = {
+ "num_heads": 8,
+ "num_head_channels": -1,
+ }
+
+ clip_vision_prefix = "cond_stage_model.model.visual."
+
+ latent_format = latent_formats.SD15
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.Stable_Zero123(self, device=device, cc_projection_weight=state_dict["cc_projection.weight"], cc_projection_bias=state_dict["cc_projection.bias"])
+ return out
+
+ def clip_target(self):
+ return None
+
+
+models = [Stable_Zero123, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXLRefiner, SDXL, SSD1B, Segmind_Vega]
+models += [SVD_img2vid]
diff --git a/comfy/supported_models_base.py b/comfy/supported_models_base.py
index 88a1d7fde..49087d23e 100644
--- a/comfy/supported_models_base.py
+++ b/comfy/supported_models_base.py
@@ -19,9 +19,11 @@ class BASE:
clip_prefix = []
clip_vision_prefix = None
noise_aug_config = None
- beta_schedule = "linear"
+ sampling_settings = {}
latent_format = latent_formats.LatentFormat
+ manual_cast_dtype = None
+
@classmethod
def matches(s, unet_config):
for k in s.unet_config:
@@ -53,6 +55,12 @@ class BASE:
def process_clip_state_dict(self, state_dict):
return state_dict
+ def process_unet_state_dict(self, state_dict):
+ return state_dict
+
+ def process_vae_state_dict(self, state_dict):
+ return state_dict
+
def process_clip_state_dict_for_saving(self, state_dict):
replace_prefix = {"": "cond_stage_model."}
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
@@ -65,3 +73,5 @@ class BASE:
replace_prefix = {"": "first_stage_model."}
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
+ def set_manual_cast(self, manual_cast_dtype):
+ self.manual_cast_dtype = manual_cast_dtype
diff --git a/comfy/taesd/taesd.py b/comfy/taesd/taesd.py
index 8df1f1609..8f96c54e5 100644
--- a/comfy/taesd/taesd.py
+++ b/comfy/taesd/taesd.py
@@ -7,9 +7,10 @@ import torch
import torch.nn as nn
import comfy.utils
+import comfy.ops
def conv(n_in, n_out, **kwargs):
- return nn.Conv2d(n_in, n_out, 3, padding=1, **kwargs)
+ return comfy.ops.disable_weight_init.Conv2d(n_in, n_out, 3, padding=1, **kwargs)
class Clamp(nn.Module):
def forward(self, x):
@@ -19,7 +20,7 @@ class Block(nn.Module):
def __init__(self, n_in, n_out):
super().__init__()
self.conv = nn.Sequential(conv(n_in, n_out), nn.ReLU(), conv(n_out, n_out), nn.ReLU(), conv(n_out, n_out))
- self.skip = nn.Conv2d(n_in, n_out, 1, bias=False) if n_in != n_out else nn.Identity()
+ self.skip = comfy.ops.disable_weight_init.Conv2d(n_in, n_out, 1, bias=False) if n_in != n_out else nn.Identity()
self.fuse = nn.ReLU()
def forward(self, x):
return self.fuse(self.conv(x) + self.skip(x))
@@ -46,15 +47,16 @@ class TAESD(nn.Module):
latent_magnitude = 3
latent_shift = 0.5
- def __init__(self, encoder_path="taesd_encoder.pth", decoder_path="taesd_decoder.pth"):
+ def __init__(self, encoder_path=None, decoder_path=None):
"""Initialize pretrained TAESD on the given device from the given checkpoints."""
super().__init__()
- self.encoder = Encoder()
- self.decoder = Decoder()
+ self.taesd_encoder = Encoder()
+ self.taesd_decoder = Decoder()
+ self.vae_scale = torch.nn.Parameter(torch.tensor(1.0))
if encoder_path is not None:
- self.encoder.load_state_dict(comfy.utils.load_torch_file(encoder_path, safe_load=True))
+ self.taesd_encoder.load_state_dict(comfy.utils.load_torch_file(encoder_path, safe_load=True))
if decoder_path is not None:
- self.decoder.load_state_dict(comfy.utils.load_torch_file(decoder_path, safe_load=True))
+ self.taesd_decoder.load_state_dict(comfy.utils.load_torch_file(decoder_path, safe_load=True))
@staticmethod
def scale_latents(x):
@@ -65,3 +67,11 @@ class TAESD(nn.Module):
def unscale_latents(x):
"""[0, 1] -> raw latents"""
return x.sub(TAESD.latent_shift).mul(2 * TAESD.latent_magnitude)
+
+ def decode(self, x):
+ x_sample = self.taesd_decoder(x * self.vae_scale)
+ x_sample = x_sample.sub(0.5).mul(2)
+ return x_sample
+
+ def encode(self, x):
+ return self.taesd_encoder(x * 0.5 + 0.5) / self.vae_scale
diff --git a/comfy/utils.py b/comfy/utils.py
index 4b484d07a..f8026ddab 100644
--- a/comfy/utils.py
+++ b/comfy/utils.py
@@ -239,6 +239,26 @@ def repeat_to_batch_size(tensor, batch_size):
return tensor.repeat([math.ceil(batch_size / tensor.shape[0])] + [1] * (len(tensor.shape) - 1))[:batch_size]
return tensor
+def resize_to_batch_size(tensor, batch_size):
+ in_batch_size = tensor.shape[0]
+ if in_batch_size == batch_size:
+ return tensor
+
+ if batch_size <= 1:
+ return tensor[:batch_size]
+
+ output = torch.empty([batch_size] + list(tensor.shape)[1:], dtype=tensor.dtype, device=tensor.device)
+ if batch_size < in_batch_size:
+ scale = (in_batch_size - 1) / (batch_size - 1)
+ for i in range(batch_size):
+ output[i] = tensor[min(round(i * scale), in_batch_size - 1)]
+ else:
+ scale = in_batch_size / batch_size
+ for i in range(batch_size):
+ output[i] = tensor[min(math.floor((i + 0.5) * scale), in_batch_size - 1)]
+
+ return output
+
def convert_sd_to(state_dict, dtype):
keys = list(state_dict.keys())
for k in keys:
@@ -258,7 +278,7 @@ def set_attr(obj, attr, value):
for name in attrs[:-1]:
obj = getattr(obj, name)
prev = getattr(obj, attrs[-1])
- setattr(obj, attrs[-1], torch.nn.Parameter(value))
+ setattr(obj, attrs[-1], torch.nn.Parameter(value, requires_grad=False))
del prev
def copy_to_param(obj, attr, value):
@@ -307,23 +327,25 @@ def bislerp(samples, width, height):
res[dot < 1e-5 - 1] = (b1 * (1.0-r) + b2 * r)[dot < 1e-5 - 1]
return res
- def generate_bilinear_data(length_old, length_new):
- coords_1 = torch.arange(length_old).reshape((1,1,1,-1)).to(torch.float32)
+ def generate_bilinear_data(length_old, length_new, device):
+ coords_1 = torch.arange(length_old, dtype=torch.float32, device=device).reshape((1,1,1,-1))
coords_1 = torch.nn.functional.interpolate(coords_1, size=(1, length_new), mode="bilinear")
ratios = coords_1 - coords_1.floor()
coords_1 = coords_1.to(torch.int64)
- coords_2 = torch.arange(length_old).reshape((1,1,1,-1)).to(torch.float32) + 1
+ coords_2 = torch.arange(length_old, dtype=torch.float32, device=device).reshape((1,1,1,-1)) + 1
coords_2[:,:,:,-1] -= 1
coords_2 = torch.nn.functional.interpolate(coords_2, size=(1, length_new), mode="bilinear")
coords_2 = coords_2.to(torch.int64)
return ratios, coords_1, coords_2
-
+
+ orig_dtype = samples.dtype
+ samples = samples.float()
n,c,h,w = samples.shape
h_new, w_new = (height, width)
#linear w
- ratios, coords_1, coords_2 = generate_bilinear_data(w, w_new)
+ ratios, coords_1, coords_2 = generate_bilinear_data(w, w_new, samples.device)
coords_1 = coords_1.expand((n, c, h, -1))
coords_2 = coords_2.expand((n, c, h, -1))
ratios = ratios.expand((n, 1, h, -1))
@@ -336,7 +358,7 @@ def bislerp(samples, width, height):
result = result.reshape(n, h, w_new, c).movedim(-1, 1)
#linear h
- ratios, coords_1, coords_2 = generate_bilinear_data(h, h_new)
+ ratios, coords_1, coords_2 = generate_bilinear_data(h, h_new, samples.device)
coords_1 = coords_1.reshape((1,1,-1,1)).expand((n, c, -1, w_new))
coords_2 = coords_2.reshape((1,1,-1,1)).expand((n, c, -1, w_new))
ratios = ratios.reshape((1,1,-1,1)).expand((n, 1, -1, w_new))
@@ -347,14 +369,14 @@ def bislerp(samples, width, height):
result = slerp(pass_1, pass_2, ratios)
result = result.reshape(n, h_new, w_new, c).movedim(-1, 1)
- return result
+ return result.to(orig_dtype)
def lanczos(samples, width, height):
images = [Image.fromarray(np.clip(255. * image.movedim(0, -1).cpu().numpy(), 0, 255).astype(np.uint8)) for image in samples]
images = [image.resize((width, height), resample=Image.Resampling.LANCZOS) for image in images]
images = [torch.from_numpy(np.array(image).astype(np.float32) / 255.0).movedim(-1, 0) for image in images]
result = torch.stack(images)
- return result
+ return result.to(samples.device, samples.dtype)
def common_upscale(samples, width, height, upscale_method, crop):
if crop == "center":
@@ -383,17 +405,17 @@ def get_tiled_scale_steps(width, height, tile_x, tile_y, overlap):
return math.ceil((height / (tile_y - overlap))) * math.ceil((width / (tile_x - overlap)))
@torch.inference_mode()
-def tiled_scale(samples, function, tile_x=64, tile_y=64, overlap = 8, upscale_amount = 4, out_channels = 3, pbar = None):
- output = torch.empty((samples.shape[0], out_channels, round(samples.shape[2] * upscale_amount), round(samples.shape[3] * upscale_amount)), device="cpu")
+def tiled_scale(samples, function, tile_x=64, tile_y=64, overlap = 8, upscale_amount = 4, out_channels = 3, output_device="cpu", pbar = None):
+ output = torch.empty((samples.shape[0], out_channels, round(samples.shape[2] * upscale_amount), round(samples.shape[3] * upscale_amount)), device=output_device)
for b in range(samples.shape[0]):
s = samples[b:b+1]
- out = torch.zeros((s.shape[0], out_channels, round(s.shape[2] * upscale_amount), round(s.shape[3] * upscale_amount)), device="cpu")
- out_div = torch.zeros((s.shape[0], out_channels, round(s.shape[2] * upscale_amount), round(s.shape[3] * upscale_amount)), device="cpu")
+ out = torch.zeros((s.shape[0], out_channels, round(s.shape[2] * upscale_amount), round(s.shape[3] * upscale_amount)), device=output_device)
+ out_div = torch.zeros((s.shape[0], out_channels, round(s.shape[2] * upscale_amount), round(s.shape[3] * upscale_amount)), device=output_device)
for y in range(0, s.shape[2], tile_y - overlap):
for x in range(0, s.shape[3], tile_x - overlap):
s_in = s[:,:,y:y+tile_y,x:x+tile_x]
- ps = function(s_in).cpu()
+ ps = function(s_in).to(output_device)
mask = torch.ones_like(ps)
feather = round(overlap * upscale_amount)
for t in range(feather):
diff --git a/comfy_extras/nodes_canny.py b/comfy_extras/nodes_canny.py
index 94d453f2c..730dded5f 100644
--- a/comfy_extras/nodes_canny.py
+++ b/comfy_extras/nodes_canny.py
@@ -291,7 +291,7 @@ class Canny:
def detect_edge(self, image, low_threshold, high_threshold):
output = canny(image.to(comfy.model_management.get_torch_device()).movedim(-1, 1), low_threshold, high_threshold)
- img_out = output[1].cpu().repeat(1, 3, 1, 1).movedim(1, -1)
+ img_out = output[1].to(comfy.model_management.intermediate_device()).repeat(1, 3, 1, 1).movedim(1, -1)
return (img_out,)
NODE_CLASS_MAPPINGS = {
diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.py
index 154ecd0d2..d5f9ba007 100644
--- a/comfy_extras/nodes_custom_sampler.py
+++ b/comfy_extras/nodes_custom_sampler.py
@@ -13,15 +13,21 @@ class BasicScheduler:
{"model": ("MODEL",),
"scheduler": (comfy.samplers.SCHEDULER_NAMES, ),
"steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
+ "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
}
}
RETURN_TYPES = ("SIGMAS",)
- CATEGORY = "sampling/custom_sampling"
+ CATEGORY = "sampling/custom_sampling/schedulers"
FUNCTION = "get_sigmas"
- def get_sigmas(self, model, scheduler, steps):
- sigmas = comfy.samplers.calculate_sigmas_scheduler(model.model, scheduler, steps).cpu()
+ def get_sigmas(self, model, scheduler, steps, denoise):
+ total_steps = steps
+ if denoise < 1.0:
+ total_steps = int(steps/denoise)
+
+ sigmas = comfy.samplers.calculate_sigmas_scheduler(model.model, scheduler, total_steps).cpu()
+ sigmas = sigmas[-(steps + 1):]
return (sigmas, )
@@ -36,7 +42,7 @@ class KarrasScheduler:
}
}
RETURN_TYPES = ("SIGMAS",)
- CATEGORY = "sampling/custom_sampling"
+ CATEGORY = "sampling/custom_sampling/schedulers"
FUNCTION = "get_sigmas"
@@ -54,7 +60,7 @@ class ExponentialScheduler:
}
}
RETURN_TYPES = ("SIGMAS",)
- CATEGORY = "sampling/custom_sampling"
+ CATEGORY = "sampling/custom_sampling/schedulers"
FUNCTION = "get_sigmas"
@@ -73,7 +79,7 @@ class PolyexponentialScheduler:
}
}
RETURN_TYPES = ("SIGMAS",)
- CATEGORY = "sampling/custom_sampling"
+ CATEGORY = "sampling/custom_sampling/schedulers"
FUNCTION = "get_sigmas"
@@ -81,6 +87,27 @@ class PolyexponentialScheduler:
sigmas = k_diffusion_sampling.get_sigmas_polyexponential(n=steps, sigma_min=sigma_min, sigma_max=sigma_max, rho=rho)
return (sigmas, )
+class SDTurboScheduler:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"model": ("MODEL",),
+ "steps": ("INT", {"default": 1, "min": 1, "max": 10}),
+ "denoise": ("FLOAT", {"default": 1.0, "min": 0, "max": 1.0, "step": 0.01}),
+ }
+ }
+ RETURN_TYPES = ("SIGMAS",)
+ CATEGORY = "sampling/custom_sampling/schedulers"
+
+ FUNCTION = "get_sigmas"
+
+ def get_sigmas(self, model, steps, denoise):
+ start_step = 10 - int(10 * denoise)
+ timesteps = torch.flip(torch.arange(1, 11) * 100 - 1, (0,))[start_step:start_step + steps]
+ sigmas = model.model.model_sampling.sigma(timesteps)
+ sigmas = torch.cat([sigmas, sigmas.new_zeros([1])])
+ return (sigmas, )
+
class VPScheduler:
@classmethod
def INPUT_TYPES(s):
@@ -92,7 +119,7 @@ class VPScheduler:
}
}
RETURN_TYPES = ("SIGMAS",)
- CATEGORY = "sampling/custom_sampling"
+ CATEGORY = "sampling/custom_sampling/schedulers"
FUNCTION = "get_sigmas"
@@ -109,7 +136,7 @@ class SplitSigmas:
}
}
RETURN_TYPES = ("SIGMAS","SIGMAS")
- CATEGORY = "sampling/custom_sampling"
+ CATEGORY = "sampling/custom_sampling/sigmas"
FUNCTION = "get_sigmas"
@@ -118,6 +145,24 @@ class SplitSigmas:
sigmas2 = sigmas[step:]
return (sigmas1, sigmas2)
+class FlipSigmas:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"sigmas": ("SIGMAS", ),
+ }
+ }
+ RETURN_TYPES = ("SIGMAS",)
+ CATEGORY = "sampling/custom_sampling/sigmas"
+
+ FUNCTION = "get_sigmas"
+
+ def get_sigmas(self, sigmas):
+ sigmas = sigmas.flip(0)
+ if sigmas[0] == 0:
+ sigmas[0] = 0.0001
+ return (sigmas,)
+
class KSamplerSelect:
@classmethod
def INPUT_TYPES(s):
@@ -126,12 +171,12 @@ class KSamplerSelect:
}
}
RETURN_TYPES = ("SAMPLER",)
- CATEGORY = "sampling/custom_sampling"
+ CATEGORY = "sampling/custom_sampling/samplers"
FUNCTION = "get_sampler"
def get_sampler(self, sampler_name):
- sampler = comfy.samplers.sampler_class(sampler_name)()
+ sampler = comfy.samplers.sampler_object(sampler_name)
return (sampler, )
class SamplerDPMPP_2M_SDE:
@@ -145,7 +190,7 @@ class SamplerDPMPP_2M_SDE:
}
}
RETURN_TYPES = ("SAMPLER",)
- CATEGORY = "sampling/custom_sampling"
+ CATEGORY = "sampling/custom_sampling/samplers"
FUNCTION = "get_sampler"
@@ -154,7 +199,7 @@ class SamplerDPMPP_2M_SDE:
sampler_name = "dpmpp_2m_sde"
else:
sampler_name = "dpmpp_2m_sde_gpu"
- sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "solver_type": solver_type})()
+ sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "solver_type": solver_type})
return (sampler, )
@@ -169,7 +214,7 @@ class SamplerDPMPP_SDE:
}
}
RETURN_TYPES = ("SAMPLER",)
- CATEGORY = "sampling/custom_sampling"
+ CATEGORY = "sampling/custom_sampling/samplers"
FUNCTION = "get_sampler"
@@ -178,7 +223,7 @@ class SamplerDPMPP_SDE:
sampler_name = "dpmpp_sde"
else:
sampler_name = "dpmpp_sde_gpu"
- sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "r": r})()
+ sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "r": r})
return (sampler, )
class SamplerCustom:
@@ -234,13 +279,15 @@ class SamplerCustom:
NODE_CLASS_MAPPINGS = {
"SamplerCustom": SamplerCustom,
+ "BasicScheduler": BasicScheduler,
"KarrasScheduler": KarrasScheduler,
"ExponentialScheduler": ExponentialScheduler,
"PolyexponentialScheduler": PolyexponentialScheduler,
"VPScheduler": VPScheduler,
+ "SDTurboScheduler": SDTurboScheduler,
"KSamplerSelect": KSamplerSelect,
"SamplerDPMPP_2M_SDE": SamplerDPMPP_2M_SDE,
"SamplerDPMPP_SDE": SamplerDPMPP_SDE,
- "BasicScheduler": BasicScheduler,
"SplitSigmas": SplitSigmas,
+ "FlipSigmas": FlipSigmas,
}
diff --git a/comfy_extras/nodes_hypertile.py b/comfy_extras/nodes_hypertile.py
index 0d7d4c954..e7446b2e5 100644
--- a/comfy_extras/nodes_hypertile.py
+++ b/comfy_extras/nodes_hypertile.py
@@ -2,9 +2,10 @@
import math
from einops import rearrange
-import random
+# Use torch rng for consistency across generations
+from torch import randint
-def random_divisor(value: int, min_value: int, /, max_options: int = 1, counter = 0) -> int:
+def random_divisor(value: int, min_value: int, /, max_options: int = 1) -> int:
min_value = min(min_value, value)
# All big divisors of value (inclusive)
@@ -12,8 +13,10 @@ def random_divisor(value: int, min_value: int, /, max_options: int = 1, counter
ns = [value // i for i in divisors[:max_options]] # has at least 1 element
- random.seed(counter)
- idx = random.randint(0, len(ns) - 1)
+ if len(ns) - 1 > 0:
+ idx = randint(low=0, high=len(ns) - 1, size=(1,)).item()
+ else:
+ idx = 0
return ns[idx]
@@ -42,7 +45,6 @@ class HyperTile:
latent_tile_size = max(32, tile_size) // 8
self.temp = None
- self.counter = 1
def hypertile_in(q, k, v, extra_options):
if q.shape[-1] in apply_to:
@@ -53,10 +55,8 @@ class HyperTile:
h, w = round(math.sqrt(hw * aspect_ratio)), round(math.sqrt(hw / aspect_ratio))
factor = 2**((q.shape[-1] // model_channels) - 1) if scale_depth else 1
- nh = random_divisor(h, latent_tile_size * factor, swap_size, self.counter)
- self.counter += 1
- nw = random_divisor(w, latent_tile_size * factor, swap_size, self.counter)
- self.counter += 1
+ nh = random_divisor(h, latent_tile_size * factor, swap_size)
+ nw = random_divisor(w, latent_tile_size * factor, swap_size)
if nh * nw > 1:
q = rearrange(q, "b (nh h nw w) c -> (b nh nw) (h w) c", h=h // nh, w=w // nw, nh=nh, nw=nw)
diff --git a/comfy_extras/nodes_images.py b/comfy_extras/nodes_images.py
new file mode 100644
index 000000000..aa80f5269
--- /dev/null
+++ b/comfy_extras/nodes_images.py
@@ -0,0 +1,175 @@
+import nodes
+import folder_paths
+from comfy.cli_args import args
+
+from PIL import Image
+from PIL.PngImagePlugin import PngInfo
+
+import numpy as np
+import json
+import os
+
+MAX_RESOLUTION = nodes.MAX_RESOLUTION
+
+class ImageCrop:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "image": ("IMAGE",),
+ "width": ("INT", {"default": 512, "min": 1, "max": MAX_RESOLUTION, "step": 1}),
+ "height": ("INT", {"default": 512, "min": 1, "max": MAX_RESOLUTION, "step": 1}),
+ "x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}),
+ "y": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}),
+ }}
+ RETURN_TYPES = ("IMAGE",)
+ FUNCTION = "crop"
+
+ CATEGORY = "image/transform"
+
+ def crop(self, image, width, height, x, y):
+ x = min(x, image.shape[2] - 1)
+ y = min(y, image.shape[1] - 1)
+ to_x = width + x
+ to_y = height + y
+ img = image[:,y:to_y, x:to_x, :]
+ return (img,)
+
+class RepeatImageBatch:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "image": ("IMAGE",),
+ "amount": ("INT", {"default": 1, "min": 1, "max": 64}),
+ }}
+ RETURN_TYPES = ("IMAGE",)
+ FUNCTION = "repeat"
+
+ CATEGORY = "image/batch"
+
+ def repeat(self, image, amount):
+ s = image.repeat((amount, 1,1,1))
+ return (s,)
+
+class SaveAnimatedWEBP:
+ def __init__(self):
+ self.output_dir = folder_paths.get_output_directory()
+ self.type = "output"
+ self.prefix_append = ""
+
+ methods = {"default": 4, "fastest": 0, "slowest": 6}
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"images": ("IMAGE", ),
+ "filename_prefix": ("STRING", {"default": "ComfyUI"}),
+ "fps": ("FLOAT", {"default": 6.0, "min": 0.01, "max": 1000.0, "step": 0.01}),
+ "lossless": ("BOOLEAN", {"default": True}),
+ "quality": ("INT", {"default": 80, "min": 0, "max": 100}),
+ "method": (list(s.methods.keys()),),
+ # "num_frames": ("INT", {"default": 0, "min": 0, "max": 8192}),
+ },
+ "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
+ }
+
+ RETURN_TYPES = ()
+ FUNCTION = "save_images"
+
+ OUTPUT_NODE = True
+
+ CATEGORY = "image/animation"
+
+ def save_images(self, images, fps, filename_prefix, lossless, quality, method, num_frames=0, prompt=None, extra_pnginfo=None):
+ method = self.methods.get(method)
+ filename_prefix += self.prefix_append
+ full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0])
+ results = list()
+ pil_images = []
+ for image in images:
+ i = 255. * image.cpu().numpy()
+ img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
+ pil_images.append(img)
+
+ metadata = pil_images[0].getexif()
+ if not args.disable_metadata:
+ if prompt is not None:
+ metadata[0x0110] = "prompt:{}".format(json.dumps(prompt))
+ if extra_pnginfo is not None:
+ inital_exif = 0x010f
+ for x in extra_pnginfo:
+ metadata[inital_exif] = "{}:{}".format(x, json.dumps(extra_pnginfo[x]))
+ inital_exif -= 1
+
+ if num_frames == 0:
+ num_frames = len(pil_images)
+
+ c = len(pil_images)
+ for i in range(0, c, num_frames):
+ file = f"{filename}_{counter:05}_.webp"
+ pil_images[i].save(os.path.join(full_output_folder, file), save_all=True, duration=int(1000.0/fps), append_images=pil_images[i + 1:i + num_frames], exif=metadata, lossless=lossless, quality=quality, method=method)
+ results.append({
+ "filename": file,
+ "subfolder": subfolder,
+ "type": self.type
+ })
+ counter += 1
+
+ animated = num_frames != 1
+ return { "ui": { "images": results, "animated": (animated,) } }
+
+class SaveAnimatedPNG:
+ def __init__(self):
+ self.output_dir = folder_paths.get_output_directory()
+ self.type = "output"
+ self.prefix_append = ""
+
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"images": ("IMAGE", ),
+ "filename_prefix": ("STRING", {"default": "ComfyUI"}),
+ "fps": ("FLOAT", {"default": 6.0, "min": 0.01, "max": 1000.0, "step": 0.01}),
+ "compress_level": ("INT", {"default": 4, "min": 0, "max": 9})
+ },
+ "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
+ }
+
+ RETURN_TYPES = ()
+ FUNCTION = "save_images"
+
+ OUTPUT_NODE = True
+
+ CATEGORY = "image/animation"
+
+ def save_images(self, images, fps, compress_level, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None):
+ filename_prefix += self.prefix_append
+ full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0])
+ results = list()
+ pil_images = []
+ for image in images:
+ i = 255. * image.cpu().numpy()
+ img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
+ pil_images.append(img)
+
+ metadata = None
+ if not args.disable_metadata:
+ metadata = PngInfo()
+ if prompt is not None:
+ metadata.add(b"comf", "prompt".encode("latin-1", "strict") + b"\0" + json.dumps(prompt).encode("latin-1", "strict"), after_idat=True)
+ if extra_pnginfo is not None:
+ for x in extra_pnginfo:
+ metadata.add(b"comf", x.encode("latin-1", "strict") + b"\0" + json.dumps(extra_pnginfo[x]).encode("latin-1", "strict"), after_idat=True)
+
+ file = f"{filename}_{counter:05}_.png"
+ pil_images[0].save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=compress_level, save_all=True, duration=int(1000.0/fps), append_images=pil_images[1:])
+ results.append({
+ "filename": file,
+ "subfolder": subfolder,
+ "type": self.type
+ })
+
+ return { "ui": { "images": results, "animated": (True,)} }
+
+NODE_CLASS_MAPPINGS = {
+ "ImageCrop": ImageCrop,
+ "RepeatImageBatch": RepeatImageBatch,
+ "SaveAnimatedWEBP": SaveAnimatedWEBP,
+ "SaveAnimatedPNG": SaveAnimatedPNG,
+}
diff --git a/comfy_extras/nodes_latent.py b/comfy_extras/nodes_latent.py
index 001de39fc..2eefc4c55 100644
--- a/comfy_extras/nodes_latent.py
+++ b/comfy_extras/nodes_latent.py
@@ -1,10 +1,9 @@
import comfy.utils
+import torch
def reshape_latent_to(target_shape, latent):
if latent.shape[1:] != target_shape[1:]:
- latent.movedim(1, -1)
latent = comfy.utils.common_upscale(latent, target_shape[3], target_shape[2], "bilinear", "center")
- latent.movedim(-1, 1)
return comfy.utils.repeat_to_batch_size(latent, target_shape[0])
@@ -67,8 +66,66 @@ class LatentMultiply:
samples_out["samples"] = s1 * multiplier
return (samples_out,)
+class LatentInterpolate:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "samples1": ("LATENT",),
+ "samples2": ("LATENT",),
+ "ratio": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
+ }}
+
+ RETURN_TYPES = ("LATENT",)
+ FUNCTION = "op"
+
+ CATEGORY = "latent/advanced"
+
+ def op(self, samples1, samples2, ratio):
+ samples_out = samples1.copy()
+
+ s1 = samples1["samples"]
+ s2 = samples2["samples"]
+
+ s2 = reshape_latent_to(s1.shape, s2)
+
+ m1 = torch.linalg.vector_norm(s1, dim=(1))
+ m2 = torch.linalg.vector_norm(s2, dim=(1))
+
+ s1 = torch.nan_to_num(s1 / m1)
+ s2 = torch.nan_to_num(s2 / m2)
+
+ t = (s1 * ratio + s2 * (1.0 - ratio))
+ mt = torch.linalg.vector_norm(t, dim=(1))
+ st = torch.nan_to_num(t / mt)
+
+ samples_out["samples"] = st * (m1 * ratio + m2 * (1.0 - ratio))
+ return (samples_out,)
+
+class LatentBatch:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "samples1": ("LATENT",), "samples2": ("LATENT",)}}
+
+ RETURN_TYPES = ("LATENT",)
+ FUNCTION = "batch"
+
+ CATEGORY = "latent/batch"
+
+ def batch(self, samples1, samples2):
+ samples_out = samples1.copy()
+ s1 = samples1["samples"]
+ s2 = samples2["samples"]
+
+ if s1.shape[1:] != s2.shape[1:]:
+ s2 = comfy.utils.common_upscale(s2, s1.shape[3], s1.shape[2], "bilinear", "center")
+ s = torch.cat((s1, s2), dim=0)
+ samples_out["samples"] = s
+ samples_out["batch_index"] = samples1.get("batch_index", [x for x in range(0, s1.shape[0])]) + samples2.get("batch_index", [x for x in range(0, s2.shape[0])])
+ return (samples_out,)
+
NODE_CLASS_MAPPINGS = {
"LatentAdd": LatentAdd,
"LatentSubtract": LatentSubtract,
"LatentMultiply": LatentMultiply,
+ "LatentInterpolate": LatentInterpolate,
+ "LatentBatch": LatentBatch,
}
diff --git a/comfy_extras/nodes_mask.py b/comfy_extras/nodes_mask.py
index d8c65c2b6..a7d164bf7 100644
--- a/comfy_extras/nodes_mask.py
+++ b/comfy_extras/nodes_mask.py
@@ -6,6 +6,7 @@ import comfy.utils
from nodes import MAX_RESOLUTION
def composite(destination, source, x, y, mask = None, multiplier = 8, resize_source = False):
+ source = source.to(destination.device)
if resize_source:
source = torch.nn.functional.interpolate(source, size=(destination.shape[2], destination.shape[3]), mode="bilinear")
@@ -20,7 +21,7 @@ def composite(destination, source, x, y, mask = None, multiplier = 8, resize_sou
if mask is None:
mask = torch.ones_like(source)
else:
- mask = mask.clone()
+ mask = mask.to(destination.device, copy=True)
mask = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(source.shape[2], source.shape[3]), mode="bilinear")
mask = comfy.utils.repeat_to_batch_size(mask, source.shape[0])
diff --git a/comfy_extras/nodes_model_advanced.py b/comfy_extras/nodes_model_advanced.py
index 399123eaa..541ce8fa5 100644
--- a/comfy_extras/nodes_model_advanced.py
+++ b/comfy_extras/nodes_model_advanced.py
@@ -17,56 +17,32 @@ class LCM(comfy.model_sampling.EPS):
return c_out * x0 + c_skip * model_input
-class ModelSamplingDiscreteLCM(torch.nn.Module):
- def __init__(self):
- super().__init__()
- self.sigma_data = 1.0
- timesteps = 1000
- beta_start = 0.00085
- beta_end = 0.012
+class ModelSamplingDiscreteDistilled(comfy.model_sampling.ModelSamplingDiscrete):
+ original_timesteps = 50
- betas = torch.linspace(beta_start**0.5, beta_end**0.5, timesteps, dtype=torch.float32) ** 2
- alphas = 1.0 - betas
- alphas_cumprod = torch.cumprod(alphas, dim=0)
+ def __init__(self, model_config=None):
+ super().__init__(model_config)
- original_timesteps = 50
- self.skip_steps = timesteps // original_timesteps
+ self.skip_steps = self.num_timesteps // self.original_timesteps
+ sigmas_valid = torch.zeros((self.original_timesteps), dtype=torch.float32)
+ for x in range(self.original_timesteps):
+ sigmas_valid[self.original_timesteps - 1 - x] = self.sigmas[self.num_timesteps - 1 - x * self.skip_steps]
- alphas_cumprod_valid = torch.zeros((original_timesteps), dtype=torch.float32)
- for x in range(original_timesteps):
- alphas_cumprod_valid[original_timesteps - 1 - x] = alphas_cumprod[timesteps - 1 - x * self.skip_steps]
-
- sigmas = ((1 - alphas_cumprod_valid) / alphas_cumprod_valid) ** 0.5
- self.set_sigmas(sigmas)
-
- def set_sigmas(self, sigmas):
- self.register_buffer('sigmas', sigmas)
- self.register_buffer('log_sigmas', sigmas.log())
-
- @property
- def sigma_min(self):
- return self.sigmas[0]
-
- @property
- def sigma_max(self):
- return self.sigmas[-1]
+ self.set_sigmas(sigmas_valid)
def timestep(self, sigma):
log_sigma = sigma.log()
dists = log_sigma.to(self.log_sigmas.device) - self.log_sigmas[:, None]
- return dists.abs().argmin(dim=0).view(sigma.shape) * self.skip_steps + (self.skip_steps - 1)
+ return (dists.abs().argmin(dim=0).view(sigma.shape) * self.skip_steps + (self.skip_steps - 1)).to(sigma.device)
def sigma(self, timestep):
- t = torch.clamp(((timestep - (self.skip_steps - 1)) / self.skip_steps).float(), min=0, max=(len(self.sigmas) - 1))
+ t = torch.clamp(((timestep.float().to(self.log_sigmas.device) - (self.skip_steps - 1)) / self.skip_steps).float(), min=0, max=(len(self.sigmas) - 1))
low_idx = t.floor().long()
high_idx = t.ceil().long()
w = t.frac()
log_sigma = (1 - w) * self.log_sigmas[low_idx] + w * self.log_sigmas[high_idx]
- return log_sigma.exp()
-
- def percent_to_sigma(self, percent):
- return self.sigma(torch.tensor(percent * 999.0))
+ return log_sigma.exp().to(timestep.device)
def rescale_zero_terminal_snr_sigmas(sigmas):
@@ -111,18 +87,48 @@ class ModelSamplingDiscrete:
sampling_type = comfy.model_sampling.V_PREDICTION
elif sampling == "lcm":
sampling_type = LCM
- sampling_base = ModelSamplingDiscreteLCM
+ sampling_base = ModelSamplingDiscreteDistilled
class ModelSamplingAdvanced(sampling_base, sampling_type):
pass
- model_sampling = ModelSamplingAdvanced()
+ model_sampling = ModelSamplingAdvanced(model.model.model_config)
if zsnr:
model_sampling.set_sigmas(rescale_zero_terminal_snr_sigmas(model_sampling.sigmas))
m.add_object_patch("model_sampling", model_sampling)
return (m, )
+class ModelSamplingContinuousEDM:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "sampling": (["v_prediction", "eps"],),
+ "sigma_max": ("FLOAT", {"default": 120.0, "min": 0.0, "max": 1000.0, "step":0.001, "round": False}),
+ "sigma_min": ("FLOAT", {"default": 0.002, "min": 0.0, "max": 1000.0, "step":0.001, "round": False}),
+ }}
+
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "advanced/model"
+
+ def patch(self, model, sampling, sigma_max, sigma_min):
+ m = model.clone()
+
+ if sampling == "eps":
+ sampling_type = comfy.model_sampling.EPS
+ elif sampling == "v_prediction":
+ sampling_type = comfy.model_sampling.V_PREDICTION
+
+ class ModelSamplingAdvanced(comfy.model_sampling.ModelSamplingContinuousEDM, sampling_type):
+ pass
+
+ model_sampling = ModelSamplingAdvanced(model.model.model_config)
+ model_sampling.set_sigma_range(sigma_min, sigma_max)
+ m.add_object_patch("model_sampling", model_sampling)
+ return (m, )
+
class RescaleCFG:
@classmethod
def INPUT_TYPES(s):
@@ -164,5 +170,6 @@ class RescaleCFG:
NODE_CLASS_MAPPINGS = {
"ModelSamplingDiscrete": ModelSamplingDiscrete,
+ "ModelSamplingContinuousEDM": ModelSamplingContinuousEDM,
"RescaleCFG": RescaleCFG,
}
diff --git a/comfy_extras/nodes_model_downscale.py b/comfy_extras/nodes_model_downscale.py
new file mode 100644
index 000000000..48bcc6892
--- /dev/null
+++ b/comfy_extras/nodes_model_downscale.py
@@ -0,0 +1,53 @@
+import torch
+import comfy.utils
+
+class PatchModelAddDownscale:
+ upscale_methods = ["bicubic", "nearest-exact", "bilinear", "area", "bislerp"]
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "block_number": ("INT", {"default": 3, "min": 1, "max": 32, "step": 1}),
+ "downscale_factor": ("FLOAT", {"default": 2.0, "min": 0.1, "max": 9.0, "step": 0.001}),
+ "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}),
+ "end_percent": ("FLOAT", {"default": 0.35, "min": 0.0, "max": 1.0, "step": 0.001}),
+ "downscale_after_skip": ("BOOLEAN", {"default": True}),
+ "downscale_method": (s.upscale_methods,),
+ "upscale_method": (s.upscale_methods,),
+ }}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "_for_testing"
+
+ def patch(self, model, block_number, downscale_factor, start_percent, end_percent, downscale_after_skip, downscale_method, upscale_method):
+ sigma_start = model.model.model_sampling.percent_to_sigma(start_percent)
+ sigma_end = model.model.model_sampling.percent_to_sigma(end_percent)
+
+ def input_block_patch(h, transformer_options):
+ if transformer_options["block"][1] == block_number:
+ sigma = transformer_options["sigmas"][0].item()
+ if sigma <= sigma_start and sigma >= sigma_end:
+ h = comfy.utils.common_upscale(h, round(h.shape[-1] * (1.0 / downscale_factor)), round(h.shape[-2] * (1.0 / downscale_factor)), downscale_method, "disabled")
+ return h
+
+ def output_block_patch(h, hsp, transformer_options):
+ if h.shape[2] != hsp.shape[2]:
+ h = comfy.utils.common_upscale(h, hsp.shape[-1], hsp.shape[-2], upscale_method, "disabled")
+ return h, hsp
+
+ m = model.clone()
+ if downscale_after_skip:
+ m.set_model_input_block_patch_after_skip(input_block_patch)
+ else:
+ m.set_model_input_block_patch(input_block_patch)
+ m.set_model_output_block_patch(output_block_patch)
+ return (m, )
+
+NODE_CLASS_MAPPINGS = {
+ "PatchModelAddDownscale": PatchModelAddDownscale,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+ # Sampling
+ "PatchModelAddDownscale": "PatchModelAddDownscale (Kohya Deep Shrink)",
+}
diff --git a/comfy_extras/nodes_perpneg.py b/comfy_extras/nodes_perpneg.py
new file mode 100644
index 000000000..45e4d418f
--- /dev/null
+++ b/comfy_extras/nodes_perpneg.py
@@ -0,0 +1,55 @@
+import torch
+import comfy.model_management
+import comfy.sample
+import comfy.samplers
+import comfy.utils
+
+
+class PerpNeg:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {"model": ("MODEL", ),
+ "empty_conditioning": ("CONDITIONING", ),
+ "neg_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0}),
+ }}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "_for_testing"
+
+ def patch(self, model, empty_conditioning, neg_scale):
+ m = model.clone()
+ nocond = comfy.sample.convert_cond(empty_conditioning)
+
+ def cfg_function(args):
+ model = args["model"]
+ noise_pred_pos = args["cond_denoised"]
+ noise_pred_neg = args["uncond_denoised"]
+ cond_scale = args["cond_scale"]
+ x = args["input"]
+ sigma = args["sigma"]
+ model_options = args["model_options"]
+ nocond_processed = comfy.samplers.encode_model_conds(model.extra_conds, nocond, x, x.device, "negative")
+
+ (noise_pred_nocond, _) = comfy.samplers.calc_cond_uncond_batch(model, nocond_processed, None, x, sigma, model_options)
+
+ pos = noise_pred_pos - noise_pred_nocond
+ neg = noise_pred_neg - noise_pred_nocond
+ perp = ((torch.mul(pos, neg).sum())/(torch.norm(neg)**2)) * neg
+ perp_neg = perp * neg_scale
+ cfg_result = noise_pred_nocond + cond_scale*(pos - perp_neg)
+ cfg_result = x - cfg_result
+ return cfg_result
+
+ m.set_model_sampler_cfg_function(cfg_function)
+
+ return (m, )
+
+
+NODE_CLASS_MAPPINGS = {
+ "PerpNeg": PerpNeg,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+ "PerpNeg": "Perp-Neg",
+}
diff --git a/comfy_extras/nodes_post_processing.py b/comfy_extras/nodes_post_processing.py
index 12704f545..71660f8a5 100644
--- a/comfy_extras/nodes_post_processing.py
+++ b/comfy_extras/nodes_post_processing.py
@@ -226,7 +226,7 @@ class Sharpen:
batch_size, height, width, channels = image.shape
kernel_size = sharpen_radius * 2 + 1
- kernel = gaussian_kernel(kernel_size, sigma) * -(alpha*10)
+ kernel = gaussian_kernel(kernel_size, sigma, device=image.device) * -(alpha*10)
center = kernel_size // 2
kernel[center, center] = kernel[center, center] - kernel.sum() + 1.0
kernel = kernel.repeat(channels, 1, 1).unsqueeze(1)
diff --git a/comfy_extras/nodes_rebatch.py b/comfy_extras/nodes_rebatch.py
index 88a4ebe29..3010fbd4b 100644
--- a/comfy_extras/nodes_rebatch.py
+++ b/comfy_extras/nodes_rebatch.py
@@ -99,10 +99,40 @@ class LatentRebatch:
return (output_list,)
+class ImageRebatch:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "images": ("IMAGE",),
+ "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
+ }}
+ RETURN_TYPES = ("IMAGE",)
+ INPUT_IS_LIST = True
+ OUTPUT_IS_LIST = (True, )
+
+ FUNCTION = "rebatch"
+
+ CATEGORY = "image/batch"
+
+ def rebatch(self, images, batch_size):
+ batch_size = batch_size[0]
+
+ output_list = []
+ all_images = []
+ for img in images:
+ for i in range(img.shape[0]):
+ all_images.append(img[i:i+1])
+
+ for i in range(0, len(all_images), batch_size):
+ output_list.append(torch.cat(all_images[i:i+batch_size], dim=0))
+
+ return (output_list,)
+
NODE_CLASS_MAPPINGS = {
"RebatchLatents": LatentRebatch,
+ "RebatchImages": ImageRebatch,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RebatchLatents": "Rebatch Latents",
-}
\ No newline at end of file
+ "RebatchImages": "Rebatch Images",
+}
diff --git a/comfy_extras/nodes_sag.py b/comfy_extras/nodes_sag.py
new file mode 100644
index 000000000..450ac3eea
--- /dev/null
+++ b/comfy_extras/nodes_sag.py
@@ -0,0 +1,168 @@
+import torch
+from torch import einsum
+import torch.nn.functional as F
+import math
+
+from einops import rearrange, repeat
+import os
+from comfy.ldm.modules.attention import optimized_attention, _ATTN_PRECISION
+import comfy.samplers
+
+# from comfy/ldm/modules/attention.py
+# but modified to return attention scores as well as output
+def attention_basic_with_sim(q, k, v, heads, mask=None):
+ b, _, dim_head = q.shape
+ dim_head //= heads
+ scale = dim_head ** -0.5
+
+ h = heads
+ q, k, v = map(
+ lambda t: t.unsqueeze(3)
+ .reshape(b, -1, heads, dim_head)
+ .permute(0, 2, 1, 3)
+ .reshape(b * heads, -1, dim_head)
+ .contiguous(),
+ (q, k, v),
+ )
+
+ # force cast to fp32 to avoid overflowing
+ if _ATTN_PRECISION =="fp32":
+ sim = einsum('b i d, b j d -> b i j', q.float(), k.float()) * scale
+ else:
+ sim = einsum('b i d, b j d -> b i j', q, k) * scale
+
+ del q, k
+
+ if mask is not None:
+ mask = rearrange(mask, 'b ... -> b (...)')
+ max_neg_value = -torch.finfo(sim.dtype).max
+ mask = repeat(mask, 'b j -> (b h) () j', h=h)
+ sim.masked_fill_(~mask, max_neg_value)
+
+ # attention, what we cannot get enough of
+ sim = sim.softmax(dim=-1)
+
+ out = einsum('b i j, b j d -> b i d', sim.to(v.dtype), v)
+ out = (
+ out.unsqueeze(0)
+ .reshape(b, heads, -1, dim_head)
+ .permute(0, 2, 1, 3)
+ .reshape(b, -1, heads * dim_head)
+ )
+ return (out, sim)
+
+def create_blur_map(x0, attn, sigma=3.0, threshold=1.0):
+ # reshape and GAP the attention map
+ _, hw1, hw2 = attn.shape
+ b, _, lh, lw = x0.shape
+ attn = attn.reshape(b, -1, hw1, hw2)
+ # Global Average Pool
+ mask = attn.mean(1, keepdim=False).sum(1, keepdim=False) > threshold
+ ratio = math.ceil(math.sqrt(lh * lw / hw1))
+ mid_shape = [math.ceil(lh / ratio), math.ceil(lw / ratio)]
+
+ # Reshape
+ mask = (
+ mask.reshape(b, *mid_shape)
+ .unsqueeze(1)
+ .type(attn.dtype)
+ )
+ # Upsample
+ mask = F.interpolate(mask, (lh, lw))
+
+ blurred = gaussian_blur_2d(x0, kernel_size=9, sigma=sigma)
+ blurred = blurred * mask + x0 * (1 - mask)
+ return blurred
+
+def gaussian_blur_2d(img, kernel_size, sigma):
+ ksize_half = (kernel_size - 1) * 0.5
+
+ x = torch.linspace(-ksize_half, ksize_half, steps=kernel_size)
+
+ pdf = torch.exp(-0.5 * (x / sigma).pow(2))
+
+ x_kernel = pdf / pdf.sum()
+ x_kernel = x_kernel.to(device=img.device, dtype=img.dtype)
+
+ kernel2d = torch.mm(x_kernel[:, None], x_kernel[None, :])
+ kernel2d = kernel2d.expand(img.shape[-3], 1, kernel2d.shape[0], kernel2d.shape[1])
+
+ padding = [kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size // 2]
+
+ img = F.pad(img, padding, mode="reflect")
+ img = F.conv2d(img, kernel2d, groups=img.shape[-3])
+ return img
+
+class SelfAttentionGuidance:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "scale": ("FLOAT", {"default": 0.5, "min": -2.0, "max": 5.0, "step": 0.1}),
+ "blur_sigma": ("FLOAT", {"default": 2.0, "min": 0.0, "max": 10.0, "step": 0.1}),
+ }}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "_for_testing"
+
+ def patch(self, model, scale, blur_sigma):
+ m = model.clone()
+
+ attn_scores = None
+
+ # TODO: make this work properly with chunked batches
+ # currently, we can only save the attn from one UNet call
+ def attn_and_record(q, k, v, extra_options):
+ nonlocal attn_scores
+ # if uncond, save the attention scores
+ heads = extra_options["n_heads"]
+ cond_or_uncond = extra_options["cond_or_uncond"]
+ b = q.shape[0] // len(cond_or_uncond)
+ if 1 in cond_or_uncond:
+ uncond_index = cond_or_uncond.index(1)
+ # do the entire attention operation, but save the attention scores to attn_scores
+ (out, sim) = attention_basic_with_sim(q, k, v, heads=heads)
+ # when using a higher batch size, I BELIEVE the result batch dimension is [uc1, ... ucn, c1, ... cn]
+ n_slices = heads * b
+ attn_scores = sim[n_slices * uncond_index:n_slices * (uncond_index+1)]
+ return out
+ else:
+ return optimized_attention(q, k, v, heads=heads)
+
+ def post_cfg_function(args):
+ nonlocal attn_scores
+ uncond_attn = attn_scores
+
+ sag_scale = scale
+ sag_sigma = blur_sigma
+ sag_threshold = 1.0
+ model = args["model"]
+ uncond_pred = args["uncond_denoised"]
+ uncond = args["uncond"]
+ cfg_result = args["denoised"]
+ sigma = args["sigma"]
+ model_options = args["model_options"]
+ x = args["input"]
+
+ # create the adversarially blurred image
+ degraded = create_blur_map(uncond_pred, uncond_attn, sag_sigma, sag_threshold)
+ degraded_noised = degraded + x - uncond_pred
+ # call into the UNet
+ (sag, _) = comfy.samplers.calc_cond_uncond_batch(model, uncond, None, degraded_noised, sigma, model_options)
+ return cfg_result + (degraded - sag) * sag_scale
+
+ m.set_model_sampler_post_cfg_function(post_cfg_function, disable_cfg1_optimization=True)
+
+ # from diffusers:
+ # unet.mid_block.attentions[0].transformer_blocks[0].attn1.patch
+ m.set_model_attn1_replace(attn_and_record, "middle", 0, 0)
+
+ return (m, )
+
+NODE_CLASS_MAPPINGS = {
+ "SelfAttentionGuidance": SelfAttentionGuidance,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+ "SelfAttentionGuidance": "Self-Attention Guidance",
+}
diff --git a/comfy_extras/nodes_stable3d.py b/comfy_extras/nodes_stable3d.py
new file mode 100644
index 000000000..c6791d8de
--- /dev/null
+++ b/comfy_extras/nodes_stable3d.py
@@ -0,0 +1,58 @@
+import torch
+import nodes
+import comfy.utils
+
+def camera_embeddings(elevation, azimuth):
+ elevation = torch.as_tensor([elevation])
+ azimuth = torch.as_tensor([azimuth])
+ embeddings = torch.stack(
+ [
+ torch.deg2rad(
+ (90 - elevation) - (90)
+ ), # Zero123 polar is 90-elevation
+ torch.sin(torch.deg2rad(azimuth)),
+ torch.cos(torch.deg2rad(azimuth)),
+ torch.deg2rad(
+ 90 - torch.full_like(elevation, 0)
+ ),
+ ], dim=-1).unsqueeze(1)
+
+ return embeddings
+
+
+class StableZero123_Conditioning:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "clip_vision": ("CLIP_VISION",),
+ "init_image": ("IMAGE",),
+ "vae": ("VAE",),
+ "width": ("INT", {"default": 256, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
+ "height": ("INT", {"default": 256, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
+ "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
+ "elevation": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0}),
+ "azimuth": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0}),
+ }}
+ RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
+ RETURN_NAMES = ("positive", "negative", "latent")
+
+ FUNCTION = "encode"
+
+ CATEGORY = "conditioning/3d_models"
+
+ def encode(self, clip_vision, init_image, vae, width, height, batch_size, elevation, azimuth):
+ output = clip_vision.encode_image(init_image)
+ pooled = output.image_embeds.unsqueeze(0)
+ pixels = comfy.utils.common_upscale(init_image.movedim(-1,1), width, height, "bilinear", "center").movedim(1,-1)
+ encode_pixels = pixels[:,:,:,:3]
+ t = vae.encode(encode_pixels)
+ cam_embeds = camera_embeddings(elevation, azimuth)
+ cond = torch.cat([pooled, cam_embeds.repeat((pooled.shape[0], 1, 1))], dim=-1)
+
+ positive = [[cond, {"concat_latent_image": t}]]
+ negative = [[torch.zeros_like(pooled), {"concat_latent_image": torch.zeros_like(t)}]]
+ latent = torch.zeros([batch_size, 4, height // 8, width // 8])
+ return (positive, negative, {"samples":latent})
+
+NODE_CLASS_MAPPINGS = {
+ "StableZero123_Conditioning": StableZero123_Conditioning,
+}
diff --git a/comfy_extras/nodes_video_model.py b/comfy_extras/nodes_video_model.py
new file mode 100644
index 000000000..26a717a38
--- /dev/null
+++ b/comfy_extras/nodes_video_model.py
@@ -0,0 +1,89 @@
+import nodes
+import torch
+import comfy.utils
+import comfy.sd
+import folder_paths
+
+
+class ImageOnlyCheckpointLoader:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "ckpt_name": (folder_paths.get_filename_list("checkpoints"), ),
+ }}
+ RETURN_TYPES = ("MODEL", "CLIP_VISION", "VAE")
+ FUNCTION = "load_checkpoint"
+
+ CATEGORY = "loaders/video_models"
+
+ def load_checkpoint(self, ckpt_name, output_vae=True, output_clip=True):
+ ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name)
+ out = comfy.sd.load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=False, output_clipvision=True, embedding_directory=folder_paths.get_folder_paths("embeddings"))
+ return (out[0], out[3], out[2])
+
+
+class SVD_img2vid_Conditioning:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "clip_vision": ("CLIP_VISION",),
+ "init_image": ("IMAGE",),
+ "vae": ("VAE",),
+ "width": ("INT", {"default": 1024, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
+ "height": ("INT", {"default": 576, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
+ "video_frames": ("INT", {"default": 14, "min": 1, "max": 4096}),
+ "motion_bucket_id": ("INT", {"default": 127, "min": 1, "max": 1023}),
+ "fps": ("INT", {"default": 6, "min": 1, "max": 1024}),
+ "augmentation_level": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.01})
+ }}
+ RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
+ RETURN_NAMES = ("positive", "negative", "latent")
+
+ FUNCTION = "encode"
+
+ CATEGORY = "conditioning/video_models"
+
+ def encode(self, clip_vision, init_image, vae, width, height, video_frames, motion_bucket_id, fps, augmentation_level):
+ output = clip_vision.encode_image(init_image)
+ pooled = output.image_embeds.unsqueeze(0)
+ pixels = comfy.utils.common_upscale(init_image.movedim(-1,1), width, height, "bilinear", "center").movedim(1,-1)
+ encode_pixels = pixels[:,:,:,:3]
+ if augmentation_level > 0:
+ encode_pixels += torch.randn_like(pixels) * augmentation_level
+ t = vae.encode(encode_pixels)
+ positive = [[pooled, {"motion_bucket_id": motion_bucket_id, "fps": fps, "augmentation_level": augmentation_level, "concat_latent_image": t}]]
+ negative = [[torch.zeros_like(pooled), {"motion_bucket_id": motion_bucket_id, "fps": fps, "augmentation_level": augmentation_level, "concat_latent_image": torch.zeros_like(t)}]]
+ latent = torch.zeros([video_frames, 4, height // 8, width // 8])
+ return (positive, negative, {"samples":latent})
+
+class VideoLinearCFGGuidance:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "min_cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.5, "round": 0.01}),
+ }}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "sampling/video_models"
+
+ def patch(self, model, min_cfg):
+ def linear_cfg(args):
+ cond = args["cond"]
+ uncond = args["uncond"]
+ cond_scale = args["cond_scale"]
+
+ scale = torch.linspace(min_cfg, cond_scale, cond.shape[0], device=cond.device).reshape((cond.shape[0], 1, 1, 1))
+ return uncond + scale * (cond - uncond)
+
+ m = model.clone()
+ m.set_model_sampler_cfg_function(linear_cfg)
+ return (m, )
+
+NODE_CLASS_MAPPINGS = {
+ "ImageOnlyCheckpointLoader": ImageOnlyCheckpointLoader,
+ "SVD_img2vid_Conditioning": SVD_img2vid_Conditioning,
+ "VideoLinearCFGGuidance": VideoLinearCFGGuidance,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+ "ImageOnlyCheckpointLoader": "Image Only Checkpoint Loader (img2vid model)",
+}
diff --git a/execution.py b/execution.py
index 918c2bc5c..53ba2e0f8 100644
--- a/execution.py
+++ b/execution.py
@@ -7,6 +7,7 @@ import threading
import heapq
import traceback
import gc
+import inspect
import torch
import nodes
@@ -382,6 +383,8 @@ class PromptExecutor:
for x in executed:
self.old_prompt[x] = copy.deepcopy(prompt[x])
self.server.last_node_id = None
+ if comfy.model_management.DISABLE_SMART_MEMORY:
+ comfy.model_management.unload_all_models()
@@ -400,6 +403,10 @@ def validate_inputs(prompt, item, validated):
errors = []
valid = True
+ validate_function_inputs = []
+ if hasattr(obj_class, "VALIDATE_INPUTS"):
+ validate_function_inputs = inspect.getfullargspec(obj_class.VALIDATE_INPUTS).args
+
for x in required_inputs:
if x not in inputs:
error = {
@@ -529,29 +536,7 @@ def validate_inputs(prompt, item, validated):
errors.append(error)
continue
- if hasattr(obj_class, "VALIDATE_INPUTS"):
- input_data_all = get_input_data(inputs, obj_class, unique_id)
- #ret = obj_class.VALIDATE_INPUTS(**input_data_all)
- ret = map_node_over_list(obj_class, input_data_all, "VALIDATE_INPUTS")
- for i, r in enumerate(ret):
- if r is not True:
- details = f"{x}"
- if r is not False:
- details += f" - {str(r)}"
-
- error = {
- "type": "custom_validation_failed",
- "message": "Custom validation failed for node",
- "details": details,
- "extra_info": {
- "input_name": x,
- "input_config": info,
- "received_value": val,
- }
- }
- errors.append(error)
- continue
- else:
+ if x not in validate_function_inputs:
if isinstance(type_input, list):
if val not in type_input:
input_config = info
@@ -578,6 +563,35 @@ def validate_inputs(prompt, item, validated):
errors.append(error)
continue
+ if len(validate_function_inputs) > 0:
+ input_data_all = get_input_data(inputs, obj_class, unique_id)
+ input_filtered = {}
+ for x in input_data_all:
+ if x in validate_function_inputs:
+ input_filtered[x] = input_data_all[x]
+
+ #ret = obj_class.VALIDATE_INPUTS(**input_filtered)
+ ret = map_node_over_list(obj_class, input_filtered, "VALIDATE_INPUTS")
+ for x in input_filtered:
+ for i, r in enumerate(ret):
+ if r is not True:
+ details = f"{x}"
+ if r is not False:
+ details += f" - {str(r)}"
+
+ error = {
+ "type": "custom_validation_failed",
+ "message": "Custom validation failed for node",
+ "details": details,
+ "extra_info": {
+ "input_name": x,
+ "input_config": info,
+ "received_value": val,
+ }
+ }
+ errors.append(error)
+ continue
+
if len(errors) > 0 or valid is not True:
ret = (False, errors, unique_id)
else:
@@ -681,6 +695,7 @@ def validate_prompt(prompt):
return (True, None, list(good_outputs), node_errors)
+MAXIMUM_HISTORY_SIZE = 10000
class PromptQueue:
def __init__(self, server):
@@ -699,10 +714,12 @@ class PromptQueue:
self.server.queue_updated()
self.not_empty.notify()
- def get(self):
+ def get(self, timeout=None):
with self.not_empty:
while len(self.queue) == 0:
- self.not_empty.wait()
+ self.not_empty.wait(timeout=timeout)
+ if timeout is not None and len(self.queue) == 0:
+ return None
item = heapq.heappop(self.queue)
i = self.task_counter
self.currently_running[i] = copy.deepcopy(item)
@@ -713,6 +730,8 @@ class PromptQueue:
def task_done(self, item_id, outputs):
with self.mutex:
prompt = self.currently_running.pop(item_id)
+ if len(self.history) > MAXIMUM_HISTORY_SIZE:
+ self.history.pop(next(iter(self.history)))
self.history[prompt[1]] = { "prompt": prompt, "outputs": {} }
for o in outputs:
self.history[prompt[1]]["outputs"][o] = outputs[o]
@@ -747,10 +766,20 @@ class PromptQueue:
return True
return False
- def get_history(self, prompt_id=None):
+ def get_history(self, prompt_id=None, max_items=None, offset=-1):
with self.mutex:
if prompt_id is None:
- return copy.deepcopy(self.history)
+ out = {}
+ i = 0
+ if offset < 0 and max_items is not None:
+ offset = len(self.history) - max_items
+ for k in self.history:
+ if i >= offset:
+ out[k] = self.history[k]
+ if max_items is not None and len(out) >= max_items:
+ break
+ i += 1
+ return out
elif prompt_id in self.history:
return {prompt_id: copy.deepcopy(self.history[prompt_id])}
else:
diff --git a/folder_paths.py b/folder_paths.py
index 4a38deec0..a8726d8dd 100644
--- a/folder_paths.py
+++ b/folder_paths.py
@@ -38,7 +38,10 @@ input_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)), "inp
filename_list_cache = {}
if not os.path.exists(input_directory):
- os.makedirs(input_directory)
+ try:
+ os.makedirs(input_directory)
+ except:
+ print("Failed to create input directory")
def set_output_directory(output_dir):
global output_directory
@@ -181,8 +184,7 @@ def cached_filename_list_(folder_name):
if folder_name not in filename_list_cache:
return None
out = filename_list_cache[folder_name]
- if time.perf_counter() < (out[2] + 0.5):
- return out
+
for x in out[1]:
time_modified = out[1][x]
folder = x
@@ -228,8 +230,12 @@ def get_save_image_path(filename_prefix, output_dir, image_width=0, image_height
full_output_folder = os.path.join(output_dir, subfolder)
if os.path.commonpath((output_dir, os.path.abspath(full_output_folder))) != output_dir:
- print("Saving image outside the output folder is not allowed.")
- return {}
+ err = "**** ERROR: Saving image outside the output folder is not allowed." + \
+ "\n full_output_folder: " + os.path.abspath(full_output_folder) + \
+ "\n output_dir: " + output_dir + \
+ "\n commonpath: " + os.path.commonpath((output_dir, os.path.abspath(full_output_folder)))
+ print(err)
+ raise Exception(err)
try:
counter = max(filter(lambda a: a[1][:-1] == filename and a[1][-1] == "_", map(map_filename, os.listdir(full_output_folder))))[0] + 1
diff --git a/latent_preview.py b/latent_preview.py
index 6e758a1a9..61754751e 100644
--- a/latent_preview.py
+++ b/latent_preview.py
@@ -22,10 +22,7 @@ class TAESDPreviewerImpl(LatentPreviewer):
self.taesd = taesd
def decode_latent_to_preview(self, x0):
- x_sample = self.taesd.decoder(x0[:1])[0].detach()
- # x_sample = self.taesd.unscale_latents(x_sample).div(4).add(0.5) # returns value in [-2, 2]
- x_sample = x_sample.sub(0.5).mul(2)
-
+ x_sample = self.taesd.decode(x0[:1])[0].detach()
x_sample = torch.clamp((x_sample + 1.0) / 2.0, min=0.0, max=1.0)
x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2)
x_sample = x_sample.astype(np.uint8)
diff --git a/main.py b/main.py
index 1100a07f4..bcf738729 100644
--- a/main.py
+++ b/main.py
@@ -64,6 +64,10 @@ if __name__ == "__main__":
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.cuda_device)
print("Set cuda device to:", args.cuda_device)
+ if args.deterministic:
+ if 'CUBLAS_WORKSPACE_CONFIG' not in os.environ:
+ os.environ['CUBLAS_WORKSPACE_CONFIG'] = ":4096:8"
+
import cuda_malloc
import comfy.utils
@@ -88,18 +92,39 @@ def cuda_malloc_warning():
def prompt_worker(q, server):
e = execution.PromptExecutor(server)
- while True:
- item, item_id = q.get()
- execution_start_time = time.perf_counter()
- prompt_id = item[1]
- e.execute(item[2], prompt_id, item[3], item[4])
- q.task_done(item_id, e.outputs_ui)
- if server.client_id is not None:
- server.send_sync("executing", { "node": None, "prompt_id": prompt_id }, server.client_id)
+ last_gc_collect = 0
+ need_gc = False
+ gc_collect_interval = 10.0
- print("Prompt executed in {:.2f} seconds".format(time.perf_counter() - execution_start_time))
- gc.collect()
- comfy.model_management.soft_empty_cache()
+ while True:
+ timeout = None
+ if need_gc:
+ timeout = max(gc_collect_interval - (current_time - last_gc_collect), 0.0)
+
+ queue_item = q.get(timeout=timeout)
+ if queue_item is not None:
+ item, item_id = queue_item
+ execution_start_time = time.perf_counter()
+ prompt_id = item[1]
+ server.last_prompt_id = prompt_id
+
+ e.execute(item[2], prompt_id, item[3], item[4])
+ need_gc = True
+ q.task_done(item_id, e.outputs_ui)
+ if server.client_id is not None:
+ server.send_sync("executing", { "node": None, "prompt_id": prompt_id }, server.client_id)
+
+ current_time = time.perf_counter()
+ execution_time = current_time - execution_start_time
+ print("Prompt executed in {:.2f} seconds".format(execution_time))
+
+ if need_gc:
+ current_time = time.perf_counter()
+ if (current_time - last_gc_collect) > gc_collect_interval:
+ gc.collect()
+ comfy.model_management.soft_empty_cache()
+ last_gc_collect = current_time
+ need_gc = False
async def run(server, address='', port=8188, verbose=True, call_on_start=None):
await asyncio.gather(server.start(address, port, verbose, call_on_start), server.publish_loop())
@@ -108,7 +133,9 @@ async def run(server, address='', port=8188, verbose=True, call_on_start=None):
def hijack_progress(server):
def hook(value, total, preview_image):
comfy.model_management.throw_exception_if_processing_interrupted()
- server.send_sync("progress", {"value": value, "max": total}, server.client_id)
+ progress = {"value": value, "max": total, "prompt_id": server.last_prompt_id, "node": server.last_node_id}
+
+ server.send_sync("progress", progress, server.client_id)
if preview_image is not None:
server.send_sync(BinaryEventTypes.UNENCODED_PREVIEW_IMAGE, preview_image, server.client_id)
comfy.utils.set_progress_bar_global_hook(hook)
diff --git a/nodes.py b/nodes.py
index 2bbfd8fe8..8e3ec947c 100644
--- a/nodes.py
+++ b/nodes.py
@@ -9,7 +9,7 @@ import math
import time
import random
-from PIL import Image, ImageOps
+from PIL import Image, ImageOps, ImageSequence
from PIL.PngImagePlugin import PngInfo
import numpy as np
import safetensors.torch
@@ -248,8 +248,8 @@ class ConditioningSetTimestepRange:
c = []
for t in conditioning:
d = t[1].copy()
- d['start_percent'] = 1.0 - start
- d['end_percent'] = 1.0 - end
+ d['start_percent'] = start
+ d['end_percent'] = end
n = [t[0], d]
c.append(n)
return (c, )
@@ -572,10 +572,69 @@ class LoraLoader:
model_lora, clip_lora = comfy.sd.load_lora_for_models(model, clip, lora, strength_model, strength_clip)
return (model_lora, clip_lora)
-class VAELoader:
+class LoraLoaderModelOnly(LoraLoader):
@classmethod
def INPUT_TYPES(s):
- return {"required": { "vae_name": (folder_paths.get_filename_list("vae"), )}}
+ return {"required": { "model": ("MODEL",),
+ "lora_name": (folder_paths.get_filename_list("loras"), ),
+ "strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}),
+ }}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "load_lora_model_only"
+
+ def load_lora_model_only(self, model, lora_name, strength_model):
+ return (self.load_lora(model, None, lora_name, strength_model, 0)[0],)
+
+class VAELoader:
+ @staticmethod
+ def vae_list():
+ vaes = folder_paths.get_filename_list("vae")
+ approx_vaes = folder_paths.get_filename_list("vae_approx")
+ sdxl_taesd_enc = False
+ sdxl_taesd_dec = False
+ sd1_taesd_enc = False
+ sd1_taesd_dec = False
+
+ for v in approx_vaes:
+ if v.startswith("taesd_decoder."):
+ sd1_taesd_dec = True
+ elif v.startswith("taesd_encoder."):
+ sd1_taesd_enc = True
+ elif v.startswith("taesdxl_decoder."):
+ sdxl_taesd_dec = True
+ elif v.startswith("taesdxl_encoder."):
+ sdxl_taesd_enc = True
+ if sd1_taesd_dec and sd1_taesd_enc:
+ vaes.append("taesd")
+ if sdxl_taesd_dec and sdxl_taesd_enc:
+ vaes.append("taesdxl")
+ return vaes
+
+ @staticmethod
+ def load_taesd(name):
+ sd = {}
+ approx_vaes = folder_paths.get_filename_list("vae_approx")
+
+ encoder = next(filter(lambda a: a.startswith("{}_encoder.".format(name)), approx_vaes))
+ decoder = next(filter(lambda a: a.startswith("{}_decoder.".format(name)), approx_vaes))
+
+ enc = comfy.utils.load_torch_file(folder_paths.get_full_path("vae_approx", encoder))
+ for k in enc:
+ sd["taesd_encoder.{}".format(k)] = enc[k]
+
+ dec = comfy.utils.load_torch_file(folder_paths.get_full_path("vae_approx", decoder))
+ for k in dec:
+ sd["taesd_decoder.{}".format(k)] = dec[k]
+
+ if name == "taesd":
+ sd["vae_scale"] = torch.tensor(0.18215)
+ elif name == "taesdxl":
+ sd["vae_scale"] = torch.tensor(0.13025)
+ return sd
+
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "vae_name": (s.vae_list(), )}}
RETURN_TYPES = ("VAE",)
FUNCTION = "load_vae"
@@ -583,8 +642,11 @@ class VAELoader:
#TODO: scale factor?
def load_vae(self, vae_name):
- vae_path = folder_paths.get_full_path("vae", vae_name)
- sd = comfy.utils.load_torch_file(vae_path)
+ if vae_name in ["taesd", "taesdxl"]:
+ sd = self.load_taesd(vae_name)
+ else:
+ vae_path = folder_paths.get_full_path("vae", vae_name)
+ sd = comfy.utils.load_torch_file(vae_path)
vae = comfy.sd.VAE(sd=sd)
return (vae,)
@@ -685,7 +747,7 @@ class ControlNetApplyAdvanced:
if prev_cnet in cnets:
c_net = cnets[prev_cnet]
else:
- c_net = control_net.copy().set_cond_hint(control_hint, strength, (1.0 - start_percent, 1.0 - end_percent))
+ c_net = control_net.copy().set_cond_hint(control_hint, strength, (start_percent, end_percent))
c_net.set_previous_controlnet(prev_cnet)
cnets[prev_cnet] = c_net
@@ -885,8 +947,8 @@ class GLIGENTextBoxApply:
return (c, )
class EmptyLatentImage:
- def __init__(self, device="cpu"):
- self.device = device
+ def __init__(self):
+ self.device = comfy.model_management.intermediate_device()
@classmethod
def INPUT_TYPES(s):
@@ -899,7 +961,7 @@ class EmptyLatentImage:
CATEGORY = "latent"
def generate(self, width, height, batch_size=1):
- latent = torch.zeros([batch_size, 4, height // 8, width // 8])
+ latent = torch.zeros([batch_size, 4, height // 8, width // 8], device=self.device)
return ({"samples":latent}, )
@@ -1275,6 +1337,7 @@ class SaveImage:
self.output_dir = folder_paths.get_output_directory()
self.type = "output"
self.prefix_append = ""
+ self.compress_level = 4
@classmethod
def INPUT_TYPES(s):
@@ -1308,7 +1371,7 @@ class SaveImage:
metadata.add_text(x, json.dumps(extra_pnginfo[x]))
file = f"{filename}_{counter:05}_.png"
- img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=4)
+ img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=self.compress_level)
results.append({
"filename": file,
"subfolder": subfolder,
@@ -1323,6 +1386,7 @@ class PreviewImage(SaveImage):
self.output_dir = folder_paths.get_temp_directory()
self.type = "temp"
self.prefix_append = "_temp_" + ''.join(random.choice("abcdefghijklmnopqrstupvxyz") for x in range(5))
+ self.compress_level = 1
@classmethod
def INPUT_TYPES(s):
@@ -1346,17 +1410,30 @@ class LoadImage:
FUNCTION = "load_image"
def load_image(self, image):
image_path = folder_paths.get_annotated_filepath(image)
- i = Image.open(image_path)
- i = ImageOps.exif_transpose(i)
- image = i.convert("RGB")
- image = np.array(image).astype(np.float32) / 255.0
- image = torch.from_numpy(image)[None,]
- if 'A' in i.getbands():
- mask = np.array(i.getchannel('A')).astype(np.float32) / 255.0
- mask = 1. - torch.from_numpy(mask)
+ img = Image.open(image_path)
+ output_images = []
+ output_masks = []
+ for i in ImageSequence.Iterator(img):
+ i = ImageOps.exif_transpose(i)
+ image = i.convert("RGB")
+ image = np.array(image).astype(np.float32) / 255.0
+ image = torch.from_numpy(image)[None,]
+ if 'A' in i.getbands():
+ mask = np.array(i.getchannel('A')).astype(np.float32) / 255.0
+ mask = 1. - torch.from_numpy(mask)
+ else:
+ mask = torch.zeros((64,64), dtype=torch.float32, device="cpu")
+ output_images.append(image)
+ output_masks.append(mask.unsqueeze(0))
+
+ if len(output_images) > 1:
+ output_image = torch.cat(output_images, dim=0)
+ output_mask = torch.cat(output_masks, dim=0)
else:
- mask = torch.zeros((64,64), dtype=torch.float32, device="cpu")
- return (image, mask.unsqueeze(0))
+ output_image = output_images[0]
+ output_mask = output_masks[0]
+
+ return (output_image, output_mask)
@classmethod
def IS_CHANGED(s, image):
@@ -1414,13 +1491,10 @@ class LoadImageMask:
return m.digest().hex()
@classmethod
- def VALIDATE_INPUTS(s, image, channel):
+ def VALIDATE_INPUTS(s, image):
if not folder_paths.exists_annotated_filepath(image):
return "Invalid image file: {}".format(image)
- if channel not in s._color_channels:
- return "Invalid color channel: {}".format(channel)
-
return True
class ImageScale:
@@ -1654,6 +1728,7 @@ NODE_CLASS_MAPPINGS = {
"ConditioningZeroOut": ConditioningZeroOut,
"ConditioningSetTimestepRange": ConditioningSetTimestepRange,
+ "LoraLoaderModelOnly": LoraLoaderModelOnly,
}
NODE_DISPLAY_NAME_MAPPINGS = {
@@ -1759,7 +1834,7 @@ def load_custom_nodes():
node_paths = folder_paths.get_folder_paths("custom_nodes")
node_import_times = []
for custom_node_path in node_paths:
- possible_modules = os.listdir(custom_node_path)
+ possible_modules = os.listdir(os.path.realpath(custom_node_path))
if "__pycache__" in possible_modules:
possible_modules.remove("__pycache__")
@@ -1799,6 +1874,12 @@ def init_custom_nodes():
"nodes_custom_sampler.py",
"nodes_hypertile.py",
"nodes_model_advanced.py",
+ "nodes_model_downscale.py",
+ "nodes_images.py",
+ "nodes_video_model.py",
+ "nodes_sag.py",
+ "nodes_perpneg.py",
+ "nodes_stable3d.py",
]
for node_file in extras_files:
diff --git a/requirements.txt b/requirements.txt
index 14524485a..e804618e7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,10 @@
torch
torchsde
+torchvision
einops
transformers>=4.25.1
safetensors>=0.3.0
aiohttp
-accelerate
pyyaml
Pillow
scipy
diff --git a/server.py b/server.py
index 11bd2a0fb..9b1e3269d 100644
--- a/server.py
+++ b/server.py
@@ -431,7 +431,10 @@ class PromptServer():
@routes.get("/history")
async def get_history(request):
- return web.json_response(self.prompt_queue.get_history())
+ max_items = request.rel_url.query.get("max_items", None)
+ if max_items is not None:
+ max_items = int(max_items)
+ return web.json_response(self.prompt_queue.get_history(max_items=max_items))
@routes.get("/history/{prompt_id}")
async def get_history(request):
@@ -573,7 +576,7 @@ class PromptServer():
bytesIO = BytesIO()
header = struct.pack(">I", type_num)
bytesIO.write(header)
- image.save(bytesIO, format=image_type, quality=95, compress_level=4)
+ image.save(bytesIO, format=image_type, quality=95, compress_level=1)
preview_bytes = bytesIO.getvalue()
await self.send_bytes(BinaryEventTypes.PREVIEW_IMAGE, preview_bytes, sid=sid)
diff --git a/tests-ui/afterSetup.js b/tests-ui/afterSetup.js
new file mode 100644
index 000000000..983f3af64
--- /dev/null
+++ b/tests-ui/afterSetup.js
@@ -0,0 +1,9 @@
+const { start } = require("./utils");
+const lg = require("./utils/litegraph");
+
+// Load things once per test file before to ensure its all warmed up for the tests
+beforeAll(async () => {
+ lg.setup(global);
+ await start({ resetEnv: true });
+ lg.teardown(global);
+});
diff --git a/tests-ui/jest.config.js b/tests-ui/jest.config.js
index b5a5d646d..86fff5057 100644
--- a/tests-ui/jest.config.js
+++ b/tests-ui/jest.config.js
@@ -2,8 +2,10 @@
const config = {
testEnvironment: "jsdom",
setupFiles: ["./globalSetup.js"],
+ setupFilesAfterEnv: ["./afterSetup.js"],
clearMocks: true,
resetModules: true,
+ testTimeout: 10000
};
module.exports = config;
diff --git a/tests-ui/setup.js b/tests-ui/setup.js
index 0f368ab22..8bbd9dcdf 100644
--- a/tests-ui/setup.js
+++ b/tests-ui/setup.js
@@ -20,6 +20,7 @@ async function setup() {
// Modify the response data to add some checkpoints
const objectInfo = JSON.parse(data);
objectInfo.CheckpointLoaderSimple.input.required.ckpt_name[0] = ["model1.safetensors", "model2.ckpt"];
+ objectInfo.VAELoader.input.required.vae_name[0] = ["vae1.safetensors", "vae2.ckpt"];
data = JSON.stringify(objectInfo, undefined, "\t");
diff --git a/tests-ui/tests/extensions.test.js b/tests-ui/tests/extensions.test.js
new file mode 100644
index 000000000..159e5113a
--- /dev/null
+++ b/tests-ui/tests/extensions.test.js
@@ -0,0 +1,196 @@
+// @ts-check
+///
+const { start } = require("../utils");
+const lg = require("../utils/litegraph");
+
+describe("extensions", () => {
+ beforeEach(() => {
+ lg.setup(global);
+ });
+
+ afterEach(() => {
+ lg.teardown(global);
+ });
+
+ it("calls each extension hook", async () => {
+ const mockExtension = {
+ name: "TestExtension",
+ init: jest.fn(),
+ setup: jest.fn(),
+ addCustomNodeDefs: jest.fn(),
+ getCustomWidgets: jest.fn(),
+ beforeRegisterNodeDef: jest.fn(),
+ registerCustomNodes: jest.fn(),
+ loadedGraphNode: jest.fn(),
+ nodeCreated: jest.fn(),
+ beforeConfigureGraph: jest.fn(),
+ afterConfigureGraph: jest.fn(),
+ };
+
+ const { app, ez, graph } = await start({
+ async preSetup(app) {
+ app.registerExtension(mockExtension);
+ },
+ });
+
+ // Basic initialisation hooks should be called once, with app
+ expect(mockExtension.init).toHaveBeenCalledTimes(1);
+ expect(mockExtension.init).toHaveBeenCalledWith(app);
+
+ // Adding custom node defs should be passed the full list of nodes
+ expect(mockExtension.addCustomNodeDefs).toHaveBeenCalledTimes(1);
+ expect(mockExtension.addCustomNodeDefs.mock.calls[0][1]).toStrictEqual(app);
+ const defs = mockExtension.addCustomNodeDefs.mock.calls[0][0];
+ expect(defs).toHaveProperty("KSampler");
+ expect(defs).toHaveProperty("LoadImage");
+
+ // Get custom widgets is called once and should return new widget types
+ expect(mockExtension.getCustomWidgets).toHaveBeenCalledTimes(1);
+ expect(mockExtension.getCustomWidgets).toHaveBeenCalledWith(app);
+
+ // Before register node def will be called once per node type
+ const nodeNames = Object.keys(defs);
+ const nodeCount = nodeNames.length;
+ expect(mockExtension.beforeRegisterNodeDef).toHaveBeenCalledTimes(nodeCount);
+ for (let i = 0; i < 10; i++) {
+ // It should be send the JS class and the original JSON definition
+ const nodeClass = mockExtension.beforeRegisterNodeDef.mock.calls[i][0];
+ const nodeDef = mockExtension.beforeRegisterNodeDef.mock.calls[i][1];
+
+ expect(nodeClass.name).toBe("ComfyNode");
+ expect(nodeClass.comfyClass).toBe(nodeNames[i]);
+ expect(nodeDef.name).toBe(nodeNames[i]);
+ expect(nodeDef).toHaveProperty("input");
+ expect(nodeDef).toHaveProperty("output");
+ }
+
+ // Register custom nodes is called once after registerNode defs to allow adding other frontend nodes
+ expect(mockExtension.registerCustomNodes).toHaveBeenCalledTimes(1);
+
+ // Before configure graph will be called here as the default graph is being loaded
+ expect(mockExtension.beforeConfigureGraph).toHaveBeenCalledTimes(1);
+ // it gets sent the graph data that is going to be loaded
+ const graphData = mockExtension.beforeConfigureGraph.mock.calls[0][0];
+
+ // A node created is fired for each node constructor that is called
+ expect(mockExtension.nodeCreated).toHaveBeenCalledTimes(graphData.nodes.length);
+ for (let i = 0; i < graphData.nodes.length; i++) {
+ expect(mockExtension.nodeCreated.mock.calls[i][0].type).toBe(graphData.nodes[i].type);
+ }
+
+ // Each node then calls loadedGraphNode to allow them to be updated
+ expect(mockExtension.loadedGraphNode).toHaveBeenCalledTimes(graphData.nodes.length);
+ for (let i = 0; i < graphData.nodes.length; i++) {
+ expect(mockExtension.loadedGraphNode.mock.calls[i][0].type).toBe(graphData.nodes[i].type);
+ }
+
+ // After configure is then called once all the setup is done
+ expect(mockExtension.afterConfigureGraph).toHaveBeenCalledTimes(1);
+
+ expect(mockExtension.setup).toHaveBeenCalledTimes(1);
+ expect(mockExtension.setup).toHaveBeenCalledWith(app);
+
+ // Ensure hooks are called in the correct order
+ const callOrder = [
+ "init",
+ "addCustomNodeDefs",
+ "getCustomWidgets",
+ "beforeRegisterNodeDef",
+ "registerCustomNodes",
+ "beforeConfigureGraph",
+ "nodeCreated",
+ "loadedGraphNode",
+ "afterConfigureGraph",
+ "setup",
+ ];
+ for (let i = 1; i < callOrder.length; i++) {
+ const fn1 = mockExtension[callOrder[i - 1]];
+ const fn2 = mockExtension[callOrder[i]];
+ expect(fn1.mock.invocationCallOrder[0]).toBeLessThan(fn2.mock.invocationCallOrder[0]);
+ }
+
+ graph.clear();
+
+ // Ensure adding a new node calls the correct callback
+ ez.LoadImage();
+ expect(mockExtension.loadedGraphNode).toHaveBeenCalledTimes(graphData.nodes.length);
+ expect(mockExtension.nodeCreated).toHaveBeenCalledTimes(graphData.nodes.length + 1);
+ expect(mockExtension.nodeCreated.mock.lastCall[0].type).toBe("LoadImage");
+
+ // Reload the graph to ensure correct hooks are fired
+ await graph.reload();
+
+ // These hooks should not be fired again
+ expect(mockExtension.init).toHaveBeenCalledTimes(1);
+ expect(mockExtension.addCustomNodeDefs).toHaveBeenCalledTimes(1);
+ expect(mockExtension.getCustomWidgets).toHaveBeenCalledTimes(1);
+ expect(mockExtension.registerCustomNodes).toHaveBeenCalledTimes(1);
+ expect(mockExtension.beforeRegisterNodeDef).toHaveBeenCalledTimes(nodeCount);
+ expect(mockExtension.setup).toHaveBeenCalledTimes(1);
+
+ // These should be called again
+ expect(mockExtension.beforeConfigureGraph).toHaveBeenCalledTimes(2);
+ expect(mockExtension.nodeCreated).toHaveBeenCalledTimes(graphData.nodes.length + 2);
+ expect(mockExtension.loadedGraphNode).toHaveBeenCalledTimes(graphData.nodes.length + 1);
+ expect(mockExtension.afterConfigureGraph).toHaveBeenCalledTimes(2);
+ }, 15000);
+
+ it("allows custom nodeDefs and widgets to be registered", async () => {
+ const widgetMock = jest.fn((node, inputName, inputData, app) => {
+ expect(node.constructor.comfyClass).toBe("TestNode");
+ expect(inputName).toBe("test_input");
+ expect(inputData[0]).toBe("CUSTOMWIDGET");
+ expect(inputData[1]?.hello).toBe("world");
+ expect(app).toStrictEqual(app);
+
+ return {
+ widget: node.addWidget("button", inputName, "hello", () => {}),
+ };
+ });
+
+ // Register our extension that adds a custom node + widget type
+ const mockExtension = {
+ name: "TestExtension",
+ addCustomNodeDefs: (nodeDefs) => {
+ nodeDefs["TestNode"] = {
+ output: [],
+ output_name: [],
+ output_is_list: [],
+ name: "TestNode",
+ display_name: "TestNode",
+ category: "Test",
+ input: {
+ required: {
+ test_input: ["CUSTOMWIDGET", { hello: "world" }],
+ },
+ },
+ };
+ },
+ getCustomWidgets: jest.fn(() => {
+ return {
+ CUSTOMWIDGET: widgetMock,
+ };
+ }),
+ };
+
+ const { graph, ez } = await start({
+ async preSetup(app) {
+ app.registerExtension(mockExtension);
+ },
+ });
+
+ expect(mockExtension.getCustomWidgets).toBeCalledTimes(1);
+
+ graph.clear();
+ expect(widgetMock).toBeCalledTimes(0);
+ const node = ez.TestNode();
+ expect(widgetMock).toBeCalledTimes(1);
+
+ // Ensure our custom widget is created
+ expect(node.inputs.length).toBe(0);
+ expect(node.widgets.length).toBe(1);
+ const w = node.widgets[0].widget;
+ expect(w.name).toBe("test_input");
+ expect(w.type).toBe("button");
+ });
+});
diff --git a/tests-ui/tests/groupNode.test.js b/tests-ui/tests/groupNode.test.js
new file mode 100644
index 000000000..e6ebedd91
--- /dev/null
+++ b/tests-ui/tests/groupNode.test.js
@@ -0,0 +1,1005 @@
+// @ts-check
+///
+
+const { start, createDefaultWorkflow, getNodeDef, checkBeforeAndAfterReload } = require("../utils");
+const lg = require("../utils/litegraph");
+
+describe("group node", () => {
+ beforeEach(() => {
+ lg.setup(global);
+ });
+
+ afterEach(() => {
+ lg.teardown(global);
+ });
+
+ /**
+ *
+ * @param {*} app
+ * @param {*} graph
+ * @param {*} name
+ * @param {*} nodes
+ * @returns { Promise> }
+ */
+ async function convertToGroup(app, graph, name, nodes) {
+ // Select the nodes we are converting
+ for (const n of nodes) {
+ n.select(true);
+ }
+
+ expect(Object.keys(app.canvas.selected_nodes).sort((a, b) => +a - +b)).toEqual(
+ nodes.map((n) => n.id + "").sort((a, b) => +a - +b)
+ );
+
+ global.prompt = jest.fn().mockImplementation(() => name);
+ const groupNode = await nodes[0].menu["Convert to Group Node"].call(false);
+
+ // Check group name was requested
+ expect(window.prompt).toHaveBeenCalled();
+
+ // Ensure old nodes are removed
+ for (const n of nodes) {
+ expect(n.isRemoved).toBeTruthy();
+ }
+
+ expect(groupNode.type).toEqual("workflow/" + name);
+
+ return graph.find(groupNode);
+ }
+
+ /**
+ * @param { Record | number[] } idMap
+ * @param { Record> } valueMap
+ */
+ function getOutput(idMap = {}, valueMap = {}) {
+ if (idMap instanceof Array) {
+ idMap = idMap.reduce((p, n) => {
+ p[n] = n + "";
+ return p;
+ }, {});
+ }
+ const expected = {
+ 1: { inputs: { ckpt_name: "model1.safetensors", ...valueMap?.[1] }, class_type: "CheckpointLoaderSimple" },
+ 2: { inputs: { text: "positive", clip: ["1", 1], ...valueMap?.[2] }, class_type: "CLIPTextEncode" },
+ 3: { inputs: { text: "negative", clip: ["1", 1], ...valueMap?.[3] }, class_type: "CLIPTextEncode" },
+ 4: { inputs: { width: 512, height: 512, batch_size: 1, ...valueMap?.[4] }, class_type: "EmptyLatentImage" },
+ 5: {
+ inputs: {
+ seed: 0,
+ steps: 20,
+ cfg: 8,
+ sampler_name: "euler",
+ scheduler: "normal",
+ denoise: 1,
+ model: ["1", 0],
+ positive: ["2", 0],
+ negative: ["3", 0],
+ latent_image: ["4", 0],
+ ...valueMap?.[5],
+ },
+ class_type: "KSampler",
+ },
+ 6: { inputs: { samples: ["5", 0], vae: ["1", 2], ...valueMap?.[6] }, class_type: "VAEDecode" },
+ 7: { inputs: { filename_prefix: "ComfyUI", images: ["6", 0], ...valueMap?.[7] }, class_type: "SaveImage" },
+ };
+
+ // Map old IDs to new at the top level
+ const mapped = {};
+ for (const oldId in idMap) {
+ mapped[idMap[oldId]] = expected[oldId];
+ delete expected[oldId];
+ }
+ Object.assign(mapped, expected);
+
+ // Map old IDs to new inside links
+ for (const k in mapped) {
+ for (const input in mapped[k].inputs) {
+ const v = mapped[k].inputs[input];
+ if (v instanceof Array) {
+ if (v[0] in idMap) {
+ v[0] = idMap[v[0]] + "";
+ }
+ }
+ }
+ }
+
+ return mapped;
+ }
+
+ test("can be created from selected nodes", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+ const group = await convertToGroup(app, graph, "test", [nodes.pos, nodes.neg, nodes.empty]);
+
+ // Ensure links are now to the group node
+ expect(group.inputs).toHaveLength(2);
+ expect(group.outputs).toHaveLength(3);
+
+ expect(group.inputs.map((i) => i.input.name)).toEqual(["clip", "CLIPTextEncode clip"]);
+ expect(group.outputs.map((i) => i.output.name)).toEqual(["LATENT", "CONDITIONING", "CLIPTextEncode CONDITIONING"]);
+
+ // ckpt clip to both clip inputs on the group
+ expect(nodes.ckpt.outputs.CLIP.connections.map((t) => [t.targetNode.id, t.targetInput.index])).toEqual([
+ [group.id, 0],
+ [group.id, 1],
+ ]);
+
+ // group conditioning to sampler
+ expect(group.outputs["CONDITIONING"].connections.map((t) => [t.targetNode.id, t.targetInput.index])).toEqual([
+ [nodes.sampler.id, 1],
+ ]);
+ // group conditioning 2 to sampler
+ expect(
+ group.outputs["CLIPTextEncode CONDITIONING"].connections.map((t) => [t.targetNode.id, t.targetInput.index])
+ ).toEqual([[nodes.sampler.id, 2]]);
+ // group latent to sampler
+ expect(group.outputs["LATENT"].connections.map((t) => [t.targetNode.id, t.targetInput.index])).toEqual([
+ [nodes.sampler.id, 3],
+ ]);
+ });
+
+ test("maintains all output links on conversion", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+ const save2 = ez.SaveImage(...nodes.decode.outputs);
+ const save3 = ez.SaveImage(...nodes.decode.outputs);
+ // Ensure an output with multiple links maintains them on convert to group
+ const group = await convertToGroup(app, graph, "test", [nodes.sampler, nodes.decode]);
+ expect(group.outputs[0].connections.length).toBe(3);
+ expect(group.outputs[0].connections[0].targetNode.id).toBe(nodes.save.id);
+ expect(group.outputs[0].connections[1].targetNode.id).toBe(save2.id);
+ expect(group.outputs[0].connections[2].targetNode.id).toBe(save3.id);
+
+ // and they're still linked when converting back to nodes
+ const newNodes = group.menu["Convert to nodes"].call();
+ const decode = graph.find(newNodes.find((n) => n.type === "VAEDecode"));
+ expect(decode.outputs[0].connections.length).toBe(3);
+ expect(decode.outputs[0].connections[0].targetNode.id).toBe(nodes.save.id);
+ expect(decode.outputs[0].connections[1].targetNode.id).toBe(save2.id);
+ expect(decode.outputs[0].connections[2].targetNode.id).toBe(save3.id);
+ });
+ test("can be be converted back to nodes", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+ const toConvert = [nodes.pos, nodes.neg, nodes.empty, nodes.sampler];
+ const group = await convertToGroup(app, graph, "test", toConvert);
+
+ // Edit some values to ensure they are set back onto the converted nodes
+ expect(group.widgets["text"].value).toBe("positive");
+ group.widgets["text"].value = "pos";
+ expect(group.widgets["CLIPTextEncode text"].value).toBe("negative");
+ group.widgets["CLIPTextEncode text"].value = "neg";
+ expect(group.widgets["width"].value).toBe(512);
+ group.widgets["width"].value = 1024;
+ expect(group.widgets["sampler_name"].value).toBe("euler");
+ group.widgets["sampler_name"].value = "ddim";
+ expect(group.widgets["control_after_generate"].value).toBe("randomize");
+ group.widgets["control_after_generate"].value = "fixed";
+
+ /** @type { Array } */
+ group.menu["Convert to nodes"].call();
+
+ // ensure widget values are set
+ const pos = graph.find(nodes.pos.id);
+ expect(pos.node.type).toBe("CLIPTextEncode");
+ expect(pos.widgets["text"].value).toBe("pos");
+ const neg = graph.find(nodes.neg.id);
+ expect(neg.node.type).toBe("CLIPTextEncode");
+ expect(neg.widgets["text"].value).toBe("neg");
+ const empty = graph.find(nodes.empty.id);
+ expect(empty.node.type).toBe("EmptyLatentImage");
+ expect(empty.widgets["width"].value).toBe(1024);
+ const sampler = graph.find(nodes.sampler.id);
+ expect(sampler.node.type).toBe("KSampler");
+ expect(sampler.widgets["sampler_name"].value).toBe("ddim");
+ expect(sampler.widgets["control_after_generate"].value).toBe("fixed");
+
+ // validate links
+ expect(nodes.ckpt.outputs.CLIP.connections.map((t) => [t.targetNode.id, t.targetInput.index])).toEqual([
+ [pos.id, 0],
+ [neg.id, 0],
+ ]);
+
+ expect(pos.outputs["CONDITIONING"].connections.map((t) => [t.targetNode.id, t.targetInput.index])).toEqual([
+ [nodes.sampler.id, 1],
+ ]);
+
+ expect(neg.outputs["CONDITIONING"].connections.map((t) => [t.targetNode.id, t.targetInput.index])).toEqual([
+ [nodes.sampler.id, 2],
+ ]);
+
+ expect(empty.outputs["LATENT"].connections.map((t) => [t.targetNode.id, t.targetInput.index])).toEqual([
+ [nodes.sampler.id, 3],
+ ]);
+ });
+ test("it can embed reroutes as inputs", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+
+ // Add and connect a reroute to the clip text encodes
+ const reroute = ez.Reroute();
+ nodes.ckpt.outputs.CLIP.connectTo(reroute.inputs[0]);
+ reroute.outputs[0].connectTo(nodes.pos.inputs[0]);
+ reroute.outputs[0].connectTo(nodes.neg.inputs[0]);
+
+ // Convert to group and ensure we only have 1 input of the correct type
+ const group = await convertToGroup(app, graph, "test", [nodes.pos, nodes.neg, nodes.empty, reroute]);
+ expect(group.inputs).toHaveLength(1);
+ expect(group.inputs[0].input.type).toEqual("CLIP");
+
+ expect((await graph.toPrompt()).output).toEqual(getOutput());
+ });
+ test("it can embed reroutes as outputs", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+
+ // Add a reroute with no output so we output IMAGE even though its used internally
+ const reroute = ez.Reroute();
+ nodes.decode.outputs.IMAGE.connectTo(reroute.inputs[0]);
+
+ // Convert to group and ensure there is an IMAGE output
+ const group = await convertToGroup(app, graph, "test", [nodes.decode, nodes.save, reroute]);
+ expect(group.outputs).toHaveLength(1);
+ expect(group.outputs[0].output.type).toEqual("IMAGE");
+ expect((await graph.toPrompt()).output).toEqual(getOutput([nodes.decode.id, nodes.save.id]));
+ });
+ test("it can embed reroutes as pipes", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+
+ // Use reroutes as a pipe
+ const rerouteModel = ez.Reroute();
+ const rerouteClip = ez.Reroute();
+ const rerouteVae = ez.Reroute();
+ nodes.ckpt.outputs.MODEL.connectTo(rerouteModel.inputs[0]);
+ nodes.ckpt.outputs.CLIP.connectTo(rerouteClip.inputs[0]);
+ nodes.ckpt.outputs.VAE.connectTo(rerouteVae.inputs[0]);
+
+ const group = await convertToGroup(app, graph, "test", [rerouteModel, rerouteClip, rerouteVae]);
+
+ expect(group.outputs).toHaveLength(3);
+ expect(group.outputs.map((o) => o.output.type)).toEqual(["MODEL", "CLIP", "VAE"]);
+
+ expect(group.outputs).toHaveLength(3);
+ expect(group.outputs.map((o) => o.output.type)).toEqual(["MODEL", "CLIP", "VAE"]);
+
+ group.outputs[0].connectTo(nodes.sampler.inputs.model);
+ group.outputs[1].connectTo(nodes.pos.inputs.clip);
+ group.outputs[1].connectTo(nodes.neg.inputs.clip);
+ });
+ test("can handle reroutes used internally", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+
+ let reroutes = [];
+ let prevNode = nodes.ckpt;
+ for (let i = 0; i < 5; i++) {
+ const reroute = ez.Reroute();
+ prevNode.outputs[0].connectTo(reroute.inputs[0]);
+ prevNode = reroute;
+ reroutes.push(reroute);
+ }
+ prevNode.outputs[0].connectTo(nodes.sampler.inputs.model);
+
+ const group = await convertToGroup(app, graph, "test", [...reroutes, ...Object.values(nodes)]);
+ expect((await graph.toPrompt()).output).toEqual(getOutput());
+
+ group.menu["Convert to nodes"].call();
+ expect((await graph.toPrompt()).output).toEqual(getOutput());
+ });
+ test("creates with widget values from inner nodes", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+
+ nodes.ckpt.widgets.ckpt_name.value = "model2.ckpt";
+ nodes.pos.widgets.text.value = "hello";
+ nodes.neg.widgets.text.value = "world";
+ nodes.empty.widgets.width.value = 256;
+ nodes.empty.widgets.height.value = 1024;
+ nodes.sampler.widgets.seed.value = 1;
+ nodes.sampler.widgets.control_after_generate.value = "increment";
+ nodes.sampler.widgets.steps.value = 8;
+ nodes.sampler.widgets.cfg.value = 4.5;
+ nodes.sampler.widgets.sampler_name.value = "uni_pc";
+ nodes.sampler.widgets.scheduler.value = "karras";
+ nodes.sampler.widgets.denoise.value = 0.9;
+
+ const group = await convertToGroup(app, graph, "test", [
+ nodes.ckpt,
+ nodes.pos,
+ nodes.neg,
+ nodes.empty,
+ nodes.sampler,
+ ]);
+
+ expect(group.widgets["ckpt_name"].value).toEqual("model2.ckpt");
+ expect(group.widgets["text"].value).toEqual("hello");
+ expect(group.widgets["CLIPTextEncode text"].value).toEqual("world");
+ expect(group.widgets["width"].value).toEqual(256);
+ expect(group.widgets["height"].value).toEqual(1024);
+ expect(group.widgets["seed"].value).toEqual(1);
+ expect(group.widgets["control_after_generate"].value).toEqual("increment");
+ expect(group.widgets["steps"].value).toEqual(8);
+ expect(group.widgets["cfg"].value).toEqual(4.5);
+ expect(group.widgets["sampler_name"].value).toEqual("uni_pc");
+ expect(group.widgets["scheduler"].value).toEqual("karras");
+ expect(group.widgets["denoise"].value).toEqual(0.9);
+
+ expect((await graph.toPrompt()).output).toEqual(
+ getOutput([nodes.ckpt.id, nodes.pos.id, nodes.neg.id, nodes.empty.id, nodes.sampler.id], {
+ [nodes.ckpt.id]: { ckpt_name: "model2.ckpt" },
+ [nodes.pos.id]: { text: "hello" },
+ [nodes.neg.id]: { text: "world" },
+ [nodes.empty.id]: { width: 256, height: 1024 },
+ [nodes.sampler.id]: {
+ seed: 1,
+ steps: 8,
+ cfg: 4.5,
+ sampler_name: "uni_pc",
+ scheduler: "karras",
+ denoise: 0.9,
+ },
+ })
+ );
+ });
+ test("group inputs can be reroutes", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+ const group = await convertToGroup(app, graph, "test", [nodes.pos, nodes.neg]);
+
+ const reroute = ez.Reroute();
+ nodes.ckpt.outputs.CLIP.connectTo(reroute.inputs[0]);
+
+ reroute.outputs[0].connectTo(group.inputs[0]);
+ reroute.outputs[0].connectTo(group.inputs[1]);
+
+ expect((await graph.toPrompt()).output).toEqual(getOutput([nodes.pos.id, nodes.neg.id]));
+ });
+ test("group outputs can be reroutes", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+ const group = await convertToGroup(app, graph, "test", [nodes.pos, nodes.neg]);
+
+ const reroute1 = ez.Reroute();
+ const reroute2 = ez.Reroute();
+ group.outputs[0].connectTo(reroute1.inputs[0]);
+ group.outputs[1].connectTo(reroute2.inputs[0]);
+
+ reroute1.outputs[0].connectTo(nodes.sampler.inputs.positive);
+ reroute2.outputs[0].connectTo(nodes.sampler.inputs.negative);
+
+ expect((await graph.toPrompt()).output).toEqual(getOutput([nodes.pos.id, nodes.neg.id]));
+ });
+ test("groups can connect to each other", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+ const group1 = await convertToGroup(app, graph, "test", [nodes.pos, nodes.neg]);
+ const group2 = await convertToGroup(app, graph, "test2", [nodes.empty, nodes.sampler]);
+
+ group1.outputs[0].connectTo(group2.inputs["positive"]);
+ group1.outputs[1].connectTo(group2.inputs["negative"]);
+
+ expect((await graph.toPrompt()).output).toEqual(
+ getOutput([nodes.pos.id, nodes.neg.id, nodes.empty.id, nodes.sampler.id])
+ );
+ });
+ test("groups can connect to each other via internal reroutes", async () => {
+ const { ez, graph, app } = await start();
+
+ const latent = ez.EmptyLatentImage();
+ const vae = ez.VAELoader();
+ const latentReroute = ez.Reroute();
+ const vaeReroute = ez.Reroute();
+
+ latent.outputs[0].connectTo(latentReroute.inputs[0]);
+ vae.outputs[0].connectTo(vaeReroute.inputs[0]);
+
+ const group1 = await convertToGroup(app, graph, "test", [latentReroute, vaeReroute]);
+ group1.menu.Clone.call();
+ expect(app.graph._nodes).toHaveLength(4);
+ const group2 = graph.find(app.graph._nodes[3]);
+ expect(group2.node.type).toEqual("workflow/test");
+ expect(group2.id).not.toEqual(group1.id);
+
+ group1.outputs.VAE.connectTo(group2.inputs.VAE);
+ group1.outputs.LATENT.connectTo(group2.inputs.LATENT);
+
+ const decode = ez.VAEDecode(group2.outputs.LATENT, group2.outputs.VAE);
+ const preview = ez.PreviewImage(decode.outputs[0]);
+
+ const output = {
+ [latent.id]: { inputs: { width: 512, height: 512, batch_size: 1 }, class_type: "EmptyLatentImage" },
+ [vae.id]: { inputs: { vae_name: "vae1.safetensors" }, class_type: "VAELoader" },
+ [decode.id]: { inputs: { samples: [latent.id + "", 0], vae: [vae.id + "", 0] }, class_type: "VAEDecode" },
+ [preview.id]: { inputs: { images: [decode.id + "", 0] }, class_type: "PreviewImage" },
+ };
+ expect((await graph.toPrompt()).output).toEqual(output);
+
+ // Ensure missing connections dont cause errors
+ group2.inputs.VAE.disconnect();
+ delete output[decode.id].inputs.vae;
+ expect((await graph.toPrompt()).output).toEqual(output);
+ });
+ test("displays generated image on group node", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+ let group = await convertToGroup(app, graph, "test", [
+ nodes.pos,
+ nodes.neg,
+ nodes.empty,
+ nodes.sampler,
+ nodes.decode,
+ nodes.save,
+ ]);
+
+ const { api } = require("../../web/scripts/api");
+
+ api.dispatchEvent(new CustomEvent("execution_start", {}));
+ api.dispatchEvent(new CustomEvent("executing", { detail: `${nodes.save.id}` }));
+ // Event should be forwarded to group node id
+ expect(+app.runningNodeId).toEqual(group.id);
+ expect(group.node["imgs"]).toBeFalsy();
+ api.dispatchEvent(
+ new CustomEvent("executed", {
+ detail: {
+ node: `${nodes.save.id}`,
+ output: {
+ images: [
+ {
+ filename: "test.png",
+ type: "output",
+ },
+ ],
+ },
+ },
+ })
+ );
+
+ // Trigger paint
+ group.node.onDrawBackground?.(app.canvas.ctx, app.canvas.canvas);
+
+ expect(group.node["images"]).toEqual([
+ {
+ filename: "test.png",
+ type: "output",
+ },
+ ]);
+
+ // Reload
+ const workflow = JSON.stringify((await graph.toPrompt()).workflow);
+ await app.loadGraphData(JSON.parse(workflow));
+ group = graph.find(group);
+
+ // Trigger inner nodes to get created
+ group.node["getInnerNodes"]();
+
+ // Check it works for internal node ids
+ api.dispatchEvent(new CustomEvent("execution_start", {}));
+ api.dispatchEvent(new CustomEvent("executing", { detail: `${group.id}:5` }));
+ // Event should be forwarded to group node id
+ expect(+app.runningNodeId).toEqual(group.id);
+ expect(group.node["imgs"]).toBeFalsy();
+ api.dispatchEvent(
+ new CustomEvent("executed", {
+ detail: {
+ node: `${group.id}:5`,
+ output: {
+ images: [
+ {
+ filename: "test2.png",
+ type: "output",
+ },
+ ],
+ },
+ },
+ })
+ );
+
+ // Trigger paint
+ group.node.onDrawBackground?.(app.canvas.ctx, app.canvas.canvas);
+
+ expect(group.node["images"]).toEqual([
+ {
+ filename: "test2.png",
+ type: "output",
+ },
+ ]);
+ });
+ test("allows widgets to be converted to inputs", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+ const group = await convertToGroup(app, graph, "test", [nodes.pos, nodes.neg]);
+ group.widgets[0].convertToInput();
+
+ const primitive = ez.PrimitiveNode();
+ primitive.outputs[0].connectTo(group.inputs["text"]);
+ primitive.widgets[0].value = "hello";
+
+ expect((await graph.toPrompt()).output).toEqual(
+ getOutput([nodes.pos.id, nodes.neg.id], {
+ [nodes.pos.id]: { text: "hello" },
+ })
+ );
+ });
+ test("can be copied", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+
+ const group1 = await convertToGroup(app, graph, "test", [
+ nodes.pos,
+ nodes.neg,
+ nodes.empty,
+ nodes.sampler,
+ nodes.decode,
+ nodes.save,
+ ]);
+
+ group1.widgets["text"].value = "hello";
+ group1.widgets["width"].value = 256;
+ group1.widgets["seed"].value = 1;
+
+ // Clone the node
+ group1.menu.Clone.call();
+ expect(app.graph._nodes).toHaveLength(3);
+ const group2 = graph.find(app.graph._nodes[2]);
+ expect(group2.node.type).toEqual("workflow/test");
+ expect(group2.id).not.toEqual(group1.id);
+
+ // Reconnect ckpt
+ nodes.ckpt.outputs.MODEL.connectTo(group2.inputs["model"]);
+ nodes.ckpt.outputs.CLIP.connectTo(group2.inputs["clip"]);
+ nodes.ckpt.outputs.CLIP.connectTo(group2.inputs["CLIPTextEncode clip"]);
+ nodes.ckpt.outputs.VAE.connectTo(group2.inputs["vae"]);
+
+ group2.widgets["text"].value = "world";
+ group2.widgets["width"].value = 1024;
+ group2.widgets["seed"].value = 100;
+
+ let i = 0;
+ expect((await graph.toPrompt()).output).toEqual({
+ ...getOutput([nodes.empty.id, nodes.pos.id, nodes.neg.id, nodes.sampler.id, nodes.decode.id, nodes.save.id], {
+ [nodes.empty.id]: { width: 256 },
+ [nodes.pos.id]: { text: "hello" },
+ [nodes.sampler.id]: { seed: 1 },
+ }),
+ ...getOutput(
+ {
+ [nodes.empty.id]: `${group2.id}:${i++}`,
+ [nodes.pos.id]: `${group2.id}:${i++}`,
+ [nodes.neg.id]: `${group2.id}:${i++}`,
+ [nodes.sampler.id]: `${group2.id}:${i++}`,
+ [nodes.decode.id]: `${group2.id}:${i++}`,
+ [nodes.save.id]: `${group2.id}:${i++}`,
+ },
+ {
+ [nodes.empty.id]: { width: 1024 },
+ [nodes.pos.id]: { text: "world" },
+ [nodes.sampler.id]: { seed: 100 },
+ }
+ ),
+ });
+
+ graph.arrange();
+ });
+ test("is embedded in workflow", async () => {
+ let { ez, graph, app } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+ let group = await convertToGroup(app, graph, "test", [nodes.pos, nodes.neg]);
+ const workflow = JSON.stringify((await graph.toPrompt()).workflow);
+
+ // Clear the environment
+ ({ ez, graph, app } = await start({
+ resetEnv: true,
+ }));
+ // Ensure the node isnt registered
+ expect(() => ez["workflow/test"]).toThrow();
+
+ // Reload the workflow
+ await app.loadGraphData(JSON.parse(workflow));
+
+ // Ensure the node is found
+ group = graph.find(group);
+
+ // Generate prompt and ensure it is as expected
+ expect((await graph.toPrompt()).output).toEqual(
+ getOutput({
+ [nodes.pos.id]: `${group.id}:0`,
+ [nodes.neg.id]: `${group.id}:1`,
+ })
+ );
+ });
+ test("shows missing node error on missing internal node when loading graph data", async () => {
+ const { graph } = await start();
+
+ const dialogShow = jest.spyOn(graph.app.ui.dialog, "show");
+ await graph.app.loadGraphData({
+ last_node_id: 3,
+ last_link_id: 1,
+ nodes: [
+ {
+ id: 3,
+ type: "workflow/testerror",
+ },
+ ],
+ links: [],
+ groups: [],
+ config: {},
+ extra: {
+ groupNodes: {
+ testerror: {
+ nodes: [
+ {
+ type: "NotKSampler",
+ },
+ {
+ type: "NotVAEDecode",
+ },
+ ],
+ },
+ },
+ },
+ });
+
+ expect(dialogShow).toBeCalledTimes(1);
+ const call = dialogShow.mock.calls[0][0].innerHTML;
+ expect(call).toContain("the following node types were not found");
+ expect(call).toContain("NotKSampler");
+ expect(call).toContain("NotVAEDecode");
+ expect(call).toContain("workflow/testerror");
+ });
+ test("maintains widget inputs on conversion back to nodes", async () => {
+ const { ez, graph, app } = await start();
+ let pos = ez.CLIPTextEncode({ text: "positive" });
+ pos.node.title = "Positive";
+ let neg = ez.CLIPTextEncode({ text: "negative" });
+ neg.node.title = "Negative";
+ pos.widgets.text.convertToInput();
+ neg.widgets.text.convertToInput();
+
+ let primitive = ez.PrimitiveNode();
+ primitive.outputs[0].connectTo(pos.inputs.text);
+ primitive.outputs[0].connectTo(neg.inputs.text);
+
+ const group = await convertToGroup(app, graph, "test", [pos, neg, primitive]);
+ // This will use a primitive widget named 'value'
+ expect(group.widgets.length).toBe(1);
+ expect(group.widgets["value"].value).toBe("positive");
+
+ const newNodes = group.menu["Convert to nodes"].call();
+ pos = graph.find(newNodes.find((n) => n.title === "Positive"));
+ neg = graph.find(newNodes.find((n) => n.title === "Negative"));
+ primitive = graph.find(newNodes.find((n) => n.type === "PrimitiveNode"));
+
+ expect(pos.inputs).toHaveLength(2);
+ expect(neg.inputs).toHaveLength(2);
+ expect(primitive.outputs[0].connections).toHaveLength(2);
+
+ expect((await graph.toPrompt()).output).toEqual({
+ 1: { inputs: { text: "positive" }, class_type: "CLIPTextEncode" },
+ 2: { inputs: { text: "positive" }, class_type: "CLIPTextEncode" },
+ });
+ });
+ test("correctly handles widget inputs", async () => {
+ const { ez, graph, app } = await start();
+ const upscaleMethods = (await getNodeDef("ImageScaleBy")).input.required["upscale_method"][0];
+
+ const image = ez.LoadImage();
+ const scale1 = ez.ImageScaleBy(image.outputs[0]);
+ const scale2 = ez.ImageScaleBy(image.outputs[0]);
+ const preview1 = ez.PreviewImage(scale1.outputs[0]);
+ const preview2 = ez.PreviewImage(scale2.outputs[0]);
+ scale1.widgets.upscale_method.value = upscaleMethods[1];
+ scale1.widgets.upscale_method.convertToInput();
+
+ const group = await convertToGroup(app, graph, "test", [scale1, scale2]);
+ expect(group.inputs.length).toBe(3);
+ expect(group.inputs[0].input.type).toBe("IMAGE");
+ expect(group.inputs[1].input.type).toBe("IMAGE");
+ expect(group.inputs[2].input.type).toBe("COMBO");
+
+ // Ensure links are maintained
+ expect(group.inputs[0].connection?.originNode?.id).toBe(image.id);
+ expect(group.inputs[1].connection?.originNode?.id).toBe(image.id);
+ expect(group.inputs[2].connection).toBeFalsy();
+
+ // Ensure primitive gets correct type
+ const primitive = ez.PrimitiveNode();
+ primitive.outputs[0].connectTo(group.inputs[2]);
+ expect(primitive.widgets.value.widget.options.values).toBe(upscaleMethods);
+ expect(primitive.widgets.value.value).toBe(upscaleMethods[1]); // Ensure value is copied
+ primitive.widgets.value.value = upscaleMethods[1];
+
+ await checkBeforeAndAfterReload(graph, async (r) => {
+ const scale1id = r ? `${group.id}:0` : scale1.id;
+ const scale2id = r ? `${group.id}:1` : scale2.id;
+ // Ensure widget value is applied to prompt
+ expect((await graph.toPrompt()).output).toStrictEqual({
+ [image.id]: { inputs: { image: "example.png", upload: "image" }, class_type: "LoadImage" },
+ [scale1id]: {
+ inputs: { upscale_method: upscaleMethods[1], scale_by: 1, image: [`${image.id}`, 0] },
+ class_type: "ImageScaleBy",
+ },
+ [scale2id]: {
+ inputs: { upscale_method: "nearest-exact", scale_by: 1, image: [`${image.id}`, 0] },
+ class_type: "ImageScaleBy",
+ },
+ [preview1.id]: { inputs: { images: [`${scale1id}`, 0] }, class_type: "PreviewImage" },
+ [preview2.id]: { inputs: { images: [`${scale2id}`, 0] }, class_type: "PreviewImage" },
+ });
+ });
+ });
+ test("adds widgets in node execution order", async () => {
+ const { ez, graph, app } = await start();
+ const scale = ez.LatentUpscale();
+ const save = ez.SaveImage();
+ const empty = ez.EmptyLatentImage();
+ const decode = ez.VAEDecode();
+
+ scale.outputs.LATENT.connectTo(decode.inputs.samples);
+ decode.outputs.IMAGE.connectTo(save.inputs.images);
+ empty.outputs.LATENT.connectTo(scale.inputs.samples);
+
+ const group = await convertToGroup(app, graph, "test", [scale, save, empty, decode]);
+ const widgets = group.widgets.map((w) => w.widget.name);
+ expect(widgets).toStrictEqual([
+ "width",
+ "height",
+ "batch_size",
+ "upscale_method",
+ "LatentUpscale width",
+ "LatentUpscale height",
+ "crop",
+ "filename_prefix",
+ ]);
+ });
+ test("adds output for external links when converting to group", async () => {
+ const { ez, graph, app } = await start();
+ const img = ez.EmptyLatentImage();
+ let decode = ez.VAEDecode(...img.outputs);
+ const preview1 = ez.PreviewImage(...decode.outputs);
+ const preview2 = ez.PreviewImage(...decode.outputs);
+
+ const group = await convertToGroup(app, graph, "test", [img, decode, preview1]);
+
+ // Ensure we have an output connected to the 2nd preview node
+ expect(group.outputs.length).toBe(1);
+ expect(group.outputs[0].connections.length).toBe(1);
+ expect(group.outputs[0].connections[0].targetNode.id).toBe(preview2.id);
+
+ // Convert back and ensure bothe previews are still connected
+ group.menu["Convert to nodes"].call();
+ decode = graph.find(decode);
+ expect(decode.outputs[0].connections.length).toBe(2);
+ expect(decode.outputs[0].connections[0].targetNode.id).toBe(preview1.id);
+ expect(decode.outputs[0].connections[1].targetNode.id).toBe(preview2.id);
+ });
+ test("adds output for external links when converting to group when nodes are not in execution order", async () => {
+ const { ez, graph, app } = await start();
+ const sampler = ez.KSampler();
+ const ckpt = ez.CheckpointLoaderSimple();
+ const empty = ez.EmptyLatentImage();
+ const pos = ez.CLIPTextEncode(ckpt.outputs.CLIP, { text: "positive" });
+ const neg = ez.CLIPTextEncode(ckpt.outputs.CLIP, { text: "negative" });
+ const decode1 = ez.VAEDecode(sampler.outputs.LATENT, ckpt.outputs.VAE);
+ const save = ez.SaveImage(decode1.outputs.IMAGE);
+ ckpt.outputs.MODEL.connectTo(sampler.inputs.model);
+ pos.outputs.CONDITIONING.connectTo(sampler.inputs.positive);
+ neg.outputs.CONDITIONING.connectTo(sampler.inputs.negative);
+ empty.outputs.LATENT.connectTo(sampler.inputs.latent_image);
+
+ const encode = ez.VAEEncode(decode1.outputs.IMAGE);
+ const vae = ez.VAELoader();
+ const decode2 = ez.VAEDecode(encode.outputs.LATENT, vae.outputs.VAE);
+ const preview = ez.PreviewImage(decode2.outputs.IMAGE);
+ vae.outputs.VAE.connectTo(encode.inputs.vae);
+
+ const group = await convertToGroup(app, graph, "test", [vae, decode1, encode, sampler]);
+
+ expect(group.outputs.length).toBe(3);
+ expect(group.outputs[0].output.name).toBe("VAE");
+ expect(group.outputs[0].output.type).toBe("VAE");
+ expect(group.outputs[1].output.name).toBe("IMAGE");
+ expect(group.outputs[1].output.type).toBe("IMAGE");
+ expect(group.outputs[2].output.name).toBe("LATENT");
+ expect(group.outputs[2].output.type).toBe("LATENT");
+
+ expect(group.outputs[0].connections.length).toBe(1);
+ expect(group.outputs[0].connections[0].targetNode.id).toBe(decode2.id);
+ expect(group.outputs[0].connections[0].targetInput.index).toBe(1);
+
+ expect(group.outputs[1].connections.length).toBe(1);
+ expect(group.outputs[1].connections[0].targetNode.id).toBe(save.id);
+ expect(group.outputs[1].connections[0].targetInput.index).toBe(0);
+
+ expect(group.outputs[2].connections.length).toBe(1);
+ expect(group.outputs[2].connections[0].targetNode.id).toBe(decode2.id);
+ expect(group.outputs[2].connections[0].targetInput.index).toBe(0);
+
+ expect((await graph.toPrompt()).output).toEqual({
+ ...getOutput({ 1: ckpt.id, 2: pos.id, 3: neg.id, 4: empty.id, 5: sampler.id, 6: decode1.id, 7: save.id }),
+ [vae.id]: { inputs: { vae_name: "vae1.safetensors" }, class_type: vae.node.type },
+ [encode.id]: { inputs: { pixels: ["6", 0], vae: [vae.id + "", 0] }, class_type: encode.node.type },
+ [decode2.id]: { inputs: { samples: [encode.id + "", 0], vae: [vae.id + "", 0] }, class_type: decode2.node.type },
+ [preview.id]: { inputs: { images: [decode2.id + "", 0] }, class_type: preview.node.type },
+ });
+ });
+ test("works with IMAGEUPLOAD widget", async () => {
+ const { ez, graph, app } = await start();
+ const img = ez.LoadImage();
+ const preview1 = ez.PreviewImage(img.outputs[0]);
+
+ const group = await convertToGroup(app, graph, "test", [img, preview1]);
+ const widget = group.widgets["upload"];
+ expect(widget).toBeTruthy();
+ expect(widget.widget.type).toBe("button");
+ });
+ test("internal primitive populates widgets for all linked inputs", async () => {
+ const { ez, graph, app } = await start();
+ const img = ez.LoadImage();
+ const scale1 = ez.ImageScale(img.outputs[0]);
+ const scale2 = ez.ImageScale(img.outputs[0]);
+ ez.PreviewImage(scale1.outputs[0]);
+ ez.PreviewImage(scale2.outputs[0]);
+
+ scale1.widgets.width.convertToInput();
+ scale2.widgets.height.convertToInput();
+
+ const primitive = ez.PrimitiveNode();
+ primitive.outputs[0].connectTo(scale1.inputs.width);
+ primitive.outputs[0].connectTo(scale2.inputs.height);
+
+ const group = await convertToGroup(app, graph, "test", [img, primitive, scale1, scale2]);
+ group.widgets.value.value = 100;
+ expect((await graph.toPrompt()).output).toEqual({
+ 1: {
+ inputs: { image: img.widgets.image.value, upload: "image" },
+ class_type: "LoadImage",
+ },
+ 2: {
+ inputs: { upscale_method: "nearest-exact", width: 100, height: 512, crop: "disabled", image: ["1", 0] },
+ class_type: "ImageScale",
+ },
+ 3: {
+ inputs: { upscale_method: "nearest-exact", width: 512, height: 100, crop: "disabled", image: ["1", 0] },
+ class_type: "ImageScale",
+ },
+ 4: { inputs: { images: ["2", 0] }, class_type: "PreviewImage" },
+ 5: { inputs: { images: ["3", 0] }, class_type: "PreviewImage" },
+ });
+ });
+ test("primitive control widgets values are copied on convert", async () => {
+ const { ez, graph, app } = await start();
+ const sampler = ez.KSampler();
+ sampler.widgets.seed.convertToInput();
+ sampler.widgets.sampler_name.convertToInput();
+
+ let p1 = ez.PrimitiveNode();
+ let p2 = ez.PrimitiveNode();
+ p1.outputs[0].connectTo(sampler.inputs.seed);
+ p2.outputs[0].connectTo(sampler.inputs.sampler_name);
+
+ p1.widgets.control_after_generate.value = "increment";
+ p2.widgets.control_after_generate.value = "decrement";
+ p2.widgets.control_filter_list.value = "/.*/";
+
+ p2.node.title = "p2";
+
+ const group = await convertToGroup(app, graph, "test", [sampler, p1, p2]);
+ expect(group.widgets.control_after_generate.value).toBe("increment");
+ expect(group.widgets["p2 control_after_generate"].value).toBe("decrement");
+ expect(group.widgets["p2 control_filter_list"].value).toBe("/.*/");
+
+ group.widgets.control_after_generate.value = "fixed";
+ group.widgets["p2 control_after_generate"].value = "randomize";
+ group.widgets["p2 control_filter_list"].value = "/.+/";
+
+ group.menu["Convert to nodes"].call();
+ p1 = graph.find(p1);
+ p2 = graph.find(p2);
+
+ expect(p1.widgets.control_after_generate.value).toBe("fixed");
+ expect(p2.widgets.control_after_generate.value).toBe("randomize");
+ expect(p2.widgets.control_filter_list.value).toBe("/.+/");
+ });
+ test("internal reroutes work with converted inputs and merge options", async () => {
+ const { ez, graph, app } = await start();
+ const vae = ez.VAELoader();
+ const latent = ez.EmptyLatentImage();
+ const decode = ez.VAEDecode(latent.outputs.LATENT, vae.outputs.VAE);
+ const scale = ez.ImageScale(decode.outputs.IMAGE);
+ ez.PreviewImage(scale.outputs.IMAGE);
+
+ const r1 = ez.Reroute();
+ const r2 = ez.Reroute();
+
+ latent.widgets.width.value = 64;
+ latent.widgets.height.value = 128;
+
+ latent.widgets.width.convertToInput();
+ latent.widgets.height.convertToInput();
+ latent.widgets.batch_size.convertToInput();
+
+ scale.widgets.width.convertToInput();
+ scale.widgets.height.convertToInput();
+
+ r1.inputs[0].input.label = "hbw";
+ r1.outputs[0].connectTo(latent.inputs.height);
+ r1.outputs[0].connectTo(latent.inputs.batch_size);
+ r1.outputs[0].connectTo(scale.inputs.width);
+
+ r2.inputs[0].input.label = "wh";
+ r2.outputs[0].connectTo(latent.inputs.width);
+ r2.outputs[0].connectTo(scale.inputs.height);
+
+ const group = await convertToGroup(app, graph, "test", [r1, r2, latent, decode, scale]);
+
+ expect(group.inputs[0].input.type).toBe("VAE");
+ expect(group.inputs[1].input.type).toBe("INT");
+ expect(group.inputs[2].input.type).toBe("INT");
+
+ const p1 = ez.PrimitiveNode();
+ const p2 = ez.PrimitiveNode();
+ p1.outputs[0].connectTo(group.inputs[1]);
+ p2.outputs[0].connectTo(group.inputs[2]);
+
+ expect(p1.widgets.value.widget.options?.min).toBe(16); // width/height min
+ expect(p1.widgets.value.widget.options?.max).toBe(4096); // batch max
+ expect(p1.widgets.value.widget.options?.step).toBe(80); // width/height step * 10
+
+ expect(p2.widgets.value.widget.options?.min).toBe(16); // width/height min
+ expect(p2.widgets.value.widget.options?.max).toBe(8192); // width/height max
+ expect(p2.widgets.value.widget.options?.step).toBe(80); // width/height step * 10
+
+ expect(p1.widgets.value.value).toBe(128);
+ expect(p2.widgets.value.value).toBe(64);
+
+ p1.widgets.value.value = 16;
+ p2.widgets.value.value = 32;
+
+ await checkBeforeAndAfterReload(graph, async (r) => {
+ const id = (v) => (r ? `${group.id}:` : "") + v;
+ expect((await graph.toPrompt()).output).toStrictEqual({
+ 1: { inputs: { vae_name: "vae1.safetensors" }, class_type: "VAELoader" },
+ [id(2)]: { inputs: { width: 32, height: 16, batch_size: 16 }, class_type: "EmptyLatentImage" },
+ [id(3)]: { inputs: { samples: [id(2), 0], vae: ["1", 0] }, class_type: "VAEDecode" },
+ [id(4)]: {
+ inputs: { upscale_method: "nearest-exact", width: 16, height: 32, crop: "disabled", image: [id(3), 0] },
+ class_type: "ImageScale",
+ },
+ 5: { inputs: { images: [id(4), 0] }, class_type: "PreviewImage" },
+ });
+ });
+ });
+ test("converted inputs with linked widgets map values correctly on creation", async () => {
+ const { ez, graph, app } = await start();
+ const k1 = ez.KSampler();
+ const k2 = ez.KSampler();
+ k1.widgets.seed.convertToInput();
+ k2.widgets.seed.convertToInput();
+
+ const rr = ez.Reroute();
+ rr.outputs[0].connectTo(k1.inputs.seed);
+ rr.outputs[0].connectTo(k2.inputs.seed);
+
+ const group = await convertToGroup(app, graph, "test", [k1, k2, rr]);
+ expect(group.widgets.steps.value).toBe(20);
+ expect(group.widgets.cfg.value).toBe(8);
+ expect(group.widgets.scheduler.value).toBe("normal");
+ expect(group.widgets["KSampler steps"].value).toBe(20);
+ expect(group.widgets["KSampler cfg"].value).toBe(8);
+ expect(group.widgets["KSampler scheduler"].value).toBe("normal");
+ });
+ test("allow multiple of the same node type to be added", async () => {
+ const { ez, graph, app } = await start();
+ const nodes = [...Array(10)].map(() => ez.ImageScaleBy());
+ const group = await convertToGroup(app, graph, "test", nodes);
+ expect(group.inputs.length).toBe(10);
+ expect(group.outputs.length).toBe(10);
+ expect(group.widgets.length).toBe(20);
+ expect(group.widgets.map((w) => w.widget.name)).toStrictEqual(
+ [...Array(10)]
+ .map((_, i) => `${i > 0 ? "ImageScaleBy " : ""}${i > 1 ? i + " " : ""}`)
+ .flatMap((p) => [`${p}upscale_method`, `${p}scale_by`])
+ );
+ });
+});
diff --git a/tests-ui/tests/widgetInputs.test.js b/tests-ui/tests/widgetInputs.test.js
index 022e54926..67e3fa341 100644
--- a/tests-ui/tests/widgetInputs.test.js
+++ b/tests-ui/tests/widgetInputs.test.js
@@ -1,7 +1,13 @@
// @ts-check
///
-const { start, makeNodeDef, checkBeforeAndAfterReload, assertNotNullOrUndefined } = require("../utils");
+const {
+ start,
+ makeNodeDef,
+ checkBeforeAndAfterReload,
+ assertNotNullOrUndefined,
+ createDefaultWorkflow,
+} = require("../utils");
const lg = require("../utils/litegraph");
/**
@@ -14,10 +20,10 @@ const lg = require("../utils/litegraph");
* @param { InstanceType } graph
* @param { InstanceType } input
* @param { string } widgetType
- * @param { boolean } hasControlWidget
+ * @param { number } controlWidgetCount
* @returns
*/
-async function connectPrimitiveAndReload(ez, graph, input, widgetType, hasControlWidget) {
+async function connectPrimitiveAndReload(ez, graph, input, widgetType, controlWidgetCount = 0) {
// Connect to primitive and ensure its still connected after
let primitive = ez.PrimitiveNode();
primitive.outputs[0].connectTo(input);
@@ -33,13 +39,17 @@ async function connectPrimitiveAndReload(ez, graph, input, widgetType, hasContro
expect(valueWidget.widget.type).toBe(widgetType);
// Check if control_after_generate should be added
- if (hasControlWidget) {
+ if (controlWidgetCount) {
const controlWidget = primitive.widgets.control_after_generate;
expect(controlWidget.widget.type).toBe("combo");
+ if (widgetType === "combo") {
+ const filterWidget = primitive.widgets.control_filter_list;
+ expect(filterWidget.widget.type).toBe("string");
+ }
}
// Ensure we dont have other widgets
- expect(primitive.node.widgets).toHaveLength(1 + +!!hasControlWidget);
+ expect(primitive.node.widgets).toHaveLength(1 + controlWidgetCount);
});
return primitive;
@@ -55,8 +65,8 @@ describe("widget inputs", () => {
});
[
- { name: "int", type: "INT", widget: "number", control: true },
- { name: "float", type: "FLOAT", widget: "number", control: true },
+ { name: "int", type: "INT", widget: "number", control: 1 },
+ { name: "float", type: "FLOAT", widget: "number", control: 1 },
{ name: "text", type: "STRING" },
{
name: "customtext",
@@ -64,7 +74,7 @@ describe("widget inputs", () => {
opt: { multiline: true },
},
{ name: "toggle", type: "BOOLEAN" },
- { name: "combo", type: ["a", "b", "c"], control: true },
+ { name: "combo", type: ["a", "b", "c"], control: 2 },
].forEach((c) => {
test(`widget conversion + primitive works on ${c.name}`, async () => {
const { ez, graph } = await start({
@@ -106,7 +116,7 @@ describe("widget inputs", () => {
n.widgets.ckpt_name.convertToInput();
expect(n.inputs.length).toEqual(inputCount + 1);
- const primitive = await connectPrimitiveAndReload(ez, graph, n.inputs.ckpt_name, "combo", true);
+ const primitive = await connectPrimitiveAndReload(ez, graph, n.inputs.ckpt_name, "combo", 2);
// Disconnect & reconnect
primitive.outputs[0].connections[0].disconnect();
@@ -198,8 +208,8 @@ describe("widget inputs", () => {
});
expect(dialogShow).toBeCalledTimes(1);
- expect(dialogShow.mock.calls[0][0]).toContain("the following node types were not found");
- expect(dialogShow.mock.calls[0][0]).toContain("TestNode");
+ expect(dialogShow.mock.calls[0][0].innerHTML).toContain("the following node types were not found");
+ expect(dialogShow.mock.calls[0][0].innerHTML).toContain("TestNode");
});
test("defaultInput widgets can be converted back to inputs", async () => {
@@ -226,7 +236,7 @@ describe("widget inputs", () => {
// Reload and ensure it still only has 1 converted widget
if (!assertNotNullOrUndefined(input)) return;
- await connectPrimitiveAndReload(ez, graph, input, "number", true);
+ await connectPrimitiveAndReload(ez, graph, input, "number", 1);
n = graph.find(n);
expect(n.widgets).toHaveLength(1);
w = n.widgets.example;
@@ -258,7 +268,7 @@ describe("widget inputs", () => {
// Reload and ensure it still only has 1 converted widget
if (assertNotNullOrUndefined(input)) {
- await connectPrimitiveAndReload(ez, graph, input, "number", true);
+ await connectPrimitiveAndReload(ez, graph, input, "number", 1);
n = graph.find(n);
expect(n.widgets).toHaveLength(1);
expect(n.widgets.example.isConvertedToInput).toBeTruthy();
@@ -304,8 +314,8 @@ describe("widget inputs", () => {
const { ez } = await start({
mockNodeDefs: {
...makeNodeDef("TestNode1", {}, [["A", "B"]]),
- ...makeNodeDef("TestNode2", { example: [["A", "B"], { forceInput: true}] }),
- ...makeNodeDef("TestNode3", { example: [["A", "B", "C"], { forceInput: true}] }),
+ ...makeNodeDef("TestNode2", { example: [["A", "B"], { forceInput: true }] }),
+ ...makeNodeDef("TestNode3", { example: [["A", "B", "C"], { forceInput: true }] }),
},
});
@@ -316,4 +326,232 @@ describe("widget inputs", () => {
n1.outputs[0].connectTo(n2.inputs[0]);
expect(() => n1.outputs[0].connectTo(n3.inputs[0])).toThrow();
});
+
+ test("combo primitive can filter list when control_after_generate called", async () => {
+ const { ez } = await start({
+ mockNodeDefs: {
+ ...makeNodeDef("TestNode1", { example: [["A", "B", "C", "D", "AA", "BB", "CC", "DD", "AAA", "BBB"], {}] }),
+ },
+ });
+
+ const n1 = ez.TestNode1();
+ n1.widgets.example.convertToInput();
+ const p = ez.PrimitiveNode();
+ p.outputs[0].connectTo(n1.inputs[0]);
+
+ const value = p.widgets.value;
+ const control = p.widgets.control_after_generate.widget;
+ const filter = p.widgets.control_filter_list;
+
+ expect(p.widgets.length).toBe(3);
+ control.value = "increment";
+ expect(value.value).toBe("A");
+
+ // Manually trigger after queue when set to increment
+ control["afterQueued"]();
+ expect(value.value).toBe("B");
+
+ // Filter to items containing D
+ filter.value = "D";
+ control["afterQueued"]();
+ expect(value.value).toBe("D");
+ control["afterQueued"]();
+ expect(value.value).toBe("DD");
+
+ // Check decrement
+ value.value = "BBB";
+ control.value = "decrement";
+ filter.value = "B";
+ control["afterQueued"]();
+ expect(value.value).toBe("BB");
+ control["afterQueued"]();
+ expect(value.value).toBe("B");
+
+ // Check regex works
+ value.value = "BBB";
+ filter.value = "/[AB]|^C$/";
+ control["afterQueued"]();
+ expect(value.value).toBe("AAA");
+ control["afterQueued"]();
+ expect(value.value).toBe("BB");
+ control["afterQueued"]();
+ expect(value.value).toBe("AA");
+ control["afterQueued"]();
+ expect(value.value).toBe("C");
+ control["afterQueued"]();
+ expect(value.value).toBe("B");
+ control["afterQueued"]();
+ expect(value.value).toBe("A");
+
+ // Check random
+ control.value = "randomize";
+ filter.value = "/D/";
+ for (let i = 0; i < 100; i++) {
+ control["afterQueued"]();
+ expect(value.value === "D" || value.value === "DD").toBeTruthy();
+ }
+
+ // Ensure it doesnt apply when fixed
+ control.value = "fixed";
+ value.value = "B";
+ filter.value = "C";
+ control["afterQueued"]();
+ expect(value.value).toBe("B");
+ });
+
+ describe("reroutes", () => {
+ async function checkOutput(graph, values) {
+ expect((await graph.toPrompt()).output).toStrictEqual({
+ 1: { inputs: { ckpt_name: "model1.safetensors" }, class_type: "CheckpointLoaderSimple" },
+ 2: { inputs: { text: "positive", clip: ["1", 1] }, class_type: "CLIPTextEncode" },
+ 3: { inputs: { text: "negative", clip: ["1", 1] }, class_type: "CLIPTextEncode" },
+ 4: {
+ inputs: { width: values.width ?? 512, height: values.height ?? 512, batch_size: values?.batch_size ?? 1 },
+ class_type: "EmptyLatentImage",
+ },
+ 5: {
+ inputs: {
+ seed: 0,
+ steps: 20,
+ cfg: 8,
+ sampler_name: "euler",
+ scheduler: values?.scheduler ?? "normal",
+ denoise: 1,
+ model: ["1", 0],
+ positive: ["2", 0],
+ negative: ["3", 0],
+ latent_image: ["4", 0],
+ },
+ class_type: "KSampler",
+ },
+ 6: { inputs: { samples: ["5", 0], vae: ["1", 2] }, class_type: "VAEDecode" },
+ 7: {
+ inputs: { filename_prefix: values.filename_prefix ?? "ComfyUI", images: ["6", 0] },
+ class_type: "SaveImage",
+ },
+ });
+ }
+
+ async function waitForWidget(node) {
+ // widgets are created slightly after the graph is ready
+ // hard to find an exact hook to get these so just wait for them to be ready
+ for (let i = 0; i < 10; i++) {
+ await new Promise((r) => setTimeout(r, 10));
+ if (node.widgets?.value) {
+ return;
+ }
+ }
+ }
+
+ it("can connect primitive via a reroute path to a widget input", async () => {
+ const { ez, graph } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+
+ nodes.empty.widgets.width.convertToInput();
+ nodes.sampler.widgets.scheduler.convertToInput();
+ nodes.save.widgets.filename_prefix.convertToInput();
+
+ let widthReroute = ez.Reroute();
+ let schedulerReroute = ez.Reroute();
+ let fileReroute = ez.Reroute();
+
+ let widthNext = widthReroute;
+ let schedulerNext = schedulerReroute;
+ let fileNext = fileReroute;
+
+ for (let i = 0; i < 5; i++) {
+ let next = ez.Reroute();
+ widthNext.outputs[0].connectTo(next.inputs[0]);
+ widthNext = next;
+
+ next = ez.Reroute();
+ schedulerNext.outputs[0].connectTo(next.inputs[0]);
+ schedulerNext = next;
+
+ next = ez.Reroute();
+ fileNext.outputs[0].connectTo(next.inputs[0]);
+ fileNext = next;
+ }
+
+ widthNext.outputs[0].connectTo(nodes.empty.inputs.width);
+ schedulerNext.outputs[0].connectTo(nodes.sampler.inputs.scheduler);
+ fileNext.outputs[0].connectTo(nodes.save.inputs.filename_prefix);
+
+ let widthPrimitive = ez.PrimitiveNode();
+ let schedulerPrimitive = ez.PrimitiveNode();
+ let filePrimitive = ez.PrimitiveNode();
+
+ widthPrimitive.outputs[0].connectTo(widthReroute.inputs[0]);
+ schedulerPrimitive.outputs[0].connectTo(schedulerReroute.inputs[0]);
+ filePrimitive.outputs[0].connectTo(fileReroute.inputs[0]);
+ expect(widthPrimitive.widgets.value.value).toBe(512);
+ widthPrimitive.widgets.value.value = 1024;
+ expect(schedulerPrimitive.widgets.value.value).toBe("normal");
+ schedulerPrimitive.widgets.value.value = "simple";
+ expect(filePrimitive.widgets.value.value).toBe("ComfyUI");
+ filePrimitive.widgets.value.value = "ComfyTest";
+
+ await checkBeforeAndAfterReload(graph, async () => {
+ widthPrimitive = graph.find(widthPrimitive);
+ schedulerPrimitive = graph.find(schedulerPrimitive);
+ filePrimitive = graph.find(filePrimitive);
+ await waitForWidget(filePrimitive);
+ expect(widthPrimitive.widgets.length).toBe(2);
+ expect(schedulerPrimitive.widgets.length).toBe(3);
+ expect(filePrimitive.widgets.length).toBe(1);
+
+ await checkOutput(graph, {
+ width: 1024,
+ scheduler: "simple",
+ filename_prefix: "ComfyTest",
+ });
+ });
+ });
+ it("can connect primitive via a reroute path to multiple widget inputs", async () => {
+ const { ez, graph } = await start();
+ const nodes = createDefaultWorkflow(ez, graph);
+
+ nodes.empty.widgets.width.convertToInput();
+ nodes.empty.widgets.height.convertToInput();
+ nodes.empty.widgets.batch_size.convertToInput();
+
+ let reroute = ez.Reroute();
+ let prevReroute = reroute;
+ for (let i = 0; i < 5; i++) {
+ const next = ez.Reroute();
+ prevReroute.outputs[0].connectTo(next.inputs[0]);
+ prevReroute = next;
+ }
+
+ const r1 = ez.Reroute(prevReroute.outputs[0]);
+ const r2 = ez.Reroute(prevReroute.outputs[0]);
+ const r3 = ez.Reroute(r2.outputs[0]);
+ const r4 = ez.Reroute(r2.outputs[0]);
+
+ r1.outputs[0].connectTo(nodes.empty.inputs.width);
+ r3.outputs[0].connectTo(nodes.empty.inputs.height);
+ r4.outputs[0].connectTo(nodes.empty.inputs.batch_size);
+
+ let primitive = ez.PrimitiveNode();
+ primitive.outputs[0].connectTo(reroute.inputs[0]);
+ expect(primitive.widgets.value.value).toBe(1);
+ primitive.widgets.value.value = 64;
+
+ await checkBeforeAndAfterReload(graph, async (r) => {
+ primitive = graph.find(primitive);
+ await waitForWidget(primitive);
+
+ // Ensure widget configs are merged
+ expect(primitive.widgets.value.widget.options?.min).toBe(16); // width/height min
+ expect(primitive.widgets.value.widget.options?.max).toBe(4096); // batch max
+ expect(primitive.widgets.value.widget.options?.step).toBe(80); // width/height step * 10
+
+ await checkOutput(graph, {
+ width: 64,
+ height: 64,
+ batch_size: 64,
+ });
+ });
+ });
+ });
});
diff --git a/tests-ui/utils/ezgraph.js b/tests-ui/utils/ezgraph.js
index 0e81fd47b..8a55246ee 100644
--- a/tests-ui/utils/ezgraph.js
+++ b/tests-ui/utils/ezgraph.js
@@ -78,6 +78,14 @@ export class EzInput extends EzSlot {
this.input = input;
}
+ get connection() {
+ const link = this.node.node.inputs?.[this.index]?.link;
+ if (link == null) {
+ return null;
+ }
+ return new EzConnection(this.node.app, this.node.app.graph.links[link]);
+ }
+
disconnect() {
this.node.node.disconnectInput(this.index);
}
@@ -117,7 +125,7 @@ export class EzOutput extends EzSlot {
const inp = input.input;
const inName = inp.name || inp.label || inp.type;
throw new Error(
- `Connecting from ${input.node.node.type}[${inName}#${input.index}] -> ${this.node.node.type}[${
+ `Connecting from ${input.node.node.type}#${input.node.id}[${inName}#${input.index}] -> ${this.node.node.type}#${this.node.id}[${
this.output.name ?? this.output.type
}#${this.index}] failed.`
);
@@ -150,7 +158,7 @@ export class EzNodeMenuItem {
if (selectNode) {
this.node.select();
}
- this.item.callback.call(this.node.node, undefined, undefined, undefined, undefined, this.node.node);
+ return this.item.callback.call(this.node.node, undefined, undefined, undefined, undefined, this.node.node);
}
}
@@ -179,6 +187,7 @@ export class EzWidget {
set value(v) {
this.widget.value = v;
+ this.widget.callback?.call?.(this.widget, v)
}
get isConvertedToInput() {
@@ -240,8 +249,12 @@ export class EzNode {
return this.#makeLookupArray(() => this.app.canvas.getNodeMenuOptions(this.node), "content", EzNodeMenuItem);
}
- select() {
- this.app.canvas.selectNode(this.node);
+ get isRemoved() {
+ return !this.app.graph.getNodeById(this.id);
+ }
+
+ select(addToSelection = false) {
+ this.app.canvas.selectNode(this.node, addToSelection);
}
// /**
@@ -275,12 +288,17 @@ export class EzNode {
if (!s) return p;
const name = s[nameProperty];
+ const item = new ctor(this, i, s);
// @ts-ignore
- if (!name || name in p) {
- throw new Error(`Unable to store ${nodeProperty} ${name} on array as name conflicts.`);
+ p.push(item);
+ if (name) {
+ // @ts-ignore
+ if (name in p) {
+ throw new Error(`Unable to store ${nodeProperty} ${name} on array as name conflicts.`);
+ }
}
// @ts-ignore
- p.push((p[name] = new ctor(this, i, s)));
+ p[name] = item;
return p;
}, Object.assign([], { $: this }));
}
@@ -310,7 +328,7 @@ export class EzGraph {
}
stringify() {
- return JSON.stringify(this.app.graph.serialize(), undefined, "\t");
+ return JSON.stringify(this.app.graph.serialize(), undefined);
}
/**
@@ -348,6 +366,19 @@ export class EzGraph {
}, 10);
});
}
+
+ /**
+ * @returns { Promise<{
+ * workflow: {},
+ * output: Record
+ * }>}> }
+ */
+ toPrompt() {
+ // @ts-ignore
+ return this.app.graphToPrompt();
+ }
}
export const Ez = {
@@ -356,12 +387,12 @@ export const Ez = {
* @example
* const { ez, graph } = Ez.graph(app);
* graph.clear();
- * const [model, clip, vae] = ez.CheckpointLoaderSimple();
- * const [pos] = ez.CLIPTextEncode(clip, { text: "positive" });
- * const [neg] = ez.CLIPTextEncode(clip, { text: "negative" });
- * const [latent] = ez.KSampler(model, pos, neg, ...ez.EmptyLatentImage());
- * const [image] = ez.VAEDecode(latent, vae);
- * const saveNode = ez.SaveImage(image).node;
+ * const [model, clip, vae] = ez.CheckpointLoaderSimple().outputs;
+ * const [pos] = ez.CLIPTextEncode(clip, { text: "positive" }).outputs;
+ * const [neg] = ez.CLIPTextEncode(clip, { text: "negative" }).outputs;
+ * const [latent] = ez.KSampler(model, pos, neg, ...ez.EmptyLatentImage().outputs).outputs;
+ * const [image] = ez.VAEDecode(latent, vae).outputs;
+ * const saveNode = ez.SaveImage(image);
* console.log(saveNode);
* graph.arrange();
* @param { app } app
diff --git a/tests-ui/utils/index.js b/tests-ui/utils/index.js
index 01c58b21f..6a08e8594 100644
--- a/tests-ui/utils/index.js
+++ b/tests-ui/utils/index.js
@@ -1,21 +1,29 @@
const { mockApi } = require("./setup");
const { Ez } = require("./ezgraph");
+const lg = require("./litegraph");
/**
*
- * @param { Parameters[0] } config
+ * @param { Parameters[0] & { resetEnv?: boolean, preSetup?(app): Promise } } config
* @returns
*/
-export async function start(config = undefined) {
+export async function start(config = {}) {
+ if(config.resetEnv) {
+ jest.resetModules();
+ jest.resetAllMocks();
+ lg.setup(global);
+ }
+
mockApi(config);
const { app } = require("../../web/scripts/app");
+ config.preSetup?.(app);
await app.setup();
- return Ez.graph(app, global["LiteGraph"], global["LGraphCanvas"]);
+ return { ...Ez.graph(app, global["LiteGraph"], global["LGraphCanvas"]), app };
}
/**
- * @param { ReturnType["graph"] } graph
- * @param { (hasReloaded: boolean) => (Promise | void) } cb
+ * @param { ReturnType["graph"] } graph
+ * @param { (hasReloaded: boolean) => (Promise | void) } cb
*/
export async function checkBeforeAndAfterReload(graph, cb) {
await cb(false);
@@ -24,10 +32,10 @@ export async function checkBeforeAndAfterReload(graph, cb) {
}
/**
- * @param { string } name
- * @param { Record } input
+ * @param { string } name
+ * @param { Record } input
* @param { (string | string[])[] | Record } output
- * @returns { Record }
+ * @returns { Record }
*/
export function makeNodeDef(name, input, output = {}) {
const nodeDef = {
@@ -37,19 +45,19 @@ export function makeNodeDef(name, input, output = {}) {
output_name: [],
output_is_list: [],
input: {
- required: {}
+ required: {},
},
};
- for(const k in input) {
+ for (const k in input) {
nodeDef.input.required[k] = typeof input[k] === "string" ? [input[k], {}] : [...input[k]];
}
- if(output instanceof Array) {
+ if (output instanceof Array) {
output = output.reduce((p, c) => {
p[c] = c;
return p;
- }, {})
+ }, {});
}
- for(const k in output) {
+ for (const k in output) {
nodeDef.output.push(output[k]);
nodeDef.output_name.push(k);
nodeDef.output_is_list.push(false);
@@ -68,4 +76,40 @@ export function assertNotNullOrUndefined(x) {
expect(x).not.toEqual(null);
expect(x).not.toEqual(undefined);
return true;
+}
+
+/**
+ *
+ * @param { ReturnType["ez"] } ez
+ * @param { ReturnType["graph"] } graph
+ */
+export function createDefaultWorkflow(ez, graph) {
+ graph.clear();
+ const ckpt = ez.CheckpointLoaderSimple();
+
+ const pos = ez.CLIPTextEncode(ckpt.outputs.CLIP, { text: "positive" });
+ const neg = ez.CLIPTextEncode(ckpt.outputs.CLIP, { text: "negative" });
+
+ const empty = ez.EmptyLatentImage();
+ const sampler = ez.KSampler(
+ ckpt.outputs.MODEL,
+ pos.outputs.CONDITIONING,
+ neg.outputs.CONDITIONING,
+ empty.outputs.LATENT
+ );
+
+ const decode = ez.VAEDecode(sampler.outputs.LATENT, ckpt.outputs.VAE);
+ const save = ez.SaveImage(decode.outputs.IMAGE);
+ graph.arrange();
+
+ return { ckpt, pos, neg, empty, sampler, decode, save };
+}
+
+export async function getNodeDefs() {
+ const { api } = require("../../web/scripts/api");
+ return api.getNodeDefs();
+}
+
+export async function getNodeDef(nodeId) {
+ return (await getNodeDefs())[nodeId];
}
\ No newline at end of file
diff --git a/tests-ui/utils/setup.js b/tests-ui/utils/setup.js
index 17e8ac1ad..dd150214a 100644
--- a/tests-ui/utils/setup.js
+++ b/tests-ui/utils/setup.js
@@ -30,16 +30,20 @@ export function mockApi({ mockExtensions, mockNodeDefs } = {}) {
mockNodeDefs = JSON.parse(fs.readFileSync(path.resolve("./data/object_info.json")));
}
+ const events = new EventTarget();
+ const mockApi = {
+ addEventListener: events.addEventListener.bind(events),
+ removeEventListener: events.removeEventListener.bind(events),
+ dispatchEvent: events.dispatchEvent.bind(events),
+ getSystemStats: jest.fn(),
+ getExtensions: jest.fn(() => mockExtensions),
+ getNodeDefs: jest.fn(() => mockNodeDefs),
+ init: jest.fn(),
+ apiURL: jest.fn((x) => "../../web/" + x),
+ };
jest.mock("../../web/scripts/api", () => ({
get api() {
- return {
- addEventListener: jest.fn(),
- getSystemStats: jest.fn(),
- getExtensions: jest.fn(() => mockExtensions),
- getNodeDefs: jest.fn(() => mockNodeDefs),
- init: jest.fn(),
- apiURL: jest.fn((x) => "../../web/" + x),
- };
+ return mockApi;
},
}));
}
diff --git a/web/extensions/core/colorPalette.js b/web/extensions/core/colorPalette.js
index 3695b08e2..b8d83613d 100644
--- a/web/extensions/core/colorPalette.js
+++ b/web/extensions/core/colorPalette.js
@@ -174,6 +174,213 @@ const colorPalettes = {
"tr-odd-bg-color": "#073642",
}
},
+ },
+ "arc": {
+ "id": "arc",
+ "name": "Arc",
+ "colors": {
+ "node_slot": {
+ "BOOLEAN": "",
+ "CLIP": "#eacb8b",
+ "CLIP_VISION": "#A8DADC",
+ "CLIP_VISION_OUTPUT": "#ad7452",
+ "CONDITIONING": "#cf876f",
+ "CONTROL_NET": "#00d78d",
+ "CONTROL_NET_WEIGHTS": "",
+ "FLOAT": "",
+ "GLIGEN": "",
+ "IMAGE": "#80a1c0",
+ "IMAGEUPLOAD": "",
+ "INT": "",
+ "LATENT": "#b38ead",
+ "LATENT_KEYFRAME": "",
+ "MASK": "#a3bd8d",
+ "MODEL": "#8978a7",
+ "SAMPLER": "",
+ "SIGMAS": "",
+ "STRING": "",
+ "STYLE_MODEL": "#C2FFAE",
+ "T2I_ADAPTER_WEIGHTS": "",
+ "TAESD": "#DCC274",
+ "TIMESTEP_KEYFRAME": "",
+ "UPSCALE_MODEL": "",
+ "VAE": "#be616b"
+ },
+ "litegraph_base": {
+ "BACKGROUND_IMAGE": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAYAAABw4pVUAAAACXBIWXMAAAsTAAALEwEAmpwYAAABcklEQVR4nO3YMUoDARgF4RfxBqZI6/0vZqFn0MYtrLIQMFN8U6V4LAtD+Jm9XG/v30OGl2e/AP7yevz4+vx45nvgF/+QGITEICQGITEIiUFIjNNC3q43u3/YnRJyPOzeQ+0e220nhRzReC8e7R7bbdvl+Jal1Bs46jEIiUFIDEJiEBKDkBhKPbZT6qHdptRTu02p53DUYxASg5AYhMQgJAYhMZR6bKfUQ7tNqad2m1LP4ajHICQGITEIiUFIDEJiKPXYTqmHdptST+02pZ7DUY9BSAxCYhASg5AYhMRQ6rGdUg/tNqWe2m1KPYejHoOQGITEICQGITEIiaHUYzulHtptSj2125R6Dkc9BiExCIlBSAxCYhASQ6nHdko9tNuUemq3KfUcjnoMQmIQEoOQGITEICSGUo/tlHpotyn11G5T6jkc9RiExCAkBiExCIlBSAylHtsp9dBuU+qp3abUczjqMQiJQUgMQmIQEoOQGITE+AHFISNQrFTGuwAAAABJRU5ErkJggg==",
+ "CLEAR_BACKGROUND_COLOR": "#2b2f38",
+ "NODE_TITLE_COLOR": "#b2b7bd",
+ "NODE_SELECTED_TITLE_COLOR": "#FFF",
+ "NODE_TEXT_SIZE": 14,
+ "NODE_TEXT_COLOR": "#AAA",
+ "NODE_SUBTEXT_SIZE": 12,
+ "NODE_DEFAULT_COLOR": "#2b2f38",
+ "NODE_DEFAULT_BGCOLOR": "#242730",
+ "NODE_DEFAULT_BOXCOLOR": "#6e7581",
+ "NODE_DEFAULT_SHAPE": "box",
+ "NODE_BOX_OUTLINE_COLOR": "#FFF",
+ "DEFAULT_SHADOW_COLOR": "rgba(0,0,0,0.5)",
+ "DEFAULT_GROUP_FONT": 22,
+ "WIDGET_BGCOLOR": "#2b2f38",
+ "WIDGET_OUTLINE_COLOR": "#6e7581",
+ "WIDGET_TEXT_COLOR": "#DDD",
+ "WIDGET_SECONDARY_TEXT_COLOR": "#b2b7bd",
+ "LINK_COLOR": "#9A9",
+ "EVENT_LINK_COLOR": "#A86",
+ "CONNECTING_LINK_COLOR": "#AFA"
+ },
+ "comfy_base": {
+ "fg-color": "#fff",
+ "bg-color": "#2b2f38",
+ "comfy-menu-bg": "#242730",
+ "comfy-input-bg": "#2b2f38",
+ "input-text": "#ddd",
+ "descrip-text": "#b2b7bd",
+ "drag-text": "#ccc",
+ "error-text": "#ff4444",
+ "border-color": "#6e7581",
+ "tr-even-bg-color": "#2b2f38",
+ "tr-odd-bg-color": "#242730"
+ }
+ },
+ },
+ "nord": {
+ "id": "nord",
+ "name": "Nord",
+ "colors": {
+ "node_slot": {
+ "BOOLEAN": "",
+ "CLIP": "#eacb8b",
+ "CLIP_VISION": "#A8DADC",
+ "CLIP_VISION_OUTPUT": "#ad7452",
+ "CONDITIONING": "#cf876f",
+ "CONTROL_NET": "#00d78d",
+ "CONTROL_NET_WEIGHTS": "",
+ "FLOAT": "",
+ "GLIGEN": "",
+ "IMAGE": "#80a1c0",
+ "IMAGEUPLOAD": "",
+ "INT": "",
+ "LATENT": "#b38ead",
+ "LATENT_KEYFRAME": "",
+ "MASK": "#a3bd8d",
+ "MODEL": "#8978a7",
+ "SAMPLER": "",
+ "SIGMAS": "",
+ "STRING": "",
+ "STYLE_MODEL": "#C2FFAE",
+ "T2I_ADAPTER_WEIGHTS": "",
+ "TAESD": "#DCC274",
+ "TIMESTEP_KEYFRAME": "",
+ "UPSCALE_MODEL": "",
+ "VAE": "#be616b"
+ },
+ "litegraph_base": {
+ "BACKGROUND_IMAGE": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAIAAAD/gAIDAAAACXBIWXMAAAsTAAALEwEAmpwYAAAFu2lUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgOS4xLWMwMDEgNzkuMTQ2Mjg5OSwgMjAyMy8wNi8yNS0yMDowMTo1NSAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczpkYz0iaHR0cDovL3B1cmwub3JnL2RjL2VsZW1lbnRzLzEuMS8iIHhtbG5zOnBob3Rvc2hvcD0iaHR0cDovL25zLmFkb2JlLmNvbS9waG90b3Nob3AvMS4wLyIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0RXZ0PSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VFdmVudCMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIDI1LjEgKFdpbmRvd3MpIiB4bXA6Q3JlYXRlRGF0ZT0iMjAyMy0xMS0xM1QwMDoxODowMiswMTowMCIgeG1wOk1vZGlmeURhdGU9IjIwMjMtMTEtMTVUMDE6MjA6NDUrMDE6MDAiIHhtcDpNZXRhZGF0YURhdGU9IjIwMjMtMTEtMTVUMDE6MjA6NDUrMDE6MDAiIGRjOmZvcm1hdD0iaW1hZ2UvcG5nIiBwaG90b3Nob3A6Q29sb3JNb2RlPSIzIiB4bXBNTTpJbnN0YW5jZUlEPSJ4bXAuaWlkOjUwNDFhMmZjLTEzNzQtMTk0ZC1hZWY4LTYxMzM1MTVmNjUwMCIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDoyMzFiMTBiMC1iNGZiLTAyNGUtYjEyZS0zMDUzMDNjZDA3YzgiIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDoyMzFiMTBiMC1iNGZiLTAyNGUtYjEyZS0zMDUzMDNjZDA3YzgiPiA8eG1wTU06SGlzdG9yeT4gPHJkZjpTZXE+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJjcmVhdGVkIiBzdEV2dDppbnN0YW5jZUlEPSJ4bXAuaWlkOjIzMWIxMGIwLWI0ZmItMDI0ZS1iMTJlLTMwNTMwM2NkMDdjOCIgc3RFdnQ6d2hlbj0iMjAyMy0xMS0xM1QwMDoxODowMiswMTowMCIgc3RFdnQ6c29mdHdhcmVBZ2VudD0iQWRvYmUgUGhvdG9zaG9wIDI1LjEgKFdpbmRvd3MpIi8+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJzYXZlZCIgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDo1MDQxYTJmYy0xMzc0LTE5NGQtYWVmOC02MTMzNTE1ZjY1MDAiIHN0RXZ0OndoZW49IjIwMjMtMTEtMTVUMDE6MjA6NDUrMDE6MDAiIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkFkb2JlIFBob3Rvc2hvcCAyNS4xIChXaW5kb3dzKSIgc3RFdnQ6Y2hhbmdlZD0iLyIvPiA8L3JkZjpTZXE+IDwveG1wTU06SGlzdG9yeT4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz73jWg/AAAAyUlEQVR42u3WKwoAIBRFQRdiMb1idv9Lsxn9gEFw4Dbb8JCTojbbXEJwjJVL2HKwYMGCBQuWLbDmjr+9zrBGjHl1WVcvy2DBggULFizTWQpewSt4HzwsgwULFiwFr7MUvMtS8D54WLBgGSxYCl7BK3iXZbBgwYIFC5bpLAWv4BW8Dx6WwYIFC5aC11kK3mUpeB88LFiwDBYsBa/gFbzLMliwYMGCBct0loJX8AreBw/LYMGCBUvB6ywF77IUvA8eFixYBgsWrNfWAZPltufdad+1AAAAAElFTkSuQmCC",
+ "CLEAR_BACKGROUND_COLOR": "#212732",
+ "NODE_TITLE_COLOR": "#999",
+ "NODE_SELECTED_TITLE_COLOR": "#e5eaf0",
+ "NODE_TEXT_SIZE": 14,
+ "NODE_TEXT_COLOR": "#bcc2c8",
+ "NODE_SUBTEXT_SIZE": 12,
+ "NODE_DEFAULT_COLOR": "#2e3440",
+ "NODE_DEFAULT_BGCOLOR": "#161b22",
+ "NODE_DEFAULT_BOXCOLOR": "#545d70",
+ "NODE_DEFAULT_SHAPE": "box",
+ "NODE_BOX_OUTLINE_COLOR": "#e5eaf0",
+ "DEFAULT_SHADOW_COLOR": "rgba(0,0,0,0.5)",
+ "DEFAULT_GROUP_FONT": 24,
+ "WIDGET_BGCOLOR": "#2e3440",
+ "WIDGET_OUTLINE_COLOR": "#545d70",
+ "WIDGET_TEXT_COLOR": "#bcc2c8",
+ "WIDGET_SECONDARY_TEXT_COLOR": "#999",
+ "LINK_COLOR": "#9A9",
+ "EVENT_LINK_COLOR": "#A86",
+ "CONNECTING_LINK_COLOR": "#AFA"
+ },
+ "comfy_base": {
+ "fg-color": "#e5eaf0",
+ "bg-color": "#2e3440",
+ "comfy-menu-bg": "#161b22",
+ "comfy-input-bg": "#2e3440",
+ "input-text": "#bcc2c8",
+ "descrip-text": "#999",
+ "drag-text": "#ccc",
+ "error-text": "#ff4444",
+ "border-color": "#545d70",
+ "tr-even-bg-color": "#2e3440",
+ "tr-odd-bg-color": "#161b22"
+ }
+ },
+ },
+ "github": {
+ "id": "github",
+ "name": "Github",
+ "colors": {
+ "node_slot": {
+ "BOOLEAN": "",
+ "CLIP": "#eacb8b",
+ "CLIP_VISION": "#A8DADC",
+ "CLIP_VISION_OUTPUT": "#ad7452",
+ "CONDITIONING": "#cf876f",
+ "CONTROL_NET": "#00d78d",
+ "CONTROL_NET_WEIGHTS": "",
+ "FLOAT": "",
+ "GLIGEN": "",
+ "IMAGE": "#80a1c0",
+ "IMAGEUPLOAD": "",
+ "INT": "",
+ "LATENT": "#b38ead",
+ "LATENT_KEYFRAME": "",
+ "MASK": "#a3bd8d",
+ "MODEL": "#8978a7",
+ "SAMPLER": "",
+ "SIGMAS": "",
+ "STRING": "",
+ "STYLE_MODEL": "#C2FFAE",
+ "T2I_ADAPTER_WEIGHTS": "",
+ "TAESD": "#DCC274",
+ "TIMESTEP_KEYFRAME": "",
+ "UPSCALE_MODEL": "",
+ "VAE": "#be616b"
+ },
+ "litegraph_base": {
+ "BACKGROUND_IMAGE": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAIAAAD/gAIDAAAACXBIWXMAAAsTAAALEwEAmpwYAAAGlmlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgOS4xLWMwMDEgNzkuMTQ2Mjg5OSwgMjAyMy8wNi8yNS0yMDowMTo1NSAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczpkYz0iaHR0cDovL3B1cmwub3JnL2RjL2VsZW1lbnRzLzEuMS8iIHhtbG5zOnBob3Rvc2hvcD0iaHR0cDovL25zLmFkb2JlLmNvbS9waG90b3Nob3AvMS4wLyIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0RXZ0PSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VFdmVudCMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIDI1LjEgKFdpbmRvd3MpIiB4bXA6Q3JlYXRlRGF0ZT0iMjAyMy0xMS0xM1QwMDoxODowMiswMTowMCIgeG1wOk1vZGlmeURhdGU9IjIwMjMtMTEtMTVUMDI6MDQ6NTkrMDE6MDAiIHhtcDpNZXRhZGF0YURhdGU9IjIwMjMtMTEtMTVUMDI6MDQ6NTkrMDE6MDAiIGRjOmZvcm1hdD0iaW1hZ2UvcG5nIiBwaG90b3Nob3A6Q29sb3JNb2RlPSIzIiB4bXBNTTpJbnN0YW5jZUlEPSJ4bXAuaWlkOmIyYzRhNjA5LWJmYTctYTg0MC1iOGFlLTk3MzE2ZjM1ZGIyNyIgeG1wTU06RG9jdW1lbnRJRD0iYWRvYmU6ZG9jaWQ6cGhvdG9zaG9wOjk0ZmNlZGU4LTE1MTctZmQ0MC04ZGU3LWYzOTgxM2E3ODk5ZiIgeG1wTU06T3JpZ2luYWxEb2N1bWVudElEPSJ4bXAuZGlkOjIzMWIxMGIwLWI0ZmItMDI0ZS1iMTJlLTMwNTMwM2NkMDdjOCI+IDx4bXBNTTpIaXN0b3J5PiA8cmRmOlNlcT4gPHJkZjpsaSBzdEV2dDphY3Rpb249ImNyZWF0ZWQiIHN0RXZ0Omluc3RhbmNlSUQ9InhtcC5paWQ6MjMxYjEwYjAtYjRmYi0wMjRlLWIxMmUtMzA1MzAzY2QwN2M4IiBzdEV2dDp3aGVuPSIyMDIzLTExLTEzVDAwOjE4OjAyKzAxOjAwIiBzdEV2dDpzb2Z0d2FyZUFnZW50PSJBZG9iZSBQaG90b3Nob3AgMjUuMSAoV2luZG93cykiLz4gPHJkZjpsaSBzdEV2dDphY3Rpb249InNhdmVkIiBzdEV2dDppbnN0YW5jZUlEPSJ4bXAuaWlkOjQ4OWY1NzlmLTJkNjUtZWQ0Zi04OTg0LTA4NGE2MGE1ZTMzNSIgc3RFdnQ6d2hlbj0iMjAyMy0xMS0xNVQwMjowNDo1OSswMTowMCIgc3RFdnQ6c29mdHdhcmVBZ2VudD0iQWRvYmUgUGhvdG9zaG9wIDI1LjEgKFdpbmRvd3MpIiBzdEV2dDpjaGFuZ2VkPSIvIi8+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJzYXZlZCIgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDpiMmM0YTYwOS1iZmE3LWE4NDAtYjhhZS05NzMxNmYzNWRiMjciIHN0RXZ0OndoZW49IjIwMjMtMTEtMTVUMDI6MDQ6NTkrMDE6MDAiIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkFkb2JlIFBob3Rvc2hvcCAyNS4xIChXaW5kb3dzKSIgc3RFdnQ6Y2hhbmdlZD0iLyIvPiA8L3JkZjpTZXE+IDwveG1wTU06SGlzdG9yeT4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz4OTe6GAAAAx0lEQVR42u3WMQoAIQxFwRzJys77X8vSLiRgITif7bYbgrwYc/mKXyBoY4VVBgsWLFiwYFmOlTv+9jfDOjHmr8u6eVkGCxYsWLBgmc5S8ApewXvgYRksWLBgKXidpeBdloL3wMOCBctgwVLwCl7BuyyDBQsWLFiwTGcpeAWv4D3wsAwWLFiwFLzOUvAuS8F74GHBgmWwYCl4Ba/gXZbBggULFixYprMUvIJX8B54WAYLFixYCl5nKXiXpeA98LBgwTJYsGC9tg1o8f4TTtqzNQAAAABJRU5ErkJggg==",
+ "CLEAR_BACKGROUND_COLOR": "#040506",
+ "NODE_TITLE_COLOR": "#999",
+ "NODE_SELECTED_TITLE_COLOR": "#e5eaf0",
+ "NODE_TEXT_SIZE": 14,
+ "NODE_TEXT_COLOR": "#bcc2c8",
+ "NODE_SUBTEXT_SIZE": 12,
+ "NODE_DEFAULT_COLOR": "#161b22",
+ "NODE_DEFAULT_BGCOLOR": "#13171d",
+ "NODE_DEFAULT_BOXCOLOR": "#30363d",
+ "NODE_DEFAULT_SHAPE": "box",
+ "NODE_BOX_OUTLINE_COLOR": "#e5eaf0",
+ "DEFAULT_SHADOW_COLOR": "rgba(0,0,0,0.5)",
+ "DEFAULT_GROUP_FONT": 24,
+ "WIDGET_BGCOLOR": "#161b22",
+ "WIDGET_OUTLINE_COLOR": "#30363d",
+ "WIDGET_TEXT_COLOR": "#bcc2c8",
+ "WIDGET_SECONDARY_TEXT_COLOR": "#999",
+ "LINK_COLOR": "#9A9",
+ "EVENT_LINK_COLOR": "#A86",
+ "CONNECTING_LINK_COLOR": "#AFA"
+ },
+ "comfy_base": {
+ "fg-color": "#e5eaf0",
+ "bg-color": "#161b22",
+ "comfy-menu-bg": "#13171d",
+ "comfy-input-bg": "#161b22",
+ "input-text": "#bcc2c8",
+ "descrip-text": "#999",
+ "drag-text": "#ccc",
+ "error-text": "#ff4444",
+ "border-color": "#30363d",
+ "tr-even-bg-color": "#161b22",
+ "tr-odd-bg-color": "#13171d"
+ }
+ },
}
};
diff --git a/web/extensions/core/groupNode.js b/web/extensions/core/groupNode.js
new file mode 100644
index 000000000..4cf1f7621
--- /dev/null
+++ b/web/extensions/core/groupNode.js
@@ -0,0 +1,1172 @@
+import { app } from "../../scripts/app.js";
+import { api } from "../../scripts/api.js";
+import { mergeIfValid } from "./widgetInputs.js";
+
+const GROUP = Symbol();
+
+const Workflow = {
+ InUse: {
+ Free: 0,
+ Registered: 1,
+ InWorkflow: 2,
+ },
+ isInUseGroupNode(name) {
+ const id = `workflow/${name}`;
+ // Check if lready registered/in use in this workflow
+ if (app.graph.extra?.groupNodes?.[name]) {
+ if (app.graph._nodes.find((n) => n.type === id)) {
+ return Workflow.InUse.InWorkflow;
+ } else {
+ return Workflow.InUse.Registered;
+ }
+ }
+ return Workflow.InUse.Free;
+ },
+ storeGroupNode(name, data) {
+ let extra = app.graph.extra;
+ if (!extra) app.graph.extra = extra = {};
+ let groupNodes = extra.groupNodes;
+ if (!groupNodes) extra.groupNodes = groupNodes = {};
+ groupNodes[name] = data;
+ },
+};
+
+class GroupNodeBuilder {
+ constructor(nodes) {
+ this.nodes = nodes;
+ }
+
+ build() {
+ const name = this.getName();
+ if (!name) return;
+
+ // Sort the nodes so they are in execution order
+ // this allows for widgets to be in the correct order when reconstructing
+ this.sortNodes();
+
+ this.nodeData = this.getNodeData();
+ Workflow.storeGroupNode(name, this.nodeData);
+
+ return { name, nodeData: this.nodeData };
+ }
+
+ getName() {
+ const name = prompt("Enter group name");
+ if (!name) return;
+ const used = Workflow.isInUseGroupNode(name);
+ switch (used) {
+ case Workflow.InUse.InWorkflow:
+ alert(
+ "An in use group node with this name already exists embedded in this workflow, please remove any instances or use a new name."
+ );
+ return;
+ case Workflow.InUse.Registered:
+ if (
+ !confirm(
+ "An group node with this name already exists embedded in this workflow, are you sure you want to overwrite it?"
+ )
+ ) {
+ return;
+ }
+ break;
+ }
+ return name;
+ }
+
+ sortNodes() {
+ // Gets the builders nodes in graph execution order
+ const nodesInOrder = app.graph.computeExecutionOrder(false);
+ this.nodes = this.nodes
+ .map((node) => ({ index: nodesInOrder.indexOf(node), node }))
+ .sort((a, b) => a.index - b.index || a.node.id - b.node.id)
+ .map(({ node }) => node);
+ }
+
+ getNodeData() {
+ const storeLinkTypes = (config) => {
+ // Store link types for dynamically typed nodes e.g. reroutes
+ for (const link of config.links) {
+ const origin = app.graph.getNodeById(link[4]);
+ const type = origin.outputs[link[1]].type;
+ link.push(type);
+ }
+ };
+
+ const storeExternalLinks = (config) => {
+ // Store any external links to the group in the config so when rebuilding we add extra slots
+ config.external = [];
+ for (let i = 0; i < this.nodes.length; i++) {
+ const node = this.nodes[i];
+ if (!node.outputs?.length) continue;
+ for (let slot = 0; slot < node.outputs.length; slot++) {
+ let hasExternal = false;
+ const output = node.outputs[slot];
+ let type = output.type;
+ if (!output.links?.length) continue;
+ for (const l of output.links) {
+ const link = app.graph.links[l];
+ if (!link) continue;
+ if (type === "*") type = link.type;
+
+ if (!app.canvas.selected_nodes[link.target_id]) {
+ hasExternal = true;
+ break;
+ }
+ }
+ if (hasExternal) {
+ config.external.push([i, slot, type]);
+ }
+ }
+ }
+ };
+
+ // Use the built in copyToClipboard function to generate the node data we need
+ const backup = localStorage.getItem("litegrapheditor_clipboard");
+ try {
+ app.canvas.copyToClipboard(this.nodes);
+ const config = JSON.parse(localStorage.getItem("litegrapheditor_clipboard"));
+
+ storeLinkTypes(config);
+ storeExternalLinks(config);
+
+ return config;
+ } finally {
+ localStorage.setItem("litegrapheditor_clipboard", backup);
+ }
+ }
+}
+
+export class GroupNodeConfig {
+ constructor(name, nodeData) {
+ this.name = name;
+ this.nodeData = nodeData;
+ this.getLinks();
+
+ this.inputCount = 0;
+ this.oldToNewOutputMap = {};
+ this.newToOldOutputMap = {};
+ this.oldToNewInputMap = {};
+ this.oldToNewWidgetMap = {};
+ this.newToOldWidgetMap = {};
+ this.primitiveDefs = {};
+ this.widgetToPrimitive = {};
+ this.primitiveToWidget = {};
+ }
+
+ async registerType(source = "workflow") {
+ this.nodeDef = {
+ output: [],
+ output_name: [],
+ output_is_list: [],
+ name: source + "/" + this.name,
+ display_name: this.name,
+ category: "group nodes" + ("/" + source),
+ input: { required: {} },
+
+ [GROUP]: this,
+ };
+
+ this.inputs = [];
+ const seenInputs = {};
+ const seenOutputs = {};
+ for (let i = 0; i < this.nodeData.nodes.length; i++) {
+ const node = this.nodeData.nodes[i];
+ node.index = i;
+ this.processNode(node, seenInputs, seenOutputs);
+ }
+
+ for (const p of this.#convertedToProcess) {
+ p();
+ }
+ this.#convertedToProcess = null;
+ await app.registerNodeDef("workflow/" + this.name, this.nodeDef);
+ }
+
+ getLinks() {
+ this.linksFrom = {};
+ this.linksTo = {};
+ this.externalFrom = {};
+
+ // Extract links for easy lookup
+ for (const l of this.nodeData.links) {
+ const [sourceNodeId, sourceNodeSlot, targetNodeId, targetNodeSlot] = l;
+
+ // Skip links outside the copy config
+ if (sourceNodeId == null) continue;
+
+ if (!this.linksFrom[sourceNodeId]) {
+ this.linksFrom[sourceNodeId] = {};
+ }
+ if (!this.linksFrom[sourceNodeId][sourceNodeSlot]) {
+ this.linksFrom[sourceNodeId][sourceNodeSlot] = [];
+ }
+ this.linksFrom[sourceNodeId][sourceNodeSlot].push(l);
+
+ if (!this.linksTo[targetNodeId]) {
+ this.linksTo[targetNodeId] = {};
+ }
+ this.linksTo[targetNodeId][targetNodeSlot] = l;
+ }
+
+ if (this.nodeData.external) {
+ for (const ext of this.nodeData.external) {
+ if (!this.externalFrom[ext[0]]) {
+ this.externalFrom[ext[0]] = { [ext[1]]: ext[2] };
+ } else {
+ this.externalFrom[ext[0]][ext[1]] = ext[2];
+ }
+ }
+ }
+ }
+
+ processNode(node, seenInputs, seenOutputs) {
+ const def = this.getNodeDef(node);
+ if (!def) return;
+
+ const inputs = { ...def.input?.required, ...def.input?.optional };
+
+ this.inputs.push(this.processNodeInputs(node, seenInputs, inputs));
+ if (def.output?.length) this.processNodeOutputs(node, seenOutputs, def);
+ }
+
+ getNodeDef(node) {
+ const def = globalDefs[node.type];
+ if (def) return def;
+
+ const linksFrom = this.linksFrom[node.index];
+ if (node.type === "PrimitiveNode") {
+ // Skip as its not linked
+ if (!linksFrom) return;
+
+ let type = linksFrom["0"][0][5];
+ if (type === "COMBO") {
+ // Use the array items
+ const source = node.outputs[0].widget.name;
+ const fromTypeName = this.nodeData.nodes[linksFrom["0"][0][2]].type;
+ const fromType = globalDefs[fromTypeName];
+ const input = fromType.input.required[source] ?? fromType.input.optional[source];
+ type = input[0];
+ }
+
+ const def = (this.primitiveDefs[node.index] = {
+ input: {
+ required: {
+ value: [type, {}],
+ },
+ },
+ output: [type],
+ output_name: [],
+ output_is_list: [],
+ });
+ return def;
+ } else if (node.type === "Reroute") {
+ const linksTo = this.linksTo[node.index];
+ if (linksTo && linksFrom && !this.externalFrom[node.index]?.[0]) {
+ // Being used internally
+ return null;
+ }
+
+ let config = {};
+ let rerouteType = "*";
+ if (linksFrom) {
+ for (const [, , id, slot] of linksFrom["0"]) {
+ const node = this.nodeData.nodes[id];
+ const input = node.inputs[slot];
+ if (rerouteType === "*") {
+ rerouteType = input.type;
+ }
+ if (input.widget) {
+ const targetDef = globalDefs[node.type];
+ const targetWidget =
+ targetDef.input.required[input.widget.name] ?? targetDef.input.optional[input.widget.name];
+
+ const widget = [targetWidget[0], config];
+ const res = mergeIfValid(
+ {
+ widget,
+ },
+ targetWidget,
+ false,
+ null,
+ widget
+ );
+ config = res?.customConfig ?? config;
+ }
+ }
+ } else if (linksTo) {
+ const [id, slot] = linksTo["0"];
+ rerouteType = this.nodeData.nodes[id].outputs[slot].type;
+ } else {
+ // Reroute used as a pipe
+ for (const l of this.nodeData.links) {
+ if (l[2] === node.index) {
+ rerouteType = l[5];
+ break;
+ }
+ }
+ if (rerouteType === "*") {
+ // Check for an external link
+ const t = this.externalFrom[node.index]?.[0];
+ if (t) {
+ rerouteType = t;
+ }
+ }
+ }
+
+ config.forceInput = true;
+ return {
+ input: {
+ required: {
+ [rerouteType]: [rerouteType, config],
+ },
+ },
+ output: [rerouteType],
+ output_name: [],
+ output_is_list: [],
+ };
+ }
+
+ console.warn("Skipping virtual node " + node.type + " when building group node " + this.name);
+ }
+
+ getInputConfig(node, inputName, seenInputs, config, extra) {
+ let name = node.inputs?.find((inp) => inp.name === inputName)?.label ?? inputName;
+ let key = name;
+ let prefix = "";
+ // Special handling for primitive to include the title if it is set rather than just "value"
+ if ((node.type === "PrimitiveNode" && node.title) || name in seenInputs) {
+ prefix = `${node.title ?? node.type} `;
+ key = name = `${prefix}${inputName}`;
+ if (name in seenInputs) {
+ name = `${prefix}${seenInputs[name]} ${inputName}`;
+ }
+ }
+ seenInputs[key] = (seenInputs[key] ?? 1) + 1;
+
+ if (inputName === "seed" || inputName === "noise_seed") {
+ if (!extra) extra = {};
+ extra.control_after_generate = `${prefix}control_after_generate`;
+ }
+ if (config[0] === "IMAGEUPLOAD") {
+ if (!extra) extra = {};
+ extra.widget = `${prefix}${config[1]?.widget ?? "image"}`;
+ }
+
+ if (extra) {
+ config = [config[0], { ...config[1], ...extra }];
+ }
+
+ return { name, config };
+ }
+
+ processWidgetInputs(inputs, node, inputNames, seenInputs) {
+ const slots = [];
+ const converted = new Map();
+ const widgetMap = (this.oldToNewWidgetMap[node.index] = {});
+ for (const inputName of inputNames) {
+ let widgetType = app.getWidgetType(inputs[inputName], inputName);
+ if (widgetType) {
+ const convertedIndex = node.inputs?.findIndex(
+ (inp) => inp.name === inputName && inp.widget?.name === inputName
+ );
+ if (convertedIndex > -1) {
+ // This widget has been converted to a widget
+ // We need to store this in the correct position so link ids line up
+ converted.set(convertedIndex, inputName);
+ widgetMap[inputName] = null;
+ } else {
+ // Normal widget
+ const { name, config } = this.getInputConfig(node, inputName, seenInputs, inputs[inputName]);
+ this.nodeDef.input.required[name] = config;
+ widgetMap[inputName] = name;
+ this.newToOldWidgetMap[name] = { node, inputName };
+ }
+ } else {
+ // Normal input
+ slots.push(inputName);
+ }
+ }
+ return { converted, slots };
+ }
+
+ checkPrimitiveConnection(link, inputName, inputs) {
+ const sourceNode = this.nodeData.nodes[link[0]];
+ if (sourceNode.type === "PrimitiveNode") {
+ // Merge link configurations
+ const [sourceNodeId, _, targetNodeId, __] = link;
+ const primitiveDef = this.primitiveDefs[sourceNodeId];
+ const targetWidget = inputs[inputName];
+ const primitiveConfig = primitiveDef.input.required.value;
+ const output = { widget: primitiveConfig };
+ const config = mergeIfValid(output, targetWidget, false, null, primitiveConfig);
+ primitiveConfig[1] = config?.customConfig ?? inputs[inputName][1] ? { ...inputs[inputName][1] } : {};
+
+ let name = this.oldToNewWidgetMap[sourceNodeId]["value"];
+ name = name.substr(0, name.length - 6);
+ primitiveConfig[1].control_after_generate = true;
+ primitiveConfig[1].control_prefix = name;
+
+ let toPrimitive = this.widgetToPrimitive[targetNodeId];
+ if (!toPrimitive) {
+ toPrimitive = this.widgetToPrimitive[targetNodeId] = {};
+ }
+ if (toPrimitive[inputName]) {
+ toPrimitive[inputName].push(sourceNodeId);
+ }
+ toPrimitive[inputName] = sourceNodeId;
+
+ let toWidget = this.primitiveToWidget[sourceNodeId];
+ if (!toWidget) {
+ toWidget = this.primitiveToWidget[sourceNodeId] = [];
+ }
+ toWidget.push({ nodeId: targetNodeId, inputName });
+ }
+ }
+
+ processInputSlots(inputs, node, slots, linksTo, inputMap, seenInputs) {
+ for (let i = 0; i < slots.length; i++) {
+ const inputName = slots[i];
+ if (linksTo[i]) {
+ this.checkPrimitiveConnection(linksTo[i], inputName, inputs);
+ // This input is linked so we can skip it
+ continue;
+ }
+
+ const { name, config } = this.getInputConfig(node, inputName, seenInputs, inputs[inputName]);
+ this.nodeDef.input.required[name] = config;
+ inputMap[i] = this.inputCount++;
+ }
+ }
+
+ processConvertedWidgets(inputs, node, slots, converted, linksTo, inputMap, seenInputs) {
+ // Add converted widgets sorted into their index order (ordered as they were converted) so link ids match up
+ const convertedSlots = [...converted.keys()].sort().map((k) => converted.get(k));
+ for (let i = 0; i < convertedSlots.length; i++) {
+ const inputName = convertedSlots[i];
+ if (linksTo[slots.length + i]) {
+ this.checkPrimitiveConnection(linksTo[slots.length + i], inputName, inputs);
+ // This input is linked so we can skip it
+ continue;
+ }
+
+ const { name, config } = this.getInputConfig(node, inputName, seenInputs, inputs[inputName], {
+ defaultInput: true,
+ });
+ this.nodeDef.input.required[name] = config;
+ this.newToOldWidgetMap[name] = { node, inputName };
+
+ if (!this.oldToNewWidgetMap[node.index]) {
+ this.oldToNewWidgetMap[node.index] = {};
+ }
+ this.oldToNewWidgetMap[node.index][inputName] = name;
+
+ inputMap[slots.length + i] = this.inputCount++;
+ }
+ }
+
+ #convertedToProcess = [];
+ processNodeInputs(node, seenInputs, inputs) {
+ const inputMapping = [];
+
+ const inputNames = Object.keys(inputs);
+ if (!inputNames.length) return;
+
+ const { converted, slots } = this.processWidgetInputs(inputs, node, inputNames, seenInputs);
+ const linksTo = this.linksTo[node.index] ?? {};
+ const inputMap = (this.oldToNewInputMap[node.index] = {});
+ this.processInputSlots(inputs, node, slots, linksTo, inputMap, seenInputs);
+
+ // Converted inputs have to be processed after all other nodes as they'll be at the end of the list
+ this.#convertedToProcess.push(() =>
+ this.processConvertedWidgets(inputs, node, slots, converted, linksTo, inputMap, seenInputs)
+ );
+
+ return inputMapping;
+ }
+
+ processNodeOutputs(node, seenOutputs, def) {
+ const oldToNew = (this.oldToNewOutputMap[node.index] = {});
+
+ // Add outputs
+ for (let outputId = 0; outputId < def.output.length; outputId++) {
+ const linksFrom = this.linksFrom[node.index];
+ if (linksFrom?.[outputId] && !this.externalFrom[node.index]?.[outputId]) {
+ // This output is linked internally so we can skip it
+ continue;
+ }
+
+ oldToNew[outputId] = this.nodeDef.output.length;
+ this.newToOldOutputMap[this.nodeDef.output.length] = { node, slot: outputId };
+ this.nodeDef.output.push(def.output[outputId]);
+ this.nodeDef.output_is_list.push(def.output_is_list[outputId]);
+
+ let label = def.output_name?.[outputId] ?? def.output[outputId];
+ const output = node.outputs.find((o) => o.name === label);
+ if (output?.label) {
+ label = output.label;
+ }
+ let name = label;
+ if (name in seenOutputs) {
+ const prefix = `${node.title ?? node.type} `;
+ name = `${prefix}${label}`;
+ if (name in seenOutputs) {
+ name = `${prefix}${node.index} ${label}`;
+ }
+ }
+ seenOutputs[name] = 1;
+
+ this.nodeDef.output_name.push(name);
+ }
+ }
+
+ static async registerFromWorkflow(groupNodes, missingNodeTypes) {
+ const clean = app.clean;
+ app.clean = function () {
+ for (const g in groupNodes) {
+ try {
+ LiteGraph.unregisterNodeType("workflow/" + g);
+ } catch (error) {}
+ }
+ app.clean = clean;
+ };
+
+ for (const g in groupNodes) {
+ const groupData = groupNodes[g];
+
+ let hasMissing = false;
+ for (const n of groupData.nodes) {
+ // Find missing node types
+ if (!(n.type in LiteGraph.registered_node_types)) {
+ missingNodeTypes.push({
+ type: n.type,
+ hint: ` (In group node 'workflow/${g}')`,
+ });
+
+ missingNodeTypes.push({
+ type: "workflow/" + g,
+ action: {
+ text: "Remove from workflow",
+ callback: (e) => {
+ delete groupNodes[g];
+ e.target.textContent = "Removed";
+ e.target.style.pointerEvents = "none";
+ e.target.style.opacity = 0.7;
+ },
+ },
+ });
+
+ hasMissing = true;
+ }
+ }
+
+ if (hasMissing) continue;
+
+ const config = new GroupNodeConfig(g, groupData);
+ await config.registerType();
+ }
+ }
+}
+
+export class GroupNodeHandler {
+ node;
+ groupData;
+
+ constructor(node) {
+ this.node = node;
+ this.groupData = node.constructor?.nodeData?.[GROUP];
+
+ this.node.setInnerNodes = (innerNodes) => {
+ this.innerNodes = innerNodes;
+
+ for (let innerNodeIndex = 0; innerNodeIndex < this.innerNodes.length; innerNodeIndex++) {
+ const innerNode = this.innerNodes[innerNodeIndex];
+
+ for (const w of innerNode.widgets ?? []) {
+ if (w.type === "converted-widget") {
+ w.serializeValue = w.origSerializeValue;
+ }
+ }
+
+ innerNode.index = innerNodeIndex;
+ innerNode.getInputNode = (slot) => {
+ // Check if this input is internal or external
+ const externalSlot = this.groupData.oldToNewInputMap[innerNode.index]?.[slot];
+ if (externalSlot != null) {
+ return this.node.getInputNode(externalSlot);
+ }
+
+ // Internal link
+ const innerLink = this.groupData.linksTo[innerNode.index]?.[slot];
+ if (!innerLink) return null;
+
+ const inputNode = innerNodes[innerLink[0]];
+ // Primitives will already apply their values
+ if (inputNode.type === "PrimitiveNode") return null;
+
+ return inputNode;
+ };
+
+ innerNode.getInputLink = (slot) => {
+ const externalSlot = this.groupData.oldToNewInputMap[innerNode.index]?.[slot];
+ if (externalSlot != null) {
+ // The inner node is connected via the group node inputs
+ const linkId = this.node.inputs[externalSlot].link;
+ let link = app.graph.links[linkId];
+
+ // Use the outer link, but update the target to the inner node
+ link = {
+ ...link,
+ target_id: innerNode.id,
+ target_slot: +slot,
+ };
+ return link;
+ }
+
+ let link = this.groupData.linksTo[innerNode.index]?.[slot];
+ if (!link) return null;
+ // Use the inner link, but update the origin node to be inner node id
+ link = {
+ origin_id: innerNodes[link[0]].id,
+ origin_slot: link[1],
+ target_id: innerNode.id,
+ target_slot: +slot,
+ };
+ return link;
+ };
+ }
+ };
+
+ this.node.updateLink = (link) => {
+ // Replace the group node reference with the internal node
+ link = { ...link };
+ const output = this.groupData.newToOldOutputMap[link.origin_slot];
+ let innerNode = this.innerNodes[output.node.index];
+ let l;
+ while (innerNode?.type === "Reroute") {
+ l = innerNode.getInputLink(0);
+ innerNode = innerNode.getInputNode(0);
+ }
+
+ if (!innerNode) {
+ return null;
+ }
+
+ if (l && GroupNodeHandler.isGroupNode(innerNode)) {
+ return innerNode.updateLink(l);
+ }
+
+ link.origin_id = innerNode.id;
+ link.origin_slot = l?.origin_slot ?? output.slot;
+ return link;
+ };
+
+ this.node.getInnerNodes = () => {
+ if (!this.innerNodes) {
+ this.node.setInnerNodes(
+ this.groupData.nodeData.nodes.map((n, i) => {
+ const innerNode = LiteGraph.createNode(n.type);
+ innerNode.configure(n);
+ innerNode.id = `${this.node.id}:${i}`;
+ return innerNode;
+ })
+ );
+ }
+
+ this.updateInnerWidgets();
+
+ return this.innerNodes;
+ };
+
+ this.node.convertToNodes = () => {
+ const addInnerNodes = () => {
+ const backup = localStorage.getItem("litegrapheditor_clipboard");
+ // Clone the node data so we dont mutate it for other nodes
+ const c = { ...this.groupData.nodeData };
+ c.nodes = [...c.nodes];
+ const innerNodes = this.node.getInnerNodes();
+ let ids = [];
+ for (let i = 0; i < c.nodes.length; i++) {
+ let id = innerNodes?.[i]?.id;
+ // Use existing IDs if they are set on the inner nodes
+ if (id == null || isNaN(id)) {
+ id = undefined;
+ } else {
+ ids.push(id);
+ }
+ c.nodes[i] = { ...c.nodes[i], id };
+ }
+ localStorage.setItem("litegrapheditor_clipboard", JSON.stringify(c));
+ app.canvas.pasteFromClipboard();
+ localStorage.setItem("litegrapheditor_clipboard", backup);
+
+ const [x, y] = this.node.pos;
+ let top;
+ let left;
+ // Configure nodes with current widget data
+ const selectedIds = ids.length ? ids : Object.keys(app.canvas.selected_nodes);
+ const newNodes = [];
+ for (let i = 0; i < selectedIds.length; i++) {
+ const id = selectedIds[i];
+ const newNode = app.graph.getNodeById(id);
+ const innerNode = innerNodes[i];
+ newNodes.push(newNode);
+
+ if (left == null || newNode.pos[0] < left) {
+ left = newNode.pos[0];
+ }
+ if (top == null || newNode.pos[1] < top) {
+ top = newNode.pos[1];
+ }
+
+ if (!newNode.widgets) continue;
+
+ const map = this.groupData.oldToNewWidgetMap[innerNode.index];
+ if (map) {
+ const widgets = Object.keys(map);
+
+ for (const oldName of widgets) {
+ const newName = map[oldName];
+ if (!newName) continue;
+
+ const widgetIndex = this.node.widgets.findIndex((w) => w.name === newName);
+ if (widgetIndex === -1) continue;
+
+ // Populate the main and any linked widgets
+ if (innerNode.type === "PrimitiveNode") {
+ for (let i = 0; i < newNode.widgets.length; i++) {
+ newNode.widgets[i].value = this.node.widgets[widgetIndex + i].value;
+ }
+ } else {
+ const outerWidget = this.node.widgets[widgetIndex];
+ const newWidget = newNode.widgets.find((w) => w.name === oldName);
+ if (!newWidget) continue;
+
+ newWidget.value = outerWidget.value;
+ for (let w = 0; w < outerWidget.linkedWidgets?.length; w++) {
+ newWidget.linkedWidgets[w].value = outerWidget.linkedWidgets[w].value;
+ }
+ }
+ }
+ }
+ }
+
+ // Shift each node
+ for (const newNode of newNodes) {
+ newNode.pos = [newNode.pos[0] - (left - x), newNode.pos[1] - (top - y)];
+ }
+
+ return { newNodes, selectedIds };
+ };
+
+ const reconnectInputs = (selectedIds) => {
+ for (const innerNodeIndex in this.groupData.oldToNewInputMap) {
+ const id = selectedIds[innerNodeIndex];
+ const newNode = app.graph.getNodeById(id);
+ const map = this.groupData.oldToNewInputMap[innerNodeIndex];
+ for (const innerInputId in map) {
+ const groupSlotId = map[innerInputId];
+ if (groupSlotId == null) continue;
+ const slot = node.inputs[groupSlotId];
+ if (slot.link == null) continue;
+ const link = app.graph.links[slot.link];
+ // connect this node output to the input of another node
+ const originNode = app.graph.getNodeById(link.origin_id);
+ originNode.connect(link.origin_slot, newNode, +innerInputId);
+ }
+ }
+ };
+
+ const reconnectOutputs = (selectedIds) => {
+ for (let groupOutputId = 0; groupOutputId < node.outputs?.length; groupOutputId++) {
+ const output = node.outputs[groupOutputId];
+ if (!output.links) continue;
+ const links = [...output.links];
+ for (const l of links) {
+ const slot = this.groupData.newToOldOutputMap[groupOutputId];
+ const link = app.graph.links[l];
+ const targetNode = app.graph.getNodeById(link.target_id);
+ const newNode = app.graph.getNodeById(selectedIds[slot.node.index]);
+ newNode.connect(slot.slot, targetNode, link.target_slot);
+ }
+ }
+ };
+
+ const { newNodes, selectedIds } = addInnerNodes();
+ reconnectInputs(selectedIds);
+ reconnectOutputs(selectedIds);
+ app.graph.remove(this.node);
+
+ return newNodes;
+ };
+
+ const getExtraMenuOptions = this.node.getExtraMenuOptions;
+ this.node.getExtraMenuOptions = function (_, options) {
+ getExtraMenuOptions?.apply(this, arguments);
+
+ let optionIndex = options.findIndex((o) => o.content === "Outputs");
+ if (optionIndex === -1) optionIndex = options.length;
+ else optionIndex++;
+ options.splice(optionIndex, 0, null, {
+ content: "Convert to nodes",
+ callback: () => {
+ return this.convertToNodes();
+ },
+ });
+ };
+
+ // Draw custom collapse icon to identity this as a group
+ const onDrawTitleBox = this.node.onDrawTitleBox;
+ this.node.onDrawTitleBox = function (ctx, height, size, scale) {
+ onDrawTitleBox?.apply(this, arguments);
+
+ const fill = ctx.fillStyle;
+ ctx.beginPath();
+ ctx.rect(11, -height + 11, 2, 2);
+ ctx.rect(14, -height + 11, 2, 2);
+ ctx.rect(17, -height + 11, 2, 2);
+ ctx.rect(11, -height + 14, 2, 2);
+ ctx.rect(14, -height + 14, 2, 2);
+ ctx.rect(17, -height + 14, 2, 2);
+ ctx.rect(11, -height + 17, 2, 2);
+ ctx.rect(14, -height + 17, 2, 2);
+ ctx.rect(17, -height + 17, 2, 2);
+
+ ctx.fillStyle = this.boxcolor || LiteGraph.NODE_DEFAULT_BOXCOLOR;
+ ctx.fill();
+ ctx.fillStyle = fill;
+ };
+
+ // Draw progress label
+ const onDrawForeground = node.onDrawForeground;
+ const groupData = this.groupData.nodeData;
+ node.onDrawForeground = function (ctx) {
+ const r = onDrawForeground?.apply?.(this, arguments);
+ if (+app.runningNodeId === this.id && this.runningInternalNodeId !== null) {
+ const n = groupData.nodes[this.runningInternalNodeId];
+ const message = `Running ${n.title || n.type} (${this.runningInternalNodeId}/${groupData.nodes.length})`;
+ ctx.save();
+ ctx.font = "12px sans-serif";
+ const sz = ctx.measureText(message);
+ ctx.fillStyle = node.boxcolor || LiteGraph.NODE_DEFAULT_BOXCOLOR;
+ ctx.beginPath();
+ ctx.roundRect(0, -LiteGraph.NODE_TITLE_HEIGHT - 20, sz.width + 12, 20, 5);
+ ctx.fill();
+
+ ctx.fillStyle = "#fff";
+ ctx.fillText(message, 6, -LiteGraph.NODE_TITLE_HEIGHT - 6);
+ ctx.restore();
+ }
+ };
+
+ // Flag this node as needing to be reset
+ const onExecutionStart = this.node.onExecutionStart;
+ this.node.onExecutionStart = function () {
+ this.resetExecution = true;
+ return onExecutionStart?.apply(this, arguments);
+ };
+
+ function handleEvent(type, getId, getEvent) {
+ const handler = ({ detail }) => {
+ const id = getId(detail);
+ if (!id) return;
+ const node = app.graph.getNodeById(id);
+ if (node) return;
+
+ const innerNodeIndex = this.innerNodes?.findIndex((n) => n.id == id);
+ if (innerNodeIndex > -1) {
+ this.node.runningInternalNodeId = innerNodeIndex;
+ api.dispatchEvent(new CustomEvent(type, { detail: getEvent(detail, this.node.id + "", this.node) }));
+ }
+ };
+ api.addEventListener(type, handler);
+ return handler;
+ }
+
+ const executing = handleEvent.call(
+ this,
+ "executing",
+ (d) => d,
+ (d, id, node) => id
+ );
+
+ const executed = handleEvent.call(
+ this,
+ "executed",
+ (d) => d?.node,
+ (d, id, node) => ({ ...d, node: id, merge: !node.resetExecution })
+ );
+
+ const onRemoved = node.onRemoved;
+ this.node.onRemoved = function () {
+ onRemoved?.apply(this, arguments);
+ api.removeEventListener("executing", executing);
+ api.removeEventListener("executed", executed);
+ };
+ }
+
+ updateInnerWidgets() {
+ for (const newWidgetName in this.groupData.newToOldWidgetMap) {
+ const newWidget = this.node.widgets.find((w) => w.name === newWidgetName);
+ if (!newWidget) continue;
+
+ const newValue = newWidget.value;
+ const old = this.groupData.newToOldWidgetMap[newWidgetName];
+ let innerNode = this.innerNodes[old.node.index];
+
+ if (innerNode.type === "PrimitiveNode") {
+ innerNode.primitiveValue = newValue;
+ const primitiveLinked = this.groupData.primitiveToWidget[old.node.index];
+ for (const linked of primitiveLinked ?? []) {
+ const node = this.innerNodes[linked.nodeId];
+ const widget = node.widgets.find((w) => w.name === linked.inputName);
+
+ if (widget) {
+ widget.value = newValue;
+ }
+ }
+ continue;
+ } else if (innerNode.type === "Reroute") {
+ const rerouteLinks = this.groupData.linksFrom[old.node.index];
+ for (const [_, , targetNodeId, targetSlot] of rerouteLinks["0"]) {
+ const node = this.innerNodes[targetNodeId];
+ const input = node.inputs[targetSlot];
+ if (input.widget) {
+ const widget = node.widgets?.find((w) => w.name === input.widget.name);
+ if (widget) {
+ widget.value = newValue;
+ }
+ }
+ }
+ }
+
+ const widget = innerNode.widgets?.find((w) => w.name === old.inputName);
+ if (widget) {
+ widget.value = newValue;
+ }
+ }
+ }
+
+ populatePrimitive(node, nodeId, oldName, i, linkedShift) {
+ // Converted widget, populate primitive if linked
+ const primitiveId = this.groupData.widgetToPrimitive[nodeId]?.[oldName];
+ if (primitiveId == null) return;
+ const targetWidgetName = this.groupData.oldToNewWidgetMap[primitiveId]["value"];
+ const targetWidgetIndex = this.node.widgets.findIndex((w) => w.name === targetWidgetName);
+ if (targetWidgetIndex > -1) {
+ const primitiveNode = this.innerNodes[primitiveId];
+ let len = primitiveNode.widgets.length;
+ if (len - 1 !== this.node.widgets[targetWidgetIndex].linkedWidgets?.length) {
+ // Fallback handling for if some reason the primitive has a different number of widgets
+ // we dont want to overwrite random widgets, better to leave blank
+ len = 1;
+ }
+ for (let i = 0; i < len; i++) {
+ this.node.widgets[targetWidgetIndex + i].value = primitiveNode.widgets[i].value;
+ }
+ }
+ return true;
+ }
+
+ populateReroute(node, nodeId, map) {
+ if (node.type !== "Reroute") return;
+
+ const link = this.groupData.linksFrom[nodeId]?.[0]?.[0];
+ if (!link) return;
+ const [, , targetNodeId, targetNodeSlot] = link;
+ const targetNode = this.groupData.nodeData.nodes[targetNodeId];
+ const inputs = targetNode.inputs;
+ const targetWidget = inputs?.[targetNodeSlot].widget;
+ if (!targetWidget) return;
+
+ const offset = inputs.length - (targetNode.widgets_values?.length ?? 0);
+ const v = targetNode.widgets_values?.[targetNodeSlot - offset];
+ if (v == null) return;
+
+ const widgetName = Object.values(map)[0];
+ const widget = this.node.widgets.find(w => w.name === widgetName);
+ if(widget) {
+ widget.value = v;
+ }
+ }
+
+
+ populateWidgets() {
+ if (!this.node.widgets) return;
+
+ for (let nodeId = 0; nodeId < this.groupData.nodeData.nodes.length; nodeId++) {
+ const node = this.groupData.nodeData.nodes[nodeId];
+ const map = this.groupData.oldToNewWidgetMap[nodeId] ?? {};
+ const widgets = Object.keys(map);
+
+ if (!node.widgets_values?.length) {
+ // special handling for populating values into reroutes
+ // this allows primitives connect to them to pick up the correct value
+ this.populateReroute(node, nodeId, map);
+ continue;
+ }
+
+ let linkedShift = 0;
+ for (let i = 0; i < widgets.length; i++) {
+ const oldName = widgets[i];
+ const newName = map[oldName];
+ const widgetIndex = this.node.widgets.findIndex((w) => w.name === newName);
+ const mainWidget = this.node.widgets[widgetIndex];
+ if (this.populatePrimitive(node, nodeId, oldName, i, linkedShift) || widgetIndex === -1) {
+ // Find the inner widget and shift by the number of linked widgets as they will have been removed too
+ const innerWidget = this.innerNodes[nodeId].widgets?.find((w) => w.name === oldName);
+ linkedShift += innerWidget?.linkedWidgets?.length ?? 0;
+ }
+ if (widgetIndex === -1) {
+ continue;
+ }
+
+ // Populate the main and any linked widget
+ mainWidget.value = node.widgets_values[i + linkedShift];
+ for (let w = 0; w < mainWidget.linkedWidgets?.length; w++) {
+ this.node.widgets[widgetIndex + w + 1].value = node.widgets_values[i + ++linkedShift];
+ }
+ }
+ }
+ }
+
+ replaceNodes(nodes) {
+ let top;
+ let left;
+
+ for (let i = 0; i < nodes.length; i++) {
+ const node = nodes[i];
+ if (left == null || node.pos[0] < left) {
+ left = node.pos[0];
+ }
+ if (top == null || node.pos[1] < top) {
+ top = node.pos[1];
+ }
+
+ this.linkOutputs(node, i);
+ app.graph.remove(node);
+ }
+
+ this.linkInputs();
+ this.node.pos = [left, top];
+ }
+
+ linkOutputs(originalNode, nodeId) {
+ if (!originalNode.outputs) return;
+
+ for (const output of originalNode.outputs) {
+ if (!output.links) continue;
+ // Clone the links as they'll be changed if we reconnect
+ const links = [...output.links];
+ for (const l of links) {
+ const link = app.graph.links[l];
+ if (!link) continue;
+
+ const targetNode = app.graph.getNodeById(link.target_id);
+ const newSlot = this.groupData.oldToNewOutputMap[nodeId]?.[link.origin_slot];
+ if (newSlot != null) {
+ this.node.connect(newSlot, targetNode, link.target_slot);
+ }
+ }
+ }
+ }
+
+ linkInputs() {
+ for (const link of this.groupData.nodeData.links ?? []) {
+ const [, originSlot, targetId, targetSlot, actualOriginId] = link;
+ const originNode = app.graph.getNodeById(actualOriginId);
+ if (!originNode) continue; // this node is in the group
+ originNode.connect(originSlot, this.node.id, this.groupData.oldToNewInputMap[targetId][targetSlot]);
+ }
+ }
+
+ static getGroupData(node) {
+ return node.constructor?.nodeData?.[GROUP];
+ }
+
+ static isGroupNode(node) {
+ return !!node.constructor?.nodeData?.[GROUP];
+ }
+
+ static async fromNodes(nodes) {
+ // Process the nodes into the stored workflow group node data
+ const builder = new GroupNodeBuilder(nodes);
+ const res = builder.build();
+ if (!res) return;
+
+ const { name, nodeData } = res;
+
+ // Convert this data into a LG node definition and register it
+ const config = new GroupNodeConfig(name, nodeData);
+ await config.registerType();
+
+ const groupNode = LiteGraph.createNode(`workflow/${name}`);
+ // Reuse the existing nodes for this instance
+ groupNode.setInnerNodes(builder.nodes);
+ groupNode[GROUP].populateWidgets();
+ app.graph.add(groupNode);
+
+ // Remove all converted nodes and relink them
+ groupNode[GROUP].replaceNodes(builder.nodes);
+ return groupNode;
+ }
+}
+
+function addConvertToGroupOptions() {
+ function addOption(options, index) {
+ const selected = Object.values(app.canvas.selected_nodes ?? {});
+ const disabled = selected.length < 2 || selected.find((n) => GroupNodeHandler.isGroupNode(n));
+ options.splice(index + 1, null, {
+ content: `Convert to Group Node`,
+ disabled,
+ callback: async () => {
+ return await GroupNodeHandler.fromNodes(selected);
+ },
+ });
+ }
+
+ // Add to canvas
+ const getCanvasMenuOptions = LGraphCanvas.prototype.getCanvasMenuOptions;
+ LGraphCanvas.prototype.getCanvasMenuOptions = function () {
+ const options = getCanvasMenuOptions.apply(this, arguments);
+ const index = options.findIndex((o) => o?.content === "Add Group") + 1 || options.length;
+ addOption(options, index);
+ return options;
+ };
+
+ // Add to nodes
+ const getNodeMenuOptions = LGraphCanvas.prototype.getNodeMenuOptions;
+ LGraphCanvas.prototype.getNodeMenuOptions = function (node) {
+ const options = getNodeMenuOptions.apply(this, arguments);
+ if (!GroupNodeHandler.isGroupNode(node)) {
+ const index = options.findIndex((o) => o?.content === "Outputs") + 1 || options.length - 1;
+ addOption(options, index);
+ }
+ return options;
+ };
+}
+
+const id = "Comfy.GroupNode";
+let globalDefs;
+const ext = {
+ name: id,
+ setup() {
+ addConvertToGroupOptions();
+ },
+ async beforeConfigureGraph(graphData, missingNodeTypes) {
+ const nodes = graphData?.extra?.groupNodes;
+ if (nodes) {
+ await GroupNodeConfig.registerFromWorkflow(nodes, missingNodeTypes);
+ }
+ },
+ addCustomNodeDefs(defs) {
+ // Store this so we can mutate it later with group nodes
+ globalDefs = defs;
+ },
+ nodeCreated(node) {
+ if (GroupNodeHandler.isGroupNode(node)) {
+ node[GROUP] = new GroupNodeHandler(node);
+ }
+ },
+};
+
+app.registerExtension(ext);
diff --git a/web/extensions/core/maskeditor.js b/web/extensions/core/maskeditor.js
index f6292b9e3..bb2f16d42 100644
--- a/web/extensions/core/maskeditor.js
+++ b/web/extensions/core/maskeditor.js
@@ -33,6 +33,18 @@ function loadedImageToBlob(image) {
return blob;
}
+function loadImage(imagePath) {
+ return new Promise((resolve, reject) => {
+ const image = new Image();
+
+ image.onload = function() {
+ resolve(image);
+ };
+
+ image.src = imagePath;
+ });
+}
+
async function uploadMask(filepath, formData) {
await api.fetchApi('/upload/mask', {
method: 'POST',
@@ -42,7 +54,7 @@ async function uploadMask(filepath, formData) {
});
ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']] = new Image();
- ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src = api.apiURL("/view?" + new URLSearchParams(filepath).toString() + app.getPreviewFormatParam());
+ ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src = api.apiURL("/view?" + new URLSearchParams(filepath).toString() + app.getPreviewFormatParam() + app.getRandParam());
if(ComfyApp.clipspace.images)
ComfyApp.clipspace.images[ComfyApp.clipspace['selectedIndex']] = filepath;
@@ -50,25 +62,25 @@ async function uploadMask(filepath, formData) {
ClipspaceDialog.invalidatePreview();
}
-function prepareRGB(image, backupCanvas, backupCtx) {
+function prepare_mask(image, maskCanvas, maskCtx) {
// paste mask data into alpha channel
- backupCtx.drawImage(image, 0, 0, backupCanvas.width, backupCanvas.height);
- const backupData = backupCtx.getImageData(0, 0, backupCanvas.width, backupCanvas.height);
+ maskCtx.drawImage(image, 0, 0, maskCanvas.width, maskCanvas.height);
+ const maskData = maskCtx.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
- // refine mask image
- for (let i = 0; i < backupData.data.length; i += 4) {
- if(backupData.data[i+3] == 255)
- backupData.data[i+3] = 0;
+ // invert mask
+ for (let i = 0; i < maskData.data.length; i += 4) {
+ if(maskData.data[i+3] == 255)
+ maskData.data[i+3] = 0;
else
- backupData.data[i+3] = 255;
+ maskData.data[i+3] = 255;
- backupData.data[i] = 0;
- backupData.data[i+1] = 0;
- backupData.data[i+2] = 0;
+ maskData.data[i] = 0;
+ maskData.data[i+1] = 0;
+ maskData.data[i+2] = 0;
}
- backupCtx.globalCompositeOperation = 'source-over';
- backupCtx.putImageData(backupData, 0, 0);
+ maskCtx.globalCompositeOperation = 'source-over';
+ maskCtx.putImageData(maskData, 0, 0);
}
class MaskEditorDialog extends ComfyDialog {
@@ -155,10 +167,6 @@ class MaskEditorDialog extends ComfyDialog {
// If it is specified as relative, using it only as a hidden placeholder for padding is recommended
// to prevent anomalies where it exceeds a certain size and goes outside of the window.
- var placeholder = document.createElement("div");
- placeholder.style.position = "relative";
- placeholder.style.height = "50px";
-
var bottom_panel = document.createElement("div");
bottom_panel.style.position = "absolute";
bottom_panel.style.bottom = "0px";
@@ -180,18 +188,16 @@ class MaskEditorDialog extends ComfyDialog {
this.brush = brush;
this.element.appendChild(imgCanvas);
this.element.appendChild(maskCanvas);
- this.element.appendChild(placeholder); // must below z-index than bottom_panel to avoid covering button
this.element.appendChild(bottom_panel);
document.body.appendChild(brush);
- var brush_size_slider = this.createLeftSlider(self, "Thickness", (event) => {
+ this.brush_size_slider = this.createLeftSlider(self, "Thickness", (event) => {
self.brush_size = event.target.value;
self.updateBrushPreview(self, null, null);
});
var clearButton = this.createLeftButton("Clear",
() => {
self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height);
- self.backupCtx.clearRect(0, 0, self.backupCanvas.width, self.backupCanvas.height);
});
var cancelButton = this.createRightButton("Cancel", () => {
document.removeEventListener("mouseup", MaskEditorDialog.handleMouseUp);
@@ -207,40 +213,42 @@ class MaskEditorDialog extends ComfyDialog {
this.element.appendChild(imgCanvas);
this.element.appendChild(maskCanvas);
- this.element.appendChild(placeholder); // must below z-index than bottom_panel to avoid covering button
this.element.appendChild(bottom_panel);
bottom_panel.appendChild(clearButton);
bottom_panel.appendChild(this.saveButton);
bottom_panel.appendChild(cancelButton);
- bottom_panel.appendChild(brush_size_slider);
+ bottom_panel.appendChild(this.brush_size_slider);
+
+ imgCanvas.style.position = "absolute";
+ maskCanvas.style.position = "absolute";
- imgCanvas.style.position = "relative";
imgCanvas.style.top = "200";
imgCanvas.style.left = "0";
- maskCanvas.style.position = "absolute";
+ maskCanvas.style.top = imgCanvas.style.top;
+ maskCanvas.style.left = imgCanvas.style.left;
}
- show() {
+ async show() {
+ this.zoom_ratio = 1.0;
+ this.pan_x = 0;
+ this.pan_y = 0;
+
if(!this.is_layout_created) {
// layout
const imgCanvas = document.createElement('canvas');
const maskCanvas = document.createElement('canvas');
- const backupCanvas = document.createElement('canvas');
imgCanvas.id = "imageCanvas";
maskCanvas.id = "maskCanvas";
- backupCanvas.id = "backupCanvas";
this.setlayout(imgCanvas, maskCanvas);
// prepare content
this.imgCanvas = imgCanvas;
this.maskCanvas = maskCanvas;
- this.backupCanvas = backupCanvas;
- this.maskCtx = maskCanvas.getContext('2d');
- this.backupCtx = backupCanvas.getContext('2d');
+ this.maskCtx = maskCanvas.getContext('2d', {willReadFrequently: true });
this.setEventHandler(maskCanvas);
@@ -252,6 +260,8 @@ class MaskEditorDialog extends ComfyDialog {
mutations.forEach(function(mutation) {
if (mutation.type === 'attributes' && mutation.attributeName === 'style') {
if(self.last_display_style && self.last_display_style != 'none' && self.element.style.display == 'none') {
+ document.removeEventListener("mouseup", MaskEditorDialog.handleMouseUp);
+ self.brush.style.display = "none";
ComfyApp.onClipspaceEditorClosed();
}
@@ -264,7 +274,8 @@ class MaskEditorDialog extends ComfyDialog {
observer.observe(this.element, config);
}
- this.setImages(this.imgCanvas, this.backupCanvas);
+ // The keydown event needs to be reconfigured when closing the dialog as it gets removed.
+ document.addEventListener('keydown', MaskEditorDialog.handleKeyDown);
if(ComfyApp.clipspace_return_node) {
this.saveButton.innerText = "Save to node";
@@ -275,97 +286,157 @@ class MaskEditorDialog extends ComfyDialog {
this.saveButton.disabled = false;
this.element.style.display = "block";
+ this.element.style.width = "85%";
+ this.element.style.margin = "0 7.5%";
+ this.element.style.height = "100vh";
+ this.element.style.top = "50%";
+ this.element.style.left = "42%";
this.element.style.zIndex = 8888; // NOTE: alert dialog must be high priority.
+
+ await this.setImages(this.imgCanvas);
+
+ this.is_visible = true;
}
isOpened() {
return this.element.style.display == "block";
}
- setImages(imgCanvas, backupCanvas) {
- const imgCtx = imgCanvas.getContext('2d');
- const backupCtx = backupCanvas.getContext('2d');
+ invalidateCanvas(orig_image, mask_image) {
+ this.imgCanvas.width = orig_image.width;
+ this.imgCanvas.height = orig_image.height;
+
+ this.maskCanvas.width = orig_image.width;
+ this.maskCanvas.height = orig_image.height;
+
+ let imgCtx = this.imgCanvas.getContext('2d', {willReadFrequently: true });
+ let maskCtx = this.maskCanvas.getContext('2d', {willReadFrequently: true });
+
+ imgCtx.drawImage(orig_image, 0, 0, orig_image.width, orig_image.height);
+ prepare_mask(mask_image, this.maskCanvas, maskCtx);
+ }
+
+ async setImages(imgCanvas) {
+ let self = this;
+
+ const imgCtx = imgCanvas.getContext('2d', {willReadFrequently: true });
const maskCtx = this.maskCtx;
const maskCanvas = this.maskCanvas;
- backupCtx.clearRect(0,0,this.backupCanvas.width,this.backupCanvas.height);
imgCtx.clearRect(0,0,this.imgCanvas.width,this.imgCanvas.height);
maskCtx.clearRect(0,0,this.maskCanvas.width,this.maskCanvas.height);
// image load
- const orig_image = new Image();
- window.addEventListener("resize", () => {
- // repositioning
- imgCanvas.width = window.innerWidth - 250;
- imgCanvas.height = window.innerHeight - 200;
-
- // redraw image
- let drawWidth = orig_image.width;
- let drawHeight = orig_image.height;
- if (orig_image.width > imgCanvas.width) {
- drawWidth = imgCanvas.width;
- drawHeight = (drawWidth / orig_image.width) * orig_image.height;
- }
-
- if (drawHeight > imgCanvas.height) {
- drawHeight = imgCanvas.height;
- drawWidth = (drawHeight / orig_image.height) * orig_image.width;
- }
-
- imgCtx.drawImage(orig_image, 0, 0, drawWidth, drawHeight);
-
- // update mask
- maskCanvas.width = drawWidth;
- maskCanvas.height = drawHeight;
- maskCanvas.style.top = imgCanvas.offsetTop + "px";
- maskCanvas.style.left = imgCanvas.offsetLeft + "px";
- backupCtx.drawImage(maskCanvas, 0, 0, maskCanvas.width, maskCanvas.height, 0, 0, backupCanvas.width, backupCanvas.height);
- maskCtx.drawImage(backupCanvas, 0, 0, backupCanvas.width, backupCanvas.height, 0, 0, maskCanvas.width, maskCanvas.height);
- });
-
const filepath = ComfyApp.clipspace.images;
- const touched_image = new Image();
-
- touched_image.onload = function() {
- backupCanvas.width = touched_image.width;
- backupCanvas.height = touched_image.height;
-
- prepareRGB(touched_image, backupCanvas, backupCtx);
- };
-
const alpha_url = new URL(ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src)
alpha_url.searchParams.delete('channel');
alpha_url.searchParams.delete('preview');
alpha_url.searchParams.set('channel', 'a');
- touched_image.src = alpha_url;
+ let mask_image = await loadImage(alpha_url);
// original image load
- orig_image.onload = function() {
- window.dispatchEvent(new Event('resize'));
- };
-
const rgb_url = new URL(ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src);
rgb_url.searchParams.delete('channel');
rgb_url.searchParams.set('channel', 'rgb');
- orig_image.src = rgb_url;
- this.image = orig_image;
+ this.image = new Image();
+ this.image.onload = function() {
+ maskCanvas.width = self.image.width;
+ maskCanvas.height = self.image.height;
+
+ self.invalidateCanvas(self.image, mask_image);
+ self.initializeCanvasPanZoom();
+ };
+ this.image.src = rgb_url;
}
- setEventHandler(maskCanvas) {
- maskCanvas.addEventListener("contextmenu", (event) => {
- event.preventDefault();
- });
+ initializeCanvasPanZoom() {
+ // set initialize
+ let drawWidth = this.image.width;
+ let drawHeight = this.image.height;
+ let width = this.element.clientWidth;
+ let height = this.element.clientHeight;
+
+ if (this.image.width > width) {
+ drawWidth = width;
+ drawHeight = (drawWidth / this.image.width) * this.image.height;
+ }
+
+ if (drawHeight > height) {
+ drawHeight = height;
+ drawWidth = (drawHeight / this.image.height) * this.image.width;
+ }
+
+ this.zoom_ratio = drawWidth/this.image.width;
+
+ const canvasX = (width - drawWidth) / 2;
+ const canvasY = (height - drawHeight) / 2;
+ this.pan_x = canvasX;
+ this.pan_y = canvasY;
+
+ this.invalidatePanZoom();
+ }
+
+
+ invalidatePanZoom() {
+ let raw_width = this.image.width * this.zoom_ratio;
+ let raw_height = this.image.height * this.zoom_ratio;
+
+ if(this.pan_x + raw_width < 10) {
+ this.pan_x = 10 - raw_width;
+ }
+
+ if(this.pan_y + raw_height < 10) {
+ this.pan_y = 10 - raw_height;
+ }
+
+ let width = `${raw_width}px`;
+ let height = `${raw_height}px`;
+
+ let left = `${this.pan_x}px`;
+ let top = `${this.pan_y}px`;
+
+ this.maskCanvas.style.width = width;
+ this.maskCanvas.style.height = height;
+ this.maskCanvas.style.left = left;
+ this.maskCanvas.style.top = top;
+
+ this.imgCanvas.style.width = width;
+ this.imgCanvas.style.height = height;
+ this.imgCanvas.style.left = left;
+ this.imgCanvas.style.top = top;
+ }
+
+
+ setEventHandler(maskCanvas) {
const self = this;
- maskCanvas.addEventListener('wheel', (event) => this.handleWheelEvent(self,event));
- maskCanvas.addEventListener('pointerdown', (event) => this.handlePointerDown(self,event));
- document.addEventListener('pointerup', MaskEditorDialog.handlePointerUp);
- maskCanvas.addEventListener('pointermove', (event) => this.draw_move(self,event));
- maskCanvas.addEventListener('touchmove', (event) => this.draw_move(self,event));
- maskCanvas.addEventListener('pointerover', (event) => { this.brush.style.display = "block"; });
- maskCanvas.addEventListener('pointerleave', (event) => { this.brush.style.display = "none"; });
- document.addEventListener('keydown', MaskEditorDialog.handleKeyDown);
+
+ if(!this.handler_registered) {
+ maskCanvas.addEventListener("contextmenu", (event) => {
+ event.preventDefault();
+ });
+
+ this.element.addEventListener('wheel', (event) => this.handleWheelEvent(self,event));
+ this.element.addEventListener('pointermove', (event) => this.pointMoveEvent(self,event));
+ this.element.addEventListener('touchmove', (event) => this.pointMoveEvent(self,event));
+
+ this.element.addEventListener('dragstart', (event) => {
+ if(event.ctrlKey) {
+ event.preventDefault();
+ }
+ });
+
+ maskCanvas.addEventListener('pointerdown', (event) => this.handlePointerDown(self,event));
+ maskCanvas.addEventListener('pointermove', (event) => this.draw_move(self,event));
+ maskCanvas.addEventListener('touchmove', (event) => this.draw_move(self,event));
+ maskCanvas.addEventListener('pointerover', (event) => { this.brush.style.display = "block"; });
+ maskCanvas.addEventListener('pointerleave', (event) => { this.brush.style.display = "none"; });
+
+ document.addEventListener('pointerup', MaskEditorDialog.handlePointerUp);
+
+ this.handler_registered = true;
+ }
}
brush_size = 10;
@@ -378,8 +449,10 @@ class MaskEditorDialog extends ComfyDialog {
const self = MaskEditorDialog.instance;
if (event.key === ']') {
self.brush_size = Math.min(self.brush_size+2, 100);
+ self.brush_slider_input.value = self.brush_size;
} else if (event.key === '[') {
self.brush_size = Math.max(self.brush_size-2, 1);
+ self.brush_slider_input.value = self.brush_size;
} else if(event.key === 'Enter') {
self.save();
}
@@ -389,6 +462,10 @@ class MaskEditorDialog extends ComfyDialog {
static handlePointerUp(event) {
event.preventDefault();
+
+ this.mousedown_x = null;
+ this.mousedown_y = null;
+
MaskEditorDialog.instance.drawing_mode = false;
}
@@ -398,24 +475,70 @@ class MaskEditorDialog extends ComfyDialog {
var centerX = self.cursorX;
var centerY = self.cursorY;
- brush.style.width = self.brush_size * 2 + "px";
- brush.style.height = self.brush_size * 2 + "px";
- brush.style.left = (centerX - self.brush_size) + "px";
- brush.style.top = (centerY - self.brush_size) + "px";
+ brush.style.width = self.brush_size * 2 * this.zoom_ratio + "px";
+ brush.style.height = self.brush_size * 2 * this.zoom_ratio + "px";
+ brush.style.left = (centerX - self.brush_size * this.zoom_ratio) + "px";
+ brush.style.top = (centerY - self.brush_size * this.zoom_ratio) + "px";
}
handleWheelEvent(self, event) {
- if(event.deltaY < 0)
- self.brush_size = Math.min(self.brush_size+2, 100);
- else
- self.brush_size = Math.max(self.brush_size-2, 1);
+ event.preventDefault();
- self.brush_slider_input.value = self.brush_size;
+ if(event.ctrlKey) {
+ // zoom canvas
+ if(event.deltaY < 0) {
+ this.zoom_ratio = Math.min(10.0, this.zoom_ratio+0.2);
+ }
+ else {
+ this.zoom_ratio = Math.max(0.2, this.zoom_ratio-0.2);
+ }
+
+ this.invalidatePanZoom();
+ }
+ else {
+ // adjust brush size
+ if(event.deltaY < 0)
+ this.brush_size = Math.min(this.brush_size+2, 100);
+ else
+ this.brush_size = Math.max(this.brush_size-2, 1);
+
+ this.brush_slider_input.value = this.brush_size;
+
+ this.updateBrushPreview(this);
+ }
+ }
+
+ pointMoveEvent(self, event) {
+ this.cursorX = event.pageX;
+ this.cursorY = event.pageY;
self.updateBrushPreview(self);
+
+ if(event.ctrlKey) {
+ event.preventDefault();
+ self.pan_move(self, event);
+ }
+ }
+
+ pan_move(self, event) {
+ if(event.buttons == 1) {
+ if(this.mousedown_x) {
+ let deltaX = this.mousedown_x - event.clientX;
+ let deltaY = this.mousedown_y - event.clientY;
+
+ self.pan_x = this.mousedown_pan_x - deltaX;
+ self.pan_y = this.mousedown_pan_y - deltaY;
+
+ self.invalidatePanZoom();
+ }
+ }
}
draw_move(self, event) {
+ if(event.ctrlKey) {
+ return;
+ }
+
event.preventDefault();
this.cursorX = event.pageX;
@@ -439,6 +562,9 @@ class MaskEditorDialog extends ComfyDialog {
y = event.targetTouches[0].clientY - maskRect.top;
}
+ x /= self.zoom_ratio;
+ y /= self.zoom_ratio;
+
var brush_size = this.brush_size;
if(event instanceof PointerEvent && event.pointerType == 'pen') {
brush_size *= event.pressure;
@@ -489,8 +615,8 @@ class MaskEditorDialog extends ComfyDialog {
}
else if(event.buttons == 2 || event.buttons == 5 || event.buttons == 32) {
const maskRect = self.maskCanvas.getBoundingClientRect();
- const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left;
- const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top;
+ const x = (event.offsetX || event.targetTouches[0].clientX - maskRect.left) / self.zoom_ratio;
+ const y = (event.offsetY || event.targetTouches[0].clientY - maskRect.top) / self.zoom_ratio;
var brush_size = this.brush_size;
if(event instanceof PointerEvent && event.pointerType == 'pen') {
@@ -540,6 +666,17 @@ class MaskEditorDialog extends ComfyDialog {
}
handlePointerDown(self, event) {
+ if(event.ctrlKey) {
+ if (event.buttons == 1) {
+ this.mousedown_x = event.clientX;
+ this.mousedown_y = event.clientY;
+
+ this.mousedown_pan_x = this.pan_x;
+ this.mousedown_pan_y = this.pan_y;
+ }
+ return;
+ }
+
var brush_size = this.brush_size;
if(event instanceof PointerEvent && event.pointerType == 'pen') {
brush_size *= event.pressure;
@@ -551,8 +688,8 @@ class MaskEditorDialog extends ComfyDialog {
event.preventDefault();
const maskRect = self.maskCanvas.getBoundingClientRect();
- const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left;
- const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top;
+ const x = (event.offsetX || event.targetTouches[0].clientX - maskRect.left) / self.zoom_ratio;
+ const y = (event.offsetY || event.targetTouches[0].clientY - maskRect.top) / self.zoom_ratio;
self.maskCtx.beginPath();
if (event.button == 0) {
@@ -570,15 +707,18 @@ class MaskEditorDialog extends ComfyDialog {
}
async save() {
- const backupCtx = this.backupCanvas.getContext('2d', {willReadFrequently:true});
+ const backupCanvas = document.createElement('canvas');
+ const backupCtx = backupCanvas.getContext('2d', {willReadFrequently:true});
+ backupCanvas.width = this.image.width;
+ backupCanvas.height = this.image.height;
- backupCtx.clearRect(0,0,this.backupCanvas.width,this.backupCanvas.height);
+ backupCtx.clearRect(0,0, backupCanvas.width, backupCanvas.height);
backupCtx.drawImage(this.maskCanvas,
0, 0, this.maskCanvas.width, this.maskCanvas.height,
- 0, 0, this.backupCanvas.width, this.backupCanvas.height);
+ 0, 0, backupCanvas.width, backupCanvas.height);
// paste mask data into alpha channel
- const backupData = backupCtx.getImageData(0, 0, this.backupCanvas.width, this.backupCanvas.height);
+ const backupData = backupCtx.getImageData(0, 0, backupCanvas.width, backupCanvas.height);
// refine mask image
for (let i = 0; i < backupData.data.length; i += 4) {
@@ -615,7 +755,7 @@ class MaskEditorDialog extends ComfyDialog {
ComfyApp.clipspace.widgets[index].value = item;
}
- const dataURL = this.backupCanvas.toDataURL();
+ const dataURL = backupCanvas.toDataURL();
const blob = dataURLToBlob(dataURL);
let original_url = new URL(this.image.src);
@@ -657,4 +797,4 @@ app.registerExtension({
const context_predicate = () => ComfyApp.clipspace && ComfyApp.clipspace.imgs && ComfyApp.clipspace.imgs.length > 0
ClipspaceDialog.registerButton("MaskEditor", context_predicate, ComfyApp.open_maskeditor);
}
-});
\ No newline at end of file
+});
diff --git a/web/extensions/core/nodeTemplates.js b/web/extensions/core/nodeTemplates.js
index b6479f454..bc9a10864 100644
--- a/web/extensions/core/nodeTemplates.js
+++ b/web/extensions/core/nodeTemplates.js
@@ -1,5 +1,6 @@
import { app } from "../../scripts/app.js";
import { ComfyDialog, $el } from "../../scripts/ui.js";
+import { GroupNodeConfig, GroupNodeHandler } from "./groupNode.js";
// Adds the ability to save and add multiple nodes as a template
// To save:
@@ -34,7 +35,7 @@ class ManageTemplates extends ComfyDialog {
type: "file",
accept: ".json",
multiple: true,
- style: {display: "none"},
+ style: { display: "none" },
parent: document.body,
onchange: () => this.importAll(),
});
@@ -109,13 +110,13 @@ class ManageTemplates extends ComfyDialog {
return;
}
- const json = JSON.stringify({templates: this.templates}, null, 2); // convert the data to a JSON string
- const blob = new Blob([json], {type: "application/json"});
+ const json = JSON.stringify({ templates: this.templates }, null, 2); // convert the data to a JSON string
+ const blob = new Blob([json], { type: "application/json" });
const url = URL.createObjectURL(blob);
const a = $el("a", {
href: url,
download: "node_templates.json",
- style: {display: "none"},
+ style: { display: "none" },
parent: document.body,
});
a.click();
@@ -163,7 +164,7 @@ class ManageTemplates extends ComfyDialog {
var prev_i = el.dataset.id;
if ( el == this.draggedEl && prev_i != i ) {
- [this.templates[i], this.templates[prev_i]] = [this.templates[prev_i], this.templates[i]];
+ this.templates.splice(i, 0, this.templates.splice(prev_i, 1)[0]);
}
el.dataset.id = i;
});
@@ -291,11 +292,11 @@ app.registerExtension({
setup() {
const manage = new ManageTemplates();
- const clipboardAction = (cb) => {
+ const clipboardAction = async (cb) => {
// We use the clipboard functions but dont want to overwrite the current user clipboard
// Restore it after we've run our callback
const old = localStorage.getItem("litegrapheditor_clipboard");
- cb();
+ await cb();
localStorage.setItem("litegrapheditor_clipboard", old);
};
@@ -309,13 +310,31 @@ app.registerExtension({
disabled: !Object.keys(app.canvas.selected_nodes || {}).length,
callback: () => {
const name = prompt("Enter name");
- if (!name || !name.trim()) return;
+ if (!name?.trim()) return;
clipboardAction(() => {
app.canvas.copyToClipboard();
+ let data = localStorage.getItem("litegrapheditor_clipboard");
+ data = JSON.parse(data);
+ const nodeIds = Object.keys(app.canvas.selected_nodes);
+ for (let i = 0; i < nodeIds.length; i++) {
+ const node = app.graph.getNodeById(nodeIds[i]);
+ const nodeData = node?.constructor.nodeData;
+
+ let groupData = GroupNodeHandler.getGroupData(node);
+ if (groupData) {
+ groupData = groupData.nodeData;
+ if (!data.groupNodes) {
+ data.groupNodes = {};
+ }
+ data.groupNodes[nodeData.name] = groupData;
+ data.nodes[i].type = nodeData.name;
+ }
+ }
+
manage.templates.push({
name,
- data: localStorage.getItem("litegrapheditor_clipboard"),
+ data: JSON.stringify(data),
});
manage.store();
});
@@ -323,15 +342,19 @@ app.registerExtension({
});
// Map each template to a menu item
- const subItems = manage.templates.map((t) => ({
- content: t.name,
- callback: () => {
- clipboardAction(() => {
- localStorage.setItem("litegrapheditor_clipboard", t.data);
- app.canvas.pasteFromClipboard();
- });
- },
- }));
+ const subItems = manage.templates.map((t) => {
+ return {
+ content: t.name,
+ callback: () => {
+ clipboardAction(async () => {
+ const data = JSON.parse(t.data);
+ await GroupNodeConfig.registerFromWorkflow(data.groupNodes, {});
+ localStorage.setItem("litegrapheditor_clipboard", t.data);
+ app.canvas.pasteFromClipboard();
+ });
+ },
+ };
+ });
subItems.push(null, {
content: "Manage",
diff --git a/web/extensions/core/rerouteNode.js b/web/extensions/core/rerouteNode.js
index 499a171da..4feff91e5 100644
--- a/web/extensions/core/rerouteNode.js
+++ b/web/extensions/core/rerouteNode.js
@@ -1,10 +1,11 @@
import { app } from "../../scripts/app.js";
+import { mergeIfValid, getWidgetConfig, setWidgetConfig } from "./widgetInputs.js";
// Node that allows you to redirect connections for cleaner graphs
app.registerExtension({
name: "Comfy.RerouteNode",
- registerCustomNodes() {
+ registerCustomNodes(app) {
class RerouteNode {
constructor() {
if (!this.properties) {
@@ -16,6 +17,12 @@ app.registerExtension({
this.addInput("", "*");
this.addOutput(this.properties.showOutputText ? "*" : "", "*");
+ this.onAfterGraphConfigured = function () {
+ requestAnimationFrame(() => {
+ this.onConnectionsChange(LiteGraph.INPUT, null, true, null);
+ });
+ };
+
this.onConnectionsChange = function (type, index, connected, link_info) {
this.applyOrientation();
@@ -47,6 +54,7 @@ app.registerExtension({
const linkId = currentNode.inputs[0].link;
if (linkId !== null) {
const link = app.graph.links[linkId];
+ if (!link) return;
const node = app.graph.getNodeById(link.origin_id);
const type = node.constructor.type;
if (type === "Reroute") {
@@ -54,8 +62,7 @@ app.registerExtension({
// We've found a circle
currentNode.disconnectInput(link.target_slot);
currentNode = null;
- }
- else {
+ } else {
// Move the previous node
currentNode = node;
}
@@ -94,8 +101,11 @@ app.registerExtension({
updateNodes.push(node);
} else {
// We've found an output
- const nodeOutType = node.inputs && node.inputs[link?.target_slot] && node.inputs[link.target_slot].type ? node.inputs[link.target_slot].type : null;
- if (inputType && nodeOutType !== inputType) {
+ const nodeOutType =
+ node.inputs && node.inputs[link?.target_slot] && node.inputs[link.target_slot].type
+ ? node.inputs[link.target_slot].type
+ : null;
+ if (inputType && inputType !== "*" && nodeOutType !== inputType) {
// The output doesnt match our input so disconnect it
node.disconnectInput(link.target_slot);
} else {
@@ -111,6 +121,9 @@ app.registerExtension({
const displayType = inputType || outputType || "*";
const color = LGraphCanvas.link_type_colors[displayType];
+ let widgetConfig;
+ let targetWidget;
+ let widgetType;
// Update the types of each node
for (const node of updateNodes) {
// If we dont have an input type we are always wildcard but we'll show the output type
@@ -125,10 +138,38 @@ app.registerExtension({
const link = app.graph.links[l];
if (link) {
link.color = color;
+
+ if (app.configuringGraph) continue;
+ const targetNode = app.graph.getNodeById(link.target_id);
+ const targetInput = targetNode.inputs?.[link.target_slot];
+ if (targetInput?.widget) {
+ const config = getWidgetConfig(targetInput);
+ if (!widgetConfig) {
+ widgetConfig = config[1] ?? {};
+ widgetType = config[0];
+ }
+ if (!targetWidget) {
+ targetWidget = targetNode.widgets?.find((w) => w.name === targetInput.widget.name);
+ }
+
+ const merged = mergeIfValid(targetInput, [config[0], widgetConfig]);
+ if (merged.customConfig) {
+ widgetConfig = merged.customConfig;
+ }
+ }
}
}
}
+ for (const node of updateNodes) {
+ if (widgetConfig && outputType) {
+ node.inputs[0].widget = { name: "value" };
+ setWidgetConfig(node.inputs[0], [widgetType ?? displayType, widgetConfig], targetWidget);
+ } else {
+ setWidgetConfig(node.inputs[0], null);
+ }
+ }
+
if (inputNode) {
const link = app.graph.links[inputNode.inputs[0].link];
if (link) {
@@ -173,8 +214,8 @@ app.registerExtension({
},
{
// naming is inverted with respect to LiteGraphNode.horizontal
- // LiteGraphNode.horizontal == true means that
- // each slot in the inputs and outputs are layed out horizontally,
+ // LiteGraphNode.horizontal == true means that
+ // each slot in the inputs and outputs are layed out horizontally,
// which is the opposite of the visual orientation of the inputs and outputs as a node
content: "Set " + (this.properties.horizontal ? "Horizontal" : "Vertical"),
callback: () => {
@@ -187,7 +228,7 @@ app.registerExtension({
applyOrientation() {
this.horizontal = this.properties.horizontal;
if (this.horizontal) {
- // we correct the input position, because LiteGraphNode.horizontal
+ // we correct the input position, because LiteGraphNode.horizontal
// doesn't account for title presence
// which reroute nodes don't have
this.inputs[0].pos = [this.size[0] / 2, 0];
diff --git a/web/extensions/core/saveImageExtraOutput.js b/web/extensions/core/saveImageExtraOutput.js
index 99e2213bf..a0506b43b 100644
--- a/web/extensions/core/saveImageExtraOutput.js
+++ b/web/extensions/core/saveImageExtraOutput.js
@@ -1,5 +1,5 @@
import { app } from "../../scripts/app.js";
-
+import { applyTextReplacements } from "../../scripts/utils.js";
// Use widget values and dates in output filenames
app.registerExtension({
@@ -7,84 +7,19 @@ app.registerExtension({
async beforeRegisterNodeDef(nodeType, nodeData, app) {
if (nodeData.name === "SaveImage") {
const onNodeCreated = nodeType.prototype.onNodeCreated;
-
- // Simple date formatter
- const parts = {
- d: (d) => d.getDate(),
- M: (d) => d.getMonth() + 1,
- h: (d) => d.getHours(),
- m: (d) => d.getMinutes(),
- s: (d) => d.getSeconds(),
- };
- const format =
- Object.keys(parts)
- .map((k) => k + k + "?")
- .join("|") + "|yyy?y?";
-
- function formatDate(text, date) {
- return text.replace(new RegExp(format, "g"), function (text) {
- if (text === "yy") return (date.getFullYear() + "").substring(2);
- if (text === "yyyy") return date.getFullYear();
- if (text[0] in parts) {
- const p = parts[text[0]](date);
- return (p + "").padStart(text.length, "0");
- }
- return text;
- });
- }
-
- // When the SaveImage node is created we want to override the serialization of the output name widget to run our S&R
+ // When the SaveImage node is created we want to override the serialization of the output name widget to run our S&R
nodeType.prototype.onNodeCreated = function () {
const r = onNodeCreated ? onNodeCreated.apply(this, arguments) : undefined;
const widget = this.widgets.find((w) => w.name === "filename_prefix");
widget.serializeValue = () => {
- return widget.value.replace(/%([^%]+)%/g, function (match, text) {
- const split = text.split(".");
- if (split.length !== 2) {
- // Special handling for dates
- if (split[0].startsWith("date:")) {
- return formatDate(split[0].substring(5), new Date());
- }
-
- if (text !== "width" && text !== "height") {
- // Dont warn on standard replacements
- console.warn("Invalid replacement pattern", text);
- }
- return match;
- }
-
- // Find node with matching S&R property name
- let nodes = app.graph._nodes.filter((n) => n.properties?.["Node name for S&R"] === split[0]);
- // If we cant, see if there is a node with that title
- if (!nodes.length) {
- nodes = app.graph._nodes.filter((n) => n.title === split[0]);
- }
- if (!nodes.length) {
- console.warn("Unable to find node", split[0]);
- return match;
- }
-
- if (nodes.length > 1) {
- console.warn("Multiple nodes matched", split[0], "using first match");
- }
-
- const node = nodes[0];
-
- const widget = node.widgets?.find((w) => w.name === split[1]);
- if (!widget) {
- console.warn("Unable to find widget", split[1], "on node", split[0], node);
- return match;
- }
-
- return ((widget.value ?? "") + "").replaceAll(/\/|\\/g, "_");
- });
+ return applyTextReplacements(app, widget.value);
};
return r;
};
} else {
- // When any other node is created add a property to alias the node
+ // When any other node is created add a property to alias the node
const onNodeCreated = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function () {
const r = onNodeCreated ? onNodeCreated.apply(this, arguments) : undefined;
diff --git a/web/extensions/core/undoRedo.js b/web/extensions/core/undoRedo.js
new file mode 100644
index 000000000..3cb137520
--- /dev/null
+++ b/web/extensions/core/undoRedo.js
@@ -0,0 +1,147 @@
+import { app } from "../../scripts/app.js";
+
+const MAX_HISTORY = 50;
+
+let undo = [];
+let redo = [];
+let activeState = null;
+let isOurLoad = false;
+function checkState() {
+ const currentState = app.graph.serialize();
+ if (!graphEqual(activeState, currentState)) {
+ undo.push(activeState);
+ if (undo.length > MAX_HISTORY) {
+ undo.shift();
+ }
+ activeState = clone(currentState);
+ redo.length = 0;
+ }
+}
+
+const loadGraphData = app.loadGraphData;
+app.loadGraphData = async function () {
+ const v = await loadGraphData.apply(this, arguments);
+ if (isOurLoad) {
+ isOurLoad = false;
+ } else {
+ checkState();
+ }
+ return v;
+};
+
+function clone(obj) {
+ try {
+ if (typeof structuredClone !== "undefined") {
+ return structuredClone(obj);
+ }
+ } catch (error) {
+ // structuredClone is stricter than using JSON.parse/stringify so fallback to that
+ }
+
+ return JSON.parse(JSON.stringify(obj));
+}
+
+function graphEqual(a, b, root = true) {
+ if (a === b) return true;
+
+ if (typeof a == "object" && a && typeof b == "object" && b) {
+ const keys = Object.getOwnPropertyNames(a);
+
+ if (keys.length != Object.getOwnPropertyNames(b).length) {
+ return false;
+ }
+
+ for (const key of keys) {
+ let av = a[key];
+ let bv = b[key];
+ if (root && key === "nodes") {
+ // Nodes need to be sorted as the order changes when selecting nodes
+ av = [...av].sort((a, b) => a.id - b.id);
+ bv = [...bv].sort((a, b) => a.id - b.id);
+ }
+ if (!graphEqual(av, bv, false)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ return false;
+}
+
+const undoRedo = async (e) => {
+ const updateState = async (source, target) => {
+ const prevState = source.pop();
+ if (prevState) {
+ target.push(activeState);
+ isOurLoad = true;
+ await app.loadGraphData(prevState, false);
+ activeState = prevState;
+ }
+ }
+ if (e.ctrlKey || e.metaKey) {
+ if (e.key === "y") {
+ updateState(redo, undo);
+ return true;
+ } else if (e.key === "z") {
+ updateState(undo, redo);
+ return true;
+ }
+ }
+};
+
+const bindInput = (activeEl) => {
+ if (activeEl?.tagName !== "CANVAS" && activeEl?.tagName !== "BODY") {
+ for (const evt of ["change", "input", "blur"]) {
+ if (`on${evt}` in activeEl) {
+ const listener = () => {
+ checkState();
+ activeEl.removeEventListener(evt, listener);
+ };
+ activeEl.addEventListener(evt, listener);
+ return true;
+ }
+ }
+ }
+};
+
+window.addEventListener(
+ "keydown",
+ (e) => {
+ requestAnimationFrame(async () => {
+ const activeEl = document.activeElement;
+ if (activeEl?.tagName === "INPUT" || activeEl?.type === "textarea") {
+ // Ignore events on inputs, they have their native history
+ return;
+ }
+
+ // Check if this is a ctrl+z ctrl+y
+ if (await undoRedo(e)) return;
+
+ // If our active element is some type of input then handle changes after they're done
+ if (bindInput(activeEl)) return;
+ checkState();
+ });
+ },
+ true
+);
+
+// Handle clicking DOM elements (e.g. widgets)
+window.addEventListener("mouseup", () => {
+ checkState();
+});
+
+// Handle litegraph clicks
+const processMouseUp = LGraphCanvas.prototype.processMouseUp;
+LGraphCanvas.prototype.processMouseUp = function (e) {
+ const v = processMouseUp.apply(this, arguments);
+ checkState();
+ return v;
+};
+const processMouseDown = LGraphCanvas.prototype.processMouseDown;
+LGraphCanvas.prototype.processMouseDown = function (e) {
+ const v = processMouseDown.apply(this, arguments);
+ checkState();
+ return v;
+};
diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js
index bad3ac3a7..3f1c1f8c1 100644
--- a/web/extensions/core/widgetInputs.js
+++ b/web/extensions/core/widgetInputs.js
@@ -1,10 +1,16 @@
-import { ComfyWidgets, addValueControlWidget } from "../../scripts/widgets.js";
+import { ComfyWidgets, addValueControlWidgets } from "../../scripts/widgets.js";
import { app } from "../../scripts/app.js";
+import { applyTextReplacements } from "../../scripts/utils.js";
const CONVERTED_TYPE = "converted-widget";
const VALID_TYPES = ["STRING", "combo", "number", "BOOLEAN"];
const CONFIG = Symbol();
const GET_CONFIG = Symbol();
+const TARGET = Symbol(); // Used for reroutes to specify the real target widget
+
+export function getWidgetConfig(slot) {
+ return slot.widget[CONFIG] ?? slot.widget[GET_CONFIG]();
+}
function getConfig(widgetName) {
const { nodeData } = this.constructor;
@@ -100,7 +106,6 @@ function getWidgetType(config) {
return { type };
}
-
function isValidCombo(combo, obj) {
// New input isnt a combo
if (!(obj instanceof Array)) {
@@ -121,6 +126,135 @@ function isValidCombo(combo, obj) {
return true;
}
+export function setWidgetConfig(slot, config, target) {
+ if (!slot.widget) return;
+ if (config) {
+ slot.widget[GET_CONFIG] = () => config;
+ slot.widget[TARGET] = target;
+ } else {
+ delete slot.widget;
+ }
+
+ if (slot.link) {
+ const link = app.graph.links[slot.link];
+ if (link) {
+ const originNode = app.graph.getNodeById(link.origin_id);
+ if (originNode.type === "PrimitiveNode") {
+ if (config) {
+ originNode.recreateWidget();
+ } else if(!app.configuringGraph) {
+ originNode.disconnectOutput(0);
+ originNode.onLastDisconnect();
+ }
+ }
+ }
+ }
+}
+
+export function mergeIfValid(output, config2, forceUpdate, recreateWidget, config1) {
+ if (!config1) {
+ config1 = output.widget[CONFIG] ?? output.widget[GET_CONFIG]();
+ }
+
+ if (config1[0] instanceof Array) {
+ if (!isValidCombo(config1[0], config2[0])) return false;
+ } else if (config1[0] !== config2[0]) {
+ // Types dont match
+ console.log(`connection rejected: types dont match`, config1[0], config2[0]);
+ return false;
+ }
+
+ const keys = new Set([...Object.keys(config1[1] ?? {}), ...Object.keys(config2[1] ?? {})]);
+
+ let customConfig;
+ const getCustomConfig = () => {
+ if (!customConfig) {
+ if (typeof structuredClone === "undefined") {
+ customConfig = JSON.parse(JSON.stringify(config1[1] ?? {}));
+ } else {
+ customConfig = structuredClone(config1[1] ?? {});
+ }
+ }
+ return customConfig;
+ };
+
+ const isNumber = config1[0] === "INT" || config1[0] === "FLOAT";
+ for (const k of keys.values()) {
+ if (k !== "default" && k !== "forceInput" && k !== "defaultInput" && k !== "control_after_generate" && k !== "multiline") {
+ let v1 = config1[1][k];
+ let v2 = config2[1]?.[k];
+
+ if (v1 === v2 || (!v1 && !v2)) continue;
+
+ if (isNumber) {
+ if (k === "min") {
+ const theirMax = config2[1]?.["max"];
+ if (theirMax != null && v1 > theirMax) {
+ console.log("connection rejected: min > max", v1, theirMax);
+ return false;
+ }
+ getCustomConfig()[k] = v1 == null ? v2 : v2 == null ? v1 : Math.max(v1, v2);
+ continue;
+ } else if (k === "max") {
+ const theirMin = config2[1]?.["min"];
+ if (theirMin != null && v1 < theirMin) {
+ console.log("connection rejected: max < min", v1, theirMin);
+ return false;
+ }
+ getCustomConfig()[k] = v1 == null ? v2 : v2 == null ? v1 : Math.min(v1, v2);
+ continue;
+ } else if (k === "step") {
+ let step;
+ if (v1 == null) {
+ // No current step
+ step = v2;
+ } else if (v2 == null) {
+ // No new step
+ step = v1;
+ } else {
+ if (v1 < v2) {
+ // Ensure v1 is larger for the mod
+ const a = v2;
+ v2 = v1;
+ v1 = a;
+ }
+ if (v1 % v2) {
+ console.log("connection rejected: steps not divisible", "current:", v1, "new:", v2);
+ return false;
+ }
+
+ step = v1;
+ }
+
+ getCustomConfig()[k] = step;
+ continue;
+ }
+ }
+
+ console.log(`connection rejected: config ${k} values dont match`, v1, v2);
+ return false;
+ }
+ }
+
+ if (customConfig || forceUpdate) {
+ if (customConfig) {
+ output.widget[CONFIG] = [config1[0], customConfig];
+ }
+
+ const widget = recreateWidget?.call(this);
+ // When deleting a node this can be null
+ if (widget) {
+ const min = widget.options.min;
+ const max = widget.options.max;
+ if (min != null && widget.value < min) widget.value = min;
+ if (max != null && widget.value > max) widget.value = max;
+ widget.callback(widget.value);
+ }
+ }
+
+ return { customConfig };
+}
+
app.registerExtension({
name: "Comfy.WidgetInputs",
async beforeRegisterNodeDef(nodeType, nodeData, app) {
@@ -301,14 +435,19 @@ app.registerExtension({
};
},
registerCustomNodes() {
+ const replacePropertyName = "Run widget replace on values";
class PrimitiveNode {
constructor() {
this.addOutput("connect to widget input", "*");
this.serialize_widgets = true;
this.isVirtualNode = true;
+
+ if (!this.properties || !(replacePropertyName in this.properties)) {
+ this.addProperty(replacePropertyName, false, "boolean");
+ }
}
- applyToGraph() {
+ applyToGraph(extraLinks = []) {
if (!this.outputs[0].links?.length) return;
function get_links(node) {
@@ -325,20 +464,30 @@ app.registerExtension({
return links;
}
- let links = get_links(this);
+ let links = [...get_links(this).map((l) => app.graph.links[l]), ...extraLinks];
+ let v = this.widgets?.[0].value;
+ if(v && this.properties[replacePropertyName]) {
+ v = applyTextReplacements(app, v);
+ }
+
// For each output link copy our value over the original widget value
- for (const l of links) {
- const linkInfo = app.graph.links[l];
+ for (const linkInfo of links) {
const node = this.graph.getNodeById(linkInfo.target_id);
const input = node.inputs[linkInfo.target_slot];
- const widgetName = input.widget.name;
- if (widgetName) {
- const widget = node.widgets.find((w) => w.name === widgetName);
- if (widget) {
- widget.value = this.widgets[0].value;
- if (widget.callback) {
- widget.callback(widget.value, app.canvas, node, app.canvas.graph_mouse, {});
- }
+ let widget;
+ if (input.widget[TARGET]) {
+ widget = input.widget[TARGET];
+ } else {
+ const widgetName = input.widget.name;
+ if (widgetName) {
+ widget = node.widgets.find((w) => w.name === widgetName);
+ }
+ }
+
+ if (widget) {
+ widget.value = v;
+ if (widget.callback) {
+ widget.callback(widget.value, app.canvas, node, app.canvas.graph_mouse, {});
}
}
}
@@ -391,26 +540,34 @@ app.registerExtension({
this.#mergeWidgetConfig();
if (!links?.length) {
- this.#onLastDisconnect();
+ this.onLastDisconnect();
}
}
}
onConnectOutput(slot, type, input, target_node, target_slot) {
// Fires before the link is made allowing us to reject it if it isn't valid
-
// No widget, we cant connect
if (!input.widget) {
if (!(input.type in ComfyWidgets)) return false;
}
if (this.outputs[slot].links?.length) {
- return this.#isValidConnection(input);
+ const valid = this.#isValidConnection(input);
+ if (valid) {
+ // On connect of additional outputs, copy our value to their widget
+ this.applyToGraph([{ target_id: target_node.id, target_slot }]);
+ }
+ return valid;
}
}
#onFirstConnection(recreating) {
// First connection can fire before the graph is ready on initial load so random things can be missing
+ if (!this.outputs[0].links) {
+ this.onLastDisconnect();
+ return;
+ }
const linkId = this.outputs[0].links[0];
const link = this.graph.links[linkId];
if (!link) return;
@@ -438,10 +595,10 @@ app.registerExtension({
this.outputs[0].name = type;
this.outputs[0].widget = widget;
- this.#createWidget(widget[CONFIG] ?? config, theirNode, widget.name, recreating);
+ this.#createWidget(widget[CONFIG] ?? config, theirNode, widget.name, recreating, widget[TARGET]);
}
- #createWidget(inputData, node, widgetName, recreating) {
+ #createWidget(inputData, node, widgetName, recreating, targetWidget) {
let type = inputData[0];
if (type instanceof Array) {
@@ -455,19 +612,33 @@ app.registerExtension({
widget = this.addWidget(type, "value", null, () => {}, {});
}
- if (node?.widgets && widget) {
+ if (targetWidget) {
+ widget.value = targetWidget.value;
+ } else if (node?.widgets && widget) {
const theirWidget = node.widgets.find((w) => w.name === widgetName);
if (theirWidget) {
widget.value = theirWidget.value;
}
}
- if (widget.type === "number" || widget.type === "combo") {
+ if (!inputData?.[1]?.control_after_generate && (widget.type === "number" || widget.type === "combo")) {
let control_value = this.widgets_values?.[1];
if (!control_value) {
control_value = "fixed";
}
- addValueControlWidget(this, widget, control_value);
+ addValueControlWidgets(this, widget, control_value, undefined, inputData);
+ let filter = this.widgets_values?.[2];
+ if (filter && this.widgets.length === 3) {
+ this.widgets[2].value = filter;
+ }
+ }
+
+ // Restore any saved control values
+ const controlValues = this.controlValues;
+ if(this.lastType === this.widgets[0].type && controlValues?.length === this.widgets.length - 1) {
+ for(let i = 0; i < controlValues.length; i++) {
+ this.widgets[i + 1].value = controlValues[i];
+ }
}
// When our value changes, update other widgets to reflect our changes
@@ -498,11 +669,14 @@ app.registerExtension({
}
}
- #recreateWidget() {
- const values = this.widgets.map((w) => w.value);
+ recreateWidget() {
+ const values = this.widgets?.map((w) => w.value);
this.#removeWidgets();
this.#onFirstConnection(true);
- for (let i = 0; i < this.widgets?.length; i++) this.widgets[i].value = values[i];
+ if (values?.length) {
+ for (let i = 0; i < this.widgets?.length; i++) this.widgets[i].value = values[i];
+ }
+ return this.widgets?.[0];
}
#mergeWidgetConfig() {
@@ -518,7 +692,7 @@ app.registerExtension({
if (links?.length < 2 && hasConfig) {
// Copy the widget options from the source
if (links.length) {
- this.#recreateWidget();
+ this.recreateWidget();
}
return;
@@ -543,108 +717,8 @@ app.registerExtension({
#isValidConnection(input, forceUpdate) {
// Only allow connections where the configs match
const output = this.outputs[0];
- const config1 = output.widget[CONFIG] ?? output.widget[GET_CONFIG]();
const config2 = input.widget[GET_CONFIG]();
-
- if (config1[0] instanceof Array) {
- if (!isValidCombo(config1[0], config2[0])) return false;
- } else if (config1[0] !== config2[0]) {
- // Types dont match
- console.log(`connection rejected: types dont match`, config1[0], config2[0]);
- return false;
- }
-
- const keys = new Set([...Object.keys(config1[1] ?? {}), ...Object.keys(config2[1] ?? {})]);
-
- let customConfig;
- const getCustomConfig = () => {
- if (!customConfig) {
- if (typeof structuredClone === "undefined") {
- customConfig = JSON.parse(JSON.stringify(config1[1] ?? {}));
- } else {
- customConfig = structuredClone(config1[1] ?? {});
- }
- }
- return customConfig;
- };
-
- const isNumber = config1[0] === "INT" || config1[0] === "FLOAT";
- for (const k of keys.values()) {
- if (k !== "default" && k !== "forceInput" && k !== "defaultInput") {
- let v1 = config1[1][k];
- let v2 = config2[1][k];
-
- if (v1 === v2 || (!v1 && !v2)) continue;
-
- if (isNumber) {
- if (k === "min") {
- const theirMax = config2[1]["max"];
- if (theirMax != null && v1 > theirMax) {
- console.log("connection rejected: min > max", v1, theirMax);
- return false;
- }
- getCustomConfig()[k] = v1 == null ? v2 : v2 == null ? v1 : Math.max(v1, v2);
- continue;
- } else if (k === "max") {
- const theirMin = config2[1]["min"];
- if (theirMin != null && v1 < theirMin) {
- console.log("connection rejected: max < min", v1, theirMin);
- return false;
- }
- getCustomConfig()[k] = v1 == null ? v2 : v2 == null ? v1 : Math.min(v1, v2);
- continue;
- } else if (k === "step") {
- let step;
- if (v1 == null) {
- // No current step
- step = v2;
- } else if (v2 == null) {
- // No new step
- step = v1;
- } else {
- if (v1 < v2) {
- // Ensure v1 is larger for the mod
- const a = v2;
- v2 = v1;
- v1 = a;
- }
- if (v1 % v2) {
- console.log("connection rejected: steps not divisible", "current:", v1, "new:", v2);
- return false;
- }
-
- step = v1;
- }
-
- getCustomConfig()[k] = step;
- continue;
- }
- }
-
- console.log(`connection rejected: config ${k} values dont match`, v1, v2);
- return false;
- }
- }
-
- if (customConfig || forceUpdate) {
- if (customConfig) {
- output.widget[CONFIG] = [config1[0], customConfig];
- }
-
- this.#recreateWidget();
-
- const widget = this.widgets[0];
- // When deleting a node this can be null
- if (widget) {
- const min = widget.options.min;
- const max = widget.options.max;
- if (min != null && widget.value < min) widget.value = min;
- if (max != null && widget.value > max) widget.value = max;
- widget.callback(widget.value);
- }
- }
-
- return true;
+ return !!mergeIfValid.call(this, output, config2, forceUpdate, this.recreateWidget);
}
#removeWidgets() {
@@ -655,11 +729,20 @@ app.registerExtension({
w.onRemove();
}
}
+
+ // Temporarily store the current values in case the node is being recreated
+ // e.g. by group node conversion
+ this.controlValues = [];
+ this.lastType = this.widgets[0]?.type;
+ for(let i = 1; i < this.widgets.length; i++) {
+ this.controlValues.push(this.widgets[i].value);
+ }
+ setTimeout(() => { delete this.lastType; delete this.controlValues }, 15);
this.widgets.length = 0;
}
}
- #onLastDisconnect() {
+ onLastDisconnect() {
// We cant remove + re-add the output here as if you drag a link over the same link
// it removes, then re-adds, causing it to break
this.outputs[0].type = "*";
diff --git a/web/lib/litegraph.core.js b/web/lib/litegraph.core.js
index e906590f5..434c4a83b 100644
--- a/web/lib/litegraph.core.js
+++ b/web/lib/litegraph.core.js
@@ -48,7 +48,7 @@
EVENT_LINK_COLOR: "#A86",
CONNECTING_LINK_COLOR: "#AFA",
- MAX_NUMBER_OF_NODES: 1000, //avoid infinite loops
+ MAX_NUMBER_OF_NODES: 10000, //avoid infinite loops
DEFAULT_POSITION: [100, 100], //default node position
VALID_SHAPES: ["default", "box", "round", "card"], //,"circle"
@@ -2533,7 +2533,7 @@
var w = this.widgets[i];
if(!w)
continue;
- if(w.options && w.options.property && this.properties[ w.options.property ])
+ if(w.options && w.options.property && (this.properties[ w.options.property ] != undefined))
w.value = JSON.parse( JSON.stringify( this.properties[ w.options.property ] ) );
}
if (info.widgets_values) {
@@ -3788,16 +3788,42 @@
/**
* returns the bounding of the object, used for rendering purposes
- * bounding is: [topleft_cornerx, topleft_cornery, width, height]
* @method getBounding
- * @return {Float32Array[4]} the total size
+ * @param out {Float32Array[4]?} [optional] a place to store the output, to free garbage
+ * @param compute_outer {boolean?} [optional] set to true to include the shadow and connection points in the bounding calculation
+ * @return {Float32Array[4]} the bounding box in format of [topleft_cornerx, topleft_cornery, width, height]
*/
- LGraphNode.prototype.getBounding = function(out) {
+ LGraphNode.prototype.getBounding = function(out, compute_outer) {
out = out || new Float32Array(4);
- out[0] = this.pos[0] - 4;
- out[1] = this.pos[1] - LiteGraph.NODE_TITLE_HEIGHT;
- out[2] = this.flags.collapsed ? (this._collapsed_width || LiteGraph.NODE_COLLAPSED_WIDTH) : this.size[0] + 4;
- out[3] = this.flags.collapsed ? LiteGraph.NODE_TITLE_HEIGHT : this.size[1] + LiteGraph.NODE_TITLE_HEIGHT;
+ const nodePos = this.pos;
+ const isCollapsed = this.flags.collapsed;
+ const nodeSize = this.size;
+
+ let left_offset = 0;
+ // 1 offset due to how nodes are rendered
+ let right_offset = 1 ;
+ let top_offset = 0;
+ let bottom_offset = 0;
+
+ if (compute_outer) {
+ // 4 offset for collapsed node connection points
+ left_offset = 4;
+ // 6 offset for right shadow and collapsed node connection points
+ right_offset = 6 + left_offset;
+ // 4 offset for collapsed nodes top connection points
+ top_offset = 4;
+ // 5 offset for bottom shadow and collapsed node connection points
+ bottom_offset = 5 + top_offset;
+ }
+
+ out[0] = nodePos[0] - left_offset;
+ out[1] = nodePos[1] - LiteGraph.NODE_TITLE_HEIGHT - top_offset;
+ out[2] = isCollapsed ?
+ (this._collapsed_width || LiteGraph.NODE_COLLAPSED_WIDTH) + right_offset :
+ nodeSize[0] + right_offset;
+ out[3] = isCollapsed ?
+ LiteGraph.NODE_TITLE_HEIGHT + bottom_offset :
+ nodeSize[1] + LiteGraph.NODE_TITLE_HEIGHT + bottom_offset;
if (this.onBounding) {
this.onBounding(out);
@@ -5714,10 +5740,10 @@ LGraphNode.prototype.executeAction = function(action)
* @method enableWebGL
**/
LGraphCanvas.prototype.enableWebGL = function() {
- if (typeof GL === undefined) {
+ if (typeof GL === "undefined") {
throw "litegl.js must be included to use a WebGL canvas";
}
- if (typeof enableWebGLCanvas === undefined) {
+ if (typeof enableWebGLCanvas === "undefined") {
throw "webglCanvas.js must be included to use this feature";
}
@@ -7110,15 +7136,16 @@ LGraphNode.prototype.executeAction = function(action)
}
};
- LGraphCanvas.prototype.copyToClipboard = function() {
+ LGraphCanvas.prototype.copyToClipboard = function(nodes) {
var clipboard_info = {
nodes: [],
links: []
};
var index = 0;
var selected_nodes_array = [];
- for (var i in this.selected_nodes) {
- var node = this.selected_nodes[i];
+ if (!nodes) nodes = this.selected_nodes;
+ for (var i in nodes) {
+ var node = nodes[i];
if (node.clonable === false)
continue;
node._relative_id = index;
@@ -7673,7 +7700,7 @@ LGraphNode.prototype.executeAction = function(action)
continue;
}
- if (!overlapBounding(this.visible_area, n.getBounding(temp))) {
+ if (!overlapBounding(this.visible_area, n.getBounding(temp, true))) {
continue;
} //out of the visible area
@@ -11335,6 +11362,7 @@ LGraphNode.prototype.executeAction = function(action)
name_element.innerText = title;
var value_element = dialog.querySelector(".value");
value_element.value = value;
+ value_element.select();
var input = value_element;
input.addEventListener("keydown", function(e) {
@@ -11702,7 +11730,7 @@ LGraphNode.prototype.executeAction = function(action)
default:
iS = 0; // try with first if no name set
}
- if (typeof options.node_from.outputs[iS] !== undefined){
+ if (typeof options.node_from.outputs[iS] !== "undefined"){
if (iS!==false && iS>-1){
options.node_from.connectByType( iS, node, options.node_from.outputs[iS].type );
}
@@ -11730,7 +11758,7 @@ LGraphNode.prototype.executeAction = function(action)
default:
iS = 0; // try with first if no name set
}
- if (typeof options.node_to.inputs[iS] !== undefined){
+ if (typeof options.node_to.inputs[iS] !== "undefined"){
if (iS!==false && iS>-1){
// try connection
options.node_to.connectByTypeOutput(iS,node,options.node_to.inputs[iS].type);
diff --git a/web/scripts/api.js b/web/scripts/api.js
index b1d245d73..9aa7528af 100644
--- a/web/scripts/api.js
+++ b/web/scripts/api.js
@@ -254,9 +254,9 @@ class ComfyApi extends EventTarget {
* Gets the prompt execution history
* @returns Prompt history including node outputs
*/
- async getHistory() {
+ async getHistory(max_items=200) {
try {
- const res = await this.fetchApi("/history");
+ const res = await this.fetchApi(`/history?max_items=${max_items}`);
return { History: Object.values(await res.json()) };
} catch (error) {
console.error(error);
diff --git a/web/scripts/app.js b/web/scripts/app.js
index be24c309b..2b818590c 100644
--- a/web/scripts/app.js
+++ b/web/scripts/app.js
@@ -4,7 +4,10 @@ import { ComfyUI, $el } from "./ui.js";
import { api } from "./api.js";
import { defaultGraph } from "./defaultGraph.js";
import { getPngMetadata, getWebpMetadata, importA1111, getLatentMetadata } from "./pnginfo.js";
+import { addDomClippingSetting } from "./domWidget.js";
+import { createImageHost, calculateImageGrid } from "./ui/imagePreview.js"
+export const ANIM_PREVIEW_WIDGET = "$$comfy_animation_preview"
function sanitizeNodeName(string) {
let entityMap = {
@@ -83,6 +86,10 @@ export class ComfyApp {
return "";
}
+ getRandParam() {
+ return "&rand=" + Math.random();
+ }
+
static isImageNode(node) {
return node.imgs || (node && node.widgets && node.widgets.findIndex(obj => obj.name === 'image') >= 0);
}
@@ -405,8 +412,10 @@ export class ComfyApp {
return shiftY;
}
- node.prototype.setSizeForImage = function () {
- if (this.inputHeight) {
+ node.prototype.setSizeForImage = function (force) {
+ if(!force && this.animatedImages) return;
+
+ if (this.inputHeight || this.freeWidgetSpace > 210) {
this.setSize(this.size);
return;
}
@@ -422,13 +431,20 @@ export class ComfyApp {
let imagesChanged = false
const output = app.nodeOutputs[this.id + ""];
- if (output && output.images) {
+ if (output?.images) {
+ this.animatedImages = output?.animated?.find(Boolean);
if (this.images !== output.images) {
this.images = output.images;
imagesChanged = true;
- imgURLs = imgURLs.concat(output.images.map(params => {
- return api.apiURL("/view?" + new URLSearchParams(params).toString() + app.getPreviewFormatParam());
- }))
+ imgURLs = imgURLs.concat(
+ output.images.map((params) => {
+ return api.apiURL(
+ "/view?" +
+ new URLSearchParams(params).toString() +
+ (this.animatedImages ? "" : app.getPreviewFormatParam()) + app.getRandParam()
+ );
+ })
+ );
}
}
@@ -507,7 +523,35 @@ export class ComfyApp {
return true;
}
- if (this.imgs && this.imgs.length) {
+ if (this.imgs?.length) {
+ const widgetIdx = this.widgets?.findIndex((w) => w.name === ANIM_PREVIEW_WIDGET);
+
+ if(this.animatedImages) {
+ // Instead of using the canvas we'll use a IMG
+ if(widgetIdx > -1) {
+ // Replace content
+ const widget = this.widgets[widgetIdx];
+ widget.options.host.updateImages(this.imgs);
+ } else {
+ const host = createImageHost(this);
+ this.setSizeForImage(true);
+ const widget = this.addDOMWidget(ANIM_PREVIEW_WIDGET, "img", host.el, {
+ host,
+ getHeight: host.getHeight,
+ onDraw: host.onDraw,
+ hideOnZoom: false
+ });
+ widget.serializeValue = () => undefined;
+ widget.options.host.updateImages(this.imgs);
+ }
+ return;
+ }
+
+ if (widgetIdx > -1) {
+ this.widgets[widgetIdx].onRemove?.();
+ this.widgets.splice(widgetIdx, 1);
+ }
+
const canvas = app.graph.list_of_graphcanvas[0];
const mouse = canvas.graph_mouse;
if (!canvas.pointer_is_down && this.pointerDown) {
@@ -547,31 +591,7 @@ export class ComfyApp {
}
else {
cell_padding = 0;
- let best = 0;
- let w = this.imgs[0].naturalWidth;
- let h = this.imgs[0].naturalHeight;
-
- // compact style
- for (let c = 1; c <= numImages; c++) {
- const rows = Math.ceil(numImages / c);
- const cW = dw / c;
- const cH = dh / rows;
- const scaleX = cW / w;
- const scaleY = cH / h;
-
- const scale = Math.min(scaleX, scaleY, 1);
- const imageW = w * scale;
- const imageH = h * scale;
- const area = imageW * imageH * numImages;
-
- if (area > best) {
- best = area;
- cellWidth = imageW;
- cellHeight = imageH;
- cols = c;
- shiftX = c * ((cW - imageW) / 2);
- }
- }
+ ({ cellWidth, cellHeight, cols, shiftX } = calculateImageGrid(this.imgs, dw, dh));
}
let anyHovered = false;
@@ -763,7 +783,7 @@ export class ComfyApp {
* Adds a handler on paste that extracts and loads images or workflows from pasted JSON data
*/
#addPasteHandler() {
- document.addEventListener("paste", (e) => {
+ document.addEventListener("paste", async (e) => {
// ctrl+shift+v is used to paste nodes with connections
// this is handled by litegraph
if(this.shiftDown) return;
@@ -811,7 +831,7 @@ export class ComfyApp {
}
if (workflow && workflow.version && workflow.nodes && workflow.extra) {
- this.loadGraphData(workflow);
+ await this.loadGraphData(workflow);
}
else {
if (e.target.type === "text" || e.target.type === "textarea") {
@@ -1161,7 +1181,19 @@ export class ComfyApp {
});
api.addEventListener("executed", ({ detail }) => {
- this.nodeOutputs[detail.node] = detail.output;
+ const output = this.nodeOutputs[detail.node];
+ if (detail.merge && output) {
+ for (const k in detail.output ?? {}) {
+ const v = output[k];
+ if (v instanceof Array) {
+ output[k] = v.concat(detail.output[k]);
+ } else {
+ output[k] = detail.output[k];
+ }
+ }
+ } else {
+ this.nodeOutputs[detail.node] = detail.output;
+ }
const node = this.graph.getNodeById(detail.node);
if (node) {
if (node.onExecuted)
@@ -1272,9 +1304,11 @@ export class ComfyApp {
canvasEl.tabIndex = "1";
document.body.prepend(canvasEl);
+ addDomClippingSetting();
this.#addProcessMouseHandler();
this.#addProcessKeyHandler();
this.#addConfigureHandler();
+ this.#addApiUpdateHandlers();
this.graph = new LGraph();
@@ -1311,7 +1345,7 @@ export class ComfyApp {
const json = localStorage.getItem("workflow");
if (json) {
const workflow = JSON.parse(json);
- this.loadGraphData(workflow);
+ await this.loadGraphData(workflow);
restored = true;
}
} catch (err) {
@@ -1320,7 +1354,7 @@ export class ComfyApp {
// We failed to restore a workflow so load the default
if (!restored) {
- this.loadGraphData();
+ await this.loadGraphData();
}
// Save current workflow automatically
@@ -1328,7 +1362,6 @@ export class ComfyApp {
this.#addDrawNodeHandler();
this.#addDrawGroupsHandler();
- this.#addApiUpdateHandlers();
this.#addDropHandler();
this.#addCopyHandler();
this.#addPasteHandler();
@@ -1348,11 +1381,95 @@ export class ComfyApp {
await this.#invokeExtensionsAsync("registerCustomNodes");
}
+ getWidgetType(inputData, inputName) {
+ const type = inputData[0];
+
+ if (Array.isArray(type)) {
+ return "COMBO";
+ } else if (`${type}:${inputName}` in this.widgets) {
+ return `${type}:${inputName}`;
+ } else if (type in this.widgets) {
+ return type;
+ } else {
+ return null;
+ }
+ }
+
+ async registerNodeDef(nodeId, nodeData) {
+ const self = this;
+ const node = Object.assign(
+ function ComfyNode() {
+ var inputs = nodeData["input"]["required"];
+ if (nodeData["input"]["optional"] != undefined) {
+ inputs = Object.assign({}, nodeData["input"]["required"], nodeData["input"]["optional"]);
+ }
+ const config = { minWidth: 1, minHeight: 1 };
+ for (const inputName in inputs) {
+ const inputData = inputs[inputName];
+ const type = inputData[0];
+
+ let widgetCreated = true;
+ const widgetType = self.getWidgetType(inputData, inputName);
+ if(widgetType) {
+ if(widgetType === "COMBO") {
+ Object.assign(config, self.widgets.COMBO(this, inputName, inputData, app) || {});
+ } else {
+ Object.assign(config, self.widgets[widgetType](this, inputName, inputData, app) || {});
+ }
+ } else {
+ // Node connection inputs
+ this.addInput(inputName, type);
+ widgetCreated = false;
+ }
+
+ if(widgetCreated && inputData[1]?.forceInput && config?.widget) {
+ if (!config.widget.options) config.widget.options = {};
+ config.widget.options.forceInput = inputData[1].forceInput;
+ }
+ if(widgetCreated && inputData[1]?.defaultInput && config?.widget) {
+ if (!config.widget.options) config.widget.options = {};
+ config.widget.options.defaultInput = inputData[1].defaultInput;
+ }
+ }
+
+ for (const o in nodeData["output"]) {
+ let output = nodeData["output"][o];
+ if(output instanceof Array) output = "COMBO";
+ const outputName = nodeData["output_name"][o] || output;
+ const outputShape = nodeData["output_is_list"][o] ? LiteGraph.GRID_SHAPE : LiteGraph.CIRCLE_SHAPE ;
+ this.addOutput(outputName, output, { shape: outputShape });
+ }
+
+ const s = this.computeSize();
+ s[0] = Math.max(config.minWidth, s[0] * 1.5);
+ s[1] = Math.max(config.minHeight, s[1]);
+ this.size = s;
+ this.serialize_widgets = true;
+
+ app.#invokeExtensionsAsync("nodeCreated", this);
+ },
+ {
+ title: nodeData.display_name || nodeData.name,
+ comfyClass: nodeData.name,
+ nodeData
+ }
+ );
+ node.prototype.comfyClass = nodeData.name;
+
+ this.#addNodeContextMenuHandler(node);
+ this.#addDrawBackgroundHandler(node, app);
+ this.#addNodeKeyHandler(node);
+
+ await this.#invokeExtensionsAsync("beforeRegisterNodeDef", node, nodeData);
+ LiteGraph.registerNodeType(nodeId, node);
+ node.category = nodeData.category;
+ }
+
async registerNodesFromDefs(defs) {
await this.#invokeExtensionsAsync("addCustomNodeDefs", defs);
// Generate list of known widgets
- const widgets = Object.assign(
+ this.widgets = Object.assign(
{},
ComfyWidgets,
...(await this.#invokeExtensionsAsync("getCustomWidgets")).filter(Boolean)
@@ -1360,75 +1477,7 @@ export class ComfyApp {
// Register a node for each definition
for (const nodeId in defs) {
- const nodeData = defs[nodeId];
- const node = Object.assign(
- function ComfyNode() {
- var inputs = nodeData["input"]["required"];
- if (nodeData["input"]["optional"] != undefined){
- inputs = Object.assign({}, nodeData["input"]["required"], nodeData["input"]["optional"])
- }
- const config = { minWidth: 1, minHeight: 1 };
- for (const inputName in inputs) {
- const inputData = inputs[inputName];
- const type = inputData[0];
-
- let widgetCreated = true;
- if (Array.isArray(type)) {
- // Enums
- Object.assign(config, widgets.COMBO(this, inputName, inputData, app) || {});
- } else if (`${type}:${inputName}` in widgets) {
- // Support custom widgets by Type:Name
- Object.assign(config, widgets[`${type}:${inputName}`](this, inputName, inputData, app) || {});
- } else if (type in widgets) {
- // Standard type widgets
- Object.assign(config, widgets[type](this, inputName, inputData, app) || {});
- } else {
- // Node connection inputs
- this.addInput(inputName, type);
- widgetCreated = false;
- }
-
- if(widgetCreated && inputData[1]?.forceInput && config?.widget) {
- if (!config.widget.options) config.widget.options = {};
- config.widget.options.forceInput = inputData[1].forceInput;
- }
- if(widgetCreated && inputData[1]?.defaultInput && config?.widget) {
- if (!config.widget.options) config.widget.options = {};
- config.widget.options.defaultInput = inputData[1].defaultInput;
- }
- }
-
- for (const o in nodeData["output"]) {
- let output = nodeData["output"][o];
- if(output instanceof Array) output = "COMBO";
- const outputName = nodeData["output_name"][o] || output;
- const outputShape = nodeData["output_is_list"][o] ? LiteGraph.GRID_SHAPE : LiteGraph.CIRCLE_SHAPE ;
- this.addOutput(outputName, output, { shape: outputShape });
- }
-
- const s = this.computeSize();
- s[0] = Math.max(config.minWidth, s[0] * 1.5);
- s[1] = Math.max(config.minHeight, s[1]);
- this.size = s;
- this.serialize_widgets = true;
-
- app.#invokeExtensionsAsync("nodeCreated", this);
- },
- {
- title: nodeData.display_name || nodeData.name,
- comfyClass: nodeData.name,
- nodeData
- }
- );
- node.prototype.comfyClass = nodeData.name;
-
- this.#addNodeContextMenuHandler(node);
- this.#addDrawBackgroundHandler(node, app);
- this.#addNodeKeyHandler(node);
-
- await this.#invokeExtensionsAsync("beforeRegisterNodeDef", node, nodeData);
- LiteGraph.registerNodeType(nodeId, node);
- node.category = nodeData.category;
+ this.registerNodeDef(nodeId, defs[nodeId]);
}
}
@@ -1470,10 +1519,37 @@ export class ComfyApp {
}
showMissingNodesError(missingNodeTypes, hasAddedNodes = true) {
+ let seenTypes = new Set();
+
this.ui.dialog.show(
- `When loading the graph, the following node types were not found: ${Array.from(new Set(missingNodeTypes)).map(
- (t) => `- ${t}
`
- ).join("")}
${hasAddedNodes ? "Nodes that have failed to load will show as red on the graph." : ""}`
+ $el("div.comfy-missing-nodes", [
+ $el("span", { textContent: "When loading the graph, the following node types were not found: " }),
+ $el(
+ "ul",
+ Array.from(new Set(missingNodeTypes)).map((t) => {
+ let children = [];
+ if (typeof t === "object") {
+ if(seenTypes.has(t.type)) return null;
+ seenTypes.add(t.type);
+ children.push($el("span", { textContent: t.type }));
+ if (t.hint) {
+ children.push($el("span", { textContent: t.hint }));
+ }
+ if (t.action) {
+ children.push($el("button", { onclick: t.action.callback, textContent: t.action.text }));
+ }
+ } else {
+ if(seenTypes.has(t)) return null;
+ seenTypes.add(t);
+ children.push($el("span", { textContent: t }));
+ }
+ return $el("li", children);
+ }).filter(Boolean)
+ ),
+ ...(hasAddedNodes
+ ? [$el("span", { textContent: "Nodes that have failed to load will show as red on the graph." })]
+ : []),
+ ])
);
this.logging.addEntry("Comfy.App", "warn", {
MissingNodes: missingNodeTypes,
@@ -1483,32 +1559,39 @@ export class ComfyApp {
/**
* Populates the graph with the specified workflow data
* @param {*} graphData A serialized graph object
+ * @param { boolean } clean If the graph state, e.g. images, should be cleared
*/
- loadGraphData(graphData) {
- this.clean();
+ async loadGraphData(graphData, clean = true) {
+ if (clean !== false) {
+ this.clean();
+ }
let reset_invalid_values = false;
if (!graphData) {
- if (typeof structuredClone === "undefined")
- {
- graphData = JSON.parse(JSON.stringify(defaultGraph));
- }else
- {
- graphData = structuredClone(defaultGraph);
- }
+ graphData = defaultGraph;
reset_invalid_values = true;
}
+ if (typeof structuredClone === "undefined")
+ {
+ graphData = JSON.parse(JSON.stringify(graphData));
+ }else
+ {
+ graphData = structuredClone(graphData);
+ }
+
const missingNodeTypes = [];
+ await this.#invokeExtensionsAsync("beforeConfigureGraph", graphData, missingNodeTypes);
for (let n of graphData.nodes) {
// Patch T2IAdapterLoader to ControlNetLoader since they are the same node now
if (n.type == "T2IAdapterLoader") n.type = "ControlNetLoader";
if (n.type == "ConditioningAverage ") n.type = "ConditioningAverage"; //typo fix
+ if (n.type == "SDV_img2vid_Conditioning") n.type = "SVD_img2vid_Conditioning"; //typo fix
// Find missing node types
if (!(n.type in LiteGraph.registered_node_types)) {
- n.type = sanitizeNodeName(n.type);
missingNodeTypes.push(n.type);
+ n.type = sanitizeNodeName(n.type);
}
}
@@ -1600,6 +1683,7 @@ export class ComfyApp {
if (missingNodeTypes.length) {
this.showMissingNodesError(missingNodeTypes);
}
+ await this.#invokeExtensionsAsync("afterConfigureGraph", missingNodeTypes);
}
/**
@@ -1607,92 +1691,110 @@ export class ComfyApp {
* @returns The workflow and node links
*/
async graphToPrompt() {
- for (const node of this.graph.computeExecutionOrder(false)) {
- if (node.isVirtualNode) {
- // Don't serialize frontend only nodes but let them make changes
- if (node.applyToGraph) {
- node.applyToGraph();
+ for (const outerNode of this.graph.computeExecutionOrder(false)) {
+ const innerNodes = outerNode.getInnerNodes ? outerNode.getInnerNodes() : [outerNode];
+ for (const node of innerNodes) {
+ if (node.isVirtualNode) {
+ // Don't serialize frontend only nodes but let them make changes
+ if (node.applyToGraph) {
+ node.applyToGraph();
+ }
}
- continue;
}
}
const workflow = this.graph.serialize();
const output = {};
// Process nodes in order of execution
- for (const node of this.graph.computeExecutionOrder(false)) {
- const n = workflow.nodes.find((n) => n.id === node.id);
+ for (const outerNode of this.graph.computeExecutionOrder(false)) {
+ const skipNode = outerNode.mode === 2 || outerNode.mode === 4;
+ const innerNodes = (!skipNode && outerNode.getInnerNodes) ? outerNode.getInnerNodes() : [outerNode];
+ for (const node of innerNodes) {
+ if (node.isVirtualNode) {
+ continue;
+ }
- if (node.isVirtualNode) {
- continue;
- }
+ if (node.mode === 2 || node.mode === 4) {
+ // Don't serialize muted nodes
+ continue;
+ }
- if (node.mode === 2 || node.mode === 4) {
- // Don't serialize muted nodes
- continue;
- }
+ const inputs = {};
+ const widgets = node.widgets;
- const inputs = {};
- const widgets = node.widgets;
-
- // Store all widget values
- if (widgets) {
- for (const i in widgets) {
- const widget = widgets[i];
- if (!widget.options || widget.options.serialize !== false) {
- inputs[widget.name] = widget.serializeValue ? await widget.serializeValue(n, i) : widget.value;
+ // Store all widget values
+ if (widgets) {
+ for (const i in widgets) {
+ const widget = widgets[i];
+ if (!widget.options || widget.options.serialize !== false) {
+ inputs[widget.name] = widget.serializeValue ? await widget.serializeValue(node, i) : widget.value;
+ }
}
}
- }
- // Store all node links
- for (let i in node.inputs) {
- let parent = node.getInputNode(i);
- if (parent) {
- let link = node.getInputLink(i);
- while (parent.mode === 4 || parent.isVirtualNode) {
- let found = false;
- if (parent.isVirtualNode) {
- link = parent.getInputLink(link.origin_slot);
- if (link) {
- parent = parent.getInputNode(link.target_slot);
- if (parent) {
- found = true;
- }
- }
- } else if (link && parent.mode === 4) {
- let all_inputs = [link.origin_slot];
- if (parent.inputs) {
- all_inputs = all_inputs.concat(Object.keys(parent.inputs))
- for (let parent_input in all_inputs) {
- parent_input = all_inputs[parent_input];
- if (parent.inputs[parent_input]?.type === node.inputs[i].type) {
- link = parent.getInputLink(parent_input);
- if (link) {
- parent = parent.getInputNode(parent_input);
- }
+ // Store all node links
+ for (let i in node.inputs) {
+ let parent = node.getInputNode(i);
+ if (parent) {
+ let link = node.getInputLink(i);
+ while (parent.mode === 4 || parent.isVirtualNode) {
+ let found = false;
+ if (parent.isVirtualNode) {
+ link = parent.getInputLink(link.origin_slot);
+ if (link) {
+ parent = parent.getInputNode(link.target_slot);
+ if (parent) {
found = true;
- break;
+ }
+ }
+ } else if (link && parent.mode === 4) {
+ let all_inputs = [link.origin_slot];
+ if (parent.inputs) {
+ all_inputs = all_inputs.concat(Object.keys(parent.inputs))
+ for (let parent_input in all_inputs) {
+ parent_input = all_inputs[parent_input];
+ if (parent.inputs[parent_input]?.type === node.inputs[i].type) {
+ link = parent.getInputLink(parent_input);
+ if (link) {
+ parent = parent.getInputNode(parent_input);
+ }
+ found = true;
+ break;
+ }
}
}
}
+
+ if (!found) {
+ break;
+ }
}
- if (!found) {
- break;
+ if (link) {
+ if (parent?.updateLink) {
+ link = parent.updateLink(link);
+ }
+ if (link) {
+ inputs[node.inputs[i].name] = [String(link.origin_id), parseInt(link.origin_slot)];
+ }
}
}
-
- if (link) {
- inputs[node.inputs[i].name] = [String(link.origin_id), parseInt(link.origin_slot)];
- }
}
- }
- output[String(node.id)] = {
- inputs,
- class_type: node.comfyClass,
- };
+ let node_data = {
+ inputs,
+ class_type: node.comfyClass,
+ };
+
+ if (this.ui.settings.getSettingValue("Comfy.DevMode")) {
+ // Ignored by the backend.
+ node_data["_meta"] = {
+ title: node.title,
+ }
+ }
+
+ output[String(node.id)] = node_data;
+ }
}
// Remove inputs connected to removed nodes
@@ -1812,7 +1914,9 @@ export class ComfyApp {
const pngInfo = await getPngMetadata(file);
if (pngInfo) {
if (pngInfo.workflow) {
- this.loadGraphData(JSON.parse(pngInfo.workflow));
+ await this.loadGraphData(JSON.parse(pngInfo.workflow));
+ } else if (pngInfo.prompt) {
+ this.loadApiJson(JSON.parse(pngInfo.prompt));
} else if (pngInfo.parameters) {
importA1111(this.graph, pngInfo.parameters);
}
@@ -1824,25 +1928,29 @@ export class ComfyApp {
this.loadGraphData(JSON.parse(pngInfo.workflow));
} else if (pngInfo.Workflow) {
this.loadGraphData(JSON.parse(pngInfo.Workflow)); // Support loading workflows from that webp custom node.
+ } else if (pngInfo.prompt) {
+ this.loadApiJson(JSON.parse(pngInfo.prompt));
}
}
} else if (file.type === "application/json" || file.name?.endsWith(".json")) {
const reader = new FileReader();
- reader.onload = () => {
+ reader.onload = async () => {
const jsonContent = JSON.parse(reader.result);
if (jsonContent?.templates) {
this.loadTemplateData(jsonContent);
} else if(this.isApiJson(jsonContent)) {
this.loadApiJson(jsonContent);
} else {
- this.loadGraphData(jsonContent);
+ await this.loadGraphData(jsonContent);
}
};
reader.readAsText(file);
} else if (file.name?.endsWith(".latent") || file.name?.endsWith(".safetensors")) {
const info = await getLatentMetadata(file);
if (info.workflow) {
- this.loadGraphData(JSON.parse(info.workflow));
+ await this.loadGraphData(JSON.parse(info.workflow));
+ } else if (info.prompt) {
+ this.loadApiJson(JSON.parse(info.prompt));
}
}
}
@@ -1863,7 +1971,7 @@ export class ComfyApp {
for (const id of ids) {
const data = apiData[id];
const node = LiteGraph.createNode(data.class_type);
- node.id = id;
+ node.id = isNaN(+id) ? id : +id;
graph.add(node);
}
@@ -1912,12 +2020,8 @@ export class ComfyApp {
async refreshComboInNodes() {
const defs = await api.getNodeDefs();
- for(const nodeId in LiteGraph.registered_node_types) {
- const node = LiteGraph.registered_node_types[nodeId];
- const nodeDef = defs[nodeId];
- if(!nodeDef) continue;
-
- node.nodeData = nodeDef;
+ for (const nodeId in defs) {
+ this.registerNodeDef(nodeId, defs[nodeId]);
}
for(let nodeNum in this.graph._nodes) {
diff --git a/web/scripts/domWidget.js b/web/scripts/domWidget.js
new file mode 100644
index 000000000..eb0742d38
--- /dev/null
+++ b/web/scripts/domWidget.js
@@ -0,0 +1,325 @@
+import { app, ANIM_PREVIEW_WIDGET } from "./app.js";
+
+const SIZE = Symbol();
+
+function intersect(a, b) {
+ const x = Math.max(a.x, b.x);
+ const num1 = Math.min(a.x + a.width, b.x + b.width);
+ const y = Math.max(a.y, b.y);
+ const num2 = Math.min(a.y + a.height, b.y + b.height);
+ if (num1 >= x && num2 >= y) return [x, y, num1 - x, num2 - y];
+ else return null;
+}
+
+function getClipPath(node, element, elRect) {
+ const selectedNode = Object.values(app.canvas.selected_nodes)[0];
+ if (selectedNode && selectedNode !== node) {
+ const MARGIN = 7;
+ const scale = app.canvas.ds.scale;
+
+ const bounding = selectedNode.getBounding();
+ const intersection = intersect(
+ { x: elRect.x / scale, y: elRect.y / scale, width: elRect.width / scale, height: elRect.height / scale },
+ {
+ x: selectedNode.pos[0] + app.canvas.ds.offset[0] - MARGIN,
+ y: selectedNode.pos[1] + app.canvas.ds.offset[1] - LiteGraph.NODE_TITLE_HEIGHT - MARGIN,
+ width: bounding[2] + MARGIN + MARGIN,
+ height: bounding[3] + MARGIN + MARGIN,
+ }
+ );
+
+ if (!intersection) {
+ return "";
+ }
+
+ const widgetRect = element.getBoundingClientRect();
+ const clipX = intersection[0] - widgetRect.x / scale + "px";
+ const clipY = intersection[1] - widgetRect.y / scale + "px";
+ const clipWidth = intersection[2] + "px";
+ const clipHeight = intersection[3] + "px";
+ const path = `polygon(0% 0%, 0% 100%, ${clipX} 100%, ${clipX} ${clipY}, calc(${clipX} + ${clipWidth}) ${clipY}, calc(${clipX} + ${clipWidth}) calc(${clipY} + ${clipHeight}), ${clipX} calc(${clipY} + ${clipHeight}), ${clipX} 100%, 100% 100%, 100% 0%)`;
+ return path;
+ }
+ return "";
+}
+
+function computeSize(size) {
+ if (this.widgets?.[0]?.last_y == null) return;
+
+ let y = this.widgets[0].last_y;
+ let freeSpace = size[1] - y;
+
+ let widgetHeight = 0;
+ let dom = [];
+ for (const w of this.widgets) {
+ if (w.type === "converted-widget") {
+ // Ignore
+ delete w.computedHeight;
+ } else if (w.computeSize) {
+ widgetHeight += w.computeSize()[1] + 4;
+ } else if (w.element) {
+ // Extract DOM widget size info
+ const styles = getComputedStyle(w.element);
+ let minHeight = w.options.getMinHeight?.() ?? parseInt(styles.getPropertyValue("--comfy-widget-min-height"));
+ let maxHeight = w.options.getMaxHeight?.() ?? parseInt(styles.getPropertyValue("--comfy-widget-max-height"));
+
+ let prefHeight = w.options.getHeight?.() ?? styles.getPropertyValue("--comfy-widget-height");
+ if (prefHeight.endsWith?.("%")) {
+ prefHeight = size[1] * (parseFloat(prefHeight.substring(0, prefHeight.length - 1)) / 100);
+ } else {
+ prefHeight = parseInt(prefHeight);
+ if (isNaN(minHeight)) {
+ minHeight = prefHeight;
+ }
+ }
+ if (isNaN(minHeight)) {
+ minHeight = 50;
+ }
+ if (!isNaN(maxHeight)) {
+ if (!isNaN(prefHeight)) {
+ prefHeight = Math.min(prefHeight, maxHeight);
+ } else {
+ prefHeight = maxHeight;
+ }
+ }
+ dom.push({
+ minHeight,
+ prefHeight,
+ w,
+ });
+ } else {
+ widgetHeight += LiteGraph.NODE_WIDGET_HEIGHT + 4;
+ }
+ }
+
+ freeSpace -= widgetHeight;
+
+ // Calculate sizes with all widgets at their min height
+ const prefGrow = []; // Nodes that want to grow to their prefd size
+ const canGrow = []; // Nodes that can grow to auto size
+ let growBy = 0;
+ for (const d of dom) {
+ freeSpace -= d.minHeight;
+ if (isNaN(d.prefHeight)) {
+ canGrow.push(d);
+ d.w.computedHeight = d.minHeight;
+ } else {
+ const diff = d.prefHeight - d.minHeight;
+ if (diff > 0) {
+ prefGrow.push(d);
+ growBy += diff;
+ d.diff = diff;
+ } else {
+ d.w.computedHeight = d.minHeight;
+ }
+ }
+ }
+
+ if (this.imgs && !this.widgets.find((w) => w.name === ANIM_PREVIEW_WIDGET)) {
+ // Allocate space for image
+ freeSpace -= 220;
+ }
+
+ this.freeWidgetSpace = freeSpace;
+
+ if (freeSpace < 0) {
+ // Not enough space for all widgets so we need to grow
+ size[1] -= freeSpace;
+ this.graph.setDirtyCanvas(true);
+ } else {
+ // Share the space between each
+ const growDiff = freeSpace - growBy;
+ if (growDiff > 0) {
+ // All pref sizes can be fulfilled
+ freeSpace = growDiff;
+ for (const d of prefGrow) {
+ d.w.computedHeight = d.prefHeight;
+ }
+ } else {
+ // We need to grow evenly
+ const shared = -growDiff / prefGrow.length;
+ for (const d of prefGrow) {
+ d.w.computedHeight = d.prefHeight - shared;
+ }
+ freeSpace = 0;
+ }
+
+ if (freeSpace > 0 && canGrow.length) {
+ // Grow any that are auto height
+ const shared = freeSpace / canGrow.length;
+ for (const d of canGrow) {
+ d.w.computedHeight += shared;
+ }
+ }
+ }
+
+ // Position each of the widgets
+ for (const w of this.widgets) {
+ w.y = y;
+ if (w.computedHeight) {
+ y += w.computedHeight;
+ } else if (w.computeSize) {
+ y += w.computeSize()[1] + 4;
+ } else {
+ y += LiteGraph.NODE_WIDGET_HEIGHT + 4;
+ }
+ }
+}
+
+// Override the compute visible nodes function to allow us to hide/show DOM elements when the node goes offscreen
+const elementWidgets = new Set();
+const computeVisibleNodes = LGraphCanvas.prototype.computeVisibleNodes;
+LGraphCanvas.prototype.computeVisibleNodes = function () {
+ const visibleNodes = computeVisibleNodes.apply(this, arguments);
+ for (const node of app.graph._nodes) {
+ if (elementWidgets.has(node)) {
+ const hidden = visibleNodes.indexOf(node) === -1;
+ for (const w of node.widgets) {
+ if (w.element) {
+ w.element.hidden = hidden;
+ w.element.style.display = hidden ? "none" : undefined;
+ if (hidden) {
+ w.options.onHide?.(w);
+ }
+ }
+ }
+ }
+ }
+
+ return visibleNodes;
+};
+
+let enableDomClipping = true;
+
+export function addDomClippingSetting() {
+ app.ui.settings.addSetting({
+ id: "Comfy.DOMClippingEnabled",
+ name: "Enable DOM element clipping (enabling may reduce performance)",
+ type: "boolean",
+ defaultValue: enableDomClipping,
+ onChange(value) {
+ enableDomClipping = !!value;
+ },
+ });
+}
+
+LGraphNode.prototype.addDOMWidget = function (name, type, element, options) {
+ options = { hideOnZoom: true, selectOn: ["focus", "click"], ...options };
+
+ if (!element.parentElement) {
+ document.body.append(element);
+ }
+
+ let mouseDownHandler;
+ if (element.blur) {
+ mouseDownHandler = (event) => {
+ if (!element.contains(event.target)) {
+ element.blur();
+ }
+ };
+ document.addEventListener("mousedown", mouseDownHandler);
+ }
+
+ const widget = {
+ type,
+ name,
+ get value() {
+ return options.getValue?.() ?? undefined;
+ },
+ set value(v) {
+ options.setValue?.(v);
+ widget.callback?.(widget.value);
+ },
+ draw: function (ctx, node, widgetWidth, y, widgetHeight) {
+ if (widget.computedHeight == null) {
+ computeSize.call(node, node.size);
+ }
+
+ const hidden =
+ node.flags?.collapsed ||
+ (!!options.hideOnZoom && app.canvas.ds.scale < 0.5) ||
+ widget.computedHeight <= 0 ||
+ widget.type === "converted-widget";
+ element.hidden = hidden;
+ element.style.display = hidden ? "none" : null;
+ if (hidden) {
+ widget.options.onHide?.(widget);
+ return;
+ }
+
+ const margin = 10;
+ const elRect = ctx.canvas.getBoundingClientRect();
+ const transform = new DOMMatrix()
+ .scaleSelf(elRect.width / ctx.canvas.width, elRect.height / ctx.canvas.height)
+ .multiplySelf(ctx.getTransform())
+ .translateSelf(margin, margin + y);
+
+ const scale = new DOMMatrix().scaleSelf(transform.a, transform.d);
+
+ Object.assign(element.style, {
+ transformOrigin: "0 0",
+ transform: scale,
+ left: `${transform.a + transform.e}px`,
+ top: `${transform.d + transform.f}px`,
+ width: `${widgetWidth - margin * 2}px`,
+ height: `${(widget.computedHeight ?? 50) - margin * 2}px`,
+ position: "absolute",
+ zIndex: app.graph._nodes.indexOf(node),
+ });
+
+ if (enableDomClipping) {
+ element.style.clipPath = getClipPath(node, element, elRect);
+ element.style.willChange = "clip-path";
+ }
+
+ this.options.onDraw?.(widget);
+ },
+ element,
+ options,
+ onRemove() {
+ if (mouseDownHandler) {
+ document.removeEventListener("mousedown", mouseDownHandler);
+ }
+ element.remove();
+ },
+ };
+
+ for (const evt of options.selectOn) {
+ element.addEventListener(evt, () => {
+ app.canvas.selectNode(this);
+ app.canvas.bringToFront(this);
+ });
+ }
+
+ this.addCustomWidget(widget);
+ elementWidgets.add(this);
+
+ const collapse = this.collapse;
+ this.collapse = function() {
+ collapse.apply(this, arguments);
+ if(this.flags?.collapsed) {
+ element.hidden = true;
+ element.style.display = "none";
+ }
+ }
+
+ const onRemoved = this.onRemoved;
+ this.onRemoved = function () {
+ element.remove();
+ elementWidgets.delete(this);
+ onRemoved?.apply(this, arguments);
+ };
+
+ if (!this[SIZE]) {
+ this[SIZE] = true;
+ const onResize = this.onResize;
+ this.onResize = function (size) {
+ options.beforeResize?.call(widget, this);
+ computeSize.call(this, size);
+ onResize?.apply(this, arguments);
+ options.afterResize?.call(widget, this);
+ };
+ }
+
+ return widget;
+};
diff --git a/web/scripts/pnginfo.js b/web/scripts/pnginfo.js
index 491caed79..83a4ebc86 100644
--- a/web/scripts/pnginfo.js
+++ b/web/scripts/pnginfo.js
@@ -24,7 +24,7 @@ export function getPngMetadata(file) {
const length = dataView.getUint32(offset);
// Get the chunk type
const type = String.fromCharCode(...pngData.slice(offset + 4, offset + 8));
- if (type === "tEXt") {
+ if (type === "tEXt" || type == "comf") {
// Get the keyword
let keyword_end = offset + 8;
while (pngData[keyword_end] !== 0) {
@@ -50,7 +50,6 @@ export function getPngMetadata(file) {
function parseExifData(exifData) {
// Check for the correct TIFF header (0x4949 for little-endian or 0x4D4D for big-endian)
const isLittleEndian = new Uint16Array(exifData.slice(0, 2))[0] === 0x4949;
- console.log(exifData);
// Function to read 16-bit and 32-bit integers from binary data
function readInt(offset, isLittleEndian, length) {
@@ -126,6 +125,9 @@ export function getWebpMetadata(file) {
const chunk_length = dataView.getUint32(offset + 4, true);
const chunk_type = String.fromCharCode(...webp.slice(offset, offset + 4));
if (chunk_type === "EXIF") {
+ if (String.fromCharCode(...webp.slice(offset + 8, offset + 8 + 6)) == "Exif\0\0") {
+ offset += 6;
+ }
let data = parseExifData(webp.slice(offset + 8, offset + 8 + chunk_length));
for (var key in data) {
var value = data[key];
diff --git a/web/scripts/ui.js b/web/scripts/ui.js
index 6f01aa5b2..ebaf86fe4 100644
--- a/web/scripts/ui.js
+++ b/web/scripts/ui.js
@@ -462,8 +462,8 @@ class ComfyList {
return $el("div", {textContent: item.prompt[0] + ": "}, [
$el("button", {
textContent: "Load",
- onclick: () => {
- app.loadGraphData(item.prompt[3].extra_pnginfo.workflow);
+ onclick: async () => {
+ await app.loadGraphData(item.prompt[3].extra_pnginfo.workflow);
if (item.outputs) {
app.nodeOutputs = item.outputs;
}
@@ -599,7 +599,7 @@ export class ComfyUI {
const fileInput = $el("input", {
id: "comfy-file-input",
type: "file",
- accept: ".json,image/png,.latent,.safetensors",
+ accept: ".json,image/png,.latent,.safetensors,image/webp",
style: {display: "none"},
parent: document.body,
onchange: () => {
@@ -784,9 +784,9 @@ export class ComfyUI {
}
}),
$el("button", {
- id: "comfy-load-default-button", textContent: "Load Default", onclick: () => {
+ id: "comfy-load-default-button", textContent: "Load Default", onclick: async () => {
if (!confirmClear.value || confirm("Load default workflow?")) {
- app.loadGraphData()
+ await app.loadGraphData()
}
}
}),
diff --git a/web/scripts/ui/imagePreview.js b/web/scripts/ui/imagePreview.js
new file mode 100644
index 000000000..2a7f66b8f
--- /dev/null
+++ b/web/scripts/ui/imagePreview.js
@@ -0,0 +1,97 @@
+import { $el } from "../ui.js";
+
+export function calculateImageGrid(imgs, dw, dh) {
+ let best = 0;
+ let w = imgs[0].naturalWidth;
+ let h = imgs[0].naturalHeight;
+ const numImages = imgs.length;
+
+ let cellWidth, cellHeight, cols, rows, shiftX;
+ // compact style
+ for (let c = 1; c <= numImages; c++) {
+ const r = Math.ceil(numImages / c);
+ const cW = dw / c;
+ const cH = dh / r;
+ const scaleX = cW / w;
+ const scaleY = cH / h;
+
+ const scale = Math.min(scaleX, scaleY, 1);
+ const imageW = w * scale;
+ const imageH = h * scale;
+ const area = imageW * imageH * numImages;
+
+ if (area > best) {
+ best = area;
+ cellWidth = imageW;
+ cellHeight = imageH;
+ cols = c;
+ rows = r;
+ shiftX = c * ((cW - imageW) / 2);
+ }
+ }
+
+ return { cellWidth, cellHeight, cols, rows, shiftX };
+}
+
+export function createImageHost(node) {
+ const el = $el("div.comfy-img-preview");
+ let currentImgs;
+ let first = true;
+
+ function updateSize() {
+ let w = null;
+ let h = null;
+
+ if (currentImgs) {
+ let elH = el.clientHeight;
+ if (first) {
+ first = false;
+ // On first run, if we are small then grow a bit
+ if (elH < 190) {
+ elH = 190;
+ }
+ el.style.setProperty("--comfy-widget-min-height", elH);
+ } else {
+ el.style.setProperty("--comfy-widget-min-height", null);
+ }
+
+ const nw = node.size[0];
+ ({ cellWidth: w, cellHeight: h } = calculateImageGrid(currentImgs, nw - 20, elH));
+ w += "px";
+ h += "px";
+
+ el.style.setProperty("--comfy-img-preview-width", w);
+ el.style.setProperty("--comfy-img-preview-height", h);
+ }
+ }
+ return {
+ el,
+ updateImages(imgs) {
+ if (imgs !== currentImgs) {
+ if (currentImgs == null) {
+ requestAnimationFrame(() => {
+ updateSize();
+ });
+ }
+ el.replaceChildren(...imgs);
+ currentImgs = imgs;
+ node.onResize(node.size);
+ node.graph.setDirtyCanvas(true, true);
+ }
+ },
+ getHeight() {
+ updateSize();
+ },
+ onDraw() {
+ // Element from point uses a hittest find elements so we need to toggle pointer events
+ el.style.pointerEvents = "all";
+ const over = document.elementFromPoint(app.canvas.mouse[0], app.canvas.mouse[1]);
+ el.style.pointerEvents = "none";
+
+ if(!over) return;
+ // Set the overIndex so Open Image etc work
+ const idx = currentImgs.indexOf(over);
+ node.overIndex = idx;
+ },
+ };
+}
diff --git a/web/scripts/utils.js b/web/scripts/utils.js
new file mode 100644
index 000000000..401aca9e4
--- /dev/null
+++ b/web/scripts/utils.js
@@ -0,0 +1,67 @@
+// Simple date formatter
+const parts = {
+ d: (d) => d.getDate(),
+ M: (d) => d.getMonth() + 1,
+ h: (d) => d.getHours(),
+ m: (d) => d.getMinutes(),
+ s: (d) => d.getSeconds(),
+};
+const format =
+ Object.keys(parts)
+ .map((k) => k + k + "?")
+ .join("|") + "|yyy?y?";
+
+function formatDate(text, date) {
+ return text.replace(new RegExp(format, "g"), function (text) {
+ if (text === "yy") return (date.getFullYear() + "").substring(2);
+ if (text === "yyyy") return date.getFullYear();
+ if (text[0] in parts) {
+ const p = parts[text[0]](date);
+ return (p + "").padStart(text.length, "0");
+ }
+ return text;
+ });
+}
+
+export function applyTextReplacements(app, value) {
+ return value.replace(/%([^%]+)%/g, function (match, text) {
+ const split = text.split(".");
+ if (split.length !== 2) {
+ // Special handling for dates
+ if (split[0].startsWith("date:")) {
+ return formatDate(split[0].substring(5), new Date());
+ }
+
+ if (text !== "width" && text !== "height") {
+ // Dont warn on standard replacements
+ console.warn("Invalid replacement pattern", text);
+ }
+ return match;
+ }
+
+ // Find node with matching S&R property name
+ let nodes = app.graph._nodes.filter((n) => n.properties?.["Node name for S&R"] === split[0]);
+ // If we cant, see if there is a node with that title
+ if (!nodes.length) {
+ nodes = app.graph._nodes.filter((n) => n.title === split[0]);
+ }
+ if (!nodes.length) {
+ console.warn("Unable to find node", split[0]);
+ return match;
+ }
+
+ if (nodes.length > 1) {
+ console.warn("Multiple nodes matched", split[0], "using first match");
+ }
+
+ const node = nodes[0];
+
+ const widget = node.widgets?.find((w) => w.name === split[1]);
+ if (!widget) {
+ console.warn("Unable to find widget", split[1], "on node", split[0], node);
+ return match;
+ }
+
+ return ((widget.value ?? "") + "").replaceAll(/\/|\\/g, "_");
+ });
+}
diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js
index 2b6747769..e2e21164d 100644
--- a/web/scripts/widgets.js
+++ b/web/scripts/widgets.js
@@ -1,4 +1,5 @@
import { api } from "./api.js"
+import "./domWidget.js";
function getNumberDefaults(inputData, defaultStep, precision, enable_rounding) {
let defaultVal = inputData[1]["default"];
@@ -22,18 +23,89 @@ function getNumberDefaults(inputData, defaultStep, precision, enable_rounding) {
return { val: defaultVal, config: { min, max, step: 10.0 * step, round, precision } };
}
-export function addValueControlWidget(node, targetWidget, defaultValue = "randomize", values) {
- const valueControl = node.addWidget("combo", "control_after_generate", defaultValue, function (v) { }, {
- values: ["fixed", "increment", "decrement", "randomize"],
- serialize: false, // Don't include this in prompt.
- });
- valueControl.afterQueued = () => {
+export function addValueControlWidget(node, targetWidget, defaultValue = "randomize", values, widgetName, inputData) {
+ let name = inputData[1]?.control_after_generate;
+ if(typeof name !== "string") {
+ name = widgetName;
+ }
+ const widgets = addValueControlWidgets(node, targetWidget, defaultValue, {
+ addFilterList: false,
+ controlAfterGenerateName: name
+ }, inputData);
+ return widgets[0];
+}
+export function addValueControlWidgets(node, targetWidget, defaultValue = "randomize", options, inputData) {
+ if (!defaultValue) defaultValue = "randomize";
+ if (!options) options = {};
+
+ const getName = (defaultName, optionName) => {
+ let name = defaultName;
+ if (options[optionName]) {
+ name = options[optionName];
+ } else if (typeof inputData?.[1]?.[defaultName] === "string") {
+ name = inputData?.[1]?.[defaultName];
+ } else if (inputData?.[1]?.control_prefix) {
+ name = inputData?.[1]?.control_prefix + " " + name
+ }
+ return name;
+ }
+
+ const widgets = [];
+ const valueControl = node.addWidget(
+ "combo",
+ getName("control_after_generate", "controlAfterGenerateName"),
+ defaultValue,
+ function () {},
+ {
+ values: ["fixed", "increment", "decrement", "randomize"],
+ serialize: false, // Don't include this in prompt.
+ }
+ );
+ widgets.push(valueControl);
+
+ const isCombo = targetWidget.type === "combo";
+ let comboFilter;
+ if (isCombo && options.addFilterList !== false) {
+ comboFilter = node.addWidget(
+ "string",
+ getName("control_filter_list", "controlFilterListName"),
+ "",
+ function () {},
+ {
+ serialize: false, // Don't include this in prompt.
+ }
+ );
+ widgets.push(comboFilter);
+ }
+
+ valueControl.afterQueued = () => {
var v = valueControl.value;
- if (targetWidget.type == "combo" && v !== "fixed") {
- let current_index = targetWidget.options.values.indexOf(targetWidget.value);
- let current_length = targetWidget.options.values.length;
+ if (isCombo && v !== "fixed") {
+ let values = targetWidget.options.values;
+ const filter = comboFilter?.value;
+ if (filter) {
+ let check;
+ if (filter.startsWith("/") && filter.endsWith("/")) {
+ try {
+ const regex = new RegExp(filter.substring(1, filter.length - 1));
+ check = (item) => regex.test(item);
+ } catch (error) {
+ console.error("Error constructing RegExp filter for node " + node.id, filter, error);
+ }
+ }
+ if (!check) {
+ const lower = filter.toLocaleLowerCase();
+ check = (item) => item.toLocaleLowerCase().includes(lower);
+ }
+ values = values.filter(item => check(item));
+ if (!values.length && targetWidget.options.values.length) {
+ console.warn("Filter for node " + node.id + " has filtered out all items", filter);
+ }
+ }
+ let current_index = values.indexOf(targetWidget.value);
+ let current_length = values.length;
switch (v) {
case "increment":
@@ -50,11 +122,12 @@ export function addValueControlWidget(node, targetWidget, defaultValue = "random
current_index = Math.max(0, current_index);
current_index = Math.min(current_length - 1, current_index);
if (current_index >= 0) {
- let value = targetWidget.options.values[current_index];
+ let value = values[current_index];
targetWidget.value = value;
targetWidget.callback(value);
}
- } else { //number
+ } else {
+ //number
let min = targetWidget.options.min;
let max = targetWidget.options.max;
// limit to something that javascript can handle
@@ -77,186 +150,68 @@ export function addValueControlWidget(node, targetWidget, defaultValue = "random
default:
break;
}
- /*check if values are over or under their respective
- * ranges and set them to min or max.*/
- if (targetWidget.value < min)
- targetWidget.value = min;
+ /*check if values are over or under their respective
+ * ranges and set them to min or max.*/
+ if (targetWidget.value < min) targetWidget.value = min;
if (targetWidget.value > max)
targetWidget.value = max;
targetWidget.callback(targetWidget.value);
}
- }
- return valueControl;
+ };
+ return widgets;
};
-function seedWidget(node, inputName, inputData, app) {
- const seed = ComfyWidgets.INT(node, inputName, inputData, app);
- const seedControl = addValueControlWidget(node, seed.widget, "randomize");
+function seedWidget(node, inputName, inputData, app, widgetName) {
+ const seed = createIntWidget(node, inputName, inputData, app, true);
+ const seedControl = addValueControlWidget(node, seed.widget, "randomize", undefined, widgetName, inputData);
seed.widget.linkedWidgets = [seedControl];
return seed;
}
-const MultilineSymbol = Symbol();
-const MultilineResizeSymbol = Symbol();
+function createIntWidget(node, inputName, inputData, app, isSeedInput) {
+ const control = inputData[1]?.control_after_generate;
+ if (!isSeedInput && control) {
+ return seedWidget(node, inputName, inputData, app, typeof control === "string" ? control : undefined);
+ }
+
+ let widgetType = isSlider(inputData[1]["display"], app);
+ const { val, config } = getNumberDefaults(inputData, 1, 0, true);
+ Object.assign(config, { precision: 0 });
+ return {
+ widget: node.addWidget(
+ widgetType,
+ inputName,
+ val,
+ function (v) {
+ const s = this.options.step / 10;
+ this.value = Math.round(v / s) * s;
+ },
+ config
+ ),
+ };
+}
function addMultilineWidget(node, name, opts, app) {
- const MIN_SIZE = 50;
+ const inputEl = document.createElement("textarea");
+ inputEl.className = "comfy-multiline-input";
+ inputEl.value = opts.defaultVal;
+ inputEl.placeholder = opts.placeholder || name;
- function computeSize(size) {
- if (node.widgets[0].last_y == null) return;
-
- let y = node.widgets[0].last_y;
- let freeSpace = size[1] - y;
-
- // Compute the height of all non customtext widgets
- let widgetHeight = 0;
- const multi = [];
- for (let i = 0; i < node.widgets.length; i++) {
- const w = node.widgets[i];
- if (w.type === "customtext") {
- multi.push(w);
- } else {
- if (w.computeSize) {
- widgetHeight += w.computeSize()[1] + 4;
- } else {
- widgetHeight += LiteGraph.NODE_WIDGET_HEIGHT + 4;
- }
- }
- }
-
- // See how large each text input can be
- freeSpace -= widgetHeight;
- freeSpace /= multi.length + (!!node.imgs?.length);
-
- if (freeSpace < MIN_SIZE) {
- // There isnt enough space for all the widgets, increase the size of the node
- freeSpace = MIN_SIZE;
- node.size[1] = y + widgetHeight + freeSpace * (multi.length + (!!node.imgs?.length));
- node.graph.setDirtyCanvas(true);
- }
-
- // Position each of the widgets
- for (const w of node.widgets) {
- w.y = y;
- if (w.type === "customtext") {
- y += freeSpace;
- w.computedHeight = freeSpace - multi.length*4;
- } else if (w.computeSize) {
- y += w.computeSize()[1] + 4;
- } else {
- y += LiteGraph.NODE_WIDGET_HEIGHT + 4;
- }
- }
-
- node.inputHeight = freeSpace;
- }
-
- const widget = {
- type: "customtext",
- name,
- get value() {
- return this.inputEl.value;
+ const widget = node.addDOMWidget(name, "customtext", inputEl, {
+ getValue() {
+ return inputEl.value;
},
- set value(x) {
- this.inputEl.value = x;
+ setValue(v) {
+ inputEl.value = v;
},
- draw: function (ctx, _, widgetWidth, y, widgetHeight) {
- if (!this.parent.inputHeight) {
- // If we are initially offscreen when created we wont have received a resize event
- // Calculate it here instead
- computeSize(node.size);
- }
- const visible = app.canvas.ds.scale > 0.5 && this.type === "customtext";
- const margin = 10;
- const elRect = ctx.canvas.getBoundingClientRect();
- const transform = new DOMMatrix()
- .scaleSelf(elRect.width / ctx.canvas.width, elRect.height / ctx.canvas.height)
- .multiplySelf(ctx.getTransform())
- .translateSelf(margin, margin + y);
-
- const scale = new DOMMatrix().scaleSelf(transform.a, transform.d)
- Object.assign(this.inputEl.style, {
- transformOrigin: "0 0",
- transform: scale,
- left: `${transform.a + transform.e}px`,
- top: `${transform.d + transform.f}px`,
- width: `${widgetWidth - (margin * 2)}px`,
- height: `${this.parent.inputHeight - (margin * 2)}px`,
- position: "absolute",
- background: (!node.color)?'':node.color,
- color: (!node.color)?'':'white',
- zIndex: app.graph._nodes.indexOf(node),
- });
- this.inputEl.hidden = !visible;
- },
- };
- widget.inputEl = document.createElement("textarea");
- widget.inputEl.className = "comfy-multiline-input";
- widget.inputEl.value = opts.defaultVal;
- widget.inputEl.placeholder = opts.placeholder || "";
- document.addEventListener("mousedown", function (event) {
- if (!widget.inputEl.contains(event.target)) {
- widget.inputEl.blur();
- }
});
- widget.parent = node;
- document.body.appendChild(widget.inputEl);
+ widget.inputEl = inputEl;
- node.addCustomWidget(widget);
-
- app.canvas.onDrawBackground = function () {
- // Draw node isnt fired once the node is off the screen
- // if it goes off screen quickly, the input may not be removed
- // this shifts it off screen so it can be moved back if the node is visible.
- for (let n in app.graph._nodes) {
- n = graph._nodes[n];
- for (let w in n.widgets) {
- let wid = n.widgets[w];
- if (Object.hasOwn(wid, "inputEl")) {
- wid.inputEl.style.left = -8000 + "px";
- wid.inputEl.style.position = "absolute";
- }
- }
- }
- };
-
- node.onRemoved = function () {
- // When removing this node we need to remove the input from the DOM
- for (let y in this.widgets) {
- if (this.widgets[y].inputEl) {
- this.widgets[y].inputEl.remove();
- }
- }
- };
-
- widget.onRemove = () => {
- widget.inputEl?.remove();
-
- // Restore original size handler if we are the last
- if (!--node[MultilineSymbol]) {
- node.onResize = node[MultilineResizeSymbol];
- delete node[MultilineSymbol];
- delete node[MultilineResizeSymbol];
- }
- };
-
- if (node[MultilineSymbol]) {
- node[MultilineSymbol]++;
- } else {
- node[MultilineSymbol] = 1;
- const onResize = (node[MultilineResizeSymbol] = node.onResize);
-
- node.onResize = function (size) {
- computeSize(size);
-
- // Call original resizer handler
- if (onResize) {
- onResize.apply(this, arguments);
- }
- };
- }
+ inputEl.addEventListener("input", () => {
+ widget.callback?.(widget.value);
+ });
return { minWidth: 400, minHeight: 200, widget };
}
@@ -288,31 +243,26 @@ export const ComfyWidgets = {
}, config) };
},
INT(node, inputName, inputData, app) {
- let widgetType = isSlider(inputData[1]["display"], app);
- const { val, config } = getNumberDefaults(inputData, 1, 0, true);
- Object.assign(config, { precision: 0 });
- return {
- widget: node.addWidget(
- widgetType,
- inputName,
- val,
- function (v) {
- const s = this.options.step / 10;
- this.value = Math.round(v / s) * s;
- },
- config
- ),
- };
+ return createIntWidget(node, inputName, inputData, app);
},
BOOLEAN(node, inputName, inputData) {
- let defaultVal = inputData[1]["default"];
+ let defaultVal = false;
+ let options = {};
+ if (inputData[1]) {
+ if (inputData[1].default)
+ defaultVal = inputData[1].default;
+ if (inputData[1].label_on)
+ options["on"] = inputData[1].label_on;
+ if (inputData[1].label_off)
+ options["off"] = inputData[1].label_off;
+ }
return {
widget: node.addWidget(
"toggle",
inputName,
defaultVal,
() => {},
- {"on": inputData[1].label_on, "off": inputData[1].label_off}
+ options,
)
};
},
@@ -338,10 +288,14 @@ export const ComfyWidgets = {
if (inputData[1] && inputData[1].default) {
defaultValue = inputData[1].default;
}
- return { widget: node.addWidget("combo", inputName, defaultValue, () => {}, { values: type }) };
+ const res = { widget: node.addWidget("combo", inputName, defaultValue, () => {}, { values: type }) };
+ if (inputData[1]?.control_after_generate) {
+ res.widget.linkedWidgets = addValueControlWidgets(node, res.widget, undefined, undefined, inputData);
+ }
+ return res;
},
IMAGEUPLOAD(node, inputName, inputData, app) {
- const imageWidget = node.widgets.find((w) => w.name === "image");
+ const imageWidget = node.widgets.find((w) => w.name === (inputData[1]?.widget ?? "image"));
let uploadWidget;
function showImage(name) {
@@ -356,7 +310,7 @@ export const ComfyWidgets = {
subfolder = name.substring(0, folder_separator);
name = name.substring(folder_separator + 1);
}
- img.src = api.apiURL(`/view?filename=${encodeURIComponent(name)}&type=input&subfolder=${subfolder}${app.getPreviewFormatParam()}`);
+ img.src = api.apiURL(`/view?filename=${encodeURIComponent(name)}&type=input&subfolder=${subfolder}${app.getPreviewFormatParam()}${app.getRandParam()}`);
node.setSizeForImage?.();
}
@@ -455,9 +409,10 @@ export const ComfyWidgets = {
document.body.append(fileInput);
// Create the button widget for selecting the files
- uploadWidget = node.addWidget("button", "choose file to upload", "image", () => {
+ uploadWidget = node.addWidget("button", inputName, "image", () => {
fileInput.click();
});
+ uploadWidget.label = "choose file to upload";
uploadWidget.serialize = false;
// Add handler to check if an image is being dragged over our node
diff --git a/web/style.css b/web/style.css
index 75c020ac9..51db5fcab 100644
--- a/web/style.css
+++ b/web/style.css
@@ -409,6 +409,26 @@ dialog::backdrop {
width: calc(100% - 10px);
}
+.comfy-img-preview {
+ pointer-events: none;
+ overflow: hidden;
+ display: flex;
+ flex-wrap: wrap;
+ align-content: flex-start;
+ justify-content: center;
+}
+
+.comfy-img-preview img {
+ object-fit: contain;
+ width: var(--comfy-img-preview-width);
+ height: var(--comfy-img-preview-height);
+}
+
+.comfy-missing-nodes li button {
+ font-size: 12px;
+ margin-left: 5px;
+}
+
/* Search box */
.litegraph.litesearchbox {